#!/usr/bin/env python3
"""Re-run the measurement behind the Phayan post "Thai Silent Letters".

Every word among the 5,000 most frequent entries of the Thai National Corpus
frequency list (as shipped with PyThaiNLP 5.4.0) that ends in a karan (U+0E4C on
the last letter) is read twice, using the pronunciation English Wiktionary records
for it ({{th-pron|...}}):

  * as the mark asks: the syllable closes where the recorded pronunciation closes it;
  * the naive way: the syllable closes on the letter under the mark.

Each reading gets a tone from the standard rule (class of the syllable's first
consonant, vowel length, live or dead ending, tone mark). The script counts the
words whose two tones differ and the share of corpus tokens they carry.

    python3 check.py            re-measure and compare with dataset.json / dataset.csv
    python3 check.py --recheck  the same; every step here takes well under a minute
    python3 check.py --write    re-measure and rewrite dataset.csv and the results
                                in dataset.json
Standard library only. Exit 0 when every stored value reproduces, 1 otherwise.
"""

import csv
import hashlib
import html
import json
import os
import re
import sys
import time
import unicodedata
import urllib.error
import urllib.parse
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
TABLE = os.path.join(HERE, "dataset.csv")

CORPUS_URL = ("https://raw.githubusercontent.com/PyThaiNLP/pythainlp/v5.4.0/"
              "pythainlp/corpus/tnc_freq.txt")
CORPUS_SHA256 = "554984d62e8735e2705bb30b06096d5710d0c9011e340ab717134e10d6dd0dda"
WIKT = "https://en.wiktionary.org/w/api.php"
UA = "phayan-blog-evidence/1.1 (re-runs the measurement in the post dataset)"

TOP_N = 5000
DAYS = ["จันทร์", "อังคาร", "พุธ", "พฤหัสบดี", "ศุกร์", "เสาร์", "อาทิตย์"]
ARTICLE_WORDS = ["สัตว์", "ศุกร์", "อาทิตย์", "จันทร์", "รถเมล์",
                 "เบียร์", "ฟิล์ม", "แสตมป์", "ยีนส์", "คอมพิวเตอร์"]

MID = set("กจฎฏดตบปอ")
HIGH = set("ขฃฉฐถผฝศษสห")
LOW = set("คฆงญณนมยรลวฅฌชซฑฒทธพภฟฮฬ")
CONS = MID | HIGH | LOW
STOP = set("บปพภฟจฉชซฌฎฏฐฑฒดตถทธศษสกขฃคฅฆ")
SONORANT = set("งญณนมยรลวฬ")
FINAL_CAPABLE = STOP | SONORANT
LEAD = set("เแโใไ")
SHORT_MARKS = set("ะัิึุ็")
SHORT_OPEN_END = set("ะิึุ")
TONE_MARKS = set("่้๊๋")
KARAN = "์"
PHINTHU = "ฺ"
TONE_NAME = {"M": "mid", "L": "low", "F": "falling", "H": "high", "R": "rising"}


def assert_classes():
    assert len(MID) + len(HIGH) + len(LOW) == 44


def tone_of(cls, dead, short, mark):
    if mark == "่":
        return {"M": "L", "H": "L", "L": "F"}[cls]
    if mark == "้":
        return {"M": "F", "H": "F", "L": "H"}[cls]
    if mark == "๊":
        return "H"
    if mark == "๋":
        return "R"
    if dead:
        if cls == "L":
            return "H" if short else "F"
        return "L"
    return {"M": "M", "H": "R", "L": "M"}[cls]


def syllable(chunk):
    """Class, vowel length, final and tone mark of one respelled syllable.

    A consonant followed by a phinthu belongs to the onset (a cluster, or a silent
    ห/อ that sets the class), so the class comes from the first onset consonant.
    """
    onset = set()
    letters = []
    for i, c in enumerate(chunk):
        if c == PHINTHU:
            if letters:
                onset.add(len(letters) - 1)
                onset.add(len(letters))
            continue
        letters.append(c)
    mark = next((c for c in letters if c in TONE_MARKS), None)
    skel = [(i, c) for i, c in enumerate(letters) if c not in TONE_MARKS]
    cons = [(i, c) for i, c in skel if c in CONS]
    if not cons:
        return None
    first = cons[0]
    onset.add(first[0])
    cls = "M" if first[1] in MID else ("H" if first[1] in HIGH else "L")
    chars = [c for _, c in skel]
    has_vowel = any(c in LEAD or c in "ะัาำิีึืุู็" for c in chars) or any(
        c == "อ" and i > first[0] for i, c in skel)
    short = any(c in SHORT_MARKS for c in chars) or not has_vowel
    last_i, last_c = skel[-1]
    final = None
    if last_c in CONS and last_i not in onset and not (last_c == "อ" and has_vowel):
        final = last_c
    if final is not None:
        dead = final in STOP
    elif chars[0] in "ไใ" or last_c == "ำ":
        dead = False
    else:
        dead = last_c in SHORT_OPEN_END
    return {"class": cls, "short": short, "final": final, "dead": dead, "mark": mark}


def reading(word, respelling):
    chunk = respelling.split("-")[-1]
    s = syllable(chunk)
    if s is None:
        return None
    naive_final = word[-2]
    t_rule = tone_of(s["class"], s["dead"], s["short"], s["mark"])
    t_naive = tone_of(s["class"], naive_final in STOP, s["short"], s["mark"])
    return {"respelling": respelling, "final": s["final"], "naive_final": naive_final,
            "tone": t_rule, "naive_tone": t_naive}


def get(url, timeout=60):
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return r.read()


def fetch_corpus():
    raw = get(CORPUS_URL, timeout=120)
    digest = hashlib.sha256(raw).hexdigest()
    if digest != CORPUS_SHA256:
        raise SystemExit("corpus changed: sha256 %s, expected %s" % (digest, CORPUS_SHA256))
    rows = []
    for line in raw.decode("utf-8").splitlines():
        p = line.split("\t")
        if len(p) == 2 and p[1].isdigit():
            rows.append((p[0], int(p[1])))
    return rows


def fetch_wikitext(words):
    out = {}
    for i in range(0, len(words), 50):
        batch = words[i:i + 50]
        url = (WIKT + "?action=query&prop=revisions&rvprop=content&rvslots=main"
               "&format=json&formatversion=2&titles=" + urllib.parse.quote("|".join(batch)))
        for attempt in range(5):
            try:
                d = json.loads(get(url))
                break
            except urllib.error.HTTPError as exc:
                if exc.code != 429 or attempt == 4:
                    raise SystemExit("wiktionary batch failed: %s" % exc)
                time.sleep(5 * (attempt + 1))
        for p in d.get("query", {}).get("pages", []):
            if "revisions" in p:
                out[p["title"]] = p["revisions"][0]["slots"]["main"]["content"]
        time.sleep(1.0)
    return out


def first_respelling(text):
    needle = "{{th-pron|"
    j = (text or "").find(needle)
    if j < 0:
        return None
    k, depth, buf = j + len(needle), 1, []
    while k < len(text):
        if text[k] == "{":
            depth += 1
        elif text[k] == "}":
            depth -= 1
            if depth == 0:
                break
        buf.append(text[k])
        k += 1
    value = "".join(buf).split("|")[0].strip()
    return value or None


def wikt_url(word):
    return "https://en.wiktionary.org/wiki/" + urllib.parse.quote(word)


def measure():
    rows = fetch_corpus()
    total_tokens = sum(f for _, f in rows)
    karan = [(i + 1, w, f) for i, (w, f) in enumerate(rows[:TOP_N]) if KARAN in w]
    last = [k for k in karan if k[1].endswith(KARAN)]
    candidates = [k for k in last if k[1][-2] in FINAL_CAPABLE]
    wt = fetch_wikitext(sorted({w for _, w, _ in candidates} | set(ARTICLE_WORDS)))

    measured, no_pron = [], []
    for rank, word, freq in candidates:
        resp = first_respelling(wt.get(word))
        res = reading(word, resp) if resp else None
        if res is None:
            no_pron.append(word)
            continue
        measured.append({"rank": rank, "word": word, "freq": freq, **res})

    changed = [m for m in measured if m["tone"] != m["naive_tone"]]
    karan_tokens = sum(f for _, _, f in karan)
    changed_tokens = sum(m["freq"] for m in changed)
    examples = []
    for w in ARTICLE_WORDS:
        resp = first_respelling(wt.get(w))
        examples.append({"word": w, **reading(w, resp)})

    metrics = {
        "corpus_tokens": total_tokens,
        "corpus_word_entries": len(rows),
        "top_n": TOP_N,
        "karan_words_top_n": len(karan),
        "karan_mark_on_last_letter": len(last),
        "karan_mark_on_last_letter_share": round(100.0 * len(last) / len(karan), 1),
        "karan_mark_not_on_last_letter": len(karan) - len(last),
        "karan_on_letter_that_can_close_a_syllable": len(candidates),
        "karan_tokens_top_n": karan_tokens,
        "measured_words": len(measured),
        "without_recorded_pronunciation": len(no_pron),
        "measured_tokens": sum(m["freq"] for m in measured),
        "tone_changes": len(changed),
        "tone_changes_share_of_measured": round(100.0 * len(changed) / len(measured), 1),
        "tone_changes_tokens": changed_tokens,
        "tone_changes_share_of_corpus": round(100.0 * changed_tokens / total_tokens, 2),
        "tone_changes_share_of_karan_tokens": round(100.0 * changed_tokens / karan_tokens, 1),
        "days_with_karan": sum(1 for d in DAYS if KARAN in d),
    }
    return metrics, measured, examples


def csv_rows(measured, examples):
    out = []
    for e in examples:
        out.append({"set": "article_example", "rank": "", "word": e["word"],
                    "corpus_frequency": "", "wiktionary_respelling": e["respelling"],
                    "tone_under_the_rule": TONE_NAME[e["tone"]],
                    "tone_if_marked_letter_read_as_final": TONE_NAME[e["naive_tone"]],
                    "changes_tone": "yes" if e["tone"] != e["naive_tone"] else "no",
                    "source": wikt_url(e["word"])})
    for m in measured:
        out.append({"set": "top5000", "rank": str(m["rank"]), "word": m["word"],
                    "corpus_frequency": str(m["freq"]), "wiktionary_respelling": m["respelling"],
                    "tone_under_the_rule": TONE_NAME[m["tone"]],
                    "tone_if_marked_letter_read_as_final": TONE_NAME[m["naive_tone"]],
                    "changes_tone": "yes" if m["tone"] != m["naive_tone"] else "no",
                    "source": wikt_url(m["word"])})
    return out


FIELDS = ["set", "rank", "word", "corpus_frequency", "wiktionary_respelling",
          "tone_under_the_rule", "tone_if_marked_letter_read_as_final", "changes_tone", "source"]


def write(dataset, metrics, rows):
    with open(TABLE, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=FIELDS, lineterminator="\n")
        w.writeheader()
        w.writerows(rows)
    dataset["results"]["measurement"] = metrics
    dataset["results"]["article_examples"] = [
        {k: r[k] for k in ("word", "wiktionary_respelling", "tone_under_the_rule",
                           "tone_if_marked_letter_read_as_final", "changes_tone", "source")}
        for r in rows if r["set"] == "article_example"]
    with open(DATASET, "w", encoding="utf-8") as f:
        json.dump(dataset, f, ensure_ascii=False, indent=2)
        f.write("\n")


def norm(s):
    s = html.unescape(re.sub(r"<[^>]+>", " ", s))
    return " ".join(unicodedata.normalize("NFKC", s).replace("’", "'").split()).lower()


def verify_quotes(dataset):
    ok = True
    for src in dataset.get("sources", []):
        try:
            body = get(src["url"], timeout=90).decode("utf-8", errors="replace")
        except (urllib.error.HTTPError, urllib.error.URLError) as exc:
            print("FAIL  %-40s not checked: %s" % (src["id"], exc))
            ok = False
            continue
        found = norm(src["quote"]) in norm(body)
        print("%s  %-40s %s" % ("ok  " if found else "FAIL", src["id"], src["url"]))
        ok = ok and found
    return ok


def compare(dataset, metrics, rows):
    ok = True
    stored = dataset.get("results", {}).get("measurement", {})
    for key in sorted(set(stored) | set(metrics)):
        if stored.get(key) != metrics.get(key):
            print("FAIL  %s: dataset %r, measured %r" % (key, stored.get(key), metrics.get(key)))
            ok = False
    if ok:
        print("ok    %d measured values match dataset.json" % len(metrics))
    try:
        with open(TABLE, newline="", encoding="utf-8") as f:
            have = list(csv.DictReader(f))
    except OSError:
        print("FAIL  dataset.csv missing")
        return False
    if have != rows:
        diff = sum(1 for a, b in zip(have, rows) if a != b) + abs(len(have) - len(rows))
        print("FAIL  dataset.csv: %d rows differ from the measurement" % diff)
        ok = False
    else:
        print("ok    %d dataset.csv rows reproduce" % len(rows))
    stored_ex = dataset.get("results", {}).get("article_examples", [])
    want_ex = [{k: r[k] for k in ("word", "wiktionary_respelling", "tone_under_the_rule",
                                  "tone_if_marked_letter_read_as_final", "changes_tone", "source")}
               for r in rows if r["set"] == "article_example"]
    if stored_ex != want_ex:
        print("FAIL  article examples in dataset.json differ from the measurement")
        ok = False
    else:
        print("ok    %d article examples match dataset.json" % len(want_ex))
    return ok


def main():
    assert_classes()
    with open(DATASET, encoding="utf-8") as f:
        dataset = json.load(f)
    mode = "write" if "--write" in sys.argv else ("recheck" if "--recheck" in sys.argv else "full")
    print("measuring (%s)" % mode)
    metrics, measured, examples = measure()
    for k, v in metrics.items():
        print("  %-44s %s" % (k, v))
    for e in examples:
        print("  %-12s %-16s %-8s naive %s" % (e["word"], e["respelling"],
                                               TONE_NAME[e["tone"]], TONE_NAME[e["naive_tone"]]))
    rows = csv_rows(measured, examples)
    if mode == "write":
        write(dataset, metrics, rows)
        print("wrote dataset.csv and dataset.json")
    ok = compare(dataset, metrics, rows)
    ok = verify_quotes(dataset) and ok
    print("\n%s" % ("all checks hold" if ok else "CHECKS FAILED"))
    return 0 if ok else 1


if __name__ == "__main__":
    sys.exit(main())
