#!/usr/bin/env python3
"""Re-runs the check behind the Phayan post "Thai Names and Nicknames: How to Read Both".

Over the 5,000 most frequent words of the Thai National Corpus frequency list (CC0-1.0,
pinned by sha256), it counts the words whose last letter is one of the five quiet or
changed endings in the post's table, weights each word by its corpus token count, and
reports the share of running text those words cover. It also measures how much of the
consonant mass in the same words the rare Sanskrit letters carry, and re-reads every
page the post cites for a third-party fact to confirm the quoted sentence is still there.

python3 check.py            compare every derived number with dataset.json, check quotes
python3 check.py --recheck  the same; the corpus step takes seconds, so nothing is skipped
python3 check.py --write    rewrite the derived parts of dataset.json and dataset.csv
Exit 0 when everything holds, 1 otherwise. Standard library only.
"""

import argparse
import csv
import hashlib
import html
import json
import os
import re
import sys
import time
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
CSV_PATH = os.path.join(HERE, "dataset.csv")

TNC_URL = "https://raw.githubusercontent.com/PyThaiNLP/pythainlp/v5.4.0/pythainlp/corpus/tnc_freq.txt"
TNC_SHA256 = "554984d62e8735e2705bb30b06096d5710d0c9011e340ab717134e10d6dd0dda"
TOP_N = 5000

UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/126.0 Safari/537.36")

KARAN = "์"
LEAD = "เแโใไ"
CONS = [chr(c) for c in range(0x0E01, 0x0E2F) if chr(c) not in "ฤฦฯ"]
CONS_SET = set(CONS)
SHORT_MARKS = set("ัิุู")
CLUSTER2 = {"ร": set("กขคปพผตบดฟ"), "ล": set("กขคปพผบฟ")}
# Hand-read words the spelling rules below get wrong: a ร silent after a closed
# syllable, or a bound prefix whose ร is read "ra" inside compounds.
NOT_CLOSING = {"เพชร", "เมตร", "กิโลเมตร", "เซนติเมตร", "เนตร", "ทอดพระเนตร", "เกษตร",
               "โทร", "สุร", "นิร", "จิร", "ธีร"}


def closes(word, letters):
    """True when the word's last letter is in `letters` and closes the last syllable."""
    x = word[-1:]
    if x not in letters or len(word) < 2 or word in NOT_CLOSING:
        return False
    a = word[-2]
    if a in LEAD or a == "ะ":                      # ไร โต แล: the letter starts a syllable
        return False
    if (a in CONS_SET and len(word) >= 3 and word[-3] in LEAD
            and (a == "ห" or a in CLUSTER2.get(x, ()))):
        return False                               # ใคร ไกล ไหล: second letter of a cluster
    if x == "ร" and a in CONS_SET and a not in "ยวอ" and len(word) >= 3:
        if word[-3] in SHORT_MARKS or (word[-3] == "า" and a == "ต"):
            return False                           # บัตร จักร บาตร: the ร is silent
    return True


PATTERNS = [
    ("silencing-mark", "์ over the last letter", "last character is ์ (U+0E4C)",
     lambda w: w.endswith(KARAN)),
    ("final-ro", "ร closing a syllable", "last letter ร, closing the last syllable",
     lambda w: closes(w, "ร")),
    ("final-lo", "ล closing a syllable", "last letter ล, closing the last syllable",
     lambda w: closes(w, "ล")),
    ("final-t-group", "ฐ ธ ศ ษ ต closing a syllable",
     "last letter one of ฐ ธ ศ ษ ต, closing the last syllable",
     lambda w: closes(w, "ฐธศษต")),
    ("tho-ro-karan", "ทร์ closing a syllable", "word ends in ทร์",
     lambda w: w.endswith("ทร" + KARAN)),
]
RARE_GROUP = "ฎฏฑฒฬฌ"
LETTERS = "ฎฏฐฑฒณฬฌษศ"


def fetch(url, timeout=45, attempts=3):
    last = None
    for n in range(attempts):
        try:
            req = urllib.request.Request(url, headers={"User-Agent": UA})
            with urllib.request.urlopen(req, timeout=timeout) as r:
                return r.read()
        except urllib.error.HTTPError as exc:
            last = exc
            if exc.code < 500:
                raise
        except (urllib.error.URLError, OSError) as exc:
            last = exc
        if n + 1 < attempts:
            time.sleep(2 * (n + 1))
    raise last


def norm(s):
    s = html.unescape(s)
    s = re.sub(r"(?is)<(script|style|noscript)\b[^>]*>.*?</\1>", " ", s)
    s = re.sub(r"(?s)<[^>]+>", " ", s)
    s = s.replace(" ", " ").replace("​", "")
    s = s.replace("’", "'").replace("‘", "'").replace("“", '"').replace("”", '"')
    s = re.sub(r"\[\s*\d+\s*\]", "", s)
    s = re.sub(r"\s+", " ", s)
    s = re.sub(r"\s+([,.;:!?)])", r"\1", s)
    s = re.sub(r"\(\s+", "(", s)
    return s.strip()


def pct(x, total):
    return round(100 * x / total, 4)


def measure():
    raw = fetch(TNC_URL)
    sha = hashlib.sha256(raw).hexdigest()
    rows = []
    for line in raw.decode("utf-8").splitlines():
        if line.strip():
            w, f = line.split("\t")[:2]
            rows.append((w, int(f)))
    corpus = sum(f for _, f in rows)
    top = rows[:TOP_N]
    top_tokens = sum(f for _, f in top)

    patterns, union = [], {}
    for key, label, rule, test in PATTERNS:
        hit = [(w, f) for w, f in top if test(w)]
        union.update(hit)
        tok = sum(f for _, f in hit)
        patterns.append({
            "item": key, "label": label, "rule": rule, "words": len(hit), "tokens": tok,
            "share_of_top5000_pct": pct(tok, top_tokens), "share_of_corpus_pct": pct(tok, corpus),
            "commonest": " ".join(w for w, _ in hit[:5]),
        })
    utok = sum(union.values())
    patterns.append({
        "item": "any-of-the-five", "label": "any of the five", "rule": "union of the five rules",
        "words": len(union), "tokens": utok,
        "share_of_top5000_pct": pct(utok, top_tokens), "share_of_corpus_pct": pct(utok, corpus),
        "commonest": "",
    })

    counts = {c: 0 for c in CONS}
    for w, f in top:
        for ch in w:
            if ch in counts:
                counts[ch] += f
    cons_total = sum(counts.values())
    letters = [{"item": "letters-" + RARE_GROUP, "letters": " ".join(RARE_GROUP),
                "tokens": sum(counts[c] for c in RARE_GROUP),
                "share_of_consonants_pct": pct(sum(counts[c] for c in RARE_GROUP), cons_total)}]
    for c in LETTERS:
        letters.append({"item": "letter-" + c, "letters": c, "tokens": counts[c],
                        "share_of_consonants_pct": pct(counts[c], cons_total)})

    totals = {"tnc_sha256": sha, "corpus_entries": len(rows), "corpus_tokens": corpus,
              "top5000_tokens": top_tokens, "top5000_consonant_tokens": cons_total}
    return totals, patterns, letters


def write_csv(patterns, letters):
    with open(CSV_PATH, "w", newline="", encoding="utf-8") as fh:
        w = csv.writer(fh, lineterminator="\n")
        w.writerow(["kind", "item", "label", "words_in_top5000", "tokens",
                    "share_of_top5000_pct", "share_of_corpus_pct", "share_of_consonants_pct",
                    "commonest_words"])
        for p in patterns:
            w.writerow(["ending", p["item"], p["label"], p["words"], p["tokens"],
                        p["share_of_top5000_pct"], p["share_of_corpus_pct"], "", p["commonest"]])
        for l in letters:
            w.writerow(["consonant", l["item"], l["letters"], "", l["tokens"], "", "",
                        l["share_of_consonants_pct"], ""])


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--recheck", action="store_true", help="re-read every cited page")
    ap.add_argument("--write", action="store_true", help="rewrite derived results")
    args = ap.parse_args()

    failures = []
    totals, patterns, letters = measure()
    if totals["tnc_sha256"] != TNC_SHA256:
        failures.append("TNC list sha256 %s, expected %s" % (totals["tnc_sha256"], TNC_SHA256))

    data = json.load(open(DATASET, encoding="utf-8"))
    if args.write:
        data["results"]["totals"] = totals
        data["results"]["endings"] = patterns
        data["results"]["consonants"] = letters
        with open(DATASET, "w", encoding="utf-8") as fh:
            json.dump(data, fh, ensure_ascii=False, indent=2)
            fh.write("\n")
        write_csv(patterns, letters)

    if data["results"].get("totals") != totals:
        failures.append("totals differ: derived %r, dataset %r" % (totals, data["results"].get("totals")))
    if data["results"].get("endings") != patterns:
        failures.append("ending rows differ from dataset.json")
    if data["results"].get("consonants") != letters:
        failures.append("consonant rows differ from dataset.json")
    with open(CSV_PATH, encoding="utf-8") as fh:
        stored = list(csv.reader(fh))
    by_item = {r[1]: r for r in stored[1:]}
    for p in patterns:
        r = by_item.get(p["item"])
        if not r or [r[3], r[4], r[5], r[6]] != [str(p["words"]), str(p["tokens"]),
                                                 str(p["share_of_top5000_pct"]), str(p["share_of_corpus_pct"])]:
            failures.append("dataset.csv row %s differs" % p["item"])
    for l in letters:
        r = by_item.get(l["item"])
        if not r or [r[4], r[7]] != [str(l["tokens"]), str(l["share_of_consonants_pct"])]:
            failures.append("dataset.csv row %s differs" % l["item"])

    print("corpus %d tokens over %d entries; top %d words carry %d tokens"
          % (totals["corpus_tokens"], totals["corpus_entries"], TOP_N, totals["top5000_tokens"]))
    for p in patterns:
        print("  %-30s words=%-4d share of running text %.4f%%  (%s)"
              % (p["label"], p["words"], p["share_of_corpus_pct"], p["commonest"]))
    for l in letters:
        print("  consonants %-12s %.4f%% of consonant tokens" % (l["letters"], l["share_of_consonants_pct"]))

    for page in data["results"]["quotes"]:
        try:
            text = norm(fetch(page["url"]).decode("utf-8", "replace"))
        except (urllib.error.URLError, urllib.error.HTTPError, OSError) as exc:
            failures.append("%s: could not fetch %s (%s)" % (page["id"], page["url"], exc))
            print("  quote %-22s NOT FETCHED" % page["id"])
            continue
        ok = norm(page["quote"]).lower() in text.lower()
        print("  quote %-22s %s" % (page["id"], "present" if ok else "GONE"))
        if not ok:
            failures.append("%s: quote gone from %s" % (page["id"], page["url"]))

    if failures:
        print("\nFAIL")
        for f in failures:
            print("  - " + f)
        return 1
    print("\nOK")
    return 0


if __name__ == "__main__":
    sys.exit(main())