#!/usr/bin/env python3
"""Re-runs every check behind dataset.json for the Phayan post `thai-words-without-spaces`.

Curriculum counts are read from the seed files of the repository this script sits in
(apps/api/src/content/seed/). The running-text figures join the curriculum lexicon to the
Thai National Corpus word list as PyThaiNLP packages it, downloaded and identified by
sha256. Every quoted sentence is looked up on its page, fetched now, or in its seed file.

    python3 check.py            measure everything, compare with dataset.json
    python3 check.py --recheck  the same; no step takes a minute, so none is skipped

Exit 0 when every row still holds, 1 otherwise. Standard library only.
"""

import argparse
import hashlib
import html
import json
import os
import re
import sys
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
REPO_ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", ".."))
SEED = os.path.join(REPO_ROOT, "apps", "api", "src", "content", "seed")
TNC_URL = ("https://raw.githubusercontent.com/PyThaiNLP/pythainlp/dev/"
           "pythainlp/corpus/tnc_freq.txt")
TNC_SHA = "554984d62e8735e2705bb30b06096d5710d0c9011e340ab717134e10d6dd0dda"
UA = "Mozilla/5.0"


# ---------------------------------------------------------------- text compare

def norm(s):
    s = html.unescape(s)
    for a, b in (("\u00a0", " "), ("\u202f", " "), ("\u2019", "'"), ("\u2018", "'"),
                 ("\u201c", '"'), ("\u201d", '"')):
        s = s.replace(a, b)
    s = re.sub(r"\s+", " ", s)
    s = re.sub(r"\s+([,.;:!?)\]}])", r"\1", s)
    s = re.sub(r"([(\[{])\s+", r"\1", s)
    return s.strip().lower()


HTML_TAG = re.compile(r"(?s)<[^>]+>")
HTML_BLOCK = re.compile(r"(?is)<(script|style)[^>]*>.*?</\1>")


def fetch_raw(url, timeout=60):
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return r.read()


def fetch_text(url):
    raw = fetch_raw(url).decode("utf-8", "replace")
    return html.unescape(HTML_TAG.sub(" ", HTML_BLOCK.sub(" ", raw)))


def seed_text(name):
    with open(os.path.join(SEED, name), encoding="utf-8") as f:
        return f.read()


# ------------------------------------------------------------- seed TS parsing

def parse_calls(src, fname):
    """Every `fname(...)` call in a TS seed file, split into positional args."""
    out, i, n, key = [], 0, len(src), fname + "("
    while True:
        j = src.find(key, i)
        if j == -1:
            break
        k = j + len(key)
        depth, args, cur, q = 1, [], [], None
        while k < n:
            c = src[k]
            if q:
                if c == "\\" and k + 1 < n:
                    cur.append(src[k + 1])
                    k += 2
                    continue
                if c == q:
                    q = None
                else:
                    cur.append(c)
                k += 1
                continue
            if c in "'\"":
                q = c
                k += 1
                continue
            if c == "(":
                depth += 1
            elif c == ")":
                depth -= 1
                if depth == 0:
                    args.append("".join(cur).strip())
                    k += 1
                    break
            if c == "," and depth == 1:
                args.append("".join(cur).strip())
                cur = []
            elif depth >= 1:
                cur.append(c)
            k += 1
        out.append(args)
        i = k
    return out


LEADING = set("เแโใไ")


def read_letters(src):
    rows = []
    for a in parse_calls(src, "letter"):
        if len(a) < 12:
            continue
        rows.append({
            "symbol": a[0], "class": a[5], "final": a[8],
            "obsolete": len(a) > 13 and a[13] == "true",
        })
    return rows


def read_vowels(src):
    rows = []
    for a in parse_calls(src, "form"):
        if len(a) < 10:
            continue
        rows.append({"symbol": a[0], "position": a[3]})
    return rows


def read_jsonl(path):
    with open(path, encoding="utf-8") as f:
        return [json.loads(l) for l in f if l.strip()]


# ---------------------------------------------------------------- measurements

class Measurements:
    def __init__(self):
        self._cache = {}

    def seed(self, name):
        if name not in self._cache:
            self._cache[name] = seed_text(name)
        return self._cache[name]

    def letters(self):
        return read_letters(self.seed("letters.ts"))

    def in_use(self):
        return [r for r in self.letters() if not r["obsolete"]]

    def with_final(self):
        return [r for r in self.in_use() if r["final"] != "null"]

    def words(self):
        if "words" not in self._cache:
            lex = read_jsonl(os.path.join(SEED, "kaikki-enriched.jsonl"))
            freq = read_jsonl(os.path.join(SEED, "word-frequency.jsonl"))
            fm = {r["word"]: r for r in freq}
            cov = [w for w in lex if fm.get(w["word"], {}).get("covered")]
            self._cache["words"] = (lex, fm, cov)
        return self._cache["words"]

    def tnc(self):
        if "tnc" not in self._cache:
            raw = fetch_raw(TNC_URL, timeout=90)
            got = hashlib.sha256(raw).hexdigest()
            if got != TNC_SHA:
                raise RuntimeError("tnc_freq.txt sha256 %s, expected %s" % (got, TNC_SHA))
            table = {}
            for line in raw.decode("utf-8").splitlines():
                parts = line.split("\t")
                if len(parts) == 2 and parts[1].isdigit():
                    table[parts[0]] = int(parts[1])
            self._cache["tnc"] = table
        return self._cache["tnc"]


def pct(num, den):
    return "%.1f" % (100.0 * num / den)


def build_checks(m):
    def letters_by_final():
        out = {}
        for r in m.with_final():
            out.setdefault(r["final"], []).append(r["symbol"])
        return out

    def words():
        return m.words()

    def covered():
        return words()[2]

    def freq():
        return words()[1]

    def one_syll():
        return [w for w in covered() if w["syllableCount"] == 1]

    def multi():
        return [w for w in covered() if w["syllableCount"] > 1]

    def tok(ws):
        f = freq()
        return sum(f[w["word"]]["frequency"] for w in ws)

    def cov_tokens():
        return sum(freq()[w["word"]]["frequency"] for w in covered())

    checks = {
        "letters-in-use": lambda: len(m.in_use()),
        "letters-with-final": lambda: len(m.with_final()),
        "letters-without-final": lambda: ", ".join(r["symbol"] for r in m.in_use() if r["final"] == "null"),
        "final-sounds-distinct": lambda: len(letters_by_final()),
        "final-t-letters": lambda: len(letters_by_final()["t"]),
        "final-t-list": lambda: " ".join(letters_by_final()["t"]),
        "final-k-letters": lambda: len(letters_by_final()["k"]),
        "final-p-letters": lambda: len(letters_by_final()["p"]),
        "final-other-letters": lambda: sum(len(v) for k, v in letters_by_final().items() if k not in ("t", "k", "p")),
        "final-other-sounds": lambda: len([k for k in letters_by_final() if k not in ("t", "k", "p")]),
        "vowel-forms-total": lambda: len(read_vowels(m.seed("vowel-forms.ts"))),
        "vowel-before": lambda: sum(1 for v in read_vowels(m.seed("vowel-forms.ts")) if v["position"] == "before"),
        "vowel-after": lambda: sum(1 for v in read_vowels(m.seed("vowel-forms.ts")) if v["position"] == "after"),
        "vowel-above": lambda: sum(1 for v in read_vowels(m.seed("vowel-forms.ts")) if v["position"] == "above"),
        "vowel-below": lambda: sum(1 for v in read_vowels(m.seed("vowel-forms.ts")) if v["position"] == "below"),
        "vowel-split": lambda: sum(1 for v in read_vowels(m.seed("vowel-forms.ts")) if v["position"] == "split"),
        "vowel-split-leading": lambda: sum(
            1 for v in read_vowels(m.seed("vowel-forms.ts"))
            if v["position"] == "split" and v["symbol"][0] in LEADING),
        "vowel-leading-total": lambda: (
            sum(1 for v in read_vowels(m.seed("vowel-forms.ts")) if v["position"] == "before")
            + sum(1 for v in read_vowels(m.seed("vowel-forms.ts"))
                  if v["position"] == "split" and v["symbol"][0] in LEADING)),
        "vowel-leading-share": lambda: pct(
            sum(1 for v in read_vowels(m.seed("vowel-forms.ts")) if v["position"] == "before")
            + sum(1 for v in read_vowels(m.seed("vowel-forms.ts"))
                  if v["position"] == "split" and v["symbol"][0] in LEADING),
            len(read_vowels(m.seed("vowel-forms.ts")))),
        "words-covered": lambda: len(covered()),
        "words-lexicon": lambda: len(words()[0]),
        "one-syll-count": lambda: len(one_syll()),
        "one-syll-share-count": lambda: pct(len(one_syll()), len(covered())),
        "one-syll-tokens": lambda: tok(one_syll()),
        "one-syll-share-tokens": lambda: pct(tok(one_syll()), cov_tokens()),
        "leading-vowel-words": lambda: len([w for w in covered() if w["word"][0] in LEADING]),
        "leading-vowel-share-count": lambda: pct(len([w for w in covered() if w["word"][0] in LEADING]), len(covered())),
        "leading-vowel-tokens": lambda: tok([w for w in covered() if w["word"][0] in LEADING]),
        "leading-vowel-share-tokens": lambda: pct(tok([w for w in covered() if w["word"][0] in LEADING]), cov_tokens()),
        "multi-syllable-words": lambda: len(multi()),
        "multi-syllable-words-all": lambda: len([w for w in words()[0] if w["syllableCount"] > 1]),
        "multi-with-internal-leading": lambda: len([w for w in multi() if any(c in LEADING for c in w["word"][1:])]),
        "multi-internal-share": lambda: pct(len([w for w in multi() if any(c in LEADING for c in w["word"][1:])]), len(multi())),
        "tokens-covered": lambda: cov_tokens(),
        "tnc-tokens-total": lambda: sum(m.tnc().values()),
        "tnc-coverage-share": lambda: pct(cov_tokens(), sum(m.tnc().values())),
        "tnc-rows": lambda: len(m.tnc()),
        "multi-syllable-at-level-8": lambda: len([w for w in words()[0] if w["syllableCount"] > 1 and w["introducedLevel"] == 8]),
        "tnc-counts-match": lambda: (
            "yes" if all(m.tnc().get(w["word"]) == freq()[w["word"]]["frequency"] for w in covered()) else "no"),
    }
    return checks


# ----------------------------------------------------------------------- main

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--recheck", action="store_true",
                    help="re-read every cited page; every step is under a minute, so nothing is skipped")
    ap.parse_args()

    dataset = json.load(open(os.path.join(HERE, "dataset.json"), encoding="utf-8"))
    m = Measurements()
    checks = build_checks(m)
    pages = {}

    failures = []
    for row in dataset["results"]:
        kind, rid = row.get("kind"), row["id"]
        if kind == "computed":
            try:
                got = checks[rid]()
            except Exception as exc:
                failures.append((rid, "compute error: %s" % exc))
                print("FAIL  %-34s compute error: %s" % (rid, exc))
                continue
            ok = str(got) == str(row["value"])
            print("%s  %-34s %s%s" % ("ok  " if ok else "FAIL", rid, got,
                                      "" if ok else "  (dataset says %s)" % row["value"]))
            if not ok:
                failures.append((rid, "got %s, dataset says %s" % (got, row["value"])))
        elif kind in ("quote", "file_quote"):
            where = row["url"] if kind == "quote" else row["file"]
            try:
                if where not in pages:
                    pages[where] = norm(fetch_text(where) if kind == "quote"
                                        else open(os.path.join(REPO_ROOT, where), encoding="utf-8").read())
                ok = norm(row["quote"]) in pages[where]
                why = "quote not found in " + where
            except Exception as exc:
                ok, why = False, "could not read %s: %s" % (where, exc)
            print("%s  %-34s %s" % ("ok  " if ok else "FAIL", rid, where))
            if not ok:
                failures.append((rid, why))
        else:
            print("WARN  %-34s unknown kind %r" % (rid, kind))
            failures.append((rid, "unknown kind"))

    print("\n%d rows, %d failed" % (len(dataset["results"]), len(failures)))
    for rid, why in failures:
        print("  %s: %s" % (rid, why))
    return 1 if failures else 0


if __name__ == "__main__":
    sys.exit(main())
