#!/usr/bin/env python3
"""Re-runs the public-corpus check behind the post `thai-transliteration`.

Downloads the kaikki.org Thai extract (a rendering of English Wiktionary), rebuilds the
frame of Thai headwords that carry both a Royal Institute (RTGS) and a Paiboon
romanisation, re-measures every row in dataset.json, re-reads every page the post quotes
and checks that each quoted sentence is still on it, confirms the product claims still
appear in the repository, and confirms dataset.csv carries the same figures.

    python3 check.py            full run
    python3 check.py --recheck  the same run; no step here is skipped

Exit 0 when every row holds, 1 otherwise. Standard library only.
"""

import argparse
import collections
import csv
import html
import json
import os
import re
import subprocess
import sys
import unicodedata
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
CSV_PATH = os.path.join(HERE, "dataset.csv")
REPO = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", ".."))
UA = "Mozilla/5.0"
THAI = re.compile(r"[฀-๿]")
VOWEL = re.compile(r"([aeiouɛɔə])\1", re.I)


# ------------------------------------------------------------------ text compare

def norm(s):
    s = html.unescape(s)
    s = unicodedata.normalize("NFC", s)
    for a, b in ((" ", " "), ("’", "'"), ("‘", "'"),
                 ("“", '"'), ("”", '"'), ("–", "-"), ("—", "-")):
        s = s.replace(a, b)
    s = re.sub(r"\s+", " ", s)
    s = re.sub(r"\s+([,.;:!?)\]])", r"\1", s)
    s = re.sub(r"([(\[])\s+", r"\1", s)
    return s.strip().lower()


HTML_TAG = re.compile(r"(?s)<[^>]+>")
HTML_BLOCK = re.compile(r"(?is)<(script|style)[^>]*>.*?</\1>")


def extract_text(kind, raw):
    """Page bytes -> the text the quote match runs on."""
    if kind == "pdf":
        tmp = os.path.join(HERE, ".recheck.pdf")
        with open(tmp, "wb") as f:
            f.write(raw)
        try:
            out = subprocess.run(["pdftotext", tmp, "-"], capture_output=True, check=True)
            return out.stdout.decode("utf-8", "replace")
        finally:
            os.remove(tmp)
    text = raw.decode("utf-8", "replace")
    if kind == "json":
        d = json.loads(text)
        r = d["results"][0]
        return (r.get("description") or "") + "\n" + (r.get("releaseNotes") or "")
    return html.unescape(HTML_TAG.sub(" ", HTML_BLOCK.sub(" ", text)))


def fetch(url):
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    with urllib.request.urlopen(req, timeout=180) as r:
        return r.read()


# ------------------------------------------------------------------ the corpus

def build_frame(raw_bytes):
    """Thai headwords carrying both a Paiboon and a Royal Institute romanisation."""
    words, order = {}, []
    for ln in raw_bytes.decode("utf-8", "replace").splitlines():
        ln = ln.strip()
        if not ln:
            continue
        d = json.loads(ln)
        w = d.get("word") or ""
        if not THAI.search(w) or " " in w:
            continue
        pai, ri = [], []
        for s in d.get("sounds", []):
            if "romanization" in (s.get("tags") or []):
                rt = s.get("raw_tags") or []
                if "Paiboon" in rt and s.get("roman"):
                    pai.append(s["roman"])
                if "Royal Institute" in rt and s.get("roman"):
                    ri.append(s["roman"])
        senses = d.get("senses") or []
        variant = bool(senses) and all(("alt_of" in sg or "form_of" in sg) for sg in senses)
        e = words.get(w)
        if e is None:
            e = words[w] = {"pai": None, "ri": None, "variant": True, "gloss": ""}
            order.append(w)
        if e["pai"] is None and pai:
            e["pai"] = pai[0]
        if e["ri"] is None and ri:
            e["ri"] = ri[0]
        e["variant"] = e["variant"] and variant
        if not e["gloss"] and senses:
            e["gloss"] = "; ".join(senses[0].get("glosses") or [])
    return [(w, words[w]) for w in order
            if words[w]["pai"] and words[w]["ri"] and not words[w]["variant"]]


def strip_tones(s):
    return "".join(c for c in unicodedata.normalize("NFD", s)
                   if not unicodedata.combining(c))


SCHEMES = collections.OrderedDict([
    ("RTGS (Royal Institute)", lambda e: e["ri"]),
    ("Paiboon without tone marks", lambda e: strip_tones(e["pai"])),
    ("Paiboon", lambda e: e["pai"]),
])


def measure(frame, fn):
    idx = collections.defaultdict(list)
    marked = doubled = 0
    for w, e in frame:
        s = fn(e)
        idx[s].append(w)
        if any(unicodedata.combining(c) for c in unicodedata.normalize("NFD", s)):
            marked += 1
        # a tone mark can sit on the first of two vowel letters, so the doubling
        # test runs on the string with every combining mark removed
        if VOWEL.search(strip_tones(s)):
            doubled += 1
    shared = {k: v for k, v in idx.items() if len(v) > 1}
    affected = set()
    for v in shared.values():
        affected.update(v)
    worst_n = max((len(v) for v in shared.values()), default=0)
    worst = sorted(k for k, v in shared.items() if len(v) == worst_n)
    return {
        "headwords": len(frame),
        "tone_marked_spellings": marked,
        "doubled_vowel_spellings": doubled,
        "distinct": len(idx),
        "shared": len(shared),
        "shared_pct": round(len(shared) / len(idx) * 100, 2),
        "affected": len(affected),
        "affected_pct": round(len(affected) / len(frame) * 100, 2),
        "worst_strings": worst,
        "worst_words": worst_n,
    }


def measure_line(option, m):
    return ("{o}: headwords={h} tone_marked={tm} doubled_vowel={dv} distinct={d} "
            "shared={s} ({sp}%) affected={a} ({ap}%) worst={w} ({n})").format(
                o=option, h=m["headwords"], tm=m["tone_marked_spellings"],
                dv=m["doubled_vowel_spellings"], d=m["distinct"], s=m["shared"],
                sp=m["shared_pct"], a=m["affected"], ap=m["affected_pct"],
                w=",".join(m["worst_strings"]), n=m["worst_words"])


# ------------------------------------------------------------------ the checks

def check_corpus(ds, frame, failures):
    for row in ds["results"]:
        option = row["option"]
        m = measure(frame, SCHEMES[option])
        line = measure_line(option, m)
        for prop in ("headwords", "tone_marked_spellings", "doubled_vowel_spellings",
                     "distinct", "shared", "shared_pct", "affected", "affected_pct",
                     "worst_words"):
            if m[prop] != row[prop]:
                failures.append("%s %s: dataset %r, measured %r"
                                % (option, prop, row[prop], m[prop]))
        if m["worst_strings"] != row["worst_strings"]:
            failures.append("%s worst_strings: dataset %r, measured %r"
                            % (option, row["worst_strings"], m["worst_strings"]))
        if row.get("quote") != line:
            failures.append("%s quote out of date: %r" % (option, row.get("quote")))
        else:
            print("ok    %s" % line)


def check_derived(ds, frame, failures):
    """The sums and point differences the post states, recomputed."""
    counts = collections.Counter(e["ri"] for _, e in frame)
    top = [n for _, n in counts.most_common(20)]
    got = {"top20_headwords": sum(top), "top10_headwords": sum(top[:10])}
    rows = {r["option"]: r for r in ds["results"]}
    gap = round(rows["RTGS (Royal Institute)"]["affected_pct"]
                - rows["Paiboon"]["affected_pct"], 2)
    tone = round(rows["Paiboon without tone marks"]["affected_pct"]
                 - rows["Paiboon"]["affected_pct"], 2)
    got["rtgs_to_paiboon_points"] = gap
    got["tone_points"] = tone
    got["vowel_length_points"] = round(gap - tone, 2)
    for key, want in ds["derived"].items():
        if got.get(key) != want:
            failures.append("derived %s: dataset %r, measured %r" % (key, want, got.get(key)))
    print("ok    derived: %s" % ", ".join("%s=%s" % (k, got[k]) for k in sorted(got)))


def check_examples(ds, frame, failures):
    """The strings and words the post names, recomputed from the frame."""
    by = {"ri": collections.defaultdict(list), "pai": collections.defaultdict(list)}
    for w, e in frame:
        by["ri"][e["ri"]].append((w, e["pai"]))
        by["pai"][e["pai"]].append((w, e["ri"]))
    for ex in ds["examples"]:
        hits = by[ex["scheme"]][ex["string"]]
        if len(hits) != ex["count"]:
            failures.append("example %s/%s: dataset %d words, measured %d"
                            % (ex["scheme"], ex["string"], ex["count"], len(hits)))
            continue
        words = {w for w, _ in hits}
        missing = [w for w in ex["words"] if w not in words]
        if missing:
            failures.append("example %s/%s: words not under it: %s"
                            % (ex["scheme"], ex["string"], ", ".join(missing)))
            continue
        print("ok    example %s/%s (%d words)" % (ex["scheme"], ex["string"], ex["count"]))

        # the romanisation the post prints for a word must be the frame's form
        for pair in ex.get("forms", []):
            if pair["thai"] not in words:
                failures.append("example form: %s is not under %s"
                                % (pair["thai"], ex["string"]))
                continue
            if pair["thai"] not in ex["words"]:
                failures.append("example form: %s is not in the dataset's word list"
                                % pair["thai"])
                continue
            forms = dict(hits)
            if forms.get(pair["thai"]) != pair["roman"]:
                failures.append("example form: %s reads %r, the frame has %r"
                                % (pair["thai"], pair["roman"], forms.get(pair["thai"])))
        print("ok    example forms for %s/%s" % (ex["scheme"], ex["string"]))


def check_pages(ds, failures):
    cache = {}
    for src in ds["inputs"]["sources"]:
        try:
            raw = fetch(src["url"])
        except Exception as exc:  # noqa: BLE001 - the reason is the report
            failures.append("%s: could not read %s (%s)" % (src["name"], src["url"], exc))
            continue
        try:
            cache[src["name"]] = norm(extract_text(src["kind"], raw))
        except Exception as exc:  # noqa: BLE001
            failures.append("%s: could not extract text from %s (%s)"
                            % (src["name"], src["url"], exc))
    for claim in ds["third_party"]:
        for q in claim["quotes"]:
            hay = cache.get(q["name"])
            if hay is None:
                continue
            if norm(q["text"]) in hay:
                print("ok    %s: %s" % (q["name"], q["text"][:70]))
            else:
                url = next(s["url"] for s in ds["inputs"]["sources"]
                           if s["name"] == q["name"])
                failures.append("%s: quote gone from %s: %r"
                                % (q["name"], url, q["text"]))


def check_product(ds, failures):
    for claim in ds["product_claims"]:
        path = os.path.join(REPO, claim["path"])
        try:
            body = open(path, encoding="utf-8").read()
        except OSError:
            failures.append("product claim: cannot read %s" % claim["path"])
            continue
        if claim["quote"] in body:
            print("ok    %s: %s" % (claim["path"], claim["quote"][:60]))
        else:
            failures.append("product claim gone from %s: %r"
                            % (claim["path"], claim["quote"]))


def check_csv(ds, failures):
    with open(CSV_PATH, newline="", encoding="utf-8") as fh:
        rows = {r["option"]: r for r in csv.DictReader(fh)}
    fields = ("headwords", "tone_marked_spellings", "doubled_vowel_spellings",
              "distinct_spellings", "shared_spellings", "shared_pct",
              "words_affected", "affected_pct", "worst_words")
    rename = {"distinct_spellings": "distinct", "shared_spellings": "shared",
              "words_affected": "affected"}
    for r in ds["results"]:
        row = rows.get(r["option"])
        if row is None:
            failures.append("dataset.csv: no row for %s" % r["option"])
            continue
        for col in fields:
            key = rename.get(col, col)
            want = r[key]
            got = row[col]
            if isinstance(want, float):
                if abs(float(got) - want) > 0.001:
                    failures.append("dataset.csv %s %s: %r vs %r"
                                    % (r["option"], col, got, want))
            elif str(want) != got:
                failures.append("dataset.csv %s %s: %r vs %r"
                                % (r["option"], col, got, want))
        want_worst = " / ".join(r["worst_strings"])
        if row["worst_strings"] != want_worst:
            failures.append("dataset.csv %s worst_strings: %r vs %r"
                            % (r["option"], row["worst_strings"], want_worst))
    print("ok    dataset.csv matches dataset.json (%d rows)" % len(rows))


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--recheck", action="store_true",
                    help="re-read every page and the corpus (this check always does)")
    ap.parse_args()
    ds = json.load(open(DATASET, encoding="utf-8"))
    failures = []
    try:
        raw = fetch(ds["inputs"]["corpus"]["url"])
    except Exception as exc:  # noqa: BLE001
        print("FAIL  corpus: %s" % exc)
        return 1
    print("ok    corpus downloaded (%d bytes)" % len(raw))
    frame = build_frame(raw)
    print("ok    frame rebuilt: %d headwords" % len(frame))
    check_corpus(ds, frame, failures)
    check_derived(ds, frame, failures)
    check_examples(ds, frame, failures)
    check_pages(ds, failures)
    check_product(ds, failures)
    check_csv(ds, failures)
    if failures:
        print("\nFAILED (%d)" % len(failures))
        for f in failures:
            print("  - " + f)
        return 1
    print("\nall rows hold")
    return 0


if __name__ == "__main__":
    sys.exit(main())