#!/usr/bin/env python3
"""Re-run the corpus check behind phayan.com/blog/thai-consonant-clusters.

Downloads the OpenSubtitles 2018 Thai frequency list (hermitdave/FrequencyWords),
verifies its hash, and counts, for each of the 18 consonant pairs in the post's
two tables, how many word forms open with the pair and what share of running
text they carry.  Compares every number with dataset.json.

    python3 check.py            verify (uses a cached copy of the list if present)
    python3 check.py --recheck  verify against a fresh download of the list
    python3 check.py --write    rewrite dataset.json and dataset.csv from a measurement

Exit 0 when every number still holds, 1 otherwise.  Standard library only.
"""

import argparse
import csv
import hashlib
import json
import os
import sys
import tempfile
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
CSV_PATH = os.path.join(HERE, "dataset.csv")
CORPUS_URL = (
    "https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/"
    "content/2018/th/th_50k.txt"
)
CORPUS_SHA256 = "20e7052f2d64222e1420c5d0b4ed6b68cd6290f0cf8b908d8bc6b0af781b6083"
CHECKED_AT = "2026-10-26"

PAIRS = [
    ("กร", "true"), ("กล", "true"), ("กว", "true"), ("ขล", "true"),
    ("ขว", "true"), ("คร", "true"), ("คล", "true"), ("คว", "true"),
    ("ตร", "true"), ("ปร", "true"), ("ปล", "true"), ("ผล", "true"),
    ("พร", "true"), ("พล", "true"),
    ("ทร", "false"), ("จร", "false"), ("ศร", "false"), ("สร", "false"),
]
# Pairs the post names in prose but keeps out of the two tables.
EXTRA_PAIRS = ["ขร", "ฟร", "ดร", "บล"]
# Example words the post's text relies on.
PROBE_WORDS = ["ครับ", "จริง", "จริงๆ", "จริงเหรอ", "ปลา", "ทราบ", "ทราย", "แทรก",
               "ทรอย", "ทราวิส", "ถนน", "ขรุขระ", "ขลาด"]

CONSONANTS = set("กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ")
PREVOWELS = set("เแโใไ")


def is_thai_token(tok):
    return bool(tok) and all("฀" <= c <= "๿" for c in tok)


def get_corpus(fresh):
    cache = os.path.join(tempfile.gettempdir(), "phayan-clusters-th_50k.txt")
    if fresh or not os.path.exists(cache):
        req = urllib.request.Request(CORPUS_URL, headers={"User-Agent": "Mozilla/5.0"})
        with urllib.request.urlopen(req, timeout=60) as resp:
            data = resp.read()
        with open(cache, "wb") as fh:
            fh.write(data)
    with open(cache, "rb") as fh:
        data = fh.read()
    rows = []
    for line in data.decode("utf-8").splitlines():
        tok, _, count = line.rpartition(" ")
        if tok and count.isdigit():
            rows.append((tok, int(count)))
    return rows, hashlib.sha256(data).hexdigest()


def initial_pair(tok):
    # A substring search would score ตกลง under กล and บอกว่า under กว.
    i = 0
    while i < len(tok) and tok[i] in PREVOWELS:
        i += 1
    if i + 1 < len(tok) and tok[i] in CONSONANTS and tok[i + 1] in CONSONANTS:
        return tok[i:i + 2]
    return None


def pct(part, whole):
    return round(100.0 * part / whole, 4) if whole else 0.0


def measure(rows):
    thai = [(t, c) for t, c in rows if is_thai_token(t)]
    total = sum(c for _, c in thai)
    counts = dict(thai)

    def pair_row(pair, group):
        hits = sorted(((t, c) for t, c in thai if initial_pair(t) == pair), key=lambda x: -x[1])
        tokens = sum(c for _, c in hits)
        return {
            "pair": pair,
            "group": group,
            "words": len(hits),
            "tokens": tokens,
            "share_pct": pct(tokens, total),
            "top_words": [{"word": w, "tokens": c} for w, c in hits[:3]],
        }

    results = sorted((pair_row(p, g + "-cluster") for p, g in PAIRS), key=lambda r: -r["tokens"])
    extras = [pair_row(p, "outside-tables") for p in EXTRA_PAIRS]
    table_tokens = sum(r["tokens"] for r in results)
    top3 = results[:3]
    return {
        "corpus_totals": {"raw_lines": len(rows), "thai_forms": len(thai), "thai_tokens": total},
        "results": results,
        "extras": extras,
        "probe_words": [{"word": w, "tokens": counts.get(w, 0)} for w in PROBE_WORDS],
        "summary": {
            "table_pairs_tokens": table_tokens,
            "table_pairs_share_of_text_pct": pct(table_tokens, total),
            "top3_pairs": [r["pair"] for r in top3],
            "top3_share_of_table_pct": round(100.0 * sum(r["tokens"] for r in top3) / table_tokens, 1),
            "krap_share_of_table_pct": round(100.0 * counts.get("ครับ", 0) / table_tokens, 1),
            "jing_three_forms_tokens": sum(counts.get(w, 0) for w in ("จริง", "จริงๆ", "จริงเหรอ")),
            "pairs_with_no_word": [r["pair"] for r in results + extras if r["words"] == 0],
        },
    }


NOTES = [
    "A word form counts for a pair when the pair is its first two consonants after any pre-posed vowel (เ แ โ ใ ไ) is stripped, so แทรก counts for ทร and ใคร for คร.",
    "The counts are of spellings, not pronunciations. Some forms that open with a listed pair do not say it as a cluster: กรุณา (ka-ru-naa), ขวบ (khuap, where ว is the vowel), ผล (phon), สรุป (sa-rup), and โทรศัพท์ (thoo-ra-sap, 236 of the ทร tokens). The table ranks how much text each spelling opens; it does not certify that every counted word is a cluster.",
    "The list is a subtitle token list: Latin-script and mixed tokens are dropped (50 000 lines to the Thai-only forms in corpus_totals), some multi-word strings count as one token, and transliterated names (ทรอย, แคลร์, เควิน) are frequent.",
    "Example words absent from the 50 000 forms (token count 0 in probe_words) are rarer than the list's cut-off; their absence says nothing about whether they exist.",
    "Not published: a coverage figure over every word-initial stop+ร/ล/ว spelling, which the collected package proposed. Its largest uncovered pairs, พว (พวก) and ดว (ด้วย), carry ว as the vowel ua, and บร (บริษัท, บรรยาย) and ตล (ตลาด) are mostly two syllables, so the figure would have measured spelling, not missing clusters.",
    "Not published: the package's checks of Ling and Drops pages. The post's sentence about those apps was cut, so no statement about either app remains to support.",
]


def build_dataset(m):
    return {
        "slug": "thai-consonant-clusters",
        "method": "public-corpus",
        "checkedAt": CHECKED_AT,
        "question": "Which of the post's consonant pairs carry the most Thai text?",
        "tools": {"python": sys.version.split()[0], "script": "check.py (standard library only)"},
        "inputs": [{
            "kind": "corpus",
            "url": CORPUS_URL,
            "sha256": CORPUS_SHA256,
            "corpus": "OpenSubtitles 2018 Thai, 50 000 most frequent word forms with counts (hermitdave/FrequencyWords)",
            "licence": "CC BY-SA 4.0 (content)",
            "downloaded": CHECKED_AT,
        }],
        **m,
        "notes": NOTES,
    }


def write_csv(results, path):
    with open(path, "w", newline="", encoding="utf-8") as fh:
        w = csv.writer(fh)
        w.writerow(["pair", "group", "words", "tokens", "share_pct", "top_words"])
        for r in results:
            w.writerow([r["pair"], r["group"], r["words"], r["tokens"], r["share_pct"],
                        "; ".join("%s %d" % (t["word"], t["tokens"]) for t in r["top_words"])])


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--recheck", action="store_true", help="download the list again before verifying")
    ap.add_argument("--write", action="store_true", help="rewrite dataset.json and dataset.csv")
    args = ap.parse_args()

    rows, digest = get_corpus(fresh=args.recheck)
    if digest != CORPUS_SHA256:
        print("FAIL corpus hash changed: %s (expected %s)" % (digest, CORPUS_SHA256))
        return 1
    m = measure(rows)

    if args.write:
        with open(DATASET, "w", encoding="utf-8") as fh:
            json.dump(build_dataset(m), fh, ensure_ascii=False, indent=2)
            fh.write("\n")
        write_csv(m["results"], CSV_PATH)
        print("wrote dataset.json and dataset.csv")
        return 0

    with open(DATASET, encoding="utf-8") as fh:
        stored = json.load(fh)
    failures = [k for k in ("corpus_totals", "results", "extras", "probe_words", "summary")
                if stored.get(k) != m[k]]
    with open(CSV_PATH, encoding="utf-8") as fh:
        csv_rows = list(csv.reader(fh))[1:]
    expected_csv = [[r["pair"], r["group"], str(r["words"]), str(r["tokens"]), str(r["share_pct"]),
                     "; ".join("%s %d" % (t["word"], t["tokens"]) for t in r["top_words"])]
                    for r in m["results"]]
    if csv_rows != expected_csv:
        failures.append("dataset.csv")

    for r in m["results"]:
        print("%s  %-14s words %4d  tokens %6d  share %.4f%%"
              % (r["pair"], r["group"], r["words"], r["tokens"], r["share_pct"]))
    if failures:
        for f in failures:
            print("FAIL %s no longer matches the measurement" % f)
        return 1
    s = m["summary"]
    print("OK %d pairs; %s carry %.1f%% of the table's tokens%s"
          % (len(m["results"]), ", ".join(s["top3_pairs"]), s["top3_share_of_table_pct"],
             " (recheck)" if args.recheck else ""))
    return 0


if __name__ == "__main__":
    sys.exit(main())
