#!/usr/bin/env python3
"""Re-runs the evidence check for Phayan's thai-alphabet-chart post.

1. Corpus (public-corpus). Phayan's word list holds 2,153 words that carry a
   Thai National Corpus rank. Every word is weighted by its corpus count; the
   script recomputes each letter's share of consonant occurrences, the number of
   words it appears in, and the share of running text a reader can decode once
   the N most frequent letters are known.
2. Published tables. Parses the two tables in the post and compares every cell
   with dataset.csv, and parses the three class tables against the seed classes.
3. Pages. Re-fetches every page cited about a named third party and checks that
   the stored sentence still appears.
4. Product. Re-reads the seed files the post's claims about Phayan rest on.

Exit 0 when every stored value and quote still holds, 1 otherwise.
  --recheck  re-reads the pages (no step here takes longer than a minute, so it
             behaves like a normal run)
  --write    recompute and store results in dataset.json and dataset.csv

Standard library only.
"""

from __future__ import annotations

import argparse
import csv
import hashlib
import html
import json
import os
import re
import sys
import unicodedata
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
CSV_PATH = os.path.join(HERE, "dataset.csv")

# apps/web/public/blog/thai-alphabet-chart -> repository root
DEFAULT_ROOT = os.path.abspath(os.path.join(HERE, *[".."] * 5))
ROOT = os.environ.get("PHYAN_REPO", DEFAULT_ROOT)

WORD_PATH = "apps/api/src/content/seed/word-frequency.jsonl"
POST_PATH = "apps/web/src/lib/blog/content-thai-alphabet-chart.tsx"
LETTERS_PATH = "apps/api/src/content/seed/letters.ts"
LEVELS_PATH = "apps/api/src/content/seed/levels.ts"
PROGRESS_PATH = "apps/web/src/lib/learn/progress-store/store.ts"

CORPUS_SHA256 = "247f0d41e351031470be2af0de4f278412eb6194a43f11f8e751e0eee7e68400"
UPSTREAM_URL = (
    "https://raw.githubusercontent.com/PyThaiNLP/pythainlp/dev/pythainlp/corpus/tnc_freq.txt"
)

# Unicode encodes the consonants in chart order inside U+0E01..U+0E2E; ฤ and ฦ
# sit in that range but are vowels.
CONSONANTS = "".join(chr(c) for c in range(0x0E01, 0x0E2F) if chr(c) not in "ฤฦ")
OBSOLETE = "ฃฅ"
RARE_IN_LAST_LEVEL = "ฆธณภญฑฒฌฬ"

PAGES = [
    {
        "party": "Ling",
        "statement": "Ling's own guide calls its romanisation toggle a support tool rather "
                     "than a permanent setting",
        "verdict": "supported, quoted on its own page",
        "quote": "The Ling app’s romanization toggle is a support tool, not a permanent setting.",
        "url": "https://ling-app.com/blog/best-app-to-learn-thai-alphabet/",
        "note": "The post cites this as what Ling says about itself. The page's section 'Turn "
                "Off Romanization Earlier Than Feels Comfortable' also tells learners to turn the "
                "toggle off after three or more sightings of a consonant.",
    },
    {
        "party": "Drops",
        "statement": "Drops announced its Thai alphabet topic in 2018 as three lessons in "
                     "alphabetical order",
        "verdict": "supported, quoted on its own page",
        "quote": "The writing system is split into three lessons, and Thai learners study the "
                 "system in alphabetical order.",
        "url": "https://languagedrops.com/blog/thai-alphabet",
        "note": "The page is dated Aug 30, 2018 and is what Drops says about its own course, so "
                "the post attributes the claim to Drops and to that date.",
    },
    {
        "party": "Thai primary-school curriculum",
        "statement": "removed: 'which is also how a Thai primary-school curriculum treats them'",
        "verdict": "wrong, cut from the post",
        "quote": "พยัญชนะไทย 44 ตัว เรียงลำดับจาก ก-ฮ",
        "url": "https://dltv.ac.th/teachplan/episode/81181",
        "note": "DLTV's Prathom 1 lesson, built on the core curriculum, teaches 44 consonants in "
                "ก-to-ฮ order, so the claim that the primary curriculum treats ฃ and ฅ as "
                "reference only was wrong. The official core-curriculum PDF could not be read: its "
                "Thai text uses a legacy font encoding, so no sentence from it could be quoted.",
    },
    {
        "party": "Anki",
        "statement": "removed: 'Anki decks can drill the letters themselves well, but a flashcard "
                     "teaches you one letter at a time and never the rule the letters feed'",
        "verdict": "cut, no page stated it",
        "quote": "",
        "url": "",
        "note": "An evaluation of a flashcard's limits, not a fact about Anki's product; no page "
                "states it, so the sentence went.",
    },
]

PRODUCT = [
    {
        "claim": "Phayan teaches the 42 and lists ฃ and ฅ as reference only",
        "path": LETTERS_PATH,
        "check": "letters-in-use",
    },
    {
        "claim": "Level 1 opens with ก, ต and ม, and first asks for มา, กา and ตา",
        "path": LEVELS_PATH,
        "check": "l1-first",
    },
    {
        "claim": "Phayan starts from the five tones",
        "path": LEVELS_PATH,
        "check": "l0-tones",
    },
    {
        "claim": "adds letters a few at a time",
        "path": LEVELS_PATH,
        "check": "letters-per-round",
    },
    {
        "claim": "the high class arrives as one block in Level 5",
        "path": LEVELS_PATH,
        "check": "high-block",
    },
    {
        "claim": "most of the low class in Levels 6 and 7",
        "path": LEVELS_PATH,
        "check": "low-block",
    },
    {
        "claim": "the nine rare letters wait for the last letter level",
        "path": LEVELS_PATH,
        "check": "rare-last",
    },
    {
        "claim": "teaches each letter with its name and with both of its sounds",
        "path": LETTERS_PATH,
        "check": "letter-name-sounds",
    },
    {
        "claim": "audio on every letter",
        "path": LETTERS_PATH,
        "check": "letter-audio",
    },
    {
        "claim": "audio on every word",
        "path": "apps/api/src/content/audio/tts/pregen-manifest.json",
        "check": "word-audio",
    },
    {
        "claim": "progress is kept on the device without an account",
        "path": PROGRESS_PATH,
        "quote": "// The localStorage-first progress store (DEC-21, R11): works with zero auth,",
    },
]


# ---------------------------------------------------------------- helpers

def norm(text: str) -> str:
    text = unicodedata.normalize("NFC", html.unescape(text))
    for a, b in (("’", "'"), ("‘", "'"), ("“", '"'), ("”", '"'), ("\xa0", " ")):
        text = text.replace(a, b)
    return re.sub(r"\s+", " ", text).strip()


def page_text(raw: str) -> str:
    raw = re.sub(r"<script[\s\S]*?</script>", " ", raw, flags=re.I)
    raw = re.sub(r"<style[\s\S]*?</style>", " ", raw, flags=re.I)
    return norm(re.sub(r"<[^>]+>", " ", raw))


def fetch(url: str, timeout: int = 60) -> str:
    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
    with urllib.request.urlopen(req, timeout=timeout) as r:
        charset = r.headers.get_content_charset() or "utf-8"
        return r.read().decode(charset, errors="replace")


def read_repo(path: str) -> str:
    with open(os.path.join(ROOT, path), encoding="utf-8") as fh:
        return fh.read()


def unq(cell: str) -> str:
    cell = cell.strip()
    if len(cell) >= 2 and cell[0] == cell[-1] and cell[0] in "'\"":
        return cell[1:-1]
    return cell


# ---------------------------------------------------------------- corpus

def load_words() -> list[dict]:
    text = read_repo(WORD_PATH)
    digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
    if digest != CORPUS_SHA256:
        raise SystemExit(f"word list changed: sha256 {digest} != {CORPUS_SHA256}")
    rows = [json.loads(line) for line in text.splitlines() if line.strip()]
    return [r for r in rows if r["covered"]]


def load_all_words() -> list[str]:
    text = read_repo(WORD_PATH)
    return [json.loads(line)["word"] for line in text.splitlines() if line.strip()]


def letter_stats(words: list[dict]) -> list[dict]:
    occ = {c: 0 for c in CONSONANTS}
    in_words = {c: 0 for c in CONSONANTS}
    running = 0
    for r in words:
        freq = int(r["frequency"])
        running += freq
        seen = set()
        for ch in r["word"]:
            if ch in CONSONANTS:
                occ[ch] += freq
                seen.add(ch)
        for ch in seen:
            in_words[ch] += 1
    total = sum(occ.values())
    order = sorted(CONSONANTS, key=lambda c: (-occ[c], c))
    rows, cum = [], 0
    for rank, ch in enumerate(order, 1):
        cum += occ[ch]
        known = set(order[:rank])
        readable = sum(
            int(r["frequency"]) for r in words
            if set(c for c in r["word"] if c in CONSONANTS) <= known
        )
        rows.append({
            "rank": rank,
            "letter": ch,
            "in_use": ch not in OBSOLETE,
            "weighted_occurrences": occ[ch],
            "occurrence_share_pct": round(100 * occ[ch] / total, 2),
            "words_containing": in_words[ch],
            "readable_share_pct": round(100 * readable / running, 2),
            "cumulative_occurrence_share_pct": round(100 * cum / total, 2),
        })
    return rows


def corpus_totals(words: list[dict]) -> dict:
    return {
        "words": len(words),
        "words_with_consonant": sum(1 for r in words if any(c in CONSONANTS for c in r["word"])),
        "running_tokens": sum(int(r["frequency"]) for r in words),
    }


# ---------------------------------------------------------------- seed files

def parse_letter_records(source: str) -> list[list[str]]:
    body = source[source.index("export const LETTERS"):]
    calls, pos = [], 0
    while True:
        start = body.find("letter(", pos)
        if start < 0:
            break
        k, depth, quote, arg, args = start + 7, 1, None, "", []
        while depth:
            ch = body[k]
            if quote:
                arg += ch
                if ch == "\\":
                    arg += body[k + 1]
                    k += 2
                    continue
                if ch == quote:
                    quote = None
            elif ch in "'\"`":
                quote = ch
                arg += ch
            elif ch == "(":
                depth += 1
                arg += ch
            elif ch == ")":
                depth -= 1
                if depth == 0:
                    args.append(arg)
                    break
                arg += ch
            elif ch == "," and depth == 1:
                args.append(arg)
                arg = ""
            else:
                arg += ch
            k += 1
        calls.append([a.strip() for a in args])
        pos = k
    return calls


def seed_letters(source: str) -> list[dict]:
    out = []
    for a in parse_letter_records(source):
        out.append({
            "letter": unq(a[0]),
            "name_thai": unq(a[1]),
            "class": unq(a[5]),
            "initial": unq(a[7]),
            "final": None if a[8] == "null" else unq(a[8]),
            "audio": len(a) > 12 and unq(a[12]) != "",
            "obsolete": len(a) > 13 and a[13] == "true",
        })
    return out


def parse_levels(source: str) -> dict:
    marks = [(m.start(), m.group(1)) for m in re.finditer(r"id: 'L(\d+)'", source)]
    levels = {}
    for i, (pos, num) in enumerate(marks):
        end = marks[i + 1][0] if i + 1 < len(marks) else len(source)
        block = source[pos:end]
        letters, words = [], []
        for arr in re.findall(r"newLetters: \[([^\]]*)\]", block):
            letters += [unq(x) for x in arr.split(",") if x.strip()]
        for arr in re.findall(r"wordIds: \[([^\]]*)\]", block):
            words += [unq(x) for x in arr.split(",") if x.strip()]
        title = re.search(r"title: '([^']*)'", block)
        levels[num] = {"letters": letters, "words": words,
                       "title": title.group(1) if title else ""}
    return levels


def word_audio() -> tuple:
    manifest = json.loads(read_repo("apps/api/src/content/audio/tts/pregen-manifest.json"))
    ids = {item["wordId"] for item in manifest["items"] if item.get("sha256")}
    words = load_all_words()
    return len(words), sum(1 for w in words if w in ids), len(manifest["failures"])


# ---------------------------------------------------------------- post tables

def parse_tables(source: str) -> list[dict]:
    out = []
    for block in re.findall(r"<table[\s\S]*?</table>", source):
        head = [norm(re.sub(r"<[^>]+>", "", h)) for h in
                re.findall(r"<th[^>]*>([\s\S]*?)</th>", block)]
        rows = []
        for row in re.findall(r"<tr>([\s\S]*?)</tr>", block):
            cells = [norm(re.sub(r"<[^>]+>", "", c)).strip() for c in
                     re.findall(r"<td[^>]*>([\s\S]*?)</td>", row)]
            if cells:
                rows.append(cells)
        out.append({"header": head, "rows": rows})
    return out


def class_tables(tables: list[dict], classes: dict) -> dict:
    """The three class tables keyed by the class their letters belong to in the seed."""
    by_class = {}
    for t in tables:
        if "Starts a syllable as" not in t["header"]:
            continue
        letters = [r[0].split()[0] for r in t["rows"]]
        names = set(classes[c] for c in letters)
        if len(names) != 1:
            raise SystemExit(f"class table mixes classes: {sorted(names)}")
        name = names.pop()
        if name in by_class:
            raise SystemExit(f"two {name} class tables in the post")
        by_class[name] = {"letters": letters, "finals": {r[0].split()[0]: r[4] for r in t["rows"]}}
    if set(by_class) != {"mid", "high", "low"}:
        raise SystemExit(f"expected mid, high and low class tables, found {sorted(by_class)}")
    return by_class


def post_counts(by_class: dict) -> dict:
    finals = {}
    for t in by_class.values():
        finals.update(t["finals"])
    in_use = {k: v for k, v in finals.items() if k not in OBSOLETE}
    by_final, in_use_by_final = {}, {}
    for sound in finals.values():
        by_final[sound] = by_final.get(sound, 0) + 1
    for sound in in_use.values():
        in_use_by_final[sound] = in_use_by_final.get(sound, 0) + 1
    sounds = sorted(s for s in in_use_by_final if s != "-")
    return {
        "class_sizes": {k: len(v["letters"]) for k, v in by_class.items()},
        "letters_total": len(finals),
        "letters_in_use": len(in_use),
        "with_final": len(in_use) - in_use_by_final.get("-", 0),
        "distinct_finals_in_use": len(sounds),
        "final_sounds_in_use": sounds,
        "by_final": by_final,
        "in_use_by_final": in_use_by_final,
        "no_final": sorted(k for k, v in finals.items() if v == "-"),
    }


def published_rows(tables: list[dict], header_key: str) -> list[list[str]]:
    for t in tables:
        if header_key in t["header"]:
            return t["rows"]
    raise SystemExit(f"no table with header {header_key!r} in the post")


def cell_str(value) -> str:
    return str(value)


# ---------------------------------------------------------------- results

def build_results() -> dict:
    words = load_words()
    totals = corpus_totals(words)
    letters = letter_stats(words)

    seed = seed_letters(read_repo(LETTERS_PATH))
    classes = {s["letter"]: s["class"] for s in seed}
    levels = parse_levels(read_repo(LEVELS_PATH))
    tables = parse_tables(read_repo(POST_PATH))
    by_class = class_tables(tables, classes)
    counts = post_counts(by_class)
    rank = {r["letter"]: r for r in letters}
    for row in letters:
        row["class"] = classes[row["letter"]]

    occ = {r["letter"]: r["weighted_occurrences"] for r in letters}
    total_occ = sum(occ.values())
    top5 = round(100 * sum(occ[c] for c in "นอ" "ก" "ง" "ม") / total_occ, 2)
    top20 = round(100 * sum(occ[r["letter"]] for r in letters[:20]) / total_occ, 2)
    ten = letters[9]
    twenty = letters[19]
    gap20 = round(twenty["cumulative_occurrence_share_pct"] - twenty["readable_share_pct"], 2)
    converge = next(
        (r["rank"] for r in letters
         if abs(r["cumulative_occurrence_share_pct"] - r["readable_share_pct"]) <= 0.1),
        None,
    )
    rare = {c: rank[c]["words_containing"] for c in "ฎฏฬฌฑฒ"}

    # Every letter in the levels that add new letters, in the order they arrive.
    letter_levels = {n: lv["letters"] for n, lv in levels.items() if lv["letters"]}
    last = max(letter_levels, key=int)
    in_use = [s for s in seed if not s["obsolete"]]
    low_in_use = [s["letter"] for s in in_use if s["class"] == "low"]
    low_block = sum(1 for c in letter_levels.get("6", []) + letter_levels.get("7", [])
                    if c in low_in_use)

    claims = [
        {"id": "opening", "asserted": "44 consonant letters, 42 of them in everyday use",
         "measured": f"{len(seed)} letters, {len(in_use)} in use", "verdict": "supported"},
        {"id": "corpus-size", "asserted": "Counted over 2,153 common words",
         "measured": totals["words"], "verdict": "supported"},
        {"id": "top-five", "asserted": "five consonants, น, อ, ก, ง and ม, make up 43 percent",
         "measured": f"{top5}% of weighted consonant occurrences", "verdict": "supported"},
        {"id": "top-twenty", "asserted": "the first twenty make up 94.7 percent",
         "measured": f"{top20}% of weighted consonant occurrences", "verdict": "supported"},
        {"id": "ten-letters", "asserted": "with ten letters you have seen two thirds of the "
                                          "consonants and can read 45 percent of running text",
         "measured": f"{ten['cumulative_occurrence_share_pct']}% of consonants, "
                     f"{ten['readable_share_pct']}% of running text readable",
         "verdict": "supported"},
        {"id": "gap-at-twenty", "asserted": "the gap there is still 3.75 points",
         "measured": gap20, "verdict": "supported"},
        {"id": "converge-at-34", "asserted": "within a tenth of a point of each other only at 34",
         "measured": converge, "verdict": "supported"},
        {"id": "half-at-eleven", "asserted": "Eleven letters is where just over half of running "
                                             "text becomes readable",
         "measured": f"{letters[10]['readable_share_pct']}% at 11 letters", "verdict": "supported"},
        {"id": "ninety-at-twenty", "asserted": "twenty is where nine words in ten are",
         "measured": f"{twenty['readable_share_pct']}% at 20 letters", "verdict": "supported"},
        {"id": "obsolete-zero", "asserted": "ฃ and ฅ appear in none of those words",
         "measured": f"ฃ {rank['ฃ']['words_containing']}, ฅ {rank['ฅ']['words_containing']}",
         "verdict": "supported"},
        {"id": "rare-six", "asserted": "ฎ, ฏ, ฬ, ฌ, ฑ and ฒ in six or fewer each",
         "measured": ", ".join(f"{k} {v}" for k, v in rare.items()), "verdict": "supported"},
        {"id": "eighteen-t", "asserted": "Eighteen letters all land on the same unreleased t",
         "measured": f"{counts['by_final'].get('t', 0)} letters", "verdict": "supported"},
        {"id": "four-k", "asserted": "Four give k",
         "measured": f"{counts['in_use_by_final'].get('k', 0)} letters in everyday use",
         "verdict": "supported"},
        {"id": "seven-p", "asserted": "seven give p",
         "measured": f"{counts['in_use_by_final'].get('p', 0)} letters", "verdict": "supported"},
        {"id": "six-n", "asserted": "six give n",
         "measured": f"{counts['in_use_by_final'].get('n', 0)} letters", "verdict": "supported"},
        {"id": "ng-m-y-w", "asserted": "ง ม ย ว give ng, m, y and w one letter each",
         "measured": ", ".join(f"{s} {counts['in_use_by_final'].get(s, 0)}"
                               for s in ("ng", "m", "y", "w")), "verdict": "supported"},
        {"id": "no-final", "asserted": "อ, ห and ฮ never end a syllable at all",
         "measured": ", ".join(counts["no_final"]), "verdict": "supported"},
        {"id": "39-and-eight", "asserted": "of the 42 letters in everyday use, 39 can close a "
                                           "syllable and they produce exactly eight sounds",
         "measured": f"{counts['with_final']} letters, {counts['distinct_finals_in_use']} sounds: "
                     f"{', '.join(counts['final_sounds_in_use'])}", "verdict": "supported"},
        {"id": "class-sizes", "asserted": "nine mid, eleven high, twenty-four low",
         "measured": ", ".join(f"{k} {v}" for k, v in counts["class_sizes"].items()),
         "verdict": "supported"},
        {"id": "42-plus-two", "asserted": "42 letters in everyday use, plus the two obsolete ones",
         "measured": f"{counts['letters_in_use']} in use, "
                     f"{counts['letters_total'] - counts['letters_in_use']} obsolete",
         "verdict": "supported"},
        {"id": "quoted-counts", "asserted": "The counts everyone quotes - 9 mid, 10 high, 23 low "
                                            "- are the same lists minus ฃ and ฅ",
         "measured": f"{counts['class_sizes']['mid']} mid, "
                     f"{counts['class_sizes']['high'] - 1} high, "
                     f"{counts['class_sizes']['low'] - 1} low in use",
         "verdict": "supported"},
    ]

    # The two published tables, cell by cell, from the same numbers.
    curve = [
        [str(n), f"{letters[n - 1]['cumulative_occurrence_share_pct']}%",
         f"{letters[n - 1]['readable_share_pct']}%"]
        for n in (5, 10, 15, 20, 25)
    ]
    ranking = [
        [str(r["rank"]), r["letter"], r["class"], f"{r['occurrence_share_pct']}%",
         str(r["words_containing"])]
        for r in letters[:20]
    ]

    totals_audio = word_audio()

    product = []
    for row in PRODUCT:
        row = dict(row)
        kind = row.get("check")
        if kind == "letters-in-use":
            row["measured"] = (f"{len(seed)} letter records, {len(in_use)} in use, "
                               f"{len(seed) - len(in_use)} obsolete")
        elif kind == "l1-first":
            row["measured"] = (f"L1 newLetters {levels['1']['letters']}, "
                               f"first words {levels['1']['words'][:3]}")
        elif kind == "l0-tones":
            row["measured"] = f"L0 title {levels['0']['title']!r}, no new letters"
        elif kind == "letters-per-round":
            sizes = [len(re.findall(r"'[^']'", a))
                     for a in re.findall(r"newLetters: \[([^\]]*)\]", read_repo(LEVELS_PATH))]
            row["measured"] = f"at most {max(sizes)} new letters in a round"
        elif kind == "high-block":
            high = [c for c in letter_levels.get("5", []) if classes[c] == "high"]
            row["measured"] = f"{len(high)} letters in Level 5, all high"
        elif kind == "low-block":
            row["measured"] = f"{low_block} of {len(low_in_use)} low-class letters in Levels 6 and 7"
        elif kind == "rare-last":
            row["measured"] = (f"last level with new letters: L{last}, "
                               f"{len(letter_levels[last])} letters: "
                               f"{' '.join(letter_levels[last])}")
        elif kind == "letter-name-sounds":
            named = sum(1 for s in seed if s["name_thai"] and s["initial"])
            with_final = sum(1 for s in seed if s["final"])
            row["measured"] = (f"{named} of {len(seed)} records carry a name and an initial "
                               f"sound; {with_final} carry a final")
        elif kind == "letter-audio":
            row["measured"] = (f"{sum(1 for s in seed if s['audio'])} of {len(seed)} letter "
                               f"records carry a non-empty audioWord")
        elif kind == "word-audio":
            row["measured"] = (f"{totals_audio[1]} of {totals_audio[0]} word list entries have a "
                               f"generated recording; {totals_audio[2]} generation failures")
        product.append(row)

    return {
        "totals": totals,
        "letters": letters,
        "post_tables": counts,
        "published_tables": {"readability": curve, "ranking": ranking},
        "claims": claims,
        "third_party": PAGES,
        "product": product,
    }


# ---------------------------------------------------------------- verification

def check_pages(stored: list[dict], verbose: bool) -> list[str]:
    failures = []
    for row in stored:
        if not row.get("url") or not row.get("quote"):
            if verbose:
                print(f"skip  {row['party']}: no page to re-read ({row['verdict']})")
            continue
        try:
            text = page_text(fetch(row["url"]))
        except Exception as exc:  # noqa: BLE001 - report, never guess
            failures.append(f"{row['party']}: could not fetch {row['url']}: {exc}")
            continue
        if norm(row["quote"]) in text:
            if verbose:
                print(f"ok    {row['party']}: quote still on {row['url']}")
        else:
            failures.append(f"{row['party']}: quote gone from {row['url']}")
    return failures


def check_product(stored: list[dict], computed: list[dict], verbose: bool) -> list[str]:
    failures = []
    expected = {r["claim"]: r for r in computed}
    sources = {}
    for row in stored:
        source = sources.get(row["path"])
        if source is None:
            try:
                source = sources[row["path"]] = read_repo(row["path"])
            except OSError:
                failures.append(f"product: {row['path']} unavailable")
                continue
        want = expected.get(row["claim"])
        if want is None:
            failures.append(f"product: claim missing from the computed set: {row['claim']}")
            continue
        if row.get("quote") and row["quote"] not in source:
            failures.append(f"product: quote gone from {row['path']}: {row['quote']}")
            continue
        if want.get("measured") != row.get("measured"):
            failures.append(f"product: {row['claim']}: {row.get('measured')} != "
                            f"{want.get('measured')}")
            continue
        if verbose:
            print(f"ok    product: {row['claim']} ({row['path']})")
    return failures


def compare(stored: dict, computed: dict) -> list[str]:
    failures = []
    if stored["totals"] != computed["totals"]:
        failures.append(f"totals differ: {stored['totals']} != {computed['totals']}")
    if stored["post_tables"] != computed["post_tables"]:
        failures.append("post tables differ from the published source")
    if stored.get("published_tables") != computed["published_tables"]:
        failures.append("published tables differ from the post")
    stored_letters = {r["letter"]: r for r in stored["letters"]}
    for row in computed["letters"]:
        old = stored_letters.get(row["letter"])
        if old is None:
            failures.append(f"letter {row['letter']} missing from dataset")
            continue
        for key in ("rank", "class", "in_use", "weighted_occurrences", "words_containing"):
            if old.get(key) != row[key]:
                failures.append(f"{row['letter']} {key}: {old.get(key)} != {row[key]}")
        for key in ("occurrence_share_pct", "readable_share_pct",
                    "cumulative_occurrence_share_pct"):
            if abs(old.get(key, -1) - row[key]) > 0.011:
                failures.append(f"{row['letter']} {key}: {old.get(key)} != {row[key]}")
    stored_claims = {c["id"]: c for c in stored["claims"]}
    for claim in computed["claims"]:
        old = stored_claims.get(claim["id"])
        if old is None:
            failures.append(f"claim {claim['id']} missing from dataset")
        elif old.get("measured") != claim["measured"] or old.get("verdict") != claim["verdict"]:
            failures.append(f"claim {claim['id']}: {old.get('measured')} != {claim['measured']}")
    return failures


def check_post_tables(computed: dict) -> list[str]:
    """The two published tables must carry exactly the values in dataset.csv."""
    failures = []
    tables = parse_tables(read_repo(POST_PATH))
    posted = {
        "readability": published_rows(tables, "Letters known"),
        "ranking": published_rows(tables, "Rank"),
    }
    for name, rows in computed["published_tables"].items():
        if posted[name] != rows:
            failures.append(
                f"{name} table in the post differs from dataset.csv:\n"
                f"  post:     {posted[name]}\n  dataset:  {rows}"
            )
    if not failures:
        print("ok    post: both published tables match dataset.csv")
    return failures


# ---------------------------------------------------------------- output

def write_dataset(computed: dict) -> None:
    dataset = {
        "slug": "thai-alphabet-chart",
        "question": "What are the Thai consonants and what sound does each make?",
        "method": "public-corpus",
        "checkedAt": "2026-10-26",
        "tools": [
            {"name": "python3", "version": "3.9.6",
             "used": "check.py, standard library only, urllib for the cited pages"},
        ],
        "inputs": [
            {
                "id": "word-list",
                "what": "Phayan's word list, filtered to the 2,153 entries that carry a Thai "
                        "National Corpus rank; each entry is one word and its corpus count.",
                "path": WORD_PATH,
                "sha256": CORPUS_SHA256,
                "checked": "every one of the 2,153 counts equals the upstream TNC list on "
                           "2026-10-26",
            },
            {
                "id": "tnc-frequency-list",
                "url": UPSTREAM_URL,
                "what": "Upstream source of the counts: the Thai National Corpus frequency list "
                        "PyThaiNLP publishes. The committed list is the input the numbers are "
                        "computed from; the counts were compared against this file.",
            },
            {
                "id": "post-source",
                "path": POST_PATH,
                "what": "The post, parsed for its three class tables, its readability table and "
                        "its 20-letter ranking table.",
            },
            {
                "id": "product-seed",
                "paths": [LETTERS_PATH, LEVELS_PATH, PROGRESS_PATH],
                "what": "The files the post's claims about Phayan are checked against.",
            },
        ],
        "methodDetail": "Every word of the 2,153-entry list is weighted by its Thai National "
                        "Corpus count. A letter's share is its weighted occurrences divided by "
                        "all weighted consonant occurrences. A word counts as readable once "
                        "every consonant in it is among the letters known, and the readable "
                        "share is the weighted share of running text. Consonants are the 44 "
                        "code points U+0E01..U+0E2E minus ฤ and ฦ. Classes come from the seed "
                        "letter records, not from the post.",
        "results": computed,
        "notes": [
            "One of the 2,153 entries contains no consonant, so 2,152 words enter the "
            "consonant counts; all 2,153 enter the running-text denominator.",
            "The earlier sentence 'The two numbers meet around twenty letters' was wrong and "
            "was replaced: at twenty letters the consonant share is 94.74 percent against "
            "90.99 percent of running text, a gap of 3.75 points, and the two come within a "
            "tenth of a point of each other only at 34 letters.",
            "Removed from the post: 'romanisation-first apps like Ling and Drops hide the "
            "distinction entirely'. Ling's own page calls romanisation a toggle and says the "
            "course teaches the script; Drops' page describes romanisation as the course's "
            "former state. Neither page supports the sentence.",
            "Removed from the post: 'which is also how a Thai primary-school curriculum treats "
            "them'. DLTV's Prathom 1 lesson teaches 44 consonants in ก-to-ฮ order, so the "
            "claim was wrong. The official core-curriculum PDF could not be quoted: pdftotext "
            "returns legacy-font mojibake, not Unicode Thai.",
            "Removed from the post: the Anki sentence. It was an evaluation of flashcards, not "
            "a fact about Anki, and no page states it.",
            "Ling and Drops were not used and not installed; only the pages linked above were "
            "read, so nothing here describes what their apps show a learner.",
            "The post states no price and no version, so nothing in it expires by date.",
        ],
    }
    with open(DATASET, "w", encoding="utf-8") as fh:
        json.dump(dataset, fh, ensure_ascii=False, indent=1)
        fh.write("\n")

    fields = ["rank", "letter", "class", "in_use", "weighted_occurrences",
              "occurrence_share_pct", "words_containing", "readable_share_pct",
              "cumulative_occurrence_share_pct"]
    with open(CSV_PATH, "w", encoding="utf-8", newline="") as fh:
        writer = csv.DictWriter(fh, fieldnames=fields)
        writer.writeheader()
        for row in computed["letters"]:
            writer.writerow({k: row[k] for k in fields})


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--recheck", action="store_true",
                        help="re-read every cited page (no step here is slow)")
    parser.add_argument("--write", action="store_true", help="store results in the dataset")
    args = parser.parse_args()

    computed = build_results()
    if args.write:
        write_dataset(computed)
        print(f"wrote {DATASET} and {CSV_PATH}")
        return 0

    if not os.path.exists(DATASET):
        print("dataset.json is missing; run with --write first")
        return 1
    with open(DATASET, encoding="utf-8") as fh:
        stored = json.load(fh)

    failures = compare(stored["results"], computed)
    failures += check_post_tables(computed)
    failures += check_product(stored["results"]["product"], computed["product"], verbose=True)
    failures += check_pages(stored["results"]["third_party"], verbose=True)

    if failures:
        print("\nFAIL")
        for line in failures:
            print(" -", line)
        return 1
    print(f"\nOK  {computed['totals']['words']} words, "
          f"{len(computed['letters'])} letters, "
          f"{len(computed['claims'])} claims, "
          f"{len(stored['results']['third_party'])} third-party statements")
    return 0


if __name__ == "__main__":
    sys.exit(main())
