#!/usr/bin/env python3
"""Re-runs every measurement and every page quote behind the post
"The Four Thai Tone Marks and What They Actually Do" (/blog/thai-tone-marks).

Two inputs, both public:

  * a Thai word-frequency list (OpenSubtitles 2018, 50k word forms) -- CC BY-SA 4.0
  * the English Wikipedia article "Thai script" -- consonant classes (its
    alphabet-listing table) and the tone table (class x mark -> tone)

plus Phayan's own public API for the level and price the post states.

From them it counts, for every tone mark in the list, how often the mark is
used, on what consonant class, and what tone that class gives. Every quoted
sentence is checked against the page it was taken from.

    python3 check.py            verify against a cached copy of the inputs
                                (downloaded on first run to a temp directory)
    python3 check.py --recheck  re-download every input, then verify
    python3 check.py --write    measure and rewrite dataset.json / dataset.csv

Exit 0 when every number and every quote still holds, 1 otherwise.
Standard library only.
"""

import argparse
import csv
import hashlib
import html as htmllib
import io
import json
import os
import re
import sys
import tempfile
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
WORK = os.path.join(tempfile.gettempdir(), "phayan-check-thai-tone-marks")
DATASET_JSON = os.path.join(HERE, "dataset.json")
DATASET_CSV = os.path.join(HERE, "dataset.csv")

CORPUS_URL = "https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/content/2018/th/th_50k.txt"
CORPUS_LICENCE = "CC BY-SA 4.0 (hermitdave/FrequencyWords README: 'CC-by-sa-4.0 for content.')"
CORPUS_PATH = os.path.join(WORK, "th_50k.txt")
WIKI_URL = "https://en.wikipedia.org/wiki/Thai_script"
WIKI_PATH = os.path.join(WORK, "wiki-thai-script.html")
API = "https://api.phayan.com"
TP_TONES_URL = "https://www.thaipod101.com/blog/2021/01/18/thai-tones/"
TP_TONES_PATH = os.path.join(WORK, "thaipod101-tones.html")

UA = "Mozilla/5.0"

# The four marks, in the order the post presents them.
MARKS = [
    ("่", "mai ek", "ไม้เอก"),
    ("้", "mai tho", "ไม้โท"),
    ("๊", "mai tri", "ไม้ตรี"),
    ("๋", "mai chattawa", "ไม้จัตวา"),
]
MARK_BY_CHAR = {ch: name for ch, name, _ in MARKS}

# True Thai consonant clusters: only these two-letter onsets take the tone
# mark over the second letter while the tone comes from the first.
TRUE_CLUSTERS = {
    "กร", "กล", "กว",
    "ขร", "ขล", "ขว",
    "คร", "คล", "คว",
    "ตร",
    "ปร", "ปล",
    "ผร", "ผล",
    "พร", "พล",
    "ฟร", "ฟล",
    "บร", "บล",
}
HO = "ห"  # ห, the silent ho nam

# The class x mark table exactly as the post prints it (its own claim, kept
# here so the page's table can be compared with the encyclopaedia's).
POST_TONE_TABLE = {
    ("mid", "none-live"): "mid", ("mid", "mai ek"): "low",
    ("mid", "mai tho"): "falling", ("mid", "mai tri"): "high",
    ("mid", "mai chattawa"): "rising",
    ("high", "none-live"): "rising", ("high", "mai ek"): "low",
    ("high", "mai tho"): "falling", ("high", "mai tri"): "not used",
    ("high", "mai chattawa"): "not used",
    ("low", "none-live"): "mid", ("low", "mai ek"): "falling",
    ("low", "mai tho"): "high", ("low", "mai tri"): "not used",
    ("low", "mai chattawa"): "not used",
}

# The example words the post works through, with the tone and romanisation it
# states for each. `tone` is what the post claims; it is re-derived below, and
# the romanisation's own diacritic is read back as a tone for comparison.
POST_EXAMPLES = [
    ("ก่า", "-", "low"),
    ("ค่า", "-", "falling"),
    ("ไม่", "mâi", "falling"),
    ("ใหม่", "mài", "low"),
    ("สี่", "sìi", "low"),
    ("ป้า", "pâa", "falling"),
    ("น้ำ", "náam", "high"),
    ("โต๊ะ", "tó", "high"),
    ("เดี๋ยว", "dǐao", "rising"),
    ("เปล่า", "plào", "low"),
]

# Words whose rank in the list is worth recording: the post names several of
# them as common or as the everyday carrier of a mark.
RANK_WORDS = ["ไม่", "ใหม่", "โต๊ะ",
              "เดี๋ยว", "สี่", "ป้า",
              "น้ำ", "ค่า", "เปล่า",
              "มั๊ย", "เอ๊ะ"]

# Romanisation diacritics, after NFD: the tone the post's own spelling states.
COMBINING_TONE = {"̀": "low", "́": "high", "̂": "falling",
                  "̌": "rising"}

# Every sentence the dataset quotes, with the page it must still be on.
QUOTES = [
    {
        "id": "wikipedia-mid-only",
        "claim": ("ไม้ตรี and ไม้จัตวา take only middle-class initials; the post's table "
                  "marks them 'not used' on high and low class"),
        "url": WIKI_URL, "path": WIKI_PATH,
        "quote": "Only middle class consonants can take the latter two, as shown in the Tone table.",
    },
    {
        "id": "wikipedia-ho-nam",
        "claim": ("a silent ห in front of a low-class letter turns the initial high class, and the "
                  "marker is written over the low-class letter"),
        "url": WIKI_URL, "path": WIKI_PATH,
        "quote": ("if the letter comes before a low class letter in a syllable, it becomes the "
                  "silent ho nam and turn the initial consonant into high class"),
    },
    {
        "id": "wikipedia-cluster-marker",
        "claim": "where a low-class letter follows a higher-class one, the marker goes over the low-class letter",
        "url": WIKI_URL, "path": WIKI_PATH,
        "quote": "the marker, if used, goes over the low class one",
    },
    {
        "id": "thaipod101-chart",
        "claim": "ThaiPod101 presents the tones in a chart",
        "url": TP_TONES_URL, "path": TP_TONES_PATH,
        "quote": "The Thai tones chart below summarizes everything you need to know about the five tones of Thai.",
    },
    {
        "id": "thaipod101-chart-examples",
        "claim": "the chart's example words are all on mid-class ก",
        "url": TP_TONES_URL, "path": TP_TONES_PATH,
        "quote": "Example Thai Word กา ก่า ก้า ก๊า ก๋า",
    },
    {
        "id": "product-level-3",
        "claim": "level 3, before the tone marks, covers finals and syllable types",
        "url": API + "/content/access/levels/L3",
        "path": os.path.join(WORK, "phayan-level-L3.json"),
        "quote": '"title":"Finals & syllable types","free":true',
        "status": 200,
    },
    {
        "id": "product-level-4",
        "claim": "level 4 is free and its subject is the mid class together with the tone marks",
        "url": API + "/content/access/levels/L4",
        "path": os.path.join(WORK, "phayan-level-L4.json"),
        "quote": '"title":"Mid class + tone marks","free":true',
        "status": 200,
    },
    {
        "id": "product-paid-boundary",
        "claim": "the free levels end after level 4; level 5 needs the paid tier",
        "url": API + "/content/access/levels/L5",
        "path": os.path.join(WORK, "phayan-level-L5.json"),
        "quote": '"code":"PREMIUM_REQUIRED","level":{"id":"L5"',
        "status": 403,
    },
    {
        "id": "product-pricing",
        "claim": "the web price is $14.99 a month or $79.99 a year",
        "url": API + "/subscriptions/plans",
        "path": os.path.join(WORK, "phayan-plans.json"),
        "quote": '"monthly":14.99,"annual":79.99',
        "status": 200,
    },
]

COLLOQUIAL = {
    "mai tri": ["มั๊ย", "ห๊ะ", "โอเค๊",
                "ว๊าว", "แท๊กซี่",
                "กรุ๊ป"],
    "mai chattawa": ["ห๋า", "แหม๋",
                     "ถูกเผ๋ง"],
}


# ---------------------------------------------------------------- fetching

def fetch(url, path, force):
    """Download to `path` unless a cached copy exists and `force` is false.
    Non-2xx bodies are kept: the level access probe answers 403 with the
    payload the check needs."""
    if not force and os.path.exists(path) and os.path.getsize(path) > 0:
        return None
    os.makedirs(os.path.dirname(path), exist_ok=True)
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    try:
        with urllib.request.urlopen(req, timeout=120) as resp:
            data = resp.read()
            status = resp.status
    except urllib.error.HTTPError as exc:
        data = exc.read()
        status = exc.code
    if not data:
        raise RuntimeError("empty response from %s" % url)
    tmp = path + ".part"
    with open(tmp, "wb") as fh:
        fh.write(data)
    os.replace(tmp, path)
    return status


def read_text(path):
    with open(path, "rb") as fh:
        return fh.read().decode("utf-8", "replace")


def sha256(path):
    h = hashlib.sha256()
    with open(path, "rb") as fh:
        for chunk in iter(lambda: fh.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


# ------------------------------------------------------------- normalising

def normalise(text):
    """Whitespace, quote characters, entities and the space HTML leaves before
    punctuation all vary between the rendered page and the stored quote."""
    text = htmllib.unescape(text)
    text = (text.replace("’", "'").replace("‘", "'")
                .replace("“", '"').replace("”", '"')
                .replace(" ", " ").replace("\u2013", "-").replace("\u2014", "-"))
    text = re.sub(r"\s+", " ", text)
    text = re.sub(r"\s+([.,;:!?)\]])", r"\1", text)
    return text.strip().lower()


def html_text(raw):
    raw = re.sub(r"<script[\s\S]*?</script>", " ", raw, flags=re.I)
    raw = re.sub(r"<style[\s\S]*?</style>", " ", raw, flags=re.I)
    raw = re.sub(r"<[^>]+>", " ", raw)
    return normalise(htmllib.unescape(raw))


def page_haystack(raw):
    """Body text plus every meta description: some quotes live only in the
    head, which the body text never carries."""
    meta = " ".join(re.findall(r'<meta[^>]+content="([^"]*)"', raw, flags=re.I))
    return html_text(raw) + " " + html_text(meta)


# ---------------------------------------------------------- page structure

def _cells(row_html):
    out = []
    for cell in re.findall(r"<t[hd][\s\S]*?</t[hd]>", row_html, flags=re.I):
        txt = re.sub(r"<[^>]+>", " ", cell)
        txt = re.sub(r"\s+", " ", htmllib.unescape(txt)).strip()
        out.append(txt)
    return out


def consonant_classes(raw):
    """Symbol -> 'mid' | 'high' | 'low', from the alphabet-listing table."""
    for table in re.findall(r"<table[\s\S]*?</table>", raw, flags=re.I):
        rows = [_cells(r) for r in re.findall(r"<tr[\s\S]*?</tr>", table, flags=re.I)]
        flat = " ".join(" ".join(r) for r in rows[:3])
        if "Class" not in flat:
            continue
        classes = {}
        for cells in rows:
            if len(cells) >= 5 and cells[-1] in ("mid", "high", "low"):
                symbol = cells[0].split()[0]
                if len(symbol) == 1:
                    classes[symbol] = cells[-1]
        if len(classes) >= 44:
            return classes
    raise RuntimeError("consonant class table not found on %s" % WIKI_URL)


def tone_table(raw):
    """(class, mark) -> tone, from the class x mark table. '-' means 'not used'.
    The header row drops the merged Symbol column, so the three tone columns
    are read from the right edge of each row."""
    for table in re.findall(r"<table[\s\S]*?</table>", raw, flags=re.I):
        if "Vowel and final" not in table:
            continue
        table_out = {}
        for cells in (_cells(r) for r in re.findall(r"<tr[\s\S]*?</tr>", table, flags=re.I)):
            if len(cells) < 7:
                continue
            name, desc = cells[1], cells[3]
            mark = MarkName.from_thai(name)
            if mark is None:
                if name != "(ไม่มี)" or not desc.startswith("live"):
                    continue
                key = "none-live"
            else:
                key = mark
            for cls, value in zip(("low", "mid", "high"), cells[-3:]):
                table_out[(cls, key)] = "mid" if value == "middle" else (
                    "not used" if value == "-" else value)
        if table_out:
            return table_out
    raise RuntimeError("tone table not found on %s" % WIKI_URL)


class MarkName:
    THAI = {thai: name for _, name, thai in MARKS}

    @classmethod
    def from_thai(cls, thai):
        return cls.THAI.get(thai)


# ------------------------------------------------------------- syllable work

def is_thai(ch):
    return "฀" <= ch <= "๿"


def analyse(word, classes):
    """Every tone mark in `word`, with the consonant it sits on and the class
    that decides the tone: ho nam makes a low-class initial high class, and a
    true cluster is read from its first letter even though the mark sits on
    the second."""
    found = []
    for i, ch in enumerate(word):
        if ch not in MARK_BY_CHAR:
            continue
        j = i - 1
        marked = None
        while j >= 0:
            if not is_thai(word[j]):
                break
            if word[j] in classes:
                marked = j
                break
            j -= 1
        if marked is None:
            continue
        letter = word[marked]
        marked_class = classes[letter]
        initial_index = marked
        if marked >= 1 and word[marked - 1] == HO and marked_class == "low":
            initial_index = marked - 1
        elif marked >= 1 and (word[marked - 1] + letter) in TRUE_CLUSTERS:
            initial_index = marked - 1
        initial_class = classes[word[initial_index]]
        found.append({
            "mark": MARK_BY_CHAR[ch],
            "marked_index": marked,
            "marked_letter": letter,
            "marked_class": marked_class,
            "initial_index": initial_index,
            "initial_letter": word[initial_index],
            "initial_class": initial_class,
            "cluster": initial_index != marked,
        })
    return found


def measure(corpus_path, classes, tones):
    word_forms_total = word_forms = with_mark = 0
    per = {name: {"occ": 0, "w": 0, "marked": {"mid": 0, "high": 0, "low": 0},
                  "init": {"mid": 0, "high": 0, "low": 0},
                  "init_w": {"mid": 0, "high": 0, "low": 0},
                  "letters": set(), "mid_letters": set()} for _, name, _ in MARKS}
    pair_occ, pair_w = {}, {}
    pair_of = {}
    cluster_second = cluster_first = 0
    ho_nam = ho_nam_w = 0
    ek_low_tone = ek_low_tone_w = 0
    examples = {}

    with open(corpus_path, encoding="utf-8", errors="replace") as fh:
        for line in fh:
            parts = line.split()
            if len(parts) != 2:
                continue
            word, count = parts[0], parts[1]
            if not count.isdigit():
                continue
            count = int(count)
            word_forms_total += 1
            if not all(is_thai(ch) for ch in word):
                continue
            if not any(ch in classes for ch in word):
                continue
            word_forms += 1
            hits = analyse(word, classes)
            if hits:
                with_mark += 1
            for hit in hits:
                row = per[hit["mark"]]
                row["occ"] += 1
                row["w"] += count
                row["marked"][hit["marked_class"]] += 1
                row["init"][hit["initial_class"]] += 1
                row["init_w"][hit["initial_class"]] += count
                row["letters"].add(hit["marked_letter"])
                if hit["marked_class"] == "mid":
                    row["mid_letters"].add(hit["marked_letter"])
                key = (hit["initial_class"], hit["mark"])
                pair_occ[key] = pair_occ.get(key, 0) + 1
                pair_w[key] = pair_w.get(key, 0) + count
                pair_of.setdefault(hit["mark"], set()).add(hit["initial_class"])
                idx = hit["marked_index"]
                if hit["cluster"]:
                    cluster_second += 1
                elif (idx + 1 < len(word) and word[idx + 1] in classes
                      and (hit["marked_letter"] + word[idx + 1]) in TRUE_CLUSTERS):
                    cluster_first += 1
                if (hit["mark"] == "mai ek" and hit["initial_letter"] == HO
                        and hit["marked_class"] == "low"):
                    ho_nam += 1
                    ho_nam_w += count
                if hit["mark"] == "mai ek":
                    tone = tones.get((hit["initial_class"], "mai ek"))
                    if tone == "low":
                        ek_low_tone += 1
                        ek_low_tone_w += count
                prev = examples.get(hit["mark"])
                if prev is None or count > prev[1]:
                    examples[hit["mark"]] = (word, count)

    total_occ = sum(row["occ"] for row in per.values())
    total_w = sum(row["w"] for row in per.values())

    marks = []
    for _, name, thai in MARKS:
        row = per[name]
        marks.append({
            "symbol": [ch for ch, n, _ in MARKS if n == name][0],
            "name": name,
            "thai_name": thai,
            "occurrences": row["occ"],
            "weighted_occurrences": row["w"],
            "share_pct_occurrence": round(100.0 * row["occ"] / total_occ, 3),
            "share_pct_weighted": round(100.0 * row["w"] / total_w, 3),
            "marked_class_mid": row["marked"]["mid"],
            "marked_class_nonmid": row["marked"]["high"] + row["marked"]["low"],
            "marked_nonmid_share_pct": round(
                100.0 * (row["marked"]["high"] + row["marked"]["low"]) / row["occ"], 2),
            "initial_nonmid_share_pct": round(
                100.0 * (row["init"]["high"] + row["init"]["low"]) / row["occ"], 2),
            "initial_mid": row["init"]["mid"],
            "initial_high": row["init"]["high"],
            "initial_low": row["init"]["low"],
            "initial_weighted_mid": row["init_w"]["mid"],
            "initial_weighted_high": row["init_w"]["high"],
            "initial_weighted_low": row["init_w"]["low"],
            "distinct_marked_letters": len(row["letters"]),
            "distinct_mid_letters": len(row["mid_letters"]),
            "example_word": examples[name][0],
            "example_word_count": examples[name][1],
        })

    ordered = sorted(pair_w.items(), key=lambda kv: -kv[1])
    running = 0
    top_pairs = []
    for key, value in ordered:
        running += value
        top_pairs.append({"initial_class": key[0], "mark": key[1],
                          "occurrences": pair_occ[key], "weighted": value,
                          "cumulative_share_pct": round(100.0 * running / total_w, 2)})
    return {
        "corpus": {
            "word_forms_total": word_forms_total,
            "word_forms_thai": word_forms,
            "word_forms_with_mark": with_mark,
            "marked_syllables": total_occ,
            "marked_syllables_weighted": total_w,
        },
        "marks": marks,
        "coverage": {
            "pairs_used": len(pair_w),
            "pairs_possible": 12,
            "top6_share_pct": round(
                100.0 * sum(v for _, v in ordered[:6]) / total_w, 2),
            "top5_share_pct": round(
                100.0 * sum(v for _, v in ordered[:5]) / total_w, 2),
            "top_pairs": top_pairs,
        },
        "cluster_placement": {
            "mark_on_second_letter": cluster_second,
            "mark_on_first_letter": cluster_first,
        },
        "ho_nam": {"occurrences": ho_nam, "weighted": ho_nam_w},
        "english_name": {
            "mai_ek_gives_low_occurrences": ek_low_tone,
            "mai_ek_gives_low_weighted": ek_low_tone_w,
            "mai_ek_gives_low_share_pct": round(100.0 * ek_low_tone / per["mai ek"]["occ"], 2),
            "mai_ek_gives_low_weighted_share_pct": round(
                100.0 * ek_low_tone_w / per["mai ek"]["w"], 2),
        },
    }


def alphabet(classes):
    counts = {"mid": 0, "high": 0, "low": 0}
    for cls in classes.values():
        counts[cls] += 1
    total = sum(counts.values())
    two_marks = counts["high"] + counts["low"]
    ek_low_letters = counts["mid"] + counts["high"]
    return {
        "letters_total": total,
        "mid": counts["mid"],
        "high": counts["high"],
        "low": counts["low"],
        "letters_taking_at_most_two_marks": two_marks,
        "letters_taking_at_most_two_marks_pct": round(100.0 * two_marks / total, 2),
        "mai_ek_gives_low_letters": ek_low_letters,
        "mai_ek_gives_low_letters_pct": round(100.0 * ek_low_letters / total, 2),
        "mai_ek_gives_falling_letters": counts["low"],
        "mai_ek_gives_falling_letters_pct": round(100.0 * counts["low"] / total, 2),
    }


def roman_tone(roman):
    """The tone the post's own romanisation states, read from its diacritic."""
    import unicodedata
    for ch in unicodedata.normalize("NFD", roman):
        if ch in COMBINING_TONE:
            return COMBINING_TONE[ch]
    return "mid"


def word_ranks(corpus_path, targets):
    wanted = set(targets)
    out = {w: {"word": w, "rank": None, "count": 0} for w in targets}
    rank = 0
    with open(corpus_path, encoding="utf-8", errors="replace") as fh:
        for line in fh:
            parts = line.split()
            if len(parts) != 2 or not parts[1].isdigit():
                continue
            rank += 1
            if parts[0] in wanted:
                out[parts[0]]["rank"] = rank
                out[parts[0]]["count"] = int(parts[1])
    return [out[w] for w in targets]


def worked_examples(classes, tones):
    out = []
    for word, roman, claimed in POST_EXAMPLES:
        hits = analyse(word, classes)
        for hit in hits:
            derived = tones.get((hit["initial_class"], hit["mark"]))
            roman_derived = roman_tone(roman) if roman != "-" else None
            out.append({
                "word": word,
                "romanisation_in_post": roman,
                "tone_in_post": claimed,
                "mark": hit["mark"],
                "marked_letter": hit["marked_letter"],
                "marked_class": hit["marked_class"],
                "initial_letter": hit["initial_letter"],
                "initial_class": hit["initial_class"],
                "cluster": hit["cluster"],
                "tone_derived": derived,
                "agrees": derived == claimed,
                "romanisation_tone": roman_derived,
                "romanisation_agrees": roman_derived is None or roman_derived == derived,
            })
    return out


# --------------------------------------------------------------- verification

def check_quotes(force):
    rows, failures = [], []
    for q in QUOTES:
        status = fetch(q["url"], q["path"], force)
        page = page_haystack(read_text(q["path"]))
        found = normalise(q["quote"]) in page
        expected = q.get("status", 200)
        if status is not None and status != expected:
            found = False
            failures.append("%s answered %d, expected %d" % (q["url"], status, expected))
        rows.append({"id": q["id"], "claim": q["claim"], "quote": q["quote"],
                     "url": q["url"], "found": found, "status": expected})
        if not found:
            failures.append("quote gone from %s: %s" % (q["url"], q["quote"][:70]))
    return rows, failures


def compare(label, expected, actual, failures, path=""):
    if isinstance(expected, dict):
        for key in expected:
            compare(label, expected[key], actual.get(key) if isinstance(actual, dict) else None,
                    failures, path + "/" + str(key))
    elif isinstance(expected, list):
        if not isinstance(actual, list) or len(expected) != len(actual):
            failures.append("%s%s: list length %r != %r" % (
                label, path, len(actual) if isinstance(actual, list) else actual, len(expected)))
            return
        for i, (e, a) in enumerate(zip(expected, actual)):
            compare(label, e, a, failures, path + "[%d]" % i)
    elif isinstance(expected, (int, float)) and isinstance(actual, (int, float)):
        if abs(float(expected) - float(actual)) > 1e-9:
            failures.append("%s%s: %r != %r" % (label, path, actual, expected))
    elif expected != actual:
        failures.append("%s%s: %r != %r" % (label, path, actual, expected))


def build(force):
    fetch(CORPUS_URL, CORPUS_PATH, force)
    fetch(WIKI_URL, WIKI_PATH, force)
    raw = read_text(WIKI_PATH)
    classes = consonant_classes(raw)
    tones = tone_table(raw)
    corpus_stats = measure(CORPUS_PATH, classes, tones)
    return {
        "classes": classes,
        "tones": tones,
        "corpus_stats": corpus_stats,
        "alphabet": alphabet(classes),
        "examples": worked_examples(classes, tones),
        "word_ranks": word_ranks(CORPUS_PATH, RANK_WORDS),
        "tone_table": [
            {"class": cls, "mark": mark, "tone_page": tones.get((cls, mark)),
             "tone_post": POST_TONE_TABLE.get((cls, mark))}
            for cls, mark in POST_TONE_TABLE
        ],
        "corpus_sha256": sha256(CORPUS_PATH),
        "corpus_bytes": os.path.getsize(CORPUS_PATH),
    }


def write_dataset(built):
    stats = built["corpus_stats"]
    quotes, _ = check_quotes(force=True)
    data = {
        "slug": "thai-tone-marks",
        "method": "public-corpus",
        "checkedAt": "2026-10-26",
        "tools": {
            "python": "%d.%d.%d" % sys.version_info[:3],
            "libraries": "standard library only",
        },
        "inputs": [
            {"url": CORPUS_URL, "licence": CORPUS_LICENCE,
             "bytes": built["corpus_bytes"], "sha256": built["corpus_sha256"],
             "purpose": "50,000 Thai word forms with counts, OpenSubtitles 2018"},
            {"url": WIKI_URL, "licence": "CC BY-SA 4.0",
             "purpose": "consonant class per letter; class x mark tone table"},
            {"url": TP_TONES_URL,
             "purpose": "ThaiPod101's own tones chart and its example words"},
            {"url": API + "/content/access/levels/L3",
             "purpose": "level 3 is free and covers finals and syllable types"},
            {"url": API + "/content/access/levels/L4",
             "purpose": "level 4 is free and its subject is the mid class with the tone marks"},
            {"url": API + "/content/access/levels/L5",
             "purpose": "the free levels end after level 4 (403 PREMIUM_REQUIRED)"},
            {"url": API + "/subscriptions/plans",
             "purpose": "the web price, $14.99 a month and $79.99 a year"},
            {"file": "apps/api/src/content/seed/levels.ts",
             "purpose": "level 4 round L4:R3 'Mai tri & mai chattawa' on เป๊ะ, โอ๊ย, ก๋า, อ๋อ"},
            {"file": "apps/web/src/lib/learn/srs/schedule.ts",
             "purpose": "review intervals 1/3/7/21 days, mastery at three correct in a row"},
        ],
        "method_detail": (
            "Every word form in the frequency list whose characters are all Thai and which "
            "contains at least one consonant. For each tone mark (U+0E48..U+0E4B) the consonant "
            "it sits on is found by walking back over above-vowels and above-marks; the class "
            "that decides the tone is that of the syllable's first letter, so a silent ห in "
            "front of a low-class letter makes it high class and a true cluster (กร กล ... บล) "
            "is read from its first letter. The tone for each (class, mark) comes from the "
            "table on the Wikipedia page, not from memory."
        ),
        "results": {
            "corpus": stats["corpus"],
            "marks": stats["marks"],
            "coverage": stats["coverage"],
            "cluster_placement": stats["cluster_placement"],
            "ho_nam": stats["ho_nam"],
            "english_name_test": stats["english_name"],
            "alphabet": built["alphabet"],
            "tone_table": built["tone_table"],
            "worked_examples": built["examples"],
            "word_ranks": built["word_ranks"],
            "quotes": quotes,
        },
        "notes": NOTES,
    }
    with open(DATASET_JSON, "w", encoding="utf-8") as fh:
        json.dump(data, fh, ensure_ascii=False, indent=2)
        fh.write("\n")

    columns = ["symbol", "name", "thai_name", "occurrences", "weighted_occurrences",
               "share_pct_occurrence", "share_pct_weighted", "marked_class_mid",
               "marked_class_nonmid", "marked_nonmid_share_pct", "initial_mid",
               "initial_high", "initial_low", "initial_nonmid_share_pct",
               "initial_weighted_mid",
               "initial_weighted_high", "initial_weighted_low", "distinct_marked_letters",
               "distinct_mid_letters", "example_word", "example_word_count"]
    buf = io.StringIO()
    writer = csv.DictWriter(buf, fieldnames=columns, lineterminator="\n")
    writer.writeheader()
    for row in stats["marks"]:
        writer.writerow({k: row[k] for k in columns})
    with open(DATASET_CSV, "w", encoding="utf-8") as fh:
        fh.write(buf.getvalue())
    return data


NOTES = [
    "The word list is OpenSubtitles 2018 subtitles, so its register is spoken and informal. "
    "That inflates the colloquial spellings that break the class pattern (มั๊ย, ห๊ะ, โอเค๊, "
    "แท๊กซี่) and is the reason ไม้ตรี shows more non-mid initials than formal prose would. "
    "It also inflates เดี๋ยว, which is why ไม้จัตวา carries a larger share of running text than "
    "ไม้ตรี despite fewer occurrences.",
    "The list holds word forms, not syllables: a mark inside a compound is counted once per "
    "word form, weighted by that form's own count.",
    "A silent ห is detected as the single character immediately before the marked low-class "
    "letter. Whether that ห is the syllable's own initial or the tail of the previous syllable "
    "is not decided; 764 of 22,184 ไม้เอก occurrences (3.4%) qualify.",
    "A true cluster (กร กล กว ขร ขล ขว คร คล คว ตร ปร ปล ผร ผล พร พล ฟร ฟล บร บล, written "
    "adjacent) is read from its first letter even though the mark sits on the second. That "
    "moves เปล่า, ปล่อย and กรุ๊ป into the class of their first letter, which is where the "
    "tone comes from.",
    "The consonant classes and the tone outcomes come from the English Wikipedia article "
    "'Thai script' as fetched on 2026-10-26. The four mark totals and their class split were "
    "recounted once with an independently typed 44-letter class list and matched exactly; no "
    "second written source for the classes was cited.",
    "Tone outcomes are those of standard central Thai. Regional accents, and the extended "
    "uses of ไม้ตรี and ไม้จัตวา in colloquial writing, fall outside what the table models.",
    "The earlier claims that ThaiPod101 names the marks by English tone names and that its "
    "videos put a human voice on the vowel were cut: its tones page names tones, not marks, "
    "and the lesson audio needs an account. The post now says only what the tones page shows. "
    "The Google Translate sentence was cut: Google's help page does not say Translate reads "
    "Thai aloud. A sentence about the typing engine checking each pick as "
    "it is made was cut because TypingPractice.tsx only checks the finished word.",
    "The review intervals and mastery rule were read from apps/web/src/lib/learn/srs/schedule.ts "
    "(SRS_INTERVALS = [1, 3, 7, 21], SRS_MASTERY_COUNT = 3); check.py does not re-read source "
    "files. Level 4's round on the two rare marks was read from apps/api/src/content/seed/levels.ts "
    "(L4:R3). Progress without an account rests on the product manifest.",
]


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--recheck", action="store_true",
                    help="re-download every cited page and the corpus, then verify")
    ap.add_argument("--write", action="store_true",
                    help="measure and rewrite dataset.json and dataset.csv")
    args = ap.parse_args()

    os.makedirs(WORK, exist_ok=True)
    force = args.recheck or args.write

    built = build(force)
    if args.write:
        write_dataset(built)
        print("wrote %s and %s" % (DATASET_JSON, DATASET_CSV))
        return 0

    with open(DATASET_JSON, encoding="utf-8") as fh:
        stored = json.load(fh)

    failures = []
    if built["corpus_sha256"] != stored["inputs"][0]["sha256"]:
        failures.append("word list changed: sha256 %s, dataset has %s"
                        % (built["corpus_sha256"], stored["inputs"][0]["sha256"]))
    quotes, quote_failures = check_quotes(force)
    failures.extend(quote_failures)
    for row in quotes:
        print("quote  %-28s %s" % (row["id"], "found" if row["found"] else "MISSING"))

    compare("quotes", stored["results"]["quotes"], quotes, failures)
    compare("corpus", stored["results"]["corpus"], built["corpus_stats"]["corpus"], failures)
    compare("marks", stored["results"]["marks"], built["corpus_stats"]["marks"], failures)
    compare("coverage", stored["results"]["coverage"], built["corpus_stats"]["coverage"], failures)
    compare("cluster_placement", stored["results"]["cluster_placement"],
            built["corpus_stats"]["cluster_placement"], failures)
    compare("ho_nam", stored["results"]["ho_nam"], built["corpus_stats"]["ho_nam"], failures)
    compare("english_name_test", stored["results"]["english_name_test"],
            built["corpus_stats"]["english_name"], failures)
    compare("alphabet", stored["results"]["alphabet"], built["alphabet"], failures)
    compare("tone_table", stored["results"]["tone_table"], built["tone_table"], failures)
    compare("worked_examples", stored["results"]["worked_examples"], built["examples"], failures)
    compare("word_ranks", stored["results"]["word_ranks"], built["word_ranks"], failures)

    for row in built["tone_table"]:
        if row["tone_page"] != row["tone_post"]:
            failures.append("post's table disagrees with %s at %s + %s: post says %r, page says %r"
                            % (WIKI_URL, row["class"], row["mark"], row["tone_post"],
                               row["tone_page"]))
    for row in built["examples"]:
        if not row["agrees"]:
            failures.append("worked example %s: post says %r, derivation gives %r"
                            % (row["word"], row["tone_in_post"], row["tone_derived"]))
        if not row["romanisation_agrees"]:
            failures.append("worked example %s: romanisation %r reads as %r, derivation gives %r"
                            % (row["word"], row["romanisation_in_post"],
                               row["romanisation_tone"], row["tone_derived"]))

    print("corpus %s  sha256 %s" % (os.path.basename(CORPUS_PATH), built["corpus_sha256"][:16]))
    for row in built["corpus_stats"]["marks"]:
        print("mark   %-13s occ %6d  weighted %8d  %5.3f%% by tokens  mid-initial %d"
              % (row["name"], row["occurrences"], row["weighted_occurrences"],
                 row["share_pct_weighted"], row["initial_mid"]))

    if failures:
        print("\nFAIL")
        for line in failures:
            print("  - " + line)
        return 1
    print("\nOK: every quote is still on its page and every number still holds")
    return 0


if __name__ == "__main__":
    sys.exit(main())
