#!/usr/bin/env python3
"""Evidence check for the Phayan post `thai-tone-rules`.

1. Corpus: the post's tone table is applied to every single-syllable word in
   Phayan's word list (apps/api/src/content/seed/kaikki-enriched.jsonl, the Thai
   entries of English Wiktionary as extracted by Kaikki.org). The table's inputs
   (consonant class, live or dead, vowel length, tone mark) are read from the
   spelling; the tone the word is actually said with is read from the tone
   numerals in the entry's own IPA. Every miss is weighted by its Thai National
   Corpus count from apps/api/src/content/seed/word-frequency.jsonl.
2. Product: the Phayan facts the post states are re-read from the plans endpoint
   and from the repository files that define them.
3. Pages: the two sentences the post quotes from ThaiPod101 and Ling are
   fetched from their own pages and matched against the recorded quote.

    python3 check.py            measure everything, compare with dataset.json and dataset.csv
    python3 check.py --recheck  the same; the corpus step takes a few seconds
    python3 check.py --write    regenerate dataset.json and dataset.csv

Exit 0 when every value and quote holds, 1 otherwise. Standard library only.
"""

import argparse
import csv
import hashlib
import html
import json
import os
import re
import subprocess
import sys
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
REPO_ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", ".."))
WORDS_FILE = "apps/api/src/content/seed/kaikki-enriched.jsonl"
FREQ_FILE = "apps/api/src/content/seed/word-frequency.jsonl"
DATASET_JSON = os.path.join(HERE, "dataset.json")
DATASET_CSV = os.path.join(HERE, "dataset.csv")
CHECKED_AT = "2026-10-26"
PLANS_URL = "https://api.phayan.com/subscriptions/plans"
UA = "PhayanBlogEvidence/1.0 (https://www.phayan.com/blog; evidence recheck script)"

# --------------------------------------------------------------- Thai spelling

MID = "กจฎฏดตบปอ"
HIGH = "ขฃฉฐถผฝศษสห"
LOW = "คฅฆงชซฌญฑฒณทธนพฟภมยรลวฮ"
MARKS = {"่": 1, "้": 2, "๊": 3, "๋": 4}
MARK_NAMES = {0: "none", 1: "mai ek", 2: "mai tho", 3: "mai tri", 4: "mai chattawa"}
SONORANT_FINAL = set("งนมยวญณรลฬ")
STOP_FINAL = set("กขคฆดฎฏตฐฒฑธทบปพภฟสศษซชฌฮ")
# The ร of a Sanskrit ตร/จร/กร ending is not pronounced; the stop before it is the
# final, which is why เมตร is a dead syllable and not a live one.
SILENT_R_AFTER = set("ฎฏดกตถทธศสปพบบภฟ")
TONE_NUMERALS = {"˧": 1, "˨˩": 2, "˥˩": 3, "˦˥": 4, "˩˩˦": 5}
TONE_NAME = {1: "mid", 2: "low", 3: "falling", 4: "high", 5: "rising"}


def consonant_class(ch):
    if ch in MID:
        return "mid"
    if ch in HIGH:
        return "high"
    if ch in LOW:
        return "low"
    return None


def syllable_inputs(word):
    """The three inputs of the post's table, read from the spelling alone."""
    word = word.replace("์", "")  # thanthakhat silences, it is not a vowel
    letters = [(i, c) for i, c in enumerate(word) if consonant_class(c)]
    if not letters:
        return None
    initial = letters[0][1]
    vowels = [i for i, c in enumerate(word) if c in "ะาำิีึืุูั็เแโใไอ"]
    after = [(i, c) for i, c in letters[1:] if vowels and i > max(vowels)]
    if not vowels:
        after = list(letters[1:])
    final = after[-1][1] if after else None
    if final == "ร" and len(after) > 1 and after[-2][1] in SILENT_R_AFTER:
        final = after[-2][1]
    has_final = final is not None

    if any(v in word for v in "ะิึุั"):
        length = "short"
    elif any(v in word for v in "าีืู"):
        length = "long"
    elif "ำ" in word or "ัว" in word or "อ" in word[1:] or "ว" in word[1:]:
        length = "long"
    elif any(v in word for v in "เแโใไ"):
        length = "short" if "ะ" in word else "long"
    elif "็" in word:
        length = "short"
    else:
        length = "long" if not has_final else "short"

    if has_final:
        if final in SONORANT_FINAL:
            dead = False
        elif final in STOP_FINAL:
            dead = True
        else:
            return None
    else:
        dead = length == "short"

    mark = next((MARKS[c] for c in word if c in MARKS), 0)
    return consonant_class(initial), dead, length, mark


def table(cls, dead, length, mark):
    """The post's table, cell for cell."""
    if mark:
        return {
            "mid": {1: 2, 2: 3, 3: 4, 4: 5},
            "high": {1: 2, 2: 3},
            "low": {1: 3, 2: 4},
        }[cls].get(mark)
    if cls == "mid":
        return 2 if dead else 1
    if cls == "high":
        return 2 if dead else 5
    if dead:
        return 4 if length == "short" else 3
    return 1


def ipa_tone(ipa):
    body = ipa.strip("/")
    runs = re.findall(r"[˥˦˧˨˩]+", body)
    if len(runs) != 1:
        return None
    return TONE_NUMERALS.get(runs[0])


# A word that fails is one Wiktionary borrows, spells after a Latin letter, or
# spells in a way the table does not cover. The category is read off the gloss,
# which is carried into dataset.csv so a reader can check the label.
CATEGORY = {
    "ก็": "native",
    "เมรุ": "native-name",
    "ฬ่อ": "obsolete-spelling",
    "ช่ะ": "native",
    "เอส": "latin-letter-name",
    "เอฟ": "latin-letter-name",
    "เอช": "latin-letter-name",
    "เฮช": "latin-letter-name",
}
DEFAULT_CATEGORY = "loanword"


def read_jsonl(rel):
    with open(os.path.join(REPO_ROOT, rel), encoding="utf-8") as fh:
        return [json.loads(line) for line in fh if line.strip()]


def run_corpus():
    words = read_jsonl(WORDS_FILE)
    freq = {r["word"]: r for r in read_jsonl(FREQ_FILE)}
    checked = correct = 0
    marked = [0, 0]
    unmarked = [0, 0]
    exceptions = []
    skipped = {"multi-syllable": 0, "no-tone-in-ipa": 0, "unparsed-spelling": 0}
    for r in words:
        ipa = r.get("ipa") or ""
        if "." in ipa or len(re.findall(r"[˥˦˧˨˩]+", ipa)) != 1:
            skipped["multi-syllable"] += 1
            continue
        tone = ipa_tone(ipa)
        if tone is None:
            skipped["no-tone-in-ipa"] += 1
            continue
        inputs = syllable_inputs(r["word"])
        if inputs is None:
            skipped["unparsed-spelling"] += 1
            continue
        cls, dead, length, mark = inputs
        predicted = table(cls, dead, length, mark)
        if predicted is None:
            skipped["unparsed-spelling"] += 1
            continue
        checked += 1
        bucket = marked if mark else unmarked
        bucket[0] += 1
        if predicted == tone:
            correct += 1
            bucket[1] += 1
        else:
            exceptions.append({
                "word": r["word"],
                "ipa": ipa,
                # Wiktionary glosses use a dash before examples; the site style has none.
                "gloss": r["gloss"].replace(" \u2014 ", "; "),
                "consonant_class": cls,
                "ending": "dead" if dead else "live",
                "vowel_length": length,
                "tone_mark": MARK_NAMES[mark],
                "predicted_tone": TONE_NAME[predicted],
                "actual_tone": TONE_NAME[tone],
                "category": CATEGORY.get(r["word"], DEFAULT_CATEGORY),
                "tnc_frequency": (freq.get(r["word"]) or {}).get("frequency"),
            })
    exceptions.sort(key=lambda e: (-(e["tnc_frequency"] or 0), e["word"]))
    tnc_total = sum((freq.get(r["word"]) or {}).get("frequency") or 0 for r in words)
    tnc_exc = sum(e["tnc_frequency"] or 0 for e in exceptions)
    for e in exceptions:
        e["share_of_exception_occurrences_pct"] = round(100 * (e["tnc_frequency"] or 0) / tnc_exc, 2)
    ko = next(e for e in exceptions if e["word"] == "ก็")
    return {
        "words_in_list": len(words),
        "checked": checked,
        "correct": correct,
        "accuracy_pct": round(100 * correct / checked, 2),
        "exceptions": len(exceptions),
        "marked_checked": marked[0],
        "marked_correct": marked[1],
        "marked_correct_pct": round(100 * marked[1] / marked[0], 2),
        "unmarked_checked": unmarked[0],
        "unmarked_correct": unmarked[1],
        "unmarked_correct_pct": round(100 * unmarked[1] / unmarked[0], 2),
        "latin_letter_names": sum(1 for e in exceptions if e["category"] == "latin-letter-name"),
        "loanwords": sum(1 for e in exceptions if e["category"] == "loanword"),
        "native_or_obsolete": sum(
            1 for e in exceptions
            if e["category"] in ("native", "native-name", "obsolete-spelling")
        ),
        "predicted_to_actual_high": sum(1 for e in exceptions if e["actual_tone"] == "high"),
        "skipped": skipped,
        "tnc_total_of_list": tnc_total,
        "tnc_of_exceptions": tnc_exc,
        "tnc_exception_share_pct": round(100 * tnc_exc / tnc_total, 3),
        "ko_tnc_frequency": ko["tnc_frequency"],
        "ko_share_of_exception_occurrences_pct": ko["share_of_exception_occurrences_pct"],
        "exception_rows": exceptions,
    }


# ------------------------------------------------------------------ product facts

PRODUCT_FACTS = [
    {
        "id": "free-levels",
        "claim": "The free path runs to level 4, where the mid class and the tone marks arrive together.",
        "file": "packages/shared/src/learn-access.ts",
        "quote": "export const FIRST_PAID_LEVEL = 5;",
    },
    {
        "id": "level-4-title",
        "claim": "Level 4 is the mid class and the tone marks.",
        "file": "apps/api/src/content/seed/levels.ts",
        "quote": "title: 'Mid class + tone marks',",
    },
    {
        "id": "level-3-title",
        "claim": "Finals and syllable types are their own free level before the tone marks.",
        "file": "apps/api/src/content/seed/levels.ts",
        "quote": "title: 'Finals & syllable types',",
    },
    {
        "id": "srs-intervals",
        "claim": "The review schedule brings an item back at 1, 3, 7 and 21 days.",
        "file": "apps/web/src/lib/learn/srs/schedule.ts",
        "quote": "export const SRS_INTERVALS = [1, 3, 7, 21] as const;",
    },
    {
        "id": "srs-mastery",
        "claim": "Three correct answers in a row mark an item as learned.",
        "file": "apps/web/src/lib/learn/srs/schedule.ts",
        "quote": "export const SRS_MASTERY_COUNT = 3;",
    },
    {
        "id": "srs-reset",
        "claim": "One wrong answer restarts the ladder.",
        "file": "apps/web/src/lib/learn/srs/schedule.ts",
        "quote": "correctCount: 0,",
    },
    {
        "id": "price",
        "claim": "Beyond the free levels it is $14.99 a month or $79.99 a year on the web.",
        "url": PLANS_URL,
        "quote": '"pricing":{"monthly":14.99,"annual":79.99,"currency":"USD"}',
    },
]


PAGE_QUOTES = [
    {
        "id": "thaipod101-rules-optional",
        "party": "ThaiPod101",
        "claim": "ThaiPod101's guide to Thai tones says you don't need to fully understand these rules and exceptions to pronounce Thai tones correctly.",
        "url": "https://www.thaipod101.com/blog/2021/01/18/thai-tones/",
        "quote": "So you don’t need to fully understand these rules and exceptions to pronounce Thai tones correctly.",
    },
    {
        "id": "thaipod101-children",
        "party": "ThaiPod101",
        "claim": "The same guide points out that Thai children meet the rules later.",
        "url": "https://www.thaipod101.com/blog/2021/01/18/thai-tones/",
        "quote": "When Thai children learn how to pronounce tones, they don’t learn about lessons like this until later.",
    },
    {
        "id": "ling-classes-and-marks",
        "party": "Ling",
        "claim": "Ling's Thai page describes consonant classes that determine tone and tone marks that interact with those classes.",
        "url": "https://ling-app.com/learn-language/thai/",
        "quote": "consonant classes that determine tone, vowels that follow positional rules, and tone marks that interact with those classes",
    },
]


def html_to_text(raw):
    raw = re.sub(r"(?is)<(script|style|noscript|svg)[^>]*>.*?</\1>", " ", raw)
    raw = re.sub(r"(?s)<!--.*?-->", " ", raw)
    raw = re.sub(r"(?s)<[^>]+>", " ", raw)
    return html.unescape(raw)


def normalise(text):
    text = text.replace("\u2018", "'").replace("\u2019", "'")
    text = text.replace("\u201c", '"').replace("\u201d", '"')
    return re.sub(r"\s+", " ", text).strip().lower()


def check_pages():
    out, cache = [], {}
    for q in PAGE_QUOTES:
        row = dict(q)
        try:
            if q["url"] not in cache:
                cache[q["url"]] = normalise(html_to_text(fetch(q["url"])))
            row["holds"] = normalise(q["quote"]) in cache[q["url"]]
        except Exception as exc:  # noqa: BLE001 - an unreadable page is a failed check
            row["holds"] = False
            row["error"] = str(exc)
        out.append(row)
    return out


def fetch(url, timeout=40):
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return r.read().decode("utf-8", "replace")


def check_facts():
    out = []
    for fact in PRODUCT_FACTS:
        row = dict(fact)
        try:
            if "url" in fact:
                text = re.sub(r"\s+", "", fetch(fact["url"]))
            else:
                with open(os.path.join(REPO_ROOT, fact["file"]), encoding="utf-8") as fh:
                    text = fh.read()
            row["holds"] = fact["quote"] in text
        except Exception as exc:  # noqa: BLE001 - an unreadable source is a failed check
            row["holds"] = False
            row["error"] = str(exc)
        out.append(row)
    return out


# ---------------------------------------------------------------------- record

CSV_FIELDS = [
    "word", "ipa", "gloss", "consonant_class", "ending", "vowel_length", "tone_mark",
    "predicted_tone", "actual_tone", "category", "tnc_frequency",
    "share_of_exception_occurrences_pct",
]


def sha256(rel):
    with open(os.path.join(REPO_ROOT, rel), "rb") as fh:
        return hashlib.sha256(fh.read()).hexdigest()


def last_commit(rel):
    res = subprocess.run(["git", "-C", REPO_ROOT, "log", "-1", "--format=%H", "--", rel],
                         capture_output=True, text=True)
    return res.stdout.strip() or None


def write():
    corpus = run_corpus()
    facts = check_facts()
    pages = check_pages()
    data = {
        "slug": "thai-tone-rules",
        "question": "How do Thai tone rules work?",
        "method": "public-corpus",
        "checkedAt": CHECKED_AT,
        "methodDetail": (
            "The post's table (consonant class x live or dead ending x tone mark, with vowel length "
            "splitting the low-class dead cell) is applied to every single-syllable word in Phayan's "
            "word list, which is built from the Thai entries of English Wiktionary as extracted by "
            "Kaikki.org. The inputs are read from the spelling; the tone the word is said with is read "
            "from the tone numerals in the entry's IPA (˧ mid, ˨˩ low, ˥˩ falling, ˦˥ high, ˩˩˦ rising). "
            "Words whose IPA has more than one syllable are skipped. Each miss is weighted by its Thai "
            "National Corpus count (PyThaiNLP tnc_freq, as committed in the repository). The Phayan "
            "facts the post states are re-read from the plans endpoint and the files that define them."
        ),
        "tools": {
            "python": "3.9.6",
            "libraries": "standard library only (json, re, csv, hashlib, urllib, subprocess)",
        },
        "inputs": [
            {
                "id": "word-list",
                "role": "Thai words with IPA, from English Wiktionary via Kaikki.org (CC BY-SA)",
                "file": WORDS_FILE,
                "sha256": sha256(WORDS_FILE),
                "commit": last_commit(WORDS_FILE),
                "upstream": "https://kaikki.org/dictionary/Thai/",
            },
            {
                "id": "tnc-frequency",
                "role": "Thai National Corpus counts for the same words, PyThaiNLP tnc_freq",
                "file": FREQ_FILE,
                "sha256": sha256(FREQ_FILE),
                "commit": last_commit(FREQ_FILE),
                "upstream": "https://github.com/PyThaiNLP/pythainlp",
            },
            {"id": "plans", "role": "Phayan web prices", "url": PLANS_URL},
            {"id": "learn-access", "role": "Phayan free/paid boundary",
             "file": "packages/shared/src/learn-access.ts"},
            {"id": "levels", "role": "Phayan level titles", "file": "apps/api/src/content/seed/levels.ts"},
            {"id": "srs", "role": "Phayan review schedule", "file": "apps/web/src/lib/learn/srs/schedule.ts"},
            {"id": "thaipod101-tones", "role": "ThaiPod101's own guide to Thai tones",
             "url": "https://www.thaipod101.com/blog/2021/01/18/thai-tones/"},
            {"id": "ling-thai", "role": "Ling's own Thai page", "url": "https://ling-app.com/learn-language/thai/"},
        ],
        "results": {
            "corpus": {k: v for k, v in corpus.items() if k != "exception_rows"},
            "exceptions": corpus["exception_rows"],
            "productFacts": [{k: v for k, v in f.items() if k != "error"} for f in facts],
            "pageQuotes": [{k: v for k, v in q.items() if k != "error"} for q in pages],
        },
        "notes": [
            "The share columns are shares of the Thai National Corpus count of the 2,249 words in the word list, not of the whole corpus.",
            "Two misses are not the table's fault: เมรุ ends in a silent ร that the parser reads as a final, and ฬ่อ is an obsolete spelling. Both stay in the list with their gloss so a reader can judge them.",
            "Categories (loanword, Latin letter name, native) are assigned by hand from each entry's gloss; the list is in CATEGORY in check.py.",
            "Glosses are Wiktionary's, with its dash before example sentences replaced by a semicolon.",
            "Not re-run: the earlier version of the post reported a 254-word check of the leading-letter rule (88% right) and a run on 1,844 words with sub-figures of 99.7% and 98.0%. Neither was published with its word list, so both figures were removed from the post.",
            "The upstream PyThaiNLP tnc_freq file was not fetched on the check date; the counts are the repository's committed snapshot, identified by its sha256 and commit.",
            "Not checked by this script: that Phayan's audio is synthesised. That comes from the product's content profile, not from a file in this repository.",
            "Removed from the post rather than kept: that ThaiPod101 publishes the table as a chart to memorise up front and teaches tone rules on video, that an Anki deck of the table teaches the table rather than the reading, and that Ling and Drops lean on romanised prompts. None had a source. ThaiPod101's own tone guide, read on the check date, says the opposite of the first (see pageQuotes).",
        ],
    }
    with open(DATASET_JSON, "w", encoding="utf-8") as fh:
        fh.write(json.dumps(data, ensure_ascii=False, indent=1) + "\n")
    with open(DATASET_CSV, "w", encoding="utf-8", newline="") as fh:
        w = csv.DictWriter(fh, fieldnames=CSV_FIELDS)
        w.writeheader()
        for row in corpus["exception_rows"]:
            w.writerow({k: row[k] for k in CSV_FIELDS})
    print("wrote %d words, %d exceptions, %d product facts, %d page quotes" % (
        corpus["checked"], corpus["exceptions"], len(facts), len(pages)))
    return 0 if all(f["holds"] for f in facts + pages) else 1


# ---------------------------------------------------------------------- verify

def verify():
    failures = []
    with open(DATASET_JSON, encoding="utf-8") as fh:
        stored = json.load(fh)
    for inp in stored["inputs"]:
        if "sha256" in inp and sha256(inp["file"]) != inp["sha256"]:
            failures.append("%s changed since the check (sha256 differs)" % inp["file"])
    live = run_corpus()
    for key, value in stored["results"]["corpus"].items():
        if live.get(key) != value:
            failures.append("corpus.%s: dataset %r != live %r" % (key, value, live.get(key)))
    if stored["results"]["exceptions"] != live["exception_rows"]:
        failures.append("results.exceptions differ from a live run")
    with open(DATASET_CSV, encoding="utf-8") as fh:
        csv_rows = list(csv.DictReader(fh))
    if len(csv_rows) != live["exceptions"]:
        failures.append("dataset.csv has %d rows, live run has %d" % (len(csv_rows), live["exceptions"]))
    for row, got in zip(csv_rows, live["exception_rows"]):
        for key in CSV_FIELDS:
            if row[key] != str(got[key]):
                failures.append("dataset.csv %s: %s %r != %r" % (row["word"], key, row[key], got[key]))
    for fact in check_facts() + check_pages():
        if not fact["holds"]:
            where = fact.get("url") or fact.get("file")
            failures.append("%s: quote gone from %s: %r %s" % (fact["id"], where, fact["quote"], fact.get("error", "")))
    if failures:
        for f in failures:
            print("FAIL", f)
        return 1
    print("ok: %d words, %d correct (%s%%), %d exceptions, ก็ %s%% of their occurrences, "
          "%d product facts and %d page quotes hold" % (
              live["checked"], live["correct"], live["accuracy_pct"], live["exceptions"],
              live["ko_share_of_exception_occurrences_pct"], len(PRODUCT_FACTS), len(PAGE_QUOTES)))
    return 0


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--recheck", action="store_true", help="re-read the pages and files (the corpus step is fast, so nothing is skipped)")
    ap.add_argument("--write", action="store_true", help="regenerate dataset.json and dataset.csv")
    args = ap.parse_args()
    return write() if args.write else verify()


if __name__ == "__main__":
    sys.exit(main())
