#!/usr/bin/env python3
"""Re-runs the check behind "Colours in Thai: สี Plus One More Word".

Method: public-corpus. For each colour word the script reads the written
spelling, finds its initial consonant and tone mark, takes the consonant class
from the Thai consonant table on Wikipedia, takes the spoken tone from the
dictionary's IPA respelling on Wiktionary, applies the standard Thai tone table
(class x live/dead x tone mark) and compares the two.

  python3 check.py            verify the live pages against dataset.json
  python3 check.py --recheck  same thing (this check has no lab step to skip)
  python3 check.py --write    rebuild dataset.json and dataset.csv

Exit 0 when every quote is still on its page and every derived tone matches
the dictionary; 1 otherwise.
"""

import concurrent.futures
import csv
import html as htmllib
import json
import os
import re
import sys
import unicodedata
import urllib.parse
import urllib.request
from html.parser import HTMLParser

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET_JSON = os.path.join(HERE, "dataset.json")
DATASET_CSV = os.path.join(HERE, "dataset.csv")

CHECKED_AT = "2026-10-26"
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) phayan-evidence-check/1.0"

WIKTIONARY = "https://en.wiktionary.org/wiki/"
# Phayan's own word list, read from the repository this file ships in.
PHAYAN_WORDS = os.path.normpath(
    os.path.join(HERE, "..", "..", "..", "..", "api", "src", "content", "seed", "words.ts")
)
PHAYAN_TONES = {"1": "mid", "2": "low", "3": "falling", "4": "high", "5": "rising"}
CONSONANT_TABLE_DOC = "https://en.wikipedia.org/wiki/Thai_script"

CONSONANTS = (
    "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ"
)
TONE_MARKS = {
    "่": "ek",        # ไม้เอก
    "้": "tho",       # ไม้โท
    "๊": "tri",       # ไม้ตรี
    "๋": "chattawa",  # ไม้จัตวา
}
TONE_CHARS = "˥˦˧˨˩"  # ˥ ˦ ˧ ˨ ˩
SONORANTS = set("mnŋɲjwɹl")
STOPS = set("ptkʔbdg")

# The standard Thai tone table, exactly the cells this corpus needs.
# Source for every cell used: the "Tone chart" on Wikipedia's Thai script
# article (class x ending x mark -> tone).
TONE_TABLE = {
    ("mid", "live", None): "mid",
    ("mid", "live", "ek"): "low",
    ("mid", "live", "tho"): "falling",
    ("mid", "live", "tri"): "high",
    ("mid", "live", "chattawa"): "rising",
    ("mid", "dead", None): "low",
    ("mid", "dead", "ek"): "low",
    ("mid", "dead", "tho"): "falling",
    ("high", "live", None): "rising",
    ("high", "live", "ek"): "low",
    ("high", "live", "tho"): "falling",
    ("high", "dead", None): "low",
    ("high", "dead", "ek"): "low",
    ("high", "dead", "tho"): "falling",
    ("low", "live", None): "mid",
    ("low", "live", "ek"): "falling",
    ("low", "live", "tho"): "high",
    ("low", "dead-short", None): "high",
    ("low", "dead-short", "ek"): "falling",
    ("low", "dead-long", None): "falling",
    ("low", "dead-long", "tho"): "high",
}

# (group, spelling, English name in the post, syllables)
WORDS = [
    ("colour", "แดง", "red", ["แดง"]),
    ("colour", "เหลือง", "yellow", ["เหลือง"]),
    ("colour", "เขียว", "green", ["เขียว"]),
    ("colour", "ฟ้า", "sky blue", ["ฟ้า"]),
    ("colour", "น้ำเงิน", "deep blue", ["น้ำ", "เงิน"]),
    ("colour", "ดำ", "black", ["ดำ"]),
    ("colour", "ขาว", "white", ["ขาว"]),
    ("colour", "ส้ม", "orange", ["ส้ม"]),
    ("colour", "ชมพู", "pink", ["ชม", "พู"]),
    ("colour", "ม่วง", "purple", ["ม่วง"]),
    ("colour", "น้ำตาล", "brown", ["น้ำ", "ตาล"]),
    ("colour", "เทา", "grey", ["เทา"]),
    ("colour", "ทอง", "gold", ["ทอง"]),
    ("mark-pair", "สี", "colour", ["สี"]),
    ("mark-pair", "สี่", "four", ["สี่"]),
]

# Statements the post makes about other companies' products. Verdicts live in
# notes.md; the quotes here are the sentences those pages actually carry today.
THIRD_PARTY = [
    {
        "post_line": 276,
        "post_quote": "Drops will drill the same eleven words as pictures far faster",
        "party": "Drops",
        "url": "https://languagedrops.com/language/learn-thai",
        "quote": "Drops helps to create memorable visual associations with new words",
        "verdict": "cut",
        "why": "the page supports pictures, not that they are faster than reading, and it never names the colour words",
    },
    {
        "post_line": 286,
        "post_quote": "ThaiPod101 covers the spoken exchange around it in more depth",
        "party": "ThaiPod101",
        "url": "https://www.thaipod101.com/thai-vocabulary-lists/colors/",
        "quote": "สีโปรดของคุณคือสีอะไร What's your favorite color?",
        "verdict": "rewritten",
        "why": "the 'spoken exchange in more depth' claim was cut; the FAQ now says only what the page shows: a 20-word Colors list that uses สีอะไร in this example sentence",
    },
    {
        "post_line": 286,
        "post_quote": "ThaiPod101's own Colors list of 20 words",
        "party": "ThaiPod101",
        "url": "https://www.thaipod101.com/thai-vocabulary-lists/colors/",
        "quote": "Colors 20 words",
        "verdict": "rewritten",
        "why": "the word count the FAQ gives for the list",
    },
    {
        "post_line": 308,
        "post_quote": "Anki is the better tool if you have the reading already",
        "party": "Anki",
        "url": "https://apps.ankiweb.net/",
        "quote": "Anki is a flashcard program that helps you spend more time on challenging material, and less on what you already know.",
        "verdict": "rewritten",
        "why": "'the better tool' was cut; the FAQ now names Anki only as what its own page says it is, a flashcard program",
    },
]


# --------------------------------------------------------------------------
# fetching and text normalisation
# --------------------------------------------------------------------------

_cache = {}


def _fetch_now(url):
    req = urllib.request.Request(
        url,
        headers={"User-Agent": USER_AGENT, "Accept-Encoding": "identity"},
    )
    with urllib.request.urlopen(req, timeout=45) as r:
        raw = r.read()
    return raw.decode("utf-8", "replace")


def fetch(url):
    if url not in _cache:
        prefetch([url])
    result = _cache[url]
    if isinstance(result, Exception):
        raise result
    return result


def prefetch(urls):
    """One pass over every page the run needs, eight at a time."""
    todo = [u for u in dict.fromkeys(urls) if u not in _cache]

    def one(url):
        try:
            return url, _fetch_now(url)
        except Exception as exc:  # noqa: BLE001
            return url, exc

    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
        for url, result in pool.map(one, todo):
            _cache[url] = result


def strip_tags(html_text):
    text = re.sub(r"(?is)<(script|style)\b.*?</\1>", " ", html_text)
    text = re.sub(r"<[^>]+>", " ", text)
    return htmllib.unescape(text)


def norm(text):
    """Whitespace, quotes and entities normalised, so a quote can be compared."""
    text = unicodedata.normalize("NFC", text)
    text = text.replace("‘", "'").replace("’", "'")
    text = text.replace("“", '"').replace("”", '"')
    text = text.replace(" ", " ")
    return re.sub(r"\s+", " ", text).strip()


def normalised_body(html_text):
    return norm(strip_tags(html_text))


# --------------------------------------------------------------------------
# orthography -> the three inputs of the tone table
# --------------------------------------------------------------------------

def initial_of(syllable):
    for ch in syllable:
        if ch in CONSONANTS:
            return ch
    raise ValueError("no consonant in %r" % syllable)


def mark_of(syllable):
    found = [TONE_MARKS[ch] for ch in syllable if ch in TONE_MARKS]
    if len(found) > 1:
        raise ValueError("two tone marks in %r" % syllable)
    return found[0] if found else None


def rime_type(ipa_syllable):
    """live / dead-short / dead-long, read off the IPA rime."""
    body = "".join(ch for ch in ipa_syllable if ch not in TONE_CHARS)
    body = body.strip("/").strip()
    if body.endswith("̃"):  # stray combining mark
        body = body[:-1]
    long_vowel = "ː" in body  # ː
    bare = body.rstrip("ː")
    last = bare[-1] if bare else ""
    if last in SONORANTS:
        return "live"
    if last in STOPS:
        return "dead-long" if long_vowel else "dead-short"
    return "live" if long_vowel else "dead-short"


def tone_of(ipa_syllable):
    marks = "".join(ch for ch in ipa_syllable if ch in TONE_CHARS)
    return {
        "˧": "mid",              # ˧
        "˩˩˦": "rising",  # ˩˩˦
        "˨˦": "rising",        # ˨˦
        "˦˥": "high",          # ˦˥
        "˥˩": "falling",       # ˥˩
        "˨˩": "low",           # ˨˩
        "˩": "low",                 # ˩
        "˨": "low",                 # ˨
        "˩˩": "low",           # ˩˩
        "˦": "high",                # ˦
        "˥": "high",                # ˥
    }.get(marks)


def ipa_syllables(ipa):
    body = ipa.strip().strip("/")
    parts = [p for p in body.split(".") if p]
    return parts


# --------------------------------------------------------------------------
# Wikipedia consonant classes
# --------------------------------------------------------------------------

def consonant_classes(html_text):
    """The Thai consonant table: letter -> mid/high/low."""
    idx = html_text.find("Symbol")
    while idx != -1:
        start = html_text.rfind("<table", 0, idx)
        end = html_text.find("</table>", idx)
        if start != -1 and end != -1:
            tbl = html_text[start:end]
            classes = {}
            for row in re.split(r"<tr", tbl):
                cells = re.findall(r"<t[dh][^>]*>(.*?)</t[dh]>", row, re.S)
                cells = [re.sub(r"\s+", " ", strip_tags(c)).strip() for c in cells]
                if len(cells) >= 9:
                    letter = cells[0].strip()
                    cls = cells[-1].strip().lower()
                    if letter in CONSONANTS and cls in ("mid", "high", "low"):
                        classes[letter] = (cls, " ".join(c for c in cells if c))
            if len(classes) >= 40:
                return classes
        idx = html_text.find("Symbol", idx + 1)
    raise RuntimeError("consonant table not found on %s" % CONSONANT_TABLE_DOC)


# --------------------------------------------------------------------------
# Wiktionary pronunciation
# --------------------------------------------------------------------------

def raw_url(word):
    return "https://en.wiktionary.org/w/index.php?title=" + urllib.parse.quote(word) + "&action=raw"


def thai_section(html_text):
    m = re.search(r'<h2[^>]*id="Thai"', html_text)
    if not m:
        raise RuntimeError("no Thai section")
    rest = html_text[m.start() + 5:]
    nxt = re.search(r"<h2", rest)
    return html_text[m.start(): m.start() + 5 + nxt.start()] if nxt else html_text[m.start():]


def dictionary_ipa(html_text):
    sec = thai_section(html_text)
    ipas = re.findall(r'<span[^>]*class="IPA"[^>]*>([^<]+)</span>', sec)
    for cand in ipas:
        text = norm(cand)
        if text.startswith("/") and text.endswith("/") and any(c in text for c in TONE_CHARS):
            return text
    raise RuntimeError("no toned IPA in the Thai section")


# --------------------------------------------------------------------------
# other senses, for the post's "what that word is otherwise" column
# --------------------------------------------------------------------------

def other_senses(wikitext):
    m = re.search(r"^==Thai==$", wikitext, re.M)
    sec = wikitext[m.start():] if m else wikitext
    n = re.search(r"^==[^=]", sec[8:], re.M)
    sec = sec[: 8 + n.start()] if n else sec
    out = []
    for line in re.findall(r"^# ([^\n]+)", sec, re.M):
        labels = re.findall(r"\{\{lb\|th\|([^}]*)\}\}", line)
        clean = re.sub(r"\{\{lb\|th\|[^}]*\}\}", " ", line)
        clean = re.sub(r"\{\{[^}]*\}\}", " ", clean)
        clean = re.sub(r"\[\[([^\]|]*\|)?([^\]]*)\]\]", r"\2", clean)
        clean = norm(clean)
        if not clean or clean in out:
            continue
        if "( )" in clean:  # a template left a hole; not quotable
            continue
        if labels:
            clean = "[" + ", ".join(labels) + "] " + clean
        out.append(clean)
    return out


# --------------------------------------------------------------------------
# Phayan's word list
# --------------------------------------------------------------------------

def phayan_entry(word):
    """(in corpus, tone) for a word in words.ts; None when the file is absent."""
    if not os.path.exists(PHAYAN_WORDS):
        return None
    with open(PHAYAN_WORDS, encoding="utf-8") as f:
        src = unicodedata.normalize("NFC", f.read())
    m = re.search(r"w\(\s*'" + re.escape(word) + r"',\s*'[^']*',\s*(\d)", src)
    if not m:
        return ("no", "")
    return ("yes", PHAYAN_TONES[m.group(1)])


# --------------------------------------------------------------------------
# the check itself
# --------------------------------------------------------------------------

def run(include_senses=True):
    wanted = [CONSONANT_TABLE_DOC] + [e["url"] for e in THIRD_PARTY]
    for _, spelling, _, _ in WORDS:
        wanted.append(WIKTIONARY + urllib.parse.quote(spelling))
        if include_senses:
            wanted.append(raw_url(spelling))
    prefetch(wanted)

    cons_page = fetch(CONSONANT_TABLE_DOC)
    classes = consonant_classes(cons_page)

    rows = []
    failures = []

    for group, spelling, english, syllables in WORDS:
        url = WIKTIONARY + urllib.parse.quote(spelling)
        page = fetch(url)
        ipa = dictionary_ipa(page)

        senses = other_senses(fetch(raw_url(spelling))) if include_senses else []

        if norm(ipa) not in norm(strip_tags(page)):
            failures.append("%s: the IPA %s is no longer on %s" % (spelling, ipa, url))

        parts = ipa_syllables(ipa)
        if len(parts) != len(syllables):
            raise RuntimeError(
                "%s: %d syllables written, %d in the IPA %s"
                % (spelling, len(syllables), len(parts), ipa)
            )

        literals, clss, types, marks, derived, spoken = [], [], [], [], [], []
        for syl, part in zip(syllables, parts):
            letter = initial_of(syl)
            if letter not in classes:
                raise RuntimeError("%s is not in the consonant table" % letter)
            cls = classes[letter][0]
            mark = mark_of(syl)
            rtype = rime_type(part)
            key = (cls, rtype, mark)
            if key not in TONE_TABLE:
                raise RuntimeError("tone table has no cell for %r" % (key,))
            tone = tone_of(part)
            literals.append(letter)
            clss.append(cls)
            types.append(rtype)
            marks.append(mark or "none")
            derived.append(TONE_TABLE[key])
            spoken.append(tone)

        matched = derived == spoken
        if not matched:
            failures.append(
                "%s: the table derives %s, the dictionary says %s (%s)"
                % (spelling, " + ".join(derived), " + ".join(str(s) for s in spoken), ipa)
            )

        entry = phayan_entry(spelling)
        row = {
            "group": group,
            "word": spelling,
            "english": english,
            "syllables": "|".join(syllables),
            "initial": "|".join(literals),
            "class": "|".join(clss),
            "syllableType": "|".join(types),
            "toneMark": "|".join(marks),
            "derivedTone": "|".join(derived),
            "dictionaryIpa": ipa,
            "dictionaryTone": "|".join(str(s) for s in spoken),
            "match": "yes" if matched else "no",
            "inPhayanWordList": entry[0] if entry else "not checked",
            "phayanTone": entry[1] if entry else "",
            "classQuote": {
                "value": "|".join(clss),
                "quote": " / ".join(classes[l][1] for l in sorted(set(literals))),
                "url": CONSONANT_TABLE_DOC,
            },
            "ipaQuote": {"value": ipa, "quote": ipa, "url": url},
            "otherSenses": senses[:6],
            "otherSensesUrl": raw_url(spelling),
        }
        rows.append(row)

    # every class row quoted must still sit on the page
    body = normalised_body(cons_page)
    checked_letters = set()
    for row in rows:
        for letter in row["initial"].split("|"):
            if letter in checked_letters:
                continue
            checked_letters.add(letter)
            rowtext = norm(classes[letter][1])
            if rowtext not in body:
                failures.append(
                    "%s: the consonant row %r is no longer on %s"
                    % (letter, rowtext, CONSONANT_TABLE_DOC)
                )

    third = []
    for entry in THIRD_PARTY:
        try:
            page = fetch(entry["url"])
            present = norm(entry["quote"]) in normalised_body(page)
        except Exception as exc:  # noqa: BLE001
            present = False
            entry = dict(entry, error=str(exc))
        third.append(dict(entry, quotePresent=present))

    return rows, failures, third


def write_package(rows, third):
    colours = [r for r in rows if r["group"] == "colour"]
    dataset = {
        "post": "thai-colour-words",
        "question": "What are the colours in Thai?",
        "method": "public-corpus",
        "checkedAt": CHECKED_AT,
        "tools": {
            "python": sys.version.split()[0],
            "curl": "8.7.1 (used to read the pages while writing this check)",
            "fetch": "urllib.request, standard library only",
        },
        "inputs": [
            {"kind": "dictionary", "source": "en.wiktionary.org", "licence": "CC BY-SA 4.0",
             "note": "Thai section pronunciation respelling (IPA with Chao tone letters) and Thai definitions"},
            {"kind": "consonant-classes", "source": CONSONANT_TABLE_DOC,
             "note": "Thai consonant table, Class column"},
            {"kind": "tone-table", "source": CONSONANT_TABLE_DOC,
             "note": "Tone chart: class x live/dead x tone mark -> tone, encoded in check.py"},
            {"kind": "file", "path": "apps/api/src/content/seed/words.ts",
             "note": "Phayan's word list: whether each colour word has an entry, and its tone number"},
        ]
        + [{"kind": "word", "word": r["word"], "url": r["ipaQuote"]["url"]} for r in rows],
        "citableLine": (
            "Applied to the 13 Thai colour words, the tone table derives the spoken tone for "
            "13 of them, and none fail."
        ),
        "summary": {
            "colourWords": len(colours),
            "matched": sum(1 for r in colours if r["match"] == "yes"),
            "syllablesChecked": sum(len(r["syllables"].split("|")) for r in rows),
            "distinctToneRulesUsed": len(
                {(c, t, m) for r in rows
                 for c, t, m in zip(r["class"].split("|"), r["syllableType"].split("|"), r["toneMark"].split("|"))}
            ),
        },
        "results": rows,
        "thirdParty": third,
        "notes": [
            "Every one of the 13 colour words and all 18 syllables checked, สี and สี่ included, are live syllables, "
            "so the live/dead half of the tone table is never tested by this corpus. The rule holds "
            "because the class and the mark alone decide it.",
            "เหลือง takes its high class from the leading ห (ho nam), not from the ล it writes over. "
            "The script reads the first consonant character, which is the one the tone table uses.",
            "Two-syllable words are split by hand in WORDS (น้ำ|เงิน, น้ำ|ตาล, ชม|พู); the IPA dot "
            "separates the same syllables, and the script errors out if the counts disagree.",
            "The post's 'what that word is otherwise' column says 'nothing else in ordinary use' for "
            "แดง and ดำ. Wiktionary lists non-archaic senses for both (แดง: colloquial 'to reveal', "
            "'money'; ดำ: 'to dive', 'to plant rice'). See notes.md.",
            "The English glosses beside each word are the post's, not Wiktionary's; the IPA and the "
            "Thai definitions are Wiktionary's.",
            "inPhayanWordList and phayanTone are read from apps/api/src/content/seed/words.ts in this "
            "repository (tone numbers 1 mid, 2 low, 3 falling, 4 high, 5 rising). Six of the 13 colour "
            "words have no entry there, and every tone it does carry agrees with Wiktionary.",
            "Not checked: how Thai speakers actually say each colour word (the tones are a dictionary's "
            "respelling, not a recording), how common each word is, and the app's audio.",
            "The post's former comparisons with Drops, ThaiPod101 and Anki were cut: each page was read "
            "on the check date and none supports the comparison the post made. The FAQ now quotes only "
            "what ThaiPod101's and Anki's own pages say about themselves, and Drops is no longer named "
            "(see thirdParty).",
        ],
    }
    with open(DATASET_JSON, "w", encoding="utf-8") as f:
        json.dump(dataset, f, ensure_ascii=False, indent=2)
        f.write("\n")

    fields = [
        "group", "word", "english", "syllables", "initial", "class", "syllableType",
        "toneMark", "derivedTone", "dictionaryIpa", "dictionaryTone", "match",
        "inPhayanWordList", "phayanTone",
    ]
    with open(DATASET_CSV, "w", encoding="utf-8", newline="") as f:
        w = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore")
        w.writeheader()
        for row in rows:
            w.writerow(row)


def verify(rows, failures, third):
    """Compare a fresh run against the stored dataset."""
    with open(DATASET_JSON, encoding="utf-8") as f:
        stored = json.load(f)
    stored_rows = {r["word"]: r for r in stored["results"]}
    fields = ["class", "syllableType", "toneMark", "derivedTone",
              "dictionaryIpa", "dictionaryTone", "match", "initial"]
    if os.path.exists(PHAYAN_WORDS):
        fields += ["inPhayanWordList", "phayanTone"]
    else:
        print("  Phayan word list not found at %s, its columns not checked" % PHAYAN_WORDS)
    for row in rows:
        old = stored_rows.get(row["word"])
        if old is None:
            failures.append("%s: not in dataset.json" % row["word"])
            continue
        for field in fields:
            if row[field] != old[field]:
                failures.append(
                    "%s: %s is %r on the page today, %r in dataset.json"
                    % (row["word"], field, row[field], old[field])
                )

    for entry in third:
        print(
            "  third party %-11s %-7s quote %s  %s"
            % (
                entry["party"],
                entry["verdict"],
                "present" if entry.get("quotePresent") else "GONE",
                entry["url"],
            )
        )

    print()
    print("%-10s %-14s %-12s %-10s %-10s %-10s %-6s %s"
          % ("word", "initial/class", "type", "mark", "derived", "dictionary", "match", "Phayan"))
    for row in rows:
        print(
            "%-10s %-14s %-12s %-10s %-10s %-10s %-6s %s"
            % (
                row["word"],
                row["initial"] + "/" + row["class"],
                row["syllableType"],
                row["toneMark"],
                row["derivedTone"],
                row["dictionaryTone"],
                row["match"],
                (row["inPhayanWordList"] + " " + row["phayanTone"]).strip(),
            )
        )
    colours = [r for r in rows if r["group"] == "colour"]
    print()
    print(
        "colour words: %d, derived tone matches the dictionary for %d"
        % (len(colours), sum(1 for r in colours if r["match"] == "yes"))
    )


def main(argv):
    write = "--write" in argv
    try:
        rows, failures, third = run(include_senses=write)
    except Exception as exc:  # noqa: BLE001
        print("check could not run: %s" % exc)
        return 1

    if write:
        write_package(rows, third)
        print("wrote %s and %s" % (DATASET_JSON, DATASET_CSV))
        for row in rows:
            print("  %-8s match %-3s Phayan %s %s" % (row["word"], row["match"], row["inPhayanWordList"], row["phayanTone"]))
        for line in failures:
            print("FAIL %s" % line)
        return 0 if not failures else 1

    verify(rows, failures, third)
    if failures:
        print()
        for line in failures:
            print("FAIL %s" % line)
        return 1
    print("OK every quote is on its page and every derived tone matches")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))