#!/usr/bin/env python3
"""Evidence check for the Phayan post `thai-two-ai-vowels`.

1. Corpus: every title in English Wiktionary's Category:Thai lemmas, split into
   the ones spelled with the rolled vowel ใ (U+0E43) and the split vowel ไ
   (U+0E44). Each lemma carrying ใ is tested against the twenty-word school
   set: is every ใ in it part of one of the twenty?
2. Product: which of the twenty have an entry in Phayan's word list
   (apps/api/src/content/seed/words.ts in this repository).
3. Pages: the sentences the post leans on (Thai Wikipedia on ไม้ม้วน, the RTGS
   vowel table, four Wiktionary entries, ThaiPod101, Anki), fetched live and
   matched against the recorded quote.

    python3 check.py            measure everything, compare with dataset.json and dataset.csv
    python3 check.py --recheck  the same; the corpus crawl takes about a minute
    python3 check.py --write    regenerate dataset.json and dataset.csv

Exit 0 when every value and quote holds, 1 otherwise. Standard library only.
"""

import argparse
import csv
import html
import json
import os
import re
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
REPO_ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", ".."))
WORDS_FILE = "apps/api/src/content/seed/words.ts"
DATASET_JSON = os.path.join(HERE, "dataset.json")
DATASET_CSV = os.path.join(HERE, "dataset.csv")
CHECKED_AT = "2026-10-26"

UA = "PhayanBlogEvidence/1.0 (https://www.phayan.com/blog; evidence recheck script)"
WIKTIONARY_API = "https://en.wiktionary.org/w/api.php"
LEMMA_CATEGORY = "Category:Thai lemmas"
CATEGORY_URL = "https://en.wiktionary.org/wiki/Category:Thai_lemmas"

ROLLED = "ใ"
SPLIT = "ไ"

# The list printed in the Thai Wikipedia article on ไม้ม้วน. The twentieth is
# the base form ใหล, which survives only inside compounds such as หลงใหล.
TWENTY = [
    "ใกล้", "ใคร", "ใคร่", "ใจ", "ใช่", "ใช้", "ใด", "ใต้", "ใน", "ใบ",
    "ใบ้", "ใฝ่", "ใย", "ใส", "ใส่", "ให้", "ใหญ่", "ใหม่", "สะใภ้", "ใหล",
]


def wikitext_url(site, page):
    return ("https://%s/w/api.php?action=parse&page=%s&prop=wikitext&format=json&formatversion=2"
            % (site, urllib.parse.quote(page)))


THWIKI = wikitext_url("th.wikipedia.org", "ไม้ม้วน")
RTGS = wikitext_url("en.wikipedia.org", "Royal Thai General System of Transcription")
THAIPOD101 = "https://www.thaipod101.com/lesson/pronunciation-3-finetuning-your-thai-vowels"
ANKI = "https://apps.ankiweb.net/"

# id, url, how the page is read, quote, what the quote supports.
# "wikitext" pages are matched after links and templates are reduced to their
# visible text; "raw" pages are matched on the wikitext as stored, because the
# quote sits inside a template.
PAGE_ROWS = [
    ("D1", THWIKI, "wikitext", "การใช้ไม้ม้วนจำกัดอยู่ที่คำศัพท์ 20 คำ มาตั้งแต่สมัยอยุธยาเป็นอย่างน้อย",
     "the rolled vowel is limited to twenty words, since Ayutthaya at least"),
    ("D2", THWIKI, "wikitext", "คำศัพท์ใหม่ ๆ ที่เกิดขึ้น จะใช้ไม้มลายทั้งสิ้น (เว้นแต่ลูกคำที่แตกจากแม่คำ ซึ่งใช้ไม้ม้วนอยู่แล้ว)",
     "new words all take ไ, except words derived from one that already has ใ"),
    ("D3", THWIKI, "wikitext", "ใหล (หลงใหล, หลับใหล, ใหลตาย)",
     "the twentieth entry is the base form ใหล"),
    ("D4", THWIKI, "wikitext", "ที่รู้จักกันดีจนปัจจุบัน",
     "the kap yani verse is well known to this day"),
    ("D5", THWIKI, "wikitext", "ผู้ใหญ่หาผ้าใหม่",
     "first line of the verse: an adult looks for new cloth"),
    ("D11", THWIKI, "wikitext", "ให้สะใภ้ใช้คล้องคอ",
     "second line: for the daughter in law to wear round the neck"),
    ("D6", THWIKI, "wikitext", "ยี่สิบม้วนจำจงดี",
     "last line of the verse"),
    ("D7", THWIKI, "wikitext", "ในสมัยกรุงศรีอยุธยา มีตำราเรียนระบุการใช้ไม้ม้วน ในคำศัพท์จำนวน 20 คำ",
     "an Ayutthaya-period textbook (Chindamani) lists the twenty"),
    ("D8", THWIKI, "wikitext", "เสียงสระของไม้ม้วนเดิมเป็นเสียงสระผสม อะ+อึ",
     "the rolled vowel was once a different diphthong"),
    ("D9", THWIKI, "wikitext", "ภาษาลาวสำเนียงหลวงพระบาง ภาษาไทใหญ่ ภาษาไทดำ และ ภาษาไทขาว",
     "Tai languages that still keep the old vowel"),
    ("D10", THWIKI, "wikitext", "ในศิลาจารึกหลักที่หนึ่ง ของพ่อขุนรามคำแหง ในคำว่า ใคร ใคร่ ใด ใส ใหญ่ ใน ให้ ใช้ ใต้",
     "the rolled vowel already appears in the Ram Khamhaeng inscription"),
    ("R1", RTGS, "raw", "{{lang|th|ใ–}}, {{lang|th|ไ–}}, {{lang|th|–ัย}}, {{lang|th|ไ–ย}}, {{lang|th|–าย}} || ai",
     "RTGS romanises ใ– and ไ– alike as ai"),
    ("A1", wikitext_url("en.wiktionary.org", "เชยีงใหม"), "raw", "{{archaic spelling of|th|เชียงใหม่}}",
     "เชยีงใหม is an archaic spelling of เชียงใหม่"),
    ("A2", wikitext_url("en.wiktionary.org", "ใชย"), "raw", "{{archaic spelling of|th|ชัย}}",
     "ใชย is an archaic spelling of ชัย"),
    ("A3", wikitext_url("en.wiktionary.org", "ใภ้"), "raw", "{{lb|th|archaic}} [[daughter-in-law]]",
     "ใภ้ is an archaic word for daughter in law"),
    ("A4", wikitext_url("en.wiktionary.org", "ใหลตาย"), "raw", "{{com|th|ใหล|t1=to [[sleep]]",
     "ใหล means to sleep"),
    ("A5", wikitext_url("en.wiktionary.org", "หลงใหล"), "raw", "# to be [[crazy]] about.",
     "หลงใหล means to be crazy about"),
    ("T1", THAIPOD101, "html", "The next two sound just the same",
     "ThaiPod101 presents ไ and ใ as one sound"),
    ("T2", ANKI, "html", "Anki is a flashcard program that helps you spend more time on challenging material",
     "Anki describes itself as a flashcard program"),
]


def fetch(url, tries=5):
    last = None
    for attempt in range(tries):
        req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Encoding": "identity"})
        try:
            with urllib.request.urlopen(req, timeout=90) as r:
                return r.read().decode("utf-8", "replace")
        except urllib.error.HTTPError as e:
            last = e
            if e.code not in (429, 503):
                break
        except Exception as e:  # noqa: BLE001
            last = e
        time.sleep(3 * (attempt + 1))
    raise RuntimeError("cannot fetch %s: %s" % (url, last))


def visible(text):
    text = re.sub(r"(?is)<(script|style)\b.*?</\1>", " ", text)
    text = re.sub(r"(?s)<[^>]+>", " ", text)
    text = re.sub(r"\[\[[^\]|]*\|([^\]]*)\]\]", r"\1", text)
    text = text.replace("[[", "").replace("]]", "")
    text = re.sub(r"\{\{[^{}]*\}\}", " ", text)
    text = text.replace("'''", "").replace("''", "")
    text = html.unescape(text).replace("’", "'")
    return re.sub(r"\s+", " ", text).strip()


def page_text(url, mode, cache):
    key = (url, mode)
    if key not in cache:
        body = fetch(url)
        if mode in ("wikitext", "raw"):
            body = json.loads(body)["parse"]["wikitext"]
        cache[key] = body if mode == "raw" else visible(body)
    return cache[key]


def fetch_lemmas():
    titles, cont = [], None
    while True:
        params = {"action": "query", "list": "categorymembers", "cmtitle": LEMMA_CATEGORY,
                  "cmlimit": "500", "cmnamespace": "0", "format": "json"}
        if cont:
            params["cmcontinue"] = cont
        data = json.loads(fetch(WIKTIONARY_API + "?" + urllib.parse.urlencode(params)))
        titles.extend(m["title"] for m in data["query"]["categorymembers"])
        cont = data.get("continue", {}).get("cmcontinue")
        if not cont:
            return titles
        time.sleep(1.0)


def carriers(word):
    """The member of the twenty that owns each ใ in `word`, or None for a ใ no member reaches."""
    owners = []
    for pos, ch in enumerate(word):
        if ch != ROLLED:
            continue
        best = None
        for member in TWENTY:
            offset = member.index(ROLLED)
            start = pos - offset
            if start >= 0 and word.startswith(member, start):
                if best is None or len(member) > len(best):
                    best = member
        owners.append(best)
    return owners


def measure_corpus():
    lemmas = fetch_lemmas()
    lemma_set = set(lemmas)
    with_rolled = [t for t in lemmas if ROLLED in t]
    outside = [t for t in with_rolled if t not in TWENTY]
    carried = {w: 0 for w in TWENTY}
    uncovered = []
    for title in outside:
        owners = carriers(title)
        if None in owners:
            uncovered.append(title)
        for member in set(o for o in owners if o):
            carried[member] += 1
    return {
        "lemmas": len(lemmas),
        "with_rolled": len(with_rolled),
        "with_split": sum(1 for t in lemmas if SPLIT in t),
        "twenty_present": sum(1 for w in TWENTY if w in lemma_set),
        "outside": len(outside),
        "outside_covered": len(outside) - len(uncovered),
        "uncovered": sorted(uncovered),
        "present": {w: w in lemma_set for w in TWENTY},
        "carried": carried,
    }


def measure_phayan():
    path = os.path.join(REPO_ROOT, WORDS_FILE)
    if not os.path.exists(path):
        return {"error": "%s not found next to this script" % WORDS_FILE, "presence": {}, "sha": None}
    source = open(path, encoding="utf-8").read()
    sha = subprocess.run(["git", "-C", REPO_ROOT, "log", "-1", "--format=%H", "--", WORDS_FILE],
                         capture_output=True, text=True).stdout.strip() or None
    presence = {w: bool(re.search(r"\bw\(\s*'" + re.escape(w) + r"'\s*,", source)) for w in TWENTY}
    return {"error": None, "presence": presence, "sha": sha}


def build():
    corpus = measure_corpus()
    phayan = measure_phayan()
    api = WIKTIONARY_API + "?action=query&list=categorymembers&cmtitle=Category:Thai_lemmas&cmnamespace=0"
    results = []

    def add(rid, subject, prop, value, quote="", url=api):
        results.append({"id": rid, "subject": subject, "property": prop,
                        "value": str(value), "quote": quote, "url": url})

    add("C1", "English Wiktionary, Category:Thai lemmas", "lemmas", corpus["lemmas"])
    add("C2", "English Wiktionary, Category:Thai lemmas", "lemmas spelled with ใ", corpus["with_rolled"])
    add("C3", "English Wiktionary, Category:Thai lemmas", "lemmas spelled with ไ", corpus["with_split"])
    add("C4", "the twenty", "present as a lemma of their own", corpus["twenty_present"])
    add("C5", "lemmas with ใ", "not one of the twenty", corpus["outside"])
    add("C6", "lemmas with ใ, not one of the twenty", "every ใ belongs to one of the twenty",
        corpus["outside_covered"])
    add("C7", "lemmas with ใ, not one of the twenty", "a ใ that none of the twenty reaches",
        len(corpus["uncovered"]), ", ".join(corpus["uncovered"]))

    rows = []
    for word in TWENTY:
        in_phayan = ("not checked" if phayan["error"]
                     else "yes" if phayan["presence"][word] else "no")
        rows.append({"word": word,
                     "own_lemma_in_wiktionary": "yes" if corpus["present"][word] else "no",
                     "other_lemmas_carrying_it": str(corpus["carried"][word]),
                     "entry_in_phayan": in_phayan})
        add("W:" + word, word, "own lemma in Wiktionary", rows[-1]["own_lemma_in_wiktionary"],
            url=CATEGORY_URL)
        add("N:" + word, word, "other Wiktionary lemmas carrying it", rows[-1]["other_lemmas_carrying_it"])
        add("P:" + word, word, "entry in Phayan's word list", in_phayan, url=WORDS_FILE)

    cache = {}
    for rid, url, mode, quote, prop in PAGE_ROWS:
        try:
            text = page_text(url, mode, cache)
            needle = quote if mode == "raw" else visible(quote)
            value = "found" if needle in text else "missing"
        except RuntimeError as exc:
            value = "not checked: %s" % exc
        add(rid, url, prop, value, quote, url)

    return results, rows, corpus, phayan


def write(results, rows, corpus, phayan):
    dataset = {
        "slug": "thai-two-ai-vowels",
        "method": "public-corpus",
        "checkedAt": CHECKED_AT,
        "tools": [
            {"name": "python3", "version": sys.version.split()[0],
             "use": "standard library only; urllib for every fetch"},
        ],
        "inputs": [
            {"what": "English Wiktionary, every member of Category:Thai lemmas (namespace 0), "
                     "via the MediaWiki API at 500 per request",
             "url": CATEGORY_URL},
            {"what": "Thai Wikipedia, ไม้ม้วน, wikitext", "url": THWIKI},
            {"what": "English Wikipedia, Royal Thai General System of Transcription, wikitext",
             "url": RTGS},
            {"what": "English Wiktionary entries เชยีงใหม, ใชย, ใภ้, ใหลตาย, หลงใหล, wikitext",
             "url": "https://en.wiktionary.org/wiki/"},
            {"what": "ThaiPod101, Pronunciation lesson 3, Fine-Tuning Your Thai Vowels", "url": THAIPOD101},
            {"what": "Anki home page", "url": ANKI},
            {"what": "Phayan word list, %s at commit %s" % (WORDS_FILE, phayan["sha"]),
             "url": WORDS_FILE},
        ],
        "twenty": TWENTY,
        "results": results,
        "notes": [
            "Counts are a snapshot of one wiki category on %s and drift as editors add lemmas; "
            "check.py re-measures rather than trusting them." % CHECKED_AT,
            "A ใ counts as belonging to one of the twenty when that member of the twenty sits at "
            "that position in the lemma; where two members fit, the longer wins (ใส่ over ใส). "
            "The lemmas outside the twenty were read by eye on %s and are all compounds or phrases "
            "built on one of the twenty, such as การตัดสินใจ and ผู้ใหญ่." % CHECKED_AT,
            "The four lemmas no member reaches are the page for the letter ใ itself and three "
            "archaic entries, rows A1 to A3: เชยีงใหม, ใชย and ใภ้.",
            "ใหล has no lemma of its own; it is counted inside หลงใหล and ใหลตาย.",
            "Before this check the post's table printed หลงใหล as the twentieth word where the "
            "list on Thai Wikipedia gives the base form ใหล; the post was corrected on %s." % CHECKED_AT,
            "Not checked: the Royal Institute Dictionary, which is the authoritative Thai word list "
            "and has no open bulk export; the result describes English Wiktionary's coverage.",
            "Not checked: pronunciation. The corpus is spelling only; that ไ and ใ sound alike in "
            "standard Thai rests on the Thai Wikipedia article (rows D8, D9).",
            "Ling was dropped from the post: its own page says it teaches writing and vocabulary in "
            "one course, which contradicts the sentence the post carried.",
        ],
    }
    with open(DATASET_JSON, "w", encoding="utf-8") as fh:
        json.dump(dataset, fh, ensure_ascii=False, indent=2)
        fh.write("\n")
    with open(DATASET_CSV, "w", encoding="utf-8", newline="") as fh:
        writer = csv.DictWriter(fh, fieldnames=list(rows[0]))
        writer.writeheader()
        writer.writerows(rows)


def compare(results, rows):
    failures = []
    with open(DATASET_JSON, encoding="utf-8") as fh:
        recorded = {r["id"]: r["value"] for r in json.load(fh)["results"]}
    for r in results:
        if recorded.get(r["id"]) != r["value"]:
            failures.append("%s %s / %s: recorded %r, now %r"
                            % (r["id"], r["subject"], r["property"], recorded.get(r["id"]), r["value"]))
        elif r["value"] != "found" and r["id"][0] in "DRAT":
            failures.append("%s quote %s on %s" % (r["id"], r["value"], r["subject"]))
    with open(DATASET_CSV, encoding="utf-8", newline="") as fh:
        if list(csv.DictReader(fh)) != rows:
            failures.append("dataset.csv differs from the measured table")
    return failures


def main():
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument("--recheck", action="store_true", help="re-read every page and the corpus")
    parser.add_argument("--write", action="store_true", help="regenerate dataset.json and dataset.csv")
    args = parser.parse_args()

    results, rows, corpus, phayan = build()
    if args.write:
        write(results, rows, corpus, phayan)
        print("wrote %d results and %d table rows" % (len(results), len(rows)))
        return 0
    if phayan["error"]:
        print("note: Phayan word list not checked (%s)" % phayan["error"], file=sys.stderr)
    failures = compare(results, rows)
    if failures:
        print("FAIL %d of %d results" % (len(failures), len(results)), file=sys.stderr)
        for line in failures:
            print("  " + line, file=sys.stderr)
        return 1
    print("OK %d results hold (%d lemmas, %d with ใ, %d outside the twenty, %d no member reaches)"
          % (len(results), corpus["lemmas"], corpus["with_rolled"], corpus["outside"],
             len(corpus["uncovered"])))
    return 0


if __name__ == "__main__":
    sys.exit(main())