#!/usr/bin/env python3
"""Evidence check for the Phayan post `thai-numbers` (https://phayan.com/blog/thai-numbers).

Three tables, one script:
  results         one row per public Thai page, measuring whether numbers are
                  printed in Thai numerals (U+0E50-U+0E59) or Arabic digits
  third_party     one row per statement the post makes about another company,
                  with the sentence on that company's own page that supports,
                  contradicts or fails to support it
  product_claims  one row per statement the post makes about Phayan, checked
                  against the repository holding this file, at HEAD

`--write` re-measures everything and rewrites dataset.json and dataset.csv.
A plain run verifies the recorded values and exits 0 only when every page still
holds its quotes and every flag, verdict and code pattern still holds.
"""

from __future__ import annotations

import argparse
import concurrent.futures as futures
import csv
import gzip
import html as html_mod
import json
import os
import re
import subprocess
import sys
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET_JSON = os.path.join(HERE, "dataset.json")
DATASET_CSV = os.path.join(HERE, "dataset.csv")
CHECKED_AT = "2026-10-26"
REF = "HEAD"


def _repo_root() -> str | None:
    try:
        out = subprocess.run(
            ["git", "-C", HERE, "rev-parse", "--show-toplevel"],
            capture_output=True, text=True, check=True,
        )
    except Exception:
        return None
    root = out.stdout.strip()
    return root if os.path.isdir(os.path.join(root, "apps", "api")) else None


REPO = _repo_root()

UA = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/126.0 Safari/537.36"
)

THAI_DIGITS = "๐๑๒๓๔๕๖๗๘๙"
RE_THAI_NUM = re.compile("[" + THAI_DIGITS + "]")
RE_ARABIC = re.compile(r"[0-9]")
RE_THAI_LETTER = re.compile("[ก-ฮ]")
# Buddhist-era year in Thai numerals: ๒๔xx or ๒๕xx (2400-2599 BE).
RE_BE_YEAR_TH = re.compile("[๒๕][๔๕][" + THAI_DIGITS + "]{2}")

MIN_THAI_LETTERS = 200  # below this the served page carries no readable Thai text

PAGES = [
    ("bank", "https://www.scb.co.th/th/personal-banking.html"),
    ("bank", "https://www.gsb.or.th/"),
    ("bank", "https://www.baac.or.th/"),
    ("bank", "https://www.lhbank.co.th/"),
    ("government-office", "https://www.moe.go.th/"),
    ("government-office", "https://www.ops.go.th/"),
    ("government-office", "https://www.mfa.go.th/th/"),
    ("government-office", "https://www.parliament.go.th/"),
    ("government-office", "https://www.nso.go.th/"),
    ("government-office", "https://www.rd.go.th/"),
    ("government-office", "https://www.thaihealth.or.th/"),
    ("utility-rate", "https://www.mea.or.th/"),
    ("utility-rate", "https://www.mwa.co.th/"),
    ("utility-rate", "https://www.pea.co.th/"),
    ("news", "https://www.thairath.co.th/home"),
    ("news", "https://prachatai.com/"),
    ("news", "https://mgronline.com/"),
    ("news", "https://www.matichon.co.th/"),
    ("news", "https://www.thaipost.net/"),
    ("retail-price", "https://www.jib.co.th/"),
    ("retail-price", "https://www.officeMate.co.th/"),
    ("retail-price", "https://www.homepro.co.th/"),
    ("retail-price", "https://www.lotuss.com/"),
    ("retail-price", "https://www.makro.co.th/"),
    ("temple-religious", "https://www.watpho.com/"),
    ("temple-religious", "https://www.84000.org/"),
    ("temple-religious", "https://www.dhammathai.org/"),
    (
        "encyclopedia",
        "https://th.wikipedia.org/wiki/%E0%B8%9B%E0%B8%A3%E0%B8%B0%E0%B9%80%E0%B8%97%E0%B8%A8%E0%B9%84%E0%B8%97%E0%B8%A2",
    ),
    (
        "encyclopedia",
        "https://th.wikipedia.org/wiki/%E0%B8%98%E0%B8%99%E0%B8%9A%E0%B8%B1%E0%B8%95%E0%B8%A3",
    ),
    (
        "encyclopedia",
        "https://th.wikipedia.org/wiki/%E0%B8%AD%E0%B8%B8%E0%B8%97%E0%B8%A2%E0%B8%B2%E0%B8%99%E0%B9%81%E0%B8%AB%E0%B9%88%E0%B8%87%E0%B8%8A%E0%B8%B2%E0%B8%95%E0%B8%B4%E0%B9%83%E0%B8%99%E0%B8%9B%E0%B8%A3%E0%B8%B0%E0%B9%80%E0%B8%97%E0%B8%A8%E0%B9%84%E0%B8%97%E0%B8%A2",
    ),
]

# Tried and refused. Written into notes, never counted in results.
REFUSED = [
    ("national-park", "https://www.dnp.go.th/", "Incapsula challenge shell, no page text (HTTP 200, 'Request unsuccessful')"),
    ("national-park", "https://www.dnp.go.th/parkreserve/index.asp", "Incapsula challenge shell, no page text"),
    ("national-park", "https://www.tourismthailand.org/Attraction", "HTTP 403 to a plain user agent"),
    ("bank", "https://www.kasikornbank.com/th/personal/Pages/index.aspx", "HTTP 403"),
    ("bank", "https://www.exim.go.th/", "served page holds 8 Thai letters, a script shell"),
    ("government-office", "https://www.cgd.go.th/", "HTTP 403"),
    ("news", "https://www.khaosod.co.th/", "HTTP 403"),
    ("retail-price", "https://www.bigc.co.th/", "HTTP 403"),
    ("retail-price", "https://www.powerbuy.co.th/", "HTTP 403"),
    ("retail-price", "https://www.tops.co.th/", "HTTP 403"),
    ("retail-price", "https://www.thaiwatsadu.com/", "HTTP 403"),
]

# Statements about another company. `verdict` is read against `quote` alone:
#   supported  the page says what the post says
#   cut        no page found that supports it; it must leave the post
GT_HELP = "https://support.google.com/translate/answer/6142483?hl=en&co=GENIE.Platform%3DAndroid"
POD_BLOG = "https://www.thaipod101.com/blog/2019/10/24/thai-numbers/"
POD_LIST = "https://www.thaipod101.com/thai-vocabulary-lists/numbers"
LING = "https://ling-app.com/thai-numbers/"

THIRD_PARTY = [
    {
        "id": "gt-camera-reads-images",
        "in_post": True,
        "post_quote": "Google Translate can translate text from a photo or through the camera",
        "party": "Google Translate",
        "claim": "the app translates text from images and through the camera",
        "url": GT_HELP,
        "quote": "In the Translate app, you can translate text from images on your phone.",
        "verdict": "supported",
        "note": "What Google says about its own app. The page says nothing specific about Thai or Thai numerals.",
    },
    {
        "id": "gt-stylized-caveat",
        "in_post": True,
        "post_quote": "Translation of small, unclear, or stylized text may not be accurate.",
        "party": "Google Translate",
        "claim": "Google's own accuracy caveat for image translation",
        "url": GT_HELP,
        "quote": "Translation of small, unclear, or stylized text may not be accurate.",
        "verdict": "supported",
        "note": "Quoted verbatim in the post as Google's own caveat.",
    },
    {
        "id": "gt-thai-errors",
        "in_post": False,
        "post_quote": "it gets Thai script wrong often enough that recognising ๒๕ yourself is the faster check",
        "party": "Google Translate",
        "claim": "the app misreads Thai script often",
        "url": GT_HELP,
        "quote": "Translation of small, unclear, or stylized text may not be accurate.",
        "verdict": "cut",
        "note": "Removed from the post on 2026-10-26. Google publishes only this generic caveat, and no page or study found states an error rate for Thai in the camera, so the frequency claim had nothing behind it.",
    },
    {
        "id": "thaipod101-arabic-digits",
        "in_post": True,
        "post_quote": "ThaiPod101 writes that most of the time Thai people use Arabic numerals in writing",
        "party": "ThaiPod101",
        "claim": "ThaiPod101's statement on which digits Thai writing uses",
        "url": POD_BLOG,
        "quote": "Most of the time, Thai people use Arabic numerals in writing.",
        "verdict": "supported",
        "note": "A learning site's statement, cited as such; the post's own count is the measurement.",
    },
    {
        "id": "thaipod101-glyph-list",
        "in_post": True,
        "post_quote": "ThaiPod101 lists each numeral beside its word",
        "party": "ThaiPod101",
        "claim": "ThaiPod101 teaches the numerals together with the number words",
        "url": POD_BLOG,
        "quote": "๑ – หนึ่ง ( nùeng )",
        "verdict": "supported",
        "note": "The same article lists ๑ to ๙ each beside its word and romanization.",
    },
    {
        "id": "thaipod101-vocabulary-list",
        "in_post": True,
        "post_quote": "ThaiPod101's numbers vocabulary list",
        "party": "ThaiPod101",
        "claim": "ThaiPod101 publishes the number words as a vocabulary list",
        "url": POD_LIST,
        "quote": "sǔun ศูนย์ (n) zero",
        "verdict": "supported",
        "note": "A vocabulary list of the number words with romanization.",
    },
    {
        "id": "ling-arabic-daily",
        "in_post": True,
        "post_quote": "Ling's guide says Thai people use Arabic numerals daily and that Thai numbers turn up in official papers and government documents",
        "party": "Ling",
        "claim": "Ling's statement on where Thai numerals are used",
        "url": LING,
        "quote": "Thai people use Arabic numerals daily, so you might have little chance to see original Thai numbers.",
        "verdict": "supported",
        "note": "A learning site's statement, cited as such.",
    },
    {
        "id": "ling-official-papers",
        "in_post": True,
        "post_quote": "Ling's guide says Thai people use Arabic numerals daily and that Thai numbers turn up in official papers and government documents",
        "party": "Ling",
        "claim": "Ling's statement on where Thai numerals are used",
        "url": LING,
        "quote": "especially in official papers or government documents",
        "verdict": "supported",
        "note": "A learning site's statement, cited as such. The government pages in this dataset print Thai numerals rarely, so the post presents it as Ling's view, not as a measured fact.",
    },
    {
        "id": "ling-numeral-table",
        "in_post": True,
        "post_quote": "Ling's Thai numbers page both teach the number words together with the numerals",
        "party": "Ling",
        "claim": "Ling teaches the numerals together with the number words",
        "url": LING,
        "quote": "Check out the Thai numerals we've written in the following table.",
        "verdict": "supported",
        "note": "Ling's page gives a table of the ten glyphs and then the words.",
    },
]

# Statements about Phayan, checked against this repository at HEAD.
# kind: contains -> the literal text must be in the file
#       count    -> the number of regex matches must equal `expect`
PRODUCT_CLAIMS = [
    {
        "id": "twenty-four-levels",
        "in_post": True,
        "post_quote": "24 levels",
        "claim": "the curriculum has 24 levels",
        "path": "apps/api/src/content/seed/levels.ts",
        "kind": "count",
        "pattern": r"^    id: 'L[0-9]+',$",
        "expect": 24,
        "verdict": "supported",
        "note": "24 level objects, numbered L0-L12, L14, L15, L17, L18, L23, L24, L26, L27, L29, L30, L31.",
    },
    {
        "id": "forty-two-letters",
        "in_post": True,
        "post_quote": "the 42 letters in current use",
        "claim": "42 letters are in current use",
        "path": "apps/api/src/content/seed/letters.ts",
        "kind": "contains",
        "pattern": "42 in use (mid 9 / high 10 / low 23)",
        "expect": 1,
        "verdict": "supported",
        "note": "44 letters authored, 42 in use, ฃ and ฅ obsolete.",
    },
    {
        "id": "five-free-levels",
        "in_post": True,
        "post_quote": "The first five levels are free.",
        "claim": "levels L0-L4 are free",
        "path": "apps/api/src/content/seed/curriculum-qa.spec.ts",
        "kind": "contains",
        "pattern": "FIRST_PAID_LEVEL === 5",
        "expect": 1,
        "verdict": "supported",
        "note": "The free range is L0-L4, five levels; L5 is the first paid one.",
    },
    {
        "id": "srs-intervals",
        "in_post": True,
        "post_quote": "come back for review after 1, 3, 7 and 21 days",
        "claim": "the review ladder is 1, 3, 7, 21 days",
        "path": "apps/web/src/lib/learn/srs/schedule.ts",
        "kind": "contains",
        "pattern": "export const SRS_INTERVALS = [1, 3, 7, 21] as const;",
        "expect": 1,
        "verdict": "supported",
        "note": "After the 21-day anchor the interval grows and a word retires near 180 days.",
    },
    {
        "id": "three-correct-mastery",
        "in_post": True,
        "post_quote": "three correct answers in a row mark an item as mastered, and a wrong answer resets that count",
        "claim": "three consecutive correct answers master an item; a wrong answer resets",
        "path": "apps/web/src/lib/learn/srs/schedule.ts",
        "kind": "contains",
        "pattern": "/** Mastery threshold — consecutive correct answers (a wrong answer resets). */",
        "expect": 1,
        "verdict": "supported",
        "note": "Mastery is not retirement; an item retires later, when the growing interval would pass 180 days.",
    },
    {
        "id": "local-first-progress",
        "in_post": True,
        "post_quote": "progress is kept on your device without an account",
        "claim": "progress works without an account",
        "path": "apps/web/src/lib/learn/progress-store/store.ts",
        "kind": "contains",
        "pattern": "The localStorage-first progress store (DEC-21, R11): works with zero auth",
        "expect": 1,
        "verdict": "supported",
        "note": "localStorage first, in-memory when storage is unavailable, sync later.",
    },
    {
        "id": "numerals-not-in-curriculum",
        "in_post": True,
        "post_quote": "the ten numerals are not a lesson of their own anywhere in it",
        "claim": "no Thai numeral glyph appears in the level definitions",
        "path": "apps/api/src/content/seed/levels.ts",
        "kind": "count",
        "pattern": "[" + THAI_DIGITS + "]",
        "expect": 0,
        "verdict": "supported",
        "note": "levels.ts holds no numeral glyph at all.",
    },
    {
        "id": "no-numerals-as-words",
        "in_post": True,
        "post_quote": "the ten numerals are not a lesson of their own anywhere in it",
        "claim": "no numeral glyph is authored as a taught word",
        "path": "apps/api/src/content/seed/words.ts",
        "kind": "count",
        "pattern": r"^\s*w\('[" + THAI_DIGITS + "]",
        "expect": 0,
        "verdict": "supported",
        "note": "Checked at the start of each word entry, so English glosses such as 'one (๑)' do not count.",
    },
    {
        "id": "typing-is-free-text",
        "in_post": False,
        "post_quote": "its typing engine makes you build each word out of letter variants and checks every pick",
        "claim": "the learner builds a word by picking letter variants",
        "path": "apps/web/src/components/learn/input/TypingInput.tsx",
        "kind": "contains",
        "pattern": "<input",
        "expect": 1,
        "verdict": "wrong",
        "note": "Removed from the post on 2026-10-26. The surface is a free-text field; nothing is built by picking letters.",
    },
    {
        "id": "counting-irregularities-not-flagged",
        "in_post": False,
        "post_quote": "Phayan flags all three the first time their number comes up, rather than leaving a learner to spot the pattern alone.",
        "claim": "the app flags เอ็ด, ยี่สิบ and the dropped หนึ่ง",
        "path": "apps/api/src/content/seed/levels.ts",
        "kind": "count",
        "pattern": "เอ็ด|ยี่สิบ",
        "expect": 0,
        "verdict": "wrong",
        "note": "Removed from the post on 2026-10-26. None of the three counting irregularities appears in the curriculum seed.",
    },
]


# ---------------------------------------------------------------- fetching


def normalize(text: str) -> str:
    """Collapse whitespace, unescape entities and unify quote characters."""
    text = re.sub(r"(?is)<(script|style)[^>]*>.*?</\1>", " ", text)
    text = re.sub(r"(?s)<!--.*?-->", " ", text)
    text = re.sub(r"(?s)<[^>]+>", " ", text)
    text = html_mod.unescape(text)
    text = (
        text.replace("‘", "'")
        .replace("’", "'")
        .replace("“", '"')
        .replace("”", '"')
        .replace("–", "-")
        .replace("—", "-")
        .replace(" ", " ")
    )
    return re.sub(r"\s+", " ", text).strip()


def fetch(url: str, timeout: int = 30, attempts: int = 3) -> str:
    last: Exception | None = None
    for _ in range(attempts):
        try:
            req = urllib.request.Request(
                url, headers={"User-Agent": UA, "Accept-Language": "th,en;q=0.8"}
            )
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                raw = resp.read()
                if resp.headers.get("Content-Encoding") == "gzip":
                    raw = gzip.decompress(raw)
                charset = resp.headers.get_content_charset() or "utf-8"
            return raw.decode(charset, errors="replace")
        except Exception as exc:  # retried, then reported
            last = exc
    raise last  # type: ignore[misc]


def measure(url: str) -> dict:
    raw = fetch(url)
    title_m = re.search(r"(?is)<title[^>]*>(.*?)</title>", raw)
    visible = normalize(raw)
    num_matches = list(RE_THAI_NUM.finditer(visible))
    snippet = ""
    if num_matches:
        m = num_matches[0]
        snippet = stable_snippet(visible, m)
    be = RE_BE_YEAR_TH.search(visible)
    return {
        "status": "checked",
        "visible": visible,
        "title": normalize(title_m.group(1)) if title_m else "",
        "thai_letter_count": len(RE_THAI_LETTER.findall(visible)),
        "thai_numeral_count": len(num_matches),
        "arabic_digit_count": len(RE_ARABIC.findall(visible)),
        "present": bool(num_matches),
        "be_year_th": bool(be),
        "numeral_snippet": snippet,
        "be_year_snippet": (
            visible[max(0, be.start() - 60) : be.end() + 60].strip() if be else ""
        ),
    }


def run_all(urls: list[str]) -> dict[str, dict]:
    out: dict[str, dict] = {}
    with futures.ThreadPoolExecutor(max_workers=8) as pool:
        jobs = {pool.submit(measure, u): u for u in urls}
        for job in futures.as_completed(jobs):
            url = jobs[job]
            try:
                out[url] = job.result()
            except Exception as exc:  # recorded, never invented
                out[url] = {"status": "error", "error": f"{type(exc).__name__}: {exc}"}
    return out


# ------------------------------------------------------------ code claims


def git_show(path: str) -> str:
    out = subprocess.run(
        ["git", "-C", REPO, "show", f"{REF}:{path}"],
        capture_output=True,
        text=True,
        check=True,
    )
    return out.stdout


ALLOWED_IN_QUOTE = re.compile("[฀-๿ .]")

# Pages where the auto-derived snippet would be a bare glyph or a rotating
# headline. Each string is read off the page and re-verified on every run.
QUOTE_OVERRIDES = {
    "https://mgronline.com/": "เมืองไทย 360 ๐",
}


def stable_snippet(visible: str, match: re.Match, span: int = 45) -> str:
    """Widen a numeral match through Thai text only, so a live counter or a
    headline digit next to it cannot make the quote move between runs."""
    start, end = match.start(), match.end()
    left = start
    while left > 0 and start - left < span and ALLOWED_IN_QUOTE.match(visible[left - 1]):
        left -= 1
    right = end
    while right < len(visible) and right - end < span and ALLOWED_IN_QUOTE.match(visible[right]):
        right += 1
    return visible[left:right].strip()


def letters_without_audio(src: str) -> int:
    """Count letter() calls whose audioWord argument (13th, index 12) is empty."""
    missing = 0
    for chunk in src.split("  letter(")[1:]:
        body = chunk.split("\n  ),")[0]
        args: list[str] = []
        cur = ""
        depth = 0
        quote_char = None
        for ch in body:
            if quote_char:
                cur += ch
                if ch == quote_char:
                    quote_char = None
                continue
            if ch in "'\"":
                quote_char = ch
                cur += ch
                continue
            if ch in "([":
                depth += 1
            if ch in ")]":
                depth -= 1
            if ch == "," and depth == 0:
                args.append(cur.strip())
                cur = ""
            else:
                cur += ch
        if cur.strip():
            args.append(cur.strip())
        if len(args) < 13 or args[12] in ("", "''", '""'):
            missing += 1
    return missing


def check_product(claim: dict) -> tuple[bool, str]:
    src = git_show(claim["path"])
    kind = claim["kind"]
    if kind == "contains":
        found = claim["pattern"] in src
        return found, f"literal {'found' if found else 'MISSING'}"
    if kind == "count":
        n = len(re.findall(claim["pattern"], src, re.M))
        return n == claim["expect"], f"{n} matches, expected {claim['expect']}"
    if kind == "letters_without_audio":
        n = letters_without_audio(src)
        return n == claim["expect"], f"{n} letters without an audio word, expected {claim['expect']}"
    raise ValueError(f"unknown kind {kind}")


# --------------------------------------------------------------- writing


def write_dataset(measured: dict[str, dict]) -> None:
    rows = []
    for category, url in PAGES:
        m = measured[url]
        if m["status"] != "checked":
            rows.append(
                {
                    "category": category,
                    "url": url,
                    "value": "not checked",
                    "quote": "",
                    "status": "not-checked",
                    "reason": m.get("error", ""),
                }
            )
            continue
        usable = m["thai_letter_count"] >= MIN_THAI_LETTERS
        rows.append(
            {
                "category": category,
                "url": url,
                "value": "thai-numerals" if m["present"] else "arabic-digits-only",
                "quote": m["title"] or m["visible"][:60],
                "thai_numeral_count": m["thai_numeral_count"],
                "arabic_digit_count": m["arabic_digit_count"],
                "thai_letter_count": m["thai_letter_count"],
                "be_year_thai_numerals": m["be_year_th"],
                "numeral_quote": (
                    QUOTE_OVERRIDES[url]
                    if url in QUOTE_OVERRIDES and QUOTE_OVERRIDES[url] in m["visible"]
                    else m["numeral_snippet"]
                ),
                "be_year_quote": m["be_year_snippet"],
                "status": "checked" if usable else "not-checked-thin-text",
                "reason": ""
                if usable
                else f"served page holds {m['thai_letter_count']} Thai letters, below {MIN_THAI_LETTERS}",
            }
        )

    totals = {}
    for category in sorted({c for c, _ in PAGES}):
        cat = [r for r in rows if r["category"] == category]
        checked = [r for r in cat if r["status"] == "checked"]
        totals[category] = {
            "pages": len(cat),
            "checked": len(checked),
            "with_thai_numerals": sum(1 for r in checked if r["value"] == "thai-numerals"),
            "with_be_year_in_thai_numerals": sum(
                1 for r in checked if r.get("be_year_thai_numerals")
            ),
        }
    checked_all = [r for r in rows if r["status"] == "checked"]
    totals["all"] = {
        "pages": len(rows),
        "checked": len(checked_all),
        "with_thai_numerals": sum(1 for r in checked_all if r["value"] == "thai-numerals"),
        "with_be_year_in_thai_numerals": sum(
            1 for r in checked_all if r.get("be_year_thai_numerals")
        ),
    }

    third = []
    for item in THIRD_PARTY:
        try:
            visible = normalize(fetch(item["url"]))
            ok = normalize(item["quote"]) in visible
            problem = "" if ok else "quote not found on the page"
        except Exception as exc:
            ok = False
            problem = f"fetch failed: {type(exc).__name__}: {exc}"
        third.append({**item, "quote_found": ok, "problem": problem})

    product = []
    for item in PRODUCT_CLAIMS if REPO else []:
        try:
            ok, detail = check_product(item)
        except Exception as exc:
            ok, detail = False, f"{type(exc).__name__}: {exc}"
        product.append({**item, "evidence_holds": ok, "detail": detail})

    contradicted = [
        {"id": p["id"], "in_post": p["in_post"], "verdict": p["verdict"], "note": p["note"]}
        for p in product
        if p["verdict"] in ("wrong", "unsupported")
    ]
    cut = [{"id": t["id"], "in_post": t["in_post"], "note": t["note"]} for t in third if t["verdict"] == "cut"]

    dataset = {
        "post": "thai-numbers",
        "project": "phayan",
        "method": "public-web",
        "checkedAt": CHECKED_AT,
        "question": "On public Thai pages, are numbers printed in Thai numerals (U+0E50-U+0E59) or in Arabic digits, and does the page's category predict which?",
        "tools": {
            "python": sys.version.split()[0],
            "http": "urllib.request, standard library only",
            "git": subprocess.run(
                ["git", "--version"], capture_output=True, text=True
            ).stdout.strip(),
            "user_agent": UA,
            "no_third_party_packages": True,
        },
        "inputs": {
            "pages": [{"category": c, "url": u} for c, u in PAGES],
            "refused": [{"category": c, "url": u, "reason": r} for c, u, r in REFUSED],
            "third_party_urls": sorted({i["url"] for i in THIRD_PARTY}),
            "product_source": {"repo": "the Phayan repository holding this file", "ref": REF},
            "generated": [],
        },
        "rules": {
            "visible_text": "HTML with <script>, <style> and comments removed, tags stripped, entities unescaped, whitespace collapsed",
            "thai_numerals": "any character in U+0E50-U+0E59",
            "be_year_thai_numerals": "a four-character run ๒๔xx or ๒๕xx, i.e. 2400-2599 Buddhist Era",
            "usable_page": f"served HTML holds at least {MIN_THAI_LETTERS} Thai letters (U+0E01-U+0E2E); below that the page is a script shell and is recorded as not checked",
            "recheck_rule": "a page fails when its title quote or its numeral quote is no longer in the served text, or when the Thai-numeral presence flag flips; character counts move with the daily page and are not asserted",
            "quote_match": "both sides normalised: tags stripped, entities unescaped, curly quotes flattened, whitespace collapsed",
            "evidence_holds": "says only that the cited code or page still contains the string quoted as evidence; for a row whose verdict is wrong it is the refutation that is intact",
        },
        "results": rows,
        "totals": totals,
        "third_party": third,
        "product_claims": product,
        "contradicted_claims": contradicted,
        "claims_to_cut": cut,
        "notes": [
            "Counting characters measures what the served HTML prints, not what a browser renders after JavaScript. Every page counted here serves readable Thai text; a page whose numerals are injected after load would be missed, and that is the main limit of the method.",
            "One counted hit is not a number: mgronline.com's two Thai numerals are the superscript in its 'เมืองไทย 360๐' column logo, carried in the page chrome of every article. Read as printed numbers, the news category is 0 of 5.",
            "The national-park category could not be measured: dnp.go.th answers every path with an Incapsula challenge page holding no text, and tourismthailand.org returns 403. The post's park fee-board claim is not covered by this dataset.",
            "Banknote faces could not be measured either; the bank category measures bank websites, a different surface from printed notes. The Thai Wikipedia article on banknotes used Arabic digits only, which speaks about the encyclopedia, not about the notes.",
            "Pages fetched and refused, or served as script shells, are listed in results with status not-checked and in inputs.refused; they are excluded from every total rather than filled from memory.",
            "Sampling is one page per site at the site root or landing page, chosen by reachability. That skews the sample toward sites with server-rendered HTML and away from the deep pages where a fee table or a notice would sit.",
            "For banknotes a primary source was tried on 2026-10-26: two Bank of Thailand banknote URLs (bot.or.th, Thai and English) returned 404, and the English Wikipedia article on Thai baht banknotes states only that the 1925 notes carried the denomination 'in both Arabic and Thai numerals', which says nothing about current notes. The post therefore makes no claim about banknotes.",
            "Claims removed from the post because nothing in this dataset or a cited source backs them: the list of places where Thai numerals 'reliably appear' (banknotes, district office notices, dual-price boards at national parks, house numbers, provincial bus route numbers), the time it takes to learn the ten shapes, and how often speakers drop the leading หนึ่ง.",
        ],
    }
    with open(DATASET_JSON, "w", encoding="utf-8") as fh:
        json.dump(dataset, fh, ensure_ascii=False, indent=2)
        fh.write("\n")

    cols = [
        "category",
        "url",
        "status",
        "value",
        "thai_numeral_count",
        "arabic_digit_count",
        "thai_letter_count",
        "be_year_thai_numerals",
        "quote",
        "reason",
    ]
    with open(DATASET_CSV, "w", encoding="utf-8", newline="") as fh:
        w = csv.DictWriter(fh, fieldnames=cols, extrasaction="ignore")
        w.writeheader()
        for r in rows:
            w.writerow(r)

    print(f"wrote {DATASET_JSON} and {DATASET_CSV}")
    for cat, t in totals.items():
        print(
            f"  {cat:20} checked {t['checked']}/{t['pages']}  "
            f"thai-numerals {t['with_thai_numerals']}  be-year {t['with_be_year_in_thai_numerals']}"
        )
    bad = [t["id"] for t in third if not t["quote_found"]]
    print(f"  third_party quotes found: {len(third) - len(bad)}/{len(third)}" + (f"  missing {bad}" if bad else ""))
    badp = [p["id"] for p in product if not p["evidence_holds"]]
    print(f"  product claims, cited code still present: {len(product) - len(badp)}/{len(product)}" + (f"  failing {badp}" if badp else ""))


# ------------------------------------------------------------ verifying


def verify() -> int:
    with open(DATASET_JSON, encoding="utf-8") as fh:
        dataset = json.load(fh)
    failures: list[tuple[str, str]] = []

    rows = [r for r in dataset["results"] if r["status"] == "checked"]
    print(f"pages: {len(rows)} recorded as checked")
    measured = run_all([r["url"] for r in rows])
    for r in rows:
        m = measured[r["url"]]
        if m["status"] != "checked":
            failures.append((r["url"], f"fetch failed: {m.get('error')}"))
            continue
        visible = m["visible"]
        problems = []
        if r["quote"] and normalize(r["quote"]) not in visible:
            problems.append("title quote gone")
        if r.get("numeral_quote") and normalize(r["numeral_quote"]) not in visible:
            problems.append("numeral quote gone")
        if m["present"] != (r["value"] == "thai-numerals"):
            problems.append(
                "presence flipped: now "
                + ("thai-numerals" if m["present"] else "arabic-digits-only")
            )
        if bool(r.get("be_year_thai_numerals")) != m["be_year_th"]:
            problems.append("be-year flag flipped")
        if problems:
            failures.append((r["url"], "; ".join(problems)))
    print(f"pages verified: {len(rows) - len([f for f in failures if f[0] in {r['url'] for r in rows}])}/{len(rows)}")

    third = dataset["third_party"]
    print(f"third-party quotes: {len(third)}")
    for item in third:
        try:
            ok = normalize(item["quote"]) in normalize(fetch(item["url"]))
        except Exception as exc:
            ok = False
            failures.append((item["id"], f"fetch failed: {type(exc).__name__}: {exc}"))
            continue
        if not ok:
            failures.append((item["id"], f"quote gone from {item['url']}"))

    product = dataset["product_claims"] if REPO else []
    print(f"product claims: {len(product)}" if REPO else "product claims: skipped, this file is not inside the Phayan repository")
    for item in product:
        try:
            ok, detail = check_product(item)
        except Exception as exc:
            failures.append((item["id"], f"{type(exc).__name__}: {exc}"))
            continue
        if not ok:
            failures.append((item["id"], f"code changed: {detail}"))

    verdicts = [i["verdict"] for i in third] + [p["verdict"] for p in product]
    print(
        "verdicts: "
        + ", ".join(f"{v} {verdicts.count(v)}" for v in sorted(set(verdicts)))
    )

    if failures:
        print("\nFAILED")
        for key, why in failures:
            print(f"  {key}: {why}")
        return 1
    print("\nall rows hold")
    return 0


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--write", action="store_true", help="re-measure and rewrite dataset.json/.csv")
    ap.add_argument("--recheck", action="store_true", help="re-read pages and code; no lab step exists, so identical to a plain run")
    args = ap.parse_args()
    if args.write:
        write_dataset(run_all([u for _, u in PAGES]))
        return 0
    return verify()


if __name__ == "__main__":
    sys.exit(main())
