#!/usr/bin/env python3
"""Re-run the check behind the Phayan post "Thai Reading Practice".

Two kinds of evidence:

  lab / public-corpus  the app's own level order and word list, read from the
                       Phayan repository this file is published in (or the one
                       PHAYAN_REPO points at), rebuilt and re-measured here.
  pages                every third-party page the post relies on, fetched and
                       tested for its exact supporting sentence.

Exit 0 when every row holds, 1 otherwise.  `--recheck` re-reads the pages and
skips any lab step that takes more than a minute (the lab steps here take
under a second, so nothing is skipped).
"""

from __future__ import annotations

import html
import json
import os
import re
import sys
import time
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")

# apps/web/public/blog/<slug>/check.py -> repository root, five levels up.
REPO = os.environ.get("PHAYAN_REPO") or os.path.abspath(os.path.join(HERE, *[".."] * 5))

LEVELS = "apps/api/src/content/seed/levels.ts"
FREQ = "apps/api/src/content/seed/word-frequency.jsonl"
ACCESS = "packages/shared/src/learn-access.ts"
LETTERS = "apps/api/src/content/seed/letters.ts"
SRS = "apps/web/src/lib/learn/srs/schedule.ts"

UA = "Mozilla/5.0"
TIMEOUT = 45
LAB_BUDGET = 60.0

# Marks and punctuation that carry no new letter: the four tone marks, mai
# taikhu, the repetition and abbreviation marks, ASCII digits, space, slash.
NONLETTER = set("่้๊๋็ฯๆ0123456789 -/")

STRINGS = [
    "ปิด",
    "เปิด",
    "ชาย",
    "หญิง",
    "ซอย",
    "๕๐ บาท",
    "ดึง",
    "ผลัก",
    "ทางออก",
    "ถนน",
    "กรุงเทพฯ",
    "ผัดไทย",
]


def git_show(path: str) -> str:
    full = os.path.join(REPO, path)
    try:
        with open(full, encoding="utf-8") as fh:
            return fh.read()
    except OSError as exc:
        raise RuntimeError("cannot read %s: %s" % (full, exc))


def normalise(text: str) -> str:
    text = html.unescape(text)
    for src, dst in (
        ("’", "'"),
        ("‘", "'"),
        ("“", '"'),
        ("”", '"'),
        ("\u2013", "-"),
        ("\u2014", "-"),
        (" ", " "),
    ):
        text = text.replace(src, dst)
    return re.sub(r"\s+", " ", text).strip().lower()


def page_text(raw: str) -> str:
    raw = re.sub(r"(?is)<(script|style|noscript)[^>]*>.*?</\1>", " ", raw)
    return normalise(re.sub(r"<[^>]+>", " ", raw))


def fetch(url: str) -> str:
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
        charset = resp.headers.get_content_charset() or "utf-8"
        return resp.read().decode(charset, "replace")


def parse_levels(src: str):
    """[(number, level_id, new letters, new vowel-sign characters)] in file order."""
    parts = re.split(r"\n  \{\n    id: '(L\d+)',", src)
    levels = []
    for i in range(1, len(parts), 2):
        lid, body = parts[i], parts[i + 1]
        num = int(re.search(r"\n    number: (\d+),", body).group(1))
        letters = [s for g in re.findall(r"newLetters: \[([^\]]*)\]", body) for s in re.findall(r"'([^']*)'", g)]
        forms = [s for g in re.findall(r"newVowelForms: \[([^\]]*)\]", body) for s in re.findall(r"'([^']*)'", g)]
        signs = [c for f in forms for c in f if c != "-"]
        levels.append((num, lid, letters, signs))
    return levels


def lab() -> dict:
    """Rebuild the corpus measurement from the pinned repository files."""
    started = time.time()
    levels = parse_levels(git_show(LEVELS))
    rows = [json.loads(line) for line in git_show(FREQ).splitlines() if line.strip()]
    weighted = [r for r in rows if r["frequency"] is not None]
    total = sum(r["frequency"] for r in weighted)

    out = {
        "levels": [],
        "strings": {},
        "meta": {
            "level_count": len(levels),
            "corpus_rows": len(rows),
            "corpus_rows_with_frequency": len(weighted),
            "corpus_total_frequency": total,
            "elapsed_s": 0.0,
        },
    }

    taught = set()
    for num, lid, letters, signs in levels:
        taught |= set("".join(letters)) | set(signs)
        if time.time() - started > LAB_BUDGET:
            out["meta"]["aborted_at"] = lid
            break
        ok = [r for r in weighted if all(c in taught or c in NONLETTER for c in r["word"])]
        out["levels"].append(
            {
                "level_id": lid,
                "level_number": num,
                "words_covered": len(ok),
                "share_pct": round(100.0 * len(ok) / len(weighted), 1),
                "freq_share_pct": round(100.0 * sum(r["frequency"] for r in ok) / total, 1)
                if total
                else 0.0,
            }
        )
        for s in STRINGS:
            if s not in out["strings"] and all(c in taught or c in NONLETTER for c in s):
                out["strings"][s] = lid

    for s in STRINGS:
        out["strings"].setdefault(s, "none")

    # Letters the course never introduces, and words that need them.
    all_taught = set()
    for num, lid, letters, signs in levels:
        all_taught |= set("".join(letters)) | set(signs)
    missing = sorted({c for r in weighted for c in r["word"] if c not in all_taught and c not in NONLETTER})
    out["meta"]["chars_never_taught"] = ["U+%04X" % ord(c) for c in missing]
    out["meta"]["words_with_never_taught_char"] = sorted(
        {r["word"] for r in weighted if any(c in missing for c in r["word"])}
    )
    out["meta"]["elapsed_s"] = round(time.time() - started, 2)
    return out


def code_facts() -> dict:
    access = git_show(ACCESS)
    letters = git_show(LETTERS)
    srs = git_show(SRS)
    levels = parse_levels(git_show(LEVELS))
    introduced = set()
    for num, lid, ls, ss in levels:
        introduced |= set("".join(ls)) | set(ss)
    return {
        "level_count": len(levels),
        "first_paid_level": int(re.search(r"FIRST_PAID_LEVEL\s*=\s*(\d+)", access).group(1)),
        "in_use_letters": int(
            re.search(r"//.*?(\d+) in use \(mid \d+ / high \d+ / low \d+\)", letters).group(1)
        ),
        "srs_intervals": [int(x) for x in re.search(r"SRS_INTERVALS = \[([^\]]*)\]", srs).group(1).split(",")],
        "srs_mastery_count": int(re.search(r"SRS_MASTERY_COUNT = (\d+)", srs).group(1)),
        "srs_retirement_days": int(re.search(r"SRS_RETIREMENT_INTERVAL_DAYS = (\d+)", srs).group(1)),
        "srs_mastered_growth": float(re.search(r"SRS_MASTERED_GROWTH = ([\d.]+)", srs).group(1)),
        "thai_digits_taught": len(introduced & set("๐๑๒๓๔๕๖๗๘๙")),
    }


def check_row(row: dict, facts: dict, corpus: dict, recheck: bool) -> tuple:
    """(ok, detail) for one dataset row."""
    group = row["group"]

    if group in ("corpus", "strings"):
        if row["property"] == "words_covered":
            got = next(
                (lv for lv in corpus["levels"] if lv["level_id"] == row["option"]),
                None,
            )
            if got is None:
                return False, "level %s missing from the rebuild" % row["option"]
            ok = got["words_covered"] == row["value"]
            return ok, "%s words_covered %s (dataset %s)" % (row["option"], got["words_covered"], row["value"])
        if row["property"] == "freq_share_pct":
            got = next(lv for lv in corpus["levels"] if lv["level_id"] == row["option"])
            ok = abs(got["freq_share_pct"] - float(row["value"])) < 0.05
            return ok, "%s freq_share_pct %s (dataset %s)" % (row["option"], got["freq_share_pct"], row["value"])
        if row["property"] == "first_level":
            got = corpus["strings"].get(row["option"], "none")
            ok = got == row["value"]
            return ok, "%s first_level %s (dataset %s)" % (row["option"], got, row["value"])
        return False, "unknown property %r" % row["property"]

    if group == "product":
        key, expected = row["option"], row["value"]
        got = facts.get(key)
        if isinstance(expected, list):
            ok = list(got) == list(expected)
        elif isinstance(expected, float):
            ok = abs(float(got) - expected) < 1e-9
        else:
            ok = got == expected
        return ok, "%s = %s (dataset %s)" % (key, got, expected)

    source = row.get("source")
    quote = row.get("quote")

    if row.get("status") == "cut" and not source:
        return True, "recorded as cut, nothing to fetch"

    if not quote:
        return False, "row has no quote"

    if source and source.startswith("http"):
        try:
            body = fetch(source)
        except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError) as exc:
            return False, "fetch failed: %s" % exc
        ok = normalise(quote) in page_text(body)
        return ok, "quote %s on %s" % ("found" if ok else "MISSING", source)

    if source and source.startswith("repo:"):
        path = source[len("repo:"):]
        try:
            body = git_show(path)
        except RuntimeError as exc:
            return False, str(exc)
        ok = normalise(quote) in normalise(body)
        return ok, "quote %s in %s" % ("found" if ok else "MISSING", path)

    return False, "no source to check against"


def main() -> int:
    recheck = "--recheck" in sys.argv[1:]
    with open(DATASET, encoding="utf-8") as fh:
        dataset = json.load(fh)

    print("dataset %s  checkedAt %s  method %s" % (dataset["slug"], dataset["checkedAt"], dataset["method"]))
    print("mode: %s" % ("recheck" if recheck else "full"))

    corpus = lab()
    facts = code_facts()

    failures = 0
    published = next(i for i in dataset["inputs"] if "newCharactersByLevel" in i)["newCharactersByLevel"]
    parsed = {
        lid: "".join(sorted(set("".join(ls)) | set(ss))) for _, lid, ls, ss in parse_levels(git_show(LEVELS))
    }
    ok = parsed == published
    if not ok:
        failures += 1
    print("%-4s %-12s %-28s %s" % ("ok" if ok else "FAIL", "inputs", "levels-new-characters",
                                   "level file parses to the published table" if ok else "level file differs from the published table"))
    for row in dataset["results"]:
        ok, detail = check_row(row, facts, corpus, recheck)
        if not ok:
            failures += 1
        print("%-4s %-12s %-28s %s" % ("ok" if ok else "FAIL", row["group"], row["id"], detail))

    print()
    print("corpus: %d levels, %d rows, %d with frequency, %d chars never taught"
          % (
              corpus["meta"]["level_count"],
              corpus["meta"]["corpus_rows"],
              corpus["meta"]["corpus_rows_with_frequency"],
              len(corpus["meta"]["chars_never_taught"]),
          ))
    print("%d rows, %d failed" % (len(dataset["results"]) + 1, failures))
    return 0 if failures == 0 else 1


if __name__ == "__main__":
    sys.exit(main())
