#!/usr/bin/env python3
"""Re-runs the evidence check for learn-thai-alphabet-methods.

1. Letter order (public corpus). Takes the Thai National Corpus word
   frequency list that PyThaiNLP publishes (CC0), walks three consonant orders
   (the traditional chart, Phayan's level order, a frequency-greedy order) and
   measures after each letter how much of the running text a reader can
   decode, a word counting as decodable once every consonant in it is known.
2. Pages. Re-fetches every page the post cites about a named service and
   checks that each supporting sentence still appears, and that Duolingo's
   public course list still holds no Thai course.

Exit 0 when every stored value and quote still holds, 1 otherwise.
  --recheck  re-read the pages; reuse a corpus copy cached in the last week
  --write    store the measured results in dataset.json and dataset.csv

Standard library only.
"""

from __future__ import annotations

import argparse
import csv
import html
import json
import os
import re
import sys
import tempfile
import time
import unicodedata
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
CSV_PATH = os.path.join(HERE, "dataset.csv")
REPO = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", ".."))
SEED = os.path.join(REPO, "apps", "api", "src", "content", "seed", "levels.ts")

CORPUS_URL = (
    "https://raw.githubusercontent.com/PyThaiNLP/pythainlp/dev/pythainlp/corpus/tnc_freq.txt"
)
DUOLINGO_COURSES = "https://www.duolingo.com/api/1/courses/list"

# Unicode encodes the consonants in chart order inside U+0E01..U+0E2E; ฤ and ฦ
# sit in that range but are vowels.
CONSONANTS = "".join(chr(c) for c in range(0x0E01, 0x0E2F) if chr(c) not in "ฤฦ")
OBSOLETE = "ฃฅ"
STEPS = (5, 10, 15, 20, 25, 30, 35, 40)
MARKS = ("25", "50", "75")

ORDERS = (
    ("traditional_chart", "Traditional chart order (ก ไก่ to ฮ นกฮูก)"),
    ("phayan_level_order", "Phayan's level order"),
    ("frequency_greedy", "Frequency-greedy order (computed from this list)"),
)


def fetch(url: str, timeout: int = 60) -> bytes:
    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return r.read()


def normalise(text: str) -> str:
    text = unicodedata.normalize("NFC", html.unescape(text))
    for a, b in (("’", "'"), ("‘", "'"), ("“", '"'), ("”", '"'), ("\xa0", " ")):
        text = text.replace(a, b)
    return re.sub(r"\s+", " ", text).strip()


# ---------------------------------------------------------------- letter order

def phayan_order() -> list[str] | None:
    if not os.path.exists(SEED):
        return None
    with open(SEED, encoding="utf-8") as fh:
        seed = fh.read()
    order: list[str] = []
    for block in re.finditer(r"newLetters: \[([^\]]*)\]", seed):
        for glyph in re.findall(r"'([^']*)'", block.group(1)):
            if glyph not in order:
                order.append(glyph)
    return order


def load_corpus(recheck: bool) -> list[tuple[str, int]]:
    cache = os.path.join(tempfile.gettempdir(), "phayan-tnc", "tnc_freq.txt")
    fresh = os.path.exists(cache) and time.time() - os.path.getmtime(cache) < 7 * 86400
    if not (recheck and fresh):
        os.makedirs(os.path.dirname(cache), exist_ok=True)
        with open(cache, "wb") as fh:
            fh.write(fetch(CORPUS_URL))
    rows = []
    with open(cache, encoding="utf-8") as fh:
        for line in fh:
            parts = line.rstrip("\n").split("\t")
            if len(parts) < 2:
                continue
            word, freq = parts[0], int(parts[1])
            if not all("฀" <= ch <= "๿" for ch in word):
                continue
            if any(ch in OBSOLETE for ch in word):
                continue
            if not any(ch in CONSONANTS for ch in word):
                continue
            rows.append((word, freq))
    return rows


def mask_of(word: str) -> int:
    m = 0
    for ch in word:
        i = CONSONANTS.find(ch)
        if i >= 0:
            m |= 1 << i
    return m


def curve(order: list[str], masks: dict[int, int], top: list[int], total: int) -> dict:
    known = 0
    shares: dict[str, float] = {}
    words: dict[str, int] = {}
    reach: dict[str, int | None] = {m: None for m in MARKS}
    for n, letter in enumerate(order, 1):
        known |= 1 << CONSONANTS.index(letter)
        share = 100.0 * sum(f for m, f in masks.items() if m & ~known == 0) / total
        if n in STEPS:
            shares[str(n)] = round(share, 1)
            words[str(n)] = sum(1 for m in top if m & ~known == 0)
        for mark in MARKS:
            if reach[mark] is None and share >= int(mark):
                reach[mark] = n
    return {"letters": len(order), "text_share": shares, "top1000_words": words, "letters_to": reach}


def greedy(masks: dict[int, int]) -> list[str]:
    known, order, left = 0, [], list(CONSONANTS)
    while left:
        def gain(c: str) -> int:
            k = known | 1 << CONSONANTS.index(c)
            return sum(f for m, f in masks.items() if m & ~k == 0)
        best = max(left, key=lambda c: (gain(c), -CONSONANTS.index(c)))
        order.append(best)
        left.remove(best)
        known |= 1 << CONSONANTS.index(best)
    return order


def measure(recheck: bool, seed_order: list[str]) -> dict:
    rows = load_corpus(recheck)
    masks: dict[int, int] = {}
    for word, freq in rows:
        m = mask_of(word)
        masks[m] = masks.get(m, 0) + freq
    total = sum(masks.values())
    top = [mask_of(w) for w, _ in rows[:1000]]
    g = greedy(masks)
    return {
        "words_in_list": len(rows),
        "tokens_in_list": total,
        "traditional_chart": curve(list(CONSONANTS), masks, top, total),
        "phayan_level_order": curve(seed_order, masks, top, total),
        "frequency_greedy": curve(g, masks, top, total),
        "orders": {
            "traditional_chart": list(CONSONANTS),
            "phayan_level_order": seed_order,
            "frequency_greedy": g,
        },
    }


def csv_rows(results: dict) -> list[dict]:
    out = []
    for key, label in ORDERS:
        c = results[key]
        row = {"order": key, "label": label, "letters_in_order": c["letters"]}
        for s in STEPS:
            row[f"text_share_after_{s}"] = c["text_share"][str(s)]
        for s in STEPS:
            row[f"top1000_words_after_{s}"] = c["top1000_words"][str(s)]
        for m in MARKS:
            row[f"letters_to_{m}pct_of_text"] = c["letters_to"][m]
        out.append(row)
    return out


def write_csv(results: dict) -> None:
    rows = csv_rows(results)
    with open(CSV_PATH, "w", encoding="utf-8", newline="") as fh:
        w = csv.DictWriter(fh, fieldnames=list(rows[0]))
        w.writeheader()
        w.writerows(rows)


def check_csv(results: dict) -> list[str]:
    with open(CSV_PATH, encoding="utf-8", newline="") as fh:
        stored = list(csv.DictReader(fh))
    want = [{k: str(v) for k, v in r.items()} for r in csv_rows(results)]
    return [] if stored == want else ["dataset.csv does not match dataset.json"]


# ---------------------------------------------------------------- pages

def check_pages(rows: list[dict]) -> list[str]:
    problems, cache = [], {}
    for row in rows:
        for ev in row["evidence"]:
            url = ev["fetch"]
            if url not in cache:
                try:
                    cache[url] = normalise(fetch(url).decode("utf-8", "replace"))
                except Exception as exc:  # noqa: BLE001 - an unreachable page fails the check
                    cache[url] = None
                    problems.append(f"{row['id']}: {url}: fetch failed ({exc})")
            body = cache[url]
            if body is None:
                continue
            if normalise(ev["quote"]) in body:
                print(f"  ok    {row['id']}: {url}")
            else:
                problems.append(f"{row['id']}: {url}: quote not found: {ev['quote'][:70]!r}")
    return problems


def check_duolingo(stored: dict) -> list[str]:
    try:
        courses = json.loads(fetch(DUOLINGO_COURSES))
    except Exception as exc:  # noqa: BLE001
        return [f"duolingo: {DUOLINGO_COURSES}: fetch failed ({exc})"]
    thai = [c for c in courses if c.get("learning_language") == "th"]
    print(f"  {'ok' if not thai else 'FAIL':<5} duolingo-no-thai: {len(courses)} courses listed "
          f"(stored {stored['courses_listed']}), {len(thai)} teach Thai")
    return [f"duolingo now lists a Thai course: {thai}"] if thai else []


# ---------------------------------------------------------------- main

def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--recheck", action="store_true")
    ap.add_argument("--write", action="store_true")
    args = ap.parse_args()

    with open(DATASET, encoding="utf-8") as fh:
        dataset = json.load(fh)
    stored = dataset["results"]["letterOrder"]
    problems: list[str] = []

    seed_order = phayan_order()
    if seed_order is None:
        print("  note: Phayan seed not found next to this file; using the stored order")
        seed_order = stored["orders"]["phayan_level_order"]
    elif not args.write and seed_order != stored["orders"]["phayan_level_order"]:
        problems.append("Phayan's level order in the seed differs from the stored order")

    print(f"corpus: {CORPUS_URL}")
    got = measure(args.recheck, seed_order)
    print(f"  {got['words_in_list']} words, {got['tokens_in_list']} tokens")
    for key, label in ORDERS:
        c = got[key]
        print(f"  {key:<20} @10={c['text_share']['10']:>5}%  @20={c['text_share']['20']:>5}%  "
              f"@30={c['text_share']['30']:>5}%  half at {c['letters_to']['50']}  "
              f"top-1000 words @20={c['top1000_words']['20']}")

    if args.write:
        dataset["results"]["letterOrder"] = got
        with open(DATASET, "w", encoding="utf-8") as fh:
            json.dump(dataset, fh, ensure_ascii=False, indent=2)
            fh.write("\n")
        write_csv(got)
        stored = got
        print("  wrote dataset.json and dataset.csv")
    elif got != stored:
        problems.append("measured letter-order results differ from dataset.json")
    problems += check_csv(stored)

    print("pages:")
    problems += check_pages(dataset["results"]["pages"])
    problems += check_duolingo(dataset["results"]["duolingo"])

    if problems:
        print("\nFAIL")
        for p in problems:
            print("  - " + p)
        return 1
    print("\nOK: every stored value and quote still holds")
    return 0


if __name__ == "__main__":
    sys.exit(main())
