#!/usr/bin/env python3
"""Re-runs the evidence check for the Phayan post thai-consonant-classes.

1. Corpus (public-corpus). Downloads the Thai National Corpus frequency list
   that PyThaiNLP publishes (CC0), keeps the 1,000 most frequent words, and
   attributes each word to a consonant class by the first of the 42 in-use
   consonants it contains. Reports, per class, how many of the 1,000 words it
   begins and how many distinct letters of that class ever begin one; plus
   which letters of the 23-strong low class never appear at all.
2. Post inventory. The three letter lists the post prints must hold 9, 10 and
   23 letters, be duplicate-free, and together be exactly the 42 consonants of
   U+0E01..U+0E2E minus the obsolete ฃ and ฅ.
3. Pages. Re-fetches the page of each named third party and checks that the
   supporting sentence still appears on it. `cut` rows carry no quote and are
   reported as not checked.

Exit 0 when every stored value and quote still holds, 1 otherwise.
  --recheck  reuse a corpus copy cached in the last week; always re-read pages
  --write    store the measurement in dataset.json and regenerate dataset.csv

Standard library only.
"""

from __future__ import annotations

import argparse
import csv
import gzip
import html
import json
import os
import re
import sys
import tempfile
import time
import unicodedata
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
CSV_PATH = os.path.join(HERE, "dataset.csv")

CORPUS_URL = (
    "https://raw.githubusercontent.com/PyThaiNLP/pythainlp/dev/pythainlp/corpus/tnc_freq.txt"
)
CORPUS_SHA256 = "554984d62e8735e2705bb30b06096d5710d0c9011e340ab717134e10d6dd0dda"

MID = "กจฎฏดตบปอ"
HIGH = "ขฉฐถผฝศษสห"
LOW = "คฆงชซฌญฑฒณทธนพฟภมยรลวฬฮ"
OBSOLETE = "ฃฅ"
CLASSES = (("mid", "อักษรกลาง", MID), ("high", "อักษรสูง", HIGH), ("low", "อักษรต่ำ", LOW))
ALL = MID + HIGH + LOW
TOP_N = 1000

# Unicode puts the consonants in chart order inside U+0E01..U+0E2E; ฤ and ฦ sit
# in that range but are vowels, not consonants.
CHART = "".join(chr(c) for c in range(0x0E01, 0x0E2F) if chr(c) not in "ฤฦ")
IN_USE = "".join(c for c in CHART if c not in OBSOLETE)

CSV_FIELDS = [
    "class",
    "thai_name",
    "letters_in_use",
    "letters_beginning_a_top1000_word",
    "letters_absent_as_initial",
    "letters_never_appearing_in_the_top1000",
    "words_in_top1000",
    "share_of_top1000_pct",
    "tokens_in_top1000",
    "token_share_pct",
]


def fetch(url: str, timeout: int = 60) -> bytes:
    req = urllib.request.Request(
        url,
        headers={
            "User-Agent": "Mozilla/5.0",
            "Accept-Encoding": "gzip",
            "Accept": "text/html,application/xhtml+xml,*/*",
        },
    )
    with urllib.request.urlopen(req, timeout=timeout) as r:
        body = r.read()
        if r.headers.get("Content-Encoding") == "gzip":
            body = gzip.decompress(body)
        return body


def normalise(text: str) -> str:
    text = unicodedata.normalize("NFC", html.unescape(text))
    for a, b in (("’", "'"), ("‘", "'"), ("“", '"'), ("”", '"'), ("\xa0", " ")):
        text = text.replace(a, b)
    return re.sub(r"\s+", " ", text).strip()


def visible(html_text: str) -> str:
    """The page as a reader sees it: no scripts, styles or tags."""
    text = re.sub(r"<script.*?</script>", " ", html_text, flags=re.S | re.I)
    text = re.sub(r"<style.*?</style>", " ", text, flags=re.S | re.I)
    text = re.sub(r"<[^>]+>", " ", text)
    return normalise(text)


def class_of(letter: str) -> str | None:
    for key, _, letters in CLASSES:
        if letter in letters:
            return key
    return None


# ---------------------------------------------------------------- corpus

def load_corpus(recheck: bool) -> list[tuple[str, int]]:
    cache = os.path.join(tempfile.gettempdir(), "phayan-tnc", "tnc_freq.txt")
    fresh = os.path.exists(cache) and time.time() - os.path.getmtime(cache) < 7 * 86400
    if not (recheck and fresh):
        os.makedirs(os.path.dirname(cache), exist_ok=True)
        with open(cache, "wb") as fh:
            fh.write(fetch(CORPUS_URL))
    rows = []
    with open(cache, encoding="utf-8") as fh:
        for line in fh:
            parts = line.rstrip("\n").split("\t")
            if len(parts) < 2:
                continue
            word, freq = parts[0], int(parts[1])
            if not all("฀" <= ch <= "๿" for ch in word):
                continue
            if any(ch in OBSOLETE for ch in word):
                continue
            if not any(ch in ALL for ch in word):
                continue
            rows.append((word, freq))
    return rows


def sha256_of(path: str) -> str:
    import hashlib

    h = hashlib.sha256()
    with open(path, "rb") as fh:
        for block in iter(lambda: fh.read(1 << 20), b""):
            h.update(block)
    return h.hexdigest()


def initial_of(word: str) -> str | None:
    for ch in word:
        if ch in ALL:
            return ch
    return None


def measure(recheck: bool) -> dict:
    rows = load_corpus(recheck)
    top = rows[:TOP_N]
    tokens_total = sum(f for _, f in top)

    per_letter_words: dict[str, int] = {c: 0 for c in ALL}
    first_seen: dict[str, int] = {}
    tokens_by_class = {key: 0 for key, _, _ in CLASSES}
    for rank, (word, freq) in enumerate(top, 1):
        initial = initial_of(word)
        if initial is None:
            continue
        per_letter_words[initial] += 1
        first_seen.setdefault(initial, rank)
        tokens_by_class[class_of(initial)] += freq

    anywhere = {c: 0 for c in ALL}
    for word, _ in top:
        for ch in set(word):
            if ch in ALL:
                anywhere[ch] += 1

    classes = {}
    for key, thai_name, letters in CLASSES:
        words = sum(per_letter_words[c] for c in letters)
        present = [c for c in letters if per_letter_words[c]]
        absent = [c for c in letters if not per_letter_words[c]]
        never = [c for c in letters if not anywhere[c]]
        classes[key] = {
            "thai_name": thai_name,
            "letters_in_use": len(letters),
            "words_in_top1000": words,
            "share_of_top1000_pct": round(100.0 * words / len(top), 1),
            "tokens_in_top1000": tokens_by_class[key],
            "token_share_pct": round(100.0 * tokens_by_class[key] / tokens_total, 1),
            "letters_beginning_a_top1000_word": len(present),
            "letters_absent_as_initial": " ".join(absent),
            "letters_never_appearing_in_the_top1000": " ".join(never),
        }

    letters = [
        {
            "letter": c,
            "class": class_of(c),
            "words_it_begins_in_top1000": per_letter_words[c],
            "rank_of_first_such_word": first_seen.get(c),
            "words_of_top1000_containing_it": anywhere[c],
        }
        for c in ALL
    ]

    return {
        "top_n": TOP_N,
        "words_in_list": len(rows),
        "frequency_of_rank_1": top[0][1],
        "frequency_of_rank_1000": top[-1][1],
        "tokens_in_top1000": tokens_total,
        "corpus_sha256": sha256_of(os.path.join(tempfile.gettempdir(), "phayan-tnc", "tnc_freq.txt")),
        "classes": classes,
        "letters": letters,
    }


def csv_rows(measurement: dict) -> list[dict]:
    out = []
    for key, _, _ in CLASSES:
        c = measurement["classes"][key]
        out.append(
            {
                "class": key,
                "thai_name": c["thai_name"],
                "letters_in_use": c["letters_in_use"],
                "letters_beginning_a_top1000_word": c["letters_beginning_a_top1000_word"],
                "letters_absent_as_initial": c["letters_absent_as_initial"],
                "letters_never_appearing_in_the_top1000": c[
                    "letters_never_appearing_in_the_top1000"
                ],
                "words_in_top1000": c["words_in_top1000"],
                "share_of_top1000_pct": c["share_of_top1000_pct"],
                "tokens_in_top1000": c["tokens_in_top1000"],
                "token_share_pct": c["token_share_pct"],
            }
        )
    return out


def write_csv(measurement: dict) -> None:
    rows = csv_rows(measurement)
    with open(CSV_PATH, "w", encoding="utf-8", newline="") as fh:
        w = csv.DictWriter(fh, fieldnames=CSV_FIELDS)
        w.writeheader()
        w.writerows(rows)


def check_csv(measurement: dict) -> list[str]:
    with open(CSV_PATH, encoding="utf-8", newline="") as fh:
        stored = [{k: ("" if v is None else v) for k, v in r.items()} for r in csv.DictReader(fh)]
    want = [{k: str(v) for k, v in r.items()} for r in csv_rows(measurement)]
    if stored == want:
        return []
    return ["dataset.csv does not match the measured values"]


# ---------------------------------------------------------------- post inventory

def check_inventory(stored: dict) -> list[str]:
    problems = []
    lists = stored["inputs"]["post_lists"]
    union: list[str] = []
    expected = {"mid": 9, "high": 10, "low": 23}
    for row in lists:
        letters = row["letters"].split()
        union += letters
        if len(letters) != expected[row["class"]]:
            problems.append(
                f"post lists {len(letters)} {row['class']} letters, the inventory says "
                f"{expected[row['class']]}"
            )
        if len(set(letters)) != len(letters):
            problems.append(f"post repeats a {row['class']} letter")
        wrong = [c for c in letters if class_of(c) != row["class"]]
        if wrong:
            problems.append(f"post files under {row['class']}: {wrong}")
    if len(set(union)) != len(union):
        problems.append("a letter appears in two of the post's classes")
    if set(union) != set(IN_USE):
        problems.append(
            f"post's 42 letters are not the chart's 42 in use: extra "
            f"{sorted(set(union) - set(IN_USE))}, missing {sorted(set(IN_USE) - set(union))}"
        )
    print(f"  ok    inventory: {len(MID)}/{len(HIGH)}/{len(LOW)} = {len(union)} letters, "
          f"the 42 of U+0E01..U+0E2E minus {OBSOLETE}")
    return problems


# ---------------------------------------------------------------- pages

def check_pages(rows: list[dict]) -> list[str]:
    problems, cache = [], {}
    for row in rows:
        if row["verdict"] == "cut":
            print(f"  skip  {row['id']}: no page to check (verdict cut)")
            continue
        url = row["url"]
        if url not in cache:
            try:
                cache[url] = visible(fetch(url).decode("utf-8", "replace"))
            except Exception as exc:  # noqa: BLE001 - an unreachable page fails the check
                cache[url] = None
                problems.append(f"{row['id']}: {url}: fetch failed ({exc})")
        body = cache[url]
        if body is None:
            continue
        if normalise(row["quote"]) in body:
            print(f"  ok    {row['id']}: {url}")
        else:
            problems.append(f"{row['id']}: {url}: quote not found: {row['quote'][:70]!r}")
    return problems


# ---------------------------------------------------------------- main

def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--recheck", action="store_true")
    ap.add_argument("--write", action="store_true")
    args = ap.parse_args()

    with open(DATASET, encoding="utf-8") as fh:
        dataset = json.load(fh)
    problems: list[str] = []

    print(f"corpus: {CORPUS_URL}")
    got = measure(args.recheck)
    print(f"  {got['words_in_list']} words in the list, top {got['top_n']} "
          f"({got['frequency_of_rank_1']} to {got['frequency_of_rank_1000']} occurrences, "
          f"{got['tokens_in_top1000']} tokens)")
    for key, _, _ in CLASSES:
        c = got["classes"][key]
        print(f"  {key:<5} {c['words_in_top1000']:>4} words ({c['share_of_top1000_pct']:>4}%), "
              f"{c['letters_beginning_a_top1000_word']:>2} of {c['letters_in_use']} letters as "
              f"initial, absent {c['letters_absent_as_initial'] or '-'}")
    if got["corpus_sha256"] != CORPUS_SHA256:
        problems.append(f"corpus file changed: sha256 {got['corpus_sha256']}")

    if args.write:
        dataset["results"]["measurement"] = got
        with open(DATASET, "w", encoding="utf-8") as fh:
            json.dump(dataset, fh, ensure_ascii=False, indent=2)
            fh.write("\n")
        write_csv(got)
        print("  wrote dataset.json and dataset.csv")
        stored = got
    else:
        stored = dataset["results"]["measurement"]
        if got["classes"] != stored["classes"]:
            for key, _, _ in CLASSES:
                if got["classes"][key] != stored["classes"][key]:
                    problems.append(f"{key} row differs from dataset.json")
        if got["letters"] != stored["letters"]:
            problems.append("per-letter counts differ from dataset.json")
    problems += check_csv(stored)

    print("post inventory:")
    problems += check_inventory(dataset)

    print("pages:")
    problems += check_pages(dataset["results"]["thirdParty"])

    if problems:
        print("\nFAIL")
        for p in problems:
            print("  - " + p)
        return 1
    print("\nOK: every stored value and quote still holds")
    return 0


if __name__ == "__main__":
    sys.exit(main())
