#!/usr/bin/env python3
"""Re-runs the public-page check for the thai-days-of-the-week dataset.

Every row names a public Thai page and one opening-hours string taken from it.
This script refetches each page, confirms the string is still there after
whitespace, entity and tag normalisation, recomputes how the seven days are
written in that string, and compares the counts with dataset.json. It also
refetches the third-party quotes, recomputes the weekday of every cited
date, and confirms dataset.csv carries the same counts. Exit 0 when everything holds, 1 otherwise.

    python3 check.py            full run
    python3 check.py --recheck  same run (there are no lab steps to skip)

Standard library only.
"""

import argparse
import csv
import datetime
import html
import json
import os
import re
import sys
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"

FULL = [("วันอาทิตย์", "Sunday"), ("วันจันทร์", "Monday"), ("วันอังคาร", "Tuesday"),
        ("วันพุธ", "Wednesday"), ("วันพฤหัสบดี", "Thursday"), ("วันศุกร์", "Friday"),
        ("วันเสาร์", "Saturday")]
BARE = [("อาทิตย์", "Sunday"), ("จันทร์", "Monday"), ("อังคาร", "Tuesday"),
        ("พุธ", "Wednesday"), ("พฤหัสบดี", "Thursday"), ("ศุกร์", "Friday"),
        ("เสาร์", "Saturday")]
ABBR = [("อา", "Sunday"), ("จ", "Monday"), ("อ", "Tuesday"), ("พ", "Wednesday"),
        ("พฤ", "Thursday"), ("ศ", "Friday"), ("ส", "Saturday")]


def normalise(raw):
    raw = re.sub(r"<(script|style)[^>]*>.*?</\1>", " ", raw, flags=re.S | re.I)
    raw = re.sub(r"<[^>]+>", " ", raw)
    raw = html.unescape(raw)
    return re.sub(r"\s+", " ", raw).strip()


def classify(quote):
    """Left-to-right scan; a full form is matched before its bare stem."""
    out = []
    i, n = 0, len(quote)
    while i < n:
        hit = None
        for pat, day in FULL:
            if quote.startswith(pat, i):
                hit = ("full", day, len(pat))
                break
        if hit is None:
            for pat, day in ABBR:
                if quote.startswith(pat, i) and i + len(pat) < n and quote[i + len(pat)] == ".":
                    if i == 0 or not ("฀" <= quote[i - 1] <= "๿"):
                        hit = ("abbreviation", day, len(pat) + 1)
                        break
        if hit is None:
            for pat, day in BARE:
                if quote.startswith(pat, i):
                    hit = ("bare", day, len(pat))
                    break
        if hit is None:
            i += 1
        else:
            out.append((hit[1], hit[0]))
            i += hit[2]
    return out


def fetch(url, cache):
    if url in cache:
        return cache[url]
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "th,en"})
    try:
        with urllib.request.urlopen(req, timeout=40) as resp:
            body = resp.read().decode("utf-8", "replace")
        cache[url] = (body, None)
    except Exception as exc:  # network, HTTP, encoding
        cache[url] = (None, "%s: %s" % (type(exc).__name__, exc))
    return cache[url]


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--recheck", action="store_true",
                    help="re-read the pages; no lab step exists, so this runs the same check")
    ap.parse_args()

    with open(DATASET, encoding="utf-8") as fh:
        data = json.load(fh)

    cache = {}
    failures = []
    totals = {"full": 0, "bare": 0, "abbreviation": 0}
    seen_urls = []

    for row in data["results"]:
        body, err = fetch(row["url"], cache)
        if err:
            failures.append("%s: fetch failed (%s)" % (row["id"], err))
            print("FAIL %-20s fetch: %s" % (row["id"], err))
            continue
        page = normalise(body)
        quote = normalise(row["quote"])
        if quote not in page:
            failures.append("%s: quote no longer on the page" % row["id"])
            print("FAIL %-20s quote missing" % row["id"])
            continue
        mentions = classify(quote)
        counts = {"full": 0, "bare": 0, "abbreviation": 0}
        for _day, form in mentions:
            counts[form] += 1
        if counts != row["counts"]:
            failures.append("%s: counts changed, stored %s recomputed %s"
                            % (row["id"], row["counts"], counts))
            print("FAIL %-20s counts %s != %s" % (row["id"], counts, row["counts"]))
            continue
        for k in totals:
            totals[k] += counts[k]
        if row["url"] not in seen_urls:
            seen_urls.append(row["url"])
        print("ok   %-20s full=%d bare=%d abbr=%d" % (row["id"], counts["full"],
                                                      counts["bare"], counts["abbreviation"]))

    summary = data["summary"]
    if totals["full"] != summary["full"] or totals["bare"] != summary["bare"] \
            or totals["abbreviation"] != summary["abbreviation"]:
        failures.append("summary totals changed: %s vs stored %s" % (totals, summary))
        print("FAIL summary totals %s != stored %s" % (totals, summary))
    else:
        print("ok   totals full=%d bare=%d abbr=%d" % (totals["full"], totals["bare"],
                                                       totals["abbreviation"]))
    if len(seen_urls) != summary["pages"]:
        failures.append("page count changed: %d vs stored %d" % (len(seen_urls), summary["pages"]))
        print("FAIL page count %d != stored %d" % (len(seen_urls), summary["pages"]))
    else:
        print("ok   pages=%d" % len(seen_urls))

    by = {}
    for row in data["results"]:
        b = by.setdefault(row["register"], {"listings": 0, "mentions": 0, "full": 0,
                                            "bare": 0, "abbreviation": 0})
        b["listings"] += 1
        for k in ("full", "bare", "abbreviation"):
            b[k] += row["counts"][k]
            b["mentions"] += row["counts"][k]
    if by != summary["byRegister"]:
        failures.append("byRegister changed: %s vs stored %s" % (by, summary["byRegister"]))
        print("FAIL byRegister %s != stored" % by)
    else:
        print("ok   byRegister %s" % ", ".join("%s=%d" % (k, v["mentions"]) for k, v in by.items()))

    with open(os.path.join(HERE, "dataset.csv"), encoding="utf-8", newline="") as fh:
        csv_rows = {r["id"]: r for r in csv.DictReader(fh)}
    for row in data["results"]:
        c = csv_rows.get(row["id"])
        if c is None or any(int(c[k]) != row["counts"][k] for k in ("full", "bare", "abbreviation")) \
                or c["quote"] != row["quote"] or c["url"] != row["url"]:
            failures.append("%s: dataset.csv disagrees with dataset.json" % row["id"])
            print("FAIL %-20s csv mismatch" % row["id"])
    if len(csv_rows) != len(data["results"]):
        failures.append("dataset.csv has %d rows, dataset.json %d" % (len(csv_rows), len(data["results"])))
    else:
        print("ok   dataset.csv matches dataset.json (%d rows)" % len(csv_rows))

    for tp in data.get("thirdParty", []):
        comp = tp.get("computed")
        if comp:
            wd = datetime.date.fromisoformat(comp["date"]).strftime("%A")
            if wd != comp["weekday"]:
                failures.append("%s: %s is a %s, stored %s" % (tp["id"], comp["date"], wd, comp["weekday"]))
                print("FAIL %-20s weekday %s" % (tp["id"], wd))
            else:
                print("ok   %-20s %s is a %s" % (tp["id"], comp["date"], wd))
        if not tp.get("quote"):
            continue
        body, err = fetch(tp["url"], cache)
        if err:
            failures.append("%s: fetch failed (%s)" % (tp["id"], err))
            print("FAIL %-20s fetch: %s" % (tp["id"], err))
            continue
        if normalise(tp["quote"]) not in normalise(body):
            failures.append("%s: supporting sentence no longer on the page" % tp["id"])
            print("FAIL %-20s quote missing" % tp["id"])
        else:
            print("ok   %-20s third-party quote present" % tp["id"])

    if failures:
        print("\n%d failure(s):" % len(failures))
        for item in failures:
            print("  - " + item)
        return 1
    print("\nall quotes hold, all counts match")
    return 0


if __name__ == "__main__":
    sys.exit(main())
