#!/usr/bin/env python3
"""Re-runs the evidence check for the post duolingo-thai-course.

Method: primary-docs. Re-reads Duolingo's own course list, the Thai page of
every app the post names as carrying Thai, the language lists of Babbel, Busuu
and Rosetta Stone, Drops' Scripts page, Anki's page and Phayan's plans
endpoint, and reports per row of dataset.csv whether its quotes still appear.

Duolingo's learner counts drift daily, so they are printed, not asserted.

Exit 0 when every stored fact still holds, 1 otherwise. --recheck re-reads the
pages; this check has no lab step, so nothing is skipped. Standard library only.
"""

from __future__ import annotations

import argparse
import csv
import html
import json
import os
import re
import sys
import urllib.error
import urllib.parse
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
CHROME_UA = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/131.0 Safari/537.36"
)
# rosettastone.com sends a Chrome user-agent to its German locale; the plain one gets English.
PLAIN_UA = "Mozilla/5.0"
DUOLINGO_URL = "https://www.duolingo.com/api/1/courses/list"
PLANS_URL = "https://api.phayan.com/subscriptions/plans"
LEARN_ACCESS = os.path.join(HERE, "..", "..", "..", "..", "..", "packages", "shared", "src", "learn-access.ts")


def fetch(url: str) -> tuple[int, str]:
    ua = PLAIN_UA if "rosettastone.com" in url else CHROME_UA
    # No Accept-Language: babbel.com answers a request carrying it with a 307 to itself.
    headers = {"User-Agent": ua, "Accept": "text/html,*/*"}
    for _ in range(6):
        req = urllib.request.Request(url, headers=headers)
        try:
            with urllib.request.urlopen(req, timeout=60) as r:
                return r.getcode(), r.read().decode("utf-8", "replace")
        except urllib.error.HTTPError as e:
            location = e.headers.get("Location") if e.headers else None
            if e.code in (301, 302, 303, 307, 308) and location:
                url = urllib.parse.urljoin(url, location)
                continue
            return e.code, e.read().decode("utf-8", "replace")
        except Exception as e:  # a network failure is a failed check, never a pass
            return 0, f"{type(e).__name__}: {e}"
    return 0, "redirect loop"


def visible(markup: str) -> str:
    markup = re.sub(r"(?is)<(script|style|noscript)[^>]*>.*?</\1>", " ", markup)
    return re.sub(r"(?s)<[^>]+>", " ", markup)


def norm(text: str) -> str:
    text = html.unescape(text)
    for a, b in (("‘", "'"), ("’", "'"), ("“", '"'), ("”", '"'),
                 ("—", "-"), ("–", "-"), (" ", " ")):
        text = text.replace(a, b)
    return re.sub(r"\s+", " ", text).strip().casefold()


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--recheck", action="store_true", help="re-read the pages (no lab step to skip)")
    ap.parse_args()

    data = json.load(open(os.path.join(HERE, "dataset.json"), encoding="utf-8"))
    rows = list(csv.DictReader(open(os.path.join(HERE, "dataset.csv"), encoding="utf-8")))
    duo = data["results"]["duolingo"]
    fails: list[str] = []
    oks = 0

    def check(label: str, ok: bool, detail: str = "") -> None:
        nonlocal oks
        if ok:
            oks += 1
            print(f"OK   {label}")
        else:
            fails.append(label)
            print(f"FAIL {label}{(': ' + detail) if detail else ''}")

    pages: dict[str, tuple[int, str]] = {}

    def page(url: str) -> tuple[int, str]:
        if url not in pages:
            pages[url] = fetch(url)
        return pages[url]

    # Duolingo's own list
    status, body = page(DUOLINGO_URL)
    if status != 200:
        check("Duolingo course list reachable", False, f"HTTP {status}")
    else:
        courses = json.loads(body)
        en = [c for c in courses if c["from_language"] == "en"]
        from_thai = [c for c in courses if c["from_language"] == "th"]
        early = [c for c in courses if c.get("phase") in (1, 2)]
        check("Duolingo: 313 courses", len(courses) == duo["courses"], f"{len(courses)}")
        check("Duolingo: 44 target languages",
              len({c["learning_language"] for c in courses}) == duo["target_languages"])
        check("Duolingo: speakers of 45 languages",
              len({c["from_language"] for c in courses}) == duo["source_languages"])
        check("Duolingo: no course teaches Thai", not any(c["learning_language"] == "th" for c in courses))
        check("Duolingo: nine courses from Thai", len(from_thai) == duo["courses_from_thai"], f"{len(from_thai)}")
        check("Duolingo: 40 courses from English", len(en) == duo["courses_from_english"], f"{len(en)}")
        check("Duolingo: 39 of them teach another language",
              sum(1 for c in en if c["learning_language"] != "en") == duo["english_courses_teaching_another_language"])
        names = {c["learning_language_name"] for c in en}
        check("Duolingo: English list has Klingon, High Valyrian, Hawaiian, Navajo, Yiddish, Zulu",
              {"Klingon", "High Valyrian", "Hawaiian", "Navajo", "Yiddish", "Zulu"} <= names)
        check("Duolingo: English list has no Lao, Khmer or Burmese", not {"Lao", "Khmer", "Burmese"} & names)
        check("Duolingo: no phase 1 or 2 course targets Thai", not any(c["learning_language"] == "th" for c in early))
        check("Duolingo: the courses from Thai are the nine in the dataset",
              {c["learning_language_name"] for c in from_thai} == set(duo["courses_from_thai_learners"]))
        largest = max(from_thai, key=lambda c: c["num_learners"]) if from_thai else None
        check("Duolingo: largest course from Thai is English", bool(largest) and largest["learning_language_name"] == "English")
        total = sum(c["num_learners"] for c in from_thai)
        print(f"note learners on courses from Thai: {total} (dataset {duo['learner_records_from_thai']}); "
              f"phase 1 or 2 courses: {len(early)} (dataset {duo['early_phase_courses']})")

    # one row per app in dataset.csv
    for row in rows:
        app, url = row["app"], row["url"]
        if app == "Duolingo":
            continue
        status, markup = page(url)
        if status != 200:
            check(f"{app}: page reachable", False, f"HTTP {status}")
            continue
        text = norm(visible(markup))
        if app == "Mondly":
            check("Mondly: Thai in the language list", re.search(r"\bthai [\d,]+ learners learn thai\b", text) is not None)
            continue
        if app == "Busuu":
            langs = set(re.findall(r"register\?learning=([a-z]{2})", markup))
            check("Busuu: 14 language courses", len(langs) == 14, f"{sorted(langs)}")
            check("Busuu: no Thai", "th" not in langs and "thai" not in text)
            continue
        for key in ("quote", "second_quote"):
            if row[key]:
                check(f"{app}: {key} on page", norm(row[key]) in text, row[key][:60])
        if app == "Babbel":
            items = {n for _, n in re.findall(r'"position":\s*(\d+)\s*,\s*"name"\s*:\s*"Learn ([^"]+)"', markup)}
            check("Babbel: 14 language courses, English among them", len(items) == 14 and "English" in items, f"{sorted(items)}")
            check("Babbel: no Thai", "thai" not in text and "Thai" not in items)
        if app == "Rosetta Stone":
            check("Rosetta Stone: no Thai", "thai" not in text)
        if row["thai_course_listed"] == "yes":
            check(f"{app}: page names Thai", "thai" in text)

    # Drops' Scripts app teaches no Thai alphabet
    status, markup = page("https://languagedrops.com/scripts")
    text = norm(visible(markup)) if status == 200 else ""
    check("Drops Scripts: 8 alphabets", "choose from one of 8 alphabets" in text)
    check("Drops Scripts: no Thai", status == 200 and "thai" not in text)

    # Anki
    status, markup = page("https://apps.ankiweb.net/")
    text = norm(visible(markup)) if status == 200 else ""
    check("Anki: free computer version", "the free computer version is available for all major platforms" in text)
    check("Anki: open source", "open source" in text)
    check("Anki: flashcards", "powerful, intelligent flashcards" in text)

    # Phayan's own price and free levels
    status, body = page(PLANS_URL)
    if status == 200:
        paid = next((p for p in json.loads(body)["plans"] if p["tier"] == "plus"), None)
        pricing = paid["pricing"] if paid else {}
        check("Phayan: $14.99 a month", pricing.get("monthly") == 14.99 and pricing.get("currency") == "USD")
        check("Phayan: $79.99 a year", pricing.get("annual") == 79.99)
    else:
        check("Phayan plans endpoint reachable", False, f"HTTP {status}")
    if os.path.exists(LEARN_ACCESS):
        src = open(LEARN_ACCESS, encoding="utf-8").read()
        check("Phayan: levels 0 to 4 free", "export const FIRST_PAID_LEVEL = 5;" in src)
    else:
        print("note Phayan free levels: learn-access.ts not found next to this script, not checked")

    print(f"\n{oks} checks passed, {len(fails)} failed")
    for f in fails:
        print(f"  failed: {f}")
    return 0 if not fails else 1


if __name__ == "__main__":
    sys.exit(main())
