#!/usr/bin/env python3
"""Re-runs the check behind the post "Is Thai Hard to Learn?".

Public corpus: the consonant table on the Wikipedia "Thai script" page. For each
consonant letter it reads the initial IPA sound and the consonant class, then
counts how many letters spell the same sound and how those duplicates spread
across the three classes.

Quote check: every row in dataset.json that carries a URL and a quote must still
find that quote on the page, after whitespace, HTML entities and quote marks are
normalised.

Exit 0 when every row holds, 1 otherwise. --recheck re-reads the same pages;
there is no lab step, so nothing is skipped and the run is identical.
"""

import argparse
import gzip
import hashlib
import html
import json
import re
import sys
import urllib.request
import zlib
from collections import Counter, defaultdict

HERE = __file__.rsplit('/', 1)[0]

WIKI = 'https://en.wikipedia.org/wiki/Thai_script'
FSI = ('https://web.archive.org/web/20251028073806id_/'
       'https://www.state.gov/foreign-service-institute/foreign-language-training/')

HEADERS = {
    'User-Agent': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
                   'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36'),
    'Accept-Encoding': 'identity',
    'Accept': 'text/html,application/xhtml+xml',
}

_cache = {}


def fetch(url):
    if url in _cache:
        return _cache[url]
    req = urllib.request.Request(url, headers=HEADERS)
    with urllib.request.urlopen(req, timeout=60) as res:
        body = res.read()
        enc = (res.headers.get('Content-Encoding') or '').lower()
        if enc == 'gzip':
            body = gzip.decompress(body)
        elif enc == 'deflate':
            body = zlib.decompress(body)
    _cache[url] = body
    return body


def normalise(s):
    for a, b in [('‘', "'"), ('’', "'"), ('“', '"'), ('”', '"'),
                 (' ', ' '), ('\u2013', '-'), ('\u2014', '-'), ('\u2011', '-')]:
        s = s.replace(a, b)
    return re.sub(r'\s+', ' ', s).strip()


def page_text(raw):
    """Visible text of a page, whitespace and quote marks normalised."""
    s = raw.decode('utf-8', 'replace')
    s = re.sub(r'(?is)<(script|style|noscript)[^>]*>.*?</\1>', ' ', s)
    s = re.sub(r'(?s)<[^>]+>', ' ', s)
    return normalise(html.unescape(s))


def cell_text(fragment):
    s = re.sub(r'(?is)<sup.*?</sup>', '', fragment)
    s = re.sub(r'(?s)<[^>]+>', '', s)
    return re.sub(r'\s+', ' ', html.unescape(s)).strip()


def parse_inventory(raw):
    """Rows of the consonant table: symbol, initial IPA, class, obsolete flag."""
    page = raw.decode('utf-8', 'replace')
    rows = []
    for table in re.findall(r'(?is)<table.*?</table>', page):
        trs = re.findall(r'(?is)<tr[^>]*>(.*?)</tr>', table)
        if len(trs) < 40:
            continue
        head = [cell_text(c) for c in re.findall(r'(?is)<t[hd][^>]*>(.*?)</t[hd]>', trs[0])]
        if head[:5] != ['Symbol', 'Name', 'RTGS', 'IPA', 'Class']:
            continue
        for tr in trs[2:]:
            cells = re.findall(r'(?is)<t[hd][^>]*>(.*?)</t[hd]>', tr)
            if len(cells) < 9:
                continue
            cls = cell_text(cells[8])
            if not cls:
                continue
            ipa = cell_text(cells[6])
            rows.append({
                'symbol': re.sub(r'\[[a-z]\]$', '', cell_text(cells[0])).strip(),
                # ฑ is listed as "/tʰ/ or /d/"; the first reading is the ordinary
                # initial-consonant value, so the split is deterministic.
                'sound': ipa.split(' or ')[0].strip(),
                'class': cls,
                'obsolete': 'obsolete' in cell_text(cells[3]).lower(),
            })
        break
    return rows


def measure(rows):
    in_use = [r for r in rows if not r['obsolete']]
    groups = defaultdict(list)
    for r in in_use:
        groups[r['sound']].append(r)

    per_sound = []
    for sound, members in groups.items():
        classes = [m['class'] for m in members]
        per_sound.append({
            'sound': sound,
            'letters': ' '.join(m['symbol'] for m in members),
            'count': str(len(members)),
            'classes': ' '.join(classes),
            'mid': str(classes.count('mid')),
            'high': str(classes.count('high')),
            'low': str(classes.count('low')),
        })
    per_sound.sort(key=lambda r: (-int(r['count']), r['sound']))

    dup = [r for r in in_use if len(groups[r['sound']]) > 1]
    dup_by_class = Counter(r['class'] for r in dup)
    largest = max(groups.values(), key=len)
    totals = Counter(r['class'] for r in in_use)
    dup_sets = [g for g in groups.values() if len(g) > 1]
    flat_sets = [g for g in dup_sets if len({m['class'] for m in g}) == 1]
    # A letter only adds tone information when no other letter has its sound and its class.
    sound_class = {(r['sound'], r['class']) for r in in_use}

    summary = {
        'total_letters': str(len(rows)),
        'letters_in_use': str(len(in_use)),
        'letters_obsolete': str(len(rows) - len(in_use)),
        'sounds': str(len(groups)),
        'duplicate_letters': str(len(dup)),
        'single_spelling_sounds': str(sum(1 for g in groups.values() if len(g) == 1)),
        'homophone_sets': str(len(dup_sets)),
        'homophone_sets_single_class': str(len(flat_sets)),
        'homophone_letters_single_class': str(sum(len(g) for g in flat_sets)),
        'sound_class_pairs': str(len(sound_class)),
        'letters_beyond_sound_and_class': str(len(in_use) - len(sound_class)),
        'largest_set_sound': largest[0]['sound'],
        'largest_set_count': str(len(largest)),
        'largest_set_letters': ' '.join(m['symbol'] for m in largest),
        'duplicate_mid': str(dup_by_class['mid']),
        'duplicate_high': str(dup_by_class['high']),
        'duplicate_low': str(dup_by_class['low']),
        'letters_mid': str(totals['mid']),
        'letters_high': str(totals['high']),
        'letters_low': str(totals['low']),
    }
    return per_sound, summary


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--recheck', action='store_true',
                    help='re-read every cited page (this check has no lab step)')
    ap.parse_args()

    with open(f'{HERE}/dataset.json', encoding='utf-8') as fh:
        dataset = json.load(fh)

    failures = []
    raw = fetch(WIKI)
    print(f'wikipedia Thai_script sha256 {hashlib.sha256(raw).hexdigest()[:16]}')
    rows = parse_inventory(raw)
    print(f'consonant rows parsed: {len(rows)}')
    if len(rows) != 44:
        failures.append(f'parsed {len(rows)} consonant rows, expected 44')

    per_sound, summary = measure(rows)
    measured = {}
    for r in per_sound:
        for prop in ('letters', 'count', 'classes', 'mid', 'high', 'low'):
            measured[(r['sound'], prop)] = r[prop]
    for k, v in summary.items():
        measured[('inventory', k)] = v

    seen = set()
    for row in dataset['results']:
        if row['group'] != 'consonant_inventory':
            continue
        key = (row['option'], row['property'])
        got = measured.get(key)
        ok = got == row['value']
        if not ok:
            failures.append(
                f'{row["option"]} {row["property"]}: dataset "{row["value"]}" != measured "{got}"')
        print(f'  {"ok" if ok else "FAIL":4} {row["option"]:10} {row["property"]}: {got}')
        if row['option'].startswith('/'):
            seen.add(row['option'])

    missing = sorted(set(r['sound'] for r in per_sound) - seen)
    if missing:
        failures.append(f'sounds on the page but not in the dataset: {missing}')

    for row in dataset['results']:
        if row['group'] == 'consonant_inventory' or not row.get('quote'):
            continue
        try:
            text = page_text(fetch(row['url']))
        except Exception as exc:
            failures.append(f'{row["option"]}: fetch failed {exc!r}')
            print(f'  FAIL {row["option"]}: fetch failed {exc!r}')
            continue
        found = normalise(row['quote']) in text
        print(f'  {"ok" if found else "FAIL":4} {row["option"]}: quote on {row["url"][:70]}')
        if not found:
            failures.append(f'{row["option"]}: quote gone from {row["url"]}')

    print()
    if failures:
        print(f'{len(failures)} failure(s)')
        for f in failures:
            print(f'  - {f}')
        return 1
    print('all rows hold')
    return 0


if __name__ == '__main__':
    sys.exit(main())
