#!/usr/bin/env python3
"""Re-runs the Thai National Corpus measurement behind the thai-r-and-l post.

Standard library only. Downloads the two PyThaiNLP corpus files (each pinned by sha256),
recomputes every value in dataset.json, compares dataset.csv and pairs.csv against the result,
and checks each quoted sentence on the page it cites.

  python3 check.py            full run
  python3 check.py --recheck  same work: no step here takes a minute, so nothing is skipped

Exit 0 when every value and quote holds, 1 otherwise.
"""

import csv
import hashlib
import html
import json
import os
import re
import sys
import unicodedata
import urllib.request

RA = 'ร'  # ro ruea, U+0E23
LA = 'ล'  # lo ling, U+0E25
UA = {'User-Agent': 'Mozilla/5.0'}

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, 'dataset.json')
DATASET_CSV = os.path.join(HERE, 'dataset.csv')
PAIRS_CSV = os.path.join(HERE, 'pairs.csv')

CORPUS = {
    'tnc': 'https://raw.githubusercontent.com/PyThaiNLP/pythainlp/v5.4.0/pythainlp/corpus/tnc_freq.txt',
    'dict': 'https://raw.githubusercontent.com/PyThaiNLP/pythainlp/v5.4.0/pythainlp/corpus/words_th.txt',
}
PAGE_LIMIT = 4000000


def get(url):
    req = urllib.request.Request(url, headers=UA)
    with urllib.request.urlopen(req, timeout=60) as r:
        return r.read(PAGE_LIMIT).decode('utf-8', 'replace')


def sha256(text):
    return hashlib.sha256(text.encode('utf-8')).hexdigest()


def norm(text):
    text = unicodedata.normalize('NFC', text)
    text = (text.replace('‘', "'").replace('’', "'")
            .replace('“', '"').replace('”', '"'))
    text = text.replace('–', '-').replace('—', '-')
    return re.sub(r'\s+', ' ', text).strip()


def page_text(raw):
    body = re.sub(r'<script.*?</script>|<style.*?</style>', ' ', raw, flags=re.S | re.I)
    body = re.sub(r'<[^>]+>', ' ', body)
    return norm(html.unescape(body))


def measure(tnc_text, dict_text):
    rows = []
    for line in tnc_text.split('\n'):
        if not line.strip():
            continue
        word, count = line.split('\t')
        rows.append((word, int(count)))
    counts = dict(rows)
    order = {w: i for i, (w, _) in enumerate(rows)}
    ranked = sorted(counts, key=lambda w: (-counts[w], order[w]))
    rank = {w: i for i, w in enumerate(ranked, 1)}
    total = sum(counts.values())
    words = set(w.strip() for w in dict_text.split('\n') if w.strip())

    pairs = set()
    for w in words:
        if RA not in w:
            continue
        for i, ch in enumerate(w):
            if ch == RA:
                swapped = w[:i] + LA + w[i + 1:]
                if swapped in words and w in counts and swapped in counts:
                    pairs.add((w, swapped))
    both = [(a, b) for a, b in pairs if rank[a] <= 3000 and rank[b] <= 3000]

    doubled = [(w, c) for w, c in rows if RA + RA in w]
    doubled_tokens = sum(c for _, c in doubled)
    stems = [(w, c) for w, c in doubled
             if 'ก' + RA + RA + 'ม' in w or 'ธ' + RA + RA + 'ม' in w]

    values = {
        'distinct_word_entries': len(rows),
        'corpus_tokens': total,
        'share_tokens_with_ro_ruea': round(100.0 * sum(c for w, c in rows if RA in w) / total, 1),
        'share_tokens_with_lo_ling': round(100.0 * sum(c for w, c in rows if LA in w) / total, 1),
        'top1000_with_ro_ruea': sum(1 for w in ranked[:1000] if RA in w),
        'dictionary_words': len(words),
        'dictionary_collision_pairs': len(pairs),
        'collision_pairs_both_top3000': len(both),
        'collision_pairs_one_side_below_3000': len(pairs) - len(both),
        'doubled_ro_ruea_words': len(doubled),
        'doubled_ro_ruea_tokens': doubled_tokens,
        'doubled_on_karrm_or_therm_words': len(stems),
        'doubled_on_karrm_or_therm_tokens': sum(c for _, c in stems),
        'doubled_on_karrm_or_therm_share': round(100.0 * sum(c for _, c in stems) / doubled_tokens, 1),
    }
    for w in TABLE_WORDS:
        values['rank:' + w] = rank.get(w)
    return values, pairs, rank, counts


TABLE_WORDS = [
    'รับ', 'ลับ', 'แรก', 'แลก', 'ร้าน', 'ล้าน', 'กรม', 'กลม',
    'ครอง', 'คลอง', 'พลิก', 'พริก', 'แปร', 'แปล', 'รา', 'ลา',
]


def rows_of(path):
    with open(path, encoding='utf-8', newline='') as f:
        return list(csv.DictReader(f))


def main(argv):
    with open(DATASET, encoding='utf-8') as f:
        data = json.load(f)
    failures = []

    tnc_text = get(CORPUS['tnc'])
    dict_text = get(CORPUS['dict'])
    for name, url, text in (('tnc_freq.txt', CORPUS['tnc'], tnc_text),
                            ('words_th.txt', CORPUS['dict'], dict_text)):
        want = next(i['sha256'] for i in data['inputs'] if i['url'] == url)
        got = sha256(text)
        if got != want:
            failures.append('%s sha256 %s, dataset says %s' % (name, got[:12], want[:12]))
        else:
            print('ok   %s sha256' % name)

    values, pairs, rank, counts = measure(tnc_text, dict_text)
    wanted = {r['id']: r['value'] for r in data['results'] if r['kind'] == 'measure'}
    for id_ in sorted(values):
        got = values[id_]
        if id_ not in wanted:
            failures.append('%s measured %s but is not in dataset.json' % (id_, got))
        elif got != wanted[id_]:
            failures.append('%s measured %s, dataset says %s' % (id_, got, wanted[id_]))
        else:
            print('ok   %-40s %s' % (id_, got))

    # dataset.csv must carry the same values as dataset.json, one row each.
    wanted_csv = {k: str(v) for k, v in wanted.items()}
    csv_rows = {r['id']: r['value'] for r in rows_of(DATASET_CSV) if r['kind'] == 'measure'}
    if csv_rows != wanted_csv:
        missing = sorted(set(wanted_csv) - set(csv_rows))
        extra = sorted(set(csv_rows) - set(wanted_csv))
        diff = sorted(k for k in set(wanted_csv) & set(csv_rows) if csv_rows[k] != wanted_csv[k])
        failures.append('dataset.csv differs from dataset.json: missing %s, extra %s, different %s'
                        % (missing, extra, diff))
    else:
        print('ok   dataset.csv %d measured values, same as dataset.json' % len(csv_rows))

    # pairs.csv must list exactly the measured pairs, with matching ranks, counts and flag.
    with open(PAIRS_CSV, encoding='utf-8', newline='') as f:
        pair_rows = list(csv.DictReader(f))
    listed = set()
    bad = 0
    for r in pair_rows:
        a, b = r['word_with_ro_ruea'], r['word_with_lo_ling']
        listed.add((a, b))
        if (r['rank_ro'], r['count_ro']) != (str(rank[a]), str(counts[a])):
            bad += 1
        if (r['rank_lo'], r['count_lo']) != (str(rank[b]), str(counts[b])):
            bad += 1
        flag = 'yes' if rank[a] <= 3000 and rank[b] <= 3000 else 'no'
        if r['both_in_top3000'] != flag:
            bad += 1
    if listed != pairs:
        failures.append('pairs.csv lists %d pairs, measured %d (%d differ)'
                        % (len(listed), len(pairs), len(listed ^ pairs)))
    elif bad:
        failures.append('pairs.csv has %d rank, count or flag cells that do not match' % bad)
    else:
        print('ok   pairs.csv %d pairs, both ranks and counts match' % len(listed))

    cache = {}
    checked = 0
    for row in data['results']:
        if row.get('kind') != 'quote':
            continue
        url = row['url']
        if url not in cache:
            cache[url] = page_text(get(url))
        checked += 1
        if norm(row['quote']) in cache[url]:
            print('ok   quote   %s' % row['id'])
        else:
            failures.append('quote gone from %s: %r' % (url, row['quote'][:80]))

    if failures:
        print('\nFAIL')
        for f in failures:
            print('  ' + f)
        return 1
    print('\nall %d values and %d quotes hold' % (len(values), checked))
    return 0


if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))