#!/usr/bin/env python3
"""Re-runs the count behind how-long-to-learn-thai-script.

Counts what Phayan's course itself contains, read from the repository this
file lives in:

1. apps/api/src/content/seed/levels.ts: per level, the rounds, the new letters,
   the new vowel forms and the words, totalled from L0 up to each milestone.
2. apps/web/src/lib/learn/srs/schedule.ts: the review ladder, replayed for a
   word answered correctly every time, to count the reviews it books before
   it retires.
3. apps/web/src (non-test files): which item kinds enter the review queue.
4. packages/shared/src/learn-access.ts: the free/paid boundary.
5. docs/specs/CURRICULUM_ARCHITECTURE.md: the pace the course is planned at.
6. The Wikipedia article on Thai script, for the size of the script.

  python3 check.py              measure and compare with dataset.json
  python3 check.py --recheck    the same (no step takes longer than a minute)
  python3 check.py --write      store the measurement in dataset.json and dataset.csv
  python3 check.py --repo PATH  measure a checkout somewhere else

Exit 0 when every count and quote still matches dataset.json, 1 otherwise.
Standard library only.
"""

import argparse
import csv
import hashlib
import html
import json
import os
import re
import sys
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, 'dataset.json')
CSV_PATH = os.path.join(HERE, 'dataset.csv')
DEFAULT_REPO = os.path.abspath(os.path.join(HERE, '..', '..', '..', '..', '..'))

FILES = {
    'levels': 'apps/api/src/content/seed/levels.ts',
    'srs': 'apps/web/src/lib/learn/srs/schedule.ts',
    'access': 'packages/shared/src/learn-access.ts',
    'plan': 'docs/specs/CURRICULUM_ARCHITECTURE.md',
}
PLAN_QUOTE = 'Per-level time is an **assumption**: one round per active day at ~10–12 min/day throughout.'

# (row id, label as the course titles the level, last level counted, last round or None)
MILESTONES = [
    ('L1R1', 'L1 First words, round 1', 1, 1),
    ('L2', 'L2 More vowels', 2, None),
    ('L3', 'L3 Finals & syllable types', 3, None),
    ('L4', 'L4 Mid class + tone marks (last free level)', 4, None),
    ('L7', 'L7 Low class II: rare letters', 7, None),
    ('L8', 'L8 Unwritten rules', 8, None),
    ('L9', 'L9 Connected reading', 9, None),
    ('L12', "L12 The letters you'll actually meet", 12, None),
    ('L30', 'L30 Menus & signs', 30, None),
    ('L31', 'L31 The everyday tail (whole course)', 31, None),
]
CSV_FIELDS = ['milestone', 'rounds_from_zero', 'letters', 'vowel_forms', 'words',
              'min_reviews_to_retire_words']

UA = 'Mozilla/5.0'
failures = []
checks = 0


def fail(msg):
    failures.append(msg)
    print('  FAIL  ' + msg)


def ok(msg):
    global checks
    checks += 1
    print('  ok    ' + msg)


# ---------------------------------------------------------------- parsing

def parse_strings(text, key):
    """Strings from `key: [ ... ]`, single- or double-quoted, escaped quotes."""
    match = re.search(r'\b' + key + r':\s*\[', text)
    if not match:
        return []
    i, depth, out, quote, buf = match.end(), 1, [], None, None
    while i < len(text):
        c = text[i]
        if quote:
            if c == '\\':
                buf.append(text[i + 1])
                i += 2
                continue
            if c == quote:
                out.append(''.join(buf))
                quote, buf = None, None
            else:
                buf.append(c)
            i += 1
            continue
        if c in '\'"':
            quote, buf = c, []
            i += 1
            continue
        if c == '[':
            depth += 1
        elif c == ']':
            depth -= 1
            if depth == 0:
                return out
        i += 1
    raise ValueError('unbalanced array after ' + key)


def parse_levels(src):
    levels = []
    starts = list(re.finditer(r"^ {4}id: '(L\d+)',$", src, re.M))
    if not starts:
        raise ValueError('no level blocks found')
    for k, match in enumerate(starts):
        end = starts[k + 1].start() if k + 1 < len(starts) else len(src)
        chunk = src[match.end():end]
        title = re.search(r"title:\s*(?:'((?:[^'\\]|\\.)*)'|\"((?:[^\"\\]|\\.)*)\")", chunk)
        title = (title.group(1) if title.group(1) is not None else title.group(2)) if title else None
        rounds = []
        rmatches = list(re.finditer(r"id: '(L\d+:R(\d+))',", chunk))
        for j, rm in enumerate(rmatches):
            rend = rmatches[j + 1].start() if j + 1 < len(rmatches) else len(chunk)
            rc = chunk[rm.end():rend]
            rounds.append({
                'round': int(rm.group(2)),
                'newLetters': parse_strings(rc, 'newLetters'),
                'newVowelForms': parse_strings(rc, 'newVowelForms'),
                'wordIds': parse_strings(rc, 'wordIds'),
            })
        levels.append({'id': match.group(1), 'number': int(match.group(1)[1:]),
                       'title': title, 'rounds': rounds})
    return levels


def read_srs(src):
    def num(name):
        m = re.search(r'\b' + name + r'\s*=\s*([\d_.]+)', src)
        return float(m.group(1).replace('_', '')) if m else None

    intervals = re.search(r'SRS_INTERVALS\s*=\s*\[([^\]]+)\]', src)
    return {
        'intervals': [int(x) for x in re.findall(r'\d+', intervals.group(1))] if intervals else [],
        'mastery': int(num('SRS_MASTERY_COUNT') or 0),
        'growth': num('SRS_MASTERED_GROWTH'),
        'retirement': num('SRS_RETIREMENT_INTERVAL_DAYS'),
        'wrong_resets_to_zero': bool(re.search(r'if \(!correct\)[\s\S]{0,400}?correctCount: 0', src)),
    }


def replay_ladder(srs):
    """applySrsAnswer for an always-correct word: the first answer meets it in a
    round, every later answer is a scheduled review, until it retires."""
    intervals, mastery = srs['intervals'], srs['mastery']
    anchor = intervals[-1]
    correct, mastered, interval, day = 1, False, intervals[0], 0.0
    offsets = []
    while True:
        day += interval
        offsets.append(round(day, 2))
        if mastered:
            nxt = anchor if interval < anchor else interval * srs['growth']
            if nxt > srs['retirement']:
                return offsets
            interval = nxt
        else:
            correct = min(correct + 1, mastery)
            mastered = correct >= mastery
            interval = intervals[min(correct, mastery) - 1]
        if len(offsets) > 50:
            raise ValueError('ladder never retires')


def srs_call_kinds(repo):
    kinds = set()
    root = os.path.join(repo, 'apps', 'web', 'src')
    for dirpath, dirnames, filenames in os.walk(root):
        dirnames[:] = [d for d in dirnames if d not in ('__tests__', 'node_modules')]
        for name in filenames:
            if not name.endswith(('.ts', '.tsx')) or re.search(r'\.(test|spec)\.', name):
                continue
            text = open(os.path.join(dirpath, name), encoding='utf-8').read()
            for m in re.finditer(r'\.recordSrsAnswer\(\s*([^,]+),', text):
                kinds.add(m.group(1).strip())
    return sorted(kinds)


def measure(repo):
    src = {k: open(os.path.join(repo, p), encoding='utf-8').read() for k, p in FILES.items()}
    levels = parse_levels(src['levels'])
    srs = read_srs(src['srs'])
    offsets = replay_ladder(srs)
    per_word = len(offsets)
    first_paid = int(re.search(r'FIRST_PAID_LEVEL\s*=\s*(\d+)', src['access']).group(1))

    def upto(last_level, last_round):
        rounds, letters, vowels, words = 0, set(), set(), set()
        for level in levels:
            if level['number'] > last_level:
                continue
            for r in level['rounds']:
                if level['number'] == last_level and last_round and r['round'] > last_round:
                    continue
                rounds += 1
                letters |= set(r['newLetters'])
                vowels |= set(r['newVowelForms'])
                words |= set(r['wordIds'])
        return {'rounds_from_zero': rounds, 'letters': len(letters), 'vowel_forms': len(vowels),
                'words': len(words), 'min_reviews_to_retire_words': len(words) * per_word}

    rows = []
    for row_id, label, last_level, last_round in MILESTONES:
        row = {'id': row_id, 'milestone': label}
        row.update(upto(last_level, last_round))
        rows.append(row)

    above = [l for l in levels if l['number'] > 12]
    counts = {
        'levels_authored': len(levels),
        'rounds_total': sum(len(l['rounds']) for l in levels),
        'first_paid_level': first_paid,
        'free_levels': len([l for l in levels if l['number'] < first_paid]),
        'level_titles_used_in_table': {l['id']: l['title'] for l in levels
                                       if l['number'] in {m[2] for m in MILESTONES}},
        'levels_above_12': len(above),
        'levels_above_12_with_new_letter_or_vowel_form': len([
            l for l in above if any(r['newLetters'] or r['newVowelForms'] for r in l['rounds'])]),
        'srs_intervals': srs['intervals'],
        'srs_mastery_count': srs['mastery'],
        'srs_wrong_answer_resets_count': srs['wrong_resets_to_zero'],
        'srs_review_day_offsets_always_correct': offsets,
        'srs_reviews_per_word_to_retire': per_word,
        'srs_item_kinds_recorded_in_app': srs_call_kinds(repo),
        'plan_pace_quote_present': PLAN_QUOTE in src['plan'],
    }
    digests = {p: hashlib.sha256(src[k].encode('utf-8')).hexdigest() for k, p in FILES.items()}
    return rows, counts, digests


# ------------------------------------------------------------- web pages

def normalise(text):
    text = re.sub(r'(?is)<(script|style)\b.*?</\1>', ' ', text)
    text = re.sub(r'(?s)<[^>]+>', ' ', text)
    text = html.unescape(text)
    for a, b in (('’', "'"), ('‘', "'"), ('“', '"'), ('”', '"'), ('–', '-'), ('\xa0', ' ')):
        text = text.replace(a, b)
    return re.sub(r'\s+', ' ', text).strip().lower()


def check_page(row):
    try:
        req = urllib.request.Request(row['url'], headers={'User-Agent': UA})
        with urllib.request.urlopen(req, timeout=45) as resp:
            page = normalise(resp.read().decode(resp.headers.get_content_charset() or 'utf-8', 'replace'))
    except (urllib.error.URLError, OSError) as exc:
        fail('%-22s %s unreachable: %s' % (row['id'], row['url'], exc))
        return
    if normalise(row['quote']) in page:
        ok('%-22s quote present %s' % (row['id'], row['url']))
    else:
        fail('%-22s quote gone from %s: %r' % (row['id'], row['url'], row['quote'][:70]))


# ------------------------------------------------------------------ main

def write_csv(rows):
    with open(CSV_PATH, 'w', newline='', encoding='utf-8') as fh:
        writer = csv.DictWriter(fh, fieldnames=CSV_FIELDS, extrasaction='ignore')
        writer.writeheader()
        writer.writerows(rows)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--recheck', action='store_true', help='re-read the pages (every step is fast)')
    parser.add_argument('--write', action='store_true', help='store the measurement')
    parser.add_argument('--repo', default=DEFAULT_REPO)
    args = parser.parse_args()

    if not os.path.exists(os.path.join(args.repo, FILES['levels'])):
        print('FAIL  no Phayan checkout at %s (pass --repo)' % args.repo)
        return 1

    with open(DATASET, encoding='utf-8') as fh:
        dataset = json.load(fh)
    rows, counts, digests = measure(args.repo)

    if args.write:
        dataset['results']['milestones'] = rows
        dataset['results']['counts'] = counts
        for entry in dataset['inputs']:
            if entry.get('path') in digests:
                entry['sha256'] = digests[entry['path']]
        with open(DATASET, 'w', encoding='utf-8') as fh:
            json.dump(dataset, fh, ensure_ascii=False, indent=2)
            fh.write('\n')
        write_csv(rows)
        print('wrote dataset.json and dataset.csv')

    print('inputs')
    for entry in dataset['inputs']:
        if entry.get('path') in digests:
            same = digests[entry['path']] == entry['sha256']
            print('  %s  %s %s' % ('same' if same else 'note', entry['path'],
                                   'unchanged since %s' % entry['commit'] if same
                                   else 'changed since %s; counts decide' % entry['commit']))

    print('milestones')
    stored = {r['id']: r for r in dataset['results']['milestones']}
    for row in rows:
        want = stored.get(row['id'])
        if want != row:
            fail('%-5s measured %s, dataset says %s' % (row['id'], row, want))
        else:
            ok('%-5s rounds %-3d letters %-2d vowel forms %-2d words %-3d reviews %d' % (
                row['id'], row['rounds_from_zero'], row['letters'], row['vowel_forms'],
                row['words'], row['min_reviews_to_retire_words']))

    with open(CSV_PATH, newline='', encoding='utf-8') as fh:
        csv_rows = list(csv.DictReader(fh))
    if csv_rows == [{k: str(r[k]) for k in CSV_FIELDS} for r in rows]:
        ok('dataset.csv matches the measurement')
    else:
        fail('dataset.csv differs from the measurement')

    print('counts')
    for key, value in counts.items():
        want = dataset['results']['counts'].get(key)
        if want != value:
            fail('%-46s measured %r, dataset says %r' % (key, value, want))
        else:
            ok('%-46s %s' % (key, json.dumps(value, ensure_ascii=False)))

    print('pages')
    for row in dataset['results']['pages']:
        check_page(row)

    print()
    if failures:
        print('%d checks, %d failed' % (checks + len(failures), len(failures)))
        return 1
    print('%d checks, 0 failed' % checks)
    return 0


if __name__ == '__main__':
    sys.exit(main())
