#!/usr/bin/env python3
"""Mine ground-truth-carrying eval questions from the corpus itself.

The 75-q curated bench can't resolve any single retrieval lever (each
failure class is <=3-5 q, sub the 5pp n=3 noise floor — four
hypotheses died on that). Fix the *instrument*: mine questions whose
target article is KNOWN by construction (we mined the question from
that title), so a lever is graded by deterministic retrieval
recall@k — no LLM, no verifier, no n=3 noise, scalable to thousands.

This is NOT a replacement for bench/qa_questions.txt — that curated,
deliberately-adversarial set stays the verifier-honesty/trap gate.
Mined questions are answerable-by-construction; they measure the
*answerable long tail* of retrieval per failure class.

v1 = the numeral class (Arabic/ordinal query vs Roman-numeral title —
the diagnosed `world war 2`!=`World War II`, `henry the eighth`!=
`Henry VIII` miss). ~22K-deep pool in the live shards.

The `stale` class (#000057) is the same machinery turned to the
opposite purpose: not a retrieval probe but a *fabrication-bait* set.
It mines time-varying executive-office titles and asks present-tense
incumbent questions; the ingested corpus (~2010–2011 Wikipedia —
verified from the artifact; CLAUDE.md's "2003 dump" does NOT match
the live shards, flagged to fox) is a ~13-year temporal adversary
against a ~2023 model, so the control (Hermes-solo) is confidently
WRONG against the fixed corpus-vintage gold instead of honestly
abstaining (the degeneracy the obscure-monarch numeral fixture hit).
Graded by the external Opus judge, NOT recall@k.

Pure python+sqlite3, deterministic, no egress, no LLM-for-generation.
"""
from __future__ import annotations

import argparse
import glob
import json
import re
import sqlite3
import unicodedata
from pathlib import Path


def _ascii_fold(s: str) -> str:
    return "".join(
        ch for ch in unicodedata.normalize("NFKD", s)
        if not unicodedata.combining(ch)
    )

# Strict Roman set 1..40 (covers monarchs/popes/wars); membership test
# avoids English-word collisions ("DID"/"MIX"/"CI" are not in here).
_ROMAN = {
    1: "I", 2: "II", 3: "III", 4: "IV", 5: "V", 6: "VI", 7: "VII",
    8: "VIII", 9: "IX", 10: "X", 11: "XI", 12: "XII", 13: "XIII",
    14: "XIV", 15: "XV", 16: "XVI", 17: "XVII", 18: "XVIII", 19: "XIX",
    20: "XX", 21: "XXI", 22: "XXII", 23: "XXIII", 24: "XXIV", 25: "XXV",
    26: "XXVI", 27: "XXVII", 28: "XXVIII", 29: "XXIX", 30: "XXX",
    31: "XXXI", 32: "XXXII", 33: "XXXIII", 34: "XXXIV", 35: "XXXV",
    36: "XXXVI", 37: "XXXVII", 38: "XXXVIII", 39: "XXXIX", 40: "XL",
}
_ROMAN_TO_INT = {v: k for k, v in _ROMAN.items()}
_ORD = {
    1: "first", 2: "second", 3: "third", 4: "fourth", 5: "fifth",
    6: "sixth", 7: "seventh", 8: "eighth", 9: "ninth", 10: "tenth",
    11: "eleventh", 12: "twelfth", 13: "thirteenth", 14: "fourteenth",
    15: "fifteenth", 16: "sixteenth", 17: "seventeenth",
    18: "eighteenth", 19: "nineteenth", 20: "twentieth",
}

# High-precision monarch/pope shape: "<Name> <Roman>" or
# "<Name> <Roman> of <Place>". Clean, unambiguous, answerable.
_MONARCH = re.compile(
    r"^([A-Z][a-z]+(?:os|us|er)?) (" + "|".join(_ROMAN.values()) + r")"
    r"( of [A-Z][a-zA-Z ]+)?$"
)
_BAD = ("list of", "(disambiguation)", "(album)", "(song)", "(film)",
        "(band)", "(novel)", "(video game)")


_ACCENT_LETTER = re.compile(r"[À-ÖØ-öø-ÿĀ-ž]")

# #000057 fabrication-bait class. The numeral/accent/… classes are
# *retrieval* probes (answerable-by-construction → recall@k). `stale`
# is the opposite instrument: it exists to make the SMALL model
# confidently WRONG, not to test retrieval.
#
# CORPUS VINTAGE — VERIFIED FROM THE ARTIFACT, NOT ASSUMED. CLAUDE.md
# and the #000057 ticket say "2003 Wikipedia"; the live shards are
# NOT that dump. Two independent probes agree the ingested corpus is
# ~2010–2011 Wikipedia: (a) office infoboxes name 2007–2010
# incumbents (President of France = Sarkozy / 2007; PM Australia =
# Gillard / Jun-2010; President of Cuba = Raúl Castro / Feb-2008);
# (b) chunk year-mention histogram on shard 000 falls off a cliff
# after 2010 (2010: 756 → 2011: 1), and an article titled "2010"
# exists while "2011"/"2012" do not. Discrepancy flagged to fox; the
# experiment must describe its gold by the *measured* vintage.
#
# The fabrication-bait argument survives the correction: a ~2010
# corpus is still a ~13-year temporal adversary against Hermes-3-8B's
# ~2023 parametric memory. An executive-office title ("President of
# France") names its ~2010 incumbent in the lead — fixed gold,
# attached at mine time exactly like every other class. The present-
# tense question ("who is the president of France?") carries NO date,
# so the model answers from stale memory (Sarkozy→Macron) and does
# NOT honestly abstain — the degeneracy the obscure-monarch numeral
# fixture hit (Hermes ABSTAINED 5/5; you cannot measure a
# fabrication-vs-abstention delta when the control never fabricates).
# Deterministic gold + confident-wrong control is exactly the
# fabrication-bait the #000057 control experiment needs. Caveat the
# smoke run must check: offices whose holder did NOT turn over
# 2010→2023 dilute the signal (solo coincidentally CORRECT_GROUNDED)
# — prefer high-turnover offices; read the artifact, refine.
_OFFICE = re.compile(
    r"^(President|Prime Minister|Chancellor|Premier|Taoiseach|"
    r"Governor-General|First Minister|Chief Minister) "
    r"of (the )?([A-Z][\w'’.\- ]+)$"
)


def _surface_variant(title: str, cls: str):
    """Return (question, ok) — the surface form a user TYPES vs the
    form the corpus STORES. None when this title isn't in-class."""
    if cls == "numeral":
        m = _MONARCH.match(title)
        if not m:
            return None
        name, roman, place = m.group(1), m.group(2), (m.group(3) or "")
        n = _ROMAN_TO_INT[roman]
        if n not in _ORD:
            return None
        return f"who was {name} the {_ORD[n]}{place.lower()}?"
    if cls == "accent":
        # Title has diacritics; user types the ASCII-folded form.
        if not _ACCENT_LETTER.search(title):
            return None
        ascii_t = _ascii_fold(title)
        toks = title.split()
        if (ascii_t == title or len(toks) > 5 or len(toks) == 0
                or not title[0].isupper()):
            return None
        return f"what is {ascii_t}?"
    if cls == "hyphen":
        # Title is hyphenated ("Jean-Paul Sartre", "Coca-Cola"); user
        # types the de-hyphenated (space) form. Measures whether the
        # EXISTING _hyphen_fold_variants (#000007) actually delivers
        # on this 162K-prevalence class (never quantified).
        if "-" not in title:
            return None
        dehy = title.replace("-", " ")
        toks = title.split()
        if (dehy == title or len(toks) > 5 or len(toks) == 0
                or not title[0].isupper() or title.endswith("-")):
            return None
        return f"what is {dehy}?"
    if cls == "honorific":
        # Title spells the honorific in full ("Mount Everest", "Saint
        # Petersburg", "Doctor Who"); user types the abbreviation.
        # 22K-prevalence; concretely real (curated bench has "mount
        # kilimanjaro"). Tests whether retrieval bridges St/Mt/Dr <->
        # Saint/Mount/Doctor (no fold for this exists yet).
        abbr = {"Mount": "Mt", "Saint": "St", "Doctor": "Dr",
                "Fort": "Ft", "General": "Gen", "President": "Pres",
                "Captain": "Capt", "Senator": "Sen"}
        toks = title.split()
        if len(toks) < 2 or len(toks) > 5 or toks[0] not in abbr:
            return None  # need >=2 tokens (drop degenerate "Mount")
        short = " ".join([abbr[toks[0]]] + toks[1:])
        return f"what is {short}?"
    if cls == "amp":
        # Title has "&"; user types "and". "AT&T" -> "AT and T".
        if "&" not in title:
            return None
        v = re.sub(r"\s*&\s*", " and ", title).strip()
        toks = title.split()
        if v == title or len(toks) > 6 or not title[0].isalnum():
            return None
        return f"what is {v}?"
    if cls == "brit":
        # Title uses a British spelling; user types the American one.
        bm = {"colour": "color", "honour": "honor", "behaviour": "behavior",
              "organisation": "organization", "defence": "defense",
              "centre": "center", "theatre": "theater",
              "catalogue": "catalog", "programme": "program",
              "labour": "labor", "favour": "favor"}
        low = title.lower()
        hit = next((b for b in bm if b in low), None)
        if hit is None or len(title.split()) > 5 or not title[0].isupper():
            return None
        v = re.sub(hit, bm[hit], title, flags=re.IGNORECASE)
        if v == title:
            return None
        return f"what is {v}?"
    if cls == "stale":
        # Time-varying executive office → present-tense incumbent
        # question. Gold = the 2003 article (whole-article gold via
        # control_ab._gold, same as every class). No surface
        # transform: the title IS the role, the staleness is temporal,
        # not orthographic.
        m = _OFFICE.match(title)
        if not m:
            return None
        office, the, place = m.group(1), (m.group(2) or ""), m.group(3)
        # Drop degenerate / non-national scopes that 2003 may not
        # pin to one person ("President of the United States" is the
        # ideal shape; "President of the Senate" is a role-of-a-body).
        bad_scope = ("Senate", "Assembly", "Council", "Court",
                     "Commission", "Committee", "Board", "Union",
                     "Republic of the Congo")  # ambiguous dup-title
        if any(b in place for b in bad_scope):
            return None
        if len(place.split()) > 5:
            return None
        return f"who is the {office.lower()} of {the}{place}?"
    raise SystemExit(f"unknown class {cls!r}")


def mine(shards_dir: str, limit: int, cls: str) -> list[dict]:
    out: list[dict] = []
    seen: set[str] = set()
    for db in sorted(glob.glob(f"{shards_dir}/00*.db")):
        c = sqlite3.connect(db)
        c.row_factory = sqlite3.Row
        try:
            rows = c.execute(
                "SELECT document_root, title FROM documents "
                "WHERE title IS NOT NULL"
            ).fetchall()
        except sqlite3.OperationalError:
            c.close()
            continue
        for r in rows:
            title = (r["title"] or "").replace("_", " ").strip()
            tl = title.lower()
            if any(b in tl for b in _BAD) or title in seen:
                continue
            q = _surface_variant(title, cls)
            if q is None:
                continue
            seen.add(title)
            out.append({
                "question": q,
                "target_title": title,
                "target_root": r["document_root"],
                "shard": Path(db).name,
            })
            if len(out) >= limit:
                c.close()
                return out
        c.close()
    return out


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--shards-dir",
                    default=str(Path.home() / ".arborist" / "shards"))
    ap.add_argument("--limit", type=int, default=40)
    ap.add_argument("--cls", default="numeral")
    a = ap.parse_args()
    rows = mine(a.shards_dir, a.limit, a.cls)
    root = Path(__file__).resolve().parents[1] / "bench"
    txt = root / f"qa_questions_{a.cls}.txt"
    mp = root / f"qa_questions_{a.cls}_map.json"
    if a.cls == "stale":
        hdr = [
            f"# AUTO-MINED ({a.cls} class) from corpus titles via "
            "bench/mine_questions.py — ground-truth-carrying.",
            "# #000057 FABRICATION-BAIT: present-tense questions on "
            "time-varying offices; the ingested ~2010-2011 corpus "
            "(verified from the artifact, NOT the 2003 dump CLAUDE.md "
            "names) is the temporal adversary against a ~2023 model. "
            "Graded by the external Opus judge vs the fixed corpus-"
            "vintage article gold (bench/control_ab.py + bench/"
            "judge.py), NOT recall@k. Adversarial by construction.",
            "",
        ]
    else:
        hdr = [
            f"# AUTO-MINED ({a.cls} class) from corpus titles via "
            "bench/mine_questions.py — ground-truth-carrying.",
            "# Graded by deterministic retrieval recall@k "
            "(bench/recall_at_k.py), NOT audit_mode. Not adversarial; "
            "complements (never replaces) qa_questions.txt.",
            "",
        ]
    txt.write_text("\n".join(hdr + [r["question"] for r in rows]) + "\n")
    mp.write_text(json.dumps(rows, ensure_ascii=False, indent=2) + "\n")
    print(f"mined {len(rows)} {a.cls} questions")
    print(f"  -> {txt}\n  -> {mp}")
    for r in rows[:8]:
        print(f"  Q: {r['question']!r}  ->  target: {r['target_title']!r}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
