arborist/bench/mine_questions.py

#!/usr/bin/env python3
"""Mine ground-truth-carrying eval questions from the corpus itself.

The 75-q curated bench can't resolve any single retrieval lever (each
failure class is <=3-5 q, sub the 5pp n=3 noise floor — four
hypotheses died on that). Fix the *instrument*: mine questions whose
target article is KNOWN by construction (we mined the question from
that title), so a lever is graded by deterministic retrieval
recall@k — no LLM, no verifier, no n=3 noise, scalable to thousands.

This is NOT a replacement for bench/qa_questions.txt — that curated,
deliberately-adversarial set stays the verifier-honesty/trap gate.
Mined questions are answerable-by-construction; they measure the
*answerable long tail* of retrieval per failure class.

v1 = the numeral class (Arabic/ordinal query vs Roman-numeral title —
the diagnosed `world war 2`!=`World War II`, `henry the eighth`!=
`Henry VIII` miss). ~22K-deep pool in the live shards.

Pure python+sqlite3, deterministic, no egress, no LLM-for-generation.
"""
from __future__ import annotations

import argparse
import glob
import json
import re
import sqlite3
from pathlib import Path

# Strict Roman set 1..40 (covers monarchs/popes/wars); membership test
# avoids English-word collisions ("DID"/"MIX"/"CI" are not in here).
_ROMAN = {
    1: "I", 2: "II", 3: "III", 4: "IV", 5: "V", 6: "VI", 7: "VII",
    8: "VIII", 9: "IX", 10: "X", 11: "XI", 12: "XII", 13: "XIII",
    14: "XIV", 15: "XV", 16: "XVI", 17: "XVII", 18: "XVIII", 19: "XIX",
    20: "XX", 21: "XXI", 22: "XXII", 23: "XXIII", 24: "XXIV", 25: "XXV",
    26: "XXVI", 27: "XXVII", 28: "XXVIII", 29: "XXIX", 30: "XXX",
    31: "XXXI", 32: "XXXII", 33: "XXXIII", 34: "XXXIV", 35: "XXXV",
    36: "XXXVI", 37: "XXXVII", 38: "XXXVIII", 39: "XXXIX", 40: "XL",
}
_ROMAN_TO_INT = {v: k for k, v in _ROMAN.items()}
_ORD = {
    1: "first", 2: "second", 3: "third", 4: "fourth", 5: "fifth",
    6: "sixth", 7: "seventh", 8: "eighth", 9: "ninth", 10: "tenth",
    11: "eleventh", 12: "twelfth", 13: "thirteenth", 14: "fourteenth",
    15: "fifteenth", 16: "sixteenth", 17: "seventeenth",
    18: "eighteenth", 19: "nineteenth", 20: "twentieth",
}

# High-precision monarch/pope shape: "<Name> <Roman>" or
# "<Name> <Roman> of <Place>". Clean, unambiguous, answerable.
_MONARCH = re.compile(
    r"^([A-Z][a-z]+(?:os|us|er)?) (" + "|".join(_ROMAN.values()) + r")"
    r"( of [A-Z][a-zA-Z ]+)?$"
)
_BAD = ("list of", "(disambiguation)", "(album)", "(song)", "(film)",
        "(band)", "(novel)", "(video game)")


def mine(shards_dir: str, limit: int) -> list[dict]:
    out: list[dict] = []
    seen: set[str] = set()
    for db in sorted(glob.glob(f"{shards_dir}/00*.db")):
        c = sqlite3.connect(db)
        c.row_factory = sqlite3.Row
        try:
            rows = c.execute(
                "SELECT document_root, title FROM documents "
                "WHERE title IS NOT NULL"
            ).fetchall()
        except sqlite3.OperationalError:
            c.close()
            continue
        for r in rows:
            title = (r["title"] or "").replace("_", " ").strip()
            tl = title.lower()
            if any(b in tl for b in _BAD) or title in seen:
                continue
            m = _MONARCH.match(title)
            if not m:
                continue
            name, roman, place = m.group(1), m.group(2), (m.group(3) or "")
            n = _ROMAN_TO_INT[roman]
            if n not in _ORD:  # keep natural ordinal phrasing only
                continue
            seen.add(title)
            # Surface variant: the Arabic/ordinal form a user types,
            # vs the Roman-numeral title the corpus stores.
            q = f"who was {name} the {_ORD[n]}{place.lower()}?"
            out.append({
                "question": q,
                "target_title": title,
                "target_root": r["document_root"],
                "shard": Path(db).name,
            })
            if len(out) >= limit:
                c.close()
                return out
        c.close()
    return out


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--shards-dir",
                    default=str(Path.home() / ".arborist" / "shards"))
    ap.add_argument("--limit", type=int, default=40)
    ap.add_argument("--cls", default="numeral")
    a = ap.parse_args()
    rows = mine(a.shards_dir, a.limit)
    root = Path(__file__).resolve().parents[1] / "bench"
    txt = root / f"qa_questions_{a.cls}.txt"
    mp = root / f"qa_questions_{a.cls}_map.json"
    hdr = [
        f"# AUTO-MINED ({a.cls} class) from corpus titles via "
        "bench/mine_questions.py — ground-truth-carrying.",
        "# Graded by deterministic retrieval recall@k "
        "(bench/recall_at_k.py), NOT audit_mode. Not adversarial; "
        "complements (never replaces) qa_questions.txt.",
        "",
    ]
    txt.write_text("\n".join(hdr + [r["question"] for r in rows]) + "\n")
    mp.write_text(json.dumps(rows, ensure_ascii=False, indent=2) + "\n")
    print(f"mined {len(rows)} {a.cls} questions")
    print(f"  -> {txt}\n  -> {mp}")
    for r in rows[:8]:
        print(f"  Q: {r['question']!r}  ->  target: {r['target_title']!r}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())