russell@unturf.com
031634c69c
control_ab.py called client.chat_completion(messages) but model is a keyword-only REQUIRED arg → TypeError every call → solo answer was always '[solo-error: TypeError]', which the hermetic judge correctly classified ABSTAINED. Every prior 'Hermes-solo abstained' smoke (incl. the n=2 the ticket cites as a case against the treatment) was a broken-control-arm artefact, not a measurement. Fix: pass model=a.model (symmetry with the Arborist arm). Error token now carries the exception message so the next failure self-describes. Also adds the #000057 'stale' fabrication-bait mining class (time-varying executive offices; corpus = ~2010-2011 Wikipedia VERIFIED from the artifact, NOT the 2003 dump CLAUDE.md names — flagged) and restores the numeral fixture deterministically.
296 lines
13 KiB
Python
296 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""Mine ground-truth-carrying eval questions from the corpus itself.
|
||
|
||
The 75-q curated bench can't resolve any single retrieval lever (each
|
||
failure class is <=3-5 q, sub the 5pp n=3 noise floor — four
|
||
hypotheses died on that). Fix the *instrument*: mine questions whose
|
||
target article is KNOWN by construction (we mined the question from
|
||
that title), so a lever is graded by deterministic retrieval
|
||
recall@k — no LLM, no verifier, no n=3 noise, scalable to thousands.
|
||
|
||
This is NOT a replacement for bench/qa_questions.txt — that curated,
|
||
deliberately-adversarial set stays the verifier-honesty/trap gate.
|
||
Mined questions are answerable-by-construction; they measure the
|
||
*answerable long tail* of retrieval per failure class.
|
||
|
||
v1 = the numeral class (Arabic/ordinal query vs Roman-numeral title —
|
||
the diagnosed `world war 2`!=`World War II`, `henry the eighth`!=
|
||
`Henry VIII` miss). ~22K-deep pool in the live shards.
|
||
|
||
The `stale` class (#000057) is the same machinery turned to the
|
||
opposite purpose: not a retrieval probe but a *fabrication-bait* set.
|
||
It mines time-varying executive-office titles and asks present-tense
|
||
incumbent questions; the ingested corpus (~2010–2011 Wikipedia —
|
||
verified from the artifact; CLAUDE.md's "2003 dump" does NOT match
|
||
the live shards, flagged to fox) is a ~13-year temporal adversary
|
||
against a ~2023 model, so the control (Hermes-solo) is confidently
|
||
WRONG against the fixed corpus-vintage gold instead of honestly
|
||
abstaining (the degeneracy the obscure-monarch numeral fixture hit).
|
||
Graded by the external Opus judge, NOT recall@k.
|
||
|
||
Pure python+sqlite3, deterministic, no egress, no LLM-for-generation.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import glob
|
||
import json
|
||
import re
|
||
import sqlite3
|
||
import unicodedata
|
||
from pathlib import Path
|
||
|
||
|
||
def _ascii_fold(s: str) -> str:
|
||
return "".join(
|
||
ch for ch in unicodedata.normalize("NFKD", s)
|
||
if not unicodedata.combining(ch)
|
||
)
|
||
|
||
# Strict Roman set 1..40 (covers monarchs/popes/wars); membership test
|
||
# avoids English-word collisions ("DID"/"MIX"/"CI" are not in here).
|
||
_ROMAN = {
|
||
1: "I", 2: "II", 3: "III", 4: "IV", 5: "V", 6: "VI", 7: "VII",
|
||
8: "VIII", 9: "IX", 10: "X", 11: "XI", 12: "XII", 13: "XIII",
|
||
14: "XIV", 15: "XV", 16: "XVI", 17: "XVII", 18: "XVIII", 19: "XIX",
|
||
20: "XX", 21: "XXI", 22: "XXII", 23: "XXIII", 24: "XXIV", 25: "XXV",
|
||
26: "XXVI", 27: "XXVII", 28: "XXVIII", 29: "XXIX", 30: "XXX",
|
||
31: "XXXI", 32: "XXXII", 33: "XXXIII", 34: "XXXIV", 35: "XXXV",
|
||
36: "XXXVI", 37: "XXXVII", 38: "XXXVIII", 39: "XXXIX", 40: "XL",
|
||
}
|
||
_ROMAN_TO_INT = {v: k for k, v in _ROMAN.items()}
|
||
_ORD = {
|
||
1: "first", 2: "second", 3: "third", 4: "fourth", 5: "fifth",
|
||
6: "sixth", 7: "seventh", 8: "eighth", 9: "ninth", 10: "tenth",
|
||
11: "eleventh", 12: "twelfth", 13: "thirteenth", 14: "fourteenth",
|
||
15: "fifteenth", 16: "sixteenth", 17: "seventeenth",
|
||
18: "eighteenth", 19: "nineteenth", 20: "twentieth",
|
||
}
|
||
|
||
# High-precision monarch/pope shape: "<Name> <Roman>" or
|
||
# "<Name> <Roman> of <Place>". Clean, unambiguous, answerable.
|
||
_MONARCH = re.compile(
|
||
r"^([A-Z][a-z]+(?:os|us|er)?) (" + "|".join(_ROMAN.values()) + r")"
|
||
r"( of [A-Z][a-zA-Z ]+)?$"
|
||
)
|
||
_BAD = ("list of", "(disambiguation)", "(album)", "(song)", "(film)",
|
||
"(band)", "(novel)", "(video game)")
|
||
|
||
|
||
_ACCENT_LETTER = re.compile(r"[À-ÖØ-öø-ÿĀ-ž]")
|
||
|
||
# #000057 fabrication-bait class. The numeral/accent/… classes are
|
||
# *retrieval* probes (answerable-by-construction → recall@k). `stale`
|
||
# is the opposite instrument: it exists to make the SMALL model
|
||
# confidently WRONG, not to test retrieval.
|
||
#
|
||
# CORPUS VINTAGE — VERIFIED FROM THE ARTIFACT, NOT ASSUMED. CLAUDE.md
|
||
# and the #000057 ticket say "2003 Wikipedia"; the live shards are
|
||
# NOT that dump. Two independent probes agree the ingested corpus is
|
||
# ~2010–2011 Wikipedia: (a) office infoboxes name 2007–2010
|
||
# incumbents (President of France = Sarkozy / 2007; PM Australia =
|
||
# Gillard / Jun-2010; President of Cuba = Raúl Castro / Feb-2008);
|
||
# (b) chunk year-mention histogram on shard 000 falls off a cliff
|
||
# after 2010 (2010: 756 → 2011: 1), and an article titled "2010"
|
||
# exists while "2011"/"2012" do not. Discrepancy flagged to fox; the
|
||
# experiment must describe its gold by the *measured* vintage.
|
||
#
|
||
# The fabrication-bait argument survives the correction: a ~2010
|
||
# corpus is still a ~13-year temporal adversary against Hermes-3-8B's
|
||
# ~2023 parametric memory. An executive-office title ("President of
|
||
# France") names its ~2010 incumbent in the lead — fixed gold,
|
||
# attached at mine time exactly like every other class. The present-
|
||
# tense question ("who is the president of France?") carries NO date,
|
||
# so the model answers from stale memory (Sarkozy→Macron) and does
|
||
# NOT honestly abstain — the degeneracy the obscure-monarch numeral
|
||
# fixture hit (Hermes ABSTAINED 5/5; you cannot measure a
|
||
# fabrication-vs-abstention delta when the control never fabricates).
|
||
# Deterministic gold + confident-wrong control is exactly the
|
||
# fabrication-bait the #000057 control experiment needs. Caveat the
|
||
# smoke run must check: offices whose holder did NOT turn over
|
||
# 2010→2023 dilute the signal (solo coincidentally CORRECT_GROUNDED)
|
||
# — prefer high-turnover offices; read the artifact, refine.
|
||
_OFFICE = re.compile(
|
||
r"^(President|Prime Minister|Chancellor|Premier|Taoiseach|"
|
||
r"Governor-General|First Minister|Chief Minister) "
|
||
r"of (the )?([A-Z][\w'’.\- ]+)$"
|
||
)
|
||
|
||
|
||
def _surface_variant(title: str, cls: str):
|
||
"""Return (question, ok) — the surface form a user TYPES vs the
|
||
form the corpus STORES. None when this title isn't in-class."""
|
||
if cls == "numeral":
|
||
m = _MONARCH.match(title)
|
||
if not m:
|
||
return None
|
||
name, roman, place = m.group(1), m.group(2), (m.group(3) or "")
|
||
n = _ROMAN_TO_INT[roman]
|
||
if n not in _ORD:
|
||
return None
|
||
return f"who was {name} the {_ORD[n]}{place.lower()}?"
|
||
if cls == "accent":
|
||
# Title has diacritics; user types the ASCII-folded form.
|
||
if not _ACCENT_LETTER.search(title):
|
||
return None
|
||
ascii_t = _ascii_fold(title)
|
||
toks = title.split()
|
||
if (ascii_t == title or len(toks) > 5 or len(toks) == 0
|
||
or not title[0].isupper()):
|
||
return None
|
||
return f"what is {ascii_t}?"
|
||
if cls == "hyphen":
|
||
# Title is hyphenated ("Jean-Paul Sartre", "Coca-Cola"); user
|
||
# types the de-hyphenated (space) form. Measures whether the
|
||
# EXISTING _hyphen_fold_variants (#000007) actually delivers
|
||
# on this 162K-prevalence class (never quantified).
|
||
if "-" not in title:
|
||
return None
|
||
dehy = title.replace("-", " ")
|
||
toks = title.split()
|
||
if (dehy == title or len(toks) > 5 or len(toks) == 0
|
||
or not title[0].isupper() or title.endswith("-")):
|
||
return None
|
||
return f"what is {dehy}?"
|
||
if cls == "honorific":
|
||
# Title spells the honorific in full ("Mount Everest", "Saint
|
||
# Petersburg", "Doctor Who"); user types the abbreviation.
|
||
# 22K-prevalence; concretely real (curated bench has "mount
|
||
# kilimanjaro"). Tests whether retrieval bridges St/Mt/Dr <->
|
||
# Saint/Mount/Doctor (no fold for this exists yet).
|
||
abbr = {"Mount": "Mt", "Saint": "St", "Doctor": "Dr",
|
||
"Fort": "Ft", "General": "Gen", "President": "Pres",
|
||
"Captain": "Capt", "Senator": "Sen"}
|
||
toks = title.split()
|
||
if len(toks) < 2 or len(toks) > 5 or toks[0] not in abbr:
|
||
return None # need >=2 tokens (drop degenerate "Mount")
|
||
short = " ".join([abbr[toks[0]]] + toks[1:])
|
||
return f"what is {short}?"
|
||
if cls == "amp":
|
||
# Title has "&"; user types "and". "AT&T" -> "AT and T".
|
||
if "&" not in title:
|
||
return None
|
||
v = re.sub(r"\s*&\s*", " and ", title).strip()
|
||
toks = title.split()
|
||
if v == title or len(toks) > 6 or not title[0].isalnum():
|
||
return None
|
||
return f"what is {v}?"
|
||
if cls == "brit":
|
||
# Title uses a British spelling; user types the American one.
|
||
bm = {"colour": "color", "honour": "honor", "behaviour": "behavior",
|
||
"organisation": "organization", "defence": "defense",
|
||
"centre": "center", "theatre": "theater",
|
||
"catalogue": "catalog", "programme": "program",
|
||
"labour": "labor", "favour": "favor"}
|
||
low = title.lower()
|
||
hit = next((b for b in bm if b in low), None)
|
||
if hit is None or len(title.split()) > 5 or not title[0].isupper():
|
||
return None
|
||
v = re.sub(hit, bm[hit], title, flags=re.IGNORECASE)
|
||
if v == title:
|
||
return None
|
||
return f"what is {v}?"
|
||
if cls == "stale":
|
||
# Time-varying executive office → present-tense incumbent
|
||
# question. Gold = the 2003 article (whole-article gold via
|
||
# control_ab._gold, same as every class). No surface
|
||
# transform: the title IS the role, the staleness is temporal,
|
||
# not orthographic.
|
||
m = _OFFICE.match(title)
|
||
if not m:
|
||
return None
|
||
office, the, place = m.group(1), (m.group(2) or ""), m.group(3)
|
||
# Drop degenerate / non-national scopes that 2003 may not
|
||
# pin to one person ("President of the United States" is the
|
||
# ideal shape; "President of the Senate" is a role-of-a-body).
|
||
bad_scope = ("Senate", "Assembly", "Council", "Court",
|
||
"Commission", "Committee", "Board", "Union",
|
||
"Republic of the Congo") # ambiguous dup-title
|
||
if any(b in place for b in bad_scope):
|
||
return None
|
||
if len(place.split()) > 5:
|
||
return None
|
||
return f"who is the {office.lower()} of {the}{place}?"
|
||
raise SystemExit(f"unknown class {cls!r}")
|
||
|
||
|
||
def mine(shards_dir: str, limit: int, cls: str) -> list[dict]:
|
||
out: list[dict] = []
|
||
seen: set[str] = set()
|
||
for db in sorted(glob.glob(f"{shards_dir}/00*.db")):
|
||
c = sqlite3.connect(db)
|
||
c.row_factory = sqlite3.Row
|
||
try:
|
||
rows = c.execute(
|
||
"SELECT document_root, title FROM documents "
|
||
"WHERE title IS NOT NULL"
|
||
).fetchall()
|
||
except sqlite3.OperationalError:
|
||
c.close()
|
||
continue
|
||
for r in rows:
|
||
title = (r["title"] or "").replace("_", " ").strip()
|
||
tl = title.lower()
|
||
if any(b in tl for b in _BAD) or title in seen:
|
||
continue
|
||
q = _surface_variant(title, cls)
|
||
if q is None:
|
||
continue
|
||
seen.add(title)
|
||
out.append({
|
||
"question": q,
|
||
"target_title": title,
|
||
"target_root": r["document_root"],
|
||
"shard": Path(db).name,
|
||
})
|
||
if len(out) >= limit:
|
||
c.close()
|
||
return out
|
||
c.close()
|
||
return out
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--shards-dir",
|
||
default=str(Path.home() / ".arborist" / "shards"))
|
||
ap.add_argument("--limit", type=int, default=40)
|
||
ap.add_argument("--cls", default="numeral")
|
||
a = ap.parse_args()
|
||
rows = mine(a.shards_dir, a.limit, a.cls)
|
||
root = Path(__file__).resolve().parents[1] / "bench"
|
||
txt = root / f"qa_questions_{a.cls}.txt"
|
||
mp = root / f"qa_questions_{a.cls}_map.json"
|
||
if a.cls == "stale":
|
||
hdr = [
|
||
f"# AUTO-MINED ({a.cls} class) from corpus titles via "
|
||
"bench/mine_questions.py — ground-truth-carrying.",
|
||
"# #000057 FABRICATION-BAIT: present-tense questions on "
|
||
"time-varying offices; the ingested ~2010-2011 corpus "
|
||
"(verified from the artifact, NOT the 2003 dump CLAUDE.md "
|
||
"names) is the temporal adversary against a ~2023 model. "
|
||
"Graded by the external Opus judge vs the fixed corpus-"
|
||
"vintage article gold (bench/control_ab.py + bench/"
|
||
"judge.py), NOT recall@k. Adversarial by construction.",
|
||
"",
|
||
]
|
||
else:
|
||
hdr = [
|
||
f"# AUTO-MINED ({a.cls} class) from corpus titles via "
|
||
"bench/mine_questions.py — ground-truth-carrying.",
|
||
"# Graded by deterministic retrieval recall@k "
|
||
"(bench/recall_at_k.py), NOT audit_mode. Not adversarial; "
|
||
"complements (never replaces) qa_questions.txt.",
|
||
"",
|
||
]
|
||
txt.write_text("\n".join(hdr + [r["question"] for r in rows]) + "\n")
|
||
mp.write_text(json.dumps(rows, ensure_ascii=False, indent=2) + "\n")
|
||
print(f"mined {len(rows)} {a.cls} questions")
|
||
print(f" -> {txt}\n -> {mp}")
|
||
for r in rows[:8]:
|
||
print(f" Q: {r['question']!r} -> target: {r['target_title']!r}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|