russell@unturf.com
a3ac6539c1
The first MEASURED, above-noise retrieval win this thread. The 75-q
n=3 audit_mode bench couldn't resolve any single lever (every failure
class <=3-5 q, sub the 5pp floor — four hypotheses died there). Fix
the instrument, not just the lever:
- bench/mine_questions.py + bench/recall_at_k.py: mine questions from
corpus titles (ground-truth target known by construction), grade by
deterministic retrieval recall@k via `query --dry-run` — no LLM, no
verifier, no n=3 noise, scalable to the 22K-deep numeral pool. The
curated qa_questions.txt stays the separate verifier-honesty/trap
gate; mined fixtures measure the answerable long tail per class.
- _numeral_fold_variants in query.py: ordinal-word ("Alexander the
second") <-> multi-char Roman ("Alexander II"), additive+symmetric,
unioned into _title_query_tokens exactly like _hyphen_fold_variants
(#000007). Strict 2..40 Roman set → no English-word collision;
single-char Romans (I/V/X) intentionally out of scope (universal
len>1 token filter — stated before building, ~4 of 10 residual
misses).
Measured on the mined numeral fixture: recall@8 22/40 (55%) -> 30/40
(75%), +20pp; 20 hits now rank-1. Discipline applied end to end:
measured-first, mirrored precedent, full-suite regression run (2482
passed, 0 regressions — numeral-fold is hot-path in
_title_query_tokens), real-path test (FakeSource->ingest->query()->
real _Hit, not a hand-built object), measured-after on a noise-free
instrument. The ~6 multi-char residual misses are a different
downstream cause the instrument now exposes for future iteration.
80 lines
2.7 KiB
Python
80 lines
2.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Deterministic retrieval recall@k on a mined ground-truth fixture.
|
|
|
|
For each mined question (surface-variant form), run retrieval ONLY
|
|
(`query --dry-run` — no LLM) and check whether the KNOWN target
|
|
article is in the top-k sources. Recall@k is a hard deterministic
|
|
number on N ground-truth questions — no verifier, no n=3 LLM noise,
|
|
no 5pp floor. This is the instrument a retrieval lever (e.g.
|
|
numeral-fold) is measured against: lever recall@k minus baseline
|
|
recall@k, on the same mined fixture.
|
|
|
|
usage: recall_at_k.py qa_questions_numeral_map.json [--k 8] [--conc 4]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import concurrent.futures as cf
|
|
import json
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
ARB = ROOT / ".venv" / "bin" / "arborist"
|
|
SHARDS = Path.home() / ".arborist" / "shards"
|
|
|
|
|
|
def _norm(t: str) -> str:
|
|
return (t or "").replace("_", " ").strip().casefold()
|
|
|
|
|
|
def probe(item: dict, k: int) -> tuple[bool, int]:
|
|
"""Return (target_in_topk, rank_or_-1). Retrieval only."""
|
|
try:
|
|
out = subprocess.run(
|
|
[str(ARB), "--shards-dir", str(SHARDS), "query", "--dry-run",
|
|
"--json", "--top-k", str(k), "--answer-mode", "claim_lattice",
|
|
item["question"]],
|
|
capture_output=True, text=True, timeout=120,
|
|
).stdout
|
|
d = json.loads(out)
|
|
except Exception:
|
|
return (False, -1)
|
|
tgt = _norm(item["target_title"])
|
|
titles = [_norm(s.get("title") or "") for s in (d.get("sources") or [])]
|
|
return (tgt in titles, titles.index(tgt) if tgt in titles else -1)
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("map_json")
|
|
ap.add_argument("--k", type=int, default=8)
|
|
ap.add_argument("--conc", type=int, default=4)
|
|
a = ap.parse_args()
|
|
items = json.loads(Path(a.map_json).read_text())
|
|
hits = 0
|
|
ranks: list[int] = []
|
|
misses: list[str] = []
|
|
with cf.ThreadPoolExecutor(max_workers=a.conc) as ex:
|
|
for it, (ok, rank) in zip(
|
|
items, ex.map(lambda i: probe(i, a.k), items)
|
|
):
|
|
if ok:
|
|
hits += 1
|
|
ranks.append(rank)
|
|
else:
|
|
misses.append(f"{it['question']!r} -> {it['target_title']!r}")
|
|
n = len(items)
|
|
print(f"recall@{a.k}: {hits}/{n} = {hits/n:.0%} "
|
|
f"(deterministic, no LLM — the instrument)")
|
|
if ranks:
|
|
print(f" of the hits, mean rank: {sum(ranks)/len(ranks):.1f} "
|
|
f"(0=top); rank-1 count: {sum(1 for r in ranks if r == 0)}")
|
|
print(f" MISSES ({len(misses)}) — target article never surfaced:")
|
|
for m in misses[:25]:
|
|
print(f" {m}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|