engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(retrieval): numeral-fold (ordinal-word <-> Roman) + mined ground-truth eval instrument

Browse source 
The first MEASURED, above-noise retrieval win this thread. The 75-q
n=3 audit_mode bench couldn't resolve any single lever (every failure
class <=3-5 q, sub the 5pp floor — four hypotheses died there). Fix
the instrument, not just the lever:

- bench/mine_questions.py + bench/recall_at_k.py: mine questions from
  corpus titles (ground-truth target known by construction), grade by
  deterministic retrieval recall@k via `query --dry-run` — no LLM, no
  verifier, no n=3 noise, scalable to the 22K-deep numeral pool. The
  curated qa_questions.txt stays the separate verifier-honesty/trap
  gate; mined fixtures measure the answerable long tail per class.

- _numeral_fold_variants in query.py: ordinal-word ("Alexander the
  second") <-> multi-char Roman ("Alexander II"), additive+symmetric,
  unioned into _title_query_tokens exactly like _hyphen_fold_variants
  (#000007). Strict 2..40 Roman set → no English-word collision;
  single-char Romans (I/V/X) intentionally out of scope (universal
  len>1 token filter — stated before building, ~4 of 10 residual
  misses).

Measured on the mined numeral fixture: recall@8 22/40 (55%) -> 30/40
(75%), +20pp; 20 hits now rank-1. Discipline applied end to end:
measured-first, mirrored precedent, full-suite regression run (2482
passed, 0 regressions — numeral-fold is hot-path in
_title_query_tokens), real-path test (FakeSource->ingest->query()->
real _Hit, not a hand-built object), measured-after on a noise-free
instrument. The ~6 multi-char residual misses are a different
downstream cause the instrument now exposes for future iteration.
This commit is contained in:
russell@unturf.com 2026-05-18 15:16:10 -04:00
parent 5d43fdc037
commit a3ac6539c1
No known key found for this signature in database
6 changed files with 635 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

133
bench/mine_questions.py Normal file
View file

@ -0,0 +1,133 @@
#!/usr/bin/env python3
"""Mine ground-truth-carrying eval questions from the corpus itself.
The 75-q curated bench can't resolve any single retrieval lever (each
failure class is <=3-5 q, sub the 5pp n=3 noise floor — four
hypotheses died on that). Fix the *instrument*: mine questions whose
target article is KNOWN by construction (we mined the question from
that title), so a lever is graded by deterministic retrieval
recall@k — no LLM, no verifier, no n=3 noise, scalable to thousands.
This is NOT a replacement for bench/qa_questions.txt — that curated,
deliberately-adversarial set stays the verifier-honesty/trap gate.
Mined questions are answerable-by-construction; they measure the
*answerable long tail* of retrieval per failure class.
v1 = the numeral class (Arabic/ordinal query vs Roman-numeral title —
the diagnosed `world war 2`!=`World War II`, `henry the eighth`!=
`Henry VIII` miss). ~22K-deep pool in the live shards.
Pure python+sqlite3, deterministic, no egress, no LLM-for-generation.
"""
from __future__ import annotations
import argparse
import glob
import json
import re
import sqlite3
from pathlib import Path
# Strict Roman set 1..40 (covers monarchs/popes/wars); membership test
# avoids English-word collisions ("DID"/"MIX"/"CI" are not in here).
_ROMAN = {
1: "I", 2: "II", 3: "III", 4: "IV", 5: "V", 6: "VI", 7: "VII",
8: "VIII", 9: "IX", 10: "X", 11: "XI", 12: "XII", 13: "XIII",
14: "XIV", 15: "XV", 16: "XVI", 17: "XVII", 18: "XVIII", 19: "XIX",
20: "XX", 21: "XXI", 22: "XXII", 23: "XXIII", 24: "XXIV", 25: "XXV",
26: "XXVI", 27: "XXVII", 28: "XXVIII", 29: "XXIX", 30: "XXX",
31: "XXXI", 32: "XXXII", 33: "XXXIII", 34: "XXXIV", 35: "XXXV",
36: "XXXVI", 37: "XXXVII", 38: "XXXVIII", 39: "XXXIX", 40: "XL",
}
_ROMAN_TO_INT = {v: k for k, v in _ROMAN.items()}
_ORD = {
1: "first", 2: "second", 3: "third", 4: "fourth", 5: "fifth",
6: "sixth", 7: "seventh", 8: "eighth", 9: "ninth", 10: "tenth",
11: "eleventh", 12: "twelfth", 13: "thirteenth", 14: "fourteenth",
15: "fifteenth", 16: "sixteenth", 17: "seventeenth",
18: "eighteenth", 19: "nineteenth", 20: "twentieth",
}
# High-precision monarch/pope shape: "<Name> <Roman>" or
# "<Name> <Roman> of <Place>". Clean, unambiguous, answerable.
_MONARCH = re.compile(
r"^([A-Z][a-z]+(?:os|us|er)?) (" + "|".join(_ROMAN.values()) + r")"
r"( of [A-Z][a-zA-Z ]+)?$"
)
_BAD = ("list of", "(disambiguation)", "(album)", "(song)", "(film)",
"(band)", "(novel)", "(video game)")
def mine(shards_dir: str, limit: int) -> list[dict]:
out: list[dict] = []
seen: set[str] = set()
for db in sorted(glob.glob(f"{shards_dir}/00*.db")):
c = sqlite3.connect(db)
c.row_factory = sqlite3.Row
try:
rows = c.execute(
"SELECT document_root, title FROM documents "
"WHERE title IS NOT NULL"
).fetchall()
except sqlite3.OperationalError:
c.close()
continue
for r in rows:
title = (r["title"] or "").replace("_", " ").strip()
tl = title.lower()
if any(b in tl for b in _BAD) or title in seen:
continue
m = _MONARCH.match(title)
if not m:
continue
name, roman, place = m.group(1), m.group(2), (m.group(3) or "")
n = _ROMAN_TO_INT[roman]
if n not in _ORD: # keep natural ordinal phrasing only
continue
seen.add(title)
# Surface variant: the Arabic/ordinal form a user types,
# vs the Roman-numeral title the corpus stores.
q = f"who was {name} the {_ORD[n]}{place.lower()}?"
out.append({
"question": q,
"target_title": title,
"target_root": r["document_root"],
"shard": Path(db).name,
})
if len(out) >= limit:
c.close()
return out
c.close()
return out
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--shards-dir",
default=str(Path.home() / ".arborist" / "shards"))
ap.add_argument("--limit", type=int, default=40)
ap.add_argument("--cls", default="numeral")
a = ap.parse_args()
rows = mine(a.shards_dir, a.limit)
root = Path(__file__).resolve().parents[1] / "bench"
txt = root / f"qa_questions_{a.cls}.txt"
mp = root / f"qa_questions_{a.cls}_map.json"
hdr = [
f"# AUTO-MINED ({a.cls} class) from corpus titles via "
"bench/mine_questions.py — ground-truth-carrying.",
"# Graded by deterministic retrieval recall@k "
"(bench/recall_at_k.py), NOT audit_mode. Not adversarial; "
"complements (never replaces) qa_questions.txt.",
"",
]
txt.write_text("\n".join(hdr + [r["question"] for r in rows]) + "\n")
mp.write_text(json.dumps(rows, ensure_ascii=False, indent=2) + "\n")
print(f"mined {len(rows)} {a.cls} questions")
print(f" -> {txt}\n -> {mp}")
for r in rows[:8]:
print(f" Q: {r['question']!r} -> target: {r['target_title']!r}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

43
bench/qa_questions_numeral.txt Normal file
View file

@ -0,0 +1,43 @@
# AUTO-MINED (numeral class) from corpus titles via bench/mine_questions.py — ground-truth-carrying.
# Graded by deterministic retrieval recall@k (bench/recall_at_k.py), NOT audit_mode. Not adversarial; complements (never replaces) qa_questions.txt.
who was Albert the third?
who was Ahmed the third?
who was Alaric the first?
who was Alexander the first of epirus?
who was Alexander the second of scotland?
who was Alexander the second?
who was Alexander the fourth?
who was Alyattes the second?
who was Afonso the fourth of portugal?
who was Alfonso the second of asturias?
who was Alfonso the fourth of aragon?
who was Alfonso the third?
who was Alfonso the fifth?
who was Anastasius the second?
who was Abbas the second of egypt?
who was Charles the fifth?
who was Constantius the second?
who was Constantine the second of scotland?
who was Charles the first of england?
who was Frederick the fifth?
who was Henry the seventh?
who was Mehmed the first?
who was Mustafa the first?
who was Mieszko the first of poland?
who was Malcolm the first of scotland?
who was Osman the second?
who was Quake the second?
who was Stephen the third?
who was Oscar the first of sweden?
who was Charles the fifteenth of sweden?
who was Sviatoslav the first of kiev?
who was Catherine the second of russia?
who was Childeric the first?
who was Rudolph the first of germany?
who was Xerxes the second of persia?
who was Richard the second of england?
who was Gustav the first of sweden?
who was Photios the first of constantinople?
who was James the fifth of scotland?
who was Basarab the first of wallachia?

242
bench/qa_questions_numeral_map.json Normal file
View file

@ -0,0 +1,242 @@
[
{
"question": "who was Albert the third?",
"target_title": "Albert III",
"target_root": "43ff05c3c851e358df34c36d7398fefc53998e5a9cd8654b5c2f8dd9524b9efd",
"shard": "000.db"
},
{
"question": "who was Ahmed the third?",
"target_title": "Ahmed III",
"target_root": "2fc91045e60b915d146f1f4a0f0c72e2df2f644efbc299ea27ca40170e6092aa",
"shard": "000.db"
},
{
"question": "who was Alaric the first?",
"target_title": "Alaric I",
"target_root": "57e7aa58a4fd36b42a920956929e7c852108f2e3d36c87487d411ec4aa7f6686",
"shard": "000.db"
},
{
"question": "who was Alexander the first of epirus?",
"target_title": "Alexander I of Epirus",
"target_root": "db479a5835381d3d94705976dd6adede8fef8e51ea88cc6c3fe4c789e1e1ae35",
"shard": "000.db"
},
{
"question": "who was Alexander the second of scotland?",
"target_title": "Alexander II of Scotland",
"target_root": "4d5885f25ab6d7ebd0054dfdb4783f00db96f210ea33cdff2c0dbb0b92b45a13",
"shard": "000.db"
},
{
"question": "who was Alexander the second?",
"target_title": "Alexander II",
"target_root": "190bf209efb2f096f9b93c7dd2d0b0be9a64d6aa469d27cf2272664e4f624de2",
"shard": "000.db"
},
{
"question": "who was Alexander the fourth?",
"target_title": "Alexander IV",
"target_root": "cc3134f31a0d69f03aa37a5821fc179082f34316e782d464b0685f3ed0e90f0a",
"shard": "000.db"
},
{
"question": "who was Alyattes the second?",
"target_title": "Alyattes II",
"target_root": "464c0878c6fcabaf67f37389aa343b180b2762325a9aa916aa0d95dd7529e853",
"shard": "000.db"
},
{
"question": "who was Afonso the fourth of portugal?",
"target_title": "Afonso IV of Portugal",
"target_root": "88c2a881546a4cfa59c8eba1e037d2092ab73c9f09feb9524801d64763dfbd3e",
"shard": "000.db"
},
{
"question": "who was Alfonso the second of asturias?",
"target_title": "Alfonso II of Asturias",
"target_root": "1b2073183930900d65f1433efb70e8080628b0aa440817a3eb235d70ddf18f61",
"shard": "000.db"
},
{
"question": "who was Alfonso the fourth of aragon?",
"target_title": "Alfonso IV of Aragon",
"target_root": "fb96fb9d074fefd80da864c412a9f07cab26b1a217b54a378442143c147316d7",
"shard": "000.db"
},
{
"question": "who was Alfonso the third?",
"target_title": "Alfonso III",
"target_root": "555a58a9f7c947f32e18dcd6dfb03fb13ba7cc29f144c5b98f8af087892d9ac0",
"shard": "000.db"
},
{
"question": "who was Alfonso the fifth?",
"target_title": "Alfonso V",
"target_root": "c7aaf0276378cefbbb1a44dc1f90b4cb5928411880c7869ea68628f09f1ebde2",
"shard": "000.db"
},
{
"question": "who was Anastasius the second?",
"target_title": "Anastasius II",
"target_root": "2cb452df76d8d15494550c8070b51bca9f369647c95574e6a4f9cb441f865c0c",
"shard": "000.db"
},
{
"question": "who was Abbas the second of egypt?",
"target_title": "Abbas II of Egypt",
"target_root": "8c3378f3375831933b0c0865074747c264599ceb2650f866c5d61d3635126035",
"shard": "000.db"
},
{
"question": "who was Charles the fifth?",
"target_title": "Charles V",
"target_root": "22d8e5eb9bebef543cecda7b33a1f0d300cd24e3a13427c6a297524c9813718f",
"shard": "000.db"
},
{
"question": "who was Constantius the second?",
"target_title": "Constantius II",
"target_root": "5f8bd920302cc83ee5aef59abfda0aafd31b6e1cb48cfc4f5c7d4c3e69bedd4a",
"shard": "000.db"
},
{
"question": "who was Constantine the second of scotland?",
"target_title": "Constantine II of Scotland",
"target_root": "24e398f6b012f01c0e1b1cb7fad7b4ee2fda8898a15975e5d1c75a4ee77bc3df",
"shard": "000.db"
},
{
"question": "who was Charles the first of england?",
"target_title": "Charles I of England",
"target_root": "ad6bc7ccd05e08e3aa9fd3db04dc85e52fc64e54c22532fe1af00ec956bef907",
"shard": "000.db"
},
{
"question": "who was Frederick the fifth?",
"target_title": "Frederick V",
"target_root": "ac3d08404d5e30a1642647512b5e5170253fb46577a244390c909dd8679922c0",
"shard": "000.db"
},
{
"question": "who was Henry the seventh?",
"target_title": "Henry VII",
"target_root": "91e7879a395c0a43c218964ccc28bcff93ec3e3e141483af3b50e2d596b185df",
"shard": "000.db"
},
{
"question": "who was Mehmed the first?",
"target_title": "Mehmed I",
"target_root": "9a526b048fe00f930da0b4128eb30dde55ebfee10490cea2322768c8d1f18023",
"shard": "000.db"
},
{
"question": "who was Mustafa the first?",
"target_title": "Mustafa I",
"target_root": "6d4d3e3d401f3b1651fc3d779d71b8084a5ff69ba2f95516dcefec63f8e6fd20",
"shard": "000.db"
},
{
"question": "who was Mieszko the first of poland?",
"target_title": "Mieszko I of Poland",
"target_root": "fee8e212dc5d6439d1b791eea1bc997d825a4b4abf3bfeeef2cfae57e948d077",
"shard": "000.db"
},
{
"question": "who was Malcolm the first of scotland?",
"target_title": "Malcolm I of Scotland",
"target_root": "1f1983a45ed9b9cfa49425103fed17a1ca0e36284189525a31a9bc73a14d2181",
"shard": "000.db"
},
{
"question": "who was Osman the second?",
"target_title": "Osman II",
"target_root": "71da21f6a76233de3ac45683f8d8b419d9962581cba0f2fb0cf81093b7e41b0a",
"shard": "000.db"
},
{
"question": "who was Quake the second?",
"target_title": "Quake II",
"target_root": "86b1189f25d0fe612423ef88be44885da48f115c27b64024078c2fd257acb66a",
"shard": "000.db"
},
{
"question": "who was Stephen the third?",
"target_title": "Stephen III",
"target_root": "07d31c9abc8c1e8626997b248c82592b09dff67e96069c31c90b95230407e4c2",
"shard": "000.db"
},
{
"question": "who was Oscar the first of sweden?",
"target_title": "Oscar I of Sweden",
"target_root": "09ab96881d295f59dd8131a2d9d15bc2d633bd9e9606c1c27d7e1c7c889cfec0",
"shard": "000.db"
},
{
"question": "who was Charles the fifteenth of sweden?",
"target_title": "Charles XV of Sweden",
"target_root": "381a3cc80ba62a7454ea87bfd56ddefb27229491e3926a88d84968fd791485bb",
"shard": "000.db"
},
{
"question": "who was Sviatoslav the first of kiev?",
"target_title": "Sviatoslav I of Kiev",
"target_root": "99066a1e171e3a3924c115f6010a306743520730d9a0e0ed4f359c0e7239a7dd",
"shard": "000.db"
},
{
"question": "who was Catherine the second of russia?",
"target_title": "Catherine II of Russia",
"target_root": "d7a6a6d1f63b0ddd4977b2c9203fdd6659b0fdcadbb43096910c028ea492a42b",
"shard": "000.db"
},
{
"question": "who was Childeric the first?",
"target_title": "Childeric I",
"target_root": "a21db476b6e4ece9aa1cc7da6b68b199474f82c297eaa3304fd3713d0b7ec682",
"shard": "000.db"
},
{
"question": "who was Rudolph the first of germany?",
"target_title": "Rudolph I of Germany",
"target_root": "7f882ac84cf32819f3af2dda446d18ad7cd818e94b201c56a3a89a0d78bf9161",
"shard": "000.db"
},
{
"question": "who was Xerxes the second of persia?",
"target_title": "Xerxes II of Persia",
"target_root": "edafe8586c5666f8fb38a499ac73f0d954fba6c2d3b869610de16ba33bcc8a3f",
"shard": "000.db"
},
{
"question": "who was Richard the second of england?",
"target_title": "Richard II of England",
"target_root": "d08b7f5eb56fb3352a77facfbc204b53b1a6cef4215d15ac7a2a97f3a30dda8d",
"shard": "000.db"
},
{
"question": "who was Gustav the first of sweden?",
"target_title": "Gustav I of Sweden",
"target_root": "4ff24ede9c8fc1bd401972bef9a3a8e4a4614b8ad289027e9e51b00f16631a04",
"shard": "000.db"
},
{
"question": "who was Photios the first of constantinople?",
"target_title": "Photios I of Constantinople",
"target_root": "1309569c1cb8f37ff5f7d2cfc00080efb5f6846e54b1a45c07357e1b0a2445f2",
"shard": "000.db"
},
{
"question": "who was James the fifth of scotland?",
"target_title": "James V of Scotland",
"target_root": "6b0e3af072a252daaad838123f5b2c2db3b0ac91bd7833faf17ff4f0a488fbe0",
"shard": "000.db"
},
{
"question": "who was Basarab the first of wallachia?",
"target_title": "Basarab I of Wallachia",
"target_root": "6f543c2b310d31ef08d43e4454a69eb2294ffc412df4cb26bafd92ea9841ca2e",
"shard": "000.db"
}
]

80
bench/recall_at_k.py Normal file
View file

@ -0,0 +1,80 @@
#!/usr/bin/env python3
"""Deterministic retrieval recall@k on a mined ground-truth fixture.
For each mined question (surface-variant form), run retrieval ONLY
(`query --dry-run` — no LLM) and check whether the KNOWN target
article is in the top-k sources. Recall@k is a hard deterministic
number on N ground-truth questions — no verifier, no n=3 LLM noise,
no 5pp floor. This is the instrument a retrieval lever (e.g.
numeral-fold) is measured against: lever recall@k minus baseline
recall@k, on the same mined fixture.
usage: recall_at_k.py qa_questions_numeral_map.json [--k 8] [--conc 4]
"""
from __future__ import annotations
import argparse
import concurrent.futures as cf
import json
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
ARB = ROOT / ".venv" / "bin" / "arborist"
SHARDS = Path.home() / ".arborist" / "shards"
def _norm(t: str) -> str:
return (t or "").replace("_", " ").strip().casefold()
def probe(item: dict, k: int) -> tuple[bool, int]:
"""Return (target_in_topk, rank_or_-1). Retrieval only."""
try:
out = subprocess.run(
[str(ARB), "--shards-dir", str(SHARDS), "query", "--dry-run",
"--json", "--top-k", str(k), "--answer-mode", "claim_lattice",
item["question"]],
capture_output=True, text=True, timeout=120,
).stdout
d = json.loads(out)
except Exception:
return (False, -1)
tgt = _norm(item["target_title"])
titles = [_norm(s.get("title") or "") for s in (d.get("sources") or [])]
return (tgt in titles, titles.index(tgt) if tgt in titles else -1)
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("map_json")
ap.add_argument("--k", type=int, default=8)
ap.add_argument("--conc", type=int, default=4)
a = ap.parse_args()
items = json.loads(Path(a.map_json).read_text())
hits = 0
ranks: list[int] = []
misses: list[str] = []
with cf.ThreadPoolExecutor(max_workers=a.conc) as ex:
for it, (ok, rank) in zip(
items, ex.map(lambda i: probe(i, a.k), items)
):
if ok:
hits += 1
ranks.append(rank)
else:
misses.append(f"{it['question']!r} -> {it['target_title']!r}")
n = len(items)
print(f"recall@{a.k}: {hits}/{n} = {hits/n:.0%} "
f"(deterministic, no LLM — the instrument)")
if ranks:
print(f" of the hits, mean rank: {sum(ranks)/len(ranks):.1f} "
f"(0=top); rank-1 count: {sum(1 for r in ranks if r == 0)}")
print(f" MISSES ({len(misses)}) — target article never surfaced:")
for m in misses[:25]:
print(f" {m}")
return 0
if __name__ == "__main__":
raise SystemExit(main())