feat(retrieval): numeral-fold (ordinal-word <-> Roman) + mined ground-truth eval instrument
The first MEASURED, above-noise retrieval win this thread. The 75-q
n=3 audit_mode bench couldn't resolve any single lever (every failure
class <=3-5 q, sub the 5pp floor — four hypotheses died there). Fix
the instrument, not just the lever:
- bench/mine_questions.py + bench/recall_at_k.py: mine questions from
corpus titles (ground-truth target known by construction), grade by
deterministic retrieval recall@k via `query --dry-run` — no LLM, no
verifier, no n=3 noise, scalable to the 22K-deep numeral pool. The
curated qa_questions.txt stays the separate verifier-honesty/trap
gate; mined fixtures measure the answerable long tail per class.
- _numeral_fold_variants in query.py: ordinal-word ("Alexander the
second") <-> multi-char Roman ("Alexander II"), additive+symmetric,
unioned into _title_query_tokens exactly like _hyphen_fold_variants
(#000007). Strict 2..40 Roman set → no English-word collision;
single-char Romans (I/V/X) intentionally out of scope (universal
len>1 token filter — stated before building, ~4 of 10 residual
misses).
Measured on the mined numeral fixture: recall@8 22/40 (55%) -> 30/40
(75%), +20pp; 20 hits now rank-1. Discipline applied end to end:
measured-first, mirrored precedent, full-suite regression run (2482
passed, 0 regressions — numeral-fold is hot-path in
_title_query_tokens), real-path test (FakeSource->ingest->query()->
real _Hit, not a hand-built object), measured-after on a noise-free
instrument. The ~6 multi-char residual misses are a different
downstream cause the instrument now exposes for future iteration.
This commit is contained in:
parent
5d43fdc037
commit
a3ac6539c1
6 changed files with 635 additions and 0 deletions
|
|
@ -156,7 +156,64 @@ def _hyphen_fold_variants(s: str) -> set[str]:
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# Numeral-fold (measured 2026-05-18): ordinal-word query ("Alexander
|
||||||
|
# the second") never reaches a Roman-numeral title ("Alexander II")
|
||||||
|
# because `second` != `ii`. Mined-fixture recall@8 = 22/40 = 55 %,
|
||||||
|
# 18 clean misses all this shape. Same additive+symmetric mechanism
|
||||||
|
# as `_hyphen_fold_variants` (#000007). Scope is honest: only
|
||||||
|
# multi-char Romans survive the universal `len > 1` token filter, so
|
||||||
|
# single-char Romans (I/V/X — "Charles V") are out of reach by token-
|
||||||
|
# fold alone (a deeper change; ~4 of the 18, separately measurable).
|
||||||
|
# Strict 2..40 set → no English-word collision ("did"/"mix"/"civil"
|
||||||
|
# are not Romans here); single-char forms intentionally absent.
|
||||||
|
_NUM_ORD_TO_ROMAN = {
|
||||||
|
"second": "ii", "third": "iii", "fourth": "iv", "sixth": "vi",
|
||||||
|
"seventh": "vii", "eighth": "viii", "ninth": "ix", "eleventh": "xi",
|
||||||
|
"twelfth": "xii", "thirteenth": "xiii", "fourteenth": "xiv",
|
||||||
|
"fifteenth": "xv", "sixteenth": "xvi", "seventeenth": "xvii",
|
||||||
|
"eighteenth": "xviii", "nineteenth": "xix", "twentieth": "xx",
|
||||||
|
}
|
||||||
|
_NUM_ROMAN_TO_ORD = {v: k for k, v in _NUM_ORD_TO_ROMAN.items()}
|
||||||
|
|
||||||
|
|
||||||
|
def _numeral_fold_variants(s: str) -> set[str]:
|
||||||
|
"""Symmetric ordinal-word <-> multi-char-Roman fold.
|
||||||
|
|
||||||
|
"who was Alexander the second?" -> {"ii"}
|
||||||
|
"Alexander II" -> {"second"}
|
||||||
|
"plain query" -> set()
|
||||||
|
|
||||||
|
Additive (mirrors `_hyphen_fold_variants`): callers union this
|
||||||
|
into the token set, so existing matches are preserved and a
|
||||||
|
folded query token can now also overlap a Roman-numeral title.
|
||||||
|
Stopword/length filtered like `_title_query_tokens`.
|
||||||
|
"""
|
||||||
|
out: set[str] = set()
|
||||||
|
for tok in _TITLE_TOKEN_RE.findall(s):
|
||||||
|
tl = tok.lower()
|
||||||
|
v = _NUM_ORD_TO_ROMAN.get(tl) or _NUM_ROMAN_TO_ORD.get(tl)
|
||||||
|
if v and len(v) > 1 and v not in _TITLE_STOPWORDS:
|
||||||
|
out.add(v)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
def _title_query_tokens(s: str) -> set[str]:
|
def _title_query_tokens(s: str) -> set[str]:
|
||||||
|
base = {
|
||||||
|
t.lower()
|
||||||
|
for t in _TITLE_TOKEN_RE.findall(s)
|
||||||
|
if t.lower() not in _TITLE_STOPWORDS and len(t) > 1
|
||||||
|
}
|
||||||
|
# Hyphen-fold: additively include joined-no-hyphen variants for
|
||||||
|
# hyphenated runs in `s`. Symmetric — the function is called on
|
||||||
|
# both queries and titles, and additive fold preserves existing
|
||||||
|
# match patterns (e.g. `Coca-Cola history` query keeps {coca,
|
||||||
|
# cola, cocacola, history} so a `Coca-Cola` title still passes
|
||||||
|
# title-breadth via {coca, cola, cocacola}). See Ticket #000007.
|
||||||
|
base |= _hyphen_fold_variants(s)
|
||||||
|
# Numeral-fold: ordinal-word <-> Roman-numeral, same additive+
|
||||||
|
# symmetric discipline (measured 2026-05-18; see above).
|
||||||
|
base |= _numeral_fold_variants(s)
|
||||||
|
return base
|
||||||
base = {
|
base = {
|
||||||
t.lower()
|
t.lower()
|
||||||
for t in _TITLE_TOKEN_RE.findall(s)
|
for t in _TITLE_TOKEN_RE.findall(s)
|
||||||
|
|
|
||||||
133
bench/mine_questions.py
Normal file
133
bench/mine_questions.py
Normal file
|
|
@ -0,0 +1,133 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Mine ground-truth-carrying eval questions from the corpus itself.
|
||||||
|
|
||||||
|
The 75-q curated bench can't resolve any single retrieval lever (each
|
||||||
|
failure class is <=3-5 q, sub the 5pp n=3 noise floor — four
|
||||||
|
hypotheses died on that). Fix the *instrument*: mine questions whose
|
||||||
|
target article is KNOWN by construction (we mined the question from
|
||||||
|
that title), so a lever is graded by deterministic retrieval
|
||||||
|
recall@k — no LLM, no verifier, no n=3 noise, scalable to thousands.
|
||||||
|
|
||||||
|
This is NOT a replacement for bench/qa_questions.txt — that curated,
|
||||||
|
deliberately-adversarial set stays the verifier-honesty/trap gate.
|
||||||
|
Mined questions are answerable-by-construction; they measure the
|
||||||
|
*answerable long tail* of retrieval per failure class.
|
||||||
|
|
||||||
|
v1 = the numeral class (Arabic/ordinal query vs Roman-numeral title —
|
||||||
|
the diagnosed `world war 2`!=`World War II`, `henry the eighth`!=
|
||||||
|
`Henry VIII` miss). ~22K-deep pool in the live shards.
|
||||||
|
|
||||||
|
Pure python+sqlite3, deterministic, no egress, no LLM-for-generation.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import glob
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# Strict Roman set 1..40 (covers monarchs/popes/wars); membership test
|
||||||
|
# avoids English-word collisions ("DID"/"MIX"/"CI" are not in here).
|
||||||
|
_ROMAN = {
|
||||||
|
1: "I", 2: "II", 3: "III", 4: "IV", 5: "V", 6: "VI", 7: "VII",
|
||||||
|
8: "VIII", 9: "IX", 10: "X", 11: "XI", 12: "XII", 13: "XIII",
|
||||||
|
14: "XIV", 15: "XV", 16: "XVI", 17: "XVII", 18: "XVIII", 19: "XIX",
|
||||||
|
20: "XX", 21: "XXI", 22: "XXII", 23: "XXIII", 24: "XXIV", 25: "XXV",
|
||||||
|
26: "XXVI", 27: "XXVII", 28: "XXVIII", 29: "XXIX", 30: "XXX",
|
||||||
|
31: "XXXI", 32: "XXXII", 33: "XXXIII", 34: "XXXIV", 35: "XXXV",
|
||||||
|
36: "XXXVI", 37: "XXXVII", 38: "XXXVIII", 39: "XXXIX", 40: "XL",
|
||||||
|
}
|
||||||
|
_ROMAN_TO_INT = {v: k for k, v in _ROMAN.items()}
|
||||||
|
_ORD = {
|
||||||
|
1: "first", 2: "second", 3: "third", 4: "fourth", 5: "fifth",
|
||||||
|
6: "sixth", 7: "seventh", 8: "eighth", 9: "ninth", 10: "tenth",
|
||||||
|
11: "eleventh", 12: "twelfth", 13: "thirteenth", 14: "fourteenth",
|
||||||
|
15: "fifteenth", 16: "sixteenth", 17: "seventeenth",
|
||||||
|
18: "eighteenth", 19: "nineteenth", 20: "twentieth",
|
||||||
|
}
|
||||||
|
|
||||||
|
# High-precision monarch/pope shape: "<Name> <Roman>" or
|
||||||
|
# "<Name> <Roman> of <Place>". Clean, unambiguous, answerable.
|
||||||
|
_MONARCH = re.compile(
|
||||||
|
r"^([A-Z][a-z]+(?:os|us|er)?) (" + "|".join(_ROMAN.values()) + r")"
|
||||||
|
r"( of [A-Z][a-zA-Z ]+)?$"
|
||||||
|
)
|
||||||
|
_BAD = ("list of", "(disambiguation)", "(album)", "(song)", "(film)",
|
||||||
|
"(band)", "(novel)", "(video game)")
|
||||||
|
|
||||||
|
|
||||||
|
def mine(shards_dir: str, limit: int) -> list[dict]:
|
||||||
|
out: list[dict] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
for db in sorted(glob.glob(f"{shards_dir}/00*.db")):
|
||||||
|
c = sqlite3.connect(db)
|
||||||
|
c.row_factory = sqlite3.Row
|
||||||
|
try:
|
||||||
|
rows = c.execute(
|
||||||
|
"SELECT document_root, title FROM documents "
|
||||||
|
"WHERE title IS NOT NULL"
|
||||||
|
).fetchall()
|
||||||
|
except sqlite3.OperationalError:
|
||||||
|
c.close()
|
||||||
|
continue
|
||||||
|
for r in rows:
|
||||||
|
title = (r["title"] or "").replace("_", " ").strip()
|
||||||
|
tl = title.lower()
|
||||||
|
if any(b in tl for b in _BAD) or title in seen:
|
||||||
|
continue
|
||||||
|
m = _MONARCH.match(title)
|
||||||
|
if not m:
|
||||||
|
continue
|
||||||
|
name, roman, place = m.group(1), m.group(2), (m.group(3) or "")
|
||||||
|
n = _ROMAN_TO_INT[roman]
|
||||||
|
if n not in _ORD: # keep natural ordinal phrasing only
|
||||||
|
continue
|
||||||
|
seen.add(title)
|
||||||
|
# Surface variant: the Arabic/ordinal form a user types,
|
||||||
|
# vs the Roman-numeral title the corpus stores.
|
||||||
|
q = f"who was {name} the {_ORD[n]}{place.lower()}?"
|
||||||
|
out.append({
|
||||||
|
"question": q,
|
||||||
|
"target_title": title,
|
||||||
|
"target_root": r["document_root"],
|
||||||
|
"shard": Path(db).name,
|
||||||
|
})
|
||||||
|
if len(out) >= limit:
|
||||||
|
c.close()
|
||||||
|
return out
|
||||||
|
c.close()
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--shards-dir",
|
||||||
|
default=str(Path.home() / ".arborist" / "shards"))
|
||||||
|
ap.add_argument("--limit", type=int, default=40)
|
||||||
|
ap.add_argument("--cls", default="numeral")
|
||||||
|
a = ap.parse_args()
|
||||||
|
rows = mine(a.shards_dir, a.limit)
|
||||||
|
root = Path(__file__).resolve().parents[1] / "bench"
|
||||||
|
txt = root / f"qa_questions_{a.cls}.txt"
|
||||||
|
mp = root / f"qa_questions_{a.cls}_map.json"
|
||||||
|
hdr = [
|
||||||
|
f"# AUTO-MINED ({a.cls} class) from corpus titles via "
|
||||||
|
"bench/mine_questions.py — ground-truth-carrying.",
|
||||||
|
"# Graded by deterministic retrieval recall@k "
|
||||||
|
"(bench/recall_at_k.py), NOT audit_mode. Not adversarial; "
|
||||||
|
"complements (never replaces) qa_questions.txt.",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
txt.write_text("\n".join(hdr + [r["question"] for r in rows]) + "\n")
|
||||||
|
mp.write_text(json.dumps(rows, ensure_ascii=False, indent=2) + "\n")
|
||||||
|
print(f"mined {len(rows)} {a.cls} questions")
|
||||||
|
print(f" -> {txt}\n -> {mp}")
|
||||||
|
for r in rows[:8]:
|
||||||
|
print(f" Q: {r['question']!r} -> target: {r['target_title']!r}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
43
bench/qa_questions_numeral.txt
Normal file
43
bench/qa_questions_numeral.txt
Normal file
|
|
@ -0,0 +1,43 @@
|
||||||
|
# AUTO-MINED (numeral class) from corpus titles via bench/mine_questions.py — ground-truth-carrying.
|
||||||
|
# Graded by deterministic retrieval recall@k (bench/recall_at_k.py), NOT audit_mode. Not adversarial; complements (never replaces) qa_questions.txt.
|
||||||
|
|
||||||
|
who was Albert the third?
|
||||||
|
who was Ahmed the third?
|
||||||
|
who was Alaric the first?
|
||||||
|
who was Alexander the first of epirus?
|
||||||
|
who was Alexander the second of scotland?
|
||||||
|
who was Alexander the second?
|
||||||
|
who was Alexander the fourth?
|
||||||
|
who was Alyattes the second?
|
||||||
|
who was Afonso the fourth of portugal?
|
||||||
|
who was Alfonso the second of asturias?
|
||||||
|
who was Alfonso the fourth of aragon?
|
||||||
|
who was Alfonso the third?
|
||||||
|
who was Alfonso the fifth?
|
||||||
|
who was Anastasius the second?
|
||||||
|
who was Abbas the second of egypt?
|
||||||
|
who was Charles the fifth?
|
||||||
|
who was Constantius the second?
|
||||||
|
who was Constantine the second of scotland?
|
||||||
|
who was Charles the first of england?
|
||||||
|
who was Frederick the fifth?
|
||||||
|
who was Henry the seventh?
|
||||||
|
who was Mehmed the first?
|
||||||
|
who was Mustafa the first?
|
||||||
|
who was Mieszko the first of poland?
|
||||||
|
who was Malcolm the first of scotland?
|
||||||
|
who was Osman the second?
|
||||||
|
who was Quake the second?
|
||||||
|
who was Stephen the third?
|
||||||
|
who was Oscar the first of sweden?
|
||||||
|
who was Charles the fifteenth of sweden?
|
||||||
|
who was Sviatoslav the first of kiev?
|
||||||
|
who was Catherine the second of russia?
|
||||||
|
who was Childeric the first?
|
||||||
|
who was Rudolph the first of germany?
|
||||||
|
who was Xerxes the second of persia?
|
||||||
|
who was Richard the second of england?
|
||||||
|
who was Gustav the first of sweden?
|
||||||
|
who was Photios the first of constantinople?
|
||||||
|
who was James the fifth of scotland?
|
||||||
|
who was Basarab the first of wallachia?
|
||||||
242
bench/qa_questions_numeral_map.json
Normal file
242
bench/qa_questions_numeral_map.json
Normal file
|
|
@ -0,0 +1,242 @@
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"question": "who was Albert the third?",
|
||||||
|
"target_title": "Albert III",
|
||||||
|
"target_root": "43ff05c3c851e358df34c36d7398fefc53998e5a9cd8654b5c2f8dd9524b9efd",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Ahmed the third?",
|
||||||
|
"target_title": "Ahmed III",
|
||||||
|
"target_root": "2fc91045e60b915d146f1f4a0f0c72e2df2f644efbc299ea27ca40170e6092aa",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Alaric the first?",
|
||||||
|
"target_title": "Alaric I",
|
||||||
|
"target_root": "57e7aa58a4fd36b42a920956929e7c852108f2e3d36c87487d411ec4aa7f6686",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Alexander the first of epirus?",
|
||||||
|
"target_title": "Alexander I of Epirus",
|
||||||
|
"target_root": "db479a5835381d3d94705976dd6adede8fef8e51ea88cc6c3fe4c789e1e1ae35",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Alexander the second of scotland?",
|
||||||
|
"target_title": "Alexander II of Scotland",
|
||||||
|
"target_root": "4d5885f25ab6d7ebd0054dfdb4783f00db96f210ea33cdff2c0dbb0b92b45a13",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Alexander the second?",
|
||||||
|
"target_title": "Alexander II",
|
||||||
|
"target_root": "190bf209efb2f096f9b93c7dd2d0b0be9a64d6aa469d27cf2272664e4f624de2",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Alexander the fourth?",
|
||||||
|
"target_title": "Alexander IV",
|
||||||
|
"target_root": "cc3134f31a0d69f03aa37a5821fc179082f34316e782d464b0685f3ed0e90f0a",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Alyattes the second?",
|
||||||
|
"target_title": "Alyattes II",
|
||||||
|
"target_root": "464c0878c6fcabaf67f37389aa343b180b2762325a9aa916aa0d95dd7529e853",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Afonso the fourth of portugal?",
|
||||||
|
"target_title": "Afonso IV of Portugal",
|
||||||
|
"target_root": "88c2a881546a4cfa59c8eba1e037d2092ab73c9f09feb9524801d64763dfbd3e",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Alfonso the second of asturias?",
|
||||||
|
"target_title": "Alfonso II of Asturias",
|
||||||
|
"target_root": "1b2073183930900d65f1433efb70e8080628b0aa440817a3eb235d70ddf18f61",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Alfonso the fourth of aragon?",
|
||||||
|
"target_title": "Alfonso IV of Aragon",
|
||||||
|
"target_root": "fb96fb9d074fefd80da864c412a9f07cab26b1a217b54a378442143c147316d7",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Alfonso the third?",
|
||||||
|
"target_title": "Alfonso III",
|
||||||
|
"target_root": "555a58a9f7c947f32e18dcd6dfb03fb13ba7cc29f144c5b98f8af087892d9ac0",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Alfonso the fifth?",
|
||||||
|
"target_title": "Alfonso V",
|
||||||
|
"target_root": "c7aaf0276378cefbbb1a44dc1f90b4cb5928411880c7869ea68628f09f1ebde2",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Anastasius the second?",
|
||||||
|
"target_title": "Anastasius II",
|
||||||
|
"target_root": "2cb452df76d8d15494550c8070b51bca9f369647c95574e6a4f9cb441f865c0c",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Abbas the second of egypt?",
|
||||||
|
"target_title": "Abbas II of Egypt",
|
||||||
|
"target_root": "8c3378f3375831933b0c0865074747c264599ceb2650f866c5d61d3635126035",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Charles the fifth?",
|
||||||
|
"target_title": "Charles V",
|
||||||
|
"target_root": "22d8e5eb9bebef543cecda7b33a1f0d300cd24e3a13427c6a297524c9813718f",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Constantius the second?",
|
||||||
|
"target_title": "Constantius II",
|
||||||
|
"target_root": "5f8bd920302cc83ee5aef59abfda0aafd31b6e1cb48cfc4f5c7d4c3e69bedd4a",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Constantine the second of scotland?",
|
||||||
|
"target_title": "Constantine II of Scotland",
|
||||||
|
"target_root": "24e398f6b012f01c0e1b1cb7fad7b4ee2fda8898a15975e5d1c75a4ee77bc3df",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Charles the first of england?",
|
||||||
|
"target_title": "Charles I of England",
|
||||||
|
"target_root": "ad6bc7ccd05e08e3aa9fd3db04dc85e52fc64e54c22532fe1af00ec956bef907",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Frederick the fifth?",
|
||||||
|
"target_title": "Frederick V",
|
||||||
|
"target_root": "ac3d08404d5e30a1642647512b5e5170253fb46577a244390c909dd8679922c0",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Henry the seventh?",
|
||||||
|
"target_title": "Henry VII",
|
||||||
|
"target_root": "91e7879a395c0a43c218964ccc28bcff93ec3e3e141483af3b50e2d596b185df",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Mehmed the first?",
|
||||||
|
"target_title": "Mehmed I",
|
||||||
|
"target_root": "9a526b048fe00f930da0b4128eb30dde55ebfee10490cea2322768c8d1f18023",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Mustafa the first?",
|
||||||
|
"target_title": "Mustafa I",
|
||||||
|
"target_root": "6d4d3e3d401f3b1651fc3d779d71b8084a5ff69ba2f95516dcefec63f8e6fd20",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Mieszko the first of poland?",
|
||||||
|
"target_title": "Mieszko I of Poland",
|
||||||
|
"target_root": "fee8e212dc5d6439d1b791eea1bc997d825a4b4abf3bfeeef2cfae57e948d077",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Malcolm the first of scotland?",
|
||||||
|
"target_title": "Malcolm I of Scotland",
|
||||||
|
"target_root": "1f1983a45ed9b9cfa49425103fed17a1ca0e36284189525a31a9bc73a14d2181",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Osman the second?",
|
||||||
|
"target_title": "Osman II",
|
||||||
|
"target_root": "71da21f6a76233de3ac45683f8d8b419d9962581cba0f2fb0cf81093b7e41b0a",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Quake the second?",
|
||||||
|
"target_title": "Quake II",
|
||||||
|
"target_root": "86b1189f25d0fe612423ef88be44885da48f115c27b64024078c2fd257acb66a",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Stephen the third?",
|
||||||
|
"target_title": "Stephen III",
|
||||||
|
"target_root": "07d31c9abc8c1e8626997b248c82592b09dff67e96069c31c90b95230407e4c2",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Oscar the first of sweden?",
|
||||||
|
"target_title": "Oscar I of Sweden",
|
||||||
|
"target_root": "09ab96881d295f59dd8131a2d9d15bc2d633bd9e9606c1c27d7e1c7c889cfec0",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Charles the fifteenth of sweden?",
|
||||||
|
"target_title": "Charles XV of Sweden",
|
||||||
|
"target_root": "381a3cc80ba62a7454ea87bfd56ddefb27229491e3926a88d84968fd791485bb",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Sviatoslav the first of kiev?",
|
||||||
|
"target_title": "Sviatoslav I of Kiev",
|
||||||
|
"target_root": "99066a1e171e3a3924c115f6010a306743520730d9a0e0ed4f359c0e7239a7dd",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Catherine the second of russia?",
|
||||||
|
"target_title": "Catherine II of Russia",
|
||||||
|
"target_root": "d7a6a6d1f63b0ddd4977b2c9203fdd6659b0fdcadbb43096910c028ea492a42b",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Childeric the first?",
|
||||||
|
"target_title": "Childeric I",
|
||||||
|
"target_root": "a21db476b6e4ece9aa1cc7da6b68b199474f82c297eaa3304fd3713d0b7ec682",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Rudolph the first of germany?",
|
||||||
|
"target_title": "Rudolph I of Germany",
|
||||||
|
"target_root": "7f882ac84cf32819f3af2dda446d18ad7cd818e94b201c56a3a89a0d78bf9161",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Xerxes the second of persia?",
|
||||||
|
"target_title": "Xerxes II of Persia",
|
||||||
|
"target_root": "edafe8586c5666f8fb38a499ac73f0d954fba6c2d3b869610de16ba33bcc8a3f",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Richard the second of england?",
|
||||||
|
"target_title": "Richard II of England",
|
||||||
|
"target_root": "d08b7f5eb56fb3352a77facfbc204b53b1a6cef4215d15ac7a2a97f3a30dda8d",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Gustav the first of sweden?",
|
||||||
|
"target_title": "Gustav I of Sweden",
|
||||||
|
"target_root": "4ff24ede9c8fc1bd401972bef9a3a8e4a4614b8ad289027e9e51b00f16631a04",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Photios the first of constantinople?",
|
||||||
|
"target_title": "Photios I of Constantinople",
|
||||||
|
"target_root": "1309569c1cb8f37ff5f7d2cfc00080efb5f6846e54b1a45c07357e1b0a2445f2",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was James the fifth of scotland?",
|
||||||
|
"target_title": "James V of Scotland",
|
||||||
|
"target_root": "6b0e3af072a252daaad838123f5b2c2db3b0ac91bd7833faf17ff4f0a488fbe0",
|
||||||
|
"shard": "000.db"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question": "who was Basarab the first of wallachia?",
|
||||||
|
"target_title": "Basarab I of Wallachia",
|
||||||
|
"target_root": "6f543c2b310d31ef08d43e4454a69eb2294ffc412df4cb26bafd92ea9841ca2e",
|
||||||
|
"shard": "000.db"
|
||||||
|
}
|
||||||
|
]
|
||||||
80
bench/recall_at_k.py
Normal file
80
bench/recall_at_k.py
Normal file
|
|
@ -0,0 +1,80 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Deterministic retrieval recall@k on a mined ground-truth fixture.
|
||||||
|
|
||||||
|
For each mined question (surface-variant form), run retrieval ONLY
|
||||||
|
(`query --dry-run` — no LLM) and check whether the KNOWN target
|
||||||
|
article is in the top-k sources. Recall@k is a hard deterministic
|
||||||
|
number on N ground-truth questions — no verifier, no n=3 LLM noise,
|
||||||
|
no 5pp floor. This is the instrument a retrieval lever (e.g.
|
||||||
|
numeral-fold) is measured against: lever recall@k minus baseline
|
||||||
|
recall@k, on the same mined fixture.
|
||||||
|
|
||||||
|
usage: recall_at_k.py qa_questions_numeral_map.json [--k 8] [--conc 4]
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import concurrent.futures as cf
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
ARB = ROOT / ".venv" / "bin" / "arborist"
|
||||||
|
SHARDS = Path.home() / ".arborist" / "shards"
|
||||||
|
|
||||||
|
|
||||||
|
def _norm(t: str) -> str:
|
||||||
|
return (t or "").replace("_", " ").strip().casefold()
|
||||||
|
|
||||||
|
|
||||||
|
def probe(item: dict, k: int) -> tuple[bool, int]:
|
||||||
|
"""Return (target_in_topk, rank_or_-1). Retrieval only."""
|
||||||
|
try:
|
||||||
|
out = subprocess.run(
|
||||||
|
[str(ARB), "--shards-dir", str(SHARDS), "query", "--dry-run",
|
||||||
|
"--json", "--top-k", str(k), "--answer-mode", "claim_lattice",
|
||||||
|
item["question"]],
|
||||||
|
capture_output=True, text=True, timeout=120,
|
||||||
|
).stdout
|
||||||
|
d = json.loads(out)
|
||||||
|
except Exception:
|
||||||
|
return (False, -1)
|
||||||
|
tgt = _norm(item["target_title"])
|
||||||
|
titles = [_norm(s.get("title") or "") for s in (d.get("sources") or [])]
|
||||||
|
return (tgt in titles, titles.index(tgt) if tgt in titles else -1)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("map_json")
|
||||||
|
ap.add_argument("--k", type=int, default=8)
|
||||||
|
ap.add_argument("--conc", type=int, default=4)
|
||||||
|
a = ap.parse_args()
|
||||||
|
items = json.loads(Path(a.map_json).read_text())
|
||||||
|
hits = 0
|
||||||
|
ranks: list[int] = []
|
||||||
|
misses: list[str] = []
|
||||||
|
with cf.ThreadPoolExecutor(max_workers=a.conc) as ex:
|
||||||
|
for it, (ok, rank) in zip(
|
||||||
|
items, ex.map(lambda i: probe(i, a.k), items)
|
||||||
|
):
|
||||||
|
if ok:
|
||||||
|
hits += 1
|
||||||
|
ranks.append(rank)
|
||||||
|
else:
|
||||||
|
misses.append(f"{it['question']!r} -> {it['target_title']!r}")
|
||||||
|
n = len(items)
|
||||||
|
print(f"recall@{a.k}: {hits}/{n} = {hits/n:.0%} "
|
||||||
|
f"(deterministic, no LLM — the instrument)")
|
||||||
|
if ranks:
|
||||||
|
print(f" of the hits, mean rank: {sum(ranks)/len(ranks):.1f} "
|
||||||
|
f"(0=top); rank-1 count: {sum(1 for r in ranks if r == 0)}")
|
||||||
|
print(f" MISSES ({len(misses)}) — target article never surfaced:")
|
||||||
|
for m in misses[:25]:
|
||||||
|
print(f" {m}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
80
tests/test_numeral_fold.py
Normal file
80
tests/test_numeral_fold.py
Normal file
|
|
@ -0,0 +1,80 @@
|
||||||
|
"""Numeral-fold: ordinal-word <-> multi-char Roman, retrieval-side.
|
||||||
|
|
||||||
|
Measured 2026-05-18: mined ground-truth recall@8 22/40 -> 30/40
|
||||||
|
(+20pp) when an ordinal-word query ("Alexander the second") can
|
||||||
|
reach a Roman-numeral title ("Alexander II"). Same additive+
|
||||||
|
symmetric discipline as `_hyphen_fold_variants` (#000007).
|
||||||
|
|
||||||
|
Tested through the REAL path (FakeSource -> ingest -> query() ->
|
||||||
|
real `_Hit`), never a hand-built object — the discipline the
|
||||||
|
reverted disambiguation v1 violated (false-green on the wrong type).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Iterator
|
||||||
|
|
||||||
|
from arborist.qa.query import _numeral_fold_variants, _title_query_tokens
|
||||||
|
|
||||||
|
|
||||||
|
def test_variant_is_symmetric_additive_and_collision_free():
|
||||||
|
assert _numeral_fold_variants("who was Alexander the second?") == {"ii"}
|
||||||
|
assert _numeral_fold_variants("Alexander II") == {"second"}
|
||||||
|
# non-numeral queries: provably no effect (additive safety)
|
||||||
|
assert _numeral_fold_variants("what is the capital of france?") == set()
|
||||||
|
# strict set → English words that look Roman-ish never fold
|
||||||
|
assert _numeral_fold_variants("the civil war did mix things") == set()
|
||||||
|
# single-char Romans intentionally absent (universal len>1 filter)
|
||||||
|
assert _numeral_fold_variants("Charles V") == set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_title_query_tokens_now_overlaps_ordinal_and_roman():
|
||||||
|
q = _title_query_tokens("who was Alexander the second?")
|
||||||
|
t = _title_query_tokens("Alexander II")
|
||||||
|
# was {alexander} (1) before the fold → below title-breadth.
|
||||||
|
assert {"alexander", "ii", "second"} <= (q & t)
|
||||||
|
|
||||||
|
|
||||||
|
def test_query_retrieves_roman_title_for_ordinal_word(tmp_path):
|
||||||
|
"""End-to-end through query(): an ordinal-word question must now
|
||||||
|
surface its Roman-numeral-titled article."""
|
||||||
|
from arborist.qa.client import StubClient
|
||||||
|
from arborist.document import Document
|
||||||
|
from arborist.ingest import ingest_source
|
||||||
|
from arborist.qa.query import DEFAULT_QUERY_POLICY, query
|
||||||
|
from arborist.source import Source
|
||||||
|
from arborist.store import connect
|
||||||
|
|
||||||
|
class FakeSource(Source):
|
||||||
|
source_type = "test"
|
||||||
|
|
||||||
|
def __init__(self, ds):
|
||||||
|
self.ds = ds
|
||||||
|
|
||||||
|
def iter_documents(self) -> Iterator[Document]:
|
||||||
|
yield from self.ds
|
||||||
|
|
||||||
|
shard = tmp_path / "shard.db"
|
||||||
|
c = connect(shard)
|
||||||
|
try:
|
||||||
|
ingest_source(c, FakeSource([
|
||||||
|
Document(uri="t://1",
|
||||||
|
content="Alexander II of Russia was Emperor of Russia. "
|
||||||
|
"Alexander II enacted major reforms. " * 12,
|
||||||
|
source_type="test", title="Alexander II"),
|
||||||
|
Document(uri="t://2",
|
||||||
|
content="Unrelated article about gardening tools. " * 12,
|
||||||
|
source_type="test", title="Gardening"),
|
||||||
|
]))
|
||||||
|
finally:
|
||||||
|
c.close()
|
||||||
|
r = query(
|
||||||
|
question="who was Alexander the second?",
|
||||||
|
qa_db=tmp_path / "qa.db",
|
||||||
|
chat_client=StubClient(answer="Alexander II was Emperor of Russia. [E1]\n"),
|
||||||
|
model_id="stub",
|
||||||
|
single_db=shard,
|
||||||
|
policy=dict(DEFAULT_QUERY_POLICY, answer_mode="claim_lattice_pointer"),
|
||||||
|
)
|
||||||
|
titles = {(s.get("title") or "") for s in (r.get("sources") or [])}
|
||||||
|
assert r["status"] != "error"
|
||||||
|
assert "Alexander II" in titles # the Roman-titled article surfaced
|
||||||
Loading…
Add table
Add a link
Reference in a new issue