feat(retrieval): numeral-fold (ordinal-word <-> Roman) + mined ground-truth eval instrument
The first MEASURED, above-noise retrieval win this thread. The 75-q
n=3 audit_mode bench couldn't resolve any single lever (every failure
class <=3-5 q, sub the 5pp floor — four hypotheses died there). Fix
the instrument, not just the lever:
- bench/mine_questions.py + bench/recall_at_k.py: mine questions from
corpus titles (ground-truth target known by construction), grade by
deterministic retrieval recall@k via `query --dry-run` — no LLM, no
verifier, no n=3 noise, scalable to the 22K-deep numeral pool. The
curated qa_questions.txt stays the separate verifier-honesty/trap
gate; mined fixtures measure the answerable long tail per class.
- _numeral_fold_variants in query.py: ordinal-word ("Alexander the
second") <-> multi-char Roman ("Alexander II"), additive+symmetric,
unioned into _title_query_tokens exactly like _hyphen_fold_variants
(#000007). Strict 2..40 Roman set → no English-word collision;
single-char Romans (I/V/X) intentionally out of scope (universal
len>1 token filter — stated before building, ~4 of 10 residual
misses).
Measured on the mined numeral fixture: recall@8 22/40 (55%) -> 30/40
(75%), +20pp; 20 hits now rank-1. Discipline applied end to end:
measured-first, mirrored precedent, full-suite regression run (2482
passed, 0 regressions — numeral-fold is hot-path in
_title_query_tokens), real-path test (FakeSource->ingest->query()->
real _Hit, not a hand-built object), measured-after on a noise-free
instrument. The ~6 multi-char residual misses are a different
downstream cause the instrument now exposes for future iteration.
This commit is contained in:
parent
5d43fdc037
commit
a3ac6539c1
6 changed files with 635 additions and 0 deletions
80
tests/test_numeral_fold.py
Normal file
80
tests/test_numeral_fold.py
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
"""Numeral-fold: ordinal-word <-> multi-char Roman, retrieval-side.
|
||||
|
||||
Measured 2026-05-18: mined ground-truth recall@8 22/40 -> 30/40
|
||||
(+20pp) when an ordinal-word query ("Alexander the second") can
|
||||
reach a Roman-numeral title ("Alexander II"). Same additive+
|
||||
symmetric discipline as `_hyphen_fold_variants` (#000007).
|
||||
|
||||
Tested through the REAL path (FakeSource -> ingest -> query() ->
|
||||
real `_Hit`), never a hand-built object — the discipline the
|
||||
reverted disambiguation v1 violated (false-green on the wrong type).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
from arborist.qa.query import _numeral_fold_variants, _title_query_tokens
|
||||
|
||||
|
||||
def test_variant_is_symmetric_additive_and_collision_free():
|
||||
assert _numeral_fold_variants("who was Alexander the second?") == {"ii"}
|
||||
assert _numeral_fold_variants("Alexander II") == {"second"}
|
||||
# non-numeral queries: provably no effect (additive safety)
|
||||
assert _numeral_fold_variants("what is the capital of france?") == set()
|
||||
# strict set → English words that look Roman-ish never fold
|
||||
assert _numeral_fold_variants("the civil war did mix things") == set()
|
||||
# single-char Romans intentionally absent (universal len>1 filter)
|
||||
assert _numeral_fold_variants("Charles V") == set()
|
||||
|
||||
|
||||
def test_title_query_tokens_now_overlaps_ordinal_and_roman():
|
||||
q = _title_query_tokens("who was Alexander the second?")
|
||||
t = _title_query_tokens("Alexander II")
|
||||
# was {alexander} (1) before the fold → below title-breadth.
|
||||
assert {"alexander", "ii", "second"} <= (q & t)
|
||||
|
||||
|
||||
def test_query_retrieves_roman_title_for_ordinal_word(tmp_path):
|
||||
"""End-to-end through query(): an ordinal-word question must now
|
||||
surface its Roman-numeral-titled article."""
|
||||
from arborist.qa.client import StubClient
|
||||
from arborist.document import Document
|
||||
from arborist.ingest import ingest_source
|
||||
from arborist.qa.query import DEFAULT_QUERY_POLICY, query
|
||||
from arborist.source import Source
|
||||
from arborist.store import connect
|
||||
|
||||
class FakeSource(Source):
|
||||
source_type = "test"
|
||||
|
||||
def __init__(self, ds):
|
||||
self.ds = ds
|
||||
|
||||
def iter_documents(self) -> Iterator[Document]:
|
||||
yield from self.ds
|
||||
|
||||
shard = tmp_path / "shard.db"
|
||||
c = connect(shard)
|
||||
try:
|
||||
ingest_source(c, FakeSource([
|
||||
Document(uri="t://1",
|
||||
content="Alexander II of Russia was Emperor of Russia. "
|
||||
"Alexander II enacted major reforms. " * 12,
|
||||
source_type="test", title="Alexander II"),
|
||||
Document(uri="t://2",
|
||||
content="Unrelated article about gardening tools. " * 12,
|
||||
source_type="test", title="Gardening"),
|
||||
]))
|
||||
finally:
|
||||
c.close()
|
||||
r = query(
|
||||
question="who was Alexander the second?",
|
||||
qa_db=tmp_path / "qa.db",
|
||||
chat_client=StubClient(answer="Alexander II was Emperor of Russia. [E1]\n"),
|
||||
model_id="stub",
|
||||
single_db=shard,
|
||||
policy=dict(DEFAULT_QUERY_POLICY, answer_mode="claim_lattice_pointer"),
|
||||
)
|
||||
titles = {(s.get("title") or "") for s in (r.get("sources") or [])}
|
||||
assert r["status"] != "error"
|
||||
assert "Alexander II" in titles # the Roman-titled article surfaced
|
||||
Loading…
Add table
Add a link
Reference in a new issue