russell@unturf.com
a3ac6539c1
The first MEASURED, above-noise retrieval win this thread. The 75-q
n=3 audit_mode bench couldn't resolve any single lever (every failure
class <=3-5 q, sub the 5pp floor — four hypotheses died there). Fix
the instrument, not just the lever:
- bench/mine_questions.py + bench/recall_at_k.py: mine questions from
corpus titles (ground-truth target known by construction), grade by
deterministic retrieval recall@k via `query --dry-run` — no LLM, no
verifier, no n=3 noise, scalable to the 22K-deep numeral pool. The
curated qa_questions.txt stays the separate verifier-honesty/trap
gate; mined fixtures measure the answerable long tail per class.
- _numeral_fold_variants in query.py: ordinal-word ("Alexander the
second") <-> multi-char Roman ("Alexander II"), additive+symmetric,
unioned into _title_query_tokens exactly like _hyphen_fold_variants
(#000007). Strict 2..40 Roman set → no English-word collision;
single-char Romans (I/V/X) intentionally out of scope (universal
len>1 token filter — stated before building, ~4 of 10 residual
misses).
Measured on the mined numeral fixture: recall@8 22/40 (55%) -> 30/40
(75%), +20pp; 20 hits now rank-1. Discipline applied end to end:
measured-first, mirrored precedent, full-suite regression run (2482
passed, 0 regressions — numeral-fold is hot-path in
_title_query_tokens), real-path test (FakeSource->ingest->query()->
real _Hit, not a hand-built object), measured-after on a noise-free
instrument. The ~6 multi-char residual misses are a different
downstream cause the instrument now exposes for future iteration.
80 lines
3.2 KiB
Python
80 lines
3.2 KiB
Python
"""Numeral-fold: ordinal-word <-> multi-char Roman, retrieval-side.
|
|
|
|
Measured 2026-05-18: mined ground-truth recall@8 22/40 -> 30/40
|
|
(+20pp) when an ordinal-word query ("Alexander the second") can
|
|
reach a Roman-numeral title ("Alexander II"). Same additive+
|
|
symmetric discipline as `_hyphen_fold_variants` (#000007).
|
|
|
|
Tested through the REAL path (FakeSource -> ingest -> query() ->
|
|
real `_Hit`), never a hand-built object — the discipline the
|
|
reverted disambiguation v1 violated (false-green on the wrong type).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Iterator
|
|
|
|
from arborist.qa.query import _numeral_fold_variants, _title_query_tokens
|
|
|
|
|
|
def test_variant_is_symmetric_additive_and_collision_free():
|
|
assert _numeral_fold_variants("who was Alexander the second?") == {"ii"}
|
|
assert _numeral_fold_variants("Alexander II") == {"second"}
|
|
# non-numeral queries: provably no effect (additive safety)
|
|
assert _numeral_fold_variants("what is the capital of france?") == set()
|
|
# strict set → English words that look Roman-ish never fold
|
|
assert _numeral_fold_variants("the civil war did mix things") == set()
|
|
# single-char Romans intentionally absent (universal len>1 filter)
|
|
assert _numeral_fold_variants("Charles V") == set()
|
|
|
|
|
|
def test_title_query_tokens_now_overlaps_ordinal_and_roman():
|
|
q = _title_query_tokens("who was Alexander the second?")
|
|
t = _title_query_tokens("Alexander II")
|
|
# was {alexander} (1) before the fold → below title-breadth.
|
|
assert {"alexander", "ii", "second"} <= (q & t)
|
|
|
|
|
|
def test_query_retrieves_roman_title_for_ordinal_word(tmp_path):
|
|
"""End-to-end through query(): an ordinal-word question must now
|
|
surface its Roman-numeral-titled article."""
|
|
from arborist.qa.client import StubClient
|
|
from arborist.document import Document
|
|
from arborist.ingest import ingest_source
|
|
from arborist.qa.query import DEFAULT_QUERY_POLICY, query
|
|
from arborist.source import Source
|
|
from arborist.store import connect
|
|
|
|
class FakeSource(Source):
|
|
source_type = "test"
|
|
|
|
def __init__(self, ds):
|
|
self.ds = ds
|
|
|
|
def iter_documents(self) -> Iterator[Document]:
|
|
yield from self.ds
|
|
|
|
shard = tmp_path / "shard.db"
|
|
c = connect(shard)
|
|
try:
|
|
ingest_source(c, FakeSource([
|
|
Document(uri="t://1",
|
|
content="Alexander II of Russia was Emperor of Russia. "
|
|
"Alexander II enacted major reforms. " * 12,
|
|
source_type="test", title="Alexander II"),
|
|
Document(uri="t://2",
|
|
content="Unrelated article about gardening tools. " * 12,
|
|
source_type="test", title="Gardening"),
|
|
]))
|
|
finally:
|
|
c.close()
|
|
r = query(
|
|
question="who was Alexander the second?",
|
|
qa_db=tmp_path / "qa.db",
|
|
chat_client=StubClient(answer="Alexander II was Emperor of Russia. [E1]\n"),
|
|
model_id="stub",
|
|
single_db=shard,
|
|
policy=dict(DEFAULT_QUERY_POLICY, answer_mode="claim_lattice_pointer"),
|
|
)
|
|
titles = {(s.get("title") or "") for s in (r.get("sources") or [])}
|
|
assert r["status"] != "error"
|
|
assert "Alexander II" in titles # the Roman-titled article surfaced
|