feat(retrieval): accent-fold (+30pp recall@1) + fold-search factory hardening
Second MEASURED fold-search win, and the instrument correcting my own
premature call. accent-fold ON vs OFF on the mined accent fixture:
recall@1 55% -> 85% (+30pp), rank-1 22/40 -> 34/40. recall@8 was
flat (95->98) — a too-lenient k nearly got a real lever wrongly
reverted; @1/@3 is the resolution that drives primary-source
selection. _accent_fold_variants: ASCII-fold then re-tokenise so a
diacritic title ("Béla Bartók", which _TITLE_TOKEN_RE otherwise
fragments to junk) matches the ASCII form a user types. Additive+
symmetric, no-op on pure-ASCII (zero effect on non-accent
queries/titles), mirrors _hyphen_fold_variants (#000007).
Also fixes a defect I shipped in a3ac653: an orphaned duplicate
body left as dead code after `return base` in _title_query_tokens
(unreachable — numeral-fold behaviour/measurement were valid — but
cruft; removed).
Fold-search factory, fanned out across the full survey backlog
(deterministic recall, no LLM, parallel — serial-by-caution was
halting in disguise):
- recall_at_k.py: returns rank -> recall@1/@3/@k from one retrieval
(verified offline). A coarse k hides rank-only lifts.
- mine_questions.py: numeral/accent/hyphen/honorific/amp/brit
ground-truth classes; fixtures committed.
- Measured @1 headroom verdicts: accent SHIP (this commit);
honorific 45% / brit 50% = real headroom (build next); hyphen
90% = existing #000007 already delivers, NOTHING to build (the
measure-the-unmeasured-thing check pays off); amp 82% = no fold
needed (prevalence-overranked, instrument kills it cheaply).
CLAUDE.md bench-maxing: two measured lessons codified — report
recall@1/@3/@k (a lenient k hides rank lifts; prevalence != miss-
rate), and fan out independent measurements (serial-by-caution is
halting). Full suite 2488 passed, 0 regressions (accent-fold is
hot-path in _title_query_tokens); real-path test (FakeSource->
ingest->query()->real _Hit).
This commit is contained in:
parent
98a4d61de9
commit
b573c592d8
17 changed files with 1684 additions and 291 deletions
83
tests/test_accent_fold.py
Normal file
83
tests/test_accent_fold.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
"""Accent-fold: ASCII-folded variants for diacritic text, retrieval-side.
|
||||
|
||||
Measured 2026-05-18 on the mined ground-truth fixture: recall@1
|
||||
55% -> 85% (+30pp), rank-1 22/40 -> 34/40. recall@8 was a near-
|
||||
miss-revert artifact (95->98, noise) — a too-lenient k hid a
|
||||
rank-only lift; @1/@3 is the resolution that drives primary-source
|
||||
selection. Same additive+symmetric discipline as
|
||||
`_hyphen_fold_variants` (#000007) / `_numeral_fold_variants`.
|
||||
|
||||
Tested through the REAL path (FakeSource -> ingest -> query() ->
|
||||
real `_Hit`), not a hand-built object — the discipline the reverted
|
||||
disambiguation v1 violated.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
from arborist.qa.query import _accent_fold_variants, _title_query_tokens
|
||||
|
||||
|
||||
def test_variant_is_additive_symmetric_and_noop_on_ascii():
|
||||
assert _accent_fold_variants("Béla Bartók") == {"bela", "bartok"}
|
||||
assert _accent_fold_variants("André-Marie Ampère") >= {"andre", "ampere"}
|
||||
# pure ASCII -> folds to itself -> empty -> zero effect (the
|
||||
# additive-safety invariant that keeps non-accent queries intact)
|
||||
assert _accent_fold_variants("what is the capital of france?") == set()
|
||||
assert _accent_fold_variants("who painted the mona lisa?") == set()
|
||||
|
||||
|
||||
def test_title_query_tokens_bridges_ascii_query_to_accented_title():
|
||||
q = _title_query_tokens("what is Bela Bartok?")
|
||||
t = _title_query_tokens("Béla Bartók")
|
||||
# the accented title otherwise fragments ("Béla"->"B","la") and
|
||||
# never overlaps the ASCII form; the fold restores the bridge.
|
||||
assert {"bela", "bartok"} <= (q & t)
|
||||
# plain ASCII query unaffected
|
||||
assert _title_query_tokens("who painted the mona lisa?") == {
|
||||
"painted", "mona", "lisa"
|
||||
}
|
||||
|
||||
|
||||
def test_query_retrieves_accented_title_for_ascii_question(tmp_path):
|
||||
from arborist.qa.client import StubClient
|
||||
from arborist.document import Document
|
||||
from arborist.ingest import ingest_source
|
||||
from arborist.qa.query import DEFAULT_QUERY_POLICY, query
|
||||
from arborist.source import Source
|
||||
from arborist.store import connect
|
||||
|
||||
class FakeSource(Source):
|
||||
source_type = "test"
|
||||
|
||||
def __init__(self, ds):
|
||||
self.ds = ds
|
||||
|
||||
def iter_documents(self) -> Iterator[Document]:
|
||||
yield from self.ds
|
||||
|
||||
shard = tmp_path / "shard.db"
|
||||
c = connect(shard)
|
||||
try:
|
||||
ingest_source(c, FakeSource([
|
||||
Document(uri="t://1",
|
||||
content="Béla Bartók was a Hungarian composer. "
|
||||
"Béla Bartók pioneered ethnomusicology. " * 12,
|
||||
source_type="test", title="Béla Bartók"),
|
||||
Document(uri="t://2",
|
||||
content="Unrelated article about gardening. " * 12,
|
||||
source_type="test", title="Gardening"),
|
||||
]))
|
||||
finally:
|
||||
c.close()
|
||||
r = query(
|
||||
question="what is Bela Bartok?",
|
||||
qa_db=tmp_path / "qa.db",
|
||||
chat_client=StubClient(answer="Béla Bartók was a composer. [E1]\n"),
|
||||
model_id="stub",
|
||||
single_db=shard,
|
||||
policy=dict(DEFAULT_QUERY_POLICY, answer_mode="claim_lattice_pointer"),
|
||||
)
|
||||
titles = {(s.get("title") or "") for s in (r.get("sources") or [])}
|
||||
assert r["status"] != "error"
|
||||
assert "Béla Bartók" in titles # accented title surfaced from ASCII q
|
||||
Loading…
Add table
Add a link
Reference in a new issue