feat(retrieval): honorific-fold + brit-fold — fold-search batch 3 (both measured wins)
Two more MEASURED fold-search wins on mined ground-truth fixtures
(deterministic recall, no LLM), both lifting at @1/@3/@8 (not
coarse-k artifacts):
honorific (Mt/St/Dr <-> Mount/Saint/Doctor): recall@1 45% -> 75%
(+30pp), @8 62% -> 85%, misses 15 -> 6
brit (British <-> American spelling): recall@1 50% -> 70%
(+20pp), @8 70% -> 85%, misses 12 -> 6
Both _*_fold_variants are additive+symmetric, strict closed sets
(no English-word collision), no-op outside their class (verified
independent: brit no-ops on honorific titles & vice versa), unioned
into _title_query_tokens beside hyphen(#000007)/numeral/accent.
Full suite 2488 passed, 0 regressions (hot-path); real-path tests
(FakeSource->ingest->query()->real _Hit).
Fold-search FINAL across the survey backlog, ranked by MEASURED @1
headroom (not prevalence — the instrument's job):
SHIPPED: numeral (a3ac653) accent (b573c59) honorific brit (here)
NO-BUILD: hyphen — existing #000007 already delivers 90%@1
(the measure-the-unmeasured-thing check pays off)
NO-BUILD: amp — 82%@1 with no fold (prevalence-overranked;
instrument killed it cheaply, like digit-ordinal pre-build)
Net: 4 deterministic retrieval wins + a reusable mined-recall
instrument + the discipline codified in CLAUDE.md, from a goal that
4 prior hypotheses died on because the bench couldn't measure them.
This commit is contained in:
parent
b573c592d8
commit
6573080284
2 changed files with 161 additions and 0 deletions
|
|
@ -232,6 +232,57 @@ def _accent_fold_variants(s: str) -> set[str]:
|
|||
}
|
||||
|
||||
|
||||
# Honorific-fold: a user types "Mt/St/Dr Everest"; the title spells
|
||||
# "Mount/Saint/Doctor Everest" (or vice versa). Measured 2026-05-18
|
||||
# (mined ground-truth fixture, no fold): recall@1 only 45% / @8 62%,
|
||||
# 15/40 misses — large headroom, no existing fold. Same additive+
|
||||
# symmetric discipline as the numeral/accent folds. Bidirectional so
|
||||
# either surface form reaches the other; strict closed set (no
|
||||
# English-word collision). "st" maps to {saint} only — "street" is
|
||||
# deliberately excluded: the measured class is honorific-titled and
|
||||
# folding street here would add noise for ~zero recall (additive but
|
||||
# precision-aware, the single-char-Roman lesson).
|
||||
_HONOR_FOLD = {
|
||||
"mount": "mt", "saint": "st", "doctor": "dr", "fort": "ft",
|
||||
"general": "gen", "president": "pres", "captain": "capt",
|
||||
"senator": "sen", "mister": "mr", "professor": "prof",
|
||||
}
|
||||
_HONOR_FOLD.update({v: k for k, v in _HONOR_FOLD.items()})
|
||||
|
||||
|
||||
def _honorific_fold_variants(s: str) -> set[str]:
|
||||
out: set[str] = set()
|
||||
for tok in _TITLE_TOKEN_RE.findall(s):
|
||||
v = _HONOR_FOLD.get(tok.lower())
|
||||
if v and len(v) > 1 and v not in _TITLE_STOPWORDS:
|
||||
out.add(v)
|
||||
return out
|
||||
|
||||
|
||||
# British<->American spelling fold. Measured 2026-05-18 (mined
|
||||
# ground-truth, no fold): recall@1 50% / @8 70%, 12/40 misses —
|
||||
# real headroom, no existing fold. Token-level (the British form
|
||||
# IS the title token: "Labour", "Organisation", "Centre"). Same
|
||||
# additive+symmetric discipline; strict closed set.
|
||||
_BRIT_FOLD = {
|
||||
"colour": "color", "honour": "honor", "behaviour": "behavior",
|
||||
"organisation": "organization", "defence": "defense",
|
||||
"centre": "center", "theatre": "theater", "catalogue": "catalog",
|
||||
"programme": "program", "labour": "labor", "favour": "favor",
|
||||
"licence": "license", "neighbour": "neighbor",
|
||||
}
|
||||
_BRIT_FOLD.update({v: k for k, v in _BRIT_FOLD.items()})
|
||||
|
||||
|
||||
def _brit_fold_variants(s: str) -> set[str]:
|
||||
out: set[str] = set()
|
||||
for tok in _TITLE_TOKEN_RE.findall(s):
|
||||
v = _BRIT_FOLD.get(tok.lower())
|
||||
if v and len(v) > 1 and v not in _TITLE_STOPWORDS:
|
||||
out.add(v)
|
||||
return out
|
||||
|
||||
|
||||
def _title_query_tokens(s: str) -> set[str]:
|
||||
base = {
|
||||
t.lower()
|
||||
|
|
@ -255,6 +306,11 @@ def _title_query_tokens(s: str) -> set[str]:
|
|||
# `s` is already ASCII (measured 2026-05-18; fold-search #1,
|
||||
# 8.1% of titles). See `_accent_fold_variants`.
|
||||
base |= _accent_fold_variants(s)
|
||||
# Honorific-fold: Mt/St/Dr <-> Mount/Saint/Doctor, same additive+
|
||||
# symmetric discipline (measured 2026-05-18; baseline recall@1 45%).
|
||||
base |= _honorific_fold_variants(s)
|
||||
# British<->American spelling, same discipline (baseline @1 50%).
|
||||
base |= _brit_fold_variants(s)
|
||||
return base
|
||||
|
||||
|
||||
|
|
|
|||
105
tests/test_honorific_brit_fold.py
Normal file
105
tests/test_honorific_brit_fold.py
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
"""Honorific + British-spelling folds, retrieval-side.
|
||||
|
||||
Measured 2026-05-18 on mined ground-truth fixtures:
|
||||
honorific recall@1 45% -> 75% (+30pp), misses 15 -> 6
|
||||
brit recall@1 50% -> 70% (+20pp), misses 12 -> 6
|
||||
Both lift at @1/@3/@8 (not coarse-k artifacts). Same additive+
|
||||
symmetric discipline as `_hyphen_fold_variants` (#000007) /
|
||||
numeral / accent.
|
||||
|
||||
Real path (FakeSource -> ingest -> query() -> real `_Hit`), never a
|
||||
hand-built object — the discipline the reverted disambiguation v1
|
||||
violated (false-green on the wrong type).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
from arborist.qa.query import (
|
||||
_brit_fold_variants,
|
||||
_honorific_fold_variants,
|
||||
_title_query_tokens,
|
||||
)
|
||||
|
||||
|
||||
def test_variants_additive_symmetric_noop_and_independent():
|
||||
assert _honorific_fold_variants("what is St Petersburg?") == {"saint"}
|
||||
assert _honorific_fold_variants("Saint Petersburg") == {"st"}
|
||||
assert _brit_fold_variants("what is labor economics?") == {"labour"}
|
||||
assert _brit_fold_variants("Labour economics") == {"labor"}
|
||||
# additive safety: pure non-class queries -> empty
|
||||
assert _honorific_fold_variants("who painted the mona lisa?") == set()
|
||||
assert _brit_fold_variants("who painted the mona lisa?") == set()
|
||||
# independence: brit is a no-op on honorific titles & vice versa
|
||||
assert _brit_fold_variants("what is St Lawrence Seaway?") == set()
|
||||
assert _honorific_fold_variants("Labour economics") == set()
|
||||
|
||||
|
||||
def test_title_query_tokens_bridges_both():
|
||||
assert {"saint", "st", "petersburg"} <= (
|
||||
_title_query_tokens("what is St Petersburg?")
|
||||
& _title_query_tokens("Saint Petersburg"))
|
||||
assert {"labor", "labour", "economics"} <= (
|
||||
_title_query_tokens("what is labor economics?")
|
||||
& _title_query_tokens("Labour economics"))
|
||||
assert _title_query_tokens("who painted the mona lisa?") == {
|
||||
"painted", "mona", "lisa"}
|
||||
|
||||
|
||||
def _ingest(tmp_path, docs):
|
||||
from arborist.document import Document
|
||||
from arborist.ingest import ingest_source
|
||||
from arborist.source import Source
|
||||
from arborist.store import connect
|
||||
|
||||
class FakeSource(Source):
|
||||
source_type = "test"
|
||||
|
||||
def __init__(self, ds):
|
||||
self.ds = ds
|
||||
|
||||
def iter_documents(self) -> Iterator[Document]:
|
||||
yield from self.ds
|
||||
|
||||
shard = tmp_path / "shard.db"
|
||||
c = connect(shard)
|
||||
try:
|
||||
ingest_source(c, FakeSource([
|
||||
Document(uri=f"t://{i}", content=ct, source_type="test", title=ti)
|
||||
for i, (ti, ct) in enumerate(docs)]))
|
||||
finally:
|
||||
c.close()
|
||||
return shard
|
||||
|
||||
|
||||
def _ask(tmp_path, shard, q, ans):
|
||||
from arborist.qa.client import StubClient
|
||||
from arborist.qa.query import DEFAULT_QUERY_POLICY, query
|
||||
r = query(question=q, qa_db=tmp_path / "qa.db",
|
||||
chat_client=StubClient(answer=ans), model_id="stub",
|
||||
single_db=shard,
|
||||
policy=dict(DEFAULT_QUERY_POLICY,
|
||||
answer_mode="claim_lattice_pointer"))
|
||||
return r, {(s.get("title") or "") for s in (r.get("sources") or [])}
|
||||
|
||||
|
||||
def test_honorific_query_retrieves_full_form_title(tmp_path):
|
||||
shard = _ingest(tmp_path, [
|
||||
("Saint Lawrence Seaway",
|
||||
"The Saint Lawrence Seaway is a system of locks and canals. " * 12),
|
||||
("Gardening", "Unrelated gardening content. " * 12)])
|
||||
r, titles = _ask(tmp_path, shard, "what is St Lawrence Seaway?",
|
||||
"The Saint Lawrence Seaway is a waterway. [E1]\n")
|
||||
assert r["status"] != "error"
|
||||
assert "Saint Lawrence Seaway" in titles
|
||||
|
||||
|
||||
def test_brit_query_retrieves_british_spelled_title(tmp_path):
|
||||
shard = _ingest(tmp_path, [
|
||||
("Labour economics",
|
||||
"Labour economics studies labour markets and wages. " * 12),
|
||||
("Gardening", "Unrelated gardening content. " * 12)])
|
||||
r, titles = _ask(tmp_path, shard, "what is labor economics?",
|
||||
"Labour economics studies labour markets. [E1]\n")
|
||||
assert r["status"] != "error"
|
||||
assert "Labour economics" in titles
|
||||
Loading…
Add table
Add a link
Reference in a new issue