russell@unturf.com
6573080284
Two more MEASURED fold-search wins on mined ground-truth fixtures
(deterministic recall, no LLM), both lifting at @1/@3/@8 (not
coarse-k artifacts):
honorific (Mt/St/Dr <-> Mount/Saint/Doctor): recall@1 45% -> 75%
(+30pp), @8 62% -> 85%, misses 15 -> 6
brit (British <-> American spelling): recall@1 50% -> 70%
(+20pp), @8 70% -> 85%, misses 12 -> 6
Both _*_fold_variants are additive+symmetric, strict closed sets
(no English-word collision), no-op outside their class (verified
independent: brit no-ops on honorific titles & vice versa), unioned
into _title_query_tokens beside hyphen(#000007)/numeral/accent.
Full suite 2488 passed, 0 regressions (hot-path); real-path tests
(FakeSource->ingest->query()->real _Hit).
Fold-search FINAL across the survey backlog, ranked by MEASURED @1
headroom (not prevalence — the instrument's job):
SHIPPED: numeral (a3ac653) accent (b573c59) honorific brit (here)
NO-BUILD: hyphen — existing #000007 already delivers 90%@1
(the measure-the-unmeasured-thing check pays off)
NO-BUILD: amp — 82%@1 with no fold (prevalence-overranked;
instrument killed it cheaply, like digit-ordinal pre-build)
Net: 4 deterministic retrieval wins + a reusable mined-recall
instrument + the discipline codified in CLAUDE.md, from a goal that
4 prior hypotheses died on because the bench couldn't measure them.
105 lines
4 KiB
Python
105 lines
4 KiB
Python
"""Honorific + British-spelling folds, retrieval-side.
|
|
|
|
Measured 2026-05-18 on mined ground-truth fixtures:
|
|
honorific recall@1 45% -> 75% (+30pp), misses 15 -> 6
|
|
brit recall@1 50% -> 70% (+20pp), misses 12 -> 6
|
|
Both lift at @1/@3/@8 (not coarse-k artifacts). Same additive+
|
|
symmetric discipline as `_hyphen_fold_variants` (#000007) /
|
|
numeral / accent.
|
|
|
|
Real path (FakeSource -> ingest -> query() -> real `_Hit`), never a
|
|
hand-built object — the discipline the reverted disambiguation v1
|
|
violated (false-green on the wrong type).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Iterator
|
|
|
|
from arborist.qa.query import (
|
|
_brit_fold_variants,
|
|
_honorific_fold_variants,
|
|
_title_query_tokens,
|
|
)
|
|
|
|
|
|
def test_variants_additive_symmetric_noop_and_independent():
|
|
assert _honorific_fold_variants("what is St Petersburg?") == {"saint"}
|
|
assert _honorific_fold_variants("Saint Petersburg") == {"st"}
|
|
assert _brit_fold_variants("what is labor economics?") == {"labour"}
|
|
assert _brit_fold_variants("Labour economics") == {"labor"}
|
|
# additive safety: pure non-class queries -> empty
|
|
assert _honorific_fold_variants("who painted the mona lisa?") == set()
|
|
assert _brit_fold_variants("who painted the mona lisa?") == set()
|
|
# independence: brit is a no-op on honorific titles & vice versa
|
|
assert _brit_fold_variants("what is St Lawrence Seaway?") == set()
|
|
assert _honorific_fold_variants("Labour economics") == set()
|
|
|
|
|
|
def test_title_query_tokens_bridges_both():
|
|
assert {"saint", "st", "petersburg"} <= (
|
|
_title_query_tokens("what is St Petersburg?")
|
|
& _title_query_tokens("Saint Petersburg"))
|
|
assert {"labor", "labour", "economics"} <= (
|
|
_title_query_tokens("what is labor economics?")
|
|
& _title_query_tokens("Labour economics"))
|
|
assert _title_query_tokens("who painted the mona lisa?") == {
|
|
"painted", "mona", "lisa"}
|
|
|
|
|
|
def _ingest(tmp_path, docs):
|
|
from arborist.document import Document
|
|
from arborist.ingest import ingest_source
|
|
from arborist.source import Source
|
|
from arborist.store import connect
|
|
|
|
class FakeSource(Source):
|
|
source_type = "test"
|
|
|
|
def __init__(self, ds):
|
|
self.ds = ds
|
|
|
|
def iter_documents(self) -> Iterator[Document]:
|
|
yield from self.ds
|
|
|
|
shard = tmp_path / "shard.db"
|
|
c = connect(shard)
|
|
try:
|
|
ingest_source(c, FakeSource([
|
|
Document(uri=f"t://{i}", content=ct, source_type="test", title=ti)
|
|
for i, (ti, ct) in enumerate(docs)]))
|
|
finally:
|
|
c.close()
|
|
return shard
|
|
|
|
|
|
def _ask(tmp_path, shard, q, ans):
|
|
from arborist.qa.client import StubClient
|
|
from arborist.qa.query import DEFAULT_QUERY_POLICY, query
|
|
r = query(question=q, qa_db=tmp_path / "qa.db",
|
|
chat_client=StubClient(answer=ans), model_id="stub",
|
|
single_db=shard,
|
|
policy=dict(DEFAULT_QUERY_POLICY,
|
|
answer_mode="claim_lattice_pointer"))
|
|
return r, {(s.get("title") or "") for s in (r.get("sources") or [])}
|
|
|
|
|
|
def test_honorific_query_retrieves_full_form_title(tmp_path):
|
|
shard = _ingest(tmp_path, [
|
|
("Saint Lawrence Seaway",
|
|
"The Saint Lawrence Seaway is a system of locks and canals. " * 12),
|
|
("Gardening", "Unrelated gardening content. " * 12)])
|
|
r, titles = _ask(tmp_path, shard, "what is St Lawrence Seaway?",
|
|
"The Saint Lawrence Seaway is a waterway. [E1]\n")
|
|
assert r["status"] != "error"
|
|
assert "Saint Lawrence Seaway" in titles
|
|
|
|
|
|
def test_brit_query_retrieves_british_spelled_title(tmp_path):
|
|
shard = _ingest(tmp_path, [
|
|
("Labour economics",
|
|
"Labour economics studies labour markets and wages. " * 12),
|
|
("Gardening", "Unrelated gardening content. " * 12)])
|
|
r, titles = _ask(tmp_path, shard, "what is labor economics?",
|
|
"Labour economics studies labour markets. [E1]\n")
|
|
assert r["status"] != "error"
|
|
assert "Labour economics" in titles
|