diff --git a/arborist/qa/query.py b/arborist/qa/query.py index 42d43b9..c1360bf 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -283,6 +283,15 @@ def _brit_fold_variants(s: str) -> set[str]: return out +# Single source of truth for the active `_title_query_tokens` fold +# set. Bound into the run-DAG retrieval plan (RetrievalPlan +# .title_token_policy) so a replay knows which token-normalization +# produced the retrieved sources (Dav1d review 2026-05-19; #000001- +# family provenance, run-DAG not governance). Bump on any fold +# add/remove/semantics change. +_TITLE_TOKEN_POLICY = "tt-v2:hyphen+numeral+accent+honorific+brit" + + def _title_query_tokens(s: str) -> set[str]: base = { t.lower() @@ -3651,6 +3660,10 @@ def query( getattr(_mt, "manifest_hash", "") if _sandwich_en_q else "" ), source_lang=(_src_lang if _sandwich_en_q else ""), + # Dav1d review 2026-05-19 — bind the active title-token + # fold set so retrieval is replay-identifiable (#000001- + # family, run-DAG not governance). + title_token_policy=_TITLE_TOKEN_POLICY, ) plan_hash = retrieval_plan_hash(plan) # Ticket #000009 — preflight node binding (nested CTI clauses diff --git a/arborist/qa/retrieval_plan.py b/arborist/qa/retrieval_plan.py index c602a74..64b50c4 100644 --- a/arborist/qa/retrieval_plan.py +++ b/arborist/qa/retrieval_plan.py @@ -72,6 +72,19 @@ class RetrievalPlan: mt_engine: str = "" mt_manifest_hash: str = "" source_lang: str = "" + # Dav1d de-novo review 2026-05-19: `_title_query_tokens` is + # hot-path (every query AND title) and its fold set + # (hyphen #000007 + numeral/accent/honorific/brit) changes which + # documents retrieve. That normalization is a *retrieval + # transform* with the SAME provenance status as + # `retrieval_keywords` / MT-engine: a replay must know which fold + # set was active. Bind its version here (run-DAG), NOT + # `governance_policy_hash` — consistent with the #000001 §5/§6 + # repo decision (the review suggested governance Option-A; repo + # precedent is run-DAG binding — surfaced as a fox call). Empty + # default → omitted from `canonical()` so every prior + # `retrieval_plan_hash` stays byte-identical (§5 zero-churn). + title_token_policy: str = "" def canonical(self) -> dict: """Sorted-key dict for canonical-JSON hashing. Empty fields @@ -92,6 +105,8 @@ class RetrievalPlan: "manifest_hash": self.mt_manifest_hash or "", "source_lang": self.source_lang or "", } + if self.title_token_policy: + out["title_token_policy"] = self.title_token_policy return out diff --git a/tests/test_title_token_provenance.py b/tests/test_title_token_provenance.py new file mode 100644 index 0000000..bb52b7a --- /dev/null +++ b/tests/test_title_token_provenance.py @@ -0,0 +1,69 @@ +"""Title-token fold provenance + edge cases (Dav1d review 2026-05-19). + +The review's central finding: `_title_query_tokens` is hot-path and +its fold set changes which docs retrieve, so a replay must know +which fold set was active. Fix is #000001-family: bind +`title_token_policy` into the run-DAG `RetrievalPlan`/`retrieval_plan +_hash` (NOT governance — repo §5/§6 precedent). Plus the review's +edge cases for the deterministic guarantee. +""" +from __future__ import annotations + +from arborist.qa.query import ( + _TITLE_TOKEN_POLICY, + _numeral_fold_variants, + _title_query_tokens, +) +from arborist.qa.retrieval_plan import RetrievalPlan, retrieval_plan_hash + + +# --- provenance binding (the review's #7/#13, the one that matters) --- + +def test_title_token_policy_binds_into_retrieval_plan_hash_zero_churn(): + base = RetrievalPlan(retrieval_keywords="", top_k=8, over_fetch=32, + max_context_chars=60000) + # empty (pre-existing records) → omitted → byte-identical hash + assert "title_token_policy" not in base.canonical() + bound = RetrievalPlan(retrieval_keywords="", top_k=8, over_fetch=32, + max_context_chars=60000, + title_token_policy=_TITLE_TOKEN_POLICY) + assert bound.canonical()["title_token_policy"] == _TITLE_TOKEN_POLICY + assert retrieval_plan_hash(bound) != retrieval_plan_hash(base) + # a different fold set → different replayable hash + other = RetrievalPlan(retrieval_keywords="", top_k=8, over_fetch=32, + max_context_chars=60000, + title_token_policy="tt-v1:hyphen") + assert retrieval_plan_hash(other) != retrieval_plan_hash(bound) + + +# --- review edge cases for the deterministic fold guarantee --- + +def test_roman_substring_inside_word_not_folded(): + # only whole-token Romans; "civil"/"mixture"/"divide" must not fold + assert _numeral_fold_variants("the civil mixture divide") == set() + + +def test_out_of_range_roman_not_folded(): + # strict 2..40; "forty-first"/"hundredth" out of the ordinal map + assert _numeral_fold_variants("the forty-first hundredth element") == set() + + +def test_unicode_roman_numerals_explicitly_unsupported(): + # ASCII-only by design (the helper keys on ASCII tokens). Document + # the boundary rather than let it silently half-work. + assert _numeral_fold_variants("Alexander Ⅱ") == set() # Ⅱ (U+2161) + + +def test_hyphen_and_numeral_folds_compose_in_title_tokens(): + # _title_query_tokens now layers hyphen + numeral (+accent/honor/ + # brit) folds; the composition surface must be exercised. + q = _title_query_tokens("anarcho-capitalism Alexander the second") + assert {"anarchocapitalism", "ii"} <= q # hyphen-fold AND numeral-fold + assert {"anarcho", "capitalism", "alexander", "second"} <= q # additive + + +def test_policy_constant_is_stable_and_descriptive(): + # the single source of truth; bumping it is the replay signal + assert _TITLE_TOKEN_POLICY.startswith("tt-") + for f in ("hyphen", "numeral", "accent", "honorific", "brit"): + assert f in _TITLE_TOKEN_POLICY