From 6291f7c19079599c05f7e4838e62aa247f706218 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 29 Apr 2026 09:06:35 -0400 Subject: [PATCH] qa: question_hash strip set covers CJK + ellipsis; pin pair-preservation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per fox: ASCII-only stripping leaves CJK full-width forms behind, so "who is X？" and "who is X" still produce different cache_keys despite being the same question. Expanded strip set: ASCII: . ? ! , ; : CJK: ？(U+FF1F) ！(U+FF01) 。(U+3002) 、(U+3001) ellipsis: …(U+2026) Pairs deliberately stay out of the set: " ' ) ] } Stripping one side of a pair breaks balance. `who said "X"?` after stripping `?` is balanced; further stripping `"` would yield `who said "X` — different equivalence class than the original. And apostrophes carry meaning: `X's` is a different question from `X`. Lifted the strip set into a module constant `_QUESTION_TRAILING_STRIP` so anyone considering an addition has a documented anchor. Tests: 11 new in tests/test_keys.py — 6 CJK / ellipsis equivalences, 5 pair-preservation cases (double-quote, single-quote, paren, bracket, brace). 24 keys tests + 297 default suite, all passing. Whitepaper sibling change in ~/git/unfirehose-nextjs-logger/whitepaper/ merkle-providence-reverse-rag-whitepaper.rst — expanded the question_hash bullet to document the equivalence class. Not committed in this commit (different repo). --- aborist/qa/keys.py | 42 ++++++++++++++++++++++------- tests/test_keys.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 10 deletions(-) diff --git a/aborist/qa/keys.py b/aborist/qa/keys.py index 3bd0db0..531fac8 100644 --- a/aborist/qa/keys.py +++ b/aborist/qa/keys.py @@ -42,21 +42,43 @@ def question_hash(question: str) -> str: 1. ``canonicalize()`` — NFC, whitespace collapse, strip ends. 2. ``.lower()`` — case-fold so "Who is X" == "who is X". - 3. ``rstrip(".?!,;:")`` — trailing punctuation carries no semantic - difference for retrieval. ``"who is X?"`` and ``"who is X"`` map - to the same cache_key, so the providence_cache hit returns the - same answer for either form. + 3. ``rstrip(_QUESTION_TRAILING_STRIP)`` — trailing punctuation + carries no semantic difference for retrieval. ``"who is X?"`` + and ``"who is X"`` map to the same cache_key, so the + providence_cache hit returns the same answer for either form. - Step 3 is question-specific — chunk leaf hashes stay verbatim because - they go through ``canonicalize()`` directly, not through this - function. Bumping this rule orphans prior cache records whose - canonical question ended in punctuation; they live as history but - won't be re-hit on lookup. + Step 3 is question-specific — chunk leaf hashes stay verbatim + because they go through ``canonicalize()`` directly, not through + this function. Bumping ``_QUESTION_TRAILING_STRIP`` orphans prior + cache records whose canonical question ended in newly-stripped + punctuation; they live as history but won't be re-hit on lookup. + + What's IN the strip set: ``.?!,;:`` (ASCII), ``？！。、`` (CJK + full-width), ``…`` (ellipsis). Pairs like ``"`` ``'`` ``)`` ``]`` + ``}`` are NOT in the set: trailing-strip on one side of a pair + breaks balance (``who said "I think therefore I am"?`` → strip + ``?`` → ``who said "I think therefore I am"`` is balanced; further + stripping ``"`` would yield an unbalanced surface). Apostrophes + aren't in the set either — ``X's`` is a different question from + ``X``. """ - canon = canonicalize(question).lower().rstrip(".?!,;:") + canon = canonicalize(question).lower().rstrip(_QUESTION_TRAILING_STRIP) return _sha256(canon) +# Trailing punctuation that carries no semantic difference at the end +# of a question. Order doesn't matter (rstrip walks char-by-char from +# the right). Repeats handled trivially: ``X???`` → ``X``. +# +# ASCII: . ? ! , ; : +# CJK: ？ U+FF1F full-width question mark +# ！ U+FF01 full-width exclamation +# 。 U+3002 ideographic full stop +# 、 U+3001 ideographic comma +# Other: … U+2026 horizontal ellipsis +_QUESTION_TRAILING_STRIP = ".?!,;:？！。、…" + + def model_profile_hash( model_id: str, revision: str = "", quantization: str = "" ) -> str: diff --git a/tests/test_keys.py b/tests/test_keys.py index 06b03f4..631ec77 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -62,6 +62,73 @@ def test_question_hash_does_not_strip_internal_punctuation(): assert question_hash("X, then Y") != question_hash("X then Y") +# CJK + ellipsis equivalence — non-ASCII trailing punctuation should +# behave identically to ASCII forms. Pin so a future "let's go ASCII-only" +# refactor can't silently regress i18n. + +def test_question_hash_strips_full_width_question_mark(): + """U+FF1F (CJK full-width question mark).""" + assert question_hash("X") == question_hash("X？") + + +def test_question_hash_strips_full_width_exclamation(): + """U+FF01.""" + assert question_hash("X") == question_hash("X！") + + +def test_question_hash_strips_ideographic_full_stop(): + """U+3002 (Chinese / Japanese sentence-ending period).""" + assert question_hash("X") == question_hash("X。") + + +def test_question_hash_strips_ideographic_comma(): + """U+3001.""" + assert question_hash("X") == question_hash("X、") + + +def test_question_hash_strips_ellipsis(): + """U+2026 — single-codepoint ellipsis, plus the three-dot ASCII form + that already worked via repeated `.` strip.""" + assert question_hash("X") == question_hash("X…") + assert question_hash("X") == question_hash("X...") + + +def test_question_hash_strips_mixed_ascii_and_cjk_trailing(): + """A question ending with both kinds of trailing punctuation strips + cleanly. rstrip walks char-by-char from the right.""" + assert question_hash("X") == question_hash("X？！.") + + +# Quotes and brackets PAIR — naive trailing-strip would break balance. +# Pin that they stay so a future "expand the strip set" refactor knows +# the design rule. + +def test_question_hash_preserves_trailing_double_quote(): + """`who said "X"?` (after stripping `?`) ends in `"`. Stripping the + quote would unbalance the surface; the equivalence class would + no longer match a balanced version.""" + assert question_hash('who said "X"') != question_hash('who said "X') + + +def test_question_hash_preserves_trailing_single_quote(): + """Apostrophes look like quotes. `X's` (possessive) must stay distinct + from `X` (the bare entity).""" + assert question_hash("who is X's") != question_hash("who is X") + + +def test_question_hash_preserves_trailing_close_paren(): + """`(foo)` — a paren pair carries grouping meaning.""" + assert question_hash("who is X (Y)") != question_hash("who is X (Y") + + +def test_question_hash_preserves_trailing_close_bracket(): + assert question_hash("who is X [a]") != question_hash("who is X [a") + + +def test_question_hash_preserves_trailing_close_brace(): + assert question_hash("who is X {a}") != question_hash("who is X {a") + + def test_question_hash_distinct_for_different_text(): """Sanity: two genuinely different questions hash to different values.""" assert question_hash("who is X") != question_hash("who is Y")