qa: question_hash strip set covers CJK + ellipsis; pin pair-preservation

Per fox: ASCII-only stripping leaves CJK full-width forms behind, so
"who is X？" and "who is X" still produce different cache_keys
despite being the same question. Expanded strip set:

  ASCII:     . ? ! , ; :
  CJK:       ？(U+FF1F)  ！(U+FF01)  。(U+3002)  、(U+3001)
  ellipsis:  …(U+2026)

Pairs deliberately stay out of the set:

  "  '  )  ]  }

Stripping one side of a pair breaks balance. `who said "X"?` after
stripping `?` is balanced; further stripping `"` would yield
`who said "X` — different equivalence class than the original. And
apostrophes carry meaning: `X's` is a different question from `X`.

Lifted the strip set into a module constant `_QUESTION_TRAILING_STRIP`
so anyone considering an addition has a documented anchor.

Tests: 11 new in tests/test_keys.py — 6 CJK / ellipsis equivalences,
5 pair-preservation cases (double-quote, single-quote, paren, bracket,
brace). 24 keys tests + 297 default suite, all passing.

Whitepaper sibling change in ~/git/unfirehose-nextjs-logger/whitepaper/
merkle-providence-reverse-rag-whitepaper.rst — expanded the
question_hash bullet to document the equivalence class. Not committed
in this commit (different repo).

aborist/qa/keys.py

@@ -42,21 +42,43 @@ def question_hash(question: str) -> str:
 
     1. ``canonicalize()`` — NFC, whitespace collapse, strip ends.
     2. ``.lower()`` — case-fold so "Who is X" == "who is X".
-    3. ``rstrip(".?!,;:")`` — trailing punctuation carries no semantic
-       difference for retrieval. ``"who is X?"`` and ``"who is X"`` map
-       to the same cache_key, so the providence_cache hit returns the
-       same answer for either form.
+    3. ``rstrip(_QUESTION_TRAILING_STRIP)`` — trailing punctuation
+       carries no semantic difference for retrieval. ``"who is X?"``
+       and ``"who is X"`` map to the same cache_key, so the
+       providence_cache hit returns the same answer for either form.
 
-    Step 3 is question-specific — chunk leaf hashes stay verbatim because
-    they go through ``canonicalize()`` directly, not through this
-    function. Bumping this rule orphans prior cache records whose
-    canonical question ended in punctuation; they live as history but
-    won't be re-hit on lookup.
+    Step 3 is question-specific — chunk leaf hashes stay verbatim
+    because they go through ``canonicalize()`` directly, not through
+    this function. Bumping ``_QUESTION_TRAILING_STRIP`` orphans prior
+    cache records whose canonical question ended in newly-stripped
+    punctuation; they live as history but won't be re-hit on lookup.
+
+    What's IN the strip set: ``.?!,;:`` (ASCII), ``？！。、`` (CJK
+    full-width), ``…`` (ellipsis). Pairs like ``"`` ``'`` ``)`` ``]``
+    ``}`` are NOT in the set: trailing-strip on one side of a pair
+    breaks balance (``who said "I think therefore I am"?`` → strip
+    ``?`` → ``who said "I think therefore I am"`` is balanced; further
+    stripping ``"`` would yield an unbalanced surface). Apostrophes
+    aren't in the set either — ``X's`` is a different question from
+    ``X``.
     """
-    canon = canonicalize(question).lower().rstrip(".?!,;:")
+    canon = canonicalize(question).lower().rstrip(_QUESTION_TRAILING_STRIP)
     return _sha256(canon)
 
 
+# Trailing punctuation that carries no semantic difference at the end
+# of a question. Order doesn't matter (rstrip walks char-by-char from
+# the right). Repeats handled trivially: ``X???`` → ``X``.
+#
+# ASCII:  . ? ! , ; :
+# CJK:    ？ U+FF1F  full-width question mark
+#         ！ U+FF01  full-width exclamation
+#         。 U+3002  ideographic full stop
+#         、 U+3001  ideographic comma
+# Other:  … U+2026  horizontal ellipsis
+_QUESTION_TRAILING_STRIP = ".?!,;:？！。、…"
+
+
 def model_profile_hash(
     model_id: str, revision: str = "", quantization: str = ""
 ) -> str:

tests/test_keys.py

@@ -62,6 +62,73 @@ def test_question_hash_does_not_strip_internal_punctuation():
     assert question_hash("X, then Y") != question_hash("X then Y")
 
 
+# CJK + ellipsis equivalence — non-ASCII trailing punctuation should
+# behave identically to ASCII forms. Pin so a future "let's go ASCII-only"
+# refactor can't silently regress i18n.
+
+def test_question_hash_strips_full_width_question_mark():
+    """U+FF1F (CJK full-width question mark)."""
+    assert question_hash("X") == question_hash("X？")
+
+
+def test_question_hash_strips_full_width_exclamation():
+    """U+FF01."""
+    assert question_hash("X") == question_hash("X！")
+
+
+def test_question_hash_strips_ideographic_full_stop():
+    """U+3002 (Chinese / Japanese sentence-ending period)."""
+    assert question_hash("X") == question_hash("X。")
+
+
+def test_question_hash_strips_ideographic_comma():
+    """U+3001."""
+    assert question_hash("X") == question_hash("X、")
+
+
+def test_question_hash_strips_ellipsis():
+    """U+2026 — single-codepoint ellipsis, plus the three-dot ASCII form
+    that already worked via repeated `.` strip."""
+    assert question_hash("X") == question_hash("X…")
+    assert question_hash("X") == question_hash("X...")
+
+
+def test_question_hash_strips_mixed_ascii_and_cjk_trailing():
+    """A question ending with both kinds of trailing punctuation strips
+    cleanly. rstrip walks char-by-char from the right."""
+    assert question_hash("X") == question_hash("X？！.")
+
+
+# Quotes and brackets PAIR — naive trailing-strip would break balance.
+# Pin that they stay so a future "expand the strip set" refactor knows
+# the design rule.
+
+def test_question_hash_preserves_trailing_double_quote():
+    """`who said "X"?` (after stripping `?`) ends in `"`. Stripping the
+    quote would unbalance the surface; the equivalence class would
+    no longer match a balanced version."""
+    assert question_hash('who said "X"') != question_hash('who said "X')
+
+
+def test_question_hash_preserves_trailing_single_quote():
+    """Apostrophes look like quotes. `X's` (possessive) must stay distinct
+    from `X` (the bare entity)."""
+    assert question_hash("who is X's") != question_hash("who is X")
+
+
+def test_question_hash_preserves_trailing_close_paren():
+    """`(foo)` — a paren pair carries grouping meaning."""
+    assert question_hash("who is X (Y)") != question_hash("who is X (Y")
+
+
+def test_question_hash_preserves_trailing_close_bracket():
+    assert question_hash("who is X [a]") != question_hash("who is X [a")
+
+
+def test_question_hash_preserves_trailing_close_brace():
+    assert question_hash("who is X {a}") != question_hash("who is X {a")
+
+
 def test_question_hash_distinct_for_different_text():
     """Sanity: two genuinely different questions hash to different values."""
     assert question_hash("who is X") != question_hash("who is Y")