diff --git a/aborist/qa/keys.py b/aborist/qa/keys.py index 5da7adf..a25cf0c 100644 --- a/aborist/qa/keys.py +++ b/aborist/qa/keys.py @@ -35,6 +35,22 @@ def _canonical_json(obj) -> str: return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) +def canonical_question(question: str) -> str: + """Canonical equivalence-class form of `question` text. + + Same four-step normalization that `question_hash` hashes, exposed + so callers can dedup BEFORE hashing — e.g. inject the canonical + form into the user message used for `conversation_hash`, while + still sending the verbatim question to the LLM. Without this split + `"who is batman"`, `"who is batman?"`, and `"who is the batman?"` + all collapse on `question_hash` but each hits `conversation_hash` + differently, missing cache. + """ + canon = canonicalize(question).lower().rstrip(_QUESTION_TRAILING_STRIP) + tokens = [t for t in canon.split() if t not in _QUESTION_ARTICLE_STRIP] + return " ".join(tokens) + + def question_hash(question: str) -> str: """SHA-256 of canonicalized + lowercased + punctuation-stripped + article-stripped question text. @@ -74,10 +90,7 @@ def question_hash(question: str) -> str: Apostrophes aren't either — ``X's`` is a different question from ``X``. """ - canon = canonicalize(question).lower().rstrip(_QUESTION_TRAILING_STRIP) - tokens = [t for t in canon.split() if t not in _QUESTION_ARTICLE_STRIP] - canon = " ".join(tokens) - return _sha256(canon) + return _sha256(canonical_question(question)) # Trailing punctuation that carries no semantic difference at the end diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 7670d96..7545b8d 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -54,6 +54,7 @@ from aborist.qa.concepts import ( ) from aborist.qa.keys import ( cache_key, + canonical_question, conversation_hash, governance_policy_hash, model_profile_hash, @@ -707,6 +708,13 @@ def query( # user-turn instructions more reliably than system-turn rules under long # context). The sources + question land last so they are the most-recent # tokens before generation begins. + # + # Decoupled hash form vs LLM form: `messages` carries the verbatim + # question (Hermes sees the user's natural phrasing); `canonical_messages` + # substitutes the canonical question (no articles, no trailing + # punctuation) so `conversation_hash` collapses on the equivalence class. + # `who is batman`, `who is batman?`, `who is the batman?` → same chash. + canonical_q = canonical_question(question) messages = [ {"role": "system", "content": policy["system_prompt"]}, ] @@ -720,10 +728,16 @@ def query( "content": f"Sources:\n\n{context}\n\n---\n\nQuestion: {question}", } ) + canonical_messages = list(messages[:-1]) + [ + { + "role": "user", + "content": f"Sources:\n\n{context}\n\n---\n\nQuestion: {canonical_q}", + } + ] context_root = _context_root([h.document_root for h in chosen]) qhash = question_hash(question) mhash = model_profile_hash(model_id, revision, quantization) - chash = conversation_hash(messages) + chash = conversation_hash(canonical_messages) ghash = governance_policy_hash(policy) ckey = cache_key( context_root, diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index c5b9fdd..e20e167 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -24,6 +24,7 @@ from aborist.merkle import MerkleTree, proof_to_dict from aborist.qa.client import ChatClient from aborist.qa.keys import ( cache_key, + canonical_question, conversation_hash, governance_policy_hash, model_profile_hash, @@ -125,6 +126,13 @@ def ask( # System sets the policy; a user-turn reminder restates the verbatim-quote # rule one message before the document arrives. Document + question land # last as the most-recent tokens before generation. + # + # Decoupled hash form vs LLM form: `messages` carries the verbatim + # question; `canonical_messages` substitutes the canonical question + # (no articles, no trailing punctuation) so `conversation_hash` + # collapses on the question equivalence class. See aborist/qa/query.py + # for the rationale. + canonical_q = canonical_question(question) messages = [ {"role": "system", "content": policy["system_prompt"]}, ] @@ -141,10 +149,19 @@ def ask( ), } ) + canonical_messages = list(messages[:-1]) + [ + { + "role": "user", + "content": ( + f"Document:\n\n{document_text}\n\n---\n\n" + f"Question: {canonical_q}" + ), + } + ] qhash = question_hash(question) mhash = model_profile_hash(model_id, revision, quantization) - chash = conversation_hash(messages) + chash = conversation_hash(canonical_messages) ghash = governance_policy_hash(policy) ckey = cache_key( document_root, diff --git a/tests/test_query.py b/tests/test_query.py index e9fdd62..1c6e2d7 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -133,6 +133,65 @@ def test_query_different_question_different_cache(tmp_path): assert len(client.calls) == 2 # both missed cache, both called client +def test_query_question_equivalence_class_dedups_cache(tmp_path): + """Fox 2026-04-29 catch: 'who is batman?', 'who is batman', and + 'who is the batman?' should all hit the same cache. Question_hash + canonicalizes correctly, but conversation_hash used to hash the + LITERAL question text in the user message — so each variant got + its own chash and missed cache. Fix: canonical_question form is + substituted into the messages list used for conversation_hash, + while the LLM still receives the verbatim question.""" + main_db = tmp_path / "corpus.db" + qa_db = tmp_path / "qa.db" + conn = connect(main_db) + try: + ingest_source(conn, FakeSource(DOCS)) + finally: + conn.close() + + captured: list = [] + + def _capture(messages, **kw): + captured.append(messages) + return "stub answer" + + # First variant — populates cache. + r1 = query( + question="What is anarchism?", + qa_db=qa_db, + chat_client=StubClient(answer=_capture), + model_id="m", + single_db=main_db, + ) + # Second & third variants — must hit the same cache_key. + r2 = query( + question="What is anarchism", # no trailing ? + qa_db=qa_db, + chat_client=StubClient(answer=_capture), + model_id="m", + single_db=main_db, + ) + r3 = query( + question="what is the anarchism?", # leading article + qa_db=qa_db, + chat_client=StubClient(answer=_capture), + model_id="m", + single_db=main_db, + ) + + assert r1["cache_key"] == r2["cache_key"] == r3["cache_key"] + assert r1["status"] == "cache_miss_then_written" + assert r2["status"] == "cache_hit" + assert r3["status"] == "cache_hit" + # Only the first call reached the LLM. + assert len(captured) == 1 + # And it received the verbatim question, not the canonical form. + user_text = "\n".join( + m["content"] for m in captured[0] if m["role"] == "user" + ) + assert "What is anarchism?" in user_text + + def test_query_no_sources_when_empty_corpus(tmp_path): main_db = tmp_path / "empty.db" qa_db = tmp_path / "qa.db"