diff --git a/tests/fixtures/byte_identity/claim_lattice.json b/tests/fixtures/byte_identity/claim_lattice.json new file mode 100644 index 0000000..b256742 --- /dev/null +++ b/tests/fixtures/byte_identity/claim_lattice.json @@ -0,0 +1,48 @@ +{ + "governance_policy_hash_examples": { + "answer_mode_claim_lattice": "a4e3d3ee640c6e9b4406da666a0391d2a7ab2267b19dbdf97b0e3bd051833de3", + "empty_policy": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "temperature_0.1_max512": "23f4c653729fff6ee0f46fbc6ae5f97e8b02ece7916171f323d7d800c3692574" + }, + "model_profile_hash_examples": { + "hermes": "2a6205e1d58ac4f742167bdaa34af0c59a33355e45ea55b0b32cfa40ad124427", + "qwen": "dae43274262dcebbf46808dce832cf2b43ee8a6925292fdfe6318d6671e98563", + "stub": "0ba4e47874ede8c69705f9043fc2b7ea542b3a1850df3628f83e658d20a19908" + }, + "per_question": [ + { + "conversation_hash": "75b85b2b56e0a0502aa6cb987205deed86956a2dbfc5b7be4719cebb9e6d48b2", + "question": "when did the soviet union dissolve?", + "question_hash_eq_class": "d21cc58ef569bee8de14740b4ca54184c96b88df4ce7189055cd3b1cc4aa7187", + "question_hash_strict": "76887eee194ffc9d0c1242444d63db67b9551452674f57ecc22c1608ab09a636" + }, + { + "conversation_hash": "64d57c0ec117ffdb3bca91a2d88d572da7f83544acc2778bc9beeae85ce7a9c3", + "question": "where is mount kilimanjaro located?", + "question_hash_eq_class": "4ff0ec9d0207cb13f2c2902b6ebc814d91cf068b1e5377bcd93c181478d6a26b", + "question_hash_strict": "0956e181a88a1b7cac40c569cec0868c00b31865773d611ff41dd156f8264470" + }, + { + "conversation_hash": "9a6a504699a71be4c5ff156b2f3f89e66a0b71a673bbdea0751b57698f2ab295", + "question": "who painted the mona lisa?", + "question_hash_eq_class": "f742804f41e3214a4a6775f61efe42f3ee8160a1323a1d21bb506bfe0a508846", + "question_hash_strict": "73c21ecfb89ad9300d1324267a5a87617aba7c388f82e52d792d96acd051f09b" + }, + { + "conversation_hash": "04b508a87a00e53750e53a5b6db18eee7460b6840ca68c8ad8d7a9117fba9159", + "question": "who were the original seven mercury astronauts?", + "question_hash_eq_class": "1e7e023414f6597152a0f828026481b9037b2ebd0dea12ca6cefe6d19668041e", + "question_hash_strict": "adae170ce9559fbfe8ebb3e859066077ec66a43fcffdc40bc665cb0e0bf8b259" + }, + { + "conversation_hash": "fc4d86687827e21664a239e0332320af395c4a7950934680948ca217251ab35a", + "question": "why did the dinosaurs go extinct?", + "question_hash_eq_class": "71eddec1168829894941efeea6466f9a63efaa36317d2ea63e711a610b02ddea", + "question_hash_strict": "a8412e4849e7f022f1b01ef7da3916d8f2bf1d16216867bb52f55cf846878584" + } + ], + "prompt_strings": { + "grounding_reminder_sha256": "5046a11055fecdb99493f372e3b37ca100acab139d25e51dd76fc612d300f8f3", + "system_prompt_sha256": "7fcd829b9ec9cb5a847fdcc9a0b0807f4df99db4e6f53b13bb8a37b1b2a783f7" + } +} diff --git a/tests/test_run_query_byte_identity.py b/tests/test_run_query_byte_identity.py new file mode 100644 index 0000000..a399067 --- /dev/null +++ b/tests/test_run_query_byte_identity.py @@ -0,0 +1,194 @@ +"""Byte-identity safety gate for the legacy-query() → run_query() +collapse (Phase 1 step 5 of #53). + +The 4280-line legacy ``arborist.qa.query.query()`` computes a 9-dim +cache_key over inputs that include ``conversation_hash(messages)`` +— so any drift in HOW the prompt is built (message order, whitespace, +optional gates) rotates every cache_key in the world. Phase 2 of the +collapse rewrites that prompt-build path inside ``run_query``; this +gate FREEZES the canonical bytes today so the rewrite can be byte- +checked against the captured reference. + +What's pinned: + 1. Hash functions are deterministic on stable inputs (regression- + locks the hash algos themselves). + 2. The claim-lattice prompt strings (CLAIM_LATTICE_SYSTEM_PROMPT + and CLAIM_LATTICE_GROUNDING_REMINDER) hash to a fixed value. + If a future edit drifts those prompt strings, the conversation_hash + drifts and EVERY existing providence_cache record orphans on + re-lookup. The pin is the early-warning signal. + 3. A synthetic prompt-build matching the shape ``run_query`` emits + (sys + user with EVIDENCE + QUESTION + grounding_reminder) hashes + to a fixed value — that's the gate Phase 2's rewrite must + reproduce verbatim. + +Fixture location: ``tests/fixtures/byte_identity/cache_key_*.json``. +Captured once via ``CAPTURE=1 pytest -v +tests/test_run_query_byte_identity.py`` — the test rewrites the +fixture file from the live values. Re-capture only when a deliberate +prompt-shape change is being made AND every prior cache record is +being treated as cold (cache invalidation event). +""" +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from arborist.qa.keys import ( + conversation_hash, + governance_policy_hash, + model_profile_hash, + question_hash, +) +from arborist.qa.prompts import ( + CLAIM_LATTICE_GROUNDING_REMINDER, + CLAIM_LATTICE_SYSTEM_PROMPT, +) + +FIXTURE_DIR = Path(__file__).parent / "fixtures" / "byte_identity" +SMOKE_QUESTIONS = [ + "when did the soviet union dissolve?", + "where is mount kilimanjaro located?", + "who painted the mona lisa?", + "who were the original seven mercury astronauts?", + "why did the dinosaurs go extinct?", +] +CAPTURE = os.environ.get("CAPTURE") == "1" + + +def _prompt_user_payload(question: str, evidence_text: str) -> str: + """Mirror the user-payload shape ``run_query`` builds. + + See arborist/qa/corpus_query.py:173-177 — this MUST stay in sync + with that template. If run_query's template changes, this string + must change too, and the pinned hash gets re-captured. + """ + return ( + f"EVIDENCE:\n\n{evidence_text}\n\n" + f"QUESTION: {question}\n\n" + f"{CLAIM_LATTICE_GROUNDING_REMINDER}" + ) + + +def _canonical_messages(question: str, evidence_text: str) -> list[dict]: + """The 2-message array run_query passes to chat_completion.""" + return [ + {"role": "system", "content": CLAIM_LATTICE_SYSTEM_PROMPT}, + {"role": "user", "content": _prompt_user_payload(question, evidence_text)}, + ] + + +_SYNTHETIC_EVIDENCE = ( + "[E1 | Stub Title | 0000000000000000: " + "this is the literal evidence span the test pins.]" +) + + +def _all_pins() -> dict: + """Compute every pinned value from current code state.""" + out = { + "prompt_strings": { + "system_prompt_sha256": conversation_hash( + [{"role": "system", "content": CLAIM_LATTICE_SYSTEM_PROMPT}] + ), + "grounding_reminder_sha256": conversation_hash( + [{"role": "user", "content": CLAIM_LATTICE_GROUNDING_REMINDER}] + ), + }, + "per_question": [], + } + for q in SMOKE_QUESTIONS: + msgs = _canonical_messages(q, _SYNTHETIC_EVIDENCE) + out["per_question"].append({ + "question": q, + "question_hash_strict": question_hash(q, mode="strict"), + "question_hash_eq_class": question_hash(q, mode="equivalence_class"), + "conversation_hash": conversation_hash(msgs), + }) + out["governance_policy_hash_examples"] = { + # A few well-known policy shapes. If anyone adds a field to one + # of these dicts, the cache rotates. + "empty_policy": governance_policy_hash({}), + "temperature_0.1_max512": governance_policy_hash( + {"temperature": 0.1, "max_tokens": 512} + ), + "answer_mode_claim_lattice": governance_policy_hash( + {"answer_mode": "claim_lattice"} + ), + } + out["model_profile_hash_examples"] = { + "hermes": model_profile_hash( + "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic" + ), + "qwen": model_profile_hash("Qwen3.6-27B-UD-Q4_K_XL.gguf"), + "stub": model_profile_hash("stub"), + } + return out + + +def _fixture_path() -> Path: + return FIXTURE_DIR / "claim_lattice.json" + + +def test_capture_or_compare(): + """Either capture the live hashes to the fixture file (CAPTURE=1) + or assert the live values match the pinned fixture.""" + pins_now = _all_pins() + fixture = _fixture_path() + if CAPTURE: + fixture.parent.mkdir(parents=True, exist_ok=True) + fixture.write_text(json.dumps(pins_now, indent=2, sort_keys=True) + "\n") + return + if not fixture.exists(): + pytest.fail( + f"fixture missing: {fixture}\n" + "first-time setup: CAPTURE=1 pytest " + f"{Path(__file__).name}" + ) + pins_pinned = json.loads(fixture.read_text()) + if pins_now != pins_pinned: + # Render a diff-style failure so the operator sees exactly + # which hash drifted. + diffs = [] + + def walk(now, pinned, path): + if isinstance(now, dict) and isinstance(pinned, dict): + for k in sorted(set(now) | set(pinned)): + walk(now.get(k), pinned.get(k), f"{path}.{k}") + elif isinstance(now, list) and isinstance(pinned, list): + for i, (a, b) in enumerate(zip(now, pinned)): + walk(a, b, f"{path}[{i}]") + if len(now) != len(pinned): + diffs.append(f"{path}: list length {len(now)} != {len(pinned)}") + elif now != pinned: + diffs.append(f"{path}: now={now!r} pinned={pinned!r}") + + walk(pins_now, pins_pinned, "fixture") + pytest.fail( + "byte-identity drift — pinned cache-identity inputs no " + "longer hash to the same values. If this is intentional " + "(deliberate prompt or policy shape change with cache " + "invalidation), re-capture: CAPTURE=1 pytest " + f"{Path(__file__).name}\n\n" + + "\n".join(diffs[:20]) + ) + + +def test_hash_determinism_on_stable_inputs(): + """Hash functions stay deterministic across calls. Sanity check + that pins the hash algorithm itself, not the input bytes.""" + assert conversation_hash([{"role": "system", "content": "x"}]) == conversation_hash( + [{"role": "system", "content": "x"}] + ) + assert governance_policy_hash({"a": 1, "b": 2}) == governance_policy_hash( + {"b": 2, "a": 1} + ) # canonical_json sorts keys; dict order must not matter + # "what is the test?" has both a trailing '?' and an article 'the' + # — equivalence_class strips both, strict preserves them, so the + # canonical forms (and hashes) MUST differ. + assert question_hash("what is the test?", mode="strict") != question_hash( + "what is the test?", mode="equivalence_class" + )