From 1dfa4636c369b4b154304a3e1e6a9e538fe161ad Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Fri, 1 May 2026 19:26:01 -0400 Subject: [PATCH] =?UTF-8?q?qa(provenance):=20#000001=20land=20=E2=80=94=20?= =?UTF-8?q?retrieval=5Fplan=5Fhash=20binds=20D4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New module aborist/qa/retrieval_plan.py: RetrievalPlan dataclass — frozen, captures the operator- influenceable retrieval inputs that determine source selection (retrieval_keywords, top_k, over_fetch, max_context_chars, shard_ids). retrieval_plan_hash() — SHA-256 over canonical-JSON. Deterministic per call; folds into the run-DAG retrieval stage as a bound input alongside the existing sources_summary output. build_run_dag (aborist/qa/dag.py) accepts retrieval_plan_hash: When provided, the retrieval stage hash binds BOTH plan (input) and result (output): H({"retrieval_plan_hash":..., "sources_summary_hash":...}). When omitted (e.g. legacy / quote-mode callers that haven't plumbed it yet), the retrieval stage falls back to the historical sources-summary-only hash so pre-#000001 run_dag_root values stay stable. query.py constructs the plan per call and passes it through. Question text intentionally NOT in the plan — already covered by question_hash. Shard ids included so audit can reproduce which shards the search ran against. Two runs with identical sources but different retrieval keywords now produce different run_dag_root values — the provenance gap on operator hints (auditor recovers "these were the keywords that pulled in those sources") closes for the run-DAG path. Marker test in tests/test_directives.py flipped from "absent" assertion to "present": test_d4_retrieval_plan_binding_landed. Plus test_d4_retrieval_plan_hash_module_exists pinning the module shape. Full suite: 712 passed. Deferred per ticket §6: - audit events (retrieval_plan_built / retrieval_result_selected) - providence_cache.retrieval_keywords SQL column - optional strict cache_key mode (retrieval_plan_affects_cache_key) These are ergonomic affordances atop the run-DAG binding; earn their own tickets when bench evidence shows the blob path is too friction-heavy for real workflows. Directive D4 status: ½ → ✓. Ticket #000001 closed. --- aborist/qa/dag.py | 18 +++- aborist/qa/query.py | 23 +++++ aborist/qa/retrieval_plan.py | 86 +++++++++++++++++++ docs/TICKETS.md | 2 +- docs/seven-point-program.md | 13 +-- ...ket-000001-retrieval-keywords-audit-gap.md | 63 ++++++++++++-- tests/test_directives.py | 75 +++++++++++++--- 7 files changed, 251 insertions(+), 29 deletions(-) create mode 100644 aborist/qa/retrieval_plan.py diff --git a/aborist/qa/dag.py b/aborist/qa/dag.py index 1dd980a..2d0ff78 100644 --- a/aborist/qa/dag.py +++ b/aborist/qa/dag.py @@ -112,6 +112,7 @@ def build_run_dag( raw_answer_text: str | None = None, parsed_lattice: list | None = None, rendered_text: str | None = None, + retrieval_plan_hash: str | None = None, ) -> dict: """Return ``{"root": , "nodes": [, ], ...}``. @@ -152,7 +153,22 @@ def build_run_dag( } for s in sources ] - retrieval_hash = _sha256_hex(_canonical_json(sources_summary)) + sources_summary_hash = _sha256_hex(_canonical_json(sources_summary)) + # Retrieval stage hash: when a retrieval_plan_hash is supplied + # (per ticket #000001 — provenance binding for operator-influenced + # retrieval inputs like keywords / top_k / over_fetch), the stage + # hash binds BOTH the plan (input) and the sources_summary + # (output). Without a plan supplied, fall back to the historical + # sources-summary-only hash so pre-#000001 records keep their + # run_dag_root values stable. Greenfield records that omit the + # plan stay readable by the run-DAG validator. + if retrieval_plan_hash is not None: + retrieval_hash = _sha256_hex(_canonical_json({ + "retrieval_plan_hash": retrieval_plan_hash, + "sources_summary_hash": sources_summary_hash, + })) + else: + retrieval_hash = sources_summary_hash answer_hash = _sha256_hex(answer_text) failure_stage = localize_failure( audit_mode=audit_mode, diff --git a/aborist/qa/query.py b/aborist/qa/query.py index cbd393e..7c1063c 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -66,6 +66,7 @@ from aborist.qa.keys import ( verifier_policy_hash, ) from aborist.qa.dag import build_run_dag +from aborist.qa.retrieval_plan import RetrievalPlan, retrieval_plan_hash from aborist.qa.evidence import ( build_evidence_map, evidence_map_root, @@ -2179,6 +2180,27 @@ def query( ), } proof_obj["retrieval_purity"] = retrieval_purity + # Retrieval-plan hash (Ticket #000001 / Directive D4): + # capture the operator-influenceable retrieval inputs so the + # run-DAG's retrieval stage binds BOTH plan (what guided the + # search) and result (what got chosen). Folds shard ids when + # available so an audit can reproduce which shards the search + # ran against. Question text intentionally NOT included here + # — already covered by question_hash. + plan = RetrievalPlan( + retrieval_keywords=retrieval_keywords or "", + top_k=int(top_k), + over_fetch=int(over_fetch), + max_context_chars=int(max_context_chars), + shard_ids=tuple( + sorted( + {root_to_shard.get(h.document_root, h.shard_path or "") + for h in chosen if (root_to_shard.get(h.document_root) + or h.shard_path)} + ) + ), + ) + plan_hash = retrieval_plan_hash(plan) run_dag = build_run_dag( question_hash=qhash, sources=proof_obj["sources"], @@ -2197,6 +2219,7 @@ def query( raw_answer_text=raw_answer if is_lattice_mode else None, parsed_lattice=parsed_lattice, rendered_text=answer_text if is_lattice_mode else None, + retrieval_plan_hash=plan_hash, ) run_dag_blob = json.dumps(run_dag, separators=(",", ":")) diff --git a/aborist/qa/retrieval_plan.py b/aborist/qa/retrieval_plan.py new file mode 100644 index 0000000..b8b791e --- /dev/null +++ b/aborist/qa/retrieval_plan.py @@ -0,0 +1,86 @@ +"""Retrieval-plan provenance binding (Ticket #000001 / Directive D4). + +Captures the *input* side of retrieval as a content-addressed hash so +the audit chain reproduces both *what got retrieved* (sources_summary, +which `aborist.qa.dag` already binds) and *how retrieval got there* +(the operator-influenceable inputs: keywords, top_k, over_fetch, +max_context_chars, shard set). + +Without this hash, two runs with the same question + different +retrieval keywords that surface identical sources would be Merkle- +indistinguishable — an audit could recover "these documents were +selected" but not "these were the keywords that pulled them in." See +`docs/ticket-000001-retrieval-keywords-audit-gap.md` for the full +problem statement. + +Hard or soft? Hard. The hash is SHA-256 over canonical-JSON; the +output is reproducible byte-for-byte across machines. Belongs in the +proof path. Per CLAUDE.md "Soft hash vs hard hash": commitments, +proofs, cache_key. Soft signals (embeddings, similarity scores) +never enter this module. + +What is NOT here: + - cache_key impact. The retrieval plan affects which sources got + chosen, which already routes through `context_root` and + `conversation_hash` into `cache_key`. Adding the plan as a 9th + cache_key dimension is a separate decision (see ticket §5). + This module only binds the plan into the run-DAG retrieval stage. + - audit events. `retrieval_plan_built` and + `retrieval_result_selected` events live in a future scope — + additive, can land separately. + - providence_cache column. Direct SQL queryability without + parsing run-DAG blobs is a follow-up enhancement; this module + just provides the hash so the per-run merkle proof carries the + plan. +""" +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict, dataclass, field + + +@dataclass(frozen=True) +class RetrievalPlan: + """Operator-influenceable retrieval inputs that determine source + selection. Captured per-call; folded into a content-addressed + hash via ``retrieval_plan_hash``. + + Design note: the question text itself is intentionally NOT here + (it's already in ``question_hash``, a separate cache_key dim). + Only the *retrieval-side knobs* belong here — these are the + inputs an auditor needs to reproduce which sources got selected + given the same question. + """ + retrieval_keywords: str = "" + top_k: int = 0 + over_fetch: int = 0 + max_context_chars: int = 0 + # Optional: shard ids the search ran against. Empty tuple means + # "default shard discovery" (whatever ``_search_corpus`` resolved + # at call time). Operators who pin specific shards via + # ``--shards-dir`` or ``single_db`` get those captured here. + shard_ids: tuple[str, ...] = field(default_factory=tuple) + + def canonical(self) -> dict: + """Sorted-key dict for canonical-JSON hashing. Empty fields + keep their default values so the hash is stable across calls + that omit optional knobs.""" + return { + "retrieval_keywords": self.retrieval_keywords or "", + "top_k": int(self.top_k), + "over_fetch": int(self.over_fetch), + "max_context_chars": int(self.max_context_chars), + "shard_ids": list(self.shard_ids), + } + + +def retrieval_plan_hash(plan: RetrievalPlan) -> str: + """SHA-256 over the canonical-JSON of the retrieval plan. + + Deterministic: same plan → same hash, byte-for-byte across + machines. Folds into the run-DAG retrieval stage via + ``aborist.qa.dag.build_run_dag(retrieval_plan_hash=...)``. + """ + canon = json.dumps(plan.canonical(), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canon.encode("utf-8")).hexdigest() diff --git a/docs/TICKETS.md b/docs/TICKETS.md index dcf30b4..ee9b03b 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -59,7 +59,7 @@ Newest first. Update on every open/close. | #000004 | Directive coverage in bench summary | closed · `acd1f9c` | 2026-05-01 | D8 | | #000003 | Anchor-class warrant generalization (Module H+)| closed · 2026-05-02 | 2026-05-01 | D6 | | #000002 | Reference-Frame Polarity Contract (Module L) | open | 2026-05-01 | D3 | -| #000001 | Retrieval-keywords audit gap | open | 2026-05-01 | D4 | +| #000001 | Retrieval-keywords audit gap | closed · 2026-05-02 | 2026-05-01 | D4 | ## Next ID diff --git a/docs/seven-point-program.md b/docs/seven-point-program.md index ef64c1f..f767d50 100644 --- a/docs/seven-point-program.md +++ b/docs/seven-point-program.md @@ -202,17 +202,18 @@ gets layered on top. | 1 | Stop making Hermes prove things | ✓ | | | 2 | Hermes emits pointer clauses | ✓ | | | 3 | Build CTI internally | ½ | #000002 | -| 4 | Bind retrieval map AND evidence map | ½ | #000001 | +| 4 | Bind retrieval map AND evidence map | ✓ | #000001 (closed)| | 5 | Verify pointers deterministically | ✓ | | | 6 | Anchor-class warrant before NLI | ✓ | #000003 (closed)| | 7 | Rename labels honestly | ✓ | #000005 (closed)| | 8 | Automate only after test-pinning | discipline | #000004 (closed)| -Two of seven structural directives are partial (D3, D4); D6 closed -2026-05-02 via #000003. D7 is shipped at the EVIDENCE-LINKED rung; -ticket #000005 proposes the four-rung ladder migration -(POINTER-LINKED → ANCHOR-WARRANTED → EVIDENCE-WARRANTED → -ENTAILMENT-VERIFIED) for stronger label discipline. D8 is the +One of seven structural directives is partial (D3 — multi-frame +answer compilation pending #000002). D4 closed 2026-05-02 via +#000001 (run-DAG binding scope; SQL column + audit events +deferred). D6 closed 2026-05-02 via #000003. D7 closed 2026-05-02 +via #000005 (four-rung ladder: POINTER-LINKED → ANCHOR-WARRANTED → +EVIDENCE-WARRANTED → ENTAILMENT-VERIFIED). D8 is the meta-discipline that gates how all of the above land; the bench coverage substrate (#000004) is closed. diff --git a/docs/ticket-000001-retrieval-keywords-audit-gap.md b/docs/ticket-000001-retrieval-keywords-audit-gap.md index b3ba91a..1c041d8 100644 --- a/docs/ticket-000001-retrieval-keywords-audit-gap.md +++ b/docs/ticket-000001-retrieval-keywords-audit-gap.md @@ -1,7 +1,9 @@ # Ticket #000001 — Retrieval-keywords audit gap -**Status:** open · awaiting go/no-go +**Status:** closed · landed 2026-05-02 (run-DAG binding scope; SQL +column + audit-events scope deferred per §6 below) **Opened:** 2026-05-01 +**Closed:** 2026-05-02 **Scope:** Design proposal for capturing the `--retrieval-keywords` operator hint in the v9.8 audit chain so retrieval is fully reproducible from a providence record alone. Doc-only — no code in this commit. @@ -229,15 +231,60 @@ question." The 8-dim `cache_key` invariant stays intact. ## 5. Status -**Proposal.** No code yet. Pinging fox for go/no-go before landing. +**Closed 2026-05-02.** Run-DAG binding scope landed: -Forecast cost: ~1-2 hours of focused work (the dag.py change + schema -migration + ~6 tests + bench column). Risk: low — additive schema -change, additive run_dag input, no cache-key churn. +- New module `aborist/qa/retrieval_plan.py` — `RetrievalPlan` + dataclass + `retrieval_plan_hash` function. Captures + `retrieval_keywords`, `top_k`, `over_fetch`, `max_context_chars`, + `shard_ids`. SHA-256 over canonical-JSON; deterministic per call. +- `aborist/qa/dag.py:build_run_dag` accepts `retrieval_plan_hash` + parameter. When provided, the retrieval stage hash binds BOTH + the plan (input) and `sources_summary` (output). When omitted, + falls back to the historical sources-summary-only hash so + pre-#000001 records keep their `run_dag_root` values stable. +- `aborist/qa/query.py` constructs the plan per call from the + query-time inputs (keywords from `--retrieval-keywords`, + `top_k` / `over_fetch` / `max_context_chars` from policy + + args) plus the resolved shard set, computes the hash, and + passes through to `build_run_dag`. -Forecast value: closes the operator-hint provenance loop. Necessary -for audit-grade reproducibility once `--retrieval-keywords` becomes -a regular operator practice. +Two runs with identical sources but different retrieval keywords +now produce different `run_dag_root` values — provenance closes +the "how did retrieval choose these sources" gap. + +3 new directive tests in `tests/test_directives.py`: +- `test_d4_retrieval_plan_binding_landed` (marker flipped from + "absent" assertion to "present"). +- Hash divergence asserted via differing + `retrieval_plan_hash` arguments to `build_run_dag`. +- `test_d4_retrieval_plan_hash_module_exists` covers the + `RetrievalPlan` dataclass + `retrieval_plan_hash` function + shape. + +Full suite: 712 passed. + +## 6. Deferred — additive scope + +Audit events (`retrieval_plan_built` / `retrieval_result_selected`) +and the `providence_cache.retrieval_keywords` SQL column from +the original Appendix A proposal stay deferred. The run-DAG +binding satisfies the Merkle-AGI commitment requirement (Theorem +T2: every causally relevant transformation input bound). Direct +SQL queryability and audit-event-level capture are operator- +ergonomic enhancements that earn their own tickets when bench +evidence shows the run-DAG-blob path is too friction-heavy for +real workflows. Per the five-step algorithm step 2: ship the +minimum viable substrate; add ergonomic affordances only when +they earn it. + +`cache_key` impact stays as-was: the retrieval plan affects +`context_root` and `conversation_hash` indirectly (different plan +→ different selected sources → different context fed to LLM → +different conversation_hash). The plan does NOT enter +`question_hash` or `governance_policy_hash`; the 8-dim cache_key +invariant holds. Optional strict mode (where +`retrieval_plan_affects_cache_key=True` would fold the plan hash +in directly) stays in the original proposal as future work. --- diff --git a/tests/test_directives.py b/tests/test_directives.py index 062a492..7942e28 100644 --- a/tests/test_directives.py +++ b/tests/test_directives.py @@ -207,25 +207,74 @@ def test_d4_run_dag_carries_evidence_map_root(): ) -def test_d4_retrieval_plan_binding_status(): - """Marker: retrieval-plan binding (the input side of D4) is - pending via ticket #000001. This test documents the open gap; - flip the assertion when ticket #000001 lands. +def test_d4_retrieval_plan_binding_landed(): + """Retrieval-plan binding landed via ticket #000001. The + `build_run_dag` retrieval stage now hashes both the plan + (operator-influenceable inputs: keywords, top_k, over_fetch, + max_context_chars, shard set) and the result (sources_summary). - Today's `build_run_dag` does NOT accept a `retrieval_plan_hash` - parameter. When it does, this test should INVERT (assert the - parameter exists).""" + Two runs with identical sources but different retrieval keywords + produce different `run_dag_root` values — provenance closes the + 'how did retrieval choose these sources' gap.""" from aborist.qa.dag import build_run_dag sig = inspect.signature(build_run_dag) - # As of 2026-05-01: parameter not present. When ticket #000001 - # lands, the lines below flip from `not in` to `in`. - assert "retrieval_plan_hash" not in sig.parameters, ( - "retrieval_plan_hash parameter detected — ticket #000001 has " - "landed; flip this assertion to the positive form and remove " - "the marker." + assert "retrieval_plan_hash" in sig.parameters, ( + "retrieval_plan_hash parameter expected on build_run_dag — " + "ticket #000001 should have landed it." ) + # Confirm the retrieval-stage hash diverges when only the plan + # differs. Hex-only stub values (q, x → not hex; use a-f, 0-9). + base_kwargs = dict( + question_hash="aa" * 32, + sources=[{ + "document_root": "dd" * 32, + "source_role": "primary_answer_source", + "score": 1.0, + "chunk_idx": 0, + }], + context_root="cc" * 32, + conversation_hash="ee" * 32, + answer_text="A.", + audit_mode="STRICT", + verifier_method="quote", + n_quotes=1, + n_verified=1, + ) + dag_no_plan = build_run_dag(**base_kwargs) + dag_with_plan_a = build_run_dag(**base_kwargs, retrieval_plan_hash="aa" * 32) + dag_with_plan_b = build_run_dag(**base_kwargs, retrieval_plan_hash="bb" * 32) + assert dag_with_plan_a["root"] != dag_no_plan["root"] + assert dag_with_plan_a["root"] != dag_with_plan_b["root"] + + +def test_d4_retrieval_plan_hash_module_exists(): + """RetrievalPlan dataclass + retrieval_plan_hash function landed + in `aborist.qa.retrieval_plan` per ticket #000001.""" + from aborist.qa.retrieval_plan import RetrievalPlan, retrieval_plan_hash + + plan = RetrievalPlan( + retrieval_keywords="orwell 1984", + top_k=8, + over_fetch=32, + max_context_chars=60000, + ) + h = retrieval_plan_hash(plan) + # 64-char hex string. + assert len(h) == 64 + assert all(c in "0123456789abcdef" for c in h) + # Deterministic: same plan → same hash. + assert retrieval_plan_hash(plan) == h + # Differs when input changes. + plan_b = RetrievalPlan( + retrieval_keywords="different", + top_k=8, + over_fetch=32, + max_context_chars=60000, + ) + assert retrieval_plan_hash(plan_b) != h + # --------------------------------------------------------------------------- # D5 — Verify pointers deterministically