engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

qa(provenance): #000001 land — retrieval_plan_hash binds D4

Browse source 
New module aborist/qa/retrieval_plan.py:
  RetrievalPlan dataclass — frozen, captures the operator-
    influenceable retrieval inputs that determine source
    selection (retrieval_keywords, top_k, over_fetch,
    max_context_chars, shard_ids).
  retrieval_plan_hash() — SHA-256 over canonical-JSON.
    Deterministic per call; folds into the run-DAG retrieval
    stage as a bound input alongside the existing
    sources_summary output.

build_run_dag (aborist/qa/dag.py) accepts retrieval_plan_hash:
  When provided, the retrieval stage hash binds BOTH plan
    (input) and result (output): H({"retrieval_plan_hash":...,
    "sources_summary_hash":...}).
  When omitted (e.g. legacy / quote-mode callers that haven't
    plumbed it yet), the retrieval stage falls back to the
    historical sources-summary-only hash so pre-#000001
    run_dag_root values stay stable.

query.py constructs the plan per call and passes it through.
Question text intentionally NOT in the plan — already covered
by question_hash. Shard ids included so audit can reproduce
which shards the search ran against.

Two runs with identical sources but different retrieval keywords
now produce different run_dag_root values — the provenance gap
on operator hints (auditor recovers "these were the keywords
that pulled in those sources") closes for the run-DAG path.

Marker test in tests/test_directives.py flipped from "absent"
assertion to "present": test_d4_retrieval_plan_binding_landed.
Plus test_d4_retrieval_plan_hash_module_exists pinning the
module shape. Full suite: 712 passed.

Deferred per ticket §6:
  - audit events (retrieval_plan_built / retrieval_result_selected)
  - providence_cache.retrieval_keywords SQL column
  - optional strict cache_key mode (retrieval_plan_affects_cache_key)

These are ergonomic affordances atop the run-DAG binding; earn
their own tickets when bench evidence shows the blob path is
too friction-heavy for real workflows.

Directive D4 status: ½ → ✓. Ticket #000001 closed.
This commit is contained in:
russell@unturf.com 2026-05-01 19:26:01 -04:00
parent 3796c238cc
commit 1dfa4636c3
No known key found for this signature in database
7 changed files with 251 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

18
aborist/qa/dag.py
View file

@ -112,6 +112,7 @@ def build_run_dag(
raw_answer_text: str | None = None,
parsed_lattice: list | None = None,
rendered_text: str | None = None,
retrieval_plan_hash: str | None = None,
) -> dict:
"""Return ``{"root": <hex>, "nodes": [<stage>, <hash>], ...}``.
@ -152,7 +153,22 @@ def build_run_dag(
}
for s in sources
]
retrieval_hash = _sha256_hex(_canonical_json(sources_summary))
sources_summary_hash = _sha256_hex(_canonical_json(sources_summary))
# Retrieval stage hash: when a retrieval_plan_hash is supplied
# (per ticket #000001 — provenance binding for operator-influenced
# retrieval inputs like keywords / top_k / over_fetch), the stage
# hash binds BOTH the plan (input) and the sources_summary
# (output). Without a plan supplied, fall back to the historical
# sources-summary-only hash so pre-#000001 records keep their
# run_dag_root values stable. Greenfield records that omit the
# plan stay readable by the run-DAG validator.
if retrieval_plan_hash is not None:
retrieval_hash = _sha256_hex(_canonical_json({
"retrieval_plan_hash": retrieval_plan_hash,
"sources_summary_hash": sources_summary_hash,
}))
else:
retrieval_hash = sources_summary_hash
answer_hash = _sha256_hex(answer_text)
failure_stage = localize_failure(
audit_mode=audit_mode,

23
aborist/qa/query.py
View file

@ -66,6 +66,7 @@ from aborist.qa.keys import (
verifier_policy_hash,
)
from aborist.qa.dag import build_run_dag
from aborist.qa.retrieval_plan import RetrievalPlan, retrieval_plan_hash
from aborist.qa.evidence import (
build_evidence_map,
evidence_map_root,
@ -2179,6 +2180,27 @@ def query(
),
}
proof_obj["retrieval_purity"] = retrieval_purity
# Retrieval-plan hash (Ticket #000001 / Directive D4):
# capture the operator-influenceable retrieval inputs so the
# run-DAG's retrieval stage binds BOTH plan (what guided the
# search) and result (what got chosen). Folds shard ids when
# available so an audit can reproduce which shards the search
# ran against. Question text intentionally NOT included here
# — already covered by question_hash.
plan = RetrievalPlan(
retrieval_keywords=retrieval_keywords or "",
top_k=int(top_k),
over_fetch=int(over_fetch),
max_context_chars=int(max_context_chars),
shard_ids=tuple(
sorted(
{root_to_shard.get(h.document_root, h.shard_path or "")
for h in chosen if (root_to_shard.get(h.document_root)
or h.shard_path)}
)
),
)
plan_hash = retrieval_plan_hash(plan)
run_dag = build_run_dag(
question_hash=qhash,
sources=proof_obj["sources"],
@ -2197,6 +2219,7 @@ def query(
raw_answer_text=raw_answer if is_lattice_mode else None,
parsed_lattice=parsed_lattice,
rendered_text=answer_text if is_lattice_mode else None,
retrieval_plan_hash=plan_hash,
)
run_dag_blob = json.dumps(run_dag, separators=(",", ":"))

86
aborist/qa/retrieval_plan.py Normal file
View file

@ -0,0 +1,86 @@
"""Retrieval-plan provenance binding (Ticket #000001 / Directive D4).
Captures the *input* side of retrieval as a content-addressed hash so
the audit chain reproduces both *what got retrieved* (sources_summary,
which `aborist.qa.dag` already binds) and *how retrieval got there*
(the operator-influenceable inputs: keywords, top_k, over_fetch,
max_context_chars, shard set).
Without this hash, two runs with the same question + different
retrieval keywords that surface identical sources would be Merkle-
indistinguishable — an audit could recover "these documents were
selected" but not "these were the keywords that pulled them in." See
`docs/ticket-000001-retrieval-keywords-audit-gap.md` for the full
problem statement.
Hard or soft? Hard. The hash is SHA-256 over canonical-JSON; the
output is reproducible byte-for-byte across machines. Belongs in the
proof path. Per CLAUDE.md "Soft hash vs hard hash": commitments,
proofs, cache_key. Soft signals (embeddings, similarity scores)
never enter this module.
What is NOT here:
- cache_key impact. The retrieval plan affects which sources got
chosen, which already routes through `context_root` and
`conversation_hash` into `cache_key`. Adding the plan as a 9th
cache_key dimension is a separate decision (see ticket §5).
This module only binds the plan into the run-DAG retrieval stage.
- audit events. `retrieval_plan_built` and
`retrieval_result_selected` events live in a future scope —
additive, can land separately.
- providence_cache column. Direct SQL queryability without
parsing run-DAG blobs is a follow-up enhancement; this module
just provides the hash so the per-run merkle proof carries the
plan.
"""
from __future__ import annotations
import hashlib
import json
from dataclasses import asdict, dataclass, field
@dataclass(frozen=True)
class RetrievalPlan:
"""Operator-influenceable retrieval inputs that determine source
selection. Captured per-call; folded into a content-addressed
hash via ``retrieval_plan_hash``.
Design note: the question text itself is intentionally NOT here
(it's already in ``question_hash``, a separate cache_key dim).
Only the *retrieval-side knobs* belong here — these are the
inputs an auditor needs to reproduce which sources got selected
given the same question.
"""
retrieval_keywords: str = ""
top_k: int = 0
over_fetch: int = 0
max_context_chars: int = 0
# Optional: shard ids the search ran against. Empty tuple means
# "default shard discovery" (whatever ``_search_corpus`` resolved
# at call time). Operators who pin specific shards via
# ``--shards-dir`` or ``single_db`` get those captured here.
shard_ids: tuple[str, ...] = field(default_factory=tuple)
def canonical(self) -> dict:
"""Sorted-key dict for canonical-JSON hashing. Empty fields
keep their default values so the hash is stable across calls
that omit optional knobs."""
return {
"retrieval_keywords": self.retrieval_keywords or "",
"top_k": int(self.top_k),
"over_fetch": int(self.over_fetch),
"max_context_chars": int(self.max_context_chars),
"shard_ids": list(self.shard_ids),
}
def retrieval_plan_hash(plan: RetrievalPlan) -> str:
"""SHA-256 over the canonical-JSON of the retrieval plan.
Deterministic: same plan → same hash, byte-for-byte across
machines. Folds into the run-DAG retrieval stage via
``aborist.qa.dag.build_run_dag(retrieval_plan_hash=...)``.
"""
canon = json.dumps(plan.canonical(), sort_keys=True, separators=(",", ":"))
return hashlib.sha256(canon.encode("utf-8")).hexdigest()

2
docs/TICKETS.md
View file

@ -59,7 +59,7 @@ Newest first. Update on every open/close.
| #000004 | Directive coverage in bench summary | closed · `acd1f9c` | 2026-05-01 | D8 |
| #000003 | Anchor-class warrant generalization (Module H+)| closed · 2026-05-02 | 2026-05-01 | D6 |
| #000002 | Reference-Frame Polarity Contract (Module L) | open | 2026-05-01 | D3 |
| #000001 | Retrieval-keywords audit gap | open | 2026-05-01 | D4 |
| #000001 | Retrieval-keywords audit gap | closed · 2026-05-02 | 2026-05-01 | D4 |
## Next ID

13
docs/seven-point-program.md
View file

@ -202,17 +202,18 @@ gets layered on top.
| 1 | Stop making Hermes prove things | ✓ | |
| 2 | Hermes emits pointer clauses | ✓ | |
| 3 | Build CTI internally | ½ | #000002 |
| 4 | Bind retrieval map AND evidence map | ½ | #000001 |
| 4 | Bind retrieval map AND evidence map | ✓ | #000001 (closed)|
| 5 | Verify pointers deterministically | ✓ | |
| 6 | Anchor-class warrant before NLI | ✓ | #000003 (closed)|
| 7 | Rename labels honestly | ✓ | #000005 (closed)|
| 8 | Automate only after test-pinning | discipline | #000004 (closed)|
Two of seven structural directives are partial (D3, D4); D6 closed
2026-05-02 via #000003. D7 is shipped at the EVIDENCE-LINKED rung;
ticket #000005 proposes the four-rung ladder migration
(POINTER-LINKED → ANCHOR-WARRANTED → EVIDENCE-WARRANTED →
ENTAILMENT-VERIFIED) for stronger label discipline. D8 is the
One of seven structural directives is partial (D3 — multi-frame
answer compilation pending #000002). D4 closed 2026-05-02 via
#000001 (run-DAG binding scope; SQL column + audit events
deferred). D6 closed 2026-05-02 via #000003. D7 closed 2026-05-02
via #000005 (four-rung ladder: POINTER-LINKED → ANCHOR-WARRANTED →
EVIDENCE-WARRANTED → ENTAILMENT-VERIFIED). D8 is the
meta-discipline that gates how all of the above land; the bench
coverage substrate (#000004) is closed.

63
docs/ticket-000001-retrieval-keywords-audit-gap.md
View file

@ -1,7 +1,9 @@
# Ticket #000001 — Retrieval-keywords audit gap
**Status:** open · awaiting go/no-go
**Status:** closed · landed 2026-05-02 (run-DAG binding scope; SQL
column + audit-events scope deferred per §6 below)
**Opened:** 2026-05-01
**Closed:** 2026-05-02
**Scope:** Design proposal for capturing the `--retrieval-keywords` operator
hint in the v9.8 audit chain so retrieval is fully reproducible from a
providence record alone. Doc-only — no code in this commit.
@ -229,15 +231,60 @@ question." The 8-dim `cache_key` invariant stays intact.
## 5. Status
**Proposal.** No code yet. Pinging fox for go/no-go before landing.
**Closed 2026-05-02.** Run-DAG binding scope landed:
Forecast cost: ~1-2 hours of focused work (the dag.py change + schema
migration + ~6 tests + bench column). Risk: low — additive schema
change, additive run_dag input, no cache-key churn.
- New module `aborist/qa/retrieval_plan.py` — `RetrievalPlan`
dataclass + `retrieval_plan_hash` function. Captures
`retrieval_keywords`, `top_k`, `over_fetch`, `max_context_chars`,
`shard_ids`. SHA-256 over canonical-JSON; deterministic per call.
- `aborist/qa/dag.py:build_run_dag` accepts `retrieval_plan_hash`
parameter. When provided, the retrieval stage hash binds BOTH
the plan (input) and `sources_summary` (output). When omitted,
falls back to the historical sources-summary-only hash so
pre-#000001 records keep their `run_dag_root` values stable.
- `aborist/qa/query.py` constructs the plan per call from the
query-time inputs (keywords from `--retrieval-keywords`,
`top_k` / `over_fetch` / `max_context_chars` from policy +
args) plus the resolved shard set, computes the hash, and
passes through to `build_run_dag`.
Forecast value: closes the operator-hint provenance loop. Necessary
for audit-grade reproducibility once `--retrieval-keywords` becomes
a regular operator practice.
Two runs with identical sources but different retrieval keywords
now produce different `run_dag_root` values — provenance closes
the "how did retrieval choose these sources" gap.
3 new directive tests in `tests/test_directives.py`:
- `test_d4_retrieval_plan_binding_landed` (marker flipped from
"absent" assertion to "present").
- Hash divergence asserted via differing
`retrieval_plan_hash` arguments to `build_run_dag`.
- `test_d4_retrieval_plan_hash_module_exists` covers the
`RetrievalPlan` dataclass + `retrieval_plan_hash` function
shape.
Full suite: 712 passed.
## 6. Deferred — additive scope
Audit events (`retrieval_plan_built` / `retrieval_result_selected`)
and the `providence_cache.retrieval_keywords` SQL column from
the original Appendix A proposal stay deferred. The run-DAG
binding satisfies the Merkle-AGI commitment requirement (Theorem
T2: every causally relevant transformation input bound). Direct
SQL queryability and audit-event-level capture are operator-
ergonomic enhancements that earn their own tickets when bench
evidence shows the run-DAG-blob path is too friction-heavy for
real workflows. Per the five-step algorithm step 2: ship the
minimum viable substrate; add ergonomic affordances only when
they earn it.
`cache_key` impact stays as-was: the retrieval plan affects
`context_root` and `conversation_hash` indirectly (different plan
→ different selected sources → different context fed to LLM →
different conversation_hash). The plan does NOT enter
`question_hash` or `governance_policy_hash`; the 8-dim cache_key
invariant holds. Optional strict mode (where
`retrieval_plan_affects_cache_key=True` would fold the plan hash
in directly) stays in the original proposal as future work.
---

75
tests/test_directives.py
View file

@ -207,25 +207,74 @@ def test_d4_run_dag_carries_evidence_map_root():
)
def test_d4_retrieval_plan_binding_status():
"""Marker: retrieval-plan binding (the input side of D4) is
pending via ticket #000001. This test documents the open gap;
flip the assertion when ticket #000001 lands.
def test_d4_retrieval_plan_binding_landed():
"""Retrieval-plan binding landed via ticket #000001. The
`build_run_dag` retrieval stage now hashes both the plan
(operator-influenceable inputs: keywords, top_k, over_fetch,
max_context_chars, shard set) and the result (sources_summary).
Today's `build_run_dag` does NOT accept a `retrieval_plan_hash`
parameter. When it does, this test should INVERT (assert the
parameter exists)."""
Two runs with identical sources but different retrieval keywords
produce different `run_dag_root` values — provenance closes the
'how did retrieval choose these sources' gap."""
from aborist.qa.dag import build_run_dag
sig = inspect.signature(build_run_dag)
# As of 2026-05-01: parameter not present. When ticket #000001
# lands, the lines below flip from `not in` to `in`.
assert "retrieval_plan_hash" not in sig.parameters, (
"retrieval_plan_hash parameter detected — ticket #000001 has "
"landed; flip this assertion to the positive form and remove "
"the marker."
assert "retrieval_plan_hash" in sig.parameters, (
"retrieval_plan_hash parameter expected on build_run_dag — "
"ticket #000001 should have landed it."
)
# Confirm the retrieval-stage hash diverges when only the plan
# differs. Hex-only stub values (q, x → not hex; use a-f, 0-9).
base_kwargs = dict(
question_hash="aa" * 32,
sources=[{
"document_root": "dd" * 32,
"source_role": "primary_answer_source",
"score": 1.0,
"chunk_idx": 0,
}],
context_root="cc" * 32,
conversation_hash="ee" * 32,
answer_text="A.",
audit_mode="STRICT",
verifier_method="quote",
n_quotes=1,
n_verified=1,
)
dag_no_plan = build_run_dag(**base_kwargs)
dag_with_plan_a = build_run_dag(**base_kwargs, retrieval_plan_hash="aa" * 32)
dag_with_plan_b = build_run_dag(**base_kwargs, retrieval_plan_hash="bb" * 32)
assert dag_with_plan_a["root"] != dag_no_plan["root"]
assert dag_with_plan_a["root"] != dag_with_plan_b["root"]
def test_d4_retrieval_plan_hash_module_exists():
"""RetrievalPlan dataclass + retrieval_plan_hash function landed
in `aborist.qa.retrieval_plan` per ticket #000001."""
from aborist.qa.retrieval_plan import RetrievalPlan, retrieval_plan_hash
plan = RetrievalPlan(
retrieval_keywords="orwell 1984",
top_k=8,
over_fetch=32,
max_context_chars=60000,
)
h = retrieval_plan_hash(plan)
# 64-char hex string.
assert len(h) == 64
assert all(c in "0123456789abcdef" for c in h)
# Deterministic: same plan → same hash.
assert retrieval_plan_hash(plan) == h
# Differs when input changes.
plan_b = RetrievalPlan(
retrieval_keywords="different",
top_k=8,
over_fetch=32,
max_context_chars=60000,
)
assert retrieval_plan_hash(plan_b) != h
# ---------------------------------------------------------------------------
# D5 — Verify pointers deterministically