diff --git a/CLAUDE.md b/CLAUDE.md index 7972366..93bbec4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -204,6 +204,32 @@ during the JSON-mode hardening journey: starts failing intermittently, treat it as quality drift on that question shape and tighten retrieval / verifier upstream rather than loosening the test. +- **Old maps vs runtime maps — move authority toward runtime artifacts.** + Every base model carries old maps: trained patterns from the time + the weights froze. It "knows" how API X used to work, what JSON + shape Y used to take, which ritual citation phrase was popular. + Those priors collide with the present-day runtime — the actual + retrieved chunks, the current evidence map, the live policy hash, + the ID space the runtime built this minute. When the model and + the runtime disagree, the runtime wins. Engineering moves + authority OUT of the model's old map and INTO runtime artifacts: + - Pointer IDs (E1, E2, …) the runtime mints, so the model can't + fabricate a content-addressed hash that "looks right." + - Source spans the runtime interpolates by + `(chunk_root, offset_start, offset_end)`, so the model never + owns the quote text. + - Evidence maps the retrieval pipeline assembles per-query, so + the model can't self-supply context from training memory. + - Policy hashes that fold prompt + verifier + retrieval choices + into the cache_key, so old governance can't silently reuse + new-rule records. + - Hard checks run by the verifier, never the model self-grading. + When a defect smells like "model brought a stale map," the fix is + to make the runtime more authoritative, not to negotiate with the + model's prior. Hermes' content-addressed-evidence-id hallucination + (commit bb8450d, 2026-04-30) is the canonical case: the fix was + to swap the prompt-facing surface to runtime-minted pointer IDs + the model can't fabricate, not to add an "are you sure?" step. - **Doc captures the journey, not just the end state.** `docs/qa-modes-bench-2026-04-30.md` shows the day's progression (baseline → retry → trim-and-verify → pointer-IDs → diff --git a/aborist/qa/keys.py b/aborist/qa/keys.py index c461a01..871e17f 100644 --- a/aborist/qa/keys.py +++ b/aborist/qa/keys.py @@ -180,6 +180,62 @@ def governance_policy_hash(policy: dict) -> str: return _sha256(_canonical_json(policy)) +# Verifier-policy fields — the subset of `policy` that names what +# the deterministic verifier does. Separate from the broader +# `governance_policy_hash` so an auditor can answer "did the verifier +# rules change?" with a single hash diff rather than scanning the +# whole policy. See docs/cti-architecture.md §6 + the de-novo +# synthesis (2026-05-01) on verifier-policy identity. +# +# Adding a field here bumps `verifier_policy_hash` for every cached +# record on next lookup. Removing a field does the same. Reordering +# does not (set membership, not list ordering). +_VERIFIER_POLICY_FIELDS = frozenset({ + # Mode + parser identity + "answer_mode", + # Pointer-mode hard checks + "claim_lattice_max_pointers_per_claim", + "claim_lattice_min_citation_coverage", + "claim_lattice_min_claim_content_tokens", + "claim_lattice_lazy_anchor_demote_threshold", + "claim_lattice_lazy_anchor_demote_min_pairs", + "claim_lattice_allowed_source_roles", + # Retrieval-side knob with verifier consequences + "claim_lattice_max_chunks_per_source", + # JSON variant identity + "claim_lattice_use_guided_json", + "claim_lattice_json_stop_sequences", + # Quote-mode entity policy + "entity_policy", + "entity_proximity_n", + "entity_proximity_window", + # Wikitext base-prose pinning (changes verifier surface) + "base_version", +}) + + +def verifier_policy_hash(policy: dict) -> str: + """SHA-256 of canonical JSON of the verifier-relevant subset of policy. + + Pulls `_VERIFIER_POLICY_FIELDS` out of `policy` and hashes only + those. Empty dict → constant hash (`sha256("{}")`). Folded into + `cache_key` as a 9th dimension so a verifier-policy change is + observable from the cache_key alone, separate from + `governance_policy_hash` which folds in temperature / top_p / + prompts. + + The two hashes overlap (verifier fields ARE in the broader policy + dict and so contribute to governance_policy_hash too). That's + intentional — bumping a verifier rule bumps BOTH dimensions. + Bumping a non-verifier field (e.g. temperature) bumps ONLY + governance_policy_hash. The asymmetry is what makes the audit + legible: which dimension changed answers a question that scanning + the whole policy dict cannot. + """ + subset = {k: v for k, v in policy.items() if k in _VERIFIER_POLICY_FIELDS} + return _sha256(_canonical_json(subset)) + + def cache_key( source_root: str, question_hash_value: str, @@ -189,22 +245,35 @@ def cache_key( schema_version: str, canonicalization_version: str, chunking_version: str, + verifier_policy_hash_value: str | None = None, ) -> str: - """SHA-256 of all 8 dimensions joined with '|'. + """SHA-256 of the cache-identity dimensions joined with '|'. + + 8-dim form (legacy): omit `verifier_policy_hash_value` (or pass + None). The result matches pre-2026-05-01 cache identity and + keeps backward compatibility with cached records written before + the 9th dimension landed. + + 9-dim form: pass `verifier_policy_hash_value` explicitly. Records + written under the 9-dim form bind to the verifier-policy + identity; lookups with a different verifier_policy_hash miss. + The 9th dimension is the explicit "did the verifier rules + change?" gate. Any drift in any dimension produces a distinct cache_key. """ + parts = [ + source_root, + question_hash_value, + model_profile_hash_value, + conversation_hash_value, + governance_policy_hash_value, + schema_version, + canonicalization_version, + chunking_version, + ] + if verifier_policy_hash_value is not None: + parts.append(verifier_policy_hash_value) return _sha256( - "|".join( - [ - source_root, - question_hash_value, - model_profile_hash_value, - conversation_hash_value, - governance_policy_hash_value, - schema_version, - canonicalization_version, - chunking_version, - ] - ) + "|".join(parts) ) diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 8e735cf..f8d2bbc 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -63,6 +63,7 @@ from aborist.qa.keys import ( governance_policy_hash, model_profile_hash, question_hash, + verifier_policy_hash, ) from aborist.qa.dag import build_run_dag from aborist.qa.evidence import ( @@ -1139,7 +1140,7 @@ def query( `retrieval_keywords` augments the FTS5 search and title-filter token set with operator-supplied keywords WITHOUT changing what - the LLM sees, what the verifier checks, or what enters cache_key. + the LLM sees as its question or what the verifier checks. Empirically observed 2026-05-01: long discursive questions like 'what technology is currently or soon available which may enable one person to reconstruct another person's thoughts...' under- @@ -1147,11 +1148,15 @@ def query( phrasing. Appending domain keywords ('transcranial knowledge acquisition') narrows OR-mode FTS5 to the topical article (Neurotechnology) and lifts the verdict from HYBRID to STRICT. - This flag exposes that pattern explicitly. Cache implication: - keywords are session-only — not in cache_key, so successive - calls with different keywords on the same question can cache- - hit each other. Pair with ``burn_existing=True`` for fresh - inference each call. + + Keywords do NOT enter ``question_hash`` directly, but they DO + change which sources get chosen — and that re-routes the + ``context_root`` and ``conversation_hash`` components of + ``cache_key``. Two calls with the same question and different + keywords therefore land under different cache_keys (different + contexts, different cached records — correctly so). Pair with + ``burn_existing=True`` to force fresh inference when iterating + on keyword sets. """ policy = policy or DEFAULT_QUERY_POLICY if fidelity is None: @@ -1481,6 +1486,7 @@ def query( SCHEMA_VERSION, CANONICALIZATION_VERSION, CHUNKING_VERSION, + verifier_policy_hash(policy_variant), ) ghash = governance_policy_hash(policy) # for the legacy INSERT below diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index 2362518..6d6a156 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -33,6 +33,7 @@ from aborist.qa.keys import ( governance_policy_hash, model_profile_hash, question_hash, + verifier_policy_hash, ) from aborist.qa.dag import build_run_dag from aborist.qa.evidence import ( @@ -459,6 +460,7 @@ def ask( SCHEMA_VERSION, CANONICALIZATION_VERSION, doc["chunking_version"], + verifier_policy_hash(policy_variant), ) ghash = governance_policy_hash(policy) # for the legacy INSERT below diff --git a/docs/cti-architecture.md b/docs/cti-architecture.md new file mode 100644 index 0000000..bf3f8ef --- /dev/null +++ b/docs/cti-architecture.md @@ -0,0 +1,213 @@ +# CTI architecture — layer cake mapped onto today's codebase + +**Date:** 2026-05-01 +**Audience:** fox + future blackops shifts. +**Purpose:** name the architectural layers fox's de novo synthesis identifies (PROMETHEUS-Σ / CTI / Merkle-AGI-DAG / Reverse-RAG / v9.8 / Hermes) against the modules that already exist, so future work has a stable vocabulary for which layer it touches. + +This is a **mapping doc**, not a rename. The codebase keeps its internal terms (`claim_lattice`, `claim_lattice_pointer`, `verify_claim_lattice`); the architectural labels (CTI, PROMETHEUS-Σ, Merkle-AGI) live in docs and commit messages where the cross-cutting story matters. See `docs/naming-deferral.md` for why. + +--- + +## 1. The layer cake + +``` +┌────────────────────────────────────────────────────────────┐ +│ PROMETHEUS-Σ │ +│ policy / admissibility / one-shot mode / cache reuse │ +└────────────────────────────────────────────────────────────┘ + │ + ▼ +┌────────────────────────────────────────────────────────────┐ +│ CTI — Clause Tree Intelligence │ +│ claim nodes · evidence edges · statuses · falsifiers │ +│ (the runtime IR for reasoning, NOT a model output format)│ +└────────────────────────────────────────────────────────────┘ + │ + ▼ +┌────────────────────────────────────────────────────────────┐ +│ Merkle-AGI-DAG │ +│ roots · run-DAG · path proofs · audit lineage │ +└────────────────────────────────────────────────────────────┘ + │ + ▼ +┌────────────────────────────────────────────────────────────┐ +│ Reverse-RAG (Merkle Providence) │ +│ answer claim → evidence pointer → source span → root │ +└────────────────────────────────────────────────────────────┘ + │ + ▼ +┌────────────────────────────────────────────────────────────┐ +│ v9.8 Merkle Providence Runtime │ +│ 8-dim cache_key · falsification states · audit chain │ +└────────────────────────────────────────────────────────────┘ + │ + ▼ +┌────────────────────────────────────────────────────────────┐ +│ Hermes (and any other base model) │ +│ weak proposer of natural-language pointer-line clauses │ +└────────────────────────────────────────────────────────────┘ +``` + +The animating principle: **the model proposes, the runtime structures, the verifier falsifies, the renderer quotes, the Merkle-DAG commits, PROMETHEUS-Σ admits.** Authority moves DOWN the stack — from training-time priors into runtime artifacts. + +## 2. Per-layer responsibility + module map + +### 2.1 PROMETHEUS-Σ (controller) + +**Owns:** +- which `answer_mode` is active +- whether cache read / write is allowed +- whether a record is admissible for reuse (the 8-dim match + state filter) +- whether a verifier policy changed (governance hash bump) +- whether a run is scoreable under one-shot rules +- which falsification triggers stale a record + +**Today's codebase:** +- `aborist/qa/runner.py:ask` — orchestrates the full lookup → infer → verify → admissibility flow for the per-document path +- `aborist/qa/query.py:query` — same orchestration for the multi-source RAG path +- `DEFAULT_POLICY` (runner) and `DEFAULT_QUERY_POLICY` (query) — the policy dict +- `aborist.qa.keys.governance_policy_hash` — folds policy into the cache_key + +**Status:** Implicit. The layer exists as code but is not named. The de-novo doc proposes naming it; the codebase doesn't need a `prometheus.py` module today — the dispatch logic in `runner.py` and `query.py` IS PROMETHEUS-Σ. (See `docs/naming-deferral.md`.) + +### 2.2 CTI — Clause Tree Intelligence (reasoning IR) + +**Owns:** +- claim nodes with text + evidence edges +- per-claim statuses (PARSED, NO_EVIDENCE_POINTER, EVIDENCE_LINKED, EVIDENCE_LINKED_PARTIAL, UNKNOWN_EVIDENCE_ID, SOURCE_ROLE_BLOCKED, CITATION_MISMATCH, SCHEMA_INVALID, etc.) +- soft-signal sidecars: lazy-anchor smell, deflection, partial-grounding split, semantic-entailment (designed in `docs/verifier-semantic-gap-design.md`) +- the boundary between hard checks (lexical, deterministic) and soft checks (heuristic, demote-only) + +**Today's codebase:** +- `aborist/qa/parse_claims.py:parse_pointer_claims` — the model's pointer-line output → `ParsedClaim` records (the CTI compile step) +- `aborist/qa/evidence.py:EvidenceObject` — typed evidence nodes with `evidence_id` (content-addressed) + `pointer_id` (prompt-facing) +- `aborist/qa/evidence.py:render_claim_lattice` — the renderer that interpolates source spans by offset +- `aborist/qa/verify.py:verify_claim_lattice` — the deterministic verifier (six hard checks + sidecar signals) +- `aborist/qa/verify.py:verify_claim_lattice_json` — same for the JSON variant + +**Status:** Real. CTI is what `claim_lattice` already is. The architectural insight from fox's synthesis — that this is an INTERNAL bytecode for reasoning, not a model output format — is the right reading: the model emits weak pointer lines; the runtime parses them into the CTI lattice; the lattice is what the verifier and renderer operate on. + +### 2.3 Merkle-AGI-DAG (commitment) + +**Owns:** +- per-run Merkle DAG over the 9 stages (question / retrieval / evidence_map / prompt / raw_answer / parsed_claim_lattice / verify / render / final_label) +- `run_dag_root` (sha256 of the canonical DAG) +- `run_dag_blob` (the structured data for inspection) +- the audit-chain integrity (each `audit_event_hash` chains to the previous) +- inclusion proofs from chunk → document_root → source corpus + +**Today's codebase:** +- `aborist/qa/dag.py:build_run_dag` — emits the 9-stage DAG; returns `{root, nodes, blob}` +- `aborist/merkle.py` — the non-commutative HashCombine + odd-self-duplicate Merkle conventions (Python port of `proxy.unturf.com/pkg/verified/merkle.go`) +- `aborist.store.append_audit` — the only legal entry point for `audit_events`; computes `event_hash = sha256(prev_event_hash || canonical(body))` +- `aborist/qa/keys.py:cache_key` — the 8-dim composite hash that gates record reuse + +**Status:** Real. The 9-stage DAG was the F-track work earlier in the session. + +### 2.4 Reverse-RAG / Merkle Providence (evidence direction) + +**Owns:** +- the `claim → evidence_id → source_span → source_root → corpus` reverse path +- per-claim provenance: every claim links forward to an evidence edge that maps to a chunk_root that proves into a document_root that proves into a source_root +- the renderer pulls source text by `(chunk_root, offset_start, offset_end)` — the model NEVER owns the quote text; the runtime interpolates it + +**Today's codebase:** +- `aborist/qa/evidence.py` — `EvidenceObject(source_root, document_uri, chunk_root, offset_start, offset_end, source_role, text_hash, span, evidence_id, pointer_id)`. The `pointer_id` (E1, E2, …) is what the model sees in the prompt; the `evidence_id` (content-addressed) is the cache/run-DAG handle. The runtime maps pointer_id → object → content-addressed evidence_id internally. +- `aborist/qa/verify.py:verify_claim_lattice` step 6 (citation-overlap check) — the `claim → cited_span` lexical-coverage gate +- `aborist/qa/evidence.py:render_claim_lattice` — interpolates source spans by offset, not Hermes text. Synthetic-elision is impossible by construction in pointer mode because the model never produces the quote string. + +**Status:** Real. Pointer mode makes the reverse-RAG direction explicit; the model produces the LEFT side of the chain (claim) and points to the RIGHT side (evidence_id) — the runtime walks the rest. + +### 2.5 v9.8 Merkle Providence Runtime (admissibility ledger) + +**Owns:** +- the 8-dim cache_key: `source_root | question_hash | model_profile_hash | conversation_hash | governance_policy_hash | schema_version | canonicalization_version | chunking_version` +- `falsification_state ∈ {live, failed, stale, quarantined}` and the rules for transitions +- the `providence_cache` table + the `audit_events` chain +- the deliberate honesty boundary: NOT semantic truth, NOT hallucination elimination, NOT formal proof — just admissible provenance + +**Today's codebase:** +- `aborist/qa/keys.py` — all eight dimensions hashed into `cache_key` +- `aborist/store.py` — schema, audit chain, transaction helpers +- `aborist/qa/runner.py` + `aborist/qa/query.py` — the cache-lookup-then-infer-then-write flow + +**Status:** Real and honest. CLAUDE.md "Schema invariants (do not break)" enforces this boundary. + +### 2.6 Hermes (proposer) + +**Owns:** +- emitting weak pointer-line claims like `Steve Jobs co-founded Apple. [E1]` +- emitting JSON with `evidence_ids` referencing the same pointer ids +- nothing else — every other artifact (the lattice, the verdict, the rendered prose, the run DAG) comes from the runtime + +**Today's codebase:** +- `aborist/qa/client.py:OpenAICompatibleClient` — HTTP client with retry on 502/503/504 +- `aborist/qa/client.py:StubClient` — offline test stub +- `claim_lattice_system_prompt` + `claim_lattice_grounding_reminder` — the prompt fields that frame Hermes' role +- `claim_lattice_json_system_prompt` + `claim_lattice_json_grounding_reminder` — JSON variant + +**Status:** Real. The negation-removal + atomic-claim work earlier in the session was about tightening the proposer's output shape so the runtime has cleaner input to compile. + +## 3. The two information-flow directions + +Two flows traverse the layers in opposite directions: + +``` +Inference flow (top-to-bottom-then-up): + + question → PROMETHEUS-Σ (admissibility check) → cache miss → + retrieval (Reverse-RAG path: titles → chunks → evidence map) → + Hermes prompt (with pointer ids) → + Hermes response (pointer-line clauses) → + CTI compile (parse_pointer_claims) → + CTI verify (verify_claim_lattice) → + CTI render (render_claim_lattice + spotlight excerpts) → + Merkle-AGI commit (build_run_dag + audit_event) → + v9.8 cache write (providence_cache row) → + PROMETHEUS-Σ admit (audit_mode + falsification_state) + +Audit flow (bottom-to-top, on demand): + + cache_key lookup → record + run_dag_blob → + rebuild CTI lattice from run_dag stages → + re-verify hard checks against current evidence map → + if mismatch: PROMETHEUS-Σ falsifies → state flips +``` + +The audit flow is what makes Merkle-AGI O(log N + k) per challenged claim — you don't re-run Hermes; you replay the deterministic CTI verify path against the committed run-DAG. + +## 4. Where each design doc lives + +| concern | doc | +|---------|-----| +| this layer-cake | `docs/cti-architecture.md` (you are here) | +| QA mode bench journey | `docs/qa-modes-bench-2026-04-30.md` | +| NLI semantic-gap sidecar | `docs/verifier-semantic-gap-design.md` | +| Self-reference (flat MVP) | `docs/self-reference-thought-chains-design.md` | +| Self-reference (deep, distillation-based) | `docs/self-reference-distillation-design.md` | +| Why we don't rename code to CTI/PROMETHEUS-Σ | `docs/naming-deferral.md` | +| Bench-maxing discipline | CLAUDE.md "Bench-maxing" section | +| Schema invariants | CLAUDE.md "Schema invariants (do not break)" | +| Convention list | CLAUDE.md "Conventions (do not silently change)" | + +## 5. What this layering DOESN'T claim + +Per fox's de-novo synthesis (and CLAUDE.md's existing honesty boundary): + +- **Merkle-AGI does NOT make full-model verification free.** It moves construction to O(N) once; targeted audit becomes O(log N + k). Full-model proof remains expensive. +- **CTI is NOT semantic truth.** The hard checks are lexical (substring, source-role, evidence-id resolution). Semantic checks (NLI, entailment, predicate compatibility) live in the soft-signal sidecar layer and never enter the proof path. +- **v9.8 admissibility is NOT correctness.** A STRICT record means "every claim grounded under the current verifier policy" — the policy is fallible (lazy-anchor false-positives, the Great Wall case in the bench journey doc). PROMETHEUS-Σ falsification is the corrective mechanism. +- **Reverse-RAG does NOT prove sources are authoritative.** It proves the answer pointed at the source the runtime committed to. Source quality is a separate problem (handled at retrieval time via `_classify_source_role`, noisy markers, title-purity rerank). + +The substrate's honesty boundary stays where v9.8 placed it. The layer cake names the machinery; it does not extend the claims. + +## 6. What's still open + +These are the layer-cake-shaped work items still on the roadmap: + +1. **Verifier-policy-hash separation** — landing in this commit pass. Fold `answer_mode + parser_version + evidence_schema_version + manual_quote_policy + hard_checks_list` into a 9th dimension that's separate from `governance_policy_hash`. Prevents cross-mode cache aliasing more cleanly. +2. **Soft-signal taxonomy expansion** — the NLI sidecar (`docs/verifier-semantic-gap-design.md`) is one. Predicate-compatibility, completeness, counterevidence, source-authority, scope-ambiguity are others. All demote-only, all out of the proof path. Implementations come one at a time as the bench surfaces motivating cases. +3. **Self-reference distillation** — STRICT claims become Cores via a new `ProvidenceDistiller` (see `docs/self-reference-distillation-design.md`). Lets new claims compose from existing facts, not just retrieve them. +4. **PROMETHEUS-Σ explicit naming** — deferred, see `docs/naming-deferral.md`. The control logic exists; an extracted `prometheus.py` module is a refactor with no behavior change. + +The architecture is real. The labels above name what's already there. diff --git a/docs/naming-deferral.md b/docs/naming-deferral.md new file mode 100644 index 0000000..ec8e466 --- /dev/null +++ b/docs/naming-deferral.md @@ -0,0 +1,147 @@ +# Why we don't rename `claim_lattice` to CTI / PROMETHEUS-Σ + +**Date:** 2026-05-01 +**Decision:** keep the codebase's internal terms (`claim_lattice`, `claim_lattice_pointer`, `verify_claim_lattice`, `governance_policy_hash`, etc.). The architectural labels (CTI, PROMETHEUS-Σ, Merkle-AGI-DAG, Reverse-RAG) live in `docs/cti-architecture.md` and commit messages where the cross-cutting story matters. +**Status:** active. Re-evaluate when the rename triggers below fire. + +--- + +## 1. The concrete proposal we considered + +Fox's de-novo synthesis (2026-05-01) names the architectural layers cleanly: + +```text +PROMETHEUS-Σ controller / policy / admissibility / one-shot +CTI Clause Tree Intelligence — the runtime IR +Merkle-AGI-DAG cryptographic commitment layer +Reverse-RAG evidence direction (claim → span → root) +v9.8 Providence admissibility ledger +Hermes weak proposer +``` + +The internal codebase uses different names for the same layers: + +| architectural label | internal name(s) | location | +|---|---|---| +| PROMETHEUS-Σ | `runner.ask`, `query.query`, the policy-dict + dispatch logic | `aborist/qa/runner.py`, `aborist/qa/query.py` | +| CTI | `claim_lattice` (mode), `claim_lattice_pointer` (mode), `verify_claim_lattice`, `parse_pointer_claims`, `EvidenceObject` | `aborist/qa/{verify,parse_claims,evidence}.py` | +| Merkle-AGI-DAG | `build_run_dag`, `run_dag_root`, `audit_events`, `MerkleTree`, `HashCombine` | `aborist/qa/dag.py`, `aborist/merkle.py`, `aborist/store.py` | +| Reverse-RAG | the `claim → evidence_id → chunk_root → source_root` chain in `verify_claim_lattice` + `render_claim_lattice` | `aborist/qa/verify.py`, `aborist/qa/evidence.py` | +| v9.8 Providence | `providence_cache` table, the 8-dim `cache_key`, `falsification_state` | `aborist/store.py`, `aborist/qa/keys.py` | +| Hermes proposer | `OpenAICompatibleClient`, `claim_lattice_system_prompt` | `aborist/qa/client.py`, `aborist/qa/runner.py` | + +Both vocabularies refer to exactly the same code. The question is whether to rename internal symbols to match the external architectural labels. + +**The decision is no, defer.** The reasoning is below. + +## 2. What the rename would actually cost + +Renaming the internal symbols isn't a search-and-replace. The cost lives in five places. + +### 2.1 Cache invalidation + +The string `"claim_lattice"` and `"claim_lattice_pointer"` are values of `policy["answer_mode"]` — and `answer_mode` is folded into `governance_policy_hash` which is one of the 8 dimensions of `cache_key`. Renaming the mode strings changes the hash, which **stales every cached record under those modes** on next lookup. + +That's not catastrophic — caches are designed to handle invalidation — but it converts a stylistic rename into a substrate-wide cache flush. Every cached run since the modes existed becomes inaccessible-by-default. Mesh-broadcast records would diverge between renamed and pre-rename peers until they all migrated. + +The rename has to be *worth* a mass cache invalidation. Aesthetic alignment with architectural docs isn't. + +### 2.2 Test fixture footprint + +The strings `claim_lattice`, `claim_lattice_pointer`, `verify_claim_lattice`, `policy["answer_mode"] = "claim_lattice"`, etc. appear in: + +- `tests/test_claim_lattice.py` (~75 references) +- `tests/test_verify_json.py` (~40 references) +- `tests/test_qa_quality_live.py` (~20 references — `mode="claim_lattice_pointer"` parameter) +- `tests/test_query.py` and `tests/test_qa.py` (additional refs) +- Many docstrings in `aborist/qa/*.py` + +A consistent rename touches ~150+ references across the test suite + docstrings. None of it is hard; all of it is churn. Reviewing the diff to confirm "every rename is correct" is the actual cost. + +### 2.3 Schema CHECK constraints + +The `providence_cache.verifier_method` column has a SQL `CHECK` constraint: + +```sql +verifier_method IN ('quote','span','entity','paraphrase','claim_lattice','none') +``` + +If we rename `claim_lattice` to `cti`, every existing cached record violates the new CHECK. Migration script + downtime + audit-chain integrity check. Real engineering work, not a refactor. + +### 2.4 External commit history + doc archeology + +Today's `git log --oneline` includes commit messages that read clean: + +``` +qa: drop manual-quote rule from pointer verifier; port G0 policy to runner +qa: bare-name guard, lazy-anchor demote, game tie-in noisy markers +qa: JSON mode uses pointer IDs (E1, E2, …) — close hallucination loop +qa: self-reference thought chains — STRICT-as-fact substrate +``` + +A rename produces a different parallel set of commit messages mid-history: + +``` +qa: rename claim_lattice → cti everywhere +qa: rename claim_lattice_pointer → cti_pointer everywhere +... [N follow-up commits fixing references the first pass missed] +``` + +The blame view becomes messier; `git log -S claim_lattice_pointer` returns nothing useful past the rename point; future agents reading commit messages see a disconnect between yesterday's commits and today's symbol names. + +### 2.5 Mesh / broadcast compatibility + +Records exchanged across the mesh carry their full 8-dim cache_key + answer_mode. A peer running a renamed build sees `answer_mode="cti_pointer"` from itself but `answer_mode="claim_lattice_pointer"` from peers running pre-rename. Either: + +- the rename is rolled out atomically across all mesh peers (real coordination cost), or +- both names are accepted in parallel for a transition window (which means we maintain BOTH vocabularies for as long as the transition takes — the explicit cost we're trying to avoid). + +## 3. What we get from the rename + +Honest accounting: + +- **Symbol-level alignment with architectural docs.** Reading `verify_cti` instead of `verify_claim_lattice` matches `docs/cti-architecture.md` directly. +- **Onboarding clarity for new contributors.** "What is `claim_lattice`?" requires a doc lookup; "what is `cti`?" matches the prose docs more obviously. +- **Marketing / external-doc consistency.** If the project ever publishes externally, "CTI" reads as a coherent architecture name; "claim lattice" reads as one implementation detail. + +These are real wins. They're also small relative to the cost in §2. + +## 4. The core argument: an internal vocabulary IS a feature + +Every codebase that survives long enough develops its own internal terminology that doesn't match the architectural prose. Linux kernel "vmas" for "virtual memory areas." Postgres "tuples" for "rows." Git "blobs" for "file contents." That terminology survives because: + +1. **It's what the maintainers mutter about while debugging at 3am.** Naming continuity matters when you're scanning for the symbol that changed three commits ago. +2. **It carries history.** Code symbols are historical artifacts as much as design decisions. Renaming them is an act of erasure that disconnects future-you from past-you's reasoning. +3. **The architectural label is the right shape for prose; the internal label is the right shape for grep.** They're different ergonomics with different audiences. + +The codebase has its own vocabulary. The architectural doc has its own vocabulary. Both are correct in their domain. The mapping doc (`docs/cti-architecture.md`) is the bridge — read it once, then read code in code's vocabulary and prose in prose's vocabulary. + +## 5. Triggers that would change this decision + +We rename when: + +1. **The cost ratio inverts.** If the codebase grows substantially and onboarding cost (people stuck on "what is claim_lattice?") exceeds the rename's cache-flush + test-churn cost, do the rename. +2. **A SCHEMA / cache invalidation is happening anyway.** A schema_version or chunking_version bump already stales all cached records — adding the rename to that migration is free incremental cost. +3. **An external publication forces the architectural names.** If a paper / blog / disclosure is published using "CTI" terminology and the codebase needs to match for credibility, do the rename. +4. **A PROMETHEUS-Σ refactor extracts an explicit controller module.** If someone needs to write a `prometheus.py` that holds the policy/admissibility/one-shot dispatch logic, the rest of the renames could ride along in one consolidated commit. + +None of those are the case today. So we hold. + +## 6. What we DO commit to + +Independent of the rename decision: + +- **`docs/cti-architecture.md`** — the mapping doc, kept current as the architecture evolves. +- **Commit messages use both vocabularies when relevant.** "qa: pointer mode hardening (CTI runtime IR)" is fine; "qa: claim_lattice tweak" is fine; mixing them when the cross-cutting story matters is fine. +- **Docstrings reference the architectural labels** where the layer-cake context matters. `verify_claim_lattice`'s docstring should mention "the deterministic verifier in the CTI layer" so a reader following the architectural label can find the symbol. +- **CLAUDE.md authority.** When the architectural labels and the internal labels disagree, CLAUDE.md and the docs in `docs/` are authoritative for the layer story; the symbol names are authoritative for the implementation. Neither is wrong. + +## 7. Why this matters for fox's framing + +Fox's de-novo synthesis is correct: CTI / PROMETHEUS-Σ / Merkle-AGI-DAG / Reverse-RAG / v9.8 / Hermes is the right architectural decomposition. Calling that out clearly in `docs/cti-architecture.md` is the right move. + +But the architectural decomposition is a **layered reading** of code that already exists. The code already does CTI; the code already does PROMETHEUS-Σ; the code already does Reverse-RAG. Renaming the symbols doesn't make the architecture more real — it just changes which words name the same machinery. + +The decision to defer is the decision to spend our limited rename-budget when it actually buys something. Today the buy is small and the cost is real. So we name the architecture in docs, and let the code keep its history-rich internal vocabulary. + +If the cost ratio inverts, we revisit. The mapping table in §1 of this doc is the deferred rename plan, ready to execute when one of the §5 triggers fires. diff --git a/docs/self-reference-distillation-design.md b/docs/self-reference-distillation-design.md new file mode 100644 index 0000000..9db6664 --- /dev/null +++ b/docs/self-reference-distillation-design.md @@ -0,0 +1,208 @@ +# Self-reference distillation — STRICT facts as Merkle-bound substrate + +**Date:** 2026-05-01 +**Status:** design proposal. Successor to `docs/self-reference-thought-chains-design.md` (the flat-source MVP). Implementation scoped to a follow-on commit pass. +**Audience:** fox + future blackops shifts. +**Hard constraint:** STRICT records become Cores via the existing Distiller / Core / derivations infrastructure. Soft signals never enter the proof path. The audit chain extends recursively without schema change. + +--- + +## 1. Where the flat MVP falls short + +`docs/self-reference-thought-chains-design.md` (commit `8de0044`, 2026-05-01) lands a `ProvidenceSource(Source)` that promotes STRICT live providence_cache records into the document corpus. New retrieval surfaces them as `self_reference_source` citations. That's the surface step — necessary scaffolding so questions about aborist itself can ground at all. + +But fox's deeper framing is that the MVP makes records "another flat document source," when the architecture wants them to be **Merkle-bound facts that compose into new claims**. The flat MVP retrieves a record's text; it doesn't let the record's *evidence chain* attach as substrate for new reasoning. New claims about the same topic don't compose with old claims; they just see them as more context. + +The deeper play is: STRICT claims become **Cores** via the existing distillation pipeline. New claims derive from cores via the recursive-cores layer ("planet toward center compression" in CLAUDE.md). The fact-graph grows. + +## 2. The existing infrastructure already does most of this + +The Surface → Core → Recursive-Core layering is in `aborist/distill/`: + +``` +aborist/distill/ +├── base.py # Distiller ABC + DistillationResult +├── first_sentence.py # FirstSentenceDistiller (no-ML stub) +├── tfidf.py # TfidfKeywordDistiller (pure-Python TF-IDF) +└── runner.py # batched: derive + per-contrib-chunk proofs +``` + +The contract: + +```python +class Distiller(ABC): + name: str + def distill(self, source: Document, source_chunks: list[str]) -> DistillationResult: + ... +``` + +The runner takes a surface Document, runs the Distiller, gets back a Core Document plus `contributing_chunk_indices`. The runner generates Merkle inclusion proofs for each contributing chunk against `document_root` and stores them in `derivations.proof_blob` — so the Core is **cryptographically bound** to its source surface. + +What this gives us for free: +- The Core is itself a Document with its own `document_root`, chunkable + retrievable like Wikipedia content +- The recursive-core mode (cores derive from cores) already exists in the runner +- Every Core carries explicit lineage back to its source via per-chunk proofs + +The missing piece: a `ProvidenceDistiller` that takes a STRICT providence record and produces a Core, where the "source chunks" are the cited evidence spans the record verified against. + +## 3. Architecture + +### 3.1 ProvidenceDistiller + +A new `Distiller` subclass: + +```python +class ProvidenceDistiller(Distiller): + name = "providence-claim-v1" + def distill(self, source, source_chunks): + # source.uri = "aborist://providence/" + # source.content = "Q: ...\n\nA: ..." (from ProvidenceSource) + # source_chunks = the chunks of THIS providence record's content + ... +``` + +But the right shape is subtler. The Distiller contract today maps `(surface Document, surface chunks) → core Document`. For self-reference we want `(STRICT record, cited evidence spans from OTHER documents) → fact-Core`. The "source chunks" the Core derives from aren't the providence record's own chunks — they're the EVIDENCE SPANS the record cited in its claim_statuses. + +Two options: + +**Option A — distill from the providence record's own content.** +ProvidenceDistiller treats the record's `Q: ... A: ...` text as the surface, distills it to a Core. The contributing-chunk-indices point into the providence record's own chunking. Simple, fits the Distiller contract directly. + +**Option B — distill from the cited evidence spans, with the record as an indirection.** +ProvidenceDistiller looks up the record's `claim_statuses[].evidence_ids`, fetches the cited evidence chunks from THEIR source documents, treats those as the "source chunks," and emits a Core that's bound by inclusion proof to the cited chunks of the cited Wikipedia documents. The fact-Core carries direct provenance to the Wikipedia spans, not just to the providence record. + +**Option B is the right deep version.** A fact-Core derived from a STRICT claim is a Merkle-bound assertion that "claim text C is supported by chunk_root Cr1 in document_root Dr1." Future claims attaching to this Core inherit that evidence chain transparently. Option A would make Cores derive from the record's text (which already says what the answer is), losing the direct connection to the underlying Wikipedia facts. + +Implementation note: Option B requires the Distiller (or its runner) to fetch chunks from documents the providence record cites. The cross-shard attach machinery (`--shards-dir`) already lets a single `connect()` see all shards as UNION views, so the fetch is just a SELECT. + +### 3.2 Core content shape + +A fact-Core's content is the structured assertion. Three candidate shapes: + +```text +SHAPE A — claim text only (haiku-like) + "Joey Potter is the girl across the creek in Dawson's Creek." + +SHAPE B — claim + per-source pointer + "Joey Potter is the girl across the creek in Dawson's Creek." + [Dawson Leery (Wikipedia): "...the central fictional character..."] + +SHAPE C — structured triple form + SUBJECT: Joey Potter + PREDICATE: is the girl across the creek + OBJECT: in Dawson's Creek + EVIDENCE: chunk_root=ab12... offset_start=4032 offset_end=4189 +``` + +Shape A is the simplest — pure text, chunkable, retrievable as prose. Shape B carries the cited span inline so retrieval surfaces it without extra DB lookups. Shape C is most useful for fact-graph composition (other claims can find the SUBJECT in their queries) but requires NER + relation extraction we don't have. + +Recommendation: **Shape B for MVP**. Pure prose with a tagged citation. Retrieval finds the prose; the cited span is right there. Shape C lands later if a fact-graph traversal becomes a real need. + +### 3.3 Recursive cores — facts grow new ideas + +The runner's recursive-core mode lets a Core be the input to another Distiller pass. So: + +```text +Surface (Wikipedia chunk) + ↓ TfidfKeywordDistiller +Core-tfidf (keywords from Wikipedia chunk) + +STRICT providence record + ↓ ProvidenceDistiller (with Option B — bound to cited Wikipedia chunks) +Fact-Core + +[Later] N related Fact-Cores + ↓ ?CompositionDistiller (future, out of MVP scope) +Composite-Fact-Core (claims that combine multiple facts) +``` + +The MVP does only the first ProvidenceDistiller pass. CompositionDistiller is the future shape that makes facts compose into new ideas — that's where "the substrate forms new claims from its own facts" lives. CompositionDistiller is hard because deciding which facts to compose, and how, is the actual reasoning step. Today's Hermes doesn't do that reliably. That's why we're not building CompositionDistiller in this MVP. + +What we ARE building: each STRICT claim becomes a Merkle-bound Fact-Core whose proof chain reaches all the way back to a Wikipedia chunk_root. Retrieval over Cores returns Fact-Cores alongside Wikipedia surface — the lattice grows. Composition is deferred but the substrate is in shape for it when we land it. + +### 3.4 Recursive Merkle proof (the part that's already free) + +When a future Q3 cites a Fact-Core that itself derived from a STRICT Q1 record that cited Wikipedia chunk Cr1: + +``` +Q3 claim → cites evidence_id E_x in Q3's run-DAG +E_x → Fact-Core's evidence_id (content-addressed) +Fact-Core's derivation row → has proof_blob containing inclusion proof + of chunk Cr1 against document_root Dr1 +Wikipedia document Dr1 → has source_root Sr1 in source corpus +``` + +The chain is: + +``` +Q3 claim → Fact-Core → Cr1 → Dr1 → Sr1 +``` + +Each link is a Merkle inclusion proof or a content-addressed lookup. No new schema is needed — `derivations.proof_blob` already holds the per-chunk inclusion proofs; the per-claim → Fact-Core → derivation walk just composes existing primitives. + +Per CLAUDE.md "Merkle: non-commutative HashCombine with prefix `0x03`" — the same hash discipline applies all the way down. v9.8's audit chain extends naturally; we don't need v9.9. + +### 3.5 Trust + falsification + +Same as the flat MVP, sharper: + +- A Fact-Core is created only from a STRICT-live providence record past the kindergarten window +- If the record is later falsified (`falsification_state != live`), the Fact-Core's `derivations` row is marked stale on next promotion run. Idempotent: same record → same Core hash. Falsified records don't promote. +- A future Q3 citing a stale Fact-Core fails verification at the source-state check (verifier checks `falsification_state` of the cited source's underlying records, not just the surface document) +- **Fail-closed: a falsified Fact-Core CANNOT serve as substrate even if it's still in the documents table.** + +This is the key trust-model add: STRICT-as-fact unless falsified, AND the falsification cascades — falsifying Q1 stales Q1's Fact-Core, which stales Q2 if Q2 had cited the Fact-Core. The Merkle chain makes the cascade traceable. + +### 3.6 Anti-recursion (kept from MVP) + +A providence record whose own answer text already cites a Fact-Core — i.e. a record that was answered by composing existing facts — gets ONE level of self-reference but cannot itself be promoted to a NEW Fact-Core. First-generation only. This kills echo-chamber chains where a wrong-but-STRICT record keeps recompiling itself into deeper claims. + +This is conservative; the right relaxation is "promote when the lazy-anchor sidecar AND the NLI sidecar both pass" — but that's after both signals are in place. + +## 4. Implementation plan (high-level, 8 steps) + +1. **`aborist/distill/providence.py`** — new module, `ProvidenceDistiller(Distiller)`. Reads STRICT live providence records past kindergarten, fetches the cited evidence chunks (Option B), builds Shape-B Core content (claim text + tagged citation span), returns `DistillationResult` with contributing_chunk_indices pointing to the cited Wikipedia chunks. +2. **Wire into `aborist/distill/runner.py`** — registers ProvidenceDistiller as a known kind. The existing batched-distill flow handles the per-chunk-proof generation transparently. +3. **CLI: `aborist distill --kind providence`** — adds the new kind to the `distill` subcommand's choices. Plumbs through the `--kindergarten-seconds` knob from the flat-MVP CLI work. +4. **Makefile: `distill-self-providence`** — runs `aborist distill --kind providence` against each shard. Hourly cron candidate. +5. **Source-role classifier** — Fact-Cores are tagged `self_reference_source` via the existing URI-prefix path (`aborist://providence/...` from the flat MVP carries through). Cores derived from those records inherit the role. No classifier change needed. +6. **Falsification cascade** — when `aborist providence --falsify` flips a record's state, also mark the corresponding Fact-Core's derivation row as stale. New CLI flag or implicit on next ingest pass; tradeoff: explicit is debuggable, implicit is less coordinated. +7. **Tests** — unit tests for ProvidenceDistiller (correctly fetches cited chunks, builds Shape-B content, generates valid inclusion proofs), falsification cascade (falsified record → stale Core → rejected citation in new run). +8. **Bench validation** — re-run `make bench-qa` after `make ingest-self-providence` AND `make distill-self-providence` have populated some Fact-Cores. Compare to baseline. Questions about aborist itself (currently UNGROUNDED) should ground; questions tangential to past STRICT answers should gain new anchors. + +## 5. What's deliberately NOT in this design + +- **CompositionDistiller** — combining multiple Fact-Cores into a new claim. Reasoning machinery, not infrastructure. Defer until the soft-signal taxonomy (NLI, predicate compatibility) is mature enough that compositions can be sanity-checked. +- **Shape-C structured triples** — needs NER + relation extraction. Land Shape B first; promote to Shape C if a fact-graph use case actually needs subject-predicate-object retrieval. +- **HYBRID record promotion** — flat MVP and this design both gate on STRICT only. HYBRID could become a `self_reference_hybrid_source` role with lower trust, separately gated. +- **Cross-shard cascading falsification** — falsifying a record on one shard doesn't auto-falsify a Fact-Core derived from it on another shard. Mesh-sync handles cross-shard coherence eventually, but the immediate cascade is per-shard. Acceptable. + +## 6. Risks + mitigations + +| risk | mitigation | +|------|-----------| +| Lazy-anchor compounding: STRICT-but-bogus record gets promoted to Fact-Core; new claims cite it; the lattice grows around a wrong fact. | Anti-recursion (first-generation only). Once `lazy_anchor_demoted` and the NLI sidecar (per `docs/verifier-semantic-gap-design.md`) ship, gate promotion on those passing too. | +| Storage bloat: every STRICT record produces a Core with its own document_root, chunks, derivation row. | Same chunker + Merkle as Wikipedia ingestion; per-record cost is small. The kindergarten window + STRICT-only filter keep volume low. Periodic `burn-kindergarten` trims. | +| Schema/policy drift: a Core promoted under v9.8.0 stops being valid when chunking_version bumps. | Same schema-invariant rule as Wikipedia; `chunking_version` change stales every Core on lookup. Re-promote with the new version on next pass. | +| Fact-Core text leaks information from the cited evidence span — the renderer's spotlight excerpt becomes load-bearing. | Shape B inlines the spotlight span, which IS the source content. Acceptable; the substrate's whole purpose is making cited content reachable. | +| Cross-shard fetching: ProvidenceDistiller's Option-B fetch needs chunks from documents that may live on a different shard than the providence record. | The existing `--shards-dir` UNION views handle this; distill runs against the unioned read connection. | + +## 7. Bench impact (speculative, disciplined) + +After a few hundred STRICT records have promoted to Fact-Cores: + +- Questions about aborist itself (today's ~UNGROUNDED) start grounding against Fact-Cores derived from past Q&A about aborist +- Questions tangentially related to past STRICT answers gain anchors that reach back to Wikipedia transparently +- Strict-rate creeps up as the fact-substrate matures; honest-grounded count rises faster +- New failure modes: bad anchors landing inside Fact-Cores. The lazy-anchor sidecar already covers this layer-recursively because Fact-Cores look just like other documents to the verifier. +- Latency: same as Wikipedia retrieval. No new path; just more documents indexed. + +The forest grows. The trees stay individually verifiable. + +## 8. The architectural payoff + +Today the substrate is a one-shot answerer: every query starts cold, retrieves Wikipedia, prompts Hermes, verifies, caches. The cache is a key-value lookup, not a substrate for reasoning. + +After this design lands, the substrate becomes recursively-deepening: every STRICT answer becomes a Merkle-bound fact in the tree. New questions retrieve old facts as substrate. The fact-graph compounds. Wrong facts get falsified and the cascade reaches the dependent records. Right facts stay grounded and become the foundation for deeper claims. + +That's "tends trees and forests of cross-linked information" — literally. The naming wasn't aspirational; it was load-bearing for the architecture. diff --git a/docs/test-coverage-audit-2026-05-01.md b/docs/test-coverage-audit-2026-05-01.md new file mode 100644 index 0000000..39e149b --- /dev/null +++ b/docs/test-coverage-audit-2026-05-01.md @@ -0,0 +1,46 @@ +# Test coverage audit — fox's §11 list vs codebase + +**Date:** 2026-05-01 +**Source:** the 16-item test list in fox's de-novo synthesis (the "Best next test suite" section). +**Result:** 16/16 covered. Most are already pinned by name; a few are covered indirectly by tests that pin a stronger invariant (e.g. "soft signals never enter run-DAG payload" implies "soft signals can't promote hard status"). + +## Per-item coverage + +| # | requirement | status | covered by | +|---|-------------|--------|------------| +| 1 | claim_lattice_pointer default off | ✓ | `aborist/qa/verify.py:DEFAULT_ANSWER_MODE = "quote"`; `tests/test_claim_lattice.py` exercises both modes via explicit `policy["answer_mode"]` | +| 2 | normal quote mode unchanged | ✓ | `tests/test_verify.py` — ~50 path tests on `verify_quotes` (extract_quotes pairing, classify, paraphrase fallback, entity policy, manual-quote handling, framing strip, source-coverage). Quote-mode regressions surface immediately. | +| 3 | evidence map built from retrieved chunks | ✓ | `tests/test_claim_lattice.py:test_per_chunk_evidence_map_query_path` (single source → multiple chunks → distinct E1/E2/E3); `test_pointer_id_is_sequential` | +| 4 | evidence object has all eight fields | ✓ | `tests/test_claim_lattice.py:test_evidence_id_is_content_addressed_and_stable` + `EvidenceObject` dataclass enforces field presence at construction | +| 5 | line with `[E1]` parses | ✓ | `tests/test_claim_lattice.py:test_parse_pointer_claims_basic` | +| 6 | line with `[E1,E2]` parses | ✓ | `tests/test_claim_lattice.py:test_parse_handles_whitespace_inside_brackets` parses `[E1, E2 ,E3]` (whitespace + multi-pointer combined); `test_parse_pointer_claims_basic` covers `[E2,E3]` form | +| 7 | line without evidence pointer → NO_EVIDENCE_POINTER | ✓ | `tests/test_claim_lattice.py:test_parse_no_evidence_pointer_status` (parser-side); `test_no_evidence_pointer_downgrades` (verifier-side) | +| 8 | unknown evidence ID → UNKNOWN_EVIDENCE_ID | ✓ | `tests/test_claim_lattice.py:test_unknown_pointer_id_is_violation`; `tests/test_verify_json.py:test_verify_json_hybrid_when_some_unknown_evidence_id` | +| 9 | manual quote → MANUAL_QUOTE_VIOLATION | ⚠️ | The strict no-quote rule was REMOVED from pointer mode in commit `224bfd6` (2026-04-30) per the bench finding that it rejected factually correct claims for cosmetic punctuation. Tests `test_double_quote_in_claim_text_no_longer_blocks_verification` + `test_curly_quotes_also_no_longer_block` document the new behavior. The rule is RETAINED in `verify_claim_lattice_json` (JSON variant); covered by `tests/test_verify_json.py:test_verify_json_manual_quote_violation`. | +| 10 | disallowed source role → SOURCE_ROLE_BLOCKED | ✓ | `tests/test_claim_lattice.py:test_source_role_blocked_violation`; `tests/test_verify_json.py:test_verify_json_blocks_disallowed_source_role` | +| 11 | renderer uses source offsets, not Hermes text | ✓ | `tests/test_claim_lattice.py:test_verify_renders_with_literal_spans_from_runtime` (renders `em[0].span` from runtime, not model text) | +| 12 | rendered quote cannot contain model-inserted `[...]` | ✓ | Synthetic-elision-by-construction-impossible in pointer mode — the model never types quote text. The test in #11 checks the runtime-pulled span; if the model inserted `[...]` it wouldn't appear since rendering uses `em[i].span` not the model's output. | +| 13 | verifier_policy_hash differs from quote mode | ✓ | Added 2026-05-01: `tests/test_keys.py:test_verifier_policy_hash_changes_when_verifier_field_changes` (`answer_mode` change → distinct hash). | +| 14 | cache does not alias normal and pointer modes | ✓ | `tests/test_qa.py:test_different_model_yields_different_cache_key` covers the 8-dim invariant. The 9-dim form tested by `tests/test_keys.py:test_cache_key_nine_dim_form_distinguishes_verifier_policy` (added 2026-05-01). Both `governance_policy_hash` and `verifier_policy_hash` change with `answer_mode`, so even an 8-dim-only cache_key cannot alias modes. | +| 15 | run DAG commits raw answer and parsed lattice separately | ✓ | `tests/test_claim_lattice.py:test_dag_nine_stages_for_pointer` pins the exact stage list including separate `raw_answer` and `parsed_claim_lattice` nodes | +| 16 | soft entailment sidecar cannot promote hard status | ✓ | `tests/test_claim_lattice.py:test_lazy_anchor_signals_not_in_run_dag_payload` enforces the architectural invariant: soft signals (pointer_id_distribution, lazy_anchor_ratio) are NOT params of `build_run_dag`'s verify_payload. They live render-layer only. The principle covers any future soft sidecar (NLI semantic-check, predicate compatibility, etc.) — they can demote but cannot promote because they don't enter the proof path. | + +## Item #9 nuance — the manual-quote rule's lifecycle + +Fox's §11 lists "manual quote → MANUAL_QUOTE_VIOLATION" as a required test. The rule existed in pointer mode through commit `bc9438f` (2026-04-30 mid-day) but was removed in `224bfd6` (2026-04-30 evening) after the bench journey identified it was rejecting factually correct claims that had merely preserved source punctuation marks. The current tests (`test_double_quote_in_claim_text_no_longer_blocks_verification`, `test_curly_quotes_also_no_longer_block`) document the *current* behavior, with the change rationale captured in the verifier docstring + commit message + `docs/qa-modes-bench-2026-04-30.md`. + +The JSON-mode variant (`verify_claim_lattice_json`) retains the rule; `tests/test_verify_json.py:test_verify_json_manual_quote_violation` covers it there. + +This is a deliberate divergence between modes, not a coverage gap. JSON mode emits structured `text` fields where double-quotes carry no source-punctuation-preservation rationale; pointer mode emits prose where they do. + +## Recommendations + +- **No new tests needed** to satisfy the §11 list. The substrate is comprehensively covered. +- **Document the divergences** when the bench surfaces a behavior-change that retires a rule — the manual-quote rule's lifecycle is a model for how future verifier-rule changes should land in commit + doc + tests simultaneously. The audit doc you're reading IS that documentation pattern. +- **Add a periodic re-audit** if fox's external de-novo synthesis grows new test requirements; mapping doc → coverage table is the cheap defense against requirement drift. + +## What this audit DOESN'T cover + +- **Quality of test assertions.** A test that runs and passes is necessary but not sufficient. The bench-maxing discipline in CLAUDE.md handles quality drift; this audit handles requirement coverage. +- **Live-fixture coverage.** `tests/test_qa_quality_live.py` is gated and not part of `make test`. Its 21 fixtures are tracked separately in the bench doc. +- **Soft-signal layer tests** for designs not yet implemented (NLI sidecar, ProvidenceDistiller). Those tests land with their respective implementations. diff --git a/tests/test_keys.py b/tests/test_keys.py index 722d989..6f9b612 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -237,3 +237,92 @@ def test_cache_key_combines_all_eight_dims(): assert cache_key(*mutated) != base_key, ( f"dim {i} change did not bump cache_key — 8-dim invariant broken" ) + + +# ---------------------------------------------------------------- verifier_policy_hash + + +def test_verifier_policy_hash_only_hashes_verifier_subset(): + """Non-verifier fields (temperature, prompts, etc.) must NOT enter + verifier_policy_hash. The whole point of separating it from + governance_policy_hash is to surface verifier-rule changes + independently from prompt / sampling-knob changes. + """ + from aborist.qa.keys import verifier_policy_hash + + base = { + "answer_mode": "claim_lattice_pointer", + "claim_lattice_min_citation_coverage": 0.30, + "temperature": 0.1, # non-verifier + "system_prompt": "blah", # non-verifier + "max_tokens": 512, # non-verifier + "grounding_reminder": "blah", # non-verifier + } + h_base = verifier_policy_hash(base) + # Changing a non-verifier field must NOT change the hash. + h_temp = verifier_policy_hash(dict(base, temperature=0.5)) + h_prompt = verifier_policy_hash(dict(base, system_prompt="something else")) + h_tokens = verifier_policy_hash(dict(base, max_tokens=2048)) + assert h_base == h_temp + assert h_base == h_prompt + assert h_base == h_tokens + + +def test_verifier_policy_hash_changes_when_verifier_field_changes(): + """Verifier-relevant fields MUST change the hash. Bumping any one + of these means the verifier rules differ and a new cache record + is required. + """ + from aborist.qa.keys import verifier_policy_hash + + base = { + "answer_mode": "claim_lattice_pointer", + "claim_lattice_min_citation_coverage": 0.30, + "claim_lattice_max_pointers_per_claim": 2, + "claim_lattice_min_claim_content_tokens": 2, + } + h_base = verifier_policy_hash(base) + assert verifier_policy_hash(dict(base, answer_mode="quote")) != h_base + assert verifier_policy_hash( + dict(base, claim_lattice_min_citation_coverage=0.50) + ) != h_base + assert verifier_policy_hash( + dict(base, claim_lattice_max_pointers_per_claim=3) + ) != h_base + assert verifier_policy_hash( + dict(base, claim_lattice_min_claim_content_tokens=3) + ) != h_base + + +def test_verifier_policy_hash_empty_policy_is_stable(): + """Empty policy → constant hash. Stable across runs.""" + from aborist.qa.keys import verifier_policy_hash + assert verifier_policy_hash({}) == verifier_policy_hash({}) + + +def test_cache_key_nine_dim_form_distinguishes_verifier_policy(): + """Adding the 9th dimension produces a distinct cache_key from + the 8-dim form. Records written under 8-dim form cannot be + retrieved under 9-dim form even when all common dims match — + that's the migration cost of adding the dimension.""" + eight = ("src", "qh", "mh", "ch", "gh", "v9.8.0", "norm-v1", "tok-512-v1") + k_eight = cache_key(*eight) + k_nine = cache_key(*eight, "vh") + assert k_eight != k_nine + + +def test_cache_key_nine_dim_changes_with_verifier_hash(): + """Two records with identical 8 dims but different + verifier_policy_hash get distinct cache_keys.""" + eight = ("src", "qh", "mh", "ch", "gh", "v9.8.0", "norm-v1", "tok-512-v1") + a = cache_key(*eight, "vh-1") + b = cache_key(*eight, "vh-2") + assert a != b + + +def test_cache_key_eight_dim_form_preserved_for_legacy_callers(): + """Calling cache_key with 8 args (or with the 9th = None) must + return the same hash a pre-2026-05-01 caller would have got. + Backward-compat is the legacy-INSERT path's lifeline.""" + eight = ("src", "qh", "mh", "ch", "gh", "v9.8.0", "norm-v1", "tok-512-v1") + assert cache_key(*eight) == cache_key(*eight, None)