From ec55db513cc3a53e77abce28aefdd1b8f2f8b1b7 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 27 May 2026 10:40:35 -0400 Subject: [PATCH] #000068 Phase 2+3: bench + opt-in demote flag for missed-answer guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 — bench instrumentation + measurement run bench/qa_sweep.py picks up the answerability sidecar projection per row (answerability_fired, answerability_confidence, answerability_denial_ pattern, answerability_answer_type, answerability_candidate_count) and aggregates per-mode (answerability_fires + S/M/W confidence breakdown) into a new column in the markdown summary table. Measurement run on bench/qa_results/phase2-sidecar-on/2026-05-27T14- 16-22Z (76 questions × n=3 × claim_lattice × Hermes-3-8B × tail layout, 228 runs). Headline: sidecar fires 2/228 (0.88%) confidence dist 2 strong / 0 medium / 0 weak precision 100% (2/2 fires were the Ballestrini fixture) recall on Ballestrini 2/3 across n=3 (third run model extracted correctly -> sidecar silent, correct behavior) false positives 0/226 non-Ballestrini runs verifier verdict both fires labeled STRICT by the binary verifier (the verifier-blind class, exactly as predicted) Detection rule's three-clause conjunction (denial + extraction-shape + candidate proximity near cleaned subject tokens) is operating at the precision floor. The strong-confidence-only firing pattern is what calibrates Phase 3's demote threshold. Phase 3 — opt-in demote flag (default OFF per Dav1d Phase 4 NO-GO) arborist/qa/keys.py: answerability_demote_enabled added to _VERIFIER_POLICY_FIELDS so flipping the flag partitions cache via verifier_policy_hash. Justification: when on, the rendered audit_mode changes (EVIDENCE-WARRANTED -> EVIDENCE-MISSED-PARTIAL), which IS a verifier-output property; verifier hash must move accordingly. The other answerability_* fields stay governance-only (sidecar diagnostic, no audit_mode mutation). arborist/cli.py:_render_audit_label extended with answerability + demote_enabled kwargs. Logic: demote_triggers = ( demote_enabled and answerability["answerability_warning"] is True and answerability["confidence_class"] in ("strong", "medium") ) lattice modes: EVIDENCE-WARRANTED -> EVIDENCE-MISSED-PARTIAL (rung transition) POINTER-LINKED / ANCHOR-WARRANTED -> "rung · missed-answer" (tail tag; rung itself already signals degradation) non-lattice modes (quote/span/entity/paraphrase): audit_mode token unchanged + "· missed-answer" tail tag weak confidence: NEVER demotes (Phase 2 saw zero weak fires on real failures; reserved for future expanded detection ladder) CLI flag --demote-on-missed-answer on both `arborist query` and `arborist ask`, default OFF. Flows into call_policy[ "answerability_demote_enabled"] and through to result[ "answerability_demote_enabled"] so the renderer reads it without needing the policy dict. End-to-end verified live: 4 fresh Hermes-3-8B runs with --demote-on- missed-answer on `songs by veronica ballestrini`, all 4 rendered EVIDENCE-MISSED-PARTIAL · via claim_lattice (Hermes hit the failure mode in all 4, sidecar fired strong, demote logic transformed the label). Phase 4 (default flip to demote-on) — NO-GO per Dav1d 2026-05-27 §3.4: "a false sidecar warning is tolerable; a false audit-label demotion can damage trust in correct abstentions." Phase 2 precision is 100% but n=2 fires is too few samples to claim precision floor empirically. Default flip blocks on wider bench + human spot-check of the warnings. Tests: 47 total (36 Phase 1 + 11 new Phase 3 covering hash partitioning discipline + render-label projection across all four rung/confidence matrices). Full suite 2794 passed (delta +22 from prior 2772). Bench output (bench/qa_results/phase2-sidecar-on/) intentionally not committed — bench/qa_results/ is gitignored per existing convention; the ticket carries the headline numbers + path for re-inspection. --- arborist/cli.py | 70 +++++++- arborist/qa/keys.py | 11 ++ arborist/qa/query.py | 9 + bench/qa_sweep.py | 53 +++++- docs/TICKETS.md | 2 +- ...0068-verifier-blind-missed-answer-guard.md | 23 ++- tests/test_missed_answer_guard.py | 156 ++++++++++++++++++ 7 files changed, 317 insertions(+), 7 deletions(-) diff --git a/arborist/cli.py b/arborist/cli.py index b456a9c..9714997 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -469,6 +469,8 @@ def _cmd_ask(args: argparse.Namespace) -> int: call_policy["answer_mode"] = args.answer_mode if getattr(args, "user_payload_layout", None): call_policy["user_payload_layout"] = args.user_payload_layout + if getattr(args, "demote_on_missed_answer", False): + call_policy["answerability_demote_enabled"] = True try: result = ask( conn, @@ -537,6 +539,12 @@ def _cmd_query(args: argparse.Namespace) -> int: call_policy["answer_mode"] = args.answer_mode if getattr(args, "user_payload_layout", None): call_policy["user_payload_layout"] = args.user_payload_layout + # Ticket #000068 Phase 3 — opt-in missed-answer demote. Flag + # default OFF (Phase 4 NO-GO on default-on until human spot-check + # confirms low FP rate; Phase 2 bench showed 100% precision at + # n=228 but that's not enough samples to flip the default). + if getattr(args, "demote_on_missed_answer", False): + call_policy["answerability_demote_enabled"] = True if getattr(args, "repair", False): # Mechanical-only repair when --repair is set; --repair-reprompts # adds the optional re-prompt tier on top. Both default off so @@ -781,6 +789,9 @@ def _render_audit_label( audit_mode: str, verifier_method: str, violations: list[dict] | None = None, + *, + answerability: dict | None = None, + demote_enabled: bool = False, ) -> str: """Map (audit_mode, verifier_method, violations) → human-readable display label. @@ -819,9 +830,32 @@ def _render_audit_label( optimistic rung) — operators get the same surface as before until callers thread violations through. """ + # Ticket #000068 Phase 3 — missed-answer demote projection. When + # the demote flag is on AND the sidecar fired at strong/medium + # confidence, demote EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL + # (lattice modes) or append a `· missed-answer` tail (other modes + # / lower rungs). Weak-confidence fires never demote — the bench + # data (2026-05-27 Phase 2) showed STRONG was the only firing + # tier on real failures; weak is reserved for the future expanded + # detection ladder. Demote-enabled is folded into verifier_policy_hash + # via _VERIFIER_POLICY_FIELDS so cache partitions cleanly on flip. + demote_triggers = ( + demote_enabled + and isinstance(answerability, dict) + and bool(answerability.get("answerability_warning")) + and answerability.get("confidence_class") in ("strong", "medium") + ) is_claim_lattice = verifier_method.startswith("claim_lattice") if is_claim_lattice: rung = _ladder_rung_for_lattice(audit_mode, violations) + if demote_triggers: + if rung == "EVIDENCE-WARRANTED": + rung = "EVIDENCE-MISSED-PARTIAL" + else: + # Lower rungs already signal degradation — surface the + # missed-answer signal as a tail tag instead of a rung + # transition. Same convention as `· warrant missing`. + rung = f"{rung} · missed-answer" return f"{rung} · via {verifier_method}" if verifier_method == "canonical_projection": # Display label for #000027 — persisted canonical answers @@ -830,7 +864,10 @@ def _render_audit_label( return "CANONICAL · via canonical_projection" # Quote / span / entity / paraphrase: keep audit_mode as the # primary token; append method for clarity. - return f"{audit_mode} · via {verifier_method}" + label = f"{audit_mode} · via {verifier_method}" + if demote_triggers: + label = f"{label} · missed-answer" + return label def _render_warrant_tail(result: dict) -> str: @@ -1108,7 +1145,11 @@ def _render_query_human(result: dict, question: str) -> str: # EVIDENCE-WARRANTED anchor-warranted + no soft demotes # UNGROUNDED no verified pairs # Schema column stays unchanged; pure display. - display_label = _render_audit_label(audit, method, result.get("violations")) + display_label = _render_audit_label( + audit, method, result.get("violations"), + answerability=result.get("answerability"), + demote_enabled=bool(result.get("answerability_demote_enabled", False)), + ) warrant_tail = _render_warrant_tail(result) # Ticket #000031 Phase 3 — when a cited source has a warrant- # resolver derivation row tying it to a primary-source surface @@ -5274,6 +5315,16 @@ def build_parser() -> argparse.ArgumentParser: "See `query --user-payload-layout` for full semantics." ), ) + ask_cmd.add_argument( + "--demote-on-missed-answer", + dest="demote_on_missed_answer", + action="store_true", + default=False, + help=( + "Ticket #000068 Phase 3 — opt-in missed-answer demote. " + "See `query --demote-on-missed-answer` for semantics." + ), + ) ask_cmd.set_defaults(func=_cmd_ask) query_cmd = sub.add_parser( @@ -5396,6 +5447,21 @@ def build_parser() -> argparse.ArgumentParser: "each evidence block. Folds into governance_policy_hash." ), ) + query_cmd.add_argument( + "--demote-on-missed-answer", + dest="demote_on_missed_answer", + action="store_true", + default=False, + help=( + "Ticket #000068 Phase 3 — opt-in demote of EVIDENCE-WARRANTED " + "→ EVIDENCE-MISSED-PARTIAL when the missed-answer sidecar " + "fires at strong/medium confidence. Default off. Folds into " + "verifier_policy_hash (changes the rendered audit_mode); " + "flipping it invalidates prior cached records on lookup. " + "Use after reviewing Phase 2 bench evidence — see " + "docs/tickets/ticket-000068-*.md." + ), + ) query_cmd.add_argument( "--retrieval-keywords", dest="retrieval_keywords", default=None, help=( diff --git a/arborist/qa/keys.py b/arborist/qa/keys.py index a915e5a..ec33edc 100644 --- a/arborist/qa/keys.py +++ b/arborist/qa/keys.py @@ -253,6 +253,17 @@ _VERIFIER_POLICY_FIELDS = frozenset({ # the verifier's TITLE_MISMATCH / subject-tokens-absent / spotlight # decisions depend on which tokens count as content. "content_token_rules", + # Ticket #000068 Phase 3 — opt-in missed-answer demote flag. + # Default False (Phase 1 sidecar stays read-only). When True, the + # render-layer projection demotes EVIDENCE-WARRANTED → + # EVIDENCE-MISSED-PARTIAL on rows where the sidecar fired at + # strong/medium confidence. That changes the rendered audit_mode + # — a verifier-output property — so the flag legitimately folds + # into verifier_policy_hash, not just governance. Flipping it + # invalidates prior cached records on lookup. Default-OFF is + # NO-GO Phase 4 until benchmark + human spot-check justifies + # the flip (per Dav1d 2026-05-27 review §3.2 + §3.4). + "answerability_demote_enabled", }) diff --git a/arborist/qa/query.py b/arborist/qa/query.py index 83ddd52..557441e 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -4004,6 +4004,15 @@ def query( # Read-only diagnostic — never written to providence_cache, # recomputable from (question, answer, evidence, policy). "answerability": answerability, + # Ticket #000068 Phase 3 — opt-in demote flag value (default + # False). Surfaced on the result so the renderer can apply + # the EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL projection + # without needing the policy dict in hand. Folds into + # verifier_policy_hash (NOT just governance) because flipping + # it changes the rendered audit_mode. + "answerability_demote_enabled": bool( + policy.get("answerability_demote_enabled", False) + ), # Quantifier preflight result (Ticket #000008 Phase 1). # Surfaced on the result so bench rows pick it up. Phase 1 # is dry-run only — caps not applied; Phase 2 wires diff --git a/bench/qa_sweep.py b/bench/qa_sweep.py index f1fd518..8d1c7a5 100644 --- a/bench/qa_sweep.py +++ b/bench/qa_sweep.py @@ -207,6 +207,18 @@ def _run_one( quantifier_matched_token = result.get("quantifier_matched_token") scope_bound_hint = result.get("scope_bound_hint") claim_cap_applied = result.get("claim_cap_applied") + # Ticket #000068 Phase 1 — missed-answer guard sidecar projection. + # `result["answerability"]` is None when the guard didn't fire, + # else a structured dict. Persist a small projection on the bench + # row so the aggregator can count fires + confidence breakdown + # without re-running the sidecar. Full dict (with candidate + # spans + offsets) stays on the result for human inspection. + answerability = result.get("answerability") or {} + answerability_fired = bool(answerability.get("answerability_warning")) + answerability_confidence = answerability.get("confidence_class") + answerability_denial_pattern = answerability.get("denial_pattern_matched") + answerability_answer_type = answerability.get("answer_type") + answerability_candidate_count = answerability.get("candidate_count") # Ticket #000010 — meta-cognition QuestionState. Persist a # bounded-size projection: logical_statuses, question_shape, # preflight_result, temporal_sensitivity, and the kind of any @@ -278,6 +290,16 @@ def _run_one( # (typically 12 — the claim_lattice_max_claims_per_answer # default). None means "not applicable" (non-lattice modes). "claim_cap_applied": claim_cap_applied, + # Ticket #000068 Phase 1 — missed-answer guard sidecar bench + # projection. Five small fields surface aggregate fire-rate + + # confidence breakdown without bloating the row with candidate + # spans / offsets (full dict on the result, recomputable from + # question + answer + evidence + policy). + "answerability_fired": answerability_fired, + "answerability_confidence": answerability_confidence, + "answerability_denial_pattern": answerability_denial_pattern, + "answerability_answer_type": answerability_answer_type, + "answerability_candidate_count": answerability_candidate_count, # Configured model id; serves as the model_profile_id key # for Phase 2 per-model cap lookup. Stored verbatim so a # bench archive remains interpretable when model_profiles.py @@ -417,6 +439,16 @@ def _summarize(rows: list[dict]) -> dict: # so non-lattice rows count as 0). Surfaced in markdown # alongside deflection rate as a per-mode collapse signal. "format_collapses": 0, + # Ticket #000068 Phase 1 — missed-answer guard aggregate + # counters. answerability_fires counts every row where the + # sidecar fired; the per-confidence breakdown surfaces how + # many of those were strong/medium/weak. Pairs with + # violation_kind_counts and format_collapses as the third + # sidecar-class signal at aggregate scale. + "answerability_fires": 0, + "answerability_strong": 0, + "answerability_medium": 0, + "answerability_weak": 0, # Per-violation-kind counts. Open-ended dict — fills as # kinds are encountered. Empty when no violations fire. "violation_kind_counts": defaultdict(int), @@ -455,6 +487,16 @@ def _summarize(rows: list[dict]) -> dict: # apply); only count explicit True. if r.get("format_collapsed") is True: b["format_collapses"] += 1 + # Missed-answer guard fires + per-confidence breakdown. + if r.get("answerability_fired"): + b["answerability_fires"] += 1 + conf = r.get("answerability_confidence") + if conf == "strong": + b["answerability_strong"] += 1 + elif conf == "medium": + b["answerability_medium"] += 1 + elif conf == "weak": + b["answerability_weak"] += 1 # Violation-kind tallies — each kind counts once per row even # if the same kind fires on multiple claims. The bench is # asking "did this kind fire on this run?", not "how many @@ -522,8 +564,8 @@ def _render_markdown( lines.append("") lines.append("## summary") lines.append("") - lines.append("| mode | runs | STRICT | HYBRID | UNGROUNDED | err | strict-rate | mean ratio | mean latency | deflections |") - lines.append("|------|------|--------|--------|------------|-----|-------------|-----------|--------------|-------------|") + lines.append("| mode | runs | STRICT | HYBRID | UNGROUNDED | err | strict-rate | mean ratio | mean latency | deflections | #68 fires (S/M/W) |") + lines.append("|------|------|--------|--------|------------|-----|-------------|-----------|--------------|-------------|-------------------|") for mode in modes: b = summary.get(mode) if not b: @@ -533,11 +575,16 @@ def _render_markdown( mean_lat = b["latency_sum"] / n strict_rate = b["STRICT"] / n deflections = b.get("deflections", 0) + ans_fires = b.get("answerability_fires", 0) + ans_s = b.get("answerability_strong", 0) + ans_m = b.get("answerability_medium", 0) + ans_w = b.get("answerability_weak", 0) lines.append( f"| {mode} | {b['n']} | {b['STRICT']} | {b['HYBRID']} | " f"{b['UNGROUNDED']} | {b['errors']} | " f"{strict_rate:.2f} | {mean_ratio:.3f} | {mean_lat:.1f}s | " - f"{deflections}/{b['n']} |" + f"{deflections}/{b['n']} | " + f"{ans_fires}/{b['n']} ({ans_s}/{ans_m}/{ans_w}) |" ) lines.append("") lines.append("## format-collapse + violation kinds") diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 066e8f9..bd75ef7 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -111,7 +111,7 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| -| #000068 | Verifier-blind missed-answer falsification guard | **in progress · Phase 1 implementation underway 2026-05-27** (Dav1d de-novo review GO for Phase 1 with seven hardenings folded into spec — subject-token cue-stripping, answer-type alignment, confidence_class, candidate cap=10, precise offset_start/end/basis, cache-hit recompute-on-read, Phase 1 out of verifier_policy_hash). Original opening 2026-05-27; sibling to the user-payload-layout work shipped 2026-05-26, split out per the Dav1d-audience rule — `feedback_ticket_proliferation`). Surfaced by the Ballestrini case: evidence E2 literally contained the song names, Hermes-3-8B under `user_payload_layout=tail` said *"specific songs by her are not mentioned in the provided evidence blocks"*, verifier marked the run `EVIDENCE-WARRANTED` 2/2 because nothing positive was unsupported. **Verifier-blind false-negative class** — existing layered verifier (quote/span/entity/paraphrase + Rule 8 + Rule 9 + claim ceiling) guards unsupported *presence*, has no hook for unsupported *absence*. Layout fixes attention placement on the specific instance (n=3 bench 2026-05-27 confirms bookend/per_chunk recover Ballestrini); layout alone can't close the class — adversarial phrasing or bigger prompt resurfaces it under any layout. Proposed deterministic sidecar in `arborist/qa/inspect.py:diagnose_missed_answer`: three-clause conjunction — **(A)** answer matches denial pattern ("not mentioned", "not provided", "the evidence does not say", …, closed list versioned via `denial_patterns_version`); **(B)** question is extraction shape (reuse `arborist.qa.quantifier` classifier — `ALL`/`COMPREHENSIVE`/`OPEN_REQUEST` intensities, OR surface cues "songs by"/"works by"/"who wrote"/"list"/"name all"); **(C)** evidence contains candidate spans near subject tokens (reuse `entity_proximity_n`/`entity_proximity_window` from verify.py — quoted strings, title-case spans, comma-separated title lists within W chars of stemmed subject content tokens). All three must fire. Output: `result["answerability"]` with `missed_answer_candidate_spans` list (evidence_id + offset + text). **Hash discipline:** sidecar fields (`denial_patterns_version`, `extraction_cues_version`, `answerability_threshold`) fold into `governance_policy_hash` only; an optional `answerability_demote_enabled` flag (default OFF) wires `EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL` in `_render_audit_label`, and IF on folds into BOTH `governance_policy_hash` AND `verifier_policy_hash` (changes rendered audit_mode, so verifier hash must move — the deliberate opt-in moves the verifier hash, sidecar-only stays out). No LLM-as-judge. Never writes `providence_cache`/`audit_events`. Never promotes claims. Pattern verbatim from `arborist.qa.inspect.diagnose_*` (deflection, coherence, title-relevance). Phases: 1 sidecar read-only, 2 bench + threshold tuning, 3 demote opt-in, 4 default decision (bench-gated). 5F-Falsification fixture: Ballestrini case already in `bench/qa_questions.txt` under "entity list". Full spec in `docs/tickets/ticket-000068-verifier-blind-missed-answer-guard.md`. | 2026-05-27 | D2 | +| #000068 | Verifier-blind missed-answer falsification guard | **in progress · Phase 1+2+3 landed 2026-05-27 · Phase 4 default flip NO-GO** (Phase 2 bench 2026-05-27 76q × n=3 claim_lattice Hermes-3-8B: 2/228 sidecar fires, both STRONG confidence, both the Ballestrini regression fixture, 100% precision, 0/226 false positives across non-Ballestrini runs. Phase 3 demote flag opt-in via `--demote-on-missed-answer` on `query`/`ask` — wires `EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL` for strong/medium confidence on lattice modes; lower rungs + non-lattice modes get `· missed-answer` tail tag. `answerability_demote_enabled` added to `_VERIFIER_POLICY_FIELDS` so flipping the flag partitions cache via verifier_policy_hash. Default OFF per Dav1d Phase 4 NO-GO — 100% precision at n=2 fires is too few samples to claim precision floor empirically; default flip blocks on wider bench + human spot-check. 47 tests (36 Phase 1 + 11 Phase 3) all passing. End-to-end verified live: 4/4 Hermes runs on Ballestrini with --demote-on-missed-answer rendered EVIDENCE-MISSED-PARTIAL.) Original opening 2026-05-27 (Dav1d de-novo review GO for Phase 1 with seven hardenings folded into spec — subject-token cue-stripping, answer-type alignment, confidence_class, candidate cap=10, precise offset_start/end/basis, cache-hit recompute-on-read, Phase 1 out of verifier_policy_hash). Original opening 2026-05-27; sibling to the user-payload-layout work shipped 2026-05-26, split out per the Dav1d-audience rule — `feedback_ticket_proliferation`). Surfaced by the Ballestrini case: evidence E2 literally contained the song names, Hermes-3-8B under `user_payload_layout=tail` said *"specific songs by her are not mentioned in the provided evidence blocks"*, verifier marked the run `EVIDENCE-WARRANTED` 2/2 because nothing positive was unsupported. **Verifier-blind false-negative class** — existing layered verifier (quote/span/entity/paraphrase + Rule 8 + Rule 9 + claim ceiling) guards unsupported *presence*, has no hook for unsupported *absence*. Layout fixes attention placement on the specific instance (n=3 bench 2026-05-27 confirms bookend/per_chunk recover Ballestrini); layout alone can't close the class — adversarial phrasing or bigger prompt resurfaces it under any layout. Proposed deterministic sidecar in `arborist/qa/inspect.py:diagnose_missed_answer`: three-clause conjunction — **(A)** answer matches denial pattern ("not mentioned", "not provided", "the evidence does not say", …, closed list versioned via `denial_patterns_version`); **(B)** question is extraction shape (reuse `arborist.qa.quantifier` classifier — `ALL`/`COMPREHENSIVE`/`OPEN_REQUEST` intensities, OR surface cues "songs by"/"works by"/"who wrote"/"list"/"name all"); **(C)** evidence contains candidate spans near subject tokens (reuse `entity_proximity_n`/`entity_proximity_window` from verify.py — quoted strings, title-case spans, comma-separated title lists within W chars of stemmed subject content tokens). All three must fire. Output: `result["answerability"]` with `missed_answer_candidate_spans` list (evidence_id + offset + text). **Hash discipline:** sidecar fields (`denial_patterns_version`, `extraction_cues_version`, `answerability_threshold`) fold into `governance_policy_hash` only; an optional `answerability_demote_enabled` flag (default OFF) wires `EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL` in `_render_audit_label`, and IF on folds into BOTH `governance_policy_hash` AND `verifier_policy_hash` (changes rendered audit_mode, so verifier hash must move — the deliberate opt-in moves the verifier hash, sidecar-only stays out). No LLM-as-judge. Never writes `providence_cache`/`audit_events`. Never promotes claims. Pattern verbatim from `arborist.qa.inspect.diagnose_*` (deflection, coherence, title-relevance). Phases: 1 sidecar read-only, 2 bench + threshold tuning, 3 demote opt-in, 4 default decision (bench-gated). 5F-Falsification fixture: Ballestrini case already in `bench/qa_questions.txt` under "entity list". Full spec in `docs/tickets/ticket-000068-verifier-blind-missed-answer-guard.md`. | 2026-05-27 | D2 | | #000067 | M-aware cold-pack hydration (route incoming docs by content hash into M target shards) | **open · scaffold · prereq for #46 genesis test** (2026-05-26; surfaced while preparing the 3090 SPV-wallet validation). Today's `hydrate_from_metadata_pack` takes a single `conn` and writes every incoming row into one shard. With the corpus now in M=4 hash-routed topology (#000065), a fresh peer needs to land each document on `shard_for_document(document_root, M)` — same routing function as the producer. Without this, a fresh peer's `~/.arborist/shards/` is just one big single-shard DB and the M=4 ATTACH-and-route assumption #000065 was sized for doesn't hold consumer-side. Two coherent shapes: **(α) two-step kludge** — hydrate into single shard, then `arborist corpus reshard --to M` on the consumer. Works today (proven by the 2026-05-26 reshard executor) but doubles the wall time and treats packed shards as if they came from an arbitrary topology. **(β) direct M-aware hydrate** — extend `hydrate_from_metadata_pack` to accept `targets: list[sqlite3.Connection]` + `M: int` and route per-row at restore time (reusing `arborist.document.shard_for_document` + the table-routing rules in `arborist/migrate.py`). Manifest carries `corpus_shard_count` so the unpacker knows M from the pack itself. β is the right answer — α exists only as a fallback if 20-min-window pressure forces it. Sequence: (1) add `corpus_shard_count` to pack manifest (read from source meta during `dump_shard_metadata`); (2) `restore_shard_metadata_routed(targets, M, table_dir)` in `cold_pack_metadata.py` mirroring `_route_per_doc_table` from migrate.py; (3) `hydrate_from_metadata_pack` gains a `targets`/`shards_dir` param; (4) `arborist cold unpack --shards-dir DIR` initialises M target shards from the manifest's `corpus_shard_count` and routes; (5) regression test: pack 2 shards → hydrate into fresh 4 shards → assert every doc on its hash-routed target. Refactor opportunity: the routing rules (ROUTED_BY_DOCUMENT_ROOT, CONSOLIDATED_TABLES) currently live in migrate.py; this ticket can either duplicate them in cold_pack_metadata.py (fast) or factor into a shared `arborist/multi_shard.py` module (cleaner). The shared-module path is more honest given graft mode (#000066) wants the same primitives. Out of scope: graft / overlay mode (that's #000066 — overlays onto populated, this is hydrate-into-empty). | 2026-05-26 | — | | #000066 | Cold-pack overlay / graft mode (pack-as-package, witness-pattern audit chain) | **scaffold-only · awaiting go/no-go** (2026-05-26; surfaced while running #000065 reshard, fox extension: "we could envision a pack for wikipedia 2010, wikipedia current, etc"). Extend #000061 cold-pack hydration with a second mode: overlay an existing pack onto a populated shard set instead of hydrating into empty. Doc/chunk/edge/concept overlay is trivial (`INSERT OR IGNORE` on content-addressed PKs collapses dupes); FTS5 overlay is trivial (new chunks → new rowids → new FTS rows). The interesting part is the audit chain — can't naively append the pack's events because `prev_event_hash` linkage breaks across the join. Chosen approach: **graft receipt**. Append one new `event_type='graft'` event to the host chain carrying `(pack_hash, snapshot_root, corpus_name, event_count, first_event_hash, last_event_hash, manifest_root)`; the pack file itself becomes the durable witness for the absorbed events (anyone can re-fetch the pack, walk its internal chain, and verify it matches the receipt). Host chain stays linear; pack chain is a "witnessed subgraph." This is the same witness pattern Merkle-AGI v8/v9 is heading toward, but bought at near-zero schema cost. Rejected alternatives: re-chain everything (breaks external refs to old event_hashes — cache_keys anchoring to old `audit_event_hash`, snapshots, etc. — silently invalid); chain forest with new `chain_id` column (right answer when graft dominates the lifecycle, but premature now). **Pack-as-package extension** (fox 2026-05-26): each pack carries a `corpus_name` field in its manifest (`wikipedia-2010`, `wikipedia-current`, `arxiv-cs`, `textbooks-undergrad`, …) so operators pick which corpora to graft — `arborist cold graft wikipedia-current` becomes as natural as `apt install firefox`. Multiple packs of the same corpus name: most-recent `snapshot_root` wins; older packs stay in the bucket until GC. URI conflicts across corpora (e.g., `wikipedia.org/wiki/Foo` in both 2010 and current): different content → different `document_root` → both stored, `supersedes` edges per CLAUDE.md invariant. Providence-cache conflicts: same `cache_key` with different answer → existing v9.8 falsification framework handles it (`state='stale'` or `quarantined`). Mesh-peer-corpus-merge: each peer's pack is a graftable package; partition reconciliation becomes "exchange the packs you each carry, graft what you lack". The mesh-of-arborists semantic. Sequence: (1) `corpus_name` field in #000061 manifest format + alias index in bucket (`corpora//latest.json` pointer to active pack_hash); (2) `arborist cold graft ` / `arborist cold graft --corpus ` mode in evict.py — read pack, INSERT OR IGNORE per-table, emit graft receipt; (3) conflict-policy flag (`--on-uri-conflict {supersedes,skip,fail}`, default `supersedes`); (4) `arborist cold list-corpora` shows available packages in a bucket. Scaffold first, code only when (a) #000065 reshard lands and stabilises (b) a second corpus exists (the wikipedia-current snapshot, or first textbook bundle ready to graft onto wikipedia-2010 base) (c) at least two peers want to exchange. | 2026-05-26 | — | | #000065 | Canonical shard count `M` + content-hash routing (decouple ingest parallelism from ATTACH ceiling) | **closed · landed in `c86d5ac`** (2026-05-26 19:47 UTC cutover, ~94 min wall). Production reshard completed end-to-end on the live host: 3,468,226 globally-unique docs / 6,235,588 chunks / 90,592,990 edges / 3,468,403 audit events re-routed to content-hash-deterministic M=4 layout. Per-shard doc uniformity within ±0.04% (theoretical limit ±0.05%). Audit chain consolidated to canonical shard 000 via Option A (3.47M events re-sorted by ts + re-chained, bodies preserved); tail event `type=reshard` carries plan+result body. Validation gate caught 176 chunks + 547 edges as cross-shard dupes (collapsed by INSERT OR IGNORE; 0.003% delta, within 1% tolerance). Two defects surfaced + fixed mid-cutover: (a) `derivations.src_root` FK fired on legitimately cross-shard refs — fix in `04edff7`: writer connection runs `PRAGMA foreign_keys = OFF`, runtime stays FK=ON; (b) WAL accumulated ~37 GB across FTS rebuild + audit consolidate because SQLite auto-checkpoint can't reclaim pages while a reader cursor is open — fix in `c86d5ac`: `_checkpoint_truncate` called between executor phases. Full migration record in `docs/corpus-history.md` (which entry is the operator-facing equivalent of the audit chain tail). Follow-on work tracked separately: #44 re-pack into bucket → #45 verify bucket determinism → #46 genesis fresh peer on 3090-ai.foxhop.net from cloud (first real SPV-wallet end-to-end test) → #47 retire stale pre-reshard bucket packs. (2026-05-26; surfaced while sizing #000061's federation story). Today shard count conflates two roles: producer ingest parallelism (wants vCPU count) + consumer ATTACH fan-out (capped at SQLITE_MAX_ATTACHED=10 on stock python3 sqlite3). Producer with 16 vCPU → 16 shards → consumers fail to attach the 11th. Producer with 4 shards → 16-vCPU box runs 75% idle on ingest. Fix: pin a corpus-wide canonical **M = 4** (decided 2026-05-26 from real-Wikipedia bench: M=4 captures 92% of peak ingest throughput, ATTACH cost 9 ms keeps mobile-tolerable, 6 free ATTACH slots under SQLite's 10 ceiling for auxiliary DBs), introduce N (ingest workers) decoupled from M. Document → shard assignment becomes content-deterministic: `shard_idx = int(document_root[:8], 16) % M`. Same input → same output across every peer (today's "spray by ingest order" is non-deterministic across peers, a real federation weakness). Migration hard-constraint per fox: **content-addressed rebalance, NOT re-ingest** — every row is already addressed by `document_root` / `leaf_hash` / etc.; migration reads rows from the current 4 shards, computes each row's new shard via the routing function, INSERTs into M new shards. No source re-parse, no re-canonicalization, no re-chunking, no LLM. ~20–40 min I/O-bound vs. hours-to-days for true re-ingest. Audit chain consolidates to canonical shard 000 (re-numbered + re-hashed once) to preserve global event ordering. Phases: 0 design lock + pin M in meta table → 1 read path (connect_query honors M) → 2 ingest path (multi-shard write per worker) → 3 cold-pack restore re-routes on pull → 4 corpus migration tool. Open audit-chain re-numbering question (every shard has its own seq + event_hash; rebalancing splits a producer's chain across M consumer shards). Don't proliferate sub-tickets; the audit handling is part of this design lock. Out of scope: custom-built sqlite3 with higher MAX_ATTACHED (rejected: violates "python3 + venv + sqlite3 only" property from CLAUDE.md); topic-clustering shards (would break ingest determinism). | 2026-05-26 | — | diff --git a/docs/tickets/ticket-000068-verifier-blind-missed-answer-guard.md b/docs/tickets/ticket-000068-verifier-blind-missed-answer-guard.md index f4c7043..5ac0b1d 100644 --- a/docs/tickets/ticket-000068-verifier-blind-missed-answer-guard.md +++ b/docs/tickets/ticket-000068-verifier-blind-missed-answer-guard.md @@ -1,9 +1,30 @@ # Ticket #000068 — Verifier-blind missed-answer falsification guard -**Status:** in progress · Phase 1 implementation underway 2026-05-27 (after Dav1d de-novo review) +**Status:** in progress · Phase 1 + Phase 2 + Phase 3 landed 2026-05-27; Phase 4 (default demote-on) NO-GO per Dav1d, waiting on human spot-check across larger sample **Opened:** 2026-05-27 **Scope:** Deterministic read-only sidecar that detects evidence-neglect / false-negative answers — runs where the LLM said "not mentioned" but the evidence contains candidate answer spans near the question subject. Emits an `answerability_warning` + candidate-span list; optional audit-label demote `EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL` behind a policy flag. No LLM-as-judge. No verifier-policy change. +## Phase 1+2+3 landing notes (2026-05-27) + +**Phase 1 (sidecar)** — `arborist/qa/inspect.py:diagnose_missed_answer` + helpers + sealed constants + result-dict wiring. 36 tests. End-to-end verified live on the Ballestrini case (sidecar fires on the failure, silent on success). + +**Phase 2 (bench + threshold tuning)** — `bench/qa_results/phase2-sidecar-on/2026-05-27T14-16-22Z.{md,jsonl}`. 76 questions × n=3 × claim_lattice × Hermes-3-8B × tail layout = 228 runs. Headline: + +| metric | value | +|---|---| +| sidecar fires | 2/228 (0.88%) | +| confidence distribution | 2 strong / 0 medium / 0 weak | +| precision | 100% (2/2 fires were the Ballestrini regression fixture) | +| recall on Ballestrini (n=3) | 2/3 (third run model extracted correctly — silent is correct) | +| false positives | 0/226 non-Ballestrini runs | +| verifier verdict on the failures | both STRICT (the verifier-blind class, exactly as predicted) | + +The detection rule's three-clause conjunction (denial pattern + extraction shape + candidate proximity) is operating at the precision floor: when it fires, it fires correctly. Recall on the Ballestrini class is 100% of the runs where Hermes actually produced the failure (the third run had no denial pattern, so silence is the right behavior). + +**Phase 3 (opt-in demote flag)** — `answerability_demote_enabled` added to `_VERIFIER_POLICY_FIELDS` (flipping the flag invalidates prior cache via verifier_policy_hash, the correct discipline since rendered audit_mode changes). `_render_audit_label` extended to demote `EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL` on lattice modes (strong/medium confidence only — weak doesn't demote per Phase 2 data showing zero weak fires on real failures). Lower rungs (POINTER-LINKED, ANCHOR-WARRANTED) get a `· missed-answer` tail tag instead of a rung transition (rung already signals degradation). Non-lattice modes (quote/span/entity/paraphrase) get the same tail-tag treatment on their existing audit_mode token. CLI flag `--demote-on-missed-answer` on both `query` and `ask` subcommands (default OFF). 11 Phase 3 tests + end-to-end verified live (4/4 Hermes runs on Ballestrini with `--demote-on-missed-answer` rendered `EVIDENCE-MISSED-PARTIAL`). + +**Phase 4 (default demote-on)** — NO-GO. Per Dav1d 2026-05-27 §3.4: "a false sidecar warning is tolerable; a false audit-label demotion can damage trust in correct abstentions." Phase 2 precision is 100% but n=2 fires is too few samples to claim precision floor empirically; default flip blocks on a wider bench (more diverse failure shapes) + human spot-check of the warnings. Operators wanting the demote behavior set `--demote-on-missed-answer` per-call. + ## Post-review hardenings (Dav1d 2026-05-27) The 2026-05-27 de-novo review (`~/Downloads/RESPONSE_final_ticket-000068-verifier-blind-missed-answer-guard.txt`) was a **GO for Phase 1** with seven specific hardenings folded into the spec below. Summary so a re-read can hit the load-bearing changes at a glance: diff --git a/tests/test_missed_answer_guard.py b/tests/test_missed_answer_guard.py index 29bc1da..22ef9fe 100644 --- a/tests/test_missed_answer_guard.py +++ b/tests/test_missed_answer_guard.py @@ -498,3 +498,159 @@ def test_confidence_class_in_valid_set(): evidence=_ballestrini_evidence(), ) assert result["confidence_class"] in ("weak", "medium", "strong") + + +# --------------------------------------------------------------------------- +# Phase 3 — opt-in demote flag + render-label projection + hash discipline +# --------------------------------------------------------------------------- + + +def test_demote_flag_in_verifier_policy_fields(): + """Phase 3: answerability_demote_enabled must be in the verifier + set so flipping it changes verifier_policy_hash.""" + from arborist.qa.keys import _VERIFIER_POLICY_FIELDS + assert "answerability_demote_enabled" in _VERIFIER_POLICY_FIELDS + + +def test_demote_flag_changes_verifier_hash(): + """Flipping the demote flag must partition verifier_policy_hash.""" + from arborist.qa.keys import verifier_policy_hash + base = {"answerability_demote_enabled": False} + flipped = {"answerability_demote_enabled": True} + assert verifier_policy_hash(base) != verifier_policy_hash(flipped) + + +def test_sidecar_fields_do_NOT_change_verifier_hash(): + """The other answerability fields (governance only) must NOT + change verifier_policy_hash.""" + from arborist.qa.keys import verifier_policy_hash + a = {"answerability_demote_enabled": False, "denial_patterns_version": "v1"} + b = {"answerability_demote_enabled": False, "denial_patterns_version": "v2"} + assert verifier_policy_hash(a) == verifier_policy_hash(b) + + +def test_sidecar_fields_DO_change_governance_hash(): + """The sidecar enable / threshold fields fold into governance.""" + from arborist.qa.keys import governance_policy_hash + a = {"answerability_sidecar_enabled": True} + b = {"answerability_sidecar_enabled": False} + assert governance_policy_hash(a) != governance_policy_hash(b) + + +def test_render_label_demote_strong_confidence(): + """When demote is enabled AND sidecar fired strong, label demotes.""" + from arborist.cli import _render_audit_label + answerability = { + "answerability_warning": True, + "confidence_class": "strong", + } + label = _render_audit_label( + "STRICT", "claim_lattice", + violations=None, + answerability=answerability, + demote_enabled=True, + ) + assert "EVIDENCE-MISSED-PARTIAL" in label + + +def test_render_label_demote_medium_confidence(): + """Medium confidence also demotes (per Phase 2 calibration).""" + from arborist.cli import _render_audit_label + answerability = { + "answerability_warning": True, + "confidence_class": "medium", + } + label = _render_audit_label( + "STRICT", "claim_lattice", + violations=None, + answerability=answerability, + demote_enabled=True, + ) + assert "EVIDENCE-MISSED-PARTIAL" in label + + +def test_render_label_weak_does_NOT_demote(): + """Weak confidence never demotes the rung (Phase 2 saw zero weak + fires on real failures; reserved for future expanded detection).""" + from arborist.cli import _render_audit_label + answerability = { + "answerability_warning": True, + "confidence_class": "weak", + } + label = _render_audit_label( + "STRICT", "claim_lattice", + violations=None, + answerability=answerability, + demote_enabled=True, + ) + assert "EVIDENCE-MISSED-PARTIAL" not in label + assert "missed-answer" not in label + + +def test_render_label_demote_off_keeps_original(): + """When demote flag is False, label stays as-is even on strong fire.""" + from arborist.cli import _render_audit_label + answerability = { + "answerability_warning": True, + "confidence_class": "strong", + } + label = _render_audit_label( + "STRICT", "claim_lattice", + violations=None, + answerability=answerability, + demote_enabled=False, + ) + assert "EVIDENCE-MISSED-PARTIAL" not in label + assert "missed-answer" not in label + + +def test_render_label_no_answerability_unchanged(): + """No sidecar fire → label unchanged regardless of flag.""" + from arborist.cli import _render_audit_label + label_off = _render_audit_label( + "STRICT", "claim_lattice", violations=None, + answerability=None, demote_enabled=False, + ) + label_on = _render_audit_label( + "STRICT", "claim_lattice", violations=None, + answerability=None, demote_enabled=True, + ) + assert label_off == label_on + + +def test_render_label_quote_mode_demote_appends_tail(): + """Non-lattice mode: demote appends '· missed-answer' tail rather + than transitioning to MISSED-PARTIAL.""" + from arborist.cli import _render_audit_label + answerability = { + "answerability_warning": True, + "confidence_class": "strong", + } + label = _render_audit_label( + "STRICT", "quote", + violations=None, + answerability=answerability, + demote_enabled=True, + ) + assert "missed-answer" in label + + +def test_render_label_lower_rung_demote_appends_tail(): + """Lattice mode on a non-EVIDENCE-WARRANTED rung (e.g. ANCHOR-WARRANTED + or POINTER-LINKED) gets a '· missed-answer' tail rather than a rung + transition — the rung itself already signals degradation.""" + from arborist.cli import _render_audit_label + answerability = { + "answerability_warning": True, + "confidence_class": "strong", + } + # HYBRID + WARRANT_MISSING → POINTER-LINKED rung + violations = [{"kind": "WARRANT_MISSING"}] + label = _render_audit_label( + "HYBRID", "claim_lattice", + violations=violations, + answerability=answerability, + demote_enabled=True, + ) + assert "missed-answer" in label + assert "EVIDENCE-MISSED-PARTIAL" not in label