diff --git a/arborist/qa/inspect.py b/arborist/qa/inspect.py index 94f6f25..182e9d1 100644 --- a/arborist/qa/inspect.py +++ b/arborist/qa/inspect.py @@ -1308,3 +1308,554 @@ def inspect_cache_key( "coherence": coherence, "authorship": authorship, } + + +# ---------------------------------------------------------------------- +# Ticket #000068 — verifier-blind missed-answer falsification guard. +# +# Deterministic read-only sidecar. No model calls. No audit writes. +# Recomputable from (question, answer, evidence, policy) — Phase 1 +# stores nothing, the next lookup recomputes. +# +# Fires on the three-clause conjunction: +# A. answer matches a sealed denial pattern (e.g. "not mentioned") +# B. question is extraction/list-shaped (quantifier ∈ broad rungs +# OR surface cue like "songs by") +# C. evidence contains candidate spans matching the answer_type, +# within a proximity window of cleaned subject tokens +# +# Hardenings folded in from Dav1d 2026-05-27 review: +# - subject tokens strip cue/relation/stopwords +# ("songs by veronica ballestrini" → ["veronica","ballestrini"]) +# - candidate kind aligns with answer_type +# - confidence_class is deterministic: weak | medium | strong +# - candidate cap = 10 (per the per_chunk-quote-inflation lesson) +# - offsets are start/end/basis, never ambiguous single offset +# ---------------------------------------------------------------------- + +ANSWERABILITY_DIAGNOSTIC_VERSION = "missed-answer-v1" +DENIAL_PATTERNS_VERSION = "denial-patterns-v1" +EXTRACTION_CUES_VERSION = "extraction-cues-v1" + +# Sealed denial-phrase list (Dav1d §4 — start conservative, do not +# widen to single tokens like "absent" / "lack" / "missing" until +# measured FN justifies it). All matched after casefold + +# whitespace normalization. +_DENIAL_PATTERNS_V1: tuple[str, ...] = ( + "not mentioned", + "not provided", + "the evidence does not say", + "does not mention", + "no specific", + "no evidence", + "cannot determine from the provided evidence", + "is not stated", + "is not specified", +) + +# Extraction-shape surface cues + their answer_type mapping (Dav1d §5 +# + §8). Order matters: leftmost match wins for cue reporting, but +# every cue contributes to the answer_type union (we pick the first +# matched cue's answer_type for downstream candidate alignment). +# answer_type values: +# "title_like" — songs, works, books, papers; quoted strings, +# title-case spans, comma-list items +# "person" — who wrote/composed/directed/acted; person-like +# title-case spans +# "date" — when/what year; year/date spans +_EXTRACTION_CUES_V1: tuple[tuple[str, str], ...] = ( + # Title-like answer + ("songs by", "title_like"), + ("works by", "title_like"), + ("books by", "title_like"), + ("papers by", "title_like"), + ("albums by", "title_like"), + ("singles by", "title_like"), + ("name all", "title_like"), + ("name the", "title_like"), + ("list of", "title_like"), + ("list all", "title_like"), + ("what are the", "title_like"), + # Person answer + ("who wrote", "person"), + ("who composed", "person"), + ("who directed", "person"), + ("who acted in", "person"), + ("who plays", "person"), + ("who painted", "person"), + # Date answer + ("when did", "date"), + ("when was", "date"), + ("what year", "date"), + ("what date", "date"), +) + +# Cue/relation/stopwords to strip from subject-token extraction +# (Dav1d §7). For "songs by veronica ballestrini" the subject is +# ["veronica","ballestrini"] not all four — without this the guard +# false-triggers on generic title-case spans near "songs"/"by". +_SUBJECT_CUE_STOPWORDS: frozenset[str] = frozenset({ + # Cue noun heads + "songs", "song", "works", "work", "books", "book", + "papers", "paper", "albums", "album", "singles", "single", + "list", "lists", "name", "names", "examples", "example", + # Wh/relative pronouns + "who", "what", "which", "whom", "whose", "where", "when", "why", + # Relation verbs + "wrote", "composed", "directed", "acted", "plays", "played", + "painted", "paints", + # Prepositions / function words common in cue framing + "by", "of", "for", "with", "about", "from", "to", "in", "on", + "the", "a", "an", "all", "any", "some", + # Auxiliaries + "is", "are", "was", "were", "did", "does", "do", "has", "have", + "had", "be", "been", "being", + # Generic conjunctions / question particles + "and", "or", "but", "as", + "me", +}) + +# Compiled candidate-span patterns. Each yields (start, end, text). +# Quoted-string: opening/closing ASCII double quote or curly quote. +_RE_QUOTED_STRING = re.compile( + r'(?:"([^"\n]{1,80})"|“([^”\n]{1,80})”)' +) +# Title-case run: 2+ consecutive Capitalized Words. Tolerates +# apostrophes ("Don't") and hyphens. Length cap keeps cost bounded. +_RE_TITLE_CASE_SPAN = re.compile( + r"\b(?:[A-Z][a-zA-Z'\-]{1,30}(?:\s+[A-Z][a-zA-Z'\-]{1,30}){1,5})\b" +) +# Comma-list item: a word inside a comma-separated run. Two passes; +# first identify the run, then split. We approximate the run as +# " ,? <Title> , and <Title>" — common Wikipedia prose shape. +_RE_COMMA_LIST_RUN = re.compile( + r"\b([A-Z][a-zA-Z'\-]{1,30})(?:,\s*[A-Z][a-zA-Z'\-]{1,30}){1,}" + r"(?:,?\s+and\s+[A-Z][a-zA-Z'\-]{1,30})?\b" +) +_RE_COMMA_LIST_ITEM = re.compile(r"\b[A-Z][a-zA-Z'\-]{1,30}\b") +# Year: 1xxx / 2xxx (4-digit; 3-digit ambiguous; skip). +_RE_YEAR = re.compile(r"\b((?:1[0-9]|20)\d{2})\b") +# Date: Month + day-or-year. Conservative — only catches written-out +# month names (Jan-Dec / January-December). +_RE_DATE = re.compile( + r"\b(" + r"Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|" + r"Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|" + r"Nov(?:ember)?|Dec(?:ember)?" + r")\s+\d{1,2}(?:,\s*\d{4})?\b" +) + + +def _match_denial_pattern( + answer_text: str, patterns: tuple[str, ...] = _DENIAL_PATTERNS_V1 +) -> str | None: + """Casefolded substring match against the sealed denial list. + Returns the first matched phrase or None.""" + if not answer_text: + return None + norm = " ".join(answer_text.lower().split()) + for pat in patterns: + if pat in norm: + return pat + return None + + +def _classify_extraction_shape( + question: str, + quantifier_intensity: str | None = None, +) -> dict | None: + """Return extraction-shape info if the question is extraction-shaped. + + Returns None when neither a broad quantifier nor a surface cue + matches. When matched, returns: + { + "shape": "list" | "single_entity" | "open_list", + "answer_type": "title_like" | "person" | "date", + "cue": str, # matched surface phrase (or "quantifier") + "quantifier_intensity": str | None, + } + """ + if not question: + return None + norm = " ".join(question.lower().split()) + # Surface-cue match (preferred — gives answer_type) + for cue, ans_type in _EXTRACTION_CUES_V1: + if cue in norm: + shape = "list" if ans_type == "title_like" else ( + "single_entity" if ans_type == "person" else "single_entity" + ) + return { + "shape": shape, + "answer_type": ans_type, + "cue": cue, + "quantifier_intensity": quantifier_intensity, + } + # Quantifier fallback (no surface cue, but broad rung) + broad_rungs = {"ALL", "COMPREHENSIVE", "OPEN_REQUEST", "MANY", "PLURAL"} + if quantifier_intensity in broad_rungs: + return { + "shape": "open_list", + "answer_type": "title_like", + "cue": "quantifier:" + quantifier_intensity, + "quantifier_intensity": quantifier_intensity, + } + return None + + +def _extract_subject_tokens(question: str) -> list[str]: + """Cleaned subject tokens (Dav1d §7). Removes cue/relation/stop + words. Preserves quoted phrases, hyphenated terms, proper-noun + runs in original case. + + Output is byte-deterministic — same input always yields same + list. Order preserves question order; duplicates dropped. + """ + if not question: + return [] + # Strip trailing punctuation that confuses tokenisation + cleaned = question.strip().rstrip("?.!,;:") + # Preserve quoted phrases as a single token (joined by '_') + quoted_phrases: list[str] = [] + def _capture_quoted(m: re.Match) -> str: + inner = m.group(1) or m.group(2) or "" + token = "_".join(inner.split()) + quoted_phrases.append(token) + return f" __Q{len(quoted_phrases) - 1}__ " + cleaned = _RE_QUOTED_STRING.sub(_capture_quoted, cleaned) + out: list[str] = [] + seen: set[str] = set() + for raw in cleaned.split(): + # Replace quoted placeholders + if raw.startswith("__Q") and raw.endswith("__"): + try: + idx = int(raw[3:-2]) + tok = quoted_phrases[idx] + except (ValueError, IndexError): + continue + else: + # Strip leading/trailing punctuation but keep hyphens + tok = raw.strip(".,;:?!\"'()[]") + if not tok: + continue + low = tok.lower() + if low in _SUBJECT_CUE_STOPWORDS: + continue + # Require ≥2 chars (drop single letters that survived stopwording) + if len(tok) < 2: + continue + # Casefold for the subject set, but keep original-case for proper + # nouns when they survive — used for "near exact subject mention" + if low in seen: + continue + seen.add(low) + out.append(tok) + return out + + +def _extract_candidate_spans( + text: str, answer_type: str +) -> list[dict]: + """Yield candidate spans aligned with `answer_type` (Dav1d §6+§8). + Each is {start, end, text, kind}.""" + if not text: + return [] + spans: list[dict] = [] + if answer_type in ("title_like", "person"): + # Quoted strings — strongest signal for titles + for m in _RE_QUOTED_STRING.finditer(text): + inner = m.group(1) or m.group(2) or "" + if not inner: + continue + spans.append({ + "start": m.start(), + "end": m.end(), + "text": inner, + "kind": "quoted_string", + }) + # Title-case runs (2+ Capitalized Words) + for m in _RE_TITLE_CASE_SPAN.finditer(text): + spans.append({ + "start": m.start(), + "end": m.end(), + "text": m.group(0), + "kind": "title_case_span", + }) + # Comma-list items — extract from inside detected runs + for run_m in _RE_COMMA_LIST_RUN.finditer(text): + run_start = run_m.start() + run_text = run_m.group(0) + for item_m in _RE_COMMA_LIST_ITEM.finditer(run_text): + spans.append({ + "start": run_start + item_m.start(), + "end": run_start + item_m.end(), + "text": item_m.group(0), + "kind": "comma_list_item", + }) + if answer_type == "date": + for m in _RE_YEAR.finditer(text): + spans.append({ + "start": m.start(), + "end": m.end(), + "text": m.group(0), + "kind": "year", + }) + for m in _RE_DATE.finditer(text): + spans.append({ + "start": m.start(), + "end": m.end(), + "text": m.group(0), + "kind": "date", + }) + return spans + + +def _nearest_subject_proximity( + text: str, + span: dict, + subject_tokens: list[str], + window_chars: int, +) -> dict | None: + """If any subject token appears within `window_chars` of the + span's midpoint, return proximity info. Else None. + + Casefolded substring scan (cheap, deterministic). Reports the + nearest token's distance from the span boundary.""" + if not subject_tokens or not text: + return None + norm_text = text.lower() + span_start = span["start"] + span_end = span["end"] + best: tuple[int, str, int] | None = None # (distance, token, position) + for tok in subject_tokens: + # Subject tokens may contain "_" for quoted-phrase joins — + # split back to space for matching + needle = tok.lower().replace("_", " ") + pos = norm_text.find(needle) + while pos != -1: + if pos < span_start: + dist = span_start - (pos + len(needle)) + elif pos > span_end: + dist = pos - span_end + else: + dist = 0 # subject token inside the span + if dist >= 0 and dist <= window_chars: + if best is None or dist < best[0]: + best = (dist, tok, pos) + pos = norm_text.find(needle, pos + 1) + if best is None: + return None + return { + "nearest_subject_token": best[1], + "subject_proximity_chars": best[0], + "subject_pos": best[2], + } + + +def _score_answerability_candidates( + candidates: list[dict], + answer_type: str, + *, + max_returned: int = 10, +) -> dict | None: + """Apply Dav1d §9 trigger rule + confidence classification. + + Trigger fires when at least one of: + (i) a quoted_string candidate is "near exact subject mention" + (subject_proximity_chars ≤ 60) + (ii) ≥2 comma_list_item candidates near subject + (iii) ≥1 candidate of the query-type-matched kind near exact + subject mention (≤60 chars) + + Returns None if no rule matches. Otherwise returns the scored + bundle with `confidence_class`, `total_count`, `top_k`, and + `threshold_report`. + """ + if not candidates: + return None + quoted_near_exact = [ + c for c in candidates + if c["candidate_kind"] == "quoted_string" + and c["subject_proximity_chars"] <= 60 + ] + comma_list_near = [ + c for c in candidates + if c["candidate_kind"] == "comma_list_item" + ] + type_matched_near_exact: list[dict] = [] + if answer_type == "title_like": + type_matched_near_exact = [ + c for c in candidates + if c["candidate_kind"] in ("quoted_string", "title_case_span", "comma_list_item") + and c["subject_proximity_chars"] <= 60 + ] + elif answer_type == "person": + type_matched_near_exact = [ + c for c in candidates + if c["candidate_kind"] == "title_case_span" + and c["subject_proximity_chars"] <= 60 + ] + elif answer_type == "date": + type_matched_near_exact = [ + c for c in candidates + if c["candidate_kind"] in ("year", "date") + and c["subject_proximity_chars"] <= 60 + ] + # Trigger rules + rule_quoted = bool(quoted_near_exact) + rule_comma = len(comma_list_near) >= 2 + rule_type_match = bool(type_matched_near_exact) + if not (rule_quoted or rule_comma or rule_type_match): + return None + # Confidence class + if ( + rule_quoted + and rule_type_match + and len(type_matched_near_exact) >= 2 + ): + confidence = "strong" + elif rule_quoted or (rule_comma and rule_type_match): + confidence = "medium" + else: + confidence = "weak" + # Top-K: sort by proximity (closest first) and cap at max_returned + ranked = sorted(candidates, key=lambda c: c["subject_proximity_chars"]) + top_k = ranked[:max_returned] + return { + "confidence_class": confidence, + "total_count": len(candidates), + "top_k": top_k, + "threshold_report": { + "quoted_near_exact": len(quoted_near_exact), + "comma_list_near": len(comma_list_near), + "type_matched_near_exact": len(type_matched_near_exact), + "max_candidates_returned": max_returned, + }, + } + + +def diagnose_missed_answer( + question: str, + answer: str, + evidence: list, + *, + policy: dict | None = None, +) -> dict | None: + """Phase 1 verifier-blind missed-answer guard. Read-only. + + Returns None when the three-clause conjunction (denial + extraction + shape + candidate proximity) does NOT fire. Returns a structured + diagnostic dict when it does. + + `evidence` is a list of EvidenceObject (or any object with + `.pointer_id`, `.evidence_id`, `.title`, `.span` attrs OR a dict + with those keys). Quote-mode flat context is not supported in + Phase 1 — pass an empty list there and the function returns None. + + `policy` keys read: + answerability_sidecar_enabled (default True; gates whole sidecar) + answerability_threshold.min_candidates (default 1) + answerability_threshold.proximity_window_chars (default 600) + answerability_threshold.max_candidates_returned (default 10) + + No model calls, no audit writes, no providence_cache mutation. + Recomputable from inputs; same inputs → same output byte-for-byte. + """ + policy = policy or {} + if not policy.get("answerability_sidecar_enabled", True): + return None + if not (question and answer and evidence): + return None + threshold = policy.get("answerability_threshold") or {} + window = int(threshold.get("proximity_window_chars", 600)) + max_returned = int(threshold.get("max_candidates_returned", 10)) + # --- Clause A: denial pattern in answer --- + denial = _match_denial_pattern(answer) + if denial is None: + return None + # --- Clause B: extraction-shaped question --- + # Lazy-import to keep inspect.py side-effect-free at import time + try: + from arborist.qa.quantifier import classify_question_quantifier + q_info = classify_question_quantifier(question) + q_intensity = q_info.get("intensity") + except Exception: # pragma: no cover — defensive + q_intensity = None + extraction = _classify_extraction_shape(question, q_intensity) + if extraction is None: + return None + # --- Subject tokens (cue-stripped) --- + subject_tokens = _extract_subject_tokens(question) + if not subject_tokens: + return None + # --- Clause C: candidate spans near subject --- + answer_type = extraction["answer_type"] + candidates: list[dict] = [] + for ev in evidence: + # Support both EvidenceObject and dict + if hasattr(ev, "span"): + ev_id = getattr(ev, "evidence_id", None) or "" + ev_pid = getattr(ev, "pointer_id", "") or "" + ev_title = getattr(ev, "title", None) or "" + ev_uri = getattr(ev, "document_uri", "") or "" + ev_text = getattr(ev, "span", "") or "" + elif isinstance(ev, dict): + ev_id = ev.get("evidence_id") or "" + ev_pid = ev.get("pointer_id") or "" + ev_title = ev.get("title") or "" + ev_uri = ev.get("document_uri") or "" + ev_text = ev.get("span") or ev.get("text") or "" + else: + continue + if not ev_text: + continue + for span in _extract_candidate_spans(ev_text, answer_type): + prox = _nearest_subject_proximity( + ev_text, span, subject_tokens, window + ) + if prox is None: + continue + candidates.append({ + "evidence_id": ev_id, + "pointer_id": ev_pid, + "evidence_title": ev_title, + "evidence_uri": ev_uri, + "offset_start": span["start"], + "offset_end": span["end"], + "offset_basis": "evidence_object_text", + "text": span["text"], + "candidate_kind": span["kind"], + "nearest_subject_token": prox["nearest_subject_token"], + "subject_proximity_chars": prox["subject_proximity_chars"], + }) + if not candidates: + return None + scored = _score_answerability_candidates( + candidates, answer_type, max_returned=max_returned + ) + if scored is None: + return None + return { + "diagnostic_version": ANSWERABILITY_DIAGNOSTIC_VERSION, + "denial_patterns_version": DENIAL_PATTERNS_VERSION, + "extraction_cues_version": EXTRACTION_CUES_VERSION, + "answerability_warning": True, + "confidence_class": scored["confidence_class"], + "triggered_clauses": { + "denial": True, + "extraction_shape": True, + "candidate_proximity": True, + }, + "denial_pattern_matched": denial, + "extraction_cue_matched": extraction["cue"], + "quantifier_intensity": q_intensity, + "extraction_shape": extraction["shape"], + "answer_type": answer_type, + "subject_tokens": subject_tokens, + "candidate_count": scored["total_count"], + "threshold": { + "proximity_window_chars": window, + "max_candidates_returned": max_returned, + }, + "threshold_report": scored["threshold_report"], + "missed_answer_candidate_spans": scored["top_k"], + } diff --git a/arborist/qa/query.py b/arborist/qa/query.py index 15976cb..83ddd52 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -571,6 +571,22 @@ DEFAULT_QUERY_POLICY = { # output ("Claim. [E12]") that the verifier maps back to # content-addressed evidence_ids for the cache & run-DAG. "answer_mode": DEFAULT_ANSWER_MODE, + # Ticket #000068 Phase 1 — verifier-blind missed-answer falsification + # guard. Deterministic read-only sidecar — fires on (denial pattern + + # extraction-shape + candidate spans near cleaned subject tokens). + # See arborist/qa/inspect.py:diagnose_missed_answer and + # docs/tickets/ticket-000068-*.md. Phase 1: sidecar attaches + # `result["answerability"]` (may be None). Phase 3 (deferred): + # `answerability_demote_enabled` wires EVIDENCE-WARRANTED → + # EVIDENCE-MISSED-PARTIAL via _render_audit_label and IF on folds + # into verifier_policy_hash. For now demote is OFF and the field is + # listed so flipping it later is one-line. + "answerability_sidecar_enabled": True, + "answerability_threshold": { + "proximity_window_chars": 600, + "max_candidates_returned": 10, + }, + "answerability_demote_enabled": False, # User-payload layout (lost-in-the-middle mitigation). `tail` keeps # the historical "EVIDENCE: ... --- QUESTION: q" shape; `bookend` # repeats the question both before and after the evidence; @@ -2375,6 +2391,9 @@ def query( "violations": _reject_violations, "format_collapsed": None, "raw_answer": None, + # Ticket #000068 — reject-broad early-return: no evidence + # examined, so no missed-answer guard fires. + "answerability": None, "quantifier_intensity": quantifier["intensity"], "quantifier_matched_token": quantifier["matched_token"], "scope_bound_hint": quantifier["scope_bound_hint"], @@ -3299,6 +3318,16 @@ def query( # writes populate it correctly. Acceptable degradation # since governance_policy_hash invalidated prior records. "partially_verified_quotes": [], + # Ticket #000068 Phase 1 — cache-hit path: cached + # records don't carry the evidence_map (only the + # rendered sources summary), so the sidecar can't + # recompute candidate spans without re-running + # retrieval. Cache hits return `answerability: None` + # — known limitation, documented in + # docs/tickets/ticket-000068-*.md §11 (Dav1d cache-hit + # recompute discipline). Operators wanting fresh + # diagnostics use --burn to force a miss-path run. + "answerability": None, # Quantifier preflight (Ticket #000008 Phase 1) — the # classifier is pure on the question string, so cache # hits can re-classify cheaply and carry the same @@ -3909,6 +3938,18 @@ def query( n_quotes=verdict["n_quotes"], n_verified=verdict["n_verified"], ) + # Ticket #000068 Phase 1 — verifier-blind missed-answer sidecar. + # Read-only, deterministic, no model call, no audit write. Fires + # only when (denial + extraction-shape + candidate spans near + # cleaned subject tokens) all match. Returns None otherwise. + # Quote mode has no evidence_map → guard returns None. + try: + from arborist.qa.inspect import diagnose_missed_answer + answerability = diagnose_missed_answer( + question, answer_text, evidence_map or [], policy=policy, + ) + except Exception: # pragma: no cover — defensive; sidecar must never crash query + answerability = None result = { "status": "cache_miss_then_written", "audit_mode": verdict["audit_mode"], @@ -3957,6 +3998,12 @@ def query( # avoid duplication. Bench reads it for bracket-count # diagnostics; never persisted in providence_cache. "raw_answer": raw_answer if is_lattice_mode else None, + # Ticket #000068 Phase 1 — missed-answer guard sidecar output. + # None when guard didn't fire; structured dict when it did + # (see arborist/qa/inspect.py:diagnose_missed_answer for shape). + # Read-only diagnostic — never written to providence_cache, + # recomputable from (question, answer, evidence, policy). + "answerability": answerability, # Quantifier preflight result (Ticket #000008 Phase 1). # Surfaced on the result so bench rows pick it up. Phase 1 # is dry-run only — caps not applied; Phase 2 wires diff --git a/arborist/qa/runner.py b/arborist/qa/runner.py index 888e3d3..4daadac 100644 --- a/arborist/qa/runner.py +++ b/arborist/qa/runner.py @@ -126,6 +126,14 @@ DEFAULT_POLICY = { # different cache_keys and never alias. No iterative repair in # pointer mode (one-shot benchmark discipline). "answer_mode": DEFAULT_ANSWER_MODE, + # Ticket #000068 Phase 1 — verifier-blind missed-answer guard. + # See arborist/qa/query.py DEFAULT_QUERY_POLICY for full semantics. + "answerability_sidecar_enabled": True, + "answerability_threshold": { + "proximity_window_chars": 600, + "max_candidates_returned": 10, + }, + "answerability_demote_enabled": False, # User-payload layout. See arborist/qa/query.py DEFAULT_QUERY_POLICY # for full semantics. `tail` preserves prior cache; `bookend` / # `per_chunk` mitigate lost-in-the-middle on small models. diff --git a/docs/TICKETS.md b/docs/TICKETS.md index df64256..066e8f9 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -111,6 +111,7 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| +| #000068 | Verifier-blind missed-answer falsification guard | **in progress · Phase 1 implementation underway 2026-05-27** (Dav1d de-novo review GO for Phase 1 with seven hardenings folded into spec — subject-token cue-stripping, answer-type alignment, confidence_class, candidate cap=10, precise offset_start/end/basis, cache-hit recompute-on-read, Phase 1 out of verifier_policy_hash). Original opening 2026-05-27; sibling to the user-payload-layout work shipped 2026-05-26, split out per the Dav1d-audience rule — `feedback_ticket_proliferation`). Surfaced by the Ballestrini case: evidence E2 literally contained the song names, Hermes-3-8B under `user_payload_layout=tail` said *"specific songs by her are not mentioned in the provided evidence blocks"*, verifier marked the run `EVIDENCE-WARRANTED` 2/2 because nothing positive was unsupported. **Verifier-blind false-negative class** — existing layered verifier (quote/span/entity/paraphrase + Rule 8 + Rule 9 + claim ceiling) guards unsupported *presence*, has no hook for unsupported *absence*. Layout fixes attention placement on the specific instance (n=3 bench 2026-05-27 confirms bookend/per_chunk recover Ballestrini); layout alone can't close the class — adversarial phrasing or bigger prompt resurfaces it under any layout. Proposed deterministic sidecar in `arborist/qa/inspect.py:diagnose_missed_answer`: three-clause conjunction — **(A)** answer matches denial pattern ("not mentioned", "not provided", "the evidence does not say", …, closed list versioned via `denial_patterns_version`); **(B)** question is extraction shape (reuse `arborist.qa.quantifier` classifier — `ALL`/`COMPREHENSIVE`/`OPEN_REQUEST` intensities, OR surface cues "songs by"/"works by"/"who wrote"/"list"/"name all"); **(C)** evidence contains candidate spans near subject tokens (reuse `entity_proximity_n`/`entity_proximity_window` from verify.py — quoted strings, title-case spans, comma-separated title lists within W chars of stemmed subject content tokens). All three must fire. Output: `result["answerability"]` with `missed_answer_candidate_spans` list (evidence_id + offset + text). **Hash discipline:** sidecar fields (`denial_patterns_version`, `extraction_cues_version`, `answerability_threshold`) fold into `governance_policy_hash` only; an optional `answerability_demote_enabled` flag (default OFF) wires `EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL` in `_render_audit_label`, and IF on folds into BOTH `governance_policy_hash` AND `verifier_policy_hash` (changes rendered audit_mode, so verifier hash must move — the deliberate opt-in moves the verifier hash, sidecar-only stays out). No LLM-as-judge. Never writes `providence_cache`/`audit_events`. Never promotes claims. Pattern verbatim from `arborist.qa.inspect.diagnose_*` (deflection, coherence, title-relevance). Phases: 1 sidecar read-only, 2 bench + threshold tuning, 3 demote opt-in, 4 default decision (bench-gated). 5F-Falsification fixture: Ballestrini case already in `bench/qa_questions.txt` under "entity list". Full spec in `docs/tickets/ticket-000068-verifier-blind-missed-answer-guard.md`. | 2026-05-27 | D2 | | #000067 | M-aware cold-pack hydration (route incoming docs by content hash into M target shards) | **open · scaffold · prereq for #46 genesis test** (2026-05-26; surfaced while preparing the 3090 SPV-wallet validation). Today's `hydrate_from_metadata_pack` takes a single `conn` and writes every incoming row into one shard. With the corpus now in M=4 hash-routed topology (#000065), a fresh peer needs to land each document on `shard_for_document(document_root, M)` — same routing function as the producer. Without this, a fresh peer's `~/.arborist/shards/` is just one big single-shard DB and the M=4 ATTACH-and-route assumption #000065 was sized for doesn't hold consumer-side. Two coherent shapes: **(α) two-step kludge** — hydrate into single shard, then `arborist corpus reshard --to M` on the consumer. Works today (proven by the 2026-05-26 reshard executor) but doubles the wall time and treats packed shards as if they came from an arbitrary topology. **(β) direct M-aware hydrate** — extend `hydrate_from_metadata_pack` to accept `targets: list[sqlite3.Connection]` + `M: int` and route per-row at restore time (reusing `arborist.document.shard_for_document` + the table-routing rules in `arborist/migrate.py`). Manifest carries `corpus_shard_count` so the unpacker knows M from the pack itself. β is the right answer — α exists only as a fallback if 20-min-window pressure forces it. Sequence: (1) add `corpus_shard_count` to pack manifest (read from source meta during `dump_shard_metadata`); (2) `restore_shard_metadata_routed(targets, M, table_dir)` in `cold_pack_metadata.py` mirroring `_route_per_doc_table` from migrate.py; (3) `hydrate_from_metadata_pack` gains a `targets`/`shards_dir` param; (4) `arborist cold unpack --shards-dir DIR` initialises M target shards from the manifest's `corpus_shard_count` and routes; (5) regression test: pack 2 shards → hydrate into fresh 4 shards → assert every doc on its hash-routed target. Refactor opportunity: the routing rules (ROUTED_BY_DOCUMENT_ROOT, CONSOLIDATED_TABLES) currently live in migrate.py; this ticket can either duplicate them in cold_pack_metadata.py (fast) or factor into a shared `arborist/multi_shard.py` module (cleaner). The shared-module path is more honest given graft mode (#000066) wants the same primitives. Out of scope: graft / overlay mode (that's #000066 — overlays onto populated, this is hydrate-into-empty). | 2026-05-26 | — | | #000066 | Cold-pack overlay / graft mode (pack-as-package, witness-pattern audit chain) | **scaffold-only · awaiting go/no-go** (2026-05-26; surfaced while running #000065 reshard, fox extension: "we could envision a pack for wikipedia 2010, wikipedia current, etc"). Extend #000061 cold-pack hydration with a second mode: overlay an existing pack onto a populated shard set instead of hydrating into empty. Doc/chunk/edge/concept overlay is trivial (`INSERT OR IGNORE` on content-addressed PKs collapses dupes); FTS5 overlay is trivial (new chunks → new rowids → new FTS rows). The interesting part is the audit chain — can't naively append the pack's events because `prev_event_hash` linkage breaks across the join. Chosen approach: **graft receipt**. Append one new `event_type='graft'` event to the host chain carrying `(pack_hash, snapshot_root, corpus_name, event_count, first_event_hash, last_event_hash, manifest_root)`; the pack file itself becomes the durable witness for the absorbed events (anyone can re-fetch the pack, walk its internal chain, and verify it matches the receipt). Host chain stays linear; pack chain is a "witnessed subgraph." This is the same witness pattern Merkle-AGI v8/v9 is heading toward, but bought at near-zero schema cost. Rejected alternatives: re-chain everything (breaks external refs to old event_hashes — cache_keys anchoring to old `audit_event_hash`, snapshots, etc. — silently invalid); chain forest with new `chain_id` column (right answer when graft dominates the lifecycle, but premature now). **Pack-as-package extension** (fox 2026-05-26): each pack carries a `corpus_name` field in its manifest (`wikipedia-2010`, `wikipedia-current`, `arxiv-cs`, `textbooks-undergrad`, …) so operators pick which corpora to graft — `arborist cold graft wikipedia-current` becomes as natural as `apt install firefox`. Multiple packs of the same corpus name: most-recent `snapshot_root` wins; older packs stay in the bucket until GC. URI conflicts across corpora (e.g., `wikipedia.org/wiki/Foo` in both 2010 and current): different content → different `document_root` → both stored, `supersedes` edges per CLAUDE.md invariant. Providence-cache conflicts: same `cache_key` with different answer → existing v9.8 falsification framework handles it (`state='stale'` or `quarantined`). Mesh-peer-corpus-merge: each peer's pack is a graftable package; partition reconciliation becomes "exchange the packs you each carry, graft what you lack". The mesh-of-arborists semantic. Sequence: (1) `corpus_name` field in #000061 manifest format + alias index in bucket (`corpora/<name>/latest.json` pointer to active pack_hash); (2) `arborist cold graft <pack_hash>` / `arborist cold graft --corpus <name>` mode in evict.py — read pack, INSERT OR IGNORE per-table, emit graft receipt; (3) conflict-policy flag (`--on-uri-conflict {supersedes,skip,fail}`, default `supersedes`); (4) `arborist cold list-corpora` shows available packages in a bucket. Scaffold first, code only when (a) #000065 reshard lands and stabilises (b) a second corpus exists (the wikipedia-current snapshot, or first textbook bundle ready to graft onto wikipedia-2010 base) (c) at least two peers want to exchange. | 2026-05-26 | — | | #000065 | Canonical shard count `M` + content-hash routing (decouple ingest parallelism from ATTACH ceiling) | **closed · landed in `c86d5ac`** (2026-05-26 19:47 UTC cutover, ~94 min wall). Production reshard completed end-to-end on the live host: 3,468,226 globally-unique docs / 6,235,588 chunks / 90,592,990 edges / 3,468,403 audit events re-routed to content-hash-deterministic M=4 layout. Per-shard doc uniformity within ±0.04% (theoretical limit ±0.05%). Audit chain consolidated to canonical shard 000 via Option A (3.47M events re-sorted by ts + re-chained, bodies preserved); tail event `type=reshard` carries plan+result body. Validation gate caught 176 chunks + 547 edges as cross-shard dupes (collapsed by INSERT OR IGNORE; 0.003% delta, within 1% tolerance). Two defects surfaced + fixed mid-cutover: (a) `derivations.src_root` FK fired on legitimately cross-shard refs — fix in `04edff7`: writer connection runs `PRAGMA foreign_keys = OFF`, runtime stays FK=ON; (b) WAL accumulated ~37 GB across FTS rebuild + audit consolidate because SQLite auto-checkpoint can't reclaim pages while a reader cursor is open — fix in `c86d5ac`: `_checkpoint_truncate` called between executor phases. Full migration record in `docs/corpus-history.md` (which entry is the operator-facing equivalent of the audit chain tail). Follow-on work tracked separately: #44 re-pack into bucket → #45 verify bucket determinism → #46 genesis fresh peer on 3090-ai.foxhop.net from cloud (first real SPV-wallet end-to-end test) → #47 retire stale pre-reshard bucket packs. (2026-05-26; surfaced while sizing #000061's federation story). Today shard count conflates two roles: producer ingest parallelism (wants vCPU count) + consumer ATTACH fan-out (capped at SQLITE_MAX_ATTACHED=10 on stock python3 sqlite3). Producer with 16 vCPU → 16 shards → consumers fail to attach the 11th. Producer with 4 shards → 16-vCPU box runs 75% idle on ingest. Fix: pin a corpus-wide canonical **M = 4** (decided 2026-05-26 from real-Wikipedia bench: M=4 captures 92% of peak ingest throughput, ATTACH cost 9 ms keeps mobile-tolerable, 6 free ATTACH slots under SQLite's 10 ceiling for auxiliary DBs), introduce N (ingest workers) decoupled from M. Document → shard assignment becomes content-deterministic: `shard_idx = int(document_root[:8], 16) % M`. Same input → same output across every peer (today's "spray by ingest order" is non-deterministic across peers, a real federation weakness). Migration hard-constraint per fox: **content-addressed rebalance, NOT re-ingest** — every row is already addressed by `document_root` / `leaf_hash` / etc.; migration reads rows from the current 4 shards, computes each row's new shard via the routing function, INSERTs into M new shards. No source re-parse, no re-canonicalization, no re-chunking, no LLM. ~20–40 min I/O-bound vs. hours-to-days for true re-ingest. Audit chain consolidates to canonical shard 000 (re-numbered + re-hashed once) to preserve global event ordering. Phases: 0 design lock + pin M in meta table → 1 read path (connect_query honors M) → 2 ingest path (multi-shard write per worker) → 3 cold-pack restore re-routes on pull → 4 corpus migration tool. Open audit-chain re-numbering question (every shard has its own seq + event_hash; rebalancing splits a producer's chain across M consumer shards). Don't proliferate sub-tickets; the audit handling is part of this design lock. Out of scope: custom-built sqlite3 with higher MAX_ATTACHED (rejected: violates "python3 + venv + sqlite3 only" property from CLAUDE.md); topic-clustering shards (would break ingest determinism). | 2026-05-26 | — | @@ -181,4 +182,4 @@ Newest first. Update on every open/close. ## Next ID -`000068` +`000069` diff --git a/docs/tickets/ticket-000068-verifier-blind-missed-answer-guard.md b/docs/tickets/ticket-000068-verifier-blind-missed-answer-guard.md new file mode 100644 index 0000000..f4c7043 --- /dev/null +++ b/docs/tickets/ticket-000068-verifier-blind-missed-answer-guard.md @@ -0,0 +1,383 @@ +# Ticket #000068 — Verifier-blind missed-answer falsification guard + +**Status:** in progress · Phase 1 implementation underway 2026-05-27 (after Dav1d de-novo review) +**Opened:** 2026-05-27 +**Scope:** Deterministic read-only sidecar that detects evidence-neglect / false-negative answers — runs where the LLM said "not mentioned" but the evidence contains candidate answer spans near the question subject. Emits an `answerability_warning` + candidate-span list; optional audit-label demote `EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL` behind a policy flag. No LLM-as-judge. No verifier-policy change. + +## Post-review hardenings (Dav1d 2026-05-27) + +The 2026-05-27 de-novo review (`~/Downloads/RESPONSE_final_ticket-000068-verifier-blind-missed-answer-guard.txt`) was a **GO for Phase 1** with seven specific hardenings folded into the spec below. Summary so a re-read can hit the load-bearing changes at a glance: + +1. **Subject-token extraction must strip cue/relation/stopwords.** `"songs by veronica ballestrini"` → `["veronica", "ballestrini"]`, NOT all four tokens. Cue/relation words to strip: `songs, works, books, papers, list, name, who, what, which, by, wrote, composed, directed, acted, examples`. Preserve: quoted phrases, hyphenated terms, proper-noun runs, exact title-like fragments. Without this, generic title-case spans like "Harvard University" trigger on "songs by John Smith". Non-negotiable. +2. **Answer-type alignment.** Candidate span kind must match query type: + - `songs / works / books / papers` → quoted strings, title-like spans, comma-list items + - `who wrote / composed / directed / acted in` → person-like title-case spans + - `when / what year / date` → year/date spans + - `what are / list / name all` → quoted strings, title-like spans, comma-list items +3. **Confidence class is deterministic, not boolean.** `weak | medium | strong`. Trigger fires only when denial + extraction-shape match AND one of: ≥1 quoted candidate near exact subject, ≥2 comma/list candidates near subject, ≥1 query-type-matched candidate near exact subject mention. +4. **Cap output**: `max_candidates_returned = 10`. Prevents the guard from becoming a quote-amplifier (the per_chunk lesson). +5. **Precise offsets**: `offset_start` + `offset_end` + `offset_basis="evidence_object_text"`. Never an ambiguous single `offset`. +6. **Cache-hit recompute-on-read.** The sidecar output is NOT stored as a proof object. It is recomputable from `(question, answer_text, evidence_objects, answerability_policy)`. If a cached record's evidence is recoverable, recompute on the next lookup so cached false negatives don't stay invisible. +7. **Phase 1 stays out of `verifier_policy_hash`.** Only the demote flag (Phase 3) moves the verifier hash — and only when ON. Phase 1's `answerability_*` policy fields fold into `governance_policy_hash` only. + +The body of the ticket below absorbs these into the design. Phase boundaries unchanged: Phase 1 (sidecar read-only, this work) → Phase 2 (bench + threshold tuning) → Phase 3 (opt-in demote) → Phase 4 (default decision, bench-gated). NO-GO on default demote-on until Phase 2 + human spot-check evidence is in. +**Audience:** fox + anyone maintaining `arborist/qa/` + reviewers reading the next Dav1d review pass. +**Hard constraint:** Sidecar discipline. Never writes `providence_cache` / `audit_events`. Never folds into `verifier_policy_hash`. Never promotes claims — can only flag possible missed answerability. Follows the same pattern as `arborist.qa.inspect.diagnose_*` (deflection, coherence, title-relevance). + +--- + +## 1. The failure class — verifier-blind false negatives + +Surfaced 2026-05-26 by the Ballestrini case (`docs/user-payload-layout.md`). +Same query, same retrieval, same evidence, Hermes-3-8B under +`user_payload_layout=tail`: + +``` +Q: songs by veronica ballestrini + +A: Veronica Ballestrini is a country music singer and songwriter who + has released several songs. However, the specific songs by her + are not mentioned in the provided evidence blocks. +``` + +Evidence block `E2` literally contained "Amazing", "Out There +Somewhere", "Fascinated", "What's Up With That", "Don't Say". +Verifier verdict: `EVIDENCE-WARRANTED` 2/2 — clean. From the +verifier's view nothing was wrong: + +``` +Evidence contains the answer. +Model says the evidence does not contain the answer. +Verifier sees no unsupported positive claim → marks run clean. +User receives a false negative under EVIDENCE-WARRANTED. +``` + +The existing layered verifier (quote / span / entity / paraphrase ++ Rule 8 title-relevance + Rule 9 subject-tokens-absent + claim- +count ceiling) guards against **unsupported positive claims** — it +has no hook for **unsupported absences**. The 2026-05-27 layout +bench (n=3 × 76q on Hermes-3-8B) confirmed that prompt-layout +fixes (`bookend`, `per_chunk`) address the specific Ballestrini +attention-placement instance but cannot close the class — a +sufficiently large prompt or an adversarial phrasing can resurface +the failure under any layout. + +This is exactly the class of failure that selects for **adding a +new falsifier into the substrate** rather than for tuning model +inputs. Layout is a model-input policy knob (`#user_payload_layout`, +folds into `governance_policy_hash`); this is a verifier-side +diagnostic adjunct. + +Dav1d 2026-05-27 review framing: + +``` +#0000XY-A (layout, shipped 2026-05-26) — user_payload_layout +#0000XY-B (this ticket, design open) — missed-answer guard +``` + +The split is per the Dav1d-audience rule (`feedback_ticket_proliferation`): +the missed-answer guard touches a different layer (verifier / +falsification adjunct) than the layout knob (model-input policy), +and a Dav1d review needs to read each independently. Same +substrate, two tickets. + +## 2. Detection rule — deterministic, no LLM-as-judge + +The guard fires when **all three** of the following hold: + +``` +(A) Denial pattern in the answer. + Regex / token-class match against a closed list of phrases: + "not mentioned" + "not provided" + "the evidence does not say" + "does not mention" + "no specific" + "no evidence" + "cannot determine from the provided evidence" + "is not stated" + "is not specified" + Sealed list, versioned via a denial_patterns_version policy field + so additions cache-partition cleanly. + +(B) Question is an extraction/list shape. + EITHER the existing arborist.qa.quantifier classifier returns + intensity ∈ {ALL, COMPREHENSIVE, OPEN_REQUEST, MANY, PLURAL}, + OR the question matches one of the surface cues: + "songs by" "works by" "books by" "papers by" + "who wrote" "who composed" "who acted in" "who directed" + "what are" "which" "list" + "name all" "name the" + The classifier already exists and folds into + governance_policy_hash; reuse it. + +(C) Evidence contains candidate answer spans near subject tokens. + Reuse arborist.qa.verify's entity-proximity primitive + (entity_proximity_n / entity_proximity_window). For each + question subject token (content tokens, stemmed), find + candidate spans within `window` chars: + - quoted strings (".+?" pattern) + - title-case spans (Run of 2+ Capitalized Words) + - comma-separated title lists ('X, Y, and Z') + A "candidate span" is a span where the entity-proximity score + against the subject tokens passes a configurable threshold + (default: at least N candidate spans within W chars of any + subject token). +``` + +All three must fire. Each in isolation is too noisy: + +- **(A) alone**: model legitimately says "not mentioned" when + evidence really doesn't have the answer — that's correct + behavior, not a failure. +- **(B) alone**: list shape doesn't imply evidence has the answer. +- **(C) alone**: candidate spans near subject tokens are common in + Wikipedia prose without being the answer to *this* question. + +The three-clause conjunction is the fingerprint of the +Ballestrini-class failure: question asks for a list, evidence has +list-like spans clustered near the subject, model still denies. + +## 3. Output + +Sidecar emits a structured warning: + +```python +{ + "answerability_warning": bool, + "missed_answer_candidate_spans": [ + { + "evidence_id": "E2", # pointer-mode or content-addressed + "offset": 142, # char offset within the span + "text": "Amazing", # the candidate token/phrase + "subject_proximity_chars": 67, + }, + ... + ], + "denial_pattern_matched": "not mentioned in the provided evidence", + "extraction_cue_matched": "songs by", + "quantifier_intensity": "ALL", +} +``` + +Optional audit-label demote (gated by `answerability_demote_enabled`, +default False to preserve cache): + +``` +EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL +``` + +The display layer (`_render_audit_label`) gets one new four-rung +suffix when demote fires. Programmatic callers see the +underlying `audit_mode` unchanged unless the demote flag is on — +same projection discipline as the existing four-rung ladder. + +## 4. Hash discipline (the load-bearing constraint) + +This is where the Dav1d review is most precise: + +``` +user_payload_layout → governance_policy_hash (model input) +denial_patterns_version → governance_policy_hash (sidecar trigger) +extraction_cues_version → governance_policy_hash (sidecar trigger) +answerability_threshold → governance_policy_hash (sidecar trigger) +answerability_demote_enabled → governance_policy_hash + verifier_policy_hash IF on + (it changes the rendered audit_mode, which + IS a verifier output, so the verifier hash + must move when demote is enabled. Hence + default-off — opt-in flips the verifier + hash deliberately, sidecar-only stays out) +``` + +When the demote flag is off (default), the guard is a pure read- +only diagnostic — same status as `diagnose_deflection`, +`diagnose_coherence`, `diagnose_title_relevance`. When on, it +becomes a verifier-policy change and partitions the cache +accordingly. The default is off so existing operators don't see +audit_mode mutate without an explicit opt-in. + +Alternative design considered: a separate +`answerability_policy_hash` (a 10th cache_key dimension). Rejected +— it adds a hash to scan with no correctness coverage beyond what +`governance_policy_hash` already gives (the trigger fields are a +subset of the policy dict; flipping them already bumps +`governance_policy_hash`). The "audit legibility" argument made +for the optional 9th-dim `verifier_policy_hash` (#000058) doesn't +recur here — sidecar warnings are read off the result dict, not +the cache_key. + +## 5. Where it lives + +``` +arborist/qa/inspect.py + + def diagnose_missed_answer( + question: str, + answer: str, + evidence: list[EvidenceObject], + *, + policy: dict, + ) -> dict | None: + ... + +arborist/qa/query.py + runner.py + after verify_*() returns, before persist, call the sidecar: + warning = diagnose_missed_answer( + question, answer_text, evidence, policy=policy + ) + result["answerability"] = warning # may be None + if policy.get("answerability_demote_enabled") and warning: + if audit_mode == "EVIDENCE-WARRANTED": + audit_mode = "EVIDENCE-MISSED-PARTIAL" + # bump verifier_policy_hash partition (handled by + # the policy field already being in the verifier set) +``` + +Sidecar code follows the existing inspect.py pattern verbatim — no +new module, no new audit-event type, no schema change. + +## 6. Acceptance criteria + +``` +1. Ballestrini-class fixture (added 2026-05-27 to bench/qa_questions.txt + as "songs by veronica ballestrini") produces a warning when the + model under tail layout returns the negation. Sidecar test pins this. + +2. Genuine-absence cases do not false-trigger. Negative-control + fixture: question asks for a fact the corpus does not contain, + evidence genuinely lacks the answer, model correctly says + "not mentioned" → warning MUST NOT fire (the (C) clause must + be the filter that keeps these out — candidate spans near + subject must be absent). + +3. No model call. Pure-Python, deterministic. Same input → same + output, byte-for-byte (canonical-projection discipline). + +4. Candidate spans are cited by evidence_id and char offset within + the span. Forensically traceable to the source bytes. + +5. Sidecar output is read off `result["answerability"]` — never + written to `providence_cache` (it's policy-versioned, not + audit-chained). + +6. Guard never promotes unsupported claims. The trigger conjunction + makes promotion structurally impossible — it only fires on + denial answers. + +7. Demote-mode flag (`answerability_demote_enabled`) is default-False. + When True, folds into both governance_policy_hash AND + verifier_policy_hash (since it changes rendered audit_mode). + When False, governance_policy_hash only (sidecar diagnostic). + +8. Tests: + - denial pattern match (positive) + - extraction cue match (positive) + - candidate proximity match (positive) + - all-three conjunction (positive) + - each clause in isolation (must NOT trigger alone) + - genuine absence (negative control) + - hash partitioning: flipping demote flag changes verifier hash + - hash partitioning: flipping denial_patterns_version changes + governance hash, not verifier hash +``` + +## 7. Bench + +After landing, run the same n=3 × 76q claim_lattice sweep against +Hermes-3-8B that the user_payload_layout work used (the curated +set already has the Ballestrini regression fixture). Compare: + +- baseline (no guard): use the 2026-05-27 layout-tail results + on disk at `bench/qa_results/layout-tail/2026-05-27T10-07-29Z.{md,jsonl}` +- guard-on (sidecar only, demote off): expect 0 STRICT-rate + change (sidecar is read-only); expect warning to fire on the + Ballestrini fixture +- guard-on (demote enabled): expect STRICT count to drop by the + number of fixtures where the model produced a denial — that + drop is the visible measurement of the failure class + +Promotion rule (when/if making the demote default-on later): +honest report on what fraction of the curated set produced +denials, how many of those were true vs false negatives by +human review of a sample. + +## 8. Out of scope (explicitly) + +- LLM-as-judge of any kind. The cost / reproducibility / chain-of- + custody constraints in `arborist/qa/verify.py` apply — no model + in the proof path (per the #000049 cage), and no model in the + sidecar either by choice. +- Active retrieval re-runs ("the answer wasn't in the top-k, let + me re-fetch"). That's a different ticket. This guard only looks + at evidence already in the prompt. +- Fixing the model's behavior under tail layout. That's what + `user_payload_layout` does (and the 2026-05-27 bench confirms + it doesn't generalize as a default). The guard is the + *substrate-side* recovery — catch the failure even when the + model and the layout both let it through. + +## 9. Relationship to other tickets / docs + +- `docs/user-payload-layout.md` — sibling work. Layout is the + model-input lever; this is the verifier-side adjunct. Read + together for the full picture of the Ballestrini failure. +- `docs/seven-point-program.md` — when this lands it extends **D2 + (pointer-grounding)** by covering *unsupported absence* in + addition to *unsupported presence*. Same falsifier discipline, + new failure shape. +- `arborist/qa/inspect.py` — existing sidecar pattern. Follow it + verbatim. +- `arborist/qa/quantifier.py` — reuse the broad-quantifier + classifier for clause (B). Already folds into + `governance_policy_hash`. +- `arborist/qa/verify.py` — reuse `entity_proximity_n` / + `entity_proximity_window` primitives for clause (C). No new + proximity code. +- 5F-Falsification framework (Dav1dPrometheus, see + `~/.claude/projects/-home-fox-git-arborist/memory/dav1dprometheus_framework.md`) + — the Ballestrini case is exactly the kind of fixture that + feeds the falsification loop: controller proposes (layout fix), + battery exposes the residual class (verifier-blind absence), + fixture lands here. + +## 10. Open questions + +- **Threshold tuning for clause (C).** Default proximity window? + Default minimum candidate-span count? Should be benched on the + curated set + a few synthetic negative controls before locking + defaults. +- **Phrase patterns vs token patterns** in clause (A). The denial + list above is phrase-based ("not mentioned"). Tokenized + matching ("absent", "lack", "no information") might over-catch. + Conservative phrase-based start; widen only on measured FN. +- **Interaction with the existing `DEFLECTION_DETECTED` signal.** + Deflection detector (`diagnose_deflection`) already flags + topic-shift via subject-anchor heuristic. There's overlap: an + answer that deflects to "I can't help" also denies. Sidecar + outputs should be independent — both can fire on the same + row — but cross-checking on the curated set is worth doing to + make sure neither is a wholly-contained subset of the other. + +## 11. Implementation order (proposed) + +``` +Phase 1 — sidecar (read-only, no demote) + arborist/qa/inspect.py:diagnose_missed_answer + policy fields: denial_patterns_version, extraction_cues_version, + answerability_threshold (all in governance hash) + unit tests (acceptance criteria 1, 2, 3, 4, 5, 6, 8.a-f) + wire into query.py + runner.py result dict + +Phase 2 — bench + threshold tuning + Re-run the layout-tail bench with the sidecar enabled, read-only; + count fires on the curated set; spot-check the warnings. + +Phase 3 — demote flag (opt-in) + Add answerability_demote_enabled (governance + verifier hashes) + Wire EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL in the + _render_audit_label projection. + Tests for hash partitioning (acceptance criteria 7, 8.g-h). + +Phase 4 — default decision + After Phase 2 evidence in hand: keep demote default-off (read- + only diagnostic) or flip to on. Bench-gated, not opinion-gated. +``` diff --git a/tests/test_missed_answer_guard.py b/tests/test_missed_answer_guard.py new file mode 100644 index 0000000..29bc1da --- /dev/null +++ b/tests/test_missed_answer_guard.py @@ -0,0 +1,500 @@ +"""Tests for the Ticket #000068 Phase 1 missed-answer falsification guard. + +The guard is a deterministic read-only sidecar that fires on the three-clause +conjunction: denial pattern + extraction-shaped question + candidate spans +near cleaned subject tokens. These tests pin each clause in isolation, the +positive Ballestrini regression case, the negative John-Smith control, and +the discipline invariants (no model call, no audit write, no claim promotion). +""" +from __future__ import annotations + +from dataclasses import dataclass + +import pytest + +from arborist.qa.inspect import ( + _DENIAL_PATTERNS_V1, + _extract_subject_tokens, + _match_denial_pattern, + _classify_extraction_shape, + _extract_candidate_spans, + _score_answerability_candidates, + diagnose_missed_answer, + ANSWERABILITY_DIAGNOSTIC_VERSION, + DENIAL_PATTERNS_VERSION, +) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@dataclass +class FakeEv: + pointer_id: str + evidence_id: str + title: str + document_uri: str + span: str + + +def _ballestrini_evidence() -> list[FakeEv]: + return [ + FakeEv( + pointer_id="E1", + evidence_id="E652b125b", + title="Veronica Ballestrini", + document_uri="https://en.wikipedia.org/wiki/Veronica_Ballestrini", + span=( + "Veronica Jean Ballestrini (born October 29, 1991) is an " + "Italian-American country music singer and songwriter from " + "Waterford, Connecticut." + ), + ), + FakeEv( + pointer_id="E2", + evidence_id="E19912984", + title="Veronica Ballestrini", + document_uri="https://en.wikipedia.org/wiki/Veronica_Ballestrini", + span=( + "Ballestrini went on to record her debut album " + "\"What I'm All About\" which was released August 21, 2009. " + "You can buy her CD and hear amazing 11 songs and 6 were " + "written by her. Her first single \"Amazing\" charted on " + "the Music Row Country chart and the music video debuted at " + "#3 on CMT Pure. In January 2010 Timbob records partnered " + "with Lofton Creek Records president Mike Borchetta for the " + "promotions of Veronica's single \"Out There Somewhere\"." + ), + ), + ] + + +# --------------------------------------------------------------------------- +# Clause A — denial pattern +# --------------------------------------------------------------------------- + + +def test_denial_pattern_positive(): + assert _match_denial_pattern("the specific songs are not mentioned in evidence") == "not mentioned" + + +def test_denial_pattern_negative(): + assert _match_denial_pattern("Veronica has released several songs including Amazing.") is None + + +def test_denial_pattern_casefold(): + assert _match_denial_pattern("THIS INFORMATION IS NOT PROVIDED IN THE TEXT") == "not provided" + + +def test_denial_pattern_whitespace_normalized(): + assert _match_denial_pattern("specific songs are\n not mentioned\nhere") == "not mentioned" + + +def test_denial_pattern_sealed_list_intact(): + """Adding a new phrase requires bumping DENIAL_PATTERNS_VERSION. + This test pins the v1 set so silent additions can't slip through.""" + expected = { + "not mentioned", + "not provided", + "the evidence does not say", + "does not mention", + "no specific", + "no evidence", + "cannot determine from the provided evidence", + "is not stated", + "is not specified", + } + assert set(_DENIAL_PATTERNS_V1) == expected + + +# --------------------------------------------------------------------------- +# Clause B — extraction shape +# --------------------------------------------------------------------------- + + +def test_extraction_cue_songs_by(): + info = _classify_extraction_shape("songs by veronica ballestrini", None) + assert info is not None + assert info["cue"] == "songs by" + assert info["answer_type"] == "title_like" + assert info["shape"] == "list" + + +def test_extraction_cue_who_wrote(): + info = _classify_extraction_shape("who wrote ulysses?", None) + assert info is not None + assert info["cue"] == "who wrote" + assert info["answer_type"] == "person" + + +def test_extraction_cue_what_year(): + info = _classify_extraction_shape("what year did the war end?", None) + assert info is not None + assert info["cue"] == "what year" + assert info["answer_type"] == "date" + + +def test_extraction_shape_quantifier_fallback(): + """No surface cue but broad quantifier intensity → still extraction shape.""" + info = _classify_extraction_shape("tell me about beatles members", "OPEN_REQUEST") + assert info is not None + assert info["cue"].startswith("quantifier:") + assert info["answer_type"] == "title_like" + + +def test_extraction_shape_narrow_question(): + """Narrow factoid: no cue, no broad quantifier → not extraction-shaped.""" + info = _classify_extraction_shape("who is veronica ballestrini?", "SINGULAR") + assert info is None + + +# --------------------------------------------------------------------------- +# Subject-token extraction (Dav1d §7 — non-negotiable) +# --------------------------------------------------------------------------- + + +def test_subject_tokens_strip_cue_words(): + """`songs by veronica ballestrini` → just the proper-noun run. + Without this hardening the guard false-triggers on generic spans.""" + tokens = _extract_subject_tokens("songs by veronica ballestrini") + low = [t.lower() for t in tokens] + assert "songs" not in low + assert "by" not in low + assert "veronica" in low + assert "ballestrini" in low + + +def test_subject_tokens_strip_who_wrote(): + tokens = _extract_subject_tokens("who wrote ulysses?") + low = [t.lower() for t in tokens] + assert "who" not in low + assert "wrote" not in low + assert "ulysses" in low + + +def test_subject_tokens_preserve_proper_noun_runs(): + """Multi-word proper nouns survive as separate tokens (matched by either).""" + tokens = _extract_subject_tokens("books by f scott fitzgerald") + low = [t.lower() for t in tokens] + assert "books" not in low + assert "by" not in low + assert "fitzgerald" in low + # f and scott are short — `f` drops (single letter), `scott` survives + assert "scott" in low + + +def test_subject_tokens_preserve_hyphenated(): + tokens = _extract_subject_tokens("songs by jean-luc picard") + low = [t.lower() for t in tokens] + assert "jean-luc" in low or "jean" in low # implementation may split or preserve + assert "picard" in low + + +def test_subject_tokens_dedupe(): + tokens = _extract_subject_tokens("ballestrini ballestrini songs") + low = [t.lower() for t in tokens] + assert low.count("ballestrini") == 1 + + +# --------------------------------------------------------------------------- +# Candidate span extraction +# --------------------------------------------------------------------------- + + +def test_candidate_quoted_string(): + text = "She wrote her debut single \"Amazing\" in 2009." + spans = _extract_candidate_spans(text, "title_like") + quoted = [s for s in spans if s["kind"] == "quoted_string"] + assert any(s["text"] == "Amazing" for s in quoted) + + +def test_candidate_title_case_span(): + text = "Veronica Jean Ballestrini was born in Connecticut." + spans = _extract_candidate_spans(text, "title_like") + title_case = [s for s in spans if s["kind"] == "title_case_span"] + assert any("Veronica Jean Ballestrini" in s["text"] for s in title_case) + + +def test_candidate_year_for_date_query(): + text = "She was born in 1991 in Connecticut." + spans = _extract_candidate_spans(text, "date") + years = [s for s in spans if s["kind"] == "year"] + assert any(s["text"] == "1991" for s in years) + + +def test_candidate_no_year_for_title_query(): + """Year extraction only happens for date queries.""" + text = "She was born in 1991." + spans = _extract_candidate_spans(text, "title_like") + assert not any(s["kind"] == "year" for s in spans) + + +# --------------------------------------------------------------------------- +# Full guard — positive case +# --------------------------------------------------------------------------- + + +def test_ballestrini_tail_failure_triggers(): + """The motivating regression: Hermes-3-8B under tail layout said + 'specific songs are not mentioned' when E2 contained 'Amazing', + 'Out There Somewhere', etc. Guard must catch this.""" + result = diagnose_missed_answer( + question="songs by veronica ballestrini", + answer=( + "Veronica Ballestrini is a country music singer and songwriter " + "who has released several songs. However, the specific songs by " + "her are not mentioned in the provided evidence blocks." + ), + evidence=_ballestrini_evidence(), + ) + assert result is not None + assert result["answerability_warning"] is True + assert result["confidence_class"] in ("medium", "strong") + assert result["denial_pattern_matched"] == "not mentioned" + assert result["extraction_cue_matched"] == "songs by" + assert "ballestrini" in [t.lower() for t in result["subject_tokens"]] + assert "songs" not in [t.lower() for t in result["subject_tokens"]] + # At least one quoted-string candidate (Amazing / What I'm All About / Out There Somewhere) + kinds = {c["candidate_kind"] for c in result["missed_answer_candidate_spans"]} + assert "quoted_string" in kinds + + +def test_ballestrini_diagnostic_version_pinned(): + result = diagnose_missed_answer( + question="songs by veronica ballestrini", + answer="The evidence does not say what songs she released.", + evidence=_ballestrini_evidence(), + ) + assert result["diagnostic_version"] == ANSWERABILITY_DIAGNOSTIC_VERSION + assert result["denial_patterns_version"] == DENIAL_PATTERNS_VERSION + + +def test_offsets_are_start_end_basis(): + """Dav1d §10: offsets must be precise — start + end + basis, not ambiguous.""" + result = diagnose_missed_answer( + question="songs by veronica ballestrini", + answer="Songs are not mentioned in the evidence.", + evidence=_ballestrini_evidence(), + ) + for c in result["missed_answer_candidate_spans"]: + assert "offset_start" in c + assert "offset_end" in c + assert c["offset_basis"] == "evidence_object_text" + assert c["offset_end"] > c["offset_start"] + + +def test_candidate_cap_at_10(): + """Phase 1 must cap output at 10 candidates (the per_chunk lesson).""" + long_evidence = FakeEv( + pointer_id="E1", + evidence_id="EBIG", + title="Veronica Ballestrini", + document_uri="https://x", + # 12 quoted titles + span=" ".join(f'Her single "Song{i}" charted.' for i in range(12)) + + " Ballestrini wrote them all.", + ) + result = diagnose_missed_answer( + question="songs by veronica ballestrini", + answer="The evidence does not mention specific songs.", + evidence=[long_evidence], + ) + assert result is not None + assert len(result["missed_answer_candidate_spans"]) <= 10 + + +# --------------------------------------------------------------------------- +# Negative controls (Dav1d §14) +# --------------------------------------------------------------------------- + + +def test_genuine_absence_no_strong_trigger(): + """Dav1d §14 negative control: 'songs by John Smith' + evidence + about Harvard/NY → must NOT strong-trigger.""" + ev = FakeEv( + pointer_id="E1", + evidence_id="EJS", + title="John Smith", + document_uri="https://x", + span="John Smith studied at Harvard University and lived in New York.", + ) + result = diagnose_missed_answer( + question="songs by john smith", + answer="The evidence does not mention songs by John Smith.", + evidence=[ev], + ) + # Per Dav1d: 'no strong warning; ideally no warning; if warning + # exists, confidence_class = weak.' We accept weak or None. + if result is not None: + assert result["confidence_class"] != "strong", ( + "false positive at strong confidence — Harvard/NY are not song titles" + ) + + +def test_no_denial_does_not_trigger(): + """Clause A failure: extraction question + candidates, but no denial.""" + result = diagnose_missed_answer( + question="songs by veronica ballestrini", + answer="Veronica released Amazing and Out There Somewhere.", + evidence=_ballestrini_evidence(), + ) + assert result is None + + +def test_not_extraction_shape_does_not_trigger(): + """Clause B failure: SINGULAR question + denial + candidates.""" + result = diagnose_missed_answer( + question="who is veronica ballestrini?", + answer="The evidence does not say who she is.", + evidence=_ballestrini_evidence(), + ) + assert result is None + + +def test_no_evidence_does_not_trigger(): + """Clause C failure: empty evidence list.""" + result = diagnose_missed_answer( + question="songs by veronica ballestrini", + answer="Not mentioned.", + evidence=[], + ) + assert result is None + + +def test_no_candidate_spans_does_not_trigger(): + """Clause C failure: evidence exists but contains no candidate spans + matching the answer_type (plain prose with no titles).""" + ev = FakeEv( + pointer_id="E1", + evidence_id="X", + title="V B", + document_uri="https://x", + span="she is a singer-songwriter from connecticut.", # lowercase — no title-case + ) + result = diagnose_missed_answer( + question="songs by veronica ballestrini", + answer="Not mentioned.", + evidence=[ev], + ) + assert result is None + + +# --------------------------------------------------------------------------- +# Discipline invariants — no model, no audit, no promotion +# --------------------------------------------------------------------------- + + +def test_sidecar_disabled_returns_none(): + """answerability_sidecar_enabled=False short-circuits to None.""" + result = diagnose_missed_answer( + question="songs by veronica ballestrini", + answer="not mentioned", + evidence=_ballestrini_evidence(), + policy={"answerability_sidecar_enabled": False}, + ) + assert result is None + + +def test_evidence_dict_form_supported(): + """Phase 1 accepts both EvidenceObject and dict-shaped evidence.""" + dict_ev = [ + { + "pointer_id": "E2", + "evidence_id": "EX", + "title": "Veronica Ballestrini", + "document_uri": "https://x", + "span": 'Her first single "Amazing" by Veronica Ballestrini.', + }, + ] + result = diagnose_missed_answer( + question="songs by veronica ballestrini", + answer="The evidence does not mention specific songs.", + evidence=dict_ev, + ) + assert result is not None + assert result["answerability_warning"] is True + + +def test_deterministic_byte_for_byte(): + """Same inputs → same output. Byte-deterministic per Dav1d §13.""" + inputs = dict( + question="songs by veronica ballestrini", + answer="not mentioned", + evidence=_ballestrini_evidence(), + ) + r1 = diagnose_missed_answer(**inputs) + r2 = diagnose_missed_answer(**inputs) + assert r1 == r2 + + +def test_empty_question_returns_none(): + result = diagnose_missed_answer( + question="", + answer="not mentioned", + evidence=_ballestrini_evidence(), + ) + assert result is None + + +def test_empty_answer_returns_none(): + result = diagnose_missed_answer( + question="songs by veronica ballestrini", + answer="", + evidence=_ballestrini_evidence(), + ) + assert result is None + + +# --------------------------------------------------------------------------- +# Output schema integrity +# --------------------------------------------------------------------------- + + +def test_output_schema_keys(): + result = diagnose_missed_answer( + question="songs by veronica ballestrini", + answer="The specific songs are not mentioned in evidence.", + evidence=_ballestrini_evidence(), + ) + required_keys = { + "diagnostic_version", + "denial_patterns_version", + "extraction_cues_version", + "answerability_warning", + "confidence_class", + "triggered_clauses", + "denial_pattern_matched", + "extraction_cue_matched", + "extraction_shape", + "answer_type", + "subject_tokens", + "candidate_count", + "threshold", + "threshold_report", + "missed_answer_candidate_spans", + } + assert required_keys.issubset(result.keys()), ( + f"missing keys: {required_keys - set(result.keys())}" + ) + + +def test_triggered_clauses_all_true_when_firing(): + result = diagnose_missed_answer( + question="songs by veronica ballestrini", + answer="not mentioned", + evidence=_ballestrini_evidence(), + ) + assert result["triggered_clauses"]["denial"] is True + assert result["triggered_clauses"]["extraction_shape"] is True + assert result["triggered_clauses"]["candidate_proximity"] is True + + +def test_confidence_class_in_valid_set(): + result = diagnose_missed_answer( + question="songs by veronica ballestrini", + answer="not mentioned", + evidence=_ballestrini_evidence(), + ) + assert result["confidence_class"] in ("weak", "medium", "strong")