diff --git a/Makefile b/Makefile index 0fd5c8f..8106ef5 100644 --- a/Makefile +++ b/Makefile @@ -167,14 +167,31 @@ ANSWER_MODE ?= claim_lattice # tell-me-everything-about-X). Pairs cleanly with the broad- # quantifier reminder which is default-on for lattice modes. # Bench (#000008 §12.10): cap-on JSON wins +14pp on STRICT-rate. + # REJECT_BROAD=1 → strict reject for ALL/COMPREHENSIVE/OPEN_REQUEST # unbounded shapes; returns UNGROUNDED before the LLM call. # ALLOW_BROAD=1 → emergent search; classifier on, caps off. -query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K="extra retrieval keywords" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1 XLANG=1 XLANG_MT=1]; JSON by default +# +# LAYOUT — user-payload-layout policy knob. Decides where the +# question text sits relative to evidence in the final user-turn +# message. See docs/user-payload-layout.md. +# Recommendation matrix (2026-05-27 n=3 × 76q bench, Hermes-3-8B, +# claim_lattice mode): +# tail safe default — preserves prior cache (proven) +# bookend safer for small-model recovery; n=3 aggregate bench +# was a wash vs tail (+0.44pp STRICT, within 5pp floor) +# per_chunk best on list/extraction shapes for small models +# (the Ballestrini case) BUT regresses -9.78pp on +# aggregate STRICT — opt-in only, never a default +# Override per-call: LAYOUT=bookend make query Q="..." +# Override session-wide: LAYOUT_DEFAULT=bookend make query Q="..." +LAYOUT_DEFAULT ?= tail +LAYOUT ?= $(LAYOUT_DEFAULT) +query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K="extra retrieval keywords" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote LAYOUT=tail|bookend|per_chunk BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1 XLANG=1 XLANG_MT=1]; JSON by default @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ - echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K=\"extra retrieval keywords\" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1 XLANG=1 XLANG_MT=1]"; exit 2; \ + echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K=\"extra retrieval keywords\" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote LAYOUT=tail|bookend|per_chunk BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1 XLANG=1 XLANG_MT=1]"; exit 2; \ fi - $(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(K),--retrieval-keywords "$(K)",) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) $(if $(WITNESS),--witness,) $(if $(XLANG),--crosslang-guard,) $(if $(XLANG_MT),--crosslang-translate,) "$(Q)" + $(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) --user-payload-layout $(LAYOUT) $(if $(K),--retrieval-keywords "$(K)",) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) $(if $(WITNESS),--witness,) $(if $(XLANG),--crosslang-guard,) $(if $(XLANG_MT),--crosslang-translate,) "$(Q)" query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON=1 BURN=1 ANSWER_MODE=... BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 XLANG=1 XLANG_MT=1] @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ diff --git a/arborist/qa/prompts.py b/arborist/qa/prompts.py index 09ccf5d..9759f9e 100644 --- a/arborist/qa/prompts.py +++ b/arborist/qa/prompts.py @@ -151,3 +151,60 @@ CLAIM_LATTICE_JSON_GROUNDING_REMINDER = ( "per claim. Each claim is one focused fact. Now answer the " "question on the next message." ) + + +USER_PAYLOAD_LAYOUTS = ("tail", "bookend", "per_chunk") + + +def format_user_payload( + question: str, + body: str, + *, + layout: str = "tail", + evidence_label: str = "EVIDENCE", + question_label: str = "QUESTION", + per_chunk_marker: str | None = "=== ", +) -> str: + """Assemble the final user-turn payload around an evidence body. + + `layout`: + - `tail` (default, preserves prior cache): question after evidence only. + - `bookend`: question repeated before AND after evidence — counters + lost-in-the-middle decay on small models (≤8B) with long contexts. + - `per_chunk`: bookend + a one-line `[for: ]` reminder + injected before each evidence block. Body must use `per_chunk_marker` + as the block-leading sentinel (lattice-mode evidence-map renderers + emit `=== E1 (...) ===` blocks joined by `\\n\\n`). When the marker + is absent (quote-mode flat context), falls back to `bookend`. + + Folds into `governance_policy_hash` via the `user_payload_layout` + policy field — flipping it cache-partitions cleanly. The corpus + (documents / chunks / FTS5 / audit-chain) is content-addressed and + untouched by this knob. + """ + if layout not in USER_PAYLOAD_LAYOUTS: + raise ValueError( + f"user_payload_layout must be one of {USER_PAYLOAD_LAYOUTS}, " + f"got {layout!r}" + ) + tail_line = f"---\n\n{question_label}: {question}" + if layout == "tail": + return f"{evidence_label}:\n\n{body}\n\n{tail_line}" + head_line = f"{question_label}: {question}" + if layout == "bookend": + return f"{head_line}\n\n{evidence_label}:\n\n{body}\n\n{tail_line}" + # per_chunk + if per_chunk_marker is None: + # No natural per-chunk boundary (e.g. quote-mode flat document) + # → fall back to bookend. + return f"{head_line}\n\n{evidence_label}:\n\n{body}\n\n{tail_line}" + sep = "\n\n" + per_chunk_marker + if sep in body: + reminder = f"[for: {question}]" + interleaved = body.replace(sep, f"\n\n{reminder}{sep}") + return ( + f"{head_line}\n\n" + f"{evidence_label}:\n\n{reminder}\n\n{interleaved}\n\n" + f"{tail_line}" + ) + return f"{head_line}\n\n{evidence_label}:\n\n{body}\n\n{tail_line}" diff --git a/arborist/qa/query.py b/arborist/qa/query.py index 32f18e5..15976cb 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -55,6 +55,8 @@ from arborist.qa.prompts import ( CLAIM_LATTICE_JSON_GROUNDING_REMINDER, CLAIM_LATTICE_JSON_SYSTEM_PROMPT, CLAIM_LATTICE_SYSTEM_PROMPT, + USER_PAYLOAD_LAYOUTS, + format_user_payload, ) from arborist.qa.concepts import ( has_compare_phrasing, @@ -569,6 +571,14 @@ DEFAULT_QUERY_POLICY = { # output ("Claim. [E12]") that the verifier maps back to # content-addressed evidence_ids for the cache & run-DAG. "answer_mode": DEFAULT_ANSWER_MODE, + # User-payload layout (lost-in-the-middle mitigation). `tail` keeps + # the historical "EVIDENCE: ... --- QUESTION: q" shape; `bookend` + # repeats the question both before and after the evidence; + # `per_chunk` adds a one-line `[for: q]` reminder before each + # evidence block. Folds into governance_policy_hash so flipping the + # layout cache-partitions cleanly. Corpus artifacts (documents / + # chunks / FTS5 / audit-chain) are content-addressed and untouched. + "user_payload_layout": "tail", "claim_lattice_system_prompt": CLAIM_LATTICE_SYSTEM_PROMPT, "claim_lattice_grounding_reminder": CLAIM_LATTICE_GROUNDING_REMINDER, # Reference-frame polarity preamble (Ticket #000002 / Module L). @@ -2919,7 +2929,10 @@ def query( rendered_evidence = render_evidence_map(evidence_map) def _user_payload(q: str) -> str: - return f"EVIDENCE:\n\n{rendered_evidence}\n\n---\n\nQUESTION: {q}" + return format_user_payload( + q, rendered_evidence, + layout=policy.get("user_payload_layout", "tail"), + ) elif answer_mode == "claim_lattice": # JSON variant — same evidence-map construction as the pointer # path, but blocks are labeled with content-addressed @@ -2984,13 +2997,22 @@ def query( rendered_evidence = render_evidence_map_for_json(evidence_map) def _user_payload(q: str) -> str: - return f"EVIDENCE:\n\n{rendered_evidence}\n\n---\n\nQUESTION: {q}" + return format_user_payload( + q, rendered_evidence, + layout=policy.get("user_payload_layout", "tail"), + ) else: sys_prompt = policy["system_prompt"] grounding_reminder = policy.get("grounding_reminder") def _user_payload(q: str) -> str: - return f"Sources:\n\n{context}\n\n---\n\nQuestion: {q}" + return format_user_payload( + q, context, + layout=policy.get("user_payload_layout", "tail"), + evidence_label="Sources", + question_label="Question", + per_chunk_marker="=== Source: ", + ) # Frame detection (Ticket #000002 / Module L). Lattice-mode only. # Surfaces whether the query is allusion-shape AND the phrase diff --git a/arborist/qa/runner.py b/arborist/qa/runner.py index ce52aae..888e3d3 100644 --- a/arborist/qa/runner.py +++ b/arborist/qa/runner.py @@ -27,6 +27,8 @@ from arborist.qa.prompts import ( CLAIM_LATTICE_JSON_GROUNDING_REMINDER, CLAIM_LATTICE_JSON_SYSTEM_PROMPT, CLAIM_LATTICE_SYSTEM_PROMPT, + USER_PAYLOAD_LAYOUTS, + format_user_payload, ) from arborist.qa.keys import ( DEFAULT_FIDELITY, @@ -124,6 +126,10 @@ DEFAULT_POLICY = { # different cache_keys and never alias. No iterative repair in # pointer mode (one-shot benchmark discipline). "answer_mode": DEFAULT_ANSWER_MODE, + # User-payload layout. See arborist/qa/query.py DEFAULT_QUERY_POLICY + # for full semantics. `tail` preserves prior cache; `bookend` / + # `per_chunk` mitigate lost-in-the-middle on small models. + "user_payload_layout": "tail", "claim_lattice_system_prompt": CLAIM_LATTICE_SYSTEM_PROMPT, "claim_lattice_grounding_reminder": CLAIM_LATTICE_GROUNDING_REMINDER, # Allowed source roles for claim-lattice verification. Roles outside @@ -549,7 +555,10 @@ def ask( rendered_evidence = render_evidence_map(evidence_map) def _user_payload(q: str) -> str: - return f"EVIDENCE:\n\n{rendered_evidence}\n\n---\n\nQUESTION: {q}" + return format_user_payload( + q, rendered_evidence, + layout=policy.get("user_payload_layout", "tail"), + ) elif answer_mode == "claim_lattice": # JSON variant — same per-chunk evidence map as pointer mode, # blocks labeled with content-addressed evidence_id (long hex) @@ -579,13 +588,22 @@ def ask( rendered_evidence = render_evidence_map_for_json(evidence_map) def _user_payload(q: str) -> str: - return f"EVIDENCE:\n\n{rendered_evidence}\n\n---\n\nQUESTION: {q}" + return format_user_payload( + q, rendered_evidence, + layout=policy.get("user_payload_layout", "tail"), + ) else: sys_prompt = policy["system_prompt"] grounding_reminder = policy.get("grounding_reminder") def _user_payload(q: str) -> str: - return f"Document:\n\n{document_text}\n\n---\n\nQuestion: {q}" + return format_user_payload( + q, document_text, + layout=policy.get("user_payload_layout", "tail"), + evidence_label="Document", + question_label="Question", + per_chunk_marker=None, + ) # System sets the policy; a user-turn reminder restates the rule one # message before the payload arrives. Payload (document or evidence diff --git a/bench/qa_questions.txt b/bench/qa_questions.txt index b04555b..8118074 100644 --- a/bench/qa_questions.txt +++ b/bench/qa_questions.txt @@ -49,6 +49,12 @@ name simpsons family members including pets? list of obelisk in connecticut what are the planets of our solar system? who were the original seven mercury astronauts? +# 2026-05-26 regression: Hermes-3-8B under user_payload_layout=tail +# returned "specific songs by her are not mentioned in the provided +# evidence" — false negation on evidence E2 that literally contains +# the song names. Same prompt under bookend/per_chunk recovers. +# See docs/user-payload-layout.md. +songs by veronica ballestrini # relationship / multi-fact who is supermans girlfriend? diff --git a/docs/user-payload-layout.md b/docs/user-payload-layout.md new file mode 100644 index 0000000..ea7a83b --- /dev/null +++ b/docs/user-payload-layout.md @@ -0,0 +1,496 @@ +# User-payload layout — where the question sits relative to evidence + +A prompt-structure knob that decides where the question text appears +in the final user-turn message relative to the evidence block. Pure +lever on the LLM's attention budget — no change to the verifier, the +audit chain, or the corpus. + +Policy field: `user_payload_layout ∈ {"tail", "bookend", "per_chunk"}`, +default `"tail"`. CLI: `--user-payload-layout` on `arborist query` +and `arborist ask`. Make: `LAYOUT=...` on `make query`. Folds into +`governance_policy_hash` so each layout cache-partitions cleanly. + +## Verdict + +``` +GO: implement static layouts as opt-in policy knobs. +NO-GO: promote bookend/per_chunk as default. The n=3×75q curated + bench (2026-05-27) found bookend statistically indistinguish- + able from tail and per_chunk regressing −9.78pp on aggregate + STRICT-rate against Hermes-3-8B. See §"Bench results". +ADD: companion missed-answer falsification guard. The Ballestrini + failure exposes a verifier-blind false-negative class that + layout fixes attention placement for but does not fully + close. See §"Verifier-blind missed-answer class". +``` + +Layout is a real but narrow lever — useful for specific failure +shapes (small-model evidence-negation under long context), not a +universal default. Keep `tail` default; route `bookend`/`per_chunk` +via the broad-quantifier classifier on query types that match the +target failure mode. + +## Why this exists — the Ballestrini case + +Same query, same corpus, same retrieval, same evidence, same model +(`hermes-3-8B`, `https://hermes.ai.unturf.com/v1`), same 22.8 KB +prompt. The default `tail` layout produced: + +> Veronica Ballestrini is a country music singer and songwriter who +> has released several songs. **However, the specific songs by her +> are not mentioned in the provided evidence blocks.** + +Evidence block `E2` literally contained the song names: "Amazing", +"Out There Somewhere", "Fascinated", and references to music videos +for "What's Up With That" and "Don't Say". The model received an +evidence chunk packed with answers and synthesized a negation. + +The verifier marked the run `EVIDENCE-WARRANTED` because the +negation literally matched no claim that needed grounding. The +guard fires on false positives — claims unsupported by evidence — +not on false negatives — answers the model declined to give. From +the verifier's view the run was clean; from a user's view it was +broken. + +Root cause is the classic lost-in-the-middle / lost-at-the-front +attention failure mode on small (≤8B) models. The final user +message under `tail` was shaped: + +``` +EVIDENCE: + +=== E1 (Veronica Ballestrini | primary_answer_source) === + + +=== E2 (Veronica Ballestrini | primary_answer_source) === + + +=== E3 ... E8 === + + +--- + +QUESTION: songs by veronica ballestrini +``` + +The question is 28 characters at the tail of a 21,176-char evidence +wall. Hermes-3-8B's attention budget loses track of which question +it was asked while reading through eight evidence chunks. By the +time generation begins, the model's prior is "summarize the +evidence I just read" — and the evidence as a whole is mostly +biographical prose, so the summary leans biographical and skirts +the specific-song question. + +## Three layouts + +### `tail` (default — preserves prior cache) + +``` +EVIDENCE: + +--- +QUESTION: +``` + +Original shape. Adequate for large models with long-context attention +(Qwen-27B, Claude, GPT-4, etc.). Brittle on small models with long +evidence blocks. + +### `bookend` (top + bottom) + +``` +QUESTION: + +EVIDENCE: + +--- +QUESTION: +``` + +Question is repeated before AND after the evidence. The leading +copy seeds the attention heads on the actual ask; the trailing copy +re-anchors right before generation. This is the standard +lost-in-the-middle mitigation (Liu et al. 2023, "Lost in the +Middle: How Language Models Use Long Contexts"). + +Effect on Hermes-3-8B for the Ballestrini case: model goes from +*"specific songs are not mentioned"* to extracting "Fascinated", +"Don't Say", album "What I'm All About", producer Cliff Downs. +Trade-off observed in this run: the model became over-eager and +conflated Veronica Ballestrini ↔ The Veronicas (Australian pop +rock duo whose articles were also in the top-8 retrieval), pulling +"This Love", "Revolution" into the answer as if they were +Ballestrini songs. + +### `per_chunk` (bookend + per-block reminder) + +``` +QUESTION: + +EVIDENCE: + +[for: ] + +=== E1 (...) === + + +[for: ] + +=== E2 (...) === + + +[for: ] + +=== E3 ... E8 === + + +--- + +QUESTION: +``` + +Bookend, plus a one-line `[for: ]` reminder injected +before each evidence block (cheap — `<28` chars × `N` blocks on a +22 KB prompt). The question reactivates per-chunk; each chunk's +attention window has the question text immediately preceding it. + +Effect on Hermes-3-8B for the Ballestrini case: model correctly +extracts "Amazing" as Ballestrini's single, then **disambiguates** +The Veronicas as a separate Australian pop rock duo with their own +songs ("Popular", "This Love", "Revolution", "Leave Me Alone"), +and additionally surfaces Elvis Costello's "Veronica" as a third +distinct entity. 6/8 evidence sources used (vs 1/8 for tail, 3/8 +for bookend). No false attribution. + +The `per_chunk` reminder is the question text verbatim, not a +paraphrase — keeping it byte-identical so the model's attention +heads activate on the same tokens repeatedly. Reminders are not +fired for the leading block in lattice mode (the leading bookend +copy already covers it) — they appear before every internal block +boundary (`\n\n=== `). + +## Where layout matters — model-size lever + +Measured 2026-05-26 on the same Ballestrini query, same retrieval, +three layouts × two models (n=1 burn each — directional, not +defensible): + +| Model | tail | bookend | per_chunk | +|--------------|---------------------------------------|----------------------------------------------------|------------------------------------------------| +| Hermes-3-8B | broken — negates evidence | works, but conflates Ballestrini ↔ The Veronicas | works, correctly disambiguates three entities | +| Qwen-27B | works | works (~identical output to tail) | works (~identical output to tail) | + +Qwen-27B is unaffected by layout on this query — its attention +budget is large enough that the question stays salient through 21 +KB of evidence regardless of where it appears. Hermes-3-8B is +broken under `tail` and recovered by either `bookend` or +`per_chunk` on this specific case. + +Layout matters most for **small models on long-context list/ +extraction queries**. For larger models, layout may be behaviorally +near-neutral in observed output — but it is **not cache-neutral or +cost-neutral**: every layout change shifts `governance_policy_hash`, +changes `cache_key`, alters token count, and can affect latency & +answer wording even when correctness is unchanged. + +## Bench results — 2026-05-27 + +Curated QA set, 76 +questions (75 at sweep start + 1 Ballestrini regression added +mid-session — won't affect already-run sweep results), n=3 samples +per cell, claim_lattice mode, +Hermes-3-8B via `https://hermes.ai.unturf.com/v1`, three layouts +sequentially with concurrency=4. 225 runs per layout, 675 total +LLM calls. Output under `bench/qa_results/layout-{tail,bookend,per_chunk}/`. + +| Layout | STRICT | HYBRID | UNGROUNDED | strict-rate | mean ratio | mean latency | Δ vs tail | +|-------------|--------|--------|------------|-------------|------------|--------------|------------------------| +| tail | 94 | 83 | 48 | 0.418 | 0.696 | 12.8s | — (control) | +| bookend | 95 | 83 | 47 | 0.422 | 0.726 | 11.9s | +0.44pp (noise) | +| per_chunk | 72 | 112 | 41 | 0.320 | 0.734 | 12.6s | **−9.78pp** (real regression) | + +Per the bench-maxing rule (5pp signal floor at n=3): + +- **bookend ≈ tail.** 1 STRICT delta on 225 runs is noise. + `mean ratio` (n_verified / n_quotes) improved 0.696 → 0.726, but + the strict-rate verdict is the gate. Bookend is a safe no-op + aggregate — no regression, no win. +- **per_chunk regresses significantly.** −22 STRICT, +29 HYBRID. + Per-chunk reminder over-anchors the model into citing more + evidence per claim: + - `TOO_MANY_EVIDENCE_IDS` violations: tail 20 → bookend 28 → + **per_chunk 54** + - mean answer chars in the 32-64 KB prompt bucket: tail 720 → + bookend 728 → **per_chunk 1,660** (2.3× longer answers) + - `WARRANT_MISSING` stable across all three (~28-30), so the + issue is **claim-volume inflation**, not deflection or + warrant failure. +- **Per-prompt-size bucket** (claim_lattice, n_runs in brackets): + + | bucket | tail [n] | bookend [n] | per_chunk [n] | + |----------|----------|-------------|---------------| + | 16-32 KB | 0.40 [141] | 0.41 [141] | 0.34 [138] | + | 32-64 KB | 0.42 [81] | 0.46 [81] | 0.26 [84] | + + Bookend nudges +4pp on the largest-prompt bucket (marginal, + ~1.5σ). Per_chunk collapses by −16pp on the same bucket. + Layout's aggregate effect is opposite to what the n=1 + Ballestrini anecdote suggested. + +### Honest read + +The Ballestrini case (small-model evidence-negation under +long-context tail) is a **real failure mode but rare in the +curated set**. Most questions don't trigger that specific +attention failure, so layout fixes don't move the aggregate +needle. `per_chunk` fixes the rare case at the cost of ~10pp +aggregate STRICT — a bad trade as a default. `bookend` is a wash. + +**Decision**: keep `tail` as the default. Ship the policy field +as an operator opt-in. The Ballestrini case becomes a regression +fixture in `bench/qa_questions.txt`. Two follow-up options: + +1. **Adaptive routing**: gate `per_chunk` behind the + broad-quantifier classifier (`arborist/qa/quantifier.py`) so + only list/extraction queries see the per-chunk reminder, + narrow factoids stay on `tail`. The classifier already exists + and folds into `governance_policy_hash`. +2. **Filtered re-bench**: run the same n=3 sweep on the + "entity list" + "broad descriptive" + "bounded universals" + subsections of `qa_questions.txt` only. Confirms whether + `per_chunk` is a net positive *when the query shape actually + matches* the failure class it targets, before wiring (1). + +(1) without (2) risks installing a class-gated layout that helps +the gated class but hasn't been measured net-positive even +within the gated class. (2) is the prudent gate before (1). + +## Caveats + +- **`per_chunk` broadens recall on small models — and that + broadening is the regression.** The per-chunk reminder activates + attention on every evidence chunk, including chunks that are + semantically unrelated. Good for "the answer is in chunk 7 of 8" + cases; harmful when chunks 4–6 are off-topic. The Rule 8 + title-mismatch verifier check catches the worst of these (drops + STRICT → HYBRID), but the answer text still grows longer and more + digressive. The 2026-05-27 bench confirms this is a net negative + on the curated set. +- **Quote-mode (`answer_mode=quote`) has no per-chunk boundary.** + The body is the flat `Sources: ...` context string. `per_chunk` + falls back to `bookend` in quote mode (same `format_user_payload` + helper, `per_chunk_marker=None` for the runner's single-document + path or `per_chunk_marker="=== Source: "` for query's multi-source + context — see `arborist/qa/prompts.py`). +- **Cache scope.** Flipping the layout changes + `governance_policy_hash` → changes `cache_key` → all prior + cached answers are bypassed on lookup. The corpus (`documents`, + `chunks`, FTS5 index, audit chain) is content-addressed and + unaffected. Re-ingestion is a no-op (same `document_root` → + idempotent upsert). +- **Verifier untouched.** `verifier_policy_hash` does not depend on + `user_payload_layout`. The same hard checks (quote / span / + entity / paraphrase verification, Rule 8 title-relevance, Rule 9 + subject-tokens-absent, claim-count ceiling) run regardless of + layout. Layout is a model-input knob, not a verification knob. + +## Verifier-blind missed-answer class + +The Ballestrini failure under `tail` is not a hallucination. It +is a **false negative**: + +``` +Evidence contains the answer. +Model says the evidence does not contain the answer. +Verifier sees no unsupported positive claim → marks run clean. +User receives a false negative under EVIDENCE-WARRANTED. +``` + +This is a verifier-blind class. The existing layered verifier +(quote / span / entity / paraphrase + Rule 8 / Rule 9 / claim +ceiling) guards against *unsupported positive claims* — it has no +hook for *unsupported absences*. Layout fixes the attention placement +that produced this specific instance, but layout alone cannot +close the class — a sufficiently large prompt or an adversarial +phrasing can resurface the failure under any layout. + +### Companion missed-answer guard (proposed sidecar) + +The right architectural fit is a **deterministic sidecar**, no +LLM-as-judge, that fires when all three of the following hold: + +``` +answer contains a denial/absence pattern (e.g. "not mentioned", + "not provided", "the evidence does not say", "no specific", + "cannot determine from the provided evidence") +AND +query is an extraction shape (list / "songs by" / "works by" / + "who wrote" / "what are" / "which" / quantifier ∈ {ALL, + COMPREHENSIVE, OPEN_REQUEST} from the existing + arborist.qa.quantifier classifier) +AND +evidence contains candidate answer spans near subject tokens + (quoted strings, comma-separated title lists, title-case + spans within a proximity window of the subject's content + tokens — same proximity-clustering primitive the entity + verifier already uses) +``` + +Output: + +``` +answerability_warning : bool +missed_answer_candidate_spans : list of (evidence_id, offset, text) +optional audit_mode demote: EVIDENCE-WARRANTED → EVIDENCE-MISSED-PARTIAL +``` + +Discipline: + +- **Sidecar, not verifier hook.** Follows the same pattern as + `arborist.qa.inspect.diagnose_*` (deflection, coherence, + title-relevance) — read-only, never writes `providence_cache` or + `audit_events`. +- **Does not fold into `verifier_policy_hash`.** If demote behavior + is wired (optional, gated), it folds into + `governance_policy_hash` (or a new `answerability_policy_hash`), + never into the verifier hash. +- **Never promotes claims.** The guard only flags possible missed + answerability; it cannot turn a HYBRID into a STRICT. + +This is a 5F-Falsification fixture: the Ballestrini case is +exactly the kind of failure that selects for adding a new +falsifier into the substrate, then propagates the new fixture +forward. + +## Usage + +```bash +# Per-call override +arborist query --user-payload-layout bookend "songs by veronica ballestrini" +arborist query --user-payload-layout per_chunk "songs by veronica ballestrini" + +# Make +make query Q="songs by veronica ballestrini" LAYOUT=bookend BURN=1 +make query Q="songs by veronica ballestrini" LAYOUT=per_chunk BURN=1 + +# Policy override in Python +policy = dict(DEFAULT_QUERY_POLICY) +policy["user_payload_layout"] = "per_chunk" +``` + +## Implementation map + +- `arborist/qa/prompts.py` — `format_user_payload(question, body, *, + layout, evidence_label, question_label, per_chunk_marker)`. Single + source of truth; raises on unknown layout. `USER_PAYLOAD_LAYOUTS` + constant lists valid values. +- `arborist/qa/query.py` — multi-source retrieval path. Three + `_user_payload` closures (one per answer_mode) all call + `format_user_payload`. Policy default added to + `DEFAULT_QUERY_POLICY`. +- `arborist/qa/runner.py` — single-document path. Three + `_user_payload` closures all call `format_user_payload`. Policy + default added to `DEFAULT_POLICY`. +- `arborist/cli.py` — `--user-payload-layout` flag on both `query` + and `ask` subcommands; flows into `call_policy`. +- `Makefile` — `LAYOUT=` plumbing on `make query`. + +## Future hardening (not yet implemented) + +These are followups flagged in the 2026-05-27 de-novo review. Not +required to ship the opt-in policy knob, but worth doing before +any default promotion or adaptive routing lands. + +- **Bounded reminder text.** `per_chunk` currently echoes the + question verbatim before each evidence block. Long or + adversarial questions can bloat the prompt and widen the + instruction-injection surface. Suggested signature extension: + + ```python + def format_user_payload( + question: str, + body: str, + *, + layout: str = "tail", + ... + max_reminder_chars: int = 512, # truncation cap + reminder_mode: str = "text", # "text" | "qid" + ) -> str: ... + ``` + + `reminder_mode="qid"` would emit `[for qid=]` — + preserves per-chunk anchor identity without re-inserting + arbitrary user text N times. Trade-off: fewer lexical attention + tokens. Worth A/B-benching against `reminder_mode="text"` on the + filtered list-shape subset. + +- **Structured evidence-block injection.** Current `per_chunk` + implementation uses `body.replace("\n\n=== ", ...)` — works + because `render_evidence_map` is deterministic, but a structured + alternative (`format_user_payload(..., evidence_blocks=[...])`) + would be more robust to upstream rendering changes. Adopt if/when + the evidence map's block boundary convention changes. + +- **Companion missed-answer falsification guard** (see + §"Verifier-blind missed-answer class"). Deterministic sidecar + that catches the failure class layout doesn't fully close. + +## Roadmap + +Phased gating — current state in **bold**. + +``` +Phase 0 — ticket finalization + Open layout ticket. Open companion missed-answer guard ticket. + Default = tail. + +Phase 1 — implementation + **DONE 2026-05-26.** format_user_payload helper, six callsites, + CLI + Make plumbing, governance_policy_hash partitioning, + USER_PAYLOAD_LAYOUTS constant, unknown-layout raises ValueError. + +Phase 2 — regression fixtures + **DONE 2026-05-27.** Ballestrini case added to bench/qa_questions.txt + under entity-list section with 4-line context comment. + Pending: The Veronicas disambiguation fixture, genuine-absence + fixture, quote-mode fallback fixture, governance/verifier hash + partitioning unit tests. + +Phase 3 — bench + **DONE 2026-05-27.** Three-layout sweep on curated 75-question + set, claim_lattice mode, n=3, Hermes-3-8B. Result: tail/bookend + are within noise; per_chunk regresses −9.78pp aggregate. See + §"Bench results". Qwen-27B sanity check still open. + +Phase 4 — selection + **NO-GO on default promotion.** Tail remains default. + Pending decision: adaptive routing via the broad-quantifier + classifier (per_chunk only on list/extraction queries), gated + by a filtered-subset re-bench to confirm net positive within + the gated class. + +Phase 5 — missed-answer guard + **DESIGN OPEN.** Implement deterministic sidecar (denial pattern + + extraction query cues + candidate-span proximity). No + LLM-as-judge. Emit warnings / optional demote to + EVIDENCE-MISSED-PARTIAL. +``` + +## Related + +- `docs/qa-modes-bench.md` — three answer modes (quote / + claim_lattice_pointer / claim_lattice). Layout is orthogonal to + mode and folds into the same `governance_policy_hash`. +- `docs/bench-maxing.md` — bench discipline. The 2026-05-27 sweep + applied the 5pp signal floor against n=3, and the verdict (no + default promotion) followed directly from that rule. +- `docs/seven-point-program.md` — north-star directives. Layout is + a D6/D7 (verifier discipline + small-model robustness) follow-up, + not a new directive. The missed-answer guard, when it lands, + would fold into D2 (pointer-grounding) by extending the + falsifier surface to cover *unsupported absence* alongside + *unsupported presence*. +- 2026-05-27 de-novo review by Dav1dPrometheus (private working + doc) — split this work into two tickets: layout (this doc) + + companion missed-answer guard. Verdict, errata, and roadmap + phasing above incorporate that review.