From 38cfea1983351d036866bb2ab23fa87ed8286e56 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sat, 2 May 2026 16:43:41 -0400 Subject: [PATCH] qa(verify): FORMAT_COLLAPSED soft-demote + open #000008 (broad-quantifier preflight) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sister rule to Rule 9 (SUBJECT_TOKENS_ABSENT) landed in the same session. Both demote STRICT → HYBRID but on orthogonal signals: Rule 9 catches premise-parroting; FORMAT_COLLAPSED catches protocol abandonment. Surfaced by fox's "winners of all major sports?" 2026-05-02 case: Hermes-3-8B melted under an under-specified broad-quantifier question, dumped 50+ free-form prose claims with zero [E\d+] pointer tags. Verifier honestly returned UNGROUNDED 0/2 (parser caught two line fragments), but operators couldn't distinguish "tried & failed to ground" from "abandoned the protocol." This soft-demote separates the two failure shapes at audit-line glance. verify_claim_lattice (pointer-mode only — JSON collapse already shows as SCHEMA_INVALID): - count meaningful_lines (>20 chars after strip) and [E\d+ regex matches in raw answer - ≥5 meaningful lines AND 0 bracket tags → FORMAT_COLLAPSED violation, soft-demote STRICT → HYBRID - format_collapsed: bool added to verdict dict Plumbing: - claim_lattice_format_collapse_check_enabled: True in DEFAULT_POLICY and DEFAULT_QUERY_POLICY - _VERIFIER_POLICY_FIELDS in keys.py adds the field so it folds into verifier_policy_hash - threaded through ask() and query() call sites CLI: - _SOFT_DEMOTE_VIOLATION_KINDS includes FORMAT_COLLAPSED so the audit-line ladder rendering treats it as a soft demote - _render_warrant_tail appends "· format collapsed" tail Bench fixture: new "under-specified 'all'" section in qa_questions.txt with `winners of all major sports?` and rationale about cross-model resilience signal. Tests: - test_format_collapsed_fires_on_bracketless_multi_line_prose - test_format_collapsed_does_not_fire_when_pointer_tags_present - CLI render coverage Full suite: 781 passed (up from 776). Open Ticket #000008 — Broad-quantifier preflight guard. Cleaner upstream fix: detect quantifier-intensity at query layer and apply a per-model claim ceiling BEFORE the 13-second LLM call. FORMAT_COLLAPSED stays as the downstream catch; #000008 proposes the upstream prevention. TICKETS.md index + Next ID 000008→000009. --- aborist/cli.py | 8 + aborist/qa/keys.py | 1 + aborist/qa/query.py | 4 + aborist/qa/runner.py | 13 + aborist/qa/verify.py | 36 +++ bench/qa_questions.txt | 11 + docs/TICKETS.md | 3 +- ...000008-broad-quantifier-preflight-guard.md | 243 ++++++++++++++++++ tests/test_claim_lattice.py | 60 +++++ tests/test_cli_render.py | 18 ++ 10 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md diff --git a/aborist/cli.py b/aborist/cli.py index acb0a3c..5772516 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -481,6 +481,12 @@ _SOFT_DEMOTE_VIOLATION_KINDS = frozenset({ "POINTER_OVERFLOW_TRIMMED", "TOO_MANY_CLAIMS", "BARE_NAME_CLAIM", + # FORMAT_COLLAPSED — model abandoned the claim_lattice_pointer + # protocol (multi-line prose, zero [E\d+] tags). Soft-demotes + # STRICT → HYBRID; pairs with the bottom UNGROUNDED rung when the + # parser found nothing groundable, but at least surfaces the + # collapse cause to the operator at audit-line glance. + "FORMAT_COLLAPSED", }) @@ -610,6 +616,8 @@ def _render_warrant_tail(result: dict) -> str: parts.append("warrant missing") if "TITLE_MISMATCH" in kinds: parts.append("title mismatch") + if "FORMAT_COLLAPSED" in kinds: + parts.append("format collapsed") if not parts: return "" return " · " + " · ".join(parts) diff --git a/aborist/qa/keys.py b/aborist/qa/keys.py index b584cc4..4c8e212 100644 --- a/aborist/qa/keys.py +++ b/aborist/qa/keys.py @@ -210,6 +210,7 @@ _VERIFIER_POLICY_FIELDS = frozenset({ # Warrant-lite (relation-question hard check, Ticket H, 2026-05-01) "claim_lattice_warrant_check_enabled", "claim_lattice_deflection_check_enabled", + "claim_lattice_format_collapse_check_enabled", # Subject-tokens-absent / premise-parroting (Ticket #000006 amend # 2026-05-02b, Rule 9). Threshold of question∩claim content tokens # absent from cited evidence union that demotes STRICT → HYBRID. diff --git a/aborist/qa/query.py b/aborist/qa/query.py index a7186f6..ab241b1 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -455,6 +455,7 @@ DEFAULT_QUERY_POLICY = { # DEFLECTION_DETECTED downgrades EVIDENCE-WARRANTED → ANCHOR- # WARRANTED via the existing soft-demote ladder path. "claim_lattice_deflection_check_enabled": True, + "claim_lattice_format_collapse_check_enabled": True, # Claim-count ceiling — see runner.DEFAULT_POLICY for rationale. # Bench finding (york-england "tell me all there is to know") # caught the runaway shape; cap of 12 admits entity-list @@ -2194,6 +2195,9 @@ def query( deflection_check_enabled=bool(policy.get( "claim_lattice_deflection_check_enabled", True )), + format_collapse_check_enabled=bool(policy.get( + "claim_lattice_format_collapse_check_enabled", True + )), ) rendered = verdict["rendered_text"] answer_text = rendered if rendered else raw_answer diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index c5971f5..739934d 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -190,6 +190,16 @@ DEFAULT_POLICY = { # at HYBRID. See aborist/qa/warrant.py. "claim_lattice_warrant_check_enabled": True, "claim_lattice_deflection_check_enabled": True, + # Format-collapse check (pointer-mode only): when the model emits + # ≥5 meaningful prose lines with zero `[E\d+]` pointer tags, it + # abandoned the claim_lattice_pointer protocol entirely. Soft-demote + # so audit display surfaces "format collapsed" vs "graceful per- + # claim refusal" — different failure shapes, same UNGROUNDED rung. + # JSON-mode collapse already shows up as SCHEMA_INVALID so this + # check is redundant there. Surfaced 2026-05-02 by fox's "Winners + # of all major sports?" case where Hermes dumped 50+ free-form + # sentences. + "claim_lattice_format_collapse_check_enabled": True, # Claim-count ceiling. Bench finding (2026-04-30 york-england): # "tell me all there is to know about X" prompted Hermes to spam # 26-59 encyclopedic claims sourced from training, only 2-4 of @@ -540,6 +550,9 @@ def ask( deflection_check_enabled=bool(policy.get( "claim_lattice_deflection_check_enabled", True )), + format_collapse_check_enabled=bool(policy.get( + "claim_lattice_format_collapse_check_enabled", True + )), ) # Rendered prose (literal spans interpolated) is the user-facing # answer text — never the model's raw pointer-line output. If diff --git a/aborist/qa/verify.py b/aborist/qa/verify.py index 0584a3b..0902216 100644 --- a/aborist/qa/verify.py +++ b/aborist/qa/verify.py @@ -1096,6 +1096,7 @@ def verify_claim_lattice( question: str | None = None, warrant_check_enabled: bool = True, deflection_check_enabled: bool = True, + format_collapse_check_enabled: bool = True, ) -> dict: """Deterministic verifier for ``answer_mode="claim_lattice_pointer"``. @@ -1629,6 +1630,40 @@ def verify_claim_lattice( if deflection_detected and audit_mode == "STRICT": audit_mode = "HYBRID" + # Format-collapse check (FORMAT_COLLAPSED soft demote). + # The "winners of all major sports?" case fox surfaced 2026-05-02: + # Hermes melted under an under-specified broad question, dumped + # 50+ free-form prose claims with ZERO `[E\d+]` pointer tags. The + # parser found 2 line-shaped fragments to count as claims; both + # ungrounded → UNGROUNDED 0/2. Verifier was honest, but operators + # couldn't tell from the audit line whether UNGROUNDED meant + # "tried to ground & failed" vs "abandoned the protocol entirely." + # This sidecar separates those two failure shapes by inspecting + # the raw answer text for the absence of bracket tags amid + # multiple meaningful prose lines. + format_collapsed = False + if format_collapse_check_enabled and answer_text: + meaningful_lines = [ + line for line in answer_text.splitlines() + if len(line.strip()) > 20 + ] + bracket_count = len(re.findall(r"\[E\d+", answer_text)) + if len(meaningful_lines) >= 5 and bracket_count == 0: + format_collapsed = True + violations.append({ + "kind": "FORMAT_COLLAPSED", + "meaningful_lines": len(meaningful_lines), + "bracket_count": bracket_count, + "rationale": ( + "model emitted multi-line prose with zero [E\\d+] " + "pointer tags — abandoned the claim_lattice_pointer " + "protocol entirely. UNGROUNDED below this signal is " + "format collapse, not graceful per-claim refusal." + ), + }) + if format_collapsed and audit_mode == "STRICT": + audit_mode = "HYBRID" + return { "n_quotes": n_pairs, "n_verified": n_pairs_verified, @@ -1646,6 +1681,7 @@ def verify_claim_lattice( "warrant_missing_claim_idxs": warrant_missing_claims, "title_mismatch_claim_idxs": title_mismatch_claims, "deflection_detected": deflection_detected, + "format_collapsed": format_collapsed, } diff --git a/bench/qa_questions.txt b/bench/qa_questions.txt index dbccbd4..b9c8092 100644 --- a/bench/qa_questions.txt +++ b/bench/qa_questions.txt @@ -22,6 +22,17 @@ tell me all there is to know about york england? tell me about the roman empire describe the structure of DNA +# under-specified "all" — the word "all" reads to Hermes-3-8B as +# license to enumerate every adjacent fact in training prior, which +# in claim_lattice_pointer mode degrades to free-form prose with zero +# `[E\d+]` tags (FORMAT_COLLAPSED soft-demote, 2026-05-02). Stronger +# models (Qwen / GPT-4 family) plausibly recover format discipline +# under the same prompt — bench coverage of this shape lets us +# measure cross-model resilience. Ticket #000008 (broad-quantifier +# preflight guard) proposes upstream classification + per-model +# claim ceiling. +winners of all major sports? + # entity list — invites lazy-anchor on a magnet chunk what dinosaurs were in the first jurassic park film? who are the members of the beatles? diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 9cee07a..451f768 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -57,6 +57,7 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| +| #000008 | Broad-quantifier preflight guard | open · awaiting go/no-go | 2026-05-02 | — | | #000007 | Query-layer hyphen folding | closed · 2026-05-02 | 2026-05-02 | — | | #000006 | Bench-emergent findings (rolling research log) | open · rolling | 2026-05-02 | — | | #000005 | Label ladder migration (POINTER-LINKED → …) | closed · 2026-05-02 | 2026-05-01 | D7 | @@ -67,4 +68,4 @@ Newest first. Update on every open/close. ## Next ID -`000008` +`000009` diff --git a/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md b/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md new file mode 100644 index 0000000..4168cce --- /dev/null +++ b/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md @@ -0,0 +1,243 @@ +# Ticket #000008 — Broad-quantifier preflight guard + +**Status:** open · awaiting go/no-go +**Opened:** 2026-05-02 +**Scope:** Detect under-specified quantifier shapes (`all`, `every`, +`everything`, etc.) at the query layer and apply a per-model claim +ceiling before the LLM call, instead of catching the resulting format +collapse downstream. +**Audience:** fox + future blackops shifts. +**Hard constraint:** Pure additive policy. No bumps to +`schema_version`, `canonicalization_version`, or `chunking_version`. +Folds into `governance_policy_hash` so opt-out invalidates prior +records on lookup. Verifier stays binary; FORMAT_COLLAPSED (already +landed) keeps owning the downstream catch. + +--- + +## 1. Problem statement + +Hermes-3-Llama-3.1-8B-FP8 melts under under-specified broad-quantifier +questions. The 2026-05-02 case fox surfaced: + +``` +make query Q="Winners of all major sports?" BURN=1 +``` + +The model interpreted "all" as license to enumerate every adjacent +fact in training prior, dumped 50+ free-form prose claims with zero +`[E\d+]` pointer tags, and the verifier returned UNGROUNDED 0/2 (the +parser caught two line-fragments). FORMAT_COLLAPSED (commit +2026-05-02b, this same session) closed the downstream signal gap — +operators now see `· format collapsed` on the audit line — but the +guardrail fires *after* a 13-second LLM call has already burned. + +The cleaner fix is upstream. The same prompt that runaways on Hermes- +3-8B plausibly stays disciplined on Qwen 3 reasoner / GPT-4-class +models, because larger models retain format-following discipline +under quantifier-induced enumeration pressure. The "right" claim +ceiling is therefore *model-dependent* — a hyperparameter we +calibrate per endpoint, similar to `max_context_chars_by_mode` +(`docs/qa-modes-bench.md`) and `claim_lattice_max_claims_per_answer` +(currently 12, runner-default). + +### 1.1 Why "all" specifically + +Fox's framing: `all` reads to a small model like a prompt injection +with positive emergent-search energy — useful when the operator +*wants* an emergent enumeration (UNGROUNDED honesty is fine, the +operator gets a wide scan), corrosive when the operator wants a +grounded answer with the protocol respected. + +The signal isn't a single keyword. It's a continuum of quantifier +intensity (Section 2). Fox sketched it as: + +> none is safe one to 10 is likely safe many is safe but we should +> have a cut off based on the model's abilities that is vibed similar +> to the max token input and other hyper params we are learning +> about. + +## 2. Quantifier intensity ladder + +| Intensity | Examples | Safe on small models? | Default cap | +|-----------|-------------------------------------------------------|------------------------|-------------| +| ABSENT | `none`, `no X`, `which X is not …` | yes — single negative claim | 1 | +| SINGULAR | `what is X`, `who is X`, `the X`, `which X` | yes — single fact | 1 | +| SMALL_NUM | `top 3`, `five biggest`, `seven X`, `the seven …` | yes — bounded by digit | match digit | +| FEW | `some`, `a few`, `several` | yes — Hermes-3-8B holds discipline at this shape | 5 | +| MANY | `many`, `various`, `multiple`, `most` | model-dependent — Hermes-8B drifts past ~10 | 8 (small) / 12 (large) | +| ALL | `all`, `every`, `each`, `complete list`, `tell me everything`, `everything you know` | unsafe on small models — collapses to runaway | 8 (small) / 12 (large) | + +The ABSENT row matters because negation is its own failure shape +(Hermes-3-8B inverts under attention — see `docs/bench-maxing.md`), +not a quantifier collapse. Capping it at 1 keeps the negation +attention narrow. + +The SMALL_NUM row reads the explicit digit and uses it as the cap +(`top 3` → cap 3, `seven mercury astronauts` → cap 7). Numbers +already specify the count; respecting them avoids both the runaway +and the artificial truncation. + +The MANY and ALL rows share a small-model cap (8) but diverge on +large models because operators wielding Qwen / GPT-4 should be able +to ask broad questions without artificial truncation. + +## 3. Design options + +### Option A — Quantifier-conditioned claim cap (recommended) + +Add a query-layer preflight that classifies the question into one of +the six rungs above, then sets `claim_lattice_max_claims_per_answer` +per call (overriding the default 12) before retrieval and LLM call. +The verifier already supports a per-call cap (`max_claims_per_answer` +parameter on `verify_claim_lattice` / +`verify_claim_lattice_json`), so this is policy-only — no verifier +changes. + +Pros: +- Reuses an existing knob. TOO_MANY_CLAIMS already demotes STRICT → + HYBRID; we'd be lowering the cap for shapes that warrant it. +- Per-model calibration lives in policy dict, not code. +- Operator can override via CLI flag (escape hatch for emergent- + search use cases — fox's "nice for emergent searches" point). + +Cons: +- Doesn't shorten the LLM call directly. The model still sees the + prompt, still tries to enumerate, still drifts. We just demote the + resulting answer. +- Doesn't help the FORMAT_COLLAPSED case where the model emits zero + brackets — TOO_MANY_CLAIMS only fires on PARSED claims. + +### Option B — Prompt-side reminder injection + +When the classifier hits MANY / ALL rungs, append a stronger +format-discipline reminder to the system prompt (e.g. "Cite at most +N claims. If the corpus does not contain enough evidence to cite N +claims, return UNGROUNDED rather than enumerating from training +prior."). N = per-model cap from the table. + +Pros: +- Targets the root cause: model behavior under broad-quantifier + pressure. +- Cheap on tokens (one extra sentence in the system reminder). + +Cons: +- Hermes-3-8B already ignores parts of the existing reminder under + enumeration pressure (that's *how* FORMAT_COLLAPSED fires). Adding + more reminder text may not change behavior. +- Folds into `governance_policy_hash` — requires bench measurement + before/after to confirm any delta is real (see + `docs/bench-maxing.md` 5pp signal floor). + +### Option C — Reject at query layer + +When the classifier hits ALL on a small model and the operator hasn't +opted in, return UNGROUNDED with a `BROAD_QUANTIFIER_REJECTED` +violation before the LLM call. Operator gets fast feedback ("your +question is too broad for this endpoint, try narrowing or use +`--allow-broad`"). + +Pros: +- Saves the LLM call entirely (~10–15s cost on Hermes-3-8B). +- Honest failure shape — UNGROUNDED on a question we know we can't + answer well. + +Cons: +- False positives are operator-hostile (some "all" questions are + genuinely answerable; e.g. "all members of the Beatles" → 4-claim + answer, easily groundable). +- Couples query layer to model capability — needs the model profile + to determine reject vs allow. + +### Option D — Hybrid (recommended composition) + +Combine A + B. Classifier sets the cap (Option A) AND injects a +mode-specific reminder (Option B). C stays available as an opt-in +flag (`--reject-broad`) but isn't on by default. + +This matches fox's framing: `all` is *useful* for emergent search, +just not on small models when the operator wants groundedness. We +keep both paths, default to grounded. + +## 4. Recommendation + +Land Option D. Concrete plan: + +1. New module `aborist/qa/quantifier.py` with + `classify_question_quantifier(question: str) -> dict` returning + `{"intensity": "ALL"|"MANY"|"FEW"|"SMALL_NUM"|"SINGULAR"|"ABSENT", + "matched_token": str, "explicit_count": int | None}`. Pure + function, no I/O. +2. New policy fields on `runner.DEFAULT_POLICY` / + `query.DEFAULT_QUERY_POLICY`: + - `quantifier_guard_enabled` (default True) + - `quantifier_caps_by_intensity` — dict mapping intensity → + int cap, with a default profile for `hermes-3-llama-3.1-8b` + and a `default` fallback that matches today's behavior (cap 12 + across the board, so opting in costs nothing). +3. Per-model profile registry. `model_profile_hash` (already in the + 8-dim cache key) gets a quantifier-cap profile attached. The + profile lives in `aborist/qa/model_profiles.py` (new file). +4. Preflight wiring in `aborist/qa/runner.py:ask` and + `aborist/qa/query.py:query`: classify the question, look up the + intensity-keyed cap from the model profile, override the + `claim_lattice_max_claims_per_answer` for this call. +5. Optional reminder injection in the prompt builder (gated on + `quantifier_reminder_enabled`, default False until bench + confirms a positive delta — see Section 5). +6. CLI escape hatch `--allow-broad` → bypasses the cap reduction + for explicit emergent-search use. +7. Folds into `governance_policy_hash` (already covered by the + existing field-list mechanism — add the new keys to + `_VERIFIER_POLICY_FIELDS` in `keys.py`). + +## 5. Bench plan + +Bench `winners of all major sports?` is queued in +`bench/qa_questions.txt` under "broad descriptive — under-specified +'all'" (commit 2026-05-02b). Bench plan: + +1. Baseline (n=3) on Hermes-3-8B with current policy (cap 12). Record + FORMAT_COLLAPSED rate, audit_mode distribution, claim count. +2. Implement Option A (cap reduction only). Bench (n=3). Compare. +3. Implement Option B (reminder injection only, no cap change). Bench + (n=3). Compare. +4. Implement Option D (A + B together). Bench (n=3). Compare. +5. Cross-model: same questions on Qwen 3 / GPT-4 (manual, not in + automated bench yet). Confirm large models hold format discipline + under broader caps without artificial truncation. + +Signal floor: 5pp per `docs/bench-maxing.md`. Decisions need at least +one of: STRICT-rate change, FORMAT_COLLAPSED rate change, claim-count +distribution shift. + +## 6. Open questions + +- Does the classifier need to handle multi-quantifier questions + ("all winners and some losers")? Initial answer: take the highest + intensity. Can refine on bench evidence. +- Should `tell me about X` (no explicit quantifier) classify as + MANY? The "tell me all there is to know" precedent (york-england + case, ticket #000006) suggests yes — operationally it produces + the same enumeration pressure. +- Does the per-model profile belong in `aborist/qa/model_profiles.py` + or extend the existing `model_profile_hash` derivation in + `keys.py`? Initial answer: new file, hash-derived from the profile + dict so changing a cap invalidates prior records. + +## 7. Scope boundaries + +- This ticket does NOT change the verifier. FORMAT_COLLAPSED stays + the downstream catch. +- This ticket does NOT add a new audit_mode token. Cap demotion + surfaces through the existing TOO_MANY_CLAIMS violation path on + the audit-line tail. +- This ticket does NOT touch retrieval. Quantifier guard runs at + policy layer only; retrieval pipeline (`aborist/qa/query.py` + Sections 1–9) stays untouched. + +## 8. Status + +Open · awaiting go/no-go on Option D. Bench-first per +`docs/bench-maxing.md` — no implementation lands without a measured +delta over baseline. diff --git a/tests/test_claim_lattice.py b/tests/test_claim_lattice.py index a75ebdb..8ab46ef 100644 --- a/tests/test_claim_lattice.py +++ b/tests/test_claim_lattice.py @@ -1165,3 +1165,63 @@ def test_too_many_claims_demotes_pointer_mode_to_hybrid(): # The cap doesn't truncate — every claim still verifies. assert v["n_verified"] == 13, \ f"all 13 should still verify; got {v['n_verified']}" + + +def test_format_collapsed_fires_on_bracketless_multi_line_prose(): + """The 'winners of all major sports?' case (2026-05-02): Hermes + melted under an under-specified broad question, dumped 50+ free-form + prose claims with ZERO `[E\\d+]` pointer tags. Parser found a couple + of fragments, both ungrounded → UNGROUNDED. Verifier was honest, but + operators couldn't tell from the audit line whether UNGROUNDED meant + 'tried to ground & failed' vs 'abandoned the protocol entirely.' + FORMAT_COLLAPSED separates the two failure shapes. + """ + em = build_evidence_map(_sample_chunks()) + answer = ( + "The 1979 FINA Men's Water Polo World Cup was won by Hungary.\n" + "The 1979 FINA Women's Water Polo World Cup was won by the USA.\n" + "The 8th Pan American Games were won by Cuba.\n" + "The 8th Mediterranean Games were won by Italy.\n" + "The Tenth Summer Universiade was won by the Soviet Union.\n" + ) + v = verify_claim_lattice(answer, em) + assert v["format_collapsed"] is True + assert any(vio["kind"] == "FORMAT_COLLAPSED" for vio in v["violations"]) + + +def test_format_collapsed_does_not_fire_when_pointer_tags_present(): + """A well-formed pointer-line answer with at least one `[E\\d+]` + bracket is a graceful protocol-following attempt — even if every + pointer is wrong, that's a per-claim verification failure, not a + format collapse. FORMAT_COLLAPSED must stay off.""" + em = build_evidence_map(_sample_chunks()) + answer = ( + "Tyrannosaurus rex appears. [E1]\n" + "Velociraptors stalk workers. [E2]\n" + ) + v = verify_claim_lattice(answer, em, min_claim_content_tokens=0) + assert v["format_collapsed"] is False + assert not any(vio["kind"] == "FORMAT_COLLAPSED" for vio in v["violations"]) + + +def test_format_collapsed_skips_short_answers(): + """Below the meaningful-line threshold (5 lines >20 chars), absence + of pointer tags is more likely a one-line refusal than a runaway + prose dump. FORMAT_COLLAPSED stays off.""" + em = build_evidence_map(_sample_chunks()) + answer = "I don't know.\n" + v = verify_claim_lattice(answer, em) + assert v["format_collapsed"] is False + + +def test_format_collapse_check_disabled_by_policy(): + """Operators can opt out via policy. With the check off, even a + 50-line bracket-free dump returns format_collapsed=False.""" + em = build_evidence_map(_sample_chunks()) + answer = "\n".join( + f"Some bracket-free prose claim number {i} with enough length." + for i in range(10) + ) + "\n" + v = verify_claim_lattice(answer, em, format_collapse_check_enabled=False) + assert v["format_collapsed"] is False + assert not any(vio["kind"] == "FORMAT_COLLAPSED" for vio in v["violations"]) diff --git a/tests/test_cli_render.py b/tests/test_cli_render.py index 69d2cd0..df72379 100644 --- a/tests/test_cli_render.py +++ b/tests/test_cli_render.py @@ -365,3 +365,21 @@ def test_render_label_quote_mode_keeps_audit_mode_token(): assert "EVIDENCE-WARRANTED" not in out assert "POINTER-LINKED" not in out assert "ANCHOR-WARRANTED" not in out + + +def test_render_label_format_collapsed_surfaces_tail(): + """FORMAT_COLLAPSED soft-demotes STRICT → HYBRID and the renderer + surfaces a `· format collapsed` tail so an operator can tell from + the audit line whether UNGROUNDED meant 'tried & failed' vs + 'abandoned the protocol entirely.' Pairs with WARRANT_MISSING / + TITLE_MISMATCH tails — multiple can coexist.""" + r = _result( + audit_mode="UNGROUNDED", + verifier_method="claim_lattice_pointer", + n_quotes=2, n_verified=0, + violations=[{"kind": "FORMAT_COLLAPSED", "meaningful_lines": 50, + "bracket_count": 0}], + ) + out = _render_query_human(r, "winners of all major sports?") + assert "format collapsed" in out + assert "UNGROUNDED" in out