qa(verify): FORMAT_COLLAPSED soft-demote + open #000008 (broad-quantifier preflight)
Sister rule to Rule 9 (SUBJECT_TOKENS_ABSENT) landed in the same session. Both demote STRICT → HYBRID but on orthogonal signals: Rule 9 catches premise-parroting; FORMAT_COLLAPSED catches protocol abandonment. Surfaced by fox's "winners of all major sports?" 2026-05-02 case: Hermes-3-8B melted under an under-specified broad-quantifier question, dumped 50+ free-form prose claims with zero [E\d+] pointer tags. Verifier honestly returned UNGROUNDED 0/2 (parser caught two line fragments), but operators couldn't distinguish "tried & failed to ground" from "abandoned the protocol." This soft-demote separates the two failure shapes at audit-line glance. verify_claim_lattice (pointer-mode only — JSON collapse already shows as SCHEMA_INVALID): - count meaningful_lines (>20 chars after strip) and [E\d+ regex matches in raw answer - ≥5 meaningful lines AND 0 bracket tags → FORMAT_COLLAPSED violation, soft-demote STRICT → HYBRID - format_collapsed: bool added to verdict dict Plumbing: - claim_lattice_format_collapse_check_enabled: True in DEFAULT_POLICY and DEFAULT_QUERY_POLICY - _VERIFIER_POLICY_FIELDS in keys.py adds the field so it folds into verifier_policy_hash - threaded through ask() and query() call sites CLI: - _SOFT_DEMOTE_VIOLATION_KINDS includes FORMAT_COLLAPSED so the audit-line ladder rendering treats it as a soft demote - _render_warrant_tail appends "· format collapsed" tail Bench fixture: new "under-specified 'all'" section in qa_questions.txt with `winners of all major sports?` and rationale about cross-model resilience signal. Tests: - test_format_collapsed_fires_on_bracketless_multi_line_prose - test_format_collapsed_does_not_fire_when_pointer_tags_present - CLI render coverage Full suite: 781 passed (up from 776). Open Ticket #000008 — Broad-quantifier preflight guard. Cleaner upstream fix: detect quantifier-intensity at query layer and apply a per-model claim ceiling BEFORE the 13-second LLM call. FORMAT_COLLAPSED stays as the downstream catch; #000008 proposes the upstream prevention. TICKETS.md index + Next ID 000008→000009.
This commit is contained in:
parent
39bebe3fdb
commit
38cfea1983
10 changed files with 396 additions and 1 deletions
|
|
@ -481,6 +481,12 @@ _SOFT_DEMOTE_VIOLATION_KINDS = frozenset({
|
|||
"POINTER_OVERFLOW_TRIMMED",
|
||||
"TOO_MANY_CLAIMS",
|
||||
"BARE_NAME_CLAIM",
|
||||
# FORMAT_COLLAPSED — model abandoned the claim_lattice_pointer
|
||||
# protocol (multi-line prose, zero [E\d+] tags). Soft-demotes
|
||||
# STRICT → HYBRID; pairs with the bottom UNGROUNDED rung when the
|
||||
# parser found nothing groundable, but at least surfaces the
|
||||
# collapse cause to the operator at audit-line glance.
|
||||
"FORMAT_COLLAPSED",
|
||||
})
|
||||
|
||||
|
||||
|
|
@ -610,6 +616,8 @@ def _render_warrant_tail(result: dict) -> str:
|
|||
parts.append("warrant missing")
|
||||
if "TITLE_MISMATCH" in kinds:
|
||||
parts.append("title mismatch")
|
||||
if "FORMAT_COLLAPSED" in kinds:
|
||||
parts.append("format collapsed")
|
||||
if not parts:
|
||||
return ""
|
||||
return " · " + " · ".join(parts)
|
||||
|
|
|
|||
|
|
@ -210,6 +210,7 @@ _VERIFIER_POLICY_FIELDS = frozenset({
|
|||
# Warrant-lite (relation-question hard check, Ticket H, 2026-05-01)
|
||||
"claim_lattice_warrant_check_enabled",
|
||||
"claim_lattice_deflection_check_enabled",
|
||||
"claim_lattice_format_collapse_check_enabled",
|
||||
# Subject-tokens-absent / premise-parroting (Ticket #000006 amend
|
||||
# 2026-05-02b, Rule 9). Threshold of question∩claim content tokens
|
||||
# absent from cited evidence union that demotes STRICT → HYBRID.
|
||||
|
|
|
|||
|
|
@ -455,6 +455,7 @@ DEFAULT_QUERY_POLICY = {
|
|||
# DEFLECTION_DETECTED downgrades EVIDENCE-WARRANTED → ANCHOR-
|
||||
# WARRANTED via the existing soft-demote ladder path.
|
||||
"claim_lattice_deflection_check_enabled": True,
|
||||
"claim_lattice_format_collapse_check_enabled": True,
|
||||
# Claim-count ceiling — see runner.DEFAULT_POLICY for rationale.
|
||||
# Bench finding (york-england "tell me all there is to know")
|
||||
# caught the runaway shape; cap of 12 admits entity-list
|
||||
|
|
@ -2194,6 +2195,9 @@ def query(
|
|||
deflection_check_enabled=bool(policy.get(
|
||||
"claim_lattice_deflection_check_enabled", True
|
||||
)),
|
||||
format_collapse_check_enabled=bool(policy.get(
|
||||
"claim_lattice_format_collapse_check_enabled", True
|
||||
)),
|
||||
)
|
||||
rendered = verdict["rendered_text"]
|
||||
answer_text = rendered if rendered else raw_answer
|
||||
|
|
|
|||
|
|
@ -190,6 +190,16 @@ DEFAULT_POLICY = {
|
|||
# at HYBRID. See aborist/qa/warrant.py.
|
||||
"claim_lattice_warrant_check_enabled": True,
|
||||
"claim_lattice_deflection_check_enabled": True,
|
||||
# Format-collapse check (pointer-mode only): when the model emits
|
||||
# ≥5 meaningful prose lines with zero `[E\d+]` pointer tags, it
|
||||
# abandoned the claim_lattice_pointer protocol entirely. Soft-demote
|
||||
# so audit display surfaces "format collapsed" vs "graceful per-
|
||||
# claim refusal" — different failure shapes, same UNGROUNDED rung.
|
||||
# JSON-mode collapse already shows up as SCHEMA_INVALID so this
|
||||
# check is redundant there. Surfaced 2026-05-02 by fox's "Winners
|
||||
# of all major sports?" case where Hermes dumped 50+ free-form
|
||||
# sentences.
|
||||
"claim_lattice_format_collapse_check_enabled": True,
|
||||
# Claim-count ceiling. Bench finding (2026-04-30 york-england):
|
||||
# "tell me all there is to know about X" prompted Hermes to spam
|
||||
# 26-59 encyclopedic claims sourced from training, only 2-4 of
|
||||
|
|
@ -540,6 +550,9 @@ def ask(
|
|||
deflection_check_enabled=bool(policy.get(
|
||||
"claim_lattice_deflection_check_enabled", True
|
||||
)),
|
||||
format_collapse_check_enabled=bool(policy.get(
|
||||
"claim_lattice_format_collapse_check_enabled", True
|
||||
)),
|
||||
)
|
||||
# Rendered prose (literal spans interpolated) is the user-facing
|
||||
# answer text — never the model's raw pointer-line output. If
|
||||
|
|
|
|||
|
|
@ -1096,6 +1096,7 @@ def verify_claim_lattice(
|
|||
question: str | None = None,
|
||||
warrant_check_enabled: bool = True,
|
||||
deflection_check_enabled: bool = True,
|
||||
format_collapse_check_enabled: bool = True,
|
||||
) -> dict:
|
||||
"""Deterministic verifier for ``answer_mode="claim_lattice_pointer"``.
|
||||
|
||||
|
|
@ -1629,6 +1630,40 @@ def verify_claim_lattice(
|
|||
if deflection_detected and audit_mode == "STRICT":
|
||||
audit_mode = "HYBRID"
|
||||
|
||||
# Format-collapse check (FORMAT_COLLAPSED soft demote).
|
||||
# The "winners of all major sports?" case fox surfaced 2026-05-02:
|
||||
# Hermes melted under an under-specified broad question, dumped
|
||||
# 50+ free-form prose claims with ZERO `[E\d+]` pointer tags. The
|
||||
# parser found 2 line-shaped fragments to count as claims; both
|
||||
# ungrounded → UNGROUNDED 0/2. Verifier was honest, but operators
|
||||
# couldn't tell from the audit line whether UNGROUNDED meant
|
||||
# "tried to ground & failed" vs "abandoned the protocol entirely."
|
||||
# This sidecar separates those two failure shapes by inspecting
|
||||
# the raw answer text for the absence of bracket tags amid
|
||||
# multiple meaningful prose lines.
|
||||
format_collapsed = False
|
||||
if format_collapse_check_enabled and answer_text:
|
||||
meaningful_lines = [
|
||||
line for line in answer_text.splitlines()
|
||||
if len(line.strip()) > 20
|
||||
]
|
||||
bracket_count = len(re.findall(r"\[E\d+", answer_text))
|
||||
if len(meaningful_lines) >= 5 and bracket_count == 0:
|
||||
format_collapsed = True
|
||||
violations.append({
|
||||
"kind": "FORMAT_COLLAPSED",
|
||||
"meaningful_lines": len(meaningful_lines),
|
||||
"bracket_count": bracket_count,
|
||||
"rationale": (
|
||||
"model emitted multi-line prose with zero [E\\d+] "
|
||||
"pointer tags — abandoned the claim_lattice_pointer "
|
||||
"protocol entirely. UNGROUNDED below this signal is "
|
||||
"format collapse, not graceful per-claim refusal."
|
||||
),
|
||||
})
|
||||
if format_collapsed and audit_mode == "STRICT":
|
||||
audit_mode = "HYBRID"
|
||||
|
||||
return {
|
||||
"n_quotes": n_pairs,
|
||||
"n_verified": n_pairs_verified,
|
||||
|
|
@ -1646,6 +1681,7 @@ def verify_claim_lattice(
|
|||
"warrant_missing_claim_idxs": warrant_missing_claims,
|
||||
"title_mismatch_claim_idxs": title_mismatch_claims,
|
||||
"deflection_detected": deflection_detected,
|
||||
"format_collapsed": format_collapsed,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,17 @@ tell me all there is to know about york england?
|
|||
tell me about the roman empire
|
||||
describe the structure of DNA
|
||||
|
||||
# under-specified "all" — the word "all" reads to Hermes-3-8B as
|
||||
# license to enumerate every adjacent fact in training prior, which
|
||||
# in claim_lattice_pointer mode degrades to free-form prose with zero
|
||||
# `[E\d+]` tags (FORMAT_COLLAPSED soft-demote, 2026-05-02). Stronger
|
||||
# models (Qwen / GPT-4 family) plausibly recover format discipline
|
||||
# under the same prompt — bench coverage of this shape lets us
|
||||
# measure cross-model resilience. Ticket #000008 (broad-quantifier
|
||||
# preflight guard) proposes upstream classification + per-model
|
||||
# claim ceiling.
|
||||
winners of all major sports?
|
||||
|
||||
# entity list — invites lazy-anchor on a magnet chunk
|
||||
what dinosaurs were in the first jurassic park film?
|
||||
who are the members of the beatles?
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ Newest first. Update on every open/close.
|
|||
|
||||
| ID | Title | Status | Opened | Directive |
|
||||
|----------|------------------------------------------------|-----------------------|------------|-----------|
|
||||
| #000008 | Broad-quantifier preflight guard | open · awaiting go/no-go | 2026-05-02 | — |
|
||||
| #000007 | Query-layer hyphen folding | closed · 2026-05-02 | 2026-05-02 | — |
|
||||
| #000006 | Bench-emergent findings (rolling research log) | open · rolling | 2026-05-02 | — |
|
||||
| #000005 | Label ladder migration (POINTER-LINKED → …) | closed · 2026-05-02 | 2026-05-01 | D7 |
|
||||
|
|
@ -67,4 +68,4 @@ Newest first. Update on every open/close.
|
|||
|
||||
## Next ID
|
||||
|
||||
`000008`
|
||||
`000009`
|
||||
|
|
|
|||
243
docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md
Normal file
243
docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md
Normal file
|
|
@ -0,0 +1,243 @@
|
|||
# Ticket #000008 — Broad-quantifier preflight guard
|
||||
|
||||
**Status:** open · awaiting go/no-go
|
||||
**Opened:** 2026-05-02
|
||||
**Scope:** Detect under-specified quantifier shapes (`all`, `every`,
|
||||
`everything`, etc.) at the query layer and apply a per-model claim
|
||||
ceiling before the LLM call, instead of catching the resulting format
|
||||
collapse downstream.
|
||||
**Audience:** fox + future blackops shifts.
|
||||
**Hard constraint:** Pure additive policy. No bumps to
|
||||
`schema_version`, `canonicalization_version`, or `chunking_version`.
|
||||
Folds into `governance_policy_hash` so opt-out invalidates prior
|
||||
records on lookup. Verifier stays binary; FORMAT_COLLAPSED (already
|
||||
landed) keeps owning the downstream catch.
|
||||
|
||||
---
|
||||
|
||||
## 1. Problem statement
|
||||
|
||||
Hermes-3-Llama-3.1-8B-FP8 melts under under-specified broad-quantifier
|
||||
questions. The 2026-05-02 case fox surfaced:
|
||||
|
||||
```
|
||||
make query Q="Winners of all major sports?" BURN=1
|
||||
```
|
||||
|
||||
The model interpreted "all" as license to enumerate every adjacent
|
||||
fact in training prior, dumped 50+ free-form prose claims with zero
|
||||
`[E\d+]` pointer tags, and the verifier returned UNGROUNDED 0/2 (the
|
||||
parser caught two line-fragments). FORMAT_COLLAPSED (commit
|
||||
2026-05-02b, this same session) closed the downstream signal gap —
|
||||
operators now see `· format collapsed` on the audit line — but the
|
||||
guardrail fires *after* a 13-second LLM call has already burned.
|
||||
|
||||
The cleaner fix is upstream. The same prompt that runaways on Hermes-
|
||||
3-8B plausibly stays disciplined on Qwen 3 reasoner / GPT-4-class
|
||||
models, because larger models retain format-following discipline
|
||||
under quantifier-induced enumeration pressure. The "right" claim
|
||||
ceiling is therefore *model-dependent* — a hyperparameter we
|
||||
calibrate per endpoint, similar to `max_context_chars_by_mode`
|
||||
(`docs/qa-modes-bench.md`) and `claim_lattice_max_claims_per_answer`
|
||||
(currently 12, runner-default).
|
||||
|
||||
### 1.1 Why "all" specifically
|
||||
|
||||
Fox's framing: `all` reads to a small model like a prompt injection
|
||||
with positive emergent-search energy — useful when the operator
|
||||
*wants* an emergent enumeration (UNGROUNDED honesty is fine, the
|
||||
operator gets a wide scan), corrosive when the operator wants a
|
||||
grounded answer with the protocol respected.
|
||||
|
||||
The signal isn't a single keyword. It's a continuum of quantifier
|
||||
intensity (Section 2). Fox sketched it as:
|
||||
|
||||
> none is safe one to 10 is likely safe many is safe but we should
|
||||
> have a cut off based on the model's abilities that is vibed similar
|
||||
> to the max token input and other hyper params we are learning
|
||||
> about.
|
||||
|
||||
## 2. Quantifier intensity ladder
|
||||
|
||||
| Intensity | Examples | Safe on small models? | Default cap |
|
||||
|-----------|-------------------------------------------------------|------------------------|-------------|
|
||||
| ABSENT | `none`, `no X`, `which X is not …` | yes — single negative claim | 1 |
|
||||
| SINGULAR | `what is X`, `who is X`, `the X`, `which X` | yes — single fact | 1 |
|
||||
| SMALL_NUM | `top 3`, `five biggest`, `seven X`, `the seven …` | yes — bounded by digit | match digit |
|
||||
| FEW | `some`, `a few`, `several` | yes — Hermes-3-8B holds discipline at this shape | 5 |
|
||||
| MANY | `many`, `various`, `multiple`, `most` | model-dependent — Hermes-8B drifts past ~10 | 8 (small) / 12 (large) |
|
||||
| ALL | `all`, `every`, `each`, `complete list`, `tell me everything`, `everything you know` | unsafe on small models — collapses to runaway | 8 (small) / 12 (large) |
|
||||
|
||||
The ABSENT row matters because negation is its own failure shape
|
||||
(Hermes-3-8B inverts under attention — see `docs/bench-maxing.md`),
|
||||
not a quantifier collapse. Capping it at 1 keeps the negation
|
||||
attention narrow.
|
||||
|
||||
The SMALL_NUM row reads the explicit digit and uses it as the cap
|
||||
(`top 3` → cap 3, `seven mercury astronauts` → cap 7). Numbers
|
||||
already specify the count; respecting them avoids both the runaway
|
||||
and the artificial truncation.
|
||||
|
||||
The MANY and ALL rows share a small-model cap (8) but diverge on
|
||||
large models because operators wielding Qwen / GPT-4 should be able
|
||||
to ask broad questions without artificial truncation.
|
||||
|
||||
## 3. Design options
|
||||
|
||||
### Option A — Quantifier-conditioned claim cap (recommended)
|
||||
|
||||
Add a query-layer preflight that classifies the question into one of
|
||||
the six rungs above, then sets `claim_lattice_max_claims_per_answer`
|
||||
per call (overriding the default 12) before retrieval and LLM call.
|
||||
The verifier already supports a per-call cap (`max_claims_per_answer`
|
||||
parameter on `verify_claim_lattice` /
|
||||
`verify_claim_lattice_json`), so this is policy-only — no verifier
|
||||
changes.
|
||||
|
||||
Pros:
|
||||
- Reuses an existing knob. TOO_MANY_CLAIMS already demotes STRICT →
|
||||
HYBRID; we'd be lowering the cap for shapes that warrant it.
|
||||
- Per-model calibration lives in policy dict, not code.
|
||||
- Operator can override via CLI flag (escape hatch for emergent-
|
||||
search use cases — fox's "nice for emergent searches" point).
|
||||
|
||||
Cons:
|
||||
- Doesn't shorten the LLM call directly. The model still sees the
|
||||
prompt, still tries to enumerate, still drifts. We just demote the
|
||||
resulting answer.
|
||||
- Doesn't help the FORMAT_COLLAPSED case where the model emits zero
|
||||
brackets — TOO_MANY_CLAIMS only fires on PARSED claims.
|
||||
|
||||
### Option B — Prompt-side reminder injection
|
||||
|
||||
When the classifier hits MANY / ALL rungs, append a stronger
|
||||
format-discipline reminder to the system prompt (e.g. "Cite at most
|
||||
N claims. If the corpus does not contain enough evidence to cite N
|
||||
claims, return UNGROUNDED rather than enumerating from training
|
||||
prior."). N = per-model cap from the table.
|
||||
|
||||
Pros:
|
||||
- Targets the root cause: model behavior under broad-quantifier
|
||||
pressure.
|
||||
- Cheap on tokens (one extra sentence in the system reminder).
|
||||
|
||||
Cons:
|
||||
- Hermes-3-8B already ignores parts of the existing reminder under
|
||||
enumeration pressure (that's *how* FORMAT_COLLAPSED fires). Adding
|
||||
more reminder text may not change behavior.
|
||||
- Folds into `governance_policy_hash` — requires bench measurement
|
||||
before/after to confirm any delta is real (see
|
||||
`docs/bench-maxing.md` 5pp signal floor).
|
||||
|
||||
### Option C — Reject at query layer
|
||||
|
||||
When the classifier hits ALL on a small model and the operator hasn't
|
||||
opted in, return UNGROUNDED with a `BROAD_QUANTIFIER_REJECTED`
|
||||
violation before the LLM call. Operator gets fast feedback ("your
|
||||
question is too broad for this endpoint, try narrowing or use
|
||||
`--allow-broad`").
|
||||
|
||||
Pros:
|
||||
- Saves the LLM call entirely (~10–15s cost on Hermes-3-8B).
|
||||
- Honest failure shape — UNGROUNDED on a question we know we can't
|
||||
answer well.
|
||||
|
||||
Cons:
|
||||
- False positives are operator-hostile (some "all" questions are
|
||||
genuinely answerable; e.g. "all members of the Beatles" → 4-claim
|
||||
answer, easily groundable).
|
||||
- Couples query layer to model capability — needs the model profile
|
||||
to determine reject vs allow.
|
||||
|
||||
### Option D — Hybrid (recommended composition)
|
||||
|
||||
Combine A + B. Classifier sets the cap (Option A) AND injects a
|
||||
mode-specific reminder (Option B). C stays available as an opt-in
|
||||
flag (`--reject-broad`) but isn't on by default.
|
||||
|
||||
This matches fox's framing: `all` is *useful* for emergent search,
|
||||
just not on small models when the operator wants groundedness. We
|
||||
keep both paths, default to grounded.
|
||||
|
||||
## 4. Recommendation
|
||||
|
||||
Land Option D. Concrete plan:
|
||||
|
||||
1. New module `aborist/qa/quantifier.py` with
|
||||
`classify_question_quantifier(question: str) -> dict` returning
|
||||
`{"intensity": "ALL"|"MANY"|"FEW"|"SMALL_NUM"|"SINGULAR"|"ABSENT",
|
||||
"matched_token": str, "explicit_count": int | None}`. Pure
|
||||
function, no I/O.
|
||||
2. New policy fields on `runner.DEFAULT_POLICY` /
|
||||
`query.DEFAULT_QUERY_POLICY`:
|
||||
- `quantifier_guard_enabled` (default True)
|
||||
- `quantifier_caps_by_intensity` — dict mapping intensity →
|
||||
int cap, with a default profile for `hermes-3-llama-3.1-8b`
|
||||
and a `default` fallback that matches today's behavior (cap 12
|
||||
across the board, so opting in costs nothing).
|
||||
3. Per-model profile registry. `model_profile_hash` (already in the
|
||||
8-dim cache key) gets a quantifier-cap profile attached. The
|
||||
profile lives in `aborist/qa/model_profiles.py` (new file).
|
||||
4. Preflight wiring in `aborist/qa/runner.py:ask` and
|
||||
`aborist/qa/query.py:query`: classify the question, look up the
|
||||
intensity-keyed cap from the model profile, override the
|
||||
`claim_lattice_max_claims_per_answer` for this call.
|
||||
5. Optional reminder injection in the prompt builder (gated on
|
||||
`quantifier_reminder_enabled`, default False until bench
|
||||
confirms a positive delta — see Section 5).
|
||||
6. CLI escape hatch `--allow-broad` → bypasses the cap reduction
|
||||
for explicit emergent-search use.
|
||||
7. Folds into `governance_policy_hash` (already covered by the
|
||||
existing field-list mechanism — add the new keys to
|
||||
`_VERIFIER_POLICY_FIELDS` in `keys.py`).
|
||||
|
||||
## 5. Bench plan
|
||||
|
||||
Bench `winners of all major sports?` is queued in
|
||||
`bench/qa_questions.txt` under "broad descriptive — under-specified
|
||||
'all'" (commit 2026-05-02b). Bench plan:
|
||||
|
||||
1. Baseline (n=3) on Hermes-3-8B with current policy (cap 12). Record
|
||||
FORMAT_COLLAPSED rate, audit_mode distribution, claim count.
|
||||
2. Implement Option A (cap reduction only). Bench (n=3). Compare.
|
||||
3. Implement Option B (reminder injection only, no cap change). Bench
|
||||
(n=3). Compare.
|
||||
4. Implement Option D (A + B together). Bench (n=3). Compare.
|
||||
5. Cross-model: same questions on Qwen 3 / GPT-4 (manual, not in
|
||||
automated bench yet). Confirm large models hold format discipline
|
||||
under broader caps without artificial truncation.
|
||||
|
||||
Signal floor: 5pp per `docs/bench-maxing.md`. Decisions need at least
|
||||
one of: STRICT-rate change, FORMAT_COLLAPSED rate change, claim-count
|
||||
distribution shift.
|
||||
|
||||
## 6. Open questions
|
||||
|
||||
- Does the classifier need to handle multi-quantifier questions
|
||||
("all winners and some losers")? Initial answer: take the highest
|
||||
intensity. Can refine on bench evidence.
|
||||
- Should `tell me about X` (no explicit quantifier) classify as
|
||||
MANY? The "tell me all there is to know" precedent (york-england
|
||||
case, ticket #000006) suggests yes — operationally it produces
|
||||
the same enumeration pressure.
|
||||
- Does the per-model profile belong in `aborist/qa/model_profiles.py`
|
||||
or extend the existing `model_profile_hash` derivation in
|
||||
`keys.py`? Initial answer: new file, hash-derived from the profile
|
||||
dict so changing a cap invalidates prior records.
|
||||
|
||||
## 7. Scope boundaries
|
||||
|
||||
- This ticket does NOT change the verifier. FORMAT_COLLAPSED stays
|
||||
the downstream catch.
|
||||
- This ticket does NOT add a new audit_mode token. Cap demotion
|
||||
surfaces through the existing TOO_MANY_CLAIMS violation path on
|
||||
the audit-line tail.
|
||||
- This ticket does NOT touch retrieval. Quantifier guard runs at
|
||||
policy layer only; retrieval pipeline (`aborist/qa/query.py`
|
||||
Sections 1–9) stays untouched.
|
||||
|
||||
## 8. Status
|
||||
|
||||
Open · awaiting go/no-go on Option D. Bench-first per
|
||||
`docs/bench-maxing.md` — no implementation lands without a measured
|
||||
delta over baseline.
|
||||
|
|
@ -1165,3 +1165,63 @@ def test_too_many_claims_demotes_pointer_mode_to_hybrid():
|
|||
# The cap doesn't truncate — every claim still verifies.
|
||||
assert v["n_verified"] == 13, \
|
||||
f"all 13 should still verify; got {v['n_verified']}"
|
||||
|
||||
|
||||
def test_format_collapsed_fires_on_bracketless_multi_line_prose():
|
||||
"""The 'winners of all major sports?' case (2026-05-02): Hermes
|
||||
melted under an under-specified broad question, dumped 50+ free-form
|
||||
prose claims with ZERO `[E\\d+]` pointer tags. Parser found a couple
|
||||
of fragments, both ungrounded → UNGROUNDED. Verifier was honest, but
|
||||
operators couldn't tell from the audit line whether UNGROUNDED meant
|
||||
'tried to ground & failed' vs 'abandoned the protocol entirely.'
|
||||
FORMAT_COLLAPSED separates the two failure shapes.
|
||||
"""
|
||||
em = build_evidence_map(_sample_chunks())
|
||||
answer = (
|
||||
"The 1979 FINA Men's Water Polo World Cup was won by Hungary.\n"
|
||||
"The 1979 FINA Women's Water Polo World Cup was won by the USA.\n"
|
||||
"The 8th Pan American Games were won by Cuba.\n"
|
||||
"The 8th Mediterranean Games were won by Italy.\n"
|
||||
"The Tenth Summer Universiade was won by the Soviet Union.\n"
|
||||
)
|
||||
v = verify_claim_lattice(answer, em)
|
||||
assert v["format_collapsed"] is True
|
||||
assert any(vio["kind"] == "FORMAT_COLLAPSED" for vio in v["violations"])
|
||||
|
||||
|
||||
def test_format_collapsed_does_not_fire_when_pointer_tags_present():
|
||||
"""A well-formed pointer-line answer with at least one `[E\\d+]`
|
||||
bracket is a graceful protocol-following attempt — even if every
|
||||
pointer is wrong, that's a per-claim verification failure, not a
|
||||
format collapse. FORMAT_COLLAPSED must stay off."""
|
||||
em = build_evidence_map(_sample_chunks())
|
||||
answer = (
|
||||
"Tyrannosaurus rex appears. [E1]\n"
|
||||
"Velociraptors stalk workers. [E2]\n"
|
||||
)
|
||||
v = verify_claim_lattice(answer, em, min_claim_content_tokens=0)
|
||||
assert v["format_collapsed"] is False
|
||||
assert not any(vio["kind"] == "FORMAT_COLLAPSED" for vio in v["violations"])
|
||||
|
||||
|
||||
def test_format_collapsed_skips_short_answers():
|
||||
"""Below the meaningful-line threshold (5 lines >20 chars), absence
|
||||
of pointer tags is more likely a one-line refusal than a runaway
|
||||
prose dump. FORMAT_COLLAPSED stays off."""
|
||||
em = build_evidence_map(_sample_chunks())
|
||||
answer = "I don't know.\n"
|
||||
v = verify_claim_lattice(answer, em)
|
||||
assert v["format_collapsed"] is False
|
||||
|
||||
|
||||
def test_format_collapse_check_disabled_by_policy():
|
||||
"""Operators can opt out via policy. With the check off, even a
|
||||
50-line bracket-free dump returns format_collapsed=False."""
|
||||
em = build_evidence_map(_sample_chunks())
|
||||
answer = "\n".join(
|
||||
f"Some bracket-free prose claim number {i} with enough length."
|
||||
for i in range(10)
|
||||
) + "\n"
|
||||
v = verify_claim_lattice(answer, em, format_collapse_check_enabled=False)
|
||||
assert v["format_collapsed"] is False
|
||||
assert not any(vio["kind"] == "FORMAT_COLLAPSED" for vio in v["violations"])
|
||||
|
|
|
|||
|
|
@ -365,3 +365,21 @@ def test_render_label_quote_mode_keeps_audit_mode_token():
|
|||
assert "EVIDENCE-WARRANTED" not in out
|
||||
assert "POINTER-LINKED" not in out
|
||||
assert "ANCHOR-WARRANTED" not in out
|
||||
|
||||
|
||||
def test_render_label_format_collapsed_surfaces_tail():
|
||||
"""FORMAT_COLLAPSED soft-demotes STRICT → HYBRID and the renderer
|
||||
surfaces a `· format collapsed` tail so an operator can tell from
|
||||
the audit line whether UNGROUNDED meant 'tried & failed' vs
|
||||
'abandoned the protocol entirely.' Pairs with WARRANT_MISSING /
|
||||
TITLE_MISMATCH tails — multiple can coexist."""
|
||||
r = _result(
|
||||
audit_mode="UNGROUNDED",
|
||||
verifier_method="claim_lattice_pointer",
|
||||
n_quotes=2, n_verified=0,
|
||||
violations=[{"kind": "FORMAT_COLLAPSED", "meaningful_lines": 50,
|
||||
"bracket_count": 0}],
|
||||
)
|
||||
out = _render_query_human(r, "winners of all major sports?")
|
||||
assert "format collapsed" in out
|
||||
assert "UNGROUNDED" in out
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue