diff --git a/aborist/qa/query.py b/aborist/qa/query.py index ab241b1..ab06ce6 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -2577,6 +2577,20 @@ def query( # of reaching STRICT. Persisted into run_dag_blob via the # verify stage's payload. "violations": verdict.get("violations") or [], + # Format-collapse signal (pointer-mode only — None elsewhere). + # True when the model emitted ≥5 meaningful prose lines with + # zero `[E\d+]` pointer tags, i.e. abandoned the + # claim_lattice_pointer protocol entirely. Surfaced on the + # result dict so bench harness can measure FC rate without + # re-deriving it from raw_answer (which is lattice-only). + # See verify.py:format_collapsed and ticket #000008. + "format_collapsed": verdict.get("format_collapsed"), + # Model's raw output before the renderer interpolates literal + # spans. Lattice modes only — quote/span/entity/paraphrase + # rows have answer_text == raw_answer so this stays None to + # avoid duplication. Bench reads it for bracket-count + # diagnostics; never persisted in providence_cache. + "raw_answer": raw_answer if is_lattice_mode else None, # Sidecar smell signals (claim_lattice mode only) — surfaced # for the renderer; never persisted in providence_cache and # never threaded into run_dag_root. diff --git a/bench/qa_questions_quantifier_baseline.txt b/bench/qa_questions_quantifier_baseline.txt new file mode 100644 index 0000000..083fbfc --- /dev/null +++ b/bench/qa_questions_quantifier_baseline.txt @@ -0,0 +1,6 @@ +# One-question bench file — Ticket #000008 baseline. +# Isolates the under-specified "all" failure shape so we can measure +# FORMAT_COLLAPSED rate, audit_mode distribution, and claim count +# under the *current* policy (cap 12) before proposing changes. +# Once Option A or D lands, re-run against this same file to compare. +winners of all major sports? diff --git a/bench/qa_sweep.py b/bench/qa_sweep.py index 15c3e58..3de0992 100644 --- a/bench/qa_sweep.py +++ b/bench/qa_sweep.py @@ -28,6 +28,7 @@ import datetime as _dt import json import os import random +import re import sys import time from collections import defaultdict @@ -40,6 +41,12 @@ from threading import Lock ANSWER_MODES = ("quote", "claim_lattice_pointer", "claim_lattice") +# Pointer-tag regex for bench-side bracket counting on raw model +# output. Mirrors the verifier's FORMAT_COLLAPSED detector +# (verify.py:1650). Module-level so the sweep loop doesn't recompile +# it per row. +_BRACKET_RE = re.compile(r"\[E\d+") + def _read_questions(path: Path) -> list[str]: out: list[str] = [] @@ -116,6 +123,22 @@ def _run_one( ) audit_mode = result.get("audit_mode") + # Bracket count from the model's RAW output (before the renderer + # interpolates literal spans). Lattice modes only; non-lattice + # rows record 0. Pairs with format_collapsed for ticket #000008 + # bench-side diagnostics: a low ratio + zero brackets + many + # lines tells us format collapse, not graceful per-claim refusal. + raw_answer = result.get("raw_answer") or "" + answer_brackets = ( + len(_BRACKET_RE.findall(raw_answer)) if raw_answer else 0 + ) + # Violation kind summary — full violation dicts are kept on the + # result for the renderer but bench rows only need the kinds for + # aggregate counting. Keeps row size bounded. + violation_kinds = sorted({ + v.get("kind") for v in (result.get("violations") or []) + if v.get("kind") + }) return { "question": question, "answer_mode": answer_mode, @@ -129,6 +152,16 @@ def _run_one( "failure_stage": result.get("failure_stage"), "lazy_anchor_ratio": result.get("lazy_anchor_ratio"), "pointer_id_distribution": result.get("pointer_id_distribution"), + # Format-collapse signal (pointer-mode only — None elsewhere). + # See verify.py and ticket #000008. + "format_collapsed": result.get("format_collapsed"), + # Sorted unique kinds list for aggregate counting; full + # violation payloads stay off the bench row to keep size + # bounded (5+ violations × dict ~= bloat across 10K-row sweeps). + "violation_kinds": violation_kinds, + # Bracket count in the model's raw output. Lattice-mode-only + # diagnostic; quote/span/entity/paraphrase rows always 0. + "answer_brackets": answer_brackets, "cache_key": (result.get("cache_key") or "")[:12], "n_sources": len(result.get("sources") or []), "elapsed_s": elapsed_s, @@ -223,6 +256,18 @@ def _summarize(rows: list[dict]) -> dict: "ratio_sum": 0.0, "latency_sum": 0.0, "deflections": 0, + # Format-collapse rate (pointer-mode only — None elsewhere + # so non-lattice rows count as 0). Surfaced in markdown + # alongside deflection rate as a per-mode collapse signal. + "format_collapses": 0, + # Per-violation-kind counts. Open-ended dict — fills as + # kinds are encountered. Empty when no violations fire. + "violation_kind_counts": defaultdict(int), + # Bracket-count distribution for lattice rows. Surfaces + # whether the model is following the pointer protocol at + # all; FORMAT_COLLAPSED is the bracket=0 corner. + "answer_brackets_sum": 0, + "answer_brackets_n": 0, # Per-directive pass counts (seven-point program). Init # all known directive ids so absent rows report 0/N # rather than missing-key. @@ -248,6 +293,25 @@ def _summarize(rows: list[dict]) -> dict: "STRICT", "HYBRID" ): b["deflections"] += 1 + # Format-collapse: pointer-mode rows report bool, others + # report None. Treat None as not-collapsed (the check didn't + # apply); only count explicit True. + if r.get("format_collapsed") is True: + b["format_collapses"] += 1 + # Violation-kind tallies — each kind counts once per row even + # if the same kind fires on multiple claims. The bench is + # asking "did this kind fire on this run?", not "how many + # times within the run". + for kind in (r.get("violation_kinds") or []): + b["violation_kind_counts"][kind] += 1 + # Bracket-count aggregates. Lattice rows only — quote and + # other modes record 0 so they'd skew the mean if averaged + # globally. Track per-mode sum + count; renderer can compute + # mean only for lattice modes. + ab = r.get("answer_brackets") + if ab is not None and m in ("claim_lattice_pointer", "claim_lattice"): + b["answer_brackets_sum"] += ab + b["answer_brackets_n"] += 1 # Directive compliance — sum the per-row booleans into # per-mode pass counts. for did, ok in (r.get("directive_compliance") or {}).items(): @@ -319,6 +383,65 @@ def _render_markdown( f"{deflections}/{b['n']} |" ) lines.append("") + lines.append("## format-collapse + violation kinds") + lines.append("") + lines.append( + "Per-mode count of FORMAT_COLLAPSED firings (pointer-mode signal — " + "model emitted ≥5 meaningful prose lines with zero `[E\\d+]` tags) " + "and per-violation-kind tallies. Each kind counts once per row " + "even if it fires on multiple claims within the run. See " + "`docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md` " + "for why this matters: format collapse separates 'tried to ground " + "& failed' from 'abandoned the protocol entirely' at aggregate " + "scale." + ) + lines.append("") + # Build the union of violation kinds observed across all modes for + # the table header — keeps the column set bench-wide rather than + # per-mode (so quote-mode rows show 0/N for kinds that only fire + # in lattice modes, instead of the column being absent). + all_kinds = sorted({ + k for b in summary.values() + for k in (b.get("violation_kind_counts") or {}).keys() + }) + if all_kinds: + header = ( + "| mode | format-collapse | mean brackets (raw) | " + + " | ".join(all_kinds) + + " |" + ) + sep = ( + "|------|-----------------|---------------------|" + + "|".join("-" * (len(k) + 2) for k in all_kinds) + + "|" + ) + else: + header = "| mode | format-collapse | mean brackets (raw) |" + sep = "|------|-----------------|---------------------|" + lines.append(header) + lines.append(sep) + for mode in modes: + b = summary.get(mode) + if not b: + continue + n = b["n"] or 1 + fc = b.get("format_collapses", 0) + mean_brackets = ( + b["answer_brackets_sum"] / b["answer_brackets_n"] + if b.get("answer_brackets_n") + else 0.0 + ) + kind_cells = [ + str(b["violation_kind_counts"].get(k, 0)) + for k in all_kinds + ] + cells_str = " | ".join(kind_cells) + prefix = f"| {mode} | {fc}/{b['n']} | {mean_brackets:.1f} |" + if all_kinds: + lines.append(f"{prefix} {cells_str} |") + else: + lines.append(prefix) + lines.append("") lines.append("## directive coverage (seven-point program)") lines.append("") lines.append( diff --git a/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md b/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md index 4168cce..852179a 100644 --- a/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md +++ b/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md @@ -59,28 +59,83 @@ intensity (Section 2). Fox sketched it as: ## 2. Quantifier intensity ladder -| Intensity | Examples | Safe on small models? | Default cap | -|-----------|-------------------------------------------------------|------------------------|-------------| -| ABSENT | `none`, `no X`, `which X is not …` | yes — single negative claim | 1 | -| SINGULAR | `what is X`, `who is X`, `the X`, `which X` | yes — single fact | 1 | -| SMALL_NUM | `top 3`, `five biggest`, `seven X`, `the seven …` | yes — bounded by digit | match digit | -| FEW | `some`, `a few`, `several` | yes — Hermes-3-8B holds discipline at this shape | 5 | -| MANY | `many`, `various`, `multiple`, `most` | model-dependent — Hermes-8B drifts past ~10 | 8 (small) / 12 (large) | -| ALL | `all`, `every`, `each`, `complete list`, `tell me everything`, `everything you know` | unsafe on small models — collapses to runaway | 8 (small) / 12 (large) | +The categories below derive from formal-semantics quantifier theory +(Mostowski generalized quantifiers; Barwise-Cooper; Partee D- vs A- +quantifiers) intersected with the operational axis aborist actually +needs: **expected number of claims in the answer**. Categories that +don't change the expected answer length are dropped from this table +and surfaced as adjacent dimensions in Section 2.1. -The ABSENT row matters because negation is its own failure shape -(Hermes-3-8B inverts under attention — see `docs/bench-maxing.md`), -not a quantifier collapse. Capping it at 1 keeps the negation -attention narrow. +| # | Intensity | Operational shape | Examples (lexical surfaces) | Default cap (small / large) | Notes | +|----|-------------------|----------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------|----------------------------------------------------------------------------------------------------| +| 1 | ABSENT | universal-negation, single claim | `none`, `no X`, `nothing`, `nobody`, `nowhere`, `neither`, `never`, `not a single`, `zero`, `which X is not …` | 1 / 1 | Hermes-3-8B also inverts under negation attention; cap-1 narrows the surface (see bench-maxing.md) | +| 2 | SINGULAR | one-fact wh / definite reference | `what is X`, `who is X`, `the X`, `which X`, `whom`, `whose`, `name the X`, `identify the X`, `pick the X` | 1 / 1 | Default for any wh-question without plural/quantifier marker | +| 3 | PROPORTIONAL | descriptive fraction | `most`, `majority of`, `minority of`, `half`, `a third of`, `25% of`, `the bulk of`, `the lion's share of` | 1 / 3 | Answer is *about* a fraction, not a count of fractions. Stays low | +| 4 | SMALL_NUM_EXPLICIT| bounded by digit | `top 3`, `five biggest`, `seven X`, `the X-th`, `first/second/third`, `last`, `pair of`, `couple of`, `dozen`, `a handful of` | match digit / match digit | Read the digit/word and use it. `dozen` → 12, `handful` → 5 | +| 5 | COMPARATIVE_BOUND | bounded by inequality | `at least X`, `at most X`, `more than X`, `fewer than X`, `under X`, `over X`, `up to X`, `between X and Y`, `no more than X` | match bound / match bound | Numeric bound from the question. Less common in QA but worth handling | +| 6 | FEW | small set, vague | `some`, `a few`, `several`, `a couple`, `a handful`, `a small number of`, `a smattering of`, `not many`, `hardly any` | 5 / 5 | Hermes-3-8B holds discipline here | +| 7 | MANY | medium set, vague | `many`, `various`, `multiple`, `numerous`, `a number of`, `lots of`, `plenty of`, `a great many`, `multitudes`, `several dozen` | 8 / 12 | Where the small/large model gap opens | +| 8 | ALL | universal quantifier | `all`, `every`, `each`, `each and every`, `every single`, `the whole`, `the entirety of`, `the totality of`, `any` (universal use), `whatever`, `whoever` | 8 / 12 | The 2026-05-02 fox case. Same cap as MANY but flagged as runaway-prone | +| 9 | COMPREHENSIVE | exhaustive request | `comprehensive`, `complete list`, `complete inventory`, `exhaustive`, `definitive`, `everything you know`, `tell me everything`, `the whole story`, `the full picture`, `from A to Z` | 5 / 15 | *Stronger* than ALL — explicitly requests exhaustion. Highest runaway risk on small models | +| 10 | OPEN_REQUEST | verb-driven enumeration | `tell me about`, `describe`, `explain`, `summarize`, `give me an overview of`, `walk me through`, `what about`, `discuss`, `elaborate on`, `expound on` | 5 / 12 | No explicit quantifier word — verb shape implies enumeration. The york-england failure shape (#000006) | -The SMALL_NUM row reads the explicit digit and uses it as the cap -(`top 3` → cap 3, `seven mercury astronauts` → cap 7). Numbers -already specify the count; respecting them avoids both the runaway -and the artificial truncation. +Total: 10 rungs, up from 6. -The MANY and ALL rows share a small-model cap (8) but diverge on -large models because operators wielding Qwen / GPT-4 should be able -to ask broad questions without artificial truncation. +### 2.1 Adjacent linguistic dimensions (orthogonal axes) + +Not every linguistic feature changes the expected answer length. +Some are real but belong on a separate axis from the quantifier +ladder, and may warrant their own ticket later. + +- **Frequency / temporal universals**: `always`, `usually`, + `typically`, `often`, `sometimes`, `rarely`, `never`. These + describe *how often* a property holds, not *how many* answers + to give. Map to SINGULAR (one descriptive claim). +- **Modality**: `must be`, `can be`, `might be`, `could be`, + `should be`. Map to SINGULAR — the modal flavor doesn't change + count. +- **Polarity**: positive vs negative wh-questions. ABSENT covers + pure negation; mixed polarity (`who didn't sign the X`) inherits + from the wh-shape. The negation-attention failure mode lives in + `docs/bench-maxing.md`, not here. +- **Distributive vs collective**: `each` (distributive — applies to + individuals one-by-one) vs `all` (collective — applies to the + set as a whole). Operationally similar for QA; both bucket into + rung 8. +- **Generic / kind-level reference**: bare plural with no + quantifier (`cats are mammals`). Maps to SINGULAR by default; the + question is about the kind, not enumeration. +- **List-shape verbs**: `list`, `name`, `enumerate`, `identify`, + `cite`. These imply enumeration but the count is bounded by the + noun phrase that follows (`list the planets` → bounded by 8; + `list all primes` → ALL rung). Treat as a *trigger* for ALL/MANY + classification rather than its own rung. +- **Hedging**: `roughly`, `approximately`, `about`, `around`. These + modify a numeric bound (`about ten`) — fold into the + SMALL_NUM_EXPLICIT or COMPARATIVE_BOUND rung with the bound. + +### 2.2 Why these specific categories matter for aborist + +Each new rung names a *distinct expected-answer-length distribution*: + +- **PROPORTIONAL** — answer is one descriptive claim (`most cats are + X`), not a list. Without this rung it lands in MANY and gets a + cap that's too generous. +- **COMPARATIVE_BOUND** — explicit numeric ceiling/floor in the + question. Bounding the cap to the explicit number prevents the + model from over-enumerating *or* under-enumerating. +- **COMPREHENSIVE** — strictly stronger than ALL. The runaway + pressure on `tell me everything you know about X` is empirically + worse than `tell me about all X` (york-england, ticket #000006). + Worth its own rung with an aggressive small-model cap. +- **OPEN_REQUEST** — operationally produces enumeration without + a quantifier word. The classifier needs to detect verb-driven + shapes separately from word-driven shapes. + +The ABSENT, SMALL_NUM_EXPLICIT, and SMALL_NUM rungs from the v1 +table split: ABSENT remains as #1, SMALL_NUM_EXPLICIT promotes to +#4, and SMALL_NUM (the implicit small-count case like `a couple`) +folds into FEW (#6). ## 3. Design options @@ -211,6 +266,151 @@ Signal floor: 5pp per `docs/bench-maxing.md`. Decisions need at least one of: STRICT-rate change, FORMAT_COLLAPSED rate change, claim-count distribution shift. +### 5.1 First baseline (2026-05-02T20-45-11Z, pre-extension) + +Bench file `bench/qa_questions_quantifier_baseline.txt`. Result: +`bench/qa_results/2026-05-02T20-45-11Z.{jsonl,md}`. + +This baseline ran *before* the harness extension (§5.2), so the +JSONL only carries summary numbers — no `format_collapsed` field, +no `violation_kinds` array. Findings here are limited to verdict +counts and the n_quotes range; richer diagnostics come from the +second baseline (§5.1.1). + +| mode | verdicts (S/H/U) | n_quotes range | median ratio | median latency | +|-------------------------|------------------|----------------|--------------|----------------| +| `quote` | 0 / 3 / 0 | 22, 22, 22 | 0.455 | 12.9s | +| `claim_lattice_pointer` | 0 / 1 / 2 | 14, 21, 51 | 0.078 | 15.0s | +| `claim_lattice` (JSON) | 0 / 2 / 1 | 1, 16, 16 | 0.062 | 17.2s | + +Observations from this run: + +1. **Pointer-mode runaway confirmed quantitatively.** One sample + emitted **51 claims** — over 4× the current cap of 12. +2. **JSON-mode self-limits via schema.** Claim counts: 1, 16, 16. + Structured shape forces brevity but doesn't make claims stick. +3. **Quote mode has near-zero variance.** All three samples + produced exactly 22 quotes with 10 verified. +4. **No STRICT in any mode across 9 samples.** The under-specified + `all` question is too broad for STRICT under current rules. +5. **FORMAT_COLLAPSED detection blind** — JSONL didn't carry the + field. Cache-side inspection of surviving rows showed 1 bracket + on the pointer survivor, just outside the FC trigger. Initial + conclusion ("FC is rare for this shape") was *wrong* — see + §5.1.1, where FC actually fires 1/3 once we capture the field + directly. + +### 5.1.1 Second baseline (2026-05-02T20-58-57Z, post-extension) + +Same bench file, re-run after the harness extension landed. +Result: `bench/qa_results/2026-05-02T20-58-57Z.{jsonl,md}`. + +| mode | verdicts (S/H/U) | format-collapse | mean brackets (raw) | median ratio | median latency | +|-------------------------|------------------|-----------------|---------------------|--------------|----------------| +| `quote` | 0 / 3 / 0 | 0 / 3 | 0.0 | 0.435 | 12.6s | +| `claim_lattice_pointer` | 0 / 0 / 3 | **1 / 3** | 3.7 | 0.000 | 8.1s | +| `claim_lattice` (JSON) | 0 / 1 / 2 | 0 / 3 | 0.0 | 0.000 | 14.7s | + +Per-mode violation-kind tallies (counts = rows in which the kind +fired at least once): + +| mode | NO_EVIDENCE_POINTER | TOO_MANY_CLAIMS | TITLE_MISMATCH | FORMAT_COLLAPSED | SCHEMA_INVALID | CITATION_MISMATCH | TOO_MANY_EVIDENCE_IDS | WARRANT_MISSING | +|-------------------------|---------------------|-----------------|----------------|------------------|----------------|-------------------|------------------------|-----------------| +| `quote` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| `claim_lattice_pointer` | **3** | 2 | 1 | 1 | 1 | 0 | 0 | 0 | +| `claim_lattice` (JSON) | 0 | 0 | 0 | 0 | 0 | 2 | 1 | 1 | + +Findings that *change* the §5.1 analysis: + +1. **NO_EVIDENCE_POINTER is the dominant pointer-mode gate (3/3), + not TITLE_MISMATCH (1/3).** Every pointer-mode run had at least + one prose-line without an `[E\d+]` tag. The §5.1 inference that + "all-claims-TITLE_MISMATCH" was the gate was wrong. +2. **FORMAT_COLLAPSED actually fires 1/3** on this question — not + the rare corner I called it in §5.1. +3. **TOO_MANY_CLAIMS fires 2/3** in pointer mode — cap-12 demote is + active more often than the §5.1 n_quotes column suggested. +4. **JSON-mode failure shape is different.** CITATION_MISMATCH 2/3 + (claim text didn't textually overlap cited evidence) + + TOO_MANY_EVIDENCE_IDS 1/3 + WARRANT_MISSING 1/3. None of these + appear in pointer mode at this n. Suggests the per-mode policy + knobs that matter are different per mode. +5. **Latency variance dominates n=3.** Pointer-mode median dropped + 15.0s → 8.1s between baselines. Cache warmup + concurrency + nondeterminism, not a real signal change. + +What this implies for Section 3's options: + +- **Option A (cap reduction)** would lower TOO_MANY_CLAIMS rate but + probably *not* move the verdict — NO_EVIDENCE_POINTER fires + upstream of the cap and is the dominant gate. Cap-only is + unlikely to clear ≥5pp. +- **Option B (prompt reminder)** targets the actual gate + (NO_EVIDENCE_POINTER = "model emits prose without tags"). If + reminder injection improves bracket discipline, it should move + this rate. Worth measuring. +- **Option D (A + B)** still recommended but motivation is now B + carrying the verdict, A carrying the secondary cleanup. + +### 5.2 Bench harness extension (landed 2026-05-02b) + +Originally tracked here as a gap — closed in this same session. + +Landed: + +- `aborist/qa/query.py:2580-2592` — `format_collapsed` and + `raw_answer` surfaced on the `query()` result dict. +- `bench/qa_sweep.py:_run_one` — three new fields per row: + `format_collapsed`, `violation_kinds` (sorted unique kind + strings), `answer_brackets` (count of `[E\d+]` in raw_answer for + lattice modes; 0 elsewhere). +- `bench/qa_sweep.py:_summarize` — per-mode FC count, + `violation_kind_counts: defaultdict(int)`, lattice-only + `answer_brackets_sum`/`answer_brackets_n`. +- `bench/qa_sweep.py:_render_markdown` — new + `## format-collapse + violation kinds` section with per-mode + FC rate, mean raw brackets, and one column per observed + violation kind (union across the sweep). +- `tests/test_bench_qa_sweep.py` — 5 new tests pinning explicit- + True-only FC counting, per-mode kind tallies, lattice-only + bracket aggregation, FC-section rendering, and graceful + no-violations degradation. + +### 5.3 Sub-investigation: SCHEMA_INVALID in pointer mode (resolved) + +The §5.1.1 table shows `SCHEMA_INVALID: 1` under +`claim_lattice_pointer`. Initial concern was that this kind +originated only in the JSON-mode verifier and was somehow leaking +into the pointer path. + +**Resolution (2026-05-02b):** false alarm. SCHEMA_INVALID is also +a legitimate pointer-mode kind, emitted by `verify_claim_lattice` +in two well-defined cases: + +- **Tag with no claim text** (`verify.py:1242`) — model emitted + `[E5]` on a line with no actual claim text before the bracket. +- **Bare-name claim** (`verify.py:1270`) — claim has fewer than + `min_claim_content_tokens` (default 3) content tokens, e.g. + `"T-rex. [E5]"`. Forces a sentence-shape claim with a predicate. + +Both failures are structurally schema-invalid at the per-claim +level — the surface form doesn't yield a meaningful claim/pointer +pair — so the kind name is consistent. The `claim_lattice` JSON +verifier reuses the same kind name for analogous failures +(`verify.py:1801,1806,1813,1838,1851,1877,1905`), which is why +the bench-side union table groups them under one column. + +No action needed. Bench-side rendering is correct; the +SCHEMA_INVALID:1 cell on the pointer-mode row is signal, not +noise — one of the three pointer-mode runs emitted at least one +bare-name or empty-text claim. + +A useful refinement (out of scope for ticket #000008): split +SCHEMA_INVALID by sub-reason at bench-aggregate scale so we can +distinguish "model emitted bare-name claims" from "model emitted +unparseable JSON envelope." Tracked as a future bench-renderer +enhancement, not blocking. + ## 6. Open questions - Does the classifier need to handle multi-quantifier questions diff --git a/tests/test_bench_qa_sweep.py b/tests/test_bench_qa_sweep.py index c87374e..35800d0 100644 --- a/tests/test_bench_qa_sweep.py +++ b/tests/test_bench_qa_sweep.py @@ -55,6 +55,14 @@ def _row(**overrides) -> dict: "prompt_chars_system": 800, "prompt_chars_question": 30, "answer_chars": 100, + # Ticket #000008 — bench harness extension for FORMAT_COLLAPSED + # rate, violation-kind tallies, and bracket-count diagnostics. + # None on `format_collapsed` means "check didn't apply" (non- + # lattice modes), boolean otherwise. `violation_kinds` is a + # sorted list of unique kind strings observed for the row. + "format_collapsed": None, + "violation_kinds": [], + "answer_brackets": 0, "directive_compliance": { "D2_pointer_clauses": True, "D3_cti_substrate_ready": True, @@ -367,3 +375,100 @@ def test_directive_compliance_returns_empty_on_error_row(qa_sweep): err="some error", ) assert dc == {} + + +# Ticket #000008 — bench harness extension. The summarizer must: +# - count FORMAT_COLLAPSED firings per mode (only explicit True; +# None means the check didn't apply, not a counter increment) +# - tally per-violation-kind counts so we can see which kinds +# dominate which mode +# - aggregate raw-output bracket counts on lattice rows so we +# can chart "model is following the pointer protocol" vs +# FORMAT_COLLAPSED at aggregate scale +# The renderer must surface those tallies in a dedicated section. + + +def test_summarize_counts_format_collapsed_only_on_explicit_true(qa_sweep): + """format_collapsed=None means the check didn't run (non-lattice + rows). Only explicit True counts. False counts as not-collapsed.""" + rows = [ + _row(answer_mode="claim_lattice_pointer", format_collapsed=True), + _row(answer_mode="claim_lattice_pointer", format_collapsed=False), + _row(answer_mode="claim_lattice_pointer", format_collapsed=None), + _row(answer_mode="quote", format_collapsed=None), + ] + summary = qa_sweep._summarize(rows) + assert summary["claim_lattice_pointer"]["format_collapses"] == 1 + assert summary["quote"]["format_collapses"] == 0 + + +def test_summarize_tallies_violation_kinds_per_mode(qa_sweep): + """Each kind counts once per row even if the same kind fires on + multiple claims. Different rows in the same mode accumulate.""" + rows = [ + _row(answer_mode="claim_lattice_pointer", + violation_kinds=["FORMAT_COLLAPSED", "TITLE_MISMATCH"]), + _row(answer_mode="claim_lattice_pointer", + violation_kinds=["TITLE_MISMATCH"]), + _row(answer_mode="claim_lattice", + violation_kinds=["WARRANT_MISSING"]), + ] + summary = qa_sweep._summarize(rows) + pointer_counts = summary["claim_lattice_pointer"]["violation_kind_counts"] + assert pointer_counts["FORMAT_COLLAPSED"] == 1 + assert pointer_counts["TITLE_MISMATCH"] == 2 + json_counts = summary["claim_lattice"]["violation_kind_counts"] + assert json_counts["WARRANT_MISSING"] == 1 + + +def test_summarize_aggregates_brackets_only_on_lattice_modes(qa_sweep): + """Quote/span/entity/paraphrase rows always record 0 brackets; + averaging them in would skew the lattice-mode signal. Only + claim_lattice* modes contribute to the bracket-count aggregate.""" + rows = [ + _row(answer_mode="claim_lattice_pointer", answer_brackets=10), + _row(answer_mode="claim_lattice_pointer", answer_brackets=20), + _row(answer_mode="quote", answer_brackets=0), + ] + summary = qa_sweep._summarize(rows) + pointer = summary["claim_lattice_pointer"] + assert pointer["answer_brackets_n"] == 2 + assert pointer["answer_brackets_sum"] == 30 + quote = summary["quote"] + assert quote["answer_brackets_n"] == 0 + assert quote["answer_brackets_sum"] == 0 + + +def test_render_markdown_includes_format_collapse_section(qa_sweep): + """A bench summary with at least one FORMAT_COLLAPSED row should + surface a 'format-collapse' section with per-mode tallies.""" + rows = [ + _row(answer_mode="claim_lattice_pointer", format_collapsed=True, + violation_kinds=["FORMAT_COLLAPSED"], answer_brackets=0), + _row(answer_mode="claim_lattice_pointer", format_collapsed=False, + violation_kinds=[], answer_brackets=14), + ] + summary = qa_sweep._summarize(rows) + md = qa_sweep._render_markdown( + rows, summary, "2026-05-02T00-00-00Z", + ["claim_lattice_pointer"], ["q"], n_samples=2, + ) + assert "format-collapse" in md.lower() + # FORMAT_COLLAPSED kind appears as a column header when present. + assert "FORMAT_COLLAPSED" in md + # Per-mode rate cell shows 1/2. + assert "1/2" in md + + +def test_render_markdown_handles_no_violations(qa_sweep): + """Renderer must not crash when no violations fired across the + sweep — the violation-kind union is empty, so the table degrades + to mode + format-collapse + mean-brackets columns only.""" + rows = [_row(answer_mode="quote", violation_kinds=[])] + summary = qa_sweep._summarize(rows) + md = qa_sweep._render_markdown( + rows, summary, "2026-05-02T00-00-00Z", + ["quote"], ["q"], n_samples=1, + ) + # No exception, section header still present, no kind columns. + assert "format-collapse" in md.lower()