diff --git a/bench/qa_sweep.py b/bench/qa_sweep.py index 3de0992..55f7796 100644 --- a/bench/qa_sweep.py +++ b/bench/qa_sweep.py @@ -46,6 +46,14 @@ ANSWER_MODES = ("quote", "claim_lattice_pointer", "claim_lattice") # (verify.py:1650). Module-level so the sweep loop doesn't recompile # it per row. _BRACKET_RE = re.compile(r"\[E\d+") +# Whole bracket-tag region (one or more `[E\d+]`-shaped tokens +# possibly separated by `,`/whitespace, optionally followed by `]`). +# Used to estimate `answer_chars_with_brackets` — the share of the +# raw answer that is bracket-tagged citation rather than prose. +_BRACKET_REGION_RE = re.compile(r"\[E\d+(?:\s*,\s*E\d+)*\s*\]") +# Distinct pointer-id extractor — counts unique `[E\d+]` ids in raw +# output regardless of whether they appear bare or in a comma list. +_POINTER_ID_RE = re.compile(r"\bE\d+\b") def _read_questions(path: Path) -> list[str]: @@ -69,6 +77,43 @@ def _ratio(verdict: dict) -> float: return (nv / nq) if nq else 0.0 +def _bracket_diagnostics(raw_answer: str) -> tuple[int, int, int, int]: + """Compute bracket-tag diagnostics from a model's raw output. + + Returns a 4-tuple: + ``(bracket_count, distinct_pointer_count, chars_with_brackets, + raw_meaningful_line_count)``. + + - ``bracket_count`` count of `[E\\d+` occurrences (mirrors + the verifier's FORMAT_COLLAPSED gate). + - ``distinct_pointer`` unique `E\\d+` ids — answers with + `[E1, E2]` show 2 ids vs 2 brackets, + same denominator either way. + - ``chars_with_brackets`` total characters inside `[E\\d+,...]` + regions; pairs with answer_chars to + compute "% of answer that is citation" + at aggregate scale. + - ``raw_meaningful_line_count`` + lines >20 chars in the raw output; the + FORMAT_COLLAPSED denominator. Mirrors + the verifier's threshold so bench-side + and verifier-side numbers line up. + + All four return 0 on empty input. Module-level regexes do the + heavy lifting; this function exists to bundle the calls in one + place for the bench row builder. + """ + if not raw_answer: + return 0, 0, 0, 0 + bracket_count = len(_BRACKET_RE.findall(raw_answer)) + distinct = len(set(_POINTER_ID_RE.findall(raw_answer))) + chars = sum(len(m) for m in _BRACKET_REGION_RE.findall(raw_answer)) + n_meaningful = sum( + 1 for line in raw_answer.splitlines() if len(line.strip()) > 20 + ) + return bracket_count, distinct, chars, n_meaningful + + def _run_one( *, question: str, @@ -123,15 +168,19 @@ def _run_one( ) audit_mode = result.get("audit_mode") - # Bracket count from the model's RAW output (before the renderer - # interpolates literal spans). Lattice modes only; non-lattice - # rows record 0. Pairs with format_collapsed for ticket #000008 - # bench-side diagnostics: a low ratio + zero brackets + many - # lines tells us format collapse, not graceful per-claim refusal. + # Bracket diagnostics from the model's RAW output (before the + # renderer interpolates literal spans). Lattice modes only; + # non-lattice rows record zeros. Pairs with format_collapsed for + # ticket #000008 bench-side diagnostics: a low ratio + zero + # brackets + many lines tells us format collapse, not graceful + # per-claim refusal. raw_answer = result.get("raw_answer") or "" - answer_brackets = ( - len(_BRACKET_RE.findall(raw_answer)) if raw_answer else 0 - ) + ( + answer_brackets, + answer_pointer_count, + answer_chars_with_brackets, + raw_meaningful_line_count, + ) = _bracket_diagnostics(raw_answer) # Violation kind summary — full violation dicts are kept on the # result for the renderer but bench rows only need the kinds for # aggregate counting. Keeps row size bounded. @@ -139,6 +188,15 @@ def _run_one( v.get("kind") for v in (result.get("violations") or []) if v.get("kind") }) + # Quantifier preflight (Phase 0.x slot — populated by the Phase 1 + # classifier once it lands; keep the keys present so the bench + # JSONL schema is stable across the rollout). model_profile_id + # captures the configured model id verbatim — Phase 2 cap lookup + # uses model_profile_hash derived from this string. + quantifier_intensity = result.get("quantifier_intensity") + quantifier_matched_token = result.get("quantifier_matched_token") + scope_bound_hint = result.get("scope_bound_hint") + claim_cap_applied = result.get("claim_cap_applied") return { "question": question, "answer_mode": answer_mode, @@ -162,6 +220,38 @@ def _run_one( # Bracket count in the model's raw output. Lattice-mode-only # diagnostic; quote/span/entity/paraphrase rows always 0. "answer_brackets": answer_brackets, + # Distinct pointer ids in raw output (`[E1, E2]` → 2 ids). + # Different from answer_brackets when the model bundles + # multiple ids inside one bracket region. + "answer_pointer_count": answer_pointer_count, + # Chars inside `[E\d+,...]` regions — divides into answer_chars + # to give "fraction of answer that is bracket-tag, not prose" + # at aggregate scale. + "answer_chars_with_brackets": answer_chars_with_brackets, + # Meaningful raw-output line count (>20 chars). The + # FORMAT_COLLAPSED denominator; surfaces how dense the raw + # output was so we can chart claim-shape vs prose-shape over + # bench cycles. + "raw_meaningful_line_count": raw_meaningful_line_count, + # Ticket #000008 quantifier preflight — Phase 0.x slot. + # Populated by the Phase 1 classifier; remain None until + # Phase 1 wires it. Keeping the keys present here makes the + # JSONL schema stable across the rollout so post-Phase-1 + # bench markdown can be re-rendered against pre-Phase-1 + # rows without column-misalignment. + "quantifier_intensity": quantifier_intensity, + "quantifier_matched_token": quantifier_matched_token, + "scope_bound_hint": scope_bound_hint, + # Cap that was actually applied in this run. Phase 2 wires + # this; until then it's whatever the policy default is + # (typically 12 — the claim_lattice_max_claims_per_answer + # default). None means "not applicable" (non-lattice modes). + "claim_cap_applied": claim_cap_applied, + # Configured model id; serves as the model_profile_id key + # for Phase 2 per-model cap lookup. Stored verbatim so a + # bench archive remains interpretable when model_profiles.py + # changes. + "model_profile_id": model, "cache_key": (result.get("cache_key") or "")[:12], "n_sources": len(result.get("sources") or []), "elapsed_s": elapsed_s, diff --git a/tests/test_bench_qa_sweep.py b/tests/test_bench_qa_sweep.py index 35800d0..44519a7 100644 --- a/tests/test_bench_qa_sweep.py +++ b/tests/test_bench_qa_sweep.py @@ -63,6 +63,16 @@ def _row(**overrides) -> dict: "format_collapsed": None, "violation_kinds": [], "answer_brackets": 0, + # Phase 0.x bench fields (commit lands ahead of Phase 1 + # classifier; keys stay present from Phase 0.x onwards). + "answer_pointer_count": 0, + "answer_chars_with_brackets": 0, + "raw_meaningful_line_count": 0, + "quantifier_intensity": None, + "quantifier_matched_token": None, + "scope_bound_hint": None, + "claim_cap_applied": None, + "model_profile_id": "test-model", "directive_compliance": { "D2_pointer_clauses": True, "D3_cti_substrate_ready": True, @@ -472,3 +482,64 @@ def test_render_markdown_handles_no_violations(qa_sweep): ) # No exception, section header still present, no kind columns. assert "format-collapse" in md.lower() + + +# Ticket #000008 Phase 0.x — bracket diagnostics helpers. Mirror the +# verifier's FORMAT_COLLAPSED detector inputs so bench-side and +# verifier-side numbers agree. + + +def test_bracket_diagnostics_empty_returns_zeros(qa_sweep): + assert qa_sweep._bracket_diagnostics("") == (0, 0, 0, 0) + assert qa_sweep._bracket_diagnostics(None) == (0, 0, 0, 0) + + +def test_bracket_diagnostics_single_pointer(qa_sweep): + raw = "Tyrannosaurus rex appears in the climactic scene. [E1]\n" + bracket_count, distinct, chars, lines = qa_sweep._bracket_diagnostics(raw) + assert bracket_count == 1 + assert distinct == 1 + # `[E1]` = 4 characters + assert chars == 4 + assert lines == 1 + + +def test_bracket_diagnostics_multi_pointer_in_one_bracket(qa_sweep): + """`[E1, E2]` is one bracket region but two distinct pointer ids. + `answer_brackets` counts `[E\\d+` openings (1); `answer_pointer + _count` counts unique ids (2).""" + raw = "Claim text. [E1, E2]\n" + bracket_count, distinct, chars, _ = qa_sweep._bracket_diagnostics(raw) + assert bracket_count == 1 + assert distinct == 2 + # `[E1, E2]` = 8 characters + assert chars == 8 + + +def test_bracket_diagnostics_multiple_pointers_separate_brackets(qa_sweep): + # Each line must be >20 chars to count as meaningful (matches the + # verifier FORMAT_COLLAPSED threshold). + raw = ( + "Tyrannosaurus rex appears in the first scene. [E1]\n" + "Velociraptors stalk the workers in the kitchen. [E2]\n" + "The T-rex breaks out of the paddock dramatically. [E1]\n" + ) + bracket_count, distinct, _, lines = qa_sweep._bracket_diagnostics(raw) + assert bracket_count == 3 # three `[E\d+` openings + assert distinct == 2 # only E1 + E2 unique + assert lines == 3 + + +def test_bracket_diagnostics_format_collapsed_shape(qa_sweep): + """The 2026-05-02 winners-of-all-major-sports case: 5+ meaningful + prose lines with zero brackets. Confirms the bench reports the + same denominator the verifier's FORMAT_COLLAPSED gate uses.""" + raw = "\n".join( + f"The 19{i} FINA Men's Water Polo World Cup was won by Hungary." + for i in range(70, 80) + ) + bracket_count, distinct, chars, lines = qa_sweep._bracket_diagnostics(raw) + assert bracket_count == 0 + assert distinct == 0 + assert chars == 0 + assert lines >= 5