arborist/bench/qa_sweep.py

"""QA-quality benchmark sweep.

Runs a fixed question set through ``arborist.qa.query.query`` under each
answer mode and tabulates STRICT / HYBRID / UNGROUNDED counts, the
``n_verified / n_quotes`` ratio, latency, and lazy-anchor signals.

Why this exists: G0 / claim-lattice-pointer mode landed in 2337b77, and
four hardenings stacked on top (smell sidecar, positive-form prompts,
source-role boost, title-purity rerank, CITATION_MISMATCH check). Every
new heuristic flies blind until we can show "this commit moved STRICT
from N to M". This is that scaffolding.

Outputs:

    bench/qa_results/<utc-iso>.jsonl   one row per (mode, question)
    bench/qa_results/<utc-iso>.md      markdown summary table

The JSONL is the durable artifact. The markdown is the thing fox reads.

Wire via ``make bench-qa``. Honors ``BURN=1`` to force fresh inference
under the current ``governance_policy_hash`` — useful after a prompt
change that doesn't bump any version field.
"""
from __future__ import annotations

import argparse
import datetime as _dt
import json
import os
import random
import re
import sys
import time
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from threading import Lock

# Defer arborist imports until argparse runs so `--help` works without
# the package installed.

ANSWER_MODES = ("quote", "claim_lattice_pointer", "claim_lattice")

# Pointer-tag regex for bench-side bracket counting on raw model
# output. Mirrors the verifier's FORMAT_COLLAPSED detector
# (verify.py:1650). Module-level so the sweep loop doesn't recompile
# it per row.
_BRACKET_RE = re.compile(r"\[E\d+")
# Whole bracket-tag region (one or more `[E\d+]`-shaped tokens
# possibly separated by `,`/whitespace, optionally followed by `]`).
# Used to estimate `answer_chars_with_brackets` — the share of the
# raw answer that is bracket-tagged citation rather than prose.
_BRACKET_REGION_RE = re.compile(r"\[E\d+(?:\s*,\s*E\d+)*\s*\]")
# Distinct pointer-id extractor — counts unique `[E\d+]` ids in raw
# output regardless of whether they appear bare or in a comma list.
_POINTER_ID_RE = re.compile(r"\bE\d+\b")


def _read_questions(path: Path) -> list[str]:
    out: list[str] = []
    for raw in path.read_text(encoding="utf-8").splitlines():
        line = raw.strip()
        if not line or line.startswith("#"):
            continue
        out.append(line)
    return out


def _utc_iso_compact() -> str:
    # YYYY-MM-DDTHH-MM-SSZ — colon-free so it's filesystem-safe.
    return _dt.datetime.now(tz=_dt.timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ")


def _ratio(verdict: dict) -> float:
    nq = verdict.get("n_quotes") or 0
    nv = verdict.get("n_verified") or 0
    return (nv / nq) if nq else 0.0


def _bracket_diagnostics(raw_answer: str) -> tuple[int, int, int, int]:
    """Compute bracket-tag diagnostics from a model's raw output.

    Returns a 4-tuple:
      ``(bracket_count, distinct_pointer_count, chars_with_brackets,
      raw_meaningful_line_count)``.

    - ``bracket_count``       count of `[E\\d+` occurrences (mirrors
                              the verifier's FORMAT_COLLAPSED gate).
    - ``distinct_pointer``    unique `E\\d+` ids — answers with
                              `[E1, E2]` show 2 ids vs 2 brackets,
                              same denominator either way.
    - ``chars_with_brackets`` total characters inside `[E\\d+,...]`
                              regions; pairs with answer_chars to
                              compute "% of answer that is citation"
                              at aggregate scale.
    - ``raw_meaningful_line_count``
                              lines >20 chars in the raw output; the
                              FORMAT_COLLAPSED denominator. Mirrors
                              the verifier's threshold so bench-side
                              and verifier-side numbers line up.

    All four return 0 on empty input. Module-level regexes do the
    heavy lifting; this function exists to bundle the calls in one
    place for the bench row builder.
    """
    if not raw_answer:
        return 0, 0, 0, 0
    bracket_count = len(_BRACKET_RE.findall(raw_answer))
    distinct = len(set(_POINTER_ID_RE.findall(raw_answer)))
    chars = sum(len(m) for m in _BRACKET_REGION_RE.findall(raw_answer))
    n_meaningful = sum(
        1 for line in raw_answer.splitlines() if len(line.strip()) > 20
    )
    return bracket_count, distinct, chars, n_meaningful


def _run_one(
    *,
    question: str,
    answer_mode: str,
    shards_dir: Path | None,
    qa_db: Path | None,
    top_k: int,
    burn: bool,
    endpoint: str,
    model: str,
    policy_overrides: dict | None = None,
) -> dict:
    """Run one (question, mode) — returns a flat record for the JSONL.

    ``policy_overrides`` is a dict of policy-field overrides applied
    after ``DEFAULT_QUERY_POLICY`` and ``answer_mode`` so that
    ``bench/qa_sweep.py --policy quantifier_reminder_enabled=true``
    can flip individual fields for an A/B cycle without touching
    the policy defaults.
    """
    from arborist.qa.client import OpenAICompatibleClient
    from arborist.qa.query import DEFAULT_QUERY_POLICY, query

    policy = dict(DEFAULT_QUERY_POLICY)
    policy["answer_mode"] = answer_mode
    if policy_overrides:
        policy.update(policy_overrides)

    api_key = os.environ.get("ARBORIST_LLM_API_KEY")
    client = OpenAICompatibleClient(base_url=endpoint, api_key=api_key)

    if qa_db is None:
        qa_db = (shards_dir / "qa.db") if shards_dir else (Path.home() / ".arborist" / "qa.db")

    t0 = time.monotonic()
    err: str | None = None
    result: dict = {}
    try:
        result = query(
            question=question,
            qa_db=qa_db,
            chat_client=client,
            model_id=model,
            shards_dir=shards_dir,
            top_k=top_k,
            policy=policy,
            burn_existing=burn,
        )
    except Exception as e:  # noqa: BLE001 — bench captures all
        err = f"{type(e).__name__}: {e}"
    elapsed_s = round(time.monotonic() - t0, 2)

    # Deflection sidecar — sidecar-only signal, never feeds the
    # verifier. Surfaces the Mars-BDFL pattern (STRICT but answer
    # never mentions the question's subject) at bench-aggregate
    # scale so a creeping "model deflects rather than refuses"
    # regression is legible across runs.
    from arborist.qa.inspect import diagnose_deflection, diagnose_metaphor_deflection
    deflection = diagnose_deflection(question, result.get("answer_text") or "")
    metaphor = diagnose_metaphor_deflection(
        question, result.get("answer_text") or ""
    )

    audit_mode = result.get("audit_mode")
    # Bracket diagnostics from the model's RAW output (before the
    # renderer interpolates literal spans). Lattice modes only;
    # non-lattice rows record zeros. Pairs with format_collapsed for
    # ticket #000008 bench-side diagnostics: a low ratio + zero
    # brackets + many lines tells us format collapse, not graceful
    # per-claim refusal.
    raw_answer = result.get("raw_answer") or ""
    (
        answer_brackets,
        answer_pointer_count,
        answer_chars_with_brackets,
        raw_meaningful_line_count,
    ) = _bracket_diagnostics(raw_answer)
    # Violation kind summary — full violation dicts are kept on the
    # result for the renderer but bench rows only need the kinds for
    # aggregate counting. Keeps row size bounded.
    violation_kinds = sorted({
        v.get("kind") for v in (result.get("violations") or [])
        if v.get("kind")
    })
    # Quantifier preflight (Phase 0.x slot — populated by the Phase 1
    # classifier once it lands; keep the keys present so the bench
    # JSONL schema is stable across the rollout). model_profile_id
    # captures the configured model id verbatim — Phase 2 cap lookup
    # uses model_profile_hash derived from this string.
    quantifier_intensity = result.get("quantifier_intensity")
    quantifier_matched_token = result.get("quantifier_matched_token")
    scope_bound_hint = result.get("scope_bound_hint")
    claim_cap_applied = result.get("claim_cap_applied")
    # Ticket #000068 Phase 1 — missed-answer guard sidecar projection.
    # `result["answerability"]` is None when the guard didn't fire,
    # else a structured dict. Persist a small projection on the bench
    # row so the aggregator can count fires + confidence breakdown
    # without re-running the sidecar. Full dict (with candidate
    # spans + offsets) stays on the result for human inspection.
    answerability = result.get("answerability") or {}
    answerability_fired = bool(answerability.get("answerability_warning"))
    answerability_confidence = answerability.get("confidence_class")
    answerability_denial_pattern = answerability.get("denial_pattern_matched")
    answerability_answer_type = answerability.get("answer_type")
    answerability_candidate_count = answerability.get("candidate_count")
    # Ticket #000010 — meta-cognition QuestionState. Persist a
    # bounded-size projection: logical_statuses, question_shape,
    # preflight_result, temporal_sensitivity, and the kind of any
    # detector hits. Full QuestionState (including
    # contradiction_pairs / false_premise_hints) stays on the
    # result dict for CLI render but is NOT persisted in bench
    # rows to keep row size bounded across long sweeps.
    qs = result.get("question_state") or {}
    preflight_logical_statuses = list(qs.get("logical_statuses") or [])
    preflight_question_shape = qs.get("question_shape")
    preflight_result = qs.get("preflight_result")
    preflight_temporal_sensitivity = qs.get("temporal_sensitivity")
    preflight_has_false_premise = bool(qs.get("false_premise_hints"))
    preflight_has_contradiction = bool(qs.get("contradiction_pairs"))
    preflight_corpus_requirement = qs.get("corpus_requirement")
    # Ticket #000009 §7.2 — preflight stage hash 12-char prefix.
    # Same truncation as cache_key. Two cache rows that differ
    # only in their preflight policy state now have different
    # 12-char prefixes — operator can grep / SQL-filter the bench
    # JSONL for cross-row policy comparison.
    preflight_hash = (result.get("preflight_hash") or "")[:12]
    row = {
        "question": question,
        "answer_mode": answer_mode,
        "status": result.get("status") or ("error" if err else "missing"),
        "audit_mode": audit_mode,
        "n_quotes": result.get("n_quotes"),
        "n_verified": result.get("n_verified"),
        "ratio": round(_ratio(result), 3),
        "verifier_method": result.get("verifier_method"),
        "lookup_path": result.get("lookup_path"),
        "failure_stage": result.get("failure_stage"),
        "lazy_anchor_ratio": result.get("lazy_anchor_ratio"),
        "pointer_id_distribution": result.get("pointer_id_distribution"),
        # Format-collapse signal (pointer-mode only — None elsewhere).
        # See verify.py and ticket #000008.
        "format_collapsed": result.get("format_collapsed"),
        # Sorted unique kinds list for aggregate counting; full
        # violation payloads stay off the bench row to keep size
        # bounded (5+ violations × dict ~= bloat across 10K-row sweeps).
        "violation_kinds": violation_kinds,
        # Bracket count in the model's raw output. Lattice-mode-only
        # diagnostic; quote/span/entity/paraphrase rows always 0.
        "answer_brackets": answer_brackets,
        # Distinct pointer ids in raw output (`[E1, E2]` → 2 ids).
        # Different from answer_brackets when the model bundles
        # multiple ids inside one bracket region.
        "answer_pointer_count": answer_pointer_count,
        # Chars inside `[E\d+,...]` regions — divides into answer_chars
        # to give "fraction of answer that is bracket-tag, not prose"
        # at aggregate scale.
        "answer_chars_with_brackets": answer_chars_with_brackets,
        # Meaningful raw-output line count (>20 chars). The
        # FORMAT_COLLAPSED denominator; surfaces how dense the raw
        # output was so we can chart claim-shape vs prose-shape over
        # bench cycles.
        "raw_meaningful_line_count": raw_meaningful_line_count,
        # Ticket #000008 quantifier preflight — Phase 0.x slot.
        # Populated by the Phase 1 classifier; remain None until
        # Phase 1 wires it. Keeping the keys present here makes the
        # JSONL schema stable across the rollout so post-Phase-1
        # bench markdown can be re-rendered against pre-Phase-1
        # rows without column-misalignment.
        "quantifier_intensity": quantifier_intensity,
        "quantifier_matched_token": quantifier_matched_token,
        "scope_bound_hint": scope_bound_hint,
        # Cap that was actually applied in this run. Phase 2 wires
        # this; until then it's whatever the policy default is
        # (typically 12 — the claim_lattice_max_claims_per_answer
        # default). None means "not applicable" (non-lattice modes).
        "claim_cap_applied": claim_cap_applied,
        # Ticket #000068 Phase 1 — missed-answer guard sidecar bench
        # projection. Five small fields surface aggregate fire-rate +
        # confidence breakdown without bloating the row with candidate
        # spans / offsets (full dict on the result, recomputable from
        # question + answer + evidence + policy).
        "answerability_fired": answerability_fired,
        "answerability_confidence": answerability_confidence,
        "answerability_denial_pattern": answerability_denial_pattern,
        "answerability_answer_type": answerability_answer_type,
        "answerability_candidate_count": answerability_candidate_count,
        # Configured model id; serves as the model_profile_id key
        # for Phase 2 per-model cap lookup. Stored verbatim so a
        # bench archive remains interpretable when model_profiles.py
        # changes.
        "model_profile_id": model,
        # Ticket #000010 — meta-cognition preflight bench fields.
        # Bounded-size projection of QuestionState. Full state
        # (contradiction_pairs / false_premise_hints) lives on the
        # result dict for CLI render; bench rows keep just the
        # kinds + shape + result so 10K-row sweeps stay
        # human-greppable.
        "preflight_logical_statuses": preflight_logical_statuses,
        "preflight_question_shape": preflight_question_shape,
        "preflight_result": preflight_result,
        "preflight_temporal_sensitivity": preflight_temporal_sensitivity,
        "preflight_has_false_premise": preflight_has_false_premise,
        "preflight_has_contradiction": preflight_has_contradiction,
        "preflight_corpus_requirement": preflight_corpus_requirement,
        # 12-char preflight stage hash prefix (#000009 §7.2).
        # Matches cache_key truncation pattern.
        "preflight_hash": preflight_hash,
        "cache_key": (result.get("cache_key") or "")[:12],
        "n_sources": len(result.get("sources") or []),
        "elapsed_s": elapsed_s,
        "deflection_kind": deflection["kind"],
        "subject_anchor": deflection["subject_anchor"],
        "subject_in_answer": deflection["subject_in_answer"],
        "metaphor_deflection_kind": metaphor["kind"],
        "metaphor_cue_count": metaphor["cue_count"],
        "metaphor_overlap_count": metaphor["answer_overlap_count"],
        # Capacity metrics — char-level proxy for prompt-token budget.
        # Surfaces "did STRICT come from a tight 5KB prompt or a 50KB
        # context-stuffed one?" at aggregate scale. Lets the bench
        # bucket strict-rate by input-size band.
        "prompt_chars_total": (result.get("prompt_chars") or {}).get("messages_total", 0),
        "prompt_chars_evidence": (result.get("prompt_chars") or {}).get("evidence_or_context", 0),
        "prompt_chars_system": (result.get("prompt_chars") or {}).get("system_prompt", 0),
        "prompt_chars_question": (result.get("prompt_chars") or {}).get("user_question", 0),
        "answer_chars": result.get("answer_chars", 0),
        # Seven-point program directive compliance — see
        # docs/seven-point-program.md. Per-row booleans where the
        # directive is observable from the result; aggregate
        # coverage shows up in the markdown summary's
        # "directive coverage" section. Directives that are global
        # properties of the substrate (D1, D5, D8) don't appear
        # per-row.
        "directive_compliance": _directive_compliance(answer_mode, result, err),
        "error": err,
    }
    # #000049 Phase 2 — when ARBORIST_NLI_SHADOW is set, query() surfaces
    # the verifier-input text; carry it (+ the answer) into the row so a
    # downstream `nli_shadow_sweep.py --input <this.jsonl>` can measure
    # the would-demote rate on bench-qa traffic (§7 #12 gate item 4). Off
    # by default — these two fields can be large; never persisted
    # otherwise. Scrub lone surrogates first: real Wikipedia context
    # occasionally carries U+D800–U+DFFF code points (mangled source
    # encoding) that `json.dumps(..., ensure_ascii=False)` then refuses
    # to UTF-8-encode — a measurement field, U+FFFD is fine.
    vit = result.get("verifier_input_text")
    if vit is not None:
        row["answer_text"] = _scrub_surrogates(result.get("answer_text"))
        row["context"] = _scrub_surrogates(vit)
    return row


def _scrub_surrogates(s):
    if not isinstance(s, str):
        return s
    return s.encode("utf-8", "replace").decode("utf-8")


def _directive_compliance(
    answer_mode: str, result: dict, err: str | None
) -> dict:
    """Compute per-row pass/fail for the seven-point directives whose
    pinning is observable from a single bench row.

    Returns a dict mapping directive id -> bool (True = pass, False =
    fail/pending). Some directives are system-global (D1 verifier
    never calls LLM, D5 verifier_method enum, D8 discipline) and don't
    appear per-row — track those once in the summary header.
    """
    if err or result.get("status") == "error":
        return {}
    audit = result.get("audit_mode")
    method = result.get("verifier_method") or ""
    is_lattice = answer_mode in ("claim_lattice_pointer", "claim_lattice")
    return {
        # D2: lattice modes emit pointer clauses (claim_lattice_pointer
        # or claim_lattice JSON variant). Quote-mode rows count as
        # n/a for D2 — they predate the directive.
        "D2_pointer_clauses": is_lattice,
        # D3: build CTI internally — proxy is "phrase route had a
        # chance to fire" (lattice mode + run_dag_root populated).
        # Module L (the answer-side multi-frame compilation) lands
        # via ticket #000002; until then this is a structural
        # readiness check, not full coverage.
        "D3_cti_substrate_ready": (
            is_lattice and bool(result.get("run_dag_root"))
        ),
        # D4: evidence_map_root + run_dag_root present in the
        # 9-stage CTI run-DAG. Retrieval-plan-hash binding pending
        # via ticket #000001.
        "D4_evidence_map_bound": bool(result.get("run_dag_root")),
        # D6: warrant ran (relation/date anchor classes — today).
        # Generalization to entity-list / count / why-cause shapes
        # pending via ticket #000003. Mark True only when the
        # verifier verdict shows the warrant was checked (today,
        # any lattice-mode row gets the warrant pass; future
        # per-shape gating sharpens this).
        "D6_warrant_fired": is_lattice,
        # D7: schema audit_mode is in the canonical enum. Renderer
        # then maps lattice-mode STRICT/HYBRID → EVIDENCE-LINKED at
        # display time (pinned by tests/test_cli_render.py). Per-row
        # signal here just confirms the row's audit token is valid —
        # the renderer side is a deterministic transformation tested
        # separately, not something each bench row can re-verify.
        "D7_honest_label": audit in ("STRICT", "HYBRID", "UNGROUNDED"),
    }


def _summarize(rows: list[dict]) -> dict:
    """Group rows by mode → STRICT/HYBRID/UNGROUNDED totals + means."""
    by_mode: dict[str, dict] = {}
    for r in rows:
        m = r["answer_mode"]
        b = by_mode.setdefault(m, {
            "n": 0,
            "STRICT": 0,
            "HYBRID": 0,
            "UNGROUNDED": 0,
            "errors": 0,
            "ratio_sum": 0.0,
            "latency_sum": 0.0,
            "deflections": 0,
            # Format-collapse rate (pointer-mode only — None elsewhere
            # so non-lattice rows count as 0). Surfaced in markdown
            # alongside deflection rate as a per-mode collapse signal.
            "format_collapses": 0,
            # Ticket #000068 Phase 1 — missed-answer guard aggregate
            # counters. answerability_fires counts every row where the
            # sidecar fired; the per-confidence breakdown surfaces how
            # many of those were strong/medium/weak. Pairs with
            # violation_kind_counts and format_collapses as the third
            # sidecar-class signal at aggregate scale.
            "answerability_fires": 0,
            "answerability_strong": 0,
            "answerability_medium": 0,
            "answerability_weak": 0,
            # Per-violation-kind counts. Open-ended dict — fills as
            # kinds are encountered. Empty when no violations fire.
            "violation_kind_counts": defaultdict(int),
            # Bracket-count distribution for lattice rows. Surfaces
            # whether the model is following the pointer protocol at
            # all; FORMAT_COLLAPSED is the bracket=0 corner.
            "answer_brackets_sum": 0,
            "answer_brackets_n": 0,
            # Per-directive pass counts (seven-point program). Init
            # all known directive ids so absent rows report 0/N
            # rather than missing-key.
            "directive_pass": {
                "D2_pointer_clauses": 0,
                "D3_cti_substrate_ready": 0,
                "D4_evidence_map_bound": 0,
                "D6_warrant_fired": 0,
                "D7_honest_label": 0,
            },
        })
        b["n"] += 1
        if r["error"] or r["status"] == "error":
            b["errors"] += 1
        elif r["audit_mode"] in ("STRICT", "HYBRID", "UNGROUNDED"):
            b[r["audit_mode"]] += 1
        b["ratio_sum"] += r["ratio"] or 0.0
        b["latency_sum"] += r["elapsed_s"] or 0.0
        # Deflection rate: STRICT/HYBRID with subject-anchor missing
        # from answer. UNGROUNDED + deflection isn't interesting (no
        # answer to deflect with), and no_question_tokens is vacuous.
        if r.get("deflection_kind") == "deflection" and r["audit_mode"] in (
            "STRICT", "HYBRID"
        ):
            b["deflections"] += 1
        # Format-collapse: pointer-mode rows report bool, others
        # report None. Treat None as not-collapsed (the check didn't
        # apply); only count explicit True.
        if r.get("format_collapsed") is True:
            b["format_collapses"] += 1
        # Missed-answer guard fires + per-confidence breakdown.
        if r.get("answerability_fired"):
            b["answerability_fires"] += 1
            conf = r.get("answerability_confidence")
            if conf == "strong":
                b["answerability_strong"] += 1
            elif conf == "medium":
                b["answerability_medium"] += 1
            elif conf == "weak":
                b["answerability_weak"] += 1
        # Violation-kind tallies — each kind counts once per row even
        # if the same kind fires on multiple claims. The bench is
        # asking "did this kind fire on this run?", not "how many
        # times within the run".
        for kind in (r.get("violation_kinds") or []):
            b["violation_kind_counts"][kind] += 1
        # Bracket-count aggregates. Lattice rows only — quote and
        # other modes record 0 so they'd skew the mean if averaged
        # globally. Track per-mode sum + count; renderer can compute
        # mean only for lattice modes.
        ab = r.get("answer_brackets")
        if ab is not None and m in ("claim_lattice_pointer", "claim_lattice"):
            b["answer_brackets_sum"] += ab
            b["answer_brackets_n"] += 1
        # Directive compliance — sum the per-row booleans into
        # per-mode pass counts.
        for did, ok in (r.get("directive_compliance") or {}).items():
            if did in b["directive_pass"] and ok:
                b["directive_pass"][did] += 1
    return by_mode


def _per_question_summary(rows: list[dict]) -> dict:
    """Group rows by (question, mode) → per-cell vote counts + medians."""
    cells: dict[tuple[str, str], dict] = {}
    for r in rows:
        key = (r["question"], r["answer_mode"])
        c = cells.setdefault(key, {
            "STRICT": 0, "HYBRID": 0, "UNGROUNDED": 0, "errors": 0,
            "ratios": [], "latencies": [],
        })
        if r["error"] or r["status"] == "error":
            c["errors"] += 1
        elif r["audit_mode"] in ("STRICT", "HYBRID", "UNGROUNDED"):
            c[r["audit_mode"]] += 1
        c["ratios"].append(r["ratio"] or 0.0)
        c["latencies"].append(r["elapsed_s"] or 0.0)
    return cells


def _median(xs: list[float]) -> float:
    if not xs:
        return 0.0
    s = sorted(xs)
    mid = len(s) // 2
    return s[mid] if len(s) % 2 else (s[mid - 1] + s[mid]) / 2


def _render_markdown(
    rows: list[dict],
    summary: dict,
    started_utc: str,
    modes: list[str],
    questions: list[str],
    n_samples: int,
) -> str:
    lines: list[str] = []
    n_runs = len(rows)
    lines.append(f"# arborist QA-quality benchmark — {started_utc}")
    lines.append("")
    lines.append(
        f"questions: {len(questions)} · modes: {len(modes)} · samples per cell: "
        f"{n_samples} · runs: {n_runs}"
    )
    lines.append("")
    lines.append("## summary")
    lines.append("")
    lines.append("| mode | runs | STRICT | HYBRID | UNGROUNDED | err | strict-rate | mean ratio | mean latency | deflections | #68 fires (S/M/W) |")
    lines.append("|------|------|--------|--------|------------|-----|-------------|-----------|--------------|-------------|-------------------|")
    for mode in modes:
        b = summary.get(mode)
        if not b:
            continue
        n = b["n"] or 1
        mean_ratio = b["ratio_sum"] / n
        mean_lat = b["latency_sum"] / n
        strict_rate = b["STRICT"] / n
        deflections = b.get("deflections", 0)
        ans_fires = b.get("answerability_fires", 0)
        ans_s = b.get("answerability_strong", 0)
        ans_m = b.get("answerability_medium", 0)
        ans_w = b.get("answerability_weak", 0)
        lines.append(
            f"| {mode} | {b['n']} | {b['STRICT']} | {b['HYBRID']} | "
            f"{b['UNGROUNDED']} | {b['errors']} | "
            f"{strict_rate:.2f} | {mean_ratio:.3f} | {mean_lat:.1f}s | "
            f"{deflections}/{b['n']} | "
            f"{ans_fires}/{b['n']} ({ans_s}/{ans_m}/{ans_w}) |"
        )
    lines.append("")
    lines.append("## format-collapse + violation kinds")
    lines.append("")
    lines.append(
        "Per-mode count of FORMAT_COLLAPSED firings (pointer-mode signal — "
        "model emitted ≥5 meaningful prose lines with zero `[E\\d+]` tags) "
        "and per-violation-kind tallies. Each kind counts once per row "
        "even if it fires on multiple claims within the run. See "
        "`docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md` "
        "for why this matters: format collapse separates 'tried to ground "
        "& failed' from 'abandoned the protocol entirely' at aggregate "
        "scale."
    )
    lines.append("")
    # Build the union of violation kinds observed across all modes for
    # the table header — keeps the column set bench-wide rather than
    # per-mode (so quote-mode rows show 0/N for kinds that only fire
    # in lattice modes, instead of the column being absent).
    all_kinds = sorted({
        k for b in summary.values()
        for k in (b.get("violation_kind_counts") or {}).keys()
    })
    if all_kinds:
        header = (
            "| mode | format-collapse | mean brackets (raw) | "
            + " | ".join(all_kinds)
            + " |"
        )
        sep = (
            "|------|-----------------|---------------------|"
            + "|".join("-" * (len(k) + 2) for k in all_kinds)
            + "|"
        )
    else:
        header = "| mode | format-collapse | mean brackets (raw) |"
        sep = "|------|-----------------|---------------------|"
    lines.append(header)
    lines.append(sep)
    for mode in modes:
        b = summary.get(mode)
        if not b:
            continue
        n = b["n"] or 1
        fc = b.get("format_collapses", 0)
        mean_brackets = (
            b["answer_brackets_sum"] / b["answer_brackets_n"]
            if b.get("answer_brackets_n")
            else 0.0
        )
        kind_cells = [
            str(b["violation_kind_counts"].get(k, 0))
            for k in all_kinds
        ]
        cells_str = " | ".join(kind_cells)
        prefix = f"| {mode} | {fc}/{b['n']} | {mean_brackets:.1f} |"
        if all_kinds:
            lines.append(f"{prefix} {cells_str} |")
        else:
            lines.append(prefix)
    lines.append("")
    lines.append("## directive coverage (seven-point program)")
    lines.append("")
    lines.append(
        "Per-mode pass-rate for the directives whose pinning is observable "
        "from a single bench row. See `docs/seven-point-program.md` for "
        "the full directive list. D1 (no LLM in verifier), D5 (verifier_method "
        "enum), and D8 (test-pinning discipline) are global properties of "
        "the substrate and don't appear per-row. ½ in the doc means some "
        "scope is implemented; bench numbers reflect the implemented portion."
    )
    lines.append("")
    lines.append("| mode | D2 pointer | D3 cti-ready | D4 ev-map bound | D6 warrant | D7 honest label |")
    lines.append("|------|-----------|--------------|-----------------|-----------|----------------|")
    for mode in modes:
        b = summary.get(mode)
        if not b:
            continue
        n = b["n"] or 1
        dp = b["directive_pass"]
        cells = []
        for did in (
            "D2_pointer_clauses",
            "D3_cti_substrate_ready",
            "D4_evidence_map_bound",
            "D6_warrant_fired",
            "D7_honest_label",
        ):
            count = dp.get(did, 0)
            cells.append(f"{count}/{n} ({count / n:.0%})")
        lines.append(f"| {mode} | " + " | ".join(cells) + " |")
    lines.append("")
    lines.append("## strict-rate by prompt size")
    lines.append("")
    lines.append(
        "Buckets capacity in `messages_total` chars (system + reminder + "
        "evidence/context + question). ~4 chars/token English prose; "
        "~2.5 chars/token JSON-evidence-heavy. Tells you whether the "
        "model strict-rate degrades with input size. Buckets log-scale "
        "from 8 KB to >= 1 MB so the same harness covers Hermes-3-8B "
        "(82K context) through 1M-context models without code change."
    )
    lines.append("")
    # Log-scale buckets — covers the entire model spectrum from
    # 8B-class models (Hermes 82K context, max useful prompt ~32-64KB)
    # through giant-context models (Gemini 1.5 / Claude 1M, useful
    # prompt potentially 100KB-800KB). Models simply don't populate
    # buckets beyond their context window; the substrate stays
    # universal. Adding new boundaries doesn't require a code change
    # for existing data — empty buckets get skipped at render.
    buckets = [
        ("<8KB", 0, 8000),
        ("8-16KB", 8000, 16000),
        ("16-32KB", 16000, 32000),
        ("32-64KB", 32000, 64000),
        ("64-128KB", 64000, 128000),
        ("128-256KB", 128000, 256000),
        ("256-512KB", 256000, 512000),
        ("512K-1M", 512000, 1048576),
        (">=1M", 1048576, 10**12),
    ]
    lines.append("| mode | bucket | runs | STRICT | strict-rate | mean evidence chars | mean answer chars |")
    lines.append("|------|--------|------|--------|-------------|---------------------|-------------------|")
    # Track per-mode (label, strict-rate, n) tuples for the
    # "recommended context budget" recommendation below.
    per_mode_buckets: dict[str, list[tuple[str, float, int]]] = {}
    for mode in modes:
        per_mode_buckets[mode] = []
        for label, lo, hi in buckets:
            sub = [
                r for r in rows
                if r["answer_mode"] == mode
                and lo <= (r.get("prompt_chars_total") or 0) < hi
                and not r["error"] and r["status"] != "error"
            ]
            if not sub:
                continue
            n_b = len(sub)
            n_strict = sum(1 for r in sub if r["audit_mode"] == "STRICT")
            mean_ev = sum((r.get("prompt_chars_evidence") or 0) for r in sub) / n_b
            mean_ans = sum((r.get("answer_chars") or 0) for r in sub) / n_b
            strict_rate = n_strict / n_b
            per_mode_buckets[mode].append((label, strict_rate, n_b))
            lines.append(
                f"| {mode} | {label} | {n_b} | {n_strict} | "
                f"{strict_rate:.2f} | {int(mean_ev)} | {int(mean_ans)} |"
            )
    lines.append("")
    lines.append("## recommended context budget (learned from this bench)")
    lines.append("")
    lines.append(
        "Per-mode peak strict-rate bucket from the table above. Bench "
        "observations feed back into a recommended `max_context_chars` "
        "default. Operator-driven landing per the five-step algorithm "
        "step 5 — surfaced here, not auto-applied. Minimum bucket size "
        "of 5 runs to be considered (smaller samples are noise). The "
        "log-scale buckets cover everything from 8B-class models "
        "through 1M-context models; a model whose context window stops "
        "at 82K simply never populates the giant buckets."
    )
    lines.append("")
    lines.append("| mode | peak bucket | strict-rate | n |")
    lines.append("|------|-------------|-------------|---|")
    for mode in modes:
        candidates = [t for t in per_mode_buckets.get(mode, []) if t[2] >= 5]
        if not candidates:
            lines.append(
                f"| {mode} | (insufficient samples; ≥5 runs/bucket "
                f"required) | n/a | n/a |"
            )
            continue
        # Peak strict-rate; tie-break on smaller bucket (less context
        # is cheaper at equivalent strict-rate).
        peak_label, peak_rate, peak_n = max(
            candidates, key=lambda t: (t[1], -buckets.index(
                next(b for b in buckets if b[0] == t[0])
            ))
        )
        lines.append(
            f"| {mode} | {peak_label} | {peak_rate:.2f} | {peak_n} |"
        )
    lines.append("")
    lines.append("## per question")
    lines.append("")
    lines.append(
        "Cell shows verdict counts across N samples (S=STRICT, H=HYBRID, U=UNGROUNDED, e=err). "
        "median ratio = median n_verified/n_quotes."
    )
    lines.append("")
    cells = _per_question_summary(rows)
    lines.append("| question | mode | verdicts (N=" + str(n_samples) + ") | median ratio | median latency |")
    lines.append("|----------|------|---------------------|--------------|----------------|")
    for q in questions:
        for mode in modes:
            c = cells.get((q, mode))
            if not c:
                continue
            verdicts = (
                f"S:{c['STRICT']} H:{c['HYBRID']} U:{c['UNGROUNDED']}"
                + (f" e:{c['errors']}" if c["errors"] else "")
            )
            mr = _median(c["ratios"])
            ml = _median(c["latencies"])
            qd = q if len(q) <= 50 else q[:47] + "..."
            lines.append(f"| {qd} | {mode} | {verdicts} | {mr:.3f} | {ml:.1f}s |")
    lines.append("")
    return "\n".join(lines)


def main(argv: list[str] | None = None) -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--questions", type=Path, default=Path("bench/qa_questions.txt"))
    ap.add_argument("--shards-dir", type=Path, default=Path.home() / ".arborist" / "shards")
    ap.add_argument("--qa-db", type=Path, default=None)
    ap.add_argument("--out-dir", type=Path, default=Path("bench/qa_results"))
    ap.add_argument("--top-k", type=int, default=8)
    ap.add_argument("--modes", default=",".join(ANSWER_MODES),
                    help="comma-separated subset of ANSWER_MODES to sweep")
    ap.add_argument("--n", dest="n_samples", type=int, default=3,
                    help="samples per (question, mode); each sample burns the cached "
                         "record so Hermes nondeterminism becomes the variance source")
    ap.add_argument("--limit", type=int, default=0,
                    help="truncate question list to N; 0 = all")
    ap.add_argument("--concurrency", type=int, default=1,
                    help="parallel sample-level tasks. Each (question, "
                         "mode, sample_idx) is one task; tasks are shuffled "
                         "via --seed and dispatched concurrently. Per-cell "
                         "Lock serializes burn+insert on the shared "
                         "cache_key so two samples of the same cell never "
                         "race. vLLM handles concurrent requests well; "
                         "4-8 is a safe starting point.")
    ap.add_argument("--seed", type=int, default=0,
                    help="deterministic shuffle seed for sample-level "
                         "task ordering. Same seed = same task order = "
                         "reproducible bench. Random ordering is the "
                         "design — uncorrelated samples give true i.i.d. "
                         "variance estimation and feed vLLM's continuous "
                         "batcher a diverse request stream.")
    ap.add_argument("--resume", type=Path, default=None,
                    help="resume an interrupted bench by appending to "
                         "an existing JSONL. Already-completed (question, "
                         "mode, sample_idx) tasks are skipped; remaining "
                         "tasks run in the original shuffled order (use "
                         "the same --seed). The same .md path is "
                         "re-rendered from the union of pre-existing + "
                         "new rows. Stop/start-able bench.")
    ap.add_argument("--endpoint", default=os.environ.get(
        "ARBORIST_LLM_ENDPOINT", "https://hermes.ai.unturf.com/v1"))
    ap.add_argument("--model", default=os.environ.get(
        "ARBORIST_LLM_MODEL", "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic"))
    ap.add_argument(
        "--policy", action="append", default=None, metavar="KEY=VALUE",
        help=(
            "policy-field override applied on top of DEFAULT_QUERY_POLICY. "
            "Repeat for multiple fields. VALUE is JSON-decoded; bare strings "
            "fall back to literal string. Example: "
            "--policy quantifier_reminder_enabled=true "
            "--policy quantifier_guard_apply_caps=true. Useful for A/B "
            "cycles per ticket #000008 §10.8 decision tree."
        ),
    )
    args = ap.parse_args(argv)

    questions = _read_questions(args.questions)
    if args.limit:
        questions = questions[: args.limit]
    if not questions:
        print(f"no questions in {args.questions}", file=sys.stderr)
        return 2

    # Parse --policy KEY=VALUE overrides into a dict. Values run through
    # json.loads so booleans, ints, lists, and quoted strings work as
    # expected; bare unquoted strings ("hermes-3") fall back to the
    # literal string.
    policy_overrides: dict | None = None
    if args.policy:
        policy_overrides = {}
        for entry in args.policy:
            if "=" not in entry:
                print(f"--policy needs KEY=VALUE, got {entry!r}", file=sys.stderr)
                return 2
            key, val = entry.split("=", 1)
            try:
                policy_overrides[key] = json.loads(val)
            except json.JSONDecodeError:
                policy_overrides[key] = val

    modes = [m.strip() for m in args.modes.split(",") if m.strip()]
    bad = [m for m in modes if m not in ANSWER_MODES]
    if bad:
        print(f"unknown mode(s): {bad}; expected subset of {list(ANSWER_MODES)}", file=sys.stderr)
        return 2

    args.out_dir.mkdir(parents=True, exist_ok=True)

    # Resume support: when --resume <existing.jsonl> is given, reuse
    # that file's stamp and rebuild done_tasks set so already-completed
    # samples skip on this run. Markdown rolls up the union of
    # pre-existing + freshly-written rows.
    existing_rows: list[dict] = []
    done_tasks: set[tuple[str, str, int]] = set()
    if args.resume:
        resume_path = Path(args.resume)
        if not resume_path.exists():
            print(f"--resume target does not exist: {resume_path}", file=sys.stderr)
            return 2
        with resume_path.open(encoding="utf-8") as rf:
            for line in rf:
                line = line.strip()
                if not line:
                    continue
                try:
                    row = json.loads(line)
                except json.JSONDecodeError:
                    continue  # skip a partial trailing line
                key = (row["question"], row["answer_mode"], int(row.get("sample_idx", 0)))
                done_tasks.add(key)
                existing_rows.append(row)
        stamp = resume_path.stem  # reuse the original "YYYY-MM-DDTHH-MM-SSZ"
        jsonl_path = resume_path
        md_path = resume_path.with_suffix(".md")
    else:
        stamp = _utc_iso_compact()
        jsonl_path = args.out_dir / f"{stamp}.jsonl"
        md_path = args.out_dir / f"{stamp}.md"

    if args.n_samples < 1:
        print(f"--n must be >= 1, got {args.n_samples}", file=sys.stderr)
        return 2

    n_runs_total = len(questions) * len(modes) * args.n_samples
    print(
        f"[bench] {len(questions)} question(s) × {len(modes)} mode(s) × "
        f"{args.n_samples} sample(s) = {n_runs_total} run(s)"
    )
    print(f"[bench] shards_dir={args.shards_dir}  top_k={args.top_k}  "
          f"burn=always (per-sample, forces fresh inference)")
    print(f"[bench] concurrency={args.concurrency}  seed={args.seed}  "
          f"(sample-level tasks; per-cell Lock serializes cache_key writes)")
    if args.resume:
        print(f"[bench] resuming from {jsonl_path}: "
              f"{len(existing_rows)} task(s) already done; skipping")
    else:
        print(f"[bench] writing → {jsonl_path}")
    print()

    # Build sample-level task list. Each task is one (question, mode,
    # sample_idx); shuffle so samples for the same cell get spread
    # across time. Two effects:
    #   - True i.i.d. n-sample variance: consecutive samples of the
    #     same cell don't share vLLM batch composition or KV-cache
    #     locality, so what looks like model nondeterminism actually
    #     IS model nondeterminism.
    #   - vLLM's continuous batcher gets a diverse request stream;
    #     under the prior cell-grouped scheduling, batches were
    #     correlated and the engine couldn't fill efficiently.
    tasks = [
        (q, mode, sample_idx)
        for q in questions
        for mode in modes
        for sample_idx in range(args.n_samples)
    ]
    rng = random.Random(args.seed)
    rng.shuffle(tasks)
    total_tasks = len(tasks)

    # Resume: filter out tasks already in the JSONL. Order preserved.
    if done_tasks:
        tasks = [t for t in tasks if t not in done_tasks]
        print(f"[bench] {len(tasks)} task(s) remaining ({total_tasks} total)")
        print()

    # Per-cell lock — burn+insert against a shared cache_key must
    # serialize. Two samples of the SAME (q, mode) ending up in
    # adjacent worker slots could race; the lock makes that path
    # safe without bottlenecking unrelated cells. Lock contention is
    # rare under shuffle (samples spread out), so the throughput cost
    # is near-zero while correctness is preserved.
    cell_locks: dict[tuple[str, str], Lock] = defaultdict(Lock)

    rows: list[dict] = list(existing_rows)  # preserve resumed rows for the rollup
    write_lock = Lock()
    print_lock = Lock()
    done = [len(existing_rows)]  # countdown reflects total, not just-this-session

    file_mode = "a" if args.resume else "w"
    with jsonl_path.open(file_mode, encoding="utf-8") as f:
        def _process(task: tuple[str, str, int]) -> None:
            q, mode, sample_idx = task
            with cell_locks[(q, mode)]:
                row = _run_one(
                    question=q,
                    answer_mode=mode,
                    shards_dir=args.shards_dir,
                    qa_db=args.qa_db,
                    top_k=args.top_k,
                    burn=True,
                    endpoint=args.endpoint,
                    model=args.model,
                    policy_overrides=policy_overrides,
                )
                row["sample_idx"] = sample_idx
            with write_lock:
                f.write(json.dumps(row, ensure_ascii=False) + "\n")
                rows.append(row)
                f.flush()
            with print_lock:
                done[0] += 1
                tag = (
                    f"err: {row['error']}" if row["error"]
                    else f"{row['audit_mode']} {row['n_verified']}/{row['n_quotes']} "
                         f"{row['elapsed_s']:.1f}s"
                )
                print(
                    f"  [{done[0]:>3}/{total_tasks}] {mode:<22} "
                    f"#{sample_idx + 1}/{args.n_samples} "
                    f"{q[:42]:<42} → {tag}",
                    flush=True,
                )

        if args.concurrency <= 1:
            for task in tasks:
                _process(task)
        else:
            with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
                futures = [pool.submit(_process, t) for t in tasks]
                for fut in as_completed(futures):
                    fut.result()  # surfaces any exception

    summary = _summarize(rows)
    md = _render_markdown(rows, summary, stamp, modes, questions, args.n_samples)
    md_path.write_text(md, encoding="utf-8")

    print()
    print(md)
    print(f"\n[bench] results: {jsonl_path}\n[bench] summary: {md_path}")
    return 0


if __name__ == "__main__":
    sys.exit(main())