diff --git a/bench/qa_questions_quantifier_subset.txt b/bench/qa_questions_quantifier_subset.txt new file mode 100644 index 0000000..71f6bf0 --- /dev/null +++ b/bench/qa_questions_quantifier_subset.txt @@ -0,0 +1,23 @@ +# Broad-quantifier bench subset — Ticket #000008 dry-run measurement. +# Runs the 9 questions that classify as broad (ALL/COMPREHENSIVE/ +# OPEN_REQUEST) plus one bounded-universal fixture pair so we can +# compare bounded vs unbounded behavior under guard-on dry-run. +# +# Used 2026-05-03 to measure post-implementation classifier and +# cap-lookup output. apply_caps stays FALSE during this run per +# §10.11.3 dry-run discipline — telemetry only. + +# unbounded universals +winners of all major sports? +tell me all there is to know about york england? + +# OPEN_REQUEST — verb-driven +tell me about connecticut +tell me about the C programming language +tell me about method man? +tell me about the roman empire +describe the structure of DNA + +# bounded universals (Finding 2 from 2026-05-03 review) +name all members of the beatles +list all planets in the solar system diff --git a/bench/qa_sweep.py b/bench/qa_sweep.py index 55f7796..dbb7574 100644 --- a/bench/qa_sweep.py +++ b/bench/qa_sweep.py @@ -124,13 +124,23 @@ def _run_one( burn: bool, endpoint: str, model: str, + policy_overrides: dict | None = None, ) -> dict: - """Run one (question, mode) — returns a flat record for the JSONL.""" + """Run one (question, mode) — returns a flat record for the JSONL. + + ``policy_overrides`` is a dict of policy-field overrides applied + after ``DEFAULT_QUERY_POLICY`` and ``answer_mode`` so that + ``bench/qa_sweep.py --policy quantifier_reminder_enabled=true`` + can flip individual fields for an A/B cycle without touching + the policy defaults. + """ from aborist.qa.client import OpenAICompatibleClient from aborist.qa.query import DEFAULT_QUERY_POLICY, query policy = dict(DEFAULT_QUERY_POLICY) policy["answer_mode"] = answer_mode + if policy_overrides: + policy.update(policy_overrides) api_key = os.environ.get("ABORIST_LLM_API_KEY") client = OpenAICompatibleClient(base_url=endpoint, api_key=api_key) @@ -721,6 +731,17 @@ def main(argv: list[str] | None = None) -> int: "ABORIST_LLM_ENDPOINT", "https://hermes.ai.unturf.com/v1")) ap.add_argument("--model", default=os.environ.get( "ABORIST_LLM_MODEL", "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic")) + ap.add_argument( + "--policy", action="append", default=None, metavar="KEY=VALUE", + help=( + "policy-field override applied on top of DEFAULT_QUERY_POLICY. " + "Repeat for multiple fields. VALUE is JSON-decoded; bare strings " + "fall back to literal string. Example: " + "--policy quantifier_reminder_enabled=true " + "--policy quantifier_guard_apply_caps=true. Useful for A/B " + "cycles per ticket #000008 §10.8 decision tree." + ), + ) args = ap.parse_args(argv) questions = _read_questions(args.questions) @@ -730,6 +751,23 @@ def main(argv: list[str] | None = None) -> int: print(f"no questions in {args.questions}", file=sys.stderr) return 2 + # Parse --policy KEY=VALUE overrides into a dict. Values run through + # json.loads so booleans, ints, lists, and quoted strings work as + # expected; bare unquoted strings ("hermes-3") fall back to the + # literal string. + policy_overrides: dict | None = None + if args.policy: + policy_overrides = {} + for entry in args.policy: + if "=" not in entry: + print(f"--policy needs KEY=VALUE, got {entry!r}", file=sys.stderr) + return 2 + key, val = entry.split("=", 1) + try: + policy_overrides[key] = json.loads(val) + except json.JSONDecodeError: + policy_overrides[key] = val + modes = [m.strip() for m in args.modes.split(",") if m.strip()] bad = [m for m in modes if m not in ANSWER_MODES] if bad: @@ -842,6 +880,7 @@ def main(argv: list[str] | None = None) -> int: burn=True, endpoint=args.endpoint, model=args.model, + policy_overrides=policy_overrides, ) row["sample_idx"] = sample_idx with write_lock: diff --git a/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md b/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md index 06c75f6..b344f39 100644 --- a/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md +++ b/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md @@ -1757,3 +1757,148 @@ but are now binding via tests + commits: - **Cross-model bench (Qwen / GPT-4)**. Manual cross-model verification of "large model holds format discipline at higher caps" not yet performed. Manual / out-of-automated-bench task. + +## 12. Live bench measurements (2026-05-03) + +First post-implementation bench cycle. Measures classifier output +end-to-end with Phase 1+2 wired in dry-run mode (caps reported, +not applied) per §10.11.3 step 2. Also catches one classifier +defect not seen in the design phase. + +### 12.1 Pre-bench: classifier distribution scan (free, no LLM) + +Ran `classify_question_quantifier()` on all 73 questions in +`bench/qa_questions.txt`. Surfaced **one defect**: `how many X?` +mis-classified as MANY (4 of 7 broad classifications wrong; 33% +false-positive rate among broad). All four were count-questions +expecting a single numeric answer ("50 states", "206 bones") — +SINGULAR is correct, MANY is wrong. + +Fix landed `d24291b`: leading-anchor count-question short-circuit. +`^\s*(?:and\s+|but\s+|so\s+)?how (?:many|much)\b` → SINGULAR. +Anchored at start so buried `how many` doesn't suppress the rest +of the question's quantifier markers. + +Post-fix distribution across the 73-question bench: + +``` +SINGULAR 65 (89.0%) was 61 (84%) +OPEN_REQUEST 5 ( 6.8%) unchanged +MANY 0 ( 0.0%) was 4 (5%) ← all moved to SINGULAR +ALL 1 ( 1.4%) unchanged +COMPREHENSIVE 1 ( 1.4%) unchanged +SMALL_NUM_EXPLICIT 1 ( 1.4%) unchanged +``` + +Also added two bounded-universal fixtures (Finding 2 from review): +`name all members of the beatles` and `list all planets in the solar +system`. Both classify ALL · `scope_bound_hint=bounded`. Without +these, the §10.1 bounded-vs-unbounded distinction had zero live +bench coverage. + +### 12.2 Live bench — broad subset (dry-run, apply_caps=False) + +Bench file `bench/qa_questions_quantifier_subset.txt` — 9 broad +questions (7 unbounded + 2 bounded). 3 modes × n=3 = 81 runs. +Result: `bench/qa_results/2026-05-03T12-29-53Z.{jsonl,md}`. + +**Per-mode summary:** + +| mode | S/H/U | strict-rate | mean ratio | latency | +|-------------------------|---------|-------------|------------|---------| +| `quote` | 15/12/0 | **0.56** | 0.900 | 16.7s | +| `claim_lattice_pointer` | 0/18/9 | 0.00 | 0.473 | 12.2s | +| `claim_lattice` (JSON) | 5/15/7 | 0.19 | 0.524 | 14.5s | + +**Key finding — bounded vs unbounded matters in practice:** + +| question | scope | quote | pointer | JSON | +|---------------------------------------|-----------|---------|---------|-----------| +| Winners of all major sports? | unbounded | H:3 | U:3 | H:2/U:1 | +| Tell me everything about York | unbounded | S:3 | H:3 | U:3 | +| **Name all members of the Beatles** | **bounded** | S:2/H:1 | U:3 | **S:3** | +| **List all planets** | **bounded** | H:3 | U:3 | S:1/H:2 | + +JSON mode hits **3/3 STRICT on the Beatles** — same model, same +verifier, different `scope_bound_hint`, fundamentally different +outcome. The §10.1 bounded vs unbounded split is empirically +real, not just architectural. + +**Pointer-mode violation distribution (27 runs, broad subset):** + +``` +CITATION_MISMATCH 14 +TITLE_MISMATCH 10 +NO_EVIDENCE_POINTER 9 +TOO_MANY_CLAIMS 7 ← cap=12 firing on broad questions +SCHEMA_INVALID 4 ← bare-name + empty-text claims (verify.py:1242,1270) +WARRANT_MISSING 4 +LAZY_ANCHOR_DEMOTE 3 +POINTER_OVERFLOW_TRIMMED 3 +FORMAT_COLLAPSED 2 +DEFLECTION_DETECTED 1 +``` + +### 12.3 Telemetry verified end-to-end + +Per-question classifier output (sampled across the 81-row JSONL): + +``` +winners of all major sports? + intensity=ALL scope=unbounded cap=8 +tell me all there is to know about york england? + intensity=COMPREHENSIVE scope=unbounded cap=5 +tell me about connecticut + intensity=OPEN_REQUEST scope=unbounded cap=5 +describe the structure of DNA + intensity=OPEN_REQUEST scope=unbounded cap=5 +name all members of the beatles + intensity=ALL scope=bounded cap=8 +list all planets in the solar system + intensity=ALL scope=bounded cap=8 +``` + +Quote-mode rows record `cap=None` (mode opted out via +`quantifier_guard_modes` default). Lattice-mode rows record the +looked-up cap (5 / 8) but the verifier used 12 (apply_caps=False +preserves dry-run discipline). + +### 12.4 §10.8 decision-tree implications + +Empirically: + +- **Cap-only (Phase 2 apply_caps=True) is unlikely to clear the + 5pp gate** for pointer mode. Pointer-mode is already 0 STRICT; + lowering cap from 12→8 would *increase* TOO_MANY_CLAIMS firings + (currently 7/27) but can't move the verdict floor below 0. + Same conclusion §5.1.1 reached at n=3, now confirmed at n=27. +- **NO_EVIDENCE_POINTER (9/27) is the load-bearing pointer-mode + failure.** This is exactly what Phase 3 reminder targets — the + reminder restates the [E\d+] citation rule one user-turn before + the question. Worth A/B testing with `quantifier_reminder + _enabled=True` next. +- **JSON mode benefits from bounded-vs-unbounded discrimination.** + 3/3 STRICT on Beatles vs 0/3 on york-england. The mode + scope + combination is what matters; cap-only doesn't help here either. +- **Quote mode is the workhorse for broad questions** at 0.56 + strict-rate. It paraphrase-verifies rather than pointer-verifies, + so it doesn't have the bracket-discipline burden. Keeping quote + out of `quantifier_guard_modes` (the default) is empirically + validated. + +### 12.5 Next bench cycles + +Per §10.8 decision tree, A/B sequence with this same broad subset: + +- [ ] **Reminder only** (apply_caps=False, reminder=True). Tests + whether reminder injection moves NO_EVIDENCE_POINTER rate. + Strongest single-knob candidate per §12.4. +- [ ] **Cap only** (apply_caps=True, reminder=False). Tests whether + cap-application alone shifts violation distribution. Expected + to move TOO_MANY_CLAIMS rate but not STRICT-rate. +- [ ] **Cap + reminder** (apply_caps=True, reminder=True). Tests + combined effect. Defaults flip on if and only if this beats + reminder-only by ≥5pp. + +5pp signal floor per `docs/bench-maxing.md` for default-flip +decisions. Each cycle adds 81 runs at ~12-17min on Hermes.