diff --git a/aborist/qa/keys.py b/aborist/qa/keys.py index f3d4b55..9bb6dd2 100644 --- a/aborist/qa/keys.py +++ b/aborist/qa/keys.py @@ -225,6 +225,7 @@ _VERIFIER_POLICY_FIELDS = frozenset({ # invalidation we want when a guard knob changes. "quantifier_guard_enabled", "quantifier_guard_apply_caps", + "quantifier_apply_caps_modes", "quantifier_caps_by_intensity", "quantifier_guard_modes", "quantifier_reminder_enabled", diff --git a/aborist/qa/query.py b/aborist/qa/query.py index b0950b5..d988521 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -462,10 +462,14 @@ DEFAULT_QUERY_POLICY = { # reported on result dict but not applied to the verifier. "quantifier_guard_enabled": True, "quantifier_guard_apply_caps": False, + # Mode allowlist when apply_caps flips True. See runner.DEFAULT_POLICY + # for the n=5 bench data driving the JSON-only default. + "quantifier_apply_caps_modes": ["claim_lattice"], "quantifier_caps_by_intensity": {}, "quantifier_guard_modes": ["claim_lattice_pointer", "claim_lattice"], - # Phase 3 — see runner.DEFAULT_POLICY for rationale. Default OFF. - "quantifier_reminder_enabled": False, + # Phase 3 — default ON for lattice modes per the 2026-05-03 bench + # A/B (#000008 §12). See runner.DEFAULT_POLICY for full rationale. + "quantifier_reminder_enabled": True, # Phase 4 — strict reject for broad-unbounded. See runner.py for # rationale. Default OFF. "quantifier_reject_broad": False, @@ -1659,14 +1663,28 @@ def query( # Effective cap that the verifier will see. Dry-run mode # (apply_caps=False) preserves the policy default; once an # operator flips apply_caps=True, the looked-up cap shadows the - # default for this call only — no policy mutation, no schema - # bump. The full cap fallback chain reads: - # 1. quantifier-guard cap (when apply_caps=True) + # default for this call only — but ONLY for modes in + # quantifier_apply_caps_modes. n=5 bench (#000008 §12.10) found + # cap-on-pointer fires TOO_MANY_CLAIMS 20× without moving the + # 0/45 STRICT floor, while cap-on-JSON wins +14pp. Default + # allowlist is ["claim_lattice"] (JSON only); empty/None falls + # back to all guard_modes. + # The full cap fallback chain reads: + # 1. quantifier-guard cap (when apply_caps=True AND mode allowed) # 2. claim_lattice_max_claims_per_answer policy field # 3. hard-coded default 12 quantifier_apply_caps = bool(policy.get("quantifier_guard_apply_caps", False)) + quantifier_apply_caps_modes = policy.get( + "quantifier_apply_caps_modes", + quantifier_guard_modes, # legacy fallback + ) or quantifier_guard_modes + quantifier_caps_mode_gated = answer_mode in (quantifier_apply_caps_modes or []) _policy_max_claims = int(policy.get("claim_lattice_max_claims_per_answer", 12)) - if quantifier_apply_caps and claim_cap_lookup is not None: + if ( + quantifier_apply_caps + and quantifier_caps_mode_gated + and claim_cap_lookup is not None + ): effective_max_claims = int(claim_cap_lookup) else: effective_max_claims = _policy_max_claims diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index 61c9695..39b7d7b 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -223,14 +223,27 @@ DEFAULT_POLICY = { # stable HYBRID 0.455 on baseline, different failure shape. "quantifier_guard_enabled": True, "quantifier_guard_apply_caps": False, + # When apply_caps flips True, this allowlist gates which modes + # actually have caps applied. n=5 verification 2026-05-03 (#000008 + # §12.10): cap on claim_lattice (JSON) wins +14pp on STRICT-rate; + # cap on claim_lattice_pointer fires TOO_MANY_CLAIMS 20× without + # moving the verdict floor (still 0 STRICT). Default to JSON only + # so flipping the master switch doesn't add wasted cap-noise on + # pointer mode. Empty list / None = honor quantifier_guard_modes + # (legacy fallback). + "quantifier_apply_caps_modes": ["claim_lattice"], "quantifier_caps_by_intensity": {}, "quantifier_guard_modes": ["claim_lattice_pointer", "claim_lattice"], - # Phase 3 — broad-quantifier reminder injection. Default OFF - # because Hermes-3-8B already ignores parts of the existing - # reminder under enumeration pressure (ticket §3 Option B con). - # Operator opts in per-call after Phase 2 dry-run telemetry - # confirms which broad-shape rows actually need the reminder. - "quantifier_reminder_enabled": False, + # Phase 3 — broad-quantifier reminder injection. Default ON for + # lattice modes (gated via quantifier_guard_modes) per the + # 2026-05-03 bench A/B (#000008 §12). Reminder eliminates + # FORMAT_COLLAPSED (2→0), reduces NO_EVIDENCE_POINTER 33%, + # boosts mean ratio +17pp on pointer / +21pp on JSON, and + # rescues JSON UNGROUNDED 7→1. n=5 verification 2026-05-03 + # confirms the compound effect with cap survives at higher + # sample size. Quote mode is mode-gated off (different failure + # shape; paraphrase verifier doesn't need pointer-tag reminders). + "quantifier_reminder_enabled": True, # Phase 4 — strict reject for broad-unbounded queries. When True # AND intensity ∈ {ALL, COMPREHENSIVE, OPEN_REQUEST} AND # scope_bound_hint == "unbounded", query()/ask() return UNGROUNDED @@ -336,8 +349,19 @@ def ask( else: claim_cap_lookup = None quantifier_apply_caps = bool(policy.get("quantifier_guard_apply_caps", False)) + quantifier_apply_caps_modes = policy.get( + "quantifier_apply_caps_modes", + quantifier_guard_modes, # legacy fallback + ) or quantifier_guard_modes + quantifier_caps_mode_gated = answer_mode_for_guard in ( + quantifier_apply_caps_modes or [] + ) _policy_max_claims = int(policy.get("claim_lattice_max_claims_per_answer", 12)) - if quantifier_apply_caps and claim_cap_lookup is not None: + if ( + quantifier_apply_caps + and quantifier_caps_mode_gated + and claim_cap_lookup is not None + ): effective_max_claims = int(claim_cap_lookup) else: effective_max_claims = _policy_max_claims diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 0c4cfbb..13f9683 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -57,7 +57,7 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| -| #000008 | Broad-quantifier preflight guard | open · phases 0–4 landed; bench A/B in progress | 2026-05-02 | — | +| #000008 | Broad-quantifier preflight guard | open · phases 0–4 landed; defaults flipped (Option A); Phase 5 DAG-binding pending | 2026-05-02 | — | | #000007 | Query-layer hyphen folding | closed · 2026-05-02 | 2026-05-02 | — | | #000006 | Bench-emergent findings (rolling research log) | open · rolling | 2026-05-02 | — | | #000005 | Label ladder migration (POINTER-LINKED → …) | closed · 2026-05-02 | 2026-05-01 | D7 | diff --git a/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md b/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md index 27c2064..2e6d530 100644 --- a/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md +++ b/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md @@ -446,9 +446,14 @@ enhancement, not blocking. ## 8. Status -**All four phases landed 2026-05-03. Mechanism complete; defaults -preserve the dry-run discipline.** Bench-first measurement of -§10.8 decision-tree thresholds is the only remaining work. +**All four phases landed 2026-05-03. Mechanism complete. Defaults +flipped per §12.11 after the four-cell A/B + n=5 verification.** +Pure-Option-A landing: reminder default-on for lattice modes, cap +default operator-opt-in with JSON-only allowlist. Pointer-mode +cap is filtered out (wasted on the 0/45 STRICT floor). Bench-side +mechanism + measurement complete; only Phase 5 (run-DAG node +binding) and cross-model Qwen/GPT-4 verification remain as +follow-ups. Implementation commit chain (all on `origin/main`): @@ -1893,7 +1898,8 @@ Per §10.8 decision tree, A/B sequence with this same broad subset: - [x] **Reminder only** (apply_caps=False, reminder=True) — §12.6. - [x] **Cap only** (apply_caps=True, reminder=False) — §12.7. - [x] **Cap + reminder** (apply_caps=True, reminder=True) — §12.8. -- [ ] **n≥5 verification** before flipping defaults (§12.9 caveat). +- [x] **n=5 verification** of cap+reminder — §12.10. +- [x] **Defaults flipped** per §12.11 landing. 5pp signal floor per `docs/bench-maxing.md` for default-flip decisions. Each cycle adds 81 runs at ~12-17min on Hermes. @@ -2154,3 +2160,105 @@ Either way, the disable hierarchy (§10.11.2) means flipping defaults is reversible per-call, per-mode, per-test. No commitment is permanent — flip the default, watch one bench cycle, revert if regression. + +### 12.10 n=5 verification (2026-05-03T19-55-14Z) + +Same 9-question broad subset, n=5 instead of n=3 (135 runs total). +Same policy: `apply_caps=True`, `reminder=True`. Tightens variance +on the cap+reminder cell to test whether the §12.8 0.30 vs §12.7 +0.33 JSON STRICT-rate gap was real or noise. + +| Metric | n=3 | **n=5** | +|---------------------------|-----------|------------| +| `JSON` STRICT-rate | 0.30 | **0.33** | +| `JSON` UNGROUNDED rate | 1/27 (4%) | **2/45 (4%)** | +| `JSON` STRICT count | 8 | **15** | +| `pointer` STRICT-rate | 0/27 | 0/45 | +| `pointer` mean ratio | 0.684 | 0.634 | +| `pointer` UNGROUNDED rate | 6/27 (22%)| 10/45 (22%)| + +**Verdict: cap+reminder JSON STRICT-rate at n=5 = 0.33, matching +cap-only (n=3 0.33).** The §12.8 0.30 reading was downward Hermes +nondeterminism. n=5 confirms the compound effect delivers cap-only's +STRICT-rate AND reminder-only's UNGROUNDED-rescue simultaneously. + +**§10.8 strict gate verdict at n=5 — MET on multiple metrics:** + +- vs reminder-only on JSON STRICT-rate: **+11pp** (clears 5pp floor) +- vs cap-only on JSON UNGROUNDED rate: **−18pp** (clears floor) +- vs reminder-only on pointer mean ratio: −1pp (within noise) +- vs cap-only on pointer mean ratio: **+12pp** (clears floor) + +Compound effect is real. But pointer-mode cap is structurally +wasted: 0/45 STRICT across all four cells; cap-on-pointer fires +TOO_MANY_CLAIMS 20× without verdict gain. + +### 12.11 Final landing — Option A defaults flipped + +The §10.8 strict gate for pure Option D ("cap + reminder default- +on for all lattice modes") is met. But the data-driven cleaner +landing is **Option A — per-mode tailored**: + +``` +Lattice modes (claim_lattice + claim_lattice_pointer): + quantifier_reminder_enabled True ← FLIPPED 2026-05-03 + quantifier_guard_apply_caps False ← unchanged (operator opt-in) + quantifier_apply_caps_modes ["claim_lattice"] ← NEW field + JSON-only allowlist for + when operator flips + apply_caps=True + +claim_lattice (JSON): + effective behavior: reminder ON; cap applies when + apply_caps=True (operator-driven) + +claim_lattice_pointer: + effective behavior: reminder ON; cap NEVER applied (filtered out + by apply_caps_modes allowlist) since cap on + pointer is structurally wasted noise. + +quote: + effective behavior: unchanged. Mode-gated off via + quantifier_guard_modes default. +``` + +Code changes (commit pending): + +- `aborist/qa/runner.py`: `quantifier_reminder_enabled=True`, + new field `quantifier_apply_caps_modes=["claim_lattice"]`. +- `aborist/qa/query.py`: same defaults. +- `aborist/qa/keys.py`: `quantifier_apply_caps_modes` added to + `_VERIFIER_POLICY_FIELDS` so flipping it bumps governance hash. +- Cap-application gate now reads: + `if apply_caps AND mode in apply_caps_modes AND cap is not None:` +- 5 new tests pin: reminder default ON for both + `runner.DEFAULT_POLICY` and `query.DEFAULT_QUERY_POLICY`, + apply_caps_modes default `["claim_lattice"]`, + governance-hash invalidation on apply_caps_modes flip, + apply_caps default still False (dry-run discipline preserved). + +**Operator commands after this landing:** + +``` +# Default behavior (no flag): reminder ON for lattice modes; cap +# OFF (operator opt-in only). +$ aborist query "winners of all major sports?" + +# Flip cap on for one call (JSON-only by default): +$ aborist query --apply-quantifier-caps "..." + +# Override the allowlist to include pointer mode too (Option D): +$ aborist query --apply-quantifier-caps \ + --policy quantifier_apply_caps_modes='["claim_lattice","claim_lattice_pointer"]' "..." + +# Disable the reminder for one call: +$ aborist query --policy quantifier_reminder_enabled=false "..." + +# Master kill: +$ aborist query --no-quantifier-guard "..." +``` + +**Six-level disable hierarchy still intact** — no flag forced upon +operators; the defaults simply move toward the §10.8-validated +position. Reverting the default is a one-line policy edit per +§10.11.5 rollback playbook. diff --git a/tests/test_quantifier_caps.py b/tests/test_quantifier_caps.py index bb6297c..488899a 100644 --- a/tests/test_quantifier_caps.py +++ b/tests/test_quantifier_caps.py @@ -219,3 +219,59 @@ def test_default_profile_present(): """Lookup-fallback target must exist or unknown models would return None on every call.""" assert "default" in PROFILES + + +# Ticket #000008 §12.10 — n=5 verification produced these defaults +# (committed 2026-05-03 post cap+reminder bench). Pin them here so a +# future drift in DEFAULT_QUERY_POLICY surfaces a failing test. + +def test_default_reminder_enabled_for_lattice_modes(): + """Phase 3 reminder default-ON per #000008 §12 bench finding: + eliminates FORMAT_COLLAPSED, reduces NO_EVIDENCE_POINTER 33%, + boosts mean ratio +17pp pointer / +21pp JSON, rescues JSON + UNGROUNDED 7→1.""" + from aborist.qa.runner import DEFAULT_POLICY as RUNNER_POLICY + from aborist.qa.query import DEFAULT_QUERY_POLICY + assert RUNNER_POLICY["quantifier_reminder_enabled"] is True + assert DEFAULT_QUERY_POLICY["quantifier_reminder_enabled"] is True + + +def test_default_apply_caps_modes_is_json_only(): + """Phase 2 cap default-allowlist per #000008 §12.10 n=5 finding: + cap-on-JSON wins +14pp on STRICT-rate; cap-on-pointer fires + TOO_MANY_CLAIMS 20× without moving the 0/45 STRICT floor. + Default the apply-caps allowlist to JSON-only so flipping + apply_caps=True doesn't add wasted cap-noise on pointer mode.""" + from aborist.qa.runner import DEFAULT_POLICY as RUNNER_POLICY + from aborist.qa.query import DEFAULT_QUERY_POLICY + assert RUNNER_POLICY["quantifier_apply_caps_modes"] == ["claim_lattice"] + assert DEFAULT_QUERY_POLICY["quantifier_apply_caps_modes"] == ["claim_lattice"] + + +def test_apply_caps_modes_is_in_verifier_policy_fields(): + """Without governance binding, an operator could change the + allowlist and silently re-use cached records written under a + different allowlist.""" + assert "quantifier_apply_caps_modes" in _VERIFIER_POLICY_FIELDS + + +def test_governance_hash_changes_when_apply_caps_modes_changes(): + base_policy = dict.fromkeys(_VERIFIER_POLICY_FIELDS, "default") + base_policy["quantifier_apply_caps_modes"] = ["claim_lattice"] + h_json_only = verifier_policy_hash(base_policy) + base_policy["quantifier_apply_caps_modes"] = [ + "claim_lattice", "claim_lattice_pointer" + ] + h_both = verifier_policy_hash(base_policy) + assert h_json_only != h_both + + +def test_apply_caps_default_off_preserves_dry_run_discipline(): + """The cap-application gate stays operator-opt-in by default + even with reminder default-on. Dry-run discipline (§10.11.3) + survives the §12 bench cycle — operators flip apply_caps via + --apply-quantifier-caps after their own bench review.""" + from aborist.qa.runner import DEFAULT_POLICY as RUNNER_POLICY + from aborist.qa.query import DEFAULT_QUERY_POLICY + assert RUNNER_POLICY["quantifier_guard_apply_caps"] is False + assert DEFAULT_QUERY_POLICY["quantifier_guard_apply_caps"] is False