qa(#000008): §12.10 n=5 verification + §12.11 defaults flipped (Option A)
n=5 verification of cap+reminder cell (135 runs):
Metric | n=3 | n=5
--------------------+-----------+------------
JSON SR | 0.30 | 0.33 ← matches cap-only
JSON UNGROUNDED rate| 1/27 (4%) | 2/45 (4%) ← matches reminder-only
pointer SR | 0/27 | 0/45 ← unchanged across all cells
The §12.8 0.30 was Hermes nondeterminism. n=5 confirms cap+reminder
delivers cap-only's STRICT-rate AND reminder-only's UNGROUNDED-rescue.
§10.8 strict gate met at n=5:
vs reminder-only on JSON SR: +11pp (clears floor)
vs cap-only on JSON UNGROUNDED: -18pp (clears floor)
vs cap-only on ptr mean ratio: +12pp (clears floor)
Defaults flipped — Option A landing (per-mode tailored):
quantifier_reminder_enabled False → True
(load-bearing on both lattice modes)
NEW field: quantifier_apply_caps_modes = ["claim_lattice"]
(allowlist for which modes apply caps
when apply_caps=True; JSON-only since
cap-on-pointer is wasted noise per
§12.10 0/45 STRICT data)
quantifier_guard_apply_caps False → False (UNCHANGED)
(operator opts in via
--apply-quantifier-caps; preserves
§10.11.3 dry-run discipline)
Cap-application gate now reads:
if apply_caps AND mode in apply_caps_modes AND cap is not None:
effective_max_claims = looked_up_cap
quantifier_apply_caps_modes folded into _VERIFIER_POLICY_FIELDS so
flipping the allowlist invalidates prior cache records.
5 new tests pin: reminder default ON for both runner.DEFAULT_POLICY
and query.DEFAULT_QUERY_POLICY; apply_caps_modes default
["claim_lattice"]; governance-hash invalidation on allowlist flip;
apply_caps default still False (dry-run preserved).
920 tests passing (5 new); 36 skipped.
Operator behavior:
$ aborist query "winners of all major sports?"
→ reminder ON, cap OFF (default after this commit)
$ aborist query --apply-quantifier-caps "..."
→ cap applies on claim_lattice (JSON) only
$ aborist query --apply-quantifier-caps \
--policy quantifier_apply_caps_modes='["claim_lattice","claim_lattice_pointer"]' "..."
→ Option D for one call
Phase 5 (run-DAG node binding for quantifier_preflight) and
cross-model Qwen/GPT-4 verification remain as follow-ups per §11.11.
This commit is contained in:
parent
9780cca4d3
commit
4f2b5a6685
6 changed files with 225 additions and 18 deletions
|
|
@ -225,6 +225,7 @@ _VERIFIER_POLICY_FIELDS = frozenset({
|
|||
# invalidation we want when a guard knob changes.
|
||||
"quantifier_guard_enabled",
|
||||
"quantifier_guard_apply_caps",
|
||||
"quantifier_apply_caps_modes",
|
||||
"quantifier_caps_by_intensity",
|
||||
"quantifier_guard_modes",
|
||||
"quantifier_reminder_enabled",
|
||||
|
|
|
|||
|
|
@ -462,10 +462,14 @@ DEFAULT_QUERY_POLICY = {
|
|||
# reported on result dict but not applied to the verifier.
|
||||
"quantifier_guard_enabled": True,
|
||||
"quantifier_guard_apply_caps": False,
|
||||
# Mode allowlist when apply_caps flips True. See runner.DEFAULT_POLICY
|
||||
# for the n=5 bench data driving the JSON-only default.
|
||||
"quantifier_apply_caps_modes": ["claim_lattice"],
|
||||
"quantifier_caps_by_intensity": {},
|
||||
"quantifier_guard_modes": ["claim_lattice_pointer", "claim_lattice"],
|
||||
# Phase 3 — see runner.DEFAULT_POLICY for rationale. Default OFF.
|
||||
"quantifier_reminder_enabled": False,
|
||||
# Phase 3 — default ON for lattice modes per the 2026-05-03 bench
|
||||
# A/B (#000008 §12). See runner.DEFAULT_POLICY for full rationale.
|
||||
"quantifier_reminder_enabled": True,
|
||||
# Phase 4 — strict reject for broad-unbounded. See runner.py for
|
||||
# rationale. Default OFF.
|
||||
"quantifier_reject_broad": False,
|
||||
|
|
@ -1659,14 +1663,28 @@ def query(
|
|||
# Effective cap that the verifier will see. Dry-run mode
|
||||
# (apply_caps=False) preserves the policy default; once an
|
||||
# operator flips apply_caps=True, the looked-up cap shadows the
|
||||
# default for this call only — no policy mutation, no schema
|
||||
# bump. The full cap fallback chain reads:
|
||||
# 1. quantifier-guard cap (when apply_caps=True)
|
||||
# default for this call only — but ONLY for modes in
|
||||
# quantifier_apply_caps_modes. n=5 bench (#000008 §12.10) found
|
||||
# cap-on-pointer fires TOO_MANY_CLAIMS 20× without moving the
|
||||
# 0/45 STRICT floor, while cap-on-JSON wins +14pp. Default
|
||||
# allowlist is ["claim_lattice"] (JSON only); empty/None falls
|
||||
# back to all guard_modes.
|
||||
# The full cap fallback chain reads:
|
||||
# 1. quantifier-guard cap (when apply_caps=True AND mode allowed)
|
||||
# 2. claim_lattice_max_claims_per_answer policy field
|
||||
# 3. hard-coded default 12
|
||||
quantifier_apply_caps = bool(policy.get("quantifier_guard_apply_caps", False))
|
||||
quantifier_apply_caps_modes = policy.get(
|
||||
"quantifier_apply_caps_modes",
|
||||
quantifier_guard_modes, # legacy fallback
|
||||
) or quantifier_guard_modes
|
||||
quantifier_caps_mode_gated = answer_mode in (quantifier_apply_caps_modes or [])
|
||||
_policy_max_claims = int(policy.get("claim_lattice_max_claims_per_answer", 12))
|
||||
if quantifier_apply_caps and claim_cap_lookup is not None:
|
||||
if (
|
||||
quantifier_apply_caps
|
||||
and quantifier_caps_mode_gated
|
||||
and claim_cap_lookup is not None
|
||||
):
|
||||
effective_max_claims = int(claim_cap_lookup)
|
||||
else:
|
||||
effective_max_claims = _policy_max_claims
|
||||
|
|
|
|||
|
|
@ -223,14 +223,27 @@ DEFAULT_POLICY = {
|
|||
# stable HYBRID 0.455 on baseline, different failure shape.
|
||||
"quantifier_guard_enabled": True,
|
||||
"quantifier_guard_apply_caps": False,
|
||||
# When apply_caps flips True, this allowlist gates which modes
|
||||
# actually have caps applied. n=5 verification 2026-05-03 (#000008
|
||||
# §12.10): cap on claim_lattice (JSON) wins +14pp on STRICT-rate;
|
||||
# cap on claim_lattice_pointer fires TOO_MANY_CLAIMS 20× without
|
||||
# moving the verdict floor (still 0 STRICT). Default to JSON only
|
||||
# so flipping the master switch doesn't add wasted cap-noise on
|
||||
# pointer mode. Empty list / None = honor quantifier_guard_modes
|
||||
# (legacy fallback).
|
||||
"quantifier_apply_caps_modes": ["claim_lattice"],
|
||||
"quantifier_caps_by_intensity": {},
|
||||
"quantifier_guard_modes": ["claim_lattice_pointer", "claim_lattice"],
|
||||
# Phase 3 — broad-quantifier reminder injection. Default OFF
|
||||
# because Hermes-3-8B already ignores parts of the existing
|
||||
# reminder under enumeration pressure (ticket §3 Option B con).
|
||||
# Operator opts in per-call after Phase 2 dry-run telemetry
|
||||
# confirms which broad-shape rows actually need the reminder.
|
||||
"quantifier_reminder_enabled": False,
|
||||
# Phase 3 — broad-quantifier reminder injection. Default ON for
|
||||
# lattice modes (gated via quantifier_guard_modes) per the
|
||||
# 2026-05-03 bench A/B (#000008 §12). Reminder eliminates
|
||||
# FORMAT_COLLAPSED (2→0), reduces NO_EVIDENCE_POINTER 33%,
|
||||
# boosts mean ratio +17pp on pointer / +21pp on JSON, and
|
||||
# rescues JSON UNGROUNDED 7→1. n=5 verification 2026-05-03
|
||||
# confirms the compound effect with cap survives at higher
|
||||
# sample size. Quote mode is mode-gated off (different failure
|
||||
# shape; paraphrase verifier doesn't need pointer-tag reminders).
|
||||
"quantifier_reminder_enabled": True,
|
||||
# Phase 4 — strict reject for broad-unbounded queries. When True
|
||||
# AND intensity ∈ {ALL, COMPREHENSIVE, OPEN_REQUEST} AND
|
||||
# scope_bound_hint == "unbounded", query()/ask() return UNGROUNDED
|
||||
|
|
@ -336,8 +349,19 @@ def ask(
|
|||
else:
|
||||
claim_cap_lookup = None
|
||||
quantifier_apply_caps = bool(policy.get("quantifier_guard_apply_caps", False))
|
||||
quantifier_apply_caps_modes = policy.get(
|
||||
"quantifier_apply_caps_modes",
|
||||
quantifier_guard_modes, # legacy fallback
|
||||
) or quantifier_guard_modes
|
||||
quantifier_caps_mode_gated = answer_mode_for_guard in (
|
||||
quantifier_apply_caps_modes or []
|
||||
)
|
||||
_policy_max_claims = int(policy.get("claim_lattice_max_claims_per_answer", 12))
|
||||
if quantifier_apply_caps and claim_cap_lookup is not None:
|
||||
if (
|
||||
quantifier_apply_caps
|
||||
and quantifier_caps_mode_gated
|
||||
and claim_cap_lookup is not None
|
||||
):
|
||||
effective_max_claims = int(claim_cap_lookup)
|
||||
else:
|
||||
effective_max_claims = _policy_max_claims
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ Newest first. Update on every open/close.
|
|||
|
||||
| ID | Title | Status | Opened | Directive |
|
||||
|----------|------------------------------------------------|-----------------------|------------|-----------|
|
||||
| #000008 | Broad-quantifier preflight guard | open · phases 0–4 landed; bench A/B in progress | 2026-05-02 | — |
|
||||
| #000008 | Broad-quantifier preflight guard | open · phases 0–4 landed; defaults flipped (Option A); Phase 5 DAG-binding pending | 2026-05-02 | — |
|
||||
| #000007 | Query-layer hyphen folding | closed · 2026-05-02 | 2026-05-02 | — |
|
||||
| #000006 | Bench-emergent findings (rolling research log) | open · rolling | 2026-05-02 | — |
|
||||
| #000005 | Label ladder migration (POINTER-LINKED → …) | closed · 2026-05-02 | 2026-05-01 | D7 |
|
||||
|
|
|
|||
|
|
@ -446,9 +446,14 @@ enhancement, not blocking.
|
|||
|
||||
## 8. Status
|
||||
|
||||
**All four phases landed 2026-05-03. Mechanism complete; defaults
|
||||
preserve the dry-run discipline.** Bench-first measurement of
|
||||
§10.8 decision-tree thresholds is the only remaining work.
|
||||
**All four phases landed 2026-05-03. Mechanism complete. Defaults
|
||||
flipped per §12.11 after the four-cell A/B + n=5 verification.**
|
||||
Pure-Option-A landing: reminder default-on for lattice modes, cap
|
||||
default operator-opt-in with JSON-only allowlist. Pointer-mode
|
||||
cap is filtered out (wasted on the 0/45 STRICT floor). Bench-side
|
||||
mechanism + measurement complete; only Phase 5 (run-DAG node
|
||||
binding) and cross-model Qwen/GPT-4 verification remain as
|
||||
follow-ups.
|
||||
|
||||
Implementation commit chain (all on `origin/main`):
|
||||
|
||||
|
|
@ -1893,7 +1898,8 @@ Per §10.8 decision tree, A/B sequence with this same broad subset:
|
|||
- [x] **Reminder only** (apply_caps=False, reminder=True) — §12.6.
|
||||
- [x] **Cap only** (apply_caps=True, reminder=False) — §12.7.
|
||||
- [x] **Cap + reminder** (apply_caps=True, reminder=True) — §12.8.
|
||||
- [ ] **n≥5 verification** before flipping defaults (§12.9 caveat).
|
||||
- [x] **n=5 verification** of cap+reminder — §12.10.
|
||||
- [x] **Defaults flipped** per §12.11 landing.
|
||||
|
||||
5pp signal floor per `docs/bench-maxing.md` for default-flip
|
||||
decisions. Each cycle adds 81 runs at ~12-17min on Hermes.
|
||||
|
|
@ -2154,3 +2160,105 @@ Either way, the disable hierarchy (§10.11.2) means flipping defaults
|
|||
is reversible per-call, per-mode, per-test. No commitment is
|
||||
permanent — flip the default, watch one bench cycle, revert if
|
||||
regression.
|
||||
|
||||
### 12.10 n=5 verification (2026-05-03T19-55-14Z)
|
||||
|
||||
Same 9-question broad subset, n=5 instead of n=3 (135 runs total).
|
||||
Same policy: `apply_caps=True`, `reminder=True`. Tightens variance
|
||||
on the cap+reminder cell to test whether the §12.8 0.30 vs §12.7
|
||||
0.33 JSON STRICT-rate gap was real or noise.
|
||||
|
||||
| Metric | n=3 | **n=5** |
|
||||
|---------------------------|-----------|------------|
|
||||
| `JSON` STRICT-rate | 0.30 | **0.33** |
|
||||
| `JSON` UNGROUNDED rate | 1/27 (4%) | **2/45 (4%)** |
|
||||
| `JSON` STRICT count | 8 | **15** |
|
||||
| `pointer` STRICT-rate | 0/27 | 0/45 |
|
||||
| `pointer` mean ratio | 0.684 | 0.634 |
|
||||
| `pointer` UNGROUNDED rate | 6/27 (22%)| 10/45 (22%)|
|
||||
|
||||
**Verdict: cap+reminder JSON STRICT-rate at n=5 = 0.33, matching
|
||||
cap-only (n=3 0.33).** The §12.8 0.30 reading was downward Hermes
|
||||
nondeterminism. n=5 confirms the compound effect delivers cap-only's
|
||||
STRICT-rate AND reminder-only's UNGROUNDED-rescue simultaneously.
|
||||
|
||||
**§10.8 strict gate verdict at n=5 — MET on multiple metrics:**
|
||||
|
||||
- vs reminder-only on JSON STRICT-rate: **+11pp** (clears 5pp floor)
|
||||
- vs cap-only on JSON UNGROUNDED rate: **−18pp** (clears floor)
|
||||
- vs reminder-only on pointer mean ratio: −1pp (within noise)
|
||||
- vs cap-only on pointer mean ratio: **+12pp** (clears floor)
|
||||
|
||||
Compound effect is real. But pointer-mode cap is structurally
|
||||
wasted: 0/45 STRICT across all four cells; cap-on-pointer fires
|
||||
TOO_MANY_CLAIMS 20× without verdict gain.
|
||||
|
||||
### 12.11 Final landing — Option A defaults flipped
|
||||
|
||||
The §10.8 strict gate for pure Option D ("cap + reminder default-
|
||||
on for all lattice modes") is met. But the data-driven cleaner
|
||||
landing is **Option A — per-mode tailored**:
|
||||
|
||||
```
|
||||
Lattice modes (claim_lattice + claim_lattice_pointer):
|
||||
quantifier_reminder_enabled True ← FLIPPED 2026-05-03
|
||||
quantifier_guard_apply_caps False ← unchanged (operator opt-in)
|
||||
quantifier_apply_caps_modes ["claim_lattice"] ← NEW field
|
||||
JSON-only allowlist for
|
||||
when operator flips
|
||||
apply_caps=True
|
||||
|
||||
claim_lattice (JSON):
|
||||
effective behavior: reminder ON; cap applies when
|
||||
apply_caps=True (operator-driven)
|
||||
|
||||
claim_lattice_pointer:
|
||||
effective behavior: reminder ON; cap NEVER applied (filtered out
|
||||
by apply_caps_modes allowlist) since cap on
|
||||
pointer is structurally wasted noise.
|
||||
|
||||
quote:
|
||||
effective behavior: unchanged. Mode-gated off via
|
||||
quantifier_guard_modes default.
|
||||
```
|
||||
|
||||
Code changes (commit pending):
|
||||
|
||||
- `aborist/qa/runner.py`: `quantifier_reminder_enabled=True`,
|
||||
new field `quantifier_apply_caps_modes=["claim_lattice"]`.
|
||||
- `aborist/qa/query.py`: same defaults.
|
||||
- `aborist/qa/keys.py`: `quantifier_apply_caps_modes` added to
|
||||
`_VERIFIER_POLICY_FIELDS` so flipping it bumps governance hash.
|
||||
- Cap-application gate now reads:
|
||||
`if apply_caps AND mode in apply_caps_modes AND cap is not None:`
|
||||
- 5 new tests pin: reminder default ON for both
|
||||
`runner.DEFAULT_POLICY` and `query.DEFAULT_QUERY_POLICY`,
|
||||
apply_caps_modes default `["claim_lattice"]`,
|
||||
governance-hash invalidation on apply_caps_modes flip,
|
||||
apply_caps default still False (dry-run discipline preserved).
|
||||
|
||||
**Operator commands after this landing:**
|
||||
|
||||
```
|
||||
# Default behavior (no flag): reminder ON for lattice modes; cap
|
||||
# OFF (operator opt-in only).
|
||||
$ aborist query "winners of all major sports?"
|
||||
|
||||
# Flip cap on for one call (JSON-only by default):
|
||||
$ aborist query --apply-quantifier-caps "..."
|
||||
|
||||
# Override the allowlist to include pointer mode too (Option D):
|
||||
$ aborist query --apply-quantifier-caps \
|
||||
--policy quantifier_apply_caps_modes='["claim_lattice","claim_lattice_pointer"]' "..."
|
||||
|
||||
# Disable the reminder for one call:
|
||||
$ aborist query --policy quantifier_reminder_enabled=false "..."
|
||||
|
||||
# Master kill:
|
||||
$ aborist query --no-quantifier-guard "..."
|
||||
```
|
||||
|
||||
**Six-level disable hierarchy still intact** — no flag forced upon
|
||||
operators; the defaults simply move toward the §10.8-validated
|
||||
position. Reverting the default is a one-line policy edit per
|
||||
§10.11.5 rollback playbook.
|
||||
|
|
|
|||
|
|
@ -219,3 +219,59 @@ def test_default_profile_present():
|
|||
"""Lookup-fallback target must exist or unknown models would
|
||||
return None on every call."""
|
||||
assert "default" in PROFILES
|
||||
|
||||
|
||||
# Ticket #000008 §12.10 — n=5 verification produced these defaults
|
||||
# (committed 2026-05-03 post cap+reminder bench). Pin them here so a
|
||||
# future drift in DEFAULT_QUERY_POLICY surfaces a failing test.
|
||||
|
||||
def test_default_reminder_enabled_for_lattice_modes():
|
||||
"""Phase 3 reminder default-ON per #000008 §12 bench finding:
|
||||
eliminates FORMAT_COLLAPSED, reduces NO_EVIDENCE_POINTER 33%,
|
||||
boosts mean ratio +17pp pointer / +21pp JSON, rescues JSON
|
||||
UNGROUNDED 7→1."""
|
||||
from aborist.qa.runner import DEFAULT_POLICY as RUNNER_POLICY
|
||||
from aborist.qa.query import DEFAULT_QUERY_POLICY
|
||||
assert RUNNER_POLICY["quantifier_reminder_enabled"] is True
|
||||
assert DEFAULT_QUERY_POLICY["quantifier_reminder_enabled"] is True
|
||||
|
||||
|
||||
def test_default_apply_caps_modes_is_json_only():
|
||||
"""Phase 2 cap default-allowlist per #000008 §12.10 n=5 finding:
|
||||
cap-on-JSON wins +14pp on STRICT-rate; cap-on-pointer fires
|
||||
TOO_MANY_CLAIMS 20× without moving the 0/45 STRICT floor.
|
||||
Default the apply-caps allowlist to JSON-only so flipping
|
||||
apply_caps=True doesn't add wasted cap-noise on pointer mode."""
|
||||
from aborist.qa.runner import DEFAULT_POLICY as RUNNER_POLICY
|
||||
from aborist.qa.query import DEFAULT_QUERY_POLICY
|
||||
assert RUNNER_POLICY["quantifier_apply_caps_modes"] == ["claim_lattice"]
|
||||
assert DEFAULT_QUERY_POLICY["quantifier_apply_caps_modes"] == ["claim_lattice"]
|
||||
|
||||
|
||||
def test_apply_caps_modes_is_in_verifier_policy_fields():
|
||||
"""Without governance binding, an operator could change the
|
||||
allowlist and silently re-use cached records written under a
|
||||
different allowlist."""
|
||||
assert "quantifier_apply_caps_modes" in _VERIFIER_POLICY_FIELDS
|
||||
|
||||
|
||||
def test_governance_hash_changes_when_apply_caps_modes_changes():
|
||||
base_policy = dict.fromkeys(_VERIFIER_POLICY_FIELDS, "default")
|
||||
base_policy["quantifier_apply_caps_modes"] = ["claim_lattice"]
|
||||
h_json_only = verifier_policy_hash(base_policy)
|
||||
base_policy["quantifier_apply_caps_modes"] = [
|
||||
"claim_lattice", "claim_lattice_pointer"
|
||||
]
|
||||
h_both = verifier_policy_hash(base_policy)
|
||||
assert h_json_only != h_both
|
||||
|
||||
|
||||
def test_apply_caps_default_off_preserves_dry_run_discipline():
|
||||
"""The cap-application gate stays operator-opt-in by default
|
||||
even with reminder default-on. Dry-run discipline (§10.11.3)
|
||||
survives the §12 bench cycle — operators flip apply_caps via
|
||||
--apply-quantifier-caps after their own bench review."""
|
||||
from aborist.qa.runner import DEFAULT_POLICY as RUNNER_POLICY
|
||||
from aborist.qa.query import DEFAULT_QUERY_POLICY
|
||||
assert RUNNER_POLICY["quantifier_guard_apply_caps"] is False
|
||||
assert DEFAULT_QUERY_POLICY["quantifier_guard_apply_caps"] is False
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue