russell@unturf.com
6f90f21d1a
Lands aborist/qa/quantifier_reminder.py with broad_quantifier_
reminder(): one-line user-turn message restating the cap and the
[E\d+] citation rule for broad-intensity questions. Two templates:
bounded universe:
"This is a broad-quantifier query with a bounded universe.
Return at most N pointer-linked claim lines. Each claim must
cite an evidence id like [E5]; do not write claim lines
without bracket citations."
unbounded universe:
"This is a broad-quantifier query with an under-specified scope.
Return at most N pointer-linked claim lines. If you cannot
ground N claims with evidence IDs, return fewer grounded
claims. Do not enumerate from training prior. Each claim must
cite an evidence id like [E5]; do not write claim lines
without bracket citations."
The bounded template skips the "do not enumerate from training
prior" clause — the corpus has the answer set. Unknown scope falls
through to the stricter unbounded template (over-warn rather than
under-warn).
Wired into both query() and runner.ask() at the same insertion
point as the existing grounding_reminder — between
grounding_reminder and the evidence/question payload, where
Hermes-3-8B's most-recent-token attention catches it.
Default OFF (`quantifier_reminder_enabled: false`). Empirical
justification: ticket §3 Option B con notes Hermes already ignores
parts of the existing reminder under enumeration pressure. The
mechanism lands so an operator can A/B test cap-only vs cap+reminder
without code changes; default flips on after bench shows ≥5pp delta
on FORMAT_COLLAPSED or pointer-loss rate per §10.8.
`quantifier_reminder_enabled` folded into _VERIFIER_POLICY_FIELDS
so flipping the switch invalidates prior cache records.
19 new tests cover: gating (non-broad → None, missing cap → None,
None intensity → None), bounded-vs-unbounded template selection,
unknown scope falls back to unbounded, cap interpolation,
[E\d+] citation rule restatement, governance-hash invalidation.
139 lines
5 KiB
Python
139 lines
5 KiB
Python
"""Broad-quantifier reminder text + governance hash binding.
|
|
|
|
Ticket #000008 Phase 3. Default-OFF mechanism — operator opts in
|
|
per-call via ``quantifier_reminder_enabled=True`` in policy. Tests
|
|
pin the reminder generator's behavior and the policy field's
|
|
inclusion in ``_VERIFIER_POLICY_FIELDS`` so flipping it bumps
|
|
``governance_policy_hash``.
|
|
|
|
Bench-first per §10.8 decision tree: enable default-on only after
|
|
a bench cycle shows ≥5pp delta on FORMAT_COLLAPSED or pointer-loss
|
|
rate.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from aborist.qa.keys import _VERIFIER_POLICY_FIELDS, verifier_policy_hash
|
|
from aborist.qa.quantifier_reminder import broad_quantifier_reminder
|
|
|
|
|
|
# ---------------------------------------------------------------- gating
|
|
|
|
@pytest.mark.parametrize("intensity", [
|
|
"SINGULAR", "PROPORTIONAL", "FEW", "MANY",
|
|
"SMALL_NUM_EXPLICIT", "COMPARATIVE_BOUND", "ABSENT",
|
|
])
|
|
def test_non_broad_intensity_returns_none(intensity):
|
|
"""Reminder only fires for ALL/COMPREHENSIVE/OPEN_REQUEST."""
|
|
out = broad_quantifier_reminder(
|
|
intensity=intensity, cap=8, scope_bound_hint="unbounded",
|
|
)
|
|
assert out is None
|
|
|
|
|
|
@pytest.mark.parametrize("intensity", ["ALL", "COMPREHENSIVE", "OPEN_REQUEST"])
|
|
def test_broad_intensity_with_cap_produces_reminder(intensity):
|
|
out = broad_quantifier_reminder(
|
|
intensity=intensity, cap=8, scope_bound_hint="unbounded",
|
|
)
|
|
assert out is not None
|
|
assert "8" in out
|
|
assert "broad-quantifier query" in out
|
|
|
|
|
|
def test_none_cap_returns_none():
|
|
"""Defensive — guard upstream may fail to extract a cap; reminder
|
|
declines rather than render an ill-formed 'at most None'."""
|
|
out = broad_quantifier_reminder(
|
|
intensity="ALL", cap=None, scope_bound_hint="unbounded",
|
|
)
|
|
assert out is None
|
|
|
|
|
|
def test_zero_or_negative_cap_returns_none():
|
|
"""Same defensive — non-positive caps shouldn't render."""
|
|
assert broad_quantifier_reminder(
|
|
intensity="ALL", cap=0, scope_bound_hint="unbounded",
|
|
) is None
|
|
assert broad_quantifier_reminder(
|
|
intensity="ALL", cap=-1, scope_bound_hint="unbounded",
|
|
) is None
|
|
|
|
|
|
def test_none_intensity_returns_none():
|
|
out = broad_quantifier_reminder(
|
|
intensity=None, cap=8, scope_bound_hint="unbounded",
|
|
)
|
|
assert out is None
|
|
|
|
|
|
# ---------------------------------------------------------------- bounded vs unbounded
|
|
|
|
def test_bounded_universe_uses_bounded_template():
|
|
"""Bounded universals (e.g. all members of the Beatles, year-
|
|
anchored) get a softer reminder — no 'do not enumerate from
|
|
training prior' clause, since the corpus has the answer set."""
|
|
bounded = broad_quantifier_reminder(
|
|
intensity="ALL", cap=4, scope_bound_hint="bounded",
|
|
)
|
|
unbounded = broad_quantifier_reminder(
|
|
intensity="ALL", cap=4, scope_bound_hint="unbounded",
|
|
)
|
|
assert "bounded universe" in bounded
|
|
assert "training prior" not in bounded
|
|
# Unbounded reminder is the stricter shape with the no-prior
|
|
# clause and the under-specified-scope warning.
|
|
assert "training prior" in unbounded
|
|
assert "under-specified" in unbounded
|
|
|
|
|
|
def test_unknown_scope_treated_as_unbounded():
|
|
"""When the classifier can't decide bounded vs unbounded, default
|
|
to the stricter (unbounded) reminder. Better to over-warn than
|
|
under-warn on a broad-intensity question."""
|
|
out = broad_quantifier_reminder(
|
|
intensity="ALL", cap=8, scope_bound_hint="unknown",
|
|
)
|
|
assert "training prior" in out
|
|
|
|
|
|
# ---------------------------------------------------------------- cap interpolation
|
|
|
|
def test_cap_appears_verbatim_in_reminder():
|
|
"""The cap N must appear in the reminder text so the model knows
|
|
its budget. Pin the literal cap string so a typo in the template
|
|
surfaces here."""
|
|
for cap in (1, 5, 8, 12, 50):
|
|
out = broad_quantifier_reminder(
|
|
intensity="ALL", cap=cap, scope_bound_hint="unbounded",
|
|
)
|
|
assert str(cap) in out
|
|
|
|
|
|
def test_reminder_demands_bracket_citations():
|
|
"""The reminder must restate the [E\\d+] citation rule so the
|
|
model doesn't drop pointer tags under enumeration pressure."""
|
|
out = broad_quantifier_reminder(
|
|
intensity="ALL", cap=8, scope_bound_hint="unbounded",
|
|
)
|
|
assert "[E5]" in out # template uses [E5] as the example
|
|
|
|
|
|
# ---------------------------------------------------------------- governance binding
|
|
|
|
def test_quantifier_reminder_enabled_is_in_verifier_policy_fields():
|
|
"""Flipping the reminder switch must invalidate prior cache
|
|
records — without this binding an operator could enable the
|
|
reminder and silently re-use cached pre-reminder verdicts."""
|
|
assert "quantifier_reminder_enabled" in _VERIFIER_POLICY_FIELDS
|
|
|
|
|
|
def test_governance_hash_changes_when_reminder_flips():
|
|
base_policy = dict.fromkeys(_VERIFIER_POLICY_FIELDS, "default")
|
|
base_policy["quantifier_reminder_enabled"] = False
|
|
h_off = verifier_policy_hash(base_policy)
|
|
base_policy["quantifier_reminder_enabled"] = True
|
|
h_on = verifier_policy_hash(base_policy)
|
|
assert h_off != h_on
|