From 84d5b5cd7671497b563d8abc350907b4f7abcd89 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sun, 3 May 2026 07:30:28 -0400 Subject: [PATCH] =?UTF-8?q?qa(#000008):=20Phase=202=20=E2=80=94=20model-pr?= =?UTF-8?q?ofile=20caps=20+=20governance=20hash=20(dry-run)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lands aborist/qa/model_profiles.py with two profiles: - adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic tight caps for broad intensities (ALL=8, COMPREHENSIVE=5, OPEN_REQUEST=5) reflecting the 2026-05-02 runaway case. - default large-reasoner-class fallback (ALL=12, COMPREHENSIVE=15, OPEN_REQUEST=12). Used when model_profile_id has no entry. cap_for_intensity() resolves caps via three-source chain: 1. policy_overrides (per-call dict, highest priority) 2. per-model profile from PROFILES 3. "default" profile fallback EXPLICIT_COUNT sentinel handles SMALL_NUM_EXPLICIT and COMPARATIVE_BOUND — cap is the question's explicit count, not a profile-set value. Defensive fallback to MANY cap if classifier fired the rung without extracting a count. Four new policy fields, all folded into governance_policy_hash via _VERIFIER_POLICY_FIELDS: - quantifier_guard_enabled master kill (default True) - quantifier_guard_apply_caps dry-run gate (default False per §10.11.3 — cap LOOKED UP and reported on result, but NOT applied to the verifier until operator flips True) - quantifier_caps_by_intensity per-call override dict - quantifier_guard_modes per-mode opt-in list (default ["claim_lattice_pointer", "claim_lattice"]; quote opts out) Six-level disable hierarchy (§10.11.2) implemented: - Per-test: policy={"quantifier_guard_enabled": False} - Per-call: --no-quantifier-guard (Phase 4) - Per-phase: each policy switch is independent - Per-mode: quantifier_guard_modes filter - Per-model: model_profiles.py lookup - Master: governance_policy_hash invalidation on flip Wired through both query() and runner.ask() — both compute effective_max_claims from the (classifier_intensity, model_profile, policy_overrides) triple and pass it as max_claims_per_answer to the verifier. Dry-run mode keeps effective_max_claims at the policy default (12) until apply_caps flips True. Result dict surfaces claim_cap_applied (the LOOKED-UP cap, even in dry-run) plus all Phase-1 quantifier fields on miss-path AND cache-hit path so bench rows stay column-aligned. 19 new tests pin: per-model selection, EXPLICIT_COUNT sentinel, override precedence, governance-hash invalidation on every cap field, profile shape (all ten rungs covered), default profile presence. --- aborist/qa/keys.py | 12 ++ aborist/qa/model_profiles.py | 132 ++++++++++++++++++++ aborist/qa/query.py | 96 ++++++++++++--- aborist/qa/runner.py | 88 +++++++++++++- tests/test_quantifier_caps.py | 221 ++++++++++++++++++++++++++++++++++ 5 files changed, 524 insertions(+), 25 deletions(-) create mode 100644 aborist/qa/model_profiles.py create mode 100644 tests/test_quantifier_caps.py diff --git a/aborist/qa/keys.py b/aborist/qa/keys.py index 4c8e212..608fef0 100644 --- a/aborist/qa/keys.py +++ b/aborist/qa/keys.py @@ -215,6 +215,18 @@ _VERIFIER_POLICY_FIELDS = frozenset({ # 2026-05-02b, Rule 9). Threshold of question∩claim content tokens # absent from cited evidence union that demotes STRICT → HYBRID. "claim_lattice_subject_tokens_absent_threshold", + # Ticket #000008 Phase 2 — quantifier preflight guard. The four + # fields together control whether broad-quantifier classification + # affects the per-call claim cap and which modes are gated. + # Adding a model to model_profiles.py PROFILES doesn't bump + # governance_policy_hash on its own (the dict isn't policy); + # but flipping any of these knobs DOES bump the hash, which + # invalidates prior cache records on lookup — exactly the + # invalidation we want when a guard knob changes. + "quantifier_guard_enabled", + "quantifier_guard_apply_caps", + "quantifier_caps_by_intensity", + "quantifier_guard_modes", # Quote-mode entity policy "entity_policy", "entity_proximity_n", diff --git a/aborist/qa/model_profiles.py b/aborist/qa/model_profiles.py new file mode 100644 index 0000000..e9ebbe3 --- /dev/null +++ b/aborist/qa/model_profiles.py @@ -0,0 +1,132 @@ +"""Per-model quantifier-cap profiles — Ticket #000008 Phase 2. + +PROMETHEUS-Σ owns the mapping ``model_profile_id → quantifier_intensity +→ claim_cap``. Each profile is a static dict; profile selection is by +exact match against ``model_profile_id`` (the configured chat-completion +model id, e.g. ``adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic``). Falls +back to ``"default"`` profile when no entry matches. + +Folds into ``governance_policy_hash`` via two policy fields surfaced in +``aborist.qa.runner.DEFAULT_POLICY`` / +``aborist.qa.query.DEFAULT_QUERY_POLICY``: + + quantifier_caps_by_intensity per-call cap dict (overrides the + profile when set on the policy) + quantifier_guard_apply_caps bool — Phase 2 dry-run gate + (default False through rollout) + +The profile table lives here; the *application* of the cap lives in the +runner (`aborist/qa/runner.py:ask`) and `aborist/qa/query.py:query`. +This file is pure data + a lookup helper. + +Per ticket §10.11.3 dry-run discipline: Phase 2 lands the cap-table and +the lookup wiring with ``quantifier_guard_apply_caps=False``. The +``claim_cap_applied`` field on the query result reports the cap that +WOULD have been applied — letting bench measure classifier+profile +behavior across the full question set before any verdict moves. + +Once dry-run telemetry is reviewed and the classifier is calibrated +(per §10.11.3 step 3), an operator flips ``quantifier_guard_apply_caps`` +to True for the targeted broad-quantifier subset. +""" + +from __future__ import annotations + +from typing import Mapping + +# Symbolic count for SMALL_NUM_EXPLICIT — the cap is the explicit +# count from the question, not a profile-set value. Resolved at +# lookup time (`cap_for_intensity` reads `explicit_count` when it +# sees this sentinel). +EXPLICIT_COUNT = "explicit_count" + + +# Per-model intensity → cap. Keys are model ids exactly as returned +# by the configured chat client (case-sensitive). Add new entries as +# we calibrate caps for new models. + +PROFILES: Mapping[str, Mapping[str, int | str]] = { + # Hermes-3 Llama-3.1-8B-FP8-Dynamic — the live production endpoint + # (hermes.ai.unturf.com). Tight caps reflect the model's broad- + # quantifier weakness surfaced 2026-05-02 by the + # "winners of all major sports?" case (51-claim runaway). + "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic": { + "ABSENT": 1, + "SINGULAR": 1, + "PROPORTIONAL": 1, + "SMALL_NUM_EXPLICIT": EXPLICIT_COUNT, + "COMPARATIVE_BOUND": EXPLICIT_COUNT, + "FEW": 5, + "MANY": 8, + "ALL": 8, + "COMPREHENSIVE": 5, + "OPEN_REQUEST": 5, + }, + # Default — large-reasoner-class models (Qwen 3 reasoner / GPT-4 / + # Claude / Gemini). Looser caps because format discipline holds + # under broad-quantifier pressure on these models. Used as the + # fallback when model_profile_id has no explicit entry. + "default": { + "ABSENT": 1, + "SINGULAR": 1, + "PROPORTIONAL": 3, + "SMALL_NUM_EXPLICIT": EXPLICIT_COUNT, + "COMPARATIVE_BOUND": EXPLICIT_COUNT, + "FEW": 5, + "MANY": 12, + "ALL": 12, + "COMPREHENSIVE": 15, + "OPEN_REQUEST": 12, + }, +} + + +def cap_for_intensity( + *, + model_profile_id: str | None, + intensity: str, + explicit_count: int | None = None, + policy_overrides: Mapping[str, int | str] | None = None, +) -> int | None: + """Return the claim cap for ``(model_profile_id, intensity)``. + + Resolution order: + 1. ``policy_overrides`` (per-call dict) wins if present and + contains the intensity. Lets the operator override the + table without touching this file. + 2. Per-model profile entry from ``PROFILES``. + 3. ``"default"`` profile fallback. + + The ``EXPLICIT_COUNT`` sentinel resolves to ``explicit_count`` + when it's an int, or the default-profile MANY cap (12) when no + explicit_count was extracted (defensive — should not happen for + SMALL_NUM_EXPLICIT / COMPARATIVE_BOUND but covers regressions). + + Returns None when the intensity is not in any profile (caller + should treat None as "no cap; use the policy default"). + """ + sources: list[Mapping[str, int | str]] = [] + if policy_overrides: + sources.append(policy_overrides) + if model_profile_id and model_profile_id in PROFILES: + sources.append(PROFILES[model_profile_id]) + sources.append(PROFILES["default"]) + + for src in sources: + if intensity in src: + cap = src[intensity] + if cap == EXPLICIT_COUNT: + if explicit_count is not None and explicit_count > 0: + return int(explicit_count) + # Defensive fallback — no explicit count parsed. + # Use the MANY cap from the same profile as a + # sensible upper bound. + return int(src.get("MANY", 12)) + return int(cap) + return None + + +def profile_id_present(model_profile_id: str | None) -> bool: + """Return True iff ``model_profile_id`` has its own entry in + PROFILES (i.e. won't fall through to the default profile).""" + return bool(model_profile_id) and model_profile_id in PROFILES diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 9a82b04..0cb2ea5 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -456,6 +456,14 @@ DEFAULT_QUERY_POLICY = { # WARRANTED via the existing soft-demote ladder path. "claim_lattice_deflection_check_enabled": True, "claim_lattice_format_collapse_check_enabled": True, + # Ticket #000008 Phase 2 — quantifier preflight guard. See + # aborist/qa/runner.py:DEFAULT_POLICY for the full rationale. + # Phase 2 lands with apply_caps=False (dry-run); cap is + # reported on result dict but not applied to the verifier. + "quantifier_guard_enabled": True, + "quantifier_guard_apply_caps": False, + "quantifier_caps_by_intensity": {}, + "quantifier_guard_modes": ["claim_lattice_pointer", "claim_lattice"], # Claim-count ceiling — see runner.DEFAULT_POLICY for rationale. # Bench finding (york-england "tell me all there is to know") # caught the runaway shape; cap of 12 admits entity-list @@ -1602,15 +1610,61 @@ def query( max_context_chars = int( by_mode.get(answer_mode, policy.get("max_context_chars", 60000)) ) - # Quantifier preflight (Ticket #000008 Phase 1). Pure lexical - # classification of the question into one of ten intensity rungs. - # Phase 1 lands in dry-run mode: the result is recorded on the - # query result dict (so bench rows pick it up) but no policy - # behavior changes — caps are NOT applied yet. Phase 2 wires the - # cap-application gate (`quantifier_guard_apply_caps`, default - # False during rollout per §10.11.3 dry-run discipline). + # Quantifier preflight (Ticket #000008 Phase 1+2). Phase 1 runs + # the lexical classifier; Phase 2 looks up the per-model cap. + # The cap is REPORTED on the result dict (claim_cap_applied) but + # only applied to the verifier when quantifier_guard_apply_caps + # is True (default False through dry-run rollout per §10.11.3). + # Six-level disable hierarchy gates each step: master + # (quantifier_guard_enabled), per-mode (quantifier_guard_modes), + # per-call (quantifier_caps_by_intensity overrides), per-test + # (policy={"quantifier_guard_enabled": False}). + from aborist.qa.model_profiles import cap_for_intensity from aborist.qa.quantifier import classify_question_quantifier - quantifier = classify_question_quantifier(question) + quantifier_guard_on = bool(policy.get("quantifier_guard_enabled", True)) + quantifier_guard_modes = policy.get( + "quantifier_guard_modes", + ["claim_lattice_pointer", "claim_lattice"], + ) + quantifier_mode_gated = answer_mode in (quantifier_guard_modes or []) + if quantifier_guard_on: + quantifier = classify_question_quantifier(question) + else: + # Master kill — emit a stub so the result schema stays + # consistent. Bench rows can still distinguish "guard off" + # from "no classification" via quantifier_intensity=None. + quantifier = { + "intensity": None, + "matched_token": None, + "explicit_count": None, + "is_broad": False, + "operational_shape": None, + "scope_bound_hint": "unknown", + "classifier_version": None, + } + if quantifier_guard_on and quantifier_mode_gated and quantifier["intensity"]: + claim_cap_lookup = cap_for_intensity( + model_profile_id=model_id, + intensity=quantifier["intensity"], + explicit_count=quantifier["explicit_count"], + policy_overrides=policy.get("quantifier_caps_by_intensity") or None, + ) + else: + claim_cap_lookup = None + # Effective cap that the verifier will see. Dry-run mode + # (apply_caps=False) preserves the policy default; once an + # operator flips apply_caps=True, the looked-up cap shadows the + # default for this call only — no policy mutation, no schema + # bump. The full cap fallback chain reads: + # 1. quantifier-guard cap (when apply_caps=True) + # 2. claim_lattice_max_claims_per_answer policy field + # 3. hard-coded default 12 + quantifier_apply_caps = bool(policy.get("quantifier_guard_apply_caps", False)) + _policy_max_claims = int(policy.get("claim_lattice_max_claims_per_answer", 12)) + if quantifier_apply_caps and claim_cap_lookup is not None: + effective_max_claims = int(claim_cap_lookup) + else: + effective_max_claims = _policy_max_claims t_start = time.monotonic() # 1. Search. @@ -2119,7 +2173,7 @@ def query( "quantifier_matched_token": quantifier["matched_token"], "scope_bound_hint": quantifier["scope_bound_hint"], "quantifier_explicit_count": quantifier["explicit_count"], - "claim_cap_applied": None, + "claim_cap_applied": claim_cap_lookup, "prompt_chars": prompt_chars, "answer_chars": len(cached["answer_text"] or ""), "timings": { @@ -2201,9 +2255,7 @@ def query( lazy_anchor_demote_min_pairs=int(policy.get( "claim_lattice_lazy_anchor_demote_min_pairs", 3 )), - max_claims_per_answer=int(policy.get( - "claim_lattice_max_claims_per_answer", 12 - )), + max_claims_per_answer=effective_max_claims, subject_tokens_absent_threshold=int(policy.get( "claim_lattice_subject_tokens_absent_threshold", 3 )), @@ -2241,9 +2293,7 @@ def query( min_citation_coverage=float(policy.get( "claim_lattice_min_citation_coverage", 0.30 )), - max_claims_per_answer=int(policy.get( - "claim_lattice_max_claims_per_answer", 12 - )), + max_claims_per_answer=effective_max_claims, subject_tokens_absent_threshold=int(policy.get( "claim_lattice_subject_tokens_absent_threshold", 3 )), @@ -2621,10 +2671,18 @@ def query( "quantifier_matched_token": quantifier["matched_token"], "scope_bound_hint": quantifier["scope_bound_hint"], "quantifier_explicit_count": quantifier["explicit_count"], - # Cap that was applied for this run. None until Phase 2 - # wires the cap-application gate; once Phase 2 lands this - # carries the per-call cap from policy + model_profile. - "claim_cap_applied": None, + # Cap that was looked up for this run. NULL semantics: + # None → guard off, mode opted out, or classifier + # returned no intensity (no cap recorded). + # int → cap that WOULD apply to the verifier when + # quantifier_guard_apply_caps=True. Reported on + # dry-run rows so bench can chart cap distribution + # before the gate flips. + # Phase 2 dry-run: cap is REPORTED but not APPLIED to the + # verifier — claim_lattice_max_claims_per_answer (default + # 12) still drives TOO_MANY_CLAIMS demote until + # quantifier_guard_apply_caps flips True. + "claim_cap_applied": claim_cap_lookup, # Sidecar smell signals (claim_lattice mode only) — surfaced # for the renderer; never persisted in providence_cache and # never threaded into run_dag_root. diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index 739934d..15601d0 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -200,6 +200,31 @@ DEFAULT_POLICY = { # of all major sports?" case where Hermes dumped 50+ free-form # sentences. "claim_lattice_format_collapse_check_enabled": True, + # Quantifier preflight guard (Ticket #000008 Phase 2). Per-call + # claim cap derived from the question's quantifier intensity and + # the configured model profile (aborist/qa/model_profiles.py). + # Phase 2 lands the lookup wiring with apply_caps=False per + # §10.11.3 dry-run discipline — claim_cap_applied is computed + # and reported on the result dict, but the verifier still uses + # claim_lattice_max_claims_per_answer as the actual cap. + # Operator flips quantifier_guard_apply_caps=True after dry-run + # bench review confirms classifier output across the full + # question set. + # + # Six-level disable hierarchy (§10.11.2): + # - quantifier_guard_enabled: master kill (False = no + # classifier output, no cap lookup, no telemetry). + # - quantifier_guard_apply_caps: dry-run gate (True = cap + # applied; False = cap reported but not applied). + # - quantifier_caps_by_intensity: per-call override dict; + # wins over the model_profiles.py table when present. + # - quantifier_guard_modes: list of answer_modes the guard + # applies to. Quote mode opts out by default — already + # stable HYBRID 0.455 on baseline, different failure shape. + "quantifier_guard_enabled": True, + "quantifier_guard_apply_caps": False, + "quantifier_caps_by_intensity": {}, + "quantifier_guard_modes": ["claim_lattice_pointer", "claim_lattice"], # Claim-count ceiling. Bench finding (2026-04-30 york-england): # "tell me all there is to know about X" prompted Hermes to spam # 26-59 encyclopedic claims sourced from training, only 2-4 of @@ -262,6 +287,45 @@ def ask( raise ValueError( f"fidelity must be one of {FIDELITY_MODES}, got {fidelity!r}" ) + # Quantifier preflight (Ticket #000008 Phase 1+2). Same wiring + # as query() — see aborist/qa/query.py for the rationale and + # disable hierarchy. + from aborist.qa.model_profiles import cap_for_intensity + from aborist.qa.quantifier import classify_question_quantifier + answer_mode_for_guard = policy.get("answer_mode", "quote") + quantifier_guard_on = bool(policy.get("quantifier_guard_enabled", True)) + quantifier_guard_modes = policy.get( + "quantifier_guard_modes", + ["claim_lattice_pointer", "claim_lattice"], + ) + quantifier_mode_gated = answer_mode_for_guard in (quantifier_guard_modes or []) + if quantifier_guard_on: + quantifier = classify_question_quantifier(question) + else: + quantifier = { + "intensity": None, + "matched_token": None, + "explicit_count": None, + "is_broad": False, + "operational_shape": None, + "scope_bound_hint": "unknown", + "classifier_version": None, + } + if quantifier_guard_on and quantifier_mode_gated and quantifier["intensity"]: + claim_cap_lookup = cap_for_intensity( + model_profile_id=model_id, + intensity=quantifier["intensity"], + explicit_count=quantifier["explicit_count"], + policy_overrides=policy.get("quantifier_caps_by_intensity") or None, + ) + else: + claim_cap_lookup = None + quantifier_apply_caps = bool(policy.get("quantifier_guard_apply_caps", False)) + _policy_max_claims = int(policy.get("claim_lattice_max_claims_per_answer", 12)) + if quantifier_apply_caps and claim_cap_lookup is not None: + effective_max_claims = int(claim_cap_lookup) + else: + effective_max_claims = _policy_max_claims t_start = time.monotonic() doc = conn.execute( @@ -463,6 +527,14 @@ def ask( else [] ), "partially_verified_quotes": [], + # Quantifier preflight (Ticket #000008 Phase 1+2). Pure + # on the question string, so cache hits re-classify + # cheaply and carry the same schema as miss-path rows. + "quantifier_intensity": quantifier["intensity"], + "quantifier_matched_token": quantifier["matched_token"], + "scope_bound_hint": quantifier["scope_bound_hint"], + "quantifier_explicit_count": quantifier["explicit_count"], + "claim_cap_applied": claim_cap_lookup, "timings": { "cache_lookup_ms": cache_lookup_ms, "llm_ms": None, @@ -537,9 +609,7 @@ def ask( lazy_anchor_demote_min_pairs=int(policy.get( "claim_lattice_lazy_anchor_demote_min_pairs", 3 )), - max_claims_per_answer=int(policy.get( - "claim_lattice_max_claims_per_answer", 12 - )), + max_claims_per_answer=effective_max_claims, subject_tokens_absent_threshold=int(policy.get( "claim_lattice_subject_tokens_absent_threshold", 3 )), @@ -581,9 +651,7 @@ def ask( min_citation_coverage=float(policy.get( "claim_lattice_min_citation_coverage", 0.30 )), - max_claims_per_answer=int(policy.get( - "claim_lattice_max_claims_per_answer", 12 - )), + max_claims_per_answer=effective_max_claims, subject_tokens_absent_threshold=int(policy.get( "claim_lattice_subject_tokens_absent_threshold", 3 )), @@ -822,6 +890,14 @@ def ask( "verifier_method": verdict["verifier_method"], "unverified_quotes": verdict["unverified_quotes"], "partially_verified_quotes": verdict.get("partially_verified_quotes") or [], + # Quantifier preflight (Ticket #000008 Phase 1+2). See query.py + # for full rationale; runner.ask carries the same schema for + # CLI-side `aborist ask` parity with `aborist query`. + "quantifier_intensity": quantifier["intensity"], + "quantifier_matched_token": quantifier["matched_token"], + "scope_bound_hint": quantifier["scope_bound_hint"], + "quantifier_explicit_count": quantifier["explicit_count"], + "claim_cap_applied": claim_cap_lookup, # Sidecar smell signals (claim_lattice mode only) — render- # layer; never persisted, never in run_dag_root. "pointer_id_distribution": verdict.get("pointer_id_distribution"), diff --git a/tests/test_quantifier_caps.py b/tests/test_quantifier_caps.py new file mode 100644 index 0000000..bb6297c --- /dev/null +++ b/tests/test_quantifier_caps.py @@ -0,0 +1,221 @@ +"""Quantifier cap table + governance hash + dry-run discipline. + +Ticket #000008 Phase 2: per-model cap profiles +(``aborist/qa/model_profiles.py``) and the four policy fields that +gate cap behavior: + + - ``quantifier_guard_enabled`` — master kill + - ``quantifier_guard_apply_caps`` — dry-run gate (default False) + - ``quantifier_caps_by_intensity`` — per-call override dict + - ``quantifier_guard_modes`` — per-mode opt-in list + +Tests pin: + 1. Per-model cap selection (Hermes profile vs default fallback). + 2. EXPLICIT_COUNT sentinel — SMALL_NUM_EXPLICIT respects the + question's count. + 3. Policy override beats per-model profile. + 4. Governance-hash invalidation on every cap-related field flip. + 5. Six-level disable hierarchy (master, dry-run, per-mode, + per-call override). +""" + +from __future__ import annotations + +import pytest + +from aborist.qa.keys import _VERIFIER_POLICY_FIELDS, verifier_policy_hash +from aborist.qa.model_profiles import ( + EXPLICIT_COUNT, + PROFILES, + cap_for_intensity, + profile_id_present, +) + + +# ---------------------------------------------------------------- profile lookup + +def test_hermes_fp8_profile_present(): + """The Hermes-3-FP8 endpoint profile must exist verbatim — the + string is what the bench harness sends as ``model_profile_id``.""" + assert profile_id_present("adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic") + + +def test_default_profile_fallback(): + """Unknown model ids fall through to the ``default`` profile.""" + assert not profile_id_present("some-future-model-v2") + cap = cap_for_intensity( + model_profile_id="some-future-model-v2", + intensity="ALL", + ) + # Default profile sets ALL = 12. + assert cap == 12 + + +def test_hermes_caps_are_tighter_than_default(): + """Hermes broad-quantifier caps must be tighter than the default + profile — that's the whole point of the per-model split. Pin the + relationship so a future profile edit can't accidentally loosen + Hermes back to default.""" + hermes_id = "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic" + for intensity in ("ALL", "COMPREHENSIVE", "OPEN_REQUEST", "MANY"): + hermes_cap = cap_for_intensity( + model_profile_id=hermes_id, intensity=intensity, + ) + default_cap = cap_for_intensity( + model_profile_id="unknown", intensity=intensity, + ) + assert hermes_cap is not None + assert default_cap is not None + assert hermes_cap <= default_cap, ( + f"Hermes {intensity} cap ({hermes_cap}) must not exceed " + f"default cap ({default_cap})" + ) + + +# ---------------------------------------------------------------- EXPLICIT_COUNT + +def test_explicit_count_resolves_from_question_count(): + """SMALL_NUM_EXPLICIT cap = the explicit count from the question.""" + cap = cap_for_intensity( + model_profile_id="adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", + intensity="SMALL_NUM_EXPLICIT", + explicit_count=3, + ) + assert cap == 3 + + +def test_explicit_count_falls_back_when_missing(): + """If the classifier reported SMALL_NUM_EXPLICIT but didn't + extract a count (defensive — shouldn't happen), fall back to the + profile's MANY cap rather than crash.""" + cap = cap_for_intensity( + model_profile_id="adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", + intensity="SMALL_NUM_EXPLICIT", + explicit_count=None, + ) + # Hermes MANY = 8. + assert cap == 8 + + +def test_comparative_bound_uses_explicit_count(): + """COMPARATIVE_BOUND also reads explicit_count (the upper bound + extracted by the classifier — for `between A and B` the upper).""" + cap = cap_for_intensity( + model_profile_id="adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", + intensity="COMPARATIVE_BOUND", + explicit_count=7, + ) + assert cap == 7 + + +# ---------------------------------------------------------------- override + +def test_policy_override_beats_per_model_profile(): + """A per-call ``quantifier_caps_by_intensity`` dict wins over the + profile table. Lets an operator override caps without editing + model_profiles.py.""" + cap = cap_for_intensity( + model_profile_id="adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", + intensity="ALL", + policy_overrides={"ALL": 4}, + ) + assert cap == 4 + + +def test_policy_override_partial_falls_back_to_profile(): + """An override dict with one intensity doesn't suppress the + others — the profile still answers for unspecified intensities.""" + cap_all = cap_for_intensity( + model_profile_id="adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", + intensity="ALL", + policy_overrides={"COMPREHENSIVE": 3}, # only COMPREHENSIVE overridden + ) + # ALL still pulls from Hermes profile (8). + assert cap_all == 8 + + +def test_unknown_intensity_returns_none(): + """Caller should treat None as 'no cap; use the policy default'.""" + cap = cap_for_intensity( + model_profile_id="adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", + intensity="MADE_UP_INTENSITY", + ) + assert cap is None + + +# ---------------------------------------------------------------- governance hash + +@pytest.mark.parametrize("field", [ + "quantifier_guard_enabled", + "quantifier_guard_apply_caps", + "quantifier_caps_by_intensity", + "quantifier_guard_modes", +]) +def test_each_quantifier_field_is_in_verifier_policy_fields(field): + """All four Phase-2 policy fields must be in + ``_VERIFIER_POLICY_FIELDS`` so flipping any of them invalidates + prior cache records on lookup. Without this binding, an operator + could change caps and silently re-use stale cached verdicts.""" + assert field in _VERIFIER_POLICY_FIELDS + + +def test_governance_hash_changes_when_apply_caps_flips(): + """Dry-run → cap-on flip MUST bump governance_policy_hash. Same + classifier output, different verifier behavior — distinct + cache_key per §10.11.2 Level 6.""" + base_policy = dict.fromkeys(_VERIFIER_POLICY_FIELDS, "default") + base_policy["quantifier_guard_apply_caps"] = False + h_dry = verifier_policy_hash(base_policy) + base_policy["quantifier_guard_apply_caps"] = True + h_apply = verifier_policy_hash(base_policy) + assert h_dry != h_apply + + +def test_governance_hash_changes_when_caps_dict_changes(): + """Editing the per-call override dict must bump the hash too.""" + base_policy = dict.fromkeys(_VERIFIER_POLICY_FIELDS, "default") + base_policy["quantifier_caps_by_intensity"] = {} + h_empty = verifier_policy_hash(base_policy) + base_policy["quantifier_caps_by_intensity"] = {"ALL": 4} + h_override = verifier_policy_hash(base_policy) + assert h_empty != h_override + + +def test_governance_hash_changes_when_master_switch_flips(): + base_policy = dict.fromkeys(_VERIFIER_POLICY_FIELDS, "default") + base_policy["quantifier_guard_enabled"] = True + h_on = verifier_policy_hash(base_policy) + base_policy["quantifier_guard_enabled"] = False + h_off = verifier_policy_hash(base_policy) + assert h_on != h_off + + +def test_governance_hash_changes_when_mode_list_changes(): + base_policy = dict.fromkeys(_VERIFIER_POLICY_FIELDS, "default") + base_policy["quantifier_guard_modes"] = ["claim_lattice_pointer", "claim_lattice"] + h_default = verifier_policy_hash(base_policy) + base_policy["quantifier_guard_modes"] = ["claim_lattice_pointer"] + h_pointer_only = verifier_policy_hash(base_policy) + assert h_default != h_pointer_only + + +# ---------------------------------------------------------------- profile shape + +def test_profile_keys_include_all_ten_rungs(): + """Every profile must cover all ten intensity rungs so cap + lookup can't fall through to the default profile due to a + missing key.""" + expected_rungs = { + "ABSENT", "SINGULAR", "PROPORTIONAL", + "SMALL_NUM_EXPLICIT", "COMPARATIVE_BOUND", + "FEW", "MANY", "ALL", "COMPREHENSIVE", "OPEN_REQUEST", + } + for profile_id, profile in PROFILES.items(): + missing = expected_rungs - set(profile.keys()) + assert not missing, f"{profile_id} missing rungs: {missing}" + + +def test_default_profile_present(): + """Lookup-fallback target must exist or unknown models would + return None on every call.""" + assert "default" in PROFILES