"""Phase 4 — broad-quantifier soft-demote tails + reject-broad path. Three new violation kinds (§10.3) added to the soft-demote ladder: - BROAD_QUANTIFIER_RUNAWAY raw_line_count >> pointer_count - BROAD_QUANTIFIER_CAP_APPLIED preflight cap fired below default - BROAD_QUANTIFIER_SCOPE_UNBOUND unbounded universal reached LLM Plus one HARD demote that returns UNGROUNDED via early-return: - BROAD_QUANTIFIER_REJECTED preflight rejected before LLM call Tests pin: tail rendering, ladder behavior, governance-hash binding on quantifier_reject_broad, and the early-return shape for the reject path. """ from __future__ import annotations import pytest from arborist.cli import ( _SOFT_DEMOTE_VIOLATION_KINDS, _ladder_rung_for_lattice, _render_query_human, _render_warrant_tail, ) from arborist.qa.keys import _VERIFIER_POLICY_FIELDS, verifier_policy_hash # ---------------------------------------------------------------- soft-demote registration @pytest.mark.parametrize("kind", [ "BROAD_QUANTIFIER_RUNAWAY", "BROAD_QUANTIFIER_CAP_APPLIED", "BROAD_QUANTIFIER_SCOPE_UNBOUND", ]) def test_broad_quantifier_kinds_in_soft_demote_set(kind): """All three soft-demote kinds must be registered so they cap the ladder at ANCHOR-WARRANTED rather than reaching EVIDENCE- WARRANTED.""" assert kind in _SOFT_DEMOTE_VIOLATION_KINDS def test_rejected_kind_not_in_soft_demote_set(): """REJECTED is a HARD demote (UNGROUNDED via early-return). Listing it as a soft demote would let UNGROUNDED rejected runs bubble up to ANCHOR-WARRANTED on the ladder.""" assert "BROAD_QUANTIFIER_REJECTED" not in _SOFT_DEMOTE_VIOLATION_KINDS # ---------------------------------------------------------------- ladder rung def test_cap_applied_demotes_to_anchor_warranted(): """CAP_APPLIED is a soft demote — claim verified, but the cap bound the answer below default. ANCHOR-WARRANTED, not EVIDENCE-WARRANTED.""" rung = _ladder_rung_for_lattice( "STRICT", violations=[{"kind": "BROAD_QUANTIFIER_CAP_APPLIED"}], ) assert rung == "ANCHOR-WARRANTED" def test_scope_unbound_demotes_to_anchor_warranted(): rung = _ladder_rung_for_lattice( "STRICT", violations=[{"kind": "BROAD_QUANTIFIER_SCOPE_UNBOUND"}], ) assert rung == "ANCHOR-WARRANTED" def test_runaway_demotes_to_anchor_warranted(): rung = _ladder_rung_for_lattice( "STRICT", violations=[{"kind": "BROAD_QUANTIFIER_RUNAWAY"}], ) assert rung == "ANCHOR-WARRANTED" # ---------------------------------------------------------------- tail rendering def test_tail_renders_broad_cap_with_count(): """Cap value must appear in the tail — operators shouldn't have to dig into violations to see what cap fired.""" tail = _render_warrant_tail({ "violations": [{"kind": "BROAD_QUANTIFIER_CAP_APPLIED"}], "claim_cap_applied": 8, }) assert "broad cap 8" in tail def test_tail_renders_broad_unbounded(): tail = _render_warrant_tail({ "violations": [{"kind": "BROAD_QUANTIFIER_SCOPE_UNBOUND"}], }) assert "broad unbounded" in tail def test_tail_renders_broad_runaway(): tail = _render_warrant_tail({ "violations": [{"kind": "BROAD_QUANTIFIER_RUNAWAY"}], }) assert "broad runaway" in tail def test_tail_renders_broad_rejected(): tail = _render_warrant_tail({ "violations": [{"kind": "BROAD_QUANTIFIER_REJECTED"}], }) assert "broad rejected" in tail def test_tail_combines_with_existing_violations(): """A run can have both a broad-quantifier soft demote AND a title mismatch — both should surface on the tail.""" tail = _render_warrant_tail({ "violations": [ {"kind": "TITLE_MISMATCH"}, {"kind": "BROAD_QUANTIFIER_CAP_APPLIED"}, ], "claim_cap_applied": 8, }) assert "title mismatch" in tail assert "broad cap 8" in tail def test_tail_omits_when_no_violations(): tail = _render_warrant_tail({"violations": []}) assert tail == "" # ---------------------------------------------------------------- audit-line render def test_render_label_with_broad_cap_tail(): """End-to-end through _render_query_human: HYBRID + broad cap surfaces both the ladder rung (ANCHOR-WARRANTED-PARTIAL) and the broad-cap tail.""" result = { "status": "cache_miss_then_written", "audit_mode": "HYBRID", "cache_key": "abc" * 21, "context_root": "ab" * 32, "answer_text": "Some bounded broad-quantifier answer.", "sources": [], "n_quotes": 8, "n_verified": 6, "verifier_method": "claim_lattice_pointer", "unverified_quotes": [], "violations": [{"kind": "BROAD_QUANTIFIER_CAP_APPLIED"}], "claim_cap_applied": 8, "timings": {"total_ms": 9000.0}, } out = _render_query_human(result, "winners of all major sports?") assert "ANCHOR-WARRANTED-PARTIAL" in out assert "broad cap 8" in out # ---------------------------------------------------------------- governance hash def test_quantifier_reject_broad_in_verifier_policy_fields(): """Flipping reject-broad must invalidate prior cache records — same question with reject_broad=False vs True should produce different cache_keys.""" assert "quantifier_reject_broad" in _VERIFIER_POLICY_FIELDS def test_governance_hash_changes_when_reject_broad_flips(): base_policy = dict.fromkeys(_VERIFIER_POLICY_FIELDS, "default") base_policy["quantifier_reject_broad"] = False h_off = verifier_policy_hash(base_policy) base_policy["quantifier_reject_broad"] = True h_on = verifier_policy_hash(base_policy) assert h_off != h_on # ---------------------------------------------------------------- reject-broad path def test_reject_broad_returns_early_for_unbounded_all(): """Integration-shape check: when policy enables reject_broad and the question is broad-unbounded, query() returns UNGROUNDED with a BROAD_QUANTIFIER_REJECTED violation BEFORE the LLM call. Skipped here because exercising the full query() path requires a populated shards-dir + qa.db which isn't set up in this unit- test scope. The behavior is exercised by the live bench cycle when --reject-broad is passed.""" pytest.skip( "integration: requires populated shards; covered by live bench" ) def test_reject_broad_does_not_fire_for_bounded_universal(): """`all members of the Beatles` is a bounded universal — should NOT reject even with quantifier_reject_broad=True. Same skip rationale as above.""" pytest.skip( "integration: requires populated shards; covered by live bench" )