From 39bebe3fdb677da343fc719bd74557ca31db8a79 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sat, 2 May 2026 15:51:58 -0400 Subject: [PATCH] =?UTF-8?q?qa(verify):=20Rule=209=20=E2=80=94=20SUBJECT=5F?= =?UTF-8?q?TOKENS=5FABSENT=20premise-parroting=20demote?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the first confirmed EVIDENCE-WARRANTED false-positive surfaced by the 200-cycle bench-emergent run on `steer/reply/correcter` (Ticket #000006 amend 2026-05-02b). The model parroted three question-distinctive tokens (correcter, steer, reply) into its claim while citing a glossary article whose 33.5K-char content contains ZERO occurrences of any of them. Generic linguistic vocabulary (language, communication, terms, relationships) carried Rule 5's citation-coverage check on its own; the actual subject tokens rode along unverified. New per-claim check `_parroted_subject_tokens_absent`: for each resolving claim, compute the question∩claim content-token set, then check substring presence in the union of cited evidence spans (lower-cased, mirroring Rule 5). When ≥ threshold parroted tokens are absent, emit `SUBJECT_TOKENS_ABSENT` and demote STRICT → HYBRID. Default threshold = 3 — single-token absence is often stem-variant noise; three+ is the parrot fingerprint. Plumbing: - New default `DEFAULT_SUBJECT_TOKENS_ABSENT_THRESHOLD = 3` - Both `verify_claim_lattice` and `verify_claim_lattice_json` gain `subject_tokens_absent_threshold` kwarg + per-claim check block (mirrors TITLE_MISMATCH plumbing, sits right after it in the rule order) - `claim_lattice_subject_tokens_absent_threshold` policy field added to `DEFAULT_QUERY_POLICY` and `DEFAULT_POLICY`; folds into `governance_policy_hash` and (via _VERIFIER_POLICY_FIELDS) `verifier_policy_hash` - All four runner/query call sites pass the policy-derived value Live verification (cache-split cleanly via policy-hash bump): pre-fix cache_key 08dbd2c1… : STRICT (false positive) post-fix cache_key 6a519636… : UNGROUNDED Three new unit tests in `tests/test_verify_json.py`: - threshold-meeting parrot demotes STRICT → HYBRID - no-op when question is None - below-threshold absence stays STRICT Full suite: 776 passed, 34 skipped. --- aborist/qa/keys.py | 4 + aborist/qa/query.py | 14 ++ aborist/qa/runner.py | 9 ++ aborist/qa/verify.py | 132 ++++++++++++++++++ .../ticket-000006-bench-emergent-findings.md | 36 +++-- tests/test_verify_json.py | 129 +++++++++++++++++ 6 files changed, 311 insertions(+), 13 deletions(-) diff --git a/aborist/qa/keys.py b/aborist/qa/keys.py index 66684ae..b584cc4 100644 --- a/aborist/qa/keys.py +++ b/aborist/qa/keys.py @@ -210,6 +210,10 @@ _VERIFIER_POLICY_FIELDS = frozenset({ # Warrant-lite (relation-question hard check, Ticket H, 2026-05-01) "claim_lattice_warrant_check_enabled", "claim_lattice_deflection_check_enabled", + # Subject-tokens-absent / premise-parroting (Ticket #000006 amend + # 2026-05-02b, Rule 9). Threshold of question∩claim content tokens + # absent from cited evidence union that demotes STRICT → HYBRID. + "claim_lattice_subject_tokens_absent_threshold", # Quote-mode entity policy "entity_policy", "entity_proximity_n", diff --git a/aborist/qa/query.py b/aborist/qa/query.py index ec33dde..a7186f6 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -309,6 +309,14 @@ DEFAULT_QUERY_POLICY = { # fold unconditionally; this flag exists to make the policy # transition observable from the cache_key alone. "hyphen_fold_v1": True, + # Ticket #000006 amend 2026-05-02b (Rule 9) — premise-parroting / + # subject-tokens-absent demote. Threshold = number of question∩ + # claim content tokens that must be absent from the union of + # cited evidence spans before STRICT → HYBRID. Default 3 keeps + # the signal unambiguous (single-token absence is often a + # stem-variant near-miss; three+ is the parroting fingerprint). + # Folds into verifier_policy_hash + governance_policy_hash. + "claim_lattice_subject_tokens_absent_threshold": 3, "system_prompt": ( "You are answering a question using ONLY the sources provided below. " "Each source is delimited by '=== Source: ===' headers.\n\n" @@ -2176,6 +2184,9 @@ def query( max_claims_per_answer=int(policy.get( "claim_lattice_max_claims_per_answer", 12 )), + subject_tokens_absent_threshold=int(policy.get( + "claim_lattice_subject_tokens_absent_threshold", 3 + )), question=question, warrant_check_enabled=bool(policy.get( "claim_lattice_warrant_check_enabled", True @@ -2210,6 +2221,9 @@ def query( max_claims_per_answer=int(policy.get( "claim_lattice_max_claims_per_answer", 12 )), + subject_tokens_absent_threshold=int(policy.get( + "claim_lattice_subject_tokens_absent_threshold", 3 + )), question=question, warrant_check_enabled=bool(policy.get( "claim_lattice_warrant_check_enabled", True diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index feae207..c5971f5 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -71,6 +71,9 @@ DEFAULT_POLICY = { # Ticket #000007 — query-layer hyphen-fold marker. See # aborist/qa/query.py:DEFAULT_QUERY_POLICY for rationale. "hyphen_fold_v1": True, + # Ticket #000006 amend 2026-05-02b (Rule 9). See + # aborist/qa/query.py:DEFAULT_QUERY_POLICY for full rationale. + "claim_lattice_subject_tokens_absent_threshold": 3, "system_prompt": ( "Answer the user's question based ONLY on the document below. " "For EVERY factual claim, include a verbatim quote from the " @@ -527,6 +530,9 @@ def ask( max_claims_per_answer=int(policy.get( "claim_lattice_max_claims_per_answer", 12 )), + subject_tokens_absent_threshold=int(policy.get( + "claim_lattice_subject_tokens_absent_threshold", 3 + )), question=question, warrant_check_enabled=bool(policy.get( "claim_lattice_warrant_check_enabled", True @@ -565,6 +571,9 @@ def ask( max_claims_per_answer=int(policy.get( "claim_lattice_max_claims_per_answer", 12 )), + subject_tokens_absent_threshold=int(policy.get( + "claim_lattice_subject_tokens_absent_threshold", 3 + )), question=question, warrant_check_enabled=bool(policy.get( "claim_lattice_warrant_check_enabled", True diff --git a/aborist/qa/verify.py b/aborist/qa/verify.py index 913ad03..0584a3b 100644 --- a/aborist/qa/verify.py +++ b/aborist/qa/verify.py @@ -922,6 +922,21 @@ def _has_manual_quote(text: str) -> bool: DEFAULT_MIN_CITATION_COVERAGE = 0.30 +# Premise-parroting / generic-vocab-ride-along threshold. When ≥ this +# many tokens shared by the question AND the claim are ABSENT from the +# union of cited evidence spans, the claim is parroting the question's +# subject without anchoring it. Surfaced by the 200-cycle bench-emergent +# delta on `steer/reply/correcter` (Ticket #000006 amend 2026-05-02b): +# claim affirmed three question-distinctive tokens (correcter, steer, +# reply) that appeared ZERO times in the cited 33.5K-char glossary. The +# generic linguistic vocabulary (language, communication, terms, +# relationships) carried Rule 5's coverage check on its own. +# +# Threshold of 3 keeps the signal unambiguous: a single absent parroted +# token is often a stem-variant near-miss; three or more is the +# parroting fingerprint. Folds into verifier_policy_hash. +DEFAULT_SUBJECT_TOKENS_ABSENT_THRESHOLD = 3 + def _claim_textually_overlaps_evidence( claim_text: str, @@ -972,6 +987,46 @@ def _claim_textually_overlaps_evidence( return coverage >= min_coverage +def _parroted_subject_tokens_absent( + question_text: str | None, + claim_text: str, + cited_spans: list[str], +) -> set[str]: + """Return claim∩question content tokens that are NOT present in + the union of cited evidence spans. + + Premise-parroting / generic-vocab-ride-along detector (Ticket + #000006 amend 2026-05-02b). The model affirms the question's + distinctive subject tokens in its claim, but those tokens are + absent from the cited evidence — the citation rode in on + overlapping generic vocabulary while the actual subject went + unverified. + + Mechanism: substring match on lowercased text, mirroring Rule 5 + (`_claim_textually_overlaps_evidence`). Stem-tolerant via the + substring rule — "polar" matches inside "bipolar", "rare" + matches "rarely", etc. + + No-question-text → empty set (skip the check). + No question∩claim overlap → empty set (claim isn't parroting). + Empty cited_spans → return the full parroted set (defensive; no + grounding at all is its own failure mode caught elsewhere). + """ + from aborist.qa.evidence import _content_tokens + + if not question_text or not claim_text: + return set() + qtok = set(_content_tokens(question_text)) + ctok = set(_content_tokens(claim_text)) + parroted = qtok & ctok + if not parroted: + return set() + union_lower = " ".join((s or "").lower() for s in cited_spans) + if not union_lower.strip(): + return parroted + return {t for t in parroted if t not in union_lower} + + DEFAULT_MAX_POINTERS_PER_CLAIM = 2 DEFAULT_MIN_CLAIM_CONTENT_TOKENS = 2 DEFAULT_LAZY_ANCHOR_DEMOTE_THRESHOLD = 0.5 @@ -1037,6 +1092,7 @@ def verify_claim_lattice( lazy_anchor_demote_threshold: float = DEFAULT_LAZY_ANCHOR_DEMOTE_THRESHOLD, lazy_anchor_demote_min_pairs: int = DEFAULT_LAZY_ANCHOR_DEMOTE_MIN_PAIRS, max_claims_per_answer: int = DEFAULT_MAX_CLAIMS_PER_ANSWER, + subject_tokens_absent_threshold: int = DEFAULT_SUBJECT_TOKENS_ABSENT_THRESHOLD, question: str | None = None, warrant_check_enabled: bool = True, deflection_check_enabled: bool = True, @@ -1498,6 +1554,47 @@ def verify_claim_lattice( ): audit_mode = "UNGROUNDED" + # Rule 9 — Subject-tokens-absent / premise-parroting check (Ticket + # #000006 amend 2026-05-02b, surfaced by `steer/reply/correcter` + # 200-cycle bench-emergent finding). For each resolving claim, + # collect the union of cited evidence spans and check whether ≥ + # subject_tokens_absent_threshold tokens shared by question AND + # claim are absent from that union. If so, the claim is parroting + # the question's distinctive subject without anchoring it — the + # citation rode in on overlapping generic vocabulary while the + # actual subject went unverified. + subject_absent_claims: list[int] = [] + if question and subject_tokens_absent_threshold > 0: + for cs in claim_statuses: + if cs.get("status") not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"): + continue + cited_eids = cs.get("evidence_ids") or [] + cited_spans = [] + for eid in cited_eids: + obj = evidence_map_by_evidence_id_local(evidence_map, eid) + if obj is not None and obj.span: + cited_spans.append(obj.span) + if not cited_spans: + continue + absent = _parroted_subject_tokens_absent( + question, cs.get("text") or "", cited_spans + ) + if len(absent) >= subject_tokens_absent_threshold: + subject_absent_claims.append(cs.get("claim_idx")) + violations.append({ + "kind": "SUBJECT_TOKENS_ABSENT", + "claim_idx": cs.get("claim_idx"), + "claim_text": (cs.get("text") or "")[:200], + "absent_tokens": sorted(absent), + "rationale": ( + f"{len(absent)} question-distinctive tokens echoed in " + f"the claim are absent from cited evidence — claim " + f"parrots question premise without anchoring it" + ), + }) + if subject_absent_claims and audit_mode == "STRICT": + audit_mode = "HYBRID" + # Deflection check (soft demote, promoted from sidecar 2026-05-02). # When the question's subject anchor is missing from the answer, # the model deflected — answered an adjacent grounded question @@ -1608,6 +1705,7 @@ def verify_claim_lattice_json( max_evidence_per_claim: int = DEFAULT_MAX_POINTERS_PER_CLAIM, min_citation_coverage: float = DEFAULT_MIN_CITATION_COVERAGE, max_claims_per_answer: int = DEFAULT_MAX_CLAIMS_PER_ANSWER, + subject_tokens_absent_threshold: int = DEFAULT_SUBJECT_TOKENS_ABSENT_THRESHOLD, question: str | None = None, warrant_check_enabled: bool = True, deflection_check_enabled: bool = True, @@ -1941,6 +2039,40 @@ def verify_claim_lattice_json( ): audit_mode = "UNGROUNDED" + # Rule 9 — Subject-tokens-absent / premise-parroting check. See + # `verify_claim_lattice` for the full rationale. + subject_absent_claims: list[int] = [] + if question and subject_tokens_absent_threshold > 0: + for cs in claim_statuses: + if cs.get("status") not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"): + continue + cited_eids = cs.get("evidence_ids") or [] + cited_spans = [] + for eid in cited_eids: + obj = evidence_map_by_evidence_id_local(evidence_map, eid) + if obj is not None and obj.span: + cited_spans.append(obj.span) + if not cited_spans: + continue + absent = _parroted_subject_tokens_absent( + question, cs.get("text") or "", cited_spans + ) + if len(absent) >= subject_tokens_absent_threshold: + subject_absent_claims.append(cs.get("claim_idx")) + violations.append({ + "kind": "SUBJECT_TOKENS_ABSENT", + "claim_idx": cs.get("claim_idx"), + "claim_text": (cs.get("text") or "")[:200], + "absent_tokens": sorted(absent), + "rationale": ( + f"{len(absent)} question-distinctive tokens echoed in " + f"the claim are absent from cited evidence — claim " + f"parrots question premise without anchoring it" + ), + }) + if subject_absent_claims and audit_mode == "STRICT": + audit_mode = "HYBRID" + # Deflection check (parallel to pointer-variant promotion). deflection_detected = False if deflection_check_enabled and question and rendered_text: diff --git a/docs/tickets/ticket-000006-bench-emergent-findings.md b/docs/tickets/ticket-000006-bench-emergent-findings.md index 238571c..40a7c32 100644 --- a/docs/tickets/ticket-000006-bench-emergent-findings.md +++ b/docs/tickets/ticket-000006-bench-emergent-findings.md @@ -345,19 +345,29 @@ that the lexical Rule-5 check cannot catch alone. ### Action items emerging from 200-cycle delta -1. **Open new ticket #000008 — PREMISE_PARROTING detector.** - Detector signal: question-distinctive tokens (rare in cited - chunk) absent from cited evidence span while generic vocabulary - carries the coverage check. Specifically: - - Compute `question_distinctive = question_content_tokens \\ - stop_corpus` (or TF-IDF top-K from question over corpus). - - For each cited evidence span: count `question_distinctive ∩ - cited_chunk_tokens`. - - If 0 question-distinctive tokens appear in cited chunk while - citation-coverage Rule 5 still passes → emit - `SUBJECT_TOKENS_ABSENT` violation, demote STRICT → HYBRID. - This stays lexical (no NLI). Folds into `verifier_policy_hash` - and `governance_policy_hash`. +1. **Rule 9 — SUBJECT_TOKENS_ABSENT detector — landed inline (no + separate ticket).** New per-claim check in + `aborist/qa/verify.py`. For each resolving claim, compute the + set of content tokens shared by question AND claim + (`_parroted_subject_tokens_absent`). For each, check substring + presence in the union of cited evidence spans (lower-cased, + same as Rule 5). When ≥ `subject_tokens_absent_threshold` (default + 3) parroted tokens are absent → emit `SUBJECT_TOKENS_ABSENT` + violation, demote STRICT → HYBRID. Stays lexical, stays binary, + folds into `verifier_policy_hash` + `governance_policy_hash`. + + **Live verification on the original case** (cache-split via + policy-hash bump): + + | cache_key | policy | audit_mode | + |---|---|---| + | `08dbd2c1…` | pre-fix | STRICT (false positive) | + | `6a519636…` | post-fix | UNGROUNDED | + + Ladder now correctly demotes the parroted answer. Three new + unit tests pin the signature: demote on threshold-meeting + parrot, no-op when question is None, no-demote below threshold. + 2. **`metaphor_deflection` sidecar** still rare (low single-digit fires across 200 cycles). Calibration deferred until 300+. diff --git a/tests/test_verify_json.py b/tests/test_verify_json.py index 19e50be..7226748 100644 --- a/tests/test_verify_json.py +++ b/tests/test_verify_json.py @@ -394,3 +394,132 @@ def test_runner_ask_json_mode_passes_guided_json_extra_body(tmp_path): assert "guided_json" in kwargs["extra_body"] finally: conn.close() + + +# ---------------------------------------------------------------- Rule 9 +# Subject-tokens-absent / premise-parroting check (Ticket #000006 amend +# 2026-05-02b). Surfaced by the 200-cycle bench-emergent finding on +# `steer/reply/correcter`: claim affirmed three question-distinctive +# tokens that appeared zero times in the cited 33.5K-char glossary, +# while generic linguistic vocabulary carried Rule 5's coverage check. + + +def test_verify_json_subject_tokens_absent_demotes_strict_to_hybrid(): + """Reproduces the steer/reply/correcter false-positive shape: + the claim parrots the question's distinctive subject tokens + (correcter, steer, reply) but the cited evidence contains only + generic linguistic vocabulary (language, communication, terms). + Rule 5 passes on the generic overlap; Rule 9 catches that the + question-distinctive tokens are absent and demotes.""" + cited_span = ( + "Sociolinguistics is the study of language in society and how " + "social factors influence communication. The exchange of meaning " + "between speakers depends on shared terms and discourse " + "relationships. Different aspects of language interact with " + "communication norms in any given community." + ) + evidence = [ + _ev( + "Eparrot01", + cited_span, + pointer_id="E1", + title="Glossary of language teaching terms and ideas", + ), + ] + answer = json.dumps({ + "claims": [ + { + # Three question-distinctive tokens (correcter, steer, + # reply) parroted from question into claim — but ZERO + # of these tokens appear in cited_span. + "text": ( + "A correcter can be used to steer a reply by " + "identifying errors in language and communication " + "between speakers, which involves the exchange of " + "meaning across discourse relationships." + ), + "evidence_ids": ["E1"], + } + ] + }) + question = ( + "How might a correcter be used to steer a reply in a " + "conversation, and what aspects of language or communication " + "do these terms encompass?" + ) + v = verify_claim_lattice_json(answer, evidence, question=question) + assert v["audit_mode"] == "HYBRID", ( + f"expected HYBRID via SUBJECT_TOKENS_ABSENT demote; got " + f"{v['audit_mode']} with violations " + f"{[vio['kind'] for vio in v['violations']]}" + ) + assert any(vio["kind"] == "SUBJECT_TOKENS_ABSENT" for vio in v["violations"]) + # The parroted-but-absent tokens should be reported. + sta = next(vio for vio in v["violations"] if vio["kind"] == "SUBJECT_TOKENS_ABSENT") + absent = set(sta["absent_tokens"]) + assert {"correcter", "steer", "reply"}.issubset(absent), ( + f"expected correcter/steer/reply in absent_tokens; got {absent}" + ) + + +def test_verify_json_subject_tokens_absent_no_question_skips_check(): + """No question text → Rule 9 is a no-op. STRICT stays STRICT + when every other check passes. Pins that the check requires + question text to operate.""" + evidence = [ + _ev( + "Enoq00001", + "Brachiosaurus appears in the Jurassic Park film as a herbivore.", + pointer_id="E1", + title="Jurassic Park (film)", + ), + ] + answer = json.dumps({ + "claims": [ + {"text": "Brachiosaurus appears in the film", "evidence_ids": ["E1"]}, + ] + }) + v = verify_claim_lattice_json(answer, evidence, question=None) + assert v["audit_mode"] == "STRICT" + assert not any(vio["kind"] == "SUBJECT_TOKENS_ABSENT" for vio in v["violations"]) + + +def test_verify_json_subject_tokens_absent_below_threshold_passes(): + """One absent parroted token → below default threshold of 3 → + no demote. Pins the threshold semantics: single-token absence + is acceptable noise, three+ is the parrot fingerprint.""" + cited_span = ( + "Brachiosaurus appears in the Jurassic Park film as a herbivore. " + "The dinosaurs in the film were rendered with practical effects " + "and CGI by Industrial Light and Magic." + ) + evidence = [ + _ev( + "Eonebelow", + cited_span, + pointer_id="E1", + title="Jurassic Park (film)", + ), + ] + # Question token "extinction" doesn't appear in cited; "brachiosaurus" + # and "film" do. Only 1 parroted-token absent → below threshold 3. + answer = json.dumps({ + "claims": [ + { + "text": ( + "Brachiosaurus appears in the film alongside other " + "dinosaurs after a long extinction" + ), + "evidence_ids": ["E1"], + } + ] + }) + question = ( + "How does Brachiosaurus appear in the Jurassic Park film " + "after extinction?" + ) + v = verify_claim_lattice_json(answer, evidence, question=question) + assert v["audit_mode"] == "STRICT", ( + f"expected STRICT (below threshold); got {v['audit_mode']} " + f"with violations {[vio['kind'] for vio in v['violations']]}" + )