qa(verify): Rule 9 — SUBJECT_TOKENS_ABSENT premise-parroting demote
Closes the first confirmed EVIDENCE-WARRANTED false-positive surfaced by the 200-cycle bench-emergent run on `steer/reply/correcter` (Ticket #000006 amend 2026-05-02b). The model parroted three question-distinctive tokens (correcter, steer, reply) into its claim while citing a glossary article whose 33.5K-char content contains ZERO occurrences of any of them. Generic linguistic vocabulary (language, communication, terms, relationships) carried Rule 5's citation-coverage check on its own; the actual subject tokens rode along unverified. New per-claim check `_parroted_subject_tokens_absent`: for each resolving claim, compute the question∩claim content-token set, then check substring presence in the union of cited evidence spans (lower-cased, mirroring Rule 5). When ≥ threshold parroted tokens are absent, emit `SUBJECT_TOKENS_ABSENT` and demote STRICT → HYBRID. Default threshold = 3 — single-token absence is often stem-variant noise; three+ is the parrot fingerprint. Plumbing: - New default `DEFAULT_SUBJECT_TOKENS_ABSENT_THRESHOLD = 3` - Both `verify_claim_lattice` and `verify_claim_lattice_json` gain `subject_tokens_absent_threshold` kwarg + per-claim check block (mirrors TITLE_MISMATCH plumbing, sits right after it in the rule order) - `claim_lattice_subject_tokens_absent_threshold` policy field added to `DEFAULT_QUERY_POLICY` and `DEFAULT_POLICY`; folds into `governance_policy_hash` and (via _VERIFIER_POLICY_FIELDS) `verifier_policy_hash` - All four runner/query call sites pass the policy-derived value Live verification (cache-split cleanly via policy-hash bump): pre-fix cache_key 08dbd2c1… : STRICT (false positive) post-fix cache_key 6a519636… : UNGROUNDED Three new unit tests in `tests/test_verify_json.py`: - threshold-meeting parrot demotes STRICT → HYBRID - no-op when question is None - below-threshold absence stays STRICT Full suite: 776 passed, 34 skipped.
This commit is contained in:
parent
d3ad520529
commit
39bebe3fdb
6 changed files with 311 additions and 13 deletions
|
|
@ -210,6 +210,10 @@ _VERIFIER_POLICY_FIELDS = frozenset({
|
|||
# Warrant-lite (relation-question hard check, Ticket H, 2026-05-01)
|
||||
"claim_lattice_warrant_check_enabled",
|
||||
"claim_lattice_deflection_check_enabled",
|
||||
# Subject-tokens-absent / premise-parroting (Ticket #000006 amend
|
||||
# 2026-05-02b, Rule 9). Threshold of question∩claim content tokens
|
||||
# absent from cited evidence union that demotes STRICT → HYBRID.
|
||||
"claim_lattice_subject_tokens_absent_threshold",
|
||||
# Quote-mode entity policy
|
||||
"entity_policy",
|
||||
"entity_proximity_n",
|
||||
|
|
|
|||
|
|
@ -309,6 +309,14 @@ DEFAULT_QUERY_POLICY = {
|
|||
# fold unconditionally; this flag exists to make the policy
|
||||
# transition observable from the cache_key alone.
|
||||
"hyphen_fold_v1": True,
|
||||
# Ticket #000006 amend 2026-05-02b (Rule 9) — premise-parroting /
|
||||
# subject-tokens-absent demote. Threshold = number of question∩
|
||||
# claim content tokens that must be absent from the union of
|
||||
# cited evidence spans before STRICT → HYBRID. Default 3 keeps
|
||||
# the signal unambiguous (single-token absence is often a
|
||||
# stem-variant near-miss; three+ is the parroting fingerprint).
|
||||
# Folds into verifier_policy_hash + governance_policy_hash.
|
||||
"claim_lattice_subject_tokens_absent_threshold": 3,
|
||||
"system_prompt": (
|
||||
"You are answering a question using ONLY the sources provided below. "
|
||||
"Each source is delimited by '=== Source: <URI> ===' headers.\n\n"
|
||||
|
|
@ -2176,6 +2184,9 @@ def query(
|
|||
max_claims_per_answer=int(policy.get(
|
||||
"claim_lattice_max_claims_per_answer", 12
|
||||
)),
|
||||
subject_tokens_absent_threshold=int(policy.get(
|
||||
"claim_lattice_subject_tokens_absent_threshold", 3
|
||||
)),
|
||||
question=question,
|
||||
warrant_check_enabled=bool(policy.get(
|
||||
"claim_lattice_warrant_check_enabled", True
|
||||
|
|
@ -2210,6 +2221,9 @@ def query(
|
|||
max_claims_per_answer=int(policy.get(
|
||||
"claim_lattice_max_claims_per_answer", 12
|
||||
)),
|
||||
subject_tokens_absent_threshold=int(policy.get(
|
||||
"claim_lattice_subject_tokens_absent_threshold", 3
|
||||
)),
|
||||
question=question,
|
||||
warrant_check_enabled=bool(policy.get(
|
||||
"claim_lattice_warrant_check_enabled", True
|
||||
|
|
|
|||
|
|
@ -71,6 +71,9 @@ DEFAULT_POLICY = {
|
|||
# Ticket #000007 — query-layer hyphen-fold marker. See
|
||||
# aborist/qa/query.py:DEFAULT_QUERY_POLICY for rationale.
|
||||
"hyphen_fold_v1": True,
|
||||
# Ticket #000006 amend 2026-05-02b (Rule 9). See
|
||||
# aborist/qa/query.py:DEFAULT_QUERY_POLICY for full rationale.
|
||||
"claim_lattice_subject_tokens_absent_threshold": 3,
|
||||
"system_prompt": (
|
||||
"Answer the user's question based ONLY on the document below. "
|
||||
"For EVERY factual claim, include a verbatim quote from the "
|
||||
|
|
@ -527,6 +530,9 @@ def ask(
|
|||
max_claims_per_answer=int(policy.get(
|
||||
"claim_lattice_max_claims_per_answer", 12
|
||||
)),
|
||||
subject_tokens_absent_threshold=int(policy.get(
|
||||
"claim_lattice_subject_tokens_absent_threshold", 3
|
||||
)),
|
||||
question=question,
|
||||
warrant_check_enabled=bool(policy.get(
|
||||
"claim_lattice_warrant_check_enabled", True
|
||||
|
|
@ -565,6 +571,9 @@ def ask(
|
|||
max_claims_per_answer=int(policy.get(
|
||||
"claim_lattice_max_claims_per_answer", 12
|
||||
)),
|
||||
subject_tokens_absent_threshold=int(policy.get(
|
||||
"claim_lattice_subject_tokens_absent_threshold", 3
|
||||
)),
|
||||
question=question,
|
||||
warrant_check_enabled=bool(policy.get(
|
||||
"claim_lattice_warrant_check_enabled", True
|
||||
|
|
|
|||
|
|
@ -922,6 +922,21 @@ def _has_manual_quote(text: str) -> bool:
|
|||
|
||||
DEFAULT_MIN_CITATION_COVERAGE = 0.30
|
||||
|
||||
# Premise-parroting / generic-vocab-ride-along threshold. When ≥ this
|
||||
# many tokens shared by the question AND the claim are ABSENT from the
|
||||
# union of cited evidence spans, the claim is parroting the question's
|
||||
# subject without anchoring it. Surfaced by the 200-cycle bench-emergent
|
||||
# delta on `steer/reply/correcter` (Ticket #000006 amend 2026-05-02b):
|
||||
# claim affirmed three question-distinctive tokens (correcter, steer,
|
||||
# reply) that appeared ZERO times in the cited 33.5K-char glossary. The
|
||||
# generic linguistic vocabulary (language, communication, terms,
|
||||
# relationships) carried Rule 5's coverage check on its own.
|
||||
#
|
||||
# Threshold of 3 keeps the signal unambiguous: a single absent parroted
|
||||
# token is often a stem-variant near-miss; three or more is the
|
||||
# parroting fingerprint. Folds into verifier_policy_hash.
|
||||
DEFAULT_SUBJECT_TOKENS_ABSENT_THRESHOLD = 3
|
||||
|
||||
|
||||
def _claim_textually_overlaps_evidence(
|
||||
claim_text: str,
|
||||
|
|
@ -972,6 +987,46 @@ def _claim_textually_overlaps_evidence(
|
|||
return coverage >= min_coverage
|
||||
|
||||
|
||||
def _parroted_subject_tokens_absent(
|
||||
question_text: str | None,
|
||||
claim_text: str,
|
||||
cited_spans: list[str],
|
||||
) -> set[str]:
|
||||
"""Return claim∩question content tokens that are NOT present in
|
||||
the union of cited evidence spans.
|
||||
|
||||
Premise-parroting / generic-vocab-ride-along detector (Ticket
|
||||
#000006 amend 2026-05-02b). The model affirms the question's
|
||||
distinctive subject tokens in its claim, but those tokens are
|
||||
absent from the cited evidence — the citation rode in on
|
||||
overlapping generic vocabulary while the actual subject went
|
||||
unverified.
|
||||
|
||||
Mechanism: substring match on lowercased text, mirroring Rule 5
|
||||
(`_claim_textually_overlaps_evidence`). Stem-tolerant via the
|
||||
substring rule — "polar" matches inside "bipolar", "rare"
|
||||
matches "rarely", etc.
|
||||
|
||||
No-question-text → empty set (skip the check).
|
||||
No question∩claim overlap → empty set (claim isn't parroting).
|
||||
Empty cited_spans → return the full parroted set (defensive; no
|
||||
grounding at all is its own failure mode caught elsewhere).
|
||||
"""
|
||||
from aborist.qa.evidence import _content_tokens
|
||||
|
||||
if not question_text or not claim_text:
|
||||
return set()
|
||||
qtok = set(_content_tokens(question_text))
|
||||
ctok = set(_content_tokens(claim_text))
|
||||
parroted = qtok & ctok
|
||||
if not parroted:
|
||||
return set()
|
||||
union_lower = " ".join((s or "").lower() for s in cited_spans)
|
||||
if not union_lower.strip():
|
||||
return parroted
|
||||
return {t for t in parroted if t not in union_lower}
|
||||
|
||||
|
||||
DEFAULT_MAX_POINTERS_PER_CLAIM = 2
|
||||
DEFAULT_MIN_CLAIM_CONTENT_TOKENS = 2
|
||||
DEFAULT_LAZY_ANCHOR_DEMOTE_THRESHOLD = 0.5
|
||||
|
|
@ -1037,6 +1092,7 @@ def verify_claim_lattice(
|
|||
lazy_anchor_demote_threshold: float = DEFAULT_LAZY_ANCHOR_DEMOTE_THRESHOLD,
|
||||
lazy_anchor_demote_min_pairs: int = DEFAULT_LAZY_ANCHOR_DEMOTE_MIN_PAIRS,
|
||||
max_claims_per_answer: int = DEFAULT_MAX_CLAIMS_PER_ANSWER,
|
||||
subject_tokens_absent_threshold: int = DEFAULT_SUBJECT_TOKENS_ABSENT_THRESHOLD,
|
||||
question: str | None = None,
|
||||
warrant_check_enabled: bool = True,
|
||||
deflection_check_enabled: bool = True,
|
||||
|
|
@ -1498,6 +1554,47 @@ def verify_claim_lattice(
|
|||
):
|
||||
audit_mode = "UNGROUNDED"
|
||||
|
||||
# Rule 9 — Subject-tokens-absent / premise-parroting check (Ticket
|
||||
# #000006 amend 2026-05-02b, surfaced by `steer/reply/correcter`
|
||||
# 200-cycle bench-emergent finding). For each resolving claim,
|
||||
# collect the union of cited evidence spans and check whether ≥
|
||||
# subject_tokens_absent_threshold tokens shared by question AND
|
||||
# claim are absent from that union. If so, the claim is parroting
|
||||
# the question's distinctive subject without anchoring it — the
|
||||
# citation rode in on overlapping generic vocabulary while the
|
||||
# actual subject went unverified.
|
||||
subject_absent_claims: list[int] = []
|
||||
if question and subject_tokens_absent_threshold > 0:
|
||||
for cs in claim_statuses:
|
||||
if cs.get("status") not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"):
|
||||
continue
|
||||
cited_eids = cs.get("evidence_ids") or []
|
||||
cited_spans = []
|
||||
for eid in cited_eids:
|
||||
obj = evidence_map_by_evidence_id_local(evidence_map, eid)
|
||||
if obj is not None and obj.span:
|
||||
cited_spans.append(obj.span)
|
||||
if not cited_spans:
|
||||
continue
|
||||
absent = _parroted_subject_tokens_absent(
|
||||
question, cs.get("text") or "", cited_spans
|
||||
)
|
||||
if len(absent) >= subject_tokens_absent_threshold:
|
||||
subject_absent_claims.append(cs.get("claim_idx"))
|
||||
violations.append({
|
||||
"kind": "SUBJECT_TOKENS_ABSENT",
|
||||
"claim_idx": cs.get("claim_idx"),
|
||||
"claim_text": (cs.get("text") or "")[:200],
|
||||
"absent_tokens": sorted(absent),
|
||||
"rationale": (
|
||||
f"{len(absent)} question-distinctive tokens echoed in "
|
||||
f"the claim are absent from cited evidence — claim "
|
||||
f"parrots question premise without anchoring it"
|
||||
),
|
||||
})
|
||||
if subject_absent_claims and audit_mode == "STRICT":
|
||||
audit_mode = "HYBRID"
|
||||
|
||||
# Deflection check (soft demote, promoted from sidecar 2026-05-02).
|
||||
# When the question's subject anchor is missing from the answer,
|
||||
# the model deflected — answered an adjacent grounded question
|
||||
|
|
@ -1608,6 +1705,7 @@ def verify_claim_lattice_json(
|
|||
max_evidence_per_claim: int = DEFAULT_MAX_POINTERS_PER_CLAIM,
|
||||
min_citation_coverage: float = DEFAULT_MIN_CITATION_COVERAGE,
|
||||
max_claims_per_answer: int = DEFAULT_MAX_CLAIMS_PER_ANSWER,
|
||||
subject_tokens_absent_threshold: int = DEFAULT_SUBJECT_TOKENS_ABSENT_THRESHOLD,
|
||||
question: str | None = None,
|
||||
warrant_check_enabled: bool = True,
|
||||
deflection_check_enabled: bool = True,
|
||||
|
|
@ -1941,6 +2039,40 @@ def verify_claim_lattice_json(
|
|||
):
|
||||
audit_mode = "UNGROUNDED"
|
||||
|
||||
# Rule 9 — Subject-tokens-absent / premise-parroting check. See
|
||||
# `verify_claim_lattice` for the full rationale.
|
||||
subject_absent_claims: list[int] = []
|
||||
if question and subject_tokens_absent_threshold > 0:
|
||||
for cs in claim_statuses:
|
||||
if cs.get("status") not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"):
|
||||
continue
|
||||
cited_eids = cs.get("evidence_ids") or []
|
||||
cited_spans = []
|
||||
for eid in cited_eids:
|
||||
obj = evidence_map_by_evidence_id_local(evidence_map, eid)
|
||||
if obj is not None and obj.span:
|
||||
cited_spans.append(obj.span)
|
||||
if not cited_spans:
|
||||
continue
|
||||
absent = _parroted_subject_tokens_absent(
|
||||
question, cs.get("text") or "", cited_spans
|
||||
)
|
||||
if len(absent) >= subject_tokens_absent_threshold:
|
||||
subject_absent_claims.append(cs.get("claim_idx"))
|
||||
violations.append({
|
||||
"kind": "SUBJECT_TOKENS_ABSENT",
|
||||
"claim_idx": cs.get("claim_idx"),
|
||||
"claim_text": (cs.get("text") or "")[:200],
|
||||
"absent_tokens": sorted(absent),
|
||||
"rationale": (
|
||||
f"{len(absent)} question-distinctive tokens echoed in "
|
||||
f"the claim are absent from cited evidence — claim "
|
||||
f"parrots question premise without anchoring it"
|
||||
),
|
||||
})
|
||||
if subject_absent_claims and audit_mode == "STRICT":
|
||||
audit_mode = "HYBRID"
|
||||
|
||||
# Deflection check (parallel to pointer-variant promotion).
|
||||
deflection_detected = False
|
||||
if deflection_check_enabled and question and rendered_text:
|
||||
|
|
|
|||
|
|
@ -345,19 +345,29 @@ that the lexical Rule-5 check cannot catch alone.
|
|||
|
||||
### Action items emerging from 200-cycle delta
|
||||
|
||||
1. **Open new ticket #000008 — PREMISE_PARROTING detector.**
|
||||
Detector signal: question-distinctive tokens (rare in cited
|
||||
chunk) absent from cited evidence span while generic vocabulary
|
||||
carries the coverage check. Specifically:
|
||||
- Compute `question_distinctive = question_content_tokens \\
|
||||
stop_corpus` (or TF-IDF top-K from question over corpus).
|
||||
- For each cited evidence span: count `question_distinctive ∩
|
||||
cited_chunk_tokens`.
|
||||
- If 0 question-distinctive tokens appear in cited chunk while
|
||||
citation-coverage Rule 5 still passes → emit
|
||||
`SUBJECT_TOKENS_ABSENT` violation, demote STRICT → HYBRID.
|
||||
This stays lexical (no NLI). Folds into `verifier_policy_hash`
|
||||
and `governance_policy_hash`.
|
||||
1. **Rule 9 — SUBJECT_TOKENS_ABSENT detector — landed inline (no
|
||||
separate ticket).** New per-claim check in
|
||||
`aborist/qa/verify.py`. For each resolving claim, compute the
|
||||
set of content tokens shared by question AND claim
|
||||
(`_parroted_subject_tokens_absent`). For each, check substring
|
||||
presence in the union of cited evidence spans (lower-cased,
|
||||
same as Rule 5). When ≥ `subject_tokens_absent_threshold` (default
|
||||
3) parroted tokens are absent → emit `SUBJECT_TOKENS_ABSENT`
|
||||
violation, demote STRICT → HYBRID. Stays lexical, stays binary,
|
||||
folds into `verifier_policy_hash` + `governance_policy_hash`.
|
||||
|
||||
**Live verification on the original case** (cache-split via
|
||||
policy-hash bump):
|
||||
|
||||
| cache_key | policy | audit_mode |
|
||||
|---|---|---|
|
||||
| `08dbd2c1…` | pre-fix | STRICT (false positive) |
|
||||
| `6a519636…` | post-fix | UNGROUNDED |
|
||||
|
||||
Ladder now correctly demotes the parroted answer. Three new
|
||||
unit tests pin the signature: demote on threshold-meeting
|
||||
parrot, no-op when question is None, no-demote below threshold.
|
||||
|
||||
2. **`metaphor_deflection` sidecar** still rare (low single-digit
|
||||
fires across 200 cycles). Calibration deferred until 300+.
|
||||
|
||||
|
|
|
|||
|
|
@ -394,3 +394,132 @@ def test_runner_ask_json_mode_passes_guided_json_extra_body(tmp_path):
|
|||
assert "guided_json" in kwargs["extra_body"]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- Rule 9
|
||||
# Subject-tokens-absent / premise-parroting check (Ticket #000006 amend
|
||||
# 2026-05-02b). Surfaced by the 200-cycle bench-emergent finding on
|
||||
# `steer/reply/correcter`: claim affirmed three question-distinctive
|
||||
# tokens that appeared zero times in the cited 33.5K-char glossary,
|
||||
# while generic linguistic vocabulary carried Rule 5's coverage check.
|
||||
|
||||
|
||||
def test_verify_json_subject_tokens_absent_demotes_strict_to_hybrid():
|
||||
"""Reproduces the steer/reply/correcter false-positive shape:
|
||||
the claim parrots the question's distinctive subject tokens
|
||||
(correcter, steer, reply) but the cited evidence contains only
|
||||
generic linguistic vocabulary (language, communication, terms).
|
||||
Rule 5 passes on the generic overlap; Rule 9 catches that the
|
||||
question-distinctive tokens are absent and demotes."""
|
||||
cited_span = (
|
||||
"Sociolinguistics is the study of language in society and how "
|
||||
"social factors influence communication. The exchange of meaning "
|
||||
"between speakers depends on shared terms and discourse "
|
||||
"relationships. Different aspects of language interact with "
|
||||
"communication norms in any given community."
|
||||
)
|
||||
evidence = [
|
||||
_ev(
|
||||
"Eparrot01",
|
||||
cited_span,
|
||||
pointer_id="E1",
|
||||
title="Glossary of language teaching terms and ideas",
|
||||
),
|
||||
]
|
||||
answer = json.dumps({
|
||||
"claims": [
|
||||
{
|
||||
# Three question-distinctive tokens (correcter, steer,
|
||||
# reply) parroted from question into claim — but ZERO
|
||||
# of these tokens appear in cited_span.
|
||||
"text": (
|
||||
"A correcter can be used to steer a reply by "
|
||||
"identifying errors in language and communication "
|
||||
"between speakers, which involves the exchange of "
|
||||
"meaning across discourse relationships."
|
||||
),
|
||||
"evidence_ids": ["E1"],
|
||||
}
|
||||
]
|
||||
})
|
||||
question = (
|
||||
"How might a correcter be used to steer a reply in a "
|
||||
"conversation, and what aspects of language or communication "
|
||||
"do these terms encompass?"
|
||||
)
|
||||
v = verify_claim_lattice_json(answer, evidence, question=question)
|
||||
assert v["audit_mode"] == "HYBRID", (
|
||||
f"expected HYBRID via SUBJECT_TOKENS_ABSENT demote; got "
|
||||
f"{v['audit_mode']} with violations "
|
||||
f"{[vio['kind'] for vio in v['violations']]}"
|
||||
)
|
||||
assert any(vio["kind"] == "SUBJECT_TOKENS_ABSENT" for vio in v["violations"])
|
||||
# The parroted-but-absent tokens should be reported.
|
||||
sta = next(vio for vio in v["violations"] if vio["kind"] == "SUBJECT_TOKENS_ABSENT")
|
||||
absent = set(sta["absent_tokens"])
|
||||
assert {"correcter", "steer", "reply"}.issubset(absent), (
|
||||
f"expected correcter/steer/reply in absent_tokens; got {absent}"
|
||||
)
|
||||
|
||||
|
||||
def test_verify_json_subject_tokens_absent_no_question_skips_check():
|
||||
"""No question text → Rule 9 is a no-op. STRICT stays STRICT
|
||||
when every other check passes. Pins that the check requires
|
||||
question text to operate."""
|
||||
evidence = [
|
||||
_ev(
|
||||
"Enoq00001",
|
||||
"Brachiosaurus appears in the Jurassic Park film as a herbivore.",
|
||||
pointer_id="E1",
|
||||
title="Jurassic Park (film)",
|
||||
),
|
||||
]
|
||||
answer = json.dumps({
|
||||
"claims": [
|
||||
{"text": "Brachiosaurus appears in the film", "evidence_ids": ["E1"]},
|
||||
]
|
||||
})
|
||||
v = verify_claim_lattice_json(answer, evidence, question=None)
|
||||
assert v["audit_mode"] == "STRICT"
|
||||
assert not any(vio["kind"] == "SUBJECT_TOKENS_ABSENT" for vio in v["violations"])
|
||||
|
||||
|
||||
def test_verify_json_subject_tokens_absent_below_threshold_passes():
|
||||
"""One absent parroted token → below default threshold of 3 →
|
||||
no demote. Pins the threshold semantics: single-token absence
|
||||
is acceptable noise, three+ is the parrot fingerprint."""
|
||||
cited_span = (
|
||||
"Brachiosaurus appears in the Jurassic Park film as a herbivore. "
|
||||
"The dinosaurs in the film were rendered with practical effects "
|
||||
"and CGI by Industrial Light and Magic."
|
||||
)
|
||||
evidence = [
|
||||
_ev(
|
||||
"Eonebelow",
|
||||
cited_span,
|
||||
pointer_id="E1",
|
||||
title="Jurassic Park (film)",
|
||||
),
|
||||
]
|
||||
# Question token "extinction" doesn't appear in cited; "brachiosaurus"
|
||||
# and "film" do. Only 1 parroted-token absent → below threshold 3.
|
||||
answer = json.dumps({
|
||||
"claims": [
|
||||
{
|
||||
"text": (
|
||||
"Brachiosaurus appears in the film alongside other "
|
||||
"dinosaurs after a long extinction"
|
||||
),
|
||||
"evidence_ids": ["E1"],
|
||||
}
|
||||
]
|
||||
})
|
||||
question = (
|
||||
"How does Brachiosaurus appear in the Jurassic Park film "
|
||||
"after extinction?"
|
||||
)
|
||||
v = verify_claim_lattice_json(answer, evidence, question=question)
|
||||
assert v["audit_mode"] == "STRICT", (
|
||||
f"expected STRICT (below threshold); got {v['audit_mode']} "
|
||||
f"with violations {[vio['kind'] for vio in v['violations']]}"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue