qa: warrant-lite — relation-question hard check (Ticket H)
Closes the lazy-anchor failure class fox surfaced in feedback-3
(2026-05-01) on "who is homer simpson's boss?":
Claim: "Homer Simpson's boss is Mr. Burns."
Cited span: voice-actor / Castellaneta bio prose
Pre-warrant: pointer-linked, source-role allowed, coverage passes
Post-warrant: WARRANT_MISSING violation, audit_mode capped at HYBRID
Pointer verification != warrant verification. A pointer resolves to
a span; a warrant requires the span to actually contain the claim's
named answer entity.
Adds aborist/qa/warrant.py:
- is_relation_question(q) — regex shape detector for "who is X's Y?",
"who is supermans girlfriend?", "who founded Z?", "boss of X" etc.
- extract_answer_anchors(claim) — multi-word proper nouns ("Mr.
Burns", "Lois Lane") with possessive 's stripping; solo-cap
fallback that skips sentence-starter false positives
- warrant_check(claim, spans) — at-least-one-anchor lexical match;
vacuous-pass when claim has no proper-noun anchor
Wired into verify_claim_lattice via new params (question,
warrant_check_enabled). Per-claim WARRANT_MISSING violations cap
audit_mode at HYBRID using the same demote pattern as
lazy_anchor_demoted. CLI renders " · warrant missing" tail when
violations present so the failure mode is legible without scanning
the violations array.
Policy field claim_lattice_warrant_check_enabled added to
DEFAULT_POLICY / DEFAULT_QUERY_POLICY and the verifier-policy hash
field set so changes bump verifier_policy_hash.
Hard or soft? Per CLAUDE.md "Soft hash vs hard hash":
substring/lexical = hard. Output is binary (warrant present or not),
violations enter the proof path through violations[] alongside
SOURCE_ROLE_BLOCKED and CITATION_MISMATCH.
Deferred (follow-on commits):
- full RELATION_PATTERNS table with relation-noun aliases
- two-hop bridge detection ("X founded Y" + "Y is Z" -> "X is Z")
- retrieval expansion for relation questions
Tests: 16 new unit tests in tests/test_warrant.py covering
is_relation_question, extract_answer_anchors, warrant_check
(happy path, lazy-anchor case, vacuous-pass, case-insensitive,
empty inputs). Full suite: 523 passed, 21 skipped.
This commit is contained in:
parent
63c5b2fd8a
commit
38d95dd21b
7 changed files with 472 additions and 1 deletions
|
|
@ -464,6 +464,23 @@ def _render_audit_label(audit_mode: str, verifier_method: str) -> str:
|
|||
return f"{audit_mode} · via {verifier_method}"
|
||||
|
||||
|
||||
def _render_warrant_tail(result: dict) -> str:
|
||||
"""If the verdict carries any WARRANT_MISSING violations
|
||||
(relation-shape question + claim names answer entity not in
|
||||
cited span), return a ` · warrant missing` tail for the
|
||||
audit-line label. Otherwise empty string.
|
||||
|
||||
Surfaces fox's feedback-3 distinction ("pointer-linked but
|
||||
relation warrant missing") at the user-facing layer without
|
||||
overloading the audit_mode field. The verifier already capped
|
||||
audit_mode at HYBRID for warrant-missing cases; this just
|
||||
names the cap reason."""
|
||||
violations = result.get("violations") or []
|
||||
if any(v.get("kind") == "WARRANT_MISSING" for v in violations):
|
||||
return " · warrant missing"
|
||||
return ""
|
||||
|
||||
|
||||
def _maybe_render_json_envelope_as_bullets(answer: str) -> str:
|
||||
"""If `answer` is a claim_lattice JSON envelope, render bullets.
|
||||
|
||||
|
|
@ -563,11 +580,12 @@ def _render_query_human(result: dict, question: str) -> str:
|
|||
# docs/verifier-semantic-gap-design.md) — never hard-checked.
|
||||
# Schema column stays unchanged; this is pure display.
|
||||
display_label = _render_audit_label(audit, method)
|
||||
warrant_tail = _render_warrant_tail(result)
|
||||
|
||||
lines: list[str] = []
|
||||
lines.append(question)
|
||||
lines.append(
|
||||
f" {display_label} {n_verified}/{n_quotes} "
|
||||
f" {display_label}{warrant_tail} {n_verified}/{n_quotes} "
|
||||
f"{elapsed} ({cache_status})"
|
||||
)
|
||||
lines.append("")
|
||||
|
|
|
|||
|
|
@ -205,6 +205,8 @@ _VERIFIER_POLICY_FIELDS = frozenset({
|
|||
# JSON variant identity
|
||||
"claim_lattice_use_guided_json",
|
||||
"claim_lattice_json_stop_sequences",
|
||||
# Warrant-lite (relation-question hard check, Ticket H, 2026-05-01)
|
||||
"claim_lattice_warrant_check_enabled",
|
||||
# Quote-mode entity policy
|
||||
"entity_policy",
|
||||
"entity_proximity_n",
|
||||
|
|
|
|||
|
|
@ -391,6 +391,7 @@ DEFAULT_QUERY_POLICY = {
|
|||
"claim_lattice_min_claim_content_tokens": 2,
|
||||
"claim_lattice_lazy_anchor_demote_threshold": 0.5,
|
||||
"claim_lattice_lazy_anchor_demote_min_pairs": 3,
|
||||
"claim_lattice_warrant_check_enabled": True,
|
||||
# Claim-count ceiling — see runner.DEFAULT_POLICY for rationale.
|
||||
# Bench finding (york-england "tell me all there is to know")
|
||||
# caught the runaway shape; cap of 12 admits entity-list
|
||||
|
|
@ -1803,6 +1804,10 @@ def query(
|
|||
max_claims_per_answer=int(policy.get(
|
||||
"claim_lattice_max_claims_per_answer", 12
|
||||
)),
|
||||
question=question,
|
||||
warrant_check_enabled=bool(policy.get(
|
||||
"claim_lattice_warrant_check_enabled", True
|
||||
)),
|
||||
)
|
||||
rendered = verdict["rendered_text"]
|
||||
answer_text = rendered if rendered else raw_answer
|
||||
|
|
@ -2144,6 +2149,12 @@ def query(
|
|||
"verifier_method": verdict["verifier_method"],
|
||||
"unverified_quotes": verdict["unverified_quotes"],
|
||||
"partially_verified_quotes": verdict.get("partially_verified_quotes") or [],
|
||||
# Violations (per-claim hard-check failures + soft demotes
|
||||
# like LAZY_ANCHOR_DEMOTE / WARRANT_MISSING). Surfaced so the
|
||||
# CLI render can name WHY a verdict capped at HYBRID instead
|
||||
# of reaching STRICT. Persisted into run_dag_blob via the
|
||||
# verify stage's payload.
|
||||
"violations": verdict.get("violations") or [],
|
||||
# Sidecar smell signals (claim_lattice mode only) — surfaced
|
||||
# for the renderer; never persisted in providence_cache and
|
||||
# never threaded into run_dag_root.
|
||||
|
|
|
|||
|
|
@ -233,6 +233,15 @@ DEFAULT_POLICY = {
|
|||
# diverse anchoring across pointers.
|
||||
"claim_lattice_lazy_anchor_demote_threshold": 0.5,
|
||||
"claim_lattice_lazy_anchor_demote_min_pairs": 3,
|
||||
# Warrant-lite — relation-question hard check (Ticket H from
|
||||
# feedback-3, 2026-05-01). Detects relation-shape questions
|
||||
# ("who is X's boss?", "who founded Y?") and requires the cited
|
||||
# span to contain at least one named answer entity (proper-noun
|
||||
# phrase) from the claim. Catches the Homer-Simpson lazy-anchor
|
||||
# case where claim asserts "Mr. Burns" but cited span is the
|
||||
# voice-actor bio. WARRANT_MISSING violations cap audit_mode
|
||||
# at HYBRID. See aborist/qa/warrant.py.
|
||||
"claim_lattice_warrant_check_enabled": True,
|
||||
# Claim-count ceiling. Bench finding (2026-04-30 york-england):
|
||||
# "tell me all there is to know about X" prompted Hermes to spam
|
||||
# 26-59 encyclopedic claims sourced from training, only 2-4 of
|
||||
|
|
@ -608,6 +617,10 @@ def ask(
|
|||
max_claims_per_answer=int(policy.get(
|
||||
"claim_lattice_max_claims_per_answer", 12
|
||||
)),
|
||||
question=question,
|
||||
warrant_check_enabled=bool(policy.get(
|
||||
"claim_lattice_warrant_check_enabled", True
|
||||
)),
|
||||
)
|
||||
# Rendered prose (literal spans interpolated) is the user-facing
|
||||
# answer text — never the model's raw pointer-line output. If
|
||||
|
|
|
|||
|
|
@ -47,6 +47,8 @@ from __future__ import annotations
|
|||
import re
|
||||
import unicodedata
|
||||
|
||||
from aborist.qa.warrant import is_relation_question, warrant_check
|
||||
|
||||
try:
|
||||
from aborist.wikitext import to_base as _wikitext_to_base
|
||||
except ImportError: # pragma: no cover
|
||||
|
|
@ -994,6 +996,8 @@ def verify_claim_lattice(
|
|||
lazy_anchor_demote_threshold: float = DEFAULT_LAZY_ANCHOR_DEMOTE_THRESHOLD,
|
||||
lazy_anchor_demote_min_pairs: int = DEFAULT_LAZY_ANCHOR_DEMOTE_MIN_PAIRS,
|
||||
max_claims_per_answer: int = DEFAULT_MAX_CLAIMS_PER_ANSWER,
|
||||
question: str | None = None,
|
||||
warrant_check_enabled: bool = True,
|
||||
) -> dict:
|
||||
"""Deterministic verifier for ``answer_mode="claim_lattice_pointer"``.
|
||||
|
||||
|
|
@ -1350,6 +1354,45 @@ def verify_claim_lattice(
|
|||
"threshold": lazy_anchor_demote_threshold,
|
||||
})
|
||||
|
||||
# Warrant-lite — relation-question hard check (Ticket H from
|
||||
# feedback-3, 2026-05-01). Claim-cited spans must contain at
|
||||
# least one of the claim's named answer entities (proper-noun
|
||||
# phrases). Catches the Homer-Simpson lazy-anchor case fox
|
||||
# surfaced — claim asserts "Mr. Burns" but cited span is
|
||||
# Castellaneta voice-actor prose. See aborist/qa/warrant.py
|
||||
# for the lexical algorithm and rationale (deterministic,
|
||||
# not NLI). Fires only when the question shape suggests a
|
||||
# relation lookup AND the lookup is enabled by policy
|
||||
# (warrant_check_enabled). Per-claim WARRANT_MISSING violations
|
||||
# cap audit_mode at HYBRID via the same demote pattern as
|
||||
# lazy_anchor_demoted.
|
||||
warrant_missing_claims: list[int] = []
|
||||
if warrant_check_enabled and question and is_relation_question(question):
|
||||
for cs in claim_statuses:
|
||||
if cs.get("status") not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"):
|
||||
continue
|
||||
cited_eids = cs.get("evidence_ids") or []
|
||||
cited_spans = [
|
||||
obj.span
|
||||
for eid in cited_eids
|
||||
for obj in [evidence_map_by_evidence_id_local(evidence_map, eid)]
|
||||
if obj is not None
|
||||
]
|
||||
ok, missing = warrant_check(cs.get("text") or "", cited_spans)
|
||||
if not ok:
|
||||
warrant_missing_claims.append(cs.get("claim_idx"))
|
||||
violations.append({
|
||||
"kind": "WARRANT_MISSING",
|
||||
"claim_idx": cs.get("claim_idx"),
|
||||
"missing_anchors": missing,
|
||||
"rationale": (
|
||||
"claim names answer entity not present in any "
|
||||
"cited span — pointer-linked but warrant missing"
|
||||
),
|
||||
})
|
||||
if warrant_missing_claims and audit_mode == "STRICT":
|
||||
audit_mode = "HYBRID"
|
||||
|
||||
return {
|
||||
"n_quotes": n_pairs,
|
||||
"n_verified": n_pairs_verified,
|
||||
|
|
@ -1364,9 +1407,22 @@ def verify_claim_lattice(
|
|||
"pointer_id_distribution": pointer_distribution,
|
||||
"lazy_anchor_ratio": lazy_anchor_ratio,
|
||||
"lazy_anchor_demoted": lazy_anchor_demoted,
|
||||
"warrant_missing_claim_idxs": warrant_missing_claims,
|
||||
}
|
||||
|
||||
|
||||
def evidence_map_by_evidence_id_local(evidence_map, eid: str):
|
||||
"""Local helper — returns the EvidenceObject whose ``evidence_id``
|
||||
matches ``eid``, or None. Avoids the import-cycle risk of pulling
|
||||
`evidence_map_by_evidence_id` into this module's hot path; the
|
||||
O(N) walk is fine since evidence maps are <30 entries.
|
||||
"""
|
||||
for obj in evidence_map or []:
|
||||
if obj.evidence_id == eid:
|
||||
return obj
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON variant — `answer_mode="claim_lattice"`. Same lattice semantics as
|
||||
# the pointer variant, but the model emits a structured JSON object
|
||||
|
|
|
|||
199
aborist/qa/warrant.py
Normal file
199
aborist/qa/warrant.py
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
"""Warrant-lite — relation-question hard check on claim-cited evidence.
|
||||
|
||||
Closes the lazy-anchor failure class fox surfaced in feedback-3
|
||||
(2026-05-01) on "who is homer simpson's boss?":
|
||||
|
||||
Claim: "Homer Simpson's boss is Mr. Burns."
|
||||
Cited span: voice-actor / Castellaneta bio prose
|
||||
→ Pointer resolves, source role allowed, coverage passes
|
||||
→ BUT span contains no "Mr. Burns" or boss/employer relation
|
||||
→ Pre-warrant: STRICT/HYBRID with bogus citation
|
||||
→ Post-warrant: WARRANT_MISSING violation, audit_mode capped
|
||||
|
||||
The principle: pointer verification ≠ warrant verification.
|
||||
A pointer resolves to a span; a warrant requires the span to
|
||||
actually contain the claim's load-bearing tokens (the named
|
||||
ANSWER ENTITY, primarily — relation aliases left as a follow-on).
|
||||
|
||||
This module implements the deterministic, lexical-only first
|
||||
pass of §11-§14. Two-hop bridge detection + the full
|
||||
RELATION_PATTERNS registry are deferred to a successor commit
|
||||
once the bench shows what answer-anchor alone catches.
|
||||
|
||||
Hard or soft? Per CLAUDE.md "Soft hash vs hard hash":
|
||||
substring/lexical = hard. The check uses `_content_tokens`
|
||||
(deterministic) and proper-noun extraction via regex
|
||||
(deterministic). Output is binary: warrant present or not.
|
||||
That makes it a hard check — its violations enter the
|
||||
proof path through `violations[]` exactly the way
|
||||
SOURCE_ROLE_BLOCKED and CITATION_MISMATCH do.
|
||||
|
||||
But unlike those, WARRANT_MISSING does not block per-pointer
|
||||
verification — it caps audit_mode at HYBRID via the same
|
||||
mechanism `lazy_anchor_demoted` uses. The lexical pointer
|
||||
checks still pass; the user/auditor sees both "pointer-linked"
|
||||
AND "warrant missing" so the failure mode is legible.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# Heuristic relation-question detector. Fires when the question
|
||||
# shape fits a "X's Y" / "who Z's W?" / "who owns Y?" / "who founded Z?"
|
||||
# pattern — the cases fox names in feedback-3. Conservative: false
|
||||
# negatives are fine (warrant just doesn't run); false positives are
|
||||
# the real cost (warrant fires on questions where it shouldn't).
|
||||
# Relation-noun aliases — used both in "who is X's <noun>?" and
|
||||
# "<noun> of X" shapes. Possessive-less variant ("who is supermans
|
||||
# girlfriend?") needs a separate pattern path since `\S+` alone won't
|
||||
# bind to a relation noun without an explicit alias list.
|
||||
_RELATION_NOUNS = (
|
||||
"boss|employer|supervisor|manager|owner|founder|director|creator|"
|
||||
"inventor|author|composer|painter|spouse|wife|husband|mother|father|"
|
||||
"parent|son|daughter|sister|brother|nephew|niece|aunt|uncle|cousin|"
|
||||
"girlfriend|boyfriend|partner|fiancee|fiance|friend|enemy|rival|"
|
||||
"successor|predecessor|mentor|teacher|student|coach|captain"
|
||||
)
|
||||
_RELATION_VERBS = (
|
||||
"owns|founded|directed|created|invented|wrote|painted|composed|"
|
||||
"discovered|killed|married|defeated|coached|hired|fired|sang|"
|
||||
"produced|designed|built|sculpted|filmed"
|
||||
)
|
||||
_RELATION_PATTERNS = (
|
||||
# "who is X's Y?" — apostrophe-s possessive (multi-word or single)
|
||||
re.compile(r"\bwho\s+is\s+\S+(?:\s+\S+)*\s*'s\s+\w+", re.IGNORECASE),
|
||||
# "who is supermans girlfriend?" — possessive without apostrophe,
|
||||
# gated on relation-noun alias list so generic nouns don't trip it
|
||||
re.compile(
|
||||
r"\bwho\s+is\s+\S+(?:\s+\S+)*\s+(?:" + _RELATION_NOUNS + r")\b",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
# "who founded microsoft?" — relation-verb
|
||||
re.compile(r"\bwho\s+(?:" + _RELATION_VERBS + r")\b", re.IGNORECASE),
|
||||
# "what is X's Y?" — possessive on a what-question
|
||||
re.compile(r"\bwhat\s+is\s+\S+(?:\s+\S+)*\s*'s\s+\w+", re.IGNORECASE),
|
||||
# "<relation-noun> of X" — bare relation-of phrase
|
||||
re.compile(r"\b(?:" + _RELATION_NOUNS + r")\s+of\b", re.IGNORECASE),
|
||||
)
|
||||
|
||||
|
||||
def is_relation_question(question: str) -> bool:
|
||||
"""Return True iff the question shape suggests a relation lookup.
|
||||
|
||||
Conservative — defaults to False on any question that doesn't
|
||||
match a known relation-shape regex. False negatives leave the
|
||||
warrant check inactive (claim verifies under the existing six
|
||||
hard checks alone); false positives risk over-firing on
|
||||
non-relation questions (worth tuning if the bench shows it).
|
||||
"""
|
||||
if not question:
|
||||
return False
|
||||
for pat in _RELATION_PATTERNS:
|
||||
if pat.search(question):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# Multi-word proper noun (Title-Case sequences) — captures "Lois
|
||||
# Lane", "Mr. Burns", "New England", "Homer Simpson's". The first-leg
|
||||
# alternation lets a token end in `\.` ONLY when followed by another
|
||||
# title-cased token (so "Mr." in "Mr. Burns" matches but "Burns." at
|
||||
# sentence end does not slurp the trailing period). Trailing `'s` /
|
||||
# stray punctuation are stripped in post-processing.
|
||||
_PROPER_NOUN_RE = re.compile(
|
||||
r"\b(?:[A-Z][A-Za-z'’\-]*\.(?=[ \t]+[A-Z])|[A-Z][A-Za-z'’\-]+)"
|
||||
r"(?:[ \t]+(?:[A-Z][A-Za-z'’\-]*\.(?=[ \t]+[A-Z])|[A-Z][A-Za-z'’\-]+))+"
|
||||
)
|
||||
|
||||
# Solo-capitalized tokens (e.g. "Apple", "Burns"). Less reliable
|
||||
# than multi-word phrases since sentence-start words look like
|
||||
# proper nouns too — used only when no multi-word phrase is
|
||||
# available.
|
||||
_SOLO_CAP_TOKEN_RE = re.compile(r"\b[A-Z][A-Za-z'’\-]{2,}\b")
|
||||
|
||||
|
||||
def _strip_anchor_tail(anchor: str) -> str:
|
||||
"""Strip possessive `'s` / `’s` and stray trailing punctuation."""
|
||||
a = anchor.rstrip(".,;:!?")
|
||||
for tail in ("'s", "’s", "'", "’"):
|
||||
if a.endswith(tail):
|
||||
a = a[: -len(tail)]
|
||||
break
|
||||
return a.strip()
|
||||
|
||||
|
||||
def extract_answer_anchors(claim_text: str) -> list[str]:
|
||||
"""Pull candidate answer-entity anchors from a claim's text.
|
||||
|
||||
Strategy:
|
||||
1. Multi-word proper-noun phrases (`"Mr. Burns"`, `"Lois Lane"`,
|
||||
`"Nineteen Eighty-Four"`) — these are reliable named
|
||||
entities even at sentence start.
|
||||
2. If none found, fall back to solo capitalized tokens that
|
||||
are not the FIRST word of the claim (skipping
|
||||
sentence-starter false positives like the leading
|
||||
"Connecticut" in "Connecticut is a state...").
|
||||
|
||||
Each anchor is post-stripped of possessive `'s` and stray
|
||||
sentence-end punctuation so downstream substring tests find
|
||||
base forms in cited spans.
|
||||
|
||||
Returns a list of anchor strings; empty list if no anchor
|
||||
detectable. Lowercase consumers should `.lower()` per call.
|
||||
"""
|
||||
if not claim_text:
|
||||
return []
|
||||
raw = _PROPER_NOUN_RE.findall(claim_text)
|
||||
if raw:
|
||||
cleaned = []
|
||||
for a in raw:
|
||||
stripped = _strip_anchor_tail(a)
|
||||
if stripped:
|
||||
cleaned.append(stripped)
|
||||
return cleaned
|
||||
# Solo-cap fallback. Skip the first token (sentence starter).
|
||||
tokens = _SOLO_CAP_TOKEN_RE.findall(claim_text)
|
||||
if len(tokens) <= 1:
|
||||
return []
|
||||
return [_strip_anchor_tail(t) for t in tokens[1:] if _strip_anchor_tail(t)]
|
||||
|
||||
|
||||
def warrant_check(
|
||||
claim_text: str,
|
||||
cited_spans: list[str],
|
||||
) -> tuple[bool, list[str]]:
|
||||
"""Hard lexical warrant check on a relation-shape claim.
|
||||
|
||||
Returns ``(warrant_ok, missing_anchors)``:
|
||||
- ``warrant_ok=True`` and ``missing_anchors=[]`` when at least
|
||||
ONE answer anchor extracted from the claim text appears as
|
||||
a substring in at least ONE cited span (case-insensitive).
|
||||
That's the "pointer cited a span that genuinely names the
|
||||
answer" gate.
|
||||
- ``warrant_ok=False`` and ``missing_anchors=[...]`` when none
|
||||
of the extracted anchors appear in any cited span. The
|
||||
list names which anchors the claim asserted but no cited
|
||||
span supports — the auditor sees what specifically was
|
||||
missing.
|
||||
|
||||
Vacuous-pass: if no answer anchor can be extracted from the
|
||||
claim text at all (the claim has no proper noun, e.g. "X is
|
||||
located here"), warrant_check returns (True, []) — there is
|
||||
no answer entity to demand from the spans. The other six
|
||||
hard checks (coverage, source_role, etc.) carry the load.
|
||||
"""
|
||||
anchors = extract_answer_anchors(claim_text)
|
||||
if not anchors:
|
||||
return True, []
|
||||
joined_lower = " ".join(s.lower() for s in cited_spans)
|
||||
found = []
|
||||
missing = []
|
||||
for anchor in anchors:
|
||||
if anchor.lower() in joined_lower:
|
||||
found.append(anchor)
|
||||
else:
|
||||
missing.append(anchor)
|
||||
if found:
|
||||
return True, []
|
||||
return False, missing
|
||||
172
tests/test_warrant.py
Normal file
172
tests/test_warrant.py
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
"""Tests for warrant-lite — relation-question hard check.
|
||||
|
||||
Covers:
|
||||
- is_relation_question detects "who is X's Y", "who founded Z",
|
||||
"who is the boss of W", etc.
|
||||
- extract_answer_anchors pulls multi-word proper nouns;
|
||||
falls back to solo caps skipping sentence-starter
|
||||
- warrant_check returns (True, []) when answer entity in cited
|
||||
span; (False, [missing]) when not; vacuous-pass when claim
|
||||
has no proper noun
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from aborist.qa.warrant import (
|
||||
extract_answer_anchors,
|
||||
is_relation_question,
|
||||
warrant_check,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- is_relation_question
|
||||
|
||||
|
||||
def test_is_relation_question_possessive_who():
|
||||
assert is_relation_question("who is homer simpson's boss?")
|
||||
assert is_relation_question("who is bilbo baggins's nephew?")
|
||||
assert is_relation_question("who is supermans girlfriend?")
|
||||
|
||||
|
||||
def test_is_relation_question_who_action_verb():
|
||||
assert is_relation_question("who founded microsoft?")
|
||||
assert is_relation_question("who painted the mona lisa?")
|
||||
assert is_relation_question("who wrote hamlet?")
|
||||
assert is_relation_question("who directed jaws?")
|
||||
assert is_relation_question("who invented the doppler effect?")
|
||||
|
||||
|
||||
def test_is_relation_question_relation_of():
|
||||
assert is_relation_question("the founder of microsoft")
|
||||
assert is_relation_question("the boss of homer simpson")
|
||||
assert is_relation_question("girlfriend of superman")
|
||||
|
||||
|
||||
def test_is_relation_question_not_a_relation():
|
||||
"""Broad-descriptive questions are NOT relation lookups."""
|
||||
assert not is_relation_question("tell me about connecticut")
|
||||
assert not is_relation_question("plot of red fish blue fish?")
|
||||
assert not is_relation_question("what is starcraft?")
|
||||
|
||||
|
||||
def test_is_relation_question_handles_empty_or_none():
|
||||
assert not is_relation_question("")
|
||||
assert not is_relation_question(None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- extract_answer_anchors
|
||||
|
||||
|
||||
def test_extract_anchors_finds_multiword_proper_nouns():
|
||||
anchors = extract_answer_anchors("Homer Simpson's boss is Mr. Burns.")
|
||||
# Multi-word proper-noun phrases captured.
|
||||
assert "Homer Simpson" in anchors
|
||||
assert "Mr. Burns" in anchors
|
||||
|
||||
|
||||
def test_extract_anchors_finds_lois_lane():
|
||||
anchors = extract_answer_anchors("Superman's girlfriend is Lois Lane.")
|
||||
assert "Lois Lane" in anchors
|
||||
|
||||
|
||||
def test_extract_anchors_falls_back_to_solo_caps_skipping_sentence_start():
|
||||
"""No multi-word phrases — falls back to solo caps but skips
|
||||
the sentence-starter to avoid 'Connecticut is a state...' false
|
||||
positives where the leading capital isn't a special-name anchor."""
|
||||
anchors = extract_answer_anchors("Connecticut is in New England region.")
|
||||
# Multi-word "New England" SHOULD be captured (region phrase).
|
||||
# Connecticut as the leading word + standalone is NOT a strong anchor.
|
||||
assert any("New England" in a for a in anchors)
|
||||
|
||||
|
||||
def test_extract_anchors_empty_when_no_proper_nouns():
|
||||
"""A claim with no capitalized words → no anchors."""
|
||||
anchors = extract_answer_anchors("the boy and the girl have many fish.")
|
||||
assert anchors == []
|
||||
|
||||
|
||||
def test_extract_anchors_handles_empty_text():
|
||||
assert extract_answer_anchors("") == []
|
||||
assert extract_answer_anchors(None) == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- warrant_check
|
||||
|
||||
|
||||
def test_warrant_passes_when_anchor_in_span():
|
||||
"""Standard happy path: claim names Mr. Burns, span has Mr. Burns."""
|
||||
ok, missing = warrant_check(
|
||||
"Homer Simpson's boss is Mr. Burns.",
|
||||
["Mr. Burns owns the Springfield Nuclear Power Plant where Homer works."],
|
||||
)
|
||||
assert ok is True
|
||||
assert missing == []
|
||||
|
||||
|
||||
def test_warrant_fails_when_answer_anchor_absent():
|
||||
"""The Homer-Simpson lazy-anchor case from feedback-3 (2026-05-01).
|
||||
Claim asserts Mr. Burns; cited span is voice-actor bio prose.
|
||||
Warrant check must catch this — that's the whole point of the
|
||||
layer."""
|
||||
ok, missing = warrant_check(
|
||||
"Homer Simpson's boss is Mr. Burns.",
|
||||
[
|
||||
"Castellaneta has won four Primetime Emmy Awards for "
|
||||
"Outstanding Voice-Over Performance."
|
||||
],
|
||||
)
|
||||
assert ok is False
|
||||
# "Mr. Burns" should be in the missing list (anchor extracted
|
||||
# from claim, not present in span).
|
||||
assert any("Burns" in a for a in missing), missing
|
||||
|
||||
|
||||
def test_warrant_passes_when_at_least_one_anchor_in_span():
|
||||
"""Multiple anchors in claim, at least one in span → warrant
|
||||
passes. Vacuous-tighter check would over-fire here."""
|
||||
ok, missing = warrant_check(
|
||||
"Mr. Burns owns the Springfield Nuclear Power Plant.",
|
||||
[
|
||||
"The Springfield Nuclear Power Plant is operated by "
|
||||
"Mr. Burns in the show."
|
||||
],
|
||||
)
|
||||
assert ok is True
|
||||
assert missing == []
|
||||
|
||||
|
||||
def test_warrant_vacuous_pass_when_no_anchors_in_claim():
|
||||
"""A claim with no proper-noun anchor (e.g. all common-noun
|
||||
description) has nothing to demand from cited spans —
|
||||
warrant_check returns (True, []) so the other six hard
|
||||
checks carry the load.
|
||||
|
||||
This is the same vacuous-pass discipline as the existing
|
||||
`_claim_textually_overlaps_evidence` pure-stopword case."""
|
||||
ok, missing = warrant_check(
|
||||
"the cake is a lie.",
|
||||
["completely unrelated chunk text"],
|
||||
)
|
||||
assert ok is True
|
||||
assert missing == []
|
||||
|
||||
|
||||
def test_warrant_case_insensitive_match():
|
||||
"""Anchor 'Mr. Burns' should match span 'mr. burns' — we
|
||||
care about presence, not capitalization."""
|
||||
ok, missing = warrant_check(
|
||||
"Homer's boss is Mr. Burns.",
|
||||
["the show features mr. burns as the plant owner"],
|
||||
)
|
||||
assert ok is True
|
||||
assert missing == []
|
||||
|
||||
|
||||
def test_warrant_handles_empty_inputs():
|
||||
"""No claim text → vacuous pass. No spans → fail (anchors
|
||||
can't match)."""
|
||||
ok, _ = warrant_check("", ["any text"])
|
||||
assert ok is True
|
||||
ok, missing = warrant_check("Mr. Burns is the boss.", [])
|
||||
assert ok is False
|
||||
assert "Mr. Burns" in missing
|
||||
Loading…
Add table
Add a link
Reference in a new issue