qa: warrant-lite — relation-question hard check (Ticket H)
Closes the lazy-anchor failure class fox surfaced in feedback-3
(2026-05-01) on "who is homer simpson's boss?":
Claim: "Homer Simpson's boss is Mr. Burns."
Cited span: voice-actor / Castellaneta bio prose
Pre-warrant: pointer-linked, source-role allowed, coverage passes
Post-warrant: WARRANT_MISSING violation, audit_mode capped at HYBRID
Pointer verification != warrant verification. A pointer resolves to
a span; a warrant requires the span to actually contain the claim's
named answer entity.
Adds aborist/qa/warrant.py:
- is_relation_question(q) — regex shape detector for "who is X's Y?",
"who is supermans girlfriend?", "who founded Z?", "boss of X" etc.
- extract_answer_anchors(claim) — multi-word proper nouns ("Mr.
Burns", "Lois Lane") with possessive 's stripping; solo-cap
fallback that skips sentence-starter false positives
- warrant_check(claim, spans) — at-least-one-anchor lexical match;
vacuous-pass when claim has no proper-noun anchor
Wired into verify_claim_lattice via new params (question,
warrant_check_enabled). Per-claim WARRANT_MISSING violations cap
audit_mode at HYBRID using the same demote pattern as
lazy_anchor_demoted. CLI renders " · warrant missing" tail when
violations present so the failure mode is legible without scanning
the violations array.
Policy field claim_lattice_warrant_check_enabled added to
DEFAULT_POLICY / DEFAULT_QUERY_POLICY and the verifier-policy hash
field set so changes bump verifier_policy_hash.
Hard or soft? Per CLAUDE.md "Soft hash vs hard hash":
substring/lexical = hard. Output is binary (warrant present or not),
violations enter the proof path through violations[] alongside
SOURCE_ROLE_BLOCKED and CITATION_MISMATCH.
Deferred (follow-on commits):
- full RELATION_PATTERNS table with relation-noun aliases
- two-hop bridge detection ("X founded Y" + "Y is Z" -> "X is Z")
- retrieval expansion for relation questions
Tests: 16 new unit tests in tests/test_warrant.py covering
is_relation_question, extract_answer_anchors, warrant_check
(happy path, lazy-anchor case, vacuous-pass, case-insensitive,
empty inputs). Full suite: 523 passed, 21 skipped.
This commit is contained in:
parent
63c5b2fd8a
commit
38d95dd21b
7 changed files with 472 additions and 1 deletions
|
|
@ -464,6 +464,23 @@ def _render_audit_label(audit_mode: str, verifier_method: str) -> str:
|
||||||
return f"{audit_mode} · via {verifier_method}"
|
return f"{audit_mode} · via {verifier_method}"
|
||||||
|
|
||||||
|
|
||||||
|
def _render_warrant_tail(result: dict) -> str:
|
||||||
|
"""If the verdict carries any WARRANT_MISSING violations
|
||||||
|
(relation-shape question + claim names answer entity not in
|
||||||
|
cited span), return a ` · warrant missing` tail for the
|
||||||
|
audit-line label. Otherwise empty string.
|
||||||
|
|
||||||
|
Surfaces fox's feedback-3 distinction ("pointer-linked but
|
||||||
|
relation warrant missing") at the user-facing layer without
|
||||||
|
overloading the audit_mode field. The verifier already capped
|
||||||
|
audit_mode at HYBRID for warrant-missing cases; this just
|
||||||
|
names the cap reason."""
|
||||||
|
violations = result.get("violations") or []
|
||||||
|
if any(v.get("kind") == "WARRANT_MISSING" for v in violations):
|
||||||
|
return " · warrant missing"
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
def _maybe_render_json_envelope_as_bullets(answer: str) -> str:
|
def _maybe_render_json_envelope_as_bullets(answer: str) -> str:
|
||||||
"""If `answer` is a claim_lattice JSON envelope, render bullets.
|
"""If `answer` is a claim_lattice JSON envelope, render bullets.
|
||||||
|
|
||||||
|
|
@ -563,11 +580,12 @@ def _render_query_human(result: dict, question: str) -> str:
|
||||||
# docs/verifier-semantic-gap-design.md) — never hard-checked.
|
# docs/verifier-semantic-gap-design.md) — never hard-checked.
|
||||||
# Schema column stays unchanged; this is pure display.
|
# Schema column stays unchanged; this is pure display.
|
||||||
display_label = _render_audit_label(audit, method)
|
display_label = _render_audit_label(audit, method)
|
||||||
|
warrant_tail = _render_warrant_tail(result)
|
||||||
|
|
||||||
lines: list[str] = []
|
lines: list[str] = []
|
||||||
lines.append(question)
|
lines.append(question)
|
||||||
lines.append(
|
lines.append(
|
||||||
f" {display_label} {n_verified}/{n_quotes} "
|
f" {display_label}{warrant_tail} {n_verified}/{n_quotes} "
|
||||||
f"{elapsed} ({cache_status})"
|
f"{elapsed} ({cache_status})"
|
||||||
)
|
)
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
|
||||||
|
|
@ -205,6 +205,8 @@ _VERIFIER_POLICY_FIELDS = frozenset({
|
||||||
# JSON variant identity
|
# JSON variant identity
|
||||||
"claim_lattice_use_guided_json",
|
"claim_lattice_use_guided_json",
|
||||||
"claim_lattice_json_stop_sequences",
|
"claim_lattice_json_stop_sequences",
|
||||||
|
# Warrant-lite (relation-question hard check, Ticket H, 2026-05-01)
|
||||||
|
"claim_lattice_warrant_check_enabled",
|
||||||
# Quote-mode entity policy
|
# Quote-mode entity policy
|
||||||
"entity_policy",
|
"entity_policy",
|
||||||
"entity_proximity_n",
|
"entity_proximity_n",
|
||||||
|
|
|
||||||
|
|
@ -391,6 +391,7 @@ DEFAULT_QUERY_POLICY = {
|
||||||
"claim_lattice_min_claim_content_tokens": 2,
|
"claim_lattice_min_claim_content_tokens": 2,
|
||||||
"claim_lattice_lazy_anchor_demote_threshold": 0.5,
|
"claim_lattice_lazy_anchor_demote_threshold": 0.5,
|
||||||
"claim_lattice_lazy_anchor_demote_min_pairs": 3,
|
"claim_lattice_lazy_anchor_demote_min_pairs": 3,
|
||||||
|
"claim_lattice_warrant_check_enabled": True,
|
||||||
# Claim-count ceiling — see runner.DEFAULT_POLICY for rationale.
|
# Claim-count ceiling — see runner.DEFAULT_POLICY for rationale.
|
||||||
# Bench finding (york-england "tell me all there is to know")
|
# Bench finding (york-england "tell me all there is to know")
|
||||||
# caught the runaway shape; cap of 12 admits entity-list
|
# caught the runaway shape; cap of 12 admits entity-list
|
||||||
|
|
@ -1803,6 +1804,10 @@ def query(
|
||||||
max_claims_per_answer=int(policy.get(
|
max_claims_per_answer=int(policy.get(
|
||||||
"claim_lattice_max_claims_per_answer", 12
|
"claim_lattice_max_claims_per_answer", 12
|
||||||
)),
|
)),
|
||||||
|
question=question,
|
||||||
|
warrant_check_enabled=bool(policy.get(
|
||||||
|
"claim_lattice_warrant_check_enabled", True
|
||||||
|
)),
|
||||||
)
|
)
|
||||||
rendered = verdict["rendered_text"]
|
rendered = verdict["rendered_text"]
|
||||||
answer_text = rendered if rendered else raw_answer
|
answer_text = rendered if rendered else raw_answer
|
||||||
|
|
@ -2144,6 +2149,12 @@ def query(
|
||||||
"verifier_method": verdict["verifier_method"],
|
"verifier_method": verdict["verifier_method"],
|
||||||
"unverified_quotes": verdict["unverified_quotes"],
|
"unverified_quotes": verdict["unverified_quotes"],
|
||||||
"partially_verified_quotes": verdict.get("partially_verified_quotes") or [],
|
"partially_verified_quotes": verdict.get("partially_verified_quotes") or [],
|
||||||
|
# Violations (per-claim hard-check failures + soft demotes
|
||||||
|
# like LAZY_ANCHOR_DEMOTE / WARRANT_MISSING). Surfaced so the
|
||||||
|
# CLI render can name WHY a verdict capped at HYBRID instead
|
||||||
|
# of reaching STRICT. Persisted into run_dag_blob via the
|
||||||
|
# verify stage's payload.
|
||||||
|
"violations": verdict.get("violations") or [],
|
||||||
# Sidecar smell signals (claim_lattice mode only) — surfaced
|
# Sidecar smell signals (claim_lattice mode only) — surfaced
|
||||||
# for the renderer; never persisted in providence_cache and
|
# for the renderer; never persisted in providence_cache and
|
||||||
# never threaded into run_dag_root.
|
# never threaded into run_dag_root.
|
||||||
|
|
|
||||||
|
|
@ -233,6 +233,15 @@ DEFAULT_POLICY = {
|
||||||
# diverse anchoring across pointers.
|
# diverse anchoring across pointers.
|
||||||
"claim_lattice_lazy_anchor_demote_threshold": 0.5,
|
"claim_lattice_lazy_anchor_demote_threshold": 0.5,
|
||||||
"claim_lattice_lazy_anchor_demote_min_pairs": 3,
|
"claim_lattice_lazy_anchor_demote_min_pairs": 3,
|
||||||
|
# Warrant-lite — relation-question hard check (Ticket H from
|
||||||
|
# feedback-3, 2026-05-01). Detects relation-shape questions
|
||||||
|
# ("who is X's boss?", "who founded Y?") and requires the cited
|
||||||
|
# span to contain at least one named answer entity (proper-noun
|
||||||
|
# phrase) from the claim. Catches the Homer-Simpson lazy-anchor
|
||||||
|
# case where claim asserts "Mr. Burns" but cited span is the
|
||||||
|
# voice-actor bio. WARRANT_MISSING violations cap audit_mode
|
||||||
|
# at HYBRID. See aborist/qa/warrant.py.
|
||||||
|
"claim_lattice_warrant_check_enabled": True,
|
||||||
# Claim-count ceiling. Bench finding (2026-04-30 york-england):
|
# Claim-count ceiling. Bench finding (2026-04-30 york-england):
|
||||||
# "tell me all there is to know about X" prompted Hermes to spam
|
# "tell me all there is to know about X" prompted Hermes to spam
|
||||||
# 26-59 encyclopedic claims sourced from training, only 2-4 of
|
# 26-59 encyclopedic claims sourced from training, only 2-4 of
|
||||||
|
|
@ -608,6 +617,10 @@ def ask(
|
||||||
max_claims_per_answer=int(policy.get(
|
max_claims_per_answer=int(policy.get(
|
||||||
"claim_lattice_max_claims_per_answer", 12
|
"claim_lattice_max_claims_per_answer", 12
|
||||||
)),
|
)),
|
||||||
|
question=question,
|
||||||
|
warrant_check_enabled=bool(policy.get(
|
||||||
|
"claim_lattice_warrant_check_enabled", True
|
||||||
|
)),
|
||||||
)
|
)
|
||||||
# Rendered prose (literal spans interpolated) is the user-facing
|
# Rendered prose (literal spans interpolated) is the user-facing
|
||||||
# answer text — never the model's raw pointer-line output. If
|
# answer text — never the model's raw pointer-line output. If
|
||||||
|
|
|
||||||
|
|
@ -47,6 +47,8 @@ from __future__ import annotations
|
||||||
import re
|
import re
|
||||||
import unicodedata
|
import unicodedata
|
||||||
|
|
||||||
|
from aborist.qa.warrant import is_relation_question, warrant_check
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from aborist.wikitext import to_base as _wikitext_to_base
|
from aborist.wikitext import to_base as _wikitext_to_base
|
||||||
except ImportError: # pragma: no cover
|
except ImportError: # pragma: no cover
|
||||||
|
|
@ -994,6 +996,8 @@ def verify_claim_lattice(
|
||||||
lazy_anchor_demote_threshold: float = DEFAULT_LAZY_ANCHOR_DEMOTE_THRESHOLD,
|
lazy_anchor_demote_threshold: float = DEFAULT_LAZY_ANCHOR_DEMOTE_THRESHOLD,
|
||||||
lazy_anchor_demote_min_pairs: int = DEFAULT_LAZY_ANCHOR_DEMOTE_MIN_PAIRS,
|
lazy_anchor_demote_min_pairs: int = DEFAULT_LAZY_ANCHOR_DEMOTE_MIN_PAIRS,
|
||||||
max_claims_per_answer: int = DEFAULT_MAX_CLAIMS_PER_ANSWER,
|
max_claims_per_answer: int = DEFAULT_MAX_CLAIMS_PER_ANSWER,
|
||||||
|
question: str | None = None,
|
||||||
|
warrant_check_enabled: bool = True,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Deterministic verifier for ``answer_mode="claim_lattice_pointer"``.
|
"""Deterministic verifier for ``answer_mode="claim_lattice_pointer"``.
|
||||||
|
|
||||||
|
|
@ -1350,6 +1354,45 @@ def verify_claim_lattice(
|
||||||
"threshold": lazy_anchor_demote_threshold,
|
"threshold": lazy_anchor_demote_threshold,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
# Warrant-lite — relation-question hard check (Ticket H from
|
||||||
|
# feedback-3, 2026-05-01). Claim-cited spans must contain at
|
||||||
|
# least one of the claim's named answer entities (proper-noun
|
||||||
|
# phrases). Catches the Homer-Simpson lazy-anchor case fox
|
||||||
|
# surfaced — claim asserts "Mr. Burns" but cited span is
|
||||||
|
# Castellaneta voice-actor prose. See aborist/qa/warrant.py
|
||||||
|
# for the lexical algorithm and rationale (deterministic,
|
||||||
|
# not NLI). Fires only when the question shape suggests a
|
||||||
|
# relation lookup AND the lookup is enabled by policy
|
||||||
|
# (warrant_check_enabled). Per-claim WARRANT_MISSING violations
|
||||||
|
# cap audit_mode at HYBRID via the same demote pattern as
|
||||||
|
# lazy_anchor_demoted.
|
||||||
|
warrant_missing_claims: list[int] = []
|
||||||
|
if warrant_check_enabled and question and is_relation_question(question):
|
||||||
|
for cs in claim_statuses:
|
||||||
|
if cs.get("status") not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"):
|
||||||
|
continue
|
||||||
|
cited_eids = cs.get("evidence_ids") or []
|
||||||
|
cited_spans = [
|
||||||
|
obj.span
|
||||||
|
for eid in cited_eids
|
||||||
|
for obj in [evidence_map_by_evidence_id_local(evidence_map, eid)]
|
||||||
|
if obj is not None
|
||||||
|
]
|
||||||
|
ok, missing = warrant_check(cs.get("text") or "", cited_spans)
|
||||||
|
if not ok:
|
||||||
|
warrant_missing_claims.append(cs.get("claim_idx"))
|
||||||
|
violations.append({
|
||||||
|
"kind": "WARRANT_MISSING",
|
||||||
|
"claim_idx": cs.get("claim_idx"),
|
||||||
|
"missing_anchors": missing,
|
||||||
|
"rationale": (
|
||||||
|
"claim names answer entity not present in any "
|
||||||
|
"cited span — pointer-linked but warrant missing"
|
||||||
|
),
|
||||||
|
})
|
||||||
|
if warrant_missing_claims and audit_mode == "STRICT":
|
||||||
|
audit_mode = "HYBRID"
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"n_quotes": n_pairs,
|
"n_quotes": n_pairs,
|
||||||
"n_verified": n_pairs_verified,
|
"n_verified": n_pairs_verified,
|
||||||
|
|
@ -1364,9 +1407,22 @@ def verify_claim_lattice(
|
||||||
"pointer_id_distribution": pointer_distribution,
|
"pointer_id_distribution": pointer_distribution,
|
||||||
"lazy_anchor_ratio": lazy_anchor_ratio,
|
"lazy_anchor_ratio": lazy_anchor_ratio,
|
||||||
"lazy_anchor_demoted": lazy_anchor_demoted,
|
"lazy_anchor_demoted": lazy_anchor_demoted,
|
||||||
|
"warrant_missing_claim_idxs": warrant_missing_claims,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def evidence_map_by_evidence_id_local(evidence_map, eid: str):
|
||||||
|
"""Local helper — returns the EvidenceObject whose ``evidence_id``
|
||||||
|
matches ``eid``, or None. Avoids the import-cycle risk of pulling
|
||||||
|
`evidence_map_by_evidence_id` into this module's hot path; the
|
||||||
|
O(N) walk is fine since evidence maps are <30 entries.
|
||||||
|
"""
|
||||||
|
for obj in evidence_map or []:
|
||||||
|
if obj.evidence_id == eid:
|
||||||
|
return obj
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# JSON variant — `answer_mode="claim_lattice"`. Same lattice semantics as
|
# JSON variant — `answer_mode="claim_lattice"`. Same lattice semantics as
|
||||||
# the pointer variant, but the model emits a structured JSON object
|
# the pointer variant, but the model emits a structured JSON object
|
||||||
|
|
|
||||||
199
aborist/qa/warrant.py
Normal file
199
aborist/qa/warrant.py
Normal file
|
|
@ -0,0 +1,199 @@
|
||||||
|
"""Warrant-lite — relation-question hard check on claim-cited evidence.
|
||||||
|
|
||||||
|
Closes the lazy-anchor failure class fox surfaced in feedback-3
|
||||||
|
(2026-05-01) on "who is homer simpson's boss?":
|
||||||
|
|
||||||
|
Claim: "Homer Simpson's boss is Mr. Burns."
|
||||||
|
Cited span: voice-actor / Castellaneta bio prose
|
||||||
|
→ Pointer resolves, source role allowed, coverage passes
|
||||||
|
→ BUT span contains no "Mr. Burns" or boss/employer relation
|
||||||
|
→ Pre-warrant: STRICT/HYBRID with bogus citation
|
||||||
|
→ Post-warrant: WARRANT_MISSING violation, audit_mode capped
|
||||||
|
|
||||||
|
The principle: pointer verification ≠ warrant verification.
|
||||||
|
A pointer resolves to a span; a warrant requires the span to
|
||||||
|
actually contain the claim's load-bearing tokens (the named
|
||||||
|
ANSWER ENTITY, primarily — relation aliases left as a follow-on).
|
||||||
|
|
||||||
|
This module implements the deterministic, lexical-only first
|
||||||
|
pass of §11-§14. Two-hop bridge detection + the full
|
||||||
|
RELATION_PATTERNS registry are deferred to a successor commit
|
||||||
|
once the bench shows what answer-anchor alone catches.
|
||||||
|
|
||||||
|
Hard or soft? Per CLAUDE.md "Soft hash vs hard hash":
|
||||||
|
substring/lexical = hard. The check uses `_content_tokens`
|
||||||
|
(deterministic) and proper-noun extraction via regex
|
||||||
|
(deterministic). Output is binary: warrant present or not.
|
||||||
|
That makes it a hard check — its violations enter the
|
||||||
|
proof path through `violations[]` exactly the way
|
||||||
|
SOURCE_ROLE_BLOCKED and CITATION_MISMATCH do.
|
||||||
|
|
||||||
|
But unlike those, WARRANT_MISSING does not block per-pointer
|
||||||
|
verification — it caps audit_mode at HYBRID via the same
|
||||||
|
mechanism `lazy_anchor_demoted` uses. The lexical pointer
|
||||||
|
checks still pass; the user/auditor sees both "pointer-linked"
|
||||||
|
AND "warrant missing" so the failure mode is legible.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
# Heuristic relation-question detector. Fires when the question
|
||||||
|
# shape fits a "X's Y" / "who Z's W?" / "who owns Y?" / "who founded Z?"
|
||||||
|
# pattern — the cases fox names in feedback-3. Conservative: false
|
||||||
|
# negatives are fine (warrant just doesn't run); false positives are
|
||||||
|
# the real cost (warrant fires on questions where it shouldn't).
|
||||||
|
# Relation-noun aliases — used both in "who is X's <noun>?" and
|
||||||
|
# "<noun> of X" shapes. Possessive-less variant ("who is supermans
|
||||||
|
# girlfriend?") needs a separate pattern path since `\S+` alone won't
|
||||||
|
# bind to a relation noun without an explicit alias list.
|
||||||
|
_RELATION_NOUNS = (
|
||||||
|
"boss|employer|supervisor|manager|owner|founder|director|creator|"
|
||||||
|
"inventor|author|composer|painter|spouse|wife|husband|mother|father|"
|
||||||
|
"parent|son|daughter|sister|brother|nephew|niece|aunt|uncle|cousin|"
|
||||||
|
"girlfriend|boyfriend|partner|fiancee|fiance|friend|enemy|rival|"
|
||||||
|
"successor|predecessor|mentor|teacher|student|coach|captain"
|
||||||
|
)
|
||||||
|
_RELATION_VERBS = (
|
||||||
|
"owns|founded|directed|created|invented|wrote|painted|composed|"
|
||||||
|
"discovered|killed|married|defeated|coached|hired|fired|sang|"
|
||||||
|
"produced|designed|built|sculpted|filmed"
|
||||||
|
)
|
||||||
|
_RELATION_PATTERNS = (
|
||||||
|
# "who is X's Y?" — apostrophe-s possessive (multi-word or single)
|
||||||
|
re.compile(r"\bwho\s+is\s+\S+(?:\s+\S+)*\s*'s\s+\w+", re.IGNORECASE),
|
||||||
|
# "who is supermans girlfriend?" — possessive without apostrophe,
|
||||||
|
# gated on relation-noun alias list so generic nouns don't trip it
|
||||||
|
re.compile(
|
||||||
|
r"\bwho\s+is\s+\S+(?:\s+\S+)*\s+(?:" + _RELATION_NOUNS + r")\b",
|
||||||
|
re.IGNORECASE,
|
||||||
|
),
|
||||||
|
# "who founded microsoft?" — relation-verb
|
||||||
|
re.compile(r"\bwho\s+(?:" + _RELATION_VERBS + r")\b", re.IGNORECASE),
|
||||||
|
# "what is X's Y?" — possessive on a what-question
|
||||||
|
re.compile(r"\bwhat\s+is\s+\S+(?:\s+\S+)*\s*'s\s+\w+", re.IGNORECASE),
|
||||||
|
# "<relation-noun> of X" — bare relation-of phrase
|
||||||
|
re.compile(r"\b(?:" + _RELATION_NOUNS + r")\s+of\b", re.IGNORECASE),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def is_relation_question(question: str) -> bool:
|
||||||
|
"""Return True iff the question shape suggests a relation lookup.
|
||||||
|
|
||||||
|
Conservative — defaults to False on any question that doesn't
|
||||||
|
match a known relation-shape regex. False negatives leave the
|
||||||
|
warrant check inactive (claim verifies under the existing six
|
||||||
|
hard checks alone); false positives risk over-firing on
|
||||||
|
non-relation questions (worth tuning if the bench shows it).
|
||||||
|
"""
|
||||||
|
if not question:
|
||||||
|
return False
|
||||||
|
for pat in _RELATION_PATTERNS:
|
||||||
|
if pat.search(question):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
# Multi-word proper noun (Title-Case sequences) — captures "Lois
|
||||||
|
# Lane", "Mr. Burns", "New England", "Homer Simpson's". The first-leg
|
||||||
|
# alternation lets a token end in `\.` ONLY when followed by another
|
||||||
|
# title-cased token (so "Mr." in "Mr. Burns" matches but "Burns." at
|
||||||
|
# sentence end does not slurp the trailing period). Trailing `'s` /
|
||||||
|
# stray punctuation are stripped in post-processing.
|
||||||
|
_PROPER_NOUN_RE = re.compile(
|
||||||
|
r"\b(?:[A-Z][A-Za-z'’\-]*\.(?=[ \t]+[A-Z])|[A-Z][A-Za-z'’\-]+)"
|
||||||
|
r"(?:[ \t]+(?:[A-Z][A-Za-z'’\-]*\.(?=[ \t]+[A-Z])|[A-Z][A-Za-z'’\-]+))+"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Solo-capitalized tokens (e.g. "Apple", "Burns"). Less reliable
|
||||||
|
# than multi-word phrases since sentence-start words look like
|
||||||
|
# proper nouns too — used only when no multi-word phrase is
|
||||||
|
# available.
|
||||||
|
_SOLO_CAP_TOKEN_RE = re.compile(r"\b[A-Z][A-Za-z'’\-]{2,}\b")
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_anchor_tail(anchor: str) -> str:
|
||||||
|
"""Strip possessive `'s` / `’s` and stray trailing punctuation."""
|
||||||
|
a = anchor.rstrip(".,;:!?")
|
||||||
|
for tail in ("'s", "’s", "'", "’"):
|
||||||
|
if a.endswith(tail):
|
||||||
|
a = a[: -len(tail)]
|
||||||
|
break
|
||||||
|
return a.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def extract_answer_anchors(claim_text: str) -> list[str]:
|
||||||
|
"""Pull candidate answer-entity anchors from a claim's text.
|
||||||
|
|
||||||
|
Strategy:
|
||||||
|
1. Multi-word proper-noun phrases (`"Mr. Burns"`, `"Lois Lane"`,
|
||||||
|
`"Nineteen Eighty-Four"`) — these are reliable named
|
||||||
|
entities even at sentence start.
|
||||||
|
2. If none found, fall back to solo capitalized tokens that
|
||||||
|
are not the FIRST word of the claim (skipping
|
||||||
|
sentence-starter false positives like the leading
|
||||||
|
"Connecticut" in "Connecticut is a state...").
|
||||||
|
|
||||||
|
Each anchor is post-stripped of possessive `'s` and stray
|
||||||
|
sentence-end punctuation so downstream substring tests find
|
||||||
|
base forms in cited spans.
|
||||||
|
|
||||||
|
Returns a list of anchor strings; empty list if no anchor
|
||||||
|
detectable. Lowercase consumers should `.lower()` per call.
|
||||||
|
"""
|
||||||
|
if not claim_text:
|
||||||
|
return []
|
||||||
|
raw = _PROPER_NOUN_RE.findall(claim_text)
|
||||||
|
if raw:
|
||||||
|
cleaned = []
|
||||||
|
for a in raw:
|
||||||
|
stripped = _strip_anchor_tail(a)
|
||||||
|
if stripped:
|
||||||
|
cleaned.append(stripped)
|
||||||
|
return cleaned
|
||||||
|
# Solo-cap fallback. Skip the first token (sentence starter).
|
||||||
|
tokens = _SOLO_CAP_TOKEN_RE.findall(claim_text)
|
||||||
|
if len(tokens) <= 1:
|
||||||
|
return []
|
||||||
|
return [_strip_anchor_tail(t) for t in tokens[1:] if _strip_anchor_tail(t)]
|
||||||
|
|
||||||
|
|
||||||
|
def warrant_check(
|
||||||
|
claim_text: str,
|
||||||
|
cited_spans: list[str],
|
||||||
|
) -> tuple[bool, list[str]]:
|
||||||
|
"""Hard lexical warrant check on a relation-shape claim.
|
||||||
|
|
||||||
|
Returns ``(warrant_ok, missing_anchors)``:
|
||||||
|
- ``warrant_ok=True`` and ``missing_anchors=[]`` when at least
|
||||||
|
ONE answer anchor extracted from the claim text appears as
|
||||||
|
a substring in at least ONE cited span (case-insensitive).
|
||||||
|
That's the "pointer cited a span that genuinely names the
|
||||||
|
answer" gate.
|
||||||
|
- ``warrant_ok=False`` and ``missing_anchors=[...]`` when none
|
||||||
|
of the extracted anchors appear in any cited span. The
|
||||||
|
list names which anchors the claim asserted but no cited
|
||||||
|
span supports — the auditor sees what specifically was
|
||||||
|
missing.
|
||||||
|
|
||||||
|
Vacuous-pass: if no answer anchor can be extracted from the
|
||||||
|
claim text at all (the claim has no proper noun, e.g. "X is
|
||||||
|
located here"), warrant_check returns (True, []) — there is
|
||||||
|
no answer entity to demand from the spans. The other six
|
||||||
|
hard checks (coverage, source_role, etc.) carry the load.
|
||||||
|
"""
|
||||||
|
anchors = extract_answer_anchors(claim_text)
|
||||||
|
if not anchors:
|
||||||
|
return True, []
|
||||||
|
joined_lower = " ".join(s.lower() for s in cited_spans)
|
||||||
|
found = []
|
||||||
|
missing = []
|
||||||
|
for anchor in anchors:
|
||||||
|
if anchor.lower() in joined_lower:
|
||||||
|
found.append(anchor)
|
||||||
|
else:
|
||||||
|
missing.append(anchor)
|
||||||
|
if found:
|
||||||
|
return True, []
|
||||||
|
return False, missing
|
||||||
172
tests/test_warrant.py
Normal file
172
tests/test_warrant.py
Normal file
|
|
@ -0,0 +1,172 @@
|
||||||
|
"""Tests for warrant-lite — relation-question hard check.
|
||||||
|
|
||||||
|
Covers:
|
||||||
|
- is_relation_question detects "who is X's Y", "who founded Z",
|
||||||
|
"who is the boss of W", etc.
|
||||||
|
- extract_answer_anchors pulls multi-word proper nouns;
|
||||||
|
falls back to solo caps skipping sentence-starter
|
||||||
|
- warrant_check returns (True, []) when answer entity in cited
|
||||||
|
span; (False, [missing]) when not; vacuous-pass when claim
|
||||||
|
has no proper noun
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from aborist.qa.warrant import (
|
||||||
|
extract_answer_anchors,
|
||||||
|
is_relation_question,
|
||||||
|
warrant_check,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- is_relation_question
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_relation_question_possessive_who():
|
||||||
|
assert is_relation_question("who is homer simpson's boss?")
|
||||||
|
assert is_relation_question("who is bilbo baggins's nephew?")
|
||||||
|
assert is_relation_question("who is supermans girlfriend?")
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_relation_question_who_action_verb():
|
||||||
|
assert is_relation_question("who founded microsoft?")
|
||||||
|
assert is_relation_question("who painted the mona lisa?")
|
||||||
|
assert is_relation_question("who wrote hamlet?")
|
||||||
|
assert is_relation_question("who directed jaws?")
|
||||||
|
assert is_relation_question("who invented the doppler effect?")
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_relation_question_relation_of():
|
||||||
|
assert is_relation_question("the founder of microsoft")
|
||||||
|
assert is_relation_question("the boss of homer simpson")
|
||||||
|
assert is_relation_question("girlfriend of superman")
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_relation_question_not_a_relation():
|
||||||
|
"""Broad-descriptive questions are NOT relation lookups."""
|
||||||
|
assert not is_relation_question("tell me about connecticut")
|
||||||
|
assert not is_relation_question("plot of red fish blue fish?")
|
||||||
|
assert not is_relation_question("what is starcraft?")
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_relation_question_handles_empty_or_none():
|
||||||
|
assert not is_relation_question("")
|
||||||
|
assert not is_relation_question(None)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- extract_answer_anchors
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_anchors_finds_multiword_proper_nouns():
|
||||||
|
anchors = extract_answer_anchors("Homer Simpson's boss is Mr. Burns.")
|
||||||
|
# Multi-word proper-noun phrases captured.
|
||||||
|
assert "Homer Simpson" in anchors
|
||||||
|
assert "Mr. Burns" in anchors
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_anchors_finds_lois_lane():
|
||||||
|
anchors = extract_answer_anchors("Superman's girlfriend is Lois Lane.")
|
||||||
|
assert "Lois Lane" in anchors
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_anchors_falls_back_to_solo_caps_skipping_sentence_start():
|
||||||
|
"""No multi-word phrases — falls back to solo caps but skips
|
||||||
|
the sentence-starter to avoid 'Connecticut is a state...' false
|
||||||
|
positives where the leading capital isn't a special-name anchor."""
|
||||||
|
anchors = extract_answer_anchors("Connecticut is in New England region.")
|
||||||
|
# Multi-word "New England" SHOULD be captured (region phrase).
|
||||||
|
# Connecticut as the leading word + standalone is NOT a strong anchor.
|
||||||
|
assert any("New England" in a for a in anchors)
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_anchors_empty_when_no_proper_nouns():
|
||||||
|
"""A claim with no capitalized words → no anchors."""
|
||||||
|
anchors = extract_answer_anchors("the boy and the girl have many fish.")
|
||||||
|
assert anchors == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_anchors_handles_empty_text():
|
||||||
|
assert extract_answer_anchors("") == []
|
||||||
|
assert extract_answer_anchors(None) == []
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- warrant_check
|
||||||
|
|
||||||
|
|
||||||
|
def test_warrant_passes_when_anchor_in_span():
|
||||||
|
"""Standard happy path: claim names Mr. Burns, span has Mr. Burns."""
|
||||||
|
ok, missing = warrant_check(
|
||||||
|
"Homer Simpson's boss is Mr. Burns.",
|
||||||
|
["Mr. Burns owns the Springfield Nuclear Power Plant where Homer works."],
|
||||||
|
)
|
||||||
|
assert ok is True
|
||||||
|
assert missing == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_warrant_fails_when_answer_anchor_absent():
|
||||||
|
"""The Homer-Simpson lazy-anchor case from feedback-3 (2026-05-01).
|
||||||
|
Claim asserts Mr. Burns; cited span is voice-actor bio prose.
|
||||||
|
Warrant check must catch this — that's the whole point of the
|
||||||
|
layer."""
|
||||||
|
ok, missing = warrant_check(
|
||||||
|
"Homer Simpson's boss is Mr. Burns.",
|
||||||
|
[
|
||||||
|
"Castellaneta has won four Primetime Emmy Awards for "
|
||||||
|
"Outstanding Voice-Over Performance."
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert ok is False
|
||||||
|
# "Mr. Burns" should be in the missing list (anchor extracted
|
||||||
|
# from claim, not present in span).
|
||||||
|
assert any("Burns" in a for a in missing), missing
|
||||||
|
|
||||||
|
|
||||||
|
def test_warrant_passes_when_at_least_one_anchor_in_span():
|
||||||
|
"""Multiple anchors in claim, at least one in span → warrant
|
||||||
|
passes. Vacuous-tighter check would over-fire here."""
|
||||||
|
ok, missing = warrant_check(
|
||||||
|
"Mr. Burns owns the Springfield Nuclear Power Plant.",
|
||||||
|
[
|
||||||
|
"The Springfield Nuclear Power Plant is operated by "
|
||||||
|
"Mr. Burns in the show."
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert ok is True
|
||||||
|
assert missing == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_warrant_vacuous_pass_when_no_anchors_in_claim():
|
||||||
|
"""A claim with no proper-noun anchor (e.g. all common-noun
|
||||||
|
description) has nothing to demand from cited spans —
|
||||||
|
warrant_check returns (True, []) so the other six hard
|
||||||
|
checks carry the load.
|
||||||
|
|
||||||
|
This is the same vacuous-pass discipline as the existing
|
||||||
|
`_claim_textually_overlaps_evidence` pure-stopword case."""
|
||||||
|
ok, missing = warrant_check(
|
||||||
|
"the cake is a lie.",
|
||||||
|
["completely unrelated chunk text"],
|
||||||
|
)
|
||||||
|
assert ok is True
|
||||||
|
assert missing == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_warrant_case_insensitive_match():
|
||||||
|
"""Anchor 'Mr. Burns' should match span 'mr. burns' — we
|
||||||
|
care about presence, not capitalization."""
|
||||||
|
ok, missing = warrant_check(
|
||||||
|
"Homer's boss is Mr. Burns.",
|
||||||
|
["the show features mr. burns as the plant owner"],
|
||||||
|
)
|
||||||
|
assert ok is True
|
||||||
|
assert missing == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_warrant_handles_empty_inputs():
|
||||||
|
"""No claim text → vacuous pass. No spans → fail (anchors
|
||||||
|
can't match)."""
|
||||||
|
ok, _ = warrant_check("", ["any text"])
|
||||||
|
assert ok is True
|
||||||
|
ok, missing = warrant_check("Mr. Burns is the boss.", [])
|
||||||
|
assert ok is False
|
||||||
|
assert "Mr. Burns" in missing
|
||||||
Loading…
Add table
Add a link
Reference in a new issue