diff --git a/aborist/cli.py b/aborist/cli.py index 89ef3a1..f032ca1 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -464,6 +464,23 @@ def _render_audit_label(audit_mode: str, verifier_method: str) -> str: return f"{audit_mode} · via {verifier_method}" +def _render_warrant_tail(result: dict) -> str: + """If the verdict carries any WARRANT_MISSING violations + (relation-shape question + claim names answer entity not in + cited span), return a ` · warrant missing` tail for the + audit-line label. Otherwise empty string. + + Surfaces fox's feedback-3 distinction ("pointer-linked but + relation warrant missing") at the user-facing layer without + overloading the audit_mode field. The verifier already capped + audit_mode at HYBRID for warrant-missing cases; this just + names the cap reason.""" + violations = result.get("violations") or [] + if any(v.get("kind") == "WARRANT_MISSING" for v in violations): + return " · warrant missing" + return "" + + def _maybe_render_json_envelope_as_bullets(answer: str) -> str: """If `answer` is a claim_lattice JSON envelope, render bullets. @@ -563,11 +580,12 @@ def _render_query_human(result: dict, question: str) -> str: # docs/verifier-semantic-gap-design.md) — never hard-checked. # Schema column stays unchanged; this is pure display. display_label = _render_audit_label(audit, method) + warrant_tail = _render_warrant_tail(result) lines: list[str] = [] lines.append(question) lines.append( - f" {display_label} {n_verified}/{n_quotes} " + f" {display_label}{warrant_tail} {n_verified}/{n_quotes} " f"{elapsed} ({cache_status})" ) lines.append("") diff --git a/aborist/qa/keys.py b/aborist/qa/keys.py index 871e17f..4a454ec 100644 --- a/aborist/qa/keys.py +++ b/aborist/qa/keys.py @@ -205,6 +205,8 @@ _VERIFIER_POLICY_FIELDS = frozenset({ # JSON variant identity "claim_lattice_use_guided_json", "claim_lattice_json_stop_sequences", + # Warrant-lite (relation-question hard check, Ticket H, 2026-05-01) + "claim_lattice_warrant_check_enabled", # Quote-mode entity policy "entity_policy", "entity_proximity_n", diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 76d967b..2f7cbde 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -391,6 +391,7 @@ DEFAULT_QUERY_POLICY = { "claim_lattice_min_claim_content_tokens": 2, "claim_lattice_lazy_anchor_demote_threshold": 0.5, "claim_lattice_lazy_anchor_demote_min_pairs": 3, + "claim_lattice_warrant_check_enabled": True, # Claim-count ceiling — see runner.DEFAULT_POLICY for rationale. # Bench finding (york-england "tell me all there is to know") # caught the runaway shape; cap of 12 admits entity-list @@ -1803,6 +1804,10 @@ def query( max_claims_per_answer=int(policy.get( "claim_lattice_max_claims_per_answer", 12 )), + question=question, + warrant_check_enabled=bool(policy.get( + "claim_lattice_warrant_check_enabled", True + )), ) rendered = verdict["rendered_text"] answer_text = rendered if rendered else raw_answer @@ -2144,6 +2149,12 @@ def query( "verifier_method": verdict["verifier_method"], "unverified_quotes": verdict["unverified_quotes"], "partially_verified_quotes": verdict.get("partially_verified_quotes") or [], + # Violations (per-claim hard-check failures + soft demotes + # like LAZY_ANCHOR_DEMOTE / WARRANT_MISSING). Surfaced so the + # CLI render can name WHY a verdict capped at HYBRID instead + # of reaching STRICT. Persisted into run_dag_blob via the + # verify stage's payload. + "violations": verdict.get("violations") or [], # Sidecar smell signals (claim_lattice mode only) — surfaced # for the renderer; never persisted in providence_cache and # never threaded into run_dag_root. diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index 6d6a156..5701f95 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -233,6 +233,15 @@ DEFAULT_POLICY = { # diverse anchoring across pointers. "claim_lattice_lazy_anchor_demote_threshold": 0.5, "claim_lattice_lazy_anchor_demote_min_pairs": 3, + # Warrant-lite — relation-question hard check (Ticket H from + # feedback-3, 2026-05-01). Detects relation-shape questions + # ("who is X's boss?", "who founded Y?") and requires the cited + # span to contain at least one named answer entity (proper-noun + # phrase) from the claim. Catches the Homer-Simpson lazy-anchor + # case where claim asserts "Mr. Burns" but cited span is the + # voice-actor bio. WARRANT_MISSING violations cap audit_mode + # at HYBRID. See aborist/qa/warrant.py. + "claim_lattice_warrant_check_enabled": True, # Claim-count ceiling. Bench finding (2026-04-30 york-england): # "tell me all there is to know about X" prompted Hermes to spam # 26-59 encyclopedic claims sourced from training, only 2-4 of @@ -608,6 +617,10 @@ def ask( max_claims_per_answer=int(policy.get( "claim_lattice_max_claims_per_answer", 12 )), + question=question, + warrant_check_enabled=bool(policy.get( + "claim_lattice_warrant_check_enabled", True + )), ) # Rendered prose (literal spans interpolated) is the user-facing # answer text — never the model's raw pointer-line output. If diff --git a/aborist/qa/verify.py b/aborist/qa/verify.py index e96bdc5..86195fa 100644 --- a/aborist/qa/verify.py +++ b/aborist/qa/verify.py @@ -47,6 +47,8 @@ from __future__ import annotations import re import unicodedata +from aborist.qa.warrant import is_relation_question, warrant_check + try: from aborist.wikitext import to_base as _wikitext_to_base except ImportError: # pragma: no cover @@ -994,6 +996,8 @@ def verify_claim_lattice( lazy_anchor_demote_threshold: float = DEFAULT_LAZY_ANCHOR_DEMOTE_THRESHOLD, lazy_anchor_demote_min_pairs: int = DEFAULT_LAZY_ANCHOR_DEMOTE_MIN_PAIRS, max_claims_per_answer: int = DEFAULT_MAX_CLAIMS_PER_ANSWER, + question: str | None = None, + warrant_check_enabled: bool = True, ) -> dict: """Deterministic verifier for ``answer_mode="claim_lattice_pointer"``. @@ -1350,6 +1354,45 @@ def verify_claim_lattice( "threshold": lazy_anchor_demote_threshold, }) + # Warrant-lite — relation-question hard check (Ticket H from + # feedback-3, 2026-05-01). Claim-cited spans must contain at + # least one of the claim's named answer entities (proper-noun + # phrases). Catches the Homer-Simpson lazy-anchor case fox + # surfaced — claim asserts "Mr. Burns" but cited span is + # Castellaneta voice-actor prose. See aborist/qa/warrant.py + # for the lexical algorithm and rationale (deterministic, + # not NLI). Fires only when the question shape suggests a + # relation lookup AND the lookup is enabled by policy + # (warrant_check_enabled). Per-claim WARRANT_MISSING violations + # cap audit_mode at HYBRID via the same demote pattern as + # lazy_anchor_demoted. + warrant_missing_claims: list[int] = [] + if warrant_check_enabled and question and is_relation_question(question): + for cs in claim_statuses: + if cs.get("status") not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"): + continue + cited_eids = cs.get("evidence_ids") or [] + cited_spans = [ + obj.span + for eid in cited_eids + for obj in [evidence_map_by_evidence_id_local(evidence_map, eid)] + if obj is not None + ] + ok, missing = warrant_check(cs.get("text") or "", cited_spans) + if not ok: + warrant_missing_claims.append(cs.get("claim_idx")) + violations.append({ + "kind": "WARRANT_MISSING", + "claim_idx": cs.get("claim_idx"), + "missing_anchors": missing, + "rationale": ( + "claim names answer entity not present in any " + "cited span — pointer-linked but warrant missing" + ), + }) + if warrant_missing_claims and audit_mode == "STRICT": + audit_mode = "HYBRID" + return { "n_quotes": n_pairs, "n_verified": n_pairs_verified, @@ -1364,9 +1407,22 @@ def verify_claim_lattice( "pointer_id_distribution": pointer_distribution, "lazy_anchor_ratio": lazy_anchor_ratio, "lazy_anchor_demoted": lazy_anchor_demoted, + "warrant_missing_claim_idxs": warrant_missing_claims, } +def evidence_map_by_evidence_id_local(evidence_map, eid: str): + """Local helper — returns the EvidenceObject whose ``evidence_id`` + matches ``eid``, or None. Avoids the import-cycle risk of pulling + `evidence_map_by_evidence_id` into this module's hot path; the + O(N) walk is fine since evidence maps are <30 entries. + """ + for obj in evidence_map or []: + if obj.evidence_id == eid: + return obj + return None + + # --------------------------------------------------------------------------- # JSON variant — `answer_mode="claim_lattice"`. Same lattice semantics as # the pointer variant, but the model emits a structured JSON object diff --git a/aborist/qa/warrant.py b/aborist/qa/warrant.py new file mode 100644 index 0000000..a1ec2ac --- /dev/null +++ b/aborist/qa/warrant.py @@ -0,0 +1,199 @@ +"""Warrant-lite — relation-question hard check on claim-cited evidence. + +Closes the lazy-anchor failure class fox surfaced in feedback-3 +(2026-05-01) on "who is homer simpson's boss?": + + Claim: "Homer Simpson's boss is Mr. Burns." + Cited span: voice-actor / Castellaneta bio prose + → Pointer resolves, source role allowed, coverage passes + → BUT span contains no "Mr. Burns" or boss/employer relation + → Pre-warrant: STRICT/HYBRID with bogus citation + → Post-warrant: WARRANT_MISSING violation, audit_mode capped + +The principle: pointer verification ≠ warrant verification. +A pointer resolves to a span; a warrant requires the span to +actually contain the claim's load-bearing tokens (the named +ANSWER ENTITY, primarily — relation aliases left as a follow-on). + +This module implements the deterministic, lexical-only first +pass of §11-§14. Two-hop bridge detection + the full +RELATION_PATTERNS registry are deferred to a successor commit +once the bench shows what answer-anchor alone catches. + +Hard or soft? Per CLAUDE.md "Soft hash vs hard hash": +substring/lexical = hard. The check uses `_content_tokens` +(deterministic) and proper-noun extraction via regex +(deterministic). Output is binary: warrant present or not. +That makes it a hard check — its violations enter the +proof path through `violations[]` exactly the way +SOURCE_ROLE_BLOCKED and CITATION_MISMATCH do. + +But unlike those, WARRANT_MISSING does not block per-pointer +verification — it caps audit_mode at HYBRID via the same +mechanism `lazy_anchor_demoted` uses. The lexical pointer +checks still pass; the user/auditor sees both "pointer-linked" +AND "warrant missing" so the failure mode is legible. +""" + +from __future__ import annotations + +import re + +# Heuristic relation-question detector. Fires when the question +# shape fits a "X's Y" / "who Z's W?" / "who owns Y?" / "who founded Z?" +# pattern — the cases fox names in feedback-3. Conservative: false +# negatives are fine (warrant just doesn't run); false positives are +# the real cost (warrant fires on questions where it shouldn't). +# Relation-noun aliases — used both in "who is X's ?" and +# " of X" shapes. Possessive-less variant ("who is supermans +# girlfriend?") needs a separate pattern path since `\S+` alone won't +# bind to a relation noun without an explicit alias list. +_RELATION_NOUNS = ( + "boss|employer|supervisor|manager|owner|founder|director|creator|" + "inventor|author|composer|painter|spouse|wife|husband|mother|father|" + "parent|son|daughter|sister|brother|nephew|niece|aunt|uncle|cousin|" + "girlfriend|boyfriend|partner|fiancee|fiance|friend|enemy|rival|" + "successor|predecessor|mentor|teacher|student|coach|captain" +) +_RELATION_VERBS = ( + "owns|founded|directed|created|invented|wrote|painted|composed|" + "discovered|killed|married|defeated|coached|hired|fired|sang|" + "produced|designed|built|sculpted|filmed" +) +_RELATION_PATTERNS = ( + # "who is X's Y?" — apostrophe-s possessive (multi-word or single) + re.compile(r"\bwho\s+is\s+\S+(?:\s+\S+)*\s*'s\s+\w+", re.IGNORECASE), + # "who is supermans girlfriend?" — possessive without apostrophe, + # gated on relation-noun alias list so generic nouns don't trip it + re.compile( + r"\bwho\s+is\s+\S+(?:\s+\S+)*\s+(?:" + _RELATION_NOUNS + r")\b", + re.IGNORECASE, + ), + # "who founded microsoft?" — relation-verb + re.compile(r"\bwho\s+(?:" + _RELATION_VERBS + r")\b", re.IGNORECASE), + # "what is X's Y?" — possessive on a what-question + re.compile(r"\bwhat\s+is\s+\S+(?:\s+\S+)*\s*'s\s+\w+", re.IGNORECASE), + # " of X" — bare relation-of phrase + re.compile(r"\b(?:" + _RELATION_NOUNS + r")\s+of\b", re.IGNORECASE), +) + + +def is_relation_question(question: str) -> bool: + """Return True iff the question shape suggests a relation lookup. + + Conservative — defaults to False on any question that doesn't + match a known relation-shape regex. False negatives leave the + warrant check inactive (claim verifies under the existing six + hard checks alone); false positives risk over-firing on + non-relation questions (worth tuning if the bench shows it). + """ + if not question: + return False + for pat in _RELATION_PATTERNS: + if pat.search(question): + return True + return False + + +# Multi-word proper noun (Title-Case sequences) — captures "Lois +# Lane", "Mr. Burns", "New England", "Homer Simpson's". The first-leg +# alternation lets a token end in `\.` ONLY when followed by another +# title-cased token (so "Mr." in "Mr. Burns" matches but "Burns." at +# sentence end does not slurp the trailing period). Trailing `'s` / +# stray punctuation are stripped in post-processing. +_PROPER_NOUN_RE = re.compile( + r"\b(?:[A-Z][A-Za-z'’\-]*\.(?=[ \t]+[A-Z])|[A-Z][A-Za-z'’\-]+)" + r"(?:[ \t]+(?:[A-Z][A-Za-z'’\-]*\.(?=[ \t]+[A-Z])|[A-Z][A-Za-z'’\-]+))+" +) + +# Solo-capitalized tokens (e.g. "Apple", "Burns"). Less reliable +# than multi-word phrases since sentence-start words look like +# proper nouns too — used only when no multi-word phrase is +# available. +_SOLO_CAP_TOKEN_RE = re.compile(r"\b[A-Z][A-Za-z'’\-]{2,}\b") + + +def _strip_anchor_tail(anchor: str) -> str: + """Strip possessive `'s` / `’s` and stray trailing punctuation.""" + a = anchor.rstrip(".,;:!?") + for tail in ("'s", "’s", "'", "’"): + if a.endswith(tail): + a = a[: -len(tail)] + break + return a.strip() + + +def extract_answer_anchors(claim_text: str) -> list[str]: + """Pull candidate answer-entity anchors from a claim's text. + + Strategy: + 1. Multi-word proper-noun phrases (`"Mr. Burns"`, `"Lois Lane"`, + `"Nineteen Eighty-Four"`) — these are reliable named + entities even at sentence start. + 2. If none found, fall back to solo capitalized tokens that + are not the FIRST word of the claim (skipping + sentence-starter false positives like the leading + "Connecticut" in "Connecticut is a state..."). + + Each anchor is post-stripped of possessive `'s` and stray + sentence-end punctuation so downstream substring tests find + base forms in cited spans. + + Returns a list of anchor strings; empty list if no anchor + detectable. Lowercase consumers should `.lower()` per call. + """ + if not claim_text: + return [] + raw = _PROPER_NOUN_RE.findall(claim_text) + if raw: + cleaned = [] + for a in raw: + stripped = _strip_anchor_tail(a) + if stripped: + cleaned.append(stripped) + return cleaned + # Solo-cap fallback. Skip the first token (sentence starter). + tokens = _SOLO_CAP_TOKEN_RE.findall(claim_text) + if len(tokens) <= 1: + return [] + return [_strip_anchor_tail(t) for t in tokens[1:] if _strip_anchor_tail(t)] + + +def warrant_check( + claim_text: str, + cited_spans: list[str], +) -> tuple[bool, list[str]]: + """Hard lexical warrant check on a relation-shape claim. + + Returns ``(warrant_ok, missing_anchors)``: + - ``warrant_ok=True`` and ``missing_anchors=[]`` when at least + ONE answer anchor extracted from the claim text appears as + a substring in at least ONE cited span (case-insensitive). + That's the "pointer cited a span that genuinely names the + answer" gate. + - ``warrant_ok=False`` and ``missing_anchors=[...]`` when none + of the extracted anchors appear in any cited span. The + list names which anchors the claim asserted but no cited + span supports — the auditor sees what specifically was + missing. + + Vacuous-pass: if no answer anchor can be extracted from the + claim text at all (the claim has no proper noun, e.g. "X is + located here"), warrant_check returns (True, []) — there is + no answer entity to demand from the spans. The other six + hard checks (coverage, source_role, etc.) carry the load. + """ + anchors = extract_answer_anchors(claim_text) + if not anchors: + return True, [] + joined_lower = " ".join(s.lower() for s in cited_spans) + found = [] + missing = [] + for anchor in anchors: + if anchor.lower() in joined_lower: + found.append(anchor) + else: + missing.append(anchor) + if found: + return True, [] + return False, missing diff --git a/tests/test_warrant.py b/tests/test_warrant.py new file mode 100644 index 0000000..6273214 --- /dev/null +++ b/tests/test_warrant.py @@ -0,0 +1,172 @@ +"""Tests for warrant-lite — relation-question hard check. + +Covers: + - is_relation_question detects "who is X's Y", "who founded Z", + "who is the boss of W", etc. + - extract_answer_anchors pulls multi-word proper nouns; + falls back to solo caps skipping sentence-starter + - warrant_check returns (True, []) when answer entity in cited + span; (False, [missing]) when not; vacuous-pass when claim + has no proper noun +""" + +from __future__ import annotations + +from aborist.qa.warrant import ( + extract_answer_anchors, + is_relation_question, + warrant_check, +) + + +# ---------------------------------------------------------------- is_relation_question + + +def test_is_relation_question_possessive_who(): + assert is_relation_question("who is homer simpson's boss?") + assert is_relation_question("who is bilbo baggins's nephew?") + assert is_relation_question("who is supermans girlfriend?") + + +def test_is_relation_question_who_action_verb(): + assert is_relation_question("who founded microsoft?") + assert is_relation_question("who painted the mona lisa?") + assert is_relation_question("who wrote hamlet?") + assert is_relation_question("who directed jaws?") + assert is_relation_question("who invented the doppler effect?") + + +def test_is_relation_question_relation_of(): + assert is_relation_question("the founder of microsoft") + assert is_relation_question("the boss of homer simpson") + assert is_relation_question("girlfriend of superman") + + +def test_is_relation_question_not_a_relation(): + """Broad-descriptive questions are NOT relation lookups.""" + assert not is_relation_question("tell me about connecticut") + assert not is_relation_question("plot of red fish blue fish?") + assert not is_relation_question("what is starcraft?") + + +def test_is_relation_question_handles_empty_or_none(): + assert not is_relation_question("") + assert not is_relation_question(None) + + +# ---------------------------------------------------------------- extract_answer_anchors + + +def test_extract_anchors_finds_multiword_proper_nouns(): + anchors = extract_answer_anchors("Homer Simpson's boss is Mr. Burns.") + # Multi-word proper-noun phrases captured. + assert "Homer Simpson" in anchors + assert "Mr. Burns" in anchors + + +def test_extract_anchors_finds_lois_lane(): + anchors = extract_answer_anchors("Superman's girlfriend is Lois Lane.") + assert "Lois Lane" in anchors + + +def test_extract_anchors_falls_back_to_solo_caps_skipping_sentence_start(): + """No multi-word phrases — falls back to solo caps but skips + the sentence-starter to avoid 'Connecticut is a state...' false + positives where the leading capital isn't a special-name anchor.""" + anchors = extract_answer_anchors("Connecticut is in New England region.") + # Multi-word "New England" SHOULD be captured (region phrase). + # Connecticut as the leading word + standalone is NOT a strong anchor. + assert any("New England" in a for a in anchors) + + +def test_extract_anchors_empty_when_no_proper_nouns(): + """A claim with no capitalized words → no anchors.""" + anchors = extract_answer_anchors("the boy and the girl have many fish.") + assert anchors == [] + + +def test_extract_anchors_handles_empty_text(): + assert extract_answer_anchors("") == [] + assert extract_answer_anchors(None) == [] + + +# ---------------------------------------------------------------- warrant_check + + +def test_warrant_passes_when_anchor_in_span(): + """Standard happy path: claim names Mr. Burns, span has Mr. Burns.""" + ok, missing = warrant_check( + "Homer Simpson's boss is Mr. Burns.", + ["Mr. Burns owns the Springfield Nuclear Power Plant where Homer works."], + ) + assert ok is True + assert missing == [] + + +def test_warrant_fails_when_answer_anchor_absent(): + """The Homer-Simpson lazy-anchor case from feedback-3 (2026-05-01). + Claim asserts Mr. Burns; cited span is voice-actor bio prose. + Warrant check must catch this — that's the whole point of the + layer.""" + ok, missing = warrant_check( + "Homer Simpson's boss is Mr. Burns.", + [ + "Castellaneta has won four Primetime Emmy Awards for " + "Outstanding Voice-Over Performance." + ], + ) + assert ok is False + # "Mr. Burns" should be in the missing list (anchor extracted + # from claim, not present in span). + assert any("Burns" in a for a in missing), missing + + +def test_warrant_passes_when_at_least_one_anchor_in_span(): + """Multiple anchors in claim, at least one in span → warrant + passes. Vacuous-tighter check would over-fire here.""" + ok, missing = warrant_check( + "Mr. Burns owns the Springfield Nuclear Power Plant.", + [ + "The Springfield Nuclear Power Plant is operated by " + "Mr. Burns in the show." + ], + ) + assert ok is True + assert missing == [] + + +def test_warrant_vacuous_pass_when_no_anchors_in_claim(): + """A claim with no proper-noun anchor (e.g. all common-noun + description) has nothing to demand from cited spans — + warrant_check returns (True, []) so the other six hard + checks carry the load. + + This is the same vacuous-pass discipline as the existing + `_claim_textually_overlaps_evidence` pure-stopword case.""" + ok, missing = warrant_check( + "the cake is a lie.", + ["completely unrelated chunk text"], + ) + assert ok is True + assert missing == [] + + +def test_warrant_case_insensitive_match(): + """Anchor 'Mr. Burns' should match span 'mr. burns' — we + care about presence, not capitalization.""" + ok, missing = warrant_check( + "Homer's boss is Mr. Burns.", + ["the show features mr. burns as the plant owner"], + ) + assert ok is True + assert missing == [] + + +def test_warrant_handles_empty_inputs(): + """No claim text → vacuous pass. No spans → fail (anchors + can't match).""" + ok, _ = warrant_check("", ["any text"]) + assert ok is True + ok, missing = warrant_check("Mr. Burns is the boss.", []) + assert ok is False + assert "Mr. Burns" in missing