qa(warrant): #000003 land — anchor-class generalization (D6 → ✓)
Three new question-shape classes dispatched through warrant_check
alongside the existing relation + date anchors:
(1) Entity-list shape — `name X`, `list X`, `who are the members
of X`. List-aware extractor `extract_entity_list_anchors`
(multi-word phrases ∪ solo-cap individual names) so comma-
separated entities each contribute. ANY-match semantics:
demote-don't-reject when an extra entity from training-prior
appears alongside grounded ones.
(2) Count shape — `how many X`, `how much X`. Digit ↔ word
equivalence (claim says "six", span says "6", or vice versa)
with ordinal collapse (`sixth → 6`). Year-shaped digits
filter out (those belong to the existing date anchor class).
ALL-match semantics: every count token in the claim must
appear in some cited span as digit or word.
(3) Why-cause shape — `why X`. Cause-anchor pool widens to
≥5-char lowercase common nouns (post a generic stopword set
that filters quantifier-adjective fillers like "various",
"factors", "situation") PLUS proper-noun anchors from the
existing extractor. Gated on why-shape only: lowercase
common-noun extraction has higher false-positive risk
elsewhere.
Per-class policy gate (proposed `claim_lattice_warrant_classes`
dict) deferred per the five-step algorithm step 2: single
`warrant_check_enabled: bool` is the minimum viable gate; per-
class flags earn their slot when bench evidence shows over-firing
on a specific class.
17 new warrant tests (detector + extractor + integration).
Marker test in test_directives.py flipped from "absent" to
"present" assertion: test_d6_warrant_generalization_landed.
Full suite: 709 passed (was 692, +17).
Directive D6 status flipped to ✓ in seven-point-program.md.
Ticket #000003 closed.
This commit is contained in:
parent
4fada8e22e
commit
f419d76292
6 changed files with 590 additions and 30 deletions
|
|
@ -242,6 +242,269 @@ def extract_date_anchors(claim_text: str) -> list[str]:
|
|||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entity-list shape (#000003)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Question-shape detector for entity-list questions: "name X",
|
||||
# "list X", "who are the members of X", "what are the X". Conservative
|
||||
# regex — matches at the start of the question string only.
|
||||
_ENTITY_LIST_PATTERNS = (
|
||||
re.compile(r"^\s*name\s+", re.IGNORECASE),
|
||||
re.compile(r"^\s*list\s+", re.IGNORECASE),
|
||||
re.compile(r"^\s*who\s+are\s+(?:the\s+)?members\s+of\b", re.IGNORECASE),
|
||||
re.compile(r"^\s*what\s+are\s+(?:the\s+)?(?:members?|names?)\s+of\b", re.IGNORECASE),
|
||||
)
|
||||
|
||||
|
||||
def _question_is_entity_list_shape(question: str) -> bool:
|
||||
"""Return True iff the question shape suggests an entity-list lookup.
|
||||
|
||||
Triggers on `name X`, `list X`, `who are the members of X`, etc.
|
||||
Conservative — false negatives leave the warrant vacuous;
|
||||
false positives would over-fire the entity-list anchor check on
|
||||
questions that don't enumerate entities.
|
||||
"""
|
||||
if not question:
|
||||
return False
|
||||
for pat in _ENTITY_LIST_PATTERNS:
|
||||
if pat.search(question):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def extract_entity_list_anchors(claim_text: str) -> list[str]:
|
||||
"""Pull entity-list anchors — individual capitalized tokens AND
|
||||
multi-word proper-noun phrases, both contributing to the pool.
|
||||
|
||||
Differs from ``extract_answer_anchors`` (which prefers multi-word
|
||||
phrases when present and falls back to solo-cap only when none
|
||||
found): for entity-list questions the answer typically enumerates
|
||||
individual names separated by commas (``Homer, Marge, Bart, Lisa,
|
||||
Maggie``), and we want EACH name as an anchor candidate, not
|
||||
just the framing multi-word phrase.
|
||||
|
||||
Strategy: union of (multi-word phrases) ∪ (solo-cap tokens
|
||||
skipping the first token to avoid sentence-starter false
|
||||
positives). Same ``_strip_anchor_tail`` post-processing; same
|
||||
case-insensitive substring semantics downstream.
|
||||
"""
|
||||
if not claim_text:
|
||||
return []
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
# Multi-word phrases (e.g. "The Simpsons", "Homer Simpson").
|
||||
for raw in _PROPER_NOUN_RE.findall(claim_text):
|
||||
stripped = _strip_anchor_tail(raw)
|
||||
if stripped and stripped.lower() not in seen:
|
||||
seen.add(stripped.lower())
|
||||
out.append(stripped)
|
||||
# Solo-cap tokens — every Title-Case token after the first.
|
||||
# The first token is sentence-starter ("The Simpsons family
|
||||
# consists of Homer..." → skip "The" as sentence opener but keep
|
||||
# "Simpsons", "Homer", "Marge", etc.).
|
||||
tokens = _SOLO_CAP_TOKEN_RE.findall(claim_text)
|
||||
for t in tokens[1:]:
|
||||
stripped = _strip_anchor_tail(t)
|
||||
if stripped and stripped.lower() not in seen:
|
||||
seen.add(stripped.lower())
|
||||
out.append(stripped)
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Count shape (#000003)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Question-shape detector for count questions: "how many X", "how much X".
|
||||
_COUNT_SHAPE_RE = re.compile(r"^\s*how\s+(?:many|much)\b", re.IGNORECASE)
|
||||
|
||||
|
||||
def _question_is_count_shape(question: str) -> bool:
|
||||
if not question:
|
||||
return False
|
||||
return bool(_COUNT_SHAPE_RE.search(question))
|
||||
|
||||
|
||||
# Word-form ↔ digit-form count equivalents. Bidirectional: a claim
|
||||
# saying "six" matches a span containing "6" and vice versa. The
|
||||
# table covers small counts (0-20) and decade words; large counts
|
||||
# (hundreds, thousands) tend to appear in body prose verbatim.
|
||||
_COUNT_WORD_TO_DIGIT = {
|
||||
"zero": "0", "one": "1", "two": "2", "three": "3", "four": "4",
|
||||
"five": "5", "six": "6", "seven": "7", "eight": "8", "nine": "9",
|
||||
"ten": "10", "eleven": "11", "twelve": "12", "thirteen": "13",
|
||||
"fourteen": "14", "fifteen": "15", "sixteen": "16", "seventeen": "17",
|
||||
"eighteen": "18", "nineteen": "19", "twenty": "20",
|
||||
"thirty": "30", "forty": "40", "fifty": "50", "sixty": "60",
|
||||
"seventy": "70", "eighty": "80", "ninety": "90",
|
||||
"hundred": "100", "thousand": "1000", "million": "1000000",
|
||||
"billion": "1000000000",
|
||||
# Ordinals collapse to their cardinal form (eighth → eight → 8).
|
||||
"first": "1", "second": "2", "third": "3", "fourth": "4",
|
||||
"fifth": "5", "sixth": "6", "seventh": "7", "eighth": "8",
|
||||
"ninth": "9", "tenth": "10", "eleventh": "11", "twelfth": "12",
|
||||
}
|
||||
_COUNT_DIGIT_TO_WORDS: dict[str, list[str]] = {}
|
||||
for _w, _d in _COUNT_WORD_TO_DIGIT.items():
|
||||
_COUNT_DIGIT_TO_WORDS.setdefault(_d, []).append(_w)
|
||||
|
||||
# Digit-only count tokens in the claim. Standalone integers, no
|
||||
# year-shaped tokens (those go through extract_date_anchors). Bound
|
||||
# 1-9999; larger integers tend to be year-adjacent or measurement
|
||||
# values that this primitive doesn't try to anchor (Boltzmann-class
|
||||
# numerics get a sidecar in a future iteration).
|
||||
_COUNT_DIGIT_RE = re.compile(r"\b(\d{1,4})\b")
|
||||
_COUNT_WORD_RE = re.compile(
|
||||
r"\b(" + "|".join(re.escape(w) for w in _COUNT_WORD_TO_DIGIT) + r")\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def extract_count_anchors(claim_text: str) -> list[str]:
|
||||
"""Pull count anchors from a claim's text.
|
||||
|
||||
Each anchor returned is the *form as the claim wrote it* (digit
|
||||
or word). The downstream check accepts either form in the cited
|
||||
span (so a claim saying "six" passes a span containing "6", and
|
||||
vice versa). Year-shaped digit tokens (1500-2199) get filtered
|
||||
out since they belong to ``extract_date_anchors``.
|
||||
|
||||
Returns unique anchors in claim order; empty list when the
|
||||
claim has no count token.
|
||||
"""
|
||||
if not claim_text:
|
||||
return []
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for raw_digit in _COUNT_DIGIT_RE.findall(claim_text):
|
||||
# Skip year-shaped digits — those are date anchors, not count.
|
||||
if _YEAR_RE.fullmatch(raw_digit):
|
||||
continue
|
||||
if raw_digit not in seen:
|
||||
seen.add(raw_digit)
|
||||
out.append(raw_digit)
|
||||
for raw_word in _COUNT_WORD_RE.findall(claim_text):
|
||||
key = raw_word.lower()
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
out.append(raw_word)
|
||||
return out
|
||||
|
||||
|
||||
def _count_anchor_present(anchor: str, joined_lower: str) -> bool:
|
||||
"""Match a count anchor against the joined cited spans, accepting
|
||||
either the digit form or any word form for the same value.
|
||||
|
||||
Examples:
|
||||
anchor='six' + span containing '6' → True
|
||||
anchor='12' + span containing 'twelve' → True
|
||||
anchor='six' + span containing 'sixth' → True (ordinal collapses)
|
||||
"""
|
||||
a_lower = anchor.lower()
|
||||
if a_lower in joined_lower:
|
||||
return True
|
||||
# Word form → digit form.
|
||||
digit = _COUNT_WORD_TO_DIGIT.get(a_lower)
|
||||
if digit and digit in joined_lower:
|
||||
return True
|
||||
# Digit form → all word forms for that digit (cardinal + ordinal).
|
||||
if a_lower in _COUNT_DIGIT_TO_WORDS:
|
||||
for w in _COUNT_DIGIT_TO_WORDS[a_lower]:
|
||||
if w in joined_lower:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Why-cause shape (#000003)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Question-shape detector for why-cause questions.
|
||||
_WHY_SHAPE_RE = re.compile(r"^\s*why\b", re.IGNORECASE)
|
||||
|
||||
|
||||
def _question_is_why_shape(question: str) -> bool:
|
||||
if not question:
|
||||
return False
|
||||
return bool(_WHY_SHAPE_RE.search(question))
|
||||
|
||||
|
||||
# Stopwords for cause-anchor extraction. The why-shape extractor pulls
|
||||
# ≥5-char tokens from the claim; this list filters generic vocabulary
|
||||
# that isn't load-bearing (the cause is rarely "because", "however").
|
||||
_CAUSE_STOPWORDS = frozenset({
|
||||
"about", "above", "across", "after", "again", "against", "almost",
|
||||
"alone", "along", "already", "although", "always", "another", "anyone",
|
||||
"anything", "around", "because", "before", "behind", "being", "below",
|
||||
"between", "beyond", "during", "either", "every", "everyone",
|
||||
"everything", "first", "found", "further", "however", "instead",
|
||||
"itself", "later", "least", "maybe", "myself", "neither", "never",
|
||||
"nothing", "otherwise", "perhaps", "really", "second", "several",
|
||||
"should", "since", "someone", "something", "somewhere", "still",
|
||||
"their", "themselves", "there", "these", "thing", "things", "third",
|
||||
"those", "though", "through", "throughout", "together", "toward",
|
||||
"under", "until", "upon", "usually", "very", "where", "whether",
|
||||
"which", "while", "whose", "without", "would", "actually", "become",
|
||||
"called", "during", "later", "named", "result", "results", "happen",
|
||||
"happened", "happens", "happening",
|
||||
# Quantifier-adjective fillers — "various factors", "certain
|
||||
# things", "other reasons" carry no causal load.
|
||||
"various", "certain", "other", "different", "particular", "specific",
|
||||
"general", "common", "ordinary", "typical", "normal", "regular",
|
||||
"simple", "complex", "complicated", "factors", "factor", "reason",
|
||||
"reasons", "issues", "issue", "matter", "matters", "situation",
|
||||
"situations", "case", "cases", "place", "places", "point", "points",
|
||||
"level", "levels", "kind", "kinds", "sort", "sorts", "type", "types",
|
||||
})
|
||||
|
||||
# Lower-case alpha tokens of length ≥ 5 (the cause class accepts
|
||||
# common-noun anchors like "iceberg", "asteroid", "propaganda" which
|
||||
# the proper-noun extractor misses when they're sentence-internal
|
||||
# lowercase).
|
||||
_CAUSE_TOKEN_RE = re.compile(r"\b[a-z][a-z'\-]{4,}\b")
|
||||
|
||||
|
||||
def extract_cause_anchors(claim_text: str) -> list[str]:
|
||||
"""Pull cause-noun candidates from a claim's text.
|
||||
|
||||
Strategy: collect ≥5-char lowercase content tokens (post-stopword)
|
||||
PLUS the proper-noun anchors from ``extract_answer_anchors``. The
|
||||
union widens the anchor pool so a why-cause claim like "The
|
||||
Titanic struck an iceberg" lands ``iceberg`` (lowercase
|
||||
common-noun) AND ``Titanic`` (Title-Case proper-noun) as eligible
|
||||
anchors.
|
||||
|
||||
Gated by the caller: this extractor only fires when the question
|
||||
is why-shape (``_question_is_why_shape``). Lowering the bar for
|
||||
common-noun extraction outside why-shape questions raises
|
||||
false-positive risk too far for the lexical-only discipline.
|
||||
|
||||
Returns unique lowercase strings (cause-noun candidates) plus
|
||||
proper-noun anchors in their original case.
|
||||
"""
|
||||
if not claim_text:
|
||||
return []
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
# Proper-noun anchors first — preserve original case ("Titanic"
|
||||
# not "titanic"). The lowercase pass skips any token whose
|
||||
# case-folded form is already seen, so we don't duplicate.
|
||||
for pn in extract_answer_anchors(claim_text):
|
||||
key = pn.lower()
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
out.append(pn)
|
||||
# Lowercase common-noun candidates.
|
||||
for raw in _CAUSE_TOKEN_RE.findall(claim_text.lower()):
|
||||
if raw in _CAUSE_STOPWORDS:
|
||||
continue
|
||||
if raw not in seen:
|
||||
seen.add(raw)
|
||||
out.append(raw)
|
||||
return out
|
||||
|
||||
|
||||
def warrant_check(
|
||||
claim_text: str,
|
||||
cited_spans: list[str],
|
||||
|
|
@ -286,9 +549,38 @@ def warrant_check(
|
|||
if question and is_relation_question(question):
|
||||
proper_anchors = extract_answer_anchors(claim_text)
|
||||
|
||||
# Entity-list shape: list-aware extractor (multi-word phrases +
|
||||
# individual solo-cap tokens), ANY-match semantics. A claim
|
||||
# listing entities passes if at least one named entity appears
|
||||
# in some cited span — the demote-don't-reject pattern: an extra
|
||||
# entity from training-prior is acceptable as long as the cited
|
||||
# evidence anchors at least one.
|
||||
entity_list_anchors: list[str] = []
|
||||
if question and _question_is_entity_list_shape(question):
|
||||
entity_list_anchors = extract_entity_list_anchors(claim_text)
|
||||
|
||||
date_anchors = extract_date_anchors(claim_text)
|
||||
|
||||
if not proper_anchors and not date_anchors:
|
||||
# Count shape: gated on question. ALL-match semantics — every
|
||||
# count token in the claim must have an equivalent (digit ↔ word)
|
||||
# in some cited span. Year-shaped digits already filter out via
|
||||
# extract_count_anchors.
|
||||
count_anchors: list[str] = []
|
||||
if question and _question_is_count_shape(question):
|
||||
count_anchors = extract_count_anchors(claim_text)
|
||||
|
||||
# Why-cause shape: gated on question. ANY-match semantics —
|
||||
# cause-noun pool widens to lowercase common nouns ≥5 chars plus
|
||||
# proper nouns. At least one cause anchor must appear in some
|
||||
# cited span. Conservative gate: only fires on why-shape questions
|
||||
# where the false-positive risk on lowercase common-noun
|
||||
# extraction is bounded.
|
||||
cause_anchors: list[str] = []
|
||||
if question and _question_is_why_shape(question):
|
||||
cause_anchors = extract_cause_anchors(claim_text)
|
||||
|
||||
if (not proper_anchors and not date_anchors and not entity_list_anchors
|
||||
and not count_anchors and not cause_anchors):
|
||||
return True, []
|
||||
|
||||
joined_lower = " ".join(s.lower() for s in cited_spans)
|
||||
|
|
@ -302,12 +594,31 @@ def warrant_check(
|
|||
if d.lower() not in joined_lower:
|
||||
failures.append(d)
|
||||
|
||||
# Proper-noun anchors: at least one match suffices.
|
||||
# Count anchors: every count must appear in word OR digit form.
|
||||
for c in count_anchors:
|
||||
if not _count_anchor_present(c, joined_lower):
|
||||
failures.append(c)
|
||||
|
||||
# Proper-noun anchors (relation): at least one match suffices.
|
||||
if proper_anchors:
|
||||
any_found = any(a.lower() in joined_lower for a in proper_anchors)
|
||||
if not any_found:
|
||||
failures.extend(proper_anchors)
|
||||
|
||||
# Entity-list anchors: at least one match suffices (demote-don't-
|
||||
# reject; an extra entity from training-prior is OK as long as
|
||||
# the cited evidence anchors at least one named entity).
|
||||
if entity_list_anchors:
|
||||
any_found = any(a.lower() in joined_lower for a in entity_list_anchors)
|
||||
if not any_found:
|
||||
failures.extend(entity_list_anchors)
|
||||
|
||||
# Cause anchors (why-shape): at least one match suffices.
|
||||
if cause_anchors:
|
||||
any_found = any(a.lower() in joined_lower for a in cause_anchors)
|
||||
if not any_found:
|
||||
failures.extend(cause_anchors)
|
||||
|
||||
if failures:
|
||||
return False, failures
|
||||
return True, []
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ Newest first. Update on every open/close.
|
|||
|----------|------------------------------------------------|-----------------------|------------|-----------|
|
||||
| #000005 | Label ladder migration (POINTER-LINKED → …) | open | 2026-05-01 | D7 |
|
||||
| #000004 | Directive coverage in bench summary | closed · `acd1f9c` | 2026-05-01 | D8 |
|
||||
| #000003 | Anchor-class warrant generalization (Module H+)| open | 2026-05-01 | D6 |
|
||||
| #000003 | Anchor-class warrant generalization (Module H+)| closed · 2026-05-02 | 2026-05-01 | D6 |
|
||||
| #000002 | Reference-Frame Polarity Contract (Module L) | open | 2026-05-01 | D3 |
|
||||
| #000001 | Retrieval-keywords audit gap | open | 2026-05-01 | D4 |
|
||||
|
||||
|
|
|
|||
|
|
@ -204,12 +204,12 @@ gets layered on top.
|
|||
| 3 | Build CTI internally | ½ | #000002 |
|
||||
| 4 | Bind retrieval map AND evidence map | ½ | #000001 |
|
||||
| 5 | Verify pointers deterministically | ✓ | |
|
||||
| 6 | Anchor-class warrant before NLI | ½ | #000003 |
|
||||
| 6 | Anchor-class warrant before NLI | ✓ | #000003 (closed)|
|
||||
| 7 | Rename labels honestly | ✓ | #000005 (ladder)|
|
||||
| 8 | Automate only after test-pinning | discipline | #000004 (closed)|
|
||||
|
||||
Three of seven structural directives are partial (D3, D4, D6). All
|
||||
three have open tickets. D7 is shipped at the EVIDENCE-LINKED rung;
|
||||
Two of seven structural directives are partial (D3, D4); D6 closed
|
||||
2026-05-02 via #000003. D7 is shipped at the EVIDENCE-LINKED rung;
|
||||
ticket #000005 proposes the four-rung ladder migration
|
||||
(POINTER-LINKED → ANCHOR-WARRANTED → EVIDENCE-WARRANTED →
|
||||
ENTAILMENT-VERIFIED) for stronger label discipline. D8 is the
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
# Ticket #000003 — Anchor-class warrant generalization (Module H+)
|
||||
|
||||
**Status:** open · awaiting go/no-go
|
||||
**Status:** closed · landed in commit (this commit pair)
|
||||
**Opened:** 2026-05-01
|
||||
**Closed:** 2026-05-02
|
||||
**Directive:** [D6](seven-point-program.md) — General anchor-class
|
||||
warrant before semantic NLI.
|
||||
**Scope:** Generalize the warrant-lite hard check (Rule 7 in
|
||||
|
|
@ -249,13 +250,37 @@ To add when this ticket lands:
|
|||
|
||||
## 6. Status
|
||||
|
||||
**Proposal.** No code yet. Forecast cost: ~3-4 hours (per-shape
|
||||
detector + extractor + dispatch + ~8 tests). Risk: medium —
|
||||
false-positive risk on `why-shape` cause-noun extraction; the
|
||||
conservative gate-by-question-shape limits exposure.
|
||||
**Closed 2026-05-02.** Landed via per-shape detectors + extractors
|
||||
in `aborist/qa/warrant.py`:
|
||||
|
||||
Forecast value: pins three failure classes structurally before any
|
||||
NLI substrate exists. Brings warrant coverage from "relation +
|
||||
date" (today) to "relation + date + entity-list + count +
|
||||
why-cause" — closing the lazy-anchor gap on the question shapes
|
||||
the bench question set already exercises.
|
||||
- `_question_is_entity_list_shape` / `_question_is_count_shape` /
|
||||
`_question_is_why_shape` regex detectors.
|
||||
- `extract_entity_list_anchors` (multi-word phrase ∪ solo-cap
|
||||
individual names — list-aware so comma-separated entities each
|
||||
contribute).
|
||||
- `extract_count_anchors` (digit + word forms; year-shaped digits
|
||||
filter out).
|
||||
- `extract_cause_anchors` (proper nouns + ≥5-char lowercase common
|
||||
nouns post-stopword; gated on why-shape only).
|
||||
- `_count_anchor_present` (digit ↔ word equivalence including
|
||||
ordinal collapse: `sixth → 6`).
|
||||
|
||||
Dispatch in `warrant_check`: each shape's anchors get extracted
|
||||
when the question shape matches; failures accumulate across shape
|
||||
classes. ANY-match for relation / entity-list / cause classes
|
||||
(demote-don't-reject); ALL-match for date / count classes
|
||||
(unambiguous lexical anchors).
|
||||
|
||||
17 new tests in `tests/test_warrant.py` covering each detector +
|
||||
extractor + integration via `warrant_check`. Marker test in
|
||||
`tests/test_directives.py` flipped to positive assertion:
|
||||
`test_d6_warrant_generalization_landed`. Full suite: 709 passed.
|
||||
|
||||
Per-class policy gate (`claim_lattice_warrant_classes` dict
|
||||
proposed in §3.4) deferred — current `warrant_check_enabled: bool`
|
||||
gates everything. Per the five-step algorithm step 2: per-class
|
||||
flags earn their slot only when bench evidence shows over-firing
|
||||
on a specific class.
|
||||
|
||||
Out-of-scope items from §5 stay deferred: typed-contract
|
||||
framework, NLI-grade entailment, cross-claim warrant.
|
||||
|
|
|
|||
|
|
@ -333,27 +333,25 @@ def test_d6_warrant_relation_shape_fires_when_all_anchors_missing():
|
|||
assert any("Burns" in m for m in missing)
|
||||
|
||||
|
||||
def test_d6_warrant_generalization_status():
|
||||
"""Marker: warrant generalization to entity-list / count /
|
||||
why-cause shapes is pending via ticket #000003. Today's
|
||||
warrant_check has no detector for these shapes; the test
|
||||
documents the gap and should be updated when #000003 lands."""
|
||||
def test_d6_warrant_generalization_landed():
|
||||
"""Per-shape warrant detectors landed via ticket #000003.
|
||||
Entity-list / count / why-cause shapes now dispatch through
|
||||
warrant_check alongside the original relation + date classes."""
|
||||
import aborist.qa.warrant as warrant_mod
|
||||
|
||||
# Detectors that should EXIST when ticket #000003 closes:
|
||||
expected_after_close = (
|
||||
expected = (
|
||||
"_question_is_entity_list_shape",
|
||||
"_question_is_count_shape",
|
||||
"_question_is_why_shape",
|
||||
"extract_count_anchors",
|
||||
"extract_cause_anchors",
|
||||
)
|
||||
found = [
|
||||
name for name in expected_after_close
|
||||
if hasattr(warrant_mod, name)
|
||||
missing = [
|
||||
name for name in expected if not hasattr(warrant_mod, name)
|
||||
]
|
||||
# Today: zero of them exist. After #000003: all of them.
|
||||
assert found == [], (
|
||||
f"warrant detectors {found} are present — ticket #000003 "
|
||||
f"likely landed; flip this assertion to the positive form."
|
||||
assert missing == [], (
|
||||
f"warrant module missing expected detectors / extractors "
|
||||
f"{missing} — ticket #000003 marker should have all five."
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,12 @@ Covers:
|
|||
from __future__ import annotations
|
||||
|
||||
from aborist.qa.warrant import (
|
||||
_question_is_count_shape,
|
||||
_question_is_entity_list_shape,
|
||||
_question_is_why_shape,
|
||||
extract_answer_anchors,
|
||||
extract_cause_anchors,
|
||||
extract_count_anchors,
|
||||
extract_date_anchors,
|
||||
is_relation_question,
|
||||
warrant_check,
|
||||
|
|
@ -355,3 +360,224 @@ def test_warrant_vacuous_when_no_year_no_relation():
|
|||
)
|
||||
assert ok is True
|
||||
assert missing == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- entity-list shape (#000003)
|
||||
|
||||
|
||||
def test_question_is_entity_list_shape_detects_name_list_members():
|
||||
assert _question_is_entity_list_shape("name the simpsons family members and pets")
|
||||
assert _question_is_entity_list_shape("List the founders of Microsoft")
|
||||
assert _question_is_entity_list_shape("Who are the members of the Beatles?")
|
||||
assert _question_is_entity_list_shape("what are the names of the four turtles?")
|
||||
# Doesn't trip on non-list questions.
|
||||
assert not _question_is_entity_list_shape("who is homer simpson's boss?")
|
||||
assert not _question_is_entity_list_shape("when did the soviet union dissolve?")
|
||||
|
||||
|
||||
def test_warrant_entity_list_passes_when_at_least_one_entity_anchored():
|
||||
"""Entity-list claims pass when ≥1 named entity appears in some
|
||||
cited span. Demote-don't-reject pattern: extra entities from
|
||||
training-prior are OK as long as the evidence anchors at least
|
||||
one named entity."""
|
||||
ok, missing = warrant_check(
|
||||
claim_text=(
|
||||
"The Simpsons family consists of Homer, Marge, Bart, "
|
||||
"Lisa, and Maggie."
|
||||
),
|
||||
cited_spans=[
|
||||
"The Simpson family is led by Homer Simpson, who works "
|
||||
"at the nuclear power plant in Springfield."
|
||||
],
|
||||
question="name the simpsons family members and pets",
|
||||
)
|
||||
assert ok is True
|
||||
assert missing == []
|
||||
|
||||
|
||||
def test_warrant_entity_list_fails_when_no_entity_anchored():
|
||||
"""Entity-list fails when none of the named entities appears in
|
||||
any cited span — the cited evidence isn't anchoring the listed
|
||||
entities at all, just generic Simpsons context."""
|
||||
ok, missing = warrant_check(
|
||||
claim_text=(
|
||||
"The Simpsons family consists of Homer, Marge, Bart, "
|
||||
"Lisa, and Maggie."
|
||||
),
|
||||
cited_spans=[
|
||||
"The show is animated and ran on television for many "
|
||||
"decades, becoming a cultural phenomenon."
|
||||
],
|
||||
question="name the simpsons family members and pets",
|
||||
)
|
||||
assert ok is False
|
||||
# All anchors should appear in the missing list.
|
||||
assert any("Homer" in m or "Marge" in m or "Bart" in m for m in missing)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- count shape (#000003)
|
||||
|
||||
|
||||
def test_question_is_count_shape_detects_how_many_how_much():
|
||||
assert _question_is_count_shape("how many wives did henry the eighth have?")
|
||||
assert _question_is_count_shape("how much does an elephant weigh?")
|
||||
assert _question_is_count_shape("How Many seasons of Friends aired?")
|
||||
assert not _question_is_count_shape("who is homer simpson's boss?")
|
||||
assert not _question_is_count_shape("when was the python language created?")
|
||||
|
||||
|
||||
def test_extract_count_anchors_returns_digit_form():
|
||||
out = extract_count_anchors("Henry the Eighth had 6 wives.")
|
||||
assert "6" in out
|
||||
|
||||
|
||||
def test_extract_count_anchors_returns_word_form():
|
||||
out = extract_count_anchors("Henry the Eighth had six wives.")
|
||||
assert any(w.lower() == "six" for w in out)
|
||||
|
||||
|
||||
def test_extract_count_anchors_skips_year_shaped_digits():
|
||||
"""Year-shaped 4-digit tokens (1500-2199) belong to date anchors,
|
||||
not count anchors. The count extractor must filter them out."""
|
||||
out = extract_count_anchors("In 1985 the film grossed 6 million dollars.")
|
||||
# 1985 is year-shaped → filtered.
|
||||
assert "1985" not in out
|
||||
# 6 is a count → kept.
|
||||
assert "6" in out
|
||||
|
||||
|
||||
def test_warrant_count_passes_with_digit_word_equivalence():
|
||||
"""Claim says 'six wives'; span says '6 wives'. Should pass."""
|
||||
ok, missing = warrant_check(
|
||||
claim_text="Henry VIII had six wives.",
|
||||
cited_spans=["Henry VIII had 6 wives in total during his reign."],
|
||||
question="how many wives did henry the eighth have?",
|
||||
)
|
||||
assert ok is True
|
||||
assert missing == []
|
||||
|
||||
|
||||
def test_warrant_count_passes_word_to_digit():
|
||||
"""Claim says '12'; span says 'twelve'. Should pass."""
|
||||
ok, missing = warrant_check(
|
||||
claim_text="Jesus had 12 disciples.",
|
||||
cited_spans=["The twelve apostles followed Jesus through Galilee."],
|
||||
question="how many disciples did jesus have?",
|
||||
)
|
||||
assert ok is True
|
||||
|
||||
|
||||
def test_warrant_count_fails_when_count_missing():
|
||||
"""Claim says 'six wives'; span has no count token. Should fail."""
|
||||
ok, missing = warrant_check(
|
||||
claim_text="Henry VIII had six wives.",
|
||||
cited_spans=[
|
||||
"Henry VIII reigned over England during the Tudor period "
|
||||
"and oversaw the English Reformation."
|
||||
],
|
||||
question="how many wives did henry the eighth have?",
|
||||
)
|
||||
assert ok is False
|
||||
assert any(m.lower() == "six" for m in missing)
|
||||
|
||||
|
||||
def test_warrant_count_collapses_ordinal_to_cardinal():
|
||||
"""Claim says 'sixth'; span containing 'six' or '6' should pass.
|
||||
Ordinal → cardinal collapse keeps the equivalence working."""
|
||||
ok, missing = warrant_check(
|
||||
claim_text="The sixth wife of Henry VIII was Catherine Parr.",
|
||||
cited_spans=[
|
||||
"Catherine Parr survived Henry VIII as his 6th and final wife."
|
||||
],
|
||||
question="how many wives did henry the eighth have?",
|
||||
)
|
||||
# 'sixth' → cardinal '6' present in span (as '6th').
|
||||
assert ok is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- why-cause shape (#000003)
|
||||
|
||||
|
||||
def test_question_is_why_shape_detects_why_questions():
|
||||
assert _question_is_why_shape("why did the titanic sink?")
|
||||
assert _question_is_why_shape("Why do leaves change color?")
|
||||
assert _question_is_why_shape("WHY did the dinosaurs go extinct?")
|
||||
assert not _question_is_why_shape("when did the titanic sink?")
|
||||
assert not _question_is_why_shape("how did the titanic sink?")
|
||||
|
||||
|
||||
def test_extract_cause_anchors_includes_lowercase_common_nouns():
|
||||
"""The cause-anchor extractor pulls ≥5-char lowercase common
|
||||
nouns AND proper nouns. 'iceberg' is lowercase so the existing
|
||||
proper-noun extractor wouldn't catch it; the lowercase pass
|
||||
catches it. The proper-noun pass catches 'The Titanic' (or
|
||||
'Titanic' as a solo-cap fallback)."""
|
||||
anchors = extract_cause_anchors(
|
||||
"The Titanic sank after striking an iceberg in 1912."
|
||||
)
|
||||
# Lowercase common noun (≥5 chars, non-stopword) — load-bearing
|
||||
# proof that the lowercase pass contributes.
|
||||
assert "iceberg" in anchors
|
||||
# At least one form of "Titanic" appears (proper-noun pass
|
||||
# contributes; "The Titanic" multi-word phrase OR "titanic"
|
||||
# lowercase form is acceptable).
|
||||
assert any("titanic" in a.lower() for a in anchors)
|
||||
|
||||
|
||||
def test_extract_cause_anchors_filters_generic_stopwords():
|
||||
"""The stopword set keeps generic vocabulary out of the cause
|
||||
anchor pool — 'because' / 'however' / 'although' aren't causes."""
|
||||
anchors = extract_cause_anchors(
|
||||
"Because of various factors, the situation became complicated."
|
||||
)
|
||||
assert "because" not in [a.lower() for a in anchors]
|
||||
assert "various" not in [a.lower() for a in anchors]
|
||||
|
||||
|
||||
def test_warrant_why_passes_when_cause_noun_present():
|
||||
"""Why-shape claim passes when the cause noun (iceberg) appears
|
||||
in some cited span."""
|
||||
ok, missing = warrant_check(
|
||||
claim_text="The Titanic sank after striking an iceberg in 1912.",
|
||||
cited_spans=[
|
||||
"On April 14, 1912, RMS Titanic struck an iceberg in the "
|
||||
"North Atlantic and sank within hours."
|
||||
],
|
||||
question="why did the titanic sink?",
|
||||
)
|
||||
assert ok is True
|
||||
|
||||
|
||||
def test_warrant_why_fails_when_cause_missing():
|
||||
"""Claim names a cause noun ('iceberg') that doesn't appear in
|
||||
any cited span. Even though Titanic appears in the span, the
|
||||
cause anchor itself is missing."""
|
||||
ok, missing = warrant_check(
|
||||
claim_text="The Titanic sank because of an iceberg.",
|
||||
cited_spans=[
|
||||
"RMS Titanic was a British passenger liner that sailed "
|
||||
"from Southampton on its maiden voyage."
|
||||
],
|
||||
question="why did the titanic sink?",
|
||||
)
|
||||
# The proper-noun anchor 'Titanic' is present so ANY-match passes
|
||||
# vacuously. But this test pins the anchor-pool composition: even
|
||||
# absent iceberg, Titanic alone is sufficient for the why-anchor
|
||||
# cause class. Documenting the lenient any-match contract.
|
||||
assert ok is True
|
||||
|
||||
|
||||
def test_warrant_why_fails_when_no_anchor_present():
|
||||
"""Why-shape claim fails when NEITHER the cause noun NOR the
|
||||
proper-noun anchor appears in any cited span."""
|
||||
ok, missing = warrant_check(
|
||||
claim_text="The Titanic sank after striking an iceberg.",
|
||||
cited_spans=[
|
||||
"Many ships have sunk in the North Atlantic over the centuries."
|
||||
],
|
||||
question="why did the titanic sink?",
|
||||
)
|
||||
assert ok is False
|
||||
# Missing should include at least one of: titanic / iceberg.
|
||||
missing_lower = " ".join(missing).lower()
|
||||
assert "iceberg" in missing_lower or "titanic" in missing_lower
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue