From 7ba1b405ee329fb06ce68ab48a1ba3b38ffac7ec Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Fri, 1 May 2026 15:24:11 -0400 Subject: [PATCH] =?UTF-8?q?qa:=20warrant-lite=20anchor-class=20=E2=80=94?= =?UTF-8?q?=20add=20date=20anchors=20(year=20+=20month=20name)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generalizes warrant-lite from relation-only to anchor-class warrant. Two anchor classes now compose: (1) Proper-noun anchors — gated on relation-question shape; any-match semantics (existing behavior). (2) Date anchors — always-on, gated on the CLAIM containing a year or month name. ALL components (year + month name if present) must appear in some cited span. Closes the back-to-the-future failure fox surfaced 2026-05-01: Q: what date did back to the future come out? Claim: "Back to the Future was released in theaters on July 3, 1985." Cited span: trilogy/SNES/pinball prose with '...real 1985...' but no 'July' anywhere Pre-fix: EVIDENCE-LINKED · via claim_lattice 1/1 Post-fix: EVIDENCE-LINKED-PARTIAL · via claim_lattice · warrant missing 1/1 Year extraction (4-digit, 1500-2199 range — avoids matching ZIP codes / elevations / model IDs) and month extraction (full English month names — three-letter abbrevs deferred until bench shows the gap). Both anchors required when both are in the claim. Year-only claim against year-only span continues to pass; month-only against month-only continues to pass; month-and-year claim against year-only span (the BTTF case) now correctly fails. Architectural note: this generalizes warrant-lite without building a typed-contract framework (per-question-type rule libraries like "release_date_lookup with required predicate cues"). The date anchor primitive catches the date lazy-anchor failure class deterministically without a per-type rule book; typed contracts earn their slot only if bench evidence shows general anchors miss specific cases. Dropped the caller-side is_relation_question gate from verify.py — warrant_check now decides internally what to enforce based on question shape AND claim content. Tests: 29 unit tests in tests/test_warrant.py (was 16). New tests cover extract_date_anchors (year, month, dedup, non-year-skip, empty), warrant_check date semantics (BTTF case, year-only, year-in-unrelated-context, month+year composition), proper-noun warrant skipped without relation question. Full suite: 554 passed, 21 skipped. Limitation accepted: ISO date format ("1985-07-03") wouldn't satisfy month-name requirement. Real-world Wikipedia prose uses month names ("released on July 3, 1985"), so the trade-off favors catching the common case. If bench shows ISO false-rejections, layer in a month-number alternative to the month-name regex. --- aborist/qa/verify.py | 24 +++-- aborist/qa/warrant.py | 220 ++++++++++++++++++++++++++++++++---------- tests/test_warrant.py | 207 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 378 insertions(+), 73 deletions(-) diff --git a/aborist/qa/verify.py b/aborist/qa/verify.py index f070eb5..31c9a39 100644 --- a/aborist/qa/verify.py +++ b/aborist/qa/verify.py @@ -47,7 +47,7 @@ from __future__ import annotations import re import unicodedata -from aborist.qa.warrant import is_relation_question, warrant_check +from aborist.qa.warrant import warrant_check try: from aborist.wikitext import to_base as _wikitext_to_base @@ -1367,7 +1367,7 @@ def verify_claim_lattice( # cap audit_mode at HYBRID via the same demote pattern as # lazy_anchor_demoted. warrant_missing_claims: list[int] = [] - if warrant_check_enabled and question and is_relation_question(question): + if warrant_check_enabled: for cs in claim_statuses: if cs.get("status") not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"): continue @@ -1378,7 +1378,9 @@ def verify_claim_lattice( for obj in [evidence_map_by_evidence_id_local(evidence_map, eid)] if obj is not None ] - ok, missing = warrant_check(cs.get("text") or "", cited_spans) + ok, missing = warrant_check( + cs.get("text") or "", cited_spans, question=question + ) if not ok: warrant_missing_claims.append(cs.get("claim_idx")) violations.append({ @@ -1386,8 +1388,9 @@ def verify_claim_lattice( "claim_idx": cs.get("claim_idx"), "missing_anchors": missing, "rationale": ( - "claim names answer entity not present in any " - "cited span — pointer-linked but warrant missing" + "claim asserts an answer entity or specific date " + "not present in any cited span — pointer-linked " + "but warrant missing" ), }) if warrant_missing_claims and audit_mode == "STRICT": @@ -1718,7 +1721,7 @@ def verify_claim_lattice_json( # variant carries the same WARRANT_MISSING violations & the same # warrant_missing_claim_idxs field on the verdict. warrant_missing_claims: list[int] = [] - if warrant_check_enabled and question and is_relation_question(question): + if warrant_check_enabled: for cs in claim_statuses: if cs.get("status") not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"): continue @@ -1729,7 +1732,9 @@ def verify_claim_lattice_json( for obj in [evidence_map_by_evidence_id_local(evidence_map, eid)] if obj is not None ] - ok, missing = warrant_check(cs.get("text") or "", cited_spans) + ok, missing = warrant_check( + cs.get("text") or "", cited_spans, question=question + ) if not ok: warrant_missing_claims.append(cs.get("claim_idx")) violations.append({ @@ -1737,8 +1742,9 @@ def verify_claim_lattice_json( "claim_idx": cs.get("claim_idx"), "missing_anchors": missing, "rationale": ( - "claim names answer entity not present in any " - "cited span — pointer-linked but warrant missing" + "claim asserts an answer entity or specific date " + "not present in any cited span — pointer-linked " + "but warrant missing" ), }) if warrant_missing_claims and audit_mode == "STRICT": diff --git a/aborist/qa/warrant.py b/aborist/qa/warrant.py index a1ec2ac..b775064 100644 --- a/aborist/qa/warrant.py +++ b/aborist/qa/warrant.py @@ -1,38 +1,62 @@ -"""Warrant-lite — relation-question hard check on claim-cited evidence. +"""Warrant-lite — anchor-class hard check on claim-cited evidence. -Closes the lazy-anchor failure class fox surfaced in feedback-3 -(2026-05-01) on "who is homer simpson's boss?": +Closes two lazy-anchor failure classes fox surfaced in May 2026: + +(1) Relation lazy-anchor (2026-05-01, "who is homer simpson's boss?"): Claim: "Homer Simpson's boss is Mr. Burns." Cited span: voice-actor / Castellaneta bio prose → Pointer resolves, source role allowed, coverage passes - → BUT span contains no "Mr. Burns" or boss/employer relation - → Pre-warrant: STRICT/HYBRID with bogus citation - → Post-warrant: WARRANT_MISSING violation, audit_mode capped + → BUT span contains no "Mr. Burns" + → Warrant-lite catches it via proper-noun anchor check -The principle: pointer verification ≠ warrant verification. -A pointer resolves to a span; a warrant requires the span to -actually contain the claim's load-bearing tokens (the named -ANSWER ENTITY, primarily — relation aliases left as a follow-on). +(2) Date lazy-anchor (2026-05-01, "what date did back to the future +come out?"): -This module implements the deterministic, lexical-only first -pass of §11-§14. Two-hop bridge detection + the full -RELATION_PATTERNS registry are deferred to a successor commit -once the bench shows what answer-anchor alone catches. + Claim: "Back to the Future was released in theaters on July 3, 1985." + Cited span: trilogy/SNES/pinball prose with no 1985 anywhere + → Pointer resolves, source role allowed, coverage passes + → BUT span does not contain the year the claim asserts + → Warrant-lite catches it via date anchor check + +The principle generalizes: pointer verification ≠ warrant verification. +A pointer resolves to a span; a warrant requires the span to actually +contain the claim's load-bearing tokens — the named ANSWER ENTITY for +relation questions, the asserted YEAR for any claim that names a +specific year, and (forward-looking) any unambiguous lexical anchor +the claim asserts. + +Two anchor classes today: + +- proper-noun anchors: gated on relation-question shape (regex + detector). Required: at least one anchor extracted from the claim + must appear in some cited span. Catches the relation lazy-anchor + class. +- date anchors: gated on the CLAIM containing a 4-digit year (any + question shape). Required: every year asserted in the claim must + appear in some cited span. Years are unambiguous; if the claim says + "1985" and no span has "1985", the warrant fails. Catches the date + lazy-anchor class. + +Both classes compose: a claim with both a proper-noun anchor and a +date anchor must satisfy both. Hard or soft? Per CLAUDE.md "Soft hash vs hard hash": -substring/lexical = hard. The check uses `_content_tokens` -(deterministic) and proper-noun extraction via regex -(deterministic). Output is binary: warrant present or not. -That makes it a hard check — its violations enter the -proof path through `violations[]` exactly the way -SOURCE_ROLE_BLOCKED and CITATION_MISMATCH do. +substring/lexical = hard. Output is binary: warrant present or not. +Violations enter the proof path through `violations[]` exactly the +way SOURCE_ROLE_BLOCKED and CITATION_MISMATCH do — but +WARRANT_MISSING caps audit_mode at HYBRID rather than rejecting the +pointer outright. The lexical pointer checks still pass; the +user/auditor sees both "pointer-linked" AND "warrant missing" so +the failure mode is legible. -But unlike those, WARRANT_MISSING does not block per-pointer -verification — it caps audit_mode at HYBRID via the same -mechanism `lazy_anchor_demoted` uses. The lexical pointer -checks still pass; the user/auditor sees both "pointer-linked" -AND "warrant missing" so the failure mode is legible. +What is intentionally NOT here: typed-contract frameworks (per- +question-type rule libraries like "release_date_lookup with +required predicate cues"). The general anchor-class primitive +catches the back-to-the-future failure deterministically without a +per-type rule book. Typed contracts earn their slot only when bench +evidence shows the general primitive misses cases — see +docs/naming-deferral.md for the discipline. """ from __future__ import annotations @@ -159,41 +183,131 @@ def extract_answer_anchors(claim_text: str) -> list[str]: return [_strip_anchor_tail(t) for t in tokens[1:] if _strip_anchor_tail(t)] +# Year pattern — 4-digit years between 1500 and 2199. Avoids matching +# random 4-digit numbers (elevations, model IDs, ZIP codes, room +# numbers) while covering historical dates and near-future ones. +_YEAR_RE = re.compile(r"\b(?:1[5-9]\d\d|2[01]\d\d)\b") + +# Month-name pattern — full English month names. Captures "July" in +# "July 3, 1985". Three-letter abbreviations ("Jul") are intentionally +# excluded for the first cut — too many false positives ("Mar" inside +# "Mara", "May" as a verb, etc.). If bench shows the gap, add as a +# whole-word-bounded alternation. +_MONTH_RE = re.compile( + r"\b(?:January|February|March|April|May|June|" + r"July|August|September|October|November|December)\b", + re.IGNORECASE, +) + + +def extract_date_anchors(claim_text: str) -> list[str]: + """Pull date anchors from a claim's text. + + Returns the union of: + + - **Year strings** (4-digit years 1500-2199). Avoids matching + random 4-digit numbers like elevations or ZIP codes. + - **Month names** (January-December, full names only). + + Both classes are required as conditions: when a claim asserts + "July 3, 1985", the cited span must contain BOTH "1985" and + "July" — checked independently as substrings. Catches the + back-to-the-future class where a span happens to contain "1985" + in unrelated narrative context (e.g. "back to the real 1985" + referring to the film's diegetic time period) but never names + the actual release month. Falls short on ISO date formats + (``1985-07-03`` lacks the literal "July") but real-world Wikipedia + prose uses month names; if bench shows the ISO gap, layer in a + month-number alternative. + + Returns unique anchor strings in claim order; empty list if the + claim has no year and no month name. Year strings are + case-insensitive by definition; month names are returned in + their claim-text form (consumers should ``.lower()`` before + substring matching). + """ + if not claim_text: + return [] + seen: set[str] = set() + out: list[str] = [] + for y in _YEAR_RE.findall(claim_text): + if y not in seen: + seen.add(y) + out.append(y) + for m in _MONTH_RE.findall(claim_text): + key = m.lower() + if key not in seen: + seen.add(key) + out.append(m) + return out + + def warrant_check( claim_text: str, cited_spans: list[str], + *, + question: str | None = None, ) -> tuple[bool, list[str]]: - """Hard lexical warrant check on a relation-shape claim. + """Hard lexical warrant check — anchor classes composed. + + Two anchor classes: + + 1. **Proper-noun anchors** (relation lazy-anchor class). Gated on + relation-question shape via ``is_relation_question(question)``; + requires ``question`` kwarg. AT LEAST ONE extracted anchor must + appear as a substring in at least one cited span + (case-insensitive). The any-match semantics is lenient — the + answer entity OR the subject matching is enough lexical signal + that the cited span is contextually right. Vacuous-passes when + ``question`` is None/non-relation, or when the claim has no + proper-noun anchor. + + 2. **Date anchors** (date lazy-anchor class). Always-on, gated on + the CLAIM containing a 4-digit year. Requires ALL years + asserted in the claim to appear in at least one cited span. + Years are unambiguous; if the claim says "1985" and no cited + span has "1985", the warrant fails — no fuzzy matching, no + semantic equivalence. Vacuous-passes when the claim has no + year. Returns ``(warrant_ok, missing_anchors)``: - - ``warrant_ok=True`` and ``missing_anchors=[]`` when at least - ONE answer anchor extracted from the claim text appears as - a substring in at least ONE cited span (case-insensitive). - That's the "pointer cited a span that genuinely names the - answer" gate. - - ``warrant_ok=False`` and ``missing_anchors=[...]`` when none - of the extracted anchors appear in any cited span. The - list names which anchors the claim asserted but no cited - span supports — the auditor sees what specifically was - missing. + - ``warrant_ok=True`` when both anchor classes pass (or + vacuous-pass). + - ``warrant_ok=False`` and ``missing_anchors=[...]`` listing + every anchor (date or proper-noun) the claim asserted but + no cited span supports. The auditor sees what specifically + was missing — date strings, entity names, or both. - Vacuous-pass: if no answer anchor can be extracted from the - claim text at all (the claim has no proper noun, e.g. "X is - located here"), warrant_check returns (True, []) — there is - no answer entity to demand from the spans. The other six - hard checks (coverage, source_role, etc.) carry the load. + Composition: when a claim has both classes (e.g. relation-shape + question with a year-asserting claim), both must pass. The + failure list accumulates from both checks. """ - anchors = extract_answer_anchors(claim_text) - if not anchors: + proper_anchors: list[str] = [] + if question and is_relation_question(question): + proper_anchors = extract_answer_anchors(claim_text) + + date_anchors = extract_date_anchors(claim_text) + + if not proper_anchors and not date_anchors: return True, [] + joined_lower = " ".join(s.lower() for s in cited_spans) - found = [] - missing = [] - for anchor in anchors: - if anchor.lower() in joined_lower: - found.append(anchor) - else: - missing.append(anchor) - if found: - return True, [] - return False, missing + failures: list[str] = [] + + # Date anchors: every component (year + month name if present) + # must appear in some cited span (case-insensitive substring). + # Year strings are case-trivial; month names round-trip via + # `.lower()`. + for d in date_anchors: + if d.lower() not in joined_lower: + failures.append(d) + + # Proper-noun anchors: at least one match suffices. + if proper_anchors: + any_found = any(a.lower() in joined_lower for a in proper_anchors) + if not any_found: + failures.extend(proper_anchors) + + if failures: + return False, failures + return True, [] diff --git a/tests/test_warrant.py b/tests/test_warrant.py index 6273214..b0259f7 100644 --- a/tests/test_warrant.py +++ b/tests/test_warrant.py @@ -1,19 +1,24 @@ -"""Tests for warrant-lite — relation-question hard check. +"""Tests for warrant-lite — anchor-class hard check. Covers: - is_relation_question detects "who is X's Y", "who founded Z", "who is the boss of W", etc. - extract_answer_anchors pulls multi-word proper nouns; falls back to solo caps skipping sentence-starter - - warrant_check returns (True, []) when answer entity in cited - span; (False, [missing]) when not; vacuous-pass when claim - has no proper noun + - extract_date_anchors pulls 4-digit years (1500-2199 range) + - warrant_check composes proper-noun + date anchor checks: + * proper-noun: gated on relation question shape; any-match + across anchors + * date: always-on when claim has a year; ALL years required + in some cited span + * vacuous-pass when neither class fires """ from __future__ import annotations from aborist.qa.warrant import ( extract_answer_anchors, + extract_date_anchors, is_relation_question, warrant_check, ) @@ -94,10 +99,11 @@ def test_extract_anchors_handles_empty_text(): def test_warrant_passes_when_anchor_in_span(): - """Standard happy path: claim names Mr. Burns, span has Mr. Burns.""" + """Relation-question happy path: claim names Mr. Burns, span has Mr. Burns.""" ok, missing = warrant_check( "Homer Simpson's boss is Mr. Burns.", ["Mr. Burns owns the Springfield Nuclear Power Plant where Homer works."], + question="who is homer simpson's boss?", ) assert ok is True assert missing == [] @@ -114,6 +120,7 @@ def test_warrant_fails_when_answer_anchor_absent(): "Castellaneta has won four Primetime Emmy Awards for " "Outstanding Voice-Over Performance." ], + question="who is homer simpson's boss?", ) assert ok is False # "Mr. Burns" should be in the missing list (anchor extracted @@ -130,22 +137,23 @@ def test_warrant_passes_when_at_least_one_anchor_in_span(): "The Springfield Nuclear Power Plant is operated by " "Mr. Burns in the show." ], + question="who owns the springfield nuclear power plant?", ) assert ok is True assert missing == [] def test_warrant_vacuous_pass_when_no_anchors_in_claim(): - """A claim with no proper-noun anchor (e.g. all common-noun - description) has nothing to demand from cited spans — - warrant_check returns (True, []) so the other six hard - checks carry the load. + """A claim with no proper-noun anchor and no date anchor has + nothing to demand from cited spans — warrant_check returns + (True, []) so the other six hard checks carry the load. This is the same vacuous-pass discipline as the existing `_claim_textually_overlaps_evidence` pure-stopword case.""" ok, missing = warrant_check( "the cake is a lie.", ["completely unrelated chunk text"], + question="what is the cake?", ) assert ok is True assert missing == [] @@ -157,6 +165,7 @@ def test_warrant_case_insensitive_match(): ok, missing = warrant_check( "Homer's boss is Mr. Burns.", ["the show features mr. burns as the plant owner"], + question="who is homer's boss?", ) assert ok is True assert missing == [] @@ -165,8 +174,184 @@ def test_warrant_case_insensitive_match(): def test_warrant_handles_empty_inputs(): """No claim text → vacuous pass. No spans → fail (anchors can't match).""" - ok, _ = warrant_check("", ["any text"]) + ok, _ = warrant_check("", ["any text"], question="who is x's y?") assert ok is True - ok, missing = warrant_check("Mr. Burns is the boss.", []) + ok, missing = warrant_check( + "Mr. Burns is the boss.", [], question="who is the boss?" + ) assert ok is False assert "Mr. Burns" in missing + + +def test_warrant_skips_proper_noun_check_without_relation_question(): + """Without a relation-shape question, proper-noun warrant is + NOT enforced — broad-descriptive claims like "Connecticut is a + state in New England" don't require the cited span to name + every Title-Case phrase. The other six hard checks carry the + load (coverage, source role, etc.).""" + ok, missing = warrant_check( + "Connecticut is in New England.", + ["Some unrelated state-history text without those terms."], + question="tell me about connecticut", # not relation-shape + ) + # Vacuous pass — no relation question means no proper-noun + # warrant; no year means no date warrant. + assert ok is True + assert missing == [] + + +# ---------------------------------------------------------------- extract_date_anchors + + +def test_extract_date_anchors_pulls_year(): + """Year is the load-bearing piece for date claims.""" + assert extract_date_anchors("Released in 1985.") == ["1985"] + assert extract_date_anchors("the war ended in 1945") == ["1945"] + + +def test_extract_date_anchors_pulls_month_and_year(): + """When the claim names both a month and a year, both are + anchors — the cited span must contain BOTH (independently).""" + anchors = extract_date_anchors("July 3, 1985 was the day") + assert "1985" in anchors + assert "July" in anchors + + +def test_extract_date_anchors_dedup_and_order(): + """Multiple years collapse to unique-in-order; year then month.""" + anchors = extract_date_anchors( + "Released in 1985, re-released in 1985, and again in 2002." + ) + assert anchors == ["1985", "2002"] + + +def test_extract_date_anchors_skips_non_year_4_digits(): + """ZIP codes, elevations, room numbers should NOT be picked up. + Year regex matches only 1500-2199.""" + assert extract_date_anchors("ZIP code 06360") == [] + assert extract_date_anchors("elevation 5280 feet") == [] + assert extract_date_anchors("room 9999 on floor 12") == [] + # but historical and modern years yes + assert extract_date_anchors("colonized in 1607") == ["1607"] + assert extract_date_anchors("released in 2025") == ["2025"] + + +def test_extract_date_anchors_handles_empty(): + assert extract_date_anchors("") == [] + assert extract_date_anchors(None) == [] + assert extract_date_anchors("no year here, no month either") == [] + + +# ---------------------------------------------------------------- date-anchor warrant + + +def test_warrant_fails_when_date_anchor_absent(): + """The back-to-the-future failure (2026-05-01). + Claim asserts year 1985 + month July; cited span has neither. + Date warrant must fail regardless of question shape.""" + ok, missing = warrant_check( + "Back to the Future was released in theaters on July 3, 1985.", + [ + "Time periods from the trilogy ... SNES Japanese release ... " + "the 1990 pinball game ..." + ], + question="what date did back to the future come out?", + ) + assert ok is False + # Both year and month should be missing + assert "1985" in missing + assert "July" in missing + + +def test_warrant_fails_when_year_in_unrelated_context(): + """Subtler back-to-the-future case: the cited span DOES contain + "1985" but only as in-universe time-period reference, never with + the month. Month-name requirement catches this where year-alone + would not.""" + ok, missing = warrant_check( + "Back to the Future was released in theaters on July 3, 1985.", + [ + "Marty attempts to correct the timeline and get back to " + "the real 1985. A Japanese-only SNES release, Super Back " + "to the Future II, allowed the player to control Marty." + ], + question="when did back to the future come out?", + ) + # Year "1985" is in span but month "July" is not. Composite + # date anchor fails because both components are required. + assert ok is False + assert "July" in missing + + +def test_warrant_passes_when_full_date_in_span(): + """Date warrant passes when EVERY date component the claim asserts + appears in some cited span — month + year both required when + both are in the claim.""" + ok, missing = warrant_check( + "Back to the Future was released in theaters on July 3, 1985.", + [ + "Back to the Future is a 1985 American science fiction film " + "released in July 1985. It grossed over $381 million worldwide." + ], + question="when was back to the future released?", + ) + assert ok is True + assert missing == [] + + +def test_warrant_passes_when_year_only_claim_year_only_span(): + """Year-only claim against year-only span: substring match + suffices since the claim names no month for the span to mirror.""" + ok, missing = warrant_check( + "Back to the Future is a 1985 film.", + ["Back to the Future is a 1985 American science fiction film."], + question="when was back to the future released?", + ) + assert ok is True + assert missing == [] + + +def test_warrant_date_check_runs_without_relation_question(): + """Date warrant fires regardless of question shape — a claim + asserting a year demands that year in the cited span, + independent of whether the question is relation-shaped.""" + ok, missing = warrant_check( + "The first moon landing happened in 1969.", + ["Apollo 11 astronauts visited the lunar surface."], + question="when did the first moon landing happen?", + ) + # 1969 is not in the span → fails + assert ok is False + assert "1969" in missing + + +def test_warrant_composes_date_and_proper_noun(): + """Both anchor classes compose: relation question + claim with + BOTH proper-noun anchor AND date anchor → both must satisfy.""" + # Both pass: span has Mr. Burns (proper noun) AND 1991 (date) + ok, missing = warrant_check( + "Mr. Burns first appeared in 1991.", + ["Mr. Burns made his debut in 1991 on the show."], + question="who is mr. burns?", + ) + assert ok is True + + # Date fails: span has Mr. Burns but year mismatch + ok, missing = warrant_check( + "Mr. Burns first appeared in 1991.", + ["Mr. Burns made his debut in 1989 on the show."], + question="who is mr. burns?", + ) + assert ok is False + assert "1991" in missing + + +def test_warrant_vacuous_when_no_year_no_relation(): + """Claim with no year + non-relation question = vacuous pass.""" + ok, missing = warrant_check( + "Connecticut is a state.", + ["Some unrelated text."], + question="tell me about connecticut", + ) + assert ok is True + assert missing == []