"""Faithfulness verifier: classify answer grounding into v9.8 audit_mode. Three layered strategies (quote → span → entity), tried in order. The first that finds evidence classifies the answer. `verifier_method` records which path fired. Tests cover the full trichotomy under each path plus regressions: - TMNT: model wrote bios not in context — must be HYBRID/UNGROUNDED - Matrix: Wikipedia infobox + paraphrased prose. Spans don't match, but every multi-word proper noun does — entity path classifies it. """ from __future__ import annotations from arborist.qa.verify import ( _entity_salient_disagrees, _is_single_sentence, _normalize, _numeric_signature, extract_claim_spans, extract_proper_nouns, extract_quotes, verify_quotes, ) # ---------------------------------------------------------------- extract def test_extract_quotes_finds_double_quoted_spans(): text = 'He said "the cat is on the mat" and then "rain in spain falls".' assert extract_quotes(text) == [ "the cat is on the mat", "rain in spain falls", ] def test_extract_quotes_skips_short_spans(): # 8-char minimum keeps "is a", "the", "ok" out. text = '"is a" "ok" "long enough span" "no"' assert extract_quotes(text) == ["long enough span"] def test_extract_quotes_sequential_pairs_skip_inter_pair_prose(): """Fox caught on 2026-04-28: model wrote `"Hackers" is as follows: "In 1988..."` and naive regex matching paired the closer of "Hackers" with the opener of "In 1988", capturing the framing prose between them as a phantom quote. Sequential pairing (1st & 2nd, 3rd & 4th, ...) treats inter-pair text as the model's framing — never as a quoted span.""" text = ( 'The plot of the 1995 film "Hackers" is as follows, directly ' 'quoted from the source:\n\n"In 1988, Seattle youth Dade Murphy."' ) quotes = extract_quotes(text) # "Hackers" is 7 chars — below MIN_QUOTE_CHARS, dropped. # "In 1988, Seattle youth Dade Murphy." pairs cleanly. # The phantom prose between them is never captured. assert quotes == ["In 1988, Seattle youth Dade Murphy."] def test_extract_quotes_handles_three_adjacent_pairs(): """`"A_____" stuff "B_____" more "C_____"` → three clean pairs.""" text = '"alpha-99" stuff "bravo-99" more "charlie9"' quotes = extract_quotes(text) assert quotes == ["alpha-99", "bravo-99", "charlie9"] def test_extract_quotes_handles_curly_quotes(): text = '“smart quoted span here” and "ascii quoted span"' quotes = extract_quotes(text) assert "smart quoted span here" in quotes assert "ascii quoted span" in quotes # ---------------------------------------------------------------- classify def test_strict_when_all_quotes_verify(): context = "Capitalism is an economic system based on private ownership." answer = 'The source defines it: "an economic system based on private ownership"' v = verify_quotes(answer, context) assert v["audit_mode"] == "STRICT" assert v["verifier_method"] == "quote" assert v["n_quotes"] == 1 assert v["n_verified"] == 1 assert v["unverified_quotes"] == [] def test_visual_when_no_quotes_no_spans_no_entities(): """Truly emergent: no double quotes, no span match, no entity match.""" context = "Apples are red." answer = "freedom rests on autonomy alone, without coercion." v = verify_quotes(answer, context) assert v["audit_mode"] == "UNGROUNDED" assert v["verifier_method"] == "none" assert v["n_quotes"] == 0 def test_visual_when_no_quote_verifies(): context = "Capitalism is an economic system based on private ownership." answer = '"this exact span is not in the source at all"' v = verify_quotes(answer, context) assert v["audit_mode"] == "UNGROUNDED" assert v["verifier_method"] == "quote" assert v["n_quotes"] == 1 assert v["n_verified"] == 0 assert v["unverified_quotes"] == ["this exact span is not in the source at all"] def test_hybrid_when_some_quotes_verify_some_dont(): context = "Capitalism is an economic system based on private ownership." answer = ( 'The source says "an economic system" but then claims ' '"Marx personally invented capitalism in 1867" which is a stretch.' ) v = verify_quotes(answer, context) assert v["audit_mode"] == "HYBRID" assert v["verifier_method"] == "quote" assert v["n_quotes"] == 2 assert v["n_verified"] == 1 assert v["unverified_quotes"] == [ "Marx personally invented capitalism in 1867" ] # ---------------------------------------------------------------- normalization def test_case_and_whitespace_insensitive_match(): context = "The Eight Forms of Capital include living, social, and intellectual." answer = '"the eight forms of capital"' # collapsed ws + lowercase + multi-space v = verify_quotes(answer, context) assert v["audit_mode"] == "STRICT" assert v["n_verified"] == 1 # ---------------------------------------------------------------- claim statuses def test_claim_statuses_quote_path_labels_each_unit(): """Per-evidence-unit status objects (toy-Hermes taxonomy 2026-04-30): quote-path returns one entry per quoted span with VERIFIED_QUOTE for matches & UNSUPPORTED for misses. method='quote'.""" context = "Capitalism is an economic system based on private ownership." answer = ( 'The source says "an economic system" but invents ' '"Marx personally launched capitalism in 1867".' ) v = verify_quotes(answer, context) statuses = v["claim_statuses"] assert len(statuses) == 2 assert {s["status"] for s in statuses} == {"VERIFIED_QUOTE", "UNSUPPORTED"} assert all(s["method"] == "quote" for s in statuses) def test_claim_statuses_paraphrase_method_flagged(): """Span path with paraphrase fallback: items that pass via token-coverage get SUPPORTED_PARAPHRASE + method='paraphrase'. Substring-verified items in the same call get VERIFIED_QUOTE.""" # Long enough source; the answer paraphrases one line and verbatim- # quotes none — so span path runs paraphrase on the single bullet. context = ( "Cloud Strife is the protagonist of Final Fantasy VII. " "He wields the Buster Sword in battle and was a SOLDIER." ) answer = "- cloud strife is the main character in final fantasy seven" v = verify_quotes(answer, context) if v["claim_statuses"]: # Either VERIFIED_QUOTE (substring match) or SUPPORTED_PARAPHRASE. s = v["claim_statuses"][0] assert s["status"] in ("VERIFIED_QUOTE", "SUPPORTED_PARAPHRASE") if s["status"] == "SUPPORTED_PARAPHRASE": assert s["method"] == "paraphrase" def test_claim_statuses_empty_when_no_evidence(): """Truly emergent answer (no quotes, no spans matching, no entities): claim_statuses is an empty list.""" context = "Apples are red." answer = "freedom rests on autonomy alone, without coercion." v = verify_quotes(answer, context) assert v["claim_statuses"] == [] assert v["audit_mode"] == "UNGROUNDED" # ---------------------------------------------------------------- regression def test_tmnt_regression_must_not_be_strict(): """Fox's catch on 2026-04-28: model emerged TMNT bios from training while only the names appeared in context (an episode plot summary). The state machine must downgrade — STRICT here would be a lie.""" context = ( "Master Splinter is framed for attempting to kill the Ultimate Daimyo. " "Raphael and Michelangelo find themselves pitted against each other in " "the tournament. Donatello and Usagi must defend Leonardo from assassins." ) answer = ( '1. Leonardo - "leader of the group, he wields a blue katana"\n' '2. Splinter - "a mutated rat who was once Hamato Yoshi"\n' '3. Plot - "Raphael and Michelangelo find themselves pitted against each other"' ) v = verify_quotes(answer, context) assert v["audit_mode"] == "HYBRID" assert v["verifier_method"] == "quote" assert v["n_verified"] == 1 assert len(v["unverified_quotes"]) == 2 assert any("blue katana" in q for q in v["unverified_quotes"]) assert any("Hamato Yoshi" in q for q in v["unverified_quotes"]) # ---------------------------------------------------------------- spans / entities def test_extract_spans_strips_bullets_and_framing(): answer = ( "Based on the provided sources, the main characters are:\n\n" "- Neo\n" "- Trinity\n" "1. Morpheus is a leader\n" ) spans = extract_claim_spans(answer) # Framing line dropped; bullet markers stripped; short single-word # bullets ("Neo", "Trinity") below MIN_SPAN_CHARS dropped. assert "Morpheus is a leader" in spans assert not any(s.startswith("Based on the") for s in spans) def test_extract_proper_nouns_picks_multi_word_phrases(): text = ( "Neo (Thomas A. Anderson), played by Keanu Reeves. " "Morpheus, played by Laurence Fishburne." ) nouns = extract_proper_nouns(text) assert "Thomas A. Anderson" in nouns assert "Keanu Reeves" in nouns assert "Laurence Fishburne" in nouns # Single-word names skipped at this layer. assert "Neo" not in nouns assert "Morpheus" not in nouns def test_span_path_classifies_when_bullet_appears_verbatim(): """A model that doesn't quote but writes lines verbatim from source still earns evidence via the span path.""" context = "Apple Inc. was founded by Steve Jobs, Steve Wozniak, and Ronald Wayne in 1976." answer = ( "- Apple Inc. was founded by Steve Jobs, Steve Wozniak, and Ronald Wayne in 1976\n" ) v = verify_quotes(answer, context) assert v["audit_mode"] == "STRICT" assert v["verifier_method"] == "span" assert v["n_verified"] >= 1 def test_matrix_entity_path_strict_under_default_proximity(): """Default entity_policy='proximity': Matrix-style answer where the source has a tight cluster of cast-list entities (infobox) classifies STRICT. Distinguishes structural grounding from incidental mention.""" context = ( "starring = [[Keanu Reeves]] [[Laurence Fishburne]] " "[[Carrie-Anne Moss]] [[Hugo Weaving]] [[Joe Pantoliano]]. " "Thomas A. Anderson is the protagonist. The film features " "Agent Smith as the antagonist. Joe Pantoliano portrays Cypher." ) answer = ( "Based on the provided sources, the main characters are:\n\n" "- Neo (Thomas A. Anderson), played by Keanu Reeves\n" "- Morpheus, played by Laurence Fishburne\n" "- Trinity, played by Carrie-Anne Moss\n" "- Agent Smith, played by Hugo Weaving\n" "- Cypher, played by Joe Pantoliano\n" ) v = verify_quotes(answer, context) assert v["audit_mode"] == "STRICT" assert v["verifier_method"] == "entity" assert v["n_verified"] >= 5 def test_tmnt_entity_path_demoted_under_default_proximity(): """Companion to the Matrix test: an answer with only one multi-word entity ("Teenage Mutant Ninja Turtles") cannot satisfy proximity's N=3 cluster requirement → demoted to HYBRID even though the entity verifies. This is what separates incidental mention from structural grounding.""" context = ( "Master Splinter is framed for attempting to kill the Ultimate Daimyo. " "Raphael and Michelangelo find themselves pitted against each other. " "Donatello and Usagi defend Leonardo. Teenage Mutant Ninja Turtles " "appears throughout the article." ) answer = ( "Based on the provided sources, the names of the Teenage Mutant Ninja " "Turtles' brothers are:\n\n" "1. Leonardo (Leo)\n" "2. Raphael (Raph)\n" "3. Donatello (Donnie)\n" "4. Michelangelo (Mikey)\n\n" "Their master's name is Splinter." ) v = verify_quotes(answer, context) assert v["audit_mode"] == "HYBRID" assert v["verifier_method"] == "entity" def test_entity_policy_strict_promotes_when_all_match(): """Legacy policy: entity_policy='strict' promotes to STRICT when all entities verify. Kept for back-compat / experimentation; overclaims.""" context = "Keanu Reeves, Laurence Fishburne, Carrie-Anne Moss appear." answer = "The cast includes Keanu Reeves, Laurence Fishburne, and Carrie-Anne Moss." v = verify_quotes(answer, context, entity_policy="strict") assert v["audit_mode"] == "STRICT" assert v["verifier_method"] == "entity" def test_entity_policy_drop_skips_path_entirely(): """drop policy: never classifies via entity path. Verified entities in source are ignored; the answer falls through to UNGROUNDED/none.""" context = "Keanu Reeves stars in this film." answer = "Keanu Reeves played the lead role." v = verify_quotes(answer, context, entity_policy="drop") assert v["audit_mode"] == "UNGROUNDED" assert v["verifier_method"] == "none" def test_entity_policy_proximity_promotes_only_on_cluster(): """proximity policy: STRICT only if N=3 verified entities cluster within W=300 chars in source. Tight infobox = STRICT. Scattered prose = HYBRID.""" # Tight cluster: 3 names within ~50 chars. tight_ctx = "Cast: Keanu Reeves, Laurence Fishburne, Carrie-Anne Moss starred." answer = "The cast: Keanu Reeves, Laurence Fishburne, and Carrie-Anne Moss." v = verify_quotes(answer, tight_ctx, entity_policy="proximity") assert v["audit_mode"] == "STRICT" assert v["verifier_method"] == "entity" # Scattered: same 3 names across 1000+ chars of unrelated prose. spread_ctx = ( "Keanu Reeves appeared briefly in scene one. " + "filler text " * 60 + "Laurence Fishburne had a cameo. " + "more filler " * 60 + "Carrie-Anne Moss made an appearance." ) v = verify_quotes(answer, spread_ctx, entity_policy="proximity") assert v["audit_mode"] == "HYBRID" assert v["verifier_method"] == "entity" def test_entity_path_hybrid_when_some_proper_nouns_unverified(): """Mixed: some proper nouns verify, others are emergent. Default 'hybrid' policy → HYBRID either way.""" context = "Keanu Reeves and Laurence Fishburne starred together." answer = "The film featured Keanu Reeves, Laurence Fishburne, and Marlon Brando." v = verify_quotes(answer, context) assert v["audit_mode"] == "HYBRID" assert v["verifier_method"] == "entity" assert "Marlon Brando" in v["unverified_quotes"] # ----------------------------------------------- wikitext-base-v1 integration def test_wikitext_context_strips_markup_before_substring_test(): """Without the strip the verifier sees [[Cloud Strife]] in source and a bare 'Cloud Strife' in the answer's quoted span; substring fails. With arborist.wikitext.to_base wired into verify_quotes, the context becomes plain prose and the model's quote verifies. This is the case from the real FF7 query that prompted wiring wikitext-base-v1 into the verifier.""" raw_wikitext_context = ( "[[Cloud Strife]], an unsociable mercenary who claims to be a former " "1st Class member of [[Shinra Electric Power Company|Shinra]]'s " "[[SOLDIER (Final Fantasy VII)|SOLDIER]] unit;Smith 2010" ) answer = ( 'The protagonist is described as: "Cloud Strife, an unsociable ' "mercenary who claims to be a former 1st Class member of Shinra's " 'SOLDIER unit;"' ) v = verify_quotes(answer, raw_wikitext_context) assert v["verifier_method"] == "quote" assert v["n_verified"] == 1 assert v["audit_mode"] == "STRICT" assert v["unverified_quotes"] == [] # --------------------------------------------------------------------------- # Trailing-citation strip — model-appended `(Source: ...)` no longer breaks # verbatim substring match. # --------------------------------------------------------------------------- def test_extract_quotes_strips_trailing_source_parenthetical(): """The Pikachu real-corpus case: model wrote a verbatim quote and appended a citation. Strip the citation so substring match works.""" answer = ( 'The model said: "Pikachu can store electricity in its cheeks. ' '(Source: https://en.wikipedia.org/wiki/Pikachu)"' ) quotes = extract_quotes(answer) assert quotes == ["Pikachu can store electricity in its cheeks."] def test_extract_claim_spans_strips_trailing_source_parenthetical(): """Span-strategy variant — sentence-level extraction also strips.""" answer = ( "Pikachu evolves from Pichu. (Source: https://example.com/p)\n" "Pikachu has yellow fur. (citing Wikipedia)" ) spans = extract_claim_spans(answer) assert any("Pikachu evolves from Pichu" in s and "Source" not in s for s in spans) assert any("Pikachu has yellow fur" in s and "citing" not in s for s in spans) def test_extract_claim_spans_strips_url_only_parenthetical(): """A bare URL parenthetical (no `Source:` cue word) is also a citation.""" answer = "The release date was 1996 (https://en.wikipedia.org/wiki/Pikachu)" spans = extract_claim_spans(answer) assert any("https" not in s for s in spans) def test_strict_recovered_with_citation_strip(): """End-to-end: a span with verbatim source content + appended citation used to flag UNGROUNDED. With the strip, it classifies STRICT.""" context = ( "Pikachu can store electricity in its cheeks and release it in " "lightning-based attacks." ) answer = ( "Pikachu can store electricity in its cheeks and release it in " "lightning-based attacks. (Source: https://en.wikipedia.org/wiki/Pikachu)" ) v = verify_quotes(answer, context) assert v["audit_mode"] == "STRICT" assert v["unverified_quotes"] == [] # --------------------------------------------------------------------------- # Paraphrase strategy (4th tier) — token-coverage fallback in span path. # --------------------------------------------------------------------------- def test_paraphrase_promotes_high_token_coverage_span(): """Model rewrites source content but every meaningful token is in the corpus. Verifier accepts as paraphrase-verified.""" # Context contains every meaningful (>=4 char) token from the answer # span, just in a different sequence — that's the paraphrase case. context = ( "Pikachu is a Pokémon species, one of many creatures from the " "Pokémon franchise developed by Game Freak." ) answer = ( "Pikachu is a species of Pokémon creatures from the Pokémon " "franchise." ) v = verify_quotes(answer, context) assert v["verifier_method"] == "paraphrase" assert v["audit_mode"] == "STRICT" assert v["unverified_quotes"] == [] def test_paraphrase_does_not_promote_low_token_coverage_span(): """A span whose content is mostly NOT in the corpus stays unverified.""" context = "Pikachu is a Pokémon species." answer = "Pikachu was elected mayor of Tokyo in 1988 by aristocrats." v = verify_quotes(answer, context) assert v["audit_mode"] == "UNGROUNDED" def test_paraphrase_method_label_set_when_any_paraphrase_used(): """If even one span verified via paraphrase (and others via substring), verifier_method flips to 'paraphrase' so an auditor knows soft signals were involved.""" # Answer line 1 is verbatim. Line 2 reorders the same content tokens # — paraphrase. Both classify as verified. context = ( "Apple Inc. was founded by Steve Jobs and Steve Wozniak. " "Apple has its main office in Cupertino, California, from where " "the company manages worldwide operations." ) answer = ( "Apple Inc. was founded by Steve Jobs and Steve Wozniak.\n" "The company manages its worldwide operations from Cupertino." ) v = verify_quotes(answer, context) assert v["verifier_method"] == "paraphrase" assert v["audit_mode"] == "STRICT" def test_paraphrase_stopword_filter_does_not_inflate_coverage(): """Stopwords (`from`, `with`, `which`, etc.) match almost any English text. Excluding them from token-coverage tightens the signal: a span where the topical content is missing scores LOWER, not higher. The fox 2026-04-29 Batman case: span 'Batman is the alias of Bruce Wayne, a wealthy businessman who resides in Gotham City' has 'wealthy/businessman/resides' missing from the corpus. With a clean stopword filter, those topical misses dominate the score and the span correctly stays UNGROUNDED rather than scraping over a lowered threshold.""" context = ( "Batman is the alias of Bruce Wayne. Batman lives in Gotham City." ) answer_with_extra_stopwords = ( "Batman, who is the alias of Bruce Wayne, lives in Gotham City." ) # Topical tokens (batman, alias, bruce, wayne, lives, gotham, city) # all present in source. Stopwords ('which', 'who', etc.) are # filtered. Coverage on filtered set = 1.0 → paraphrase verifies. v = verify_quotes(answer_with_extra_stopwords, context) assert v["audit_mode"] == "STRICT" def test_paraphrase_rejects_when_topical_tokens_missing(): """Confirms that lowering the bar wouldn't be a quick win — Q1's Batman case (missing 'wealthy/businessman/resides') stays UNGROUNDED because the topical content isn't in source. Stopword filter doesn't rescue it.""" context = "Batman is Bruce Wayne. Batman fights crime in Gotham." answer = ( "Batman is the alias of Bruce Wayne, a wealthy businessman who " "resides in Gotham City." ) # 'wealthy', 'businessman', 'resides', 'alias' all absent from # context. Topical content missing — coverage drops below 0.85. v = verify_quotes(answer, context) # Either UNGROUNDED (substring fail) or HYBRID (some entities) — # the key constraint is NOT STRICT (would mean fabrication promoted). assert v["audit_mode"] != "STRICT" def test_paraphrase_strategy_keeps_quote_strategy_strict(): """Quote strategy stays verbatim-only — quotes ARE quotes, paraphrase- in-quotes is the model's mistake. Paraphrase fallback applies to span strategy only.""" context = "The actual sentence is something specific about cats." answer = '"This is a totally different sentence about dogs and rats"' v = verify_quotes(answer, context) assert v["verifier_method"] == "quote" assert v["audit_mode"] == "UNGROUNDED" def test_wikitext_strip_does_not_rescue_genuine_hallucination(): """A quote the model invented stays unverified even after stripping.""" raw_wikitext_context = ( "[[Cloud Strife]] is the [[protagonist]] of [[Final Fantasy VII]]." ) answer = ( 'The story is: "Set in a dystopian world, Final Fantasy VII\'s story ' 'centers on mercenary Cloud Strife..."' ) v = verify_quotes(answer, raw_wikitext_context) assert v["verifier_method"] == "quote" assert v["n_verified"] == 0 assert v["audit_mode"] == "UNGROUNDED" assert len(v["unverified_quotes"]) == 1 # --- #000046 — paraphrase numeric-agreement gate ----------------- def test_numeric_signature_comma_normalized(): assert _numeric_signature("8,849 meters") == frozenset({"8849"}) assert _numeric_signature("300,000 km") == frozenset({"300000"}) assert _numeric_signature("the year 1789, the year 1889") == frozenset({"1789", "1889"}) assert _numeric_signature("no digit numbers at all") == frozenset() # decimals split at the dot — fine for the near-miss patterns we target assert _numeric_signature("pi is 3.14 plus 2") == frozenset({"3", "14", "2"}) def test_numeric_signature_subset_matches_comma_variant(): # a span number written without a thousands comma is "in" a source # number written with one (both collapse), so legitimate # number-paraphrases still verify. assert _numeric_signature("8849") <= _numeric_signature("Mount Everest is 8,849 m tall") def test_paraphrase_numeric_disagreement_rejected(): # All content tokens of "Water boils at 50 degrees Celsius at sea # level" are in the source (which says 100), so token coverage is # 100% — but the number is wrong. The numeric-agreement gate # demotes it from paraphrase-STRICT to UNGROUNDED. v = verify_quotes( "Water boils at 50 degrees Celsius at sea level.", "Water boils at 100 degrees Celsius at sea level.", ) assert v["audit_mode"] == "UNGROUNDED" # The magnitude case too: 300 vs 300,000 (1000x off). v2 = verify_quotes( "The speed of light is approximately 300 kilometers per second.", "The speed of light is approximately 300,000 kilometers per second in a vacuum.", ) assert v2["audit_mode"] == "UNGROUNDED" def test_paraphrase_numeric_gate_is_narrow_no_number_unaffected(): # The gate fires ONLY on a digit-number the source lacks. A # recombined-but-no-number near-miss ("Mercury is the largest" vs # source "Jupiter is the largest; Mercury is the smallest") is NOT # caught — that's the headroom a future verifier upgrade closes, # not this gate's job. Pinning the narrowness so a later widening # is a deliberate choice. v = verify_quotes( "Mercury is the largest planet in the Solar System.", "Jupiter is the largest planet in the Solar System; Mercury is the smallest.", ) assert v["audit_mode"] == "STRICT" assert v["verifier_method"] == "paraphrase" def test_paraphrase_number_present_in_source_still_verifies(): # A genuine paraphrase whose number IS in the source verifies as # before — the gate doesn't touch it. v = verify_quotes( "The French Revolution started in 1789.", "The French Revolution began in 1789, with the storming of the Bastille.", ) # quote/span/entity may classify first; what matters is it's not # demoted to UNGROUNDED by a spurious numeric mismatch. assert v["audit_mode"] in ("STRICT", "HYBRID") # --- #000048 step 2.1 — entity salient-token-disagreement gate ---- def test_is_single_sentence_helper(): assert _is_single_sentence("Insulin was discovered by Alexander Fleming.") assert _is_single_sentence("The Eiffel Tower is in Berlin.") assert _is_single_sentence("The Titanic sank in the Pacific Ocean.\n") # trailing ws stripped assert not _is_single_sentence("One claim. Another claim.") assert not _is_single_sentence("1. Leonardo\n2. Raphael\n3. Donatello") assert not _is_single_sentence("Was it true? She asked.") def test_entity_salient_disagrees_helper(): # Swapped subject: "Insulin" is capitalized, >4 chars, absent from # the source → disagrees. assert _entity_salient_disagrees( "Insulin was discovered by Alexander Fleming.", _normalize("Penicillin was discovered by Alexander Fleming in 1928."), ) # Swapped city: "London" absent. assert _entity_salient_disagrees( "The Eiffel Tower is in London.", _normalize("The Eiffel Tower is in Paris France."), ) # Swapped year: a digit-number absent. assert _entity_salient_disagrees( "The French Revolution began in 1889.", _normalize("The French Revolution began in 1789, with the storming of the Bastille."), ) # Legit entity claim: every capitalized token is in the source → # does NOT disagree (the prose differing — "stars in" vs "cast" — # is fine; only capitalized tokens / numbers are checked). assert not _entity_salient_disagrees( "Keanu Reeves stars in The Matrix.", _normalize("The Matrix cast: Keanu Reeves as Neo, Laurence Fishburne as Morpheus."), ) def test_entity_path_swapped_subject_demoted_to_ungrounded(): """The entity strategy used to grant HYBRID for "Insulin was discovered by Alexander Fleming" against "Penicillin was discovered by Alexander Fleming" on the shared "Alexander Fleming". #000048's salient gate (single short sentence, lone non-clustered match, a capitalized token the source lacks) declines that grounding.""" v = verify_quotes( "Insulin was discovered by Alexander Fleming.", "Penicillin was discovered by Alexander Fleming in 1928.", ) assert v["audit_mode"] == "UNGROUNDED" v2 = verify_quotes( "The Eiffel Tower is in Berlin.", "The Eiffel Tower is in Paris, designed by Gustave Eiffel.", ) assert v2["audit_mode"] == "UNGROUNDED" def test_entity_gate_narrow_multi_claim_summary_unaffected(): """The gate fires only on the single-sentence-lone-match shape — a structured multi-claim summary that the source partly grounds (a cast list with model-added accurate detail) is NOT demoted. The Matrix / TMNT regression tests above are the full version; this pins the principle on a minimal case: two sentences, so even a capitalized token the source lacks ("Morpheus") doesn't trip the gate.""" v = verify_quotes( "The cast includes Keanu Reeves. Morpheus is played by Laurence Fishburne.", "Cast of The Matrix: Keanu Reeves, Laurence Fishburne, Carrie-Anne Moss.", ) # Not single-sentence → entity gate doesn't fire → still grounded # at HYBRID (or STRICT if a cluster forms — either way, not # UNGROUNDED-by-the-gate). assert v["audit_mode"] in ("STRICT", "HYBRID")