arborist/tests/test_verify.py

"""Faithfulness verifier: classify answer grounding into v9.8 audit_mode.

Three layered strategies (quote → span → entity), tried in order. The
first that finds evidence classifies the answer. `verifier_method`
records which path fired.

Tests cover the full trichotomy under each path plus regressions:
    - TMNT: model wrote bios not in context — must be HYBRID/UNGROUNDED
    - Matrix: Wikipedia infobox + paraphrased prose. Spans don't match,
      but every multi-word proper noun does — entity path classifies it.
"""

from __future__ import annotations

from arborist.qa.verify import (
    extract_claim_spans,
    extract_proper_nouns,
    extract_quotes,
    verify_quotes,
)


# ---------------------------------------------------------------- extract


def test_extract_quotes_finds_double_quoted_spans():
    text = 'He said "the cat is on the mat" and then "rain in spain falls".'
    assert extract_quotes(text) == [
        "the cat is on the mat",
        "rain in spain falls",
    ]


def test_extract_quotes_skips_short_spans():
    # 8-char minimum keeps "is a", "the", "ok" out.
    text = '"is a" "ok" "long enough span" "no"'
    assert extract_quotes(text) == ["long enough span"]


def test_extract_quotes_sequential_pairs_skip_inter_pair_prose():
    """Fox caught on 2026-04-28: model wrote `"Hackers" is as follows: "In 1988..."`
    and naive regex matching paired the closer of "Hackers" with the
    opener of "In 1988", capturing the framing prose between them as a
    phantom quote. Sequential pairing (1st & 2nd, 3rd & 4th, ...) treats
    inter-pair text as the model's framing — never as a quoted span."""
    text = (
        'The plot of the 1995 film "Hackers" is as follows, directly '
        'quoted from the source:\n\n"In 1988, Seattle youth Dade Murphy."'
    )
    quotes = extract_quotes(text)
    # "Hackers" is 7 chars — below MIN_QUOTE_CHARS, dropped.
    # "In 1988, Seattle youth Dade Murphy." pairs cleanly.
    # The phantom prose between them is never captured.
    assert quotes == ["In 1988, Seattle youth Dade Murphy."]


def test_extract_quotes_handles_three_adjacent_pairs():
    """`"A_____" stuff "B_____" more "C_____"` → three clean pairs."""
    text = '"alpha-99" stuff "bravo-99" more "charlie9"'
    quotes = extract_quotes(text)
    assert quotes == ["alpha-99", "bravo-99", "charlie9"]


def test_extract_quotes_handles_curly_quotes():
    text = '“smart quoted span here” and "ascii quoted span"'
    quotes = extract_quotes(text)
    assert "smart quoted span here" in quotes
    assert "ascii quoted span" in quotes


# ---------------------------------------------------------------- classify


def test_strict_when_all_quotes_verify():
    context = "Capitalism is an economic system based on private ownership."
    answer = 'The source defines it: "an economic system based on private ownership"'
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "STRICT"
    assert v["verifier_method"] == "quote"
    assert v["n_quotes"] == 1
    assert v["n_verified"] == 1
    assert v["unverified_quotes"] == []


def test_visual_when_no_quotes_no_spans_no_entities():
    """Truly emergent: no double quotes, no span match, no entity match."""
    context = "Apples are red."
    answer = "freedom rests on autonomy alone, without coercion."
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "UNGROUNDED"
    assert v["verifier_method"] == "none"
    assert v["n_quotes"] == 0


def test_visual_when_no_quote_verifies():
    context = "Capitalism is an economic system based on private ownership."
    answer = '"this exact span is not in the source at all"'
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "UNGROUNDED"
    assert v["verifier_method"] == "quote"
    assert v["n_quotes"] == 1
    assert v["n_verified"] == 0
    assert v["unverified_quotes"] == ["this exact span is not in the source at all"]


def test_hybrid_when_some_quotes_verify_some_dont():
    context = "Capitalism is an economic system based on private ownership."
    answer = (
        'The source says "an economic system" but then claims '
        '"Marx personally invented capitalism in 1867" which is a stretch.'
    )
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "HYBRID"
    assert v["verifier_method"] == "quote"
    assert v["n_quotes"] == 2
    assert v["n_verified"] == 1
    assert v["unverified_quotes"] == [
        "Marx personally invented capitalism in 1867"
    ]


# ---------------------------------------------------------------- normalization


def test_case_and_whitespace_insensitive_match():
    context = "The Eight Forms of Capital include living, social, and intellectual."
    answer = '"the   eight forms of capital"'  # collapsed ws + lowercase + multi-space
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "STRICT"
    assert v["n_verified"] == 1


# ---------------------------------------------------------------- claim statuses


def test_claim_statuses_quote_path_labels_each_unit():
    """Per-evidence-unit status objects (toy-Hermes taxonomy 2026-04-30):
    quote-path returns one entry per quoted span with VERIFIED_QUOTE for
    matches & UNSUPPORTED for misses. method='quote'."""
    context = "Capitalism is an economic system based on private ownership."
    answer = (
        'The source says "an economic system" but invents '
        '"Marx personally launched capitalism in 1867".'
    )
    v = verify_quotes(answer, context)
    statuses = v["claim_statuses"]
    assert len(statuses) == 2
    assert {s["status"] for s in statuses} == {"VERIFIED_QUOTE", "UNSUPPORTED"}
    assert all(s["method"] == "quote" for s in statuses)


def test_claim_statuses_paraphrase_method_flagged():
    """Span path with paraphrase fallback: items that pass via
    token-coverage get SUPPORTED_PARAPHRASE + method='paraphrase'.
    Substring-verified items in the same call get VERIFIED_QUOTE."""
    # Long enough source; the answer paraphrases one line and verbatim-
    # quotes none — so span path runs paraphrase on the single bullet.
    context = (
        "Cloud Strife is the protagonist of Final Fantasy VII. "
        "He wields the Buster Sword in battle and was a SOLDIER."
    )
    answer = "- cloud strife is the main character in final fantasy seven"
    v = verify_quotes(answer, context)
    if v["claim_statuses"]:
        # Either VERIFIED_QUOTE (substring match) or SUPPORTED_PARAPHRASE.
        s = v["claim_statuses"][0]
        assert s["status"] in ("VERIFIED_QUOTE", "SUPPORTED_PARAPHRASE")
        if s["status"] == "SUPPORTED_PARAPHRASE":
            assert s["method"] == "paraphrase"


def test_claim_statuses_empty_when_no_evidence():
    """Truly emergent answer (no quotes, no spans matching, no entities):
    claim_statuses is an empty list."""
    context = "Apples are red."
    answer = "freedom rests on autonomy alone, without coercion."
    v = verify_quotes(answer, context)
    assert v["claim_statuses"] == []
    assert v["audit_mode"] == "UNGROUNDED"


# ---------------------------------------------------------------- regression


def test_tmnt_regression_must_not_be_strict():
    """Fox's catch on 2026-04-28: model emerged TMNT bios from training
    while only the names appeared in context (an episode plot summary).
    The state machine must downgrade — STRICT here would be a lie."""
    context = (
        "Master Splinter is framed for attempting to kill the Ultimate Daimyo. "
        "Raphael and Michelangelo find themselves pitted against each other in "
        "the tournament. Donatello and Usagi must defend Leonardo from assassins."
    )
    answer = (
        '1. Leonardo - "leader of the group, he wields a blue katana"\n'
        '2. Splinter - "a mutated rat who was once Hamato Yoshi"\n'
        '3. Plot - "Raphael and Michelangelo find themselves pitted against each other"'
    )
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "HYBRID"
    assert v["verifier_method"] == "quote"
    assert v["n_verified"] == 1
    assert len(v["unverified_quotes"]) == 2
    assert any("blue katana" in q for q in v["unverified_quotes"])
    assert any("Hamato Yoshi" in q for q in v["unverified_quotes"])


# ---------------------------------------------------------------- spans / entities


def test_extract_spans_strips_bullets_and_framing():
    answer = (
        "Based on the provided sources, the main characters are:\n\n"
        "- Neo\n"
        "- Trinity\n"
        "1. Morpheus is a leader\n"
    )
    spans = extract_claim_spans(answer)
    # Framing line dropped; bullet markers stripped; short single-word
    # bullets ("Neo", "Trinity") below MIN_SPAN_CHARS dropped.
    assert "Morpheus is a leader" in spans
    assert not any(s.startswith("Based on the") for s in spans)


def test_extract_proper_nouns_picks_multi_word_phrases():
    text = (
        "Neo (Thomas A. Anderson), played by Keanu Reeves. "
        "Morpheus, played by Laurence Fishburne."
    )
    nouns = extract_proper_nouns(text)
    assert "Thomas A. Anderson" in nouns
    assert "Keanu Reeves" in nouns
    assert "Laurence Fishburne" in nouns
    # Single-word names skipped at this layer.
    assert "Neo" not in nouns
    assert "Morpheus" not in nouns


def test_span_path_classifies_when_bullet_appears_verbatim():
    """A model that doesn't quote but writes lines verbatim from source
    still earns evidence via the span path."""
    context = "Apple Inc. was founded by Steve Jobs, Steve Wozniak, and Ronald Wayne in 1976."
    answer = (
        "- Apple Inc. was founded by Steve Jobs, Steve Wozniak, and Ronald Wayne in 1976\n"
    )
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "STRICT"
    assert v["verifier_method"] == "span"
    assert v["n_verified"] >= 1


def test_matrix_entity_path_strict_under_default_proximity():
    """Default entity_policy='proximity': Matrix-style answer where the
    source has a tight cluster of cast-list entities (infobox) classifies
    STRICT. Distinguishes structural grounding from incidental mention."""
    context = (
        "starring = [[Keanu Reeves]] [[Laurence Fishburne]] "
        "[[Carrie-Anne Moss]] [[Hugo Weaving]] [[Joe Pantoliano]]. "
        "Thomas A. Anderson is the protagonist. The film features "
        "Agent Smith as the antagonist. Joe Pantoliano portrays Cypher."
    )
    answer = (
        "Based on the provided sources, the main characters are:\n\n"
        "- Neo (Thomas A. Anderson), played by Keanu Reeves\n"
        "- Morpheus, played by Laurence Fishburne\n"
        "- Trinity, played by Carrie-Anne Moss\n"
        "- Agent Smith, played by Hugo Weaving\n"
        "- Cypher, played by Joe Pantoliano\n"
    )
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "STRICT"
    assert v["verifier_method"] == "entity"
    assert v["n_verified"] >= 5


def test_tmnt_entity_path_demoted_under_default_proximity():
    """Companion to the Matrix test: an answer with only one multi-word
    entity ("Teenage Mutant Ninja Turtles") cannot satisfy proximity's
    N=3 cluster requirement → demoted to HYBRID even though the entity
    verifies. This is what separates incidental mention from structural
    grounding."""
    context = (
        "Master Splinter is framed for attempting to kill the Ultimate Daimyo. "
        "Raphael and Michelangelo find themselves pitted against each other. "
        "Donatello and Usagi defend Leonardo. Teenage Mutant Ninja Turtles "
        "appears throughout the article."
    )
    answer = (
        "Based on the provided sources, the names of the Teenage Mutant Ninja "
        "Turtles' brothers are:\n\n"
        "1. Leonardo (Leo)\n"
        "2. Raphael (Raph)\n"
        "3. Donatello (Donnie)\n"
        "4. Michelangelo (Mikey)\n\n"
        "Their master's name is Splinter."
    )
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "HYBRID"
    assert v["verifier_method"] == "entity"


def test_entity_policy_strict_promotes_when_all_match():
    """Legacy policy: entity_policy='strict' promotes to STRICT when all
    entities verify. Kept for back-compat / experimentation; overclaims."""
    context = "Keanu Reeves, Laurence Fishburne, Carrie-Anne Moss appear."
    answer = "The cast includes Keanu Reeves, Laurence Fishburne, and Carrie-Anne Moss."
    v = verify_quotes(answer, context, entity_policy="strict")
    assert v["audit_mode"] == "STRICT"
    assert v["verifier_method"] == "entity"


def test_entity_policy_drop_skips_path_entirely():
    """drop policy: never classifies via entity path. Verified entities
    in source are ignored; the answer falls through to UNGROUNDED/none."""
    context = "Keanu Reeves stars in this film."
    answer = "Keanu Reeves played the lead role."
    v = verify_quotes(answer, context, entity_policy="drop")
    assert v["audit_mode"] == "UNGROUNDED"
    assert v["verifier_method"] == "none"


def test_entity_policy_proximity_promotes_only_on_cluster():
    """proximity policy: STRICT only if N=3 verified entities cluster
    within W=300 chars in source. Tight infobox = STRICT.
    Scattered prose = HYBRID."""
    # Tight cluster: 3 names within ~50 chars.
    tight_ctx = "Cast: Keanu Reeves, Laurence Fishburne, Carrie-Anne Moss starred."
    answer = "The cast: Keanu Reeves, Laurence Fishburne, and Carrie-Anne Moss."
    v = verify_quotes(answer, tight_ctx, entity_policy="proximity")
    assert v["audit_mode"] == "STRICT"
    assert v["verifier_method"] == "entity"

    # Scattered: same 3 names across 1000+ chars of unrelated prose.
    spread_ctx = (
        "Keanu Reeves appeared briefly in scene one. "
        + "filler text " * 60
        + "Laurence Fishburne had a cameo. "
        + "more filler " * 60
        + "Carrie-Anne Moss made an appearance."
    )
    v = verify_quotes(answer, spread_ctx, entity_policy="proximity")
    assert v["audit_mode"] == "HYBRID"
    assert v["verifier_method"] == "entity"


def test_entity_path_hybrid_when_some_proper_nouns_unverified():
    """Mixed: some proper nouns verify, others are emergent.
    Default 'hybrid' policy → HYBRID either way."""
    context = "Keanu Reeves and Laurence Fishburne starred together."
    answer = "The film featured Keanu Reeves, Laurence Fishburne, and Marlon Brando."
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "HYBRID"
    assert v["verifier_method"] == "entity"
    assert "Marlon Brando" in v["unverified_quotes"]


# ----------------------------------------------- wikitext-base-v1 integration


def test_wikitext_context_strips_markup_before_substring_test():
    """Without the strip the verifier sees [[Cloud Strife]] in source and a
    bare 'Cloud Strife' in the answer's quoted span; substring fails. With
    arborist.wikitext.to_base wired into verify_quotes, the context becomes
    plain prose and the model's quote verifies. This is the case from the
    real FF7 query that prompted wiring wikitext-base-v1 into the verifier."""
    raw_wikitext_context = (
        "[[Cloud Strife]], an unsociable mercenary who claims to be a former "
        "1st Class member of [[Shinra Electric Power Company|Shinra]]'s "
        "[[SOLDIER (Final Fantasy VII)|SOLDIER]] unit;<ref>Smith 2010</ref>"
    )
    answer = (
        'The protagonist is described as: "Cloud Strife, an unsociable '
        "mercenary who claims to be a former 1st Class member of Shinra's "
        'SOLDIER unit;"'
    )
    v = verify_quotes(answer, raw_wikitext_context)
    assert v["verifier_method"] == "quote"
    assert v["n_verified"] == 1
    assert v["audit_mode"] == "STRICT"
    assert v["unverified_quotes"] == []


# ---------------------------------------------------------------------------
# Trailing-citation strip — model-appended `(Source: ...)` no longer breaks
# verbatim substring match.
# ---------------------------------------------------------------------------


def test_extract_quotes_strips_trailing_source_parenthetical():
    """The Pikachu real-corpus case: model wrote a verbatim quote and
    appended a citation. Strip the citation so substring match works."""
    answer = (
        'The model said: "Pikachu can store electricity in its cheeks. '
        '(Source: https://en.wikipedia.org/wiki/Pikachu)"'
    )
    quotes = extract_quotes(answer)
    assert quotes == ["Pikachu can store electricity in its cheeks."]


def test_extract_claim_spans_strips_trailing_source_parenthetical():
    """Span-strategy variant — sentence-level extraction also strips."""
    answer = (
        "Pikachu evolves from Pichu. (Source: https://example.com/p)\n"
        "Pikachu has yellow fur. (citing Wikipedia)"
    )
    spans = extract_claim_spans(answer)
    assert any("Pikachu evolves from Pichu" in s and "Source" not in s for s in spans)
    assert any("Pikachu has yellow fur" in s and "citing" not in s for s in spans)


def test_extract_claim_spans_strips_url_only_parenthetical():
    """A bare URL parenthetical (no `Source:` cue word) is also a citation."""
    answer = "The release date was 1996 (https://en.wikipedia.org/wiki/Pikachu)"
    spans = extract_claim_spans(answer)
    assert any("https" not in s for s in spans)


def test_strict_recovered_with_citation_strip():
    """End-to-end: a span with verbatim source content + appended citation
    used to flag UNGROUNDED. With the strip, it classifies STRICT."""
    context = (
        "Pikachu can store electricity in its cheeks and release it in "
        "lightning-based attacks."
    )
    answer = (
        "Pikachu can store electricity in its cheeks and release it in "
        "lightning-based attacks. (Source: https://en.wikipedia.org/wiki/Pikachu)"
    )
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "STRICT"
    assert v["unverified_quotes"] == []


# ---------------------------------------------------------------------------
# Paraphrase strategy (4th tier) — token-coverage fallback in span path.
# ---------------------------------------------------------------------------


def test_paraphrase_promotes_high_token_coverage_span():
    """Model rewrites source content but every meaningful token is in
    the corpus. Verifier accepts as paraphrase-verified."""
    # Context contains every meaningful (>=4 char) token from the answer
    # span, just in a different sequence — that's the paraphrase case.
    context = (
        "Pikachu is a Pokémon species, one of many creatures from the "
        "Pokémon franchise developed by Game Freak."
    )
    answer = (
        "Pikachu is a species of Pokémon creatures from the Pokémon "
        "franchise."
    )
    v = verify_quotes(answer, context)
    assert v["verifier_method"] == "paraphrase"
    assert v["audit_mode"] == "STRICT"
    assert v["unverified_quotes"] == []


def test_paraphrase_does_not_promote_low_token_coverage_span():
    """A span whose content is mostly NOT in the corpus stays unverified."""
    context = "Pikachu is a Pokémon species."
    answer = "Pikachu was elected mayor of Tokyo in 1988 by aristocrats."
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "UNGROUNDED"


def test_paraphrase_method_label_set_when_any_paraphrase_used():
    """If even one span verified via paraphrase (and others via substring),
    verifier_method flips to 'paraphrase' so an auditor knows soft signals
    were involved."""
    # Answer line 1 is verbatim. Line 2 reorders the same content tokens
    # — paraphrase. Both classify as verified.
    context = (
        "Apple Inc. was founded by Steve Jobs and Steve Wozniak. "
        "Apple has its main office in Cupertino, California, from where "
        "the company manages worldwide operations."
    )
    answer = (
        "Apple Inc. was founded by Steve Jobs and Steve Wozniak.\n"
        "The company manages its worldwide operations from Cupertino."
    )
    v = verify_quotes(answer, context)
    assert v["verifier_method"] == "paraphrase"
    assert v["audit_mode"] == "STRICT"


def test_paraphrase_stopword_filter_does_not_inflate_coverage():
    """Stopwords (`from`, `with`, `which`, etc.) match almost any English
    text. Excluding them from token-coverage tightens the signal: a span
    where the topical content is missing scores LOWER, not higher.

    The fox 2026-04-29 Batman case: span 'Batman is the alias of Bruce
    Wayne, a wealthy businessman who resides in Gotham City' has
    'wealthy/businessman/resides' missing from the corpus. With a clean
    stopword filter, those topical misses dominate the score and the
    span correctly stays UNGROUNDED rather than scraping over a lowered
    threshold."""
    context = (
        "Batman is the alias of Bruce Wayne. Batman lives in Gotham City."
    )
    answer_with_extra_stopwords = (
        "Batman, who is the alias of Bruce Wayne, lives in Gotham City."
    )
    # Topical tokens (batman, alias, bruce, wayne, lives, gotham, city)
    # all present in source. Stopwords ('which', 'who', etc.) are
    # filtered. Coverage on filtered set = 1.0 → paraphrase verifies.
    v = verify_quotes(answer_with_extra_stopwords, context)
    assert v["audit_mode"] == "STRICT"


def test_paraphrase_rejects_when_topical_tokens_missing():
    """Confirms that lowering the bar wouldn't be a quick win — Q1's
    Batman case (missing 'wealthy/businessman/resides') stays
    UNGROUNDED because the topical content isn't in source. Stopword
    filter doesn't rescue it."""
    context = "Batman is Bruce Wayne. Batman fights crime in Gotham."
    answer = (
        "Batman is the alias of Bruce Wayne, a wealthy businessman who "
        "resides in Gotham City."
    )
    # 'wealthy', 'businessman', 'resides', 'alias' all absent from
    # context. Topical content missing — coverage drops below 0.85.
    v = verify_quotes(answer, context)
    # Either UNGROUNDED (substring fail) or HYBRID (some entities) —
    # the key constraint is NOT STRICT (would mean fabrication promoted).
    assert v["audit_mode"] != "STRICT"


def test_paraphrase_strategy_keeps_quote_strategy_strict():
    """Quote strategy stays verbatim-only — quotes ARE quotes, paraphrase-
    in-quotes is the model's mistake. Paraphrase fallback applies to span
    strategy only."""
    context = "The actual sentence is something specific about cats."
    answer = '"This is a totally different sentence about dogs and rats"'
    v = verify_quotes(answer, context)
    assert v["verifier_method"] == "quote"
    assert v["audit_mode"] == "UNGROUNDED"


def test_wikitext_strip_does_not_rescue_genuine_hallucination():
    """A quote the model invented stays unverified even after stripping."""
    raw_wikitext_context = (
        "[[Cloud Strife]] is the [[protagonist]] of [[Final Fantasy VII]]."
    )
    answer = (
        'The story is: "Set in a dystopian world, Final Fantasy VII\'s story '
        'centers on mercenary Cloud Strife..."'
    )
    v = verify_quotes(answer, raw_wikitext_context)
    assert v["verifier_method"] == "quote"
    assert v["n_verified"] == 0
    assert v["audit_mode"] == "UNGROUNDED"
    assert len(v["unverified_quotes"]) == 1