arborist/tests/test_verify.py

"""Faithfulness verifier: classify answer grounding into v9.8 audit_mode.

Three layered strategies (quote → span → entity), tried in order. The
first that finds evidence classifies the answer. `verifier_method`
records which path fired.

Tests cover the full trichotomy under each path plus regressions:
    - TMNT: model wrote bios not in context — must be HYBRID/UNGROUNDED
    - Matrix: Wikipedia infobox + paraphrased prose. Spans don't match,
      but every multi-word proper noun does — entity path classifies it.
"""

from __future__ import annotations

from aborist.qa.verify import (
    extract_claim_spans,
    extract_proper_nouns,
    extract_quotes,
    verify_quotes,
)


# ---------------------------------------------------------------- extract


def test_extract_quotes_finds_double_quoted_spans():
    text = 'He said "the cat is on the mat" and then "rain in spain falls".'
    assert extract_quotes(text) == [
        "the cat is on the mat",
        "rain in spain falls",
    ]


def test_extract_quotes_skips_short_spans():
    # 8-char minimum keeps "is a", "the", "ok" out.
    text = '"is a" "ok" "long enough span" "no"'
    assert extract_quotes(text) == ["long enough span"]


def test_extract_quotes_sequential_pairs_skip_inter_pair_prose():
    """Fox caught on 2026-04-28: model wrote `"Hackers" is as follows: "In 1988..."`
    and naive regex matching paired the closer of "Hackers" with the
    opener of "In 1988", capturing the framing prose between them as a
    phantom quote. Sequential pairing (1st & 2nd, 3rd & 4th, ...) treats
    inter-pair text as the model's framing — never as a quoted span."""
    text = (
        'The plot of the 1995 film "Hackers" is as follows, directly '
        'quoted from the source:\n\n"In 1988, Seattle youth Dade Murphy."'
    )
    quotes = extract_quotes(text)
    # "Hackers" is 7 chars — below MIN_QUOTE_CHARS, dropped.
    # "In 1988, Seattle youth Dade Murphy." pairs cleanly.
    # The phantom prose between them is never captured.
    assert quotes == ["In 1988, Seattle youth Dade Murphy."]


def test_extract_quotes_handles_three_adjacent_pairs():
    """`"A_____" stuff "B_____" more "C_____"` → three clean pairs."""
    text = '"alpha-99" stuff "bravo-99" more "charlie9"'
    quotes = extract_quotes(text)
    assert quotes == ["alpha-99", "bravo-99", "charlie9"]


def test_extract_quotes_handles_curly_quotes():
    text = '“smart quoted span here” and "ascii quoted span"'
    quotes = extract_quotes(text)
    assert "smart quoted span here" in quotes
    assert "ascii quoted span" in quotes


# ---------------------------------------------------------------- classify


def test_strict_when_all_quotes_verify():
    context = "Capitalism is an economic system based on private ownership."
    answer = 'The source defines it: "an economic system based on private ownership"'
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "STRICT"
    assert v["verifier_method"] == "quote"
    assert v["n_quotes"] == 1
    assert v["n_verified"] == 1
    assert v["unverified_quotes"] == []


def test_visual_when_no_quotes_no_spans_no_entities():
    """Truly emergent: no double quotes, no span match, no entity match."""
    context = "Apples are red."
    answer = "freedom rests on autonomy alone, without coercion."
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "UNGROUNDED"
    assert v["verifier_method"] == "none"
    assert v["n_quotes"] == 0


def test_visual_when_no_quote_verifies():
    context = "Capitalism is an economic system based on private ownership."
    answer = '"this exact span is not in the source at all"'
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "UNGROUNDED"
    assert v["verifier_method"] == "quote"
    assert v["n_quotes"] == 1
    assert v["n_verified"] == 0
    assert v["unverified_quotes"] == ["this exact span is not in the source at all"]


def test_hybrid_when_some_quotes_verify_some_dont():
    context = "Capitalism is an economic system based on private ownership."
    answer = (
        'The source says "an economic system" but then claims '
        '"Marx personally invented capitalism in 1867" which is a stretch.'
    )
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "HYBRID"
    assert v["verifier_method"] == "quote"
    assert v["n_quotes"] == 2
    assert v["n_verified"] == 1
    assert v["unverified_quotes"] == [
        "Marx personally invented capitalism in 1867"
    ]


# ---------------------------------------------------------------- normalization


def test_case_and_whitespace_insensitive_match():
    context = "The Eight Forms of Capital include living, social, and intellectual."
    answer = '"the   eight forms of capital"'  # collapsed ws + lowercase + multi-space
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "STRICT"
    assert v["n_verified"] == 1


# ---------------------------------------------------------------- regression


def test_tmnt_regression_must_not_be_strict():
    """Fox's catch on 2026-04-28: model emerged TMNT bios from training
    while only the names appeared in context (an episode plot summary).
    The state machine must downgrade — STRICT here would be a lie."""
    context = (
        "Master Splinter is framed for attempting to kill the Ultimate Daimyo. "
        "Raphael and Michelangelo find themselves pitted against each other in "
        "the tournament. Donatello and Usagi must defend Leonardo from assassins."
    )
    answer = (
        '1. Leonardo - "leader of the group, he wields a blue katana"\n'
        '2. Splinter - "a mutated rat who was once Hamato Yoshi"\n'
        '3. Plot - "Raphael and Michelangelo find themselves pitted against each other"'
    )
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "HYBRID"
    assert v["verifier_method"] == "quote"
    assert v["n_verified"] == 1
    assert len(v["unverified_quotes"]) == 2
    assert any("blue katana" in q for q in v["unverified_quotes"])
    assert any("Hamato Yoshi" in q for q in v["unverified_quotes"])


# ---------------------------------------------------------------- spans / entities


def test_extract_spans_strips_bullets_and_framing():
    answer = (
        "Based on the provided sources, the main characters are:\n\n"
        "- Neo\n"
        "- Trinity\n"
        "1. Morpheus is a leader\n"
    )
    spans = extract_claim_spans(answer)
    # Framing line dropped; bullet markers stripped; short single-word
    # bullets ("Neo", "Trinity") below MIN_SPAN_CHARS dropped.
    assert "Morpheus is a leader" in spans
    assert not any(s.startswith("Based on the") for s in spans)


def test_extract_proper_nouns_picks_multi_word_phrases():
    text = (
        "Neo (Thomas A. Anderson), played by Keanu Reeves. "
        "Morpheus, played by Laurence Fishburne."
    )
    nouns = extract_proper_nouns(text)
    assert "Thomas A. Anderson" in nouns
    assert "Keanu Reeves" in nouns
    assert "Laurence Fishburne" in nouns
    # Single-word names skipped at this layer.
    assert "Neo" not in nouns
    assert "Morpheus" not in nouns


def test_span_path_classifies_when_bullet_appears_verbatim():
    """A model that doesn't quote but writes lines verbatim from source
    still earns evidence via the span path."""
    context = "Apple Inc. was founded by Steve Jobs, Steve Wozniak, and Ronald Wayne in 1976."
    answer = (
        "- Apple Inc. was founded by Steve Jobs, Steve Wozniak, and Ronald Wayne in 1976\n"
    )
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "STRICT"
    assert v["verifier_method"] == "span"
    assert v["n_verified"] >= 1


def test_matrix_entity_path_strict_under_default_proximity():
    """Default entity_policy='proximity': Matrix-style answer where the
    source has a tight cluster of cast-list entities (infobox) classifies
    STRICT. Distinguishes structural grounding from incidental mention."""
    context = (
        "starring = [[Keanu Reeves]] [[Laurence Fishburne]] "
        "[[Carrie-Anne Moss]] [[Hugo Weaving]] [[Joe Pantoliano]]. "
        "Thomas A. Anderson is the protagonist. The film features "
        "Agent Smith as the antagonist. Joe Pantoliano portrays Cypher."
    )
    answer = (
        "Based on the provided sources, the main characters are:\n\n"
        "- Neo (Thomas A. Anderson), played by Keanu Reeves\n"
        "- Morpheus, played by Laurence Fishburne\n"
        "- Trinity, played by Carrie-Anne Moss\n"
        "- Agent Smith, played by Hugo Weaving\n"
        "- Cypher, played by Joe Pantoliano\n"
    )
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "STRICT"
    assert v["verifier_method"] == "entity"
    assert v["n_verified"] >= 5


def test_tmnt_entity_path_demoted_under_default_proximity():
    """Companion to the Matrix test: an answer with only one multi-word
    entity ("Teenage Mutant Ninja Turtles") cannot satisfy proximity's
    N=3 cluster requirement → demoted to HYBRID even though the entity
    verifies. This is what separates incidental mention from structural
    grounding."""
    context = (
        "Master Splinter is framed for attempting to kill the Ultimate Daimyo. "
        "Raphael and Michelangelo find themselves pitted against each other. "
        "Donatello and Usagi defend Leonardo. Teenage Mutant Ninja Turtles "
        "appears throughout the article."
    )
    answer = (
        "Based on the provided sources, the names of the Teenage Mutant Ninja "
        "Turtles' brothers are:\n\n"
        "1. Leonardo (Leo)\n"
        "2. Raphael (Raph)\n"
        "3. Donatello (Donnie)\n"
        "4. Michelangelo (Mikey)\n\n"
        "Their master's name is Splinter."
    )
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "HYBRID"
    assert v["verifier_method"] == "entity"


def test_entity_policy_strict_promotes_when_all_match():
    """Legacy policy: entity_policy='strict' promotes to STRICT when all
    entities verify. Kept for back-compat / experimentation; overclaims."""
    context = "Keanu Reeves, Laurence Fishburne, Carrie-Anne Moss appear."
    answer = "The cast includes Keanu Reeves, Laurence Fishburne, and Carrie-Anne Moss."
    v = verify_quotes(answer, context, entity_policy="strict")
    assert v["audit_mode"] == "STRICT"
    assert v["verifier_method"] == "entity"


def test_entity_policy_drop_skips_path_entirely():
    """drop policy: never classifies via entity path. Verified entities
    in source are ignored; the answer falls through to UNGROUNDED/none."""
    context = "Keanu Reeves stars in this film."
    answer = "Keanu Reeves played the lead role."
    v = verify_quotes(answer, context, entity_policy="drop")
    assert v["audit_mode"] == "UNGROUNDED"
    assert v["verifier_method"] == "none"


def test_entity_policy_proximity_promotes_only_on_cluster():
    """proximity policy: STRICT only if N=3 verified entities cluster
    within W=300 chars in source. Tight infobox = STRICT.
    Scattered prose = HYBRID."""
    # Tight cluster: 3 names within ~50 chars.
    tight_ctx = "Cast: Keanu Reeves, Laurence Fishburne, Carrie-Anne Moss starred."
    answer = "The cast: Keanu Reeves, Laurence Fishburne, and Carrie-Anne Moss."
    v = verify_quotes(answer, tight_ctx, entity_policy="proximity")
    assert v["audit_mode"] == "STRICT"
    assert v["verifier_method"] == "entity"

    # Scattered: same 3 names across 1000+ chars of unrelated prose.
    spread_ctx = (
        "Keanu Reeves appeared briefly in scene one. "
        + "filler text " * 60
        + "Laurence Fishburne had a cameo. "
        + "more filler " * 60
        + "Carrie-Anne Moss made an appearance."
    )
    v = verify_quotes(answer, spread_ctx, entity_policy="proximity")
    assert v["audit_mode"] == "HYBRID"
    assert v["verifier_method"] == "entity"


def test_entity_path_hybrid_when_some_proper_nouns_unverified():
    """Mixed: some proper nouns verify, others are emergent.
    Default 'hybrid' policy → HYBRID either way."""
    context = "Keanu Reeves and Laurence Fishburne starred together."
    answer = "The film featured Keanu Reeves, Laurence Fishburne, and Marlon Brando."
    v = verify_quotes(answer, context)
    assert v["audit_mode"] == "HYBRID"
    assert v["verifier_method"] == "entity"
    assert "Marlon Brando" in v["unverified_quotes"]


# ----------------------------------------------- wikitext-base-v1 integration


def test_wikitext_context_strips_markup_before_substring_test():
    """Without the strip the verifier sees [[Cloud Strife]] in source and a
    bare 'Cloud Strife' in the answer's quoted span; substring fails. With
    aborist.wikitext.to_base wired into verify_quotes, the context becomes
    plain prose and the model's quote verifies. This is the case from the
    real FF7 query that prompted wiring wikitext-base-v1 into the verifier."""
    raw_wikitext_context = (
        "[[Cloud Strife]], an unsociable mercenary who claims to be a former "
        "1st Class member of [[Shinra Electric Power Company|Shinra]]'s "
        "[[SOLDIER (Final Fantasy VII)|SOLDIER]] unit;<ref>Smith 2010</ref>"
    )
    answer = (
        'The protagonist is described as: "Cloud Strife, an unsociable '
        "mercenary who claims to be a former 1st Class member of Shinra's "
        'SOLDIER unit;"'
    )
    v = verify_quotes(answer, raw_wikitext_context)
    assert v["verifier_method"] == "quote"
    assert v["n_verified"] == 1
    assert v["audit_mode"] == "STRICT"
    assert v["unverified_quotes"] == []


def test_wikitext_strip_does_not_rescue_genuine_hallucination():
    """A quote the model invented stays unverified even after stripping."""
    raw_wikitext_context = (
        "[[Cloud Strife]] is the [[protagonist]] of [[Final Fantasy VII]]."
    )
    answer = (
        'The story is: "Set in a dystopian world, Final Fantasy VII\'s story '
        'centers on mercenary Cloud Strife..."'
    )
    v = verify_quotes(answer, raw_wikitext_context)
    assert v["verifier_method"] == "quote"
    assert v["n_verified"] == 0
    assert v["audit_mode"] == "UNGROUNDED"
    assert len(v["unverified_quotes"]) == 1