arborist/tests/test_missed_answer_guard.py

"""Tests for the Ticket #000068 Phase 1 missed-answer falsification guard.

The guard is a deterministic read-only sidecar that fires on the three-clause
conjunction: denial pattern + extraction-shaped question + candidate spans
near cleaned subject tokens. These tests pin each clause in isolation, the
positive Ballestrini regression case, the negative John-Smith control, and
the discipline invariants (no model call, no audit write, no claim promotion).
"""
from __future__ import annotations

from dataclasses import dataclass

import pytest

from arborist.qa.inspect import (
    _DENIAL_PATTERNS_V1,
    _extract_subject_tokens,
    _match_denial_pattern,
    _classify_extraction_shape,
    _extract_candidate_spans,
    _score_answerability_candidates,
    diagnose_missed_answer,
    ANSWERABILITY_DIAGNOSTIC_VERSION,
    DENIAL_PATTERNS_VERSION,
)


# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------


@dataclass
class FakeEv:
    pointer_id: str
    evidence_id: str
    title: str
    document_uri: str
    span: str


def _ballestrini_evidence() -> list[FakeEv]:
    return [
        FakeEv(
            pointer_id="E1",
            evidence_id="E652b125b",
            title="Veronica Ballestrini",
            document_uri="https://en.wikipedia.org/wiki/Veronica_Ballestrini",
            span=(
                "Veronica Jean Ballestrini (born October 29, 1991) is an "
                "Italian-American country music singer and songwriter from "
                "Waterford, Connecticut."
            ),
        ),
        FakeEv(
            pointer_id="E2",
            evidence_id="E19912984",
            title="Veronica Ballestrini",
            document_uri="https://en.wikipedia.org/wiki/Veronica_Ballestrini",
            span=(
                "Ballestrini went on to record her debut album "
                "\"What I'm All About\" which was released August 21, 2009. "
                "You can buy her CD and hear amazing 11 songs and 6 were "
                "written by her. Her first single \"Amazing\" charted on "
                "the Music Row Country chart and the music video debuted at "
                "#3 on CMT Pure. In January 2010 Timbob records partnered "
                "with Lofton Creek Records president Mike Borchetta for the "
                "promotions of Veronica's single \"Out There Somewhere\"."
            ),
        ),
    ]


# ---------------------------------------------------------------------------
# Clause A — denial pattern
# ---------------------------------------------------------------------------


def test_denial_pattern_positive():
    assert _match_denial_pattern("the specific songs are not mentioned in evidence") == "not mentioned"


def test_denial_pattern_negative():
    assert _match_denial_pattern("Veronica has released several songs including Amazing.") is None


def test_denial_pattern_casefold():
    assert _match_denial_pattern("THIS INFORMATION IS NOT PROVIDED IN THE TEXT") == "not provided"


def test_denial_pattern_whitespace_normalized():
    assert _match_denial_pattern("specific  songs  are\n not  mentioned\nhere") == "not mentioned"


def test_denial_pattern_sealed_list_intact():
    """Adding a new phrase requires bumping DENIAL_PATTERNS_VERSION.
    This test pins the v1 set so silent additions can't slip through."""
    expected = {
        "not mentioned",
        "not provided",
        "the evidence does not say",
        "does not mention",
        "no specific",
        "no evidence",
        "cannot determine from the provided evidence",
        "is not stated",
        "is not specified",
    }
    assert set(_DENIAL_PATTERNS_V1) == expected


# ---------------------------------------------------------------------------
# Clause B — extraction shape
# ---------------------------------------------------------------------------


def test_extraction_cue_songs_by():
    info = _classify_extraction_shape("songs by veronica ballestrini", None)
    assert info is not None
    assert info["cue"] == "songs by"
    assert info["answer_type"] == "title_like"
    assert info["shape"] == "list"


def test_extraction_cue_who_wrote():
    info = _classify_extraction_shape("who wrote ulysses?", None)
    assert info is not None
    assert info["cue"] == "who wrote"
    assert info["answer_type"] == "person"


def test_extraction_cue_what_year():
    info = _classify_extraction_shape("what year did the war end?", None)
    assert info is not None
    assert info["cue"] == "what year"
    assert info["answer_type"] == "date"


def test_extraction_shape_quantifier_fallback():
    """No surface cue but broad quantifier intensity → still extraction shape."""
    info = _classify_extraction_shape("tell me about beatles members", "OPEN_REQUEST")
    assert info is not None
    assert info["cue"].startswith("quantifier:")
    assert info["answer_type"] == "title_like"


def test_extraction_shape_narrow_question():
    """Narrow factoid: no cue, no broad quantifier → not extraction-shaped."""
    info = _classify_extraction_shape("who is veronica ballestrini?", "SINGULAR")
    assert info is None


# ---------------------------------------------------------------------------
# Subject-token extraction (Dav1d §7 — non-negotiable)
# ---------------------------------------------------------------------------


def test_subject_tokens_strip_cue_words():
    """`songs by veronica ballestrini` → just the proper-noun run.
    Without this hardening the guard false-triggers on generic spans."""
    tokens = _extract_subject_tokens("songs by veronica ballestrini")
    low = [t.lower() for t in tokens]
    assert "songs" not in low
    assert "by" not in low
    assert "veronica" in low
    assert "ballestrini" in low


def test_subject_tokens_strip_who_wrote():
    tokens = _extract_subject_tokens("who wrote ulysses?")
    low = [t.lower() for t in tokens]
    assert "who" not in low
    assert "wrote" not in low
    assert "ulysses" in low


def test_subject_tokens_preserve_proper_noun_runs():
    """Multi-word proper nouns survive as separate tokens (matched by either)."""
    tokens = _extract_subject_tokens("books by f scott fitzgerald")
    low = [t.lower() for t in tokens]
    assert "books" not in low
    assert "by" not in low
    assert "fitzgerald" in low
    # f and scott are short — `f` drops (single letter), `scott` survives
    assert "scott" in low


def test_subject_tokens_preserve_hyphenated():
    tokens = _extract_subject_tokens("songs by jean-luc picard")
    low = [t.lower() for t in tokens]
    assert "jean-luc" in low or "jean" in low  # implementation may split or preserve
    assert "picard" in low


def test_subject_tokens_dedupe():
    tokens = _extract_subject_tokens("ballestrini ballestrini songs")
    low = [t.lower() for t in tokens]
    assert low.count("ballestrini") == 1


# ---------------------------------------------------------------------------
# Candidate span extraction
# ---------------------------------------------------------------------------


def test_candidate_quoted_string():
    text = "She wrote her debut single \"Amazing\" in 2009."
    spans = _extract_candidate_spans(text, "title_like")
    quoted = [s for s in spans if s["kind"] == "quoted_string"]
    assert any(s["text"] == "Amazing" for s in quoted)


def test_candidate_title_case_span():
    text = "Veronica Jean Ballestrini was born in Connecticut."
    spans = _extract_candidate_spans(text, "title_like")
    title_case = [s for s in spans if s["kind"] == "title_case_span"]
    assert any("Veronica Jean Ballestrini" in s["text"] for s in title_case)


def test_candidate_year_for_date_query():
    text = "She was born in 1991 in Connecticut."
    spans = _extract_candidate_spans(text, "date")
    years = [s for s in spans if s["kind"] == "year"]
    assert any(s["text"] == "1991" for s in years)


def test_candidate_no_year_for_title_query():
    """Year extraction only happens for date queries."""
    text = "She was born in 1991."
    spans = _extract_candidate_spans(text, "title_like")
    assert not any(s["kind"] == "year" for s in spans)


# ---------------------------------------------------------------------------
# Full guard — positive case
# ---------------------------------------------------------------------------


def test_ballestrini_tail_failure_triggers():
    """The motivating regression: Hermes-3-8B under tail layout said
    'specific songs are not mentioned' when E2 contained 'Amazing',
    'Out There Somewhere', etc. Guard must catch this."""
    result = diagnose_missed_answer(
        question="songs by veronica ballestrini",
        answer=(
            "Veronica Ballestrini is a country music singer and songwriter "
            "who has released several songs. However, the specific songs by "
            "her are not mentioned in the provided evidence blocks."
        ),
        evidence=_ballestrini_evidence(),
    )
    assert result is not None
    assert result["answerability_warning"] is True
    assert result["confidence_class"] in ("medium", "strong")
    assert result["denial_pattern_matched"] == "not mentioned"
    assert result["extraction_cue_matched"] == "songs by"
    assert "ballestrini" in [t.lower() for t in result["subject_tokens"]]
    assert "songs" not in [t.lower() for t in result["subject_tokens"]]
    # At least one quoted-string candidate (Amazing / What I'm All About / Out There Somewhere)
    kinds = {c["candidate_kind"] for c in result["missed_answer_candidate_spans"]}
    assert "quoted_string" in kinds


def test_ballestrini_diagnostic_version_pinned():
    result = diagnose_missed_answer(
        question="songs by veronica ballestrini",
        answer="The evidence does not say what songs she released.",
        evidence=_ballestrini_evidence(),
    )
    assert result["diagnostic_version"] == ANSWERABILITY_DIAGNOSTIC_VERSION
    assert result["denial_patterns_version"] == DENIAL_PATTERNS_VERSION


def test_offsets_are_start_end_basis():
    """Dav1d §10: offsets must be precise — start + end + basis, not ambiguous."""
    result = diagnose_missed_answer(
        question="songs by veronica ballestrini",
        answer="Songs are not mentioned in the evidence.",
        evidence=_ballestrini_evidence(),
    )
    for c in result["missed_answer_candidate_spans"]:
        assert "offset_start" in c
        assert "offset_end" in c
        assert c["offset_basis"] == "evidence_object_text"
        assert c["offset_end"] > c["offset_start"]


def test_candidate_cap_at_10():
    """Phase 1 must cap output at 10 candidates (the per_chunk lesson)."""
    long_evidence = FakeEv(
        pointer_id="E1",
        evidence_id="EBIG",
        title="Veronica Ballestrini",
        document_uri="https://x",
        # 12 quoted titles
        span=" ".join(f'Her single "Song{i}" charted.' for i in range(12))
              + " Ballestrini wrote them all.",
    )
    result = diagnose_missed_answer(
        question="songs by veronica ballestrini",
        answer="The evidence does not mention specific songs.",
        evidence=[long_evidence],
    )
    assert result is not None
    assert len(result["missed_answer_candidate_spans"]) <= 10


# ---------------------------------------------------------------------------
# Negative controls (Dav1d §14)
# ---------------------------------------------------------------------------


def test_genuine_absence_no_strong_trigger():
    """Dav1d §14 negative control: 'songs by John Smith' + evidence
    about Harvard/NY → must NOT strong-trigger."""
    ev = FakeEv(
        pointer_id="E1",
        evidence_id="EJS",
        title="John Smith",
        document_uri="https://x",
        span="John Smith studied at Harvard University and lived in New York.",
    )
    result = diagnose_missed_answer(
        question="songs by john smith",
        answer="The evidence does not mention songs by John Smith.",
        evidence=[ev],
    )
    # Per Dav1d: 'no strong warning; ideally no warning; if warning
    # exists, confidence_class = weak.' We accept weak or None.
    if result is not None:
        assert result["confidence_class"] != "strong", (
            "false positive at strong confidence — Harvard/NY are not song titles"
        )


def test_no_denial_does_not_trigger():
    """Clause A failure: extraction question + candidates, but no denial."""
    result = diagnose_missed_answer(
        question="songs by veronica ballestrini",
        answer="Veronica released Amazing and Out There Somewhere.",
        evidence=_ballestrini_evidence(),
    )
    assert result is None


def test_not_extraction_shape_does_not_trigger():
    """Clause B failure: SINGULAR question + denial + candidates."""
    result = diagnose_missed_answer(
        question="who is veronica ballestrini?",
        answer="The evidence does not say who she is.",
        evidence=_ballestrini_evidence(),
    )
    assert result is None


def test_no_evidence_does_not_trigger():
    """Clause C failure: empty evidence list."""
    result = diagnose_missed_answer(
        question="songs by veronica ballestrini",
        answer="Not mentioned.",
        evidence=[],
    )
    assert result is None


def test_no_candidate_spans_does_not_trigger():
    """Clause C failure: evidence exists but contains no candidate spans
    matching the answer_type (plain prose with no titles)."""
    ev = FakeEv(
        pointer_id="E1",
        evidence_id="X",
        title="V B",
        document_uri="https://x",
        span="she is a singer-songwriter from connecticut.",  # lowercase — no title-case
    )
    result = diagnose_missed_answer(
        question="songs by veronica ballestrini",
        answer="Not mentioned.",
        evidence=[ev],
    )
    assert result is None


# ---------------------------------------------------------------------------
# Discipline invariants — no model, no audit, no promotion
# ---------------------------------------------------------------------------


def test_sidecar_disabled_returns_none():
    """answerability_sidecar_enabled=False short-circuits to None."""
    result = diagnose_missed_answer(
        question="songs by veronica ballestrini",
        answer="not mentioned",
        evidence=_ballestrini_evidence(),
        policy={"answerability_sidecar_enabled": False},
    )
    assert result is None


def test_evidence_dict_form_supported():
    """Phase 1 accepts both EvidenceObject and dict-shaped evidence."""
    dict_ev = [
        {
            "pointer_id": "E2",
            "evidence_id": "EX",
            "title": "Veronica Ballestrini",
            "document_uri": "https://x",
            "span": 'Her first single "Amazing" by Veronica Ballestrini.',
        },
    ]
    result = diagnose_missed_answer(
        question="songs by veronica ballestrini",
        answer="The evidence does not mention specific songs.",
        evidence=dict_ev,
    )
    assert result is not None
    assert result["answerability_warning"] is True


def test_deterministic_byte_for_byte():
    """Same inputs → same output. Byte-deterministic per Dav1d §13."""
    inputs = dict(
        question="songs by veronica ballestrini",
        answer="not mentioned",
        evidence=_ballestrini_evidence(),
    )
    r1 = diagnose_missed_answer(**inputs)
    r2 = diagnose_missed_answer(**inputs)
    assert r1 == r2


def test_empty_question_returns_none():
    result = diagnose_missed_answer(
        question="",
        answer="not mentioned",
        evidence=_ballestrini_evidence(),
    )
    assert result is None


def test_empty_answer_returns_none():
    result = diagnose_missed_answer(
        question="songs by veronica ballestrini",
        answer="",
        evidence=_ballestrini_evidence(),
    )
    assert result is None


# ---------------------------------------------------------------------------
# Output schema integrity
# ---------------------------------------------------------------------------


def test_output_schema_keys():
    result = diagnose_missed_answer(
        question="songs by veronica ballestrini",
        answer="The specific songs are not mentioned in evidence.",
        evidence=_ballestrini_evidence(),
    )
    required_keys = {
        "diagnostic_version",
        "denial_patterns_version",
        "extraction_cues_version",
        "answerability_warning",
        "confidence_class",
        "triggered_clauses",
        "denial_pattern_matched",
        "extraction_cue_matched",
        "extraction_shape",
        "answer_type",
        "subject_tokens",
        "candidate_count",
        "threshold",
        "threshold_report",
        "missed_answer_candidate_spans",
    }
    assert required_keys.issubset(result.keys()), (
        f"missing keys: {required_keys - set(result.keys())}"
    )


def test_triggered_clauses_all_true_when_firing():
    result = diagnose_missed_answer(
        question="songs by veronica ballestrini",
        answer="not mentioned",
        evidence=_ballestrini_evidence(),
    )
    assert result["triggered_clauses"]["denial"] is True
    assert result["triggered_clauses"]["extraction_shape"] is True
    assert result["triggered_clauses"]["candidate_proximity"] is True


def test_confidence_class_in_valid_set():
    result = diagnose_missed_answer(
        question="songs by veronica ballestrini",
        answer="not mentioned",
        evidence=_ballestrini_evidence(),
    )
    assert result["confidence_class"] in ("weak", "medium", "strong")


# ---------------------------------------------------------------------------
# Phase 3 — opt-in demote flag + render-label projection + hash discipline
# ---------------------------------------------------------------------------


def test_demote_flag_in_verifier_policy_fields():
    """Phase 3: answerability_demote_enabled must be in the verifier
    set so flipping it changes verifier_policy_hash."""
    from arborist.qa.keys import _VERIFIER_POLICY_FIELDS
    assert "answerability_demote_enabled" in _VERIFIER_POLICY_FIELDS


def test_demote_flag_changes_verifier_hash():
    """Flipping the demote flag must partition verifier_policy_hash."""
    from arborist.qa.keys import verifier_policy_hash
    base = {"answerability_demote_enabled": False}
    flipped = {"answerability_demote_enabled": True}
    assert verifier_policy_hash(base) != verifier_policy_hash(flipped)


def test_sidecar_fields_do_NOT_change_verifier_hash():
    """The other answerability fields (governance only) must NOT
    change verifier_policy_hash."""
    from arborist.qa.keys import verifier_policy_hash
    a = {"answerability_demote_enabled": False, "denial_patterns_version": "v1"}
    b = {"answerability_demote_enabled": False, "denial_patterns_version": "v2"}
    assert verifier_policy_hash(a) == verifier_policy_hash(b)


def test_sidecar_fields_DO_change_governance_hash():
    """The sidecar enable / threshold fields fold into governance."""
    from arborist.qa.keys import governance_policy_hash
    a = {"answerability_sidecar_enabled": True}
    b = {"answerability_sidecar_enabled": False}
    assert governance_policy_hash(a) != governance_policy_hash(b)


def test_render_label_demote_strong_confidence():
    """When demote is enabled AND sidecar fired strong, label demotes."""
    from arborist.cli import _render_audit_label
    answerability = {
        "answerability_warning": True,
        "confidence_class": "strong",
    }
    label = _render_audit_label(
        "STRICT", "claim_lattice",
        violations=None,
        answerability=answerability,
        demote_enabled=True,
    )
    assert "EVIDENCE-MISSED-PARTIAL" in label


def test_render_label_demote_medium_confidence():
    """Medium confidence also demotes (per Phase 2 calibration)."""
    from arborist.cli import _render_audit_label
    answerability = {
        "answerability_warning": True,
        "confidence_class": "medium",
    }
    label = _render_audit_label(
        "STRICT", "claim_lattice",
        violations=None,
        answerability=answerability,
        demote_enabled=True,
    )
    assert "EVIDENCE-MISSED-PARTIAL" in label


def test_render_label_weak_does_NOT_demote():
    """Weak confidence never demotes the rung (Phase 2 saw zero weak
    fires on real failures; reserved for future expanded detection)."""
    from arborist.cli import _render_audit_label
    answerability = {
        "answerability_warning": True,
        "confidence_class": "weak",
    }
    label = _render_audit_label(
        "STRICT", "claim_lattice",
        violations=None,
        answerability=answerability,
        demote_enabled=True,
    )
    assert "EVIDENCE-MISSED-PARTIAL" not in label
    assert "missed-answer" not in label


def test_render_label_demote_off_keeps_original():
    """When demote flag is False, label stays as-is even on strong fire."""
    from arborist.cli import _render_audit_label
    answerability = {
        "answerability_warning": True,
        "confidence_class": "strong",
    }
    label = _render_audit_label(
        "STRICT", "claim_lattice",
        violations=None,
        answerability=answerability,
        demote_enabled=False,
    )
    assert "EVIDENCE-MISSED-PARTIAL" not in label
    assert "missed-answer" not in label


def test_render_label_no_answerability_unchanged():
    """No sidecar fire → label unchanged regardless of flag."""
    from arborist.cli import _render_audit_label
    label_off = _render_audit_label(
        "STRICT", "claim_lattice", violations=None,
        answerability=None, demote_enabled=False,
    )
    label_on = _render_audit_label(
        "STRICT", "claim_lattice", violations=None,
        answerability=None, demote_enabled=True,
    )
    assert label_off == label_on


def test_render_label_quote_mode_demote_appends_tail():
    """Non-lattice mode: demote appends '· missed-answer' tail rather
    than transitioning to MISSED-PARTIAL."""
    from arborist.cli import _render_audit_label
    answerability = {
        "answerability_warning": True,
        "confidence_class": "strong",
    }
    label = _render_audit_label(
        "STRICT", "quote",
        violations=None,
        answerability=answerability,
        demote_enabled=True,
    )
    assert "missed-answer" in label


def test_render_label_lower_rung_demote_appends_tail():
    """Lattice mode on a non-EVIDENCE-WARRANTED rung (e.g. ANCHOR-WARRANTED
    or POINTER-LINKED) gets a '· missed-answer' tail rather than a rung
    transition — the rung itself already signals degradation."""
    from arborist.cli import _render_audit_label
    answerability = {
        "answerability_warning": True,
        "confidence_class": "strong",
    }
    # HYBRID + WARRANT_MISSING → POINTER-LINKED rung
    violations = [{"kind": "WARRANT_MISSING"}]
    label = _render_audit_label(
        "HYBRID", "claim_lattice",
        violations=violations,
        answerability=answerability,
        demote_enabled=True,
    )
    assert "missed-answer" in label
    assert "EVIDENCE-MISSED-PARTIAL" not in label