arborist/tests/test_relevance_shadow.py

"""Tests for the #000052 §3.2 relevance shadow scaffold.

Mirrors `tests/test_nli_shadow.py`'s discipline: pure-Python parts +
graceful degradation when the ``[nli]`` extra is absent. The real
model path is exercised opportunistically (skipped when deps missing).
"""
from __future__ import annotations

import json
from pathlib import Path

import pytest

from arborist.qa.relevance import (
    ShadowRelevance, ShadowRelevanceResult, load_manifest,
    shadow_check_question_answer, shadow_check_claim_source,
)
from arborist.qa.relevance.shadow import _resolve_relevance_index


# --- manifest --------------------------------------------------------------

def test_manifest_has_required_fields():
    m = load_manifest()
    for key in ("relevance_model_version", "hf_repo", "license", "max_length",
                "score_shape", "alternates", "demote_below_score",
                "demote_below_score_comment"):
        assert key in m, key
    # demote_below_score MUST be null until bench-set (§7 #18→#27 lesson)
    assert m["demote_below_score"] is None


def test_manifest_is_valid_json_file():
    p = Path(__file__).resolve().parents[1] / "arborist" / "qa" / "relevance" / "manifest.json"
    json.loads(p.read_text())  # raises on malformed


# --- label-index resolution ------------------------------------------------

@pytest.mark.parametrize("id2label,expect", [
    ({0: "LABEL_0"}, 0),                                         # single-output reranker (typical)
    ({0: "irrelevant", 1: "relevant"}, 1),                       # 2-class with "relevant" label
    ({0: "negative", 1: "positive"}, 1),                          # 2-class with "positive" label
    ({0: "LABEL_0", 1: "LABEL_1"}, 1),                           # fallback to "label_1" substring
    ({0: "score_0", 1: "score_1"}, 0),                           # no relevant/positive → fallback to 0
])
def test_resolve_relevance_index(id2label, expect):
    assert _resolve_relevance_index(id2label) == expect


# --- ShadowRelevance contract ----------------------------------------------

def test_shadowrelevance_construction_never_raises_and_starts_unavailable():
    r = ShadowRelevance()
    assert r.available is False
    assert r.model_version == load_manifest()["relevance_model_version"]
    assert r.demote_below_score is None      # no hardcoded threshold (§7 #18→#27 discipline)
    assert r.device_pref in ("auto", "cpu", "cuda")
    assert r.batch_size >= 1
    assert r.backend is None
    # _ensure_loaded never raises even when the extra is missing
    r._ensure_loaded()
    assert r.available in (True, False)


def test_shadow_check_question_answer_returns_result_and_degrades_gracefully():
    res = shadow_check_question_answer("who painted the mona lisa?",
                                       "Leonardo da Vinci painted the Mona Lisa.")
    assert isinstance(res, ShadowRelevanceResult)
    assert res.pair_kind == "question_answer"
    assert res.would_demote in (True, False)
    d = res.as_dict()
    assert set(d) >= {"available", "relevance_score", "would_demote",
                      "demote_below_score", "model_version", "pair_kind", "reason"}
    if not res.available:
        assert res.would_demote is False
        assert "deps_missing" in res.reason or "load_failed" in res.reason


def test_shadow_check_claim_source_returns_result_and_degrades_gracefully():
    res = shadow_check_claim_source("Leonardo da Vinci painted the Mona Lisa.",
                                    "The Mona Lisa is a half-length portrait painting by Italian artist Leonardo da Vinci.")
    assert isinstance(res, ShadowRelevanceResult)
    assert res.pair_kind == "claim_source"


def test_no_hardcoded_threshold_means_would_demote_stays_False():
    # the §7 #18→#27 discipline: demote_below_score must be null until
    # a shadow sweep sets it. The contract: would_demote=False until then.
    r = ShadowRelevance()
    r._ensure_loaded()
    if not r.available:
        pytest.skip("[nli] extra not installed — would_demote-False path only reachable when loaded")
    # any pair the model scores should still produce would_demote=False
    # because demote_below_score is None at scaffold time
    res = r.check_question_answer("capital of france?", "Paris.")
    assert res.would_demote is False
    assert res.demote_below_score is None


def test_empty_input_when_available_is_no_demote():
    r = ShadowRelevance()
    r._ensure_loaded()
    if not r.available:
        pytest.skip("[nli] extra not installed")
    assert r.check_question_answer("", "non-empty answer").reason == "empty_input"
    assert r.check_question_answer("non-empty question", "").reason == "empty_input"


def test_zionist_entity_field_case_when_available():
    """The motivating case (#000052 §1): a garbled claim about 'phrase
    used as the entity' against a 'Zionist entity' source. The
    relevance score for the (claim, source) pair should be LOWER than
    for a well-aligned (claim, source) pair — that is the signal the
    veto would act on if the threshold were set."""
    r = ShadowRelevance()
    r._ensure_loaded()
    if not r.available:
        pytest.skip("[nli] extra not installed")
    # off-topic: a claim about the abstract noun 'entity' against a source about Israel
    off = r.check_claim_source(
        "the phrase 'Zionist entity' is sometimes used as the entity, referring to the State of Israel",
        "Wikipedia article 'Phrase' — In grammar, a phrase is a group of words functioning as a single unit in the syntax of a sentence.",
    )
    on = r.check_claim_source(
        "the phrase 'Zionist entity' is sometimes used as a pejorative for the State of Israel",
        "Zionist entity () is a phrase sometimes used by Arabs and Muslims as a pejorative for the State of Israel.",
    )
    # both available
    assert off.available and on.available
    # on-topic should score strictly higher than off-topic — that's the discriminator
    assert on.relevance_score > off.relevance_score