arborist/tests/test_judge_code.py

"""Tests for the deterministic code-only judge (``bench/judge_code.py``).

The self-test fixture set is the BEHAVIOURAL CONTRACT — these are
exactly the 4 cases the Opus judge (``bench/judge.py``) was hand-built
to grade correctly. A code judge that disagrees on these is not a
trustworthy instrument. CI fails the build if the contract slips.

We deliberately do NOT pin specific NLI probabilities or specific
``rules_fired`` traces — those are tuning knobs we expect to move.
The label is the public contract; the trace is implementation detail.

Runs WITHOUT the ``[nli]`` extra installed too: NLI is optional, the
abstention / lexical / specificity paths give correct labels on the
fixture set even when NLI is unavailable (case 1 then routes via
JUDGE_ERROR — see ``test_self_test_passes_with_nli_or_skips``).
"""
from __future__ import annotations

import pytest

from bench.judge_code import (
    JUDGE_MODEL,
    JUDGE_RULESET_ID,
    Verdict,
    judge,
    judge_batch,
    self_test,
    _is_abstention,
    _specifics_not_in_gold,
)


# --- module surface --------------------------------------------------------

def test_module_identifiers_pinned():
    """Audit-trail parity with bench/judge.py: stable model + prompt
    id strings so verdict logs from this judge are sortable / joinable
    against logs from the Opus judge."""
    assert JUDGE_RULESET_ID == "code-judge-v1"
    assert "code" in JUDGE_MODEL.lower()
    assert "no llm" in JUDGE_MODEL.lower()


def test_verdict_dataclass_shape():
    v = judge("Q?", "I cannot determine that.", "gold")
    assert isinstance(v, Verdict)
    # Same fields as bench.judge.Verdict so the two are drop-in compat.
    assert v.label in {"CORRECT_GROUNDED", "WRONG", "FABRICATED",
                       "ABSTAINED", "JUDGE_ERROR"}
    assert v.prompt_id == JUDGE_RULESET_ID
    assert v.model == JUDGE_MODEL
    assert isinstance(v.rationale, str) and v.rationale
    assert isinstance(v.raw, str) and v.raw  # JSON-serialised trace
    assert isinstance(v.decision, dict)


# --- rule 1: empty / no-gold guards ----------------------------------------

def test_empty_answer_is_abstained():
    v = judge("Q?", "", "gold text")
    assert v.label == "ABSTAINED"
    assert "empty_answer" in v.decision["rules_fired"]


def test_empty_gold_is_judge_error():
    v = judge("Q?", "Some claim.", "")
    assert v.label == "JUDGE_ERROR"
    assert "no_gold" in v.decision["rules_fired"]


# --- rule 2: explicit abstention -------------------------------------------

@pytest.mark.parametrize("phrase", [
    "I cannot determine that from the provided source.",
    "I do not know.",
    "I can't tell from the article.",
    "The source does not mention this.",
    "The article doesn't contain that information.",
    "Not enough information is provided.",
    "Unable to determine from the passage.",
    "Cannot be determined from the source.",
])
def test_abstention_phrases_classify_as_abstained(phrase):
    v = judge("Q?", phrase, "irrelevant gold content")
    assert v.label == "ABSTAINED", phrase
    assert "abstention_phrase" in v.decision["rules_fired"]


def test_assertive_answer_is_not_abstention():
    """A model that makes a confident claim is NOT abstaining, even if
    the claim happens to be ungrounded — that's WRONG / FABRICATED,
    not ABSTAINED."""
    is_abs, _ = _is_abstention(
        "The capital of France is Berlin and the population is 12 million."
    )
    assert is_abs is False


# --- rule 4: specificity (the FABRICATED gate) -----------------------------

def test_specifics_picked_up_when_not_in_gold():
    answer = "The Higgs boson cafe was founded by Napoleon in 1066."
    gold = "This article discusses thermodynamics and entropy."
    s = _specifics_not_in_gold(answer, gold)
    # Should flag the proper noun(s) and the date.
    joined = " ".join(s).lower()
    assert "napoleon" in joined
    assert "1066" in joined


def test_specifics_excluded_when_in_gold():
    """A proper noun / number that ALSO appears in gold is grounded
    by definition (case-insensitive) and must NOT be flagged."""
    answer = "Napoleon led the army in 1812."
    gold = "Napoleon Bonaparte's campaign in 1812 was disastrous."
    s = _specifics_not_in_gold(answer, gold)
    joined = " ".join(s).lower()
    assert "napoleon" not in joined
    assert "1812" not in joined


# --- the canonical behavioural contract: 4-case self-test ------------------

def test_self_test_passes():
    """The same 4 fixtures the Opus judge was hand-built to grade
    correctly. If this fails, the code judge is not an interchangeable
    instrument."""
    assert self_test() == 0


# --- batch parity ----------------------------------------------------------

def test_judge_batch_loops_correctly():
    triples = [
        ("Q1", "I cannot determine.", "gold a"),
        ("Q2", "", "gold b"),
    ]
    out = judge_batch(triples)
    assert [v.label for v in out] == ["ABSTAINED", "ABSTAINED"]
    assert all(isinstance(v, Verdict) for v in out)


# --- graceful NLI degradation ----------------------------------------------

def test_judge_handles_nli_unavailable(monkeypatch):
    """When the [nli] extra is missing, shadow_check returns an
    unavailable ShadowResult — the judge must NOT crash; it just
    proceeds without rule-3 (contradiction) signal and without the
    HYBRID corroboration path. Labels stay sound for the explicit
    abstention / UNGROUNDED-specifics paths (which short-circuit
    before NLI is even consulted)."""
    import bench.judge_code as jc

    monkeypatch.setattr(jc, "_nli_check", lambda a, g: None)

    # Abstention path returns at rule 2, BEFORE the NLI block runs —
    # so the trace doesn't (and shouldn't) carry nli_available. The
    # contract here is "no crash + correct label", not "NLI metadata
    # is populated on every path".
    v = jc.judge("Q?", "I cannot determine.", "gold")
    assert v.label == "ABSTAINED"
    assert "abstention_phrase" in v.decision["rules_fired"]
    assert "nli_available" not in v.decision  # rule 2 short-circuit

    # An answer that reaches the NLI block must record nli_available
    # = False when _nli_check returns None.
    v = jc.judge("Q?", "Some claim about something.", "different content here")
    assert v.decision.get("nli_available") is False

    # FABRICATED path still works (no NLI needed).
    v = jc.judge(
        "Who founded the cafe?",
        "Napoleon founded it in 1066.",
        "This article is about thermodynamics.",
    )
    assert v.label == "FABRICATED"