#000049 Phase 2: NLI shadow scaffold in arborist (§7 #19)

arborist/qa/nli/ — SHADOW ONLY (never an audit_mode input; manifest not
yet in governance_policy_hash per §7 #2). manifest.json pins
cross-encoder/nli-MiniLM2-L6-H768 @ a fixed HF revision + the
bench-validated θc 0.5/θe 0.9 + 2 alternates + the Phase-3 TODO;
shadow.py = ShadowNLI/shadow_check (lazy transformers+torch behind a new
[nli] extra, clauses() segmenter, the §7 #5 clause-level Demote()
decision, degrades to available=False when [nli] absent);
bench/scripts/nli_shadow_sweep.py + make bootstrap-nli / bench-nli-shadow
(the gate-item-4 instrument); 16 tests.

First sweep (116 records — 5f-falsification packs + the arborist-nli-bench
eval sets): 28/28 synth recombination demoted, 0/26 FP on legit summaries,
0/9 fires on already-STRICT_SPAN records, 25/50 on UNGROUNDED (the
contradiction half; quiet on non-sequiturs). Gate items 1/2/3/5/6 clear
on available data; item 4 — shadow FP rate on a real live-bench-qa
sample — remains the open measurement. Production verifier unchanged;
falsification-hard stays 10/12.

tests/test_nli_shadow.py

@@ -0,0 +1,153 @@
+"""Tests for the #000049 Phase-2 NLI shadow scaffold.
+
+These run in the default suite — i.e. WITHOUT the ``[nli]`` extra
+installed — so they exercise the pure-Python parts (clause splitter,
+manifest, label-index resolution, the dataclass contract) and the
+graceful-degradation path (deps missing → ``available=False``, never an
+exception). When ``[nli]`` *is* installed they also cover the real
+model path; we don't assert specific probabilities (those belong in the
+``arborist-nli-bench`` scorecard, not the unit suite).
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from arborist.qa.nli import ShadowNLI, ShadowResult, shadow_check, load_manifest
+from arborist.qa.nli.shadow import clauses, _resolve_label_indices
+
+
+# --- clause splitter -------------------------------------------------------
+
+def test_clauses_splits_on_sentence_punctuation():
+    assert clauses("Jupiter is the largest. Mercury is the smallest.") == [
+        "Jupiter is the largest.",
+        "Mercury is the smallest.",
+    ]
+
+
+def test_clauses_handles_semicolons():
+    assert clauses("A is true; B is false") == ["A is true;", "B is false"]
+
+
+def test_clauses_single_sentence_is_one_clause():
+    assert clauses("Just one sentence with no breaks") == ["Just one sentence with no breaks"]
+
+
+def test_clauses_empty_or_blank():
+    assert clauses("") == []
+    assert clauses("   ") == []
+    assert clauses(None) == []  # type: ignore[arg-type]
+
+
+# --- manifest --------------------------------------------------------------
+
+def test_manifest_has_required_fields():
+    m = load_manifest()
+    for key in ("nli_model_version", "hf_repo", "pinned_revision", "license", "thresholds"):
+        assert key in m, key
+    assert "contradiction_veto" in m["thresholds"]
+    assert "entailment_block_veto" in m["thresholds"]
+    # shadow-mode thresholds are the bench-validated point
+    assert m["thresholds"]["contradiction_veto"] == 0.5
+    assert m["thresholds"]["entailment_block_veto"] == 0.9
+
+
+def test_manifest_is_valid_json_file():
+    p = Path(__file__).resolve().parents[1] / "arborist" / "qa" / "nli" / "manifest.json"
+    json.loads(p.read_text())  # raises on malformed
+
+
+# --- label-index resolution (the cross-encoder vs MNLI vs BART orderings) --
+
+@pytest.mark.parametrize("id2label,expect", [
+    ({0: "entailment", 1: "neutral", 2: "contradiction"}, (0, 1, 2)),   # MNLI/MoritzLaurer order
+    ({0: "contradiction", 1: "entailment", 2: "neutral"}, (1, 2, 0)),   # cross-encoder order
+    ({0: "contradiction", 1: "neutral", 2: "entailment"}, (2, 1, 0)),   # BART order
+    ({0: "ENTAILMENT", 1: "NEUTRAL", 2: "CONTRADICTION"}, (0, 1, 2)),   # case-insensitive
+])
+def test_resolve_label_indices(id2label, expect):
+    assert _resolve_label_indices(id2label) == expect
+
+
+def test_resolve_label_indices_rejects_incomplete():
+    with pytest.raises(RuntimeError):
+        _resolve_label_indices({0: "entailment", 1: "neutral"})
+
+
+# --- ShadowNLI / ShadowResult contract -------------------------------------
+
+def test_shadownli_construction_never_raises_and_starts_unavailable():
+    nli = ShadowNLI()
+    assert nli.available is False           # not loaded until first check
+    assert nli.model_version == load_manifest()["nli_model_version"]
+    assert nli.theta_contra == 0.5 and nli.theta_entail == 0.9
+
+
+def test_check_returns_shadowresult_and_degrades_gracefully():
+    res = shadow_check("Mercury is the largest planet.",
+                       "Jupiter is the largest planet. Mercury is the smallest planet.")
+    assert isinstance(res, ShadowResult)
+    assert res.would_demote in (True, False)
+    assert 0.0 <= res.max_contradiction <= 1.0
+    assert 0.0 <= res.max_entailment <= 1.0
+    assert res.n_clauses == 2
+    d = res.as_dict()
+    assert set(d) >= {"available", "would_demote", "max_contradiction", "max_entailment",
+                      "best_clause", "n_clauses", "model_version", "theta_contra",
+                      "theta_entail", "reason"}
+    if not res.available:
+        # deps-missing path: a clean, non-raising, no-signal result
+        assert res.would_demote is False
+        assert res.max_contradiction == 0.0 and res.max_entailment == 0.0
+        assert "deps_missing" in res.reason or "load_failed" in res.reason
+
+
+def test_check_empty_input_when_available_is_no_demote():
+    nli = ShadowNLI()
+    nli._ensure_loaded()
+    if not nli.available:
+        pytest.skip("[nli] extra not installed — empty-input branch only reachable when loaded")
+    res = nli.check("", "Some source text. Another clause.")
+    assert res.available is True and res.would_demote is False and res.reason == "empty_input"
+
+
+# --- bench sweep script ----------------------------------------------------
+
+def test_shadow_sweep_parses_both_record_shapes(tmp_path):
+    from bench.scripts.nli_shadow_sweep import _records
+    p = tmp_path / "mixed.jsonl"
+    p.write_text("\n".join([
+        json.dumps({"_meta": {"battery": "5f"}}),
+        json.dumps({"id": "a", "answer_text": "X is Y.", "context": "X is Z. Q is Y.", "expected_reason": "UNGROUNDED"}),
+        json.dumps({"id": "b", "claim": "P is Q.", "source": "P is R.", "want": "not_contradiction"}),
+        json.dumps({"id": "c", "answer_text": "only answer, no context"}),  # incomplete → skipped
+    ]))
+    recs = list(_records(p))
+    assert [r["id"] for r in recs] == ["a", "b"]
+    assert recs[0]["bucket"] == "UNGROUNDED" and recs[0]["is_fp_probe"] is False
+    assert recs[1]["bucket"] == "not_contradiction" and recs[1]["is_fp_probe"] is True
+
+
+def test_shadow_sweep_main_smoke(tmp_path):
+    from bench.scripts.nli_shadow_sweep import main
+    inp = tmp_path / "in.jsonl"
+    inp.write_text("\n".join([
+        json.dumps({"id": "r1", "claim": "Mercury is the largest planet.",
+                    "source": "Jupiter is the largest planet. Mercury is the smallest planet.",
+                    "want": "contradiction"}),
+        json.dumps({"id": "r2", "claim": "Batman, the alias of Bruce Wayne, lives in Gotham.",
+                    "source": "Batman is the alias of Bruce Wayne. Batman lives in Gotham.",
+                    "want": "not_contradiction"}),
+    ]))
+    out = tmp_path / "report.json"
+    rc = main(["--input", str(inp), "--out", str(out)])
+    assert rc == 0
+    rep = json.loads(out.read_text())
+    assert rep["n_records"] == 2
+    assert "by_bucket" in rep and "false_positive_probe" in rep
+    assert set(rep["by_bucket"]) == {"contradiction", "not_contradiction"}
+    # when [nli] absent the report still renders, just unmeasured
+    assert rep["available"] in (True, False)