russell@unturf.com
5f4f4ceb1c
§3.1 diagnose_coherence (now 19 tests, +10 from the parallel session's 9):
- 3 more positive shapes (multi-sentence vacuous, named-entity circular,
grammar-term phrase_component_reuse).
- 5 xfail regression tests for SHAPES THAT FALSE-FIRE on real bench-qa
STRICT data (44/808 = 5.4% FP rate measured on the pooled n=1+3+5
STRICT answers). Each xfail names the exact shape + why it should
ideally be 'ok' + which rule needs tightening:
* 'The chemical symbol for gold is Au.' → vacuous (short predicate)
* 'Michael Jordan's Restaurant was a restaurant ... named after
Michael Jordan.' → circular (named-after re-use)
* 'The Western X was the western half of the X' → circular
* 'The name <Phrase> is a translation ... of the <derivative>' →
phrase_component_reuse (translation/etymology)
* claim-lattice [E1 | … …"] tails → vacuous (truncated bracket
fragment)
* 'The term <X>' → phrase_component_reuse (idiomatic English)
- 1 load-bearing real-traffic test: FP rate on 808-cell pooled STRICT
must stay ≤ 7% (current 5.4%) — fires loud if a future change
regresses it. Skips on fresh-checkout (bench/qa_results/ gitignored).
§3.2 ShadowRelevance (now 20 tests, +7 from the round-1 scaffold):
- Manifest tests for round-2 primary (bge-reranker-large), the size
spectrum coverage (50-560MB), the candidate-bench findings block
(biggest-within-family / not-across-families / deeper-not-better /
capacity-floor).
- Pair-kind distinction (question_answer vs claim_source recorded
separately for downstream telemetry / governance hashing).
- Batch-order preservation (_score_batch must return scores in input
order — load-bearing for downstream zip-back).
- Empty-input handling (Q empty, D empty, whitespace-only).
- Zionist-entity discriminator sanity (on-topic > off-topic logit).
- demote_below_score-stays-null invariant (the §7 #18→#27 discipline:
no hardcoded threshold; must come from a real-traffic shadow sweep).
Total: 101 passed + 6 xfailed (5 §3.1 regressions documented + 1 from
parallel session). The 5 xfails are the bench-maxing receipts — they
document EXACTLY which shapes §3.1 false-fires on, with the rule that
needs tightening named in each reason.
251 lines
12 KiB
Python
251 lines
12 KiB
Python
"""Tests for the #000052 §3.2 relevance shadow scaffold.
|
||
|
||
Mirrors `tests/test_nli_shadow.py`'s discipline: pure-Python parts +
|
||
graceful degradation when the ``[nli]`` extra is absent. The real
|
||
model path is exercised opportunistically (skipped when deps missing).
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from arborist.qa.relevance import (
|
||
ShadowRelevance, ShadowRelevanceResult, load_manifest,
|
||
shadow_check_question_answer, shadow_check_claim_source,
|
||
)
|
||
from arborist.qa.relevance.shadow import _resolve_relevance_index
|
||
|
||
|
||
# --- manifest --------------------------------------------------------------
|
||
|
||
def test_manifest_has_required_fields():
|
||
m = load_manifest()
|
||
for key in ("relevance_model_version", "hf_repo", "license", "max_length",
|
||
"score_shape", "alternates", "demote_below_score",
|
||
"demote_below_score_comment"):
|
||
assert key in m, key
|
||
# demote_below_score MUST be null until bench-set (§7 #18→#27 lesson)
|
||
assert m["demote_below_score"] is None
|
||
|
||
|
||
def test_manifest_is_valid_json_file():
|
||
p = Path(__file__).resolve().parents[1] / "arborist" / "qa" / "relevance" / "manifest.json"
|
||
json.loads(p.read_text()) # raises on malformed
|
||
|
||
|
||
# --- label-index resolution ------------------------------------------------
|
||
|
||
@pytest.mark.parametrize("id2label,expect", [
|
||
({0: "LABEL_0"}, 0), # single-output reranker (typical)
|
||
({0: "irrelevant", 1: "relevant"}, 1), # 2-class with "relevant" label
|
||
({0: "negative", 1: "positive"}, 1), # 2-class with "positive" label
|
||
({0: "LABEL_0", 1: "LABEL_1"}, 1), # fallback to "label_1" substring
|
||
({0: "score_0", 1: "score_1"}, 0), # no relevant/positive → fallback to 0
|
||
])
|
||
def test_resolve_relevance_index(id2label, expect):
|
||
assert _resolve_relevance_index(id2label) == expect
|
||
|
||
|
||
# --- ShadowRelevance contract ----------------------------------------------
|
||
|
||
def test_shadowrelevance_construction_never_raises_and_starts_unavailable():
|
||
r = ShadowRelevance()
|
||
assert r.available is False
|
||
assert r.model_version == load_manifest()["relevance_model_version"]
|
||
assert r.demote_below_score is None # no hardcoded threshold (§7 #18→#27 discipline)
|
||
assert r.device_pref in ("auto", "cpu", "cuda")
|
||
assert r.batch_size >= 1
|
||
assert r.backend is None
|
||
# _ensure_loaded never raises even when the extra is missing
|
||
r._ensure_loaded()
|
||
assert r.available in (True, False)
|
||
|
||
|
||
def test_shadow_check_question_answer_returns_result_and_degrades_gracefully():
|
||
res = shadow_check_question_answer("who painted the mona lisa?",
|
||
"Leonardo da Vinci painted the Mona Lisa.")
|
||
assert isinstance(res, ShadowRelevanceResult)
|
||
assert res.pair_kind == "question_answer"
|
||
assert res.would_demote in (True, False)
|
||
d = res.as_dict()
|
||
assert set(d) >= {"available", "relevance_score", "would_demote",
|
||
"demote_below_score", "model_version", "pair_kind", "reason"}
|
||
if not res.available:
|
||
assert res.would_demote is False
|
||
assert "deps_missing" in res.reason or "load_failed" in res.reason
|
||
|
||
|
||
def test_shadow_check_claim_source_returns_result_and_degrades_gracefully():
|
||
res = shadow_check_claim_source("Leonardo da Vinci painted the Mona Lisa.",
|
||
"The Mona Lisa is a half-length portrait painting by Italian artist Leonardo da Vinci.")
|
||
assert isinstance(res, ShadowRelevanceResult)
|
||
assert res.pair_kind == "claim_source"
|
||
|
||
|
||
def test_no_hardcoded_threshold_means_would_demote_stays_False():
|
||
# the §7 #18→#27 discipline: demote_below_score must be null until
|
||
# a shadow sweep sets it. The contract: would_demote=False until then.
|
||
r = ShadowRelevance()
|
||
r._ensure_loaded()
|
||
if not r.available:
|
||
pytest.skip("[nli] extra not installed — would_demote-False path only reachable when loaded")
|
||
# any pair the model scores should still produce would_demote=False
|
||
# because demote_below_score is None at scaffold time
|
||
res = r.check_question_answer("capital of france?", "Paris.")
|
||
assert res.would_demote is False
|
||
assert res.demote_below_score is None
|
||
|
||
|
||
def test_empty_input_when_available_is_no_demote():
|
||
r = ShadowRelevance()
|
||
r._ensure_loaded()
|
||
if not r.available:
|
||
pytest.skip("[nli] extra not installed")
|
||
assert r.check_question_answer("", "non-empty answer").reason == "empty_input"
|
||
assert r.check_question_answer("non-empty question", "").reason == "empty_input"
|
||
|
||
|
||
def test_zionist_entity_field_case_when_available():
|
||
"""The motivating case (#000052 §1): a garbled claim about 'phrase
|
||
used as the entity' against a 'Zionist entity' source. The
|
||
relevance score for the (claim, source) pair should be LOWER than
|
||
for a well-aligned (claim, source) pair — that is the signal the
|
||
veto would act on if the threshold were set."""
|
||
r = ShadowRelevance()
|
||
r._ensure_loaded()
|
||
if not r.available:
|
||
pytest.skip("[nli] extra not installed")
|
||
# off-topic: a claim about the abstract noun 'entity' against a source about Israel
|
||
off = r.check_claim_source(
|
||
"the phrase 'Zionist entity' is sometimes used as the entity, referring to the State of Israel",
|
||
"Wikipedia article 'Phrase' — In grammar, a phrase is a group of words functioning as a single unit in the syntax of a sentence.",
|
||
)
|
||
on = r.check_claim_source(
|
||
"the phrase 'Zionist entity' is sometimes used as a pejorative for the State of Israel",
|
||
"Zionist entity () is a phrase sometimes used by Arabs and Muslims as a pejorative for the State of Israel.",
|
||
)
|
||
# both available
|
||
assert off.available and on.available
|
||
# on-topic should score strictly higher than off-topic — that's the discriminator
|
||
assert on.relevance_score > off.relevance_score
|
||
|
||
|
||
# --- broader §3.2 coverage (round-2 candidate-bench, 2026-05-13) ----------
|
||
# Added after extending the size spectrum 30MB→560MB (#000052 §3.2.1
|
||
# round-2): more tests covering pair-kind-specific behavior, batch
|
||
# correctness, and the new manifest entries.
|
||
|
||
def test_manifest_round2_primary_is_bge_reranker_large():
|
||
m = load_manifest()
|
||
assert m["relevance_model_version"] == "relevance-shadow-v1-bge-reranker-large"
|
||
assert m["hf_repo"] == "BAAI/bge-reranker-large"
|
||
# the round-2 candidate-bench block is recorded
|
||
assert "candidate_bench_results_2026_05_13_round2" in m
|
||
cb = m["candidate_bench_results_2026_05_13_round2"]
|
||
assert cb["rank_by_sep_margin"][0]["model"] == "bge-reranker-large"
|
||
# capacity-floor finding is documented
|
||
floor = cb["rank_by_sep_margin"][-1]
|
||
assert floor["model"] == "ms-marco-MiniLM-L-2-v2"
|
||
assert floor["margin"] < 0 # NOT clean-separable
|
||
# findings block exists with the key science
|
||
assert "findings" in cb
|
||
for key in ("biggest_is_best_within_a_family", "but_not_across_families",
|
||
"deeper_is_not_better_within_MS_MARCO_MiniLM",
|
||
"capacity_floor_between_L_2_and_L_4"):
|
||
assert key in cb["findings"]
|
||
|
||
|
||
def test_manifest_alternates_cover_size_spectrum():
|
||
m = load_manifest()
|
||
sizes = sorted(a["approx_mb"] for a in m["alternates"])
|
||
# the alternates must span the spectrum from ≤50MB (small) to ≥280MB (large)
|
||
assert sizes[0] <= 50, f"smallest alternate is {sizes[0]}MB — needs a ≤50MB datapoint"
|
||
assert sizes[-1] >= 280, f"largest alternate is {sizes[-1]}MB — needs a ≥280MB datapoint"
|
||
|
||
|
||
def test_pair_kind_recorded_separately_for_qa_vs_claim_source():
|
||
# The result records WHICH method produced the score — same model,
|
||
# same primitive, but the bookkeeping must distinguish the two
|
||
# for downstream telemetry / governance hashing.
|
||
r = ShadowRelevance()
|
||
res_qa = r.check_question_answer("who painted the mona lisa?",
|
||
"Leonardo da Vinci painted the Mona Lisa.")
|
||
res_cs = r.check_claim_source("Leonardo da Vinci painted the Mona Lisa.",
|
||
"The Mona Lisa is a portrait by Leonardo da Vinci.")
|
||
assert res_qa.pair_kind == "question_answer"
|
||
assert res_cs.pair_kind == "claim_source"
|
||
|
||
|
||
def test_batch_score_preserves_input_order():
|
||
"""_score_batch returns scores in input order (any padding/batching
|
||
must not reorder). This is load-bearing for downstream bench
|
||
scripts that zip scores back onto input records."""
|
||
r = ShadowRelevance()
|
||
r._ensure_loaded()
|
||
if not r.available:
|
||
pytest.skip("[nli] extra not installed")
|
||
pairs = [
|
||
("very short", "very short"),
|
||
("query about the painting Mona Lisa by Leonardo da Vinci in the 16th century",
|
||
"Leonardo da Vinci painted the Mona Lisa around 1503–1519, an Italian Renaissance portrait."),
|
||
("a", "b"),
|
||
]
|
||
a = r._score_batch(pairs)
|
||
# scoring each pair individually should give the same answers (modulo
|
||
# padding noise, which for fp32 should be 0)
|
||
individual = [r._score_batch([p])[0] for p in pairs]
|
||
assert len(a) == 3 == len(individual)
|
||
for batched, single in zip(a, individual):
|
||
# padding can introduce tiny numerical drift on fp16/fp32; allow
|
||
# 0.05 logit slack (well below any threshold we'd ever set)
|
||
assert abs(batched - single) < 0.05, f"batched {batched} vs single {single}"
|
||
|
||
|
||
def test_empty_string_query_or_document_returns_empty_input():
|
||
r = ShadowRelevance()
|
||
r._ensure_loaded()
|
||
if not r.available:
|
||
pytest.skip("[nli] extra not installed")
|
||
assert r.check_question_answer("", "some answer").reason == "empty_input"
|
||
assert r.check_claim_source("a claim", "").reason == "empty_input"
|
||
# whitespace-only too
|
||
assert r.check_question_answer(" ", "answer").reason == "empty_input"
|
||
|
||
|
||
def test_score_sign_distinguishes_on_topic_from_off_topic():
|
||
"""Sanity: on the Zionist field case (the motivating example),
|
||
the on-topic (claim, source) pair MUST score strictly higher than
|
||
the off-topic one. This is the load-bearing discrimination — if
|
||
this assertion fails on the manifest's primary, the whole §3.2
|
||
approach is invalidated and we walk back."""
|
||
r = ShadowRelevance()
|
||
r._ensure_loaded()
|
||
if not r.available:
|
||
pytest.skip("[nli] extra not installed")
|
||
on = r.check_claim_source(
|
||
"the phrase 'Zionist entity' is sometimes used as a pejorative for the State of Israel",
|
||
"Zionist entity () is a phrase sometimes used by Arabs and Muslims as a pejorative for the State of Israel.",
|
||
)
|
||
off = r.check_claim_source(
|
||
"the phrase 'Zionist entity' is sometimes used as the entity, referring to the State of Israel",
|
||
"In grammar, a phrase is a group of words functioning as a single unit in the syntax of a sentence.",
|
||
)
|
||
assert on.available and off.available
|
||
assert on.relevance_score > off.relevance_score
|
||
|
||
|
||
def test_demote_below_score_remains_null_until_bench_sets_it():
|
||
"""Load-bearing invariant from #000049 §7 #18→#27: the demote
|
||
threshold MUST be set by a real-traffic shadow sweep, not by a
|
||
candidate-bench number or literature default. If anyone sets
|
||
demote_below_score in the manifest before step 2 lands, this
|
||
test fires."""
|
||
m = load_manifest()
|
||
assert m["demote_below_score"] is None, \
|
||
"demote_below_score must be null until §3.2.2 step 2 sets it from a real-traffic shadow sweep"
|
||
# the round-2 candidate-bench DOES record candidate θ values, but
|
||
# explicitly notes they are not the runtime threshold
|
||
cb = m["candidate_bench_results_2026_05_13_round2"]
|
||
assert "note" in cb and "CANDIDATE-BENCH ONLY" in cb["note"]
|