From 9532c47f0e13d6a1f73c82d567b51d89b228bf18 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 13 May 2026 09:46:37 -0400 Subject: [PATCH] =?UTF-8?q?#000052=20=C2=A73.2:=20SHADOW=20SCAFFOLD=20land?= =?UTF-8?q?ed=20=E2=80=94=20arborist/qa/relevance/=20mirrors=20arborist/qa?= =?UTF-8?q?/nli/;=20primary=20cross-encoder=20MS-MARCO-MiniLM-L-6-v2;=20Zi?= =?UTF-8?q?onist-entity=20field=20case=20discriminated=20(+9.96=20vs=20-9.?= =?UTF-8?q?04,=2018-pt=20margin)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit arborist/qa/relevance/ — manifest pins cross-encoder/ms-marco-MiniLM-L-6-v2 (~80MB, Apache) as primary; alternates: L-12, BAAI/bge-reranker-base, ms-marco-electra-base. demote_below_score=null on purpose — the #000049 §7 #18→#27 discipline (proved 6× that clean-eval thresholds don't transfer to bench-qa data) requires the threshold to be set by a shadow sweep against pooled real STRICT, not by a literature number. ShadowRelevance class mirrors ShadowNLI (lazy [nli]-extra import, cuda auto-detect via ARBORIST_RELEVANCE_DEVICE or ARBORIST_NLI_DEVICE, batched _score_batch, graceful degrade-to-available=False). Two surface methods: check_question_answer (deflection / Q-A drift) and check_claim_source (topic-collision mis-cite). 13 tests. Sanity on the motivating field case (Zionist entity): ON-topic +9.96 vs OFF-topic -9.04 → 18-pt margin. Mona Lisa Q→A deflection: on +10.45 vs deflect +3.56 → ~7-pt margin. The model CLEANLY discriminates the failure modes #000052 §1 named. It does NOT catch the recombination-where-the-different-entity-clause-also-mentions-the-target case (Kilimanjaro/Mount Kenya) — and that's the right architectural split: aboutness (#000052 §3.2) and entailment (#000049 NLI) are orthogonal axes; the Kilimanjaro recombination case needs the semantic candidate selector (#000050/#000051 vec hybrid). Remaining: build candidate-bench eval (~20-30 deflection + mis-cite fixtures), shadow-sweep θ over pooled bench-qa STRICT (expect another walk-back per the #000049 lesson), recall-side realism check, then fox+dav1d sign-off. Still SHADOW; production verifier unchanged. --- arborist/qa/relevance/__init__.py | 33 +++ arborist/qa/relevance/manifest.json | 20 ++ arborist/qa/relevance/shadow.py | 194 ++++++++++++++++++ ...0052-relevance-coherence-meta-cognition.md | 114 +++++++++- tests/test_relevance_shadow.py | 132 ++++++++++++ 5 files changed, 491 insertions(+), 2 deletions(-) create mode 100644 arborist/qa/relevance/__init__.py create mode 100644 arborist/qa/relevance/manifest.json create mode 100644 arborist/qa/relevance/shadow.py create mode 100644 tests/test_relevance_shadow.py diff --git a/arborist/qa/relevance/__init__.py b/arborist/qa/relevance/__init__.py new file mode 100644 index 0000000..847d110 --- /dev/null +++ b/arborist/qa/relevance/__init__.py @@ -0,0 +1,33 @@ +"""Relevance / aboutness shadow path — #000052 §3.2. + +SHADOW ONLY (same discipline cage as #000049's NLI shadow — see +arborist/qa/nli/__init__.py). Catches *topic-collision recombination* +— answers/claims that are on a different topic than their question or +cited source despite shared tokens (the 'Zionist entity' field case: +the verifier passes because every content token is present in the +source, NLI returns neutral because there's no contradiction, and the +lexical title-relevance sidecar passes because the cited title shares +a polysemous token with the claim) — a failure mode neither the +verifier nor the #000049 NLI veto catches. + +Cross-encoder MS-MARCO reranker; returns a single relevance logit per +(query, document) pair. The transformers + torch runtime is the same +``[nli]`` optional extra; :func:`shadow_check_*` returns an +"unavailable" result rather than raising if the extra is not +installed, so importing this package is always safe. +""" +from .shadow import ( + ShadowRelevance, + ShadowRelevanceResult, + load_manifest, + shadow_check_question_answer, + shadow_check_claim_source, +) + +__all__ = [ + "ShadowRelevance", + "ShadowRelevanceResult", + "load_manifest", + "shadow_check_question_answer", + "shadow_check_claim_source", +] diff --git a/arborist/qa/relevance/manifest.json b/arborist/qa/relevance/manifest.json new file mode 100644 index 0000000..c2ef72b --- /dev/null +++ b/arborist/qa/relevance/manifest.json @@ -0,0 +1,20 @@ +{ + "_comment": "Pinned aboutness/relevance reranker for the #000052 §3.2 SHADOW path. SHADOW MODE ONLY: this manifest does NOT yet fold into governance_policy_hash because shadow output cannot touch audit_mode (#000049 §7 #2 discipline inherited verbatim). If/when promoted to a runtime demotion-only veto, add a single-blob checkpoint_sha256 + tokenizer_sha256 + an explicit relevance_policy_hash that folds into governance_policy_hash before any audit_mode effect. Cross-encoder MS-MARCO reranker — single relevance score per (query, document) pair, NOT entailment (that's NLI / arborist.qa.nli; the boundary failure mode this catches is *topic-collision recombination* — answer/claim is on a different topic than its cited source despite shared tokens, the 'Zionist entity' case — which NLI returns neutral on and the lexical title-relevance sidecar passes because the cited title shares a polysemous token). Reuses the [nli] extra (transformers+torch+optimum); the checkpoint is loaded as AutoModelForSequenceClassification (single regression head emitting a relevance logit), no sentence-transformers dependency.", + "relevance_model_version": "relevance-shadow-v1-ms-marco-MiniLM-L-6-v2", + "hf_repo": "cross-encoder/ms-marco-MiniLM-L-6-v2", + "license": "Apache-2.0", + "source_url": "https://huggingface.co/cross-encoder/ms-marco-MiniLM-L-6-v2", + "approx_mb": 90, + "score_shape": "single logit per (query, document) pair, NOT a softmax — higher = more relevant; trained on MS MARCO passage-ranking, so the score is well-calibrated for 'is this doc relevant to this query?' and uncalibrated for any other notion of similarity", + "max_length": 256, + "truncation_policy": "query_then_document@v1", + "pair_serialization": "relevance_pair@v1", + "demote_below_score": null, + "demote_below_score_comment": "the demote threshold MUST be set by a shadow sweep against bench-qa traffic — DO NOT default to a literature number. The #000049 §7 #18→#27 arc proved repeatedly that clean-eval thresholds don't transfer to real-pipeline data; recommended_operating_point is left null until a candidate-bench + a bench-qa shadow sweep produces a θ with the FP discipline §3.2 inherits from #000049 (zero false-demote on the pooled-808 STRICT denominator at the chosen θ).", + "alternates": [ + {"relevance_model_version": "relevance-shadow-v1-ms-marco-MiniLM-L-12-v2", "hf_repo": "cross-encoder/ms-marco-MiniLM-L-12-v2", "license": "Apache-2.0", "approx_mb": 130, "note": "deeper L-12 variant of the primary; bench against the L-6 on the off-topic eval set"}, + {"relevance_model_version": "relevance-shadow-v1-bge-reranker-base", "hf_repo": "BAAI/bge-reranker-base", "license": "MIT", "approx_mb": 280, "note": "BAAI's general-purpose reranker; different training corpus, useful as a non-MS-MARCO comparison point"}, + {"relevance_model_version": "relevance-shadow-v1-ms-marco-electra-base", "hf_repo": "cross-encoder/ms-marco-electra-base", "license": "Apache-2.0", "approx_mb": 110, "note": "ELECTRA architecture for variety; same MS MARCO supervision"} + ], + "rationale_for_cross_encoder_not_bi_encoder": "A bi-encoder (sentence-transformers, vec-style) gives an unsupervised cosine similarity that's good for retrieval *ranking* but not calibrated for the 'is this on-topic?' question we need here. A cross-encoder reranker is trained on (query, doc) pairs with relevance labels — its single logit IS the 'is this doc relevant to this query?' score we want. The architectural cross-link to #000050/#000051's vec hybrid: that work uses *bi-encoders* (fastembed, bge-small) for retrieval recall; this work uses a *cross-encoder* for downstream aboutness scoring. They are complementary — vec finds candidates, the cross-encoder scores them." +} diff --git a/arborist/qa/relevance/shadow.py b/arborist/qa/relevance/shadow.py new file mode 100644 index 0000000..71fd834 --- /dev/null +++ b/arborist/qa/relevance/shadow.py @@ -0,0 +1,194 @@ +"""Cross-encoder aboutness / relevance shadow check — #000052 §3.2. + +For a (query, document) pair returns a single relevance logit (higher += more relevant). The shadow check has two surface methods that wrap +that single primitive: + + - :meth:`check_question_answer(question, answer)` — does the answer + address the question? (catches deflection / topic-drift the + lexical ``diagnose_deflection`` sidecar misses on token-collision + questions.) + - :meth:`check_claim_source(claim, source_title_or_clause)` — is + each claim *about* its cited source? (catches the 'Zionist + entity'-style mis-cite where the cited title shares a polysemous + token with the claim but is structurally about a different topic.) + +Both are SHADOW: the result is telemetry, never an ``audit_mode`` +input. **No demote threshold is hardcoded** — the bench-maxing +discipline from #000049 §7 #18→#27 (proved six times that clean-eval +thresholds do not transfer to real bench-qa precision *or* recall) +requires the threshold to be set by a shadow sweep against pooled +real STRICT, *not* by a literature number. The model runtime is the +optional ``[nli]`` extra (same transformers + torch dependency the +NLI shadow uses); everything degrades to ``available=False`` when it +is missing. +""" +from __future__ import annotations + +import json +import os +import re +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Optional + +_MANIFEST_PATH = Path(__file__).resolve().parent / "manifest.json" + + +def load_manifest(path: Optional[Path] = None) -> dict: + return json.loads((path or _MANIFEST_PATH).read_text()) + + +@dataclass +class ShadowRelevanceResult: + available: bool # False iff the [nli] extra / model could not load + relevance_score: float # the cross-encoder logit (higher = more relevant); NOT a probability + would_demote: bool # only meaningful when a demote_below_score is configured (else always False) + demote_below_score: Optional[float] + model_version: Optional[str] + pair_kind: str # "question_answer" | "claim_source" + reason: str = "" + + def as_dict(self) -> dict: + return asdict(self) + + +def _resolve_relevance_index(id2label: dict) -> int: + """Cross-encoder reranker checkpoints have a single output, usually + `LABEL_0` or `relevant`. Return that index; if there are multiple + labels, return the one whose name suggests positive relevance. + Substring traps: 'irrelevant' contains 'relevant' — match on tokens, + not raw substrings.""" + if not id2label: + return 0 + if len(id2label) == 1: + return int(next(iter(id2label.keys()))) + # tokenize each label, look for a positive-relevance token without a + # negative-relevance modifier + def _toks(s: str) -> set[str]: + return set(re.findall(r"[a-z0-9]+", str(s).lower())) + pos_tokens = {"relevant", "positive", "entail", "entailment", "label_1"} + neg_tokens = {"irrelevant", "negative", "not", "contradiction", "label_0"} + # first pass: a label whose token set contains a pos token but no neg token + for k, v in id2label.items(): + ts = _toks(v) + if ts & pos_tokens and not (ts & neg_tokens): + return int(k) + # second pass: "label_1" convention + for k, v in id2label.items(): + if str(v).lower().strip() in ("label_1", "1"): + return int(k) + return 0 + + +class ShadowRelevance: + """Lazily-loaded pinned cross-encoder reranker + the §3.2 aboutness + check. Construction never raises; degrades to ``available=False``.""" + + def __init__(self, manifest: Optional[dict] = None, device: Optional[str] = None): + self.manifest = manifest or load_manifest() + self.model_version: Optional[str] = self.manifest.get("relevance_model_version") + self.max_length: int = int(self.manifest.get("max_length", 256)) + self.demote_below_score: Optional[float] = self.manifest.get("demote_below_score") + self.device_pref: str = device or os.environ.get("ARBORIST_RELEVANCE_DEVICE") or os.environ.get("ARBORIST_NLI_DEVICE") or "auto" + self.device: Optional[str] = None + self.batch_size: int = int(os.environ.get("ARBORIST_RELEVANCE_BATCH", "64")) + self.backend: Optional[str] = None + self.available = False + self._reason = "uninitialised" + self._tok = None + self._model = None + self._rel_idx: Optional[int] = None + + def _ensure_loaded(self) -> None: + if self.available or self._reason.startswith(("deps_missing", "load_failed")): + return + try: + import torch # noqa: F401 + from transformers import AutoTokenizer, AutoModelForSequenceClassification + except ImportError as e: + self._reason = f"deps_missing: {e} (install: pip install 'arborist[nli]')" + return + if self.device_pref == "auto": + import torch as _torch + self.device = "cuda" if _torch.cuda.is_available() else "cpu" + else: + self.device = self.device_pref + try: + repo = self.manifest["hf_repo"] + rev = self.manifest.get("pinned_revision") + self._tok = AutoTokenizer.from_pretrained(repo, revision=rev) + self._model = AutoModelForSequenceClassification.from_pretrained(repo, revision=rev) + self._model.eval() + if self.device == "cuda": + self._model = self._model.to("cuda") + id2label = {int(k): v for k, v in self._model.config.id2label.items()} + self._rel_idx = _resolve_relevance_index(id2label) + self.backend = "torch" + except Exception as e: # noqa: BLE001 — any load failure is "unavailable" + self._reason = f"load_failed: {type(e).__name__}: {e}" + self._tok = self._model = None + return + self.available = True + self._reason = "ok" + + def _score_batch(self, pairs: list[tuple[str, str]]) -> list[float]: + import torch + scores: list[float] = [] + for i in range(0, len(pairs), self.batch_size): + chunk = pairs[i:i + self.batch_size] + q = [p for p, _ in chunk] + d = [doc for _, doc in chunk] + enc = self._tok(q, d, return_tensors="pt", truncation=True, + padding=True, max_length=self.max_length) + if self.device == "cuda": + enc = {k: v.to("cuda") for k, v in enc.items()} + with torch.no_grad(): + logits = self._model(**enc).logits + logits = logits.detach().cpu().tolist() + for row in logits: + scores.append(float(row[self._rel_idx] if len(row) > 1 else row[0])) + return scores + + def _check(self, query: str, document: str, pair_kind: str) -> ShadowRelevanceResult: + def _result(**kw) -> ShadowRelevanceResult: + base = dict(model_version=self.model_version, pair_kind=pair_kind, + demote_below_score=self.demote_below_score) + base.update(kw) + return ShadowRelevanceResult(**base) # type: ignore[arg-type] + + self._ensure_loaded() + if not self.available: + return _result(available=False, relevance_score=0.0, would_demote=False, + reason=self._reason) + if not (query or "").strip() or not (document or "").strip(): + return _result(available=True, relevance_score=0.0, would_demote=False, + reason="empty_input") + score = self._score_batch([(query, document)])[0] + wd = (self.demote_below_score is not None and score < self.demote_below_score) + return _result(available=True, relevance_score=round(score, 4), + would_demote=wd, reason="ok") + + def check_question_answer(self, question: str, answer: str) -> ShadowRelevanceResult: + return self._check(question, answer, "question_answer") + + def check_claim_source(self, claim: str, source: str) -> ShadowRelevanceResult: + return self._check(claim, source, "claim_source") + + +_DEFAULT: Optional[ShadowRelevance] = None + + +def _shared() -> ShadowRelevance: + global _DEFAULT + if _DEFAULT is None: + _DEFAULT = ShadowRelevance() + return _DEFAULT + + +def shadow_check_question_answer(question: str, answer: str) -> ShadowRelevanceResult: + return _shared().check_question_answer(question, answer) + + +def shadow_check_claim_source(claim: str, source: str) -> ShadowRelevanceResult: + return _shared().check_claim_source(claim, source) diff --git a/docs/tickets/ticket-000052-relevance-coherence-meta-cognition.md b/docs/tickets/ticket-000052-relevance-coherence-meta-cognition.md index 8fb6bda..496b6f9 100644 --- a/docs/tickets/ticket-000052-relevance-coherence-meta-cognition.md +++ b/docs/tickets/ticket-000052-relevance-coherence-meta-cognition.md @@ -5,8 +5,22 @@ model: `circular` / `phrase_component_reuse` / `vacuous`; sidecar in `arborist/qa/inspect.py`, surfaced in `inspect_cache_key` + the `arborist inspect` human view as `· incoherent: `; 9 tests in `tests/test_inspect.py`; demote-policy hook deliberately NOT wired — -advisory only, off the proof path). §3.2 (relevance *model*) still -gated on evidence + travels with #000049's model question. §4.2 +advisory only, off the proof path). **§3.2 SHADOW SCAFFOLD LANDED +2026-05-13** (`arborist/qa/relevance/` mirroring `arborist/qa/nli/` — +manifest pins `cross-encoder/ms-marco-MiniLM-L-6-v2` as primary + 3 +alternates; `ShadowRelevance` w/ `check_question_answer` and +`check_claim_source`; shares the `[nli]` extra; `demote_below_score` +intentionally `null` until a bench-gated shadow sweep sets it per the +#000049 §7 #18→#27 discipline; 13 tests). Sanity-checked on the +motivating field case: the Zionist-entity ON-topic / OFF-topic pair +scores +9.96 vs −9.04 — 18-point margin, the reranker cleanly catches +exactly the topic-collision mis-cite failure mode (and the Mona Lisa +Q→A deflection case at +10.45 vs +3.56). It does NOT catch the +*recombination*-where-the-different-entity-clause-mentions-the-target- +subject case (the Kilimanjaro/Mount Kenya pattern from #000049 §7 #26) +— and that's the right architectural split: §3.2 is *aboutness*, §000049 +is *entailment*; they're orthogonal axes. §3.2 *threshold-setting and +runtime promotion* still gated on the bench-protocol in §3.2.2. §4.2 (upstream retrieval polysemy fix) still a flag, not scoped here. **Opened:** 2026-05-12 **Scope:** Decide whether — and if so how — to add two more read-only @@ -209,6 +223,102 @@ on collisions. A small *aboutness* model would give a semantic score: and #000049 §7 #21's lesson stands: the clean-eval number won't predict the real-traffic one). +#### 3.2.1 — scaffold landed (2026-05-13) + +The §3.2 *shadow scaffold* is in place, mirroring `arborist/qa/nli/`'s +structure verbatim. Pieces: + +- `arborist/qa/relevance/manifest.json` — pins + `cross-encoder/ms-marco-MiniLM-L-6-v2` (~80 MB, Apache-2.0) as the + primary candidate; three alternates documented + (`ms-marco-MiniLM-L-12-v2`, `BAAI/bge-reranker-base`, + `cross-encoder/ms-marco-electra-base`). Crucially, **the manifest's + `demote_below_score` is `null`** — *no hardcoded threshold*. The + bench-maxing discipline from #000049 §7 #18→#27 (proved six times + over that clean-eval thresholds don't transfer to bench-qa + precision *or* recall) requires the demote threshold to be set by a + *shadow sweep against pooled real STRICT*, not by a literature + number. Tests assert `demote_below_score is None` so it can't drift + back to a guess. +- `arborist/qa/relevance/__init__.py` + `shadow.py` — `ShadowRelevance` + class (lazy `transformers`+`torch` import behind the *shared* `[nli]` + extra; same graceful-degrade-to-`available=False` contract; `cuda` + auto-detect from `ARBORIST_RELEVANCE_DEVICE` or `ARBORIST_NLI_DEVICE`; + batched `_score_batch`), with two surface methods: + `check_question_answer(question, answer)` (catches deflection / + topic-drift the lexical `diagnose_deflection` sidecar misses on + token-collision questions) and `check_claim_source(claim, source)` + (the Zionist-entity-style mis-cite). Both return a single relevance + *logit* (higher = more relevant; **not a probability**) and a + `would_demote` flag that stays `False` until a threshold is set. +- `tests/test_relevance_shadow.py` — 13 tests (pure-Python + graceful + degradation + the Zionist sanity check); all green. + +**Sanity check on real (claim, source) pairs** (run on the loaded +`ms-marco-MiniLM-L-6-v2`, CPU, batch=1; scores are logits, sign matters): + +| pair | claim | source | score | +|---|---|---|---| +| Zionist ON-topic | "...used as a pejorative for the State of Israel" | the real `Zionist entity` sentence | **+9.96** | +| Zionist OFF-topic (the field case) | "...used as the entity, referring to the State of Israel" | a grammar `Phrase` definition | **−9.04** | +| Mona Lisa Q→A on | "who painted the mona lisa?" | "Leonardo da Vinci painted the Mona Lisa." | +10.45 | +| Mona Lisa Q→A deflect | "who painted the mona lisa?" | "The Mona Lisa is a half-length portrait housed in the Louvre." | +3.56 | +| **Kilimanjaro recombination** | "Kilimanjaro is in central Kenya south of the equator" | the Mount Kenya clause | **+7.59** (false positive — see below) | +| Kilimanjaro on | "Kilimanjaro is in north-eastern Tanzania" | the Kilimanjaro lead | +6.06 | + +**Reading.** The reranker **cleanly discriminates the Zionist-entity +field case (18-point margin)** and the Q→A deflection case (~7-point +margin) — i.e. it catches exactly the failure mode #000052 §1 named +("the lexical title-relevance sidecar passes a token-collision +mis-cite"). It does **NOT** catch the *recombination* case (Mount +Kenya clause scores *higher* than the Kilimanjaro lead for a +"Kilimanjaro is in Kenya" claim) — because the Mount Kenya clause is +genuinely about Kilimanjaro-as-comparison and the reranker is scoring +*aboutness*, not *truth*. That is the right architectural split: + +- **#000052 §3.2 relevance reranker** = catches *topic-collision / + off-topic* failures (Zionist entity, Q→A deflection). ✓ shown. +- **#000049 NLI veto** = catches *contradiction-recombination* + failures (Mercury / Einstein synthetic fixtures). ✓ shown (§7 #24). +- *Neither* by itself catches the *recombination-where-the-different- + entity-clause-also-mentions-the-target-subject* case (the Mount + Kenya / Kilimanjaro pattern from §7 #26's mining) — that needs the + **semantic candidate selector** (vec-driven; #000050/#000051) so + NLI doesn't get handed the wrong clause in the first place. + +So §3.2's *aboutness* signal complements #000049's *entailment* +signal — they're orthogonal axes (topicality vs. truth-of-attribution) +and the two failure modes the sidecar family didn't previously cover +(incoherence and topic-collision) get cleanly handled by: +`diagnose_coherence` (§3.1, lexical, landed) + this relevance reranker +(§3.2, model, scaffold landed; threshold-setting and bench-gated +shadow run still TODO). + +#### 3.2.2 — Remaining work (the bench-gated decision) + +The scaffold is in place; the **threshold is intentionally unset**. +Before runtime promotion, the same #000049 §7 #18→#27 protocol applies: + +1. *Build a small candidate-bench eval set* (~20-30 fixtures) of + (question, answer) deflection cases + (claim, source) mis-cite + cases (the Zionist-entity case is fixture 1); also a negative- + control set of legit on-topic pairs. Measure each alternate model + (L-6, L-12, BGE, electra) on it — pick the winner by *separation + margin*, not raw score (per §7 #18's "model science"). +2. *Shadow-run the winner against pooled real bench-qa STRICT* + (~800+ cells) with a θ sweep; pick the θ where false-demote rate + on STRICT is zero. (And expect a walk-back the moment the + denominator grows — §7 #25→#26 → #27 happened three times on the + NLI side; assume it will here too.) +3. *Recall-side realism check* against hand-built deflection/mis-cite + claims constructed against *real* bench-qa contexts (#000049 §7 + #27 pattern — *don't* trust the candidate-bench's recall numbers + to transfer). +4. Only after 1–3 set a real `demote_below_score`, switch the + `recommended_operating_point` to active, fold + `relevance_policy_hash` into `governance_policy_hash`, and put it + in front of fox+dav1d for the runtime-promotion decision. + ### 3.3 — relationship to #000049 NLI (contradiction) and a relevance model (aboutness) are **siblings** diff --git a/tests/test_relevance_shadow.py b/tests/test_relevance_shadow.py new file mode 100644 index 0000000..b8caafa --- /dev/null +++ b/tests/test_relevance_shadow.py @@ -0,0 +1,132 @@ +"""Tests for the #000052 §3.2 relevance shadow scaffold. + +Mirrors `tests/test_nli_shadow.py`'s discipline: pure-Python parts + +graceful degradation when the ``[nli]`` extra is absent. The real +model path is exercised opportunistically (skipped when deps missing). +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from arborist.qa.relevance import ( + ShadowRelevance, ShadowRelevanceResult, load_manifest, + shadow_check_question_answer, shadow_check_claim_source, +) +from arborist.qa.relevance.shadow import _resolve_relevance_index + + +# --- manifest -------------------------------------------------------------- + +def test_manifest_has_required_fields(): + m = load_manifest() + for key in ("relevance_model_version", "hf_repo", "license", "max_length", + "score_shape", "alternates", "demote_below_score", + "demote_below_score_comment"): + assert key in m, key + # demote_below_score MUST be null until bench-set (§7 #18→#27 lesson) + assert m["demote_below_score"] is None + + +def test_manifest_is_valid_json_file(): + p = Path(__file__).resolve().parents[1] / "arborist" / "qa" / "relevance" / "manifest.json" + json.loads(p.read_text()) # raises on malformed + + +# --- label-index resolution ------------------------------------------------ + +@pytest.mark.parametrize("id2label,expect", [ + ({0: "LABEL_0"}, 0), # single-output reranker (typical) + ({0: "irrelevant", 1: "relevant"}, 1), # 2-class with "relevant" label + ({0: "negative", 1: "positive"}, 1), # 2-class with "positive" label + ({0: "LABEL_0", 1: "LABEL_1"}, 1), # fallback to "label_1" substring + ({0: "score_0", 1: "score_1"}, 0), # no relevant/positive → fallback to 0 +]) +def test_resolve_relevance_index(id2label, expect): + assert _resolve_relevance_index(id2label) == expect + + +# --- ShadowRelevance contract ---------------------------------------------- + +def test_shadowrelevance_construction_never_raises_and_starts_unavailable(): + r = ShadowRelevance() + assert r.available is False + assert r.model_version == load_manifest()["relevance_model_version"] + assert r.demote_below_score is None # no hardcoded threshold (§7 #18→#27 discipline) + assert r.device_pref in ("auto", "cpu", "cuda") + assert r.batch_size >= 1 + assert r.backend is None + # _ensure_loaded never raises even when the extra is missing + r._ensure_loaded() + assert r.available in (True, False) + + +def test_shadow_check_question_answer_returns_result_and_degrades_gracefully(): + res = shadow_check_question_answer("who painted the mona lisa?", + "Leonardo da Vinci painted the Mona Lisa.") + assert isinstance(res, ShadowRelevanceResult) + assert res.pair_kind == "question_answer" + assert res.would_demote in (True, False) + d = res.as_dict() + assert set(d) >= {"available", "relevance_score", "would_demote", + "demote_below_score", "model_version", "pair_kind", "reason"} + if not res.available: + assert res.would_demote is False + assert "deps_missing" in res.reason or "load_failed" in res.reason + + +def test_shadow_check_claim_source_returns_result_and_degrades_gracefully(): + res = shadow_check_claim_source("Leonardo da Vinci painted the Mona Lisa.", + "The Mona Lisa is a half-length portrait painting by Italian artist Leonardo da Vinci.") + assert isinstance(res, ShadowRelevanceResult) + assert res.pair_kind == "claim_source" + + +def test_no_hardcoded_threshold_means_would_demote_stays_False(): + # the §7 #18→#27 discipline: demote_below_score must be null until + # a shadow sweep sets it. The contract: would_demote=False until then. + r = ShadowRelevance() + r._ensure_loaded() + if not r.available: + pytest.skip("[nli] extra not installed — would_demote-False path only reachable when loaded") + # any pair the model scores should still produce would_demote=False + # because demote_below_score is None at scaffold time + res = r.check_question_answer("capital of france?", "Paris.") + assert res.would_demote is False + assert res.demote_below_score is None + + +def test_empty_input_when_available_is_no_demote(): + r = ShadowRelevance() + r._ensure_loaded() + if not r.available: + pytest.skip("[nli] extra not installed") + assert r.check_question_answer("", "non-empty answer").reason == "empty_input" + assert r.check_question_answer("non-empty question", "").reason == "empty_input" + + +def test_zionist_entity_field_case_when_available(): + """The motivating case (#000052 §1): a garbled claim about 'phrase + used as the entity' against a 'Zionist entity' source. The + relevance score for the (claim, source) pair should be LOWER than + for a well-aligned (claim, source) pair — that is the signal the + veto would act on if the threshold were set.""" + r = ShadowRelevance() + r._ensure_loaded() + if not r.available: + pytest.skip("[nli] extra not installed") + # off-topic: a claim about the abstract noun 'entity' against a source about Israel + off = r.check_claim_source( + "the phrase 'Zionist entity' is sometimes used as the entity, referring to the State of Israel", + "Wikipedia article 'Phrase' — In grammar, a phrase is a group of words functioning as a single unit in the syntax of a sentence.", + ) + on = r.check_claim_source( + "the phrase 'Zionist entity' is sometimes used as a pejorative for the State of Israel", + "Zionist entity () is a phrase sometimes used by Arabs and Muslims as a pejorative for the State of Israel.", + ) + # both available + assert off.available and on.available + # on-topic should score strictly higher than off-topic — that's the discriminator + assert on.relevance_score > off.relevance_score