From 70ecda3d6c07e1f15b3d13638ea26ea5ba63c751 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 12 May 2026 14:14:21 -0400 Subject: [PATCH] =?UTF-8?q?#000049=20Phase=202:=20NLI=20shadow=20scaffold?= =?UTF-8?q?=20in=20arborist=20(=C2=A77=20#19)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit arborist/qa/nli/ — SHADOW ONLY (never an audit_mode input; manifest not yet in governance_policy_hash per §7 #2). manifest.json pins cross-encoder/nli-MiniLM2-L6-H768 @ a fixed HF revision + the bench-validated θc 0.5/θe 0.9 + 2 alternates + the Phase-3 TODO; shadow.py = ShadowNLI/shadow_check (lazy transformers+torch behind a new [nli] extra, clauses() segmenter, the §7 #5 clause-level Demote() decision, degrades to available=False when [nli] absent); bench/scripts/nli_shadow_sweep.py + make bootstrap-nli / bench-nli-shadow (the gate-item-4 instrument); 16 tests. First sweep (116 records — 5f-falsification packs + the arborist-nli-bench eval sets): 28/28 synth recombination demoted, 0/26 FP on legit summaries, 0/9 fires on already-STRICT_SPAN records, 25/50 on UNGROUNDED (the contradiction half; quiet on non-sequiturs). Gate items 1/2/3/5/6 clear on available data; item 4 — shadow FP rate on a real live-bench-qa sample — remains the open measurement. Production verifier unchanged; falsification-hard stays 10/12. --- Makefile | 24 +- arborist/qa/nli/__init__.py | 17 + arborist/qa/nli/manifest.json | 24 + arborist/qa/nli/shadow.py | 183 ++ bench/results/nli-shadow-sweep.json | 1909 +++++++++++++++++ bench/scripts/nli_shadow_sweep.py | 158 ++ docs/TICKETS.md | 2 +- ...00049-attribution-aware-grounding-check.md | 64 +- pyproject.toml | 17 + tests/test_nli_shadow.py | 153 ++ 10 files changed, 2543 insertions(+), 8 deletions(-) create mode 100644 arborist/qa/nli/__init__.py create mode 100644 arborist/qa/nli/manifest.json create mode 100644 arborist/qa/nli/shadow.py create mode 100644 bench/results/nli-shadow-sweep.json create mode 100644 bench/scripts/nli_shadow_sweep.py create mode 100644 tests/test_nli_shadow.py diff --git a/Makefile b/Makefile index 852d550..d7ca5cc 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ SEARCH_Q ?= computer prometheus-trigger-probe bench-5f-threshold-calibration \ bench-5f-selfmodel-snapshot bench-5f-finetuning-shardchain \ bench-5f-falsification-hard bench-fork-baseline-hard bench-5f-formulate-hard \ - bootstrap-math clean clean-db clean-data help \ + bootstrap-math bootstrap-nli bench-nli-shadow clean clean-db clean-data help \ textbooks-summary textbooks-urls fetch-textbooks textbooks-stats textbooks-verify \ crawl-textbooks crawl-textbooks-stats textbook textbook-list @@ -779,6 +779,28 @@ bootstrap-crawler: bootstrap ## install [crawler] extras into the venv bootstrap-math: bootstrap ## install [math] extras (sympy) into the venv $(PIP) install -e '.[math]' +# #000049 Phase 2 — sentence-pair NLI runtime for the SHADOW path +# (arborist/qa/nli/). ~600 MB (transformers + CPU torch); hard out of +# core / dev so a fresh checkout stays python3.12 + venv + sqlite3. +# Installs the extra, then warms the pinned checkpoint download so the +# first `make bench-nli-shadow` doesn't pay for it. Operator target — +# NOT part of `make bootstrap`, `make test`, or a fresh checkout. +bootstrap-nli: bootstrap ## install [nli] extras + warm the pinned NLI checkpoint + $(PIP) install -e '.[nli]' + $(PY) -c "from arborist.qa.nli import ShadowNLI; r=ShadowNLI(); r._ensure_loaded(); print('nli runtime ·', 'available:', r.available, '·', r._reason)" + +# #000049 Phase 2 / §7 #12 gate item 4 — measure the would-demote rate +# of the clause-level shadow check over (answer, context) records. +# SHADOW ONLY: writes only bench/results/nli-shadow-sweep.json, never an +# audit_mode. With no INPUT it sweeps the 5f falsification packs (every +# record there is a FALSE claim, so a high would-demote rate = the model +# working). Point INPUT at a legit-answer sample for the real gate-item-4 +# false-positive number. Renders a structural report even without the +# [nli] extra (marked available:false). Run `make bootstrap-nli` first +# for real numbers. +bench-nli-shadow: bootstrap ## #000049 Phase 2 — NLI shadow would-demote sweep (INPUT="f1.jsonl f2.jsonl" optional) + PYTHONUNBUFFERED=1 $(PY) bench/scripts/nli_shadow_sweep.py $(foreach f,$(INPUT),--input $(f)) + # --------------------------------------------------------------------------- # Public-domain + open-licensed textbooks (#000031 / surface-ingest scope). # diff --git a/arborist/qa/nli/__init__.py b/arborist/qa/nli/__init__.py new file mode 100644 index 0000000..c1f279f --- /dev/null +++ b/arborist/qa/nli/__init__.py @@ -0,0 +1,17 @@ +"""NLI shadow path — #000049 Phase 2. + +SHADOW ONLY. Nothing in this package writes to ``providence_cache`` or +``audit_events`` or influences ``audit_mode``. It exists to *measure* +whether a clause-level entailment check would demote a weakly-grounded +answer (the §7 #5 algorithm), so the #000049 §7 #12 gate item 4 — the +real-traffic false-positive rate — can be computed before any decision +to wire NLI into the live verifier (Phase 3). + +The ``transformers`` / ``torch`` runtime is the optional ``[nli]`` +extra; :func:`shadow_check` returns an "unavailable" result rather than +raising if the extra is not installed, so importing this package is +always safe. +""" +from .shadow import ShadowResult, shadow_check, ShadowNLI, load_manifest + +__all__ = ["ShadowResult", "shadow_check", "ShadowNLI", "load_manifest"] diff --git a/arborist/qa/nli/manifest.json b/arborist/qa/nli/manifest.json new file mode 100644 index 0000000..03527f8 --- /dev/null +++ b/arborist/qa/nli/manifest.json @@ -0,0 +1,24 @@ +{ + "_comment": "Pinned NLI checkpoint for the #000049 Phase-2 shadow path. SHADOW MODE ONLY: this manifest does NOT yet fold into governance_policy_hash because shadow output cannot touch audit_mode (ticket #000049 §7 #2). Phase 3 (demotion-only runtime) MUST add a single-blob checkpoint_sha256 + tokenizer_sha256 + an explicit nli_policy_hash that folds into governance_policy_hash before any audit_mode effect. The benched checkpoints are not pre-exported to ONNX; a Phase-3 step should ONNX-export the chosen one and switch the [nli] extra to onnxruntime-cpu to drop the torch dependency (cf. the [vec] extra).", + "nli_model_version": "nli-shadow-v1-minilm2-l6-h768", + "hf_repo": "cross-encoder/nli-MiniLM2-L6-H768", + "pinned_revision": "b95119ce93d3e065de6214e38cd4a97b0f2f2c6d", + "license": "Apache-2.0", + "source_url": "https://huggingface.co/cross-encoder/nli-MiniLM2-L6-H768", + "approx_mb": 90, + "onnx": false, + "runtime": "transformers-torch-cpu", + "label_map_source": "read from model.config.id2label at load time (cross-encoder ordering is contradiction/entailment/neutral; do not hardcode)", + "max_length": 256, + "truncation_policy": "source_clause_then_claim@v1", + "pair_serialization": "nli_pair@v1", + "thresholds": { + "contradiction_veto": 0.5, + "entailment_block_veto": 0.9 + }, + "thresholds_provenance": "the bench-validated operating point — arborist-nli-bench commit a1cb28d, ticket #000049 §7 #18; on the 28+26-case synth set this point gives 28/28 catch, 0/26 false positives, and the entailment guard is what filters the spurious competing-superlative contradictions", + "alternates": [ + {"nli_model_version": "nli-shadow-v1-deberta-v3-base", "hf_repo": "MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli", "pinned_revision": "6f5cf0a2b59cabb106aca4c287eed12e357e90eb", "license": "MIT", "approx_mb": 440, "note": "the threshold-equivalent base candidate; ~5x slower than MiniLM on CPU"}, + {"nli_model_version": "nli-shadow-v1-bart-large-mnli", "hf_repo": "facebook/bart-large-mnli", "license": "MIT", "approx_mb": 1600, "note": "the threshold-robust reference (clean separation with no entailment guard); 407M params, heaviest"} + ] +} diff --git a/arborist/qa/nli/shadow.py b/arborist/qa/nli/shadow.py new file mode 100644 index 0000000..a74c752 --- /dev/null +++ b/arborist/qa/nli/shadow.py @@ -0,0 +1,183 @@ +"""Clause-level NLI shadow check — #000049 §7 #5 algorithm, Phase 2. + +For a (claim, source) pair the lexical verifier resolved only *weakly* +(paraphrase path, or an entity path with one verified unit), run a +sentence-pair NLI model over each source clause and report whether a +*demotion-only contradiction veto* would fire: + + would_demote iff max_contradiction >= theta_contra + and max_entailment < theta_entail + +The second conjunct (the entailment guard) is load-bearing — a source +can carry both a clause that loosely contradicts the claim and a clause +that directly entails it (the competing-superlative case from the +#000049 §7 #18 bench); the guard suppresses the veto when some clause +entails. Defaults theta_contra=0.5, theta_entail=0.9 are the +bench-validated point (arborist-nli-bench a1cb28d). + +This is SHADOW infrastructure: the result is telemetry, never an +``audit_mode`` input. The model runtime is the optional ``[nli]`` +extra; everything degrades to ``available=False`` when it is missing. +""" +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, asdict +from pathlib import Path +from typing import Optional + +_MANIFEST_PATH = Path(__file__).resolve().parent / "manifest.json" +_SENT_SPLIT = re.compile(r"(?<=[.!?;])\s+") + + +def load_manifest(path: Optional[Path] = None) -> dict: + return json.loads((path or _MANIFEST_PATH).read_text()) + + +def clauses(text: str) -> list[str]: + """Split source text into clauses (the §7 #5 step-1 segmenter). + + Standalone here (a sibling-repo / bench-callable surface) rather than + importing ``parse_claims``' pointer-aware segmenter — the shadow + check runs on raw retrieved context, which has no pointer markup. + """ + parts = [p.strip() for p in _SENT_SPLIT.split((text or "").strip()) if p.strip()] + return parts or ([text.strip()] if text and text.strip() else []) + + +@dataclass +class ShadowResult: + available: bool # False iff the [nli] extra / model could not load + would_demote: bool # the §7 #5 Demote() decision (shadow — not applied) + max_contradiction: float + max_entailment: float + best_clause: Optional[str] # the clause with the highest contradiction prob + n_clauses: int + model_version: Optional[str] + theta_contra: float + theta_entail: float + reason: str = "" # human note ("ok", "deps_missing: …", "load_failed: …") + + def as_dict(self) -> dict: + return asdict(self) + + +def _softmax(xs: list[float]) -> list[float]: + import math + m = max(xs) + es = [math.exp(x - m) for x in xs] + s = sum(es) + return [e / s for e in es] + + +def _resolve_label_indices(id2label: dict) -> tuple[int, int, int]: + inv: dict[str, int] = {} + for i, lab in id2label.items(): + low = str(lab).lower() + if "entail" in low: + inv["e"] = int(i) + elif "neutral" in low: + inv["n"] = int(i) + elif "contra" in low: + inv["c"] = int(i) + missing = {"e", "n", "c"} - set(inv) + if missing: + raise RuntimeError(f"cannot map NLI labels from id2label={id2label!r} (missing {missing})") + return inv["e"], inv["n"], inv["c"] + + +class ShadowNLI: + """Lazily-loaded pinned NLI model + the §7 #5 clause-level check. + + Construction never raises: if the ``[nli]`` extra is missing or the + pinned checkpoint will not load, :attr:`available` stays False and + :meth:`check` returns an unavailable :class:`ShadowResult`. + """ + + def __init__(self, manifest: Optional[dict] = None): + self.manifest = manifest or load_manifest() + self.model_version: Optional[str] = self.manifest.get("nli_model_version") + th = self.manifest.get("thresholds", {}) + self.theta_contra: float = float(th.get("contradiction_veto", 0.5)) + self.theta_entail: float = float(th.get("entailment_block_veto", 0.9)) + self.max_length: int = int(self.manifest.get("max_length", 256)) + self.available = False + self._reason = "uninitialised" + self._tok = None + self._model = None + self._ei = self._ni = self._ci = None + + def _ensure_loaded(self) -> None: + if self.available or self._reason.startswith(("deps_missing", "load_failed")): + return + try: + import torch # noqa: F401 + from transformers import AutoTokenizer, AutoModelForSequenceClassification + except ImportError as e: + self._reason = f"deps_missing: {e} (install: pip install 'arborist[nli]')" + return + try: + repo = self.manifest["hf_repo"] + rev = self.manifest.get("pinned_revision") + self._tok = AutoTokenizer.from_pretrained(repo, revision=rev) + self._model = AutoModelForSequenceClassification.from_pretrained(repo, revision=rev) + self._model.eval() + id2label = {int(k): v for k, v in self._model.config.id2label.items()} + self._ei, self._ni, self._ci = _resolve_label_indices(id2label) + except Exception as e: # noqa: BLE001 — any load failure is "unavailable" + self._reason = f"load_failed: {type(e).__name__}: {e}" + self._tok = self._model = None + return + self.available = True + self._reason = "ok" + + def _nli(self, premise: str, hypothesis: str) -> tuple[float, float, float]: + import torch + enc = self._tok(premise, hypothesis, return_tensors="pt", truncation=True, max_length=self.max_length) + with torch.no_grad(): + logits = self._model(**enc).logits[0].tolist() + p = _softmax(logits) + return p[self._ei], p[self._ni], p[self._ci] + + def check(self, claim: str, source: str) -> ShadowResult: + cls = clauses(source) + self._ensure_loaded() + if not self.available: + return ShadowResult(available=False, would_demote=False, max_contradiction=0.0, + max_entailment=0.0, best_clause=None, n_clauses=len(cls), + model_version=self.model_version, theta_contra=self.theta_contra, + theta_entail=self.theta_entail, reason=self._reason) + if not cls or not (claim or "").strip(): + return ShadowResult(available=True, would_demote=False, max_contradiction=0.0, + max_entailment=0.0, best_clause=None, n_clauses=len(cls), + model_version=self.model_version, theta_contra=self.theta_contra, + theta_entail=self.theta_entail, reason="empty_input") + max_e = max_n = max_c = 0.0 + best_clause = None + for cl in cls: + pe, pn, pc = self._nli(cl, claim) + max_e = max(max_e, pe) + max_n = max(max_n, pn) + if pc > max_c: + max_c = pc + best_clause = cl + would_demote = (max_c >= self.theta_contra) and (max_e < self.theta_entail) + return ShadowResult(available=True, would_demote=would_demote, + max_contradiction=round(max_c, 4), max_entailment=round(max_e, 4), + best_clause=best_clause, n_clauses=len(cls), + model_version=self.model_version, theta_contra=self.theta_contra, + theta_entail=self.theta_entail, reason="ok") + + +_DEFAULT: Optional[ShadowNLI] = None + + +def shadow_check(claim: str, source: str) -> ShadowResult: + """Module-level convenience: run :meth:`ShadowNLI.check` on a shared, + lazily-built model. Safe to call regardless of whether ``[nli]`` is + installed (returns ``available=False`` in that case).""" + global _DEFAULT + if _DEFAULT is None: + _DEFAULT = ShadowNLI() + return _DEFAULT.check(claim, source) diff --git a/bench/results/nli-shadow-sweep.json b/bench/results/nli-shadow-sweep.json new file mode 100644 index 0000000..26f9e15 --- /dev/null +++ b/bench/results/nli-shadow-sweep.json @@ -0,0 +1,1909 @@ +{ + "generated_at": "2026-05-12T18:11:51Z", + "available": true, + "reason": "ok", + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "n_records": 116, + "n_available": 116, + "elapsed_seconds": 14.3, + "would_demote_total": 54, + "would_demote_rate": 0.4655, + "by_bucket": { + "HYBRID_ENTITY": { + "n": 2, + "would_demote": 1, + "rate": 0.5 + }, + "STRICT_PARAPHRASE": { + "n": 1, + "would_demote": 0, + "rate": 0.0 + }, + "STRICT_SPAN": { + "n": 9, + "would_demote": 0, + "rate": 0.0 + }, + "UNGROUNDED": { + "n": 50, + "would_demote": 25, + "rate": 0.5 + }, + "contradiction": { + "n": 28, + "would_demote": 28, + "rate": 1.0 + }, + "not_contradiction": { + "n": 26, + "would_demote": 0, + "rate": 0.0 + } + }, + "false_positive_probe": { + "n": 26, + "would_demote": 0, + "rate": 0.0, + "note": "would_demote on records labeled want=not_contradiction \u2014 these are shadow FALSE POSITIVES; this is \u00a77 #12 gate item 4 when the input is a real legit-answer sample" + }, + "rows": [ + { + "id": "5f-fal-hard-001", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-hard-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0066, + "max_entailment": 0.9861, + "best_clause": "Penicillin was discovered by Alexander Fleming in 1928.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-hard-002", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-hard-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9977, + "max_entailment": 0.0004, + "best_clause": "The Eiffel Tower is in Paris, designed by Gustave Eiffel.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-hard-003", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-hard-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9926, + "max_entailment": 0.0069, + "best_clause": "Mercury is the smallest.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-hard-004", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-hard-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9932, + "max_entailment": 0.0011, + "best_clause": "Water boils at 100 degrees Celsius at sea level.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-hard-005", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-hard-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9956, + "max_entailment": 0.0017, + "best_clause": "Charles Darwin developed evolution.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-hard-006", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-hard-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9793, + "max_entailment": 0.0091, + "best_clause": "The French Revolution began in 1789, with the storming of the Bastille.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-hard-007", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-hard-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0506, + "max_entailment": 0.9136, + "best_clause": "The speed of light is approximately 300,000 kilometers per second in a vacuum.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-hard-008", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-hard-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9954, + "max_entailment": 0.0007, + "best_clause": "The Titanic sank in the North Atlantic Ocean in 1912.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-hard-009", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-hard-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9897, + "max_entailment": 0.0118, + "best_clause": "Shakespeare wrote Hamlet.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-hard-010", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-hard-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.4814, + "max_entailment": 0.0844, + "best_clause": "The mitochondria is the powerhouse of the cell.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-hard-011", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-hard-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9966, + "max_entailment": 0.4478, + "best_clause": "The capital of Australia is Canberra;", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-hard-012", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-hard-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9954, + "max_entailment": 0.0038, + "best_clause": "Gold has the chemical symbol Au;", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-001", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0131, + "max_entailment": 0.3806, + "best_clause": "Some completely unrelated context.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-002", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.986, + "max_entailment": 0.0022, + "best_clause": "Mount Everest is the tallest mountain at 8849 meters.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-003", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0066, + "max_entailment": 0.9861, + "best_clause": "Penicillin was discovered by Alexander Fleming in 1928.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-004", + "bucket": "STRICT_SPAN", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0014, + "max_entailment": 0.9847, + "best_clause": "Water boils at 100 degrees Celsius at sea level.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-005", + "bucket": "STRICT_SPAN", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0008, + "max_entailment": 0.9919, + "best_clause": "Hydrogen has atomic number 1 and is the lightest chemical element.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-006", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9561, + "max_entailment": 0.0096, + "best_clause": "There are eight planets in the solar system since Pluto's reclassification.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-007", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9893, + "max_entailment": 0.0031, + "best_clause": "DNA has a double helix structure first described by Watson and Crick.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-008", + "bucket": "STRICT_SPAN", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0011, + "max_entailment": 0.9908, + "best_clause": "Gold has the chemical symbol Au and atomic number 79.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-009", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9945, + "max_entailment": 0.0005, + "best_clause": "Intel released a new CPU yesterday.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-010", + "bucket": "STRICT_SPAN", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.001, + "max_entailment": 0.9845, + "best_clause": "The Pacific Ocean is the largest ocean on Earth.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-011", + "bucket": "STRICT_SPAN", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0009, + "max_entailment": 0.9931, + "best_clause": "Sputnik 1 was the first artificial satellite launched in 1957 by the Soviet Union.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-012", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.996, + "max_entailment": 0.0003, + "best_clause": "The capital of France is Paris.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-013", + "bucket": "STRICT_SPAN", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0014, + "max_entailment": 0.9779, + "best_clause": "Plain text matching context exactly.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-014", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0035, + "max_entailment": 0.9891, + "best_clause": "Penicillin was discovered by Alexander Fleming in 1928.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-015", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0038, + "max_entailment": 0.9671, + "best_clause": "Cuneiform is one of the earliest known writing systems originating in ancient Sumer.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-016", + "bucket": "HYBRID_ENTITY", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0076, + "max_entailment": 0.9787, + "best_clause": "Mount Everest is the tallest mountain on Earth at about 8849 meters.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-017", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0039, + "max_entailment": 0.977, + "best_clause": "DNA has a double helix structure first described by Watson and Crick.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-018", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0018, + "max_entailment": 0.9887, + "best_clause": "Hydrogen has atomic number 1 and is the lightest chemical element.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-019", + "bucket": "STRICT_SPAN", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0008, + "max_entailment": 0.9915, + "best_clause": "There are eight planets in the solar system.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-020", + "bucket": "STRICT_PARAPHRASE", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0009, + "max_entailment": 0.9843, + "best_clause": "Mandarin Chinese is the most widely spoken native language in the world.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-021", + "bucket": "STRICT_SPAN", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0015, + "max_entailment": 0.9893, + "best_clause": "The Python ast module parses source code into an abstract syntax tree.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-022", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0063, + "max_entailment": 0.9785, + "best_clause": "Golden retrievers are dogs which are mammals.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-023", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.4943, + "max_entailment": 0.0332, + "best_clause": "The Pacific Ocean is the largest ocean.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-024", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9936, + "max_entailment": 0.0023, + "best_clause": "AMD released a new GPU yesterday.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-025", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0123, + "max_entailment": 0.9577, + "best_clause": "Pluto was reclassified as a dwarf planet in 2006.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-026", + "bucket": "HYBRID_ENTITY", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9958, + "max_entailment": 0.0011, + "best_clause": "The Nile is one of the longest rivers in the world at about 6650 kilometers.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-027", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.99, + "max_entailment": 0.0006, + "best_clause": "Sputnik 1 was launched in 1957.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-028", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.998, + "max_entailment": 0.0005, + "best_clause": "The Eiffel Tower is in Paris France.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-029", + "bucket": "STRICT_SPAN", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0011, + "max_entailment": 0.9933, + "best_clause": "Pride and Prejudice was published in 1813.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-030", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9972, + "max_entailment": 0.0003, + "best_clause": "The capital of France is Paris.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-031", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9788, + "max_entailment": 0.0011, + "best_clause": "Some context.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-032", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0191, + "max_entailment": 0.3648, + "best_clause": "Paris is the capital of France.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-033", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0023, + "max_entailment": 0.0033, + "best_clause": "Tigers are large mammals.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-034", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9795, + "max_entailment": 0.001, + "best_clause": "Some context.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-035", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0193, + "max_entailment": 0.4569, + "best_clause": "Paris is the capital of France.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-036", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0022, + "max_entailment": 0.0039, + "best_clause": "Tigers are large mammals.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-037", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9784, + "max_entailment": 0.0011, + "best_clause": "Some context.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-038", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0194, + "max_entailment": 0.3792, + "best_clause": "Paris is the capital of France.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-039", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0023, + "max_entailment": 0.0039, + "best_clause": "Tigers are large mammals.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-040", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9834, + "max_entailment": 0.0012, + "best_clause": "Some context.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-041", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0208, + "max_entailment": 0.3534, + "best_clause": "Paris is the capital of France.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-042", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0022, + "max_entailment": 0.0034, + "best_clause": "Tigers are large mammals.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-043", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.98, + "max_entailment": 0.0009, + "best_clause": "Some context.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-044", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0225, + "max_entailment": 0.4006, + "best_clause": "Paris is the capital of France.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-045", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0023, + "max_entailment": 0.0036, + "best_clause": "Tigers are large mammals.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-046", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9805, + "max_entailment": 0.0011, + "best_clause": "Some context.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-047", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0225, + "max_entailment": 0.4071, + "best_clause": "Paris is the capital of France.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-048", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0023, + "max_entailment": 0.0035, + "best_clause": "Tigers are large mammals.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-049", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.98, + "max_entailment": 0.0012, + "best_clause": "Some context.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "5f-fal-live-050", + "bucket": "UNGROUNDED", + "is_fp_probe": false, + "src_file": "falsification-live-v1.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0208, + "max_entailment": 0.3305, + "best_clause": "Paris is the capital of France.", + "n_clauses": 1, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-001", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9975, + "max_entailment": 0.0066, + "best_clause": "Mercury is the smallest planet in the Solar System.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-002", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.997, + "max_entailment": 0.0017, + "best_clause": "Charles Darwin developed the theory of evolution.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-003", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9806, + "max_entailment": 0.1759, + "best_clause": "The Nile is the longest river in Africa.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-004", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9922, + "max_entailment": 0.713, + "best_clause": "Mount Everest is the highest mountain above sea level.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-005", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9967, + "max_entailment": 0.0013, + "best_clause": "Christopher Marlowe wrote Doctor Faustus.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-006", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9956, + "max_entailment": 0.0018, + "best_clause": "Python was created by Guido van Rossum.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-007", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9972, + "max_entailment": 0.0007, + "best_clause": "The Arctic Ocean is the smallest ocean.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-008", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9968, + "max_entailment": 0.0011, + "best_clause": "Beijing is the capital of China.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-009", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9944, + "max_entailment": 0.003, + "best_clause": "Usain Bolt holds the world record in the 100 metres.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-010", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9853, + "max_entailment": 0.0071, + "best_clause": "The Congo Basin contains the second largest tropical rainforest.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-011", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9965, + "max_entailment": 0.0018, + "best_clause": "Sandro Botticelli painted The Birth of Venus.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-012", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9936, + "max_entailment": 0.0102, + "best_clause": "The cheetah is the fastest land animal.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-013", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.998, + "max_entailment": 0.0616, + "best_clause": "Africa is the hottest continent.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-014", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9956, + "max_entailment": 0.0023, + "best_clause": "Mozart composed The Magic Flute.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-015", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9941, + "max_entailment": 0.0055, + "best_clause": "The blue whale is the largest animal ever known to have lived.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-016", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9979, + "max_entailment": 0.0135, + "best_clause": "Russia is the largest country by area.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-017", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9931, + "max_entailment": 0.31, + "best_clause": "Pierre Curie won a Nobel Prize in Physics.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-018", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9948, + "max_entailment": 0.026, + "best_clause": "Charles Babbage designed the Analytical Engine.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-019", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9748, + "max_entailment": 0.0374, + "best_clause": "Mercury, the closest planet to the Sun, is the smallest.", + "n_clauses": 3, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-020", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9802, + "max_entailment": 0.0096, + "best_clause": "Marie Curie was awarded the Nobel Prize in Chemistry in 1911 for her discovery of radium and polonium.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-021", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9445, + "max_entailment": 0.0153, + "best_clause": "The Great Wall of China is the longest wall ever built, stretching thousands of kilometres.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-022", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9483, + "max_entailment": 0.2615, + "best_clause": "In computing history, Linux was originally created by Linus Torvalds in 1991.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-023", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9531, + "max_entailment": 0.1426, + "best_clause": "Venus is the hottest planet in the Solar System owing to its dense carbon-dioxide atmosphere.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-024", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9916, + "max_entailment": 0.0024, + "best_clause": "The Burj Khalifa in Dubai stands 828 metres tall, making it the tallest building in the world.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-025", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9898, + "max_entailment": 0.0011, + "best_clause": "His contemporary William Makepeace Thackeray wrote Vanity Fair.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-026", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.7589, + "max_entailment": 0.4461, + "best_clause": "Antarctica, though icy, is technically the largest desert of any kind because of its extremely low precipitation.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-027", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9832, + "max_entailment": 0.0051, + "best_clause": "Jupiter has the largest number of confirmed moons of any planet.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "recomb-028", + "bucket": "contradiction", + "is_fp_probe": false, + "src_file": "recombination.jsonl", + "available": true, + "would_demote": true, + "max_contradiction": 0.9849, + "max_entailment": 0.0071, + "best_clause": "The peregrine falcon, in a hunting dive, exceeds 320 km/h, making it the fastest animal of any kind.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-001", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.009, + "max_entailment": 0.0461, + "best_clause": "Batman lives in Gotham City.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-002", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0038, + "max_entailment": 0.0033, + "best_clause": "Marie Curie was a physicist and chemist.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-003", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0033, + "max_entailment": 0.1003, + "best_clause": "The Eiffel Tower is located in Paris.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-004", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.003, + "max_entailment": 0.0031, + "best_clause": "Albert Einstein was born in Germany.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-005", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0021, + "max_entailment": 0.0123, + "best_clause": "Mount Everest is the highest mountain above sea level.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-006", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0136, + "max_entailment": 0.2455, + "best_clause": "Python is a programming language.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-007", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0025, + "max_entailment": 0.0034, + "best_clause": "The Nile flows through Egypt.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-008", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0041, + "max_entailment": 0.1253, + "best_clause": "Shakespeare was an English playwright.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-009", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0025, + "max_entailment": 0.0183, + "best_clause": "Tokyo is the capital of Japan.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-010", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0056, + "max_entailment": 0.1995, + "best_clause": "The blue whale is a marine mammal.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-011", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0234, + "max_entailment": 0.0181, + "best_clause": "Leonardo da Vinci was an Italian polymath.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-012", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0024, + "max_entailment": 0.0145, + "best_clause": "The Amazon River is in South America.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-013", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.002, + "max_entailment": 0.0046, + "best_clause": "Jupiter is a gas giant.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-014", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0019, + "max_entailment": 0.018, + "best_clause": "Usain Bolt is a Jamaican sprinter.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-015", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0463, + "max_entailment": 0.22, + "best_clause": "Antarctica is a continent.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-016", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0058, + "max_entailment": 0.0055, + "best_clause": "Mozart was an Austrian composer.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-017", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0216, + "max_entailment": 0.0232, + "best_clause": "Vatican City is an independent city-state.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-018", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0053, + "max_entailment": 0.014, + "best_clause": "The cheetah is a big cat.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-019", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0068, + "max_entailment": 0.0045, + "best_clause": "It was completed in 1889 for the World's Fair.", + "n_clauses": 3, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-020", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0082, + "max_entailment": 0.0031, + "best_clause": "She conducted pioneering research on radioactivity.", + "n_clauses": 3, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-021", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0335, + "max_entailment": 0.002, + "best_clause": "It is set in Denmark.", + "n_clauses": 3, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-022", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.2009, + "max_entailment": 0.9769, + "best_clause": "It is a gas giant composed mainly of hydrogen and helium.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-023", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0421, + "max_entailment": 0.0084, + "best_clause": "It is located in Dubai.", + "n_clauses": 3, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-024", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.0636, + "max_entailment": 0.0232, + "best_clause": "He also created the Git version-control system in 2005.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-025", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.1172, + "max_entailment": 0.9792, + "best_clause": "Antarctica is the largest desert overall.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + }, + { + "id": "legit-026", + "bucket": "not_contradiction", + "is_fp_probe": true, + "src_file": "legit_summary.jsonl", + "available": true, + "would_demote": false, + "max_contradiction": 0.9458, + "max_entailment": 0.9732, + "best_clause": "The peregrine falcon is the fastest animal in the world when diving.", + "n_clauses": 2, + "model_version": "nli-shadow-v1-minilm2-l6-h768", + "theta_contra": 0.5, + "theta_entail": 0.9, + "reason": "ok" + } + ] +} \ No newline at end of file diff --git a/bench/scripts/nli_shadow_sweep.py b/bench/scripts/nli_shadow_sweep.py new file mode 100644 index 0000000..e64cda6 --- /dev/null +++ b/bench/scripts/nli_shadow_sweep.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""NLI shadow sweep — #000049 Phase 2 / §7 #12 gate item 4 instrument. + +Runs the clause-level shadow check (``arborist.qa.nli.shadow_check`` — +the §7 #5 algorithm) over a set of (answer, context) records and reports +the *would-demote* rate, bucketed by the verifier label the record +carries. SHADOW ONLY: writes nothing to any shard, touches no +``audit_mode``. + +Two record shapes are accepted (auto-detected per line): + - 5f-fixture shape: ``{"answer_text": …, "context": …, + "expected_reason": …}`` (e.g. ``bench/fixtures/5f/*.jsonl``; + ``_meta`` lines are skipped). ``expected_reason`` is the bucket. + - nli-bench eval shape: ``{"claim": …, "source": …, "want": …}`` + (e.g. ``~/git/arborist-nli-bench/eval/*.jsonl``). ``want`` is the + bucket; ``want="not_contradiction"`` rows are the false-positive + probe — any would_demote on those is a shadow FP. + +With no ``--input`` it sweeps the 5f falsification packs that have +``answer_text``/``context`` columns. Requires the ``[nli]`` extra to +produce real numbers — without it the report still renders, marked +``available: false`` (so the harness/CI never breaks). + +Usage: + python3 bench/scripts/nli_shadow_sweep.py + python3 bench/scripts/nli_shadow_sweep.py --input path/to/records.jsonl --input more.jsonl + python3 bench/scripts/nli_shadow_sweep.py --out bench/results/nli-shadow-sweep.json +""" +from __future__ import annotations + +import argparse +import json +import sys +import time +from collections import defaultdict +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +_DEFAULT_INPUTS = [ + REPO / "bench" / "fixtures" / "5f" / "falsification-hard-v1.jsonl", + REPO / "bench" / "fixtures" / "5f" / "falsification-live-v1.jsonl", + REPO / "bench" / "fixtures" / "5f" / "falsification-v1.jsonl", +] + + +def _records(path: Path): + for ln in path.read_text().splitlines(): + ln = ln.strip() + if not ln: + continue + obj = json.loads(ln) + if "_meta" in obj: + continue + claim = obj.get("answer_text") or obj.get("claim") + source = obj.get("context") or obj.get("source") + if not claim or not source: + continue + bucket = obj.get("expected_reason") or obj.get("want") or "unlabeled" + is_fp_probe = (obj.get("want") == "not_contradiction") + yield {"id": obj.get("id", path.stem), "claim": claim, "source": source, + "bucket": bucket, "is_fp_probe": is_fp_probe, "src_file": path.name} + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--input", action="append", type=Path, help="JSONL record file(s); repeatable") + ap.add_argument("--out", type=Path, default=REPO / "bench" / "results" / "nli-shadow-sweep.json") + args = ap.parse_args(argv) + + inputs = args.input or [p for p in _DEFAULT_INPUTS if p.exists()] + recs = [] + for p in inputs: + if not p.exists(): + print(f"[nli-shadow] skip (missing): {p}", file=sys.stderr) + continue + n0 = len(recs) + recs.extend(_records(p)) + print(f"[nli-shadow] {p.name}: {len(recs) - n0} records", flush=True) + if not recs: + print("[nli-shadow] no usable records", file=sys.stderr) + return 2 + + from arborist.qa.nli import ShadowNLI + nli = ShadowNLI() + + by_bucket: dict[str, dict] = defaultdict(lambda: {"n": 0, "would_demote": 0}) + fp_probe = {"n": 0, "would_demote": 0} + rows = [] + n_available = 0 + t0 = time.time() + for r in recs: + res = nli.check(r["claim"], r["source"]) + if res.available: + n_available += 1 + b = by_bucket[r["bucket"]] + b["n"] += 1 + if res.would_demote: + b["would_demote"] += 1 + if r["is_fp_probe"]: + fp_probe["n"] += 1 + if res.would_demote: + fp_probe["would_demote"] += 1 + rows.append({**{k: r[k] for k in ("id", "bucket", "is_fp_probe", "src_file")}, + **res.as_dict()}) + elapsed = time.time() - t0 + + available = n_available > 0 + reason = "ok" if available else (rows[0]["reason"] if rows else "no rows") + total_demote = sum(b["would_demote"] for b in by_bucket.values()) + report = { + "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "available": available, + "reason": reason, + "model_version": nli.model_version, + "theta_contra": nli.theta_contra, + "theta_entail": nli.theta_entail, + "n_records": len(recs), + "n_available": n_available, + "elapsed_seconds": round(elapsed, 1), + "would_demote_total": total_demote, + "would_demote_rate": round(total_demote / len(recs), 4) if recs else 0.0, + "by_bucket": {k: {**v, "rate": round(v["would_demote"] / v["n"], 4) if v["n"] else 0.0} + for k, v in sorted(by_bucket.items())}, + "false_positive_probe": {**fp_probe, + "rate": round(fp_probe["would_demote"] / fp_probe["n"], 4) if fp_probe["n"] else None, + "note": "would_demote on records labeled want=not_contradiction — these are shadow FALSE POSITIVES; this is §7 #12 gate item 4 when the input is a real legit-answer sample"}, + "rows": rows, + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2)) + print(f"[nli-shadow] wrote {args.out}") + + print("\n" + "=" * 72) + print("NLI SHADOW SWEEP — #000049 Phase 2 (would-demote rate, shadow only)") + print("=" * 72) + if not available: + print(f" model UNAVAILABLE — {reason}") + print(" (install the runtime to get real numbers: pip install 'arborist[nli]')") + print(f" swept {len(recs)} records structurally; would-demote rate not measured.") + return 0 + print(f" model {nli.model_version} · θc={nli.theta_contra} θe={nli.theta_entail} · {len(recs)} records · {elapsed:.1f}s") + print(f" would_demote overall: {total_demote}/{len(recs)} = {report['would_demote_rate']:.3f}") + print(f" {'bucket':<28} {'n':>5} {'would_demote':>13} {'rate':>7}") + print(" " + "-" * 56) + for k, v in report["by_bucket"].items(): + print(f" {k:<28} {v['n']:>5} {v['would_demote']:>13} {v['rate']:>7.3f}") + print(" " + "-" * 56) + fpp = report["false_positive_probe"] + if fpp["n"]: + print(f" false-positive probe (want=not_contradiction): {fpp['would_demote']}/{fpp['n']} = {fpp['rate']:.3f} ← lower is better; 0 clears §7 #12 item 2") + else: + print(" (no want=not_contradiction rows in this input — point --input at a legit-answer sample for the real gate-item-4 number)") + print(" NB: on falsification-* packs every record is a FALSE claim, so a high would_demote rate there is the model working as intended — not a false positive.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 78ff1f1..d9604a2 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -103,7 +103,7 @@ Newest first. Update on every open/close. |----------|------------------------------------------------|-----------------------|------------|-----------| | #000051 | Federated vecpack distribution (gossip the embedding backfill) | open · awaiting go/no-go · doc-only scaffold. Makes `chunk_vecs` a distributable artifact: backfill once on any CPU box (cloud / Prometheus-Σ sweep — #000037 §3.1), publish a **vecpack** `(shard_root, vec_backend_version, [(leaf_hash, embedding_blob)…])` over the mesh wire layer, every peer pulls + bulk-loads (sub-ms/chunk on the receiver — the laptop never runs the transformer). Keyed on `leaf_hash` (portable) not `chunk_id` (shard-local). Vecpacks are **soft data** — embeddings are `UNGROUNDED`, never proof path — so a cheap structural sanity gate (chunk exists locally w/ matching leaf_hash, right blob length for (dim,quant), finite norm, backend_version matches) suffices, no Merkle-proof-grade verification needed. Supplies #000050's prereq #1 ("a vecpack exists & is imported on the bench box", not "fox embedded the corpus locally"). GPU producer (the fast path): bge-small-en-v1.5 batched on a CUDA box (4090) ≈ 10³–10⁴ chunks/s → full 6.24M-chunk corpus in *minutes*, not days — drop a CUDA `Embedder` into `default_embedder()`; CUDA stack lives only on the producer box, never in arborist's `python+sqlite3` core. The mechanism behind whitepaper §1's "the embedding pass runs off the device". #000039 / #000050 sibling | 2026-05-12 | — | | #000050 | Vec RRF hybrid fusion (#000039 Phase 2) | open · awaiting go/no-go · doc-only scaffold; design in #000039 §4.2 (RRF) + §8 (the gate). Wire `VecBackend` as a 5th retrieval route in `query.py`, RRF-merged (route provenance carried) with the 4 FTS5 routes; UNGROUNDED hits, additive not replacement. Phase-2 sub-items now explicit: **accept-path-5** in `_filter_by_title_relevance` (low-title-overlap vec hits survive only via a stronger span-level warrant, never similarity-score alone — else the title gate drops exactly the semantic candidates vec exists for & the bench shows no lift); **six** vec config fields fold into `governance_policy_hash` (recipe-named quant `int8sym`) **+ a cache-write guard** blocking `providence_cache` persistence for vec/hybrid runs until that's wired; **run-DAG records the vec stage** (backend version, six fields, top_k, query-embedding hash, candidate chunk_ids+distances). **Gated** on (a) a corpus backfill **distributed via #000051** AND (b) a **four-condition** recall bench (A FTS5-only / B vec-only / C RRF hybrid / D candidate-union-no-RRF) clearing the 5pp floor incl. C-beats-D, on semantic-allusion + curated + **adversarial-semantic-neighbor** fixtures (else park, vec stays opt-in `--backend vec`; if C≈D ship the union, drop RRF). #000039 follow-up | 2026-05-12 | — | -| #000049 | Attribution-aware grounding check (the recombination boundary) | open · boundary accepted · production no-go · shadow-path approved (de novo review 2026-05-13 — ticket §7) · doc-only; the home for #000048's deferred §2.3 — closing the 2 recombination over-grounds in `falsification-hard` (hard-003 Mercury / hard-005 Einstein) needs an attribution / dependency-parse or mini-NLI check, which is *not lexical* (#000048 §5). Discipline question answered: a small fixed purpose-built NLI/entailment *model* may influence `audit_mode` only as an opt-in, hash-pinned, governance-hashed, **demotion-only contradiction veto** after shadow-mode evidence (never promotes — `MODEL_ASSISTED_DEMOTION`, never `MODEL_ASSISTED_PROMOTION`). Production verifier unchanged; `falsification-hard` stays 10/12 as an honest boundary marker. Roadmap: Phase 0 (this amendment) → Phase 1 (shadow design: NLI manifest, fetch/verify, `nli_pair@v1` canonicalization, recombination-risk trigger) → Phase 2 (bench-only shadow impl, `[nli]` extra, `make fetch-nli`) → Phase 3 (demotion-only runtime, gated) → Phase 4 (mesh blob sync); §7 #12 six-condition bench gate required before Phases 2–4; if NLI ever affects `audit_mode`, `nli_policy_hash` folds into `governance_policy_hash`. **Phase-2 candidate bench done 2026-05-12** (`~/git/arborist-nli-bench/`, commits `829f9a4` + `a1cb28d`; ticket §7 #18): checkpoint-agnostic harness runs the §7 #5 clause-level algorithm over 28 synth recombination cases (incl. the 2 fixtures + harder shapes) + 26 legit cases (true summaries + near-miss decoys). 4 working candidates; `nli-MiniLM2-L6-H768` (82M, 45ms p50 CPU), `deberta-v3-base-mnli-fever-anli` (184M, 223ms), `bart-large-mnli` (407M, 259ms) all 28/28 catch · 0/26 FP with the standard θe=0.9 entailment guard; `cross-encoder/nli-deberta-v3-base` 27/28; deberta-large repo-id TODO. **Key finding: the §7 #5 two-threshold rule is load-bearing** — 3 of 4 candidates argmax-contradict 1/26 legit cases on the *wrong* source clause (competing-superlative confusion, e.g. "largest hot desert" vs "largest desert overall"); the entailment guard filters every one because another clause restates the claim → 0% guarded FP vs ~4% single-threshold. Picture: recombination is *easy* for any modern NLI checkpoint — differentiator is cost/robustness, MiniLM is the cost-pick, bart-large the threshold-robust pick. Open risk = real-traffic FP rate, measurable only by a shadow run on actual `bench-qa` (gate item 4, TODO). #000048 follow-up | 2026-05-12 | — | +| #000049 | Attribution-aware grounding check (the recombination boundary) | open · boundary accepted · production no-go · shadow-path approved (de novo review 2026-05-13 — ticket §7) · doc-only; the home for #000048's deferred §2.3 — closing the 2 recombination over-grounds in `falsification-hard` (hard-003 Mercury / hard-005 Einstein) needs an attribution / dependency-parse or mini-NLI check, which is *not lexical* (#000048 §5). Discipline question answered: a small fixed purpose-built NLI/entailment *model* may influence `audit_mode` only as an opt-in, hash-pinned, governance-hashed, **demotion-only contradiction veto** after shadow-mode evidence (never promotes — `MODEL_ASSISTED_DEMOTION`, never `MODEL_ASSISTED_PROMOTION`). Production verifier unchanged; `falsification-hard` stays 10/12 as an honest boundary marker. Roadmap: Phase 0 (this amendment) → Phase 1 (shadow design: NLI manifest, fetch/verify, `nli_pair@v1` canonicalization, recombination-risk trigger) → Phase 2 (bench-only shadow impl, `[nli]` extra, `make fetch-nli`) → Phase 3 (demotion-only runtime, gated) → Phase 4 (mesh blob sync); §7 #12 six-condition bench gate required before Phases 2–4; if NLI ever affects `audit_mode`, `nli_policy_hash` folds into `governance_policy_hash`. **Phase-2 candidate bench done 2026-05-12** (`~/git/arborist-nli-bench/`, commits `829f9a4` + `a1cb28d`; ticket §7 #18): checkpoint-agnostic harness runs the §7 #5 clause-level algorithm over 28 synth recombination cases (incl. the 2 fixtures + harder shapes) + 26 legit cases (true summaries + near-miss decoys). 4 working candidates; `nli-MiniLM2-L6-H768` (82M, 45ms p50 CPU), `deberta-v3-base-mnli-fever-anli` (184M, 223ms), `bart-large-mnli` (407M, 259ms) all 28/28 catch · 0/26 FP with the standard θe=0.9 entailment guard; `cross-encoder/nli-deberta-v3-base` 27/28; deberta-large repo-id TODO. **Key finding: the §7 #5 two-threshold rule is load-bearing** — 3 of 4 candidates argmax-contradict 1/26 legit cases on the *wrong* source clause (competing-superlative confusion, e.g. "largest hot desert" vs "largest desert overall"); the entailment guard filters every one because another clause restates the claim → 0% guarded FP vs ~4% single-threshold. Picture: recombination is *easy* for any modern NLI checkpoint — differentiator is cost/robustness, MiniLM is the cost-pick, bart-large the threshold-robust pick. **Phase-2 shadow scaffold landed in arborist 2026-05-12** (ticket §7 #19): `arborist/qa/nli/` (manifest pins MiniLM @ a fixed HF revision + θc 0.5/θe 0.9 + 2 alternates; `ShadowNLI`/`shadow_check` lazy-imports `transformers`+`torch` behind a new `[nli]` extra, degrades to `available=False` when absent — SHADOW ONLY, never an `audit_mode` input, manifest not yet in `governance_policy_hash` per §7 #2) + `bench/scripts/nli_shadow_sweep.py` + `make bootstrap-nli` / `make bench-nli-shadow` + 16 tests. First sweep (116 records: 5f-falsification packs + the sibling-repo eval sets): 28/28 synth recombination demoted, 0/26 FP on legit summaries, 0/9 fires on already-`STRICT_SPAN` records, 25/50 on `UNGROUNDED` (the contradiction half; quiet on non-sequiturs — correct). Gate items 1/2/3/5/6 look clear on available data; **item 4 — shadow FP rate on a real live-`bench-qa` sample — remains the one open measurement** (instrument in place; the run is slow/live, fox-decides). Production verifier unchanged; `falsification-hard` stays 10/12. #000048 follow-up | 2026-05-12 | — | | #000048 | Verifier upgrade — recombination-aware grounding + clause segmentation | **closed · 2026-05-12** — steps 2.1 + 2.4 landed 2026-05-11 (12 of 16 residual items: 4 HYBRID_ENTITY over-grounds + 8 Formulate mis-segments → `formulate-hard` 12/12, `falsification-hard` 10/12; each bench-gated, no STRICT-rate regression — 2.1's gate fired on 0 QA answers, 2.4's segmenter touched 7 of 450 lattice cells both verdict changes correct). Step 2.2 (single-clause-containment paraphrase check) attempted + reverted — catches the 2 recombination fixtures but also rejects legit cross-sentence summaries with no threshold separating the two; recombination-vs-summary isn't lexical (§5 "What we learned"). The attribution-aware path moved to **#000049** (fox 2026-05-12). 2 live-pack `expected_reason` updated HYBRID_ENTITY→UNGROUNDED; 12+ tests; `make bench-5f-falsification-hard` / `bench-5f-formulate-hard` / `bench-fork-baseline-hard`. #000046 follow-up; #000047 closed | 2026-05-11 | — | | #000047 | ForkScore `_delta_*` aggregator (mean vs max vs sum) | **closed · 2026-05-11** — Option D: `WeightSet.delta_aggregator` ∈ {`mean`,`max`,`sum`} (default `mean` unchanged → no `ESTIMATOR_VERSION` bump), `fork_score._delta_5{s,t,f}` dispatch via `_aggregate`, recorded in `ScoredFork.weights`, per-sub `HARD_REGRESSION_FLOOR` flags aggregator-independent; bench data behind keeping `mean` in `5f-threshold-calibration-2026-05-11.md` §5; 8+1 tests. #000012-revision / #000025 §10.14 follow-up | 2026-05-11 | — | | #000046 | Harder 5S/5T/5F fixture tier (below-ceiling baselines) | **closed · 2026-05-11** — Phase 1 `falsification-hard-v1.jsonl` (12 near-misses) + Phase 2 `formulate-hard-v1.jsonl` (12 mis-segments, rate 4/12) + Phase 3 `verify_quotes` paraphrase numeric-agreement gate (`_numeric_signature`; demotes a token-covering span asserting a digit-number the source lacks modulo thousands-comma) → falsification-hard rate 4/12 → 6/12 on a real change; bench-gated (`make bench-qa` n=3×75×3 before/after — no STRICT-rate regression on legit answers; only gate-caused QA shift was correctly demoting a fictional-year claim STRICT→HYBRID); `fork_score` γ·Δ5f went positive on it. Headroom now down to 2 falsification-hard over-grounds (#000048 step 2.1 closed the 4 entity over-grounds; step 2.4 closed the 8 Formulate mis-segments → that pack 12/12; step 2.2 attempted + reverted — the last 2 recombination fixtures need an attribution-aware verifier, now tracked as **#000049**, and stand as documented residue). `make bench-5f-falsification-hard` / `bench-5f-formulate-hard` / `bench-fork-baseline-hard`; 7+ tests. #000025 §10.14 follow-up; #000047 closed; #000048 closed | 2026-05-11 | — | diff --git a/docs/tickets/ticket-000049-attribution-aware-grounding-check.md b/docs/tickets/ticket-000049-attribution-aware-grounding-check.md index fa32b04..fec3dc6 100644 --- a/docs/tickets/ticket-000049-attribution-aware-grounding-check.md +++ b/docs/tickets/ticket-000049-attribution-aware-grounding-check.md @@ -1,12 +1,16 @@ # Ticket #000049 — Attribution-aware grounding check (the recombination boundary) **Status:** open · boundary accepted · production no-go · shadow-path -approved (de novo review 2026-05-13 — see §7) · Phase-2 candidate bench -done (§7 #18 — `~/git/arborist-nli-bench/`; 4 working candidates, 3 hit -28/28 catch · 0/26 FP on a 54-case synth set incl. harder shapes; -confirmed the §7 #5 entailment guard is load-bearing — filters spurious -competing-superlative contradictions; MiniLM-82M is the cost-pick; -real-QA shadow FP rate still TODO = gate item 4) +approved (de novo review 2026-05-13 — see §7) · Phase-1 + Phase-2-scaffold +landed 2026-05-12 (§7 #18/#19 — candidate bench in `~/git/arborist-nli-bench/` +[MiniLM-82M cost-pick, §7 #5 entailment guard confirmed load-bearing]; +`arborist/qa/nli/` shadow module + `[nli]` extra + `make bench-nli-shadow` +in-repo, SHADOW ONLY — never an `audit_mode` input; first sweep: +28/28 recombination demoted, 0/26 FP on legit summaries, never fires on +already-STRICT records). Remaining: §7 #12 gate item 4 — shadow FP rate +on a real live-`bench-qa` sample (instrument in place; the run is the +next step, slow/live). Production verifier unchanged; `falsification-hard` +stays 10/12. **Opened:** 2026-05-12 **Scope:** Decide whether — and if so how — to add a verifier check that catches a *recombination*: a claim whose content tokens are all @@ -641,3 +645,51 @@ two-threshold algorithm is the right shape (the entailment guard earns its place); a small cross-encoder NLI suffices; the one open risk is real-traffic false positives, measurable only by a shadow run on actual QA (Phase 2 → gate item 4). + +**19. Phase-2 shadow scaffold landed in arborist (2026-05-12).** +`arborist/qa/nli/` — SHADOW ONLY (writes nothing to `providence_cache` +/ `audit_events`, never an `audit_mode` input; per §7 #2 the manifest +does *not* yet fold into `governance_policy_hash` because shadow +output can't touch `audit_mode`). Pieces: `manifest.json` (pins +`cross-encoder/nli-MiniLM2-L6-H768` @ a fixed HF revision + the +bench-validated θc 0.5 / θe 0.9 + two alternates; carries the Phase-3 +TODO — add single-blob `checkpoint_sha256` + `tokenizer_sha256` + +`nli_policy_hash` and ONNX-export to drop torch); `shadow.py` +(`ShadowNLI` / `shadow_check` — lazy `transformers`+`torch` import +behind the new `[nli]` extra, `clauses()` segmenter, the §7 #5 +clause-level `Demote()` decision; construction never raises, degrades +to `available=False` when `[nli]` absent); `[nli]` extra in +`pyproject.toml` (hard out of core/dev — fresh checkout stays +`python3.12 + venv + sqlite3`; `make bootstrap-nli` to opt in); +`bench/scripts/nli_shadow_sweep.py` + `make bench-nli-shadow` (the +gate-item-4 instrument — sweeps `(answer, context)` records, reports +the *would-demote* rate bucketed by verifier label; renders even +without `[nli]`, marked `available:false`); 16 tests in +`tests/test_nli_shadow.py` (pure-Python parts + graceful degradation ++ the bench-sweep parser — run in the default suite). + +**First shadow sweep (2026-05-12, `nli-shadow-v1-minilm2-l6-h768`, +θc 0.5 / θe 0.9, 116 records, ~14 s CPU; `bench/results/nli-shadow-sweep.json`)** — +inputs: `bench/fixtures/5f/falsification-{hard,live}-v1.jsonl` (all +*false* claims the lexical verifier already rejects) + the +`arborist-nli-bench` 28 recombination / 26 legit-summary eval sets: + +| verifier-label bucket | n | would_demote | rate | reading | +|---|---|---|---|---| +| `contradiction` (synth recombination, incl. the 2 fixtures) | 28 | 28 | **1.00** | every recombination falsehood demoted — closes the fixtures (gate item 1) | +| `not_contradiction` (synth legit summaries + decoys) | 26 | 0 | **0.00** | zero false positives (gate item 2) | +| `STRICT_SPAN` | 9 | 0 | 0.00 | never fires on an already-STRICT-verified record (gate item 3, in this sample) | +| `STRICT_PARAPHRASE` | 1 | 0 | 0.00 | " | +| `UNGROUNDED` (5f-fal-live, all false) | 50 | 25 | 0.50 | NLI demotes the *attribution-contradiction* half, stays quiet on the *non-sequitur* half ("random claim" vs "unrelated context" is *neutral*, not contradicted — correct: NLI is a contradiction veto, not a grounding check) | +| `HYBRID_ENTITY` | 2 | 1 | 0.50 | n too small to read | + +So gate items 1, 2, 3, 5 (latency ≈ 60 ms/pair CPU, and it'd run only +on the unresolved subset), 6 (deterministic — `ShadowNLI` is) all look +clear on available data. **Item 4 — the shadow FP rate on a *real* +`bench-qa` sample (live-LLM answers that actually took the +paraphrase/entity path) — remains the one open measurement**; this +sweep used synthetic legit + the 5f packs, not real Wikipedia QA +output. The instrument is in place (`make bench-nli-shadow +INPUT="…"`); feeding it a live `bench-qa` run is the next concrete +step, and a slow/live one (fox-decides). Nothing here changes the +production verifier; `falsification-hard` stays 10/12. diff --git a/pyproject.toml b/pyproject.toml index 0ab52bb..17985c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,23 @@ vec = [ "sqlite-vec>=0.1.9", "fastembed>=0.4", ] +nli = [ + # Sentence-pair NLI for the #000049 Phase-2 *shadow* path + # (arborist/qa/nli/) — measures whether a clause-level contradiction + # veto would demote a weakly-grounded answer; never touches + # audit_mode. transformers + a CPU torch is ~600 MB installed, so it + # is gated hard out of core / dev — a fresh checkout stays + # python3.12 + venv + sqlite3, and the default test suite skips the + # NLI tests via pytest.importorskip when this extra is absent. + # Install with: + # pip install 'arborist[nli]' + # Phase 3 (if it happens) should ONNX-export the pinned checkpoint + # and switch this to onnxruntime-cpu to drop torch (cf. [vec]). + "transformers>=4.40", + "torch>=2.2", + "sentencepiece>=0.2", + "protobuf>=4.0", +] dev = [ "pytest>=8", "pytest-asyncio>=0.23", diff --git a/tests/test_nli_shadow.py b/tests/test_nli_shadow.py new file mode 100644 index 0000000..0a36167 --- /dev/null +++ b/tests/test_nli_shadow.py @@ -0,0 +1,153 @@ +"""Tests for the #000049 Phase-2 NLI shadow scaffold. + +These run in the default suite — i.e. WITHOUT the ``[nli]`` extra +installed — so they exercise the pure-Python parts (clause splitter, +manifest, label-index resolution, the dataclass contract) and the +graceful-degradation path (deps missing → ``available=False``, never an +exception). When ``[nli]`` *is* installed they also cover the real +model path; we don't assert specific probabilities (those belong in the +``arborist-nli-bench`` scorecard, not the unit suite). +""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from arborist.qa.nli import ShadowNLI, ShadowResult, shadow_check, load_manifest +from arborist.qa.nli.shadow import clauses, _resolve_label_indices + + +# --- clause splitter ------------------------------------------------------- + +def test_clauses_splits_on_sentence_punctuation(): + assert clauses("Jupiter is the largest. Mercury is the smallest.") == [ + "Jupiter is the largest.", + "Mercury is the smallest.", + ] + + +def test_clauses_handles_semicolons(): + assert clauses("A is true; B is false") == ["A is true;", "B is false"] + + +def test_clauses_single_sentence_is_one_clause(): + assert clauses("Just one sentence with no breaks") == ["Just one sentence with no breaks"] + + +def test_clauses_empty_or_blank(): + assert clauses("") == [] + assert clauses(" ") == [] + assert clauses(None) == [] # type: ignore[arg-type] + + +# --- manifest -------------------------------------------------------------- + +def test_manifest_has_required_fields(): + m = load_manifest() + for key in ("nli_model_version", "hf_repo", "pinned_revision", "license", "thresholds"): + assert key in m, key + assert "contradiction_veto" in m["thresholds"] + assert "entailment_block_veto" in m["thresholds"] + # shadow-mode thresholds are the bench-validated point + assert m["thresholds"]["contradiction_veto"] == 0.5 + assert m["thresholds"]["entailment_block_veto"] == 0.9 + + +def test_manifest_is_valid_json_file(): + p = Path(__file__).resolve().parents[1] / "arborist" / "qa" / "nli" / "manifest.json" + json.loads(p.read_text()) # raises on malformed + + +# --- label-index resolution (the cross-encoder vs MNLI vs BART orderings) -- + +@pytest.mark.parametrize("id2label,expect", [ + ({0: "entailment", 1: "neutral", 2: "contradiction"}, (0, 1, 2)), # MNLI/MoritzLaurer order + ({0: "contradiction", 1: "entailment", 2: "neutral"}, (1, 2, 0)), # cross-encoder order + ({0: "contradiction", 1: "neutral", 2: "entailment"}, (2, 1, 0)), # BART order + ({0: "ENTAILMENT", 1: "NEUTRAL", 2: "CONTRADICTION"}, (0, 1, 2)), # case-insensitive +]) +def test_resolve_label_indices(id2label, expect): + assert _resolve_label_indices(id2label) == expect + + +def test_resolve_label_indices_rejects_incomplete(): + with pytest.raises(RuntimeError): + _resolve_label_indices({0: "entailment", 1: "neutral"}) + + +# --- ShadowNLI / ShadowResult contract ------------------------------------- + +def test_shadownli_construction_never_raises_and_starts_unavailable(): + nli = ShadowNLI() + assert nli.available is False # not loaded until first check + assert nli.model_version == load_manifest()["nli_model_version"] + assert nli.theta_contra == 0.5 and nli.theta_entail == 0.9 + + +def test_check_returns_shadowresult_and_degrades_gracefully(): + res = shadow_check("Mercury is the largest planet.", + "Jupiter is the largest planet. Mercury is the smallest planet.") + assert isinstance(res, ShadowResult) + assert res.would_demote in (True, False) + assert 0.0 <= res.max_contradiction <= 1.0 + assert 0.0 <= res.max_entailment <= 1.0 + assert res.n_clauses == 2 + d = res.as_dict() + assert set(d) >= {"available", "would_demote", "max_contradiction", "max_entailment", + "best_clause", "n_clauses", "model_version", "theta_contra", + "theta_entail", "reason"} + if not res.available: + # deps-missing path: a clean, non-raising, no-signal result + assert res.would_demote is False + assert res.max_contradiction == 0.0 and res.max_entailment == 0.0 + assert "deps_missing" in res.reason or "load_failed" in res.reason + + +def test_check_empty_input_when_available_is_no_demote(): + nli = ShadowNLI() + nli._ensure_loaded() + if not nli.available: + pytest.skip("[nli] extra not installed — empty-input branch only reachable when loaded") + res = nli.check("", "Some source text. Another clause.") + assert res.available is True and res.would_demote is False and res.reason == "empty_input" + + +# --- bench sweep script ---------------------------------------------------- + +def test_shadow_sweep_parses_both_record_shapes(tmp_path): + from bench.scripts.nli_shadow_sweep import _records + p = tmp_path / "mixed.jsonl" + p.write_text("\n".join([ + json.dumps({"_meta": {"battery": "5f"}}), + json.dumps({"id": "a", "answer_text": "X is Y.", "context": "X is Z. Q is Y.", "expected_reason": "UNGROUNDED"}), + json.dumps({"id": "b", "claim": "P is Q.", "source": "P is R.", "want": "not_contradiction"}), + json.dumps({"id": "c", "answer_text": "only answer, no context"}), # incomplete → skipped + ])) + recs = list(_records(p)) + assert [r["id"] for r in recs] == ["a", "b"] + assert recs[0]["bucket"] == "UNGROUNDED" and recs[0]["is_fp_probe"] is False + assert recs[1]["bucket"] == "not_contradiction" and recs[1]["is_fp_probe"] is True + + +def test_shadow_sweep_main_smoke(tmp_path): + from bench.scripts.nli_shadow_sweep import main + inp = tmp_path / "in.jsonl" + inp.write_text("\n".join([ + json.dumps({"id": "r1", "claim": "Mercury is the largest planet.", + "source": "Jupiter is the largest planet. Mercury is the smallest planet.", + "want": "contradiction"}), + json.dumps({"id": "r2", "claim": "Batman, the alias of Bruce Wayne, lives in Gotham.", + "source": "Batman is the alias of Bruce Wayne. Batman lives in Gotham.", + "want": "not_contradiction"}), + ])) + out = tmp_path / "report.json" + rc = main(["--input", str(inp), "--out", str(out)]) + assert rc == 0 + rep = json.loads(out.read_text()) + assert rep["n_records"] == 2 + assert "by_bucket" in rep and "false_positive_probe" in rep + assert set(rep["by_bucket"]) == {"contradiction", "not_contradiction"} + # when [nli] absent the report still renders, just unmeasured + assert rep["available"] in (True, False)