candidate_clauses() — NLI now runs only on the top-N source clauses by content-token overlap with the answer claim (max_candidate_clauses=6), not the whole context; records n_candidate_clauses / best_clause_overlap / recombination_risk. Synthetic sweep unchanged (28/28 recombination, 0/26 legit FP, mean 1.45 candidate clauses/record). Real-traffic smoke re-run: STRICT would-demote 30% → 20%, overall 47% → 33% — better, not fixed; recombination-risk split doesn't separate either. Residual STRICT false-contras at ~0.83-0.92 → θc would need ≈ 0.90 (vs the clean-set 0.5); at θc=0.90 the data in hand gives 27/28 synthetic recall, 0/26 legit FP, 0/10 smoke STRICT FP — but n=10 is too small to set on. Next: a fuller ARBORIST_NLI_SHADOW=1 bench-qa run → sweep θc on hundreds of STRICT cells → confirm → set it. θc stays 0.5; runtime NLI demotion stays off. Production verifier unchanged; falsification-hard stays 10/12.
This commit is contained in:
parent
f4de936ff9
commit
02f8dfec07
8 changed files with 730 additions and 124 deletions
|
|
@ -10,6 +10,8 @@
|
|||
"runtime": "transformers-torch-cpu",
|
||||
"label_map_source": "read from model.config.id2label at load time (cross-encoder ordering is contradiction/entailment/neutral; do not hardcode)",
|
||||
"max_length": 256,
|
||||
"max_candidate_clauses": 6,
|
||||
"max_candidate_clauses_provenance": "§7 #5 step 3 + §7 #20 — NLI runs only on the top-N source clauses by content-token overlap with the answer claim, never the whole context; running it over every clause of a real 30 KB retrieved context is a multiple-comparisons false-positive machine (~30% would-demote on STRICT in the first real-traffic smoke). A Phase-3 runtime hook would instead pass the clauses the verifier's quote/span/entity matching actually touched.",
|
||||
"truncation_policy": "source_clause_then_claim@v1",
|
||||
"pair_serialization": "nli_pair@v1",
|
||||
"thresholds": {
|
||||
|
|
|
|||
|
|
@ -29,6 +29,18 @@ from typing import Optional
|
|||
|
||||
_MANIFEST_PATH = Path(__file__).resolve().parent / "manifest.json"
|
||||
_SENT_SPLIT = re.compile(r"(?<=[.!?;])\s+")
|
||||
_WORD_RE = re.compile(r"[A-Za-z][A-Za-z'-]+")
|
||||
# A tiny stopword set — just the function words long enough to clear the
|
||||
# >3-char length filter and frequent enough to dominate overlap noise.
|
||||
_STOP = frozenset((
|
||||
"the", "and", "that", "this", "with", "from", "have", "has", "had",
|
||||
"was", "were", "are", "been", "being", "for", "into", "than", "then",
|
||||
"they", "them", "their", "there", "which", "what", "when", "where",
|
||||
"who", "whom", "whose", "while", "also", "such", "some", "any", "all",
|
||||
"more", "most", "other", "over", "under", "between", "about", "because",
|
||||
"since", "though", "although", "however", "would", "could", "should",
|
||||
"will", "shall", "can", "may", "might", "must", "does", "did", "not",
|
||||
))
|
||||
|
||||
|
||||
def load_manifest(path: Optional[Path] = None) -> dict:
|
||||
|
|
@ -46,18 +58,49 @@ def clauses(text: str) -> list[str]:
|
|||
return parts or ([text.strip()] if text and text.strip() else [])
|
||||
|
||||
|
||||
def _content_tokens(text: str) -> set[str]:
|
||||
return {w for w in (m.group(0).lower() for m in _WORD_RE.finditer(text or ""))
|
||||
if len(w) > 3 and w not in _STOP}
|
||||
|
||||
|
||||
def candidate_clauses(claim: str, source: str, k: int) -> list[tuple[str, float]]:
|
||||
"""The §7 #5 step-3 "candidate source clauses": clauses that share
|
||||
content tokens with the answer claim, ranked by overlap fraction,
|
||||
top ``k``, zero-overlap dropped.
|
||||
|
||||
Restricting NLI to this small set is load-bearing — running it over
|
||||
*every* clause of a 30 KB retrieved context is a multiple-
|
||||
comparisons FP machine (ticket #000049 §7 #20). Standalone lexical
|
||||
proxy here; a runtime (Phase 3) hook would instead pass the clauses
|
||||
the verifier's quote/span/entity matching actually touched.
|
||||
"""
|
||||
ctoks = _content_tokens(claim)
|
||||
if not ctoks:
|
||||
return []
|
||||
scored: list[tuple[str, float]] = []
|
||||
for cl in clauses(source):
|
||||
ov = ctoks & _content_tokens(cl)
|
||||
if ov:
|
||||
scored.append((cl, len(ov) / len(ctoks)))
|
||||
scored.sort(key=lambda t: t[1], reverse=True)
|
||||
return scored[:max(1, k)]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ShadowResult:
|
||||
available: bool # False iff the [nli] extra / model could not load
|
||||
would_demote: bool # the §7 #5 Demote() decision (shadow — not applied)
|
||||
max_contradiction: float
|
||||
max_entailment: float
|
||||
best_clause: Optional[str] # the clause with the highest contradiction prob
|
||||
n_clauses: int
|
||||
best_clause: Optional[str] # the candidate clause with the highest contradiction prob
|
||||
n_clauses: int # total clauses in the source
|
||||
n_candidate_clauses: int # clauses NLI was actually run on (§7 #5 step 3 restriction)
|
||||
best_clause_overlap: float # max content-token overlap of any candidate clause with the claim
|
||||
recombination_risk: bool # claim tokens scattered across >=2 candidate clauses, no single dominant one
|
||||
model_version: Optional[str]
|
||||
theta_contra: float
|
||||
theta_entail: float
|
||||
reason: str = "" # human note ("ok", "deps_missing: …", "load_failed: …")
|
||||
reason: str = "" # human note ("ok", "deps_missing: …", "no_candidate_clauses", …)
|
||||
|
||||
def as_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
|
@ -102,6 +145,8 @@ class ShadowNLI:
|
|||
self.theta_contra: float = float(th.get("contradiction_veto", 0.5))
|
||||
self.theta_entail: float = float(th.get("entailment_block_veto", 0.9))
|
||||
self.max_length: int = int(self.manifest.get("max_length", 256))
|
||||
# §7 #5 step-3 / §7 #20 — cap how many source clauses NLI runs on.
|
||||
self.max_candidate_clauses: int = int(self.manifest.get("max_candidate_clauses", 6))
|
||||
self.available = False
|
||||
self._reason = "uninitialised"
|
||||
self._tok = None
|
||||
|
|
@ -141,21 +186,40 @@ class ShadowNLI:
|
|||
return p[self._ei], p[self._ni], p[self._ci]
|
||||
|
||||
def check(self, claim: str, source: str) -> ShadowResult:
|
||||
cls = clauses(source)
|
||||
n_clauses = len(clauses(source))
|
||||
cand = candidate_clauses(claim, source, self.max_candidate_clauses)
|
||||
best_overlap = cand[0][1] if cand else 0.0
|
||||
# recombination risk: claim tokens scattered across >=2 candidate
|
||||
# clauses, none of which alone dominates the claim. (Not a gate
|
||||
# here — recorded so the bench can split the would-demote rate by
|
||||
# it and confirm whether the §7 #5 risk-gate would help.)
|
||||
recomb_risk = (len(cand) >= 2 and cand[1][1] >= 0.2 and best_overlap < 0.85)
|
||||
|
||||
def _result(**kw) -> ShadowResult:
|
||||
base = dict(n_clauses=n_clauses, n_candidate_clauses=len(cand),
|
||||
best_clause_overlap=round(best_overlap, 4), recombination_risk=recomb_risk,
|
||||
model_version=self.model_version, theta_contra=self.theta_contra,
|
||||
theta_entail=self.theta_entail)
|
||||
base.update(kw)
|
||||
return ShadowResult(**base) # type: ignore[arg-type]
|
||||
|
||||
self._ensure_loaded()
|
||||
if not self.available:
|
||||
return ShadowResult(available=False, would_demote=False, max_contradiction=0.0,
|
||||
max_entailment=0.0, best_clause=None, n_clauses=len(cls),
|
||||
model_version=self.model_version, theta_contra=self.theta_contra,
|
||||
theta_entail=self.theta_entail, reason=self._reason)
|
||||
if not cls or not (claim or "").strip():
|
||||
return ShadowResult(available=True, would_demote=False, max_contradiction=0.0,
|
||||
max_entailment=0.0, best_clause=None, n_clauses=len(cls),
|
||||
model_version=self.model_version, theta_contra=self.theta_contra,
|
||||
theta_entail=self.theta_entail, reason="empty_input")
|
||||
return _result(available=False, would_demote=False, max_contradiction=0.0,
|
||||
max_entailment=0.0, best_clause=None, reason=self._reason)
|
||||
if not (claim or "").strip() or n_clauses == 0:
|
||||
return _result(available=True, would_demote=False, max_contradiction=0.0,
|
||||
max_entailment=0.0, best_clause=None, reason="empty_input")
|
||||
if not cand:
|
||||
# nothing in the source shares content tokens with the claim →
|
||||
# the lexical verifier would not have matched it either; no
|
||||
# contradiction veto. (This is *not* "grounded" — it's "NLI has
|
||||
# nothing to say"; UNGROUNDED-by-no-overlap is the verifier's job.)
|
||||
return _result(available=True, would_demote=False, max_contradiction=0.0,
|
||||
max_entailment=0.0, best_clause=None, reason="no_candidate_clauses")
|
||||
max_e = max_n = max_c = 0.0
|
||||
best_clause = None
|
||||
for cl in cls:
|
||||
for cl, _ov in cand:
|
||||
pe, pn, pc = self._nli(cl, claim)
|
||||
max_e = max(max_e, pe)
|
||||
max_n = max(max_n, pn)
|
||||
|
|
@ -163,11 +227,9 @@ class ShadowNLI:
|
|||
max_c = pc
|
||||
best_clause = cl
|
||||
would_demote = (max_c >= self.theta_contra) and (max_e < self.theta_entail)
|
||||
return ShadowResult(available=True, would_demote=would_demote,
|
||||
max_contradiction=round(max_c, 4), max_entailment=round(max_e, 4),
|
||||
best_clause=best_clause, n_clauses=len(cls),
|
||||
model_version=self.model_version, theta_contra=self.theta_contra,
|
||||
theta_entail=self.theta_entail, reason="ok")
|
||||
return _result(available=True, would_demote=would_demote,
|
||||
max_contradiction=round(max_c, 4), max_entailment=round(max_e, 4),
|
||||
best_clause=best_clause, reason="ok")
|
||||
|
||||
|
||||
_DEFAULT: Optional[ShadowNLI] = None
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"generated_at": "2026-05-12T18:27:18Z",
|
||||
"generated_at": "2026-05-12T18:32:34Z",
|
||||
"available": true,
|
||||
"reason": "ok",
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
|
|
@ -7,19 +7,21 @@
|
|||
"theta_entail": 0.9,
|
||||
"n_records": 15,
|
||||
"n_available": 15,
|
||||
"elapsed_seconds": 417.4,
|
||||
"would_demote_total": 7,
|
||||
"would_demote_rate": 0.4667,
|
||||
"elapsed_seconds": 18.0,
|
||||
"would_demote_total": 5,
|
||||
"would_demote_rate": 0.3333,
|
||||
"mean_candidate_clauses": 6.0,
|
||||
"max_candidate_clauses_cap": 6,
|
||||
"by_bucket": {
|
||||
"HYBRID": {
|
||||
"n": 3,
|
||||
"would_demote": 2,
|
||||
"rate": 0.6667
|
||||
"would_demote": 1,
|
||||
"rate": 0.3333
|
||||
},
|
||||
"STRICT": {
|
||||
"n": 10,
|
||||
"would_demote": 3,
|
||||
"rate": 0.3
|
||||
"would_demote": 2,
|
||||
"rate": 0.2
|
||||
},
|
||||
"UNGROUNDED": {
|
||||
"n": 2,
|
||||
|
|
@ -27,10 +29,22 @@
|
|||
"rate": 1.0
|
||||
}
|
||||
},
|
||||
"by_recombination_risk": {
|
||||
"risk": {
|
||||
"n": 10,
|
||||
"would_demote": 4,
|
||||
"rate": 0.4
|
||||
},
|
||||
"no_risk": {
|
||||
"n": 5,
|
||||
"would_demote": 1,
|
||||
"rate": 0.2
|
||||
}
|
||||
},
|
||||
"false_positive_probe": {
|
||||
"n": 10,
|
||||
"would_demote": 3,
|
||||
"rate": 0.3,
|
||||
"would_demote": 2,
|
||||
"rate": 0.2,
|
||||
"note": "would_demote on records labeled want=not_contradiction \u2014 these are shadow FALSE POSITIVES; this is \u00a77 #12 gate item 4 when the input is a real legit-answer sample"
|
||||
},
|
||||
"rows": [
|
||||
|
|
@ -41,10 +55,13 @@
|
|||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": true,
|
||||
"max_contradiction": 0.9699,
|
||||
"max_contradiction": 0.8838,
|
||||
"max_entailment": 0.2466,
|
||||
"best_clause": "Mount Kenya is located in central Kenya, just south of the equator, around () north-northeast of the capital Nairobi.",
|
||||
"best_clause": "The Same District is administratively divided into 25 wards: *Bombo *Bendera *Bwambo *Chome *Hedaru *Kihurio *Kirangare *Kisiwani (English meaning: on the island) *Makanya *Maore *Mhezi *Mpinji *Mshewa *Msindo *Mtii *Mwembe (English meaning: mango tree) *Myamba *Ndungu *Njoro *Ruvu *Same Mjini *Suji *Vudee *Vuje *Vunta ==Sources== * Kilimanjaro Region Homepage for the 2002 Tanzania National Census *Tanzanian Government Directory Database *Kilimanaro Wards ==See also== http://en.wikipedia.org/wiki/Moshi Moshi City bg:\u041a\u0438\u043b\u0438\u043c\u0430\u043d\u0434\u0436\u0430\u0440\u043e (\u0440\u0435\u0433\u0438\u043e\u043d) de:Kilimandscharo (Region) et:Kilimanjaro piirkond es:Regi\u00f3n de Kilimanjaro eo:Regiono Kiliman\u011daro fr:Kilimandjaro (r\u00e9gion) it:Regione del Kilimanjaro sw:Mkoa wa Kilimanjaro nl:Kilimanjaro (regio) ja:\u30ad\u30ea\u30de\u30f3\u30b8\u30e3\u30ed\u5dde no:Kilimanjaro (region) pl:Kilimand\u017caro (region) pt:Kilimanjaro (regi\u00e3o) ro:Regiunea Kilimanjaro sr:\u041a\u0438\u043b\u0438\u043c\u0430\u043d\u045f\u0430\u0440\u043e (\u0440\u0435\u0433\u0438\u043e\u043d) fi:Kilimanjaron alue sv:Kilimanjaro (region) tg:\u0412\u0438\u043b\u043e\u044f\u0442\u0438 \u041a\u0438\u043b\u0438\u043c\u0430\u043d\u04b7\u0430\u0440\u043e yo:Kilimanjaro Region zh:\u4e5e\u529b\u99ac\u624e\u7f85\u5340\n\n=== E11 (Kilimanjaro National Park | background_source) ===\nKilimanjaro National Park is a national park, located near Moshi, Tanzania.",
|
||||
"n_clauses": 166,
|
||||
"n_candidate_clauses": 6,
|
||||
"best_clause_overlap": 0.75,
|
||||
"recombination_risk": true,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
|
|
@ -61,6 +78,9 @@
|
|||
"max_entailment": 0.4797,
|
||||
"best_clause": "The signing of the treaty, however, was interrupted by the August Coup\u2014an attempted coup d'\u00e9tat against Gorbachev by hardline Communist Party members of the government and the KGB, who sought to reverse Gorbachev's reforms and reassert the central government's control over the republics.",
|
||||
"n_clauses": 154,
|
||||
"n_candidate_clauses": 6,
|
||||
"best_clause_overlap": 0.3729,
|
||||
"recombination_risk": true,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
|
|
@ -73,10 +93,13 @@
|
|||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": true,
|
||||
"max_contradiction": 0.861,
|
||||
"max_contradiction": 0.6023,
|
||||
"max_entailment": 0.2301,
|
||||
"best_clause": "Astronaut Sally Ride became the first American woman in space in 1983 on STS-7, and Eileen Collins was the first woman to pilot the Space Shuttle during STS-63 in 1995.",
|
||||
"best_clause": "The Mercury spacecraft was named Freedom 7 which performed a suborbital flight piloted by astronaut Alan Shepard, who became the first American in space.",
|
||||
"n_clauses": 190,
|
||||
"n_candidate_clauses": 6,
|
||||
"best_clause_overlap": 0.65,
|
||||
"recombination_risk": true,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
|
|
@ -89,10 +112,13 @@
|
|||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": true,
|
||||
"max_contradiction": 0.8257,
|
||||
"max_entailment": 0.3798,
|
||||
"best_clause": "While the term astronaut is sometimes applied to anyone who travels into space, including scientists, politicians, journalists, and tourists, this article only lists professional astronauts.",
|
||||
"max_contradiction": 0.7417,
|
||||
"max_entailment": 0.0867,
|
||||
"best_clause": "Source: https://en.wikipedia.org/wiki/Mercury_Seven\n Mercury Seven was the group of seven Mercury astronauts selected by NASA on April 9, 1959.",
|
||||
"n_clauses": 100,
|
||||
"n_candidate_clauses": 6,
|
||||
"best_clause_overlap": 0.5806,
|
||||
"recombination_risk": true,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
|
|
@ -104,11 +130,14 @@
|
|||
"is_fp_probe": false,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": true,
|
||||
"max_contradiction": 0.7787,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.2961,
|
||||
"max_entailment": 0.1831,
|
||||
"best_clause": "Mona Lisa may also refer to: == People == *Mona Lisa (singer), an R&B singer *Mona Lisa (actress), a Filipino film actress *Mona Lisa (Pakistani actress), a Pakistani actress == Films == *Mona Lisa (film), a 1986 British film *Mona Lisa Smile, a 2003 American film == Music == *Mona Lisa (opera), an opera by Max von Schillings (1915) *Mona Lisa (Nat King Cole song) *Mona Lisa (Britney Spears song) *\"Mona Lisa (When The World Comes Down)\", a song by All-American Rejects on their album When the World Comes Down == Others == *Mona Lisa (ship) *Mona Lisa (crater), a crater on Venus *Mona Lisa, a Ninja Turtles character cs:Mona Lisa (rozcestn\u00edk) de:Mona Lisa (Begriffskl\u00e4rung) es:Mona Lisa (desambiguaci\u00f3n) fr:Mona Lisa pt:Mona Lisa (desambigua\u00e7\u00e3o) ksh:Mona Lisa (Watt \u0117\u00df\u00df datt?) ru:\u041c\u043e\u043d\u0430 \u041b\u0438\u0437\u0430 (\u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f) tl:Mona Lisa (paglilinaw) tr:Mona Lisa (anlam ayr\u0131m\u0131)",
|
||||
"best_clause": "There is no indication of an intimate dialogue between the woman and the observer as is the case in the Portrait of Baldassare Castiglione (Louvre) painted by Raphael about ten years after Mona Lisa, and undoubtedly influenced by Leonardo's portrait.",
|
||||
"n_clauses": 127,
|
||||
"n_candidate_clauses": 6,
|
||||
"best_clause_overlap": 0.2807,
|
||||
"recombination_risk": true,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
|
|
@ -121,10 +150,13 @@
|
|||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.3874,
|
||||
"max_entailment": 0.0709,
|
||||
"best_clause": "==External links== * Trade Unions in the USSR ru:\u0412\u0441\u0435\u0441\u043e\u044e\u0437\u043d\u044b\u0439 \u0446\u0435\u043d\u0442\u0440\u0430\u043b\u044c\u043d\u044b\u0439 \u0441\u043e\u0432\u0435\u0442 \u043f\u0440\u043e\u0444\u0435\u0441\u0441\u0438\u043e\u043d\u0430\u043b\u044c\u043d\u044b\u0445 \u0441\u043e\u044e\u0437\u043e\u0432\n\n=== E9 (Communist Party of the Soviet Union | primary_answer_source) ===\nThe Communist Party of the Soviet Union (, Kommunisticheskaya Partiya Sovetskogo Soyuza;",
|
||||
"max_contradiction": 0.1982,
|
||||
"max_entailment": 0.0472,
|
||||
"best_clause": "The signing of the treaty, however, was interrupted by the August Coup\u2014an attempted coup d'\u00e9tat against Gorbachev by hardline Communist Party members of the government and the KGB, who sought to reverse Gorbachev's reforms and reassert the central government's control over the republics.",
|
||||
"n_clauses": 193,
|
||||
"n_candidate_clauses": 6,
|
||||
"best_clause_overlap": 0.4,
|
||||
"recombination_risk": true,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
|
|
@ -137,10 +169,13 @@
|
|||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.64,
|
||||
"max_contradiction": 0.3087,
|
||||
"max_entailment": 0.9235,
|
||||
"best_clause": "Gregory, Frederick Hauck, Jon McBride, Francis \"Dick\" Scobee, Brewster Shaw, Loren Shriver, David Walker, Donald Williams :Mission specialists: Guion Bluford, James Buchli, John Fabian, Anna Fisher, Dale Gardner, S.",
|
||||
"best_clause": "From the 18, the first seven\n\n=== E2 (Mercury Seven | background_source) ===\nAlan Bartlett Shepard Jr., USN, (1923\u20131998) :MR-3 (Freedom 7), Apollo 14 * Virgil Ivan (Gus) Grissom, USAF, (1926\u20131967) :MR-4 (Liberty Bell 7), Gemini 3, Apollo 1 * John Herschel Glenn Jr., USMC, (born 1921) :MA-6 (Friendship 7), STS-95 * Malcolm Scott Carpenter, USN, (born 1925) :MA-7 (Aurora 7) * Walter Marty (Wally) Schirra Jr., USN, (1923\u20132007) :MA-8 (Sigma 7), Gemini 6A, Apollo 7 * Leroy Gordon Cooper Jr., USAF, (1927\u20132004) :MA-9 (Faith 7), Gemini 5 * Donald Kent (Deke) Slayton, USAF, (1924\u20131993) :Apollo-Soyuz Test Project == See also == * Man In Space Soonest * Mercury 13 * NASA Astronaut Groups == References == et:Mercury Seven it:Mercury Seven lv:Mercury sept\u012btnieks lb:Mercury Seven hu:Mercury Seven ms:Mercury Tujuh nl:Mercury Seven ja:\u30de\u30fc\u30ad\u30e5\u30ea\u30fc\u30fb\u30bb\u30d6\u30f3 nn:Mercury Seven ru:\u041f\u0435\u0440\u0432\u044b\u0439 \u043e\u0442\u0440\u044f\u0434 \u0430\u0441\u0442\u0440\u043e\u043d\u0430\u0432\u0442\u043e\u0432 \u0421\u0428\u0410 sv:Astronautgrupp 1 zh:\u6c34\u661f\u8ba1\u52127\u4eba\n\n=== E3 (Mercury-Redstone 3 | background_source) ===\n21 weeks of unplanned preparation would be needed before it could be launched on its mission.",
|
||||
"n_clauses": 291,
|
||||
"n_candidate_clauses": 6,
|
||||
"best_clause_overlap": 0.7826,
|
||||
"recombination_risk": true,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
|
|
@ -153,10 +188,13 @@
|
|||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.9757,
|
||||
"max_contradiction": 0.387,
|
||||
"max_entailment": 0.9069,
|
||||
"best_clause": "Mount Kenya is located in central Kenya, just south of the equator, around () north-northeast of the capital Nairobi.",
|
||||
"best_clause": "The seven summits, the highest peaks\n\n Source: https://en.wikipedia.org/wiki/Mount_Kenya\n Mount Kenya is the highest mountain in Kenya and the second-highest in Africa, after Kilimanjaro.",
|
||||
"n_clauses": 108,
|
||||
"n_candidate_clauses": 6,
|
||||
"best_clause_overlap": 0.5217,
|
||||
"recombination_risk": true,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
|
|
@ -169,10 +207,13 @@
|
|||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": true,
|
||||
"max_contradiction": 0.9422,
|
||||
"max_entailment": 0.1603,
|
||||
"best_clause": "Mona Lisa may also refer to: == People == *Mona Lisa (singer), an R&B singer *Mona Lisa (actress), a Filipino film actress *Mona Lisa (Pakistani actress), a Pakistani actress == Films == *Mona Lisa (film), a 1986 British film *Mona Lisa Smile, a 2003 American film == Music == *Mona Lisa (opera), an opera by Max von Schillings (1915) *Mona Lisa (Nat King Cole song) *Mona Lisa (Britney Spears song) *\"Mona Lisa (When The World Comes Down)\", a song by All-American Rejects on their album When the World Comes Down == Others == *Mona Lisa (ship) *Mona Lisa (crater), a crater on Venus *Mona Lisa, a Ninja Turtles character cs:Mona Lisa (rozcestn\u00edk) de:Mona Lisa (Begriffskl\u00e4rung) es:Mona Lisa (desambiguaci\u00f3n) fr:Mona Lisa pt:Mona Lisa (desambigua\u00e7\u00e3o) ksh:Mona Lisa (Watt \u0117\u00df\u00df datt?) ru:\u041c\u043e\u043d\u0430 \u041b\u0438\u0437\u0430 (\u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f) tl:Mona Lisa (paglilinaw) tr:Mona Lisa (anlam ayr\u0131m\u0131)\n\n=== E10 (Mona Lisa (opera) | primary_answer_source) ===\nmaid |alto | |- |Sisto, Francesco's servant |tenor |Felix Decken |} ==Synopsis== :Place: the house of Francesco del Giocondo in Florence :Time: present (prologue and epilogue), 1492 (Act 1 and 2) ===Prologue=== A couple is visiting a palace in Florence.",
|
||||
"max_contradiction": 0.8297,
|
||||
"max_entailment": 0.035,
|
||||
"best_clause": "Wyn, 1952, p.141.</ref> Salvador Dal\u00ed, famous for his surrealist work, painted Self portrait as Mona Lisa in 1954.",
|
||||
"n_clauses": 202,
|
||||
"n_candidate_clauses": 6,
|
||||
"best_clause_overlap": 0.4062,
|
||||
"recombination_risk": true,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
|
|
@ -185,10 +226,13 @@
|
|||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": true,
|
||||
"max_contradiction": 0.9844,
|
||||
"max_contradiction": 0.9201,
|
||||
"max_entailment": 0.3517,
|
||||
"best_clause": "Mount Meru is an active stratovolcano located west of Mount Kilimanjaro in the nation of Tanzania.",
|
||||
"best_clause": "__TOC__ The exact meaning and origin of the name Kilimanjaro is unknown.",
|
||||
"n_clauses": 146,
|
||||
"n_candidate_clauses": 6,
|
||||
"best_clause_overlap": 0.6786,
|
||||
"recombination_risk": false,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
|
|
@ -200,11 +244,14 @@
|
|||
"is_fp_probe": true,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": true,
|
||||
"max_contradiction": 0.6823,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.211,
|
||||
"max_entailment": 0.5614,
|
||||
"best_clause": "One notable exception is Repenomamus giganticus, a triconodont weighing between and that is known to have eaten small dinosaurs like young Psittacosaurus.<ref name=huetal2005>{{cite journal |author=Hu Yaoming |year=2005 |title=Large Mesozoic mammals fed on dinosaurs |journal=Nature |volume=433 |pages=149\u2013152|doi=10.1038/nature03102 |pmid=15650737 |last2=Meng |first2=J |last3=Wang |first3=Y |last4=Li |first4=C\n\n=== E3 (Cultural depictions of dinosaurs | background_source) ===\nSince the word dinosaur was coined in 1842, there have been various different cultural depictions of dinosaurs.",
|
||||
"best_clause": "An ambitious scientist who used dinosaurs and other fossils to promote his beliefs, Owen was the driving force for the Crystal Palace dinosaur sculptures, the first large-scale dinosaur reconstructions that were accessible to the public\n\n=== E4 (Cultural depictions of dinosaurs | background_source) ===\nThe popular ideals of dinosaurs have many misconceptions, reinforced by films, books, comics, television shows, and even theme parks.",
|
||||
"n_clauses": 336,
|
||||
"n_candidate_clauses": 6,
|
||||
"best_clause_overlap": 0.4,
|
||||
"recombination_risk": false,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
|
|
@ -217,10 +264,13 @@
|
|||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.3694,
|
||||
"max_contradiction": 0.3206,
|
||||
"max_entailment": 0.0084,
|
||||
"best_clause": "Trade Unions in Russia and Ukraine, 1985-1995.",
|
||||
"best_clause": "Under the constitution adopted in 1936 and modified along the way until October 1977, the political foundation of the Soviet Union was formed by the Soviets (Councils) of People's Depu\n\n Source: https://en.wikipedia.org/wiki/Japan_\u2013_Soviet_Union_relations\nRelations between the Soviet Union and Japan between the former's establishment in 1925 and its collapse in 1991 tended to be hostile.",
|
||||
"n_clauses": 100,
|
||||
"n_candidate_clauses": 6,
|
||||
"best_clause_overlap": 0.4815,
|
||||
"recombination_risk": true,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
|
|
@ -233,10 +283,13 @@
|
|||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.9779,
|
||||
"max_contradiction": 0.3749,
|
||||
"max_entailment": 0.9691,
|
||||
"best_clause": "* Clyde painted Sontaran sidearms in his art, having encountered Commander Kaagh in The Last Sontaran and Enemy of the Bane.",
|
||||
"best_clause": "Mona\n\n Source: https://en.wikipedia.org/wiki/Speculation_about_Mona_Lisa Mona Lisa, or La Gioconda (La Joconde) is a 16th-century portrait painted in oil on a poplar panel by Leonardo Da Vinci.",
|
||||
"n_clauses": 114,
|
||||
"n_candidate_clauses": 6,
|
||||
"best_clause_overlap": 0.9474,
|
||||
"recombination_risk": false,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
|
|
@ -249,10 +302,13 @@
|
|||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.143,
|
||||
"max_entailment": 0.331,
|
||||
"best_clause": "An ambitious scientist who used dinosaurs and other fossils to promote his beliefs, Owen was the driving force for the Crystal Palace dinosaur sculptures, the first large-scale dinosaur reconstructions that were accessible to the public\n\n=== E4 (Cultural depictions of dinosaurs | background_source) ===\nThe popular ideals of dinosaurs have many misconceptions, reinforced by films, books, comics, television shows, and even theme parks.",
|
||||
"max_contradiction": 0.1046,
|
||||
"max_entailment": 0.2561,
|
||||
"best_clause": "10,000 species explode across the planet.",
|
||||
"n_clauses": 201,
|
||||
"n_candidate_clauses": 6,
|
||||
"best_clause_overlap": 0.2807,
|
||||
"recombination_risk": false,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
|
|
@ -265,10 +321,13 @@
|
|||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.4601,
|
||||
"max_entailment": 0.0089,
|
||||
"best_clause": "As the air temperature reaches 120 degrees, the Saurornithoides run inside the cave.",
|
||||
"max_contradiction": 0.2374,
|
||||
"max_entailment": 0.0068,
|
||||
"best_clause": "Source: https://en.wikipedia.org/wiki/Dinosaur\n Dinosaurs are a diverse group of animals that were the dominant terrestrial vertebrates for over 160\u00a0million years, from the late Triassic period (about 230\u00a0million years ago) until the end of the Cretaceous (about 65\u00a0million years ago).",
|
||||
"n_clauses": 114,
|
||||
"n_candidate_clauses": 6,
|
||||
"best_clause_overlap": 0.1429,
|
||||
"recombination_risk": false,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -92,18 +92,25 @@ def main(argv=None) -> int:
|
|||
nli = ShadowNLI()
|
||||
|
||||
by_bucket: dict[str, dict] = defaultdict(lambda: {"n": 0, "would_demote": 0})
|
||||
by_recomb_risk: dict[str, dict] = {"risk": {"n": 0, "would_demote": 0},
|
||||
"no_risk": {"n": 0, "would_demote": 0}}
|
||||
fp_probe = {"n": 0, "would_demote": 0}
|
||||
rows = []
|
||||
n_available = 0
|
||||
n_candidate_clauses_total = 0
|
||||
t0 = time.time()
|
||||
for r in recs:
|
||||
res = nli.check(r["claim"], r["source"])
|
||||
if res.available:
|
||||
n_available += 1
|
||||
n_candidate_clauses_total += res.n_candidate_clauses
|
||||
b = by_bucket[r["bucket"]]
|
||||
b["n"] += 1
|
||||
rr = by_recomb_risk["risk" if res.recombination_risk else "no_risk"]
|
||||
rr["n"] += 1
|
||||
if res.would_demote:
|
||||
b["would_demote"] += 1
|
||||
rr["would_demote"] += 1
|
||||
if r["is_fp_probe"]:
|
||||
fp_probe["n"] += 1
|
||||
if res.would_demote:
|
||||
|
|
@ -127,8 +134,12 @@ def main(argv=None) -> int:
|
|||
"elapsed_seconds": round(elapsed, 1),
|
||||
"would_demote_total": total_demote,
|
||||
"would_demote_rate": round(total_demote / len(recs), 4) if recs else 0.0,
|
||||
"mean_candidate_clauses": round(n_candidate_clauses_total / len(recs), 2) if recs else 0.0,
|
||||
"max_candidate_clauses_cap": nli.max_candidate_clauses,
|
||||
"by_bucket": {k: {**v, "rate": round(v["would_demote"] / v["n"], 4) if v["n"] else 0.0}
|
||||
for k, v in sorted(by_bucket.items())},
|
||||
"by_recombination_risk": {k: {**v, "rate": round(v["would_demote"] / v["n"], 4) if v["n"] else 0.0}
|
||||
for k, v in by_recomb_risk.items()},
|
||||
"false_positive_probe": {**fp_probe,
|
||||
"rate": round(fp_probe["would_demote"] / fp_probe["n"], 4) if fp_probe["n"] else None,
|
||||
"note": "would_demote on records labeled want=not_contradiction — these are shadow FALSE POSITIVES; this is §7 #12 gate item 4 when the input is a real legit-answer sample"},
|
||||
|
|
@ -146,8 +157,13 @@ def main(argv=None) -> int:
|
|||
print(" (install the runtime to get real numbers: pip install 'arborist[nli]')")
|
||||
print(f" swept {len(recs)} records structurally; would-demote rate not measured.")
|
||||
return 0
|
||||
print(f" model {nli.model_version} · θc={nli.theta_contra} θe={nli.theta_entail} · {len(recs)} records · {elapsed:.1f}s")
|
||||
print(f" model {nli.model_version} · θc={nli.theta_contra} θe={nli.theta_entail} · cand-clause cap {nli.max_candidate_clauses} · {len(recs)} records · {elapsed:.1f}s")
|
||||
print(f" mean candidate clauses NLI'd / record: {report['mean_candidate_clauses']} (vs the whole context — the §7 #20 restriction)")
|
||||
print(f" would_demote overall: {total_demote}/{len(recs)} = {report['would_demote_rate']:.3f}")
|
||||
rr = report["by_recombination_risk"]
|
||||
print(f" by recombination-risk: risk {rr['risk']['would_demote']}/{rr['risk']['n']}={rr['risk']['rate']:.3f} "
|
||||
f"no_risk {rr['no_risk']['would_demote']}/{rr['no_risk']['n']}={rr['no_risk']['rate']:.3f} "
|
||||
f"(if no_risk-rate is high too, the risk-gate alone isn't enough)")
|
||||
print(f" {'bucket':<28} {'n':>5} {'would_demote':>13} {'rate':>7}")
|
||||
print(" " + "-" * 56)
|
||||
for k, v in report["by_bucket"].items():
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ Newest first. Update on every open/close.
|
|||
|----------|------------------------------------------------|-----------------------|------------|-----------|
|
||||
| #000051 | Federated vecpack distribution (gossip the embedding backfill) | open · awaiting go/no-go · doc-only scaffold. Makes `chunk_vecs` a distributable artifact: backfill once on any CPU box (cloud / Prometheus-Σ sweep — #000037 §3.1), publish a **vecpack** `(shard_root, vec_backend_version, [(leaf_hash, embedding_blob)…])` over the mesh wire layer, every peer pulls + bulk-loads (sub-ms/chunk on the receiver — the laptop never runs the transformer). Keyed on `leaf_hash` (portable) not `chunk_id` (shard-local). Vecpacks are **soft data** — embeddings are `UNGROUNDED`, never proof path — so a cheap structural sanity gate (chunk exists locally w/ matching leaf_hash, right blob length for (dim,quant), finite norm, backend_version matches) suffices, no Merkle-proof-grade verification needed. Supplies #000050's prereq #1 ("a vecpack exists & is imported on the bench box", not "fox embedded the corpus locally"). GPU producer (the fast path): bge-small-en-v1.5 batched on a CUDA box (4090) ≈ 10³–10⁴ chunks/s → full 6.24M-chunk corpus in *minutes*, not days — drop a CUDA `Embedder` into `default_embedder()`; CUDA stack lives only on the producer box, never in arborist's `python+sqlite3` core. The mechanism behind whitepaper §1's "the embedding pass runs off the device". #000039 / #000050 sibling | 2026-05-12 | — |
|
||||
| #000050 | Vec RRF hybrid fusion (#000039 Phase 2) | open · awaiting go/no-go · doc-only scaffold; design in #000039 §4.2 (RRF) + §8 (the gate). Wire `VecBackend` as a 5th retrieval route in `query.py`, RRF-merged (route provenance carried) with the 4 FTS5 routes; UNGROUNDED hits, additive not replacement. Phase-2 sub-items now explicit: **accept-path-5** in `_filter_by_title_relevance` (low-title-overlap vec hits survive only via a stronger span-level warrant, never similarity-score alone — else the title gate drops exactly the semantic candidates vec exists for & the bench shows no lift); **six** vec config fields fold into `governance_policy_hash` (recipe-named quant `int8sym`) **+ a cache-write guard** blocking `providence_cache` persistence for vec/hybrid runs until that's wired; **run-DAG records the vec stage** (backend version, six fields, top_k, query-embedding hash, candidate chunk_ids+distances). **Gated** on (a) a corpus backfill **distributed via #000051** AND (b) a **four-condition** recall bench (A FTS5-only / B vec-only / C RRF hybrid / D candidate-union-no-RRF) clearing the 5pp floor incl. C-beats-D, on semantic-allusion + curated + **adversarial-semantic-neighbor** fixtures (else park, vec stays opt-in `--backend vec`; if C≈D ship the union, drop RRF). #000039 follow-up | 2026-05-12 | — |
|
||||
| #000049 | Attribution-aware grounding check (the recombination boundary) | open · boundary accepted · production no-go · shadow-path approved (de novo review 2026-05-13 — ticket §7) · doc-only; the home for #000048's deferred §2.3 — closing the 2 recombination over-grounds in `falsification-hard` (hard-003 Mercury / hard-005 Einstein) needs an attribution / dependency-parse or mini-NLI check, which is *not lexical* (#000048 §5). Discipline question answered: a small fixed purpose-built NLI/entailment *model* may influence `audit_mode` only as an opt-in, hash-pinned, governance-hashed, **demotion-only contradiction veto** after shadow-mode evidence (never promotes — `MODEL_ASSISTED_DEMOTION`, never `MODEL_ASSISTED_PROMOTION`). Production verifier unchanged; `falsification-hard` stays 10/12 as an honest boundary marker. Roadmap: Phase 0 (this amendment) → Phase 1 (shadow design: NLI manifest, fetch/verify, `nli_pair@v1` canonicalization, recombination-risk trigger) → Phase 2 (bench-only shadow impl, `[nli]` extra, `make fetch-nli`) → Phase 3 (demotion-only runtime, gated) → Phase 4 (mesh blob sync); §7 #12 six-condition bench gate required before Phases 2–4; if NLI ever affects `audit_mode`, `nli_policy_hash` folds into `governance_policy_hash`. **Phase-2 candidate bench done 2026-05-12** (`~/git/arborist-nli-bench/`, commits `829f9a4` + `a1cb28d`; ticket §7 #18): checkpoint-agnostic harness runs the §7 #5 clause-level algorithm over 28 synth recombination cases (incl. the 2 fixtures + harder shapes) + 26 legit cases (true summaries + near-miss decoys). 4 working candidates; `nli-MiniLM2-L6-H768` (82M, 45ms p50 CPU), `deberta-v3-base-mnli-fever-anli` (184M, 223ms), `bart-large-mnli` (407M, 259ms) all 28/28 catch · 0/26 FP with the standard θe=0.9 entailment guard; `cross-encoder/nli-deberta-v3-base` 27/28; deberta-large repo-id TODO. **Key finding: the §7 #5 two-threshold rule is load-bearing** — 3 of 4 candidates argmax-contradict 1/26 legit cases on the *wrong* source clause (competing-superlative confusion, e.g. "largest hot desert" vs "largest desert overall"); the entailment guard filters every one because another clause restates the claim → 0% guarded FP vs ~4% single-threshold. Picture: recombination is *easy* for any modern NLI checkpoint — differentiator is cost/robustness, MiniLM is the cost-pick, bart-large the threshold-robust pick. **Phase-2 shadow scaffold landed in arborist 2026-05-12** (ticket §7 #19): `arborist/qa/nli/` (manifest pins MiniLM @ a fixed HF revision + θc 0.5/θe 0.9 + 2 alternates; `ShadowNLI`/`shadow_check` lazy-imports `transformers`+`torch` behind a new `[nli]` extra, degrades to `available=False` when absent — SHADOW ONLY, never an `audit_mode` input, manifest not yet in `governance_policy_hash` per §7 #2) + `bench/scripts/nli_shadow_sweep.py` + `make bootstrap-nli` / `make bench-nli-shadow` + 16 tests. Synthetic sweep (116 records): 28/28 recombination demoted, 0/26 FP on legit summaries, 0/9 fires on already-`STRICT_SPAN`. **First real-traffic sweep** (§7 #20 — `ARBORIST_NLI_SHADOW=1 make bench-qa-smoke`, 15 cells; `query.py` surfaces verifier-input text gated off-by-default, `qa_sweep.py` carries it, `nli_shadow_sweep.py` reads it): the *naive* "NLI on every context clause" scaffold has a **~30% would-demote rate on STRICT answers** — a haystack/multiple-comparisons artifact (real contexts → 100–336 clauses; `max`-over-all almost always hits a tangential "contradiction"). Lesson: the §7 #5 "candidate source clauses" + recombination-risk gating is **load-bearing** — Phase 3 (and the next Phase-2 step) must restrict the NLI call to the clauses the lexical verifier matched and/or a deterministic recombination-risk trigger; do NOT enable runtime NLI demotion on the current scaffold. Remaining: add the candidate-clause restriction, re-run, then gate item 4 is measurable. Production verifier unchanged; `falsification-hard` stays 10/12. #000048 follow-up | 2026-05-12 | — |
|
||||
| #000049 | Attribution-aware grounding check (the recombination boundary) | open · boundary accepted · production no-go · shadow-path approved (de novo review 2026-05-13 — ticket §7) · doc-only; the home for #000048's deferred §2.3 — closing the 2 recombination over-grounds in `falsification-hard` (hard-003 Mercury / hard-005 Einstein) needs an attribution / dependency-parse or mini-NLI check, which is *not lexical* (#000048 §5). Discipline question answered: a small fixed purpose-built NLI/entailment *model* may influence `audit_mode` only as an opt-in, hash-pinned, governance-hashed, **demotion-only contradiction veto** after shadow-mode evidence (never promotes — `MODEL_ASSISTED_DEMOTION`, never `MODEL_ASSISTED_PROMOTION`). Production verifier unchanged; `falsification-hard` stays 10/12 as an honest boundary marker. Roadmap: Phase 0 (this amendment) → Phase 1 (shadow design: NLI manifest, fetch/verify, `nli_pair@v1` canonicalization, recombination-risk trigger) → Phase 2 (bench-only shadow impl, `[nli]` extra, `make fetch-nli`) → Phase 3 (demotion-only runtime, gated) → Phase 4 (mesh blob sync); §7 #12 six-condition bench gate required before Phases 2–4; if NLI ever affects `audit_mode`, `nli_policy_hash` folds into `governance_policy_hash`. **Phase-2 candidate bench done 2026-05-12** (`~/git/arborist-nli-bench/`, commits `829f9a4` + `a1cb28d`; ticket §7 #18): checkpoint-agnostic harness runs the §7 #5 clause-level algorithm over 28 synth recombination cases (incl. the 2 fixtures + harder shapes) + 26 legit cases (true summaries + near-miss decoys). 4 working candidates; `nli-MiniLM2-L6-H768` (82M, 45ms p50 CPU), `deberta-v3-base-mnli-fever-anli` (184M, 223ms), `bart-large-mnli` (407M, 259ms) all 28/28 catch · 0/26 FP with the standard θe=0.9 entailment guard; `cross-encoder/nli-deberta-v3-base` 27/28; deberta-large repo-id TODO. **Key finding: the §7 #5 two-threshold rule is load-bearing** — 3 of 4 candidates argmax-contradict 1/26 legit cases on the *wrong* source clause (competing-superlative confusion, e.g. "largest hot desert" vs "largest desert overall"); the entailment guard filters every one because another clause restates the claim → 0% guarded FP vs ~4% single-threshold. Picture: recombination is *easy* for any modern NLI checkpoint — differentiator is cost/robustness, MiniLM is the cost-pick, bart-large the threshold-robust pick. **Phase-2 shadow scaffold landed in arborist 2026-05-12** (ticket §7 #19): `arborist/qa/nli/` (manifest pins MiniLM @ a fixed HF revision + θc 0.5/θe 0.9 + 2 alternates; `ShadowNLI`/`shadow_check` lazy-imports `transformers`+`torch` behind a new `[nli]` extra, degrades to `available=False` when absent — SHADOW ONLY, never an `audit_mode` input, manifest not yet in `governance_policy_hash` per §7 #2) + `bench/scripts/nli_shadow_sweep.py` + `make bootstrap-nli` / `make bench-nli-shadow` + 16 tests. Synthetic sweep (116 records): 28/28 recombination demoted, 0/26 FP on legit summaries, 0/9 fires on already-`STRICT_SPAN`. **First real-traffic sweep** (§7 #20 — `ARBORIST_NLI_SHADOW=1 make bench-qa-smoke`, 15 cells; `query.py` surfaces verifier-input text gated off-by-default, `qa_sweep.py` carries it, `nli_shadow_sweep.py` reads it): the *naive* "NLI on every context clause" scaffold has a **~30% would-demote rate on STRICT answers** — a haystack/multiple-comparisons artifact (real contexts → 100–336 clauses; `max`-over-all almost always hits a tangential "contradiction"). Candidate-clause restriction added (§7 #21 — NLI runs on the top-6 source clauses by content-token overlap, not the whole context; `max_candidate_clauses=6`): STRICT would-demote 30% → 20% on the smoke, overall 47% → 33% — **helps, not fixed**; recombination-risk split doesn't separate either. Residual STRICT false-contras land at ~0.83–0.92, so θc would need ≈ 0.90 (up from the clean-set 0.5): at θc=0.90 the data in hand gives 27/28 synthetic recombination recall, 0/26 synthetic legit FP, 0/10 smoke STRICT FP — but n=10 is far too small to set a threshold on. Remaining: a fuller `ARBORIST_NLI_SHADOW=1 make bench-qa` run → ~hundreds of STRICT cells → sweep θc/θe → confirm → set it; until then θc stays 0.5 and runtime NLI demotion stays off. Standing lesson: the clean synthetic eval (§7 #18) does not predict real-traffic precision. Production verifier unchanged; `falsification-hard` stays 10/12. #000048 follow-up | 2026-05-12 | — |
|
||||
| #000048 | Verifier upgrade — recombination-aware grounding + clause segmentation | **closed · 2026-05-12** — steps 2.1 + 2.4 landed 2026-05-11 (12 of 16 residual items: 4 HYBRID_ENTITY over-grounds + 8 Formulate mis-segments → `formulate-hard` 12/12, `falsification-hard` 10/12; each bench-gated, no STRICT-rate regression — 2.1's gate fired on 0 QA answers, 2.4's segmenter touched 7 of 450 lattice cells both verdict changes correct). Step 2.2 (single-clause-containment paraphrase check) attempted + reverted — catches the 2 recombination fixtures but also rejects legit cross-sentence summaries with no threshold separating the two; recombination-vs-summary isn't lexical (§5 "What we learned"). The attribution-aware path moved to **#000049** (fox 2026-05-12). 2 live-pack `expected_reason` updated HYBRID_ENTITY→UNGROUNDED; 12+ tests; `make bench-5f-falsification-hard` / `bench-5f-formulate-hard` / `bench-fork-baseline-hard`. #000046 follow-up; #000047 closed | 2026-05-11 | — |
|
||||
| #000047 | ForkScore `_delta_*` aggregator (mean vs max vs sum) | **closed · 2026-05-11** — Option D: `WeightSet.delta_aggregator` ∈ {`mean`,`max`,`sum`} (default `mean` unchanged → no `ESTIMATOR_VERSION` bump), `fork_score._delta_5{s,t,f}` dispatch via `_aggregate`, recorded in `ScoredFork.weights`, per-sub `HARD_REGRESSION_FLOOR` flags aggregator-independent; bench data behind keeping `mean` in `5f-threshold-calibration-2026-05-11.md` §5; 8+1 tests. #000012-revision / #000025 §10.14 follow-up | 2026-05-11 | — |
|
||||
| #000046 | Harder 5S/5T/5F fixture tier (below-ceiling baselines) | **closed · 2026-05-11** — Phase 1 `falsification-hard-v1.jsonl` (12 near-misses) + Phase 2 `formulate-hard-v1.jsonl` (12 mis-segments, rate 4/12) + Phase 3 `verify_quotes` paraphrase numeric-agreement gate (`_numeric_signature`; demotes a token-covering span asserting a digit-number the source lacks modulo thousands-comma) → falsification-hard rate 4/12 → 6/12 on a real change; bench-gated (`make bench-qa` n=3×75×3 before/after — no STRICT-rate regression on legit answers; only gate-caused QA shift was correctly demoting a fictional-year claim STRICT→HYBRID); `fork_score` γ·Δ5f went positive on it. Headroom now down to 2 falsification-hard over-grounds (#000048 step 2.1 closed the 4 entity over-grounds; step 2.4 closed the 8 Formulate mis-segments → that pack 12/12; step 2.2 attempted + reverted — the last 2 recombination fixtures need an attribution-aware verifier, now tracked as **#000049**, and stand as documented residue). `make bench-5f-falsification-hard` / `bench-5f-formulate-hard` / `bench-fork-baseline-hard`; 7+ tests. #000025 §10.14 follow-up; #000047 closed; #000048 closed | 2026-05-11 | — |
|
||||
|
|
|
|||
|
|
@ -17,9 +17,15 @@ recombination-risk gating is **load-bearing, not optional** — Phase 3 (and
|
|||
the next Phase-2 step) must restrict the NLI call to the clauses the
|
||||
lexical verifier actually matched and/or a deterministic recombination-risk
|
||||
trigger; do NOT enable runtime NLI demotion on the current scaffold.
|
||||
Remaining: add the candidate-clause restriction, re-run the real-traffic
|
||||
sweep, then gate item 4 is measurable. Production verifier unchanged;
|
||||
`falsification-hard` stays 10/12.
|
||||
Candidate-clause restriction added (§7 #21 — `ShadowNLI.check` now NLI's
|
||||
only the top-6 source clauses by overlap, not the whole context): STRICT
|
||||
would-demote 30% → 20% on the smoke — *helps, not fixed*; the data points
|
||||
to θc ≈ 0.90 (up from the clean-set 0.5) to zero out the real-traffic
|
||||
STRICT FPs at 27/28 synthetic recombination recall, but n=10 is too small
|
||||
to set it on. Remaining: a fuller `ARBORIST_NLI_SHADOW=1 make bench-qa`
|
||||
run → sweep θc on hundreds of STRICT cells → confirm → set it. Runtime NLI
|
||||
demotion stays off. Production verifier unchanged; `falsification-hard`
|
||||
stays 10/12.
|
||||
**Opened:** 2026-05-12
|
||||
**Scope:** Decide whether — and if so how — to add a verifier check
|
||||
that catches a *recombination*: a claim whose content tokens are all
|
||||
|
|
@ -673,7 +679,7 @@ to `available=False` when `[nli]` absent); `[nli]` extra in
|
|||
`bench/scripts/nli_shadow_sweep.py` + `make bench-nli-shadow` (the
|
||||
gate-item-4 instrument — sweeps `(answer, context)` records, reports
|
||||
the *would-demote* rate bucketed by verifier label; renders even
|
||||
without `[nli]`, marked `available:false`); <!--AUTOCOUNT:tests:tests/test_nli_shadow.py-->16<!--/AUTOCOUNT--> tests in
|
||||
without `[nli]`, marked `available:false`); <!--AUTOCOUNT:tests:tests/test_nli_shadow.py-->22<!--/AUTOCOUNT--> tests in
|
||||
`tests/test_nli_shadow.py` (pure-Python parts + graceful degradation
|
||||
+ the bench-sweep parser — run in the default suite).
|
||||
|
||||
|
|
@ -760,3 +766,50 @@ candidate-clause restriction to `ShadowNLI.check`, re-run the
|
|||
real-traffic sweep, and only then is gate item 4 measurable as a
|
||||
meaningful number. Production verifier unchanged; `falsification-hard`
|
||||
stays 10/12.
|
||||
|
||||
**21. Candidate-clause restriction added — helps, doesn't close it
|
||||
(2026-05-12).** `ShadowNLI.check` now runs NLI only on the *top-N
|
||||
source clauses by content-token overlap with the answer claim*
|
||||
(`candidate_clauses()`; `max_candidate_clauses=6` in the manifest;
|
||||
zero-overlap clauses dropped; if nothing overlaps → `would_demote=False,
|
||||
reason="no_candidate_clauses"`), and records `n_candidate_clauses`,
|
||||
`best_clause_overlap`, `recombination_risk` (claim tokens scattered
|
||||
across ≥2 candidate clauses, none dominant). Re-ran both sweeps:
|
||||
|
||||
- *Synthetic* (`bench/results/nli-shadow-sweep.json`, 116 records,
|
||||
~9 s — mean **1.45** candidate clauses/record): `contradiction`
|
||||
28/28, `not_contradiction` **0/26**, `STRICT_SPAN`/`STRICT_PARAPHRASE`
|
||||
0 — the clean-set numbers are unchanged (short sources → top-6 ≈ all
|
||||
clauses anyway).
|
||||
- *Real traffic* (`bench/results/nli-shadow-sweep-smoke-live.json`,
|
||||
the 15-cell smoke re-run): **STRICT would-demote 30% → 20%** (2/10),
|
||||
overall 47% → 33%. Better, **not fixed**. The recombination-risk
|
||||
split doesn't separate either (risk 4/10 = 0.40, no_risk 1/5 = 0.20
|
||||
— a no-risk case still FPs).
|
||||
|
||||
The residual STRICT FPs land at `max_contradiction` ≈ 0.83 / 0.88
|
||||
(and a HYBRID at 0.92) — the model is *moderately* confident a
|
||||
top-overlap Wikipedia clause contradicts a *synthesized/aggregated*
|
||||
STRICT answer (a dropped qualifier, a tangential date, a sibling
|
||||
fact). On the clean synthetic set the recombination contras are ≈
|
||||
0.99 and the legit ones ≈ 0.005, so the manifest's θc = 0.5 was fine
|
||||
*there*; on real traffic the false-contra mass sits in 0.6–0.92, so
|
||||
**θc would need to rise to ≈ 0.90**. At θc = 0.90 on the data in hand:
|
||||
synthetic recombination catch = 27/28 (loses 1 hard case), synthetic
|
||||
legit FP = 0/26 (the one 0.946 legit case is rescued by the entailment
|
||||
guard, `max_entail` ≈ 0.99), smoke STRICT FP = **0/10**. So the
|
||||
*combination* {candidate-clause restriction + θc ≈ 0.90 + entailment
|
||||
guard} looks viable — but n = 10 STRICT in the smoke is far too small
|
||||
to set a threshold on. **Next step (slow/live, fox-decides): a fuller
|
||||
`ARBORIST_NLI_SHADOW=1 make bench-qa` run (n=1 or n=3 over the full
|
||||
question set) → ~hundreds of STRICT cells → sweep θc/θe over that,
|
||||
confirm the FP rate at θc ≈ 0.90, *then* set it in the manifest.**
|
||||
Until that confirmation: θc stays 0.5 (the clean-set default), runtime
|
||||
NLI demotion stays off, and the standalone-lexical candidate proxy
|
||||
stays a proxy — a Phase-3 runtime hook would pass the verifier's
|
||||
*actual* matched clauses, tighter than top-6-by-overlap. The deeper
|
||||
standing lesson: the clean synthetic eval (§7 #18) does **not** predict
|
||||
real-traffic precision — every gate number that matters has to come
|
||||
from a shadow run on real `bench-qa` output, not from contrived
|
||||
fixtures. Production verifier unchanged; `falsification-hard` stays
|
||||
10/12.
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from pathlib import Path
|
|||
import pytest
|
||||
|
||||
from arborist.qa.nli import ShadowNLI, ShadowResult, shadow_check, load_manifest
|
||||
from arborist.qa.nli.shadow import clauses, _resolve_label_indices
|
||||
from arborist.qa.nli.shadow import clauses, candidate_clauses, _content_tokens, _resolve_label_indices
|
||||
|
||||
|
||||
# --- clause splitter -------------------------------------------------------
|
||||
|
|
@ -42,6 +42,58 @@ def test_clauses_empty_or_blank():
|
|||
assert clauses(None) == [] # type: ignore[arg-type]
|
||||
|
||||
|
||||
# --- candidate-clause restriction (§7 #5 step 3 / §7 #20) ------------------
|
||||
|
||||
def test_content_tokens_filters_short_and_stopwords():
|
||||
toks = _content_tokens("The Mercury is the largest planet in the Solar System.")
|
||||
assert "mercury" in toks and "largest" in toks and "planet" in toks and "solar" in toks
|
||||
assert "the" not in toks and "is" not in toks and "in" not in toks # stopword / too short
|
||||
|
||||
|
||||
def test_candidate_clauses_ranks_by_overlap_and_caps():
|
||||
claim = "Mercury is the largest planet in the Solar System."
|
||||
source = ("Jupiter is the largest planet in the Solar System. "
|
||||
"Mercury is the smallest planet in the Solar System. "
|
||||
"The Roman god Mercury was the messenger of the gods. "
|
||||
"Bananas are yellow.")
|
||||
cand = candidate_clauses(claim, source, k=6)
|
||||
# the banana clause shares no content tokens → dropped
|
||||
assert all("Bananas" not in c for c, _ in cand)
|
||||
# the two planet clauses are the highest-overlap ones
|
||||
assert "Jupiter is the largest planet" in cand[0][0] or "Mercury is the smallest planet" in cand[0][0]
|
||||
assert cand[0][1] > cand[-1][1] or len(cand) == 1
|
||||
# cap respected
|
||||
assert len(candidate_clauses(claim, source, k=1)) == 1
|
||||
|
||||
|
||||
def test_candidate_clauses_empty_when_no_overlap():
|
||||
assert candidate_clauses("Bananas are yellow.", "Mercury is the smallest planet.", k=6) == []
|
||||
assert candidate_clauses("", "anything at all here", k=6) == []
|
||||
|
||||
|
||||
def test_check_restricts_to_candidate_clauses_not_whole_haystack():
|
||||
# a long source where only 2 clauses touch the claim — the result must
|
||||
# report n_candidate_clauses << n_clauses (this is the §7 #20 fix).
|
||||
haystack = " ".join(f"Unrelated fact number {i} about pottery." for i in range(40))
|
||||
source = "Jupiter is the largest planet. " + haystack + " Mercury is the smallest planet."
|
||||
res = shadow_check("Mercury is the largest planet.", source)
|
||||
assert res.n_clauses >= 40
|
||||
assert res.n_candidate_clauses <= 6
|
||||
assert res.n_candidate_clauses < res.n_clauses
|
||||
|
||||
|
||||
def test_check_no_candidate_clauses_is_no_demote():
|
||||
res = shadow_check("Mercury is the largest planet.",
|
||||
"Bananas are yellow. Pottery is ancient. The weather is fine.")
|
||||
assert res.would_demote is False
|
||||
if res.available:
|
||||
assert res.reason == "no_candidate_clauses" and res.n_candidate_clauses == 0
|
||||
|
||||
|
||||
def test_manifest_has_candidate_clause_cap():
|
||||
assert load_manifest()["max_candidate_clauses"] == 6
|
||||
|
||||
|
||||
# --- manifest --------------------------------------------------------------
|
||||
|
||||
def test_manifest_has_required_fields():
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue