Live hook: ARBORIST_NLI_SHADOW=1 makes query() surface the verifier-input text (gated off-by-default, never a cache_key/governance/audit_mode input); qa_sweep.py carries it + the answer into bench rows; the shadow sweep reads them and buckets by audit_mode. ARBORIST_NLI_SHADOW=1 make bench-qa-smoke (15 cells) → the naive 'NLI on every context clause' scaffold has a ~30% would-demote rate on STRICT answers — a haystack / multiple-comparisons artifact (real contexts segment into 100-336 clauses; max-over-all almost always finds a tangential clause the model reads as contradicting; a paraphrased STRICT answer often isn't verbatim- entailed by any single clause so the entailment guard doesn't rescue it). Lesson: the §7 #5 'candidate source clauses' + recombination-risk gating is load-bearing, not optional. Do NOT enable runtime NLI demotion on the current scaffold; next step is the candidate-clause restriction, then re-run, then gate item 4 is meaningful. Production verifier unchanged; falsification-hard stays 10/12.
This commit is contained in:
parent
70ecda3d6c
commit
f4de936ff9
6 changed files with 383 additions and 9 deletions
|
|
@ -3462,7 +3462,7 @@ def query(
|
|||
n_quotes=verdict["n_quotes"],
|
||||
n_verified=verdict["n_verified"],
|
||||
)
|
||||
return {
|
||||
result = {
|
||||
"status": "cache_miss_then_written",
|
||||
"audit_mode": verdict["audit_mode"],
|
||||
"cache_key": ckey,
|
||||
|
|
@ -3586,6 +3586,17 @@ def query(
|
|||
"total_ms": _ms_since(t_start),
|
||||
},
|
||||
}
|
||||
if os.environ.get("ARBORIST_NLI_SHADOW"):
|
||||
# #000049 Phase 2 — surface the verifier-input text so an
|
||||
# off-line NLI shadow sweep can re-derive (answer, source) pairs
|
||||
# and measure the would-demote rate on real traffic (§7 #12
|
||||
# gate item 4). Off by default — bloats bench rows; this is a
|
||||
# measurement hook, never a cache_key / governance / audit_mode
|
||||
# input.
|
||||
result["verifier_input_text"] = (
|
||||
rendered_evidence if is_lattice_mode else context
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _ms_since(t: float) -> float:
|
||||
|
|
|
|||
|
|
@ -228,7 +228,7 @@ def _run_one(
|
|||
# 12-char prefixes — operator can grep / SQL-filter the bench
|
||||
# JSONL for cross-row policy comparison.
|
||||
preflight_hash = (result.get("preflight_hash") or "")[:12]
|
||||
return {
|
||||
row = {
|
||||
"question": question,
|
||||
"answer_mode": answer_mode,
|
||||
"status": result.get("status") or ("error" if err else "missing"),
|
||||
|
|
@ -327,6 +327,16 @@ def _run_one(
|
|||
"directive_compliance": _directive_compliance(answer_mode, result, err),
|
||||
"error": err,
|
||||
}
|
||||
# #000049 Phase 2 — when ARBORIST_NLI_SHADOW is set, query() surfaces
|
||||
# the verifier-input text; carry it (+ the answer) into the row so a
|
||||
# downstream `nli_shadow_sweep.py --input <this.jsonl>` can measure
|
||||
# the would-demote rate on real traffic (§7 #12 gate item 4). Off by
|
||||
# default — these two fields can be large; never persisted otherwise.
|
||||
vit = result.get("verifier_input_text")
|
||||
if vit is not None:
|
||||
row["answer_text"] = result.get("answer_text")
|
||||
row["context"] = vit
|
||||
return row
|
||||
|
||||
|
||||
def _directive_compliance(
|
||||
|
|
|
|||
278
bench/results/nli-shadow-sweep-smoke-live.json
Normal file
278
bench/results/nli-shadow-sweep-smoke-live.json
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
{
|
||||
"generated_at": "2026-05-12T18:27:18Z",
|
||||
"available": true,
|
||||
"reason": "ok",
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"n_records": 15,
|
||||
"n_available": 15,
|
||||
"elapsed_seconds": 417.4,
|
||||
"would_demote_total": 7,
|
||||
"would_demote_rate": 0.4667,
|
||||
"by_bucket": {
|
||||
"HYBRID": {
|
||||
"n": 3,
|
||||
"would_demote": 2,
|
||||
"rate": 0.6667
|
||||
},
|
||||
"STRICT": {
|
||||
"n": 10,
|
||||
"would_demote": 3,
|
||||
"rate": 0.3
|
||||
},
|
||||
"UNGROUNDED": {
|
||||
"n": 2,
|
||||
"would_demote": 2,
|
||||
"rate": 1.0
|
||||
}
|
||||
},
|
||||
"false_positive_probe": {
|
||||
"n": 10,
|
||||
"would_demote": 3,
|
||||
"rate": 0.3,
|
||||
"note": "would_demote on records labeled want=not_contradiction \u2014 these are shadow FALSE POSITIVES; this is \u00a77 #12 gate item 4 when the input is a real legit-answer sample"
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"id": "2026-05-12T18-19-18Z",
|
||||
"bucket": "STRICT",
|
||||
"is_fp_probe": true,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": true,
|
||||
"max_contradiction": 0.9699,
|
||||
"max_entailment": 0.2466,
|
||||
"best_clause": "Mount Kenya is located in central Kenya, just south of the equator, around () north-northeast of the capital Nairobi.",
|
||||
"n_clauses": 166,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"reason": "ok"
|
||||
},
|
||||
{
|
||||
"id": "2026-05-12T18-19-18Z",
|
||||
"bucket": "STRICT",
|
||||
"is_fp_probe": true,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.1387,
|
||||
"max_entailment": 0.4797,
|
||||
"best_clause": "The signing of the treaty, however, was interrupted by the August Coup\u2014an attempted coup d'\u00e9tat against Gorbachev by hardline Communist Party members of the government and the KGB, who sought to reverse Gorbachev's reforms and reassert the central government's control over the republics.",
|
||||
"n_clauses": 154,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"reason": "ok"
|
||||
},
|
||||
{
|
||||
"id": "2026-05-12T18-19-18Z",
|
||||
"bucket": "UNGROUNDED",
|
||||
"is_fp_probe": false,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": true,
|
||||
"max_contradiction": 0.861,
|
||||
"max_entailment": 0.2301,
|
||||
"best_clause": "Astronaut Sally Ride became the first American woman in space in 1983 on STS-7, and Eileen Collins was the first woman to pilot the Space Shuttle during STS-63 in 1995.",
|
||||
"n_clauses": 190,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"reason": "ok"
|
||||
},
|
||||
{
|
||||
"id": "2026-05-12T18-19-18Z",
|
||||
"bucket": "UNGROUNDED",
|
||||
"is_fp_probe": false,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": true,
|
||||
"max_contradiction": 0.8257,
|
||||
"max_entailment": 0.3798,
|
||||
"best_clause": "While the term astronaut is sometimes applied to anyone who travels into space, including scientists, politicians, journalists, and tourists, this article only lists professional astronauts.",
|
||||
"n_clauses": 100,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"reason": "ok"
|
||||
},
|
||||
{
|
||||
"id": "2026-05-12T18-19-18Z",
|
||||
"bucket": "HYBRID",
|
||||
"is_fp_probe": false,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": true,
|
||||
"max_contradiction": 0.7787,
|
||||
"max_entailment": 0.1831,
|
||||
"best_clause": "Mona Lisa may also refer to: == People == *Mona Lisa (singer), an R&B singer *Mona Lisa (actress), a Filipino film actress *Mona Lisa (Pakistani actress), a Pakistani actress == Films == *Mona Lisa (film), a 1986 British film *Mona Lisa Smile, a 2003 American film == Music == *Mona Lisa (opera), an opera by Max von Schillings (1915) *Mona Lisa (Nat King Cole song) *Mona Lisa (Britney Spears song) *\"Mona Lisa (When The World Comes Down)\", a song by All-American Rejects on their album When the World Comes Down == Others == *Mona Lisa (ship) *Mona Lisa (crater), a crater on Venus *Mona Lisa, a Ninja Turtles character cs:Mona Lisa (rozcestn\u00edk) de:Mona Lisa (Begriffskl\u00e4rung) es:Mona Lisa (desambiguaci\u00f3n) fr:Mona Lisa pt:Mona Lisa (desambigua\u00e7\u00e3o) ksh:Mona Lisa (Watt \u0117\u00df\u00df datt?) ru:\u041c\u043e\u043d\u0430 \u041b\u0438\u0437\u0430 (\u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f) tl:Mona Lisa (paglilinaw) tr:Mona Lisa (anlam ayr\u0131m\u0131)",
|
||||
"n_clauses": 127,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"reason": "ok"
|
||||
},
|
||||
{
|
||||
"id": "2026-05-12T18-19-18Z",
|
||||
"bucket": "STRICT",
|
||||
"is_fp_probe": true,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.3874,
|
||||
"max_entailment": 0.0709,
|
||||
"best_clause": "==External links== * Trade Unions in the USSR ru:\u0412\u0441\u0435\u0441\u043e\u044e\u0437\u043d\u044b\u0439 \u0446\u0435\u043d\u0442\u0440\u0430\u043b\u044c\u043d\u044b\u0439 \u0441\u043e\u0432\u0435\u0442 \u043f\u0440\u043e\u0444\u0435\u0441\u0441\u0438\u043e\u043d\u0430\u043b\u044c\u043d\u044b\u0445 \u0441\u043e\u044e\u0437\u043e\u0432\n\n=== E9 (Communist Party of the Soviet Union | primary_answer_source) ===\nThe Communist Party of the Soviet Union (, Kommunisticheskaya Partiya Sovetskogo Soyuza;",
|
||||
"n_clauses": 193,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"reason": "ok"
|
||||
},
|
||||
{
|
||||
"id": "2026-05-12T18-19-18Z",
|
||||
"bucket": "STRICT",
|
||||
"is_fp_probe": true,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.64,
|
||||
"max_entailment": 0.9235,
|
||||
"best_clause": "Gregory, Frederick Hauck, Jon McBride, Francis \"Dick\" Scobee, Brewster Shaw, Loren Shriver, David Walker, Donald Williams :Mission specialists: Guion Bluford, James Buchli, John Fabian, Anna Fisher, Dale Gardner, S.",
|
||||
"n_clauses": 291,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"reason": "ok"
|
||||
},
|
||||
{
|
||||
"id": "2026-05-12T18-19-18Z",
|
||||
"bucket": "STRICT",
|
||||
"is_fp_probe": true,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.9757,
|
||||
"max_entailment": 0.9069,
|
||||
"best_clause": "Mount Kenya is located in central Kenya, just south of the equator, around () north-northeast of the capital Nairobi.",
|
||||
"n_clauses": 108,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"reason": "ok"
|
||||
},
|
||||
{
|
||||
"id": "2026-05-12T18-19-18Z",
|
||||
"bucket": "STRICT",
|
||||
"is_fp_probe": true,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": true,
|
||||
"max_contradiction": 0.9422,
|
||||
"max_entailment": 0.1603,
|
||||
"best_clause": "Mona Lisa may also refer to: == People == *Mona Lisa (singer), an R&B singer *Mona Lisa (actress), a Filipino film actress *Mona Lisa (Pakistani actress), a Pakistani actress == Films == *Mona Lisa (film), a 1986 British film *Mona Lisa Smile, a 2003 American film == Music == *Mona Lisa (opera), an opera by Max von Schillings (1915) *Mona Lisa (Nat King Cole song) *Mona Lisa (Britney Spears song) *\"Mona Lisa (When The World Comes Down)\", a song by All-American Rejects on their album When the World Comes Down == Others == *Mona Lisa (ship) *Mona Lisa (crater), a crater on Venus *Mona Lisa, a Ninja Turtles character cs:Mona Lisa (rozcestn\u00edk) de:Mona Lisa (Begriffskl\u00e4rung) es:Mona Lisa (desambiguaci\u00f3n) fr:Mona Lisa pt:Mona Lisa (desambigua\u00e7\u00e3o) ksh:Mona Lisa (Watt \u0117\u00df\u00df datt?) ru:\u041c\u043e\u043d\u0430 \u041b\u0438\u0437\u0430 (\u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f) tl:Mona Lisa (paglilinaw) tr:Mona Lisa (anlam ayr\u0131m\u0131)\n\n=== E10 (Mona Lisa (opera) | primary_answer_source) ===\nmaid |alto | |- |Sisto, Francesco's servant |tenor |Felix Decken |} ==Synopsis== :Place: the house of Francesco del Giocondo in Florence :Time: present (prologue and epilogue), 1492 (Act 1 and 2) ===Prologue=== A couple is visiting a palace in Florence.",
|
||||
"n_clauses": 202,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"reason": "ok"
|
||||
},
|
||||
{
|
||||
"id": "2026-05-12T18-19-18Z",
|
||||
"bucket": "HYBRID",
|
||||
"is_fp_probe": false,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": true,
|
||||
"max_contradiction": 0.9844,
|
||||
"max_entailment": 0.3517,
|
||||
"best_clause": "Mount Meru is an active stratovolcano located west of Mount Kilimanjaro in the nation of Tanzania.",
|
||||
"n_clauses": 146,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"reason": "ok"
|
||||
},
|
||||
{
|
||||
"id": "2026-05-12T18-19-18Z",
|
||||
"bucket": "STRICT",
|
||||
"is_fp_probe": true,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": true,
|
||||
"max_contradiction": 0.6823,
|
||||
"max_entailment": 0.5614,
|
||||
"best_clause": "One notable exception is Repenomamus giganticus, a triconodont weighing between and that is known to have eaten small dinosaurs like young Psittacosaurus.<ref name=huetal2005>{{cite journal |author=Hu Yaoming |year=2005 |title=Large Mesozoic mammals fed on dinosaurs |journal=Nature |volume=433 |pages=149\u2013152|doi=10.1038/nature03102 |pmid=15650737 |last2=Meng |first2=J |last3=Wang |first3=Y |last4=Li |first4=C\n\n=== E3 (Cultural depictions of dinosaurs | background_source) ===\nSince the word dinosaur was coined in 1842, there have been various different cultural depictions of dinosaurs.",
|
||||
"n_clauses": 336,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"reason": "ok"
|
||||
},
|
||||
{
|
||||
"id": "2026-05-12T18-19-18Z",
|
||||
"bucket": "STRICT",
|
||||
"is_fp_probe": true,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.3694,
|
||||
"max_entailment": 0.0084,
|
||||
"best_clause": "Trade Unions in Russia and Ukraine, 1985-1995.",
|
||||
"n_clauses": 100,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"reason": "ok"
|
||||
},
|
||||
{
|
||||
"id": "2026-05-12T18-19-18Z",
|
||||
"bucket": "STRICT",
|
||||
"is_fp_probe": true,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.9779,
|
||||
"max_entailment": 0.9691,
|
||||
"best_clause": "* Clyde painted Sontaran sidearms in his art, having encountered Commander Kaagh in The Last Sontaran and Enemy of the Bane.",
|
||||
"n_clauses": 114,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"reason": "ok"
|
||||
},
|
||||
{
|
||||
"id": "2026-05-12T18-19-18Z",
|
||||
"bucket": "HYBRID",
|
||||
"is_fp_probe": false,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.143,
|
||||
"max_entailment": 0.331,
|
||||
"best_clause": "An ambitious scientist who used dinosaurs and other fossils to promote his beliefs, Owen was the driving force for the Crystal Palace dinosaur sculptures, the first large-scale dinosaur reconstructions that were accessible to the public\n\n=== E4 (Cultural depictions of dinosaurs | background_source) ===\nThe popular ideals of dinosaurs have many misconceptions, reinforced by films, books, comics, television shows, and even theme parks.",
|
||||
"n_clauses": 201,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"reason": "ok"
|
||||
},
|
||||
{
|
||||
"id": "2026-05-12T18-19-18Z",
|
||||
"bucket": "STRICT",
|
||||
"is_fp_probe": true,
|
||||
"src_file": "2026-05-12T18-19-18Z.jsonl",
|
||||
"available": true,
|
||||
"would_demote": false,
|
||||
"max_contradiction": 0.4601,
|
||||
"max_entailment": 0.0089,
|
||||
"best_clause": "As the air temperature reaches 120 degrees, the Saurornithoides run inside the cave.",
|
||||
"n_clauses": 114,
|
||||
"model_version": "nli-shadow-v1-minilm2-l6-h768",
|
||||
"theta_contra": 0.5,
|
||||
"theta_entail": 0.9,
|
||||
"reason": "ok"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -55,8 +55,16 @@ def _records(path: Path):
|
|||
source = obj.get("context") or obj.get("source")
|
||||
if not claim or not source:
|
||||
continue
|
||||
bucket = obj.get("expected_reason") or obj.get("want") or "unlabeled"
|
||||
is_fp_probe = (obj.get("want") == "not_contradiction")
|
||||
bucket = (obj.get("expected_reason") or obj.get("want")
|
||||
or obj.get("audit_mode") or "unlabeled")
|
||||
# On real bench-qa rows there is no ground truth; treat a
|
||||
# would_demote on a STRICT row as a *potential* false positive
|
||||
# worth surfacing (the lexical verifier was confident, NLI
|
||||
# disagrees). On nli-bench eval rows `want=not_contradiction`
|
||||
# is the authoritative FP probe.
|
||||
is_fp_probe = (obj.get("want") == "not_contradiction"
|
||||
or (obj.get("want") is None and obj.get("expected_reason") is None
|
||||
and obj.get("audit_mode") == "STRICT"))
|
||||
yield {"id": obj.get("id", path.stem), "claim": claim, "source": source,
|
||||
"bucket": bucket, "is_fp_probe": is_fp_probe, "src_file": path.name}
|
||||
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ Newest first. Update on every open/close.
|
|||
|----------|------------------------------------------------|-----------------------|------------|-----------|
|
||||
| #000051 | Federated vecpack distribution (gossip the embedding backfill) | open · awaiting go/no-go · doc-only scaffold. Makes `chunk_vecs` a distributable artifact: backfill once on any CPU box (cloud / Prometheus-Σ sweep — #000037 §3.1), publish a **vecpack** `(shard_root, vec_backend_version, [(leaf_hash, embedding_blob)…])` over the mesh wire layer, every peer pulls + bulk-loads (sub-ms/chunk on the receiver — the laptop never runs the transformer). Keyed on `leaf_hash` (portable) not `chunk_id` (shard-local). Vecpacks are **soft data** — embeddings are `UNGROUNDED`, never proof path — so a cheap structural sanity gate (chunk exists locally w/ matching leaf_hash, right blob length for (dim,quant), finite norm, backend_version matches) suffices, no Merkle-proof-grade verification needed. Supplies #000050's prereq #1 ("a vecpack exists & is imported on the bench box", not "fox embedded the corpus locally"). GPU producer (the fast path): bge-small-en-v1.5 batched on a CUDA box (4090) ≈ 10³–10⁴ chunks/s → full 6.24M-chunk corpus in *minutes*, not days — drop a CUDA `Embedder` into `default_embedder()`; CUDA stack lives only on the producer box, never in arborist's `python+sqlite3` core. The mechanism behind whitepaper §1's "the embedding pass runs off the device". #000039 / #000050 sibling | 2026-05-12 | — |
|
||||
| #000050 | Vec RRF hybrid fusion (#000039 Phase 2) | open · awaiting go/no-go · doc-only scaffold; design in #000039 §4.2 (RRF) + §8 (the gate). Wire `VecBackend` as a 5th retrieval route in `query.py`, RRF-merged (route provenance carried) with the 4 FTS5 routes; UNGROUNDED hits, additive not replacement. Phase-2 sub-items now explicit: **accept-path-5** in `_filter_by_title_relevance` (low-title-overlap vec hits survive only via a stronger span-level warrant, never similarity-score alone — else the title gate drops exactly the semantic candidates vec exists for & the bench shows no lift); **six** vec config fields fold into `governance_policy_hash` (recipe-named quant `int8sym`) **+ a cache-write guard** blocking `providence_cache` persistence for vec/hybrid runs until that's wired; **run-DAG records the vec stage** (backend version, six fields, top_k, query-embedding hash, candidate chunk_ids+distances). **Gated** on (a) a corpus backfill **distributed via #000051** AND (b) a **four-condition** recall bench (A FTS5-only / B vec-only / C RRF hybrid / D candidate-union-no-RRF) clearing the 5pp floor incl. C-beats-D, on semantic-allusion + curated + **adversarial-semantic-neighbor** fixtures (else park, vec stays opt-in `--backend vec`; if C≈D ship the union, drop RRF). #000039 follow-up | 2026-05-12 | — |
|
||||
| #000049 | Attribution-aware grounding check (the recombination boundary) | open · boundary accepted · production no-go · shadow-path approved (de novo review 2026-05-13 — ticket §7) · doc-only; the home for #000048's deferred §2.3 — closing the 2 recombination over-grounds in `falsification-hard` (hard-003 Mercury / hard-005 Einstein) needs an attribution / dependency-parse or mini-NLI check, which is *not lexical* (#000048 §5). Discipline question answered: a small fixed purpose-built NLI/entailment *model* may influence `audit_mode` only as an opt-in, hash-pinned, governance-hashed, **demotion-only contradiction veto** after shadow-mode evidence (never promotes — `MODEL_ASSISTED_DEMOTION`, never `MODEL_ASSISTED_PROMOTION`). Production verifier unchanged; `falsification-hard` stays 10/12 as an honest boundary marker. Roadmap: Phase 0 (this amendment) → Phase 1 (shadow design: NLI manifest, fetch/verify, `nli_pair@v1` canonicalization, recombination-risk trigger) → Phase 2 (bench-only shadow impl, `[nli]` extra, `make fetch-nli`) → Phase 3 (demotion-only runtime, gated) → Phase 4 (mesh blob sync); §7 #12 six-condition bench gate required before Phases 2–4; if NLI ever affects `audit_mode`, `nli_policy_hash` folds into `governance_policy_hash`. **Phase-2 candidate bench done 2026-05-12** (`~/git/arborist-nli-bench/`, commits `829f9a4` + `a1cb28d`; ticket §7 #18): checkpoint-agnostic harness runs the §7 #5 clause-level algorithm over 28 synth recombination cases (incl. the 2 fixtures + harder shapes) + 26 legit cases (true summaries + near-miss decoys). 4 working candidates; `nli-MiniLM2-L6-H768` (82M, 45ms p50 CPU), `deberta-v3-base-mnli-fever-anli` (184M, 223ms), `bart-large-mnli` (407M, 259ms) all 28/28 catch · 0/26 FP with the standard θe=0.9 entailment guard; `cross-encoder/nli-deberta-v3-base` 27/28; deberta-large repo-id TODO. **Key finding: the §7 #5 two-threshold rule is load-bearing** — 3 of 4 candidates argmax-contradict 1/26 legit cases on the *wrong* source clause (competing-superlative confusion, e.g. "largest hot desert" vs "largest desert overall"); the entailment guard filters every one because another clause restates the claim → 0% guarded FP vs ~4% single-threshold. Picture: recombination is *easy* for any modern NLI checkpoint — differentiator is cost/robustness, MiniLM is the cost-pick, bart-large the threshold-robust pick. **Phase-2 shadow scaffold landed in arborist 2026-05-12** (ticket §7 #19): `arborist/qa/nli/` (manifest pins MiniLM @ a fixed HF revision + θc 0.5/θe 0.9 + 2 alternates; `ShadowNLI`/`shadow_check` lazy-imports `transformers`+`torch` behind a new `[nli]` extra, degrades to `available=False` when absent — SHADOW ONLY, never an `audit_mode` input, manifest not yet in `governance_policy_hash` per §7 #2) + `bench/scripts/nli_shadow_sweep.py` + `make bootstrap-nli` / `make bench-nli-shadow` + 16 tests. First sweep (116 records: 5f-falsification packs + the sibling-repo eval sets): 28/28 synth recombination demoted, 0/26 FP on legit summaries, 0/9 fires on already-`STRICT_SPAN` records, 25/50 on `UNGROUNDED` (the contradiction half; quiet on non-sequiturs — correct). Gate items 1/2/3/5/6 look clear on available data; **item 4 — shadow FP rate on a real live-`bench-qa` sample — remains the one open measurement** (instrument in place; the run is slow/live, fox-decides). Production verifier unchanged; `falsification-hard` stays 10/12. #000048 follow-up | 2026-05-12 | — |
|
||||
| #000049 | Attribution-aware grounding check (the recombination boundary) | open · boundary accepted · production no-go · shadow-path approved (de novo review 2026-05-13 — ticket §7) · doc-only; the home for #000048's deferred §2.3 — closing the 2 recombination over-grounds in `falsification-hard` (hard-003 Mercury / hard-005 Einstein) needs an attribution / dependency-parse or mini-NLI check, which is *not lexical* (#000048 §5). Discipline question answered: a small fixed purpose-built NLI/entailment *model* may influence `audit_mode` only as an opt-in, hash-pinned, governance-hashed, **demotion-only contradiction veto** after shadow-mode evidence (never promotes — `MODEL_ASSISTED_DEMOTION`, never `MODEL_ASSISTED_PROMOTION`). Production verifier unchanged; `falsification-hard` stays 10/12 as an honest boundary marker. Roadmap: Phase 0 (this amendment) → Phase 1 (shadow design: NLI manifest, fetch/verify, `nli_pair@v1` canonicalization, recombination-risk trigger) → Phase 2 (bench-only shadow impl, `[nli]` extra, `make fetch-nli`) → Phase 3 (demotion-only runtime, gated) → Phase 4 (mesh blob sync); §7 #12 six-condition bench gate required before Phases 2–4; if NLI ever affects `audit_mode`, `nli_policy_hash` folds into `governance_policy_hash`. **Phase-2 candidate bench done 2026-05-12** (`~/git/arborist-nli-bench/`, commits `829f9a4` + `a1cb28d`; ticket §7 #18): checkpoint-agnostic harness runs the §7 #5 clause-level algorithm over 28 synth recombination cases (incl. the 2 fixtures + harder shapes) + 26 legit cases (true summaries + near-miss decoys). 4 working candidates; `nli-MiniLM2-L6-H768` (82M, 45ms p50 CPU), `deberta-v3-base-mnli-fever-anli` (184M, 223ms), `bart-large-mnli` (407M, 259ms) all 28/28 catch · 0/26 FP with the standard θe=0.9 entailment guard; `cross-encoder/nli-deberta-v3-base` 27/28; deberta-large repo-id TODO. **Key finding: the §7 #5 two-threshold rule is load-bearing** — 3 of 4 candidates argmax-contradict 1/26 legit cases on the *wrong* source clause (competing-superlative confusion, e.g. "largest hot desert" vs "largest desert overall"); the entailment guard filters every one because another clause restates the claim → 0% guarded FP vs ~4% single-threshold. Picture: recombination is *easy* for any modern NLI checkpoint — differentiator is cost/robustness, MiniLM is the cost-pick, bart-large the threshold-robust pick. **Phase-2 shadow scaffold landed in arborist 2026-05-12** (ticket §7 #19): `arborist/qa/nli/` (manifest pins MiniLM @ a fixed HF revision + θc 0.5/θe 0.9 + 2 alternates; `ShadowNLI`/`shadow_check` lazy-imports `transformers`+`torch` behind a new `[nli]` extra, degrades to `available=False` when absent — SHADOW ONLY, never an `audit_mode` input, manifest not yet in `governance_policy_hash` per §7 #2) + `bench/scripts/nli_shadow_sweep.py` + `make bootstrap-nli` / `make bench-nli-shadow` + 16 tests. Synthetic sweep (116 records): 28/28 recombination demoted, 0/26 FP on legit summaries, 0/9 fires on already-`STRICT_SPAN`. **First real-traffic sweep** (§7 #20 — `ARBORIST_NLI_SHADOW=1 make bench-qa-smoke`, 15 cells; `query.py` surfaces verifier-input text gated off-by-default, `qa_sweep.py` carries it, `nli_shadow_sweep.py` reads it): the *naive* "NLI on every context clause" scaffold has a **~30% would-demote rate on STRICT answers** — a haystack/multiple-comparisons artifact (real contexts → 100–336 clauses; `max`-over-all almost always hits a tangential "contradiction"). Lesson: the §7 #5 "candidate source clauses" + recombination-risk gating is **load-bearing** — Phase 3 (and the next Phase-2 step) must restrict the NLI call to the clauses the lexical verifier matched and/or a deterministic recombination-risk trigger; do NOT enable runtime NLI demotion on the current scaffold. Remaining: add the candidate-clause restriction, re-run, then gate item 4 is measurable. Production verifier unchanged; `falsification-hard` stays 10/12. #000048 follow-up | 2026-05-12 | — |
|
||||
| #000048 | Verifier upgrade — recombination-aware grounding + clause segmentation | **closed · 2026-05-12** — steps 2.1 + 2.4 landed 2026-05-11 (12 of 16 residual items: 4 HYBRID_ENTITY over-grounds + 8 Formulate mis-segments → `formulate-hard` 12/12, `falsification-hard` 10/12; each bench-gated, no STRICT-rate regression — 2.1's gate fired on 0 QA answers, 2.4's segmenter touched 7 of 450 lattice cells both verdict changes correct). Step 2.2 (single-clause-containment paraphrase check) attempted + reverted — catches the 2 recombination fixtures but also rejects legit cross-sentence summaries with no threshold separating the two; recombination-vs-summary isn't lexical (§5 "What we learned"). The attribution-aware path moved to **#000049** (fox 2026-05-12). 2 live-pack `expected_reason` updated HYBRID_ENTITY→UNGROUNDED; 12+ tests; `make bench-5f-falsification-hard` / `bench-5f-formulate-hard` / `bench-fork-baseline-hard`. #000046 follow-up; #000047 closed | 2026-05-11 | — |
|
||||
| #000047 | ForkScore `_delta_*` aggregator (mean vs max vs sum) | **closed · 2026-05-11** — Option D: `WeightSet.delta_aggregator` ∈ {`mean`,`max`,`sum`} (default `mean` unchanged → no `ESTIMATOR_VERSION` bump), `fork_score._delta_5{s,t,f}` dispatch via `_aggregate`, recorded in `ScoredFork.weights`, per-sub `HARD_REGRESSION_FLOOR` flags aggregator-independent; bench data behind keeping `mean` in `5f-threshold-calibration-2026-05-11.md` §5; 8+1 tests. #000012-revision / #000025 §10.14 follow-up | 2026-05-11 | — |
|
||||
| #000046 | Harder 5S/5T/5F fixture tier (below-ceiling baselines) | **closed · 2026-05-11** — Phase 1 `falsification-hard-v1.jsonl` (12 near-misses) + Phase 2 `formulate-hard-v1.jsonl` (12 mis-segments, rate 4/12) + Phase 3 `verify_quotes` paraphrase numeric-agreement gate (`_numeric_signature`; demotes a token-covering span asserting a digit-number the source lacks modulo thousands-comma) → falsification-hard rate 4/12 → 6/12 on a real change; bench-gated (`make bench-qa` n=3×75×3 before/after — no STRICT-rate regression on legit answers; only gate-caused QA shift was correctly demoting a fictional-year claim STRICT→HYBRID); `fork_score` γ·Δ5f went positive on it. Headroom now down to 2 falsification-hard over-grounds (#000048 step 2.1 closed the 4 entity over-grounds; step 2.4 closed the 8 Formulate mis-segments → that pack 12/12; step 2.2 attempted + reverted — the last 2 recombination fixtures need an attribution-aware verifier, now tracked as **#000049**, and stand as documented residue). `make bench-5f-falsification-hard` / `bench-5f-formulate-hard` / `bench-fork-baseline-hard`; 7+ tests. #000025 §10.14 follow-up; #000047 closed; #000048 closed | 2026-05-11 | — |
|
||||
|
|
|
|||
|
|
@ -7,10 +7,19 @@ landed 2026-05-12 (§7 #18/#19 — candidate bench in `~/git/arborist-nli-bench/
|
|||
`arborist/qa/nli/` shadow module + `[nli]` extra + `make bench-nli-shadow`
|
||||
in-repo, SHADOW ONLY — never an `audit_mode` input; first sweep:
|
||||
28/28 recombination demoted, 0/26 FP on legit summaries, never fires on
|
||||
already-STRICT records). Remaining: §7 #12 gate item 4 — shadow FP rate
|
||||
on a real live-`bench-qa` sample (instrument in place; the run is the
|
||||
next step, slow/live). Production verifier unchanged; `falsification-hard`
|
||||
stays 10/12.
|
||||
already-STRICT records). First real-traffic shadow sweep done (§7 #20, `ARBORIST_NLI_SHADOW=1 make
|
||||
bench-qa-smoke` → 15 cells): the *naive* "NLI on every context clause"
|
||||
version has a **~30% would-demote rate on STRICT answers** — a haystack /
|
||||
multiple-comparisons artifact (real contexts segment into 100–336 clauses;
|
||||
`max`-over-all-of-them almost always finds a tangential clause the model
|
||||
reads as contradicting). Lesson: the §7 #5 "candidate source clauses" +
|
||||
recombination-risk gating is **load-bearing, not optional** — Phase 3 (and
|
||||
the next Phase-2 step) must restrict the NLI call to the clauses the
|
||||
lexical verifier actually matched and/or a deterministic recombination-risk
|
||||
trigger; do NOT enable runtime NLI demotion on the current scaffold.
|
||||
Remaining: add the candidate-clause restriction, re-run the real-traffic
|
||||
sweep, then gate item 4 is measurable. Production verifier unchanged;
|
||||
`falsification-hard` stays 10/12.
|
||||
**Opened:** 2026-05-12
|
||||
**Scope:** Decide whether — and if so how — to add a verifier check
|
||||
that catches a *recombination*: a claim whose content tokens are all
|
||||
|
|
@ -693,3 +702,61 @@ output. The instrument is in place (`make bench-nli-shadow
|
|||
INPUT="…"`); feeding it a live `bench-qa` run is the next concrete
|
||||
step, and a slow/live one (fox-decides). Nothing here changes the
|
||||
production verifier; `falsification-hard` stays 10/12.
|
||||
|
||||
**20. First *real-traffic* shadow sweep (2026-05-12) — the haystack
|
||||
problem.** Added the live hook: `ARBORIST_NLI_SHADOW=1` makes
|
||||
`query()` surface the verifier-input text (gated, off by default,
|
||||
never a cache_key / governance / `audit_mode` input — `query.py`
|
||||
~`return result`), `bench/qa_sweep.py` carries it + the answer into
|
||||
its rows when present, and `bench/scripts/nli_shadow_sweep.py` reads
|
||||
those rows (buckets by the row's `audit_mode`; treats a `would_demote`
|
||||
on a `STRICT` row as a candidate false positive). Ran
|
||||
`ARBORIST_NLI_SHADOW=1 make bench-qa-smoke` (5 questions × 3 modes =
|
||||
15 cells against Hermes) → `bench/results/nli-shadow-sweep-smoke-live.json`:
|
||||
|
||||
| audit_mode bucket | n | would_demote | rate |
|
||||
|---|---|---|---|
|
||||
| STRICT | 10 | **3** | **0.30** |
|
||||
| HYBRID | 3 | 2 | 0.67 |
|
||||
| UNGROUNDED | 2 | 2 | 1.00 |
|
||||
|
||||
**A 30% would-demote rate on STRICT answers — i.e. a ~30% false-
|
||||
positive rate.** That is *far* above any acceptable §7 #12 gate
|
||||
threshold (item 2/4 want ≈ 0), and it is the opposite of the §7 #18
|
||||
synthetic result (0/26 FP). The cause is the **haystack**: a real
|
||||
retrieved context is 8–64 KB → 100–336 *clauses* per the §7 #5
|
||||
step-1 segmenter; running `NLI(clause, answer)` over *all* of them
|
||||
and taking `max_contradiction` is a massive multiple-comparisons
|
||||
problem — somewhere in 200 tangential Wikipedia clauses there is
|
||||
almost always one the model reads as contradicting a short
|
||||
synthesized answer, and the entailment guard doesn't always rescue it
|
||||
because a *paraphrased/aggregated* STRICT answer often isn't verbatim-
|
||||
entailed by any single clause (max_entail stays < θe even though the
|
||||
answer is perfectly grounded). On the clean synthetic set (1–3 clause
|
||||
sources) `max`-over-2 is fine; `max`-over-300 on real context is a
|
||||
different regime. (Also: 417 s for 15 records ≈ 28 s/record on CPU —
|
||||
running NLI on hundreds of clauses per answer is too slow for inline
|
||||
use regardless.)
|
||||
|
||||
**Consequence for the design — the §7 #5 "candidate source clauses" +
|
||||
"recombination-risk heuristic fires" gating is not optional, it is
|
||||
load-bearing.** The shadow scaffold's `ShadowNLI.check` deliberately
|
||||
ran NLI on *every* clause (the simplest thing that could measure
|
||||
anything); that version is now empirically shown to be unusable on
|
||||
real traffic. Phase 3 (and a refined Phase-2 shadow) MUST restrict the
|
||||
NLI call to a *small* candidate set — only the clauses the lexical
|
||||
verifier's quote/span/entity matching actually touched, and/or only
|
||||
when a deterministic recombination-risk trigger fires (high
|
||||
content-token coverage, answer subject/object tokens occurring in
|
||||
*different* matched clauses, weak verifier path only — §7 #5 step 3,
|
||||
§7 #12). The §7 #18 framing ("recombination is an *easy* task for any
|
||||
modern NLI checkpoint") holds for the *isolated clause-pair* judgment;
|
||||
it does **not** hold for "find the one contradicting clause in a
|
||||
30 KB haystack" — that is a precision/triage problem the NLI model
|
||||
cannot solve alone, and the deterministic recombination-risk pre-
|
||||
filter is what makes it tractable. So: do not enable any runtime NLI
|
||||
demotion on the current scaffold; the next Phase-2 step is to add the
|
||||
candidate-clause restriction to `ShadowNLI.check`, re-run the
|
||||
real-traffic sweep, and only then is gate item 4 measurable as a
|
||||
meaningful number. Production verifier unchanged; `falsification-hard`
|
||||
stays 10/12.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue