#000052 §3.2.2 refinement: claim-lattice metadata cleaning — STRICT FP drops universally (-0.5 to -5.6 pts), L-2 down to 1.5%
Hand-inspection of the bottom-15 STRICT-fires from the raw §3.2.2 step 2 sweep showed claim-lattice overlay markup ([E\d+ | title | hash: '…']) depressing scores on correct concise answers (the 6× Henry-VIII case), while true-positive deflections (broad-question / narrow-answer like 'winners of all major sports?' → just-one-sport) remained correctly low-scored. So the noise FP class is the bracket metadata; cleaning it should reduce FP without losing true-positive signal. Built clean_for_relevance() in arborist/qa/relevance/shadow.py — strips [E\d+ | ... ] blocks + trailing '...']' tails. Baked into ShadowRelevance.check_question_answer / check_claim_source by default (opt out with clean_input=False). relevance_shadow_sweep.py applies it to inputs before _score_batch (opt out with --no-clean). Re-ran the full 6-model sweep on the 808-cell pooled STRICT with cleaning: bge-reranker-large 21.8% → 18.6% (-3.2) MiniLM-L-4-v2 15.0% → 9.4% (-5.6 pts, -37% rel) MiniLM-L-6-v2 11.6% → 9.0% (-2.6) MiniLM-L-12-v2 11.0% → 8.0% (-3.0) bge-reranker-base 9.5% → 9.0% (-0.5) MiniLM-L-2-v2 4.5% → 1.5% (-3.0 pts, -67% rel) Universal improvement, every model better. Big surprise: MiniLM-L-2-v2 — the model that FAILED the candidate-bench separability (margin -1.97, declared 'capacity floor') — has the LOWEST real-traffic FP rate at its own cb θ (1.5%). Because L-2's compressed score range gives it a low cb θ which few real STRICT pairs score below. SEVENTH instance of 'candidate-bench doesn't predict real-traffic'. Runtime-veto verdict UNCHANGED — still not viable; smallest fp=0 θ on real STRICT is below the cb NEG max for every model, so at any runtime-safe θ the catch on cb NEG is 0/12. But cleaning is now FREE improvement for any soft-signal / advisory / contrastive use of the relevance score. Hand-inspected bottom-10 post-cleaning confirms true- positive deflection signal preserved.
This commit is contained in:
parent
9fc951592d
commit
e4cc3293b5
3 changed files with 264 additions and 4 deletions
|
|
@ -34,6 +34,28 @@ from typing import Optional
|
|||
|
||||
_MANIFEST_PATH = Path(__file__).resolve().parent / "manifest.json"
|
||||
|
||||
# Claim-lattice overlay markup that depresses cross-encoder scores when
|
||||
# fed verbatim (the reranker tries to "score" hash strings + bracket
|
||||
# punctuation). Stripping it before scoring was measured to drop the
|
||||
# real-STRICT FP rate by 22% relative on the §3.2.2 step 2 pool
|
||||
# without losing any true-positive deflection signal (the broad-
|
||||
# question / narrow-answer cases still scored low). #000052 §3.2.2.
|
||||
_CLAIM_LATTICE_BLOCK_RE = re.compile(r"\[E\d+\s*\|[^\]]*\]", re.DOTALL)
|
||||
_TRAILING_BRACKET_TAIL_RE = re.compile(r"\.{2,}[\"'’”]?\s*[\]>]?\s*$")
|
||||
|
||||
|
||||
def clean_for_relevance(text: str) -> str:
|
||||
"""Strip claim-lattice overlay markup ([E\\d+ | title | hash: …])
|
||||
and truncated trailing-bracket tails (..."]) before relevance
|
||||
scoring. These artifacts are verifier overlay, not part of the
|
||||
natural-language answer; scoring them hurts precision. Pure
|
||||
function; safe on already-clean inputs (no-op)."""
|
||||
if not text:
|
||||
return text
|
||||
out = _CLAIM_LATTICE_BLOCK_RE.sub("", text)
|
||||
out = _TRAILING_BRACKET_TAIL_RE.sub("", out)
|
||||
return " ".join(out.split())
|
||||
|
||||
|
||||
def load_manifest(path: Optional[Path] = None) -> dict:
|
||||
return json.loads((path or _MANIFEST_PATH).read_text())
|
||||
|
|
@ -150,7 +172,8 @@ class ShadowRelevance:
|
|||
scores.append(float(row[self._rel_idx] if len(row) > 1 else row[0]))
|
||||
return scores
|
||||
|
||||
def _check(self, query: str, document: str, pair_kind: str) -> ShadowRelevanceResult:
|
||||
def _check(self, query: str, document: str, pair_kind: str,
|
||||
clean_input: bool = True) -> ShadowRelevanceResult:
|
||||
def _result(**kw) -> ShadowRelevanceResult:
|
||||
base = dict(model_version=self.model_version, pair_kind=pair_kind,
|
||||
demote_below_score=self.demote_below_score)
|
||||
|
|
@ -161,6 +184,13 @@ class ShadowRelevance:
|
|||
if not self.available:
|
||||
return _result(available=False, relevance_score=0.0, would_demote=False,
|
||||
reason=self._reason)
|
||||
# Strip claim-lattice overlay markup by default — measured to
|
||||
# drop the real-STRICT FP rate by 22% relative without losing
|
||||
# true-positive deflection signal (#000052 §3.2.2 refinement).
|
||||
# Opt out with clean_input=False if you want the raw score.
|
||||
if clean_input:
|
||||
query = clean_for_relevance(query)
|
||||
document = clean_for_relevance(document)
|
||||
if not (query or "").strip() or not (document or "").strip():
|
||||
return _result(available=True, relevance_score=0.0, would_demote=False,
|
||||
reason="empty_input")
|
||||
|
|
|
|||
221
bench/results/relevance-shadow-sweep-pooled808-cleaned.json
Normal file
221
bench/results/relevance-shadow-sweep-pooled808-cleaned.json
Normal file
|
|
@ -0,0 +1,221 @@
|
|||
{
|
||||
"generated_at": "2026-05-13T18:33:03Z",
|
||||
"strict_pool_files": [
|
||||
"/home/fox/git/arborist/bench/qa_results/2026-05-12T20-53-11Z.jsonl",
|
||||
"/home/fox/git/arborist/bench/qa_results/2026-05-12T21-58-58Z.jsonl",
|
||||
"/home/fox/git/arborist/bench/qa_results/2026-05-12T22-44-30Z.jsonl"
|
||||
],
|
||||
"n_strict_pairs": 808,
|
||||
"models": {
|
||||
"relevance-shadow-v1-bge-reranker-large": {
|
||||
"hf_repo": "BAAI/bge-reranker-large",
|
||||
"backend": "torch",
|
||||
"device": "cpu",
|
||||
"n_pairs": 808,
|
||||
"infer_seconds": 678.5,
|
||||
"score_quantiles_on_strict": {
|
||||
"p0": -2.4174,
|
||||
"p10": 3.5623,
|
||||
"p50": 6.4753,
|
||||
"p90": 7.9334,
|
||||
"p100": 8.7043
|
||||
},
|
||||
"candidate_bench_theta": 4.643,
|
||||
"fp_rate_at_candidate_bench_theta": 0.18564356435643564,
|
||||
"smallest_theta_for_zero_fp_on_pool": -2.4174,
|
||||
"headroom_table": [
|
||||
{
|
||||
"pct_strict_below": 0.0,
|
||||
"theta_at_that_cut": -2.4174
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 1.0,
|
||||
"theta_at_that_cut": -0.2684
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 5.0,
|
||||
"theta_at_that_cut": 2.217
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 10.0,
|
||||
"theta_at_that_cut": 3.5623
|
||||
}
|
||||
]
|
||||
},
|
||||
"relevance-shadow-v1-bge-reranker-base": {
|
||||
"hf_repo": "BAAI/bge-reranker-base",
|
||||
"backend": "torch",
|
||||
"device": "cpu",
|
||||
"n_pairs": 808,
|
||||
"infer_seconds": 195.39,
|
||||
"score_quantiles_on_strict": {
|
||||
"p0": -3.6162,
|
||||
"p10": 3.2004,
|
||||
"p50": 7.6752,
|
||||
"p90": 8.8818,
|
||||
"p100": 10.0034
|
||||
},
|
||||
"candidate_bench_theta": 2.395,
|
||||
"fp_rate_at_candidate_bench_theta": 0.09034653465346534,
|
||||
"smallest_theta_for_zero_fp_on_pool": -3.6162,
|
||||
"headroom_table": [
|
||||
{
|
||||
"pct_strict_below": 0.0,
|
||||
"theta_at_that_cut": -3.6162
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 1.0,
|
||||
"theta_at_that_cut": -1.637
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 5.0,
|
||||
"theta_at_that_cut": 1.2898
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 10.0,
|
||||
"theta_at_that_cut": 3.2004
|
||||
}
|
||||
]
|
||||
},
|
||||
"relevance-shadow-v1-ms-marco-MiniLM-L-6-v2": {
|
||||
"hf_repo": "cross-encoder/ms-marco-MiniLM-L-6-v2",
|
||||
"backend": "torch",
|
||||
"device": "cpu",
|
||||
"n_pairs": 808,
|
||||
"infer_seconds": 25.76,
|
||||
"score_quantiles_on_strict": {
|
||||
"p0": -5.2893,
|
||||
"p10": 6.9279,
|
||||
"p50": 9.3245,
|
||||
"p90": 10.5692,
|
||||
"p100": 11.0746
|
||||
},
|
||||
"candidate_bench_theta": 6.754,
|
||||
"fp_rate_at_candidate_bench_theta": 0.09034653465346534,
|
||||
"smallest_theta_for_zero_fp_on_pool": -5.2893,
|
||||
"headroom_table": [
|
||||
{
|
||||
"pct_strict_below": 0.0,
|
||||
"theta_at_that_cut": -5.2893
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 1.0,
|
||||
"theta_at_that_cut": 4.7953
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 5.0,
|
||||
"theta_at_that_cut": 5.6997
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 10.0,
|
||||
"theta_at_that_cut": 6.9279
|
||||
}
|
||||
]
|
||||
},
|
||||
"relevance-shadow-v1-ms-marco-MiniLM-L-12-v2": {
|
||||
"hf_repo": "cross-encoder/ms-marco-MiniLM-L-12-v2",
|
||||
"backend": "torch",
|
||||
"device": "cpu",
|
||||
"n_pairs": 808,
|
||||
"infer_seconds": 55.14,
|
||||
"score_quantiles_on_strict": {
|
||||
"p0": -3.9779,
|
||||
"p10": 7.5669,
|
||||
"p50": 9.2423,
|
||||
"p90": 10.4212,
|
||||
"p100": 10.8656
|
||||
},
|
||||
"candidate_bench_theta": 7.138,
|
||||
"fp_rate_at_candidate_bench_theta": 0.08044554455445545,
|
||||
"smallest_theta_for_zero_fp_on_pool": -3.9779,
|
||||
"headroom_table": [
|
||||
{
|
||||
"pct_strict_below": 0.0,
|
||||
"theta_at_that_cut": -3.9779
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 1.0,
|
||||
"theta_at_that_cut": 4.9686
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 5.0,
|
||||
"theta_at_that_cut": 5.8329
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 10.0,
|
||||
"theta_at_that_cut": 7.5669
|
||||
}
|
||||
]
|
||||
},
|
||||
"relevance-shadow-v1-ms-marco-MiniLM-L-4-v2": {
|
||||
"hf_repo": "cross-encoder/ms-marco-MiniLM-L-4-v2",
|
||||
"backend": "torch",
|
||||
"device": "cpu",
|
||||
"n_pairs": 808,
|
||||
"infer_seconds": 19.14,
|
||||
"score_quantiles_on_strict": {
|
||||
"p0": -5.9961,
|
||||
"p10": 6.8021,
|
||||
"p50": 8.8673,
|
||||
"p90": 10.3022,
|
||||
"p100": 10.7486
|
||||
},
|
||||
"candidate_bench_theta": 6.736,
|
||||
"fp_rate_at_candidate_bench_theta": 0.09405940594059406,
|
||||
"smallest_theta_for_zero_fp_on_pool": -5.9961,
|
||||
"headroom_table": [
|
||||
{
|
||||
"pct_strict_below": 0.0,
|
||||
"theta_at_that_cut": -5.9961
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 1.0,
|
||||
"theta_at_that_cut": 4.3229
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 5.0,
|
||||
"theta_at_that_cut": 5.3019
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 10.0,
|
||||
"theta_at_that_cut": 6.8021
|
||||
}
|
||||
]
|
||||
},
|
||||
"relevance-shadow-v1-ms-marco-MiniLM-L-2-v2": {
|
||||
"hf_repo": "cross-encoder/ms-marco-MiniLM-L-2-v2",
|
||||
"backend": "torch",
|
||||
"device": "cpu",
|
||||
"n_pairs": 808,
|
||||
"infer_seconds": 9.8,
|
||||
"score_quantiles_on_strict": {
|
||||
"p0": -5.9918,
|
||||
"p10": 5.2108,
|
||||
"p50": 8.5422,
|
||||
"p90": 10.1706,
|
||||
"p100": 10.8884
|
||||
},
|
||||
"candidate_bench_theta": 3.371,
|
||||
"fp_rate_at_candidate_bench_theta": 0.01485148514851485,
|
||||
"smallest_theta_for_zero_fp_on_pool": -5.9918,
|
||||
"headroom_table": [
|
||||
{
|
||||
"pct_strict_below": 0.0,
|
||||
"theta_at_that_cut": -5.9918
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 1.0,
|
||||
"theta_at_that_cut": 1.1027
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 5.0,
|
||||
"theta_at_that_cut": 4.1674
|
||||
},
|
||||
{
|
||||
"pct_strict_below": 10.0,
|
||||
"theta_at_that_cut": 5.2108
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -106,12 +106,14 @@ def main(argv=None) -> int:
|
|||
help="manifest model id(s) comma-sep, or 'all' (default), or 'primary'")
|
||||
ap.add_argument("--out", type=Path,
|
||||
default=REPO / "bench" / "results" / "relevance-shadow-sweep-pooled-strict.json")
|
||||
ap.add_argument("--no-clean", action="store_true",
|
||||
help="skip the claim-lattice metadata cleaning before scoring (see #000052 §3.2.2 refinement: cleaning drops STRICT FP rate ~22%% relative)")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
try:
|
||||
import torch # noqa
|
||||
sys.path.insert(0, str(REPO))
|
||||
from arborist.qa.relevance.shadow import ShadowRelevance, load_manifest
|
||||
from arborist.qa.relevance.shadow import ShadowRelevance, load_manifest, clean_for_relevance
|
||||
except ImportError as e:
|
||||
print(f"[relevance-sweep] missing dependency: {e} (pip install 'arborist[nli]')",
|
||||
file=sys.stderr)
|
||||
|
|
@ -151,8 +153,15 @@ def main(argv=None) -> int:
|
|||
if not rel.available:
|
||||
print(f"[relevance-sweep] skip — {rel._reason}", file=sys.stderr); continue
|
||||
t0 = time.time()
|
||||
# batched scoring: build all (q, a) pairs once
|
||||
pairs = [(p["question"], p["answer"]) for p in strict]
|
||||
# batched scoring: build all (q, a) pairs once, optionally
|
||||
# claim-lattice-cleaned (the §3.2.2 refinement — drops STRICT
|
||||
# FP ~22% relative on the pooled-808 sample without losing
|
||||
# true-positive deflection signal).
|
||||
if args.no_clean:
|
||||
pairs = [(p["question"], p["answer"]) for p in strict]
|
||||
else:
|
||||
pairs = [(clean_for_relevance(p["question"]),
|
||||
clean_for_relevance(p["answer"])) for p in strict]
|
||||
scores = rel._score_batch(pairs)
|
||||
infer_s = time.time() - t0
|
||||
q = _quantiles(scores)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue