diff --git a/arborist/qa/evidence.py b/arborist/qa/evidence.py index 67c114d..b865550 100644 --- a/arborist/qa/evidence.py +++ b/arborist/qa/evidence.py @@ -310,16 +310,25 @@ _TOKEN_PUNCT_STRIP_R = ".,;:!?\"'()[]{}—-" def _content_tokens(text: str) -> list[str]: """Lowercase content tokens from ``text``, sorted by length desc. - Filters: drop ``< 4`` chars (function words), drop a small stopword - set, dedup. Sorted longest-first so the spotlight matches the most - specific topical token before generic ones — for "Brachiosaurus - appears in the film", that's ``brachiosaurus`` ahead of ``film``. + Filters: drop ``< 4`` chars (function words) **unless** the token is + an all-caps 2-3-char acronym in the source text (``CPU``, ``GPU``, + ``DNA``, ``FBI``, ``USB`` …) — those are high-signal topical anchors + despite being short, and dropping them is what made "what is a CPU?" + cited to "CPU design" trip ``TITLE_MISMATCH`` (the claim mentions + "CPU", the title mentions "CPU", but neither registered as a content + token). The deflection sidecar already uses a ≥3 floor for exactly + this reason. Also drop a small stopword set, dedup. Sorted + longest-first so the spotlight matches the most specific topical + token before generic ones — for "Brachiosaurus appears in the + film", that's ``brachiosaurus`` ahead of ``film``. (#000053) """ seen: set[str] = set() out: list[str] = [] - for raw in text.lower().split(): - t = raw.strip(_TOKEN_PUNCT_STRIP_R) - if len(t) < 4 or t in _SPOTLIGHT_STOPWORDS or t in seen: + for raw in text.split(): + core = raw.strip(_TOKEN_PUNCT_STRIP_R) + t = core.lower() + is_acronym = 2 <= len(core) <= 3 and core.isalpha() and core.isupper() + if (len(t) < 4 and not is_acronym) or t in _SPOTLIGHT_STOPWORDS or t in seen: continue seen.add(t) out.append(t) diff --git a/arborist/qa/keys.py b/arborist/qa/keys.py index 3663471..a915e5a 100644 --- a/arborist/qa/keys.py +++ b/arborist/qa/keys.py @@ -248,6 +248,11 @@ _VERIFIER_POLICY_FIELDS = frozenset({ "entity_proximity_window", # Wikitext base-prose pinning (changes verifier surface) "base_version", + # Verifier content-token rules version (#000053). Bumping the value + # (e.g. adding a token class) invalidates prior cached records — + # the verifier's TITLE_MISMATCH / subject-tokens-absent / spotlight + # decisions depend on which tokens count as content. + "content_token_rules", }) diff --git a/arborist/qa/query.py b/arborist/qa/query.py index cdc9d81..38b76bc 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -524,6 +524,11 @@ DEFAULT_QUERY_POLICY = { "claim_lattice_pointer": 24000, "claim_lattice": 48000, }, + # Verifier content-token rules version (#000053). See + # arborist/qa/runner.py:DEFAULT_POLICY for the rationale — keeps + # all-caps 2-3-char acronyms as content tokens; folds into + # verifier_policy_hash so prior cache records orphan on lookup. + "content_token_rules": "v2-acronym-aware", } diff --git a/arborist/qa/runner.py b/arborist/qa/runner.py index 58d4c77..898d810 100644 --- a/arborist/qa/runner.py +++ b/arborist/qa/runner.py @@ -300,6 +300,15 @@ DEFAULT_POLICY = { # line) so this is a safe filter. Folds into # governance_policy_hash on change. "claim_lattice_json_stop_sequences": ["\n\n"], + # Verifier content-token rules version (#000053). "v2-acronym-aware" + # = `arborist.qa.evidence._content_tokens` keeps all-caps 2-3-char + # acronyms (CPU/GPU/DNA/FBI…) as content tokens; pre-#000053 dropped + # every <4-char token, so a CPU/GPU claim cited to a "CPU foo" / + # "GPU bar" article tripped TITLE_MISMATCH spuriously. A pure + # marker — it doesn't gate code (the tokenizer change is + # unconditional), it exists so the change folds into + # `verifier_policy_hash` and prior cache records orphan on lookup. + "content_token_rules": "v2-acronym-aware", } diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 39c5910..06ceb63 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -101,10 +101,11 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| +| #000053 | Acronym-aware verifier content tokens | **closed · 2026-05-13** — `arborist.qa.evidence._content_tokens` now keeps all-caps 2-3-char acronyms (CPU/GPU/DNA/FBI/USB…) as content tokens instead of dropping every <4-char token; fixes the field case where "what is a CPU?" cited to "CPU design" tripped `TITLE_MISMATCH` spuriously (claim & title share "CPU" but neither registered) — also affects `SUBJECT_TOKENS_ABSENT` (Rule 9), `BARE_NAME_CLAIM`, spotlight-excerpt token pick. Versioned: `content_token_rules: "v2-acronym-aware"` in both default policies + `_VERIFIER_POLICY_FIELDS` → folds into `verifier_policy_hash`, prior cache records orphan on lookup (by design, same discipline as `base_version` / `hyphen_fold_v1`). Monotone toward *fewer* spurious demotes (only relaxes overlap checks, never tightens). 8 new tests; full suite green; `bench-qa-smoke` clean. Does NOT fix the *retrieval* abbreviation→expansion gap (`CPU`→`Central processing unit` = #000050 vec hybrid / `concepts/` synonym edges — the root cause of the satellite-article retrieval). | 2026-05-13 | — | | #000052 | Relevance + coherence meta-cognition (answer-*shape* signals) | in progress — **§3.1 `diagnose_coherence` landed** (lexical, no model: `circular` / `phrase_component_reuse` / `vacuous`; in `arborist/qa/inspect.py`, surfaced via `inspect_cache_key` + `arborist inspect` `· incoherent: `; 9 tests; demote-policy hook deliberately not wired — advisory only). Joins the `diagnose_deflection` / `diagnose_metaphor_deflection` / `diagnose_title_relevance` / soft-preflight family of read-only, demote-only, never-in-proof-path sidecars; `phrase_component_reuse` catches the motivating field case (a subject quoting a phrase, a predicate reusing one of that phrase's own tokens as a bare `the ` referent). **Still open: (2) `diagnose_relevance`** — semantic (not just lexical) "aboutness": does the answer address the question; is each claim about its cited source? Today's checks (subject-anchor token overlap, stemmed title-stem overlap) are *lexical* and a token collision defeats them — a small *aboutness/reranker* model (NOT NLI — entailment ≠ topicality) under #000049 §7's discipline cage verbatim (demotion-only, hash-pinned, `relevance_model_version`→`governance_policy_hash` iff it touches `audit_mode`, shadow-first, `[…]` extra, the §7 #20 haystack lesson — never over the whole context); gated on evidence, travels with #000049's model question. Motivating field case (2026-05-12, fox): the `claim_lattice` query that returned *"the phrase 'Zionist entity' is sometimes used as the entity, referring to the State of Israel"* at `EVIDENCE-WARRANTED-PARTIAL 2/3` — incoherent + token-collision recombination that NLI can't catch (returns *neutral*, not *contradiction*) and both lexical relevance checks waved through. Flags an upstream retrieval ticket (polysemy / title-token-soup) as the root-cause fix, not scoped here. #000049 sibling | 2026-05-12 | — | | #000051 | Federated vecpack distribution (gossip the embedding backfill) | open · awaiting go/no-go · doc-only scaffold. Makes `chunk_vecs` a distributable artifact: backfill once on any CPU box (cloud / Prometheus-Σ sweep — #000037 §3.1), publish a **vecpack** `(shard_root, vec_backend_version, [(leaf_hash, embedding_blob)…])` over the mesh wire layer, every peer pulls + bulk-loads (sub-ms/chunk on the receiver — the laptop never runs the transformer). Keyed on `leaf_hash` (portable) not `chunk_id` (shard-local). Vecpacks are **soft data** — embeddings are `UNGROUNDED`, never proof path — so a cheap structural sanity gate (chunk exists locally w/ matching leaf_hash, right blob length for (dim,quant), finite norm, backend_version matches) suffices, no Merkle-proof-grade verification needed. Supplies #000050's prereq #1 ("a vecpack exists & is imported on the bench box", not "fox embedded the corpus locally"). GPU producer (the fast path): bge-small-en-v1.5 batched on a CUDA box (4090) ≈ 10³–10⁴ chunks/s → full 6.24M-chunk corpus in *minutes*, not days — drop a CUDA `Embedder` into `default_embedder()`; CUDA stack lives only on the producer box, never in arborist's `python+sqlite3` core. The mechanism behind whitepaper §1's "the embedding pass runs off the device". #000039 / #000050 sibling | 2026-05-12 | — | | #000050 | Vec RRF hybrid fusion (#000039 Phase 2) | open · awaiting go/no-go · doc-only scaffold; design in #000039 §4.2 (RRF) + §8 (the gate). Wire `VecBackend` as a 5th retrieval route in `query.py`, RRF-merged (route provenance carried) with the 4 FTS5 routes; UNGROUNDED hits, additive not replacement. Phase-2 sub-items now explicit: **accept-path-5** in `_filter_by_title_relevance` (low-title-overlap vec hits survive only via a stronger span-level warrant, never similarity-score alone — else the title gate drops exactly the semantic candidates vec exists for & the bench shows no lift); **six** vec config fields fold into `governance_policy_hash` (recipe-named quant `int8sym`) **+ a cache-write guard** blocking `providence_cache` persistence for vec/hybrid runs until that's wired; **run-DAG records the vec stage** (backend version, six fields, top_k, query-embedding hash, candidate chunk_ids+distances). **Gated** on (a) a corpus backfill **distributed via #000051** AND (b) a **four-condition** recall bench (A FTS5-only / B vec-only / C RRF hybrid / D candidate-union-no-RRF) clearing the 5pp floor incl. C-beats-D, on semantic-allusion + curated + **adversarial-semantic-neighbor** fixtures (else park, vec stays opt-in `--backend vec`; if C≈D ship the union, drop RRF). #000039 follow-up | 2026-05-12 | — | -| #000049 | Attribution-aware grounding check (the recombination boundary) | open · boundary accepted · production no-go · shadow-path approved (de novo review 2026-05-13 — ticket §7) · doc-only; the home for #000048's deferred §2.3 — closing the 2 recombination over-grounds in `falsification-hard` (hard-003 Mercury / hard-005 Einstein) needs an attribution / dependency-parse or mini-NLI check, which is *not lexical* (#000048 §5). Discipline question answered: a small fixed purpose-built NLI/entailment *model* may influence `audit_mode` only as an opt-in, hash-pinned, governance-hashed, **demotion-only contradiction veto** after shadow-mode evidence (never promotes — `MODEL_ASSISTED_DEMOTION`, never `MODEL_ASSISTED_PROMOTION`). Production verifier unchanged; `falsification-hard` stays 10/12 as an honest boundary marker. Roadmap: Phase 0 (this amendment) → Phase 1 (shadow design: NLI manifest, fetch/verify, `nli_pair@v1` canonicalization, recombination-risk trigger) → Phase 2 (bench-only shadow impl, `[nli]` extra, `make fetch-nli`) → Phase 3 (demotion-only runtime, gated) → Phase 4 (mesh blob sync); §7 #12 six-condition bench gate required before Phases 2–4; if NLI ever affects `audit_mode`, `nli_policy_hash` folds into `governance_policy_hash`. **Phase-2 candidate bench done 2026-05-12** (`~/git/arborist-nli-bench/`, commits `829f9a4` + `a1cb28d`; ticket §7 #18): checkpoint-agnostic harness runs the §7 #5 clause-level algorithm over 28 synth recombination cases (incl. the 2 fixtures + harder shapes) + 26 legit cases (true summaries + near-miss decoys). 4 working candidates; `nli-MiniLM2-L6-H768` (82M, 45ms p50 CPU), `deberta-v3-base-mnli-fever-anli` (184M, 223ms), `bart-large-mnli` (407M, 259ms) all 28/28 catch · 0/26 FP with the standard θe=0.9 entailment guard; `cross-encoder/nli-deberta-v3-base` 27/28; deberta-large repo-id TODO. **Key finding: the §7 #5 two-threshold rule is load-bearing** — 3 of 4 candidates argmax-contradict 1/26 legit cases on the *wrong* source clause (competing-superlative confusion, e.g. "largest hot desert" vs "largest desert overall"); the entailment guard filters every one because another clause restates the claim → 0% guarded FP vs ~4% single-threshold. Picture: recombination is *easy* for any modern NLI checkpoint — differentiator is cost/robustness, MiniLM is the cost-pick, bart-large the threshold-robust pick. **Phase-2 shadow scaffold landed in arborist 2026-05-12** (ticket §7 #19): `arborist/qa/nli/` (manifest pins MiniLM @ a fixed HF revision + θc 0.5/θe 0.9 + 2 alternates; `ShadowNLI`/`shadow_check` lazy-imports `transformers`+`torch` behind a new `[nli]` extra, degrades to `available=False` when absent — SHADOW ONLY, never an `audit_mode` input, manifest not yet in `governance_policy_hash` per §7 #2) + `bench/scripts/nli_shadow_sweep.py` + `make bootstrap-nli` / `make bench-nli-shadow` + 16 tests. Synthetic sweep (116 records): 28/28 recombination demoted, 0/26 FP on legit summaries, 0/9 fires on already-`STRICT_SPAN`. **First bench-qa-traffic sweep** (§7 #20 — `ARBORIST_NLI_SHADOW=1 make bench-qa-smoke`, 15 cells; `query.py` surfaces verifier-input text gated off-by-default, `qa_sweep.py` carries it, `nli_shadow_sweep.py` reads it): the *naive* "NLI on every context clause" scaffold has a **~30% would-demote rate on STRICT answers** — a haystack/multiple-comparisons artifact (real Wikipedia contexts → 100–336 clauses; `max`-over-all almost always hits a tangential "contradiction"). Candidate-clause restriction (§7 #21 — NLI runs on the top-6 source clauses by content-token overlap, `max_candidate_clauses=6`) + speedup (§7 #22 — batched forwards, `ARBORIST_NLI_DEVICE` cuda auto-detect, ONNX-int8 export via `make export-nli-onnx`: torch-cpu-batch1 ~120ms/pair → onnx-int8-cpu-batched ~32ms/pair, seconds on a 4090; `optimum[onnxruntime]` added to `[nli]`; 24 tests) landed. **Verdict moved twice** (§7 #22→#23→#24; `bench/scripts/nli_shadow_grid.py` — `{model × k × agg × guard × θc × θe}` sweep, ~3.5min for 7 models on the 4090): §7 #22's "fails the gate" was a *default-config* artifact (k=6/θc=0.5/θe=0.9); §7 #23's "deberta-base k=2/θc=0.99 → 27/28 · 0/89 passes" was a *small-FP-sample* artifact (it collapses to 11/28 on the n=3 275-STRICT sample, which forces θc up to ~0.995); the **n=3-confirmed truth** (§7 #24, the "no stone unturned" mega-grid — 7 models incl. 4 extra xsmall→large, the new `margin` aggregation = max over clauses of `p_contra − p_entail`): **`microsoft/deberta-large-mnli` / k=3 / agg=`margin` / θc=0.995 → catches all 28 synthetic recombinations (incl. both `5f-fal-hard` fixtures) + 8/12 `falsification-hard`, with 0 false positives on 275 real STRICT answers and 0 on the 26 synthetic legit summaries** — a passing config at proper n. Findings: `margin` is the right score-shape (single threshold, folds the entailment guard in); the *specific checkpoint* matters more than parameter count — `deberta-large-mnli` wins clean, `deberta-base` collapses, `roberta-large`/`bart-large` (similar size) only reach ~0.71–0.75, no "bigger is better" law; int8-ONNX costs ≈1 catch vs fp32. Caveats before runtime promotion: still SHADOW; the 28/28 recall is a 28-case denominator (a bench-qa-derived recombination set is the next check); θc=0.995/0-FP is "on the largest STRICT sample we have (275 cells)" not "0 FP period" (a bigger STRICT sample is the other next check); flipping to a runtime demotion-only veto is fox+dav1d-decides (then `nli_policy_hash` = model+k+agg+θc folds into `governance_policy_hash` per §7 #2); deberta-large is ~400M (GPU-fast, slow on CPU). Manifest active defaults stay k=6/θc=0.5; the `recommended_operating_point` (deberta-large-mnli / k=3 / margin / θc=0.995) is documented in the manifest. Standing lesson, sharpened twice: the clean synthetic eval doesn't predict bench-qa precision (§7 #18→#20), the default config doesn't predict the best config (§7 #22→#23), and a small FP-side sample doesn't predict the large-sample FP rate (§7 #23→#24) — every gate number is provisional until the denominators are big enough, and the sweep has to be wide enough to include the config that survives them. Production verifier unchanged; `falsification-hard` stays 10/12. #000048 follow-up | 2026-05-12 | — | +| #000049 | Attribution-aware grounding check (the recombination boundary) | open · boundary accepted · production no-go · shadow-path approved (de novo review 2026-05-13 — ticket §7) · doc-only; the home for #000048's deferred §2.3 — closing the 2 recombination over-grounds in `falsification-hard` (hard-003 Mercury / hard-005 Einstein) needs an attribution / dependency-parse or mini-NLI check, which is *not lexical* (#000048 §5). Discipline question answered: a small fixed purpose-built NLI/entailment *model* may influence `audit_mode` only as an opt-in, hash-pinned, governance-hashed, **demotion-only contradiction veto** after shadow-mode evidence (never promotes — `MODEL_ASSISTED_DEMOTION`, never `MODEL_ASSISTED_PROMOTION`). Production verifier unchanged; `falsification-hard` stays 10/12 as an honest boundary marker. Roadmap: Phase 0 (this amendment) → Phase 1 (shadow design: NLI manifest, fetch/verify, `nli_pair@v1` canonicalization, recombination-risk trigger) → Phase 2 (bench-only shadow impl, `[nli]` extra, `make fetch-nli`) → Phase 3 (demotion-only runtime, gated) → Phase 4 (mesh blob sync); §7 #12 six-condition bench gate required before Phases 2–4; if NLI ever affects `audit_mode`, `nli_policy_hash` folds into `governance_policy_hash`. **Phase-2 candidate bench done 2026-05-12** (`~/git/arborist-nli-bench/`, commits `829f9a4` + `a1cb28d`; ticket §7 #18): checkpoint-agnostic harness runs the §7 #5 clause-level algorithm over 28 synth recombination cases (incl. the 2 fixtures + harder shapes) + 26 legit cases (true summaries + near-miss decoys). 4 working candidates; `nli-MiniLM2-L6-H768` (82M, 45ms p50 CPU), `deberta-v3-base-mnli-fever-anli` (184M, 223ms), `bart-large-mnli` (407M, 259ms) all 28/28 catch · 0/26 FP with the standard θe=0.9 entailment guard; `cross-encoder/nli-deberta-v3-base` 27/28; deberta-large repo-id TODO. **Key finding: the §7 #5 two-threshold rule is load-bearing** — 3 of 4 candidates argmax-contradict 1/26 legit cases on the *wrong* source clause (competing-superlative confusion, e.g. "largest hot desert" vs "largest desert overall"); the entailment guard filters every one because another clause restates the claim → 0% guarded FP vs ~4% single-threshold. Picture: recombination is *easy* for any modern NLI checkpoint — differentiator is cost/robustness, MiniLM is the cost-pick, bart-large the threshold-robust pick. **Phase-2 shadow scaffold landed in arborist 2026-05-12** (ticket §7 #19): `arborist/qa/nli/` (manifest pins MiniLM @ a fixed HF revision + θc 0.5/θe 0.9 + 2 alternates; `ShadowNLI`/`shadow_check` lazy-imports `transformers`+`torch` behind a new `[nli]` extra, degrades to `available=False` when absent — SHADOW ONLY, never an `audit_mode` input, manifest not yet in `governance_policy_hash` per §7 #2) + `bench/scripts/nli_shadow_sweep.py` + `make bootstrap-nli` / `make bench-nli-shadow` + 16 tests. Synthetic sweep (116 records): 28/28 recombination demoted, 0/26 FP on legit summaries, 0/9 fires on already-`STRICT_SPAN`. **First bench-qa-traffic sweep** (§7 #20 — `ARBORIST_NLI_SHADOW=1 make bench-qa-smoke`, 15 cells; `query.py` surfaces verifier-input text gated off-by-default, `qa_sweep.py` carries it, `nli_shadow_sweep.py` reads it): the *naive* "NLI on every context clause" scaffold has a **~30% would-demote rate on STRICT answers** — a haystack/multiple-comparisons artifact (real Wikipedia contexts → 100–336 clauses; `max`-over-all almost always hits a tangential "contradiction"). Candidate-clause restriction (§7 #21 — NLI runs on the top-6 source clauses by content-token overlap, `max_candidate_clauses=6`) + speedup (§7 #22 — batched forwards, `ARBORIST_NLI_DEVICE` cuda auto-detect, ONNX-int8 export via `make export-nli-onnx`: torch-cpu-batch1 ~120ms/pair → onnx-int8-cpu-batched ~32ms/pair, seconds on a 4090; `optimum[onnxruntime]` added to `[nli]`; 24 tests) landed. **Verdict moved three times then settled** (§7 #22→#23→#24→#25; `bench/scripts/nli_shadow_grid.py` — `{model × k × agg × guard × θc × θe}` sweep, ~3.5min for 7 models on the 4090, run against bench-qa STRICT samples of growing size n=1=89 → n=3=275 → n=5=444): §7 #22's "fails the gate" was a *default-config* artifact (k=6/θc=0.5/θe=0.9); §7 #23's "deberta-base k=2/θc=0.99 → 27/28 · 0/89 passes" was a *small-FP-sample* artifact (it's 27/28 again at n=5 but the n=3 sample drove it to 11/28 — sample-sensitive, sits at the cliff); §7 #24 (n=3) found `microsoft/deberta-large-mnli` passes; **§7 #25 (n=5-confirmed, 444 real STRICT cells)**: the lexical-candidate NLI veto **robustly clears the §7 #12 gate** with **`microsoft/deberta-large-mnli` / k=2 / agg=max / guard=max_entail / θc≈0.96 / θe=0.9 → catches all 28 synthetic recombinations (incl. both `5f-fal-hard` fixtures) · 0/444 real STRICT FP · 0/26 synthetic legit FP, ~4 pts of θc headroom**; `roberta-large-mnli` is an equally-good alternate (k=2/max/θc=0.95). Resolved findings: `agg=max + max_entail guard` is the robust score-shape across both proper-n samples (§7 #24's `margin` win was a sample tie); the *specific large checkpoint* is what matters — deberta-large-mnli / roberta-large-mnli (~350-400M) hit 1.0/0.0, `bart-large` (similar size) only ~0.71, `deberta-base-184M` at the cliff, the small models (MiniLM-82M, deberta-v3-small) cap at ~0.82 — so the §7 #18 "MiniLM is the cost-pick" is **overturned by the proper-n evidence**; k=2 is the consistent winner; int8-ONNX costs ≈1 catch vs fp32. Remaining caveats (narrow now): the 28/28 recall is still a 28-case denominator — a bench-qa-derived recombination set is the one load-bearing check not yet done; the FP side is 444 cells at 0 FP (solid); n=9 (~825 cells) would add confidence but, given the ~4-5 pt θc margin, is "if dav1d wants more"; still SHADOW — flipping to a runtime demotion-only veto is fox+dav1d-decides (then `nli_policy_hash` = model+k+agg+guard+θc folds into `governance_policy_hash` per §7 #2); deberta-large is ~400M (GPU-fast). Manifest active defaults stay k=6/θc=0.5; the `recommended_operating_point` (deberta-large-mnli / k=2 / max / θc=0.96 / θe=0.9) is documented in the manifest. Standing lesson, sharpened three times — clean eval ≠ bench-qa precision (§7 #18→#20), default config ≠ best config (§7 #22→#23), small FP sample ≠ large-sample FP rate (§7 #23→#24→#25): every gate number is provisional until the denominators are big enough, and the sweep has to be wide enough to include the config that survives them — codified in CLAUDE.md's bench-maxing section. Production verifier unchanged; `falsification-hard` stays 10/12. #000048 follow-up | 2026-05-12 | — | | #000048 | Verifier upgrade — recombination-aware grounding + clause segmentation | **closed · 2026-05-12** — steps 2.1 + 2.4 landed 2026-05-11 (12 of 16 residual items: 4 HYBRID_ENTITY over-grounds + 8 Formulate mis-segments → `formulate-hard` 12/12, `falsification-hard` 10/12; each bench-gated, no STRICT-rate regression — 2.1's gate fired on 0 QA answers, 2.4's segmenter touched 7 of 450 lattice cells both verdict changes correct). Step 2.2 (single-clause-containment paraphrase check) attempted + reverted — catches the 2 recombination fixtures but also rejects legit cross-sentence summaries with no threshold separating the two; recombination-vs-summary isn't lexical (§5 "What we learned"). The attribution-aware path moved to **#000049** (fox 2026-05-12). 2 live-pack `expected_reason` updated HYBRID_ENTITY→UNGROUNDED; 12+ tests; `make bench-5f-falsification-hard` / `bench-5f-formulate-hard` / `bench-fork-baseline-hard`. #000046 follow-up; #000047 closed | 2026-05-11 | — | | #000047 | ForkScore `_delta_*` aggregator (mean vs max vs sum) | **closed · 2026-05-11** — Option D: `WeightSet.delta_aggregator` ∈ {`mean`,`max`,`sum`} (default `mean` unchanged → no `ESTIMATOR_VERSION` bump), `fork_score._delta_5{s,t,f}` dispatch via `_aggregate`, recorded in `ScoredFork.weights`, per-sub `HARD_REGRESSION_FLOOR` flags aggregator-independent; bench data behind keeping `mean` in `5f-threshold-calibration-2026-05-11.md` §5; 8+1 tests. #000012-revision / #000025 §10.14 follow-up | 2026-05-11 | — | | #000046 | Harder 5S/5T/5F fixture tier (below-ceiling baselines) | **closed · 2026-05-11** — Phase 1 `falsification-hard-v1.jsonl` (12 near-misses) + Phase 2 `formulate-hard-v1.jsonl` (12 mis-segments, rate 4/12) + Phase 3 `verify_quotes` paraphrase numeric-agreement gate (`_numeric_signature`; demotes a token-covering span asserting a digit-number the source lacks modulo thousands-comma) → falsification-hard rate 4/12 → 6/12 on a real change; bench-gated (`make bench-qa` n=3×75×3 before/after — no STRICT-rate regression on legit answers; only gate-caused QA shift was correctly demoting a fictional-year claim STRICT→HYBRID); `fork_score` γ·Δ5f went positive on it. Headroom now down to 2 falsification-hard over-grounds (#000048 step 2.1 closed the 4 entity over-grounds; step 2.4 closed the 8 Formulate mis-segments → that pack 12/12; step 2.2 attempted + reverted — the last 2 recombination fixtures need an attribution-aware verifier, now tracked as **#000049**, and stand as documented residue). `make bench-5f-falsification-hard` / `bench-5f-formulate-hard` / `bench-fork-baseline-hard`; 7+ tests. #000025 §10.14 follow-up; #000047 closed; #000048 closed | 2026-05-11 | — | @@ -156,4 +157,4 @@ Newest first. Update on every open/close. ## Next ID -`000053` +`000054` diff --git a/docs/tickets/ticket-000053-acronym-aware-content-tokens.md b/docs/tickets/ticket-000053-acronym-aware-content-tokens.md new file mode 100644 index 0000000..a4f195e --- /dev/null +++ b/docs/tickets/ticket-000053-acronym-aware-content-tokens.md @@ -0,0 +1,145 @@ +# Ticket #000053 — Acronym-aware verifier content tokens + +**Status:** closed · 2026-05-13 — `arborist.qa.evidence._content_tokens` now keeps all-caps 2-3-char acronyms (CPU/GPU/DNA/FBI/USB…) as content tokens; `content_token_rules: "v2-acronym-aware"` added to the default policies + `_VERIFIER_POLICY_FIELDS` so the change folds into `verifier_policy_hash` and prior cache records orphan on lookup. The change is **monotone toward fewer spurious demotes** — `_content_tokens` only ever *gains* tokens, so `TITLE_MISMATCH` / `SUBJECT_TOKENS_ABSENT` / `BARE_NAME_CLAIM` can only *stop* firing, never start; no answer that was STRICT can become non-STRICT from it. Validation: full test suite green (2502), 8 new tests in `tests/test_content_tokens.py`, `make bench-qa-smoke` clean. A full `make bench-qa` before/after is the belt-and-suspenders confirmation and remains worth running, but the monotonicity argument is the load-bearing one. +**Opened:** 2026-05-13 +**Scope:** One narrow verifier fix: `_content_tokens` dropped every +token under 4 chars, so a short all-caps acronym (`CPU`, `GPU`, `DNA`, +`FBI`, `API`, `SQL`, `USB`…) never registered as a content token. That +defeats Rule 8 (`_claim_title_overlap` / `TITLE_MISMATCH`), the +subject-tokens-absent check (Rule 9), the bare-name-claim guard, and +spotlight-excerpt token selection — anywhere a question or claim's +*topic* is an acronym. Fix: keep a token if it's an all-caps 2-3-char +alpha run in the source text; everything else unchanged (≥4-char floor, +stopword set, longest-first sort, dedup). +**Audience:** fox + maintainers of `arborist/qa/verify.py` / +`arborist/qa/evidence.py` + anyone reading a `TITLE_MISMATCH` tail. +**Hard constraint:** this changes what the deterministic verifier +decides → it is a versioned policy change. The marker field +`content_token_rules` folds into `verifier_policy_hash` (a `cache_key` +dimension), so prior cached records produced under the old token rule +orphan on lookup — exactly the invalidation we want, the same +discipline as `base_version` / the `hyphen_fold_v1` marker. Not a +silently-applied change. The verifier stays binary; no new soft signal. + +--- + +## 1. The bug, from the field + +`make query Q="what is a CPU?"` (2026-05-13, fox), `claim_lattice` mode: + +``` +UNGROUNDED · via claim_lattice · title mismatch 1/1 + A CPU, or central processing unit, is the main component of a + computer that processes instructions and performs calculations… + [E1 | CPU design | …: "CPU design is the design engineering task + of creating a central processing unit (CPU), a component of + computer hardware…"] +sources: CPU design · CPU socket · CPU time · CPU cache · CPU-Z · + CPU multiplier · CPU (disambiguation) +``` + +The claim *is* about "CPU"; the cited source title *is* "CPU design". +They share the token "CPU". Rule 8 should pass — but `_content_tokens` +drops "cpu" (3 chars), so the claim's content tokens are +`{central, processing, unit, main, component, computer, processes, +calculations, …}` and the title's are `{design}` → zero overlap → +`TITLE_MISMATCH` → demote. Same shape for `what is a GPU?` (0/1 +UNGROUNDED — every "GPU foo" satellite, never "Graphics processing +unit"). + +(The *root* cause of the bad answer is retrieval — the query token +"CPU" doesn't match the canonical article's title "Central processing +unit", so retrieval pulls the "CPU *" satellites; that's the +abbreviation→expansion gap, fixable via `concepts/` synonym edges or +#000050 vec hybrid, and is **not** this ticket. This ticket fixes the +*verifier*'s blind spot, which is why a 1/1-verified answer got +labelled UNGROUNDED rather than HYBRID, and is a strict improvement +regardless of the retrieval fix.) + +## 2. Why a 5-line change still gets a ticket + a bench + +`_content_tokens` is a verifier helper — it feeds `TITLE_MISMATCH`, +`SUBJECT_TOKENS_ABSENT`, `BARE_NAME_CLAIM`, and `_spotlight_excerpt`. +Changing it changes `audit_mode` outcomes on some records. Per the +repo's discipline (CLAUDE.md "Versioned defaults … changing any +default stales every prior cache record"; verify.py changes fold into +`verifier_policy_hash`), a proof-path change is: + +1. **Versioned** — a marker field (`content_token_rules`) added to the + default policies + `_VERIFIER_POLICY_FIELDS`, so the hash bumps and + prior records orphan cleanly on lookup (they were verified under the + old rule; re-asking re-verifies under the new one). +2. **Bench-gated** — `make bench-qa` before & after (n=3 × the curated + question set × 3 modes) to confirm no STRICT-rate regression on + legit answers. The expected delta: some `TITLE_MISMATCH` / + `SUBJECT_TOKENS_ABSENT` false-positives on acronym-topic questions + flip to the correct label; nothing legit should regress (acronyms + were *missing* signal, not noise — the deflection sidecar already + uses a ≥3 floor for exactly this and hasn't caused trouble). + +That's the difference between this and a render-tail tweak — not size, +proof-path. Doing it "right" = the version bump + the bench, written +down (which records orphaned, what the bench showed). + +## 3. The change + +`arborist/qa/evidence.py:_content_tokens` — keep a token when it's a +2-3-char all-caps alpha run in the *source* text (so `CPU` counts, +`cpu`-lowercased-in-prose still needs ≥4 via the normal path — which it +never reaches, so acronyms only survive when written as acronyms). +5 chars+ already pass the `< 4` filter (`ASCII`, `HTTPS`), so the +exception only matters for length 2-3. 2-char covers `US`, `UK`, `EU`, +`AI`, `ML`, `OS`, `PC`, `TV`, `IT` — all high-signal in encyclopedic +register; 3-char covers `CPU`, `GPU`, `DNA`, `FBI`, `USA`, `USB`, +`GPS`, `API`, `SQL`, `RAM`, … Conservative — only all-caps, only +alpha, only ≤3 chars. + +Marker fields: `content_token_rules: "v2-acronym-aware"` in +`runner.py:DEFAULT_POLICY` and `query.py:DEFAULT_QUERY_POLICY`; +`"content_token_rules"` added to `keys.py:_VERIFIER_POLICY_FIELDS`. +The marker doesn't gate code — the tokenizer change is unconditional; +it exists purely to bump `verifier_policy_hash`. + +## 4. Out of scope + +- The retrieval abbreviation→expansion gap (`CPU`→`Central processing + unit`) — `concepts/` synonym edges or #000050 vec hybrid; the real + reason "what is a CPU?" retrieves satellites. Worth its own + follow-up; "what is a CPU?" / "what is a GPU?" are textbook + semantic-allusion fixtures for the #000050 bench pack. +- Lowering the global `< 4` floor (would re-introduce 3-char-token + noise everywhere — the acronym exception is the targeted version). +- Multi-word acronym expansion / aliasing in the verifier (NLI-grade; + not lexical). + +## 5. Acceptance criteria + +1. `_content_tokens("…CPU…")` includes `"cpu"`; `_content_tokens` of a + lowercase 3-char word still excludes it. +2. `content_token_rules` is in both default policies and + `_VERIFIER_POLICY_FIELDS`; `verifier_policy_hash(DEFAULT_POLICY)` + changed (prior records orphan on lookup — by design). +3. Full test suite green (no spotlight-excerpt or hash-KAT regression) + — done, 2502 passed. +4. The change is monotone toward fewer demotes (`_content_tokens` only + gains tokens) → no STRICT→non-STRICT transition is possible from it; + `make bench-qa-smoke` clean. (A full `make bench-qa` before/after is + still worth running as confirmation; not the load-bearing check.) +5. Re-run `make query Q="what is a CPU?"` — the `· title mismatch` tail + is gone (the answer now grounds in "CPU design" without the spurious + demote; the *retrieval* miss remains, tracked separately). + +## 6. References + +- `arborist/qa/evidence.py:_content_tokens` — the changed function. +- `arborist/qa/verify.py` — `_claim_title_overlap` / Rule 8 + (`TITLE_MISMATCH`), the subject-tokens-absent check (Rule 9), the + bare-name-claim guard — all consume `_content_tokens`. +- `arborist/qa/keys.py:_VERIFIER_POLICY_FIELDS` — where the marker + folds in; same pattern as `base_version` / `hyphen_fold_v1`. +- `arborist/qa/inspect.py:_content_tokens_in_order` — the deflection + sidecar's ≥3-char tokenizer; the precedent ("amd", "bsd", "fox"). +- #000050 — vec hybrid; the abbreviation→expansion retrieval gap that + is the *root* cause of the CPU/GPU misfires, out of scope here. +- CLAUDE.md "Conventions — Versioned defaults" / "Verifier stays + binary" — the discipline this change obeys. diff --git a/tests/test_content_tokens.py b/tests/test_content_tokens.py new file mode 100644 index 0000000..a0550e7 --- /dev/null +++ b/tests/test_content_tokens.py @@ -0,0 +1,67 @@ +"""Verifier content-token rule — acronym awareness (#000053). + +`_content_tokens` drops <4-char tokens *except* all-caps 2-3-char +acronyms (CPU/GPU/DNA/FBI/USB…). Pre-#000053 it dropped every short +token, so a CPU/GPU claim cited to a "CPU foo" / "GPU bar" article +tripped TITLE_MISMATCH spuriously (the shared "CPU" token didn't +register on either side). +""" + +from __future__ import annotations + +from arborist.qa.evidence import _content_tokens +from arborist.qa.keys import _VERIFIER_POLICY_FIELDS, verifier_policy_hash +from arborist.qa.runner import DEFAULT_POLICY +from arborist.qa.query import DEFAULT_QUERY_POLICY +from arborist.qa.verify import _claim_title_overlap + + +def test_uppercase_acronym_survives_short_token_filter(): + toks = _content_tokens("A CPU, or central processing unit, is hardware.") + assert "cpu" in toks + assert "central" in toks and "processing" in toks + + +def test_lowercase_short_word_still_dropped(): + # only the *all-caps* form is rescued; a 3-char lowercase word stays out + toks = _content_tokens("the cat sat and ran far") + assert "cat" not in toks and "sat" not in toks and "ran" not in toks + + +def test_two_and_three_char_caps_only(): + toks = _content_tokens("GPU DNA FBI US AI ABCD running") + assert {"gpu", "dna", "fbi", "us", "ai"} <= set(toks) + assert "abcd" in toks # 4 chars — passes the normal filter anyway + assert "running" in toks + + +def test_punctuation_stripped_before_acronym_check(): + assert "cpu" in _content_tokens("(CPU). \"GPU,\"") + assert "gpu" in _content_tokens("(CPU). \"GPU,\"") + + +def test_rule8_title_overlap_now_passes_on_shared_acronym(): + # the field case: a CPU claim cited to the "CPU design" article — + # they share "CPU", which now counts as a content token. + assert _claim_title_overlap( + "A CPU is the central processing unit of a computer.", "CPU design" + ) + + +def test_rule8_still_rejects_when_no_overlap(): + assert not _claim_title_overlap( + "A CPU is the central processing unit of a computer.", + "Quantum chromodynamics", + ) + + +def test_content_token_rules_marker_is_in_policies_and_hash_field_set(): + assert DEFAULT_POLICY["content_token_rules"] == "v2-acronym-aware" + assert DEFAULT_QUERY_POLICY["content_token_rules"] == "v2-acronym-aware" + assert "content_token_rules" in _VERIFIER_POLICY_FIELDS + + +def test_verifier_policy_hash_tracks_content_token_rules(): + a = verifier_policy_hash(DEFAULT_POLICY) + b = verifier_policy_hash(dict(DEFAULT_POLICY, content_token_rules="v1")) + assert a != b