From 37ac523bba769efc75c31ac5a5c9893a2ee0146b Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 29 Apr 2026 21:28:28 -0400 Subject: [PATCH] =?UTF-8?q?qa:=20lazy-anchor=20smell=20sidecar=20=E2=80=94?= =?UTF-8?q?=20surface=20when=20one=20pointer=20carries=20every=20claim?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit verify_claim_lattice now derives pointer_id_distribution and lazy_anchor_ratio from the verified-or-partial claim_statuses and returns them in the verdict. Surfaced through ask()/query() result dicts; CLI human-render displays a one-liner when ratio >= 0.5 (threshold for "model is lazy-anchoring"). Sidecar discipline preserved: never persisted in providence_cache, never threaded into build_run_dag's verify_payload (test pins this invariant — sidecar values must not enter run_dag_root). The distribution is recoverable from claim_statuses which IS persisted, so future sidecar tools can recompute on demand without enlarging the audit chain. JP dinosaurs benchmark currently triggers the warning at 11/11 ratio 1.00 — Hermes-3-8B cites [E1] for every claim even with G0.3's relevance ranking putting the dinosaur-mention chunk near the top. The render-layer signal now makes that lazy-anchor behavior explicit instead of leaving it for the auditor to spot from the spotlight excerpts. Tests: +4 covering the ratio across distributions (single-anchor / diversified / no-verified-pairs) plus the run_dag isolation invariant. Total 442 passing. --- aborist/cli.py | 21 +++++++++++++++++ aborist/qa/query.py | 5 ++++ aborist/qa/runner.py | 4 ++++ aborist/qa/verify.py | 26 ++++++++++++++++++++ tests/test_claim_lattice.py | 47 +++++++++++++++++++++++++++++++++++++ 5 files changed, 103 insertions(+) diff --git a/aborist/cli.py b/aborist/cli.py index b913438..d27907d 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -467,6 +467,27 @@ def _render_query_human(result: dict, question: str) -> str: lines.append(f' - "{qtxt}"') lines.append("") + # Anchor-smell sidecar (claim_lattice mode only). Soft signal — + # never persisted, never in cache_key. Surfaces when ≥50% of + # verified claim-pointer pairs share one pointer_id, which is + # the lazy-anchor failure mode the spotlight render also exposes. + ratio = result.get("lazy_anchor_ratio") + distribution = result.get("pointer_id_distribution") or {} + if ( + method == "claim_lattice" + and isinstance(ratio, (int, float)) + and ratio >= 0.5 + and distribution + ): + top_pid, top_count = max(distribution.items(), key=lambda kv: kv[1]) + total = sum(distribution.values()) + lines.append( + f"lazy-anchor smell: {top_count} of {total} verified " + f"pairs cite [{top_pid}] (ratio {ratio:.2f}); " + f"distinct pointers cited: {len(distribution)}" + ) + lines.append("") + cache_key = (result.get("cache_key") or "")[:8] lines.append(f"cache_key: {cache_key}… ") return "\n".join(lines) diff --git a/aborist/qa/query.py b/aborist/qa/query.py index fb82035..36f27e3 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -1519,6 +1519,11 @@ def query( "n_verified": verdict["n_verified"], "verifier_method": verdict["verifier_method"], "unverified_quotes": verdict["unverified_quotes"], + # Sidecar smell signals (claim_lattice mode only) — surfaced + # for the renderer; never persisted in providence_cache and + # never threaded into run_dag_root. + "pointer_id_distribution": verdict.get("pointer_id_distribution"), + "lazy_anchor_ratio": verdict.get("lazy_anchor_ratio"), "timings": { "search_ms": search_ms, "context_ms": context_ms, diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index 211f81a..51ccd64 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -625,6 +625,10 @@ def ask( "n_verified": verdict["n_verified"], "verifier_method": verdict["verifier_method"], "unverified_quotes": verdict["unverified_quotes"], + # Sidecar smell signals (claim_lattice mode only) — render- + # layer; never persisted, never in run_dag_root. + "pointer_id_distribution": verdict.get("pointer_id_distribution"), + "lazy_anchor_ratio": verdict.get("lazy_anchor_ratio"), "timings": { "cache_lookup_ms": cache_lookup_ms, "llm_ms": llm_ms, diff --git a/aborist/qa/verify.py b/aborist/qa/verify.py index 11638cc..b822f6c 100644 --- a/aborist/qa/verify.py +++ b/aborist/qa/verify.py @@ -912,6 +912,30 @@ def verify_claim_lattice( else: audit_mode = "UNGROUNDED" + # Anchor-smell sidecar (render-layer only — never persisted as a + # v9.8 field). Counts how many distinct pointer_ids the model used + # across the verified-or-partial claims. ``lazy_anchor_ratio`` is + # the max share any single pointer claimed: 1.0 = every claim cites + # the same pointer (Hermes-3-8B's lazy-anchor habit on the JP- + # dinosaurs benchmark), 1/N = every claim cites a unique pointer. + # The distribution is recoverable from ``claim_statuses`` which + # IS persisted in run_dag_blob; we surface the derived numbers in + # the verdict for the human renderer, but they never thread back + # into ``build_run_dag``'s verify_payload, so ``run_dag_root`` + # stays clean. + pointer_distribution: dict[str, int] = {} + for cs in claim_statuses: + if cs["status"] not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"): + continue + for pid in cs.get("pointer_ids") or []: + pointer_distribution[pid] = pointer_distribution.get(pid, 0) + 1 + total_pointers = sum(pointer_distribution.values()) + lazy_anchor_ratio = ( + max(pointer_distribution.values()) / total_pointers + if total_pointers + else 0.0 + ) + return { "n_quotes": n_pairs, "n_verified": n_pairs_verified, @@ -922,4 +946,6 @@ def verify_claim_lattice( "violations": violations, "rendered_text": rendered_text, "evidence_id_pairs": evidence_id_pairs, + "pointer_id_distribution": pointer_distribution, + "lazy_anchor_ratio": lazy_anchor_ratio, } diff --git a/tests/test_claim_lattice.py b/tests/test_claim_lattice.py index 4b319ef..d30a50b 100644 --- a/tests/test_claim_lattice.py +++ b/tests/test_claim_lattice.py @@ -863,6 +863,53 @@ def test_query_orders_chunks_by_relevance_g03(tmp_path): assert scored[0][0] != 0 +def test_lazy_anchor_ratio_one_when_all_share_pointer(): + em = build_evidence_map(_sample_chunks()) + answer = ( + "Trex appears. [E1]\n" + "T-rex roars. [E1]\n" + "T-rex breaks out. [E1]\n" + ) + v = verify_claim_lattice(answer, em) + assert v["lazy_anchor_ratio"] == 1.0 + assert v["pointer_id_distribution"] == {"E1": 3} + + +def test_lazy_anchor_ratio_one_third_when_diversified(): + em = build_evidence_map(_sample_chunks()) + answer = ( + "Trex appears. [E1]\n" + "Velociraptors stalk. [E2]\n" + "Brachiosaurus visible. [E1]\n" + ) + v = verify_claim_lattice(answer, em) + # 2 cite E1, 1 cites E2 → max share 2/3. + assert v["pointer_id_distribution"] == {"E1": 2, "E2": 1} + assert abs(v["lazy_anchor_ratio"] - 2 / 3) < 1e-9 + + +def test_lazy_anchor_ratio_zero_when_no_verified_pairs(): + em = build_evidence_map(_sample_chunks()) + # Unknown pointer → 0 verified pairs. + answer = "Imaginary claim. [E99]\n" + v = verify_claim_lattice(answer, em) + assert v["lazy_anchor_ratio"] == 0.0 + assert v["pointer_id_distribution"] == {} + + +def test_lazy_anchor_signals_not_in_run_dag_payload(): + """Sidecar invariant: pointer_id_distribution and lazy_anchor_ratio + must NOT enter build_run_dag's verify_payload — they're soft + signals recoverable from claim_statuses, and threading them in + would make run_dag_root depend on the model's anchoring habits.""" + import inspect as _inspect + from aborist.qa.dag import build_run_dag + sig = _inspect.signature(build_run_dag) + params = set(sig.parameters) + assert "pointer_id_distribution" not in params + assert "lazy_anchor_ratio" not in params + + def test_per_chunk_evidence_map_uses_distinct_chunk_roots(tmp_path): """Chunk-level evidence_ids must be content-addressed per chunk — different chunks of the same source produce different evidence_ids