qa: lazy-anchor smell sidecar — surface when one pointer carries every claim

verify_claim_lattice now derives pointer_id_distribution and
lazy_anchor_ratio from the verified-or-partial claim_statuses and
returns them in the verdict. Surfaced through ask()/query() result
dicts; CLI human-render displays a one-liner when ratio >= 0.5
(threshold for "model is lazy-anchoring").

Sidecar discipline preserved: never persisted in providence_cache,
never threaded into build_run_dag's verify_payload (test pins this
invariant — sidecar values must not enter run_dag_root). The
distribution is recoverable from claim_statuses which IS persisted,
so future sidecar tools can recompute on demand without enlarging
the audit chain.

JP dinosaurs benchmark currently triggers the warning at 11/11
ratio 1.00 — Hermes-3-8B cites [E1] for every claim even with
G0.3's relevance ranking putting the dinosaur-mention chunk near
the top. The render-layer signal now makes that lazy-anchor
behavior explicit instead of leaving it for the auditor to spot
from the spotlight excerpts.

Tests: +4 covering the ratio across distributions (single-anchor /
diversified / no-verified-pairs) plus the run_dag isolation
invariant. Total 442 passing.
This commit is contained in:
russell@unturf.com 2026-04-29 21:28:28 -04:00
parent fd54031882
commit 37ac523bba
No known key found for this signature in database
5 changed files with 103 additions and 0 deletions

View file

@ -467,6 +467,27 @@ def _render_query_human(result: dict, question: str) -> str:
lines.append(f' - "{qtxt}"')
lines.append("")
# Anchor-smell sidecar (claim_lattice mode only). Soft signal —
# never persisted, never in cache_key. Surfaces when ≥50% of
# verified claim-pointer pairs share one pointer_id, which is
# the lazy-anchor failure mode the spotlight render also exposes.
ratio = result.get("lazy_anchor_ratio")
distribution = result.get("pointer_id_distribution") or {}
if (
method == "claim_lattice"
and isinstance(ratio, (int, float))
and ratio >= 0.5
and distribution
):
top_pid, top_count = max(distribution.items(), key=lambda kv: kv[1])
total = sum(distribution.values())
lines.append(
f"lazy-anchor smell: {top_count} of {total} verified "
f"pairs cite [{top_pid}] (ratio {ratio:.2f}); "
f"distinct pointers cited: {len(distribution)}"
)
lines.append("")
cache_key = (result.get("cache_key") or "")[:8]
lines.append(f"cache_key: {cache_key}… <run with --json for full record>")
return "\n".join(lines)

View file

@ -1519,6 +1519,11 @@ def query(
"n_verified": verdict["n_verified"],
"verifier_method": verdict["verifier_method"],
"unverified_quotes": verdict["unverified_quotes"],
# Sidecar smell signals (claim_lattice mode only) — surfaced
# for the renderer; never persisted in providence_cache and
# never threaded into run_dag_root.
"pointer_id_distribution": verdict.get("pointer_id_distribution"),
"lazy_anchor_ratio": verdict.get("lazy_anchor_ratio"),
"timings": {
"search_ms": search_ms,
"context_ms": context_ms,

View file

@ -625,6 +625,10 @@ def ask(
"n_verified": verdict["n_verified"],
"verifier_method": verdict["verifier_method"],
"unverified_quotes": verdict["unverified_quotes"],
# Sidecar smell signals (claim_lattice mode only) — render-
# layer; never persisted, never in run_dag_root.
"pointer_id_distribution": verdict.get("pointer_id_distribution"),
"lazy_anchor_ratio": verdict.get("lazy_anchor_ratio"),
"timings": {
"cache_lookup_ms": cache_lookup_ms,
"llm_ms": llm_ms,

View file

@ -912,6 +912,30 @@ def verify_claim_lattice(
else:
audit_mode = "UNGROUNDED"
# Anchor-smell sidecar (render-layer only — never persisted as a
# v9.8 field). Counts how many distinct pointer_ids the model used
# across the verified-or-partial claims. ``lazy_anchor_ratio`` is
# the max share any single pointer claimed: 1.0 = every claim cites
# the same pointer (Hermes-3-8B's lazy-anchor habit on the JP-
# dinosaurs benchmark), 1/N = every claim cites a unique pointer.
# The distribution is recoverable from ``claim_statuses`` which
# IS persisted in run_dag_blob; we surface the derived numbers in
# the verdict for the human renderer, but they never thread back
# into ``build_run_dag``'s verify_payload, so ``run_dag_root``
# stays clean.
pointer_distribution: dict[str, int] = {}
for cs in claim_statuses:
if cs["status"] not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"):
continue
for pid in cs.get("pointer_ids") or []:
pointer_distribution[pid] = pointer_distribution.get(pid, 0) + 1
total_pointers = sum(pointer_distribution.values())
lazy_anchor_ratio = (
max(pointer_distribution.values()) / total_pointers
if total_pointers
else 0.0
)
return {
"n_quotes": n_pairs,
"n_verified": n_pairs_verified,
@ -922,4 +946,6 @@ def verify_claim_lattice(
"violations": violations,
"rendered_text": rendered_text,
"evidence_id_pairs": evidence_id_pairs,
"pointer_id_distribution": pointer_distribution,
"lazy_anchor_ratio": lazy_anchor_ratio,
}

View file

@ -863,6 +863,53 @@ def test_query_orders_chunks_by_relevance_g03(tmp_path):
assert scored[0][0] != 0
def test_lazy_anchor_ratio_one_when_all_share_pointer():
em = build_evidence_map(_sample_chunks())
answer = (
"Trex appears. [E1]\n"
"T-rex roars. [E1]\n"
"T-rex breaks out. [E1]\n"
)
v = verify_claim_lattice(answer, em)
assert v["lazy_anchor_ratio"] == 1.0
assert v["pointer_id_distribution"] == {"E1": 3}
def test_lazy_anchor_ratio_one_third_when_diversified():
em = build_evidence_map(_sample_chunks())
answer = (
"Trex appears. [E1]\n"
"Velociraptors stalk. [E2]\n"
"Brachiosaurus visible. [E1]\n"
)
v = verify_claim_lattice(answer, em)
# 2 cite E1, 1 cites E2 → max share 2/3.
assert v["pointer_id_distribution"] == {"E1": 2, "E2": 1}
assert abs(v["lazy_anchor_ratio"] - 2 / 3) < 1e-9
def test_lazy_anchor_ratio_zero_when_no_verified_pairs():
em = build_evidence_map(_sample_chunks())
# Unknown pointer → 0 verified pairs.
answer = "Imaginary claim. [E99]\n"
v = verify_claim_lattice(answer, em)
assert v["lazy_anchor_ratio"] == 0.0
assert v["pointer_id_distribution"] == {}
def test_lazy_anchor_signals_not_in_run_dag_payload():
"""Sidecar invariant: pointer_id_distribution and lazy_anchor_ratio
must NOT enter build_run_dag's verify_payload — they're soft
signals recoverable from claim_statuses, and threading them in
would make run_dag_root depend on the model's anchoring habits."""
import inspect as _inspect
from aborist.qa.dag import build_run_dag
sig = _inspect.signature(build_run_dag)
params = set(sig.parameters)
assert "pointer_id_distribution" not in params
assert "lazy_anchor_ratio" not in params
def test_per_chunk_evidence_map_uses_distinct_chunk_roots(tmp_path):
"""Chunk-level evidence_ids must be content-addressed per chunk —
different chunks of the same source produce different evidence_ids