qa: lazy-anchor smell sidecar — surface when one pointer carries every claim
verify_claim_lattice now derives pointer_id_distribution and lazy_anchor_ratio from the verified-or-partial claim_statuses and returns them in the verdict. Surfaced through ask()/query() result dicts; CLI human-render displays a one-liner when ratio >= 0.5 (threshold for "model is lazy-anchoring"). Sidecar discipline preserved: never persisted in providence_cache, never threaded into build_run_dag's verify_payload (test pins this invariant — sidecar values must not enter run_dag_root). The distribution is recoverable from claim_statuses which IS persisted, so future sidecar tools can recompute on demand without enlarging the audit chain. JP dinosaurs benchmark currently triggers the warning at 11/11 ratio 1.00 — Hermes-3-8B cites [E1] for every claim even with G0.3's relevance ranking putting the dinosaur-mention chunk near the top. The render-layer signal now makes that lazy-anchor behavior explicit instead of leaving it for the auditor to spot from the spotlight excerpts. Tests: +4 covering the ratio across distributions (single-anchor / diversified / no-verified-pairs) plus the run_dag isolation invariant. Total 442 passing.
This commit is contained in:
parent
fd54031882
commit
37ac523bba
5 changed files with 103 additions and 0 deletions
|
|
@ -467,6 +467,27 @@ def _render_query_human(result: dict, question: str) -> str:
|
|||
lines.append(f' - "{qtxt}"')
|
||||
lines.append("")
|
||||
|
||||
# Anchor-smell sidecar (claim_lattice mode only). Soft signal —
|
||||
# never persisted, never in cache_key. Surfaces when ≥50% of
|
||||
# verified claim-pointer pairs share one pointer_id, which is
|
||||
# the lazy-anchor failure mode the spotlight render also exposes.
|
||||
ratio = result.get("lazy_anchor_ratio")
|
||||
distribution = result.get("pointer_id_distribution") or {}
|
||||
if (
|
||||
method == "claim_lattice"
|
||||
and isinstance(ratio, (int, float))
|
||||
and ratio >= 0.5
|
||||
and distribution
|
||||
):
|
||||
top_pid, top_count = max(distribution.items(), key=lambda kv: kv[1])
|
||||
total = sum(distribution.values())
|
||||
lines.append(
|
||||
f"lazy-anchor smell: {top_count} of {total} verified "
|
||||
f"pairs cite [{top_pid}] (ratio {ratio:.2f}); "
|
||||
f"distinct pointers cited: {len(distribution)}"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
cache_key = (result.get("cache_key") or "")[:8]
|
||||
lines.append(f"cache_key: {cache_key}… <run with --json for full record>")
|
||||
return "\n".join(lines)
|
||||
|
|
|
|||
|
|
@ -1519,6 +1519,11 @@ def query(
|
|||
"n_verified": verdict["n_verified"],
|
||||
"verifier_method": verdict["verifier_method"],
|
||||
"unverified_quotes": verdict["unverified_quotes"],
|
||||
# Sidecar smell signals (claim_lattice mode only) — surfaced
|
||||
# for the renderer; never persisted in providence_cache and
|
||||
# never threaded into run_dag_root.
|
||||
"pointer_id_distribution": verdict.get("pointer_id_distribution"),
|
||||
"lazy_anchor_ratio": verdict.get("lazy_anchor_ratio"),
|
||||
"timings": {
|
||||
"search_ms": search_ms,
|
||||
"context_ms": context_ms,
|
||||
|
|
|
|||
|
|
@ -625,6 +625,10 @@ def ask(
|
|||
"n_verified": verdict["n_verified"],
|
||||
"verifier_method": verdict["verifier_method"],
|
||||
"unverified_quotes": verdict["unverified_quotes"],
|
||||
# Sidecar smell signals (claim_lattice mode only) — render-
|
||||
# layer; never persisted, never in run_dag_root.
|
||||
"pointer_id_distribution": verdict.get("pointer_id_distribution"),
|
||||
"lazy_anchor_ratio": verdict.get("lazy_anchor_ratio"),
|
||||
"timings": {
|
||||
"cache_lookup_ms": cache_lookup_ms,
|
||||
"llm_ms": llm_ms,
|
||||
|
|
|
|||
|
|
@ -912,6 +912,30 @@ def verify_claim_lattice(
|
|||
else:
|
||||
audit_mode = "UNGROUNDED"
|
||||
|
||||
# Anchor-smell sidecar (render-layer only — never persisted as a
|
||||
# v9.8 field). Counts how many distinct pointer_ids the model used
|
||||
# across the verified-or-partial claims. ``lazy_anchor_ratio`` is
|
||||
# the max share any single pointer claimed: 1.0 = every claim cites
|
||||
# the same pointer (Hermes-3-8B's lazy-anchor habit on the JP-
|
||||
# dinosaurs benchmark), 1/N = every claim cites a unique pointer.
|
||||
# The distribution is recoverable from ``claim_statuses`` which
|
||||
# IS persisted in run_dag_blob; we surface the derived numbers in
|
||||
# the verdict for the human renderer, but they never thread back
|
||||
# into ``build_run_dag``'s verify_payload, so ``run_dag_root``
|
||||
# stays clean.
|
||||
pointer_distribution: dict[str, int] = {}
|
||||
for cs in claim_statuses:
|
||||
if cs["status"] not in ("EVIDENCE_LINKED", "EVIDENCE_LINKED_PARTIAL"):
|
||||
continue
|
||||
for pid in cs.get("pointer_ids") or []:
|
||||
pointer_distribution[pid] = pointer_distribution.get(pid, 0) + 1
|
||||
total_pointers = sum(pointer_distribution.values())
|
||||
lazy_anchor_ratio = (
|
||||
max(pointer_distribution.values()) / total_pointers
|
||||
if total_pointers
|
||||
else 0.0
|
||||
)
|
||||
|
||||
return {
|
||||
"n_quotes": n_pairs,
|
||||
"n_verified": n_pairs_verified,
|
||||
|
|
@ -922,4 +946,6 @@ def verify_claim_lattice(
|
|||
"violations": violations,
|
||||
"rendered_text": rendered_text,
|
||||
"evidence_id_pairs": evidence_id_pairs,
|
||||
"pointer_id_distribution": pointer_distribution,
|
||||
"lazy_anchor_ratio": lazy_anchor_ratio,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -863,6 +863,53 @@ def test_query_orders_chunks_by_relevance_g03(tmp_path):
|
|||
assert scored[0][0] != 0
|
||||
|
||||
|
||||
def test_lazy_anchor_ratio_one_when_all_share_pointer():
|
||||
em = build_evidence_map(_sample_chunks())
|
||||
answer = (
|
||||
"Trex appears. [E1]\n"
|
||||
"T-rex roars. [E1]\n"
|
||||
"T-rex breaks out. [E1]\n"
|
||||
)
|
||||
v = verify_claim_lattice(answer, em)
|
||||
assert v["lazy_anchor_ratio"] == 1.0
|
||||
assert v["pointer_id_distribution"] == {"E1": 3}
|
||||
|
||||
|
||||
def test_lazy_anchor_ratio_one_third_when_diversified():
|
||||
em = build_evidence_map(_sample_chunks())
|
||||
answer = (
|
||||
"Trex appears. [E1]\n"
|
||||
"Velociraptors stalk. [E2]\n"
|
||||
"Brachiosaurus visible. [E1]\n"
|
||||
)
|
||||
v = verify_claim_lattice(answer, em)
|
||||
# 2 cite E1, 1 cites E2 → max share 2/3.
|
||||
assert v["pointer_id_distribution"] == {"E1": 2, "E2": 1}
|
||||
assert abs(v["lazy_anchor_ratio"] - 2 / 3) < 1e-9
|
||||
|
||||
|
||||
def test_lazy_anchor_ratio_zero_when_no_verified_pairs():
|
||||
em = build_evidence_map(_sample_chunks())
|
||||
# Unknown pointer → 0 verified pairs.
|
||||
answer = "Imaginary claim. [E99]\n"
|
||||
v = verify_claim_lattice(answer, em)
|
||||
assert v["lazy_anchor_ratio"] == 0.0
|
||||
assert v["pointer_id_distribution"] == {}
|
||||
|
||||
|
||||
def test_lazy_anchor_signals_not_in_run_dag_payload():
|
||||
"""Sidecar invariant: pointer_id_distribution and lazy_anchor_ratio
|
||||
must NOT enter build_run_dag's verify_payload — they're soft
|
||||
signals recoverable from claim_statuses, and threading them in
|
||||
would make run_dag_root depend on the model's anchoring habits."""
|
||||
import inspect as _inspect
|
||||
from aborist.qa.dag import build_run_dag
|
||||
sig = _inspect.signature(build_run_dag)
|
||||
params = set(sig.parameters)
|
||||
assert "pointer_id_distribution" not in params
|
||||
assert "lazy_anchor_ratio" not in params
|
||||
|
||||
|
||||
def test_per_chunk_evidence_map_uses_distinct_chunk_roots(tmp_path):
|
||||
"""Chunk-level evidence_ids must be content-addressed per chunk —
|
||||
different chunks of the same source produce different evidence_ids
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue