qa/corpus_query: run_query gains policy= kwarg (verifier kwargs only)

Phase 1 step 6a of #53. Smallest additive change to set up Phase 2's
cache wrapper. Existing callers (corpus-query, cloud query) pass no
policy and behave IDENTICALLY to before — byte-identity gate from
step 5 stays green.

When policy IS provided, twelve recognized verifier kwargs forward
through to verify_claim_lattice:

  allowed_source_roles, max_pointers_per_claim, min_citation_coverage,
  min_claim_content_tokens, lazy_anchor_demote_threshold,
  lazy_anchor_demote_min_pairs, max_claims_per_answer,
  subject_tokens_absent_threshold, warrant_check_enabled,
  deflection_check_enabled, format_collapse_check_enabled,
  warrant_chain_roots

UNKNOWN keys are silently ignored — a policy dict shared with legacy
query() may carry fields (base_version, retrieval_keywords, etc.) the
orchestrator doesn't yet honor; ignoring them keeps the call site
clean instead of requiring callers to filter.

Steps 6b/c/d extend the policy surface:
  6b: role-classified + role-weighted context budget
  6c: multi-route retrieval (title + phrase + core_keyword + body merge)
  6d: wikitext-strip

Tests cover three contracts:
  - policy=None is byte-identical to pre-6a (262 tests including the
    step-5 byte-identity fixture pass)
  - policy={"max_claims_per_answer": 0} actually trips TOO_MANY_CLAIMS
    (proves the kwarg reaches the verifier, not just the function
    signature)
  - policy with unknown keys (e.g. base_version) doesn't blow up
This commit is contained in:
russell@unturf.com 2026-05-31 12:38:18 -04:00
parent 182274194c
commit 72d111796f
No known key found for this signature in database
2 changed files with 108 additions and 6 deletions

View file

@ -59,6 +59,7 @@ def run_query(
temperature: float = 0.1,
max_tokens: int = 512,
warrant_check_enabled: bool = False,
policy: dict | None = None,
) -> dict:
"""End-to-end claim-lattice query over any Corpus.
@ -73,6 +74,27 @@ def run_query(
Times each phase (search, context, llm, verify, total) into
timings dict so callers / benchmarks can attribute cost.
``policy`` (Phase 1 step 6a of #53): optional dict that forwards
verifier kwargs into ``verify_claim_lattice``. When None, the
default minimal-pipeline behavior is preserved byte-for-byte
(existing ``corpus-query`` and ``cloud query`` callers pass no
policy and stay unchanged). When provided, the following keys are
plumbed through to the verifier any UNKNOWN keys are SILENTLY
IGNORED so a policy dict shared with legacy query() doesn't blow
up on fields like ``base_version`` that this orchestrator doesn't
yet honor:
allowed_source_roles, max_pointers_per_claim,
min_citation_coverage, min_claim_content_tokens,
lazy_anchor_demote_threshold, lazy_anchor_demote_min_pairs,
max_claims_per_answer, subject_tokens_absent_threshold,
warrant_check_enabled, deflection_check_enabled,
format_collapse_check_enabled, warrant_chain_roots
Steps 6b/c/d extend the policy surface to retrieval routes + role
weighting + wikitext-strip; for now this is the minimal additive
pass.
"""
timings: dict[str, float] = {}
t_start = _time.time()
@ -213,13 +235,28 @@ def run_query(
}
timings["llm"] = _time.time() - ts
# 5. Verify (claim_lattice).
# 5. Verify (claim_lattice). When `policy` is provided, forward
# every recognized verifier kwarg from it; otherwise use the
# defaults that have been baked in since this orchestrator was
# written. Existing callers (no policy) get IDENTICAL behavior.
ts = _time.time()
verdict = verify_claim_lattice(
answer, evidence_map,
question=question,
warrant_check_enabled=warrant_check_enabled,
)
verify_kwargs: dict = {
"question": question,
"warrant_check_enabled": warrant_check_enabled,
}
if policy:
_VERIFIER_KWARGS = (
"allowed_source_roles", "max_pointers_per_claim",
"min_citation_coverage", "min_claim_content_tokens",
"lazy_anchor_demote_threshold", "lazy_anchor_demote_min_pairs",
"max_claims_per_answer", "subject_tokens_absent_threshold",
"warrant_check_enabled", "deflection_check_enabled",
"format_collapse_check_enabled", "warrant_chain_roots",
)
for k in _VERIFIER_KWARGS:
if k in policy:
verify_kwargs[k] = policy[k]
verdict = verify_claim_lattice(answer, evidence_map, **verify_kwargs)
timings["verify"] = _time.time() - ts
# 6. Annotate sources with used / used_pointer_ids from the model's

View file

@ -139,3 +139,68 @@ def test_run_query_corpus_name_surfaces(corpus):
model_id="stub", top_k=1,
)
assert result.get("corpus_name") == "sqlite-shard"
# ---------------------------------------------------------------------------
# policy= kwarg (Phase 1 step 6a of #53)
# ---------------------------------------------------------------------------
def test_run_query_no_policy_default_behavior(corpus):
"""policy=None (default) must produce the SAME verdict as before
step 6a this is the byte-identity gate at the orchestrator level."""
answer = 'Anarchism is a "stateless society". [E1]'
r_no_policy = run_query(
corpus, "anarchism", StubClient(answer=answer),
model_id="stub", top_k=1,
)
# Same call again to confirm determinism + capture the legacy shape.
r_again = run_query(
corpus, "anarchism", StubClient(answer=answer),
model_id="stub", top_k=1,
)
assert r_no_policy["audit_mode"] == r_again["audit_mode"]
assert r_no_policy["n_verified"] == r_again["n_verified"]
assert r_no_policy["raw_answer"] == r_again["raw_answer"]
def test_run_query_policy_forwards_verifier_kwarg(corpus):
"""policy={"max_claims_per_answer": 0} should trip TOO_MANY_CLAIMS
on any non-empty answer (default cap is 12). Proves policy actually
reaches the verifier."""
answer = 'Anarchism is a "stateless society". [E1]'
r_strict = run_query(
corpus, "anarchism", StubClient(answer=answer),
model_id="stub", top_k=1,
)
r_capped = run_query(
corpus, "anarchism", StubClient(answer=answer),
model_id="stub", top_k=1,
policy={"max_claims_per_answer": 0},
)
# Default-policy run should STRICT or higher; capped run should
# have at least one TOO_MANY_CLAIMS violation.
assert r_strict["audit_mode"] in ("STRICT", "HYBRID")
violations = r_capped.get("violations") or []
assert any(
v.get("kind") == "TOO_MANY_CLAIMS" or v.get("violation_type") == "TOO_MANY_CLAIMS"
for v in violations
), f"expected TOO_MANY_CLAIMS violation in capped run; got {violations!r}"
def test_run_query_policy_ignores_unknown_keys(corpus):
"""policy with verifier-irrelevant keys (e.g. base_version that
Phase 1 step 6a doesn't honor yet) must not blow up — unknown keys
are silently ignored so a legacy DEFAULT_QUERY_POLICY can be
passed without filtering."""
answer = 'Anarchism is a "stateless society". [E1]'
result = run_query(
corpus, "anarchism", StubClient(answer=answer),
model_id="stub", top_k=1,
policy={
"base_version": "wikitext-base-v1", # ignored (step 6d work)
"some_future_field": 42, # ignored
"max_claims_per_answer": 12, # honored — default
},
)
assert result["audit_mode"] in ("STRICT", "HYBRID", "UNGROUNDED")