qa/corpus_query: run_query gains policy= kwarg (verifier kwargs only)
Phase 1 step 6a of #53. Smallest additive change to set up Phase 2's cache wrapper. Existing callers (corpus-query, cloud query) pass no policy and behave IDENTICALLY to before — byte-identity gate from step 5 stays green. When policy IS provided, twelve recognized verifier kwargs forward through to verify_claim_lattice: allowed_source_roles, max_pointers_per_claim, min_citation_coverage, min_claim_content_tokens, lazy_anchor_demote_threshold, lazy_anchor_demote_min_pairs, max_claims_per_answer, subject_tokens_absent_threshold, warrant_check_enabled, deflection_check_enabled, format_collapse_check_enabled, warrant_chain_roots UNKNOWN keys are silently ignored — a policy dict shared with legacy query() may carry fields (base_version, retrieval_keywords, etc.) the orchestrator doesn't yet honor; ignoring them keeps the call site clean instead of requiring callers to filter. Steps 6b/c/d extend the policy surface: 6b: role-classified + role-weighted context budget 6c: multi-route retrieval (title + phrase + core_keyword + body merge) 6d: wikitext-strip Tests cover three contracts: - policy=None is byte-identical to pre-6a (262 tests including the step-5 byte-identity fixture pass) - policy={"max_claims_per_answer": 0} actually trips TOO_MANY_CLAIMS (proves the kwarg reaches the verifier, not just the function signature) - policy with unknown keys (e.g. base_version) doesn't blow up
This commit is contained in:
parent
182274194c
commit
72d111796f
2 changed files with 108 additions and 6 deletions
|
|
@ -139,3 +139,68 @@ def test_run_query_corpus_name_surfaces(corpus):
|
|||
model_id="stub", top_k=1,
|
||||
)
|
||||
assert result.get("corpus_name") == "sqlite-shard"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# policy= kwarg (Phase 1 step 6a of #53)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_run_query_no_policy_default_behavior(corpus):
|
||||
"""policy=None (default) must produce the SAME verdict as before
|
||||
step 6a — this is the byte-identity gate at the orchestrator level."""
|
||||
answer = 'Anarchism is a "stateless society". [E1]'
|
||||
r_no_policy = run_query(
|
||||
corpus, "anarchism", StubClient(answer=answer),
|
||||
model_id="stub", top_k=1,
|
||||
)
|
||||
# Same call again to confirm determinism + capture the legacy shape.
|
||||
r_again = run_query(
|
||||
corpus, "anarchism", StubClient(answer=answer),
|
||||
model_id="stub", top_k=1,
|
||||
)
|
||||
assert r_no_policy["audit_mode"] == r_again["audit_mode"]
|
||||
assert r_no_policy["n_verified"] == r_again["n_verified"]
|
||||
assert r_no_policy["raw_answer"] == r_again["raw_answer"]
|
||||
|
||||
|
||||
def test_run_query_policy_forwards_verifier_kwarg(corpus):
|
||||
"""policy={"max_claims_per_answer": 0} should trip TOO_MANY_CLAIMS
|
||||
on any non-empty answer (default cap is 12). Proves policy actually
|
||||
reaches the verifier."""
|
||||
answer = 'Anarchism is a "stateless society". [E1]'
|
||||
r_strict = run_query(
|
||||
corpus, "anarchism", StubClient(answer=answer),
|
||||
model_id="stub", top_k=1,
|
||||
)
|
||||
r_capped = run_query(
|
||||
corpus, "anarchism", StubClient(answer=answer),
|
||||
model_id="stub", top_k=1,
|
||||
policy={"max_claims_per_answer": 0},
|
||||
)
|
||||
# Default-policy run should STRICT or higher; capped run should
|
||||
# have at least one TOO_MANY_CLAIMS violation.
|
||||
assert r_strict["audit_mode"] in ("STRICT", "HYBRID")
|
||||
violations = r_capped.get("violations") or []
|
||||
assert any(
|
||||
v.get("kind") == "TOO_MANY_CLAIMS" or v.get("violation_type") == "TOO_MANY_CLAIMS"
|
||||
for v in violations
|
||||
), f"expected TOO_MANY_CLAIMS violation in capped run; got {violations!r}"
|
||||
|
||||
|
||||
def test_run_query_policy_ignores_unknown_keys(corpus):
|
||||
"""policy with verifier-irrelevant keys (e.g. base_version that
|
||||
Phase 1 step 6a doesn't honor yet) must not blow up — unknown keys
|
||||
are silently ignored so a legacy DEFAULT_QUERY_POLICY can be
|
||||
passed without filtering."""
|
||||
answer = 'Anarchism is a "stateless society". [E1]'
|
||||
result = run_query(
|
||||
corpus, "anarchism", StubClient(answer=answer),
|
||||
model_id="stub", top_k=1,
|
||||
policy={
|
||||
"base_version": "wikitext-base-v1", # ignored (step 6d work)
|
||||
"some_future_field": 42, # ignored
|
||||
"max_claims_per_answer": 12, # honored — default
|
||||
},
|
||||
)
|
||||
assert result["audit_mode"] in ("STRICT", "HYBRID", "UNGROUNDED")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue