qa/corpus_query: run_query gains policy= kwarg (verifier kwargs only)
Phase 1 step 6a of #53. Smallest additive change to set up Phase 2's cache wrapper. Existing callers (corpus-query, cloud query) pass no policy and behave IDENTICALLY to before — byte-identity gate from step 5 stays green. When policy IS provided, twelve recognized verifier kwargs forward through to verify_claim_lattice: allowed_source_roles, max_pointers_per_claim, min_citation_coverage, min_claim_content_tokens, lazy_anchor_demote_threshold, lazy_anchor_demote_min_pairs, max_claims_per_answer, subject_tokens_absent_threshold, warrant_check_enabled, deflection_check_enabled, format_collapse_check_enabled, warrant_chain_roots UNKNOWN keys are silently ignored — a policy dict shared with legacy query() may carry fields (base_version, retrieval_keywords, etc.) the orchestrator doesn't yet honor; ignoring them keeps the call site clean instead of requiring callers to filter. Steps 6b/c/d extend the policy surface: 6b: role-classified + role-weighted context budget 6c: multi-route retrieval (title + phrase + core_keyword + body merge) 6d: wikitext-strip Tests cover three contracts: - policy=None is byte-identical to pre-6a (262 tests including the step-5 byte-identity fixture pass) - policy={"max_claims_per_answer": 0} actually trips TOO_MANY_CLAIMS (proves the kwarg reaches the verifier, not just the function signature) - policy with unknown keys (e.g. base_version) doesn't blow up
This commit is contained in:
parent
182274194c
commit
72d111796f
2 changed files with 108 additions and 6 deletions
|
|
@ -59,6 +59,7 @@ def run_query(
|
|||
temperature: float = 0.1,
|
||||
max_tokens: int = 512,
|
||||
warrant_check_enabled: bool = False,
|
||||
policy: dict | None = None,
|
||||
) -> dict:
|
||||
"""End-to-end claim-lattice query over any Corpus.
|
||||
|
||||
|
|
@ -73,6 +74,27 @@ def run_query(
|
|||
|
||||
Times each phase (search, context, llm, verify, total) into
|
||||
timings dict so callers / benchmarks can attribute cost.
|
||||
|
||||
``policy`` (Phase 1 step 6a of #53): optional dict that forwards
|
||||
verifier kwargs into ``verify_claim_lattice``. When None, the
|
||||
default minimal-pipeline behavior is preserved byte-for-byte
|
||||
(existing ``corpus-query`` and ``cloud query`` callers pass no
|
||||
policy and stay unchanged). When provided, the following keys are
|
||||
plumbed through to the verifier — any UNKNOWN keys are SILENTLY
|
||||
IGNORED so a policy dict shared with legacy query() doesn't blow
|
||||
up on fields like ``base_version`` that this orchestrator doesn't
|
||||
yet honor:
|
||||
|
||||
allowed_source_roles, max_pointers_per_claim,
|
||||
min_citation_coverage, min_claim_content_tokens,
|
||||
lazy_anchor_demote_threshold, lazy_anchor_demote_min_pairs,
|
||||
max_claims_per_answer, subject_tokens_absent_threshold,
|
||||
warrant_check_enabled, deflection_check_enabled,
|
||||
format_collapse_check_enabled, warrant_chain_roots
|
||||
|
||||
Steps 6b/c/d extend the policy surface to retrieval routes + role
|
||||
weighting + wikitext-strip; for now this is the minimal additive
|
||||
pass.
|
||||
"""
|
||||
timings: dict[str, float] = {}
|
||||
t_start = _time.time()
|
||||
|
|
@ -213,13 +235,28 @@ def run_query(
|
|||
}
|
||||
timings["llm"] = _time.time() - ts
|
||||
|
||||
# 5. Verify (claim_lattice).
|
||||
# 5. Verify (claim_lattice). When `policy` is provided, forward
|
||||
# every recognized verifier kwarg from it; otherwise use the
|
||||
# defaults that have been baked in since this orchestrator was
|
||||
# written. Existing callers (no policy) get IDENTICAL behavior.
|
||||
ts = _time.time()
|
||||
verdict = verify_claim_lattice(
|
||||
answer, evidence_map,
|
||||
question=question,
|
||||
warrant_check_enabled=warrant_check_enabled,
|
||||
)
|
||||
verify_kwargs: dict = {
|
||||
"question": question,
|
||||
"warrant_check_enabled": warrant_check_enabled,
|
||||
}
|
||||
if policy:
|
||||
_VERIFIER_KWARGS = (
|
||||
"allowed_source_roles", "max_pointers_per_claim",
|
||||
"min_citation_coverage", "min_claim_content_tokens",
|
||||
"lazy_anchor_demote_threshold", "lazy_anchor_demote_min_pairs",
|
||||
"max_claims_per_answer", "subject_tokens_absent_threshold",
|
||||
"warrant_check_enabled", "deflection_check_enabled",
|
||||
"format_collapse_check_enabled", "warrant_chain_roots",
|
||||
)
|
||||
for k in _VERIFIER_KWARGS:
|
||||
if k in policy:
|
||||
verify_kwargs[k] = policy[k]
|
||||
verdict = verify_claim_lattice(answer, evidence_map, **verify_kwargs)
|
||||
timings["verify"] = _time.time() - ts
|
||||
|
||||
# 6. Annotate sources with used / used_pointer_ids from the model's
|
||||
|
|
|
|||
|
|
@ -139,3 +139,68 @@ def test_run_query_corpus_name_surfaces(corpus):
|
|||
model_id="stub", top_k=1,
|
||||
)
|
||||
assert result.get("corpus_name") == "sqlite-shard"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# policy= kwarg (Phase 1 step 6a of #53)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_run_query_no_policy_default_behavior(corpus):
|
||||
"""policy=None (default) must produce the SAME verdict as before
|
||||
step 6a — this is the byte-identity gate at the orchestrator level."""
|
||||
answer = 'Anarchism is a "stateless society". [E1]'
|
||||
r_no_policy = run_query(
|
||||
corpus, "anarchism", StubClient(answer=answer),
|
||||
model_id="stub", top_k=1,
|
||||
)
|
||||
# Same call again to confirm determinism + capture the legacy shape.
|
||||
r_again = run_query(
|
||||
corpus, "anarchism", StubClient(answer=answer),
|
||||
model_id="stub", top_k=1,
|
||||
)
|
||||
assert r_no_policy["audit_mode"] == r_again["audit_mode"]
|
||||
assert r_no_policy["n_verified"] == r_again["n_verified"]
|
||||
assert r_no_policy["raw_answer"] == r_again["raw_answer"]
|
||||
|
||||
|
||||
def test_run_query_policy_forwards_verifier_kwarg(corpus):
|
||||
"""policy={"max_claims_per_answer": 0} should trip TOO_MANY_CLAIMS
|
||||
on any non-empty answer (default cap is 12). Proves policy actually
|
||||
reaches the verifier."""
|
||||
answer = 'Anarchism is a "stateless society". [E1]'
|
||||
r_strict = run_query(
|
||||
corpus, "anarchism", StubClient(answer=answer),
|
||||
model_id="stub", top_k=1,
|
||||
)
|
||||
r_capped = run_query(
|
||||
corpus, "anarchism", StubClient(answer=answer),
|
||||
model_id="stub", top_k=1,
|
||||
policy={"max_claims_per_answer": 0},
|
||||
)
|
||||
# Default-policy run should STRICT or higher; capped run should
|
||||
# have at least one TOO_MANY_CLAIMS violation.
|
||||
assert r_strict["audit_mode"] in ("STRICT", "HYBRID")
|
||||
violations = r_capped.get("violations") or []
|
||||
assert any(
|
||||
v.get("kind") == "TOO_MANY_CLAIMS" or v.get("violation_type") == "TOO_MANY_CLAIMS"
|
||||
for v in violations
|
||||
), f"expected TOO_MANY_CLAIMS violation in capped run; got {violations!r}"
|
||||
|
||||
|
||||
def test_run_query_policy_ignores_unknown_keys(corpus):
|
||||
"""policy with verifier-irrelevant keys (e.g. base_version that
|
||||
Phase 1 step 6a doesn't honor yet) must not blow up — unknown keys
|
||||
are silently ignored so a legacy DEFAULT_QUERY_POLICY can be
|
||||
passed without filtering."""
|
||||
answer = 'Anarchism is a "stateless society". [E1]'
|
||||
result = run_query(
|
||||
corpus, "anarchism", StubClient(answer=answer),
|
||||
model_id="stub", top_k=1,
|
||||
policy={
|
||||
"base_version": "wikitext-base-v1", # ignored (step 6d work)
|
||||
"some_future_field": 42, # ignored
|
||||
"max_claims_per_answer": 12, # honored — default
|
||||
},
|
||||
)
|
||||
assert result["audit_mode"] in ("STRICT", "HYBRID", "UNGROUNDED")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue