From 72d111796f515977b6e1a330146567f3b09d8184 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sun, 31 May 2026 12:38:18 -0400 Subject: [PATCH] qa/corpus_query: run_query gains policy= kwarg (verifier kwargs only) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 step 6a of #53. Smallest additive change to set up Phase 2's cache wrapper. Existing callers (corpus-query, cloud query) pass no policy and behave IDENTICALLY to before — byte-identity gate from step 5 stays green. When policy IS provided, twelve recognized verifier kwargs forward through to verify_claim_lattice: allowed_source_roles, max_pointers_per_claim, min_citation_coverage, min_claim_content_tokens, lazy_anchor_demote_threshold, lazy_anchor_demote_min_pairs, max_claims_per_answer, subject_tokens_absent_threshold, warrant_check_enabled, deflection_check_enabled, format_collapse_check_enabled, warrant_chain_roots UNKNOWN keys are silently ignored — a policy dict shared with legacy query() may carry fields (base_version, retrieval_keywords, etc.) the orchestrator doesn't yet honor; ignoring them keeps the call site clean instead of requiring callers to filter. Steps 6b/c/d extend the policy surface: 6b: role-classified + role-weighted context budget 6c: multi-route retrieval (title + phrase + core_keyword + body merge) 6d: wikitext-strip Tests cover three contracts: - policy=None is byte-identical to pre-6a (262 tests including the step-5 byte-identity fixture pass) - policy={"max_claims_per_answer": 0} actually trips TOO_MANY_CLAIMS (proves the kwarg reaches the verifier, not just the function signature) - policy with unknown keys (e.g. base_version) doesn't blow up --- arborist/qa/corpus_query.py | 49 ++++++++++++++++++++++---- tests/test_qa_corpus_query.py | 65 +++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 6 deletions(-) diff --git a/arborist/qa/corpus_query.py b/arborist/qa/corpus_query.py index b38f3b3..da7713e 100644 --- a/arborist/qa/corpus_query.py +++ b/arborist/qa/corpus_query.py @@ -59,6 +59,7 @@ def run_query( temperature: float = 0.1, max_tokens: int = 512, warrant_check_enabled: bool = False, + policy: dict | None = None, ) -> dict: """End-to-end claim-lattice query over any Corpus. @@ -73,6 +74,27 @@ def run_query( Times each phase (search, context, llm, verify, total) into timings dict so callers / benchmarks can attribute cost. + + ``policy`` (Phase 1 step 6a of #53): optional dict that forwards + verifier kwargs into ``verify_claim_lattice``. When None, the + default minimal-pipeline behavior is preserved byte-for-byte + (existing ``corpus-query`` and ``cloud query`` callers pass no + policy and stay unchanged). When provided, the following keys are + plumbed through to the verifier — any UNKNOWN keys are SILENTLY + IGNORED so a policy dict shared with legacy query() doesn't blow + up on fields like ``base_version`` that this orchestrator doesn't + yet honor: + + allowed_source_roles, max_pointers_per_claim, + min_citation_coverage, min_claim_content_tokens, + lazy_anchor_demote_threshold, lazy_anchor_demote_min_pairs, + max_claims_per_answer, subject_tokens_absent_threshold, + warrant_check_enabled, deflection_check_enabled, + format_collapse_check_enabled, warrant_chain_roots + + Steps 6b/c/d extend the policy surface to retrieval routes + role + weighting + wikitext-strip; for now this is the minimal additive + pass. """ timings: dict[str, float] = {} t_start = _time.time() @@ -213,13 +235,28 @@ def run_query( } timings["llm"] = _time.time() - ts - # 5. Verify (claim_lattice). + # 5. Verify (claim_lattice). When `policy` is provided, forward + # every recognized verifier kwarg from it; otherwise use the + # defaults that have been baked in since this orchestrator was + # written. Existing callers (no policy) get IDENTICAL behavior. ts = _time.time() - verdict = verify_claim_lattice( - answer, evidence_map, - question=question, - warrant_check_enabled=warrant_check_enabled, - ) + verify_kwargs: dict = { + "question": question, + "warrant_check_enabled": warrant_check_enabled, + } + if policy: + _VERIFIER_KWARGS = ( + "allowed_source_roles", "max_pointers_per_claim", + "min_citation_coverage", "min_claim_content_tokens", + "lazy_anchor_demote_threshold", "lazy_anchor_demote_min_pairs", + "max_claims_per_answer", "subject_tokens_absent_threshold", + "warrant_check_enabled", "deflection_check_enabled", + "format_collapse_check_enabled", "warrant_chain_roots", + ) + for k in _VERIFIER_KWARGS: + if k in policy: + verify_kwargs[k] = policy[k] + verdict = verify_claim_lattice(answer, evidence_map, **verify_kwargs) timings["verify"] = _time.time() - ts # 6. Annotate sources with used / used_pointer_ids from the model's diff --git a/tests/test_qa_corpus_query.py b/tests/test_qa_corpus_query.py index 850cb15..b13a5f4 100644 --- a/tests/test_qa_corpus_query.py +++ b/tests/test_qa_corpus_query.py @@ -139,3 +139,68 @@ def test_run_query_corpus_name_surfaces(corpus): model_id="stub", top_k=1, ) assert result.get("corpus_name") == "sqlite-shard" + + +# --------------------------------------------------------------------------- +# policy= kwarg (Phase 1 step 6a of #53) +# --------------------------------------------------------------------------- + + +def test_run_query_no_policy_default_behavior(corpus): + """policy=None (default) must produce the SAME verdict as before + step 6a — this is the byte-identity gate at the orchestrator level.""" + answer = 'Anarchism is a "stateless society". [E1]' + r_no_policy = run_query( + corpus, "anarchism", StubClient(answer=answer), + model_id="stub", top_k=1, + ) + # Same call again to confirm determinism + capture the legacy shape. + r_again = run_query( + corpus, "anarchism", StubClient(answer=answer), + model_id="stub", top_k=1, + ) + assert r_no_policy["audit_mode"] == r_again["audit_mode"] + assert r_no_policy["n_verified"] == r_again["n_verified"] + assert r_no_policy["raw_answer"] == r_again["raw_answer"] + + +def test_run_query_policy_forwards_verifier_kwarg(corpus): + """policy={"max_claims_per_answer": 0} should trip TOO_MANY_CLAIMS + on any non-empty answer (default cap is 12). Proves policy actually + reaches the verifier.""" + answer = 'Anarchism is a "stateless society". [E1]' + r_strict = run_query( + corpus, "anarchism", StubClient(answer=answer), + model_id="stub", top_k=1, + ) + r_capped = run_query( + corpus, "anarchism", StubClient(answer=answer), + model_id="stub", top_k=1, + policy={"max_claims_per_answer": 0}, + ) + # Default-policy run should STRICT or higher; capped run should + # have at least one TOO_MANY_CLAIMS violation. + assert r_strict["audit_mode"] in ("STRICT", "HYBRID") + violations = r_capped.get("violations") or [] + assert any( + v.get("kind") == "TOO_MANY_CLAIMS" or v.get("violation_type") == "TOO_MANY_CLAIMS" + for v in violations + ), f"expected TOO_MANY_CLAIMS violation in capped run; got {violations!r}" + + +def test_run_query_policy_ignores_unknown_keys(corpus): + """policy with verifier-irrelevant keys (e.g. base_version that + Phase 1 step 6a doesn't honor yet) must not blow up — unknown keys + are silently ignored so a legacy DEFAULT_QUERY_POLICY can be + passed without filtering.""" + answer = 'Anarchism is a "stateless society". [E1]' + result = run_query( + corpus, "anarchism", StubClient(answer=answer), + model_id="stub", top_k=1, + policy={ + "base_version": "wikitext-base-v1", # ignored (step 6d work) + "some_future_field": 42, # ignored + "max_claims_per_answer": 12, # honored — default + }, + ) + assert result["audit_mode"] in ("STRICT", "HYBRID", "UNGROUNDED")