diff --git a/arborist/qa/corpus_query.py b/arborist/qa/corpus_query.py index b38f3b3..da7713e 100644 --- a/arborist/qa/corpus_query.py +++ b/arborist/qa/corpus_query.py @@ -59,6 +59,7 @@ def run_query( temperature: float = 0.1, max_tokens: int = 512, warrant_check_enabled: bool = False, + policy: dict | None = None, ) -> dict: """End-to-end claim-lattice query over any Corpus. @@ -73,6 +74,27 @@ def run_query( Times each phase (search, context, llm, verify, total) into timings dict so callers / benchmarks can attribute cost. + + ``policy`` (Phase 1 step 6a of #53): optional dict that forwards + verifier kwargs into ``verify_claim_lattice``. When None, the + default minimal-pipeline behavior is preserved byte-for-byte + (existing ``corpus-query`` and ``cloud query`` callers pass no + policy and stay unchanged). When provided, the following keys are + plumbed through to the verifier — any UNKNOWN keys are SILENTLY + IGNORED so a policy dict shared with legacy query() doesn't blow + up on fields like ``base_version`` that this orchestrator doesn't + yet honor: + + allowed_source_roles, max_pointers_per_claim, + min_citation_coverage, min_claim_content_tokens, + lazy_anchor_demote_threshold, lazy_anchor_demote_min_pairs, + max_claims_per_answer, subject_tokens_absent_threshold, + warrant_check_enabled, deflection_check_enabled, + format_collapse_check_enabled, warrant_chain_roots + + Steps 6b/c/d extend the policy surface to retrieval routes + role + weighting + wikitext-strip; for now this is the minimal additive + pass. """ timings: dict[str, float] = {} t_start = _time.time() @@ -213,13 +235,28 @@ def run_query( } timings["llm"] = _time.time() - ts - # 5. Verify (claim_lattice). + # 5. Verify (claim_lattice). When `policy` is provided, forward + # every recognized verifier kwarg from it; otherwise use the + # defaults that have been baked in since this orchestrator was + # written. Existing callers (no policy) get IDENTICAL behavior. ts = _time.time() - verdict = verify_claim_lattice( - answer, evidence_map, - question=question, - warrant_check_enabled=warrant_check_enabled, - ) + verify_kwargs: dict = { + "question": question, + "warrant_check_enabled": warrant_check_enabled, + } + if policy: + _VERIFIER_KWARGS = ( + "allowed_source_roles", "max_pointers_per_claim", + "min_citation_coverage", "min_claim_content_tokens", + "lazy_anchor_demote_threshold", "lazy_anchor_demote_min_pairs", + "max_claims_per_answer", "subject_tokens_absent_threshold", + "warrant_check_enabled", "deflection_check_enabled", + "format_collapse_check_enabled", "warrant_chain_roots", + ) + for k in _VERIFIER_KWARGS: + if k in policy: + verify_kwargs[k] = policy[k] + verdict = verify_claim_lattice(answer, evidence_map, **verify_kwargs) timings["verify"] = _time.time() - ts # 6. Annotate sources with used / used_pointer_ids from the model's diff --git a/tests/test_qa_corpus_query.py b/tests/test_qa_corpus_query.py index 850cb15..b13a5f4 100644 --- a/tests/test_qa_corpus_query.py +++ b/tests/test_qa_corpus_query.py @@ -139,3 +139,68 @@ def test_run_query_corpus_name_surfaces(corpus): model_id="stub", top_k=1, ) assert result.get("corpus_name") == "sqlite-shard" + + +# --------------------------------------------------------------------------- +# policy= kwarg (Phase 1 step 6a of #53) +# --------------------------------------------------------------------------- + + +def test_run_query_no_policy_default_behavior(corpus): + """policy=None (default) must produce the SAME verdict as before + step 6a — this is the byte-identity gate at the orchestrator level.""" + answer = 'Anarchism is a "stateless society". [E1]' + r_no_policy = run_query( + corpus, "anarchism", StubClient(answer=answer), + model_id="stub", top_k=1, + ) + # Same call again to confirm determinism + capture the legacy shape. + r_again = run_query( + corpus, "anarchism", StubClient(answer=answer), + model_id="stub", top_k=1, + ) + assert r_no_policy["audit_mode"] == r_again["audit_mode"] + assert r_no_policy["n_verified"] == r_again["n_verified"] + assert r_no_policy["raw_answer"] == r_again["raw_answer"] + + +def test_run_query_policy_forwards_verifier_kwarg(corpus): + """policy={"max_claims_per_answer": 0} should trip TOO_MANY_CLAIMS + on any non-empty answer (default cap is 12). Proves policy actually + reaches the verifier.""" + answer = 'Anarchism is a "stateless society". [E1]' + r_strict = run_query( + corpus, "anarchism", StubClient(answer=answer), + model_id="stub", top_k=1, + ) + r_capped = run_query( + corpus, "anarchism", StubClient(answer=answer), + model_id="stub", top_k=1, + policy={"max_claims_per_answer": 0}, + ) + # Default-policy run should STRICT or higher; capped run should + # have at least one TOO_MANY_CLAIMS violation. + assert r_strict["audit_mode"] in ("STRICT", "HYBRID") + violations = r_capped.get("violations") or [] + assert any( + v.get("kind") == "TOO_MANY_CLAIMS" or v.get("violation_type") == "TOO_MANY_CLAIMS" + for v in violations + ), f"expected TOO_MANY_CLAIMS violation in capped run; got {violations!r}" + + +def test_run_query_policy_ignores_unknown_keys(corpus): + """policy with verifier-irrelevant keys (e.g. base_version that + Phase 1 step 6a doesn't honor yet) must not blow up — unknown keys + are silently ignored so a legacy DEFAULT_QUERY_POLICY can be + passed without filtering.""" + answer = 'Anarchism is a "stateless society". [E1]' + result = run_query( + corpus, "anarchism", StubClient(answer=answer), + model_id="stub", top_k=1, + policy={ + "base_version": "wikitext-base-v1", # ignored (step 6d work) + "some_future_field": 42, # ignored + "max_claims_per_answer": 12, # honored — default + }, + ) + assert result["audit_mode"] in ("STRICT", "HYBRID", "UNGROUNDED")