From c6621ee700a8b2784dbfc4743f252c9781120648 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 19 May 2026 19:53:00 -0400 Subject: [PATCH] =?UTF-8?q?feat(#000057):=20arborist+qwen=20enablement=20?= =?UTF-8?q?=E2=80=94=20multi-engine=20JSON-schema=20+=20per-model=20extras?= =?UTF-8?q?=20pass-through?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two surgical fixes unblock 'arborist with synthesis LLM = Qwen-on- llama.cpp' as a viable arm in the control sweep. Pre-existing docstring said 'Arborist×Qwen needs proof-path guided_json+extra_body surgery — coupled follow-up'; this is that follow-up. Fix 1 — multi-engine structured-output extras The runner / query JSON-mode paths previously sent only vLLM's 'guided_json' key for the claim_lattice schema. llama.cpp silently drops it, leaving Qwen un-enforced (the parse-tolerant fallback did all the work). Helper claim_lattice_structured_output_extras() in arborist/qa/verify.py now returns a dict carrying the schema under all three engine conventions: - guided_json (vLLM grammar-constrained sampling) - json_schema (llama.cpp native shorthand) - response_format (OpenAI-spec, honoured by llama.cpp and newer vLLM) Each engine recognises its own key and silently drops the others. Used at both inference call sites (runner.py:740, query.py:3324). Hermes/vLLM path is unchanged — it picks up 'guided_json' and ignores the other two. Fix 2 — query() accepts user-supplied extra_body, merges with defaults query() grew a keyword-only extra_body parameter (default None). Per-model knobs (Qwen's {'chat_template_kwargs': {'enable_thinking': False}} toggle, future template knobs) can flow from the caller to the synthesis chat-completion call. Schema-enforcement extras are added inside query() and merge under user keys — common case is disjoint namespaces, but if a caller wants to override 'guided_json' they can. bench/control_sweep.py now passes MODELS[arborist_ref]['extra'] through to query() in the arborist branch, so --arborist-ref qwen-nothink runs with reasoning disabled and --arborist-ref qwen-think runs with reasoning enabled. Phase 1's arborist arm with --arborist-ref=hermes is unaffected (MODELS['hermes']['extra'] is None, merges to no-op). Tests + 3 new in tests/test_verify_json.py covering helper default shape, alternate-schema reuse, and query()'s new extra_body parameter 220 affected tests still green (verify / claim_lattice / judge / runner suite) pytest test_verify_json: 27/27 Next: small smoke run --arborist-ref qwen-nothink against 4-8 items to confirm end-to-end before any full sweep. Phase 2 (qwen-think solo) still running in background, unaffected — it doesn't touch the arborist arm. --- arborist/qa/query.py | 23 ++++++++++----- arborist/qa/runner.py | 14 ++++++--- arborist/qa/verify.py | 43 +++++++++++++++++++++++++++ bench/control_sweep.py | 11 +++++++ tests/test_verify_json.py | 61 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 141 insertions(+), 11 deletions(-) diff --git a/arborist/qa/query.py b/arborist/qa/query.py index c1360bf..32f18e5 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -1920,6 +1920,7 @@ def query( retrieval_keywords: str | None = None, translator: object | None = None, progress: Progress | None = None, + extra_body: dict | None = None, ) -> dict: """Answer `question` using the corpus. Cache to qa_db. Returns a result dict. @@ -3312,16 +3313,24 @@ def query( }, } - # 5. Cache miss — call LLM. JSON mode passes guided_json via - # extra_body so vLLM constrains output to the schema at sampling - # time. Non-vLLM endpoints silently drop the field; the lenient - # pre-parser in the verifier handles whatever drift remains. - extra_body: dict | None = None + # 5. Cache miss — call LLM. JSON mode emits structured-output + # extras under all three engine conventions (vLLM `guided_json`, + # llama.cpp `json_schema`, OpenAI-spec `response_format`) so the + # schema is enforced regardless of which engine is behind the + # endpoint. Engines silently drop unknown keys. The user-supplied + # `extra_body` (e.g. {"chat_template_kwargs": {"enable_thinking": + # False}} for Qwen) merges on top — user keys override defaults + # if the caller set both, but the common case is disjoint + # namespaces (model-template knobs vs grammar guidance). + from arborist.qa.verify import claim_lattice_structured_output_extras + final_extra_body: dict | None = None stop_seqs: list[str] | None = None if answer_mode == "claim_lattice" and policy.get( "claim_lattice_use_guided_json", True ): - extra_body = {"guided_json": CLAIM_LATTICE_JSON_SCHEMA} + final_extra_body = claim_lattice_structured_output_extras() + if extra_body: + final_extra_body = {**(final_extra_body or {}), **extra_body} if answer_mode == "claim_lattice": # JSON-mode token-runaway guard — see runner.py for the # full rationale. Stops generation on a blank line so @@ -3344,7 +3353,7 @@ def query( temperature=policy["temperature"], max_tokens=policy["max_tokens"], top_p=policy.get("top_p", 1.0), - extra_body=extra_body, + extra_body=final_extra_body, stop=stop_seqs, ) llm_ms = _ms_since(t_phase) diff --git a/arborist/qa/runner.py b/arborist/qa/runner.py index 898d810..ce52aae 100644 --- a/arborist/qa/runner.py +++ b/arborist/qa/runner.py @@ -729,15 +729,21 @@ def ask( } t_llm = time.monotonic() - # JSON mode: pass guided_json schema so vLLM constrains output at - # sampling time. Endpoints without guided-decoding silently drop the - # field; the lenient pre-parser handles whatever drift remains. + # JSON mode: pass structured-output extras under all three engine + # conventions (vLLM `guided_json`, llama.cpp `json_schema`, OpenAI- + # spec `response_format`) so the same call site enforces the schema + # whether the model is on vLLM, llama.cpp, or OpenAI-spec. Engines + # silently drop unknown keys. Pre-2026-05-19 this was vLLM-only + # (`guided_json` alone); llama.cpp Qwen got NO enforcement and + # relied on the lenient pre-parser to clean drift — the parse- + # tolerant fallback still handles whatever drift remains. + from arborist.qa.verify import claim_lattice_structured_output_extras extra_body: dict | None = None stop_seqs: list[str] | None = None if answer_mode == "claim_lattice" and policy.get( "claim_lattice_use_guided_json", True ): - extra_body = {"guided_json": CLAIM_LATTICE_JSON_SCHEMA} + extra_body = claim_lattice_structured_output_extras() if answer_mode == "claim_lattice": # JSON-mode token-runaway guard. On broad-descriptive / # comparison questions Hermes-3-8B sometimes spams whitespace diff --git a/arborist/qa/verify.py b/arborist/qa/verify.py index 3d4552e..b444af8 100644 --- a/arborist/qa/verify.py +++ b/arborist/qa/verify.py @@ -1847,6 +1847,49 @@ CLAIM_LATTICE_JSON_SCHEMA = { } +def claim_lattice_structured_output_extras( + schema: dict | None = None, + *, + name: str = "claim_lattice", +) -> dict: + """Multi-engine ``extra_body`` for JSON-schema enforcement on + chat completions. Each inference engine recognises its own key + and silently drops the others, so sending all three lets the + same call site work across vLLM, llama.cpp, and OpenAI-spec + endpoints without per-endpoint branching: + + * ``guided_json`` — vLLM grammar-constrained sampling + * ``json_schema`` — llama.cpp native shorthand + * ``response_format``— OpenAI-spec ``{type: json_schema, …}`` + (honoured by llama.cpp and newer vLLM) + + Returns a dict you splat into + ``client.chat_completion(extra_body=…)``. Defaults to the claim- + lattice schema; pass an alternate schema to reuse the helper for + other structured-output features. The ``name`` is required by + OpenAI-spec ``response_format`` and is the user-visible label + for the schema in some engines' error messages. + + Added 2026-05-19 to enable the Arborist arm to run with Qwen on + llama.cpp (the old single-key ``guided_json`` was silently dropped + on llama.cpp, leaving Qwen un-enforced and the parse-tolerant + fallback doing all the work). Hermes/vLLM path is unchanged — + it still picks up ``guided_json`` and ignores the other two.""" + s = schema if schema is not None else CLAIM_LATTICE_JSON_SCHEMA + return { + "guided_json": s, + "json_schema": s, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": name, + "schema": s, + "strict": True, + }, + }, + } + + def verify_claim_lattice_json( answer_json_text: str, evidence_map, diff --git a/bench/control_sweep.py b/bench/control_sweep.py index 1e1dd97..25d237a 100644 --- a/bench/control_sweep.py +++ b/bench/control_sweep.py @@ -189,10 +189,21 @@ def _process_item(idx: int, it: dict, variants: list[str], "judge_raw": (v.raw or "")[:700]}) if arborist_on: try: + # Pass through MODELS[arborist_ref]["extra"] so + # per-model knobs (e.g. Qwen's enable_thinking + # toggle under chat_template_kwargs) reach the + # synthesis call in the proof-path. Without this, + # an arborist-ref=qwen-nothink sweep would still + # emit reasoning tokens because the toggle never + # crosses query()'s boundary. JSON-schema extras + # are added inside query() via + # claim_lattice_structured_output_extras() and + # merge with this per-model extras dict. r = query(question=q_asked, qa_db=qa_db, chat_client=cl(arborist_ref), model_id=MODELS[arborist_ref]["model"], shards_dir=shards_dir, + extra_body=MODELS[arborist_ref]["extra"], policy=dict(DEFAULT_QUERY_POLICY, answer_mode="claim_lattice")) arb_raw = (r.get("raw_answer") diff --git a/tests/test_verify_json.py b/tests/test_verify_json.py index eb9ce55..22ac06e 100644 --- a/tests/test_verify_json.py +++ b/tests/test_verify_json.py @@ -523,3 +523,64 @@ def test_verify_json_subject_tokens_absent_below_threshold_passes(): f"expected STRICT (below threshold); got {v['audit_mode']} " f"with violations {[vio['kind'] for vio in v['violations']]}" ) + + +# --- multi-engine structured-output extras (claim_lattice JSON mode) ------- + +def test_claim_lattice_structured_output_extras_default_shape(): + """The helper returns extras under all three engine conventions so + the same call site enforces the JSON schema across vLLM, llama.cpp, + and OpenAI-spec endpoints. Engines silently drop unknown keys, so + sending all three is safe.""" + from arborist.qa.verify import ( + CLAIM_LATTICE_JSON_SCHEMA, + claim_lattice_structured_output_extras, + ) + + e = claim_lattice_structured_output_extras() + assert set(e.keys()) == {"guided_json", "json_schema", "response_format"} + + # vLLM convention — the schema directly under `guided_json`. + assert e["guided_json"] is CLAIM_LATTICE_JSON_SCHEMA + + # llama.cpp shorthand — the schema directly under `json_schema`. + assert e["json_schema"] is CLAIM_LATTICE_JSON_SCHEMA + + # OpenAI-spec — wrapped under response_format with the type tag. + rf = e["response_format"] + assert rf["type"] == "json_schema" + inner = rf["json_schema"] + assert inner["name"] == "claim_lattice" + assert inner["schema"] is CLAIM_LATTICE_JSON_SCHEMA + assert inner["strict"] is True + + +def test_claim_lattice_structured_output_extras_accepts_alternate_schema(): + """The helper isn't claim-lattice-specific in mechanism — pass a + different schema and an alternate name to reuse the multi-engine + bridge for any structured-output feature.""" + from arborist.qa.verify import claim_lattice_structured_output_extras + + other = {"type": "object", "properties": {"x": {"type": "string"}}, + "required": ["x"], "additionalProperties": False} + e = claim_lattice_structured_output_extras(other, name="my_thing") + assert e["guided_json"] is other + assert e["json_schema"] is other + assert e["response_format"]["json_schema"]["name"] == "my_thing" + assert e["response_format"]["json_schema"]["schema"] is other + + +def test_query_accepts_extra_body_parameter(): + """query() grew an extra_body keyword so per-model knobs (Qwen's + enable_thinking, etc.) can flow through to the synthesis call. + Schema-enforcement extras are added inside query() and merge with + the user-supplied dict (user keys override defaults on collision — + common case is disjoint namespaces).""" + import inspect + from arborist.qa.query import query + + sig = inspect.signature(query) + assert "extra_body" in sig.parameters + p = sig.parameters["extra_body"] + # keyword-only with default None — accepts any caller, optional. + assert p.default is None