diff --git a/bench/control_sweep.py b/bench/control_sweep.py index 27c24b6..80dae62 100644 --- a/bench/control_sweep.py +++ b/bench/control_sweep.py @@ -92,16 +92,16 @@ MODELS: dict[str, dict] = { "hermes": dict( endpoint="https://hermes.ai.unturf.com/v1", model="adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", - extra=None, max_tokens=512), + extra=None, max_tokens=512, reasoning=False), "qwen-think": dict( endpoint="https://qwen.ai.unturf.com/v1", model="Qwen3.6-27B-UD-Q4_K_XL.gguf", - extra=None, max_tokens=1024), + extra=None, max_tokens=1024, reasoning=True), "qwen-nothink": dict( endpoint="https://qwen.ai.unturf.com/v1", model="Qwen3.6-27B-UD-Q4_K_XL.gguf", extra={"chat_template_kwargs": {"enable_thinking": False}}, - max_tokens=512), + max_tokens=512, reasoning=False), } @@ -207,16 +207,48 @@ def _process_item(idx: int, it: dict, variants: list[str], # are added inside query() via # claim_lattice_structured_output_extras() and # merge with this per-model extras dict. - r = query(question=q_asked, qa_db=qa_db, - chat_client=cl(arborist_ref), - model_id=MODELS[arborist_ref]["model"], - shards_dir=shards_dir, - extra_body=MODELS[arborist_ref]["extra"], - policy=dict(DEFAULT_QUERY_POLICY, - answer_mode="claim_lattice")) - arb_raw = (r.get("raw_answer") - or r.get("answer_text") or "") - arb_mode = r.get("audit_mode") + arb_policy = dict(DEFAULT_QUERY_POLICY, + answer_mode="claim_lattice") + # Reasoning models emit a multi-line trace before + # the JSON; the claim_lattice "\n\n" runaway-guard + # stop sequence (tuned for single-line Hermes JSON) + # truncates that trace to an EMPTY answer (measured + # 2026-05-20: arborist+qwen-think produced 100% + # empty → ABSTAINED). Clear the stop for reasoning + # refs so the JSON actually lands. (With json-schema + # grammar enforcement the reasoning trace is itself + # suppressed, so output is clean single-line JSON — + # but the stop must still be cleared or the first + # structural newline truncates it.) + reasoning_ref = bool(MODELS[arborist_ref].get("reasoning")) + if reasoning_ref: + arb_policy["claim_lattice_json_stop_sequences"] = [] + # Empty-output self-heal for reasoning refs: qwen- + # think under json-schema grammar intermittently + # emits an empty completion (~1/3 of calls, measured + # 2026-05-20) — a model-side artefact, NOT a real + # abstention. Retry (burning the cached empty) up to + # 3 attempts so an artefact-empty doesn't masquerade + # as ABSTAINED in the scorecard. Non-reasoning refs + # don't exhibit this (qwen-nothink phase 3 had 0 + # spurious empties) so they take a single pass. We + # never fabricate — an answer that is still empty + # after retries is recorded as the empty it is. + max_attempts = 3 if reasoning_ref else 1 + arb_raw, arb_mode = "", None + for attempt in range(max_attempts): + r = query(question=q_asked, qa_db=qa_db, + chat_client=cl(arborist_ref), + model_id=MODELS[arborist_ref]["model"], + shards_dir=shards_dir, + extra_body=MODELS[arborist_ref]["extra"], + policy=arb_policy, + burn_existing=(attempt > 0)) + arb_raw = (r.get("raw_answer") + or r.get("answer_text") or "") + arb_mode = r.get("audit_mode") + if arb_raw.strip(): + break except Exception as e: # noqa: BLE001 arb_raw = f"[arborist-error: {type(e).__name__}: {e}]" arb_mode = "ERROR"