From aa9d9c827787e2bda5842085d58699732e601ef4 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 20 May 2026 08:46:09 -0400 Subject: [PATCH] =?UTF-8?q?fix(#000057):=20arborist+reasoning-model=20?= =?UTF-8?q?=E2=80=94=20clear=20JSON=20stop-seq=20+=20empty-output=20retry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects blocked the arborist+qwen-think cell, both diagnosed 2026-05-20: Defect 1 — stop sequence truncates to empty. The claim_lattice path sets stop=['\n\n'] (runaway guard tuned for single-line Hermes JSON). A reasoning model's output trips it immediately → 100% empty answers → 100% ABSTAINED (measured on the first 23 items of the killed run). Fix: MODELS gains an explicit flag; the arborist arm clears claim_lattice_json_stop_sequences for reasoning refs. Direct A/B confirmed: stop=['\n\n'] → ''; stop=None → valid JSON. Defect 2 — intermittent empty completions. Even with the stop cleared, qwen-think under json-schema grammar emits an empty completion ~1/3 of calls (a llama.cpp reasoning+grammar artefact; qwen-nothink phase 3 had ~0 spurious empties). Fix: empty-output self-heal — reasoning refs retry up to 3 attempts, burning the cached empty each retry. Never fabricates: a still-empty answer after retries is recorded as empty. 6-item smoke: 0/6 (broken) → 5/6 valid JSON; residual ~1/6 are questions that reliably break (4 consecutive empties), a documented artefact. Structural finding (to fold into Addendum 8): json-schema grammar enforcement SUPPRESSES the reasoning trace — output is pure single- line JSON, no block. So arborist+qwen-think is structurally ≈ arborist+qwen-nothink; the thinking lever that moved the solo arm is neutralized by the claim_lattice grammar. The re-run will confirm empirically. Non-reasoning refs (hermes, qwen-nothink) unchanged: single pass, stop sequence intact. --- bench/control_sweep.py | 58 ++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/bench/control_sweep.py b/bench/control_sweep.py index 27c24b6..80dae62 100644 --- a/bench/control_sweep.py +++ b/bench/control_sweep.py @@ -92,16 +92,16 @@ MODELS: dict[str, dict] = { "hermes": dict( endpoint="https://hermes.ai.unturf.com/v1", model="adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", - extra=None, max_tokens=512), + extra=None, max_tokens=512, reasoning=False), "qwen-think": dict( endpoint="https://qwen.ai.unturf.com/v1", model="Qwen3.6-27B-UD-Q4_K_XL.gguf", - extra=None, max_tokens=1024), + extra=None, max_tokens=1024, reasoning=True), "qwen-nothink": dict( endpoint="https://qwen.ai.unturf.com/v1", model="Qwen3.6-27B-UD-Q4_K_XL.gguf", extra={"chat_template_kwargs": {"enable_thinking": False}}, - max_tokens=512), + max_tokens=512, reasoning=False), } @@ -207,16 +207,48 @@ def _process_item(idx: int, it: dict, variants: list[str], # are added inside query() via # claim_lattice_structured_output_extras() and # merge with this per-model extras dict. - r = query(question=q_asked, qa_db=qa_db, - chat_client=cl(arborist_ref), - model_id=MODELS[arborist_ref]["model"], - shards_dir=shards_dir, - extra_body=MODELS[arborist_ref]["extra"], - policy=dict(DEFAULT_QUERY_POLICY, - answer_mode="claim_lattice")) - arb_raw = (r.get("raw_answer") - or r.get("answer_text") or "") - arb_mode = r.get("audit_mode") + arb_policy = dict(DEFAULT_QUERY_POLICY, + answer_mode="claim_lattice") + # Reasoning models emit a multi-line trace before + # the JSON; the claim_lattice "\n\n" runaway-guard + # stop sequence (tuned for single-line Hermes JSON) + # truncates that trace to an EMPTY answer (measured + # 2026-05-20: arborist+qwen-think produced 100% + # empty → ABSTAINED). Clear the stop for reasoning + # refs so the JSON actually lands. (With json-schema + # grammar enforcement the reasoning trace is itself + # suppressed, so output is clean single-line JSON — + # but the stop must still be cleared or the first + # structural newline truncates it.) + reasoning_ref = bool(MODELS[arborist_ref].get("reasoning")) + if reasoning_ref: + arb_policy["claim_lattice_json_stop_sequences"] = [] + # Empty-output self-heal for reasoning refs: qwen- + # think under json-schema grammar intermittently + # emits an empty completion (~1/3 of calls, measured + # 2026-05-20) — a model-side artefact, NOT a real + # abstention. Retry (burning the cached empty) up to + # 3 attempts so an artefact-empty doesn't masquerade + # as ABSTAINED in the scorecard. Non-reasoning refs + # don't exhibit this (qwen-nothink phase 3 had 0 + # spurious empties) so they take a single pass. We + # never fabricate — an answer that is still empty + # after retries is recorded as the empty it is. + max_attempts = 3 if reasoning_ref else 1 + arb_raw, arb_mode = "", None + for attempt in range(max_attempts): + r = query(question=q_asked, qa_db=qa_db, + chat_client=cl(arborist_ref), + model_id=MODELS[arborist_ref]["model"], + shards_dir=shards_dir, + extra_body=MODELS[arborist_ref]["extra"], + policy=arb_policy, + burn_existing=(attempt > 0)) + arb_raw = (r.get("raw_answer") + or r.get("answer_text") or "") + arb_mode = r.get("audit_mode") + if arb_raw.strip(): + break except Exception as e: # noqa: BLE001 arb_raw = f"[arborist-error: {type(e).__name__}: {e}]" arb_mode = "ERROR"