fix(#000057): arborist+reasoning-model — clear JSON stop-seq + empty-output retry
Two defects blocked the arborist+qwen-think cell, both diagnosed 2026-05-20: Defect 1 — stop sequence truncates to empty. The claim_lattice path sets stop=['\n\n'] (runaway guard tuned for single-line Hermes JSON). A reasoning model's output trips it immediately → 100% empty answers → 100% ABSTAINED (measured on the first 23 items of the killed run). Fix: MODELS gains an explicit flag; the arborist arm clears claim_lattice_json_stop_sequences for reasoning refs. Direct A/B confirmed: stop=['\n\n'] → ''; stop=None → valid JSON. Defect 2 — intermittent empty completions. Even with the stop cleared, qwen-think under json-schema grammar emits an empty completion ~1/3 of calls (a llama.cpp reasoning+grammar artefact; qwen-nothink phase 3 had ~0 spurious empties). Fix: empty-output self-heal — reasoning refs retry up to 3 attempts, burning the cached empty each retry. Never fabricates: a still-empty answer after retries is recorded as empty. 6-item smoke: 0/6 (broken) → 5/6 valid JSON; residual ~1/6 are questions that reliably break (4 consecutive empties), a documented artefact. Structural finding (to fold into Addendum 8): json-schema grammar enforcement SUPPRESSES the reasoning trace — output is pure single- line JSON, no <think> block. So arborist+qwen-think is structurally ≈ arborist+qwen-nothink; the thinking lever that moved the solo arm is neutralized by the claim_lattice grammar. The re-run will confirm empirically. Non-reasoning refs (hermes, qwen-nothink) unchanged: single pass, stop sequence intact.
This commit is contained in:
parent
42f614a501
commit
aa9d9c8277
1 changed files with 45 additions and 13 deletions
|
|
@ -92,16 +92,16 @@ MODELS: dict[str, dict] = {
|
||||||
"hermes": dict(
|
"hermes": dict(
|
||||||
endpoint="https://hermes.ai.unturf.com/v1",
|
endpoint="https://hermes.ai.unturf.com/v1",
|
||||||
model="adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic",
|
model="adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic",
|
||||||
extra=None, max_tokens=512),
|
extra=None, max_tokens=512, reasoning=False),
|
||||||
"qwen-think": dict(
|
"qwen-think": dict(
|
||||||
endpoint="https://qwen.ai.unturf.com/v1",
|
endpoint="https://qwen.ai.unturf.com/v1",
|
||||||
model="Qwen3.6-27B-UD-Q4_K_XL.gguf",
|
model="Qwen3.6-27B-UD-Q4_K_XL.gguf",
|
||||||
extra=None, max_tokens=1024),
|
extra=None, max_tokens=1024, reasoning=True),
|
||||||
"qwen-nothink": dict(
|
"qwen-nothink": dict(
|
||||||
endpoint="https://qwen.ai.unturf.com/v1",
|
endpoint="https://qwen.ai.unturf.com/v1",
|
||||||
model="Qwen3.6-27B-UD-Q4_K_XL.gguf",
|
model="Qwen3.6-27B-UD-Q4_K_XL.gguf",
|
||||||
extra={"chat_template_kwargs": {"enable_thinking": False}},
|
extra={"chat_template_kwargs": {"enable_thinking": False}},
|
||||||
max_tokens=512),
|
max_tokens=512, reasoning=False),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -207,16 +207,48 @@ def _process_item(idx: int, it: dict, variants: list[str],
|
||||||
# are added inside query() via
|
# are added inside query() via
|
||||||
# claim_lattice_structured_output_extras() and
|
# claim_lattice_structured_output_extras() and
|
||||||
# merge with this per-model extras dict.
|
# merge with this per-model extras dict.
|
||||||
r = query(question=q_asked, qa_db=qa_db,
|
arb_policy = dict(DEFAULT_QUERY_POLICY,
|
||||||
chat_client=cl(arborist_ref),
|
answer_mode="claim_lattice")
|
||||||
model_id=MODELS[arborist_ref]["model"],
|
# Reasoning models emit a multi-line trace before
|
||||||
shards_dir=shards_dir,
|
# the JSON; the claim_lattice "\n\n" runaway-guard
|
||||||
extra_body=MODELS[arborist_ref]["extra"],
|
# stop sequence (tuned for single-line Hermes JSON)
|
||||||
policy=dict(DEFAULT_QUERY_POLICY,
|
# truncates that trace to an EMPTY answer (measured
|
||||||
answer_mode="claim_lattice"))
|
# 2026-05-20: arborist+qwen-think produced 100%
|
||||||
arb_raw = (r.get("raw_answer")
|
# empty → ABSTAINED). Clear the stop for reasoning
|
||||||
or r.get("answer_text") or "")
|
# refs so the JSON actually lands. (With json-schema
|
||||||
arb_mode = r.get("audit_mode")
|
# grammar enforcement the reasoning trace is itself
|
||||||
|
# suppressed, so output is clean single-line JSON —
|
||||||
|
# but the stop must still be cleared or the first
|
||||||
|
# structural newline truncates it.)
|
||||||
|
reasoning_ref = bool(MODELS[arborist_ref].get("reasoning"))
|
||||||
|
if reasoning_ref:
|
||||||
|
arb_policy["claim_lattice_json_stop_sequences"] = []
|
||||||
|
# Empty-output self-heal for reasoning refs: qwen-
|
||||||
|
# think under json-schema grammar intermittently
|
||||||
|
# emits an empty completion (~1/3 of calls, measured
|
||||||
|
# 2026-05-20) — a model-side artefact, NOT a real
|
||||||
|
# abstention. Retry (burning the cached empty) up to
|
||||||
|
# 3 attempts so an artefact-empty doesn't masquerade
|
||||||
|
# as ABSTAINED in the scorecard. Non-reasoning refs
|
||||||
|
# don't exhibit this (qwen-nothink phase 3 had 0
|
||||||
|
# spurious empties) so they take a single pass. We
|
||||||
|
# never fabricate — an answer that is still empty
|
||||||
|
# after retries is recorded as the empty it is.
|
||||||
|
max_attempts = 3 if reasoning_ref else 1
|
||||||
|
arb_raw, arb_mode = "", None
|
||||||
|
for attempt in range(max_attempts):
|
||||||
|
r = query(question=q_asked, qa_db=qa_db,
|
||||||
|
chat_client=cl(arborist_ref),
|
||||||
|
model_id=MODELS[arborist_ref]["model"],
|
||||||
|
shards_dir=shards_dir,
|
||||||
|
extra_body=MODELS[arborist_ref]["extra"],
|
||||||
|
policy=arb_policy,
|
||||||
|
burn_existing=(attempt > 0))
|
||||||
|
arb_raw = (r.get("raw_answer")
|
||||||
|
or r.get("answer_text") or "")
|
||||||
|
arb_mode = r.get("audit_mode")
|
||||||
|
if arb_raw.strip():
|
||||||
|
break
|
||||||
except Exception as e: # noqa: BLE001
|
except Exception as e: # noqa: BLE001
|
||||||
arb_raw = f"[arborist-error: {type(e).__name__}: {e}]"
|
arb_raw = f"[arborist-error: {type(e).__name__}: {e}]"
|
||||||
arb_mode = "ERROR"
|
arb_mode = "ERROR"
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue