From f23d3a306762d81346aef6f472ff9d5da43bc2e9 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 30 Apr 2026 16:15:07 -0400 Subject: [PATCH] qa: JSON-mode stop-sequence guards against post-brace token runaway MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bench data shows residual JSON-mode token runaway after the pointer-ID switch: ~4 cases out of 66 land UNGROUNDED 0/0 at 12-15s instead of ~2-4s normal. Pattern: Hermes-3-8B emits a valid claim object, then keeps generating whitespace / blank lines until max_tokens (512) exhausts. The truncated payload won't parse and the lenient pre-parser returns no claims. Concrete instances in the latest bench (2026-04-30T19-55-11Z): - tell me about the apollo program (3/3 samples runaway) - tell me about the python programming language (1/3) Fix: pass `stop=["\n\n"]` to the chat completion in JSON mode so vLLM cuts generation at the first blank line. Well-formed JSON- mode output never legitimately contains a blank line — Hermes emits one object on a single line (or with simple internal newlines), never `\n\n`. The stop sequence is the runaway signature itself. Plumbing: - OpenAICompatibleClient.chat_completion: new `stop` kwarg, injects into request payload when non-empty - StubClient already absorbs **kwargs; no change needed - DEFAULT_POLICY (runner) + DEFAULT_QUERY_POLICY (query) gain `claim_lattice_json_stop_sequences = ["\n\n"]`. Folds into governance_policy_hash so changing the stop list invalidates prior cached records. - Both call sites in runner.py / query.py read the policy field and pass it only on JSON mode (pointer + quote modes don't need it). Defensive measure: worst case the stop sequence never fires; best case the apollo/python residuals recover and JSON's strict-rate climbs further. --- aborist/qa/client.py | 3 +++ aborist/qa/query.py | 10 ++++++++++ aborist/qa/runner.py | 24 ++++++++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/aborist/qa/client.py b/aborist/qa/client.py index da88fe4..cda3b69 100644 --- a/aborist/qa/client.py +++ b/aborist/qa/client.py @@ -101,6 +101,7 @@ class OpenAICompatibleClient: max_tokens: int = 512, top_p: float = 1.0, extra_body: dict | None = None, + stop: list[str] | None = None, ) -> str: import httpx import time as _time @@ -115,6 +116,8 @@ class OpenAICompatibleClient: "max_tokens": max_tokens, "top_p": top_p, } + if stop: + payload["stop"] = list(stop) # extra_body merges into the payload root — vLLM accepts knobs # like {"guided_json": {...schema...}} or {"guided_grammar": "..."}. # Endpoints that don't recognize a key silently drop it. diff --git a/aborist/qa/query.py b/aborist/qa/query.py index b278d6c..26b7430 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -420,6 +420,7 @@ DEFAULT_QUERY_POLICY = { "Now answer the question on the next message." ), "claim_lattice_use_guided_json": True, + "claim_lattice_json_stop_sequences": ["\n\n"], } @@ -1492,10 +1493,18 @@ def query( # time. Non-vLLM endpoints silently drop the field; the lenient # pre-parser in the verifier handles whatever drift remains. extra_body: dict | None = None + stop_seqs: list[str] | None = None if answer_mode == "claim_lattice" and policy.get( "claim_lattice_use_guided_json", True ): extra_body = {"guided_json": CLAIM_LATTICE_JSON_SCHEMA} + if answer_mode == "claim_lattice": + # JSON-mode token-runaway guard — see runner.py for the + # full rationale. Stops generation on a blank line so + # post-JSON whitespace spam doesn't blow max_tokens. + stop_seqs = list(policy.get( + "claim_lattice_json_stop_sequences", ["\n\n"] + )) t_phase = time.monotonic() raw_answer = chat_client.chat_completion( messages, @@ -1504,6 +1513,7 @@ def query( max_tokens=policy["max_tokens"], top_p=policy.get("top_p", 1.0), extra_body=extra_body, + stop=stop_seqs, ) llm_ms = _ms_since(t_phase) diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index 1cb2b94..23eec7e 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -252,6 +252,15 @@ DEFAULT_POLICY = { "characters. Now answer the question on the next message." ), "claim_lattice_use_guided_json": True, + # JSON-mode stop sequences. Hermes-3-8B sometimes spams whitespace + # / newlines after the closing brace on broad-descriptive shapes + # ("plot of X", "tell me about Y") — the response runs out the + # max_tokens budget and the lenient parser sees truncated JSON. + # Stopping on a blank line cuts the runaway. JSON-mode output + # never legitimately contains a blank line (single object, single + # line) so this is a safe filter. Folds into + # governance_policy_hash on change. + "claim_lattice_json_stop_sequences": ["\n\n"], } @@ -500,10 +509,24 @@ def ask( # sampling time. Endpoints without guided-decoding silently drop the # field; the lenient pre-parser handles whatever drift remains. extra_body: dict | None = None + stop_seqs: list[str] | None = None if answer_mode == "claim_lattice" and policy.get( "claim_lattice_use_guided_json", True ): extra_body = {"guided_json": CLAIM_LATTICE_JSON_SCHEMA} + if answer_mode == "claim_lattice": + # JSON-mode token-runaway guard. On broad-descriptive / + # comparison questions Hermes-3-8B sometimes spams whitespace + # / newlines after the closing brace until max_tokens + # exhausts; the resulting truncated payload won't parse and + # the run lands UNGROUNDED 0/0 at 12-15s instead of 2-4s. + # Stopping on a blank line (\n\n) cuts the runaway — + # well-formed JSON-mode output never contains a blank line + # since the model emits a single object on one line (or + # with simple internal newlines). + stop_seqs = list(policy.get( + "claim_lattice_json_stop_sequences", ["\n\n"] + )) raw_answer = client.chat_completion( messages, model=model_id, @@ -511,6 +534,7 @@ def ask( max_tokens=policy["max_tokens"], top_p=policy.get("top_p", 1.0), extra_body=extra_body, + stop=stop_seqs, ) llm_ms = _ms_since(t_llm)