diff --git a/aborist/qa/client.py b/aborist/qa/client.py index da88fe4..cda3b69 100644 --- a/aborist/qa/client.py +++ b/aborist/qa/client.py @@ -101,6 +101,7 @@ class OpenAICompatibleClient: max_tokens: int = 512, top_p: float = 1.0, extra_body: dict | None = None, + stop: list[str] | None = None, ) -> str: import httpx import time as _time @@ -115,6 +116,8 @@ class OpenAICompatibleClient: "max_tokens": max_tokens, "top_p": top_p, } + if stop: + payload["stop"] = list(stop) # extra_body merges into the payload root — vLLM accepts knobs # like {"guided_json": {...schema...}} or {"guided_grammar": "..."}. # Endpoints that don't recognize a key silently drop it. diff --git a/aborist/qa/query.py b/aborist/qa/query.py index b278d6c..26b7430 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -420,6 +420,7 @@ DEFAULT_QUERY_POLICY = { "Now answer the question on the next message." ), "claim_lattice_use_guided_json": True, + "claim_lattice_json_stop_sequences": ["\n\n"], } @@ -1492,10 +1493,18 @@ def query( # time. Non-vLLM endpoints silently drop the field; the lenient # pre-parser in the verifier handles whatever drift remains. extra_body: dict | None = None + stop_seqs: list[str] | None = None if answer_mode == "claim_lattice" and policy.get( "claim_lattice_use_guided_json", True ): extra_body = {"guided_json": CLAIM_LATTICE_JSON_SCHEMA} + if answer_mode == "claim_lattice": + # JSON-mode token-runaway guard — see runner.py for the + # full rationale. Stops generation on a blank line so + # post-JSON whitespace spam doesn't blow max_tokens. + stop_seqs = list(policy.get( + "claim_lattice_json_stop_sequences", ["\n\n"] + )) t_phase = time.monotonic() raw_answer = chat_client.chat_completion( messages, @@ -1504,6 +1513,7 @@ def query( max_tokens=policy["max_tokens"], top_p=policy.get("top_p", 1.0), extra_body=extra_body, + stop=stop_seqs, ) llm_ms = _ms_since(t_phase) diff --git a/aborist/qa/runner.py b/aborist/qa/runner.py index 1cb2b94..23eec7e 100644 --- a/aborist/qa/runner.py +++ b/aborist/qa/runner.py @@ -252,6 +252,15 @@ DEFAULT_POLICY = { "characters. Now answer the question on the next message." ), "claim_lattice_use_guided_json": True, + # JSON-mode stop sequences. Hermes-3-8B sometimes spams whitespace + # / newlines after the closing brace on broad-descriptive shapes + # ("plot of X", "tell me about Y") — the response runs out the + # max_tokens budget and the lenient parser sees truncated JSON. + # Stopping on a blank line cuts the runaway. JSON-mode output + # never legitimately contains a blank line (single object, single + # line) so this is a safe filter. Folds into + # governance_policy_hash on change. + "claim_lattice_json_stop_sequences": ["\n\n"], } @@ -500,10 +509,24 @@ def ask( # sampling time. Endpoints without guided-decoding silently drop the # field; the lenient pre-parser handles whatever drift remains. extra_body: dict | None = None + stop_seqs: list[str] | None = None if answer_mode == "claim_lattice" and policy.get( "claim_lattice_use_guided_json", True ): extra_body = {"guided_json": CLAIM_LATTICE_JSON_SCHEMA} + if answer_mode == "claim_lattice": + # JSON-mode token-runaway guard. On broad-descriptive / + # comparison questions Hermes-3-8B sometimes spams whitespace + # / newlines after the closing brace until max_tokens + # exhausts; the resulting truncated payload won't parse and + # the run lands UNGROUNDED 0/0 at 12-15s instead of 2-4s. + # Stopping on a blank line (\n\n) cuts the runaway — + # well-formed JSON-mode output never contains a blank line + # since the model emits a single object on one line (or + # with simple internal newlines). + stop_seqs = list(policy.get( + "claim_lattice_json_stop_sequences", ["\n\n"] + )) raw_answer = client.chat_completion( messages, model=model_id, @@ -511,6 +534,7 @@ def ask( max_tokens=policy["max_tokens"], top_p=policy.get("top_p", 1.0), extra_body=extra_body, + stop=stop_seqs, ) llm_ms = _ms_since(t_llm)