From e50fe6e8303c59a807af52dcbadd380f68888c08 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 25 May 2026 14:49:43 -0400 Subject: [PATCH] streaming: forward reasoning_content to client as separate channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Qwen3.x / Deepseek-R1 / o1 stream model thinking via delta.reasoning_content during SSE; the final answer arrives later via delta.content. Both server streaming paths (OpenAI client + llama-cpp Python lib) now extract reasoning_content per chunk and emit it via the same socketio "message_chunk" event with a `reasoning_content` field (distinct from `content`). Reasoning is forwarded but NOT accumulated into the buffer that persists to the DB — it's transient model-private state, not part of the saved message. Frontend (templates/chat.html): the message_chunk handler dispatches on payload shape. On `reasoning_content`: lazy-create a
block at the top of the message wrapper, append text, return early (no buffer/markdown render). On `content`: if a thinking block exists and is still open, auto-collapse it to "thinking (click to expand)" before standard content rendering. If a response never thinks, no thinking DOM element is created — no visual artefact at all. Compatible with all existing message_chunk listeners; reasoning_content is purely additive. --- app.py | 29 ++++++++++++++++++++-- templates/chat.html | 59 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 81 insertions(+), 7 deletions(-) diff --git a/app.py b/app.py index 8ffd082..e7ccea4 100644 --- a/app.py +++ b/app.py @@ -2241,7 +2241,21 @@ def chat_gpt(username, room_name, model_name="gpt-4o-mini"): del cancellation_requests[msg_id] break - content = chunk.choices[0].delta.content + delta = chunk.choices[0].delta + # Qwen3.x / Deepseek-R1 / o1: thinking streams via delta.reasoning_content, + # final answer via delta.content. Forward reasoning deltas to the client + # so it can lazy-create a collapsible thinking block; do NOT persist them + # (transient, model-private intermediate state). + reasoning = getattr(delta, "reasoning_content", None) + if reasoning: + socketio.emit( + "message_chunk", + {"id": msg_id, "reasoning_content": reasoning}, + room=room_name, + ) + socketio.sleep(0) + + content = delta.content if content: buffer += content # Accumulate content @@ -2358,7 +2372,18 @@ def chat_llama(username, room_name, model_name="mistral-7b-instruct-v0.2.Q3_K_L. del cancellation_requests[msg_id] break - content = chunk["choices"][0]["delta"].get("content") + delta = chunk["choices"][0]["delta"] + # See OpenAI-client path above for rationale on reasoning_content. + reasoning = delta.get("reasoning_content") + if reasoning: + socketio.emit( + "message_chunk", + {"id": msg_id, "reasoning_content": reasoning}, + room=room_name, + ) + socketio.sleep(0) + + content = delta.get("content") if content: buffer += content # Accumulate content diff --git a/templates/chat.html b/templates/chat.html index 0ac3a0c..91ddf75 100644 --- a/templates/chat.html +++ b/templates/chat.html @@ -1444,18 +1444,58 @@ socket.on("message_chunk", (data) => { document.getElementById("chat").appendChild(messageWrapper); } + // Reasoning channel: lazy-create a collapsible
block above + // the message-content div. Only appears if the server actually streams + // delta.reasoning_content (i.e., thinking is on AND the model is using it). + // Auto-collapses below on the first content delta. + if (data.reasoning_content) { + let thinkingDetails = messageWrapper.querySelector(".message-thinking"); + if (!thinkingDetails) { + // Ensure a message-body wrapper exists to anchor against + let messageBodyWrapper = messageWrapper.querySelector(".message-body"); + if (!messageBodyWrapper) { + messageBodyWrapper = document.createElement("div"); + messageBodyWrapper.className = "message-body"; + messageWrapper.appendChild(messageBodyWrapper); + } + thinkingDetails = document.createElement("details"); + thinkingDetails.className = "message-thinking"; + thinkingDetails.open = true; + const summary = document.createElement("summary"); + summary.textContent = "thinking…"; + thinkingDetails.appendChild(summary); + const body = document.createElement("div"); + body.className = "message-thinking-body"; + body.style.opacity = "0.6"; + body.style.fontStyle = "italic"; + body.style.whiteSpace = "pre-wrap"; + thinkingDetails.appendChild(body); + // Insert at top of message-body so thinking appears above the answer + messageBodyWrapper.insertBefore(thinkingDetails, messageBodyWrapper.firstChild); + } + const body = thinkingDetails.querySelector(".message-thinking-body"); + body.textContent += data.reasoning_content; + if (!userHasScrolledUp) { + document.getElementById("chat").scrollTop = document.getElementById("chat").scrollHeight; + } + return; // reasoning deltas don't touch buffer/markdown render + } + // If the message-content div doesn't exist, create it if (!messageWrapper.querySelector(".message-content")) { // Create a message body wrapper to contain both header and content - const messageBodyWrapper = document.createElement("div"); - messageBodyWrapper.className = "message-body"; - messageWrapper.appendChild(messageBodyWrapper); - + let messageBodyWrapper = messageWrapper.querySelector(".message-body"); + if (!messageBodyWrapper) { + messageBodyWrapper = document.createElement("div"); + messageBodyWrapper.className = "message-body"; + messageWrapper.appendChild(messageBodyWrapper); + } + // Create header element for username/model const headerElement = document.createElement("div"); headerElement.className = "message-header"; messageBodyWrapper.appendChild(headerElement); - + // Create content element for actual message content targetMessageElement = document.createElement("div"); targetMessageElement.className = "message-content"; @@ -1464,6 +1504,15 @@ socket.on("message_chunk", (data) => { targetMessageElement = messageWrapper.querySelector(".message-content"); } + // First content delta after thinking: auto-collapse the thinking block + // so the answer is the visible focus. Thinking stays one click away. + const thinkingDetails = messageWrapper.querySelector(".message-thinking"); + if (thinkingDetails && thinkingDetails.open) { + thinkingDetails.open = false; + const summary = thinkingDetails.querySelector("summary"); + if (summary) summary.textContent = "thinking (click to expand)"; + } + // If the message buffer for this ID doesn't exist, create it if (!messageBuffers[data.id]) { messageBuffers[data.id] = "";