engineering/opencompletion.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

streaming: forward reasoning_content to client as separate channel

Browse source 
Qwen3.x / Deepseek-R1 / o1 stream model thinking via
delta.reasoning_content during SSE; the final answer arrives later via
delta.content. Both server streaming paths (OpenAI client + llama-cpp
Python lib) now extract reasoning_content per chunk and emit it via the
same socketio "message_chunk" event with a `reasoning_content` field
(distinct from `content`). Reasoning is forwarded but NOT accumulated
into the buffer that persists to the DB — it's transient model-private
state, not part of the saved message.

Frontend (templates/chat.html): the message_chunk handler dispatches
on payload shape. On `reasoning_content`: lazy-create a
<details class="message-thinking"> block at the top of the message
wrapper, append text, return early (no buffer/markdown render). On
`content`: if a thinking block exists and is still open, auto-collapse
it to "thinking (click to expand)" before standard content rendering.

If a response never thinks, no thinking DOM element is created — no
visual artefact at all. Compatible with all existing message_chunk
listeners; reasoning_content is purely additive.
This commit is contained in:
russell@unturf.com 2026-05-25 14:49:43 -04:00
parent 8110080a4f
commit e50fe6e830
No known key found for this signature in database
2 changed files with 81 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

29
app.py
View file

@ -2241,7 +2241,21 @@ def chat_gpt(username, room_name, model_name="gpt-4o-mini"):
del cancellation_requests[msg_id]
break
content = chunk.choices[0].delta.content
delta = chunk.choices[0].delta
# Qwen3.x / Deepseek-R1 / o1: thinking streams via delta.reasoning_content,
# final answer via delta.content. Forward reasoning deltas to the client
# so it can lazy-create a collapsible thinking block; do NOT persist them
# (transient, model-private intermediate state).
reasoning = getattr(delta, "reasoning_content", None)
if reasoning:
socketio.emit(
"message_chunk",
{"id": msg_id, "reasoning_content": reasoning},
room=room_name,
)
socketio.sleep(0)
content = delta.content
if content:
buffer += content # Accumulate content
@ -2358,7 +2372,18 @@ def chat_llama(username, room_name, model_name="mistral-7b-instruct-v0.2.Q3_K_L.
del cancellation_requests[msg_id]
break
content = chunk["choices"][0]["delta"].get("content")
delta = chunk["choices"][0]["delta"]
# See OpenAI-client path above for rationale on reasoning_content.
reasoning = delta.get("reasoning_content")
if reasoning:
socketio.emit(
"message_chunk",
{"id": msg_id, "reasoning_content": reasoning},
room=room_name,
)
socketio.sleep(0)
content = delta.get("content")
if content:
buffer += content # Accumulate content

59
templates/chat.html
View file

@ -1444,18 +1444,58 @@ socket.on("message_chunk", (data) => {
document.getElementById("chat").appendChild(messageWrapper);
}
// Reasoning channel: lazy-create a collapsible <details> block above
// the message-content div. Only appears if the server actually streams
// delta.reasoning_content (i.e., thinking is on AND the model is using it).
// Auto-collapses below on the first content delta.
if (data.reasoning_content) {
let thinkingDetails = messageWrapper.querySelector(".message-thinking");
if (!thinkingDetails) {
// Ensure a message-body wrapper exists to anchor against
let messageBodyWrapper = messageWrapper.querySelector(".message-body");
if (!messageBodyWrapper) {
messageBodyWrapper = document.createElement("div");
messageBodyWrapper.className = "message-body";
messageWrapper.appendChild(messageBodyWrapper);
}
thinkingDetails = document.createElement("details");
thinkingDetails.className = "message-thinking";
thinkingDetails.open = true;
const summary = document.createElement("summary");
summary.textContent = "thinking…";
thinkingDetails.appendChild(summary);
const body = document.createElement("div");
body.className = "message-thinking-body";
body.style.opacity = "0.6";
body.style.fontStyle = "italic";
body.style.whiteSpace = "pre-wrap";
thinkingDetails.appendChild(body);
// Insert at top of message-body so thinking appears above the answer
messageBodyWrapper.insertBefore(thinkingDetails, messageBodyWrapper.firstChild);
}
const body = thinkingDetails.querySelector(".message-thinking-body");
body.textContent += data.reasoning_content;
if (!userHasScrolledUp) {
document.getElementById("chat").scrollTop = document.getElementById("chat").scrollHeight;
}
return; // reasoning deltas don't touch buffer/markdown render
}
// If the message-content div doesn't exist, create it
if (!messageWrapper.querySelector(".message-content")) {
// Create a message body wrapper to contain both header and content
const messageBodyWrapper = document.createElement("div");
messageBodyWrapper.className = "message-body";
messageWrapper.appendChild(messageBodyWrapper);
let messageBodyWrapper = messageWrapper.querySelector(".message-body");
if (!messageBodyWrapper) {
messageBodyWrapper = document.createElement("div");
messageBodyWrapper.className = "message-body";
messageWrapper.appendChild(messageBodyWrapper);
}
// Create header element for username/model
const headerElement = document.createElement("div");
headerElement.className = "message-header";
messageBodyWrapper.appendChild(headerElement);
// Create content element for actual message content
targetMessageElement = document.createElement("div");
targetMessageElement.className = "message-content";
@ -1464,6 +1504,15 @@ socket.on("message_chunk", (data) => {
targetMessageElement = messageWrapper.querySelector(".message-content");
}
// First content delta after thinking: auto-collapse the thinking block
// so the answer is the visible focus. Thinking stays one click away.
const thinkingDetails = messageWrapper.querySelector(".message-thinking");
if (thinkingDetails && thinkingDetails.open) {
thinkingDetails.open = false;
const summary = thinkingDetails.querySelector("summary");
if (summary) summary.textContent = "thinking (click to expand)";
}
// If the message buffer for this ID doesn't exist, create it
if (!messageBuffers[data.id]) {
messageBuffers[data.id] = "";