From 471d61176de7744b42f533fb622cfb9bdf226435 Mon Sep 17 00:00:00 2001
From: Russell Ballestrini <russell.ballestrini@gmail.com>
Date: Sat, 12 Jul 2025 13:31:39 -0400
Subject: [PATCH] allow chat responses to use full available space for coherent
 answers - cap at 4096 tokens max

---
 src/chat.js | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/chat.js b/src/chat.js
index 803a556..353e747 100644
--- a/src/chat.js
+++ b/src/chat.js
@@ -110,8 +110,15 @@ export async function* sendMessage(message) {
 	
 	// Cap to model's max completion tokens limit
 	const maxCompletionTokens = getSelectedModelMaxCompletionTokens();
+	// For chat, prefer using the full calculated available space for coherent responses
+	// Only cap if we're exceeding the model's hard limit
 	const finalAvailable = Math.min(calculatedAvailable, maxCompletionTokens);
-	console.log("Chat using tokens:", finalAvailable, "calculated:", calculatedAvailable, "max completion:", maxCompletionTokens);
+	
+	// However, for very large available token counts, cap at a reasonable max to prevent rambling
+	const reasonableMax = 4096; // Enough for very thorough responses without being excessive
+	const actualFinal = Math.min(finalAvailable, reasonableMax);
+	
+	console.log("Chat using tokens:", actualFinal, "calculated:", calculatedAvailable, "max completion:", maxCompletionTokens);
 	
 	const response = await fetch(apiUrl, {
 		method: "POST",
@@ -120,7 +127,7 @@ export async function* sendMessage(message) {
 			model: model,
 			messages: chatHistory,
 			temperature: 0.5,
-			max_tokens: finalAvailable,
+			max_tokens: actualFinal,
 			stream: true,
 		}),
 	});