diff --git a/src/chat.js b/src/chat.js index 803a556..353e747 100644 --- a/src/chat.js +++ b/src/chat.js @@ -110,8 +110,15 @@ export async function* sendMessage(message) { // Cap to model's max completion tokens limit const maxCompletionTokens = getSelectedModelMaxCompletionTokens(); + // For chat, prefer using the full calculated available space for coherent responses + // Only cap if we're exceeding the model's hard limit const finalAvailable = Math.min(calculatedAvailable, maxCompletionTokens); - console.log("Chat using tokens:", finalAvailable, "calculated:", calculatedAvailable, "max completion:", maxCompletionTokens); + + // However, for very large available token counts, cap at a reasonable max to prevent rambling + const reasonableMax = 4096; // Enough for very thorough responses without being excessive + const actualFinal = Math.min(finalAvailable, reasonableMax); + + console.log("Chat using tokens:", actualFinal, "calculated:", calculatedAvailable, "max completion:", maxCompletionTokens); const response = await fetch(apiUrl, { method: "POST", @@ -120,7 +127,7 @@ export async function* sendMessage(message) { model: model, messages: chatHistory, temperature: 0.5, - max_tokens: finalAvailable, + max_tokens: actualFinal, stream: true, }), });