engineering/uncloseai.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remove artificial response caps - let models use their full completion token limits

Browse source
This commit is contained in:
Russell Ballestrini 2025-07-12 13:32:56 -04:00
parent 471d61176d
commit 04fa0fb05f
1 changed files with 3 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

11
src/chat.js
View file

@ -110,15 +110,10 @@ export async function* sendMessage(message) {
// Cap to model's max completion tokens limit
const maxCompletionTokens = getSelectedModelMaxCompletionTokens();
// For chat, prefer using the full calculated available space for coherent responses
// Only cap if we're exceeding the model's hard limit
// Use the full calculated available space or the model's limit, whichever is smaller
const finalAvailable = Math.min(calculatedAvailable, maxCompletionTokens);
// However, for very large available token counts, cap at a reasonable max to prevent rambling
const reasonableMax = 4096; // Enough for very thorough responses without being excessive
const actualFinal = Math.min(finalAvailable, reasonableMax);
console.log("Chat using tokens:", actualFinal, "calculated:", calculatedAvailable, "max completion:", maxCompletionTokens);
console.log("Chat using tokens:", finalAvailable, "calculated:", calculatedAvailable, "max completion:", maxCompletionTokens);
const response = await fetch(apiUrl, {
method: "POST",
@ -127,7 +122,7 @@ export async function* sendMessage(message) {
model: model,
messages: chatHistory,
temperature: 0.5,
max_tokens: actualFinal,
max_tokens: finalAvailable,
stream: true,
}),
});