optimize chat token allocation for maximum input/output space

- replace aggressive 1.5x buffer with smart model-based allocation
- 128k+ models: only 2k buffer (was wasting 10k+ tokens)
- 32k models: 1.5k buffer max
- 8k models: conservative buffer
- follow-up responses now have maximum room for both history and output
- especially important for long conversations on large context models
This commit is contained in:
Russell Ballestrini 2025-07-12 13:29:58 -04:00
parent 539121a165
commit 8467f58d6b

View file

@ -13,7 +13,21 @@ function estimateTokens(text) {
function calculateAvailableTokens(chatHistory, maxTokens) {
const inputText = chatHistory.map(msg => msg.content).join('');
const inputTokens = estimateTokens(inputText);
const buffer = Math.max(2048, Math.floor(inputTokens * 1.5)); // Use 1.5x input tokens as buffer, minimum 2048
// For large context models (>32k), use a more reasonable buffer
// For smaller models, be more conservative
let buffer;
if (maxTokens > 32000) {
// Large context models: just reserve 10% or 2k tokens for safety
buffer = Math.min(2000, Math.floor(maxTokens * 0.1));
} else if (maxTokens > 8000) {
// Medium models: reserve 20% or 1.5k tokens
buffer = Math.min(1500, Math.floor(maxTokens * 0.2));
} else {
// Small models: keep conservative approach
buffer = Math.max(1000, Math.floor(inputTokens * 0.5));
}
const availableTokens = Math.max(100, maxTokens - inputTokens - buffer);
console.log(`Token calculation: max=${maxTokens}, input≈${inputTokens}, buffer=${buffer}, available≈${availableTokens}`);
return availableTokens;