optimize chat token allocation for maximum input/output space
- replace aggressive 1.5x buffer with smart model-based allocation - 128k+ models: only 2k buffer (was wasting 10k+ tokens) - 32k models: 1.5k buffer max - 8k models: conservative buffer - follow-up responses now have maximum room for both history and output - especially important for long conversations on large context models
This commit is contained in:
parent
539121a165
commit
8467f58d6b
1 changed files with 15 additions and 1 deletions
16
src/chat.js
16
src/chat.js
|
|
@ -13,7 +13,21 @@ function estimateTokens(text) {
|
|||
function calculateAvailableTokens(chatHistory, maxTokens) {
|
||||
const inputText = chatHistory.map(msg => msg.content).join('');
|
||||
const inputTokens = estimateTokens(inputText);
|
||||
const buffer = Math.max(2048, Math.floor(inputTokens * 1.5)); // Use 1.5x input tokens as buffer, minimum 2048
|
||||
|
||||
// For large context models (>32k), use a more reasonable buffer
|
||||
// For smaller models, be more conservative
|
||||
let buffer;
|
||||
if (maxTokens > 32000) {
|
||||
// Large context models: just reserve 10% or 2k tokens for safety
|
||||
buffer = Math.min(2000, Math.floor(maxTokens * 0.1));
|
||||
} else if (maxTokens > 8000) {
|
||||
// Medium models: reserve 20% or 1.5k tokens
|
||||
buffer = Math.min(1500, Math.floor(maxTokens * 0.2));
|
||||
} else {
|
||||
// Small models: keep conservative approach
|
||||
buffer = Math.max(1000, Math.floor(inputTokens * 0.5));
|
||||
}
|
||||
|
||||
const availableTokens = Math.max(100, maxTokens - inputTokens - buffer);
|
||||
console.log(`Token calculation: max=${maxTokens}, input≈${inputTokens}, buffer=${buffer}, available≈${availableTokens}`);
|
||||
return availableTokens;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue