engineering/uncloseai.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix token buffer calculation causing chunking to fail

Browse source 
Previous buffer calculation used 1.5x input tokens which exceeded model
context window for large content. Changed to fixed 1000 token reserve
and improved chunk sizing to use reasonable portions of model capacity.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Russell Ballestrini 2025-07-11 21:38:47 -04:00
parent 7b34e60f63
commit 04f0ac766b
2 changed files with 10 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/translate-modal.js
View file

@ -336,8 +336,12 @@ export function openTranslateModal() {
// Calculate actual tokens that will be sent to AI (including preservation and chunking)
let totalTokens = 0;
// Import all needed functions
const { getSelectedModelMaxTokens, getSelectedModel } = await import("./models.js");
const { countTokensWithUpstream } = await import("./token_estimator.js");
const { preserveSpecialContent, estimateProcessingTime, SUPPORTED_LANGUAGES } = await import("./translation.js");
// Simulate the same chunking logic that translateHTML will use
const { getSelectedModelMaxTokens } = await import("./models.js");
const maxTokens = getSelectedModelMaxTokens() || 8192;
const reserveTokens = 1000;
const availableTokens = Math.max(maxTokens - reserveTokens, 2000);

10
src/translation.js
View file

@ -125,12 +125,12 @@ async function* sendMessageWithHistory(messageHistory) {
const modelMaxTokens = getSelectedModelMaxTokens();
const inputText = messageHistory.map(msg => msg.content).join('');
const inputTokens = Math.ceil(inputText.length / 4); // Simple token estimation
const buffer = Math.max(2048, Math.floor(inputTokens * 1.5)); // Use 1.5x input tokens as buffer, minimum 2048
const availableTokens = Math.max(100, modelMaxTokens - inputTokens - buffer);
const reserveTokens = 1000; // Reserve for output tokens
const availableTokens = Math.max(100, modelMaxTokens - inputTokens - reserveTokens);
console.log("Translation using endpoint:", apiUrl);
console.log("Translation using model:", model);
console.log("Translation token calculation: max=" + modelMaxTokens + ", input≈" + inputTokens + ", buffer=" + buffer + ", available≈" + availableTokens);
console.log("Translation token calculation: max=" + modelMaxTokens + ", input≈" + inputTokens + ", reserve=" + reserveTokens + ", available≈" + availableTokens);
const response = await fetch(apiUrl, {
method: "POST",
@ -542,8 +542,8 @@ export async function translateHTML(htmlContent, targetLanguage, sourceUrl = nul
console.log(`Content requires chunking: ${totalEstimatedTokens} total tokens > ${availableTokens} limit`);
// Split the original HTML content into semantic chunks
// Reduce chunk size to account for translation expansion
const maxChunkInputTokens = Math.floor(availableTokens / 2.8); // Account for 1.8x expansion + overhead
// Use reasonable chunk size for the model's context window
const maxChunkInputTokens = Math.floor((maxTokens - 1500) / 2.5); // Leave room for prompt + expansion
const chunks = await splitHTMLIntoChunks(processedContent, maxChunkInputTokens, getSelectedModel());
console.log(`Split into ${chunks.length} chunks for parallel translation`);