From a7dddf24d8c1855b1417666e1b4096317626c6c2 Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Fri, 11 Jul 2025 21:02:23 -0400 Subject: [PATCH] fix small model cutoffs by accounting for translation output expansion in chunking logic --- src/translation.js | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/translation.js b/src/translation.js index 675bb8e..608243c 100644 --- a/src/translation.js +++ b/src/translation.js @@ -519,24 +519,32 @@ export async function translateHTML(htmlContent, targetLanguage, sourceUrl = nul const contentTokensResult = await countTokens(processedContent, getSelectedModel()); const contentTokens = typeof contentTokensResult === 'object' ? contentTokensResult.count || contentTokensResult.totalTokens || 0 : contentTokensResult; + // Account for translation expansion (typically 1.5-2x input size) + const estimatedOutputTokens = Math.floor(contentTokens * 1.8); + const totalEstimatedTokens = contentTokens + estimatedOutputTokens; + console.log(`=== CHUNKING DECISION ===`); console.log(`Max tokens: ${maxTokens}`); console.log(`Reserve tokens: ${reserveTokens}`); console.log(`Available tokens: ${availableTokens}`); console.log(`Content tokens: ${contentTokens}`); - console.log(`Should chunk: ${contentTokens > availableTokens}`); + console.log(`Estimated output tokens: ${estimatedOutputTokens}`); + console.log(`Total estimated tokens: ${totalEstimatedTokens}`); + console.log(`Should chunk: ${totalEstimatedTokens > availableTokens}`); let translatedContent; - if (contentTokens <= availableTokens) { + if (totalEstimatedTokens <= availableTokens) { // Content fits in one request - use normal translation with preservation console.log(`Content fits in single request: ${contentTokens} tokens`); translatedContent = await translateText(processedContent, targetLanguage); } else { // Content needs chunking - chunk BEFORE preservation to maintain HTML structure - console.log(`Content requires chunking: ${contentTokens} tokens > ${availableTokens} limit`); + console.log(`Content requires chunking: ${totalEstimatedTokens} total tokens > ${availableTokens} limit`); // Split the original HTML content into semantic chunks - const chunks = await splitHTMLIntoChunks(processedContent, availableTokens, getSelectedModel()); + // Reduce chunk size to account for translation expansion + const maxChunkInputTokens = Math.floor(availableTokens / 2.8); // Account for 1.8x expansion + overhead + const chunks = await splitHTMLIntoChunks(processedContent, maxChunkInputTokens, getSelectedModel()); console.log(`Split into ${chunks.length} chunks for parallel translation`); // Debug: log chunk content to see what we're actually translating