engineering/uncloseai.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix small model cutoffs by accounting for translation output expansion in chunking logic

Browse source
This commit is contained in:
Russell Ballestrini 2025-07-11 21:02:23 -04:00
parent a43714f82d
commit a7dddf24d8
1 changed files with 12 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

16
src/translation.js
View file

@ -519,24 +519,32 @@ export async function translateHTML(htmlContent, targetLanguage, sourceUrl = nul
const contentTokensResult = await countTokens(processedContent, getSelectedModel());
const contentTokens = typeof contentTokensResult === 'object' ? contentTokensResult.count || contentTokensResult.totalTokens || 0 : contentTokensResult;
// Account for translation expansion (typically 1.5-2x input size)
const estimatedOutputTokens = Math.floor(contentTokens * 1.8);
const totalEstimatedTokens = contentTokens + estimatedOutputTokens;
console.log(`=== CHUNKING DECISION ===`);
console.log(`Max tokens: ${maxTokens}`);
console.log(`Reserve tokens: ${reserveTokens}`);
console.log(`Available tokens: ${availableTokens}`);
console.log(`Content tokens: ${contentTokens}`);
console.log(`Should chunk: ${contentTokens > availableTokens}`);
console.log(`Estimated output tokens: ${estimatedOutputTokens}`);
console.log(`Total estimated tokens: ${totalEstimatedTokens}`);
console.log(`Should chunk: ${totalEstimatedTokens > availableTokens}`);
let translatedContent;
if (contentTokens <= availableTokens) {
if (totalEstimatedTokens <= availableTokens) {
// Content fits in one request - use normal translation with preservation
console.log(`Content fits in single request: ${contentTokens} tokens`);
translatedContent = await translateText(processedContent, targetLanguage);
} else {
// Content needs chunking - chunk BEFORE preservation to maintain HTML structure
console.log(`Content requires chunking: ${contentTokens} tokens > ${availableTokens} limit`);
console.log(`Content requires chunking: ${totalEstimatedTokens} total tokens > ${availableTokens} limit`);
// Split the original HTML content into semantic chunks
const chunks = await splitHTMLIntoChunks(processedContent, availableTokens, getSelectedModel());
// Reduce chunk size to account for translation expansion
const maxChunkInputTokens = Math.floor(availableTokens / 2.8); // Account for 1.8x expansion + overhead
const chunks = await splitHTMLIntoChunks(processedContent, maxChunkInputTokens, getSelectedModel());
console.log(`Split into ${chunks.length} chunks for parallel translation`);
// Debug: log chunk content to see what we're actually translating