engineering/uncloseai.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix chunking to work on original HTML before preservation, not placeholder soup

Browse source
This commit is contained in:
Russell Ballestrini 2025-07-11 20:29:05 -04:00
parent a2a786040e
commit bbac26b487
1 changed files with 7 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

20
src/translation.js
View file

@ -505,33 +505,27 @@ export async function translateHTML(htmlContent, targetLanguage, sourceUrl = nul
const reserveTokens = 1000; // Reserve tokens for prompt overhead
const availableTokens = Math.max(maxTokens - reserveTokens, 2000); // Ensure minimum
// Check if content needs chunking based on token count
// Check if content needs chunking based on token count (on raw HTML, not preserved)
const contentTokens = await countTokens(processedContent, getSelectedModel());
let translatedContent;
if (contentTokens <= availableTokens) {
// Content fits in one request
// Content fits in one request - use normal translation with preservation
console.log(`Content fits in single request: ${contentTokens} tokens`);
translatedContent = await translateText(processedContent, targetLanguage);
} else {
// Content needs chunking - translate all chunks in parallel
// Content needs chunking - chunk BEFORE preservation to maintain HTML structure
console.log(`Content requires chunking: ${contentTokens} tokens > ${availableTokens} limit`);
// Split the original HTML content into semantic chunks
const chunks = await splitHTMLIntoChunks(processedContent, availableTokens, getSelectedModel());
console.log(`Split into ${chunks.length} chunks for parallel translation`);
// Apply HTML preservation to each chunk individually before translation
// Translate all chunks in parallel using the normal translateText function
const translationPromises = chunks.map((chunk, index) => {
console.log(`Queued chunk ${index + 1}/${chunks.length} for translation`);
// Each chunk gets its own preservation/restoration cycle
const { preservedText, preservations } = preserveSpecialContent(chunk);
// Translate the preserved chunk using raw function (no double preservation)
return translateTextRaw(preservedText, targetLanguage).then(translatedChunk => {
// Restore special content for this chunk
return restoreSpecialContent(translatedChunk, preservations);
});
// Use full translateText which handles preservation/restoration per chunk
return translateText(chunk, targetLanguage);
});
// Wait for all translations to complete