engineering/uncloseai.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add debug logging to see why chunking is being triggered when it shouldn't

Browse source
This commit is contained in:
Russell Ballestrini 2025-07-11 20:46:44 -04:00
parent f476b6e476
commit af6108e988
1 changed files with 31 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

40
src/translation.js
View file

@ -513,6 +513,13 @@ export async function translateHTML(htmlContent, targetLanguage, sourceUrl = nul
// Check if content needs chunking based on token count (on raw HTML, not preserved)
const contentTokens = await countTokens(processedContent, getSelectedModel());
console.log(`=== CHUNKING DECISION ===`);
console.log(`Max tokens: ${maxTokens}`);
console.log(`Reserve tokens: ${reserveTokens}`);
console.log(`Available tokens: ${availableTokens}`);
console.log(`Content tokens: ${contentTokens}`);
console.log(`Should chunk: ${contentTokens > availableTokens}`);
let translatedContent;
if (contentTokens <= availableTokens) {
// Content fits in one request - use normal translation with preservation
@ -531,16 +538,31 @@ export async function translateHTML(htmlContent, targetLanguage, sourceUrl = nul
console.log(`Chunk ${index + 1} preview:`, chunk.substring(0, 200) + (chunk.length > 200 ? '...' : ''));
});
// Translate all chunks in parallel using the normal translateText function
const translationPromises = chunks.map((chunk, index) => {
console.log(`Queued chunk ${index + 1}/${chunks.length} for translation`);
// Use full translateText which handles preservation/restoration per chunk
return translateText(chunk, targetLanguage);
});
// Process chunks in batches to avoid overwhelming the API
const batchSize = 2; // Limit concurrent requests
const translatedChunks = [];
// Wait for all translations to complete
const translatedChunks = await Promise.all(translationPromises);
console.log(`All ${chunks.length} chunks translated successfully in parallel`);
for (let i = 0; i < chunks.length; i += batchSize) {
const batch = chunks.slice(i, i + batchSize);
console.log(`Processing batch ${Math.floor(i/batchSize) + 1}/${Math.ceil(chunks.length/batchSize)} (chunks ${i + 1}-${Math.min(i + batchSize, chunks.length)})`);
const batchPromises = batch.map((chunk, batchIndex) => {
const chunkIndex = i + batchIndex + 1;
console.log(`Translating chunk ${chunkIndex}/${chunks.length}`);
return translateText(chunk, targetLanguage);
});
try {
const batchResults = await Promise.all(batchPromises);
translatedChunks.push(...batchResults);
console.log(`Batch ${Math.floor(i/batchSize) + 1} completed successfully`);
} catch (error) {
console.error(`Batch ${Math.floor(i/batchSize) + 1} failed:`, error);
throw error;
}
}
console.log(`All ${chunks.length} chunks translated successfully in batches`);
// Reassemble the chunks into the original document structure
const originalParser = new DOMParser();