diff --git a/src/translation.js b/src/translation.js index 8e35ecd..3478c65 100644 --- a/src/translation.js +++ b/src/translation.js @@ -488,14 +488,39 @@ export async function translateHTML(htmlContent, targetLanguage, sourceUrl = nul console.log('Converted relative URLs using base URL:', sourceUrl); } - // Limit content length to avoid overwhelming the AI - const maxLength = 50000; // Much higher limit for complete page translation - const contentToTranslate = - processedContent.length > maxLength - ? `${processedContent.substring(0, maxLength)}...` - : processedContent; - - // Translation content is already handled above (chunked or single request) + // Use actual token limits instead of character limits + const maxTokens = getSelectedModelMaxTokens(); + const reserveTokens = 1000; // Reserve tokens for prompt overhead + const availableTokens = Math.max(maxTokens - reserveTokens, 2000); // Ensure minimum + + // Check if content needs chunking based on token count + const contentTokens = await countTokens(processedContent, getSelectedModel()); + + let translatedContent; + if (contentTokens <= availableTokens) { + // Content fits in one request + console.log(`Content fits in single request: ${contentTokens} tokens`); + translatedContent = await translateText(processedContent, targetLanguage); + } else { + // Content needs chunking - translate all chunks in parallel + console.log(`Content requires chunking: ${contentTokens} tokens > ${availableTokens} limit`); + + const chunks = await splitHTMLIntoChunks(processedContent, availableTokens, getSelectedModel()); + console.log(`Split into ${chunks.length} chunks for parallel translation`); + + // Translate all chunks in parallel for much faster processing + const translationPromises = chunks.map((chunk, index) => { + console.log(`Queued chunk ${index + 1}/${chunks.length} for translation`); + return translateText(chunk, targetLanguage); + }); + + // Wait for all translations to complete + const translatedChunks = await Promise.all(translationPromises); + console.log(`All ${chunks.length} chunks translated successfully in parallel`); + + // Reassemble the chunks in order + translatedContent = translatedChunks.join(''); + } // Parse the content to add base tag for URL resolution const parser = new DOMParser();