diff --git a/src/translation.js b/src/translation.js index a283609..4dc2985 100644 --- a/src/translation.js +++ b/src/translation.js @@ -31,8 +31,15 @@ async function splitHTMLIntoChunks(htmlContent, maxTokensPerChunk, model) { const parser = new DOMParser(); const doc = parser.parseFromString(htmlContent, 'text/html'); - // Find semantic boundaries (paragraphs, sections, articles, divs) - const elements = doc.querySelectorAll('p, section, article, div, h1, h2, h3, h4, h5, h6, li, blockquote'); + // Extract only the body content for chunking to avoid duplicating html/head/body tags + const bodyContent = doc.body ? doc.body.innerHTML : doc.documentElement.innerHTML; + + // Create a temporary container to work with body content + const tempDoc = new DOMParser().parseFromString(`
${bodyContent}
`, 'text/html'); + const container = tempDoc.querySelector('div'); + + // Find semantic boundaries within the body content + const elements = container.querySelectorAll('p, section, article, div, h1, h2, h3, h4, h5, h6, li, blockquote'); let currentChunk = ''; let currentTokens = 0; @@ -57,12 +64,12 @@ async function splitHTMLIntoChunks(htmlContent, maxTokensPerChunk, model) { chunks.push(currentChunk); } - // If no semantic elements found, fall back to character-based splitting + // If no semantic elements found, fall back to character-based splitting of body content if (chunks.length === 0) { - const totalTokens = await countTokens(htmlContent, model); - const chunkSize = Math.floor(htmlContent.length / Math.ceil(totalTokens / maxTokensPerChunk)); - for (let i = 0; i < htmlContent.length; i += chunkSize) { - chunks.push(htmlContent.substring(i, i + chunkSize)); + const totalTokens = await countTokens(bodyContent, model); + const chunkSize = Math.floor(bodyContent.length / Math.ceil(totalTokens / maxTokensPerChunk)); + for (let i = 0; i < bodyContent.length; i += chunkSize) { + chunks.push(bodyContent.substring(i, i + chunkSize)); } } @@ -532,8 +539,20 @@ export async function translateHTML(htmlContent, targetLanguage, sourceUrl = nul const translatedChunks = await Promise.all(translationPromises); console.log(`All ${chunks.length} chunks translated successfully in parallel`); - // Reassemble the chunks in order - translatedContent = translatedChunks.join(''); + // Reassemble the chunks into the original document structure + const originalParser = new DOMParser(); + const originalDoc = originalParser.parseFromString(processedContent, 'text/html'); + + // Replace the body content with reassembled translated chunks + const reassembledBodyContent = translatedChunks.join(''); + if (originalDoc.body) { + originalDoc.body.innerHTML = reassembledBodyContent; + } else { + // If no body tag, replace documentElement content + originalDoc.documentElement.innerHTML = `${originalDoc.head ? originalDoc.head.innerHTML : ''}${reassembledBodyContent}`; + } + + translatedContent = originalDoc.documentElement.outerHTML; } // Parse the content to add base tag for URL resolution