engineering/uncloseai.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix DOM duplication by chunking only body content and reassembling properly

Browse source
This commit is contained in:
Russell Ballestrini 2025-07-11 20:39:15 -04:00
parent bbac26b487
commit 8e78df00c0
1 changed files with 28 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

37
src/translation.js
View file

@ -31,8 +31,15 @@ async function splitHTMLIntoChunks(htmlContent, maxTokensPerChunk, model) {
const parser = new DOMParser();
const doc = parser.parseFromString(htmlContent, 'text/html');
// Find semantic boundaries (paragraphs, sections, articles, divs)
const elements = doc.querySelectorAll('p, section, article, div, h1, h2, h3, h4, h5, h6, li, blockquote');
// Extract only the body content for chunking to avoid duplicating html/head/body tags
const bodyContent = doc.body ? doc.body.innerHTML : doc.documentElement.innerHTML;
// Create a temporary container to work with body content
const tempDoc = new DOMParser().parseFromString(`<div>${bodyContent}</div>`, 'text/html');
const container = tempDoc.querySelector('div');
// Find semantic boundaries within the body content
const elements = container.querySelectorAll('p, section, article, div, h1, h2, h3, h4, h5, h6, li, blockquote');
let currentChunk = '';
let currentTokens = 0;
@ -57,12 +64,12 @@ async function splitHTMLIntoChunks(htmlContent, maxTokensPerChunk, model) {
chunks.push(currentChunk);
}
// If no semantic elements found, fall back to character-based splitting
// If no semantic elements found, fall back to character-based splitting of body content
if (chunks.length === 0) {
const totalTokens = await countTokens(htmlContent, model);
const chunkSize = Math.floor(htmlContent.length / Math.ceil(totalTokens / maxTokensPerChunk));
for (let i = 0; i < htmlContent.length; i += chunkSize) {
chunks.push(htmlContent.substring(i, i + chunkSize));
const totalTokens = await countTokens(bodyContent, model);
const chunkSize = Math.floor(bodyContent.length / Math.ceil(totalTokens / maxTokensPerChunk));
for (let i = 0; i < bodyContent.length; i += chunkSize) {
chunks.push(bodyContent.substring(i, i + chunkSize));
}
}
@ -532,8 +539,20 @@ export async function translateHTML(htmlContent, targetLanguage, sourceUrl = nul
const translatedChunks = await Promise.all(translationPromises);
console.log(`All ${chunks.length} chunks translated successfully in parallel`);
// Reassemble the chunks in order
translatedContent = translatedChunks.join('');
// Reassemble the chunks into the original document structure
const originalParser = new DOMParser();
const originalDoc = originalParser.parseFromString(processedContent, 'text/html');
// Replace the body content with reassembled translated chunks
const reassembledBodyContent = translatedChunks.join('');
if (originalDoc.body) {
originalDoc.body.innerHTML = reassembledBodyContent;
} else {
// If no body tag, replace documentElement content
originalDoc.documentElement.innerHTML = `<head>${originalDoc.head ? originalDoc.head.innerHTML : ''}</head><body>${reassembledBodyContent}</body>`;
}
translatedContent = originalDoc.documentElement.outerHTML;
}
// Parse the content to add base tag for URL resolution