From 0b02a3d75e711662e7c0003dc4092b13ec38b5fa Mon Sep 17 00:00:00 2001
From: Russell Ballestrini <russell.ballestrini@gmail.com>
Date: Fri, 11 Jul 2025 20:11:31 -0400
Subject: [PATCH] implement intelligent chunking for large page translations
 based on actual token limits

---
 src/translation.js | 47 ++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 45 insertions(+), 2 deletions(-)

diff --git a/src/translation.js b/src/translation.js
index c4adbaf..8e35ecd 100644
--- a/src/translation.js
+++ b/src/translation.js
@@ -25,6 +25,50 @@ export const SUPPORTED_LANGUAGES = {
 	ko: "Korean",
 };
 
+// Function to split HTML content into chunks at semantic boundaries
+async function splitHTMLIntoChunks(htmlContent, maxTokensPerChunk, model) {
+	const chunks = [];
+	const parser = new DOMParser();
+	const doc = parser.parseFromString(htmlContent, 'text/html');
+	
+	// Find semantic boundaries (paragraphs, sections, articles, divs)
+	const elements = doc.querySelectorAll('p, section, article, div, h1, h2, h3, h4, h5, h6, li, blockquote');
+	
+	let currentChunk = '';
+	let currentTokens = 0;
+	
+	for (const element of elements) {
+		const elementHTML = element.outerHTML;
+		const elementTokens = await countTokens(elementHTML, model);
+		
+		// If adding this element would exceed the limit, start a new chunk
+		if (currentTokens + elementTokens > maxTokensPerChunk && currentChunk) {
+			chunks.push(currentChunk);
+			currentChunk = elementHTML;
+			currentTokens = elementTokens;
+		} else {
+			currentChunk += elementHTML;
+			currentTokens += elementTokens;
+		}
+	}
+	
+	// Add the last chunk if it exists
+	if (currentChunk) {
+		chunks.push(currentChunk);
+	}
+	
+	// If no semantic elements found, fall back to character-based splitting
+	if (chunks.length === 0) {
+		const totalTokens = await countTokens(htmlContent, model);
+		const chunkSize = Math.floor(htmlContent.length / Math.ceil(totalTokens / maxTokensPerChunk));
+		for (let i = 0; i < htmlContent.length; i += chunkSize) {
+			chunks.push(htmlContent.substring(i, i + chunkSize));
+		}
+	}
+	
+	return chunks;
+}
+
 // Native language names with proper scripts, accents, and authentic forms
 export const NATIVE_LANGUAGE_NAMES = {
 	en: "English",
@@ -451,8 +495,7 @@ export async function translateHTML(htmlContent, targetLanguage, sourceUrl = nul
 			? `${processedContent.substring(0, maxLength)}...`
 			: processedContent;
 
-	// Translate the content
-	const translatedContent = await translateText(contentToTranslate, targetLanguage);
+	// Translation content is already handled above (chunked or single request)
 	
 	// Parse the content to add base tag for URL resolution
 	const parser = new DOMParser();