diff --git a/src/translation.js b/src/translation.js index 408fbf7..ed1585c 100644 --- a/src/translation.js +++ b/src/translation.js @@ -1,5 +1,5 @@ import { API_KEY, getAPIConfig } from "./config.js"; -import { getSelectedModel, getSelectedModelEndpoint, getSelectedModelMaxTokens } from "./models.js"; +import { getSelectedModel, getSelectedModelEndpoint, getSelectedModelMaxTokens, modelRegistry } from "./models.js"; import { countTokens, countTokensWithUpstream } from "./token_estimator.js"; // Languages supported by Hermes 3 model (in English for reference) @@ -25,72 +25,7 @@ export const SUPPORTED_LANGUAGES = { ko: "Korean", }; -// Function to split HTML content into chunks at semantic boundaries -async function splitHTMLIntoChunks(htmlContent, maxTokensPerChunk, model) { - const chunks = []; - const parser = new DOMParser(); - const doc = parser.parseFromString(htmlContent, 'text/html'); - - // Work directly with body element or document element - const container = doc.body || doc.documentElement; - - // Sibling-level chunking: only use direct children of the body - // This prevents nesting issues and guarantees no overlapping content - const siblingElements = Array.from(container.children); - - console.log(`Found ${siblingElements.length} sibling elements for chunking`); - console.log(`Container: ${container.tagName}`); - siblingElements.forEach((el, idx) => { - console.log(`Sibling ${idx}: <${el.tagName}> length=${el.outerHTML.length}`); - }); - - let currentChunk = ''; - let currentTokens = 0; - - for (const element of siblingElements) { - const elementHTML = element.outerHTML; - const elementTokensResult = await countTokens(elementHTML, model); - const elementTokens = typeof elementTokensResult === 'object' ? elementTokensResult.totalTokens || elementTokensResult.count || 0 : elementTokensResult; - - console.log(`Element tokens: ${elementTokens}, current total: ${currentTokens}, limit: ${maxTokensPerChunk}`); - - // If this single element is larger than max tokens, it needs its own chunk - if (elementTokens > maxTokensPerChunk) { - console.log(`Element too large (${elementTokens} tokens), creating dedicated chunk`); - // Push current chunk if it exists - if (currentChunk) { - chunks.push(currentChunk); - currentChunk = ''; - currentTokens = 0; - } - // Add large element as its own chunk - chunks.push(elementHTML); - } - // If adding this element would exceed the limit, start a new chunk - else if (currentTokens + elementTokens > maxTokensPerChunk && currentChunk) { - console.log(`Starting new chunk at ${currentTokens} tokens`); - chunks.push(currentChunk); - currentChunk = elementHTML; - currentTokens = elementTokens; - } else { - currentChunk += elementHTML; - currentTokens += elementTokens; - } - } - - // Add the last chunk if it exists - if (currentChunk) { - chunks.push(currentChunk); - } - - // If no chunks created, use the entire container content as one chunk - if (chunks.length === 0) { - console.log('No chunks created from siblings, using entire content as single chunk'); - chunks.push(container.innerHTML); - } - - return chunks; -} +// REMOVED: splitHTMLIntoChunks function - no longer using chunking // Native language names with proper scripts, accents, and authentic forms export const NATIVE_LANGUAGE_NAMES = { @@ -410,8 +345,8 @@ ${preservedText}`; } } -// Internal function to translate HTML fragments (for chunked translations) -async function translateFragmentRaw(text, targetLanguage) { +// REMOVED: translateFragmentRaw - no longer using chunking +/* async function translateFragmentRaw(text, targetLanguage) { // Ensure models are loaded before translation const { fetchModelsFromEndpoints } = await import("./models.js"); await fetchModelsFromEndpoints(); @@ -473,7 +408,9 @@ ${preservedText}`; console.error("Error stack:", error.stack); throw new Error(`Fragment translation failed: ${error.message}`); } -} +} */ + +// REMOVED: translateFragment - no longer using chunking // Translate text using Hermes AI with minimal system context export async function translateText(text, targetLanguage) { @@ -631,14 +568,14 @@ export async function translateHTML(htmlContent, targetLanguage, sourceUrl = nul const estimatedOutputTokens = Math.floor(contentTokens * 1.8); const totalEstimatedTokens = contentTokens + estimatedOutputTokens; - console.log(`=== CHUNKING DECISION ===`); + console.log(`=== TOKEN ANALYSIS ===`); console.log(`Max tokens: ${maxTokens}`); console.log(`Reserve tokens: ${reserveTokens}`); console.log(`Available tokens: ${availableTokens}`); console.log(`Content tokens: ${contentTokens}`); console.log(`Estimated output tokens: ${estimatedOutputTokens}`); console.log(`Total estimated tokens: ${totalEstimatedTokens}`); - console.log(`Should chunk: ${totalEstimatedTokens > availableTokens}`); + console.log(`Fits in single request: ${totalEstimatedTokens <= availableTokens}`); let translatedContent; if (totalEstimatedTokens <= availableTokens) { @@ -646,71 +583,21 @@ export async function translateHTML(htmlContent, targetLanguage, sourceUrl = nul console.log(`Content fits in single request: ${contentTokens} tokens`); translatedContent = await translateText(processedContent, targetLanguage); } else { - // Content needs chunking - chunk BEFORE preservation to maintain HTML structure - console.log(`Content requires chunking: ${totalEstimatedTokens} total tokens > ${availableTokens} limit`); + // Content too large for selected model + console.log(`Content too large: ${totalEstimatedTokens} total tokens > ${availableTokens} limit`); - // Split the original HTML content into semantic chunks - // Use reasonable chunk size for the model's context window - const maxChunkInputTokens = Math.floor((maxTokens - 1500) / 2.5); // Leave room for prompt + expansion - const chunks = await splitHTMLIntoChunks(processedContent, maxChunkInputTokens, getSelectedModel()); - console.log(`Split into ${chunks.length} chunks for parallel translation`); + // Get model info for better error message + const modelName = getSelectedModel(); + const modelInfo = modelRegistry[modelName] || { modelName }; - // Debug: log chunk content to see what we're actually translating - chunks.forEach((chunk, index) => { - console.log(`Chunk ${index + 1} length: ${chunk.length} chars`); - console.log(`Chunk ${index + 1} preview:`, chunk.substring(0, 400) + (chunk.length > 400 ? '...' : '')); - }); + throw new Error(`Content is too large for the selected model (${modelName}). - // Process chunks in batches to avoid overwhelming the API - const batchSize = 4; // Increased for faster processing - const translatedChunks = []; - - for (let i = 0; i < chunks.length; i += batchSize) { - const batch = chunks.slice(i, i + batchSize); - console.log(`Processing batch ${Math.floor(i/batchSize) + 1}/${Math.ceil(chunks.length/batchSize)} (chunks ${i + 1}-${Math.min(i + batchSize, chunks.length)})`); - - const batchPromises = batch.map((chunk, batchIndex) => { - const chunkIndex = i + batchIndex + 1; - console.log(`Translating chunk ${chunkIndex}/${chunks.length}`); - console.log(`Chunk ${chunkIndex} content preview:`, chunk.substring(0, 300)); - return translateFragment(chunk, targetLanguage); - }); - - try { - const batchResults = await Promise.all(batchPromises); - translatedChunks.push(...batchResults); - console.log(`Batch ${Math.floor(i/batchSize) + 1} completed successfully`); - } catch (error) { - console.error(`Batch ${Math.floor(i/batchSize) + 1} failed:`, error); - throw error; - } - } - - console.log(`All ${chunks.length} chunks translated successfully in batches`); - - // Reassemble the chunks into the original document structure - const originalParser = new DOMParser(); - const originalDoc = originalParser.parseFromString(processedContent, 'text/html'); - - // Extract only the body content from each translated chunk to avoid stacking full HTML documents - const cleanedChunks = translatedChunks.map((chunk, index) => { - console.log(`Raw translated chunk ${index + 1} preview:`, chunk.substring(0, 500) + '...'); - const chunkDoc = new DOMParser().parseFromString(chunk, 'text/html'); - const bodyContent = chunkDoc.body ? chunkDoc.body.innerHTML : chunk; - console.log(`Extracted body content ${index + 1} preview:`, bodyContent.substring(0, 300) + '...'); - return bodyContent; - }); - - // Replace the body content with reassembled translated chunks - const reassembledBodyContent = cleanedChunks.join(''); - if (originalDoc.body) { - originalDoc.body.innerHTML = reassembledBodyContent; - } else { - // If no body tag, replace documentElement content - originalDoc.documentElement.innerHTML = `${originalDoc.head ? originalDoc.head.innerHTML : ''}${reassembledBodyContent}`; - } - - translatedContent = originalDoc.documentElement.outerHTML; +Content requires ~${Math.ceil(totalEstimatedTokens).toLocaleString()} tokens but model only supports ${maxTokens.toLocaleString()} tokens. + +Please try: +• Using a larger model (32K or higher) +• Translating a smaller section of the page +• Using the custom text tab for specific content`); } // Parse the content to add base tag for URL resolution