remove chunking complexity, use simple fit-or-error approach
This commit is contained in:
parent
50a0487bcd
commit
fd71442331
1 changed files with 21 additions and 134 deletions
|
|
@ -1,5 +1,5 @@
|
|||
import { API_KEY, getAPIConfig } from "./config.js";
|
||||
import { getSelectedModel, getSelectedModelEndpoint, getSelectedModelMaxTokens } from "./models.js";
|
||||
import { getSelectedModel, getSelectedModelEndpoint, getSelectedModelMaxTokens, modelRegistry } from "./models.js";
|
||||
import { countTokens, countTokensWithUpstream } from "./token_estimator.js";
|
||||
|
||||
// Languages supported by Hermes 3 model (in English for reference)
|
||||
|
|
@ -25,72 +25,7 @@ export const SUPPORTED_LANGUAGES = {
|
|||
ko: "Korean",
|
||||
};
|
||||
|
||||
// Function to split HTML content into chunks at semantic boundaries
|
||||
async function splitHTMLIntoChunks(htmlContent, maxTokensPerChunk, model) {
|
||||
const chunks = [];
|
||||
const parser = new DOMParser();
|
||||
const doc = parser.parseFromString(htmlContent, 'text/html');
|
||||
|
||||
// Work directly with body element or document element
|
||||
const container = doc.body || doc.documentElement;
|
||||
|
||||
// Sibling-level chunking: only use direct children of the body
|
||||
// This prevents nesting issues and guarantees no overlapping content
|
||||
const siblingElements = Array.from(container.children);
|
||||
|
||||
console.log(`Found ${siblingElements.length} sibling elements for chunking`);
|
||||
console.log(`Container: ${container.tagName}`);
|
||||
siblingElements.forEach((el, idx) => {
|
||||
console.log(`Sibling ${idx}: <${el.tagName}> length=${el.outerHTML.length}`);
|
||||
});
|
||||
|
||||
let currentChunk = '';
|
||||
let currentTokens = 0;
|
||||
|
||||
for (const element of siblingElements) {
|
||||
const elementHTML = element.outerHTML;
|
||||
const elementTokensResult = await countTokens(elementHTML, model);
|
||||
const elementTokens = typeof elementTokensResult === 'object' ? elementTokensResult.totalTokens || elementTokensResult.count || 0 : elementTokensResult;
|
||||
|
||||
console.log(`Element tokens: ${elementTokens}, current total: ${currentTokens}, limit: ${maxTokensPerChunk}`);
|
||||
|
||||
// If this single element is larger than max tokens, it needs its own chunk
|
||||
if (elementTokens > maxTokensPerChunk) {
|
||||
console.log(`Element too large (${elementTokens} tokens), creating dedicated chunk`);
|
||||
// Push current chunk if it exists
|
||||
if (currentChunk) {
|
||||
chunks.push(currentChunk);
|
||||
currentChunk = '';
|
||||
currentTokens = 0;
|
||||
}
|
||||
// Add large element as its own chunk
|
||||
chunks.push(elementHTML);
|
||||
}
|
||||
// If adding this element would exceed the limit, start a new chunk
|
||||
else if (currentTokens + elementTokens > maxTokensPerChunk && currentChunk) {
|
||||
console.log(`Starting new chunk at ${currentTokens} tokens`);
|
||||
chunks.push(currentChunk);
|
||||
currentChunk = elementHTML;
|
||||
currentTokens = elementTokens;
|
||||
} else {
|
||||
currentChunk += elementHTML;
|
||||
currentTokens += elementTokens;
|
||||
}
|
||||
}
|
||||
|
||||
// Add the last chunk if it exists
|
||||
if (currentChunk) {
|
||||
chunks.push(currentChunk);
|
||||
}
|
||||
|
||||
// If no chunks created, use the entire container content as one chunk
|
||||
if (chunks.length === 0) {
|
||||
console.log('No chunks created from siblings, using entire content as single chunk');
|
||||
chunks.push(container.innerHTML);
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
// REMOVED: splitHTMLIntoChunks function - no longer using chunking
|
||||
|
||||
// Native language names with proper scripts, accents, and authentic forms
|
||||
export const NATIVE_LANGUAGE_NAMES = {
|
||||
|
|
@ -410,8 +345,8 @@ ${preservedText}`;
|
|||
}
|
||||
}
|
||||
|
||||
// Internal function to translate HTML fragments (for chunked translations)
|
||||
async function translateFragmentRaw(text, targetLanguage) {
|
||||
// REMOVED: translateFragmentRaw - no longer using chunking
|
||||
/* async function translateFragmentRaw(text, targetLanguage) {
|
||||
// Ensure models are loaded before translation
|
||||
const { fetchModelsFromEndpoints } = await import("./models.js");
|
||||
await fetchModelsFromEndpoints();
|
||||
|
|
@ -473,7 +408,9 @@ ${preservedText}`;
|
|||
console.error("Error stack:", error.stack);
|
||||
throw new Error(`Fragment translation failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
} */
|
||||
|
||||
// REMOVED: translateFragment - no longer using chunking
|
||||
|
||||
// Translate text using Hermes AI with minimal system context
|
||||
export async function translateText(text, targetLanguage) {
|
||||
|
|
@ -631,14 +568,14 @@ export async function translateHTML(htmlContent, targetLanguage, sourceUrl = nul
|
|||
const estimatedOutputTokens = Math.floor(contentTokens * 1.8);
|
||||
const totalEstimatedTokens = contentTokens + estimatedOutputTokens;
|
||||
|
||||
console.log(`=== CHUNKING DECISION ===`);
|
||||
console.log(`=== TOKEN ANALYSIS ===`);
|
||||
console.log(`Max tokens: ${maxTokens}`);
|
||||
console.log(`Reserve tokens: ${reserveTokens}`);
|
||||
console.log(`Available tokens: ${availableTokens}`);
|
||||
console.log(`Content tokens: ${contentTokens}`);
|
||||
console.log(`Estimated output tokens: ${estimatedOutputTokens}`);
|
||||
console.log(`Total estimated tokens: ${totalEstimatedTokens}`);
|
||||
console.log(`Should chunk: ${totalEstimatedTokens > availableTokens}`);
|
||||
console.log(`Fits in single request: ${totalEstimatedTokens <= availableTokens}`);
|
||||
|
||||
let translatedContent;
|
||||
if (totalEstimatedTokens <= availableTokens) {
|
||||
|
|
@ -646,71 +583,21 @@ export async function translateHTML(htmlContent, targetLanguage, sourceUrl = nul
|
|||
console.log(`Content fits in single request: ${contentTokens} tokens`);
|
||||
translatedContent = await translateText(processedContent, targetLanguage);
|
||||
} else {
|
||||
// Content needs chunking - chunk BEFORE preservation to maintain HTML structure
|
||||
console.log(`Content requires chunking: ${totalEstimatedTokens} total tokens > ${availableTokens} limit`);
|
||||
// Content too large for selected model
|
||||
console.log(`Content too large: ${totalEstimatedTokens} total tokens > ${availableTokens} limit`);
|
||||
|
||||
// Split the original HTML content into semantic chunks
|
||||
// Use reasonable chunk size for the model's context window
|
||||
const maxChunkInputTokens = Math.floor((maxTokens - 1500) / 2.5); // Leave room for prompt + expansion
|
||||
const chunks = await splitHTMLIntoChunks(processedContent, maxChunkInputTokens, getSelectedModel());
|
||||
console.log(`Split into ${chunks.length} chunks for parallel translation`);
|
||||
// Get model info for better error message
|
||||
const modelName = getSelectedModel();
|
||||
const modelInfo = modelRegistry[modelName] || { modelName };
|
||||
|
||||
// Debug: log chunk content to see what we're actually translating
|
||||
chunks.forEach((chunk, index) => {
|
||||
console.log(`Chunk ${index + 1} length: ${chunk.length} chars`);
|
||||
console.log(`Chunk ${index + 1} preview:`, chunk.substring(0, 400) + (chunk.length > 400 ? '...' : ''));
|
||||
});
|
||||
throw new Error(`Content is too large for the selected model (${modelName}).
|
||||
|
||||
// Process chunks in batches to avoid overwhelming the API
|
||||
const batchSize = 4; // Increased for faster processing
|
||||
const translatedChunks = [];
|
||||
|
||||
for (let i = 0; i < chunks.length; i += batchSize) {
|
||||
const batch = chunks.slice(i, i + batchSize);
|
||||
console.log(`Processing batch ${Math.floor(i/batchSize) + 1}/${Math.ceil(chunks.length/batchSize)} (chunks ${i + 1}-${Math.min(i + batchSize, chunks.length)})`);
|
||||
|
||||
const batchPromises = batch.map((chunk, batchIndex) => {
|
||||
const chunkIndex = i + batchIndex + 1;
|
||||
console.log(`Translating chunk ${chunkIndex}/${chunks.length}`);
|
||||
console.log(`Chunk ${chunkIndex} content preview:`, chunk.substring(0, 300));
|
||||
return translateFragment(chunk, targetLanguage);
|
||||
});
|
||||
|
||||
try {
|
||||
const batchResults = await Promise.all(batchPromises);
|
||||
translatedChunks.push(...batchResults);
|
||||
console.log(`Batch ${Math.floor(i/batchSize) + 1} completed successfully`);
|
||||
} catch (error) {
|
||||
console.error(`Batch ${Math.floor(i/batchSize) + 1} failed:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`All ${chunks.length} chunks translated successfully in batches`);
|
||||
|
||||
// Reassemble the chunks into the original document structure
|
||||
const originalParser = new DOMParser();
|
||||
const originalDoc = originalParser.parseFromString(processedContent, 'text/html');
|
||||
|
||||
// Extract only the body content from each translated chunk to avoid stacking full HTML documents
|
||||
const cleanedChunks = translatedChunks.map((chunk, index) => {
|
||||
console.log(`Raw translated chunk ${index + 1} preview:`, chunk.substring(0, 500) + '...');
|
||||
const chunkDoc = new DOMParser().parseFromString(chunk, 'text/html');
|
||||
const bodyContent = chunkDoc.body ? chunkDoc.body.innerHTML : chunk;
|
||||
console.log(`Extracted body content ${index + 1} preview:`, bodyContent.substring(0, 300) + '...');
|
||||
return bodyContent;
|
||||
});
|
||||
|
||||
// Replace the body content with reassembled translated chunks
|
||||
const reassembledBodyContent = cleanedChunks.join('');
|
||||
if (originalDoc.body) {
|
||||
originalDoc.body.innerHTML = reassembledBodyContent;
|
||||
} else {
|
||||
// If no body tag, replace documentElement content
|
||||
originalDoc.documentElement.innerHTML = `<head>${originalDoc.head ? originalDoc.head.innerHTML : ''}</head><body>${reassembledBodyContent}</body>`;
|
||||
}
|
||||
|
||||
translatedContent = originalDoc.documentElement.outerHTML;
|
||||
Content requires ~${Math.ceil(totalEstimatedTokens).toLocaleString()} tokens but model only supports ${maxTokens.toLocaleString()} tokens.
|
||||
|
||||
Please try:
|
||||
• Using a larger model (32K or higher)
|
||||
• Translating a smaller section of the page
|
||||
• Using the custom text tab for specific content`);
|
||||
}
|
||||
|
||||
// Parse the content to add base tag for URL resolution
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue