uncloseai.com/src/translation.js

620 lines
21 KiB
JavaScript

import { API_KEY, getAPIConfig } from "./config.js";
import { getSelectedModel, getSelectedModelEndpoint, getSelectedModelMaxTokens } from "./models.js";
import { countTokens, countTokensWithUpstream } from "./token_estimator.js";
// Languages supported by Hermes 3 model (in English for reference)
export const SUPPORTED_LANGUAGES = {
en: "English",
zh: "Chinese (Simplified)",
hi: "Hindi",
es: "Spanish",
fr: "French",
ar: "Arabic",
bn: "Bengali",
ru: "Russian",
pt: "Portuguese",
ur: "Urdu",
id: "Indonesian",
de: "German",
ja: "Japanese",
sw: "Swahili",
mr: "Marathi",
te: "Telugu",
tr: "Turkish",
"zh-tw": "Chinese (Traditional)",
ko: "Korean",
};
// Function to split HTML content into chunks at semantic boundaries
async function splitHTMLIntoChunks(htmlContent, maxTokensPerChunk, model) {
const chunks = [];
const parser = new DOMParser();
const doc = parser.parseFromString(htmlContent, 'text/html');
// Extract only the body content for chunking to avoid duplicating html/head/body tags
const bodyContent = doc.body ? doc.body.innerHTML : doc.documentElement.innerHTML;
// Create a temporary container to work with body content
const tempDoc = new DOMParser().parseFromString(`<div>${bodyContent}</div>`, 'text/html');
const container = tempDoc.querySelector('div');
// Find semantic boundaries within the body content
const elements = container.querySelectorAll('p, section, article, div, h1, h2, h3, h4, h5, h6, li, blockquote');
let currentChunk = '';
let currentTokens = 0;
for (const element of elements) {
const elementHTML = element.outerHTML;
const elementTokens = await countTokens(elementHTML, model);
// If adding this element would exceed the limit, start a new chunk
if (currentTokens + elementTokens > maxTokensPerChunk && currentChunk) {
chunks.push(currentChunk);
currentChunk = elementHTML;
currentTokens = elementTokens;
} else {
currentChunk += elementHTML;
currentTokens += elementTokens;
}
}
// Add the last chunk if it exists
if (currentChunk) {
chunks.push(currentChunk);
}
// If no semantic elements found, fall back to character-based splitting of body content
if (chunks.length === 0) {
const totalTokens = await countTokens(bodyContent, model);
const chunkSize = Math.floor(bodyContent.length / Math.ceil(totalTokens / maxTokensPerChunk));
for (let i = 0; i < bodyContent.length; i += chunkSize) {
chunks.push(bodyContent.substring(i, i + chunkSize));
}
}
return chunks;
}
// Native language names with proper scripts, accents, and authentic forms
export const NATIVE_LANGUAGE_NAMES = {
en: "English",
zh: "中文(简体)",
hi: "हिंदी",
es: "Español",
fr: "Français",
ar: "العَرَبِيَّة",
bn: "বাংলা",
ru: "Русский",
pt: "Português",
ur: "اُردُو",
id: "Bahasa Indonesia",
de: "Deutsch",
ja: "日本語",
sw: "Kiswahili",
mr: "मराठी",
te: "తెలుగు",
tr: "Türkçe",
"zh-tw": "中文(繁體)",
ko: "한국어",
};
// Send message with custom history (for translation with minimal system prompt)
async function* sendMessageWithHistory(messageHistory) {
// Get API configuration (custom or default)
const apiConfig = await getAPIConfig();
let apiUrl, headers, model;
if (apiConfig.isCustom) {
// Use custom API configuration
apiUrl = `${apiConfig.endpoint}/chat/completions`;
headers = apiConfig.headers;
model = apiConfig.model;
} else {
// Use default Hermes configuration
apiUrl = `${getSelectedModelEndpoint()}/chat/completions`;
headers = {
"Content-Type": "application/json",
Authorization: `Bearer ${API_KEY}`,
};
model = getSelectedModel();
}
// Calculate available tokens based on input length
const modelMaxTokens = getSelectedModelMaxTokens();
const inputText = messageHistory.map(msg => msg.content).join('');
const inputTokens = Math.ceil(inputText.length / 4); // Simple token estimation
const buffer = Math.max(2048, Math.floor(inputTokens * 1.5)); // Use 1.5x input tokens as buffer, minimum 2048
const availableTokens = Math.max(100, modelMaxTokens - inputTokens - buffer);
console.log("Translation using endpoint:", apiUrl);
console.log("Translation using model:", model);
console.log("Translation token calculation: max=" + modelMaxTokens + ", input≈" + inputTokens + ", buffer=" + buffer + ", available≈" + availableTokens);
const response = await fetch(apiUrl, {
method: "POST",
headers: headers,
body: JSON.stringify({
model: model,
messages: messageHistory,
temperature: 0.3, // Lower temperature for more consistent translation
max_tokens: availableTokens,
stream: true,
}),
});
if (!response.ok) {
const errorText = await response.text();
console.error("Translation API Error:", {
status: response.status,
url: apiUrl,
response: errorText
});
throw new Error(`HTTP error! status: ${response.status} from ${apiUrl}: ${errorText}`);
}
const reader = response.body.getReader();
const decoder = new TextDecoder();
let streamBuffer = "";
while (true) {
const { done, value } = await reader.read();
if (done) break;
streamBuffer += decoder.decode(value, { stream: true });
const lines = streamBuffer.split("\n");
for (let i = 0; i < lines.length - 1; i++) {
const line = lines[i].trim();
if (line.startsWith("data: ")) {
const jsonData = line.slice(6);
if (jsonData === "[DONE]") continue;
try {
const parsedData = JSON.parse(jsonData);
const content = parsedData.choices[0].delta.content;
if (content) {
yield content;
}
} catch (error) {
console.error("Error parsing JSON:", error);
}
}
}
streamBuffer = lines[lines.length - 1];
}
}
// Preserve code blocks, URLs, and special formatting during translation
export function preserveSpecialContent(text) {
const preservations = [];
let preservedText = text;
// HTML code elements (most important in HTML context)
preservedText = preservedText.replace(
/<script[\s\S]*?<\/script>/gi,
(match) => {
const index = preservations.length;
const placeholder = `__CODE_${index}__`;
preservations.push({ type: "CODE", content: match, placeholder });
return placeholder;
},
);
preservedText = preservedText.replace(/<pre[\s\S]*?<\/pre>/gi, (match) => {
const index = preservations.length;
const placeholder = `__CODE_${index}__`;
preservations.push({ type: "CODE", content: match, placeholder });
return placeholder;
});
preservedText = preservedText.replace(/<code[\s\S]*?<\/code>/gi, (match) => {
const index = preservations.length;
const placeholder = `__CODE_${index}__`;
preservations.push({ type: "CODE", content: match, placeholder });
return placeholder;
});
// Other code-related HTML elements
preservedText = preservedText.replace(
/<(kbd|samp|var)[\s\S]*?<\/\1>/gi,
(match) => {
const index = preservations.length;
const placeholder = `__CODE_${index}__`;
preservations.push({ type: "CODE", content: match, placeholder });
return placeholder;
},
);
// Markdown code blocks (for mixed content)
preservedText = preservedText.replace(/```[\s\S]*?```/g, (match) => {
const index = preservations.length;
const placeholder = `__CODE_${index}__`;
preservations.push({ type: "CODE", content: match, placeholder });
return placeholder;
});
// Inline markdown code
preservedText = preservedText.replace(/`[^`\n]+`/g, (match) => {
const index = preservations.length;
const placeholder = `__CODE_${index}__`;
preservations.push({ type: "CODE", content: match, placeholder });
return placeholder;
});
// URLs and URIs
preservedText = preservedText.replace(
/https?:\/\/[^\s<>"{}|\\^`[\]]+/g,
(match) => {
const index = preservations.length;
const placeholder = `__URI_${index}__`;
preservations.push({ type: "URI", content: match, placeholder });
return placeholder;
},
);
// Email addresses
preservedText = preservedText.replace(
/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g,
(match) => {
const index = preservations.length;
const placeholder = `__URI_${index}__`;
preservations.push({ type: "URI", content: match, placeholder });
return placeholder;
},
);
// Other HTML tags (after code elements are preserved)
preservedText = preservedText.replace(/<[^>]+>/g, (match) => {
const index = preservations.length;
const placeholder = `__HTML_${index}__`;
preservations.push({ type: "HTML", content: match, placeholder });
return placeholder;
});
return { preservedText, preservations };
}
// Restore preserved content after translation
export function restoreSpecialContent(translatedText, preservations) {
let restoredText = translatedText;
// By reversing the array, we restore the outer elements (like HTML tags) first,
// which then reveals the inner placeholders (like URIs and code) for subsequent replacement.
for (let i = preservations.length - 1; i >= 0; i--) {
const item = preservations[i];
restoredText = restoredText.replace(
new RegExp(item.placeholder, "g"),
item.content,
);
}
return restoredText;
}
// Calculate processing time estimate based on real vLLM metrics
export function estimateProcessingTime(tokenCount) {
// Real vLLM performance metrics from production:
// - Prompt processing: ~3,144 tokens/second
// - Generation: ~121 tokens/second
// - Translation typically generates 1-2x input tokens
const promptProcessingTime = tokenCount / 3144; // seconds for input processing
const estimatedOutputTokens = tokenCount * 1.2; // Translation usually 1.2x input length
const generationTime = estimatedOutputTokens / 121; // seconds for generation
// Add some buffer for network latency and processing overhead
const totalTime = (promptProcessingTime + generationTime) * 1.3;
return Math.ceil(totalTime);
}
// Internal function to translate text without preservation (for chunked translations)
async function translateTextRaw(text, targetLanguage) {
// Ensure models are loaded before translation
const { fetchModelsFromEndpoints } = await import("./models.js");
await fetchModelsFromEndpoints();
// Skip preservation - assume text is already preserved
const preservedText = text;
// Build the actual prompt that will be sent to the AI
const prompt = `Translate the following HTML content into ${SUPPORTED_LANGUAGES[targetLanguage]}. The input is in English and formatted as HTML, which includes formatting, code blocks, and technical content. Ensure the output preserves the structure, syntax, and formatting of the original. Do not translate or modify placeholders like __CODE_0__, __URI_0__, __HTML_0__, etc., as they represent code blocks, URLs, or HTML elements that should remain unchanged. Only translate the surrounding text. Here is the text to translate:
${preservedText}`;
// Get accurate token count and timing estimate using upstream tokenizer
// Tokenize the actual prompt that will be sent to the AI
const tokenInfo = await countTokensWithUpstream(prompt, getSelectedModel());
const estimatedSeconds = estimateProcessingTime(tokenInfo.totalTokens);
console.log("=== TRANSLATION ANALYSIS ===");
console.log(`Input tokens: ${tokenInfo.totalTokens}`);
console.log(`Text length: ${tokenInfo.textLength} characters`);
console.log(`Estimated processing time: ${estimatedSeconds} seconds`);
console.log("Token breakdown:", tokenInfo.breakdown);
// Debug logging
console.log("=== RAW TRANSLATION DEBUG ===");
console.log(
"Input text:",
text.substring(0, 300) + (text.length > 300 ? "..." : ""),
);
console.log(
"Preserved text:",
preservedText.substring(0, 300) + (preservedText.length > 300 ? "..." : ""),
);
// Use minimal system message for translation - focused purely on the task
const translationHistory = [
{
role: "system",
content:
"You are a translation assistant. Translate text accurately while preserving all formatting and placeholders exactly as provided.",
},
{
role: "user",
content: prompt,
},
];
let translatedText = "";
try {
for await (const chunk of sendMessageWithHistory(translationHistory)) {
translatedText += chunk;
}
// Clean up the response (remove any extra explanations)
translatedText = translatedText.trim();
console.log("AI response length:", translatedText.length);
console.log(
"AI response:",
translatedText.substring(0, 300) +
(translatedText.length > 300 ? "..." : ""),
);
// Validate response
if (!translatedText || translatedText.length === 0) {
throw new Error(
"Translation API returned empty response. Please try again.",
);
}
// Return raw translated text without restoration (for chunked translations)
console.log("=== RAW TRANSLATION RESULT ===");
console.log(`Raw translated length: ${translatedText.length} characters`);
return translatedText;
} catch (error) {
console.error("Translation error:", error);
throw new Error("Translation failed. Please try again.");
}
}
// Translate text using Hermes AI with minimal system context
export async function translateText(text, targetLanguage) {
const { preservedText, preservations } = preserveSpecialContent(text);
// Use raw translation function
const translatedText = await translateTextRaw(preservedText, targetLanguage);
// Check if placeholders are still in the response
preservations.forEach((item, _index) => {
const found = translatedText.includes(item.placeholder);
console.log(
` Placeholder ${item.placeholder} found in response: ${found}`,
);
});
// Restore preserved content
const restored = restoreSpecialContent(translatedText, preservations);
console.log("Restored text length:", restored.length);
// Only log full text in debug mode, otherwise just show length
if (restored.length < 500) {
console.log("Restored text:", restored);
} else {
console.log("Restored text preview:", restored.substring(0, 100) + "...");
}
// Final validation
if (!restored || restored.length === 0) {
throw new Error("Translation processing failed. Please try again.");
}
console.log("=== FINAL RESULT ===");
console.log(`Final translated length: ${restored.length} characters`);
return restored;
}
// Extract page content for translation
export function extractPageContent() {
// Clone the document to avoid modifying the original
const documentClone = document.cloneNode(true);
// Remove any open modals/dialogs that shouldn't be in the translation
const modalsToRemove = documentClone.querySelectorAll(
'dialog[open], #uncloseai-embedded-modal, [id*="modal"], [class*="modal"]',
);
modalsToRemove.forEach((modal) => modal.remove());
// Return the entire HTML of the document to preserve head, styles, and scripts
return documentClone.documentElement.outerHTML;
}
// Convert relative URLs to absolute URLs based on source URL
function convertRelativeToAbsolute(htmlContent, sourceUrl) {
if (!sourceUrl) {
return htmlContent;
}
try {
const baseUrl = new URL(sourceUrl);
const parser = new DOMParser();
const doc = parser.parseFromString(htmlContent, 'text/html');
// Convert relative URLs in various attributes
const elementsWithUrls = [
{ selector: 'a[href]', attr: 'href' },
{ selector: 'img[src]', attr: 'src' },
{ selector: 'link[href]', attr: 'href' },
{ selector: 'script[src]', attr: 'src' },
{ selector: 'iframe[src]', attr: 'src' },
{ selector: 'source[src]', attr: 'src' },
{ selector: 'audio[src]', attr: 'src' },
{ selector: 'video[src]', attr: 'src' },
{ selector: 'embed[src]', attr: 'src' },
{ selector: 'object[data]', attr: 'data' },
{ selector: 'form[action]', attr: 'action' }
];
elementsWithUrls.forEach(({ selector, attr }) => {
const elements = doc.querySelectorAll(selector);
elements.forEach(element => {
const url = element.getAttribute(attr);
if (url && !url.startsWith('http') && !url.startsWith('//') && !url.startsWith('#') && !url.startsWith('mailto:') && !url.startsWith('tel:')) {
try {
const absoluteUrl = new URL(url, baseUrl).href;
element.setAttribute(attr, absoluteUrl);
} catch (e) {
// If URL conversion fails, keep the original
console.warn('Failed to convert relative URL:', url, e);
}
}
});
});
return doc.documentElement.outerHTML;
} catch (error) {
console.warn('Failed to convert relative URLs:', error);
return htmlContent;
}
}
// Translate any HTML content with optional URL resolution
export async function translateHTML(htmlContent, targetLanguage, sourceUrl = null) {
if (!htmlContent || htmlContent.length < 10) {
throw new Error("Unable to extract meaningful content from provided HTML.");
}
// Skip URL conversion during chunking to avoid performance overhead
// We'll add base tag instead which is more efficient
let processedContent = htmlContent;
console.log('Skipping URL conversion for chunked translation - will use base tag instead');
// Use actual token limits instead of character limits
const maxTokens = getSelectedModelMaxTokens();
const reserveTokens = 1000; // Reserve tokens for prompt overhead
const availableTokens = Math.max(maxTokens - reserveTokens, 2000); // Ensure minimum
// Check if content needs chunking based on token count (on raw HTML, not preserved)
const contentTokens = await countTokens(processedContent, getSelectedModel());
console.log(`=== CHUNKING DECISION ===`);
console.log(`Max tokens: ${maxTokens}`);
console.log(`Reserve tokens: ${reserveTokens}`);
console.log(`Available tokens: ${availableTokens}`);
console.log(`Content tokens: ${contentTokens}`);
console.log(`Should chunk: ${contentTokens > availableTokens}`);
let translatedContent;
if (contentTokens <= availableTokens) {
// Content fits in one request - use normal translation with preservation
console.log(`Content fits in single request: ${contentTokens} tokens`);
translatedContent = await translateText(processedContent, targetLanguage);
} else {
// Content needs chunking - chunk BEFORE preservation to maintain HTML structure
console.log(`Content requires chunking: ${contentTokens} tokens > ${availableTokens} limit`);
// Split the original HTML content into semantic chunks
const chunks = await splitHTMLIntoChunks(processedContent, availableTokens, getSelectedModel());
console.log(`Split into ${chunks.length} chunks for parallel translation`);
// Debug: log chunk content to see what we're actually translating
chunks.forEach((chunk, index) => {
console.log(`Chunk ${index + 1} preview:`, chunk.substring(0, 200) + (chunk.length > 200 ? '...' : ''));
});
// Process chunks in batches to avoid overwhelming the API
const batchSize = 2; // Limit concurrent requests
const translatedChunks = [];
for (let i = 0; i < chunks.length; i += batchSize) {
const batch = chunks.slice(i, i + batchSize);
console.log(`Processing batch ${Math.floor(i/batchSize) + 1}/${Math.ceil(chunks.length/batchSize)} (chunks ${i + 1}-${Math.min(i + batchSize, chunks.length)})`);
const batchPromises = batch.map((chunk, batchIndex) => {
const chunkIndex = i + batchIndex + 1;
console.log(`Translating chunk ${chunkIndex}/${chunks.length}`);
return translateText(chunk, targetLanguage);
});
try {
const batchResults = await Promise.all(batchPromises);
translatedChunks.push(...batchResults);
console.log(`Batch ${Math.floor(i/batchSize) + 1} completed successfully`);
} catch (error) {
console.error(`Batch ${Math.floor(i/batchSize) + 1} failed:`, error);
throw error;
}
}
console.log(`All ${chunks.length} chunks translated successfully in batches`);
// Reassemble the chunks into the original document structure
const originalParser = new DOMParser();
const originalDoc = originalParser.parseFromString(processedContent, 'text/html');
// Replace the body content with reassembled translated chunks
const reassembledBodyContent = translatedChunks.join('');
if (originalDoc.body) {
originalDoc.body.innerHTML = reassembledBodyContent;
} else {
// If no body tag, replace documentElement content
originalDoc.documentElement.innerHTML = `<head>${originalDoc.head ? originalDoc.head.innerHTML : ''}</head><body>${reassembledBodyContent}</body>`;
}
translatedContent = originalDoc.documentElement.outerHTML;
}
// Parse the content to add base tag for URL resolution
const parser = new DOMParser();
const doc = parser.parseFromString(translatedContent, 'text/html');
// Add base tag for URL resolution
let baseUrl;
if (sourceUrl) {
// Use the source URL as base
baseUrl = new URL('.', sourceUrl).href; // Get base URL (without filename)
} else {
// Use current page URL as base (like current page translation)
const currentUrl = window.location.href;
baseUrl = new URL('.', currentUrl).href;
}
// Remove any existing base tag
const existingBase = doc.querySelector('base');
if (existingBase) {
existingBase.remove();
}
// Add new base tag to head
const baseTag = doc.createElement('base');
baseTag.setAttribute('href', baseUrl);
doc.head.insertBefore(baseTag, doc.head.firstChild);
console.log('Added base tag with href:', baseUrl);
return doc.documentElement.outerHTML;
}
// Translate current page content
export async function translateCurrentPage(targetLanguage) {
const pageContent = extractPageContent();
// Use the new translateHTML function for current page translation
return await translateHTML(pageContent, targetLanguage);
}