uncloseai.com/src/translation.js

import { API_KEY } from "./config.js";
import { getSelectedModel, getSelectedModelEndpoint } from "./models.js";
import { countTokens } from "./token_estimator.js";

// Languages supported by Hermes 3 model (in English for reference)
export const SUPPORTED_LANGUAGES = {
	en: "English",
	zh: "Chinese (Simplified)",
	hi: "Hindi",
	es: "Spanish",
	fr: "French",
	ar: "Arabic",
	bn: "Bengali",
	ru: "Russian",
	pt: "Portuguese",
	ur: "Urdu",
	id: "Indonesian",
	de: "German",
	ja: "Japanese",
	sw: "Swahili",
	mr: "Marathi",
	te: "Telugu",
	tr: "Turkish",
	"zh-tw": "Chinese (Traditional)",
	ko: "Korean",
};

// Native language names with proper scripts, accents, and authentic forms
export const NATIVE_LANGUAGE_NAMES = {
	en: "English",
	zh: "中文（简体）",
	hi: "हिंदी",
	es: "Español",
	fr: "Français",
	ar: "العَرَبِيَّة",
	bn: "বাংলা",
	ru: "Русский",
	pt: "Português",
	ur: "اُردُو",
	id: "Bahasa Indonesia",
	de: "Deutsch",
	ja: "日本語",
	sw: "Kiswahili",
	mr: "मराठी",
	te: "తెలుగు",
	tr: "Türkçe",
	"zh-tw": "中文（繁體）",
	ko: "한국어",
};

// Send message with custom history (for translation with minimal system prompt)
async function* sendMessageWithHistory(messageHistory) {
	const apiUrl = `${getSelectedModelEndpoint()}/chat/completions`;

	console.log("Translation using endpoint:", apiUrl);
	console.log("Translation using model:", getSelectedModel());

	const response = await fetch(apiUrl, {
		method: "POST",
		headers: {
			"Content-Type": "application/json",
			Authorization: `Bearer ${API_KEY}`,
		},
		body: JSON.stringify({
			model: getSelectedModel(),
			messages: messageHistory,
			temperature: 0.3, // Lower temperature for more consistent translation
			max_tokens: 70000,
			stream: true,
		}),
	});

	if (!response.ok) {
		throw new Error(`HTTP error! status: ${response.status} from ${apiUrl}`);
	}

	const reader = response.body.getReader();
	const decoder = new TextDecoder();
	let buffer = "";

	while (true) {
		const { done, value } = await reader.read();
		if (done) break;

		buffer += decoder.decode(value, { stream: true });
		const lines = buffer.split("\n");

		for (let i = 0; i < lines.length - 1; i++) {
			const line = lines[i].trim();
			if (line.startsWith("data: ")) {
				const jsonData = line.slice(6);
				if (jsonData === "[DONE]") continue;

				try {
					const parsedData = JSON.parse(jsonData);
					const content = parsedData.choices[0].delta.content;
					if (content) {
						yield content;
					}
				} catch (error) {
					console.error("Error parsing JSON:", error);
				}
			}
		}

		buffer = lines[lines.length - 1];
	}
}

// Preserve code blocks, URLs, and special formatting during translation
export function preserveSpecialContent(text) {
	const preservations = [];
	let preservedText = text;

	// HTML code elements (most important in HTML context)
	preservedText = preservedText.replace(
		/<script[\s\S]*?<\/script>/gi,
		(match) => {
			const index = preservations.length;
			const placeholder = `__CODE_${index}__`;
			preservations.push({ type: "CODE", content: match, placeholder });
			return placeholder;
		},
	);

	preservedText = preservedText.replace(/<pre[\s\S]*?<\/pre>/gi, (match) => {
		const index = preservations.length;
		const placeholder = `__CODE_${index}__`;
		preservations.push({ type: "CODE", content: match, placeholder });
		return placeholder;
	});

	preservedText = preservedText.replace(/<code[\s\S]*?<\/code>/gi, (match) => {
		const index = preservations.length;
		const placeholder = `__CODE_${index}__`;
		preservations.push({ type: "CODE", content: match, placeholder });
		return placeholder;
	});

	// Other code-related HTML elements
	preservedText = preservedText.replace(
		/<(kbd|samp|var)[\s\S]*?<\/\1>/gi,
		(match) => {
			const index = preservations.length;
			const placeholder = `__CODE_${index}__`;
			preservations.push({ type: "CODE", content: match, placeholder });
			return placeholder;
		},
	);

	// Markdown code blocks (for mixed content)
	preservedText = preservedText.replace(/```[\s\S]*?```/g, (match) => {
		const index = preservations.length;
		const placeholder = `__CODE_${index}__`;
		preservations.push({ type: "CODE", content: match, placeholder });
		return placeholder;
	});

	// Inline markdown code
	preservedText = preservedText.replace(/`[^`\n]+`/g, (match) => {
		const index = preservations.length;
		const placeholder = `__CODE_${index}__`;
		preservations.push({ type: "CODE", content: match, placeholder });
		return placeholder;
	});

	// URLs and URIs
	preservedText = preservedText.replace(
		/https?:\/\/[^\s<>"{}|\\^`[\]]+/g,
		(match) => {
			const index = preservations.length;
			const placeholder = `__URI_${index}__`;
			preservations.push({ type: "URI", content: match, placeholder });
			return placeholder;
		},
	);

	// Email addresses
	preservedText = preservedText.replace(
		/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g,
		(match) => {
			const index = preservations.length;
			const placeholder = `__URI_${index}__`;
			preservations.push({ type: "URI", content: match, placeholder });
			return placeholder;
		},
	);

	// Other HTML tags (after code elements are preserved)
	preservedText = preservedText.replace(/<[^>]+>/g, (match) => {
		const index = preservations.length;
		const placeholder = `__HTML_${index}__`;
		preservations.push({ type: "HTML", content: match, placeholder });
		return placeholder;
	});

	return { preservedText, preservations };
}

// Restore preserved content after translation
export function restoreSpecialContent(translatedText, preservations) {
	let restoredText = translatedText;

	// By reversing the array, we restore the outer elements (like HTML tags) first,
	// which then reveals the inner placeholders (like URIs and code) for subsequent replacement.
	for (let i = preservations.length - 1; i >= 0; i--) {
		const item = preservations[i];
		restoredText = restoredText.replace(
			new RegExp(item.placeholder, "g"),
			item.content,
		);
	}

	return restoredText;
}

// Calculate processing time estimate based on real vLLM metrics
export function estimateProcessingTime(tokenCount) {
	// Real vLLM performance metrics from production:
	// - Prompt processing: ~3,144 tokens/second
	// - Generation: ~121 tokens/second
	// - Translation typically generates 1-2x input tokens

	const promptProcessingTime = tokenCount / 3144; // seconds for input processing
	const estimatedOutputTokens = tokenCount * 1.2; // Translation usually 1.2x input length
	const generationTime = estimatedOutputTokens / 121; // seconds for generation

	// Add some buffer for network latency and processing overhead
	const totalTime = (promptProcessingTime + generationTime) * 1.3;

	return Math.ceil(totalTime);
}

// Translate text using Hermes AI with minimal system context
export async function translateText(text, targetLanguage) {
	// Ensure models are loaded before translation
	const { fetchModelsFromEndpoints } = await import("./models.js");
	await fetchModelsFromEndpoints();

	// Get accurate token count and timing estimate
	const tokenInfo = countTokens(text);
	const estimatedSeconds = estimateProcessingTime(tokenInfo.totalTokens);

	console.log("=== TRANSLATION ANALYSIS ===");
	console.log(`Input tokens: ${tokenInfo.totalTokens}`);
	console.log(`Text length: ${tokenInfo.textLength} characters`);
	console.log(`Estimated processing time: ${estimatedSeconds} seconds`);
	console.log("Token breakdown:", tokenInfo.breakdown);

	const { preservedText, preservations } = preserveSpecialContent(text);

	// Debug logging
	console.log("=== TRANSLATION DEBUG ===");
	console.log(
		"Original text:",
		text.substring(0, 300) + (text.length > 300 ? "..." : ""),
	);
	console.log(
		"Preserved text:",
		preservedText.substring(0, 300) + (preservedText.length > 300 ? "..." : ""),
	);
	console.log("Preservations found:", preservations.length);
	preservations.forEach((item, index) => {
		console.log(
			`  ${index}: ${item.placeholder} -> ${item.content.substring(0, 50)}${item.content.length > 50 ? "..." : ""}`,
		);
	});

	const prompt = `Translate the following HTML content into ${SUPPORTED_LANGUAGES[targetLanguage]}. The input is in English and formatted as HTML, which includes formatting, code blocks, and technical content. Ensure the output preserves the structure, syntax, and formatting of the original. Do not translate or modify placeholders like __CODE_0__, __URI_0__, __HTML_0__, etc., as they represent code blocks, URLs, or HTML elements that should remain unchanged. Only translate the surrounding text. Here is the text to translate:

${preservedText}`;

	// Use minimal system message for translation - focused purely on the task
	const translationHistory = [
		{
			role: "system",
			content:
				"You are a translation assistant. Translate text accurately while preserving all formatting and placeholders exactly as provided.",
		},
		{
			role: "user",
			content: prompt,
		},
	];

	let translatedText = "";

	try {
		for await (const chunk of sendMessageWithHistory(translationHistory)) {
			translatedText += chunk;
		}

		// Clean up the response (remove any extra explanations)
		translatedText = translatedText.trim();
		console.log("AI response length:", translatedText.length);
		console.log(
			"AI response:",
			translatedText.substring(0, 300) +
				(translatedText.length > 300 ? "..." : ""),
		);

		// Validate response
		if (!translatedText || translatedText.length === 0) {
			throw new Error(
				"Translation API returned empty response. Please try again.",
			);
		}

		// Check if placeholders are still in the response
		preservations.forEach((item, _index) => {
			const found = translatedText.includes(item.placeholder);
			console.log(
				`  Placeholder ${item.placeholder} found in response: ${found}`,
			);
		});

		// Restore preserved content
		const restored = restoreSpecialContent(translatedText, preservations);
		console.log("Restored text length:", restored.length);
		console.log(
			"Restored text:",
			restored.substring(0, 300) + (restored.length > 300 ? "..." : ""),
		);

		// Final validation
		if (!restored || restored.length === 0) {
			throw new Error("Translation processing failed. Please try again.");
		}

		return restored;
	} catch (error) {
		console.error("Translation error:", error);
		throw new Error("Translation failed. Please try again.");
	}
}

// Extract page content for translation
export function extractPageContent() {
	// Clone the document to avoid modifying the original
	const documentClone = document.cloneNode(true);

	// Remove any open modals/dialogs that shouldn't be in the translation
	const modalsToRemove = documentClone.querySelectorAll(
		'dialog[open], #uncloseai-embedded-modal, [id*="modal"], [class*="modal"]',
	);
	modalsToRemove.forEach((modal) => modal.remove());

	// Return the entire HTML of the document to preserve head, styles, and scripts
	return documentClone.documentElement.outerHTML;
}

// Translate current page content
export async function translateCurrentPage(targetLanguage) {
	const pageContent = extractPageContent();

	if (!pageContent || pageContent.length < 10) {
		throw new Error("Unable to extract meaningful content from this page.");
	}

	// Limit content length to avoid overwhelming the AI
	const maxLength = 50000; // Much higher limit for complete page translation
	const contentToTranslate =
		pageContent.length > maxLength
			? `${pageContent.substring(0, maxLength)}...`
			: pageContent;

	return await translateText(contentToTranslate, targetLanguage);
}