uncloseai.com/src/language-detection.js

// Language detection functionality using AI and fallback methods

import { API_KEY } from "./config.js";

// Fast AI-powered language detection using Hermes
export async function detectPageLanguage() {
	try {
		// Get page content as markdown
		const pageHtml = document.documentElement.outerHTML;
		const pageMarkdown = htmlToMarkdown(pageHtml);

		// Take a sample (first 1000 characters) for speed
		const sample = pageMarkdown.substring(0, 1000);

		// Import required modules
		const { getSelectedModel, getSelectedModelEndpoint } = await import(
			"./models.js"
		);
		const apiUrl = `${getSelectedModelEndpoint()}/chat/completions`;

		const response = await fetch(apiUrl, {
			method: "POST",
			headers: {
				"Content-Type": "application/json",
				Authorization: `Bearer ${API_KEY}`,
			},
			body: JSON.stringify({
				model: getSelectedModel(),
				messages: [
					{
						role: "system",
						content:
							"You are a language classifier. Respond with ONLY the 2-letter ISO language code (en, es, fr, de, etc.) for the detected language. No explanations, just the code.",
					},
					{
						role: "user",
						content: `Detect the language of this text:\n\n${sample}`,
					},
				],
				temperature: 0.1,
				max_tokens: 10,
				stream: false,
			}),
		});

		if (!response.ok) {
			console.warn(
				"Language detection failed, falling back to browser language",
			);
			return fallbackLanguageDetection();
		}

		const data = await response.json();
		const detectedLang = data.choices[0].message.content.trim().toLowerCase();

		// Validate it's a reasonable language code (2-3 chars)
		if (/^[a-z]{2,3}$/.test(detectedLang)) {
			console.log("AI detected language:", detectedLang);
			return detectedLang;
		}
		console.warn("Invalid language code from AI:", detectedLang);
		return fallbackLanguageDetection();
	} catch (error) {
		console.warn("Language detection error:", error);
		return fallbackLanguageDetection();
	}
}

// Fallback language detection using DOM attributes
export function fallbackLanguageDetection() {
	// Check html lang attribute first
	const htmlLang = document.documentElement.lang;
	if (htmlLang) {
		const langCode = htmlLang.split("-")[0].toLowerCase();
		return langCode;
	}

	// Fallback to browser language
	const browserLang = navigator.language || navigator.userLanguage;
	if (browserLang) {
		const langCode = browserLang.split("-")[0].toLowerCase();
		return langCode;
	}

	// Final fallback
	return "en";
}

// Simple HTML to markdown conversion (for language detection)
function htmlToMarkdown(html) {
	// Create a temporary element to parse HTML
	const tempDiv = document.createElement("div");
	tempDiv.innerHTML = html;

	// Extract text content
	return tempDiv.textContent || tempDiv.innerText || "";
}