feat(ui): add comprehensive language localization system
- Add complete UI translations for all 19 supported languages - Add language preference dropdown in modal settings - Store language preference in localStorage - Inject language preference into Hermes system prompts - Add smart translation dropdown with AI-powered language detection - Keep both original translation modal and new smart dropdown - Remove non-functional upload button from modal - Add biome.json config to ignore third-party CSS files
This commit is contained in:
parent
cae51b3578
commit
eee33986cf
11 changed files with 1962 additions and 867 deletions
59
biome.json
Normal file
59
biome.json
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
{
|
||||
"$schema": "https://biomejs.dev/schemas/2.0.6/schema.json",
|
||||
"files": {
|
||||
"includes": [
|
||||
"**",
|
||||
"!**/css/pico.classless.min.css",
|
||||
"!**/node_modules",
|
||||
"!**/*.min.js",
|
||||
"!**/*.min.css"
|
||||
]
|
||||
},
|
||||
"linter": {
|
||||
"enabled": true,
|
||||
"rules": {
|
||||
"recommended": true,
|
||||
"suspicious": {
|
||||
"noAssignInExpressions": "off"
|
||||
},
|
||||
"style": {
|
||||
"useTemplate": "warn",
|
||||
"noParameterAssign": "error",
|
||||
"useAsConstAssertion": "error",
|
||||
"useDefaultParameterLast": "error",
|
||||
"useEnumInitializers": "error",
|
||||
"useSelfClosingElements": "error",
|
||||
"useSingleVarDeclarator": "error",
|
||||
"noUnusedTemplateLiteral": "error",
|
||||
"useNumberNamespace": "error",
|
||||
"noInferrableTypes": "error",
|
||||
"noUselessElse": "error"
|
||||
},
|
||||
"complexity": {
|
||||
"useOptionalChain": "warn"
|
||||
}
|
||||
}
|
||||
},
|
||||
"formatter": {
|
||||
"enabled": true,
|
||||
"formatWithErrors": false,
|
||||
"indentStyle": "tab",
|
||||
"indentWidth": 2,
|
||||
"lineWidth": 80,
|
||||
"lineEnding": "lf",
|
||||
"includes": ["**", "!**/css/pico.classless.min.css"]
|
||||
},
|
||||
"assist": { "actions": { "source": { "organizeImports": "on" } } },
|
||||
"javascript": {
|
||||
"formatter": {
|
||||
"quoteStyle": "double",
|
||||
"jsxQuoteStyle": "double",
|
||||
"quoteProperties": "asNeeded",
|
||||
"trailingCommas": "all",
|
||||
"semicolons": "always",
|
||||
"arrowParentheses": "always",
|
||||
"bracketSpacing": true,
|
||||
"bracketSameLine": false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -3,8 +3,8 @@ import { marked } from "https://cdn.jsdelivr.net/npm/marked/lib/marked.esm.js";
|
|||
import hljs from "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/es/highlight.min.js";
|
||||
import { API_KEY } from "./config.js";
|
||||
import { getSelectedModel, getSelectedModelEndpoint } from "./models.js";
|
||||
import { speakText, generateTitleForTTS } from "./tts.js";
|
||||
import { saveConversationHistory, initializeChatHistory } from "./storage.js";
|
||||
import { initializeChatHistory, saveConversationHistory } from "./storage.js";
|
||||
import { generateTitleForTTS, speakText } from "./tts.js";
|
||||
|
||||
// Initialize chat history
|
||||
export let chatHistory = initializeChatHistory();
|
||||
|
|
|
|||
|
|
@ -17,11 +17,50 @@ export function setSystemMessageAppend(appendText) {
|
|||
SYSTEM_MESSAGE_APPEND = appendText;
|
||||
}
|
||||
|
||||
// Function to get the complete system message
|
||||
// Function to get the complete system message with language preference
|
||||
export function getSystemMessage() {
|
||||
return SYSTEM_MESSAGE_APPEND
|
||||
// Get user's language preference from localStorage
|
||||
let userLang = "en";
|
||||
try {
|
||||
userLang = localStorage.getItem("uncloseai_language") || "en";
|
||||
} catch (error) {
|
||||
console.warn("Failed to read language preference:", error);
|
||||
}
|
||||
|
||||
// Add language instruction to the system message
|
||||
let languageInstruction = "";
|
||||
if (userLang !== "en") {
|
||||
// Map language codes to full names for clarity
|
||||
const langNames = {
|
||||
es: "Spanish",
|
||||
zh: "Chinese (Simplified)",
|
||||
hi: "Hindi",
|
||||
fr: "French",
|
||||
ar: "Arabic",
|
||||
bn: "Bengali",
|
||||
ru: "Russian",
|
||||
pt: "Portuguese",
|
||||
ur: "Urdu",
|
||||
id: "Indonesian",
|
||||
de: "German",
|
||||
ja: "Japanese",
|
||||
sw: "Swahili",
|
||||
mr: "Marathi",
|
||||
te: "Telugu",
|
||||
tr: "Turkish",
|
||||
"zh-tw": "Chinese (Traditional)",
|
||||
ko: "Korean",
|
||||
};
|
||||
|
||||
const langName = langNames[userLang] || userLang;
|
||||
languageInstruction = `\n\nIMPORTANT: The user has set their language preference to ${langName}. Please respond in ${langName} unless the user explicitly asks for another language. Maintain natural, fluent communication in ${langName}.`;
|
||||
}
|
||||
|
||||
const baseMessage = SYSTEM_MESSAGE_APPEND
|
||||
? `${SYSTEM_MESSAGE_BASE}\n\n${SYSTEM_MESSAGE_APPEND}`
|
||||
: SYSTEM_MESSAGE_BASE;
|
||||
|
||||
return baseMessage + languageInstruction;
|
||||
}
|
||||
|
||||
// Dynamic Endpoints Configuration for Chat API
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ export function extractWebpageContent() {
|
|||
// Recursively extract text and links from the body content
|
||||
function getTextWithLinks(element) {
|
||||
if (element.nodeType === Node.TEXT_NODE) {
|
||||
content += element.textContent + " ";
|
||||
content += `${element.textContent} `;
|
||||
} else if (element.nodeType === Node.ELEMENT_NODE) {
|
||||
if (element.tagName.toLowerCase() === "a") {
|
||||
// If it's a link, append the text and the href
|
||||
|
|
|
|||
|
|
@ -1,9 +1,6 @@
|
|||
// File upload and processing functionality
|
||||
import { marked } from "https://cdn.jsdelivr.net/npm/marked/lib/marked.esm.js";
|
||||
import hljs from "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/es/highlight.min.js";
|
||||
import { chatHistory } from "./chat.js";
|
||||
import { MEGAPARCE_API_URL } from "./config.js";
|
||||
import { speakText } from "./tts.js";
|
||||
import { chatHistory } from "./chat.js";
|
||||
|
||||
// Progress indicator functions
|
||||
export function showProgressIndicator(message) {
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ export async function createModelSelectionDropdown() {
|
|||
});
|
||||
|
||||
// Insert the dropdown directly above the chat input box.
|
||||
if (userInput && userInput.parentNode) {
|
||||
if (userInput?.parentNode) {
|
||||
userInput.parentNode.parentNode.insertBefore(
|
||||
dropdown,
|
||||
userInput.parentNode,
|
||||
|
|
@ -135,7 +135,7 @@ export function addRefreshModelsButton() {
|
|||
container.id = "refresh-models-container";
|
||||
|
||||
// Position it near the model selection dropdown
|
||||
if (userInput && userInput.parentNode) {
|
||||
if (userInput?.parentNode) {
|
||||
userInput.parentNode.parentNode.insertBefore(
|
||||
container,
|
||||
userInput.parentNode,
|
||||
|
|
@ -146,7 +146,7 @@ export function addRefreshModelsButton() {
|
|||
const refreshButton = document.createElement("button");
|
||||
refreshButton.textContent = "Refresh Models";
|
||||
refreshButton.style.margin = "10px";
|
||||
refreshButton.onclick = async function () {
|
||||
refreshButton.onclick = async () => {
|
||||
refreshButton.textContent = "Refreshing...";
|
||||
refreshButton.disabled = true;
|
||||
|
||||
|
|
@ -189,7 +189,7 @@ export function addRefreshModelsButton() {
|
|||
export function getSelectedModel() {
|
||||
// Check modal dropdown first
|
||||
const modalDropdown = document.getElementById("hermes-model-selection");
|
||||
if (modalDropdown && modalDropdown.value) {
|
||||
if (modalDropdown?.value) {
|
||||
// Extract just the model name, not the unique ID
|
||||
const selectedId = modalDropdown.value;
|
||||
if (modelRegistry[selectedId]) {
|
||||
|
|
@ -200,7 +200,7 @@ export function getSelectedModel() {
|
|||
|
||||
// Fallback to main dropdown
|
||||
const dropdown = document.getElementById("model-selection");
|
||||
if (dropdown && dropdown.value) {
|
||||
if (dropdown?.value) {
|
||||
// Extract just the model name, not the unique ID
|
||||
const selectedId = dropdown.value;
|
||||
if (modelRegistry[selectedId]) {
|
||||
|
|
@ -223,13 +223,13 @@ export function getSelectedModel() {
|
|||
export function getSelectedModelEndpoint() {
|
||||
// Check modal dropdown first
|
||||
const modalDropdown = document.getElementById("modal-model-selection");
|
||||
if (modalDropdown && modalDropdown.value && modelRegistry[modalDropdown.value]) {
|
||||
if (modalDropdown?.value && modelRegistry[modalDropdown.value]) {
|
||||
return modelRegistry[modalDropdown.value].url;
|
||||
}
|
||||
|
||||
// Fallback to main dropdown
|
||||
const dropdown = document.getElementById("model-selection");
|
||||
if (dropdown && dropdown.value && modelRegistry[dropdown.value]) {
|
||||
if (dropdown?.value && modelRegistry[dropdown.value]) {
|
||||
return modelRegistry[dropdown.value].url;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,51 +10,132 @@ export class SimpleBPETokenizer {
|
|||
this.initializePatterns();
|
||||
this.initializeCommonTokens();
|
||||
}
|
||||
|
||||
|
||||
initializePatterns() {
|
||||
// Simplified tokenization patterns based on tiktoken's approach
|
||||
this.patterns = [
|
||||
// Contractions (case insensitive)
|
||||
/'(?:s|t|m|d|ll|ve|re)/gi,
|
||||
|
||||
|
||||
// Letters (word boundaries)
|
||||
/[a-zA-Z]+/g,
|
||||
|
||||
|
||||
// Numbers (1-3 digits groups)
|
||||
/\d{1,3}/g,
|
||||
|
||||
|
||||
// Punctuation and symbols
|
||||
/[^\s\w]/g,
|
||||
|
||||
|
||||
// Whitespace (including newlines)
|
||||
/\s+/g
|
||||
/\s+/g,
|
||||
];
|
||||
|
||||
|
||||
// Compiled regex for faster processing
|
||||
this.mainPattern = /'(?:s|t|m|d|ll|ve|re)|[a-zA-Z]+|\d{1,3}|[^\s\w]|\s+/gi;
|
||||
}
|
||||
|
||||
|
||||
initializeCommonTokens() {
|
||||
// Common subword patterns from BPE training (simplified set)
|
||||
this.commonSubwords = new Set([
|
||||
// Common prefixes
|
||||
'un', 're', 'in', 'dis', 'en', 'non', 'pre', 'over', 'mis', 'sub',
|
||||
'anti', 'auto', 'co', 'de', 'ex', 'inter', 'multi', 'out', 'under',
|
||||
|
||||
// Common suffixes
|
||||
'ing', 'ed', 'er', 'est', 'ly', 'tion', 'ness', 'ment', 'able', 'ible',
|
||||
'ful', 'less', 'ship', 'ward', 'wise', 'like', 'ous', 'ive', 'ate',
|
||||
|
||||
"un",
|
||||
"re",
|
||||
"in",
|
||||
"dis",
|
||||
"en",
|
||||
"non",
|
||||
"pre",
|
||||
"over",
|
||||
"mis",
|
||||
"sub",
|
||||
"anti",
|
||||
"auto",
|
||||
"co",
|
||||
"de",
|
||||
"ex",
|
||||
"inter",
|
||||
"multi",
|
||||
"out",
|
||||
"under",
|
||||
|
||||
// Common suffixes
|
||||
"ing",
|
||||
"ed",
|
||||
"er",
|
||||
"est",
|
||||
"ly",
|
||||
"tion",
|
||||
"ness",
|
||||
"ment",
|
||||
"able",
|
||||
"ible",
|
||||
"ful",
|
||||
"less",
|
||||
"ship",
|
||||
"ward",
|
||||
"wise",
|
||||
"like",
|
||||
"ous",
|
||||
"ive",
|
||||
"ate",
|
||||
|
||||
// Common words (high frequency)
|
||||
'the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can', 'had',
|
||||
'her', 'was', 'one', 'our', 'out', 'day', 'get', 'use', 'man', 'new',
|
||||
'now', 'way', 'may', 'say', 'see', 'him', 'two', 'how', 'its', 'who',
|
||||
|
||||
"the",
|
||||
"and",
|
||||
"for",
|
||||
"are",
|
||||
"but",
|
||||
"not",
|
||||
"you",
|
||||
"all",
|
||||
"can",
|
||||
"had",
|
||||
"her",
|
||||
"was",
|
||||
"one",
|
||||
"our",
|
||||
"out",
|
||||
"day",
|
||||
"get",
|
||||
"use",
|
||||
"man",
|
||||
"new",
|
||||
"now",
|
||||
"way",
|
||||
"may",
|
||||
"say",
|
||||
"see",
|
||||
"him",
|
||||
"two",
|
||||
"how",
|
||||
"its",
|
||||
"who",
|
||||
|
||||
// Common letter combinations
|
||||
'th', 'er', 'on', 'an', 're', 'he', 'in', 'ed', 'nd', 'ha', 'at',
|
||||
'en', 'es', 'of', 'or', 'nt', 'ea', 'ti', 'to', 'it', 'st', 'io'
|
||||
"th",
|
||||
"er",
|
||||
"on",
|
||||
"an",
|
||||
"re",
|
||||
"he",
|
||||
"in",
|
||||
"ed",
|
||||
"nd",
|
||||
"ha",
|
||||
"at",
|
||||
"en",
|
||||
"es",
|
||||
"of",
|
||||
"or",
|
||||
"nt",
|
||||
"ea",
|
||||
"ti",
|
||||
"to",
|
||||
"it",
|
||||
"st",
|
||||
"io",
|
||||
]);
|
||||
|
||||
|
||||
// Token length estimates for common patterns
|
||||
this.tokenWeights = {
|
||||
word: 1,
|
||||
|
|
@ -62,29 +143,29 @@ export class SimpleBPETokenizer {
|
|||
punctuation: 1,
|
||||
whitespace: 0.3,
|
||||
contraction: 0.5,
|
||||
subword: 0.6
|
||||
subword: 0.6,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
// Main tokenization method
|
||||
encode(text) {
|
||||
if (!text || typeof text !== 'string') return [];
|
||||
|
||||
if (!text || typeof text !== "string") return [];
|
||||
|
||||
// Split text using main pattern
|
||||
const rawTokens = text.match(this.mainPattern) || [];
|
||||
const processedTokens = [];
|
||||
|
||||
|
||||
for (const token of rawTokens) {
|
||||
processedTokens.push(...this.processToken(token));
|
||||
}
|
||||
|
||||
|
||||
return processedTokens;
|
||||
}
|
||||
|
||||
|
||||
processToken(token) {
|
||||
const trimmed = token.trim();
|
||||
if (!trimmed) return [token]; // Keep whitespace as-is
|
||||
|
||||
|
||||
// Check token type and split if needed
|
||||
if (this.isWord(trimmed)) {
|
||||
return this.splitWord(trimmed);
|
||||
|
|
@ -94,25 +175,25 @@ export class SimpleBPETokenizer {
|
|||
return [token]; // Punctuation, symbols, etc.
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
isWord(token) {
|
||||
return /^[a-zA-Z]+$/.test(token);
|
||||
}
|
||||
|
||||
|
||||
isNumber(token) {
|
||||
return /^\d+$/.test(token);
|
||||
}
|
||||
|
||||
|
||||
splitWord(word) {
|
||||
if (word.length <= 3) return [word];
|
||||
|
||||
|
||||
const tokens = [];
|
||||
let remaining = word.toLowerCase();
|
||||
|
||||
|
||||
// Try to find common subwords
|
||||
while (remaining.length > 0) {
|
||||
let found = false;
|
||||
|
||||
|
||||
// Look for longest matching subword first
|
||||
for (let len = Math.min(remaining.length, 6); len >= 2; len--) {
|
||||
const substr = remaining.substring(0, len);
|
||||
|
|
@ -123,7 +204,7 @@ export class SimpleBPETokenizer {
|
|||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (!found) {
|
||||
// Split remaining into chunks
|
||||
if (remaining.length <= 4) {
|
||||
|
|
@ -137,10 +218,10 @@ export class SimpleBPETokenizer {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
|
||||
splitNumber(number) {
|
||||
// Split numbers into groups of 1-3 digits
|
||||
const chunks = [];
|
||||
|
|
@ -149,71 +230,72 @@ export class SimpleBPETokenizer {
|
|||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
|
||||
// Quick token count (main method for estimation)
|
||||
count(text) {
|
||||
if (!text || typeof text !== 'string') return 0;
|
||||
|
||||
if (!text || typeof text !== "string") return 0;
|
||||
|
||||
// Handle special content first
|
||||
const specialTokens = this.countSpecialContent(text);
|
||||
|
||||
|
||||
// Remove special content and count regular tokens
|
||||
const cleanText = this.removeSpecialContent(text);
|
||||
const regularTokens = this.encode(cleanText);
|
||||
|
||||
|
||||
return {
|
||||
totalTokens: specialTokens.count + regularTokens.length,
|
||||
regularTokens: regularTokens.length,
|
||||
specialTokens: specialTokens.count,
|
||||
breakdown: {
|
||||
...specialTokens.breakdown,
|
||||
words: regularTokens.filter(t => this.isWord(t.trim())).length,
|
||||
numbers: regularTokens.filter(t => this.isNumber(t.trim())).length,
|
||||
punctuation: regularTokens.filter(t => /^[^\s\w]+$/.test(t.trim())).length,
|
||||
whitespace: regularTokens.filter(t => /^\s+$/.test(t)).length
|
||||
words: regularTokens.filter((t) => this.isWord(t.trim())).length,
|
||||
numbers: regularTokens.filter((t) => this.isNumber(t.trim())).length,
|
||||
punctuation: regularTokens.filter((t) => /^[^\s\w]+$/.test(t.trim()))
|
||||
.length,
|
||||
whitespace: regularTokens.filter((t) => /^\s+$/.test(t)).length,
|
||||
},
|
||||
textLength: text.length
|
||||
textLength: text.length,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
countSpecialContent(text) {
|
||||
let count = 0;
|
||||
const breakdown = { htmlTags: 0, codeBlocks: 0, inlineCode: 0, urls: 0 };
|
||||
|
||||
|
||||
// HTML tags
|
||||
const htmlMatches = text.match(/<[^>]+>/g) || [];
|
||||
breakdown.htmlTags = htmlMatches.length;
|
||||
count += htmlMatches.length * 3; // ~3 tokens per tag
|
||||
|
||||
|
||||
// Code blocks
|
||||
const codeBlockMatches = text.match(/```[\s\S]*?```/g) || [];
|
||||
breakdown.codeBlocks = codeBlockMatches.length;
|
||||
codeBlockMatches.forEach(block => {
|
||||
const content = block.replace(/```[\w]*\n?/g, '').replace(/```$/g, '');
|
||||
codeBlockMatches.forEach((block) => {
|
||||
const content = block.replace(/```[\w]*\n?/g, "").replace(/```$/g, "");
|
||||
count += Math.ceil(content.length / 2.5); // Code is denser
|
||||
});
|
||||
|
||||
|
||||
// Inline code
|
||||
const inlineMatches = text.match(/`[^`]+`/g) || [];
|
||||
breakdown.inlineCode = inlineMatches.length;
|
||||
inlineMatches.forEach(code => {
|
||||
inlineMatches.forEach((code) => {
|
||||
count += Math.ceil(code.length / 3);
|
||||
});
|
||||
|
||||
|
||||
// URLs
|
||||
const urlMatches = text.match(/https?:\/\/[^\s<>"{}|\\^`\[\]]+/g) || [];
|
||||
const urlMatches = text.match(/https?:\/\/[^\s<>"{}|\\^`[\]]+/g) || [];
|
||||
breakdown.urls = urlMatches.length;
|
||||
count += urlMatches.length * 4; // URLs are typically 2-6 tokens
|
||||
|
||||
|
||||
return { count, breakdown };
|
||||
}
|
||||
|
||||
|
||||
removeSpecialContent(text) {
|
||||
return text
|
||||
.replace(/```[\s\S]*?```/g, ' ') // Replace code blocks
|
||||
.replace(/`[^`]+`/g, ' ') // Replace inline code
|
||||
.replace(/<[^>]+>/g, ' ') // Replace HTML tags
|
||||
.replace(/https?:\/\/[^\s<>"{}|\\^`\[\]]+/g, ' '); // Replace URLs
|
||||
.replace(/```[\s\S]*?```/g, " ") // Replace code blocks
|
||||
.replace(/`[^`]+`/g, " ") // Replace inline code
|
||||
.replace(/<[^>]+>/g, " ") // Replace HTML tags
|
||||
.replace(/https?:\/\/[^\s<>"{}|\\^`[\]]+/g, " "); // Replace URLs
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -228,4 +310,4 @@ export function countTokens(text) {
|
|||
export function estimateTokens(text) {
|
||||
const result = tokenizer.count(text);
|
||||
return result.totalTokens;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
// Translation functionality with code block preservation
|
||||
import { sendMessage } from "./chat.js";
|
||||
import { API_KEY } from "./config.js";
|
||||
import { getSelectedModel, getSelectedModelEndpoint } from "./models.js";
|
||||
import { countTokens } from "./token_estimator.js";
|
||||
|
|
@ -168,7 +166,7 @@ export function preserveSpecialContent(text) {
|
|||
|
||||
// URLs and URIs
|
||||
preservedText = preservedText.replace(
|
||||
/https?:\/\/[^\s<>"{}|\\^`\[\]]+/g,
|
||||
/https?:\/\/[^\s<>"{}|\\^`[\]]+/g,
|
||||
(match) => {
|
||||
const index = preservations.length;
|
||||
const placeholder = `__URI_${index}__`;
|
||||
|
|
@ -219,17 +217,17 @@ export function restoreSpecialContent(translatedText, preservations) {
|
|||
// Calculate processing time estimate based on real vLLM metrics
|
||||
export function estimateProcessingTime(tokenCount) {
|
||||
// Real vLLM performance metrics from production:
|
||||
// - Prompt processing: ~3,144 tokens/second
|
||||
// - Prompt processing: ~3,144 tokens/second
|
||||
// - Generation: ~121 tokens/second
|
||||
// - Translation typically generates 1-2x input tokens
|
||||
|
||||
|
||||
const promptProcessingTime = tokenCount / 3144; // seconds for input processing
|
||||
const estimatedOutputTokens = tokenCount * 1.2; // Translation usually 1.2x input length
|
||||
const generationTime = estimatedOutputTokens / 121; // seconds for generation
|
||||
|
||||
|
||||
// Add some buffer for network latency and processing overhead
|
||||
const totalTime = (promptProcessingTime + generationTime) * 1.3;
|
||||
|
||||
|
||||
return Math.ceil(totalTime);
|
||||
}
|
||||
|
||||
|
|
@ -238,17 +236,17 @@ export async function translateText(text, targetLanguage) {
|
|||
// Ensure models are loaded before translation
|
||||
const { fetchModelsFromEndpoints } = await import("./models.js");
|
||||
await fetchModelsFromEndpoints();
|
||||
|
||||
|
||||
// Get accurate token count and timing estimate
|
||||
const tokenInfo = countTokens(text);
|
||||
const estimatedSeconds = estimateProcessingTime(tokenInfo.totalTokens);
|
||||
|
||||
|
||||
console.log("=== TRANSLATION ANALYSIS ===");
|
||||
console.log(`Input tokens: ${tokenInfo.totalTokens}`);
|
||||
console.log(`Text length: ${tokenInfo.textLength} characters`);
|
||||
console.log(`Estimated processing time: ${estimatedSeconds} seconds`);
|
||||
console.log(`Token breakdown:`, tokenInfo.breakdown);
|
||||
|
||||
|
||||
const { preservedText, preservations } = preserveSpecialContent(text);
|
||||
|
||||
// Debug logging
|
||||
|
|
@ -303,11 +301,13 @@ ${preservedText}`;
|
|||
|
||||
// Validate response
|
||||
if (!translatedText || translatedText.length === 0) {
|
||||
throw new Error("Translation API returned empty response. Please try again.");
|
||||
throw new Error(
|
||||
"Translation API returned empty response. Please try again.",
|
||||
);
|
||||
}
|
||||
|
||||
// Check if placeholders are still in the response
|
||||
preservations.forEach((item, index) => {
|
||||
preservations.forEach((item, _index) => {
|
||||
const found = translatedText.includes(item.placeholder);
|
||||
console.log(
|
||||
` Placeholder ${item.placeholder} found in response: ${found}`,
|
||||
|
|
@ -338,11 +338,13 @@ ${preservedText}`;
|
|||
export function extractPageContent() {
|
||||
// Clone the document to avoid modifying the original
|
||||
const documentClone = document.cloneNode(true);
|
||||
|
||||
|
||||
// Remove any open modals/dialogs that shouldn't be in the translation
|
||||
const modalsToRemove = documentClone.querySelectorAll('dialog[open], #uncloseai-embedded-modal, [id*="modal"], [class*="modal"]');
|
||||
modalsToRemove.forEach(modal => modal.remove());
|
||||
|
||||
const modalsToRemove = documentClone.querySelectorAll(
|
||||
'dialog[open], #uncloseai-embedded-modal, [id*="modal"], [class*="modal"]',
|
||||
);
|
||||
modalsToRemove.forEach((modal) => modal.remove());
|
||||
|
||||
// Return the entire HTML of the document to preserve head, styles, and scripts
|
||||
return documentClone.documentElement.outerHTML;
|
||||
}
|
||||
|
|
@ -359,7 +361,7 @@ export async function translateCurrentPage(targetLanguage) {
|
|||
const maxLength = 50000; // Much higher limit for complete page translation
|
||||
const contentToTranslate =
|
||||
pageContent.length > maxLength
|
||||
? pageContent.substring(0, maxLength) + "..."
|
||||
? `${pageContent.substring(0, maxLength)}...`
|
||||
: pageContent;
|
||||
|
||||
return await translateText(contentToTranslate, targetLanguage);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
// Text-to-speech functionality
|
||||
import { TTS_API_URL, API_KEY, MODEL, setLastTTS } from "./config.js";
|
||||
import { API_KEY, MODEL, setLastTTS, TTS_API_URL } from "./config.js";
|
||||
import { getSelectedModel, getSelectedModelEndpoint } from "./models.js";
|
||||
|
||||
// Function to read text using TTS
|
||||
|
|
|
|||
33
uncloseai.js
33
uncloseai.js
|
|
@ -5,21 +5,17 @@
|
|||
* This maintains backward compatibility while enabling better code organization
|
||||
*/
|
||||
|
||||
// External dependencies
|
||||
import { marked } from "https://cdn.jsdelivr.net/npm/marked/lib/marked.esm.js";
|
||||
import hljs from "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/es/highlight.min.js";
|
||||
|
||||
import * as Chat from "./src/chat.js";
|
||||
// Internal modules
|
||||
import * as Config from "./src/config.js";
|
||||
import * as Models from "./src/models.js";
|
||||
import * as Content from "./src/content.js";
|
||||
import * as TTS from "./src/tts.js";
|
||||
import * as Chat from "./src/chat.js";
|
||||
import * as FileUpload from "./src/file-upload.js";
|
||||
import * as Storage from "./src/storage.js";
|
||||
import * as Models from "./src/models.js";
|
||||
import * as PageReader from "./src/page-reader.js";
|
||||
import * as UI from "./src/ui.js";
|
||||
import * as Storage from "./src/storage.js";
|
||||
import * as Translation from "./src/translation.js";
|
||||
import * as TTS from "./src/tts.js";
|
||||
import * as UI from "./src/ui.js";
|
||||
|
||||
// -------------------------
|
||||
// Public API - Main Functions
|
||||
|
|
@ -105,7 +101,8 @@ window.speakText = TTS.speakText;
|
|||
window.uploadFile = FileUpload.uploadFile;
|
||||
window.openTTSModal = UI.openTTSModal;
|
||||
window.openTranslateModal = UI.openTranslateModal;
|
||||
window.toggleUncloseaiEmbeddedModal = async () => await UI.toggleUncloseaiEmbeddedModal();
|
||||
window.toggleUncloseaiEmbeddedModal = async () =>
|
||||
await UI.toggleUncloseaiEmbeddedModal();
|
||||
window.extractWebpageContent = Content.extractWebpageContent;
|
||||
window.getSelectedModel = Models.getSelectedModel;
|
||||
window.getSelectedModelEndpoint = Models.getSelectedModelEndpoint;
|
||||
|
|
@ -118,12 +115,16 @@ window.handleSmartTranslate = UI.handleSmartTranslate;
|
|||
// Initialize on page load
|
||||
window.addEventListener("load", () => {
|
||||
console.log("uncloseai.js: window.onload event fired.");
|
||||
|
||||
|
||||
// Check skip init flag first
|
||||
if (window.UNCLOSEAI_SKIP_INIT === true) {
|
||||
console.log("uncloseai.js: Skipping full initialization as requested by flag.");
|
||||
console.log("uncloseai.js: Creating floating button only for preview windows.");
|
||||
|
||||
console.log(
|
||||
"uncloseai.js: Skipping full initialization as requested by flag.",
|
||||
);
|
||||
console.log(
|
||||
"uncloseai.js: Creating floating button only for preview windows.",
|
||||
);
|
||||
|
||||
// Still create floating button for preview windows, but skip other initialization
|
||||
const SHOW_FLOATING_BUTTON = window.UNCLOSEAI_FLOATING_BUTTON !== false;
|
||||
if (SHOW_FLOATING_BUTTON) {
|
||||
|
|
@ -131,12 +132,12 @@ window.addEventListener("load", () => {
|
|||
}
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
UI.initializeSystem();
|
||||
});
|
||||
|
||||
// Export helper functions for class-based integrations
|
||||
window.handleCustomChat = async function (button) {
|
||||
window.handleCustomChat = async (button) => {
|
||||
const container = button.parentElement;
|
||||
const input = container.querySelector("[data-chat-input]");
|
||||
const chatBox = container.querySelector("[data-chat-box]");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue