feat(ui): add comprehensive language localization system
- Add complete UI translations for all 19 supported languages - Add language preference dropdown in modal settings - Store language preference in localStorage - Inject language preference into Hermes system prompts - Add smart translation dropdown with AI-powered language detection - Keep both original translation modal and new smart dropdown - Remove non-functional upload button from modal - Add biome.json config to ignore third-party CSS files
This commit is contained in:
parent
cae51b3578
commit
eee33986cf
11 changed files with 1962 additions and 867 deletions
59
biome.json
Normal file
59
biome.json
Normal file
|
|
@ -0,0 +1,59 @@
|
||||||
|
{
|
||||||
|
"$schema": "https://biomejs.dev/schemas/2.0.6/schema.json",
|
||||||
|
"files": {
|
||||||
|
"includes": [
|
||||||
|
"**",
|
||||||
|
"!**/css/pico.classless.min.css",
|
||||||
|
"!**/node_modules",
|
||||||
|
"!**/*.min.js",
|
||||||
|
"!**/*.min.css"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"linter": {
|
||||||
|
"enabled": true,
|
||||||
|
"rules": {
|
||||||
|
"recommended": true,
|
||||||
|
"suspicious": {
|
||||||
|
"noAssignInExpressions": "off"
|
||||||
|
},
|
||||||
|
"style": {
|
||||||
|
"useTemplate": "warn",
|
||||||
|
"noParameterAssign": "error",
|
||||||
|
"useAsConstAssertion": "error",
|
||||||
|
"useDefaultParameterLast": "error",
|
||||||
|
"useEnumInitializers": "error",
|
||||||
|
"useSelfClosingElements": "error",
|
||||||
|
"useSingleVarDeclarator": "error",
|
||||||
|
"noUnusedTemplateLiteral": "error",
|
||||||
|
"useNumberNamespace": "error",
|
||||||
|
"noInferrableTypes": "error",
|
||||||
|
"noUselessElse": "error"
|
||||||
|
},
|
||||||
|
"complexity": {
|
||||||
|
"useOptionalChain": "warn"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"formatter": {
|
||||||
|
"enabled": true,
|
||||||
|
"formatWithErrors": false,
|
||||||
|
"indentStyle": "tab",
|
||||||
|
"indentWidth": 2,
|
||||||
|
"lineWidth": 80,
|
||||||
|
"lineEnding": "lf",
|
||||||
|
"includes": ["**", "!**/css/pico.classless.min.css"]
|
||||||
|
},
|
||||||
|
"assist": { "actions": { "source": { "organizeImports": "on" } } },
|
||||||
|
"javascript": {
|
||||||
|
"formatter": {
|
||||||
|
"quoteStyle": "double",
|
||||||
|
"jsxQuoteStyle": "double",
|
||||||
|
"quoteProperties": "asNeeded",
|
||||||
|
"trailingCommas": "all",
|
||||||
|
"semicolons": "always",
|
||||||
|
"arrowParentheses": "always",
|
||||||
|
"bracketSpacing": true,
|
||||||
|
"bracketSameLine": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -3,8 +3,8 @@ import { marked } from "https://cdn.jsdelivr.net/npm/marked/lib/marked.esm.js";
|
||||||
import hljs from "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/es/highlight.min.js";
|
import hljs from "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/es/highlight.min.js";
|
||||||
import { API_KEY } from "./config.js";
|
import { API_KEY } from "./config.js";
|
||||||
import { getSelectedModel, getSelectedModelEndpoint } from "./models.js";
|
import { getSelectedModel, getSelectedModelEndpoint } from "./models.js";
|
||||||
import { speakText, generateTitleForTTS } from "./tts.js";
|
import { initializeChatHistory, saveConversationHistory } from "./storage.js";
|
||||||
import { saveConversationHistory, initializeChatHistory } from "./storage.js";
|
import { generateTitleForTTS, speakText } from "./tts.js";
|
||||||
|
|
||||||
// Initialize chat history
|
// Initialize chat history
|
||||||
export let chatHistory = initializeChatHistory();
|
export let chatHistory = initializeChatHistory();
|
||||||
|
|
|
||||||
|
|
@ -17,11 +17,50 @@ export function setSystemMessageAppend(appendText) {
|
||||||
SYSTEM_MESSAGE_APPEND = appendText;
|
SYSTEM_MESSAGE_APPEND = appendText;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Function to get the complete system message
|
// Function to get the complete system message with language preference
|
||||||
export function getSystemMessage() {
|
export function getSystemMessage() {
|
||||||
return SYSTEM_MESSAGE_APPEND
|
// Get user's language preference from localStorage
|
||||||
|
let userLang = "en";
|
||||||
|
try {
|
||||||
|
userLang = localStorage.getItem("uncloseai_language") || "en";
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("Failed to read language preference:", error);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add language instruction to the system message
|
||||||
|
let languageInstruction = "";
|
||||||
|
if (userLang !== "en") {
|
||||||
|
// Map language codes to full names for clarity
|
||||||
|
const langNames = {
|
||||||
|
es: "Spanish",
|
||||||
|
zh: "Chinese (Simplified)",
|
||||||
|
hi: "Hindi",
|
||||||
|
fr: "French",
|
||||||
|
ar: "Arabic",
|
||||||
|
bn: "Bengali",
|
||||||
|
ru: "Russian",
|
||||||
|
pt: "Portuguese",
|
||||||
|
ur: "Urdu",
|
||||||
|
id: "Indonesian",
|
||||||
|
de: "German",
|
||||||
|
ja: "Japanese",
|
||||||
|
sw: "Swahili",
|
||||||
|
mr: "Marathi",
|
||||||
|
te: "Telugu",
|
||||||
|
tr: "Turkish",
|
||||||
|
"zh-tw": "Chinese (Traditional)",
|
||||||
|
ko: "Korean",
|
||||||
|
};
|
||||||
|
|
||||||
|
const langName = langNames[userLang] || userLang;
|
||||||
|
languageInstruction = `\n\nIMPORTANT: The user has set their language preference to ${langName}. Please respond in ${langName} unless the user explicitly asks for another language. Maintain natural, fluent communication in ${langName}.`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const baseMessage = SYSTEM_MESSAGE_APPEND
|
||||||
? `${SYSTEM_MESSAGE_BASE}\n\n${SYSTEM_MESSAGE_APPEND}`
|
? `${SYSTEM_MESSAGE_BASE}\n\n${SYSTEM_MESSAGE_APPEND}`
|
||||||
: SYSTEM_MESSAGE_BASE;
|
: SYSTEM_MESSAGE_BASE;
|
||||||
|
|
||||||
|
return baseMessage + languageInstruction;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Dynamic Endpoints Configuration for Chat API
|
// Dynamic Endpoints Configuration for Chat API
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,7 @@ export function extractWebpageContent() {
|
||||||
// Recursively extract text and links from the body content
|
// Recursively extract text and links from the body content
|
||||||
function getTextWithLinks(element) {
|
function getTextWithLinks(element) {
|
||||||
if (element.nodeType === Node.TEXT_NODE) {
|
if (element.nodeType === Node.TEXT_NODE) {
|
||||||
content += element.textContent + " ";
|
content += `${element.textContent} `;
|
||||||
} else if (element.nodeType === Node.ELEMENT_NODE) {
|
} else if (element.nodeType === Node.ELEMENT_NODE) {
|
||||||
if (element.tagName.toLowerCase() === "a") {
|
if (element.tagName.toLowerCase() === "a") {
|
||||||
// If it's a link, append the text and the href
|
// If it's a link, append the text and the href
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,6 @@
|
||||||
// File upload and processing functionality
|
import { chatHistory } from "./chat.js";
|
||||||
import { marked } from "https://cdn.jsdelivr.net/npm/marked/lib/marked.esm.js";
|
|
||||||
import hljs from "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/es/highlight.min.js";
|
|
||||||
import { MEGAPARCE_API_URL } from "./config.js";
|
import { MEGAPARCE_API_URL } from "./config.js";
|
||||||
import { speakText } from "./tts.js";
|
import { speakText } from "./tts.js";
|
||||||
import { chatHistory } from "./chat.js";
|
|
||||||
|
|
||||||
// Progress indicator functions
|
// Progress indicator functions
|
||||||
export function showProgressIndicator(message) {
|
export function showProgressIndicator(message) {
|
||||||
|
|
|
||||||
|
|
@ -105,7 +105,7 @@ export async function createModelSelectionDropdown() {
|
||||||
});
|
});
|
||||||
|
|
||||||
// Insert the dropdown directly above the chat input box.
|
// Insert the dropdown directly above the chat input box.
|
||||||
if (userInput && userInput.parentNode) {
|
if (userInput?.parentNode) {
|
||||||
userInput.parentNode.parentNode.insertBefore(
|
userInput.parentNode.parentNode.insertBefore(
|
||||||
dropdown,
|
dropdown,
|
||||||
userInput.parentNode,
|
userInput.parentNode,
|
||||||
|
|
@ -135,7 +135,7 @@ export function addRefreshModelsButton() {
|
||||||
container.id = "refresh-models-container";
|
container.id = "refresh-models-container";
|
||||||
|
|
||||||
// Position it near the model selection dropdown
|
// Position it near the model selection dropdown
|
||||||
if (userInput && userInput.parentNode) {
|
if (userInput?.parentNode) {
|
||||||
userInput.parentNode.parentNode.insertBefore(
|
userInput.parentNode.parentNode.insertBefore(
|
||||||
container,
|
container,
|
||||||
userInput.parentNode,
|
userInput.parentNode,
|
||||||
|
|
@ -146,7 +146,7 @@ export function addRefreshModelsButton() {
|
||||||
const refreshButton = document.createElement("button");
|
const refreshButton = document.createElement("button");
|
||||||
refreshButton.textContent = "Refresh Models";
|
refreshButton.textContent = "Refresh Models";
|
||||||
refreshButton.style.margin = "10px";
|
refreshButton.style.margin = "10px";
|
||||||
refreshButton.onclick = async function () {
|
refreshButton.onclick = async () => {
|
||||||
refreshButton.textContent = "Refreshing...";
|
refreshButton.textContent = "Refreshing...";
|
||||||
refreshButton.disabled = true;
|
refreshButton.disabled = true;
|
||||||
|
|
||||||
|
|
@ -189,7 +189,7 @@ export function addRefreshModelsButton() {
|
||||||
export function getSelectedModel() {
|
export function getSelectedModel() {
|
||||||
// Check modal dropdown first
|
// Check modal dropdown first
|
||||||
const modalDropdown = document.getElementById("hermes-model-selection");
|
const modalDropdown = document.getElementById("hermes-model-selection");
|
||||||
if (modalDropdown && modalDropdown.value) {
|
if (modalDropdown?.value) {
|
||||||
// Extract just the model name, not the unique ID
|
// Extract just the model name, not the unique ID
|
||||||
const selectedId = modalDropdown.value;
|
const selectedId = modalDropdown.value;
|
||||||
if (modelRegistry[selectedId]) {
|
if (modelRegistry[selectedId]) {
|
||||||
|
|
@ -200,7 +200,7 @@ export function getSelectedModel() {
|
||||||
|
|
||||||
// Fallback to main dropdown
|
// Fallback to main dropdown
|
||||||
const dropdown = document.getElementById("model-selection");
|
const dropdown = document.getElementById("model-selection");
|
||||||
if (dropdown && dropdown.value) {
|
if (dropdown?.value) {
|
||||||
// Extract just the model name, not the unique ID
|
// Extract just the model name, not the unique ID
|
||||||
const selectedId = dropdown.value;
|
const selectedId = dropdown.value;
|
||||||
if (modelRegistry[selectedId]) {
|
if (modelRegistry[selectedId]) {
|
||||||
|
|
@ -223,13 +223,13 @@ export function getSelectedModel() {
|
||||||
export function getSelectedModelEndpoint() {
|
export function getSelectedModelEndpoint() {
|
||||||
// Check modal dropdown first
|
// Check modal dropdown first
|
||||||
const modalDropdown = document.getElementById("modal-model-selection");
|
const modalDropdown = document.getElementById("modal-model-selection");
|
||||||
if (modalDropdown && modalDropdown.value && modelRegistry[modalDropdown.value]) {
|
if (modalDropdown?.value && modelRegistry[modalDropdown.value]) {
|
||||||
return modelRegistry[modalDropdown.value].url;
|
return modelRegistry[modalDropdown.value].url;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback to main dropdown
|
// Fallback to main dropdown
|
||||||
const dropdown = document.getElementById("model-selection");
|
const dropdown = document.getElementById("model-selection");
|
||||||
if (dropdown && dropdown.value && modelRegistry[dropdown.value]) {
|
if (dropdown?.value && modelRegistry[dropdown.value]) {
|
||||||
return modelRegistry[dropdown.value].url;
|
return modelRegistry[dropdown.value].url;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,51 +10,132 @@ export class SimpleBPETokenizer {
|
||||||
this.initializePatterns();
|
this.initializePatterns();
|
||||||
this.initializeCommonTokens();
|
this.initializeCommonTokens();
|
||||||
}
|
}
|
||||||
|
|
||||||
initializePatterns() {
|
initializePatterns() {
|
||||||
// Simplified tokenization patterns based on tiktoken's approach
|
// Simplified tokenization patterns based on tiktoken's approach
|
||||||
this.patterns = [
|
this.patterns = [
|
||||||
// Contractions (case insensitive)
|
// Contractions (case insensitive)
|
||||||
/'(?:s|t|m|d|ll|ve|re)/gi,
|
/'(?:s|t|m|d|ll|ve|re)/gi,
|
||||||
|
|
||||||
// Letters (word boundaries)
|
// Letters (word boundaries)
|
||||||
/[a-zA-Z]+/g,
|
/[a-zA-Z]+/g,
|
||||||
|
|
||||||
// Numbers (1-3 digits groups)
|
// Numbers (1-3 digits groups)
|
||||||
/\d{1,3}/g,
|
/\d{1,3}/g,
|
||||||
|
|
||||||
// Punctuation and symbols
|
// Punctuation and symbols
|
||||||
/[^\s\w]/g,
|
/[^\s\w]/g,
|
||||||
|
|
||||||
// Whitespace (including newlines)
|
// Whitespace (including newlines)
|
||||||
/\s+/g
|
/\s+/g,
|
||||||
];
|
];
|
||||||
|
|
||||||
// Compiled regex for faster processing
|
// Compiled regex for faster processing
|
||||||
this.mainPattern = /'(?:s|t|m|d|ll|ve|re)|[a-zA-Z]+|\d{1,3}|[^\s\w]|\s+/gi;
|
this.mainPattern = /'(?:s|t|m|d|ll|ve|re)|[a-zA-Z]+|\d{1,3}|[^\s\w]|\s+/gi;
|
||||||
}
|
}
|
||||||
|
|
||||||
initializeCommonTokens() {
|
initializeCommonTokens() {
|
||||||
// Common subword patterns from BPE training (simplified set)
|
// Common subword patterns from BPE training (simplified set)
|
||||||
this.commonSubwords = new Set([
|
this.commonSubwords = new Set([
|
||||||
// Common prefixes
|
// Common prefixes
|
||||||
'un', 're', 'in', 'dis', 'en', 'non', 'pre', 'over', 'mis', 'sub',
|
"un",
|
||||||
'anti', 'auto', 'co', 'de', 'ex', 'inter', 'multi', 'out', 'under',
|
"re",
|
||||||
|
"in",
|
||||||
// Common suffixes
|
"dis",
|
||||||
'ing', 'ed', 'er', 'est', 'ly', 'tion', 'ness', 'ment', 'able', 'ible',
|
"en",
|
||||||
'ful', 'less', 'ship', 'ward', 'wise', 'like', 'ous', 'ive', 'ate',
|
"non",
|
||||||
|
"pre",
|
||||||
|
"over",
|
||||||
|
"mis",
|
||||||
|
"sub",
|
||||||
|
"anti",
|
||||||
|
"auto",
|
||||||
|
"co",
|
||||||
|
"de",
|
||||||
|
"ex",
|
||||||
|
"inter",
|
||||||
|
"multi",
|
||||||
|
"out",
|
||||||
|
"under",
|
||||||
|
|
||||||
|
// Common suffixes
|
||||||
|
"ing",
|
||||||
|
"ed",
|
||||||
|
"er",
|
||||||
|
"est",
|
||||||
|
"ly",
|
||||||
|
"tion",
|
||||||
|
"ness",
|
||||||
|
"ment",
|
||||||
|
"able",
|
||||||
|
"ible",
|
||||||
|
"ful",
|
||||||
|
"less",
|
||||||
|
"ship",
|
||||||
|
"ward",
|
||||||
|
"wise",
|
||||||
|
"like",
|
||||||
|
"ous",
|
||||||
|
"ive",
|
||||||
|
"ate",
|
||||||
|
|
||||||
// Common words (high frequency)
|
// Common words (high frequency)
|
||||||
'the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can', 'had',
|
"the",
|
||||||
'her', 'was', 'one', 'our', 'out', 'day', 'get', 'use', 'man', 'new',
|
"and",
|
||||||
'now', 'way', 'may', 'say', 'see', 'him', 'two', 'how', 'its', 'who',
|
"for",
|
||||||
|
"are",
|
||||||
|
"but",
|
||||||
|
"not",
|
||||||
|
"you",
|
||||||
|
"all",
|
||||||
|
"can",
|
||||||
|
"had",
|
||||||
|
"her",
|
||||||
|
"was",
|
||||||
|
"one",
|
||||||
|
"our",
|
||||||
|
"out",
|
||||||
|
"day",
|
||||||
|
"get",
|
||||||
|
"use",
|
||||||
|
"man",
|
||||||
|
"new",
|
||||||
|
"now",
|
||||||
|
"way",
|
||||||
|
"may",
|
||||||
|
"say",
|
||||||
|
"see",
|
||||||
|
"him",
|
||||||
|
"two",
|
||||||
|
"how",
|
||||||
|
"its",
|
||||||
|
"who",
|
||||||
|
|
||||||
// Common letter combinations
|
// Common letter combinations
|
||||||
'th', 'er', 'on', 'an', 're', 'he', 'in', 'ed', 'nd', 'ha', 'at',
|
"th",
|
||||||
'en', 'es', 'of', 'or', 'nt', 'ea', 'ti', 'to', 'it', 'st', 'io'
|
"er",
|
||||||
|
"on",
|
||||||
|
"an",
|
||||||
|
"re",
|
||||||
|
"he",
|
||||||
|
"in",
|
||||||
|
"ed",
|
||||||
|
"nd",
|
||||||
|
"ha",
|
||||||
|
"at",
|
||||||
|
"en",
|
||||||
|
"es",
|
||||||
|
"of",
|
||||||
|
"or",
|
||||||
|
"nt",
|
||||||
|
"ea",
|
||||||
|
"ti",
|
||||||
|
"to",
|
||||||
|
"it",
|
||||||
|
"st",
|
||||||
|
"io",
|
||||||
]);
|
]);
|
||||||
|
|
||||||
// Token length estimates for common patterns
|
// Token length estimates for common patterns
|
||||||
this.tokenWeights = {
|
this.tokenWeights = {
|
||||||
word: 1,
|
word: 1,
|
||||||
|
|
@ -62,29 +143,29 @@ export class SimpleBPETokenizer {
|
||||||
punctuation: 1,
|
punctuation: 1,
|
||||||
whitespace: 0.3,
|
whitespace: 0.3,
|
||||||
contraction: 0.5,
|
contraction: 0.5,
|
||||||
subword: 0.6
|
subword: 0.6,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// Main tokenization method
|
// Main tokenization method
|
||||||
encode(text) {
|
encode(text) {
|
||||||
if (!text || typeof text !== 'string') return [];
|
if (!text || typeof text !== "string") return [];
|
||||||
|
|
||||||
// Split text using main pattern
|
// Split text using main pattern
|
||||||
const rawTokens = text.match(this.mainPattern) || [];
|
const rawTokens = text.match(this.mainPattern) || [];
|
||||||
const processedTokens = [];
|
const processedTokens = [];
|
||||||
|
|
||||||
for (const token of rawTokens) {
|
for (const token of rawTokens) {
|
||||||
processedTokens.push(...this.processToken(token));
|
processedTokens.push(...this.processToken(token));
|
||||||
}
|
}
|
||||||
|
|
||||||
return processedTokens;
|
return processedTokens;
|
||||||
}
|
}
|
||||||
|
|
||||||
processToken(token) {
|
processToken(token) {
|
||||||
const trimmed = token.trim();
|
const trimmed = token.trim();
|
||||||
if (!trimmed) return [token]; // Keep whitespace as-is
|
if (!trimmed) return [token]; // Keep whitespace as-is
|
||||||
|
|
||||||
// Check token type and split if needed
|
// Check token type and split if needed
|
||||||
if (this.isWord(trimmed)) {
|
if (this.isWord(trimmed)) {
|
||||||
return this.splitWord(trimmed);
|
return this.splitWord(trimmed);
|
||||||
|
|
@ -94,25 +175,25 @@ export class SimpleBPETokenizer {
|
||||||
return [token]; // Punctuation, symbols, etc.
|
return [token]; // Punctuation, symbols, etc.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
isWord(token) {
|
isWord(token) {
|
||||||
return /^[a-zA-Z]+$/.test(token);
|
return /^[a-zA-Z]+$/.test(token);
|
||||||
}
|
}
|
||||||
|
|
||||||
isNumber(token) {
|
isNumber(token) {
|
||||||
return /^\d+$/.test(token);
|
return /^\d+$/.test(token);
|
||||||
}
|
}
|
||||||
|
|
||||||
splitWord(word) {
|
splitWord(word) {
|
||||||
if (word.length <= 3) return [word];
|
if (word.length <= 3) return [word];
|
||||||
|
|
||||||
const tokens = [];
|
const tokens = [];
|
||||||
let remaining = word.toLowerCase();
|
let remaining = word.toLowerCase();
|
||||||
|
|
||||||
// Try to find common subwords
|
// Try to find common subwords
|
||||||
while (remaining.length > 0) {
|
while (remaining.length > 0) {
|
||||||
let found = false;
|
let found = false;
|
||||||
|
|
||||||
// Look for longest matching subword first
|
// Look for longest matching subword first
|
||||||
for (let len = Math.min(remaining.length, 6); len >= 2; len--) {
|
for (let len = Math.min(remaining.length, 6); len >= 2; len--) {
|
||||||
const substr = remaining.substring(0, len);
|
const substr = remaining.substring(0, len);
|
||||||
|
|
@ -123,7 +204,7 @@ export class SimpleBPETokenizer {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!found) {
|
if (!found) {
|
||||||
// Split remaining into chunks
|
// Split remaining into chunks
|
||||||
if (remaining.length <= 4) {
|
if (remaining.length <= 4) {
|
||||||
|
|
@ -137,10 +218,10 @@ export class SimpleBPETokenizer {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return tokens;
|
return tokens;
|
||||||
}
|
}
|
||||||
|
|
||||||
splitNumber(number) {
|
splitNumber(number) {
|
||||||
// Split numbers into groups of 1-3 digits
|
// Split numbers into groups of 1-3 digits
|
||||||
const chunks = [];
|
const chunks = [];
|
||||||
|
|
@ -149,71 +230,72 @@ export class SimpleBPETokenizer {
|
||||||
}
|
}
|
||||||
return chunks;
|
return chunks;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Quick token count (main method for estimation)
|
// Quick token count (main method for estimation)
|
||||||
count(text) {
|
count(text) {
|
||||||
if (!text || typeof text !== 'string') return 0;
|
if (!text || typeof text !== "string") return 0;
|
||||||
|
|
||||||
// Handle special content first
|
// Handle special content first
|
||||||
const specialTokens = this.countSpecialContent(text);
|
const specialTokens = this.countSpecialContent(text);
|
||||||
|
|
||||||
// Remove special content and count regular tokens
|
// Remove special content and count regular tokens
|
||||||
const cleanText = this.removeSpecialContent(text);
|
const cleanText = this.removeSpecialContent(text);
|
||||||
const regularTokens = this.encode(cleanText);
|
const regularTokens = this.encode(cleanText);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
totalTokens: specialTokens.count + regularTokens.length,
|
totalTokens: specialTokens.count + regularTokens.length,
|
||||||
regularTokens: regularTokens.length,
|
regularTokens: regularTokens.length,
|
||||||
specialTokens: specialTokens.count,
|
specialTokens: specialTokens.count,
|
||||||
breakdown: {
|
breakdown: {
|
||||||
...specialTokens.breakdown,
|
...specialTokens.breakdown,
|
||||||
words: regularTokens.filter(t => this.isWord(t.trim())).length,
|
words: regularTokens.filter((t) => this.isWord(t.trim())).length,
|
||||||
numbers: regularTokens.filter(t => this.isNumber(t.trim())).length,
|
numbers: regularTokens.filter((t) => this.isNumber(t.trim())).length,
|
||||||
punctuation: regularTokens.filter(t => /^[^\s\w]+$/.test(t.trim())).length,
|
punctuation: regularTokens.filter((t) => /^[^\s\w]+$/.test(t.trim()))
|
||||||
whitespace: regularTokens.filter(t => /^\s+$/.test(t)).length
|
.length,
|
||||||
|
whitespace: regularTokens.filter((t) => /^\s+$/.test(t)).length,
|
||||||
},
|
},
|
||||||
textLength: text.length
|
textLength: text.length,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
countSpecialContent(text) {
|
countSpecialContent(text) {
|
||||||
let count = 0;
|
let count = 0;
|
||||||
const breakdown = { htmlTags: 0, codeBlocks: 0, inlineCode: 0, urls: 0 };
|
const breakdown = { htmlTags: 0, codeBlocks: 0, inlineCode: 0, urls: 0 };
|
||||||
|
|
||||||
// HTML tags
|
// HTML tags
|
||||||
const htmlMatches = text.match(/<[^>]+>/g) || [];
|
const htmlMatches = text.match(/<[^>]+>/g) || [];
|
||||||
breakdown.htmlTags = htmlMatches.length;
|
breakdown.htmlTags = htmlMatches.length;
|
||||||
count += htmlMatches.length * 3; // ~3 tokens per tag
|
count += htmlMatches.length * 3; // ~3 tokens per tag
|
||||||
|
|
||||||
// Code blocks
|
// Code blocks
|
||||||
const codeBlockMatches = text.match(/```[\s\S]*?```/g) || [];
|
const codeBlockMatches = text.match(/```[\s\S]*?```/g) || [];
|
||||||
breakdown.codeBlocks = codeBlockMatches.length;
|
breakdown.codeBlocks = codeBlockMatches.length;
|
||||||
codeBlockMatches.forEach(block => {
|
codeBlockMatches.forEach((block) => {
|
||||||
const content = block.replace(/```[\w]*\n?/g, '').replace(/```$/g, '');
|
const content = block.replace(/```[\w]*\n?/g, "").replace(/```$/g, "");
|
||||||
count += Math.ceil(content.length / 2.5); // Code is denser
|
count += Math.ceil(content.length / 2.5); // Code is denser
|
||||||
});
|
});
|
||||||
|
|
||||||
// Inline code
|
// Inline code
|
||||||
const inlineMatches = text.match(/`[^`]+`/g) || [];
|
const inlineMatches = text.match(/`[^`]+`/g) || [];
|
||||||
breakdown.inlineCode = inlineMatches.length;
|
breakdown.inlineCode = inlineMatches.length;
|
||||||
inlineMatches.forEach(code => {
|
inlineMatches.forEach((code) => {
|
||||||
count += Math.ceil(code.length / 3);
|
count += Math.ceil(code.length / 3);
|
||||||
});
|
});
|
||||||
|
|
||||||
// URLs
|
// URLs
|
||||||
const urlMatches = text.match(/https?:\/\/[^\s<>"{}|\\^`\[\]]+/g) || [];
|
const urlMatches = text.match(/https?:\/\/[^\s<>"{}|\\^`[\]]+/g) || [];
|
||||||
breakdown.urls = urlMatches.length;
|
breakdown.urls = urlMatches.length;
|
||||||
count += urlMatches.length * 4; // URLs are typically 2-6 tokens
|
count += urlMatches.length * 4; // URLs are typically 2-6 tokens
|
||||||
|
|
||||||
return { count, breakdown };
|
return { count, breakdown };
|
||||||
}
|
}
|
||||||
|
|
||||||
removeSpecialContent(text) {
|
removeSpecialContent(text) {
|
||||||
return text
|
return text
|
||||||
.replace(/```[\s\S]*?```/g, ' ') // Replace code blocks
|
.replace(/```[\s\S]*?```/g, " ") // Replace code blocks
|
||||||
.replace(/`[^`]+`/g, ' ') // Replace inline code
|
.replace(/`[^`]+`/g, " ") // Replace inline code
|
||||||
.replace(/<[^>]+>/g, ' ') // Replace HTML tags
|
.replace(/<[^>]+>/g, " ") // Replace HTML tags
|
||||||
.replace(/https?:\/\/[^\s<>"{}|\\^`\[\]]+/g, ' '); // Replace URLs
|
.replace(/https?:\/\/[^\s<>"{}|\\^`[\]]+/g, " "); // Replace URLs
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -228,4 +310,4 @@ export function countTokens(text) {
|
||||||
export function estimateTokens(text) {
|
export function estimateTokens(text) {
|
||||||
const result = tokenizer.count(text);
|
const result = tokenizer.count(text);
|
||||||
return result.totalTokens;
|
return result.totalTokens;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,3 @@
|
||||||
// Translation functionality with code block preservation
|
|
||||||
import { sendMessage } from "./chat.js";
|
|
||||||
import { API_KEY } from "./config.js";
|
import { API_KEY } from "./config.js";
|
||||||
import { getSelectedModel, getSelectedModelEndpoint } from "./models.js";
|
import { getSelectedModel, getSelectedModelEndpoint } from "./models.js";
|
||||||
import { countTokens } from "./token_estimator.js";
|
import { countTokens } from "./token_estimator.js";
|
||||||
|
|
@ -168,7 +166,7 @@ export function preserveSpecialContent(text) {
|
||||||
|
|
||||||
// URLs and URIs
|
// URLs and URIs
|
||||||
preservedText = preservedText.replace(
|
preservedText = preservedText.replace(
|
||||||
/https?:\/\/[^\s<>"{}|\\^`\[\]]+/g,
|
/https?:\/\/[^\s<>"{}|\\^`[\]]+/g,
|
||||||
(match) => {
|
(match) => {
|
||||||
const index = preservations.length;
|
const index = preservations.length;
|
||||||
const placeholder = `__URI_${index}__`;
|
const placeholder = `__URI_${index}__`;
|
||||||
|
|
@ -219,17 +217,17 @@ export function restoreSpecialContent(translatedText, preservations) {
|
||||||
// Calculate processing time estimate based on real vLLM metrics
|
// Calculate processing time estimate based on real vLLM metrics
|
||||||
export function estimateProcessingTime(tokenCount) {
|
export function estimateProcessingTime(tokenCount) {
|
||||||
// Real vLLM performance metrics from production:
|
// Real vLLM performance metrics from production:
|
||||||
// - Prompt processing: ~3,144 tokens/second
|
// - Prompt processing: ~3,144 tokens/second
|
||||||
// - Generation: ~121 tokens/second
|
// - Generation: ~121 tokens/second
|
||||||
// - Translation typically generates 1-2x input tokens
|
// - Translation typically generates 1-2x input tokens
|
||||||
|
|
||||||
const promptProcessingTime = tokenCount / 3144; // seconds for input processing
|
const promptProcessingTime = tokenCount / 3144; // seconds for input processing
|
||||||
const estimatedOutputTokens = tokenCount * 1.2; // Translation usually 1.2x input length
|
const estimatedOutputTokens = tokenCount * 1.2; // Translation usually 1.2x input length
|
||||||
const generationTime = estimatedOutputTokens / 121; // seconds for generation
|
const generationTime = estimatedOutputTokens / 121; // seconds for generation
|
||||||
|
|
||||||
// Add some buffer for network latency and processing overhead
|
// Add some buffer for network latency and processing overhead
|
||||||
const totalTime = (promptProcessingTime + generationTime) * 1.3;
|
const totalTime = (promptProcessingTime + generationTime) * 1.3;
|
||||||
|
|
||||||
return Math.ceil(totalTime);
|
return Math.ceil(totalTime);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -238,17 +236,17 @@ export async function translateText(text, targetLanguage) {
|
||||||
// Ensure models are loaded before translation
|
// Ensure models are loaded before translation
|
||||||
const { fetchModelsFromEndpoints } = await import("./models.js");
|
const { fetchModelsFromEndpoints } = await import("./models.js");
|
||||||
await fetchModelsFromEndpoints();
|
await fetchModelsFromEndpoints();
|
||||||
|
|
||||||
// Get accurate token count and timing estimate
|
// Get accurate token count and timing estimate
|
||||||
const tokenInfo = countTokens(text);
|
const tokenInfo = countTokens(text);
|
||||||
const estimatedSeconds = estimateProcessingTime(tokenInfo.totalTokens);
|
const estimatedSeconds = estimateProcessingTime(tokenInfo.totalTokens);
|
||||||
|
|
||||||
console.log("=== TRANSLATION ANALYSIS ===");
|
console.log("=== TRANSLATION ANALYSIS ===");
|
||||||
console.log(`Input tokens: ${tokenInfo.totalTokens}`);
|
console.log(`Input tokens: ${tokenInfo.totalTokens}`);
|
||||||
console.log(`Text length: ${tokenInfo.textLength} characters`);
|
console.log(`Text length: ${tokenInfo.textLength} characters`);
|
||||||
console.log(`Estimated processing time: ${estimatedSeconds} seconds`);
|
console.log(`Estimated processing time: ${estimatedSeconds} seconds`);
|
||||||
console.log(`Token breakdown:`, tokenInfo.breakdown);
|
console.log(`Token breakdown:`, tokenInfo.breakdown);
|
||||||
|
|
||||||
const { preservedText, preservations } = preserveSpecialContent(text);
|
const { preservedText, preservations } = preserveSpecialContent(text);
|
||||||
|
|
||||||
// Debug logging
|
// Debug logging
|
||||||
|
|
@ -303,11 +301,13 @@ ${preservedText}`;
|
||||||
|
|
||||||
// Validate response
|
// Validate response
|
||||||
if (!translatedText || translatedText.length === 0) {
|
if (!translatedText || translatedText.length === 0) {
|
||||||
throw new Error("Translation API returned empty response. Please try again.");
|
throw new Error(
|
||||||
|
"Translation API returned empty response. Please try again.",
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if placeholders are still in the response
|
// Check if placeholders are still in the response
|
||||||
preservations.forEach((item, index) => {
|
preservations.forEach((item, _index) => {
|
||||||
const found = translatedText.includes(item.placeholder);
|
const found = translatedText.includes(item.placeholder);
|
||||||
console.log(
|
console.log(
|
||||||
` Placeholder ${item.placeholder} found in response: ${found}`,
|
` Placeholder ${item.placeholder} found in response: ${found}`,
|
||||||
|
|
@ -338,11 +338,13 @@ ${preservedText}`;
|
||||||
export function extractPageContent() {
|
export function extractPageContent() {
|
||||||
// Clone the document to avoid modifying the original
|
// Clone the document to avoid modifying the original
|
||||||
const documentClone = document.cloneNode(true);
|
const documentClone = document.cloneNode(true);
|
||||||
|
|
||||||
// Remove any open modals/dialogs that shouldn't be in the translation
|
// Remove any open modals/dialogs that shouldn't be in the translation
|
||||||
const modalsToRemove = documentClone.querySelectorAll('dialog[open], #uncloseai-embedded-modal, [id*="modal"], [class*="modal"]');
|
const modalsToRemove = documentClone.querySelectorAll(
|
||||||
modalsToRemove.forEach(modal => modal.remove());
|
'dialog[open], #uncloseai-embedded-modal, [id*="modal"], [class*="modal"]',
|
||||||
|
);
|
||||||
|
modalsToRemove.forEach((modal) => modal.remove());
|
||||||
|
|
||||||
// Return the entire HTML of the document to preserve head, styles, and scripts
|
// Return the entire HTML of the document to preserve head, styles, and scripts
|
||||||
return documentClone.documentElement.outerHTML;
|
return documentClone.documentElement.outerHTML;
|
||||||
}
|
}
|
||||||
|
|
@ -359,7 +361,7 @@ export async function translateCurrentPage(targetLanguage) {
|
||||||
const maxLength = 50000; // Much higher limit for complete page translation
|
const maxLength = 50000; // Much higher limit for complete page translation
|
||||||
const contentToTranslate =
|
const contentToTranslate =
|
||||||
pageContent.length > maxLength
|
pageContent.length > maxLength
|
||||||
? pageContent.substring(0, maxLength) + "..."
|
? `${pageContent.substring(0, maxLength)}...`
|
||||||
: pageContent;
|
: pageContent;
|
||||||
|
|
||||||
return await translateText(contentToTranslate, targetLanguage);
|
return await translateText(contentToTranslate, targetLanguage);
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
// Text-to-speech functionality
|
// Text-to-speech functionality
|
||||||
import { TTS_API_URL, API_KEY, MODEL, setLastTTS } from "./config.js";
|
import { API_KEY, MODEL, setLastTTS, TTS_API_URL } from "./config.js";
|
||||||
import { getSelectedModel, getSelectedModelEndpoint } from "./models.js";
|
import { getSelectedModel, getSelectedModelEndpoint } from "./models.js";
|
||||||
|
|
||||||
// Function to read text using TTS
|
// Function to read text using TTS
|
||||||
|
|
|
||||||
33
uncloseai.js
33
uncloseai.js
|
|
@ -5,21 +5,17 @@
|
||||||
* This maintains backward compatibility while enabling better code organization
|
* This maintains backward compatibility while enabling better code organization
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// External dependencies
|
import * as Chat from "./src/chat.js";
|
||||||
import { marked } from "https://cdn.jsdelivr.net/npm/marked/lib/marked.esm.js";
|
|
||||||
import hljs from "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/es/highlight.min.js";
|
|
||||||
|
|
||||||
// Internal modules
|
// Internal modules
|
||||||
import * as Config from "./src/config.js";
|
import * as Config from "./src/config.js";
|
||||||
import * as Models from "./src/models.js";
|
|
||||||
import * as Content from "./src/content.js";
|
import * as Content from "./src/content.js";
|
||||||
import * as TTS from "./src/tts.js";
|
|
||||||
import * as Chat from "./src/chat.js";
|
|
||||||
import * as FileUpload from "./src/file-upload.js";
|
import * as FileUpload from "./src/file-upload.js";
|
||||||
import * as Storage from "./src/storage.js";
|
import * as Models from "./src/models.js";
|
||||||
import * as PageReader from "./src/page-reader.js";
|
import * as PageReader from "./src/page-reader.js";
|
||||||
import * as UI from "./src/ui.js";
|
import * as Storage from "./src/storage.js";
|
||||||
import * as Translation from "./src/translation.js";
|
import * as Translation from "./src/translation.js";
|
||||||
|
import * as TTS from "./src/tts.js";
|
||||||
|
import * as UI from "./src/ui.js";
|
||||||
|
|
||||||
// -------------------------
|
// -------------------------
|
||||||
// Public API - Main Functions
|
// Public API - Main Functions
|
||||||
|
|
@ -105,7 +101,8 @@ window.speakText = TTS.speakText;
|
||||||
window.uploadFile = FileUpload.uploadFile;
|
window.uploadFile = FileUpload.uploadFile;
|
||||||
window.openTTSModal = UI.openTTSModal;
|
window.openTTSModal = UI.openTTSModal;
|
||||||
window.openTranslateModal = UI.openTranslateModal;
|
window.openTranslateModal = UI.openTranslateModal;
|
||||||
window.toggleUncloseaiEmbeddedModal = async () => await UI.toggleUncloseaiEmbeddedModal();
|
window.toggleUncloseaiEmbeddedModal = async () =>
|
||||||
|
await UI.toggleUncloseaiEmbeddedModal();
|
||||||
window.extractWebpageContent = Content.extractWebpageContent;
|
window.extractWebpageContent = Content.extractWebpageContent;
|
||||||
window.getSelectedModel = Models.getSelectedModel;
|
window.getSelectedModel = Models.getSelectedModel;
|
||||||
window.getSelectedModelEndpoint = Models.getSelectedModelEndpoint;
|
window.getSelectedModelEndpoint = Models.getSelectedModelEndpoint;
|
||||||
|
|
@ -118,12 +115,16 @@ window.handleSmartTranslate = UI.handleSmartTranslate;
|
||||||
// Initialize on page load
|
// Initialize on page load
|
||||||
window.addEventListener("load", () => {
|
window.addEventListener("load", () => {
|
||||||
console.log("uncloseai.js: window.onload event fired.");
|
console.log("uncloseai.js: window.onload event fired.");
|
||||||
|
|
||||||
// Check skip init flag first
|
// Check skip init flag first
|
||||||
if (window.UNCLOSEAI_SKIP_INIT === true) {
|
if (window.UNCLOSEAI_SKIP_INIT === true) {
|
||||||
console.log("uncloseai.js: Skipping full initialization as requested by flag.");
|
console.log(
|
||||||
console.log("uncloseai.js: Creating floating button only for preview windows.");
|
"uncloseai.js: Skipping full initialization as requested by flag.",
|
||||||
|
);
|
||||||
|
console.log(
|
||||||
|
"uncloseai.js: Creating floating button only for preview windows.",
|
||||||
|
);
|
||||||
|
|
||||||
// Still create floating button for preview windows, but skip other initialization
|
// Still create floating button for preview windows, but skip other initialization
|
||||||
const SHOW_FLOATING_BUTTON = window.UNCLOSEAI_FLOATING_BUTTON !== false;
|
const SHOW_FLOATING_BUTTON = window.UNCLOSEAI_FLOATING_BUTTON !== false;
|
||||||
if (SHOW_FLOATING_BUTTON) {
|
if (SHOW_FLOATING_BUTTON) {
|
||||||
|
|
@ -131,12 +132,12 @@ window.addEventListener("load", () => {
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
UI.initializeSystem();
|
UI.initializeSystem();
|
||||||
});
|
});
|
||||||
|
|
||||||
// Export helper functions for class-based integrations
|
// Export helper functions for class-based integrations
|
||||||
window.handleCustomChat = async function (button) {
|
window.handleCustomChat = async (button) => {
|
||||||
const container = button.parentElement;
|
const container = button.parentElement;
|
||||||
const input = container.querySelector("[data-chat-input]");
|
const input = container.querySelector("[data-chat-input]");
|
||||||
const chatBox = container.querySelector("[data-chat-box]");
|
const chatBox = container.querySelector("[data-chat-box]");
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue