diff --git a/src/token_estimator.js b/src/token_estimator.js index 4019402..1ba9a3b 100644 --- a/src/token_estimator.js +++ b/src/token_estimator.js @@ -301,6 +301,87 @@ export class SimpleBPETokenizer { // Export singleton instance export const tokenizer = new SimpleBPETokenizer(); +// Use upstream tokenizer API for accurate token counting +export async function countTokensWithUpstream(text, model) { + const { API_KEY } = await import("./config.js"); + const { getSelectedModel, getSelectedModelEndpoint } = await import("./models.js"); + + try { + const response = await fetch(`${getSelectedModelEndpoint()}/tokenize`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${API_KEY}`, + }, + body: JSON.stringify({ + model: model || getSelectedModel(), + prompt: text, + add_special_tokens: true, + }), + }); + + if (!response.ok) { + console.warn(`Upstream tokenizer failed (${response.status}), falling back to JS tokenizer`); + return tokenizer.count(text); + } + + const tokens = await response.json(); + // Simply count the length of the token array + const tokenCount = tokens.length; + + console.log(`Upstream tokenizer: ${tokenCount.toLocaleString()} tokens for ${text.length.toLocaleString()} chars`); + + // Return in same format as JS tokenizer for compatibility + return { + totalTokens: tokenCount, + regularTokens: tokenCount, + specialTokens: 0, + breakdown: { + words: Math.ceil(tokenCount * 0.7), + numbers: Math.ceil(tokenCount * 0.1), + punctuation: Math.ceil(tokenCount * 0.1), + whitespace: Math.ceil(tokenCount * 0.1), + }, + textLength: text.length, + }; + } catch (error) { + console.warn("Error with upstream tokenizer, falling back to JS tokenizer:", error); + return tokenizer.count(text); + } +} + +// Get just the token count as a number from upstream +export async function getTokenCount(text, model) { + const { API_KEY } = await import("./config.js"); + const { getSelectedModel, getSelectedModelEndpoint } = await import("./models.js"); + + try { + const response = await fetch(`${getSelectedModelEndpoint()}/tokenize`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${API_KEY}`, + }, + body: JSON.stringify({ + model: model || getSelectedModel(), + prompt: text, + add_special_tokens: true, + }), + }); + + if (!response.ok) { + throw new Error(`Tokenizer API error: ${response.status}`); + } + + const tokens = await response.json(); + return tokens.length; + } catch (error) { + console.warn("Upstream tokenizer error:", error); + // Fallback to JS estimate + return tokenizer.count(text).totalTokens; + } +} + // Convenience functions export function countTokens(text) { return tokenizer.count(text); @@ -310,3 +391,8 @@ export function estimateTokens(text) { const result = tokenizer.count(text); return result.totalTokens; } + +// Format token count as a string with commas (e.g., "8,432 tokens") +export function formatTokenCount(count) { + return `${count.toLocaleString()} tokens`; +} diff --git a/src/translate-modal.js b/src/translate-modal.js index 055459d..4179731 100644 --- a/src/translate-modal.js +++ b/src/translate-modal.js @@ -284,13 +284,14 @@ export function openTranslateModal() { translateButton.disabled = true; resultContainer.style.display = "none"; - // Get token count and estimated time for progress - const { countTokens } = await import("./token_estimator.js"); + // Get token count and estimated time for progress using upstream tokenizer + const { countTokensWithUpstream } = await import("./token_estimator.js"); const { estimateProcessingTime } = await import("./translation.js"); + const { getSelectedModel } = await import("./models.js"); const textToAnalyze = isPageMode ? document.documentElement.outerHTML : textArea.value.trim(); - const tokenInfo = countTokens(textToAnalyze); + const tokenInfo = await countTokensWithUpstream(textToAnalyze, getSelectedModel()); const estimatedSeconds = estimateProcessingTime(tokenInfo.totalTokens); // Create progress indicator diff --git a/src/translation.js b/src/translation.js index e04ddc2..7b533ff 100644 --- a/src/translation.js +++ b/src/translation.js @@ -1,6 +1,6 @@ import { API_KEY } from "./config.js"; import { getSelectedModel, getSelectedModelEndpoint } from "./models.js"; -import { countTokens } from "./token_estimator.js"; +import { countTokens, countTokensWithUpstream } from "./token_estimator.js"; // Languages supported by Hermes 3 model (in English for reference) export const SUPPORTED_LANGUAGES = { @@ -237,8 +237,8 @@ export async function translateText(text, targetLanguage) { const { fetchModelsFromEndpoints } = await import("./models.js"); await fetchModelsFromEndpoints(); - // Get accurate token count and timing estimate - const tokenInfo = countTokens(text); + // Get accurate token count and timing estimate using upstream tokenizer + const tokenInfo = await countTokensWithUpstream(text, getSelectedModel()); const estimatedSeconds = estimateProcessingTime(tokenInfo.totalTokens); console.log("=== TRANSLATION ANALYSIS ==="); diff --git a/src/tts-modal.js b/src/tts-modal.js index 7d8f8b1..cd78aed 100644 --- a/src/tts-modal.js +++ b/src/tts-modal.js @@ -191,16 +191,16 @@ export function openTTSModal() { const speedLabel = document.createElement("label"); const speedValue = document.createElement("span"); - speedValue.textContent = "1.0"; + speedValue.textContent = "0.9"; speedLabel.textContent = `${getUIText("speedLabel")} ${speedValue.textContent}`; speedLabel.setAttribute("data-i18n", "speedLabel"); const speedSlider = document.createElement("input"); speedSlider.type = "range"; speedSlider.min = "0.25"; - speedSlider.max = "4.0"; - speedSlider.step = "0.25"; - speedSlider.value = "1.0"; + speedSlider.max = "2.0"; + speedSlider.step = "0.05"; + speedSlider.value = "0.9"; speedSlider.oninput = () => { speedValue.textContent = speedSlider.value; speedLabel.textContent = `${getUIText("speedLabel")} ${speedSlider.value}`; @@ -286,6 +286,7 @@ export function openTTSModal() { // Create hidden audio element (no controls) const audio = new Audio(result.blobUrl); audio.preload = "auto"; + audio.playbackRate = speed; // Apply the speed setting! // Create custom control container const controlsContainer = document.createElement("div"); diff --git a/src/uncloseai-embed-modal.js b/src/uncloseai-embed-modal.js index 8d8700b..73b8f04 100644 --- a/src/uncloseai-embed-modal.js +++ b/src/uncloseai-embed-modal.js @@ -121,7 +121,42 @@ function addCodeBlockCopyButtons(element) { pre.appendChild(copyBtn); } }); -}async function openUncloseaiEmbeddedModalNew() { +} + +// Function to refresh modal UI when language changes +function refreshModalUI() { + // Refresh all elements with data-i18n attributes in the modal + const modal = document.getElementById("uncloseai-embedded-modal"); + if (!modal) return; + + const elementsWithI18n = modal.querySelectorAll("[data-i18n]"); + elementsWithI18n.forEach((element) => { + const key = element.getAttribute("data-i18n"); + const translatedText = getUIText(key); + + if (element.tagName === "INPUT" && element.type === "text") { + element.placeholder = translatedText; + } else if (element.tagName === "BUTTON" || element.tagName === "LABEL") { + // Preserve any icons/emojis at the start + const currentText = element.textContent; + const iconMatch = currentText.match(/^[^\w\s]+\s*/); + const icon = iconMatch ? iconMatch[0] : ''; + element.textContent = icon + translatedText; + } else { + element.textContent = translatedText; + } + }); + + // Update any specific elements that need special handling + const titleElement = modal.querySelector("h1"); + if (titleElement) { + titleElement.innerHTML = `uncloseai. ${getUIText("hermesTitle")}`; + } + + console.log("Modal UI language refreshed"); +} + +async function openUncloseaiEmbeddedModalNew() { // Ensure ChunkFive font is loaded initializeChunkFiveFont(); @@ -495,6 +530,9 @@ function addCodeBlockCopyButtons(element) { // Show confirmation const selectedOption = languageSelect.options[languageSelect.selectedIndex]; console.log(`Language changed to: ${selectedOption.textContent}`); + + // Refresh all UI text in the modal + refreshModalUI(); // Notify user to refresh for full effect setTimeout(() => {