engineering/uncloseai.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: use exact tokenization in translate modal UI instead of rough estimate

Browse source
This commit is contained in:
Russell Ballestrini 2025-07-03 22:01:01 -04:00
parent e300a93545
commit 28d0b516f1
1 changed files with 26 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

33
src/translate-modal.js
View file

@ -284,13 +284,32 @@ export function openTranslateModal() {
translateButton.disabled = true;
resultContainer.style.display = "none";
// Note: Accurate tokenization happens in the translation function
// This is just a rough estimate for the UI progress indicator
const textToAnalyze = isPageMode
? document.documentElement.outerHTML
// Get exact tokenization for the UI progress indicator
const { extractPageContent } = await import("./translation.js");
const { preserveSpecialContent, estimateProcessingTime } = await import("./translation.js");
const { countTokensWithUpstream } = await import("./token_estimator.js");
const { getSelectedModel } = await import("./models.js");
const { SUPPORTED_LANGUAGES } = await import("./translation.js");
const rawContent = isPageMode
? extractPageContent()
: textArea.value.trim();
const roughTokenEstimate = Math.ceil(textToAnalyze.length / 3);
const estimatedSeconds = Math.ceil(roughTokenEstimate / 100) * 10;
// Limit content length to avoid overwhelming the AI
const maxLength = 50000;
const contentToProcess = rawContent.length > maxLength
? `${rawContent.substring(0, maxLength)}...`
: rawContent;
const { preservedText } = preserveSpecialContent(contentToProcess);
// Build the actual prompt that will be sent to the AI
const prompt = `Translate the following HTML content into ${SUPPORTED_LANGUAGES[targetLanguageCode]}. The input is in English and formatted as HTML, which includes formatting, code blocks, and technical content. Ensure the output preserves the structure, syntax, and formatting of the original. Do not translate or modify placeholders like __CODE_0__, __URI_0__, __HTML_0__, etc., as they represent code blocks, URLs, or HTML elements that should remain unchanged. Only translate the surrounding text. Here is the text to translate:
${preservedText}`;
const tokenInfo = await countTokensWithUpstream(prompt, getSelectedModel());
const estimatedSeconds = estimateProcessingTime(tokenInfo.totalTokens);
// Create progress indicator
const progressContainer = document.createElement("div");
@ -306,7 +325,7 @@ export function openTranslateModal() {
const progressText = document.createElement("div");
progressText.style.cssText =
"font-weight: bold; color: #4CAF50; margin-bottom: 10px;";
progressText.textContent = getUIText("processingTokens", { tokens: roughTokenEstimate });
progressText.textContent = getUIText("processingTokens", { tokens: tokenInfo.totalTokens });
const etaText = document.createElement("div");
etaText.style.cssText = "color: #666; margin-bottom: 15px;";