diff --git a/src/token_estimator.js b/src/token_estimator.js index af84deb..9a790cb 100644 --- a/src/token_estimator.js +++ b/src/token_estimator.js @@ -301,6 +301,9 @@ export class SimpleBPETokenizer { // Export singleton instance export const tokenizer = new SimpleBPETokenizer(); +// Cache endpoints that don't support tokenization +const failedTokenizeEndpoints = new Set(); + // Use upstream tokenizer API for accurate token counting export async function countTokensWithUpstream(text, model) { const { API_KEY } = await import("./config.js"); @@ -308,6 +311,13 @@ export async function countTokensWithUpstream(text, model) { try { const endpoint = getSelectedModelEndpoint().replace('/v1', ''); + + // Skip if this endpoint previously failed tokenization + if (failedTokenizeEndpoints.has(endpoint)) { + console.log(`Endpoint ${endpoint} previously failed tokenization, using JS tokenizer`); + return tokenizer.count(text); + } + console.log(`Trying tokenization at: ${endpoint}/tokenize`); const response = await fetch(`${endpoint}/tokenize`, { @@ -325,6 +335,8 @@ export async function countTokensWithUpstream(text, model) { if (!response.ok) { console.warn(`Upstream tokenizer failed (${response.status}) at ${endpoint}/tokenize, falling back to JS tokenizer`); + // Cache this endpoint as failed for future requests + failedTokenizeEndpoints.add(endpoint); return tokenizer.count(text); } @@ -350,6 +362,9 @@ export async function countTokensWithUpstream(text, model) { }; } catch (error) { console.warn("Error with upstream tokenizer, falling back to JS tokenizer:", error); + // Cache this endpoint as failed for future requests + const endpoint = getSelectedModelEndpoint().replace('/v1', ''); + failedTokenizeEndpoints.add(endpoint); return tokenizer.count(text); } } @@ -361,6 +376,13 @@ export async function getTokenCount(text, model) { try { const endpoint = getSelectedModelEndpoint().replace('/v1', ''); + + // Skip if this endpoint previously failed tokenization + if (failedTokenizeEndpoints.has(endpoint)) { + console.log(`Endpoint ${endpoint} previously failed tokenization, using JS tokenizer`); + return tokenizer.count(text).totalTokens; + } + console.log(`Trying tokenization at: ${endpoint}/tokenize`); const response = await fetch(`${endpoint}/tokenize`, { @@ -377,6 +399,8 @@ export async function getTokenCount(text, model) { }); if (!response.ok) { + // Cache this endpoint as failed for future requests + failedTokenizeEndpoints.add(endpoint); throw new Error(`Tokenizer API error: ${response.status} at ${endpoint}/tokenize`); } @@ -385,6 +409,9 @@ export async function getTokenCount(text, model) { return Array.isArray(tokens) ? tokens.length : (tokens.count || Object.keys(tokens).length); } catch (error) { console.warn("Upstream tokenizer error:", error); + // Cache this endpoint as failed for future requests + const endpoint = getSelectedModelEndpoint().replace('/v1', ''); + failedTokenizeEndpoints.add(endpoint); // Fallback to JS estimate return tokenizer.count(text).totalTokens; } diff --git a/src/translation.js b/src/translation.js index f07227d..06f72a1 100644 --- a/src/translation.js +++ b/src/translation.js @@ -46,10 +46,14 @@ async function splitHTMLIntoChunks(htmlContent, maxTokensPerChunk, model) { for (const element of elements) { const elementHTML = element.outerHTML; - const elementTokens = await countTokens(elementHTML, model); + const elementTokensResult = await countTokens(elementHTML, model); + const elementTokens = typeof elementTokensResult === 'object' ? elementTokensResult.totalTokens || elementTokensResult.count || 0 : elementTokensResult; + + console.log(`Element tokens: ${elementTokens}, current total: ${currentTokens}, limit: ${maxTokensPerChunk}`); // If adding this element would exceed the limit, start a new chunk if (currentTokens + elementTokens > maxTokensPerChunk && currentChunk) { + console.log(`Starting new chunk at ${currentTokens} tokens`); chunks.push(currentChunk); currentChunk = elementHTML; currentTokens = elementTokens;