From a8a4359e96b5acf1be566852a346272d80da55c4 Mon Sep 17 00:00:00 2001
From: Russell Ballestrini <russell.ballestrini@gmail.com>
Date: Fri, 11 Jul 2025 21:52:24 -0400
Subject: [PATCH] auto-detect and cache endpoints without tokenization support
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When upstream tokenization fails (404, errors), cache the endpoint
and automatically use JS tokenizer for future requests. This fixes
chunking for APIs like Groq that don't have tokenize endpoints.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 src/token_estimator.js | 27 +++++++++++++++++++++++++++
 src/translation.js     |  6 +++++-
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/src/token_estimator.js b/src/token_estimator.js
index af84deb..9a790cb 100644
--- a/src/token_estimator.js
+++ b/src/token_estimator.js
@@ -301,6 +301,9 @@ export class SimpleBPETokenizer {
 // Export singleton instance
 export const tokenizer = new SimpleBPETokenizer();
 
+// Cache endpoints that don't support tokenization
+const failedTokenizeEndpoints = new Set();
+
 // Use upstream tokenizer API for accurate token counting
 export async function countTokensWithUpstream(text, model) {
 	const { API_KEY } = await import("./config.js");
@@ -308,6 +311,13 @@ export async function countTokensWithUpstream(text, model) {
 	
 	try {
 		const endpoint = getSelectedModelEndpoint().replace('/v1', '');
+		
+		// Skip if this endpoint previously failed tokenization
+		if (failedTokenizeEndpoints.has(endpoint)) {
+			console.log(`Endpoint ${endpoint} previously failed tokenization, using JS tokenizer`);
+			return tokenizer.count(text);
+		}
+		
 		console.log(`Trying tokenization at: ${endpoint}/tokenize`);
 		
 		const response = await fetch(`${endpoint}/tokenize`, {
@@ -325,6 +335,8 @@ export async function countTokensWithUpstream(text, model) {
 
 		if (!response.ok) {
 			console.warn(`Upstream tokenizer failed (${response.status}) at ${endpoint}/tokenize, falling back to JS tokenizer`);
+			// Cache this endpoint as failed for future requests
+			failedTokenizeEndpoints.add(endpoint);
 			return tokenizer.count(text);
 		}
 
@@ -350,6 +362,9 @@ export async function countTokensWithUpstream(text, model) {
 		};
 	} catch (error) {
 		console.warn("Error with upstream tokenizer, falling back to JS tokenizer:", error);
+		// Cache this endpoint as failed for future requests
+		const endpoint = getSelectedModelEndpoint().replace('/v1', '');
+		failedTokenizeEndpoints.add(endpoint);
 		return tokenizer.count(text);
 	}
 }
@@ -361,6 +376,13 @@ export async function getTokenCount(text, model) {
 	
 	try {
 		const endpoint = getSelectedModelEndpoint().replace('/v1', '');
+		
+		// Skip if this endpoint previously failed tokenization
+		if (failedTokenizeEndpoints.has(endpoint)) {
+			console.log(`Endpoint ${endpoint} previously failed tokenization, using JS tokenizer`);
+			return tokenizer.count(text).totalTokens;
+		}
+		
 		console.log(`Trying tokenization at: ${endpoint}/tokenize`);
 		
 		const response = await fetch(`${endpoint}/tokenize`, {
@@ -377,6 +399,8 @@ export async function getTokenCount(text, model) {
 		});
 
 		if (!response.ok) {
+			// Cache this endpoint as failed for future requests
+			failedTokenizeEndpoints.add(endpoint);
 			throw new Error(`Tokenizer API error: ${response.status} at ${endpoint}/tokenize`);
 		}
 
@@ -385,6 +409,9 @@ export async function getTokenCount(text, model) {
 		return Array.isArray(tokens) ? tokens.length : (tokens.count || Object.keys(tokens).length);
 	} catch (error) {
 		console.warn("Upstream tokenizer error:", error);
+		// Cache this endpoint as failed for future requests
+		const endpoint = getSelectedModelEndpoint().replace('/v1', '');
+		failedTokenizeEndpoints.add(endpoint);
 		// Fallback to JS estimate
 		return tokenizer.count(text).totalTokens;
 	}
diff --git a/src/translation.js b/src/translation.js
index f07227d..06f72a1 100644
--- a/src/translation.js
+++ b/src/translation.js
@@ -46,10 +46,14 @@ async function splitHTMLIntoChunks(htmlContent, maxTokensPerChunk, model) {
 	
 	for (const element of elements) {
 		const elementHTML = element.outerHTML;
-		const elementTokens = await countTokens(elementHTML, model);
+		const elementTokensResult = await countTokens(elementHTML, model);
+		const elementTokens = typeof elementTokensResult === 'object' ? elementTokensResult.totalTokens || elementTokensResult.count || 0 : elementTokensResult;
+		
+		console.log(`Element tokens: ${elementTokens}, current total: ${currentTokens}, limit: ${maxTokensPerChunk}`);
 		
 		// If adding this element would exceed the limit, start a new chunk
 		if (currentTokens + elementTokens > maxTokensPerChunk && currentChunk) {
+			console.log(`Starting new chunk at ${currentTokens} tokens`);
 			chunks.push(currentChunk);
 			currentChunk = elementHTML;
 			currentTokens = elementTokens;