engineering/uncloseai.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

auto-detect and cache endpoints without tokenization support

Browse source 
When upstream tokenization fails (404, errors), cache the endpoint
and automatically use JS tokenizer for future requests. This fixes
chunking for APIs like Groq that don't have tokenize endpoints.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Russell Ballestrini 2025-07-11 21:52:24 -04:00
parent e50b0b8118
commit a8a4359e96
2 changed files with 32 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

27
src/token_estimator.js
View file

@ -301,6 +301,9 @@ export class SimpleBPETokenizer {
// Export singleton instance
export const tokenizer = new SimpleBPETokenizer();
// Cache endpoints that don't support tokenization
const failedTokenizeEndpoints = new Set();
// Use upstream tokenizer API for accurate token counting
export async function countTokensWithUpstream(text, model) {
const { API_KEY } = await import("./config.js");
@ -308,6 +311,13 @@ export async function countTokensWithUpstream(text, model) {
try {
const endpoint = getSelectedModelEndpoint().replace('/v1', '');
// Skip if this endpoint previously failed tokenization
if (failedTokenizeEndpoints.has(endpoint)) {
console.log(`Endpoint ${endpoint} previously failed tokenization, using JS tokenizer`);
return tokenizer.count(text);
}
console.log(`Trying tokenization at: ${endpoint}/tokenize`);
const response = await fetch(`${endpoint}/tokenize`, {
@ -325,6 +335,8 @@ export async function countTokensWithUpstream(text, model) {
if (!response.ok) {
console.warn(`Upstream tokenizer failed (${response.status}) at ${endpoint}/tokenize, falling back to JS tokenizer`);
// Cache this endpoint as failed for future requests
failedTokenizeEndpoints.add(endpoint);
return tokenizer.count(text);
}
@ -350,6 +362,9 @@ export async function countTokensWithUpstream(text, model) {
};
} catch (error) {
console.warn("Error with upstream tokenizer, falling back to JS tokenizer:", error);
// Cache this endpoint as failed for future requests
const endpoint = getSelectedModelEndpoint().replace('/v1', '');
failedTokenizeEndpoints.add(endpoint);
return tokenizer.count(text);
}
}
@ -361,6 +376,13 @@ export async function getTokenCount(text, model) {
try {
const endpoint = getSelectedModelEndpoint().replace('/v1', '');
// Skip if this endpoint previously failed tokenization
if (failedTokenizeEndpoints.has(endpoint)) {
console.log(`Endpoint ${endpoint} previously failed tokenization, using JS tokenizer`);
return tokenizer.count(text).totalTokens;
}
console.log(`Trying tokenization at: ${endpoint}/tokenize`);
const response = await fetch(`${endpoint}/tokenize`, {
@ -377,6 +399,8 @@ export async function getTokenCount(text, model) {
});
if (!response.ok) {
// Cache this endpoint as failed for future requests
failedTokenizeEndpoints.add(endpoint);
throw new Error(`Tokenizer API error: ${response.status} at ${endpoint}/tokenize`);
}
@ -385,6 +409,9 @@ export async function getTokenCount(text, model) {
return Array.isArray(tokens) ? tokens.length : (tokens.count || Object.keys(tokens).length);
} catch (error) {
console.warn("Upstream tokenizer error:", error);
// Cache this endpoint as failed for future requests
const endpoint = getSelectedModelEndpoint().replace('/v1', '');
failedTokenizeEndpoints.add(endpoint);
// Fallback to JS estimate
return tokenizer.count(text).totalTokens;
}

6
src/translation.js
View file

@ -46,10 +46,14 @@ async function splitHTMLIntoChunks(htmlContent, maxTokensPerChunk, model) {
for (const element of elements) {
const elementHTML = element.outerHTML;
const elementTokens = await countTokens(elementHTML, model);
const elementTokensResult = await countTokens(elementHTML, model);
const elementTokens = typeof elementTokensResult === 'object' ? elementTokensResult.totalTokens || elementTokensResult.count || 0 : elementTokensResult;
console.log(`Element tokens: ${elementTokens}, current total: ${currentTokens}, limit: ${maxTokensPerChunk}`);
// If adding this element would exceed the limit, start a new chunk
if (currentTokens + elementTokens > maxTokensPerChunk && currentChunk) {
console.log(`Starting new chunk at ${currentTokens} tokens`);
chunks.push(currentChunk);
currentChunk = elementHTML;
currentTokens = elementTokens;