diff --git a/apps/web/package.json b/apps/web/package.json index d61372d..3fc22ee 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -31,7 +31,6 @@ "rehype-raw": "^7.0.0", "rehype-sanitize": "^6.0.0", "remark-gfm": "^4.0.1", - "tiktoken": "^1.0.22", "zod": "^3.25.76" }, "devDependencies": { diff --git a/apps/web/src/lib/ai-agent/tool-executor-agent.ts b/apps/web/src/lib/ai-agent/tool-executor-agent.ts index f48da69..2175b5a 100644 --- a/apps/web/src/lib/ai-agent/tool-executor-agent.ts +++ b/apps/web/src/lib/ai-agent/tool-executor-agent.ts @@ -7,7 +7,6 @@ import { openai } from '@ai-sdk/openai'; import type { Tool } from '@tpmjs/db'; import { executePackage } from '@tpmjs/package-executor'; import { type CoreMessage, tool as aiTool, streamText } from 'ai'; -import { encoding_for_model } from 'tiktoken'; import { z } from 'zod'; /** @@ -121,19 +120,12 @@ export function createToolDefinition(tool: Tool) { } /** - * Count tokens in text using tiktoken + * Count tokens in text using character estimation + * Uses rough estimation: ~4 characters per token + * This is used instead of tiktoken to avoid WASM dependency issues in serverless */ -function countTokens(text: string, model = 'gpt-4'): number { - try { - // biome-ignore lint/suspicious/noExplicitAny: tiktoken type compatibility workaround - const encoder = encoding_for_model(model as any); - const tokens = encoder.encode(text); - encoder.free(); - return tokens.length; - } catch { - // Fallback to rough estimation: ~4 characters per token - return Math.ceil(text.length / 4); - } +function countTokens(text: string): number { + return Math.ceil(text.length / 4); } /**