Fix tiktoken SSL/gevent conflict in tests: use fallback token counting

This commit is contained in:
Russell Ballestrini 2026-01-11 08:42:51 -05:00
parent 4cc7e61c8b
commit 9fb7047866
2 changed files with 68 additions and 3 deletions

View file

@ -1,7 +1,13 @@
from flask_sqlalchemy import SQLAlchemy
from datetime import datetime, timedelta
import tiktoken
import os
try:
import tiktoken
TIKTOKEN_AVAILABLE = True
except Exception:
TIKTOKEN_AVAILABLE = False
tiktoken = None
import json
@ -119,9 +125,16 @@ class Message(db.Model):
if self.token_count is None:
if self.is_base64_image():
self.token_count = 0
elif not TIKTOKEN_AVAILABLE or os.environ.get("TESTING"):
# Fallback: estimate ~4 chars per token when tiktoken unavailable
self.token_count = len(self.content) // 4 + 1
else:
encoding = tiktoken.encoding_for_model("gpt-4")
self.token_count = len(encoding.encode(self.content))
try:
encoding = tiktoken.encoding_for_model("gpt-4")
self.token_count = len(encoding.encode(self.content))
except Exception:
# Fallback on any tiktoken error (network, SSL, etc.)
self.token_count = len(self.content) // 4 + 1
return self.token_count
def is_base64_image(self):