engineering/spider.unturf.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

modified: duck_duck_go_hermes_unturf.py

Browse source
This commit is contained in:
Russell Ballestrini 2025-04-26 10:26:52 -04:00
parent b3f12dffbb
commit 5b823155a6
1 changed files with 140 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

212
duck_duck_go_hermes_unturf.py
View file

@ -34,8 +34,8 @@ logger = logging.getLogger(__name__)
# Default Hermes endpoints # Default Hermes endpoints
DEFAULT_HERMES_ENDPOINTS = [ DEFAULT_HERMES_ENDPOINTS = [
"https://hermes.ai.unturf.com/v1", "https://hermes.ai.unturf.com/v1", # 80K tokens
"https://hermes2.ai.unturf.com/v1", "https://hermes2.ai.unturf.com/v1", # 48K tokens
] ]
# Max characters to send to extraction # Max characters to send to extraction
MAX_HTML_INPUT_CHARS = 50000 MAX_HTML_INPUT_CHARS = 50000
@ -244,77 +244,111 @@ class SQLAlchemyDuckDuckGoCrawler:
return None return None
def _fanout_call(self, messages, max_tokens, prefer_hermes=False): def _fanout_call(self, messages, max_tokens, prefer_hermes=False):
def call_client(client, endpoint): def estimate_input_tokens(messages):
# More accurate: ~5 chars per word, 4 chars per token
total_chars = sum(len(m["content"]) for m in messages)
return (total_chars // 5 + 1) // 4 + 1
def call_client(client, endpoint, messages, max_tokens):
try: try:
adjusted_max_tokens = max_tokens input_tokens = estimate_input_tokens(messages)
if endpoint == "https://hermes2.ai.unturf.com/v1": total_tokens = input_tokens + max_tokens
input_tokens = sum(len(m["content"]) // 4 + 1 for m in messages) if (
if input_tokens + max_tokens > HERMES2_TOKEN_LIMIT: endpoint == "https://hermes2.ai.unturf.com/v1"
adjusted_max_tokens = max( and total_tokens > HERMES2_TOKEN_LIMIT
1000, HERMES2_TOKEN_LIMIT - input_tokens ):
) logger.warning(
logger.info( f"Request exceeds {HERMES2_TOKEN_LIMIT} tokens for {endpoint}: {total_tokens}"
f"Adjusted max_tokens to {adjusted_max_tokens} for {endpoint} due to context limit" )
) return None, endpoint
if (
endpoint == "https://hermes.ai.unturf.com/v1"
and total_tokens > HERMES_TOKEN_LIMIT
):
logger.warning(
f"Request exceeds {HERMES_TOKEN_LIMIT} tokens for {endpoint}: {total_tokens}"
)
return None, endpoint
response = client.chat.completions.create( response = client.chat.completions.create(
model=self.model, model=self.model,
messages=messages, messages=messages,
temperature=0, temperature=0,
max_tokens=adjusted_max_tokens, max_tokens=max_tokens,
) )
return response.choices[0].message.content, endpoint return response.choices[0].message.content, endpoint
except Exception as e: except Exception as e:
logger.error(f"Error calling endpoint {endpoint}: {e}") logger.error(f"Error calling endpoint {endpoint}: {e}")
return None, endpoint return None, endpoint
if prefer_hermes: # Prefer hermes.ai.unturf.com/v1 (80K limit)
hermes_endpoint = "https://hermes.ai.unturf.com/v1" hermes_endpoint = "https://hermes.ai.unturf.com/v1"
hermes_client = next( hermes2_endpoint = "https://hermes2.ai.unturf.com/v1"
( hermes_client = next(
c (
for c, e in zip(self.clients, self.hermes_endpoints) c
if e == hermes_endpoint for c, e in zip(self.clients, self.hermes_endpoints)
), if e == hermes_endpoint
None, ),
None,
)
hermes2_client = next(
(
c
for c, e in zip(self.clients, self.hermes_endpoints)
if e == hermes2_endpoint
),
None,
)
if prefer_hermes and hermes_client:
logger.info(f"Attempting {hermes_endpoint} for final answer")
result, used_endpoint = call_client(
hermes_client, hermes_endpoint, messages, max_tokens
) )
if hermes_client: if result is not None:
logger.info(f"Attempting {hermes_endpoint} for final answer") logger.info(f"Success using {used_endpoint}")
result, used_endpoint = call_client(hermes_client, hermes_endpoint) return result
logger.warning(
f"Fallback: {hermes_endpoint} failed, trying {hermes2_endpoint}"
)
if hermes2_client:
result, used_endpoint = call_client(
hermes2_client, hermes2_endpoint, messages, max_tokens
)
if result is not None: if result is not None:
logger.info(f"Success using {used_endpoint}") logger.info(f"Success using {used_endpoint}")
return result return result
logger.warning(
f"Fallback: {hermes_endpoint} failed, trying other endpoints"
)
other_clients = [
(c, e)
for c, e in zip(self.clients, self.hermes_endpoints)
if e != hermes_endpoint
]
with ThreadPoolExecutor(max_workers=len(other_clients)) as executor:
futures = [
executor.submit(call_client, c, e) for c, e in other_clients
]
for future in as_completed(futures):
result, used_endpoint = future.result()
if result is not None:
logger.info(f"Success using {used_endpoint}")
return result
else:
logger.warning(
f"Warning: {hermes_endpoint} not in endpoints, using parallel execution"
)
else: else:
with ThreadPoolExecutor(max_workers=len(self.clients)) as executor: # Try hermes.ai.unturf.com/v1 first
futures = [ if hermes_client:
executor.submit(call_client, c, e) logger.info(f"Attempting {hermes_endpoint}")
for c, e in zip(self.clients, self.hermes_endpoints) result, used_endpoint = call_client(
] hermes_client, hermes_endpoint, messages, max_tokens
for future in as_completed(futures): )
result, used_endpoint = future.result() if result is not None:
logger.info(f"Success using {used_endpoint}")
return result
# Retry with hermes2 if hermes fails or if hermes2 is preferred
if hermes2_client:
logger.info(f"Attempting {hermes2_endpoint}")
result, used_endpoint = call_client(
hermes2_client, hermes2_endpoint, messages, max_tokens
)
if result is not None:
logger.info(f"Success using {used_endpoint}")
return result
# If hermes2 fails due to context length, retry with hermes
if hermes_client and "maximum context length" in str(result):
logger.info(
f"Retrying with {hermes_endpoint} due to context length error"
)
result, used_endpoint = call_client(
hermes_client, hermes_endpoint, messages, max_tokens
)
if result is not None: if result is not None:
logger.info(f"Success using {used_endpoint}") logger.info(f"Success using {used_endpoint}")
return result return result
raise RuntimeError("All Hermes endpoints failed.") raise RuntimeError("All Hermes endpoints failed.")
def extract_with_hermes(self, html): def extract_with_hermes(self, html):
@ -497,10 +531,12 @@ class SQLAlchemyDuckDuckGoCrawler:
logger.info(f"Retrieved {len(cache_results)} articles from SQLite cache") logger.info(f"Retrieved {len(cache_results)} articles from SQLite cache")
def estimate_tokens(text): def estimate_tokens(text):
return len(text) // 4 + 1 # ~5 chars per word, 4 chars per token
return (len(text) // 5 + 1) // 4 + 1
combined_content = [] combined_content = []
total_words = 0 total_words = 0
total_tokens = 0
prompt_template = f"Based on these article excerpts:\n{{}}\n\nProvide a comprehensive answer to: {query}" prompt_template = f"Based on these article excerpts:\n{{}}\n\nProvide a comprehensive answer to: {query}"
template_tokens = estimate_tokens(prompt_template.format("")) template_tokens = estimate_tokens(prompt_template.format(""))
@ -509,36 +545,68 @@ class SQLAlchemyDuckDuckGoCrawler:
content = result["content"] content = result["content"]
word_count = len(content.split()) word_count = len(content.split())
content_tokens = estimate_tokens(content) content_tokens = estimate_tokens(content)
if total_words + word_count + template_tokens <= CONTEXT_WINDOW_WORDS: endpoint_limit = (
HERMES2_INPUT_TOKENS
if total_tokens + content_tokens + template_tokens
> HERMES2_INPUT_TOKENS
else HERMES_INPUT_TOKENS
)
if (
total_words + word_count <= CONTEXT_WINDOW_WORDS
and total_tokens + content_tokens + template_tokens <= endpoint_limit
):
combined_content.append( combined_content.append(
f"From {result['url']} (Title: {result['title']}):\n{content}" f"From {result['url']} (Title: {result['title']}):\n{content}"
) )
total_words += word_count total_words += word_count
total_tokens += content_tokens
logger.info( logger.info(
f"Included article {result['url']} (words: {word_count}, tokens: {content_tokens})" f"Included article {result['url']} (words: {word_count}, tokens: {content_tokens})"
) )
else: else:
# Chunk if content exceeds context window # Trim or chunk content
chunks = self._split_into_chunks(content, len(content)) if total_tokens + template_tokens < endpoint_limit:
for chunk in chunks: remaining_tokens = endpoint_limit - total_tokens - template_tokens
chunk_words = len(chunk.split()) remaining_words = min(word_count, remaining_tokens * 4 // 5)
chunk_tokens = estimate_tokens(chunk) trimmed_content = " ".join(content.split()[:remaining_words])
if ( trimmed_tokens = estimate_tokens(trimmed_content)
total_words + chunk_words + template_tokens if trimmed_tokens <= remaining_tokens:
<= CONTEXT_WINDOW_WORDS
):
combined_content.append( combined_content.append(
f"From {result['url']} (Title: {result['title']}):\n{chunk}" f"From {result['url']} (Title: {result['title']}):\n{trimmed_content}"
) )
total_words += chunk_words total_words += remaining_words
total_tokens += trimmed_tokens
logger.info( logger.info(
f"Included chunk from {result['url']} (words: {chunk_words}, tokens: {chunk_tokens})" f"Included trimmed article {result['url']} (words: {remaining_words}, tokens: {trimmed_tokens})"
) )
else: else:
logger.info( # Chunk if trimming still exceeds
f"Skipping chunk from {result['url']} (exceeds context window: {total_words + chunk_words})" chunks = self._split_into_chunks(content, len(content))
) for chunk in chunks:
break chunk_words = len(chunk.split())
chunk_tokens = estimate_tokens(chunk)
if (
total_words + chunk_words <= CONTEXT_WINDOW_WORDS
and total_tokens + chunk_tokens + template_tokens
<= endpoint_limit
):
combined_content.append(
f"From {result['url']} (Title: {result['title']}):\n{chunk}"
)
total_words += chunk_words
total_tokens += chunk_tokens
logger.info(
f"Included chunk from {result['url']} (words: {chunk_words}, tokens: {chunk_tokens})"
)
else:
logger.info(
f"Skipping chunk from {result['url']} (exceeds limit: {total_tokens + chunk_tokens + template_tokens})"
)
break
else:
logger.info(
f"Skipping article {result['url']} (exceeds limit: {total_tokens + content_tokens + template_tokens})"
)
if not combined_content and search_results: if not combined_content and search_results:
logger.warning( logger.warning(