From 5b823155a6e1bfc1e843a00cdeadd42d5f2356f2 Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Sat, 26 Apr 2025 10:26:52 -0400 Subject: [PATCH] modified: duck_duck_go_hermes_unturf.py --- duck_duck_go_hermes_unturf.py | 212 ++++++++++++++++++++++------------ 1 file changed, 140 insertions(+), 72 deletions(-) diff --git a/duck_duck_go_hermes_unturf.py b/duck_duck_go_hermes_unturf.py index e80d1e5..23d6b5b 100644 --- a/duck_duck_go_hermes_unturf.py +++ b/duck_duck_go_hermes_unturf.py @@ -34,8 +34,8 @@ logger = logging.getLogger(__name__) # Default Hermes endpoints DEFAULT_HERMES_ENDPOINTS = [ - "https://hermes.ai.unturf.com/v1", - "https://hermes2.ai.unturf.com/v1", + "https://hermes.ai.unturf.com/v1", # 80K tokens + "https://hermes2.ai.unturf.com/v1", # 48K tokens ] # Max characters to send to extraction MAX_HTML_INPUT_CHARS = 50000 @@ -244,77 +244,111 @@ class SQLAlchemyDuckDuckGoCrawler: return None def _fanout_call(self, messages, max_tokens, prefer_hermes=False): - def call_client(client, endpoint): + def estimate_input_tokens(messages): + # More accurate: ~5 chars per word, 4 chars per token + total_chars = sum(len(m["content"]) for m in messages) + return (total_chars // 5 + 1) // 4 + 1 + + def call_client(client, endpoint, messages, max_tokens): try: - adjusted_max_tokens = max_tokens - if endpoint == "https://hermes2.ai.unturf.com/v1": - input_tokens = sum(len(m["content"]) // 4 + 1 for m in messages) - if input_tokens + max_tokens > HERMES2_TOKEN_LIMIT: - adjusted_max_tokens = max( - 1000, HERMES2_TOKEN_LIMIT - input_tokens - ) - logger.info( - f"Adjusted max_tokens to {adjusted_max_tokens} for {endpoint} due to context limit" - ) + input_tokens = estimate_input_tokens(messages) + total_tokens = input_tokens + max_tokens + if ( + endpoint == "https://hermes2.ai.unturf.com/v1" + and total_tokens > HERMES2_TOKEN_LIMIT + ): + logger.warning( + f"Request exceeds {HERMES2_TOKEN_LIMIT} tokens for {endpoint}: {total_tokens}" + ) + return None, endpoint + if ( + endpoint == "https://hermes.ai.unturf.com/v1" + and total_tokens > HERMES_TOKEN_LIMIT + ): + logger.warning( + f"Request exceeds {HERMES_TOKEN_LIMIT} tokens for {endpoint}: {total_tokens}" + ) + return None, endpoint response = client.chat.completions.create( model=self.model, messages=messages, temperature=0, - max_tokens=adjusted_max_tokens, + max_tokens=max_tokens, ) return response.choices[0].message.content, endpoint except Exception as e: logger.error(f"Error calling endpoint {endpoint}: {e}") return None, endpoint - if prefer_hermes: - hermes_endpoint = "https://hermes.ai.unturf.com/v1" - hermes_client = next( - ( - c - for c, e in zip(self.clients, self.hermes_endpoints) - if e == hermes_endpoint - ), - None, + # Prefer hermes.ai.unturf.com/v1 (80K limit) + hermes_endpoint = "https://hermes.ai.unturf.com/v1" + hermes2_endpoint = "https://hermes2.ai.unturf.com/v1" + hermes_client = next( + ( + c + for c, e in zip(self.clients, self.hermes_endpoints) + if e == hermes_endpoint + ), + None, + ) + hermes2_client = next( + ( + c + for c, e in zip(self.clients, self.hermes_endpoints) + if e == hermes2_endpoint + ), + None, + ) + + if prefer_hermes and hermes_client: + logger.info(f"Attempting {hermes_endpoint} for final answer") + result, used_endpoint = call_client( + hermes_client, hermes_endpoint, messages, max_tokens ) - if hermes_client: - logger.info(f"Attempting {hermes_endpoint} for final answer") - result, used_endpoint = call_client(hermes_client, hermes_endpoint) + if result is not None: + logger.info(f"Success using {used_endpoint}") + return result + logger.warning( + f"Fallback: {hermes_endpoint} failed, trying {hermes2_endpoint}" + ) + if hermes2_client: + result, used_endpoint = call_client( + hermes2_client, hermes2_endpoint, messages, max_tokens + ) if result is not None: logger.info(f"Success using {used_endpoint}") return result - logger.warning( - f"Fallback: {hermes_endpoint} failed, trying other endpoints" - ) - other_clients = [ - (c, e) - for c, e in zip(self.clients, self.hermes_endpoints) - if e != hermes_endpoint - ] - with ThreadPoolExecutor(max_workers=len(other_clients)) as executor: - futures = [ - executor.submit(call_client, c, e) for c, e in other_clients - ] - for future in as_completed(futures): - result, used_endpoint = future.result() - if result is not None: - logger.info(f"Success using {used_endpoint}") - return result - else: - logger.warning( - f"Warning: {hermes_endpoint} not in endpoints, using parallel execution" - ) else: - with ThreadPoolExecutor(max_workers=len(self.clients)) as executor: - futures = [ - executor.submit(call_client, c, e) - for c, e in zip(self.clients, self.hermes_endpoints) - ] - for future in as_completed(futures): - result, used_endpoint = future.result() + # Try hermes.ai.unturf.com/v1 first + if hermes_client: + logger.info(f"Attempting {hermes_endpoint}") + result, used_endpoint = call_client( + hermes_client, hermes_endpoint, messages, max_tokens + ) + if result is not None: + logger.info(f"Success using {used_endpoint}") + return result + # Retry with hermes2 if hermes fails or if hermes2 is preferred + if hermes2_client: + logger.info(f"Attempting {hermes2_endpoint}") + result, used_endpoint = call_client( + hermes2_client, hermes2_endpoint, messages, max_tokens + ) + if result is not None: + logger.info(f"Success using {used_endpoint}") + return result + # If hermes2 fails due to context length, retry with hermes + if hermes_client and "maximum context length" in str(result): + logger.info( + f"Retrying with {hermes_endpoint} due to context length error" + ) + result, used_endpoint = call_client( + hermes_client, hermes_endpoint, messages, max_tokens + ) if result is not None: logger.info(f"Success using {used_endpoint}") return result + raise RuntimeError("All Hermes endpoints failed.") def extract_with_hermes(self, html): @@ -497,10 +531,12 @@ class SQLAlchemyDuckDuckGoCrawler: logger.info(f"Retrieved {len(cache_results)} articles from SQLite cache") def estimate_tokens(text): - return len(text) // 4 + 1 + # ~5 chars per word, 4 chars per token + return (len(text) // 5 + 1) // 4 + 1 combined_content = [] total_words = 0 + total_tokens = 0 prompt_template = f"Based on these article excerpts:\n{{}}\n\nProvide a comprehensive answer to: {query}" template_tokens = estimate_tokens(prompt_template.format("")) @@ -509,36 +545,68 @@ class SQLAlchemyDuckDuckGoCrawler: content = result["content"] word_count = len(content.split()) content_tokens = estimate_tokens(content) - if total_words + word_count + template_tokens <= CONTEXT_WINDOW_WORDS: + endpoint_limit = ( + HERMES2_INPUT_TOKENS + if total_tokens + content_tokens + template_tokens + > HERMES2_INPUT_TOKENS + else HERMES_INPUT_TOKENS + ) + if ( + total_words + word_count <= CONTEXT_WINDOW_WORDS + and total_tokens + content_tokens + template_tokens <= endpoint_limit + ): combined_content.append( f"From {result['url']} (Title: {result['title']}):\n{content}" ) total_words += word_count + total_tokens += content_tokens logger.info( f"Included article {result['url']} (words: {word_count}, tokens: {content_tokens})" ) else: - # Chunk if content exceeds context window - chunks = self._split_into_chunks(content, len(content)) - for chunk in chunks: - chunk_words = len(chunk.split()) - chunk_tokens = estimate_tokens(chunk) - if ( - total_words + chunk_words + template_tokens - <= CONTEXT_WINDOW_WORDS - ): + # Trim or chunk content + if total_tokens + template_tokens < endpoint_limit: + remaining_tokens = endpoint_limit - total_tokens - template_tokens + remaining_words = min(word_count, remaining_tokens * 4 // 5) + trimmed_content = " ".join(content.split()[:remaining_words]) + trimmed_tokens = estimate_tokens(trimmed_content) + if trimmed_tokens <= remaining_tokens: combined_content.append( - f"From {result['url']} (Title: {result['title']}):\n{chunk}" + f"From {result['url']} (Title: {result['title']}):\n{trimmed_content}" ) - total_words += chunk_words + total_words += remaining_words + total_tokens += trimmed_tokens logger.info( - f"Included chunk from {result['url']} (words: {chunk_words}, tokens: {chunk_tokens})" + f"Included trimmed article {result['url']} (words: {remaining_words}, tokens: {trimmed_tokens})" ) else: - logger.info( - f"Skipping chunk from {result['url']} (exceeds context window: {total_words + chunk_words})" - ) - break + # Chunk if trimming still exceeds + chunks = self._split_into_chunks(content, len(content)) + for chunk in chunks: + chunk_words = len(chunk.split()) + chunk_tokens = estimate_tokens(chunk) + if ( + total_words + chunk_words <= CONTEXT_WINDOW_WORDS + and total_tokens + chunk_tokens + template_tokens + <= endpoint_limit + ): + combined_content.append( + f"From {result['url']} (Title: {result['title']}):\n{chunk}" + ) + total_words += chunk_words + total_tokens += chunk_tokens + logger.info( + f"Included chunk from {result['url']} (words: {chunk_words}, tokens: {chunk_tokens})" + ) + else: + logger.info( + f"Skipping chunk from {result['url']} (exceeds limit: {total_tokens + chunk_tokens + template_tokens})" + ) + break + else: + logger.info( + f"Skipping article {result['url']} (exceeds limit: {total_tokens + content_tokens + template_tokens})" + ) if not combined_content and search_results: logger.warning(