modified: duck_duck_go_hermes_unturf.py
This commit is contained in:
parent
b3f12dffbb
commit
5b823155a6
1 changed files with 140 additions and 72 deletions
|
|
@ -34,8 +34,8 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
# Default Hermes endpoints
|
||||
DEFAULT_HERMES_ENDPOINTS = [
|
||||
"https://hermes.ai.unturf.com/v1",
|
||||
"https://hermes2.ai.unturf.com/v1",
|
||||
"https://hermes.ai.unturf.com/v1", # 80K tokens
|
||||
"https://hermes2.ai.unturf.com/v1", # 48K tokens
|
||||
]
|
||||
# Max characters to send to extraction
|
||||
MAX_HTML_INPUT_CHARS = 50000
|
||||
|
|
@ -244,31 +244,45 @@ class SQLAlchemyDuckDuckGoCrawler:
|
|||
return None
|
||||
|
||||
def _fanout_call(self, messages, max_tokens, prefer_hermes=False):
|
||||
def call_client(client, endpoint):
|
||||
def estimate_input_tokens(messages):
|
||||
# More accurate: ~5 chars per word, 4 chars per token
|
||||
total_chars = sum(len(m["content"]) for m in messages)
|
||||
return (total_chars // 5 + 1) // 4 + 1
|
||||
|
||||
def call_client(client, endpoint, messages, max_tokens):
|
||||
try:
|
||||
adjusted_max_tokens = max_tokens
|
||||
if endpoint == "https://hermes2.ai.unturf.com/v1":
|
||||
input_tokens = sum(len(m["content"]) // 4 + 1 for m in messages)
|
||||
if input_tokens + max_tokens > HERMES2_TOKEN_LIMIT:
|
||||
adjusted_max_tokens = max(
|
||||
1000, HERMES2_TOKEN_LIMIT - input_tokens
|
||||
input_tokens = estimate_input_tokens(messages)
|
||||
total_tokens = input_tokens + max_tokens
|
||||
if (
|
||||
endpoint == "https://hermes2.ai.unturf.com/v1"
|
||||
and total_tokens > HERMES2_TOKEN_LIMIT
|
||||
):
|
||||
logger.warning(
|
||||
f"Request exceeds {HERMES2_TOKEN_LIMIT} tokens for {endpoint}: {total_tokens}"
|
||||
)
|
||||
logger.info(
|
||||
f"Adjusted max_tokens to {adjusted_max_tokens} for {endpoint} due to context limit"
|
||||
return None, endpoint
|
||||
if (
|
||||
endpoint == "https://hermes.ai.unturf.com/v1"
|
||||
and total_tokens > HERMES_TOKEN_LIMIT
|
||||
):
|
||||
logger.warning(
|
||||
f"Request exceeds {HERMES_TOKEN_LIMIT} tokens for {endpoint}: {total_tokens}"
|
||||
)
|
||||
return None, endpoint
|
||||
response = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=messages,
|
||||
temperature=0,
|
||||
max_tokens=adjusted_max_tokens,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
return response.choices[0].message.content, endpoint
|
||||
except Exception as e:
|
||||
logger.error(f"Error calling endpoint {endpoint}: {e}")
|
||||
return None, endpoint
|
||||
|
||||
if prefer_hermes:
|
||||
# Prefer hermes.ai.unturf.com/v1 (80K limit)
|
||||
hermes_endpoint = "https://hermes.ai.unturf.com/v1"
|
||||
hermes2_endpoint = "https://hermes2.ai.unturf.com/v1"
|
||||
hermes_client = next(
|
||||
(
|
||||
c
|
||||
|
|
@ -277,44 +291,64 @@ class SQLAlchemyDuckDuckGoCrawler:
|
|||
),
|
||||
None,
|
||||
)
|
||||
if hermes_client:
|
||||
hermes2_client = next(
|
||||
(
|
||||
c
|
||||
for c, e in zip(self.clients, self.hermes_endpoints)
|
||||
if e == hermes2_endpoint
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
if prefer_hermes and hermes_client:
|
||||
logger.info(f"Attempting {hermes_endpoint} for final answer")
|
||||
result, used_endpoint = call_client(hermes_client, hermes_endpoint)
|
||||
result, used_endpoint = call_client(
|
||||
hermes_client, hermes_endpoint, messages, max_tokens
|
||||
)
|
||||
if result is not None:
|
||||
logger.info(f"Success using {used_endpoint}")
|
||||
return result
|
||||
logger.warning(
|
||||
f"Fallback: {hermes_endpoint} failed, trying other endpoints"
|
||||
f"Fallback: {hermes_endpoint} failed, trying {hermes2_endpoint}"
|
||||
)
|
||||
if hermes2_client:
|
||||
result, used_endpoint = call_client(
|
||||
hermes2_client, hermes2_endpoint, messages, max_tokens
|
||||
)
|
||||
other_clients = [
|
||||
(c, e)
|
||||
for c, e in zip(self.clients, self.hermes_endpoints)
|
||||
if e != hermes_endpoint
|
||||
]
|
||||
with ThreadPoolExecutor(max_workers=len(other_clients)) as executor:
|
||||
futures = [
|
||||
executor.submit(call_client, c, e) for c, e in other_clients
|
||||
]
|
||||
for future in as_completed(futures):
|
||||
result, used_endpoint = future.result()
|
||||
if result is not None:
|
||||
logger.info(f"Success using {used_endpoint}")
|
||||
return result
|
||||
else:
|
||||
logger.warning(
|
||||
f"Warning: {hermes_endpoint} not in endpoints, using parallel execution"
|
||||
# Try hermes.ai.unturf.com/v1 first
|
||||
if hermes_client:
|
||||
logger.info(f"Attempting {hermes_endpoint}")
|
||||
result, used_endpoint = call_client(
|
||||
hermes_client, hermes_endpoint, messages, max_tokens
|
||||
)
|
||||
else:
|
||||
with ThreadPoolExecutor(max_workers=len(self.clients)) as executor:
|
||||
futures = [
|
||||
executor.submit(call_client, c, e)
|
||||
for c, e in zip(self.clients, self.hermes_endpoints)
|
||||
]
|
||||
for future in as_completed(futures):
|
||||
result, used_endpoint = future.result()
|
||||
if result is not None:
|
||||
logger.info(f"Success using {used_endpoint}")
|
||||
return result
|
||||
# Retry with hermes2 if hermes fails or if hermes2 is preferred
|
||||
if hermes2_client:
|
||||
logger.info(f"Attempting {hermes2_endpoint}")
|
||||
result, used_endpoint = call_client(
|
||||
hermes2_client, hermes2_endpoint, messages, max_tokens
|
||||
)
|
||||
if result is not None:
|
||||
logger.info(f"Success using {used_endpoint}")
|
||||
return result
|
||||
# If hermes2 fails due to context length, retry with hermes
|
||||
if hermes_client and "maximum context length" in str(result):
|
||||
logger.info(
|
||||
f"Retrying with {hermes_endpoint} due to context length error"
|
||||
)
|
||||
result, used_endpoint = call_client(
|
||||
hermes_client, hermes_endpoint, messages, max_tokens
|
||||
)
|
||||
if result is not None:
|
||||
logger.info(f"Success using {used_endpoint}")
|
||||
return result
|
||||
|
||||
raise RuntimeError("All Hermes endpoints failed.")
|
||||
|
||||
def extract_with_hermes(self, html):
|
||||
|
|
@ -497,10 +531,12 @@ class SQLAlchemyDuckDuckGoCrawler:
|
|||
logger.info(f"Retrieved {len(cache_results)} articles from SQLite cache")
|
||||
|
||||
def estimate_tokens(text):
|
||||
return len(text) // 4 + 1
|
||||
# ~5 chars per word, 4 chars per token
|
||||
return (len(text) // 5 + 1) // 4 + 1
|
||||
|
||||
combined_content = []
|
||||
total_words = 0
|
||||
total_tokens = 0
|
||||
prompt_template = f"Based on these article excerpts:\n{{}}\n\nProvide a comprehensive answer to: {query}"
|
||||
template_tokens = estimate_tokens(prompt_template.format(""))
|
||||
|
||||
|
|
@ -509,36 +545,68 @@ class SQLAlchemyDuckDuckGoCrawler:
|
|||
content = result["content"]
|
||||
word_count = len(content.split())
|
||||
content_tokens = estimate_tokens(content)
|
||||
if total_words + word_count + template_tokens <= CONTEXT_WINDOW_WORDS:
|
||||
endpoint_limit = (
|
||||
HERMES2_INPUT_TOKENS
|
||||
if total_tokens + content_tokens + template_tokens
|
||||
> HERMES2_INPUT_TOKENS
|
||||
else HERMES_INPUT_TOKENS
|
||||
)
|
||||
if (
|
||||
total_words + word_count <= CONTEXT_WINDOW_WORDS
|
||||
and total_tokens + content_tokens + template_tokens <= endpoint_limit
|
||||
):
|
||||
combined_content.append(
|
||||
f"From {result['url']} (Title: {result['title']}):\n{content}"
|
||||
)
|
||||
total_words += word_count
|
||||
total_tokens += content_tokens
|
||||
logger.info(
|
||||
f"Included article {result['url']} (words: {word_count}, tokens: {content_tokens})"
|
||||
)
|
||||
else:
|
||||
# Chunk if content exceeds context window
|
||||
# Trim or chunk content
|
||||
if total_tokens + template_tokens < endpoint_limit:
|
||||
remaining_tokens = endpoint_limit - total_tokens - template_tokens
|
||||
remaining_words = min(word_count, remaining_tokens * 4 // 5)
|
||||
trimmed_content = " ".join(content.split()[:remaining_words])
|
||||
trimmed_tokens = estimate_tokens(trimmed_content)
|
||||
if trimmed_tokens <= remaining_tokens:
|
||||
combined_content.append(
|
||||
f"From {result['url']} (Title: {result['title']}):\n{trimmed_content}"
|
||||
)
|
||||
total_words += remaining_words
|
||||
total_tokens += trimmed_tokens
|
||||
logger.info(
|
||||
f"Included trimmed article {result['url']} (words: {remaining_words}, tokens: {trimmed_tokens})"
|
||||
)
|
||||
else:
|
||||
# Chunk if trimming still exceeds
|
||||
chunks = self._split_into_chunks(content, len(content))
|
||||
for chunk in chunks:
|
||||
chunk_words = len(chunk.split())
|
||||
chunk_tokens = estimate_tokens(chunk)
|
||||
if (
|
||||
total_words + chunk_words + template_tokens
|
||||
<= CONTEXT_WINDOW_WORDS
|
||||
total_words + chunk_words <= CONTEXT_WINDOW_WORDS
|
||||
and total_tokens + chunk_tokens + template_tokens
|
||||
<= endpoint_limit
|
||||
):
|
||||
combined_content.append(
|
||||
f"From {result['url']} (Title: {result['title']}):\n{chunk}"
|
||||
)
|
||||
total_words += chunk_words
|
||||
total_tokens += chunk_tokens
|
||||
logger.info(
|
||||
f"Included chunk from {result['url']} (words: {chunk_words}, tokens: {chunk_tokens})"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"Skipping chunk from {result['url']} (exceeds context window: {total_words + chunk_words})"
|
||||
f"Skipping chunk from {result['url']} (exceeds limit: {total_tokens + chunk_tokens + template_tokens})"
|
||||
)
|
||||
break
|
||||
else:
|
||||
logger.info(
|
||||
f"Skipping article {result['url']} (exceeds limit: {total_tokens + content_tokens + template_tokens})"
|
||||
)
|
||||
|
||||
if not combined_content and search_results:
|
||||
logger.warning(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue