commit 923c4a4f9e3d1bd85658f8a63b49bf7af44f7924 Author: Russell Ballestrini Date: Sat Apr 19 13:20:58 2025 -0400 version 0.0.1 new file: .gitignore new file: duck_duck_go_hermes_unturf.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8fce603 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +data/ diff --git a/duck_duck_go_hermes_unturf.py b/duck_duck_go_hermes_unturf.py new file mode 100644 index 0000000..c400737 --- /dev/null +++ b/duck_duck_go_hermes_unturf.py @@ -0,0 +1,207 @@ +import os +import requests +import random +import time +import re +from urllib.parse import quote_plus +from datetime import datetime, timezone +from concurrent.futures import ThreadPoolExecutor, as_completed + +from sqlalchemy import ( + create_engine, + Column, + Integer, + String, + Text, + DateTime, + UniqueConstraint, +) +from sqlalchemy.orm import declarative_base, sessionmaker +from openai import OpenAI +from bs4 import BeautifulSoup + +# Default Hermes endpoints +DEFAULT_HERMES_ENDPOINTS = [ + "https://hermes.ai.unturf.com/v1", + "https://hermes2.ai.unturf.com/v1", +] +# Max characters to send to extraction to avoid context overflow +MAX_HTML_INPUT_CHARS = 50000 + +Base = declarative_base() + +class Article(Base): + __tablename__ = 'articles' + id = Column(Integer, primary_key=True) + url = Column(String, unique=True, nullable=False) + title = Column(String, nullable=False) + raw_html = Column(Text, nullable=False) + extracted_content = Column(Text, nullable=False) + summary = Column(Text, nullable=False) + fetched_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + __table_args__ = (UniqueConstraint('url', name='_url_uc'),) + +class SQLAlchemyDuckDuckGoCrawler: + def __init__( + self, + api_key, + model, + db_path="data/articles.db", + hermes_endpoints=None, + ): + # Setup Hermes clients + self.hermes_endpoints = hermes_endpoints or DEFAULT_HERMES_ENDPOINTS + self.clients = [OpenAI(base_url=ep, api_key=api_key) for ep in self.hermes_endpoints] + self.api_key = api_key + self.model = model + self.db_path = db_path + + # HTTP session for DuckDuckGo and page fetches + self.session = requests.Session() + self.session.headers.update({ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/91.0.4472.124 Safari/537.36" + ) + }) + + # Initialize DB + self.engine = create_engine(f"sqlite:///{self.db_path}", echo=False, future=True) + Base.metadata.create_all(self.engine) + SessionLocal = sessionmaker(bind=self.engine, autoflush=False, autocommit=False) + self.db = SessionLocal() + + def search_duckduckgo(self, query, max_results=10): + encoded = quote_plus(query) + url = f"https://html.duckduckgo.com/html/?q={encoded}" + resp = self.session.get(url, timeout=10) + resp.raise_for_status() + soup = BeautifulSoup(resp.text, 'html.parser') + results = [] + for link in soup.select('.result__title a'): + if len(results) >= max_results: + break + results.append((link.get_text(strip=True), link.get('href'))) + return results + + def fetch_webpage(self, url): + try: + time.sleep(random.uniform(1, 3)) + r = self.session.get(url, timeout=15) + r.raise_for_status() + return r.text + except Exception as e: + print(f"Error fetching {url}: {e}") + return None + + def _fanout_call(self, messages, max_tokens): + def call_client(client): + try: + response = client.chat.completions.create( + model=self.model, + messages=messages, + temperature=0, + max_tokens=max_tokens + ) + return response.choices[0].message.content + except Exception: + return None + + with ThreadPoolExecutor(max_workers=len(self.clients)) as executor: + futures = [executor.submit(call_client, c) for c in self.clients] + for future in as_completed(futures): + content = future.result() + if content is not None: + for f in futures: + if not f.done(): + f.cancel() + return content + raise RuntimeError("All Hermes endpoints failed.") + + def extract_with_hermes(self, html): + text = BeautifulSoup(html, 'html.parser').get_text(separator='\n') + if len(text) > MAX_HTML_INPUT_CHARS: + text = text[:MAX_HTML_INPUT_CHARS] + messages = [ + {"role": "system", "content": ( + "Extract the main article content, preserving formatting and structure, " + "excluding ads and navigation. Return only the full text." + )}, + {"role": "user", "content": text} + ] + try: + return self._fanout_call(messages, max_tokens=10000) + except RuntimeError: + shortened = text[:MAX_HTML_INPUT_CHARS//2] + messages[1]['content'] = shortened + return self._fanout_call(messages, max_tokens=5000) + + def summarize_with_hermes(self, content): + messages = [ + {"role": "system", "content": ( + "Summarize the following article concisely as bullet points, keeping all factual details and structure. Avoid hallucination." + )}, + {"role": "user", "content": content} + ] + return self._fanout_call(messages, max_tokens=10000) + + def cache_article(self, url, title, html, extracted, summary): + art = Article( + url=url, + title=title, + raw_html=html, + extracted_content=extracted, + summary=summary, + fetched_at=datetime.now(timezone.utc) + ) + self.db.add(art) + self.db.commit() + + def process_url(self, title_url): + title, url = title_url + if self.db.query(Article).filter_by(url=url).first(): + print(f"Already cached: {url}") + return + print(f"Fetching: {url}") + html = self.fetch_webpage(url) + if not html: + return + extracted = self.extract_with_hermes(html) + summary = self.summarize_with_hermes(extracted) + self.cache_article(url, title, html, extracted, summary) + + def aggregate_and_answer(self, query): + summaries = [art.summary for art in self.db.query(Article).all()] + combined = "\n\n".join(summaries) + messages = [{"role": "user", "content": ( + f"Based on these article summaries:\n{combined}\n\n" + f"Provide a comprehensive answer to: {query}" + )}] + return self._fanout_call(messages, max_tokens=30000) + + def run(self, query, max_results=10): + hits = self.search_duckduckgo(query, max_results) + # Parallel fetch/extract/summarize using endpoints count + with ThreadPoolExecutor(max_workers=len(self.clients)) as executor: + futures = [executor.submit(self.process_url, hit) for hit in hits] + for future in as_completed(futures): + future.result() + print("Generating comprehensive answer...") + print(self.aggregate_and_answer(query)) + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("query", help="Search and deep-query prompt") + parser.add_argument("--api-key", default="dummy-api-key", help="OpenAI API key") + parser.add_argument("--model", default="adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", help="Model ID") + parser.add_argument("--max-results", type=int, default=10, help="Max search results to process") + args = parser.parse_args() + + crawler = SQLAlchemyDuckDuckGoCrawler( + api_key=args.api_key, + model=args.model + ) + crawler.run(args.query, args.max_results)