version 0.0.1

new file:   .gitignore
	new file:   duck_duck_go_hermes_unturf.py

.gitignore

@@ -0,0 +1 @@
+data/

duck_duck_go_hermes_unturf.py

@@ -0,0 +1,207 @@
+import os
+import requests
+import random
+import time
+import re
+from urllib.parse import quote_plus
+from datetime import datetime, timezone
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from sqlalchemy import (
+    create_engine,
+    Column,
+    Integer,
+    String,
+    Text,
+    DateTime,
+    UniqueConstraint,
+)
+from sqlalchemy.orm import declarative_base, sessionmaker
+from openai import OpenAI
+from bs4 import BeautifulSoup
+
+# Default Hermes endpoints
+DEFAULT_HERMES_ENDPOINTS = [
+    "https://hermes.ai.unturf.com/v1",
+    "https://hermes2.ai.unturf.com/v1",
+]
+# Max characters to send to extraction to avoid context overflow
+MAX_HTML_INPUT_CHARS = 50000
+
+Base = declarative_base()
+
+class Article(Base):
+    __tablename__ = 'articles'
+    id = Column(Integer, primary_key=True)
+    url = Column(String, unique=True, nullable=False)
+    title = Column(String, nullable=False)
+    raw_html = Column(Text, nullable=False)
+    extracted_content = Column(Text, nullable=False)
+    summary = Column(Text, nullable=False)
+    fetched_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
+    __table_args__ = (UniqueConstraint('url', name='_url_uc'),)
+
+class SQLAlchemyDuckDuckGoCrawler:
+    def __init__(
+        self,
+        api_key,
+        model,
+        db_path="data/articles.db",
+        hermes_endpoints=None,
+    ):
+        # Setup Hermes clients
+        self.hermes_endpoints = hermes_endpoints or DEFAULT_HERMES_ENDPOINTS
+        self.clients = [OpenAI(base_url=ep, api_key=api_key) for ep in self.hermes_endpoints]
+        self.api_key = api_key
+        self.model = model
+        self.db_path = db_path
+
+        # HTTP session for DuckDuckGo and page fetches
+        self.session = requests.Session()
+        self.session.headers.update({
+            "User-Agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/91.0.4472.124 Safari/537.36"
+            )
+        })
+
+        # Initialize DB
+        self.engine = create_engine(f"sqlite:///{self.db_path}", echo=False, future=True)
+        Base.metadata.create_all(self.engine)
+        SessionLocal = sessionmaker(bind=self.engine, autoflush=False, autocommit=False)
+        self.db = SessionLocal()
+
+    def search_duckduckgo(self, query, max_results=10):
+        encoded = quote_plus(query)
+        url = f"https://html.duckduckgo.com/html/?q={encoded}"
+        resp = self.session.get(url, timeout=10)
+        resp.raise_for_status()
+        soup = BeautifulSoup(resp.text, 'html.parser')
+        results = []
+        for link in soup.select('.result__title a'):
+            if len(results) >= max_results:
+                break
+            results.append((link.get_text(strip=True), link.get('href')))
+        return results
+
+    def fetch_webpage(self, url):
+        try:
+            time.sleep(random.uniform(1, 3))
+            r = self.session.get(url, timeout=15)
+            r.raise_for_status()
+            return r.text
+        except Exception as e:
+            print(f"Error fetching {url}: {e}")
+            return None
+
+    def _fanout_call(self, messages, max_tokens):
+        def call_client(client):
+            try:
+                response = client.chat.completions.create(
+                    model=self.model,
+                    messages=messages,
+                    temperature=0,
+                    max_tokens=max_tokens
+                )
+                return response.choices[0].message.content
+            except Exception:
+                return None
+
+        with ThreadPoolExecutor(max_workers=len(self.clients)) as executor:
+            futures = [executor.submit(call_client, c) for c in self.clients]
+            for future in as_completed(futures):
+                content = future.result()
+                if content is not None:
+                    for f in futures:
+                        if not f.done():
+                            f.cancel()
+                    return content
+        raise RuntimeError("All Hermes endpoints failed.")
+
+    def extract_with_hermes(self, html):
+        text = BeautifulSoup(html, 'html.parser').get_text(separator='\n')
+        if len(text) > MAX_HTML_INPUT_CHARS:
+            text = text[:MAX_HTML_INPUT_CHARS]
+        messages = [
+            {"role": "system", "content": (
+                "Extract the main article content, preserving formatting and structure, "
+                "excluding ads and navigation. Return only the full text."
+            )},
+            {"role": "user", "content": text}
+        ]
+        try:
+            return self._fanout_call(messages, max_tokens=10000)
+        except RuntimeError:
+            shortened = text[:MAX_HTML_INPUT_CHARS//2]
+            messages[1]['content'] = shortened
+            return self._fanout_call(messages, max_tokens=5000)
+
+    def summarize_with_hermes(self, content):
+        messages = [
+            {"role": "system", "content": (
+                "Summarize the following article concisely as bullet points, keeping all factual details and structure. Avoid hallucination."
+            )},
+            {"role": "user", "content": content}
+        ]
+        return self._fanout_call(messages, max_tokens=10000)
+
+    def cache_article(self, url, title, html, extracted, summary):
+        art = Article(
+            url=url,
+            title=title,
+            raw_html=html,
+            extracted_content=extracted,
+            summary=summary,
+            fetched_at=datetime.now(timezone.utc)
+        )
+        self.db.add(art)
+        self.db.commit()
+
+    def process_url(self, title_url):
+        title, url = title_url
+        if self.db.query(Article).filter_by(url=url).first():
+            print(f"Already cached: {url}")
+            return
+        print(f"Fetching: {url}")
+        html = self.fetch_webpage(url)
+        if not html:
+            return
+        extracted = self.extract_with_hermes(html)
+        summary = self.summarize_with_hermes(extracted)
+        self.cache_article(url, title, html, extracted, summary)
+
+    def aggregate_and_answer(self, query):
+        summaries = [art.summary for art in self.db.query(Article).all()]
+        combined = "\n\n".join(summaries)
+        messages = [{"role": "user", "content": (
+            f"Based on these article summaries:\n{combined}\n\n"
+            f"Provide a comprehensive answer to: {query}"
+        )}]
+        return self._fanout_call(messages, max_tokens=30000)
+
+    def run(self, query, max_results=10):
+        hits = self.search_duckduckgo(query, max_results)
+        # Parallel fetch/extract/summarize using endpoints count
+        with ThreadPoolExecutor(max_workers=len(self.clients)) as executor:
+            futures = [executor.submit(self.process_url, hit) for hit in hits]
+            for future in as_completed(futures):
+                future.result()
+        print("Generating comprehensive answer...")
+        print(self.aggregate_and_answer(query))
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("query", help="Search and deep-query prompt")
+    parser.add_argument("--api-key", default="dummy-api-key", help="OpenAI API key")
+    parser.add_argument("--model", default="adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", help="Model ID")
+    parser.add_argument("--max-results", type=int, default=10, help="Max search results to process")
+    args = parser.parse_args()
+
+    crawler = SQLAlchemyDuckDuckGoCrawler(
+        api_key=args.api_key,
+        model=args.model
+    )
+    crawler.run(args.query, args.max_results)