# requirements.txt
# Specifies Python dependencies for unturf_spider.py.
# Install with: pip install -r requirements.txt
# Versions pinned for compatibility with Python 3.13.

# HTTP requests for web crawling and API calls
# for HTTP requests to fetch pages and call APIs (e.g., DuckDuckGo, Hermes).
requests

# HTML parsing for extracting content from web pages
# Used for parsing HTML with BeautifulSoup to extract article text.
beautifulsoup4

# SQL database management for caching articles
# Used for SQLite database operations to store and query articles.
sqlalchemy

# OpenAI client for interacting with Hermes API endpoints
# Used for API calls to Hermes endpoints (hermes.ai.unturf.com/v1, hermes2).
openai

# URL parsing and robots.txt handling
# Used for URL parsing and robots.txt via urllib.robotparser.
urllib3

# Text analysis for TF-IDF similarity search
# Used for TfidfVectorizer and cosine_similarity to rank articles.
scikit-learn

# Numerical computations for text analysis
# Used by scikit-learn for TF-IDF and cosine similarity calculations.
numpy

# SSL certificate verification for secure HTTP requests
# Used to ensure secure HTTPS connections with updated SSL certificates.
certifi