From a5e709a47017a1591159b4b6ab392bd546546142 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 28 Apr 2026 20:23:36 -0400 Subject: [PATCH] crawler: verbatim lift from agents.ai.unturf.com (off by default) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lifts the async web fetcher into aborist as an opt-in source. The implementation comes directly from ~/git/agents.ai.unturf.com/core (rev 2026-04-28); aborist's adaptations are minimal and documented in aborist/sources/crawler/__init__.py: core/async_web_fetcher.py -> aborist/sources/crawler/async_web_fetcher.py core/web_fetch.py -> aborist/sources/crawler/web_fetch.py Two source-side changes during the lift: 1. Heavy deps (aiohttp, bs4, miniuri) wrapped in try/except so a bare `import aborist.sources.crawler` raises ImportError with the install hint instead of leaking AttributeErrors deep in user code. 2. Chat-bot fetch triggers (`has_fresh_fetch_trigger`, `has_web_fetch_trigger` from agents.ai.unturf.com/core/keywords) replaced with NotImplementedError stubs. Aborist has no chat surface — fetch intent is detected at the application layer. The two test classes that exercised these triggers are `@pytest.mark.skip`'d with the same rationale. Not lifted: web_cache_manager.py — it backs page caching with SQLAlchemy. Aborist has its own content-addressed cache via providence_cache; no need to carry SQLAlchemy as a dep just for crawled-page memoization. Off by default: - `[crawler]` extras section in pyproject.toml carries the heavy deps. `[dev]` pulls them in so the crawler tests can run. - `make test` ignores tests/crawler/ entirely. - `make bootstrap-crawler` installs the extras into the venv. - `make test-crawler` runs only the lifted tests after extras land. Tests: 74 passed, 9 skipped (the chat-bot trigger tests deliberately dropped). Default `make test` stays at 273 passed, 1 skipped. --- Makefile | 15 +- aborist/sources/crawler/__init__.py | 88 + aborist/sources/crawler/async_web_fetcher.py | 2755 ++++++++++++++++++ aborist/sources/crawler/web_fetch.py | 520 ++++ pyproject.toml | 18 + tests/crawler/test_async_web_fetcher.py | 470 +++ tests/crawler/test_web_fetch.py | 343 +++ 7 files changed, 4206 insertions(+), 3 deletions(-) create mode 100644 aborist/sources/crawler/__init__.py create mode 100644 aborist/sources/crawler/async_web_fetcher.py create mode 100644 aborist/sources/crawler/web_fetch.py create mode 100644 tests/crawler/test_async_web_fetcher.py create mode 100644 tests/crawler/test_web_fetch.py diff --git a/Makefile b/Makefile index 0427f4a..58408fb 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ SEARCH_Q ?= computer ingest-grok ingest-grok-media \ ingest-self ingest-git ingest-hg \ verify search stats test docs chain-check chain-check-shards \ - falsify burn clean clean-db clean-data help + falsify burn bootstrap-crawler test-crawler clean clean-db clean-data help all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats @@ -309,8 +309,17 @@ search: bootstrap ## keyword search; override SEARCH_Q (or pass Q=...) stats: bootstrap ## counts: documents, chunks, edges, audit chain $(ABORIST) --db $(DB) stats -test: bootstrap ## run pytest suite - $(VENV)/bin/pytest -q +test: bootstrap ## run pytest suite (excludes opt-in crawler tests) + $(VENV)/bin/pytest -q --ignore=tests/crawler + +# Crawler tests are off-by-default — they hit the network in many cases +# and require the heavy [crawler] extras (aiohttp, bs4, lxml, etc.). +# Bootstrap installs the extras into the existing venv idempotently. +bootstrap-crawler: bootstrap ## install [crawler] extras into the venv + $(PIP) install -e '.[crawler]' + +test-crawler: bootstrap-crawler ## run only the lifted crawler tests + $(VENV)/bin/pytest -q tests/crawler DOT_SRCS := $(wildcard docs/diagrams/*.dot) DOT_PNGS := $(DOT_SRCS:.dot=.png) diff --git a/aborist/sources/crawler/__init__.py b/aborist/sources/crawler/__init__.py new file mode 100644 index 0000000..8928083 --- /dev/null +++ b/aborist/sources/crawler/__init__.py @@ -0,0 +1,88 @@ +"""Crawler — ethical async web fetching with robots.txt compliance. + +Verbatim lift from ``~/git/agents.ai.unturf.com/core/`` (rev as of 2026-04-28): + + core/async_web_fetcher.py -> aborist/sources/crawler/async_web_fetcher.py + core/web_fetch.py -> aborist/sources/crawler/web_fetch.py + +Two source-side adaptations during the lift: + +1. Chat-bot fetch triggers (``has_fresh_fetch_trigger`` / + ``has_web_fetch_trigger`` from ``core.keywords``) were dropped. Aborist + has no chat surface — fetch intent is detected at the application + layer, not from message content. + +2. ``web_cache_manager.py`` (SQLAlchemy-backed page cache) was NOT + lifted. Aborist has its own content-addressed store; cache through + that path instead of carrying SQLAlchemy as a dep. + +**Off by default.** Heavy dependencies (aiohttp, beautifulsoup4, lxml, +html5lib, html2text, miniuri, feedparser, Pillow, cairosvg, pypdf) ship +as the ``[crawler]`` extras. Importing this module without the extras +raises ``ImportError`` with the install hint. The default test suite +(``make test``) does not exercise the crawler; ``make test-crawler`` +runs the lifted tests with the extras installed. + +Public surface — same as the agents repo: + + from aborist.sources.crawler import AsyncWebFetcher, CrawlMode + from aborist.sources.crawler import ( + URIContentType, + extract_all_uris_from_content, + extract_url_from_content, + build_sources_footer, + fetch_and_cache, + ) + +Adopters should pass ``user_agent`` reflecting their deployment and +honor ``robots.txt`` — that's what the underlying parser already does +when given a URL. +""" + +from aborist.sources.crawler.async_web_fetcher import ( + AsyncWebFetcher, + CrawlMode, + FeedItem, + MediaItem, + MediaMetadata, + discover_feeds, + extract_media_from_html, + fetch_robots_txt, + parse_atom, + parse_feed, + parse_rss, + parse_sitemap, +) +from aborist.sources.crawler.web_fetch import ( + URIContentType, + build_sources_footer, + detect_uri_content_type, + extract_all_uris_from_content, + extract_domain, + extract_url_from_content, + fetch_and_cache, + format_progress_message, +) + +__all__ = [ + "AsyncWebFetcher", + "CrawlMode", + "FeedItem", + "MediaItem", + "MediaMetadata", + "URIContentType", + "build_sources_footer", + "detect_uri_content_type", + "discover_feeds", + "extract_all_uris_from_content", + "extract_domain", + "extract_media_from_html", + "extract_url_from_content", + "fetch_and_cache", + "fetch_robots_txt", + "format_progress_message", + "parse_atom", + "parse_feed", + "parse_rss", + "parse_sitemap", +] diff --git a/aborist/sources/crawler/async_web_fetcher.py b/aborist/sources/crawler/async_web_fetcher.py new file mode 100644 index 0000000..fb6b30b --- /dev/null +++ b/aborist/sources/crawler/async_web_fetcher.py @@ -0,0 +1,2755 @@ +#!/usr/bin/env python3 +""" +Async Web Fetcher for Discord Bot +Ethical web scraping with robots.txt compliance, async/await compatible + +Supports multiple crawl modes: +- text: Extract text content (default, current behavior) +- images: Collect images only +- videos: Collect videos only +- media: Collect all media (images + videos + audio) +- all: Uber crawl - text + all media, full domain slurp +""" + +import os +import io +import hashlib +import logging +import asyncio +import re +from dataclasses import dataclass, field +from datetime import datetime, timezone +from enum import Enum +from typing import Optional, Dict, List, Tuple, Set, Any +from urllib.parse import urljoin +from urllib.robotparser import RobotFileParser +import time + +# Heavy crawler dependencies are optional. Install via: +# pip install 'aborist[crawler]' +# Off by default per CLAUDE.md — the core aborist test suite does not +# touch the network and must not import this module unless asked. +try: + import aiohttp + from bs4 import BeautifulSoup + from miniuri import Uri +except ImportError as e: # pragma: no cover + raise ImportError( + "crawler module requires extras: pip install 'aborist[crawler]'" + ) from e + + +class CrawlMode(Enum): + """Crawl modes for different content types.""" + TEXT = "text" # Extract text content (default) + IMAGES = "images" # Collect images only + VIDEOS = "videos" # Collect videos only + MEDIA = "media" # All media (images + videos + audio) + ALL = "all" # Uber crawl: text + all media + + +# File extensions by media type +IMAGE_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.bmp', '.ico', '.tiff', '.tif', '.avif'} +VIDEO_EXTENSIONS = {'.mp4', '.webm', '.mov', '.avi', '.mkv', '.m4v', '.ogv', '.flv', '.wmv'} +AUDIO_EXTENSIONS = {'.mp3', '.wav', '.ogg', '.m4a', '.flac', '.aac', '.wma'} + +# Code file extensions +CODE_EXTENSIONS = { + '.py', '.js', '.ts', '.jsx', '.tsx', '.mjs', '.cjs', # Python, JavaScript, TypeScript + '.rs', '.go', '.rb', '.php', '.pl', '.pm', # Rust, Go, Ruby, PHP, Perl + '.java', '.kt', '.kts', '.scala', '.groovy', # JVM languages + '.c', '.h', '.cpp', '.hpp', '.cc', '.cxx', '.hxx', # C/C++ + '.cs', '.fs', '.fsx', # .NET + '.swift', '.m', '.mm', # Apple + '.lua', '.r', '.R', '.jl', # Lua, R, Julia + '.sh', '.bash', '.zsh', '.fish', '.ps1', # Shell + '.sql', '.graphql', '.gql', # Query languages + '.yaml', '.yml', '.toml', '.json', '.xml', # Config + '.md', '.rst', '.txt', # Docs + '.zig', '.nim', '.d', '.v', # Modern systems langs + '.ex', '.exs', '.erl', '.hrl', # Erlang/Elixir + '.clj', '.cljs', '.cljc', '.edn', # Clojure + '.hs', '.lhs', # Haskell + '.ml', '.mli', '.re', '.rei', # OCaml/ReasonML + '.lisp', '.cl', '.el', '.scm', '.rkt', # Lisps + '.f90', '.f95', '.f03', '.for', # Fortran + '.asm', '.s', # Assembly + '.cob', '.cbl', # COBOL + '.pro', # Prolog + '.tcl', # Tcl + '.dart', # Dart + '.raku', '.p6', # Raku + '.cr', # Crystal + '.vue', '.svelte', # Frontend frameworks + '.tf', '.hcl', # Terraform + '.dockerfile', '.makefile', # Build files +} + +# Font file extensions +FONT_EXTENSIONS = {'.woff', '.woff2', '.ttf', '.otf', '.eot', '.sfnt'} + +# Style file extensions +STYLE_EXTENSIONS = {'.css', '.scss', '.sass', '.less', '.styl'} + +# MIME types by media type +IMAGE_MIME_PREFIXES = ('image/',) +VIDEO_MIME_PREFIXES = ('video/',) +AUDIO_MIME_PREFIXES = ('audio/',) +CODE_MIME_TYPES = { + 'text/x-python', 'application/x-python', 'text/x-python-script', + 'text/javascript', 'application/javascript', 'application/x-javascript', + 'text/typescript', 'application/typescript', + 'text/x-rust', 'text/x-go', 'text/x-ruby', 'application/x-ruby', + 'text/x-java-source', 'text/x-kotlin', 'text/x-scala', + 'text/x-c', 'text/x-c++', 'text/x-csrc', 'text/x-c++src', + 'text/x-csharp', 'text/x-fsharp', + 'text/x-swift', 'text/x-objective-c', + 'text/x-lua', 'text/x-r', 'text/x-julia', + 'text/x-shellscript', 'application/x-sh', 'text/x-bash', + 'application/sql', 'application/graphql', + 'application/json', 'application/xml', 'text/xml', + 'text/yaml', 'application/x-yaml', 'text/x-yaml', + 'text/markdown', 'text/x-markdown', + 'text/plain', # Often used for code +} +FONT_MIME_TYPES = { + 'font/woff', 'font/woff2', 'font/ttf', 'font/otf', 'font/sfnt', + 'application/font-woff', 'application/font-woff2', + 'application/x-font-ttf', 'application/x-font-otf', + 'application/vnd.ms-fontobject', +} +STYLE_MIME_TYPES = {'text/css', 'text/x-scss', 'text/x-sass', 'text/x-less'} + + +@dataclass +class MediaItem: + """Represents a discovered media item.""" + url: str + source_page: str + media_type: str # 'image', 'video', 'audio' + mime_type: Optional[str] = None + md5_hash: Optional[str] = None + file_size: Optional[int] = None + alt_text: Optional[str] = None + title: Optional[str] = None + width: Optional[int] = None + height: Optional[int] = None + discovered_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) + + +class MediaMetadata: + """ + Accumulates metadata from all sources without clobbering. + + "Never clobber, always append" - collects ALL metadata from: + - img.alt, img.title + - a.title, a.text (link text) + - figcaption + - nearby headings + - page title/h1 + + Produces a combined searchable_text for full-text search. + """ + + def __init__(self): + self.titles: List[str] = [] + self.alt_texts: List[str] = [] + self.descriptions: List[str] = [] + self.captions: List[str] = [] + self.headings: List[str] = [] + self.link_texts: List[str] = [] + self.link_titles: List[str] = [] + + def _add_unique(self, lst: List[str], value: str, max_len: int = 500) -> None: + """Add value to list if non-empty and not duplicate.""" + if value and value.strip(): + clean = value.strip()[:max_len] + if clean not in lst: + lst.append(clean) + + def add_img_alt(self, alt: str) -> None: + """Add img alt attribute.""" + self._add_unique(self.alt_texts, alt) + + def add_img_title(self, title: str) -> None: + """Add img title attribute.""" + self._add_unique(self.titles, title) + + def add_link_title(self, title: str) -> None: + """Add attribute.""" + self._add_unique(self.link_titles, title) + + def add_link_text(self, text: str) -> None: + """Add inner text.""" + self._add_unique(self.link_texts, text) + + def add_figcaption(self, caption: str) -> None: + """Add figcaption text.""" + self._add_unique(self.captions, caption) + + def add_heading(self, heading: str) -> None: + """Add nearby heading (h1-h6).""" + self._add_unique(self.headings, heading) + + def add_description(self, desc: str) -> None: + """Add description (og:description, meta description, etc.).""" + self._add_unique(self.descriptions, desc) + + def add_page_title(self, title: str) -> None: + """Add page title.""" + self._add_unique(self.titles, title) + + def get_best_title(self) -> Optional[str]: + """Get best title for display (first non-empty).""" + for lst in [self.titles, self.alt_texts, self.link_titles, + self.captions, self.link_texts, self.headings]: + if lst: + return lst[0] + return None + + def get_best_alt(self) -> Optional[str]: + """Get best alt text for accessibility.""" + if self.alt_texts: + return self.alt_texts[0] + return self.get_best_title() + + def to_searchable_text(self) -> str: + """ + Combine ALL collected metadata into searchable text. + + This enables finding images by ANY associated text: + - "find images of cats" matches img alt="cute cat" + - "find images from blog post about python" matches page content + """ + all_parts = [] + # Dedupe while preserving order + seen = set() + for lst in [self.titles, self.alt_texts, self.descriptions, + self.captions, self.headings, self.link_texts, self.link_titles]: + for item in lst: + if item and item not in seen: + all_parts.append(item) + seen.add(item) + return ' | '.join(all_parts) + + def to_dict(self) -> Dict[str, Any]: + """Export all collected metadata as dict.""" + return { + 'titles': self.titles, + 'alt_texts': self.alt_texts, + 'descriptions': self.descriptions, + 'captions': self.captions, + 'headings': self.headings, + 'link_texts': self.link_texts, + 'link_titles': self.link_titles, + 'searchable_text': self.to_searchable_text(), + 'best_title': self.get_best_title(), + 'best_alt': self.get_best_alt(), + } + + +# PDF text extraction +try: + from pypdf import PdfReader + PDF_SUPPORT = True +except ImportError: + PDF_SUPPORT = False + +logger = logging.getLogger(__name__) + +# Default crawl delay +DEFAULT_CRAWL_DELAY = 2.0 + +# Global last error storage (for error reporting without log parsing) +# Format: {'type': 'robots_txt'|'http_error'|'timeout'|'dns'|'ssl'|'unknown', 'details': str, 'url': str} +LAST_FETCH_ERROR = None + + +def get_last_fetch_error() -> Optional[Dict]: + """Get the last fetch error, if any.""" + return LAST_FETCH_ERROR + + +def strip_uri_fragment(uri_str: str) -> str: + """ + Strip the fragment (#anchor) from a URI using miniuri. + + Args: + uri_str: URI that may contain a fragment + + Returns: + URI without fragment + """ + if not uri_str: + return uri_str + + uri = Uri(uri_str) + # Reconstruct URI without fragment + result = f"{uri.scheme}://{uri.authority}{uri.path or ''}" + if uri.query: + result += f"?{uri.query}" + return result + + +def normalize_link(link) -> Tuple[str, str]: + """ + Extract URI and anchor text from a link, stripping fragments. + + Args: + link: Either a dict with 'url' and optional 'anchor_text', or a string URI + + Returns: + Tuple of (normalized_uri, anchor_text) + """ + if isinstance(link, dict): + uri = strip_uri_fragment(link.get('url', '')) + anchor_text = link.get('anchor_text', '') + else: + uri = strip_uri_fragment(link) + anchor_text = '' + return uri, anchor_text + + +# Alias for backwards compatibility +strip_url_fragment = strip_uri_fragment + + +def extract_text_from_pdf(pdf_bytes: bytes) -> Optional[str]: + """ + Extract text from PDF binary data. + + Args: + pdf_bytes: Raw PDF file bytes + + Returns: + Extracted text or None if extraction fails + """ + if not PDF_SUPPORT: + logger.warning("PDF support not available (pypdf not installed)") + return None + + try: + reader = PdfReader(io.BytesIO(pdf_bytes)) + text_parts = [] + for page in reader.pages: + page_text = page.extract_text() + if page_text: + text_parts.append(page_text) + if text_parts: + return "\n\n".join(text_parts) + return None + except Exception as e: + logger.error(f"Failed to extract text from PDF: {e}") + return None + + +def clear_last_fetch_error(): + """Clear the last fetch error.""" + global LAST_FETCH_ERROR + LAST_FETCH_ERROR = None + + +async def fetch_robots_txt(url: str, user_agent: str = "uncloseai.com/1.42") -> Optional[str]: + """ + Fetch raw robots.txt content for analysis and error reporting. + + Args: + url: Any URL on the domain (robots.txt will be fetched from root) + user_agent: User agent string + + Returns: + Raw robots.txt content or None if not available + """ + parsed = Uri(url) + robots_url = f"{parsed.scheme}://{parsed.hostname}/robots.txt" + + try: + async with aiohttp.ClientSession() as session: + async with session.get( + robots_url, + headers={"User-Agent": user_agent}, + timeout=aiohttp.ClientTimeout(total=5) + ) as response: + if response.status == 200: + return await response.text() + except Exception as e: + logger.warning(f"Could not fetch robots.txt for error analysis: {e}") + + return None + + +def analyze_robots_txt_blocking(robots_txt: str, blocked_url: str, user_agent: str = "uncloseai.com/1.42") -> str: + """ + Analyze robots.txt to explain why a URL is blocked. + + Args: + robots_txt: Raw robots.txt content + blocked_url: The URL that was blocked + user_agent: User agent to check against + + Returns: + Human-readable explanation of the blocking rule + """ + parsed = Uri(blocked_url) + path = parsed.path or "/" + + # Parse robots.txt manually for detailed analysis + lines = robots_txt.strip().split('\n') + current_agent = None + blocking_rule = None + all_rules = [] + + for line in lines: + line = line.strip() + if not line or line.startswith('#'): + continue + + if ':' in line: + key, value = line.split(':', 1) + key = key.strip().lower() + value = value.strip() + + if key == 'user-agent': + current_agent = value + elif key == 'disallow' and current_agent: + # Check if this rule applies to us + agent_matches = ( + current_agent == '*' or + user_agent.lower().startswith(current_agent.lower()) or + current_agent.lower() in user_agent.lower() + ) + if agent_matches: + all_rules.append(f"Disallow: {value}") + # Check if this rule blocks our path + if value and path.startswith(value): + blocking_rule = f"Disallow: {value}" + + if blocking_rule: + result = f"**Blocked by rule:** `{blocking_rule}`\n" + result += f"**Path requested:** `{path}`\n" + result += f"**User-Agent:** `{user_agent}`\n\n" + + # Show relevant rules + if all_rules: + result += "**Applicable rules:**\n" + for rule in all_rules[:10]: # Limit to first 10 + result += f" • `{rule}`\n" + + return result + elif all_rules: + return f"Blocked by robots.txt (exact rule unclear). Rules found:\n" + "\n".join(f" • `{r}`" for r in all_rules[:10]) + else: + return "Blocked by robots.txt (no matching disallow rule found - may be a catch-all)" + + +def get_media_type_from_extension(url: str) -> Optional[str]: + """ + Determine media type from URL extension. + + Returns: + 'image', 'video', 'audio', 'code', 'font', 'style', or None + """ + parsed = Uri(url) + if not parsed.path: + return None + path = parsed.path.lower() + + # Check for extension match + for ext in IMAGE_EXTENSIONS: + if path.endswith(ext): + return 'image' + for ext in VIDEO_EXTENSIONS: + if path.endswith(ext): + return 'video' + for ext in AUDIO_EXTENSIONS: + if path.endswith(ext): + return 'audio' + for ext in CODE_EXTENSIONS: + if path.endswith(ext): + return 'code' + for ext in FONT_EXTENSIONS: + if path.endswith(ext): + return 'font' + for ext in STYLE_EXTENSIONS: + if path.endswith(ext): + return 'style' + return None + + +def get_media_type_from_mime(mime_type: str) -> Optional[str]: + """ + Determine media type from MIME type. + + Returns: + 'image', 'video', 'audio', 'code', 'font', 'style', or None + """ + if not mime_type: + return None + + mime_lower = mime_type.lower() + # Check prefixes first + if mime_lower.startswith(IMAGE_MIME_PREFIXES): + return 'image' + if mime_lower.startswith(VIDEO_MIME_PREFIXES): + return 'video' + if mime_lower.startswith(AUDIO_MIME_PREFIXES): + return 'audio' + # Check exact matches for code/font/style + if mime_lower in CODE_MIME_TYPES: + return 'code' + if mime_lower in FONT_MIME_TYPES: + return 'font' + if mime_lower in STYLE_MIME_TYPES: + return 'style' + return None + + +def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMode.MEDIA) -> List[Dict[str, Any]]: + """ + Extract media URLs from HTML content. + + Extracts from: + - and + - + - pointing to media files + - data-src, data-lazy-src (lazy loading) + - Open Graph and Twitter card meta tags + + Args: + html: HTML content + base_url: Base URL for resolving relative URLs + mode: CrawlMode to filter what types to extract + + Returns: + List of dicts with 'url', 'media_type', 'alt_text', 'title', etc. + """ + media_items = [] + seen_urls: Set[str] = set() + + # Determine which types to collect based on mode + collect_images = mode in (CrawlMode.IMAGES, CrawlMode.MEDIA, CrawlMode.ALL) + collect_videos = mode in (CrawlMode.VIDEOS, CrawlMode.MEDIA, CrawlMode.ALL) + collect_audio = mode in (CrawlMode.MEDIA, CrawlMode.ALL) + + soup = BeautifulSoup(html, "html.parser") + + # Extract page-level metadata for inference + page_title = '' + title_tag = soup.find('title') + if title_tag: + page_title = title_tag.get_text(strip=True) + + # Get h1 as fallback title + h1_tag = soup.find('h1') + page_h1 = h1_tag.get_text(strip=True) if h1_tag else '' + + # Extract meta description and keywords for searchability + page_description = '' + page_keywords = '' + for meta in soup.find_all('meta'): + name = (meta.get('name') or meta.get('property') or '').lower() + content = meta.get('content', '') + if name == 'description' or name == 'og:description': + page_description = content[:500] # Limit length + elif name == 'keywords': + page_keywords = content[:500] + + # Extract page content (body text) for full-text search + # This enables blog images to be searchable by post content + # IMPORTANT: Parse a fresh copy so we don't remove header/nav images from the soup + # that will be used for media extraction below + page_content = '' + body = soup.find('body') + if body: + body_soup = BeautifulSoup(str(body), 'html.parser') + # Remove script, style, nav, footer elements from the COPY only + for tag in body_soup.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside']): + tag.decompose() + page_content = body_soup.get_text(separator=' ', strip=True)[:10000] # Limit to 10k chars + + def get_context_for_element(element) -> dict: + """Extract contextual metadata from surrounding HTML elements.""" + context = { + 'page_title': page_title, + 'page_h1': page_h1, + 'figure_caption': None, + 'nearby_heading': None, + 'link_text': None, + 'link_title': None, # attribute + } + + # Check if inside a
with
+ figure = element.find_parent('figure') + if figure: + figcaption = figure.find('figcaption') + if figcaption: + context['figure_caption'] = figcaption.get_text(strip=True)[:200] + + # Check if inside an tag with text and/or title + link = element.find_parent('a') + if link: + link_text = link.get_text(strip=True) + if link_text and link_text != element.get('alt', ''): + context['link_text'] = link_text[:200] + # Also extract title attribute from tag (tooltip text) + link_title = link.get('title', '').strip() + if link_title: + context['link_title'] = link_title[:200] + + # Find nearest heading (h1-h6) before this element + for heading_tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']: + # Look for preceding siblings and parents + for prev in element.find_all_previous(heading_tag, limit=3): + heading_text = prev.get_text(strip=True) + if heading_text: + context['nearby_heading'] = heading_text[:200] + break + if context['nearby_heading']: + break + + return context + + def infer_title(alt_text: str, title: str, context: dict) -> str: + """Infer best title from available metadata.""" + # Priority: explicit title > alt text > link title > figure caption > link text > nearby heading > page h1 > page title + if title and title.strip(): + return title.strip() + if alt_text and alt_text.strip() and len(alt_text) > 3: + return alt_text.strip() + if context.get('link_title'): # attribute + return context['link_title'] + if context.get('figure_caption'): + return context['figure_caption'] + if context.get('link_text'): + return context['link_text'] + if context.get('nearby_heading'): + return context['nearby_heading'] + if context.get('page_h1'): + return context['page_h1'] + if context.get('page_title'): + return context['page_title'] + return None + + def add_media(url: str, media_type: str, alt_text: str = None, title: str = None, width: int = None, height: int = None, element=None, detail_page_url: str = None): + """Helper to add media item if not already seen.""" + if not url or url in seen_urls: + return + if url.startswith('data:'): # Skip data URIs for now + return + + # Resolve relative URLs + absolute_url = urljoin(base_url, url) + if absolute_url in seen_urls: + return + + seen_urls.add(absolute_url) + + # Get context from surrounding elements + context = get_context_for_element(element) if element else {'page_title': page_title, 'page_h1': page_h1} + + # Build metadata accumulator - "never clobber, always append" + metadata = MediaMetadata() + + # Add from img element + if alt_text: + metadata.add_img_alt(alt_text) + if title: + metadata.add_img_title(title) + + # Add from context + if context.get('figure_caption'): + metadata.add_figcaption(context['figure_caption']) + if context.get('link_text'): + metadata.add_link_text(context['link_text']) + if context.get('link_title'): + metadata.add_link_title(context['link_title']) + if context.get('nearby_heading'): + metadata.add_heading(context['nearby_heading']) + if context.get('page_title'): + metadata.add_page_title(context['page_title']) + if context.get('page_h1'): + metadata.add_heading(context['page_h1']) + + # Add page-level metadata + if page_description: + metadata.add_description(page_description) + + # Get best values for backward compatibility + best_title = metadata.get_best_title() + best_alt = metadata.get_best_alt() + + # Check if this image is wrapped in an tag pointing to a detail page + # (Pinterest-style galleries where thumbnail links to detail page with canonical image) + resolved_detail_url = None + if element and not detail_page_url: + parent_link = element.find_parent('a', href=True) + if parent_link: + href = parent_link.get('href', '') + # Only consider internal links (not direct image links) + if href and not get_media_type_from_extension(href): + resolved_detail_url = urljoin(base_url, href) + # Only track same-domain detail pages + base_domain = Uri(base_url).hostname + detail_domain = Uri(resolved_detail_url).hostname + if base_domain != detail_domain: + resolved_detail_url = None + elif detail_page_url: + resolved_detail_url = urljoin(base_url, detail_page_url) + + media_items.append({ + 'url': absolute_url, + 'media_type': media_type, + 'alt_text': best_alt, + 'title': best_title, + 'width': width, + 'height': height, + 'source_page': base_url, + 'page_title': page_title, + 'page_description': page_description, + 'page_keywords': page_keywords, + 'page_content': page_content, # Full text for blog post searchability + 'figure_caption': context.get('figure_caption'), + 'nearby_heading': context.get('nearby_heading'), + 'link_text': context.get('link_text'), + 'link_title': context.get('link_title'), # attribute + 'detail_page_url': resolved_detail_url, # URL to fetch for canonical image + richer metadata + # Accumulated metadata for full-text search + 'searchable_text': metadata.to_searchable_text(), + 'metadata': metadata.to_dict(), # Full breakdown for debugging/analysis + }) + + # Extract from tags + if collect_images: + for img in soup.find_all('img'): + src = img.get('src') + alt = img.get('alt', '') + title = img.get('title', '') + width = img.get('width') + height = img.get('height') + + if src: + add_media(src, 'image', alt, title, + int(width) if width and width.isdigit() else None, + int(height) if height and height.isdigit() else None, + element=img) + + # Handle srcset + srcset = img.get('srcset', '') + for src_entry in srcset.split(','): + src_entry = src_entry.strip() + if src_entry: + src_url = src_entry.split()[0] # Get URL part, ignore size descriptor + add_media(src_url, 'image', alt, title, element=img) + + # Lazy loading attributes + for attr in ['data-src', 'data-lazy-src', 'data-original', 'data-srcset']: + lazy_src = img.get(attr) + if lazy_src: + if attr.endswith('srcset'): + for src_entry in lazy_src.split(','): + src_entry = src_entry.strip() + if src_entry: + add_media(src_entry.split()[0], 'image', alt, title, element=img) + else: + add_media(lazy_src, 'image', alt, title, element=img) + + # Extract from tags + if collect_images: + for picture in soup.find_all('picture'): + for source in picture.find_all('source'): + srcset = source.get('srcset', '') + for src_entry in srcset.split(','): + src_entry = src_entry.strip() + if src_entry: + add_media(src_entry.split()[0], 'image') + + # Extract from pointing to media/code/font/style files + for a in soup.find_all('a', href=True): + href = a['href'] + media_type = get_media_type_from_extension(href) + if media_type: + if (media_type == 'image' and collect_images) or \ + (media_type == 'video' and collect_videos) or \ + (media_type == 'audio' and collect_audio) or \ + media_type in ('code', 'font', 'style'): # Always collect code/font/style + add_media(href, media_type, alt_text=a.get_text(strip=True)[:100]) + + # Extract from tags for stylesheets and fonts + for link in soup.find_all('link', href=True): + href = link.get('href') + rel = link.get('rel', []) + as_attr = link.get('as', '') + + if 'stylesheet' in rel: + add_media(href, 'style', alt_text='stylesheet') + elif 'preload' in rel and as_attr == 'font': + add_media(href, 'font', alt_text='preload font') + elif 'preload' in rel and as_attr == 'style': + add_media(href, 'style', alt_text='preload style') + else: + # Check by extension + media_type = get_media_type_from_extension(href) + if media_type in ('font', 'style'): + add_media(href, media_type, alt_text=f'link {media_type}') + + # Extract from + + + + ''' + text = fetcher.extract_text_content(html) + + assert "Hello World" in text + assert "test paragraph" in text + assert "alert" not in text # Script content should be removed + assert ".hidden" not in text # Style content should be removed + + def test_extract_title(self): + """Test title extraction from HTML.""" + fetcher = AsyncWebFetcher() + html = 'Page Title' + title = fetcher.extract_title(html) + assert title == "Page Title" + + def test_extract_title_missing(self): + """Test title extraction when missing.""" + fetcher = AsyncWebFetcher() + html = '' + title = fetcher.extract_title(html) + # Returns "Untitled" when no title present + assert title == "Untitled" or title == "" + + +class TestAsyncWebFetcherScoring: + """Test link and page scoring functionality.""" + + def test_score_link_keyword_match(self): + """Test link scoring with keyword matches.""" + fetcher = AsyncWebFetcher() + + # Link with keyword in URL + score = fetcher._score_link( + "https://example.com/pricing", + "View Pricing", + query_keywords=["pricing", "cost"], + ) + assert score > 0 # Should have positive score + + # Link without keywords + score_none = fetcher._score_link( + "https://example.com/about", + "About Us", + query_keywords=["pricing", "cost"], + ) + assert score > score_none # Keyword match should score higher + + def test_score_link_anchor_text(self): + """Test link scoring with anchor text.""" + fetcher = AsyncWebFetcher() + + # Anchor text with keyword + score = fetcher._score_link( + "https://example.com/page", + "Our Pricing Plans", + query_keywords=["pricing"], + ) + assert score > 0 + + def test_score_page_keyword_match(self): + """Test page scoring with keyword matches.""" + fetcher = AsyncWebFetcher() + + page_data = { + "url": "https://example.com/pricing", + "title": "Pricing Plans", + "extracted_text": "Our pricing is competitive. See our cost breakdown.", + } + + score = fetcher._score_page( + page_data, + query_keywords=["pricing", "cost"], + ) + assert score > 0 + + +class TestAsyncWebFetcherAsync: + """Test async methods of AsyncWebFetcher.""" + + @pytest.mark.asyncio + async def test_can_fetch_respects_robots(self): + """Test that _can_fetch respects robots.txt.""" + fetcher = AsyncWebFetcher() + + # Mock the robots.txt fetch to return a parser that disallows + with patch.object(fetcher, '_fetch_robots_txt', new_callable=AsyncMock) as mock_robots: + mock_parser = MagicMock() + mock_parser.can_fetch.return_value = False + mock_robots.return_value = mock_parser + + result = await fetcher._can_fetch("https://example.com/blocked") + assert result is False + + @pytest.mark.asyncio + async def test_can_fetch_allows_when_parser_allows(self): + """Test _can_fetch when robots parser allows.""" + fetcher = AsyncWebFetcher() + + with patch.object(fetcher, '_fetch_robots_txt', new_callable=AsyncMock) as mock_robots: + mock_parser = MagicMock() + mock_parser.can_fetch.return_value = True + mock_robots.return_value = mock_parser + + result = await fetcher._can_fetch("https://example.com/allowed") + assert result is True + + @pytest.mark.asyncio + async def test_enforce_crawl_delay(self): + """Test crawl delay enforcement.""" + fetcher = AsyncWebFetcher(default_crawl_delay=0.1) + domain = "example.com" + + # First request should not delay + start = asyncio.get_event_loop().time() + await fetcher._enforce_crawl_delay(domain) + first_time = asyncio.get_event_loop().time() - start + + # Second request should delay + start = asyncio.get_event_loop().time() + await fetcher._enforce_crawl_delay(domain) + second_time = asyncio.get_event_loop().time() - start + + # Second call should have some delay (at least part of 0.1s) + assert second_time >= 0.05 # Allow some tolerance + + @pytest.mark.asyncio + async def test_check_media_url_signature(self): + """Test check_media_url function signature.""" + fetcher = AsyncWebFetcher() + + # Verify the method exists and has correct signature + import inspect + sig = inspect.signature(fetcher.check_media_url) + params = list(sig.parameters.keys()) + + # Should have url parameter + assert 'url' in params or len(params) >= 1 + + +class TestErrorHandling: + """Test error handling functionality.""" + + def test_get_last_fetch_error_initially_none(self): + """Test that last fetch error is None initially.""" + clear_last_fetch_error() + error = get_last_fetch_error() + assert error is None + + def test_clear_last_fetch_error(self): + """Test clearing the last fetch error.""" + clear_last_fetch_error() + assert get_last_fetch_error() is None + + +class TestExtensionSets: + """Test extension and MIME type sets.""" + + def test_image_extensions(self): + """Test image extension set.""" + assert '.jpg' in IMAGE_EXTENSIONS + assert '.jpeg' in IMAGE_EXTENSIONS + assert '.png' in IMAGE_EXTENSIONS + assert '.gif' in IMAGE_EXTENSIONS + assert '.webp' in IMAGE_EXTENSIONS + assert '.svg' in IMAGE_EXTENSIONS + + def test_video_extensions(self): + """Test video extension set.""" + assert '.mp4' in VIDEO_EXTENSIONS + assert '.webm' in VIDEO_EXTENSIONS + assert '.mov' in VIDEO_EXTENSIONS + assert '.avi' in VIDEO_EXTENSIONS + + def test_audio_extensions(self): + """Test audio extension set.""" + assert '.mp3' in AUDIO_EXTENSIONS + assert '.wav' in AUDIO_EXTENSIONS + assert '.ogg' in AUDIO_EXTENSIONS + assert '.flac' in AUDIO_EXTENSIONS + + +if __name__ == '__main__': + pytest.main([__file__, '-v']) diff --git a/tests/crawler/test_web_fetch.py b/tests/crawler/test_web_fetch.py new file mode 100644 index 0000000..21f0db0 --- /dev/null +++ b/tests/crawler/test_web_fetch.py @@ -0,0 +1,343 @@ +""" +Tests for core.web_fetch module. + +Tests URL extraction, sources footer building, and web fetch helpers. +""" + +import pytest +from unittest.mock import Mock, AsyncMock, patch +from aborist.sources.crawler.web_fetch import ( + extract_url_from_content, + is_web_fetch_request, + build_sources_footer, + detect_fresh_fetch_intent, + extract_domain, + format_progress_message, + format_answering_message, + generate_keyword_stems, +) + + +class TestURLExtraction: + """Test URL extraction from message content.""" + + def test_extract_url_with_protocol(self): + """Test extracting URL with http:// or https://""" + content = "Check out https://example.com for more info" + url = extract_url_from_content(content) + assert url == "https://example.com" + + def test_extract_url_http_protocol(self): + """Test extracting URL with http:// protocol""" + content = "Visit http://insecure.example.com" + url = extract_url_from_content(content) + assert url == "http://insecure.example.com" + + def test_extract_url_without_protocol(self): + """Test extracting URL without protocol (adds https://)""" + content = "Go to example.com" + url = extract_url_from_content(content) + assert url == "https://example.com" + + def test_extract_subdomain_url(self): + """Test extracting URL with subdomain""" + content = "Check media.unturf.com/test" + url = extract_url_from_content(content) + assert url == "https://media.unturf.com/test" + + def test_extract_url_with_path(self): + """Test extracting URL with path""" + content = "Read https://docs.example.com/api/v1/reference" + url = extract_url_from_content(content) + assert url == "https://docs.example.com/api/v1/reference" + + def test_extract_url_with_query(self): + """Test extracting URL with query parameters""" + content = "See https://example.com/search?q=test&lang=en" + url = extract_url_from_content(content) + assert url == "https://example.com/search?q=test&lang=en" + + def test_no_url_in_content(self): + """Test when no URL is present""" + content = "This is just a regular message without any URLs" + url = extract_url_from_content(content) + assert url is None + + def test_extract_first_url_when_multiple(self): + """Test that first URL is extracted when multiple are present""" + content = "Check https://first.com and https://second.com" + url = extract_url_from_content(content) + assert url == "https://first.com" + + def test_extract_url_with_www(self): + """Test extracting URL with www prefix""" + content = "Visit www.example.com" + url = extract_url_from_content(content) + assert url == "https://www.example.com" + + def test_extract_url_ignores_discord_embeds(self): + """Test that angle brackets don't break URL extraction""" + content = "Check out " + url = extract_url_from_content(content) + # Angle brackets should be part of the URL match + assert "example.com" in url + + +@pytest.mark.skip( + reason="chat-bot fetch triggers (has_web_fetch_trigger) were " + "dropped during the aborist lift — aborist has no chat surface; " + "see aborist/sources/crawler/__init__.py for rationale." +) +class TestWebFetchDetection: + """Test web fetch request detection.""" + + def test_is_web_fetch_request_with_url(self): + """Test detection when URL is present""" + assert is_web_fetch_request("Fetch https://example.com") is True + + def test_is_web_fetch_request_without_url(self): + """Test detection when no URL is present""" + assert is_web_fetch_request("Tell me about Python") is False + + def test_is_web_fetch_request_domain_only(self): + """Test detection with domain-only URL""" + assert is_web_fetch_request("What's on example.com?") is True + + +@pytest.mark.skip( + reason="chat-bot fresh-fetch triggers (has_fresh_fetch_trigger) were " + "dropped during the aborist lift — aborist has no chat surface; " + "see aborist/sources/crawler/__init__.py for rationale." +) +class TestFreshFetchDetection: + """Test fresh fetch intent detection.""" + + def test_detect_fresh_with_keyword(self): + """Test detection with 'fresh' keyword""" + assert detect_fresh_fetch_intent("Get fresh data from example.com") is True + + def test_detect_refresh_with_keyword(self): + """Test detection with 'refresh' keyword""" + assert detect_fresh_fetch_intent("Refresh the content") is True + + def test_detect_reload_with_keyword(self): + """Test detection with 'reload' keyword""" + assert detect_fresh_fetch_intent("Reload example.com") is True + + def test_detect_latest_with_keyword(self): + """Test detection with 'latest' keyword""" + assert detect_fresh_fetch_intent("Get the latest from example.com") is True + + def test_no_fresh_intent(self): + """Test when no fresh fetch keywords present""" + assert detect_fresh_fetch_intent("Fetch example.com") is False + + def test_fresh_detection_case_insensitive(self): + """Test that fresh detection is case insensitive""" + assert detect_fresh_fetch_intent("FRESH data please") is True + + +class TestSourcesFooter: + """Test sources footer building.""" + + def test_build_sources_footer_single_source(self): + """Test building footer with single source""" + sources = [{'url': 'https://example.com', 'title': 'Example Page'}] + footer = build_sources_footer(sources) + assert 'Sources:' in footer + assert 'Example Page' in footer + assert '' in footer + + def test_build_sources_footer_multiple_sources(self): + """Test building footer with multiple sources""" + sources = [ + {'url': 'https://example.com', 'title': 'Example Page'}, + {'url': 'https://test.com', 'title': 'Test Page'} + ] + footer = build_sources_footer(sources) + assert 'Example Page' in footer + assert 'Test Page' in footer + assert footer.count('<') == 2 # Two URLs with angle brackets + + def test_build_sources_footer_empty_list(self): + """Test building footer with empty source list""" + footer = build_sources_footer([]) + assert footer == "" + + def test_build_sources_footer_missing_title(self): + """Test building footer when title is missing""" + sources = [{'url': 'https://example.com'}] + footer = build_sources_footer(sources) + assert 'Untitled' in footer + assert '' in footer + + def test_build_sources_footer_missing_url(self): + """Test building footer when URL is missing""" + sources = [{'title': 'Example Page'}] + footer = build_sources_footer(sources) + # Should not include entry if URL is missing + assert footer == '**Sources:**\n' + + def test_build_sources_footer_angle_brackets_suppress_embeds(self): + """Test that angle brackets are used to suppress Discord embeds""" + sources = [{'url': 'https://example.com', 'title': 'Test'}] + footer = build_sources_footer(sources) + # URLs should be in angle brackets + assert '' in footer + assert 'https://example.com>' not in footer.replace('', '') + + +class TestDomainExtraction: + """Test domain extraction from URLs.""" + + def test_extract_domain_basic(self): + """Test extracting domain from basic URL""" + domain = extract_domain("https://example.com/path") + assert domain == "example.com" + + def test_extract_domain_with_subdomain(self): + """Test extracting domain with subdomain""" + domain = extract_domain("https://api.example.com") + assert domain == "api.example.com" + + def test_extract_domain_with_port(self): + """Test extracting domain with port""" + domain = extract_domain("https://example.com:8080/path") + assert domain == "example.com:8080" + + def test_extract_domain_invalid_url(self): + """Test extracting domain from invalid URL""" + domain = extract_domain("not a url") + # Should return None or handle gracefully + assert domain is None or isinstance(domain, str) + + +class TestProgressFormatting: + """Test progress message formatting.""" + + def test_format_progress_message_basic(self): + """Test basic progress message formatting""" + msg = format_progress_message( + url="https://example.com", + task="Find pricing", + keywords=["pricing", "cost"], + depth=2 + ) + assert "example.com" in msg + assert "pricing" in msg + assert "cost" in msg + assert "**Depth:** 2" in msg + + def test_format_progress_message_with_crawl_delay(self): + """Test progress message with crawl delay""" + msg = format_progress_message( + url="https://example.com", + crawl_delay=5.0 + ) + assert "**Crawl delay:** 5.0s" in msg + + def test_format_progress_message_no_crawl_delay_if_default(self): + """Test that default crawl delay is not shown""" + msg = format_progress_message( + url="https://example.com", + crawl_delay=2.0 + ) + assert "Crawl delay" not in msg + + def test_format_progress_message_fresh_mode(self): + """Test progress message with fresh mode""" + msg = format_progress_message( + url="https://example.com", + cache_mode="fresh" + ) + assert "**Mode:** fresh" in msg + + def test_format_answering_message(self): + """Test answering phase message formatting""" + msg = format_answering_message(5) + assert "5 page(s)" in msg + assert "Analyzing" in msg + + +class TestKeywordStemming: + """Test keyword stemming functionality.""" + + def test_generate_stems_basic(self): + """Test generating stems from keywords""" + stems = generate_keyword_stems(["running", "tests"], []) + assert "run" in stems or "test" in stems + + def test_generate_stems_no_duplicates(self): + """Test that stems don't duplicate keywords""" + stems = generate_keyword_stems(["run"], ["running"]) + # "run" is already in keywords, so stem shouldn't be added + assert "run" not in stems + + def test_generate_stems_variations(self): + """Test stem generation from variations""" + stems = generate_keyword_stems([], ["pricing", "prices"]) + # Should generate stems if they're different from variations + assert isinstance(stems, list) + + def test_generate_stems_no_stemming_needed(self): + """Test when no stemming is needed""" + stems = generate_keyword_stems(["api", "cli"], []) + # Short words shouldn't be stemmed + assert len(stems) == 0 + + def test_generate_stems_phrase(self): + """Test stem generation from phrase""" + stems = generate_keyword_stems(["pull requests"], []) + # Should split phrase and stem individual words + assert isinstance(stems, list) + + +class TestWebFetchIntegration: + """Test web fetch integration with mock web cache manager.""" + + @pytest.mark.asyncio + async def test_fetch_and_cache_success(self): + """Test successful fetch and cache operation""" + # This would require mocking web_cache_manager + # For now, just verify the function exists and has correct signature + from aborist.sources.crawler.web_fetch import fetch_and_cache + import inspect + sig = inspect.signature(fetch_and_cache) + assert 'url' in sig.parameters + assert 'web_cache_manager' in sig.parameters + assert 'keywords' in sig.parameters + assert 'depth' in sig.parameters + assert 'fresh' in sig.parameters + + @pytest.mark.asyncio + async def test_fetch_and_cache_with_progress_callback(self): + """Test fetch with progress callback""" + from aborist.sources.crawler.web_fetch import fetch_and_cache + + # Mock web cache manager + mock_manager = Mock() + mock_manager.get_or_fetch = AsyncMock(return_value={ + 'url': 'https://example.com', + 'title': 'Test', + 'extracted_text': 'Content', + 'source_urls': [] + }) + + # Mock progress callback + progress_calls = [] + async def progress_callback(msg): + progress_calls.append(msg) + + result = await fetch_and_cache( + url='https://example.com', + web_cache_manager=mock_manager, + progress_callback=progress_callback + ) + + assert result is not None + assert result['url'] == 'https://example.com' + mock_manager.get_or_fetch.assert_called_once() + + +if __name__ == '__main__': + pytest.main([__file__, '-v'])