engineering/pig.py
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
pig.py/neopig/async_web_fetcher.py

2765 lines
117 KiB
Python
Raw Blame History

#!/usr/bin/env python3
# This is free software for the public good of a permacomputer hosted at
# permacomputer.com, an always-on computer by the people, for the people.
# One which is durable, easy to repair, & distributed like tap water
# for machine learning intelligence.
#
# The permacomputer is community-owned infrastructure optimized around
# four values:
#
# TRUTH First principles, math & science, open source code freely distributed
# FREEDOM Voluntary partnerships, freedom from tyranny & corporate control
# HARMONY Minimal waste, self-renewing systems with diverse thriving connections
# LOVE Be yourself without hurting others, cooperation through natural law
#
# This software contributes to that vision by archiving the web, preserving digital knowledge before it disappears.
# Code is seeds to sprout on any abandoned technology.
"""
Async Web Fetcher for Discord Bot
Ethical web scraping with robots.txt compliance, async/await compatible
Supports multiple crawl modes:
- text: Extract text content (default, current behavior)
- images: Collect images only
- videos: Collect videos only
- media: Collect all media (images + videos + audio)
- all: Uber crawl - text + all media, full domain slurp
# inputs > process > outputs
# The universe is logical. - The Sign Maker
"""
import os
import io
import hashlib
import logging
import aiohttp
import asyncio
import re
from dataclasses import dataclass, field
from datetime import datetime, timezone
from enum import Enum
from typing import Optional, Dict, List, Tuple, Set, Any
from urllib.parse import urljoin
from urllib.robotparser import RobotFileParser
from bs4 import BeautifulSoup
import time
from miniuri import Uri
class CrawlMode(Enum):
"""Crawl modes for different content types."""
TEXT = "text" # Extract text content (default)
IMAGES = "images" # Collect images only
VIDEOS = "videos" # Collect videos only
MEDIA = "media" # All media (images + videos + audio)
ALL = "all" # Uber crawl: text + all media
# File extensions by media type
IMAGE_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.bmp', '.ico', '.tiff', '.tif', '.avif'}
VIDEO_EXTENSIONS = {'.mp4', '.webm', '.mov', '.avi', '.mkv', '.m4v', '.ogv', '.flv', '.wmv'}
AUDIO_EXTENSIONS = {'.mp3', '.wav', '.ogg', '.m4a', '.flac', '.aac', '.wma'}
# Code file extensions
CODE_EXTENSIONS = {
'.py', '.js', '.ts', '.jsx', '.tsx', '.mjs', '.cjs', # Python, JavaScript, TypeScript
'.rs', '.go', '.rb', '.php', '.pl', '.pm', # Rust, Go, Ruby, PHP, Perl
'.java', '.kt', '.kts', '.scala', '.groovy', # JVM languages
'.c', '.h', '.cpp', '.hpp', '.cc', '.cxx', '.hxx', # C/C++
'.cs', '.fs', '.fsx', # .NET
'.swift', '.m', '.mm', # Apple
'.lua', '.r', '.R', '.jl', # Lua, R, Julia
'.sh', '.bash', '.zsh', '.fish', '.ps1', # Shell
'.sql', '.graphql', '.gql', # Query languages
'.yaml', '.yml', '.toml', '.json', '.xml', # Config
'.md', '.rst', '.txt', # Docs
'.zig', '.nim', '.d', '.v', # Modern systems langs
'.ex', '.exs', '.erl', '.hrl', # Erlang/Elixir
'.clj', '.cljs', '.cljc', '.edn', # Clojure
'.hs', '.lhs', # Haskell
'.ml', '.mli', '.re', '.rei', # OCaml/ReasonML
'.lisp', '.cl', '.el', '.scm', '.rkt', # Lisps
'.f90', '.f95', '.f03', '.for', # Fortran
'.asm', '.s', # Assembly
'.cob', '.cbl', # COBOL
'.pro', # Prolog
'.tcl', # Tcl
'.dart', # Dart
'.raku', '.p6', # Raku
'.cr', # Crystal
'.vue', '.svelte', # Frontend frameworks
'.tf', '.hcl', # Terraform
'.dockerfile', '.makefile', # Build files
}
# Font file extensions
FONT_EXTENSIONS = {'.woff', '.woff2', '.ttf', '.otf', '.eot', '.sfnt'}
# Style file extensions
STYLE_EXTENSIONS = {'.css', '.scss', '.sass', '.less', '.styl'}
# MIME types by media type
IMAGE_MIME_PREFIXES = ('image/',)
VIDEO_MIME_PREFIXES = ('video/',)
AUDIO_MIME_PREFIXES = ('audio/',)
CODE_MIME_TYPES = {
'text/x-python', 'application/x-python', 'text/x-python-script',
'text/javascript', 'application/javascript', 'application/x-javascript',
'text/typescript', 'application/typescript',
'text/x-rust', 'text/x-go', 'text/x-ruby', 'application/x-ruby',
'text/x-java-source', 'text/x-kotlin', 'text/x-scala',
'text/x-c', 'text/x-c++', 'text/x-csrc', 'text/x-c++src',
'text/x-csharp', 'text/x-fsharp',
'text/x-swift', 'text/x-objective-c',
'text/x-lua', 'text/x-r', 'text/x-julia',
'text/x-shellscript', 'application/x-sh', 'text/x-bash',
'application/sql', 'application/graphql',
'application/json', 'application/xml', 'text/xml',
'text/yaml', 'application/x-yaml', 'text/x-yaml',
'text/markdown', 'text/x-markdown',
'text/plain', # Often used for code
}
FONT_MIME_TYPES = {
'font/woff', 'font/woff2', 'font/ttf', 'font/otf', 'font/sfnt',
'application/font-woff', 'application/font-woff2',
'application/x-font-ttf', 'application/x-font-otf',
'application/vnd.ms-fontobject',
}
STYLE_MIME_TYPES = {'text/css', 'text/x-scss', 'text/x-sass', 'text/x-less'}
@dataclass
class MediaItem:
"""Represents a discovered media item."""
url: str
source_page: str
media_type: str # 'image', 'video', 'audio'
mime_type: Optional[str] = None
md5_hash: Optional[str] = None
file_size: Optional[int] = None
alt_text: Optional[str] = None
title: Optional[str] = None
width: Optional[int] = None
height: Optional[int] = None
discovered_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
class MediaMetadata:
"""
Accumulates metadata from all sources without clobbering.
"Never clobber, always append" - collects ALL metadata from:
- img.alt, img.title
- a.title, a.text (link text)
- figcaption
- nearby headings
- page title/h1
Produces a combined searchable_text for full-text search.
"""
def __init__(self):
self.titles: List[str] = []
self.alt_texts: List[str] = []
self.descriptions: List[str] = []
self.captions: List[str] = []
self.headings: List[str] = []
self.link_texts: List[str] = []
self.link_titles: List[str] = []
def _add_unique(self, lst: List[str], value: str, max_len: int = 500) -> None:
"""Add value to list if non-empty and not duplicate."""
if value and value.strip():
clean = value.strip()[:max_len]
if clean not in lst:
lst.append(clean)
def add_img_alt(self, alt: str) -> None:
"""Add img alt attribute."""
self._add_unique(self.alt_texts, alt)
def add_img_title(self, title: str) -> None:
"""Add img title attribute."""
self._add_unique(self.titles, title)
def add_link_title(self, title: str) -> None:
"""Add <a title="..."> attribute."""
self._add_unique(self.link_titles, title)
def add_link_text(self, text: str) -> None:
"""Add <a> inner text."""
self._add_unique(self.link_texts, text)
def add_figcaption(self, caption: str) -> None:
"""Add figcaption text."""
self._add_unique(self.captions, caption)
def add_heading(self, heading: str) -> None:
"""Add nearby heading (h1-h6)."""
self._add_unique(self.headings, heading)
def add_description(self, desc: str) -> None:
"""Add description (og:description, meta description, etc.)."""
self._add_unique(self.descriptions, desc)
def add_page_title(self, title: str) -> None:
"""Add page title."""
self._add_unique(self.titles, title)
def get_best_title(self) -> Optional[str]:
"""Get best title for display (first non-empty)."""
for lst in [self.titles, self.alt_texts, self.link_titles,
self.captions, self.link_texts, self.headings]:
if lst:
return lst[0]
return None
def get_best_alt(self) -> Optional[str]:
"""Get best alt text for accessibility."""
if self.alt_texts:
return self.alt_texts[0]
return self.get_best_title()
def to_searchable_text(self) -> str:
"""
Combine ALL collected metadata into searchable text.
This enables finding images by ANY associated text:
- "find images of cats" matches img alt="cute cat"
- "find images from blog post about python" matches page content
"""
all_parts = []
# Dedupe while preserving order
seen = set()
for lst in [self.titles, self.alt_texts, self.descriptions,
self.captions, self.headings, self.link_texts, self.link_titles]:
for item in lst:
if item and item not in seen:
all_parts.append(item)
seen.add(item)
return ' | '.join(all_parts)
def to_dict(self) -> Dict[str, Any]:
"""Export all collected metadata as dict."""
return {
'titles': self.titles,
'alt_texts': self.alt_texts,
'descriptions': self.descriptions,
'captions': self.captions,
'headings': self.headings,
'link_texts': self.link_texts,
'link_titles': self.link_titles,
'searchable_text': self.to_searchable_text(),
'best_title': self.get_best_title(),
'best_alt': self.get_best_alt(),
}
# PDF text extraction
try:
from pypdf import PdfReader
PDF_SUPPORT = True
except ImportError:
PDF_SUPPORT = False
logger = logging.getLogger(__name__)
# Default crawl delay
DEFAULT_CRAWL_DELAY = 2.0
# Global last error storage (for error reporting without log parsing)
# Format: {'type': 'robots_txt'|'http_error'|'timeout'|'dns'|'ssl'|'unknown', 'details': str, 'url': str}
LAST_FETCH_ERROR = None
def get_last_fetch_error() -> Optional[Dict]:
"""Get the last fetch error, if any."""
return LAST_FETCH_ERROR
def strip_uri_fragment(uri_str: str) -> str:
"""
Strip the fragment (#anchor) from a URI using miniuri.
Args:
uri_str: URI that may contain a fragment
Returns:
URI without fragment
"""
if not uri_str:
return uri_str
uri = Uri(uri_str)
# Reconstruct URI without fragment
result = f"{uri.scheme}://{uri.authority}{uri.path or ''}"
if uri.query:
result += f"?{uri.query}"
return result
def normalize_link(link) -> Tuple[str, str]:
"""
Extract URI and anchor text from a link, stripping fragments.
Args:
link: Either a dict with 'url' and optional 'anchor_text', or a string URI
Returns:
Tuple of (normalized_uri, anchor_text)
"""
if isinstance(link, dict):
uri = strip_uri_fragment(link.get('url', ''))
anchor_text = link.get('anchor_text', '')
else:
uri = strip_uri_fragment(link)
anchor_text = ''
return uri, anchor_text
# Alias for backwards compatibility
strip_url_fragment = strip_uri_fragment
def extract_text_from_pdf(pdf_bytes: bytes) -> Optional[str]:
"""
Extract text from PDF binary data.
Args:
pdf_bytes: Raw PDF file bytes
Returns:
Extracted text or None if extraction fails
"""
if not PDF_SUPPORT:
logger.warning("PDF support not available (pypdf not installed)")
return None
try:
reader = PdfReader(io.BytesIO(pdf_bytes))
text_parts = []
for page in reader.pages:
page_text = page.extract_text()
if page_text:
text_parts.append(page_text)
if text_parts:
return "\n\n".join(text_parts)
return None
except Exception as e:
logger.error(f"Failed to extract text from PDF: {e}")
return None
def clear_last_fetch_error():
"""Clear the last fetch error."""
global LAST_FETCH_ERROR
LAST_FETCH_ERROR = None
async def fetch_robots_txt(url: str, user_agent: str = "uncloseai.com/1.42") -> Optional[str]:
"""
Fetch raw robots.txt content for analysis and error reporting.
Args:
url: Any URL on the domain (robots.txt will be fetched from root)
user_agent: User agent string
Returns:
Raw robots.txt content or None if not available
"""
parsed = Uri(url)
robots_url = f"{parsed.scheme}://{parsed.hostname}/robots.txt"
try:
async with aiohttp.ClientSession() as session:
async with session.get(
robots_url,
headers={"User-Agent": user_agent},
timeout=aiohttp.ClientTimeout(total=5)
) as response:
if response.status == 200:
return await response.text()
except Exception as e:
logger.warning(f"Could not fetch robots.txt for error analysis: {e}")
return None
def analyze_robots_txt_blocking(robots_txt: str, blocked_url: str, user_agent: str = "uncloseai.com/1.42") -> str:
"""
Analyze robots.txt to explain why a URL is blocked.
Args:
robots_txt: Raw robots.txt content
blocked_url: The URL that was blocked
user_agent: User agent to check against
Returns:
Human-readable explanation of the blocking rule
"""
parsed = Uri(blocked_url)
path = parsed.path or "/"
# Parse robots.txt manually for detailed analysis
lines = robots_txt.strip().split('\n')
current_agent = None
blocking_rule = None
all_rules = []
for line in lines:
line = line.strip()
if not line or line.startswith('#'):
continue
if ':' in line:
key, value = line.split(':', 1)
key = key.strip().lower()
value = value.strip()
if key == 'user-agent':
current_agent = value
elif key == 'disallow' and current_agent:
# Check if this rule applies to us
agent_matches = (
current_agent == '*' or
user_agent.lower().startswith(current_agent.lower()) or
current_agent.lower() in user_agent.lower()
)
if agent_matches:
all_rules.append(f"Disallow: {value}")
# Check if this rule blocks our path
if value and path.startswith(value):
blocking_rule = f"Disallow: {value}"
if blocking_rule:
result = f"**Blocked by rule:** `{blocking_rule}`\n"
result += f"**Path requested:** `{path}`\n"
result += f"**User-Agent:** `{user_agent}`\n\n"
# Show relevant rules
if all_rules:
result += "**Applicable rules:**\n"
for rule in all_rules[:10]: # Limit to first 10
result += f" • `{rule}`\n"
return result
elif all_rules:
return f"Blocked by robots.txt (exact rule unclear). Rules found:\n" + "\n".join(f" • `{r}`" for r in all_rules[:10])
else:
return "Blocked by robots.txt (no matching disallow rule found - may be a catch-all)"
def get_media_type_from_extension(url: str) -> Optional[str]:
"""
Determine media type from URL extension.
Returns:
'image', 'video', 'audio', 'code', 'font', 'style', or None
"""
parsed = Uri(url)
if not parsed.path:
return None
path = parsed.path.lower()
# Check for extension match
for ext in IMAGE_EXTENSIONS:
if path.endswith(ext):
return 'image'
for ext in VIDEO_EXTENSIONS:
if path.endswith(ext):
return 'video'
for ext in AUDIO_EXTENSIONS:
if path.endswith(ext):
return 'audio'
for ext in CODE_EXTENSIONS:
if path.endswith(ext):
return 'code'
for ext in FONT_EXTENSIONS:
if path.endswith(ext):
return 'font'
for ext in STYLE_EXTENSIONS:
if path.endswith(ext):
return 'style'
return None
def get_media_type_from_mime(mime_type: str) -> Optional[str]:
"""
Determine media type from MIME type.
Returns:
'image', 'video', 'audio', 'code', 'font', 'style', or None
"""
if not mime_type:
return None
mime_lower = mime_type.lower()
# Check prefixes first
if mime_lower.startswith(IMAGE_MIME_PREFIXES):
return 'image'
if mime_lower.startswith(VIDEO_MIME_PREFIXES):
return 'video'
if mime_lower.startswith(AUDIO_MIME_PREFIXES):
return 'audio'
# Check exact matches for code/font/style
if mime_lower in CODE_MIME_TYPES:
return 'code'
if mime_lower in FONT_MIME_TYPES:
return 'font'
if mime_lower in STYLE_MIME_TYPES:
return 'style'
return None
def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMode.MEDIA) -> List[Dict[str, Any]]:
"""
Extract media URLs from HTML content.
Extracts from:
- <img src="..."> and <img srcset="...">
- <picture><source srcset="..."></picture>
- <video src="..."> and <video poster="...">
- <source src="..."> (within video/audio)
- <audio src="...">
- CSS background-image: url(...)
- <a href="..."> pointing to media files
- data-src, data-lazy-src (lazy loading)
- Open Graph and Twitter card meta tags
Args:
html: HTML content
base_url: Base URL for resolving relative URLs
mode: CrawlMode to filter what types to extract
Returns:
List of dicts with 'url', 'media_type', 'alt_text', 'title', etc.
"""
media_items = []
seen_urls: Set[str] = set()
# Determine which types to collect based on mode
collect_images = mode in (CrawlMode.IMAGES, CrawlMode.MEDIA, CrawlMode.ALL)
collect_videos = mode in (CrawlMode.VIDEOS, CrawlMode.MEDIA, CrawlMode.ALL)
collect_audio = mode in (CrawlMode.MEDIA, CrawlMode.ALL)
soup = BeautifulSoup(html, "html.parser")
# Extract page-level metadata for inference
page_title = ''
title_tag = soup.find('title')
if title_tag:
page_title = title_tag.get_text(strip=True)
# Get h1 as fallback title
h1_tag = soup.find('h1')
page_h1 = h1_tag.get_text(strip=True) if h1_tag else ''
# Extract meta description and keywords for searchability
page_description = ''
page_keywords = ''
for meta in soup.find_all('meta'):
name = (meta.get('name') or meta.get('property') or '').lower()
content = meta.get('content', '')
if name == 'description' or name == 'og:description':
page_description = content[:500] # Limit length
elif name == 'keywords':
page_keywords = content[:500]
# Extract page content (body text) for full-text search
# This enables blog images to be searchable by post content
# IMPORTANT: Parse a fresh copy so we don't remove header/nav images from the soup
# that will be used for media extraction below
page_content = ''
body = soup.find('body')
if body:
body_soup = BeautifulSoup(str(body), 'html.parser')
# Remove script, style, nav, footer elements from the COPY only
for tag in body_soup.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside']):
tag.decompose()
page_content = body_soup.get_text(separator=' ', strip=True)[:10000] # Limit to 10k chars
def get_context_for_element(element) -> dict:
"""Extract contextual metadata from surrounding HTML elements."""
context = {
'page_title': page_title,
'page_h1': page_h1,
'figure_caption': None,
'nearby_heading': None,
'link_text': None,
'link_title': None, # <a title="..."> attribute
}
# Check if inside a <figure> with <figcaption>
figure = element.find_parent('figure')
if figure:
figcaption = figure.find('figcaption')
if figcaption:
context['figure_caption'] = figcaption.get_text(strip=True)[:200]
# Check if inside an <a> tag with text and/or title
link = element.find_parent('a')
if link:
link_text = link.get_text(strip=True)
if link_text and link_text != element.get('alt', ''):
context['link_text'] = link_text[:200]
# Also extract title attribute from <a> tag (tooltip text)
link_title = link.get('title', '').strip()
if link_title:
context['link_title'] = link_title[:200]
# Find nearest heading (h1-h6) before this element
for heading_tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
# Look for preceding siblings and parents
for prev in element.find_all_previous(heading_tag, limit=3):
heading_text = prev.get_text(strip=True)
if heading_text:
context['nearby_heading'] = heading_text[:200]
break
if context['nearby_heading']:
break
return context
def infer_title(alt_text: str, title: str, context: dict) -> str:
"""Infer best title from available metadata."""
# Priority: explicit title > alt text > link title > figure caption > link text > nearby heading > page h1 > page title
if title and title.strip():
return title.strip()
if alt_text and alt_text.strip() and len(alt_text) > 3:
return alt_text.strip()
if context.get('link_title'): # <a title="..."> attribute
return context['link_title']
if context.get('figure_caption'):
return context['figure_caption']
if context.get('link_text'):
return context['link_text']
if context.get('nearby_heading'):
return context['nearby_heading']
if context.get('page_h1'):
return context['page_h1']
if context.get('page_title'):
return context['page_title']
return None
def add_media(url: str, media_type: str, alt_text: str = None, title: str = None, width: int = None, height: int = None, element=None, detail_page_url: str = None):
"""Helper to add media item if not already seen."""
if not url or url in seen_urls:
return
if url.startswith('data:'): # Skip data URIs for now
return
# Resolve relative URLs
absolute_url = urljoin(base_url, url)
if absolute_url in seen_urls:
return
seen_urls.add(absolute_url)
# Get context from surrounding elements
context = get_context_for_element(element) if element else {'page_title': page_title, 'page_h1': page_h1}
# Build metadata accumulator - "never clobber, always append"
metadata = MediaMetadata()
# Add from img element
if alt_text:
metadata.add_img_alt(alt_text)
if title:
metadata.add_img_title(title)
# Add from context
if context.get('figure_caption'):
metadata.add_figcaption(context['figure_caption'])
if context.get('link_text'):
metadata.add_link_text(context['link_text'])
if context.get('link_title'):
metadata.add_link_title(context['link_title'])
if context.get('nearby_heading'):
metadata.add_heading(context['nearby_heading'])
if context.get('page_title'):
metadata.add_page_title(context['page_title'])
if context.get('page_h1'):
metadata.add_heading(context['page_h1'])
# Add page-level metadata
if page_description:
metadata.add_description(page_description)
# Get best values for backward compatibility
best_title = metadata.get_best_title()
best_alt = metadata.get_best_alt()
# Check if this image is wrapped in an <a> tag pointing to a detail page
# (Pinterest-style galleries where thumbnail links to detail page with canonical image)
resolved_detail_url = None
if element and not detail_page_url:
parent_link = element.find_parent('a', href=True)
if parent_link:
href = parent_link.get('href', '')
# Only consider internal links (not direct image links)
if href and not get_media_type_from_extension(href):
resolved_detail_url = urljoin(base_url, href)
# Only track same-domain detail pages
base_domain = Uri(base_url).hostname
detail_domain = Uri(resolved_detail_url).hostname
if base_domain != detail_domain:
resolved_detail_url = None
elif detail_page_url:
resolved_detail_url = urljoin(base_url, detail_page_url)
media_items.append({
'url': absolute_url,
'media_type': media_type,
'alt_text': best_alt,
'title': best_title,
'width': width,
'height': height,
'source_page': base_url,
'page_title': page_title,
'page_description': page_description,
'page_keywords': page_keywords,
'page_content': page_content, # Full text for blog post searchability
'figure_caption': context.get('figure_caption'),
'nearby_heading': context.get('nearby_heading'),
'link_text': context.get('link_text'),
'link_title': context.get('link_title'), # <a title="..."> attribute
'detail_page_url': resolved_detail_url, # URL to fetch for canonical image + richer metadata
# Accumulated metadata for full-text search
'searchable_text': metadata.to_searchable_text(),
'metadata': metadata.to_dict(), # Full breakdown for debugging/analysis
})
# Extract from <img> tags
if collect_images:
for img in soup.find_all('img'):
src = img.get('src')
alt = img.get('alt', '')
title = img.get('title', '')
width = img.get('width')
height = img.get('height')
if src:
add_media(src, 'image', alt, title,
int(width) if width and width.isdigit() else None,
int(height) if height and height.isdigit() else None,
element=img)
# Handle srcset
srcset = img.get('srcset', '')
for src_entry in srcset.split(','):
src_entry = src_entry.strip()
if src_entry:
src_url = src_entry.split()[0] # Get URL part, ignore size descriptor
add_media(src_url, 'image', alt, title, element=img)
# Lazy loading attributes
for attr in ['data-src', 'data-lazy-src', 'data-original', 'data-srcset']:
lazy_src = img.get(attr)
if lazy_src:
if attr.endswith('srcset'):
for src_entry in lazy_src.split(','):
src_entry = src_entry.strip()
if src_entry:
add_media(src_entry.split()[0], 'image', alt, title, element=img)
else:
add_media(lazy_src, 'image', alt, title, element=img)
# Extract from <picture><source> tags
if collect_images:
for picture in soup.find_all('picture'):
for source in picture.find_all('source'):
srcset = source.get('srcset', '')
for src_entry in srcset.split(','):
src_entry = src_entry.strip()
if src_entry:
add_media(src_entry.split()[0], 'image')
# Extract from <video> tags
if collect_videos:
for video in soup.find_all('video'):
src = video.get('src')
poster = video.get('poster')
title = video.get('title', '')
if src:
add_media(src, 'video', title=title, element=video)
if poster and collect_images:
add_media(poster, 'image', title=f"Video poster: {title}", element=video)
# Sources within video
for source in video.find_all('source'):
src = source.get('src')
if src:
add_media(src, 'video', title=title)
# Extract from <audio> tags
if collect_audio:
for audio in soup.find_all('audio'):
src = audio.get('src')
title = audio.get('title', '')
if src:
add_media(src, 'audio', title=title)
for source in audio.find_all('source'):
src = source.get('src')
if src:
add_media(src, 'audio', title=title)
# Extract from <a href="..."> pointing to media/code/font/style files
for a in soup.find_all('a', href=True):
href = a['href']
media_type = get_media_type_from_extension(href)
if media_type:
if (media_type == 'image' and collect_images) or \
(media_type == 'video' and collect_videos) or \
(media_type == 'audio' and collect_audio) or \
media_type in ('code', 'font', 'style'): # Always collect code/font/style
add_media(href, media_type, alt_text=a.get_text(strip=True)[:100])
# Extract from <link> tags for stylesheets and fonts
for link in soup.find_all('link', href=True):
href = link.get('href')
rel = link.get('rel', [])
as_attr = link.get('as', '')
if 'stylesheet' in rel:
add_media(href, 'style', alt_text='stylesheet')
elif 'preload' in rel and as_attr == 'font':
add_media(href, 'font', alt_text='preload font')
elif 'preload' in rel and as_attr == 'style':
add_media(href, 'style', alt_text='preload style')
else:
# Check by extension
media_type = get_media_type_from_extension(href)
if media_type in ('font', 'style'):
add_media(href, media_type, alt_text=f'link {media_type}')
# Extract from <script src="..."> for JavaScript files
for script in soup.find_all('script', src=True):
src = script.get('src')
if src:
add_media(src, 'code', alt_text='script')
# Extract font URLs from @font-face in <style> blocks
font_face_pattern = re.compile(r'@font-face\s*\{[^}]*url\(["\']?([^"\')\s]+)["\']?\)', re.IGNORECASE | re.DOTALL)
for style_tag in soup.find_all('style'):
if style_tag.string:
for match in font_face_pattern.findall(style_tag.string):
add_media(match, 'font', alt_text='font-face')
# Extract from CSS background-image: url(...)
if collect_images:
bg_pattern = re.compile(r'background(?:-image)?\s*:\s*url\(["\']?([^"\')\s]+)["\']?\)', re.IGNORECASE)
# Inline styles
for elem in soup.find_all(style=True):
style = elem.get('style', '')
for match in bg_pattern.findall(style):
add_media(match, 'image')
# <style> blocks
for style_tag in soup.find_all('style'):
if style_tag.string:
for match in bg_pattern.findall(style_tag.string):
add_media(match, 'image')
# Extract from Open Graph and Twitter meta tags
# Also look for og:title and og:image:alt for metadata
og_title = None
og_image_alt = None
media_filename = None
media_url = None
media_type_meta = None
for meta in soup.find_all('meta'):
prop = meta.get('property', '') or meta.get('name', '')
content = meta.get('content', '')
prop_lower = prop.lower()
# Extract metadata for richer media info
if prop_lower == 'og:title':
og_title = content
elif prop_lower == 'og:image:alt':
og_image_alt = content
elif prop_lower == 'media:filename':
media_filename = content
elif prop_lower == 'media:url':
media_url = content
elif prop_lower == 'media:type':
media_type_meta = content
# Process OG/Twitter image tags with enriched metadata
if collect_images:
for meta in soup.find_all('meta'):
prop = meta.get('property', '') or meta.get('name', '')
content = meta.get('content', '')
if content and prop.lower() in ('og:image', 'og:image:url', 'twitter:image', 'twitter:image:src'):
title = og_title or media_filename or 'Open Graph/Twitter image'
alt = og_image_alt or og_title or media_filename
add_media(content, 'image', alt_text=alt, title=title)
if collect_videos:
for meta in soup.find_all('meta'):
prop = meta.get('property', '') or meta.get('name', '')
content = meta.get('content', '')
if content and prop.lower() in ('og:video', 'og:video:url', 'twitter:player'):
title = og_title or media_filename or 'Open Graph/Twitter video'
add_media(content, 'video', title=title)
# Also check custom media:url meta tag (PyraFiles specific)
if media_url:
detected_type = media_type_meta or 'image' # Default to image
if detected_type in ('image', 'video', 'audio'):
should_add = (
(detected_type == 'image' and collect_images) or
(detected_type == 'video' and collect_videos) or
(detected_type == 'audio' and collect_audio)
)
if should_add:
title = og_title or media_filename
add_media(media_url, detected_type, alt_text=og_title, title=title)
logger.debug(f"Extracted {len(media_items)} media items from {base_url}")
return media_items
# ============================================================================
# Hydra Mode: Feed/Sitemap Discovery and Parsing
# ============================================================================
@dataclass
class FeedItem:
"""Represents an item discovered from RSS/Atom/Sitemap."""
url: str
title: Optional[str] = None
published: Optional[str] = None # ISO date string
updated: Optional[str] = None
priority: float = 0.5 # Sitemap priority (0.0-1.0)
def discover_feeds(html: str, base_url: str) -> List[str]:
"""
Discover RSS/Atom feed and sitemap URLs from HTML.
Looks for:
- <link rel="alternate" type="application/rss+xml">
- <link rel="alternate" type="application/atom+xml">
- <link rel="sitemap" href="...">
- Common sitemap locations: /sitemap.xml, /sitemap_index.xml
"""
feeds = []
soup = BeautifulSoup(html, 'html.parser')
# RSS/Atom feeds from link tags
for link in soup.find_all('link', rel='alternate'):
link_type = link.get('type', '')
href = link.get('href', '')
if href and ('rss' in link_type or 'atom' in link_type):
feeds.append(urljoin(base_url, href))
# Sitemap from link tag
for link in soup.find_all('link', rel='sitemap'):
href = link.get('href', '')
if href:
feeds.append(urljoin(base_url, href))
# Common sitemap locations
parsed = Uri(base_url)
base = f"{parsed.scheme}://{parsed.hostname}"
if parsed.port and parsed.port not in (80, 443):
base += f":{parsed.port}"
common_sitemaps = [
'/sitemap.xml',
'/sitemap_index.xml',
'/sitemap-index.xml',
'/sitemaps.xml',
]
for path in common_sitemaps:
sitemap_url = base + path
if sitemap_url not in feeds:
feeds.append(sitemap_url)
return feeds
def parse_rss(xml_content: str, base_url: str) -> List[FeedItem]:
"""Parse RSS feed and extract items."""
items = []
try:
soup = BeautifulSoup(xml_content, 'xml')
for item in soup.find_all('item'):
link = item.find('link')
title = item.find('title')
pub_date = item.find('pubDate')
if link:
url = link.get_text().strip() if link.get_text() else link.get('href', '')
if url:
items.append(FeedItem(
url=urljoin(base_url, url),
title=title.get_text().strip() if title else None,
published=pub_date.get_text().strip() if pub_date else None,
))
except Exception as e:
logger.debug(f"RSS parse error: {e}")
return items
def parse_atom(xml_content: str, base_url: str) -> List[FeedItem]:
"""Parse Atom feed and extract entries."""
items = []
try:
soup = BeautifulSoup(xml_content, 'xml')
for entry in soup.find_all('entry'):
# Atom uses <link href="..."> or <link>url</link>
link = entry.find('link')
title = entry.find('title')
published = entry.find('published')
updated = entry.find('updated')
url = None
if link:
url = link.get('href') or link.get_text().strip()
if url:
items.append(FeedItem(
url=urljoin(base_url, url),
title=title.get_text().strip() if title else None,
published=published.get_text().strip() if published else None,
updated=updated.get_text().strip() if updated else None,
))
except Exception as e:
logger.debug(f"Atom parse error: {e}")
return items
def parse_sitemap(xml_content: str, base_url: str) -> Tuple[List[FeedItem], List[str]]:
"""
Parse sitemap and extract URLs.
Returns:
Tuple of (items, sub_sitemaps) - items are page URLs, sub_sitemaps are nested sitemap URLs
"""
items = []
sub_sitemaps = []
try:
soup = BeautifulSoup(xml_content, 'xml')
# Check for sitemap index (contains other sitemaps)
for sitemap in soup.find_all('sitemap'):
loc = sitemap.find('loc')
if loc:
sub_sitemaps.append(loc.get_text().strip())
# Regular sitemap URLs
for url_tag in soup.find_all('url'):
loc = url_tag.find('loc')
lastmod = url_tag.find('lastmod')
priority = url_tag.find('priority')
if loc:
url = loc.get_text().strip()
items.append(FeedItem(
url=urljoin(base_url, url),
updated=lastmod.get_text().strip() if lastmod else None,
priority=float(priority.get_text().strip()) if priority else 0.5,
))
except Exception as e:
logger.debug(f"Sitemap parse error: {e}")
return items, sub_sitemaps
def parse_feed(content: str, url: str) -> Tuple[List[FeedItem], List[str]]:
"""
Auto-detect feed type and parse.
Returns:
Tuple of (items, sub_feeds) - sub_feeds for sitemap indexes
"""
content_lower = content[:1000].lower()
if '<urlset' in content_lower or '<sitemapindex' in content_lower:
return parse_sitemap(content, url)
elif '<feed' in content_lower:
return parse_atom(content, url), []
elif '<rss' in content_lower or '<channel' in content_lower:
return parse_rss(content, url), []
else:
logger.debug(f"Unknown feed format: {url}")
return [], []
class AsyncWebFetcher:
"""
Async web fetcher that respects robots.txt and crawl delays.
Designed for use with Discord bot's async event loop.
"""
def __init__(
self,
user_agent: str = "uncloseai.com/1.42 (ethical web crawler; +https://uncloseai.com)",
default_crawl_delay: float = DEFAULT_CRAWL_DELAY,
fast_mode: bool = False
):
self.user_agent = user_agent
self.default_crawl_delay = default_crawl_delay
self.fast_mode = fast_mode
# Timeouts: 5s in fast mode, 60s normally
self.media_timeout = 5 if fast_mode else 60
self.page_timeout = 5 if fast_mode else 15
# Parallel workers for page fetching: 3x CPU in fast mode, 1 otherwise
import multiprocessing
self.max_page_workers = multiprocessing.cpu_count() * 3 if fast_mode else 1
# Caches for robots.txt and crawl delays per domain
self.robot_parsers: Dict[str, Optional[RobotFileParser]] = {}
self.domain_crawl_delays: Dict[str, float] = {}
self.domain_last_fetched: Dict[str, float] = {}
# Page cache: {url: (html, links, timestamp)}
self.page_cache: Dict[str, Tuple[str, List, float]] = {}
# Domain skip list: domains with too many consecutive timeouts
self.skip_domains: Set[str] = set()
self.domain_timeout_counts: Dict[str, int] = {}
self.MAX_CONSECUTIVE_TIMEOUTS = 5
logger.info(f"AsyncWebFetcher initialized with user-agent: {self.user_agent}")
def _get_domain(self, url: str) -> str:
"""Extract domain from URL"""
parsed = Uri(url)
return parsed.hostname
async def _fetch_robots_txt(self, domain: str) -> Optional[RobotFileParser]:
"""
Fetch and parse robots.txt for a domain.
Returns RobotFileParser or None if not available.
"""
if domain in self.robot_parsers:
logger.debug(f"Using cached robots.txt for {domain}")
return self.robot_parsers[domain]
logger.info(f"Fetching robots.txt for {domain}")
robots_url = f"https://{domain}/robots.txt"
parser = RobotFileParser()
parser.set_url(robots_url)
try:
async with aiohttp.ClientSession() as session:
async with session.get(
robots_url,
headers={"User-Agent": self.user_agent},
timeout=aiohttp.ClientTimeout(total=5)
) as response:
if response.status == 200:
text = await response.text()
parser.parse(text.splitlines())
self.robot_parsers[domain] = parser
# Extract crawl delay (ignored in fast mode)
if self.fast_mode:
self.domain_crawl_delays[domain] = 0.0
else:
delay = parser.crawl_delay(self.user_agent)
self.domain_crawl_delays[domain] = (
delay if delay is not None else self.default_crawl_delay
)
logger.info(f"Parsed robots.txt for {domain}, crawl delay: {self.domain_crawl_delays[domain]}s")
return parser
else:
logger.info(f"No robots.txt for {domain} (status {response.status})")
self.robot_parsers[domain] = None
self.domain_crawl_delays[domain] = self.default_crawl_delay
return None
except Exception as e:
logger.warning(f"Could not fetch robots.txt for {domain}: {e}")
self.robot_parsers[domain] = None
self.domain_crawl_delays[domain] = self.default_crawl_delay
return None
async def _can_fetch(self, url: str) -> bool:
"""Check if URL can be fetched according to robots.txt"""
domain = self._get_domain(url)
parser = await self._fetch_robots_txt(domain)
if parser is None:
# No robots.txt means we can fetch
return True
can_fetch = parser.can_fetch(self.user_agent, url)
if not can_fetch:
logger.warning(f"Blocked by robots.txt: {url}")
return can_fetch
async def _enforce_crawl_delay(self, domain: str):
"""Enforce crawl delay for a domain"""
delay = self.domain_crawl_delays.get(domain, self.default_crawl_delay)
last_fetched = self.domain_last_fetched.get(domain, 0)
elapsed = time.time() - last_fetched
if elapsed < delay:
sleep_time = delay - elapsed
logger.info(f"Enforcing crawl delay for {domain}: sleeping {sleep_time:.2f}s")
await asyncio.sleep(sleep_time)
self.domain_last_fetched[domain] = time.time()
async def check_media_url(
self,
url: str,
session: Optional[aiohttp.ClientSession] = None
) -> Optional[Dict[str, Any]]:
"""
Check if a URL points to media content using HEAD request.
Useful for extensionless URLs where we can't determine media type from path.
Args:
url: URL to check
session: Optional aiohttp session to reuse
Returns:
Dict with 'media_type', 'mime_type', 'content_length' if media, None otherwise
"""
try:
close_session = session is None
if session is None:
session = aiohttp.ClientSession()
try:
async with session.head(
url,
headers={"User-Agent": self.user_agent},
timeout=aiohttp.ClientTimeout(total=10),
allow_redirects=True
) as response:
if response.status != 200:
return None
content_type = response.headers.get('Content-Type', '').lower()
mime_type = content_type.split(';')[0].strip()
media_type = get_media_type_from_mime(mime_type)
if media_type:
content_length = response.headers.get('Content-Length')
return {
'media_type': media_type,
'mime_type': mime_type,
'content_length': int(content_length) if content_length else None,
'url': str(response.url) # Final URL after redirects
}
return None
finally:
if close_session:
await session.close()
except Exception as e:
logger.debug(f"HEAD check failed for {url}: {e}")
return None
async def resolve_canonical_image(
self,
detail_page_url: str,
thumbnail_url: str,
embedding_title: Optional[str] = None,
session: Optional[aiohttp.ClientSession] = None
) -> Optional[Dict[str, Any]]:
"""
Fetch a detail page and extract the canonical (full-res) image URL.
Universal Algorithm - no hardcoded strings, purely structural detection:
1. REST pattern: If an <img> src matches detail_page_url minus last path segment
2. Wrapped links: <a> tags wrapping <img> where href has query params (versioned)
3. Download links: <a download> pointing to media
4. Path similarity: <img> src sharing path structure with detail page
5. og:image fallback
Title: Prefers embedding_title (from listing page where image was found).
Falls back to detail page metadata only if embedding_title not provided.
Args:
detail_page_url: URL of the detail page to fetch
thumbnail_url: Original thumbnail URL (to avoid returning same URL)
embedding_title: Title from the page that linked here (preferred)
session: Optional aiohttp session to reuse
Returns:
Dict with 'canonical_url', 'title', 'description', 'og_image' or None
"""
try:
close_session = session is None
if session is None:
session = aiohttp.ClientSession()
try:
# Check robots.txt
if not await self._can_fetch(detail_page_url):
return None
# Enforce crawl delay
domain = self._get_domain(detail_page_url)
await self._enforce_crawl_delay(domain)
async with session.get(
detail_page_url,
headers={"User-Agent": self.user_agent},
timeout=aiohttp.ClientTimeout(total=15),
allow_redirects=True
) as response:
if response.status != 200:
return None
html = await response.text()
# Parse detail page in thread to not block event loop
def _parse_detail_page():
soup = BeautifulSoup(html, "html.parser")
result = {
'detail_page_url': detail_page_url,
'canonical_url': None,
'embedding_title': embedding_title, # from listing page
'detail_title': None, # from detail page
'detail_content': None, # body text from detail page
'title': None, # best available
'description': None,
'og_image': None,
}
# ========================================
# COLLECT BOTH TITLES (skeleton key approach)
# ========================================
# Always extract detail page title
detail_title = None
# 1. og:title
og_title = soup.find('meta', property='og:title')
if og_title:
detail_title = og_title.get('content', '').strip()
# 2. First image alt text
if not detail_title:
for img in soup.find_all('img', alt=True):
alt = img.get('alt', '').strip()
if alt and len(alt) > 2:
detail_title = alt
break
# 3. h1 tag
if not detail_title:
h1_tag = soup.find('h1')
if h1_tag:
detail_title = h1_tag.get_text(strip=True)
# 4. title tag
if not detail_title:
title_tag = soup.find('title')
if title_tag:
detail_title = title_tag.get_text(strip=True)
result['detail_title'] = detail_title
# Primary title: prefer embedding, fallback to detail
result['title'] = embedding_title or detail_title
# ========================================
# DESCRIPTION EXTRACTION
# ========================================
og_desc = soup.find('meta', property='og:description')
if og_desc:
result['description'] = og_desc.get('content', '')
if not result['description']:
meta_desc = soup.find('meta', attrs={'name': 'description'})
if meta_desc:
result['description'] = meta_desc.get('content', '')
# Extract og:image
og_image = soup.find('meta', property='og:image')
if og_image:
result['og_image'] = og_image.get('content', '')
# ========================================
# DETAIL CONTENT EXTRACTION (body text)
# ========================================
# Extract body text from detail page for full-text searchability
# This enables Pinterest-style galleries where detail pages
# have richer descriptions than thumbnails on listing pages
body = soup.find('body')
if body:
# Remove non-content elements
for tag in body.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside']):
tag.decompose()
detail_content = body.get_text(separator=' ', strip=True)[:10000]
result['detail_content'] = detail_content
# ========================================
# CANONICAL URL DETECTION (universal patterns)
# ========================================
canonical_url = None
detail_parsed = Uri(detail_page_url)
# Pattern 1: REST-style - check if img src matches URL minus last segment
# e.g., /media/ID/details has img pointing to /media/ID
path_segments = detail_parsed.path.rstrip('/').split('/')
if len(path_segments) > 1:
parent_path = '/'.join(path_segments[:-1])
parent_url = f"{detail_parsed.scheme}://{detail_parsed.hostname}{parent_path}"
for img in soup.find_all('img', src=True):
src = img.get('src', '')
full_src = urljoin(detail_page_url, src)
if full_src == parent_url or full_src.rstrip('/') == parent_url:
if full_src != thumbnail_url:
canonical_url = full_src
break
# Pattern 2: Find <a> tags wrapping images with query params (versioned URLs)
# The href with ?param=value suggests a cache-busted/versioned canonical
if not canonical_url:
for a_tag in soup.find_all('a', href=True):
href = a_tag.get('href', '')
full_href = urljoin(detail_page_url, href)
# Must have query params (indicates versioned/timestamped)
if '?' not in full_href:
continue
# Must wrap or be near an image
img_inside = a_tag.find('img')
if not img_inside:
continue
# Check it's a media URL or same-domain endpoint
href_parsed = Uri(full_href)
media_type = get_media_type_from_extension(full_href)
if media_type == 'image':
canonical_url = full_href
break
# Same domain with query params - likely image endpoint
if href_parsed.hostname == detail_parsed.hostname or not href_parsed.hostname:
canonical_url = full_href
break
# Pattern 3: <a download> attribute - semantic HTML for downloadable content
if not canonical_url:
for a_tag in soup.find_all('a', href=True, download=True):
href = a_tag.get('href', '')
if href:
full_href = urljoin(detail_page_url, href)
media_type = get_media_type_from_extension(full_href)
if media_type == 'image':
canonical_url = full_href
break
# Pattern 4: First image sharing path structure with detail page
if not canonical_url:
for img in soup.find_all('img', src=True):
src = img.get('src', '')
full_src = urljoin(detail_page_url, src)
if full_src == thumbnail_url:
continue
img_parsed = Uri(full_src)
# Same host
if img_parsed.hostname == detail_parsed.hostname:
# Count shared path segments
detail_parts = detail_parsed.path.rstrip('/').split('/')
img_parts = img_parsed.path.rstrip('/').split('/')
common = sum(1 for d, i in zip(detail_parts, img_parts) if d == i)
# At least 2 shared segments suggests same resource
if common >= 2:
canonical_url = full_src
break
# Pattern 5: og:image fallback (if different from thumbnail)
if not canonical_url and result['og_image']:
if result['og_image'] != thumbnail_url:
canonical_url = result['og_image']
result['canonical_url'] = canonical_url
return result
result = await asyncio.to_thread(_parse_detail_page)
logger.info(f"Resolved canonical image from {detail_page_url}: {result['canonical_url']}, title: {result['title'][:50] if result['title'] else 'None'}")
return result
finally:
if close_session:
await session.close()
except Exception as e:
logger.debug(f"Failed to resolve canonical image from {detail_page_url}: {e}")
return None
def _record_timeout(self, domain: str):
"""Record a timeout for a domain. After MAX_CONSECUTIVE_TIMEOUTS, add to skip list."""
self.domain_timeout_counts[domain] = self.domain_timeout_counts.get(domain, 0) + 1
if self.domain_timeout_counts[domain] >= self.MAX_CONSECUTIVE_TIMEOUTS:
if domain not in self.skip_domains:
self.skip_domains.add(domain)
logger.warning(f"Skipping domain {domain} after {self.MAX_CONSECUTIVE_TIMEOUTS} consecutive timeouts")
def _record_success(self, domain: str):
"""Record a successful fetch, resetting timeout count."""
self.domain_timeout_counts[domain] = 0
async def fetch_media(
self,
url: str,
session: Optional[aiohttp.ClientSession] = None,
max_size: int = 100 * 1024 * 1024 # 100MB default limit
) -> Optional[Dict[str, Any]]:
"""
Fetch media content and compute MD5 hash.
Args:
url: URL of media to fetch
session: Optional aiohttp session to reuse
max_size: Maximum file size in bytes (default 100MB)
Returns:
Dict with 'data', 'md5_hash', 'mime_type', 'size', 'url' or None on failure
"""
global LAST_FETCH_ERROR
# Check if domain is in skip list
domain = self._get_domain(url)
if domain in self.skip_domains:
LAST_FETCH_ERROR = {'type': 'skip_domain', 'details': f'Domain {domain} skipped (too many timeouts)', 'url': url}
logger.debug(f"Skipping {url}: domain {domain} in skip list")
return None
# Check robots.txt
if not await self._can_fetch(url):
LAST_FETCH_ERROR = {'type': 'robots_txt', 'details': 'Blocked by robots.txt', 'url': url}
return None
# Enforce crawl delay
await self._enforce_crawl_delay(domain)
try:
close_session = session is None
if session is None:
session = aiohttp.ClientSession()
try:
async with session.get(
url,
headers={"User-Agent": self.user_agent},
timeout=aiohttp.ClientTimeout(total=self.media_timeout),
allow_redirects=True
) as response:
if response.status != 200:
LAST_FETCH_ERROR = {'type': 'http_error', 'details': f'HTTP {response.status}', 'url': url}
return None
# Check content length before downloading
content_length = response.headers.get('Content-Length')
if content_length and int(content_length) > max_size:
LAST_FETCH_ERROR = {'type': 'too_large', 'details': f'File too large: {int(content_length)} bytes', 'url': url}
logger.warning(f"Skipping {url}: file too large ({int(content_length)} bytes)")
return None
content_type = response.headers.get('Content-Type', '').lower()
mime_type = content_type.split(';')[0].strip()
# Download content
data = await response.read()
if len(data) > max_size:
LAST_FETCH_ERROR = {'type': 'too_large', 'details': f'File too large: {len(data)} bytes', 'url': url}
return None
# Compute MD5 hash
md5_hash = hashlib.md5(data).hexdigest()
# Determine media type
media_type = get_media_type_from_mime(mime_type) or get_media_type_from_extension(url)
# Success - reset timeout count
self._record_success(domain)
logger.info(f"Fetched media {url}: {len(data)} bytes, MD5: {md5_hash}, type: {media_type}")
return {
'data': data,
'md5_hash': md5_hash,
'mime_type': mime_type,
'media_type': media_type,
'size': len(data),
'url': str(response.url) # Final URL after redirects
}
finally:
if close_session:
await session.close()
except asyncio.TimeoutError:
self._record_timeout(domain)
LAST_FETCH_ERROR = {'type': 'timeout', 'details': f'Download timeout ({self.media_timeout}s)', 'url': url}
logger.error(f"Timeout fetching media {url} ({self.media_timeout}s)")
return None
except Exception as e:
LAST_FETCH_ERROR = {'type': 'unknown', 'details': str(e), 'url': url}
logger.error(f"Error fetching media {url}: {e}")
return None
async def fetch_webpage(
self,
url: str,
extract_links: bool = False,
extract_anchor_text: bool = False,
cache_check_callback = None
) -> Tuple[Optional[str], List]:
"""
Fetch a webpage, respecting robots.txt and crawl delays.
Uses cache to avoid redundant fetches and delays.
Args:
url: URL to fetch
extract_links: Whether to extract links from the page
extract_anchor_text: Whether to extract anchor text with links
cache_check_callback: Optional async callback to check external cache (e.g., SQLite3)
Should return (html, links) tuple if cached, None otherwise
Returns:
If extract_anchor_text=True: Tuple of (html_content, list of dicts with 'url' and 'anchor_text')
If extract_anchor_text=False: Tuple of (html_content, list of URLs)
Returns (None, []) if fetch fails
"""
global LAST_FETCH_ERROR
# Validate URL
if not url.startswith(("http://", "https://")):
logger.error(f"Invalid URL scheme: {url}")
return None, []
# Check external cache first (e.g., SQLite3 via WebCacheManager)
if cache_check_callback:
try:
cached_result = await cache_check_callback(url)
if cached_result:
html, links = cached_result
# If we need links but cache has none, re-extract from HTML (in thread to not block)
if extract_links and not links and html:
logger.info(f"💾 Cache hit but no links cached, re-extracting from HTML: {url}")
def _reextract_links():
soup = BeautifulSoup(html, "html.parser")
result = []
for a_tag in soup.find_all("a", href=True):
href = a_tag["href"]
if href.startswith(("javascript:", "#")):
continue
rel = a_tag.get("rel", [])
if isinstance(rel, str):
rel = rel.split()
if "nofollow" in rel:
continue
absolute_url = strip_url_fragment(urljoin(url, href))
parsed = Uri(absolute_url)
if parsed.scheme in ("http", "https"):
if extract_anchor_text:
anchor_text = a_tag.get_text(strip=True)
result.append({"url": absolute_url, "anchor_text": anchor_text})
else:
result.append(absolute_url)
return result
links = await asyncio.to_thread(_reextract_links)
logger.info(f"Re-extracted {len(links)} links from cached HTML")
else:
logger.info(f"💾 Using SQLite3 cached page: {url} (skipping robots.txt + crawl delay)")
return html, links
except Exception as e:
logger.warning(f"Cache check callback failed for {url}: {e}")
# Check in-memory cache (fallback for when no external cache available)
if url in self.page_cache:
html, cached_links, cached_time = self.page_cache[url]
# If we need links but cache has none, re-extract from HTML (in thread to not block)
if extract_links and not cached_links and html:
logger.info(f"💾 In-memory cache hit but no links, re-extracting from HTML: {url}")
def _reextract_cached_links():
soup = BeautifulSoup(html, "html.parser")
result = []
for a_tag in soup.find_all("a", href=True):
href = a_tag["href"]
if href.startswith(("javascript:", "#")):
continue
rel = a_tag.get("rel", [])
if isinstance(rel, str):
rel = rel.split()
if "nofollow" in rel:
continue
absolute_url = strip_url_fragment(urljoin(url, href))
parsed_link = Uri(absolute_url)
if parsed_link.scheme in ("http", "https"):
if extract_anchor_text:
anchor_text = a_tag.get_text(strip=True)
result.append({"url": absolute_url, "anchor_text": anchor_text})
else:
result.append(absolute_url)
return result
cached_links = await asyncio.to_thread(_reextract_cached_links)
logger.info(f"Re-extracted {len(cached_links)} links from in-memory cached HTML")
else:
logger.info(f"💾 Using in-memory cached page: {url} (cached {time.time() - cached_time:.0f}s ago, skipping crawl delay)")
return html, cached_links
# Check robots.txt
if not await self._can_fetch(url):
LAST_FETCH_ERROR = {'type': 'robots_txt', 'details': 'Blocked by robots.txt', 'url': url}
return None, []
# Enforce crawl delay (only for fresh fetches)
domain = self._get_domain(url)
await self._enforce_crawl_delay(domain)
# Fetch the page
try:
async with aiohttp.ClientSession() as session:
# HEAD request first to check content type without downloading
try:
async with session.head(
url,
headers={"User-Agent": self.user_agent},
timeout=aiohttp.ClientTimeout(total=10),
allow_redirects=True
) as head_response:
head_content_type = head_response.headers.get('Content-Type', '').lower()
mime_type = head_content_type.split(';')[0].strip()
logger.info(f"HEAD {url} -> {head_response.status}, Content-Type: {mime_type}")
# Skip images, video, audio - these need vision mode, not text fetch
if mime_type.startswith(('image/', 'video/', 'audio/')):
LAST_FETCH_ERROR = {
'type': 'binary_content',
'details': f'Content is {mime_type} - requires vision mode for images',
'url': url,
'content_type': mime_type
}
logger.info(f"Skipping {mime_type} content (not text-extractable): {url}")
return None, []
except Exception as e:
# HEAD failed, continue with GET anyway
logger.debug(f"HEAD request failed for {url}: {e}, continuing with GET")
async with session.get(
url,
headers={"User-Agent": self.user_agent},
timeout=aiohttp.ClientTimeout(total=15),
allow_redirects=True,
max_redirects=3
) as response:
if response.status != 200:
LAST_FETCH_ERROR = {'type': 'http_error', 'details': f'HTTP {response.status}', 'url': url, 'status_code': response.status}
logger.warning(f"Failed to fetch {url}: HTTP {response.status}")
return None, []
# Check content type for binary files (PDF, etc.)
content_type = response.headers.get('Content-Type', '').lower()
mime_type = content_type.split(';')[0].strip()
# Double-check for binary content that HEAD might have missed
if mime_type.startswith(('image/', 'video/', 'audio/')):
LAST_FETCH_ERROR = {
'type': 'binary_content',
'details': f'Content is {mime_type} - requires vision mode for images',
'url': url,
'content_type': mime_type
}
logger.info(f"Skipping {mime_type} content (not text-extractable): {url}")
return None, []
is_pdf = 'application/pdf' in content_type or url.lower().endswith('.pdf')
if is_pdf:
# Handle PDF: read as binary and extract text
pdf_bytes = await response.read()
logger.info(f"Fetched PDF {url} ({len(pdf_bytes)} bytes)")
pdf_text = extract_text_from_pdf(pdf_bytes)
if pdf_text:
# Wrap extracted text in minimal HTML for consistent processing
# Extract filename from URL for title
pdf_filename = url.split('/')[-1].split('?')[0] or "PDF Document"
html = f"<html><head><title>{pdf_filename}</title></head><body><pre>{pdf_text}</pre></body></html>"
logger.info(f"Extracted {len(pdf_text)} chars of text from PDF")
else:
LAST_FETCH_ERROR = {'type': 'pdf_extraction_failed', 'details': 'Could not extract text from PDF', 'url': url}
logger.warning(f"Failed to extract text from PDF: {url}")
return None, []
else:
html = await response.text()
logger.info(f"Successfully fetched {url} ({len(html)} bytes)")
# Extract links if requested (run in thread to not block event loop)
links = []
if extract_links:
def _extract_links():
result = []
soup = BeautifulSoup(html, "html.parser")
for a_tag in soup.find_all("a", href=True):
href = a_tag["href"]
if href.startswith(("javascript:", "#")):
continue
rel = a_tag.get("rel", [])
if isinstance(rel, str):
rel = rel.split()
if "nofollow" in rel:
continue
absolute_url = strip_url_fragment(urljoin(url, href))
parsed = Uri(absolute_url)
if parsed.scheme in ("http", "https"):
if extract_anchor_text:
anchor_text = a_tag.get_text(strip=True)
result.append({"url": absolute_url, "anchor_text": anchor_text})
else:
result.append(absolute_url)
return result
links = await asyncio.to_thread(_extract_links)
logger.info(f"Extracted {len(links)} links from {url}")
# Cache in memory for future fetches in this session
self.page_cache[url] = (html, links, time.time())
return html, links
except asyncio.TimeoutError:
LAST_FETCH_ERROR = {'type': 'timeout', 'details': 'Connection timeout (>15 seconds)', 'url': url}
logger.error(f"Timeout fetching {url}")
return None, []
except Exception as e:
error_str = str(e)
if 'SSL' in error_str or 'certificate' in error_str.lower():
LAST_FETCH_ERROR = {'type': 'ssl', 'details': str(e), 'url': url}
elif 'Name or service not known' in error_str or 'DNS' in error_str:
LAST_FETCH_ERROR = {'type': 'dns', 'details': 'DNS resolution failed', 'url': url}
else:
LAST_FETCH_ERROR = {'type': 'unknown', 'details': str(e), 'url': url}
logger.error(f"Error fetching {url}: {e}")
return None, []
def extract_text_content(self, html: str) -> str:
"""
Extract main text content from HTML using BeautifulSoup.
Args:
html: HTML content
Returns:
Extracted text
"""
soup = BeautifulSoup(html, "html.parser")
# Remove script and style elements
for script in soup(["script", "style"]):
script.extract()
# Block-level elements that should create paragraph breaks
import re
block_tags = {'p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
'section', 'article', 'header', 'footer', 'main',
'li', 'tr', 'blockquote', 'pre', 'br', 'hr'}
# Insert markers before/after block elements
for tag in soup.find_all(block_tags):
tag.insert_before('\n\n')
tag.insert_after('\n\n')
# Get text with spaces between inline elements
text = soup.get_text(separator=' ')
# Clean up: split on newlines, strip lines, filter empties
lines = []
for line in text.split('\n'):
# Collapse multiple spaces to single space
cleaned = re.sub(r' +', ' ', line.strip())
if cleaned:
lines.append(cleaned)
# Join with single newlines, then collapse 2+ to paragraph breaks
text = '\n'.join(lines)
text = re.sub(r'\n{2,}', '\n\n', text)
return text
def extract_title(self, html: str) -> str:
"""Extract title from HTML"""
soup = BeautifulSoup(html, "html.parser")
if soup.title and soup.title.string:
return soup.title.string.strip()
return "Untitled"
def _simple_stem(self, word: str) -> str:
"""
Simple suffix-stripping stemmer for common English suffixes.
Args:
word: Word to stem
Returns:
Stemmed word
"""
word = word.lower()
# Common suffixes in order of longest first
suffixes = ['ings', 'ing', 'ed', 'es', 's', 'ly', 'er', 'est', 'tion', 'sion', 'ness', 'ment', 'able', 'ible']
for suffix in suffixes:
if word.endswith(suffix) and len(word) > len(suffix) + 2: # Keep at least 3 chars in stem
return word[:-len(suffix)]
return word
def _score_link(self, link_url: str, anchor_text: str, query_keywords: Optional[List[str]] = None, keyword_variations: Optional[List[str]] = None) -> float:
"""
Score a link based on URL and anchor text keyword matches.
Uses both exact and stemmed matching with different scoring weights.
Scoring weights:
- Exact primary keyword match: 10.0 (URL) / 5.0 (anchor)
- Stemmed primary keyword match: 7.0 (URL) / 3.5 (anchor) [70% of exact]
- Exact variation match: 6.0 (URL) / 3.0 (anchor) [60% of exact primary]
- Stemmed variation match: 4.0 (URL) / 2.0 (anchor) [40% of exact primary]
Args:
link_url: URL of the link
anchor_text: Anchor text of the link
query_keywords: Primary keywords to match against (full score)
keyword_variations: Keyword variations/synonyms (reduced score)
Returns:
Float score (higher = more relevant to follow)
"""
if not query_keywords and not keyword_variations:
return 0.0
score = 0.0
url_lower = link_url.lower()
anchor_lower = anchor_text.lower()
# Score primary keywords (exact and stemmed matching)
if query_keywords:
for keyword in query_keywords:
keyword_lower = keyword.lower()
keyword_stem = self._simple_stem(keyword_lower)
# Exact match for keywords in URL path
if keyword_lower in url_lower:
score += 10.0
logger.debug(f"Link URL contains exact primary keyword '{keyword_lower}': {link_url}")
# Stemmed match for keywords in URL path (70% of exact)
elif keyword_stem in url_lower and keyword_stem != keyword_lower:
score += 7.0
logger.debug(f"Link URL contains stemmed primary keyword '{keyword_stem}' (from '{keyword_lower}'): {link_url}")
# Exact match for keywords in anchor text
if keyword_lower in anchor_lower:
score += 5.0
logger.debug(f"Link anchor contains exact primary keyword '{keyword_lower}': {anchor_text}")
# Stemmed match for keywords in anchor text (70% of exact)
elif keyword_stem in anchor_lower and keyword_stem != keyword_lower:
score += 3.5
logger.debug(f"Link anchor contains stemmed primary keyword '{keyword_stem}' (from '{keyword_lower}'): {anchor_text}")
# Score keyword variations (exact and stemmed matching, reduced weight)
if keyword_variations:
for variation in keyword_variations:
variation_lower = variation.lower()
variation_stem = self._simple_stem(variation_lower)
# Exact match for variations in URL path (60% of exact primary)
if variation_lower in url_lower:
score += 6.0
logger.debug(f"Link URL contains exact keyword variation '{variation_lower}': {link_url}")
# Stemmed match for variations in URL path (40% of exact primary)
elif variation_stem in url_lower and variation_stem != variation_lower:
score += 4.0
logger.debug(f"Link URL contains stemmed keyword variation '{variation_stem}' (from '{variation_lower}'): {link_url}")
# Exact match for variations in anchor text (60% of exact primary)
if variation_lower in anchor_lower:
score += 3.0
logger.debug(f"Link anchor contains exact keyword variation '{variation_lower}': {anchor_text}")
# Stemmed match for variations in anchor text (40% of exact primary)
elif variation_stem in anchor_lower and variation_stem != variation_lower:
score += 2.0
logger.debug(f"Link anchor contains stemmed keyword variation '{variation_stem}' (from '{variation_lower}'): {anchor_text}")
return score
def _score_page(self, page_data: Dict[str, str], query_keywords: Optional[List[str]] = None) -> float:
"""
Score a page based on content quality metrics and optional query relevance.
Higher scores indicate more valuable content.
Scoring factors:
- Content length (more content = higher score)
- Title quality (descriptive titles = higher score)
- URL quality (cleaner URLs = higher score)
- Content density (unique words = higher score)
- Query relevance (if keywords provided, matching content scores higher)
Args:
page_data: Dict with 'text', 'title', 'url' keys
query_keywords: Optional list of keywords from user's query for relevance scoring
Returns:
Float score (0-100+, can exceed 100 with relevance bonus)
"""
score = 0.0
# Factor 1: Content length (0-30 points - reduced from 40 to make room for relevance)
# Award points for substantial content, max at 5000 chars
text_len = len(page_data.get('text', ''))
score += min(30, text_len / 167) # 5000 chars = 30 points
# Factor 2: Title quality (0-15 points - reduced from 20)
# Descriptive titles with meaningful length get higher scores
title = page_data.get('title', '')
if title and title != "Untitled":
title_len = len(title)
if 10 <= title_len <= 100: # Sweet spot for titles
score += 15
elif 5 <= title_len < 10 or 100 < title_len <= 150:
score += 8
else:
score += 3
# Factor 3: URL quality (0-15 points - reduced from 20)
# Prefer shorter, cleaner URLs over long query-string heavy URLs
url = page_data.get('url', '')
if url:
# Penalize query strings and fragments
if '?' in url:
score += 3 # Query strings often mean dynamic/less important content
elif '#' in url:
score += 8 # Fragments are slightly better
else:
score += 15 # Clean URLs are best
# Bonus for human-readable paths
path_parts = url.split('/')
if any(len(part) > 3 and part.replace('-', '').replace('_', '').isalnum() for part in path_parts):
score += 3 # Readable path segments
# Factor 4: Content density (0-15 points - reduced from 20)
# Reward pages with substantial, non-repetitive content
words = [] # Initialize words for use in Factor 5
if text_len > 0:
# Check for unique words (simple heuristic for content quality)
words = page_data.get('text', '').lower().split()
unique_words = len(set(words))
if len(words) > 0:
uniqueness_ratio = unique_words / len(words)
score += uniqueness_ratio * 15 # Higher uniqueness = better content
# Factor 5: Query relevance (0-40 points - NEW!)
# If user's query keywords are provided, boost pages that contain them
if query_keywords and len(query_keywords) > 0:
text_lower = page_data.get('text', '').lower()
title_lower = title.lower()
url_lower = url.lower()
keyword_matches = 0
keyword_density = 0.0
for keyword in query_keywords:
keyword_lower = keyword.lower()
# Count occurrences in different sections with different weights
text_count = text_lower.count(keyword_lower)
title_count = title_lower.count(keyword_lower)
url_count = url_lower.count(keyword_lower)
if text_count > 0:
keyword_matches += 1
keyword_density += text_count
# Bonus for keywords in title (very relevant)
if title_count > 0:
score += 5 * title_count # Up to 5 points per title match
# HIGH BONUS for keywords in URL (strong signal for topic-specific pages)
# URLs are hand-crafted structure, keywords there mean this page is ABOUT that topic
if url_count > 0:
score += 15 * url_count # Up to 15 points per URL match (5x higher than before!)
logger.debug(f"URL contains keyword '{keyword_lower}' ({url_count}x) - boosting score by {15 * url_count} points")
# Award points based on what percentage of query keywords matched
if len(query_keywords) > 0:
match_ratio = keyword_matches / len(query_keywords)
score += match_ratio * 20 # Up to 20 points for matching all keywords
# Award points for keyword density (how often keywords appear)
if len(words) > 0 and keyword_density > 0:
density_score = min(10, (keyword_density / len(words)) * 1000) # Up to 10 points
score += density_score
return score
async def fetch_feeds(
self,
base_url: str,
html: str = None,
max_sitemaps: int = 10,
feed_urls: List[str] = None,
) -> List[FeedItem]:
"""
Hydra mode: Fetch RSS/Atom/Sitemap feeds.
Args:
base_url: Base URL to discover feeds from
html: Optional HTML content (if already fetched)
max_sitemaps: Maximum nested sitemaps to follow
feed_urls: Optional list of known feed URLs to fetch directly
Returns:
List of FeedItems with discovered URLs
"""
all_items = []
processed_feeds = set()
# Use provided feed URLs or discover them
if feed_urls:
# Use known feeds directly
feeds_to_check = list(feed_urls)
self._last_discovered_feeds = [] # No new discovery
elif html:
feeds_to_check = discover_feeds(html, base_url)
self._last_discovered_feeds = list(feeds_to_check) # Track discovered
else:
# Try common sitemap locations
parsed = Uri(base_url)
base = f"{parsed.scheme}://{parsed.hostname}"
if parsed.port and parsed.port not in (80, 443):
base += f":{parsed.port}"
feeds_to_check = [
base + '/sitemap.xml',
base + '/sitemap_index.xml',
base + '/feed/',
base + '/rss/',
base + '/atom.xml',
]
self._last_discovered_feeds = [] # Will track which ones actually work
# Queue for processing (includes nested sitemaps)
to_process = list(feeds_to_check)
sitemap_count = 0
while to_process and sitemap_count < max_sitemaps:
feed_url = to_process.pop(0)
if feed_url in processed_feeds:
continue
processed_feeds.add(feed_url)
try:
# Fetch feed content
async with self.session.get(
feed_url,
headers={'User-Agent': self.user_agent},
timeout=aiohttp.ClientTimeout(total=30),
allow_redirects=True,
) as response:
if response.status != 200:
logger.debug(f"Feed {feed_url} returned {response.status}")
continue
content = await response.text()
# Parse feed
items, sub_feeds = parse_feed(content, feed_url)
all_items.extend(items)
# Add nested sitemaps to queue
for sub_feed in sub_feeds:
if sub_feed not in processed_feeds:
to_process.append(sub_feed)
if items:
logger.info(f"Hydra: Found {len(items)} URLs in {feed_url}")
# Track working feeds for persistence
if not feed_urls and feed_url not in self._last_discovered_feeds:
self._last_discovered_feeds.append(feed_url)
sitemap_count += 1
except asyncio.TimeoutError:
logger.debug(f"Timeout fetching feed: {feed_url}")
except Exception as e:
logger.debug(f"Error fetching feed {feed_url}: {e}")
logger.info(f"Hydra mode: Discovered {len(all_items)} URLs from {len(processed_feeds)} feeds")
return all_items
async def fetch_with_depth(
self,
start_url: str,
depth: int = 2,
max_pages: int = 10,
query_keywords: Optional[List[str]] = None,
keyword_variations: Optional[List[str]] = None,
progress_callback = None,
cache_check_callback = None,
should_continue_callback = None, # Deprecated - kept for compatibility
user_query: Optional[str] = None, # Deprecated - kept for compatibility
mode: CrawlMode = CrawlMode.TEXT,
media_callback = None, # Callback for discovered media: async fn(media_item: Dict) -> None
page_callback = None, # Callback for page HTML: async fn(url: str, html: str) -> None
uris_total_callback = None, # Callback for URI total updates: fn(total: int) -> None
initial_visited: Optional[Set[str]] = None, # Pre-visited URLs for resume support
hydra_urls: Optional[List[str]] = None, # Priority URLs from feeds/sitemaps (hydra mode)
) -> List[Dict[str, str]]:
"""
Intelligent keyword-driven crawl strategy with domain prioritization.
Supports multiple crawl modes:
- TEXT (default): Extract text content only
- IMAGES: Collect images from pages
- VIDEOS: Collect videos from pages
- MEDIA: Collect all media (images + videos + audio)
- ALL: Uber crawl - text + all media
Crawl strategy:
1. Always fetch target URI first (the URL user asked about) and extract ALL links
2. Register links in link_registry with seen_count (boost for repeated links)
3. Sort links: ALL same-domain before ANY cross-domain
4. Crawl high-scoring links automatically up to max_pages
5. Cross-domain links only crawled if score >= 30 (exceptionally high relevance)
Example: Searching "dropbox encryption" on https://dropbox.com
- Target page fetched → extracts links (same-domain and cross-domain)
- Same-domain links with keyword matches crawled first
- Cross-domain links only crawled if score >= 30
- Continues until max_pages or runs out of high-scoring links
Args:
start_url: Starting URL
depth: How many levels deep to crawl (0 = just this page, 1 = page + links, etc.)
Use -1 for unlimited depth (full domain crawl)
max_pages: Maximum number of pages to fetch (default 10, use -1 for unlimited)
query_keywords: Optional list of primary keywords from user's query (full scoring weight)
keyword_variations: Optional list of keyword variations/synonyms (60% scoring weight)
progress_callback: Optional callback for progress updates
cache_check_callback: Optional callback for cache checking (not currently used)
should_continue_callback: DEPRECATED - no longer used
user_query: DEPRECATED - no longer used
mode: CrawlMode - what type of content to extract (default: TEXT)
media_callback: Optional async callback for discovered media items.
Called with dict containing 'url', 'media_type', 'source_page', etc.
Use this to process/store media as it's discovered.
Returns:
List of dicts with keys: url, title, text, fetched_at, score, depth, links_from_page
If mode includes media, each page dict also contains 'media' list
"""
# Handle unlimited depth/pages
unlimited_depth = (depth == -1)
unlimited_pages = (max_pages == -1)
if unlimited_depth:
depth = 999999 # Effectively unlimited
if unlimited_pages:
max_pages = 999999 # Effectively unlimited
all_pages = []
visited = set(initial_visited) if initial_visited else set()
base_domain = self._get_domain(start_url)
# Log resume info
if initial_visited:
logger.info(f"Resuming with {len(initial_visited)} previously visited pages")
# Track all discovered links: {url: {'anchor_texts': [str], 'seen_count': int, 'total_link_score': float, 'domain': str, 'is_same_domain': bool}}
link_registry = {}
# Hydra mode: Inject priority URLs from feeds/sitemaps
if hydra_urls:
logger.info(f"Hydra mode: Injecting {len(hydra_urls)} priority URLs from feeds/sitemaps")
for url in hydra_urls:
if url not in visited:
domain = self._get_domain(url)
is_same = (domain == base_domain)
link_registry[url] = {
'anchor_texts': ['[hydra]'],
'seen_count': 100, # High priority
'total_link_score': 100.0, # High score to ensure they're crawled first
'domain': domain,
'is_same_domain': is_same,
'from_hydra': True,
}
# Log if using query-aware scoring
if query_keywords or keyword_variations:
logger.info(f"Using query-aware scoring with keywords: {query_keywords}, variations: {keyword_variations}")
logger.info(f"Starting intelligent crawl with depth={depth}, max_pages={max_pages}, primary_domain={base_domain}")
# PHASE 1: Always fetch target URI first
logger.info(f"Phase 1: Fetching target URI: {start_url}")
html, links = await self.fetch_webpage(
start_url,
extract_links=True,
extract_anchor_text=bool(query_keywords), # Extract anchor text only if we have keywords
cache_check_callback=cache_check_callback # Pass through SQLite3 cache callback
)
if html is None:
logger.error(f"Failed to fetch target URI {start_url}")
return []
# TEMPORARY: If HTML is empty but we got it from cache, we can still extract text/title from it
# Eventually we should store raw HTML in cache for all pages to enable proper virtual crawling
if not html:
logger.warning(f"Got empty HTML for {start_url} (likely from cache without raw_html), cannot extract links for virtual crawl")
# For now, continue with empty HTML - we'll at least process the target page even if we can't crawl links
# TODO: Store raw_html in cache for all pages to enable full virtual crawling
# Detect and skip RSS/XML feeds
if start_url.endswith(('.xml', '.rss', '.atom', 'feed', 'feeds')) or \
'/feed' in start_url or '/rss' in start_url or 'atom.xml' in start_url:
logger.warning(f"Skipping RSS/XML feed: {start_url} (detected feed pattern)")
return []
if html.strip().startswith('<?xml') or '<rss' in html[:500] or '<feed' in html[:500]:
logger.warning(f"Skipping RSS/XML feed: {start_url} (detected XML content)")
return []
# Extract target page content (in thread to not block event loop)
def _extract_content():
return self.extract_title(html), self.extract_text_content(html)
title, text = await asyncio.to_thread(_extract_content)
target_page = {
"url": start_url,
"title": title,
"text": text,
"html": html, # Include raw HTML for cache storage (enables virtual crawling)
"fetched_at": datetime.now(timezone.utc).isoformat(),
"depth": 0,
"links_from_page": [l['url'] if isinstance(l, dict) else l for l in links] # Store for CSV
}
# Call page callback to archive raw HTML
if page_callback and html:
try:
await page_callback(start_url, html)
except Exception as e:
logger.warning(f"Page callback failed for {start_url}: {e}")
# Extract media if mode requires it
if mode in (CrawlMode.IMAGES, CrawlMode.VIDEOS, CrawlMode.MEDIA, CrawlMode.ALL):
media_items = await asyncio.to_thread(extract_media_from_html, html, start_url, mode)
target_page["media"] = media_items
logger.info(f"Extracted {len(media_items)} media items from target page")
# Call media callbacks concurrently (don't block on each one)
if media_callback:
await asyncio.gather(*[media_callback(item) for item in media_items], return_exceptions=True)
# Score the target page
target_page["score"] = self._score_page(target_page, query_keywords=query_keywords)
all_pages.append(target_page)
visited.add(start_url)
media_count = len(target_page.get("media", []))
logger.info(f"Crawled target page (depth 0, score {target_page['score']:.1f}): {len(text)} chars, {len(links)} links, {media_count} media extracted")
# Update progress callback
if progress_callback:
await progress_callback(f"📄 **Fetched page 1**\n{title[:50]}\n<{start_url}>\n~{len(text)//4} tokens (estimated)")
# PHASE 2: Register ALL links from target page (both same-domain and cross-domain)
logger.info(f"Phase 2: Registering {len(links)} links from target page")
same_domain_count = 0
cross_domain_count = 0
for link in links:
link_url, anchor_text = normalize_link(link)
# Filter: not already visited
if link_url in visited:
continue
# Track domain
link_domain = self._get_domain(link_url)
is_same_domain = (link_domain == base_domain)
if is_same_domain:
same_domain_count += 1
else:
cross_domain_count += 1
# Register or update link
if link_url not in link_registry:
link_score = self._score_link(link_url, anchor_text, query_keywords, keyword_variations) if (query_keywords or keyword_variations) else 0.0
link_registry[link_url] = {
'anchor_texts': [anchor_text],
'seen_count': 1,
'total_link_score': link_score,
'domain': link_domain,
'is_same_domain': is_same_domain
}
else:
# Link seen again - increment count and update score
link_registry[link_url]['anchor_texts'].append(anchor_text)
link_registry[link_url]['seen_count'] += 1
if query_keywords or keyword_variations:
link_score = self._score_link(link_url, anchor_text, query_keywords, keyword_variations)
link_registry[link_url]['total_link_score'] += link_score
logger.info(f"Registered {len(link_registry)} unique links from target page (same-domain: {same_domain_count}, cross-domain: {cross_domain_count})")
if uris_total_callback:
uris_total_callback(len(link_registry) + 1) # +1 for target page
# PHASE 3: Decide whether to crawl deeper
# If depth=0 or we're at max_pages, stop
if depth == 0 or (max_pages > 0 and len(all_pages) >= max_pages):
logger.info(f"Stopping crawl (depth={depth}, pages={len(all_pages)})")
return self._finalize_results(all_pages)
# If target page has high score (good keyword matches), maybe we're done
# But if it has low score, we should explore links
# Threshold: if target page scores < 50, explore links
if target_page['score'] >= 50 and query_keywords:
logger.info(f"Target page has high score ({target_page['score']:.1f}), may not need to crawl deeper")
# Still crawl a few top links if they have strong keyword matches
else:
logger.info(f"Target page has low score ({target_page['score']:.1f}), will explore keyword-matching links")
# PHASE 4: Sort links by combined score (keyword_score + seen_count_boost)
# STRATEGY: Prioritize same-domain links, only allow cross-domain if exceptionally high score
# seen_count_boost = seen_count * 2 (each additional sighting adds 2 points)
scored_links = []
for link_url, meta in link_registry.items():
seen_count_boost = (meta['seen_count'] - 1) * 2 # First sighting doesn't get boost
combined_score = meta['total_link_score'] + seen_count_boost
scored_links.append({
'url': link_url,
'anchor_texts': meta['anchor_texts'],
'seen_count': meta['seen_count'],
'link_score': meta['total_link_score'],
'combined_score': combined_score,
'domain': meta['domain'],
'is_same_domain': meta['is_same_domain']
})
# Sort by: same_domain first (True sorts before False), then by combined_score
# This ensures ALL same-domain links are processed before ANY cross-domain links
scored_links.sort(key=lambda x: (not x['is_same_domain'], -x['combined_score']))
# Log top scoring links by domain
same_domain_links = [l for l in scored_links if l['is_same_domain'] and l['combined_score'] > 0][:10]
cross_domain_links = [l for l in scored_links if not l['is_same_domain'] and l['combined_score'] > 0][:5]
if same_domain_links:
logger.info(f"Top {len(same_domain_links)} same-domain scoring links:")
for i, link in enumerate(same_domain_links, 1):
logger.info(f" {i}. [{link['combined_score']:.1f}] {link['url']} (keyword={link['link_score']:.1f}, seen={link['seen_count']}x)")
if cross_domain_links:
logger.info(f"Top {len(cross_domain_links)} cross-domain scoring links (will only crawl if score > 30):")
for i, link in enumerate(cross_domain_links, 1):
logger.info(f" {i}. [{link['combined_score']:.1f}] {link['url']} ({link['domain']}, keyword={link['link_score']:.1f})")
# PHASE 5: Crawl links iteratively for each depth level (1 to depth)
# Handle unlimited depth (-1) by using large upper bound
max_depth = depth if depth > 0 else 10000
for current_depth in range(1, max_depth + 1):
if max_pages > 0 and len(all_pages) >= max_pages:
break
remaining = "unlimited" if max_pages < 0 else str(max_pages - len(all_pages))
logger.info(f"Phase 5.{current_depth}: Crawling depth-{current_depth} links (max {remaining} more pages)")
# Re-score and sort all unvisited links
candidates = []
for link_url, meta in link_registry.items():
if link_url in visited:
continue
seen_count_boost = (meta['seen_count'] - 1) * 2
combined_score = meta['total_link_score'] + seen_count_boost
candidates.append({
'url': link_url,
'anchor_texts': meta['anchor_texts'],
'seen_count': meta['seen_count'],
'link_score': meta['total_link_score'],
'combined_score': combined_score,
'domain': meta['domain'],
'is_same_domain': meta['is_same_domain']
})
if not candidates:
logger.info(f"No more unvisited links at depth {current_depth}")
break
# Sort: same_domain first, then by combined_score
candidates.sort(key=lambda x: (not x['is_same_domain'], -x['combined_score']))
# Log top candidates for this depth
top_same = [l for l in candidates if l['is_same_domain'] and l['combined_score'] > 0][:10]
if top_same:
logger.info(f"Top {len(top_same)} depth-{current_depth} same-domain candidates:")
for i, link in enumerate(top_same, 1):
logger.info(f" {i}. [{link['combined_score']:.1f}] {link['url']} (keyword={link['link_score']:.1f}, seen={link['seen_count']}x)")
# Track pages added at this depth level
pages_at_this_depth = 0
# Pre-filter candidates to get valid URLs
valid_candidates = []
for link_data in candidates:
if max_pages > 0 and len(all_pages) + len(valid_candidates) >= max_pages:
break
# Apply score thresholds
if link_data['is_same_domain']:
if query_keywords and link_data['link_score'] <= 0:
continue
else:
if link_data['combined_score'] < 30:
continue
link_url = link_data['url']
if link_url in visited:
continue
valid_candidates.append(link_data)
visited.add(link_url) # Mark as visited to prevent duplicates in batch
if not valid_candidates:
logger.info(f"No valid candidates at depth {current_depth}")
break
# Parallel fetching with semaphore (fast_mode uses multiple workers)
semaphore = asyncio.Semaphore(self.max_page_workers)
if self.fast_mode and self.max_page_workers > 1:
logger.info(f"Parallel fetch: {len(valid_candidates)} pages with {self.max_page_workers} workers")
async def fetch_one(link_data):
"""Fetch a single page with semaphore control."""
async with semaphore:
link_url = link_data['url']
try:
html, child_links = await self.fetch_webpage(
link_url,
extract_links=True,
extract_anchor_text=bool(query_keywords),
cache_check_callback=cache_check_callback
)
return {'url': link_url, 'html': html, 'child_links': child_links, 'link_data': link_data, 'error': None}
except Exception as e:
return {'url': link_url, 'html': None, 'child_links': [], 'link_data': link_data, 'error': str(e)}
# Fetch pages in parallel, process each as it completes (fractal/streaming)
fetch_tasks = [asyncio.create_task(fetch_one(c)) for c in valid_candidates]
logger.info(f"Streaming {len(fetch_tasks)} fetches (processing as each completes)")
# Process results as they arrive (not waiting for all to complete)
processed_count = 0
for coro in asyncio.as_completed(fetch_tasks):
result = await coro
if max_pages > 0 and len(all_pages) >= max_pages:
logger.info(f"Reached max_pages limit ({max_pages}), stopping")
break
link_url = result['url']
html = result['html']
child_links = result['child_links']
link_data = result['link_data']
processed_count += 1
if not html:
if result['error']:
logger.warning(f"Skipping {link_url} (fetch failed: {result['error']})")
else:
logger.warning(f"Skipping {link_url} (fetch failed)")
continue
# Skip RSS/XML feeds
if link_url.endswith(('.xml', '.rss', '.atom', 'feed', 'feeds')) or \
'/feed' in link_url or '/rss' in link_url or 'atom.xml' in link_url:
logger.warning(f"Skipping RSS/XML feed: {link_url}")
continue
if html.strip().startswith('<?xml') or '<rss' in html[:500] or '<feed' in html[:500]:
logger.warning(f"Skipping RSS/XML feed: {link_url}")
continue
# Extract content (in thread to not block event loop)
def _extract_page_content():
return self.extract_title(html), self.extract_text_content(html)
title, text = await asyncio.to_thread(_extract_page_content)
page_data = {
"url": link_url,
"title": title,
"text": text,
"html": html,
"fetched_at": datetime.now(timezone.utc).isoformat(),
"depth": current_depth,
"links_from_page": [l['url'] if isinstance(l, dict) else l for l in child_links]
}
# Call page callback to archive raw HTML
if page_callback and html:
try:
await page_callback(link_url, html)
except Exception as e:
logger.warning(f"Page callback failed for {link_url}: {e}")
# Extract media if mode requires it
if mode in (CrawlMode.IMAGES, CrawlMode.VIDEOS, CrawlMode.MEDIA, CrawlMode.ALL):
media_items = await asyncio.to_thread(extract_media_from_html, html, link_url, mode)
page_data["media"] = media_items
# Call media callbacks concurrently (don't block on each one)
if media_callback:
await asyncio.gather(*[media_callback(item) for item in media_items], return_exceptions=True)
page_data["score"] = self._score_page(page_data, query_keywords=query_keywords)
all_pages.append(page_data)
pages_at_this_depth += 1
media_count = len(page_data.get("media", []))
logger.info(f"Crawled {link_url} (depth {current_depth}, score {page_data['score']:.1f}): {len(text)} chars, {len(child_links)} links, {media_count} media")
if progress_callback:
await progress_callback(f"📄 **Fetched page {len(all_pages)}** (depth {current_depth})\n{title[:50]}\n<{link_url}>")
# Register newly discovered links for next depth level
for link in child_links:
child_url, anchor_text = normalize_link(link)
if child_url in visited:
continue
child_domain = self._get_domain(child_url)
is_same_domain = (child_domain == base_domain)
if child_url not in link_registry:
link_score = self._score_link(child_url, anchor_text, query_keywords, keyword_variations) if (query_keywords or keyword_variations) else 0.0
link_registry[child_url] = {
'anchor_texts': [anchor_text],
'seen_count': 1,
'total_link_score': link_score,
'domain': child_domain,
'is_same_domain': is_same_domain
}
else:
link_registry[child_url]['anchor_texts'].append(anchor_text)
link_registry[child_url]['seen_count'] += 1
if query_keywords or keyword_variations:
link_score = self._score_link(child_url, anchor_text, query_keywords, keyword_variations)
link_registry[child_url]['total_link_score'] += link_score
# Update total after processing child links
if uris_total_callback:
uris_total_callback(len(link_registry) + 1)
logger.info(f"Depth {current_depth} complete: processed {processed_count}/{len(fetch_tasks)} results, crawled {pages_at_this_depth} pages")
# Stop if no pages were crawled at this depth (all links exhausted or skipped)
if pages_at_this_depth == 0:
logger.info("No pages crawled at this depth - stopping crawl")
break
return self._finalize_results(all_pages)
def _finalize_results(self, all_pages: List[Dict]) -> List[Dict]:
"""Sort and finalize crawl results"""
# Sort by depth first (main page first), then by score within each depth
all_pages.sort(key=lambda p: (p.get("depth", 0), -p["score"]))
logger.info(f"Crawl complete: fetched {len(all_pages)} pages")
if len(all_pages) > 1:
logger.info(f"Score range: {all_pages[-1]['score']:.1f} to {all_pages[0]['score']:.1f}")
return all_pages
def select_pages_by_token_budget(
self,
pages: list,
token_budget: int,
reserve_tokens: int = 1000
) -> list:
"""
Select highest-value pages that fit within token budget using greedy knapsack algorithm.
Args:
pages: List of page dicts with 'score', 'tokens', 'url', 'title', 'text' keys
token_budget: Maximum tokens available for page content
reserve_tokens: Tokens to reserve for output and system prompts
Returns:
List of selected pages (sorted by score descending) that fit in budget
"""
if not pages:
return []
# Adjust budget for reserved tokens
effective_budget = max(0, token_budget - reserve_tokens)
if effective_budget <= 0:
logger.warning(f"Token budget too low after reserving {reserve_tokens} tokens")
return []
# Calculate value/token ratio for each page (efficiency metric)
pages_with_ratio = []
for page in pages:
tokens = page.get('tokens', len(page.get('text', '')) // 4) # Fallback estimate
score = page.get('score', 0)
if tokens > 0:
ratio = score / tokens
pages_with_ratio.append({
**page,
'tokens': tokens,
'value_per_token': ratio
})
if not pages_with_ratio:
return []
# Greedy algorithm: Sort by value/token ratio (descending)
# This maximizes value while minimizing token usage
pages_with_ratio.sort(key=lambda p: p['value_per_token'], reverse=True)
# Select pages greedily until budget exhausted
selected = []
total_tokens = 0
for page in pages_with_ratio:
page_tokens = page['tokens']
if total_tokens + page_tokens <= effective_budget:
selected.append(page)
total_tokens += page_tokens
logger.debug(f"Selected page: {page['title']} (score={page['score']:.1f}, tokens={page_tokens}, ratio={page['value_per_token']:.3f})")
else:
logger.debug(f"Skipped page: {page['title']} (would exceed budget: {total_tokens + page_tokens} > {effective_budget})")
# Sort selected pages by score (highest first) for presentation
selected.sort(key=lambda p: p['score'], reverse=True)
logger.info(f"Selected {len(selected)}/{len(pages)} pages using {total_tokens}/{effective_budget} tokens (budget utilization: {total_tokens/effective_budget*100:.1f}%)")
return selected
if __name__ == "__main__":
# Test the async web fetcher
async def test():
logging.basicConfig(level=logging.INFO)
fetcher = AsyncWebFetcher()
# Test single page fetch
print("Testing single page fetch...")
results = await fetcher.fetch_with_depth("https://example.com", depth=0)
if results:
print(f"\nFetched: {results[0]['title']}")
print(f"Content length: {len(results[0]['text'])} chars")
print(f"Preview: {results[0]['text'][:200]}...")
else:
print("Failed to fetch")
asyncio.run(test())
Reference in a new issue View git blame Copy permalink