Russell Ballestrini
416b3cb760
- Bootstrap C program extracts serve.py and runs from tarball - Archives now include neopig source files for self-contained crawling - --upgrade-neopig flag to update neopig in existing archives - html2md.py: smart HTML-to-markdown converter for forums/blogs/Q&A - Fix vault path defaults (data/vault instead of vault) - Streaming tar.gz creation without temp copies - URL rewriting for local media references in archives
2305 lines
98 KiB
Python
2305 lines
98 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Async Web Fetcher for Discord Bot
|
|
Ethical web scraping with robots.txt compliance, async/await compatible
|
|
|
|
Supports multiple crawl modes:
|
|
- text: Extract text content (default, current behavior)
|
|
- images: Collect images only
|
|
- videos: Collect videos only
|
|
- media: Collect all media (images + videos + audio)
|
|
- all: Uber crawl - text + all media, full domain slurp
|
|
"""
|
|
|
|
import os
|
|
import io
|
|
import hashlib
|
|
import logging
|
|
import aiohttp
|
|
import asyncio
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from enum import Enum
|
|
from typing import Optional, Dict, List, Tuple, Set, Any
|
|
from urllib.parse import urlparse, urljoin
|
|
from urllib.robotparser import RobotFileParser
|
|
from bs4 import BeautifulSoup
|
|
import time
|
|
|
|
from miniuri import Uri
|
|
|
|
|
|
class CrawlMode(Enum):
|
|
"""Crawl modes for different content types."""
|
|
TEXT = "text" # Extract text content (default)
|
|
IMAGES = "images" # Collect images only
|
|
VIDEOS = "videos" # Collect videos only
|
|
MEDIA = "media" # All media (images + videos + audio)
|
|
ALL = "all" # Uber crawl: text + all media
|
|
|
|
|
|
# File extensions by media type
|
|
IMAGE_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.bmp', '.ico', '.tiff', '.tif', '.avif'}
|
|
VIDEO_EXTENSIONS = {'.mp4', '.webm', '.mov', '.avi', '.mkv', '.m4v', '.ogv', '.flv', '.wmv'}
|
|
AUDIO_EXTENSIONS = {'.mp3', '.wav', '.ogg', '.m4a', '.flac', '.aac', '.wma'}
|
|
|
|
# MIME types by media type
|
|
IMAGE_MIME_PREFIXES = ('image/',)
|
|
VIDEO_MIME_PREFIXES = ('video/',)
|
|
AUDIO_MIME_PREFIXES = ('audio/',)
|
|
|
|
|
|
@dataclass
|
|
class MediaItem:
|
|
"""Represents a discovered media item."""
|
|
url: str
|
|
source_page: str
|
|
media_type: str # 'image', 'video', 'audio'
|
|
mime_type: Optional[str] = None
|
|
md5_hash: Optional[str] = None
|
|
file_size: Optional[int] = None
|
|
alt_text: Optional[str] = None
|
|
title: Optional[str] = None
|
|
width: Optional[int] = None
|
|
height: Optional[int] = None
|
|
discovered_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
|
|
|
|
|
|
class MediaMetadata:
|
|
"""
|
|
Accumulates metadata from all sources without clobbering.
|
|
|
|
"Never clobber, always append" - collects ALL metadata from:
|
|
- img.alt, img.title
|
|
- a.title, a.text (link text)
|
|
- figcaption
|
|
- nearby headings
|
|
- page title/h1
|
|
|
|
Produces a combined searchable_text for full-text search.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.titles: List[str] = []
|
|
self.alt_texts: List[str] = []
|
|
self.descriptions: List[str] = []
|
|
self.captions: List[str] = []
|
|
self.headings: List[str] = []
|
|
self.link_texts: List[str] = []
|
|
self.link_titles: List[str] = []
|
|
|
|
def _add_unique(self, lst: List[str], value: str, max_len: int = 500) -> None:
|
|
"""Add value to list if non-empty and not duplicate."""
|
|
if value and value.strip():
|
|
clean = value.strip()[:max_len]
|
|
if clean not in lst:
|
|
lst.append(clean)
|
|
|
|
def add_img_alt(self, alt: str) -> None:
|
|
"""Add img alt attribute."""
|
|
self._add_unique(self.alt_texts, alt)
|
|
|
|
def add_img_title(self, title: str) -> None:
|
|
"""Add img title attribute."""
|
|
self._add_unique(self.titles, title)
|
|
|
|
def add_link_title(self, title: str) -> None:
|
|
"""Add <a title="..."> attribute."""
|
|
self._add_unique(self.link_titles, title)
|
|
|
|
def add_link_text(self, text: str) -> None:
|
|
"""Add <a> inner text."""
|
|
self._add_unique(self.link_texts, text)
|
|
|
|
def add_figcaption(self, caption: str) -> None:
|
|
"""Add figcaption text."""
|
|
self._add_unique(self.captions, caption)
|
|
|
|
def add_heading(self, heading: str) -> None:
|
|
"""Add nearby heading (h1-h6)."""
|
|
self._add_unique(self.headings, heading)
|
|
|
|
def add_description(self, desc: str) -> None:
|
|
"""Add description (og:description, meta description, etc.)."""
|
|
self._add_unique(self.descriptions, desc)
|
|
|
|
def add_page_title(self, title: str) -> None:
|
|
"""Add page title."""
|
|
self._add_unique(self.titles, title)
|
|
|
|
def get_best_title(self) -> Optional[str]:
|
|
"""Get best title for display (first non-empty)."""
|
|
for lst in [self.titles, self.alt_texts, self.link_titles,
|
|
self.captions, self.link_texts, self.headings]:
|
|
if lst:
|
|
return lst[0]
|
|
return None
|
|
|
|
def get_best_alt(self) -> Optional[str]:
|
|
"""Get best alt text for accessibility."""
|
|
if self.alt_texts:
|
|
return self.alt_texts[0]
|
|
return self.get_best_title()
|
|
|
|
def to_searchable_text(self) -> str:
|
|
"""
|
|
Combine ALL collected metadata into searchable text.
|
|
|
|
This enables finding images by ANY associated text:
|
|
- "find images of cats" matches img alt="cute cat"
|
|
- "find images from blog post about python" matches page content
|
|
"""
|
|
all_parts = []
|
|
# Dedupe while preserving order
|
|
seen = set()
|
|
for lst in [self.titles, self.alt_texts, self.descriptions,
|
|
self.captions, self.headings, self.link_texts, self.link_titles]:
|
|
for item in lst:
|
|
if item and item not in seen:
|
|
all_parts.append(item)
|
|
seen.add(item)
|
|
return ' | '.join(all_parts)
|
|
|
|
def to_dict(self) -> Dict[str, Any]:
|
|
"""Export all collected metadata as dict."""
|
|
return {
|
|
'titles': self.titles,
|
|
'alt_texts': self.alt_texts,
|
|
'descriptions': self.descriptions,
|
|
'captions': self.captions,
|
|
'headings': self.headings,
|
|
'link_texts': self.link_texts,
|
|
'link_titles': self.link_titles,
|
|
'searchable_text': self.to_searchable_text(),
|
|
'best_title': self.get_best_title(),
|
|
'best_alt': self.get_best_alt(),
|
|
}
|
|
|
|
|
|
# PDF text extraction
|
|
try:
|
|
from pypdf import PdfReader
|
|
PDF_SUPPORT = True
|
|
except ImportError:
|
|
PDF_SUPPORT = False
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Default crawl delay
|
|
DEFAULT_CRAWL_DELAY = 2.0
|
|
|
|
# Global last error storage (for error reporting without log parsing)
|
|
# Format: {'type': 'robots_txt'|'http_error'|'timeout'|'dns'|'ssl'|'unknown', 'details': str, 'url': str}
|
|
LAST_FETCH_ERROR = None
|
|
|
|
|
|
def get_last_fetch_error() -> Optional[Dict]:
|
|
"""Get the last fetch error, if any."""
|
|
return LAST_FETCH_ERROR
|
|
|
|
|
|
def strip_uri_fragment(uri_str: str) -> str:
|
|
"""
|
|
Strip the fragment (#anchor) from a URI using miniuri.
|
|
|
|
Args:
|
|
uri_str: URI that may contain a fragment
|
|
|
|
Returns:
|
|
URI without fragment
|
|
"""
|
|
if not uri_str:
|
|
return uri_str
|
|
|
|
uri = Uri(uri_str)
|
|
# Reconstruct URI without fragment
|
|
result = f"{uri.scheme}://{uri.authority}{uri.path or ''}"
|
|
if uri.query:
|
|
result += f"?{uri.query}"
|
|
return result
|
|
|
|
|
|
def normalize_link(link) -> Tuple[str, str]:
|
|
"""
|
|
Extract URI and anchor text from a link, stripping fragments.
|
|
|
|
Args:
|
|
link: Either a dict with 'url' and optional 'anchor_text', or a string URI
|
|
|
|
Returns:
|
|
Tuple of (normalized_uri, anchor_text)
|
|
"""
|
|
if isinstance(link, dict):
|
|
uri = strip_uri_fragment(link.get('url', ''))
|
|
anchor_text = link.get('anchor_text', '')
|
|
else:
|
|
uri = strip_uri_fragment(link)
|
|
anchor_text = ''
|
|
return uri, anchor_text
|
|
|
|
|
|
# Alias for backwards compatibility
|
|
strip_url_fragment = strip_uri_fragment
|
|
|
|
|
|
def extract_text_from_pdf(pdf_bytes: bytes) -> Optional[str]:
|
|
"""
|
|
Extract text from PDF binary data.
|
|
|
|
Args:
|
|
pdf_bytes: Raw PDF file bytes
|
|
|
|
Returns:
|
|
Extracted text or None if extraction fails
|
|
"""
|
|
if not PDF_SUPPORT:
|
|
logger.warning("PDF support not available (pypdf not installed)")
|
|
return None
|
|
|
|
try:
|
|
reader = PdfReader(io.BytesIO(pdf_bytes))
|
|
text_parts = []
|
|
for page in reader.pages:
|
|
page_text = page.extract_text()
|
|
if page_text:
|
|
text_parts.append(page_text)
|
|
if text_parts:
|
|
return "\n\n".join(text_parts)
|
|
return None
|
|
except Exception as e:
|
|
logger.error(f"Failed to extract text from PDF: {e}")
|
|
return None
|
|
|
|
|
|
def clear_last_fetch_error():
|
|
"""Clear the last fetch error."""
|
|
global LAST_FETCH_ERROR
|
|
LAST_FETCH_ERROR = None
|
|
|
|
|
|
async def fetch_robots_txt(url: str, user_agent: str = "uncloseai.com/1.42") -> Optional[str]:
|
|
"""
|
|
Fetch raw robots.txt content for analysis and error reporting.
|
|
|
|
Args:
|
|
url: Any URL on the domain (robots.txt will be fetched from root)
|
|
user_agent: User agent string
|
|
|
|
Returns:
|
|
Raw robots.txt content or None if not available
|
|
"""
|
|
parsed = urlparse(url)
|
|
robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
|
|
|
|
try:
|
|
async with aiohttp.ClientSession() as session:
|
|
async with session.get(
|
|
robots_url,
|
|
headers={"User-Agent": user_agent},
|
|
timeout=aiohttp.ClientTimeout(total=5)
|
|
) as response:
|
|
if response.status == 200:
|
|
return await response.text()
|
|
except Exception as e:
|
|
logger.warning(f"Could not fetch robots.txt for error analysis: {e}")
|
|
|
|
return None
|
|
|
|
|
|
def analyze_robots_txt_blocking(robots_txt: str, blocked_url: str, user_agent: str = "uncloseai.com/1.42") -> str:
|
|
"""
|
|
Analyze robots.txt to explain why a URL is blocked.
|
|
|
|
Args:
|
|
robots_txt: Raw robots.txt content
|
|
blocked_url: The URL that was blocked
|
|
user_agent: User agent to check against
|
|
|
|
Returns:
|
|
Human-readable explanation of the blocking rule
|
|
"""
|
|
parsed = urlparse(blocked_url)
|
|
path = parsed.path or "/"
|
|
|
|
# Parse robots.txt manually for detailed analysis
|
|
lines = robots_txt.strip().split('\n')
|
|
current_agent = None
|
|
blocking_rule = None
|
|
all_rules = []
|
|
|
|
for line in lines:
|
|
line = line.strip()
|
|
if not line or line.startswith('#'):
|
|
continue
|
|
|
|
if ':' in line:
|
|
key, value = line.split(':', 1)
|
|
key = key.strip().lower()
|
|
value = value.strip()
|
|
|
|
if key == 'user-agent':
|
|
current_agent = value
|
|
elif key == 'disallow' and current_agent:
|
|
# Check if this rule applies to us
|
|
agent_matches = (
|
|
current_agent == '*' or
|
|
user_agent.lower().startswith(current_agent.lower()) or
|
|
current_agent.lower() in user_agent.lower()
|
|
)
|
|
if agent_matches:
|
|
all_rules.append(f"Disallow: {value}")
|
|
# Check if this rule blocks our path
|
|
if value and path.startswith(value):
|
|
blocking_rule = f"Disallow: {value}"
|
|
|
|
if blocking_rule:
|
|
result = f"**Blocked by rule:** `{blocking_rule}`\n"
|
|
result += f"**Path requested:** `{path}`\n"
|
|
result += f"**User-Agent:** `{user_agent}`\n\n"
|
|
|
|
# Show relevant rules
|
|
if all_rules:
|
|
result += "**Applicable rules:**\n"
|
|
for rule in all_rules[:10]: # Limit to first 10
|
|
result += f" • `{rule}`\n"
|
|
|
|
return result
|
|
elif all_rules:
|
|
return f"Blocked by robots.txt (exact rule unclear). Rules found:\n" + "\n".join(f" • `{r}`" for r in all_rules[:10])
|
|
else:
|
|
return "Blocked by robots.txt (no matching disallow rule found - may be a catch-all)"
|
|
|
|
|
|
def get_media_type_from_extension(url: str) -> Optional[str]:
|
|
"""
|
|
Determine media type from URL extension.
|
|
|
|
Returns:
|
|
'image', 'video', 'audio', or None
|
|
"""
|
|
parsed = urlparse(url)
|
|
path = parsed.path.lower()
|
|
|
|
for ext in IMAGE_EXTENSIONS:
|
|
if path.endswith(ext):
|
|
return 'image'
|
|
for ext in VIDEO_EXTENSIONS:
|
|
if path.endswith(ext):
|
|
return 'video'
|
|
for ext in AUDIO_EXTENSIONS:
|
|
if path.endswith(ext):
|
|
return 'audio'
|
|
return None
|
|
|
|
|
|
def get_media_type_from_mime(mime_type: str) -> Optional[str]:
|
|
"""
|
|
Determine media type from MIME type.
|
|
|
|
Returns:
|
|
'image', 'video', 'audio', or None
|
|
"""
|
|
if not mime_type:
|
|
return None
|
|
|
|
mime_lower = mime_type.lower()
|
|
if mime_lower.startswith(IMAGE_MIME_PREFIXES):
|
|
return 'image'
|
|
if mime_lower.startswith(VIDEO_MIME_PREFIXES):
|
|
return 'video'
|
|
if mime_lower.startswith(AUDIO_MIME_PREFIXES):
|
|
return 'audio'
|
|
return None
|
|
|
|
|
|
def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMode.MEDIA) -> List[Dict[str, Any]]:
|
|
"""
|
|
Extract media URLs from HTML content.
|
|
|
|
Extracts from:
|
|
- <img src="..."> and <img srcset="...">
|
|
- <picture><source srcset="..."></picture>
|
|
- <video src="..."> and <video poster="...">
|
|
- <source src="..."> (within video/audio)
|
|
- <audio src="...">
|
|
- CSS background-image: url(...)
|
|
- <a href="..."> pointing to media files
|
|
- data-src, data-lazy-src (lazy loading)
|
|
- Open Graph and Twitter card meta tags
|
|
|
|
Args:
|
|
html: HTML content
|
|
base_url: Base URL for resolving relative URLs
|
|
mode: CrawlMode to filter what types to extract
|
|
|
|
Returns:
|
|
List of dicts with 'url', 'media_type', 'alt_text', 'title', etc.
|
|
"""
|
|
media_items = []
|
|
seen_urls: Set[str] = set()
|
|
|
|
# Determine which types to collect based on mode
|
|
collect_images = mode in (CrawlMode.IMAGES, CrawlMode.MEDIA, CrawlMode.ALL)
|
|
collect_videos = mode in (CrawlMode.VIDEOS, CrawlMode.MEDIA, CrawlMode.ALL)
|
|
collect_audio = mode in (CrawlMode.MEDIA, CrawlMode.ALL)
|
|
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
|
|
# Extract page-level metadata for inference
|
|
page_title = ''
|
|
title_tag = soup.find('title')
|
|
if title_tag:
|
|
page_title = title_tag.get_text(strip=True)
|
|
|
|
# Get h1 as fallback title
|
|
h1_tag = soup.find('h1')
|
|
page_h1 = h1_tag.get_text(strip=True) if h1_tag else ''
|
|
|
|
# Extract meta description and keywords for searchability
|
|
page_description = ''
|
|
page_keywords = ''
|
|
for meta in soup.find_all('meta'):
|
|
name = (meta.get('name') or meta.get('property') or '').lower()
|
|
content = meta.get('content', '')
|
|
if name == 'description' or name == 'og:description':
|
|
page_description = content[:500] # Limit length
|
|
elif name == 'keywords':
|
|
page_keywords = content[:500]
|
|
|
|
# Extract page content (body text) for full-text search
|
|
# This enables blog images to be searchable by post content
|
|
page_content = ''
|
|
body = soup.find('body')
|
|
if body:
|
|
# Remove script, style, nav, footer elements
|
|
for tag in body.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside']):
|
|
tag.decompose()
|
|
page_content = body.get_text(separator=' ', strip=True)[:10000] # Limit to 10k chars
|
|
|
|
def get_context_for_element(element) -> dict:
|
|
"""Extract contextual metadata from surrounding HTML elements."""
|
|
context = {
|
|
'page_title': page_title,
|
|
'page_h1': page_h1,
|
|
'figure_caption': None,
|
|
'nearby_heading': None,
|
|
'link_text': None,
|
|
'link_title': None, # <a title="..."> attribute
|
|
}
|
|
|
|
# Check if inside a <figure> with <figcaption>
|
|
figure = element.find_parent('figure')
|
|
if figure:
|
|
figcaption = figure.find('figcaption')
|
|
if figcaption:
|
|
context['figure_caption'] = figcaption.get_text(strip=True)[:200]
|
|
|
|
# Check if inside an <a> tag with text and/or title
|
|
link = element.find_parent('a')
|
|
if link:
|
|
link_text = link.get_text(strip=True)
|
|
if link_text and link_text != element.get('alt', ''):
|
|
context['link_text'] = link_text[:200]
|
|
# Also extract title attribute from <a> tag (tooltip text)
|
|
link_title = link.get('title', '').strip()
|
|
if link_title:
|
|
context['link_title'] = link_title[:200]
|
|
|
|
# Find nearest heading (h1-h6) before this element
|
|
for heading_tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
|
|
# Look for preceding siblings and parents
|
|
for prev in element.find_all_previous(heading_tag, limit=3):
|
|
heading_text = prev.get_text(strip=True)
|
|
if heading_text:
|
|
context['nearby_heading'] = heading_text[:200]
|
|
break
|
|
if context['nearby_heading']:
|
|
break
|
|
|
|
return context
|
|
|
|
def infer_title(alt_text: str, title: str, context: dict) -> str:
|
|
"""Infer best title from available metadata."""
|
|
# Priority: explicit title > alt text > link title > figure caption > link text > nearby heading > page h1 > page title
|
|
if title and title.strip():
|
|
return title.strip()
|
|
if alt_text and alt_text.strip() and len(alt_text) > 3:
|
|
return alt_text.strip()
|
|
if context.get('link_title'): # <a title="..."> attribute
|
|
return context['link_title']
|
|
if context.get('figure_caption'):
|
|
return context['figure_caption']
|
|
if context.get('link_text'):
|
|
return context['link_text']
|
|
if context.get('nearby_heading'):
|
|
return context['nearby_heading']
|
|
if context.get('page_h1'):
|
|
return context['page_h1']
|
|
if context.get('page_title'):
|
|
return context['page_title']
|
|
return None
|
|
|
|
def add_media(url: str, media_type: str, alt_text: str = None, title: str = None, width: int = None, height: int = None, element=None, detail_page_url: str = None):
|
|
"""Helper to add media item if not already seen."""
|
|
if not url or url in seen_urls:
|
|
return
|
|
if url.startswith('data:'): # Skip data URIs for now
|
|
return
|
|
|
|
# Resolve relative URLs
|
|
absolute_url = urljoin(base_url, url)
|
|
if absolute_url in seen_urls:
|
|
return
|
|
|
|
seen_urls.add(absolute_url)
|
|
|
|
# Get context from surrounding elements
|
|
context = get_context_for_element(element) if element else {'page_title': page_title, 'page_h1': page_h1}
|
|
|
|
# Build metadata accumulator - "never clobber, always append"
|
|
metadata = MediaMetadata()
|
|
|
|
# Add from img element
|
|
if alt_text:
|
|
metadata.add_img_alt(alt_text)
|
|
if title:
|
|
metadata.add_img_title(title)
|
|
|
|
# Add from context
|
|
if context.get('figure_caption'):
|
|
metadata.add_figcaption(context['figure_caption'])
|
|
if context.get('link_text'):
|
|
metadata.add_link_text(context['link_text'])
|
|
if context.get('link_title'):
|
|
metadata.add_link_title(context['link_title'])
|
|
if context.get('nearby_heading'):
|
|
metadata.add_heading(context['nearby_heading'])
|
|
if context.get('page_title'):
|
|
metadata.add_page_title(context['page_title'])
|
|
if context.get('page_h1'):
|
|
metadata.add_heading(context['page_h1'])
|
|
|
|
# Add page-level metadata
|
|
if page_description:
|
|
metadata.add_description(page_description)
|
|
|
|
# Get best values for backward compatibility
|
|
best_title = metadata.get_best_title()
|
|
best_alt = metadata.get_best_alt()
|
|
|
|
# Check if this image is wrapped in an <a> tag pointing to a detail page
|
|
# (Pinterest-style galleries where thumbnail links to detail page with canonical image)
|
|
resolved_detail_url = None
|
|
if element and not detail_page_url:
|
|
parent_link = element.find_parent('a', href=True)
|
|
if parent_link:
|
|
href = parent_link.get('href', '')
|
|
# Only consider internal links (not direct image links)
|
|
if href and not get_media_type_from_extension(href):
|
|
resolved_detail_url = urljoin(base_url, href)
|
|
# Only track same-domain detail pages
|
|
base_domain = urlparse(base_url).netloc
|
|
detail_domain = urlparse(resolved_detail_url).netloc
|
|
if base_domain != detail_domain:
|
|
resolved_detail_url = None
|
|
elif detail_page_url:
|
|
resolved_detail_url = urljoin(base_url, detail_page_url)
|
|
|
|
media_items.append({
|
|
'url': absolute_url,
|
|
'media_type': media_type,
|
|
'alt_text': best_alt,
|
|
'title': best_title,
|
|
'width': width,
|
|
'height': height,
|
|
'source_page': base_url,
|
|
'page_title': page_title,
|
|
'page_description': page_description,
|
|
'page_keywords': page_keywords,
|
|
'page_content': page_content, # Full text for blog post searchability
|
|
'figure_caption': context.get('figure_caption'),
|
|
'nearby_heading': context.get('nearby_heading'),
|
|
'link_text': context.get('link_text'),
|
|
'link_title': context.get('link_title'), # <a title="..."> attribute
|
|
'detail_page_url': resolved_detail_url, # URL to fetch for canonical image + richer metadata
|
|
# Accumulated metadata for full-text search
|
|
'searchable_text': metadata.to_searchable_text(),
|
|
'metadata': metadata.to_dict(), # Full breakdown for debugging/analysis
|
|
})
|
|
|
|
# Extract from <img> tags
|
|
if collect_images:
|
|
for img in soup.find_all('img'):
|
|
src = img.get('src')
|
|
alt = img.get('alt', '')
|
|
title = img.get('title', '')
|
|
width = img.get('width')
|
|
height = img.get('height')
|
|
|
|
if src:
|
|
add_media(src, 'image', alt, title,
|
|
int(width) if width and width.isdigit() else None,
|
|
int(height) if height and height.isdigit() else None,
|
|
element=img)
|
|
|
|
# Handle srcset
|
|
srcset = img.get('srcset', '')
|
|
for src_entry in srcset.split(','):
|
|
src_entry = src_entry.strip()
|
|
if src_entry:
|
|
src_url = src_entry.split()[0] # Get URL part, ignore size descriptor
|
|
add_media(src_url, 'image', alt, title, element=img)
|
|
|
|
# Lazy loading attributes
|
|
for attr in ['data-src', 'data-lazy-src', 'data-original', 'data-srcset']:
|
|
lazy_src = img.get(attr)
|
|
if lazy_src:
|
|
if attr.endswith('srcset'):
|
|
for src_entry in lazy_src.split(','):
|
|
src_entry = src_entry.strip()
|
|
if src_entry:
|
|
add_media(src_entry.split()[0], 'image', alt, title, element=img)
|
|
else:
|
|
add_media(lazy_src, 'image', alt, title, element=img)
|
|
|
|
# Extract from <picture><source> tags
|
|
if collect_images:
|
|
for picture in soup.find_all('picture'):
|
|
for source in picture.find_all('source'):
|
|
srcset = source.get('srcset', '')
|
|
for src_entry in srcset.split(','):
|
|
src_entry = src_entry.strip()
|
|
if src_entry:
|
|
add_media(src_entry.split()[0], 'image')
|
|
|
|
# Extract from <video> tags
|
|
if collect_videos:
|
|
for video in soup.find_all('video'):
|
|
src = video.get('src')
|
|
poster = video.get('poster')
|
|
title = video.get('title', '')
|
|
|
|
if src:
|
|
add_media(src, 'video', title=title, element=video)
|
|
if poster and collect_images:
|
|
add_media(poster, 'image', title=f"Video poster: {title}", element=video)
|
|
|
|
# Sources within video
|
|
for source in video.find_all('source'):
|
|
src = source.get('src')
|
|
if src:
|
|
add_media(src, 'video', title=title)
|
|
|
|
# Extract from <audio> tags
|
|
if collect_audio:
|
|
for audio in soup.find_all('audio'):
|
|
src = audio.get('src')
|
|
title = audio.get('title', '')
|
|
|
|
if src:
|
|
add_media(src, 'audio', title=title)
|
|
|
|
for source in audio.find_all('source'):
|
|
src = source.get('src')
|
|
if src:
|
|
add_media(src, 'audio', title=title)
|
|
|
|
# Extract from <a href="..."> pointing to media files
|
|
for a in soup.find_all('a', href=True):
|
|
href = a['href']
|
|
media_type = get_media_type_from_extension(href)
|
|
if media_type:
|
|
if (media_type == 'image' and collect_images) or \
|
|
(media_type == 'video' and collect_videos) or \
|
|
(media_type == 'audio' and collect_audio):
|
|
add_media(href, media_type, alt_text=a.get_text(strip=True)[:100])
|
|
|
|
# Extract from CSS background-image: url(...)
|
|
if collect_images:
|
|
bg_pattern = re.compile(r'background(?:-image)?\s*:\s*url\(["\']?([^"\')\s]+)["\']?\)', re.IGNORECASE)
|
|
|
|
# Inline styles
|
|
for elem in soup.find_all(style=True):
|
|
style = elem.get('style', '')
|
|
for match in bg_pattern.findall(style):
|
|
add_media(match, 'image')
|
|
|
|
# <style> blocks
|
|
for style_tag in soup.find_all('style'):
|
|
if style_tag.string:
|
|
for match in bg_pattern.findall(style_tag.string):
|
|
add_media(match, 'image')
|
|
|
|
# Extract from Open Graph and Twitter meta tags
|
|
# Also look for og:title and og:image:alt for metadata
|
|
og_title = None
|
|
og_image_alt = None
|
|
media_filename = None
|
|
media_url = None
|
|
media_type_meta = None
|
|
|
|
for meta in soup.find_all('meta'):
|
|
prop = meta.get('property', '') or meta.get('name', '')
|
|
content = meta.get('content', '')
|
|
prop_lower = prop.lower()
|
|
|
|
# Extract metadata for richer media info
|
|
if prop_lower == 'og:title':
|
|
og_title = content
|
|
elif prop_lower == 'og:image:alt':
|
|
og_image_alt = content
|
|
elif prop_lower == 'media:filename':
|
|
media_filename = content
|
|
elif prop_lower == 'media:url':
|
|
media_url = content
|
|
elif prop_lower == 'media:type':
|
|
media_type_meta = content
|
|
|
|
# Process OG/Twitter image tags with enriched metadata
|
|
if collect_images:
|
|
for meta in soup.find_all('meta'):
|
|
prop = meta.get('property', '') or meta.get('name', '')
|
|
content = meta.get('content', '')
|
|
|
|
if content and prop.lower() in ('og:image', 'og:image:url', 'twitter:image', 'twitter:image:src'):
|
|
title = og_title or media_filename or 'Open Graph/Twitter image'
|
|
alt = og_image_alt or og_title or media_filename
|
|
add_media(content, 'image', alt_text=alt, title=title)
|
|
|
|
if collect_videos:
|
|
for meta in soup.find_all('meta'):
|
|
prop = meta.get('property', '') or meta.get('name', '')
|
|
content = meta.get('content', '')
|
|
|
|
if content and prop.lower() in ('og:video', 'og:video:url', 'twitter:player'):
|
|
title = og_title or media_filename or 'Open Graph/Twitter video'
|
|
add_media(content, 'video', title=title)
|
|
|
|
# Also check custom media:url meta tag (PyraFiles specific)
|
|
if media_url:
|
|
detected_type = media_type_meta or 'image' # Default to image
|
|
if detected_type in ('image', 'video', 'audio'):
|
|
should_add = (
|
|
(detected_type == 'image' and collect_images) or
|
|
(detected_type == 'video' and collect_videos) or
|
|
(detected_type == 'audio' and collect_audio)
|
|
)
|
|
if should_add:
|
|
title = og_title or media_filename
|
|
add_media(media_url, detected_type, alt_text=og_title, title=title)
|
|
|
|
logger.debug(f"Extracted {len(media_items)} media items from {base_url}")
|
|
return media_items
|
|
|
|
|
|
class AsyncWebFetcher:
|
|
"""
|
|
Async web fetcher that respects robots.txt and crawl delays.
|
|
Designed for use with Discord bot's async event loop.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
user_agent: str = "uncloseai.com/1.42 (ethical web crawler; +https://uncloseai.com)",
|
|
default_crawl_delay: float = DEFAULT_CRAWL_DELAY,
|
|
fast_mode: bool = False
|
|
):
|
|
self.user_agent = user_agent
|
|
self.default_crawl_delay = default_crawl_delay
|
|
self.fast_mode = fast_mode
|
|
|
|
# Timeouts: 5s in fast mode, 60s normally
|
|
self.media_timeout = 5 if fast_mode else 60
|
|
self.page_timeout = 5 if fast_mode else 15
|
|
|
|
# Caches for robots.txt and crawl delays per domain
|
|
self.robot_parsers: Dict[str, Optional[RobotFileParser]] = {}
|
|
self.domain_crawl_delays: Dict[str, float] = {}
|
|
self.domain_last_fetched: Dict[str, float] = {}
|
|
|
|
# Page cache: {url: (html, links, timestamp)}
|
|
self.page_cache: Dict[str, Tuple[str, List, float]] = {}
|
|
|
|
# Domain skip list: domains with too many consecutive timeouts
|
|
self.skip_domains: Set[str] = set()
|
|
self.domain_timeout_counts: Dict[str, int] = {}
|
|
self.MAX_CONSECUTIVE_TIMEOUTS = 5
|
|
|
|
logger.info(f"AsyncWebFetcher initialized with user-agent: {self.user_agent}")
|
|
|
|
def _get_domain(self, url: str) -> str:
|
|
"""Extract domain from URL"""
|
|
parsed = urlparse(url)
|
|
return parsed.netloc
|
|
|
|
async def _fetch_robots_txt(self, domain: str) -> Optional[RobotFileParser]:
|
|
"""
|
|
Fetch and parse robots.txt for a domain.
|
|
Returns RobotFileParser or None if not available.
|
|
"""
|
|
if domain in self.robot_parsers:
|
|
logger.debug(f"Using cached robots.txt for {domain}")
|
|
return self.robot_parsers[domain]
|
|
|
|
logger.info(f"Fetching robots.txt for {domain}")
|
|
robots_url = f"https://{domain}/robots.txt"
|
|
parser = RobotFileParser()
|
|
parser.set_url(robots_url)
|
|
|
|
try:
|
|
async with aiohttp.ClientSession() as session:
|
|
async with session.get(
|
|
robots_url,
|
|
headers={"User-Agent": self.user_agent},
|
|
timeout=aiohttp.ClientTimeout(total=5)
|
|
) as response:
|
|
if response.status == 200:
|
|
text = await response.text()
|
|
parser.parse(text.splitlines())
|
|
self.robot_parsers[domain] = parser
|
|
|
|
# Extract crawl delay
|
|
delay = parser.crawl_delay(self.user_agent)
|
|
self.domain_crawl_delays[domain] = (
|
|
delay if delay is not None else self.default_crawl_delay
|
|
)
|
|
|
|
logger.info(f"Parsed robots.txt for {domain}, crawl delay: {self.domain_crawl_delays[domain]}s")
|
|
return parser
|
|
else:
|
|
logger.info(f"No robots.txt for {domain} (status {response.status})")
|
|
self.robot_parsers[domain] = None
|
|
self.domain_crawl_delays[domain] = self.default_crawl_delay
|
|
return None
|
|
except Exception as e:
|
|
logger.warning(f"Could not fetch robots.txt for {domain}: {e}")
|
|
self.robot_parsers[domain] = None
|
|
self.domain_crawl_delays[domain] = self.default_crawl_delay
|
|
return None
|
|
|
|
async def _can_fetch(self, url: str) -> bool:
|
|
"""Check if URL can be fetched according to robots.txt"""
|
|
domain = self._get_domain(url)
|
|
parser = await self._fetch_robots_txt(domain)
|
|
|
|
if parser is None:
|
|
# No robots.txt means we can fetch
|
|
return True
|
|
|
|
can_fetch = parser.can_fetch(self.user_agent, url)
|
|
if not can_fetch:
|
|
logger.warning(f"Blocked by robots.txt: {url}")
|
|
|
|
return can_fetch
|
|
|
|
async def _enforce_crawl_delay(self, domain: str):
|
|
"""Enforce crawl delay for a domain"""
|
|
delay = self.domain_crawl_delays.get(domain, self.default_crawl_delay)
|
|
last_fetched = self.domain_last_fetched.get(domain, 0)
|
|
elapsed = time.time() - last_fetched
|
|
|
|
if elapsed < delay:
|
|
sleep_time = delay - elapsed
|
|
logger.info(f"Enforcing crawl delay for {domain}: sleeping {sleep_time:.2f}s")
|
|
await asyncio.sleep(sleep_time)
|
|
|
|
self.domain_last_fetched[domain] = time.time()
|
|
|
|
async def check_media_url(
|
|
self,
|
|
url: str,
|
|
session: Optional[aiohttp.ClientSession] = None
|
|
) -> Optional[Dict[str, Any]]:
|
|
"""
|
|
Check if a URL points to media content using HEAD request.
|
|
|
|
Useful for extensionless URLs where we can't determine media type from path.
|
|
|
|
Args:
|
|
url: URL to check
|
|
session: Optional aiohttp session to reuse
|
|
|
|
Returns:
|
|
Dict with 'media_type', 'mime_type', 'content_length' if media, None otherwise
|
|
"""
|
|
try:
|
|
close_session = session is None
|
|
if session is None:
|
|
session = aiohttp.ClientSession()
|
|
|
|
try:
|
|
async with session.head(
|
|
url,
|
|
headers={"User-Agent": self.user_agent},
|
|
timeout=aiohttp.ClientTimeout(total=10),
|
|
allow_redirects=True
|
|
) as response:
|
|
if response.status != 200:
|
|
return None
|
|
|
|
content_type = response.headers.get('Content-Type', '').lower()
|
|
mime_type = content_type.split(';')[0].strip()
|
|
media_type = get_media_type_from_mime(mime_type)
|
|
|
|
if media_type:
|
|
content_length = response.headers.get('Content-Length')
|
|
return {
|
|
'media_type': media_type,
|
|
'mime_type': mime_type,
|
|
'content_length': int(content_length) if content_length else None,
|
|
'url': str(response.url) # Final URL after redirects
|
|
}
|
|
return None
|
|
finally:
|
|
if close_session:
|
|
await session.close()
|
|
|
|
except Exception as e:
|
|
logger.debug(f"HEAD check failed for {url}: {e}")
|
|
return None
|
|
|
|
async def resolve_canonical_image(
|
|
self,
|
|
detail_page_url: str,
|
|
thumbnail_url: str,
|
|
embedding_title: Optional[str] = None,
|
|
session: Optional[aiohttp.ClientSession] = None
|
|
) -> Optional[Dict[str, Any]]:
|
|
"""
|
|
Fetch a detail page and extract the canonical (full-res) image URL.
|
|
|
|
Universal Algorithm - no hardcoded strings, purely structural detection:
|
|
|
|
1. REST pattern: If an <img> src matches detail_page_url minus last path segment
|
|
2. Wrapped links: <a> tags wrapping <img> where href has query params (versioned)
|
|
3. Download links: <a download> pointing to media
|
|
4. Path similarity: <img> src sharing path structure with detail page
|
|
5. og:image fallback
|
|
|
|
Title: Prefers embedding_title (from listing page where image was found).
|
|
Falls back to detail page metadata only if embedding_title not provided.
|
|
|
|
Args:
|
|
detail_page_url: URL of the detail page to fetch
|
|
thumbnail_url: Original thumbnail URL (to avoid returning same URL)
|
|
embedding_title: Title from the page that linked here (preferred)
|
|
session: Optional aiohttp session to reuse
|
|
|
|
Returns:
|
|
Dict with 'canonical_url', 'title', 'description', 'og_image' or None
|
|
"""
|
|
try:
|
|
close_session = session is None
|
|
if session is None:
|
|
session = aiohttp.ClientSession()
|
|
|
|
try:
|
|
# Check robots.txt
|
|
if not await self._can_fetch(detail_page_url):
|
|
return None
|
|
|
|
# Enforce crawl delay
|
|
domain = self._get_domain(detail_page_url)
|
|
await self._enforce_crawl_delay(domain)
|
|
|
|
async with session.get(
|
|
detail_page_url,
|
|
headers={"User-Agent": self.user_agent},
|
|
timeout=aiohttp.ClientTimeout(total=15),
|
|
allow_redirects=True
|
|
) as response:
|
|
if response.status != 200:
|
|
return None
|
|
|
|
html = await response.text()
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
|
|
result = {
|
|
'detail_page_url': detail_page_url,
|
|
'canonical_url': None,
|
|
'embedding_title': embedding_title, # from listing page
|
|
'detail_title': None, # from detail page
|
|
'detail_content': None, # body text from detail page
|
|
'title': None, # best available
|
|
'description': None,
|
|
'og_image': None,
|
|
}
|
|
|
|
# ========================================
|
|
# COLLECT BOTH TITLES (skeleton key approach)
|
|
# ========================================
|
|
|
|
# Always extract detail page title
|
|
detail_title = None
|
|
|
|
# 1. og:title
|
|
og_title = soup.find('meta', property='og:title')
|
|
if og_title:
|
|
detail_title = og_title.get('content', '').strip()
|
|
|
|
# 2. First image alt text
|
|
if not detail_title:
|
|
for img in soup.find_all('img', alt=True):
|
|
alt = img.get('alt', '').strip()
|
|
if alt and len(alt) > 2:
|
|
detail_title = alt
|
|
break
|
|
|
|
# 3. h1 tag
|
|
if not detail_title:
|
|
h1_tag = soup.find('h1')
|
|
if h1_tag:
|
|
detail_title = h1_tag.get_text(strip=True)
|
|
|
|
# 4. title tag
|
|
if not detail_title:
|
|
title_tag = soup.find('title')
|
|
if title_tag:
|
|
detail_title = title_tag.get_text(strip=True)
|
|
|
|
result['detail_title'] = detail_title
|
|
# Primary title: prefer embedding, fallback to detail
|
|
result['title'] = embedding_title or detail_title
|
|
|
|
# ========================================
|
|
# DESCRIPTION EXTRACTION
|
|
# ========================================
|
|
og_desc = soup.find('meta', property='og:description')
|
|
if og_desc:
|
|
result['description'] = og_desc.get('content', '')
|
|
if not result['description']:
|
|
meta_desc = soup.find('meta', attrs={'name': 'description'})
|
|
if meta_desc:
|
|
result['description'] = meta_desc.get('content', '')
|
|
|
|
# Extract og:image
|
|
og_image = soup.find('meta', property='og:image')
|
|
if og_image:
|
|
result['og_image'] = og_image.get('content', '')
|
|
|
|
# ========================================
|
|
# DETAIL CONTENT EXTRACTION (body text)
|
|
# ========================================
|
|
# Extract body text from detail page for full-text searchability
|
|
# This enables Pinterest-style galleries where detail pages
|
|
# have richer descriptions than thumbnails on listing pages
|
|
body = soup.find('body')
|
|
if body:
|
|
# Remove non-content elements
|
|
for tag in body.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside']):
|
|
tag.decompose()
|
|
detail_content = body.get_text(separator=' ', strip=True)[:10000]
|
|
result['detail_content'] = detail_content
|
|
|
|
# ========================================
|
|
# CANONICAL URL DETECTION (universal patterns)
|
|
# ========================================
|
|
canonical_url = None
|
|
detail_parsed = urlparse(detail_page_url)
|
|
|
|
# Pattern 1: REST-style - check if img src matches URL minus last segment
|
|
# e.g., /media/ID/details has img pointing to /media/ID
|
|
path_segments = detail_parsed.path.rstrip('/').split('/')
|
|
if len(path_segments) > 1:
|
|
parent_path = '/'.join(path_segments[:-1])
|
|
parent_url = f"{detail_parsed.scheme}://{detail_parsed.netloc}{parent_path}"
|
|
for img in soup.find_all('img', src=True):
|
|
src = img.get('src', '')
|
|
full_src = urljoin(detail_page_url, src)
|
|
if full_src == parent_url or full_src.rstrip('/') == parent_url:
|
|
if full_src != thumbnail_url:
|
|
canonical_url = full_src
|
|
break
|
|
|
|
# Pattern 2: Find <a> tags wrapping images with query params (versioned URLs)
|
|
# The href with ?param=value suggests a cache-busted/versioned canonical
|
|
if not canonical_url:
|
|
for a_tag in soup.find_all('a', href=True):
|
|
href = a_tag.get('href', '')
|
|
full_href = urljoin(detail_page_url, href)
|
|
# Must have query params (indicates versioned/timestamped)
|
|
if '?' not in full_href:
|
|
continue
|
|
# Must wrap or be near an image
|
|
img_inside = a_tag.find('img')
|
|
if not img_inside:
|
|
continue
|
|
# Check it's a media URL or same-domain endpoint
|
|
href_parsed = urlparse(full_href)
|
|
media_type = get_media_type_from_extension(full_href)
|
|
if media_type == 'image':
|
|
canonical_url = full_href
|
|
break
|
|
# Same domain with query params - likely image endpoint
|
|
if href_parsed.netloc == detail_parsed.netloc or not href_parsed.netloc:
|
|
canonical_url = full_href
|
|
break
|
|
|
|
# Pattern 3: <a download> attribute - semantic HTML for downloadable content
|
|
if not canonical_url:
|
|
for a_tag in soup.find_all('a', href=True, download=True):
|
|
href = a_tag.get('href', '')
|
|
if href:
|
|
full_href = urljoin(detail_page_url, href)
|
|
media_type = get_media_type_from_extension(full_href)
|
|
if media_type == 'image':
|
|
canonical_url = full_href
|
|
break
|
|
|
|
# Pattern 4: First image sharing path structure with detail page
|
|
if not canonical_url:
|
|
for img in soup.find_all('img', src=True):
|
|
src = img.get('src', '')
|
|
full_src = urljoin(detail_page_url, src)
|
|
if full_src == thumbnail_url:
|
|
continue
|
|
img_parsed = urlparse(full_src)
|
|
# Same host
|
|
if img_parsed.netloc == detail_parsed.netloc:
|
|
# Count shared path segments
|
|
detail_parts = detail_parsed.path.rstrip('/').split('/')
|
|
img_parts = img_parsed.path.rstrip('/').split('/')
|
|
common = sum(1 for d, i in zip(detail_parts, img_parts) if d == i)
|
|
# At least 2 shared segments suggests same resource
|
|
if common >= 2:
|
|
canonical_url = full_src
|
|
break
|
|
|
|
# Pattern 5: og:image fallback (if different from thumbnail)
|
|
if not canonical_url and result['og_image']:
|
|
if result['og_image'] != thumbnail_url:
|
|
canonical_url = result['og_image']
|
|
|
|
result['canonical_url'] = canonical_url
|
|
|
|
logger.info(f"Resolved canonical image from {detail_page_url}: {canonical_url}, title: {result['title'][:50] if result['title'] else 'None'}")
|
|
return result
|
|
|
|
finally:
|
|
if close_session:
|
|
await session.close()
|
|
|
|
except Exception as e:
|
|
logger.debug(f"Failed to resolve canonical image from {detail_page_url}: {e}")
|
|
return None
|
|
|
|
def _record_timeout(self, domain: str):
|
|
"""Record a timeout for a domain. After MAX_CONSECUTIVE_TIMEOUTS, add to skip list."""
|
|
self.domain_timeout_counts[domain] = self.domain_timeout_counts.get(domain, 0) + 1
|
|
if self.domain_timeout_counts[domain] >= self.MAX_CONSECUTIVE_TIMEOUTS:
|
|
if domain not in self.skip_domains:
|
|
self.skip_domains.add(domain)
|
|
logger.warning(f"Skipping domain {domain} after {self.MAX_CONSECUTIVE_TIMEOUTS} consecutive timeouts")
|
|
|
|
def _record_success(self, domain: str):
|
|
"""Record a successful fetch, resetting timeout count."""
|
|
self.domain_timeout_counts[domain] = 0
|
|
|
|
async def fetch_media(
|
|
self,
|
|
url: str,
|
|
session: Optional[aiohttp.ClientSession] = None,
|
|
max_size: int = 100 * 1024 * 1024 # 100MB default limit
|
|
) -> Optional[Dict[str, Any]]:
|
|
"""
|
|
Fetch media content and compute MD5 hash.
|
|
|
|
Args:
|
|
url: URL of media to fetch
|
|
session: Optional aiohttp session to reuse
|
|
max_size: Maximum file size in bytes (default 100MB)
|
|
|
|
Returns:
|
|
Dict with 'data', 'md5_hash', 'mime_type', 'size', 'url' or None on failure
|
|
"""
|
|
global LAST_FETCH_ERROR
|
|
|
|
# Check if domain is in skip list
|
|
domain = self._get_domain(url)
|
|
if domain in self.skip_domains:
|
|
LAST_FETCH_ERROR = {'type': 'skip_domain', 'details': f'Domain {domain} skipped (too many timeouts)', 'url': url}
|
|
logger.debug(f"Skipping {url}: domain {domain} in skip list")
|
|
return None
|
|
|
|
# Check robots.txt
|
|
if not await self._can_fetch(url):
|
|
LAST_FETCH_ERROR = {'type': 'robots_txt', 'details': 'Blocked by robots.txt', 'url': url}
|
|
return None
|
|
|
|
# Enforce crawl delay
|
|
await self._enforce_crawl_delay(domain)
|
|
|
|
try:
|
|
close_session = session is None
|
|
if session is None:
|
|
session = aiohttp.ClientSession()
|
|
|
|
try:
|
|
async with session.get(
|
|
url,
|
|
headers={"User-Agent": self.user_agent},
|
|
timeout=aiohttp.ClientTimeout(total=self.media_timeout),
|
|
allow_redirects=True
|
|
) as response:
|
|
if response.status != 200:
|
|
LAST_FETCH_ERROR = {'type': 'http_error', 'details': f'HTTP {response.status}', 'url': url}
|
|
return None
|
|
|
|
# Check content length before downloading
|
|
content_length = response.headers.get('Content-Length')
|
|
if content_length and int(content_length) > max_size:
|
|
LAST_FETCH_ERROR = {'type': 'too_large', 'details': f'File too large: {int(content_length)} bytes', 'url': url}
|
|
logger.warning(f"Skipping {url}: file too large ({int(content_length)} bytes)")
|
|
return None
|
|
|
|
content_type = response.headers.get('Content-Type', '').lower()
|
|
mime_type = content_type.split(';')[0].strip()
|
|
|
|
# Download content
|
|
data = await response.read()
|
|
|
|
if len(data) > max_size:
|
|
LAST_FETCH_ERROR = {'type': 'too_large', 'details': f'File too large: {len(data)} bytes', 'url': url}
|
|
return None
|
|
|
|
# Compute MD5 hash
|
|
md5_hash = hashlib.md5(data).hexdigest()
|
|
|
|
# Determine media type
|
|
media_type = get_media_type_from_mime(mime_type) or get_media_type_from_extension(url)
|
|
|
|
# Success - reset timeout count
|
|
self._record_success(domain)
|
|
|
|
logger.info(f"Fetched media {url}: {len(data)} bytes, MD5: {md5_hash}, type: {media_type}")
|
|
|
|
return {
|
|
'data': data,
|
|
'md5_hash': md5_hash,
|
|
'mime_type': mime_type,
|
|
'media_type': media_type,
|
|
'size': len(data),
|
|
'url': str(response.url) # Final URL after redirects
|
|
}
|
|
|
|
finally:
|
|
if close_session:
|
|
await session.close()
|
|
|
|
except asyncio.TimeoutError:
|
|
self._record_timeout(domain)
|
|
LAST_FETCH_ERROR = {'type': 'timeout', 'details': f'Download timeout ({self.media_timeout}s)', 'url': url}
|
|
logger.error(f"Timeout fetching media {url} ({self.media_timeout}s)")
|
|
return None
|
|
except Exception as e:
|
|
LAST_FETCH_ERROR = {'type': 'unknown', 'details': str(e), 'url': url}
|
|
logger.error(f"Error fetching media {url}: {e}")
|
|
return None
|
|
|
|
async def fetch_webpage(
|
|
self,
|
|
url: str,
|
|
extract_links: bool = False,
|
|
extract_anchor_text: bool = False,
|
|
cache_check_callback = None
|
|
) -> Tuple[Optional[str], List]:
|
|
"""
|
|
Fetch a webpage, respecting robots.txt and crawl delays.
|
|
Uses cache to avoid redundant fetches and delays.
|
|
|
|
Args:
|
|
url: URL to fetch
|
|
extract_links: Whether to extract links from the page
|
|
extract_anchor_text: Whether to extract anchor text with links
|
|
cache_check_callback: Optional async callback to check external cache (e.g., SQLite3)
|
|
Should return (html, links) tuple if cached, None otherwise
|
|
|
|
Returns:
|
|
If extract_anchor_text=True: Tuple of (html_content, list of dicts with 'url' and 'anchor_text')
|
|
If extract_anchor_text=False: Tuple of (html_content, list of URLs)
|
|
Returns (None, []) if fetch fails
|
|
"""
|
|
global LAST_FETCH_ERROR
|
|
|
|
# Validate URL
|
|
if not url.startswith(("http://", "https://")):
|
|
logger.error(f"Invalid URL scheme: {url}")
|
|
return None, []
|
|
|
|
# Check external cache first (e.g., SQLite3 via WebCacheManager)
|
|
if cache_check_callback:
|
|
try:
|
|
cached_result = await cache_check_callback(url)
|
|
if cached_result:
|
|
html, links = cached_result
|
|
# If we need links but cache has none, re-extract from HTML
|
|
if extract_links and not links and html:
|
|
logger.info(f"💾 Cache hit but no links cached, re-extracting from HTML: {url}")
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
links = []
|
|
for a_tag in soup.find_all("a", href=True):
|
|
href = a_tag["href"]
|
|
if href.startswith(("javascript:", "#")):
|
|
continue
|
|
rel = a_tag.get("rel", [])
|
|
if isinstance(rel, str):
|
|
rel = rel.split()
|
|
if "nofollow" in rel:
|
|
continue
|
|
absolute_url = strip_url_fragment(urljoin(url, href))
|
|
parsed = urlparse(absolute_url)
|
|
if parsed.scheme in ("http", "https"):
|
|
if extract_anchor_text:
|
|
anchor_text = a_tag.get_text(strip=True)
|
|
links.append({"url": absolute_url, "anchor_text": anchor_text})
|
|
else:
|
|
links.append(absolute_url)
|
|
logger.info(f"Re-extracted {len(links)} links from cached HTML")
|
|
else:
|
|
logger.info(f"💾 Using SQLite3 cached page: {url} (skipping robots.txt + crawl delay)")
|
|
return html, links
|
|
except Exception as e:
|
|
logger.warning(f"Cache check callback failed for {url}: {e}")
|
|
|
|
# Check in-memory cache (fallback for when no external cache available)
|
|
if url in self.page_cache:
|
|
html, cached_links, cached_time = self.page_cache[url]
|
|
# If we need links but cache has none, re-extract from HTML
|
|
if extract_links and not cached_links and html:
|
|
logger.info(f"💾 In-memory cache hit but no links, re-extracting from HTML: {url}")
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
cached_links = []
|
|
for a_tag in soup.find_all("a", href=True):
|
|
href = a_tag["href"]
|
|
if href.startswith(("javascript:", "#")):
|
|
continue
|
|
rel = a_tag.get("rel", [])
|
|
if isinstance(rel, str):
|
|
rel = rel.split()
|
|
if "nofollow" in rel:
|
|
continue
|
|
absolute_url = strip_url_fragment(urljoin(url, href))
|
|
parsed_link = urlparse(absolute_url)
|
|
if parsed_link.scheme in ("http", "https"):
|
|
if extract_anchor_text:
|
|
anchor_text = a_tag.get_text(strip=True)
|
|
cached_links.append({"url": absolute_url, "anchor_text": anchor_text})
|
|
else:
|
|
cached_links.append(absolute_url)
|
|
logger.info(f"Re-extracted {len(cached_links)} links from in-memory cached HTML")
|
|
else:
|
|
logger.info(f"💾 Using in-memory cached page: {url} (cached {time.time() - cached_time:.0f}s ago, skipping crawl delay)")
|
|
return html, cached_links
|
|
|
|
# Check robots.txt
|
|
if not await self._can_fetch(url):
|
|
LAST_FETCH_ERROR = {'type': 'robots_txt', 'details': 'Blocked by robots.txt', 'url': url}
|
|
return None, []
|
|
|
|
# Enforce crawl delay (only for fresh fetches)
|
|
domain = self._get_domain(url)
|
|
await self._enforce_crawl_delay(domain)
|
|
|
|
# Fetch the page
|
|
try:
|
|
async with aiohttp.ClientSession() as session:
|
|
# HEAD request first to check content type without downloading
|
|
try:
|
|
async with session.head(
|
|
url,
|
|
headers={"User-Agent": self.user_agent},
|
|
timeout=aiohttp.ClientTimeout(total=10),
|
|
allow_redirects=True
|
|
) as head_response:
|
|
head_content_type = head_response.headers.get('Content-Type', '').lower()
|
|
mime_type = head_content_type.split(';')[0].strip()
|
|
logger.info(f"HEAD {url} -> {head_response.status}, Content-Type: {mime_type}")
|
|
|
|
# Skip images, video, audio - these need vision mode, not text fetch
|
|
if mime_type.startswith(('image/', 'video/', 'audio/')):
|
|
LAST_FETCH_ERROR = {
|
|
'type': 'binary_content',
|
|
'details': f'Content is {mime_type} - requires vision mode for images',
|
|
'url': url,
|
|
'content_type': mime_type
|
|
}
|
|
logger.info(f"Skipping {mime_type} content (not text-extractable): {url}")
|
|
return None, []
|
|
except Exception as e:
|
|
# HEAD failed, continue with GET anyway
|
|
logger.debug(f"HEAD request failed for {url}: {e}, continuing with GET")
|
|
|
|
async with session.get(
|
|
url,
|
|
headers={"User-Agent": self.user_agent},
|
|
timeout=aiohttp.ClientTimeout(total=15),
|
|
allow_redirects=True,
|
|
max_redirects=3
|
|
) as response:
|
|
if response.status != 200:
|
|
LAST_FETCH_ERROR = {'type': 'http_error', 'details': f'HTTP {response.status}', 'url': url, 'status_code': response.status}
|
|
logger.warning(f"Failed to fetch {url}: HTTP {response.status}")
|
|
return None, []
|
|
|
|
# Check content type for binary files (PDF, etc.)
|
|
content_type = response.headers.get('Content-Type', '').lower()
|
|
mime_type = content_type.split(';')[0].strip()
|
|
|
|
# Double-check for binary content that HEAD might have missed
|
|
if mime_type.startswith(('image/', 'video/', 'audio/')):
|
|
LAST_FETCH_ERROR = {
|
|
'type': 'binary_content',
|
|
'details': f'Content is {mime_type} - requires vision mode for images',
|
|
'url': url,
|
|
'content_type': mime_type
|
|
}
|
|
logger.info(f"Skipping {mime_type} content (not text-extractable): {url}")
|
|
return None, []
|
|
|
|
is_pdf = 'application/pdf' in content_type or url.lower().endswith('.pdf')
|
|
|
|
if is_pdf:
|
|
# Handle PDF: read as binary and extract text
|
|
pdf_bytes = await response.read()
|
|
logger.info(f"Fetched PDF {url} ({len(pdf_bytes)} bytes)")
|
|
pdf_text = extract_text_from_pdf(pdf_bytes)
|
|
if pdf_text:
|
|
# Wrap extracted text in minimal HTML for consistent processing
|
|
# Extract filename from URL for title
|
|
pdf_filename = url.split('/')[-1].split('?')[0] or "PDF Document"
|
|
html = f"<html><head><title>{pdf_filename}</title></head><body><pre>{pdf_text}</pre></body></html>"
|
|
logger.info(f"Extracted {len(pdf_text)} chars of text from PDF")
|
|
else:
|
|
LAST_FETCH_ERROR = {'type': 'pdf_extraction_failed', 'details': 'Could not extract text from PDF', 'url': url}
|
|
logger.warning(f"Failed to extract text from PDF: {url}")
|
|
return None, []
|
|
else:
|
|
html = await response.text()
|
|
logger.info(f"Successfully fetched {url} ({len(html)} bytes)")
|
|
|
|
# Extract links if requested
|
|
links = []
|
|
if extract_links:
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
for a_tag in soup.find_all("a", href=True):
|
|
href = a_tag["href"]
|
|
if href.startswith(("javascript:", "#")):
|
|
continue
|
|
# Respect rel="nofollow" - skip links marked as nofollow
|
|
rel = a_tag.get("rel", [])
|
|
if isinstance(rel, str):
|
|
rel = rel.split()
|
|
if "nofollow" in rel:
|
|
logger.debug(f"Skipping nofollow link: {href}")
|
|
continue
|
|
absolute_url = strip_url_fragment(urljoin(url, href))
|
|
parsed = urlparse(absolute_url)
|
|
if parsed.scheme in ("http", "https"):
|
|
if extract_anchor_text:
|
|
# Extract anchor text for keyword matching
|
|
anchor_text = a_tag.get_text(strip=True)
|
|
links.append({
|
|
"url": absolute_url,
|
|
"anchor_text": anchor_text
|
|
})
|
|
else:
|
|
links.append(absolute_url)
|
|
logger.info(f"Extracted {len(links)} links from {url}")
|
|
|
|
# Cache in memory for future fetches in this session
|
|
self.page_cache[url] = (html, links, time.time())
|
|
|
|
return html, links
|
|
|
|
except asyncio.TimeoutError:
|
|
LAST_FETCH_ERROR = {'type': 'timeout', 'details': 'Connection timeout (>15 seconds)', 'url': url}
|
|
logger.error(f"Timeout fetching {url}")
|
|
return None, []
|
|
except Exception as e:
|
|
error_str = str(e)
|
|
if 'SSL' in error_str or 'certificate' in error_str.lower():
|
|
LAST_FETCH_ERROR = {'type': 'ssl', 'details': str(e), 'url': url}
|
|
elif 'Name or service not known' in error_str or 'DNS' in error_str:
|
|
LAST_FETCH_ERROR = {'type': 'dns', 'details': 'DNS resolution failed', 'url': url}
|
|
else:
|
|
LAST_FETCH_ERROR = {'type': 'unknown', 'details': str(e), 'url': url}
|
|
logger.error(f"Error fetching {url}: {e}")
|
|
return None, []
|
|
|
|
def extract_text_content(self, html: str) -> str:
|
|
"""
|
|
Extract main text content from HTML using BeautifulSoup.
|
|
|
|
Args:
|
|
html: HTML content
|
|
|
|
Returns:
|
|
Extracted text
|
|
"""
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
|
|
# Remove script and style elements
|
|
for script in soup(["script", "style"]):
|
|
script.extract()
|
|
|
|
# Block-level elements that should create paragraph breaks
|
|
import re
|
|
block_tags = {'p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
|
|
'section', 'article', 'header', 'footer', 'main',
|
|
'li', 'tr', 'blockquote', 'pre', 'br', 'hr'}
|
|
|
|
# Insert markers before/after block elements
|
|
for tag in soup.find_all(block_tags):
|
|
tag.insert_before('\n\n')
|
|
tag.insert_after('\n\n')
|
|
|
|
# Get text with spaces between inline elements
|
|
text = soup.get_text(separator=' ')
|
|
|
|
# Clean up: split on newlines, strip lines, filter empties
|
|
lines = []
|
|
for line in text.split('\n'):
|
|
# Collapse multiple spaces to single space
|
|
cleaned = re.sub(r' +', ' ', line.strip())
|
|
if cleaned:
|
|
lines.append(cleaned)
|
|
|
|
# Join with single newlines, then collapse 2+ to paragraph breaks
|
|
text = '\n'.join(lines)
|
|
text = re.sub(r'\n{2,}', '\n\n', text)
|
|
|
|
return text
|
|
|
|
def extract_title(self, html: str) -> str:
|
|
"""Extract title from HTML"""
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
if soup.title and soup.title.string:
|
|
return soup.title.string.strip()
|
|
return "Untitled"
|
|
|
|
def _simple_stem(self, word: str) -> str:
|
|
"""
|
|
Simple suffix-stripping stemmer for common English suffixes.
|
|
|
|
Args:
|
|
word: Word to stem
|
|
|
|
Returns:
|
|
Stemmed word
|
|
"""
|
|
word = word.lower()
|
|
# Common suffixes in order of longest first
|
|
suffixes = ['ings', 'ing', 'ed', 'es', 's', 'ly', 'er', 'est', 'tion', 'sion', 'ness', 'ment', 'able', 'ible']
|
|
for suffix in suffixes:
|
|
if word.endswith(suffix) and len(word) > len(suffix) + 2: # Keep at least 3 chars in stem
|
|
return word[:-len(suffix)]
|
|
return word
|
|
|
|
def _score_link(self, link_url: str, anchor_text: str, query_keywords: Optional[List[str]] = None, keyword_variations: Optional[List[str]] = None) -> float:
|
|
"""
|
|
Score a link based on URL and anchor text keyword matches.
|
|
Uses both exact and stemmed matching with different scoring weights.
|
|
|
|
Scoring weights:
|
|
- Exact primary keyword match: 10.0 (URL) / 5.0 (anchor)
|
|
- Stemmed primary keyword match: 7.0 (URL) / 3.5 (anchor) [70% of exact]
|
|
- Exact variation match: 6.0 (URL) / 3.0 (anchor) [60% of exact primary]
|
|
- Stemmed variation match: 4.0 (URL) / 2.0 (anchor) [40% of exact primary]
|
|
|
|
Args:
|
|
link_url: URL of the link
|
|
anchor_text: Anchor text of the link
|
|
query_keywords: Primary keywords to match against (full score)
|
|
keyword_variations: Keyword variations/synonyms (reduced score)
|
|
|
|
Returns:
|
|
Float score (higher = more relevant to follow)
|
|
"""
|
|
if not query_keywords and not keyword_variations:
|
|
return 0.0
|
|
|
|
score = 0.0
|
|
url_lower = link_url.lower()
|
|
anchor_lower = anchor_text.lower()
|
|
|
|
# Score primary keywords (exact and stemmed matching)
|
|
if query_keywords:
|
|
for keyword in query_keywords:
|
|
keyword_lower = keyword.lower()
|
|
keyword_stem = self._simple_stem(keyword_lower)
|
|
|
|
# Exact match for keywords in URL path
|
|
if keyword_lower in url_lower:
|
|
score += 10.0
|
|
logger.debug(f"Link URL contains exact primary keyword '{keyword_lower}': {link_url}")
|
|
# Stemmed match for keywords in URL path (70% of exact)
|
|
elif keyword_stem in url_lower and keyword_stem != keyword_lower:
|
|
score += 7.0
|
|
logger.debug(f"Link URL contains stemmed primary keyword '{keyword_stem}' (from '{keyword_lower}'): {link_url}")
|
|
|
|
# Exact match for keywords in anchor text
|
|
if keyword_lower in anchor_lower:
|
|
score += 5.0
|
|
logger.debug(f"Link anchor contains exact primary keyword '{keyword_lower}': {anchor_text}")
|
|
# Stemmed match for keywords in anchor text (70% of exact)
|
|
elif keyword_stem in anchor_lower and keyword_stem != keyword_lower:
|
|
score += 3.5
|
|
logger.debug(f"Link anchor contains stemmed primary keyword '{keyword_stem}' (from '{keyword_lower}'): {anchor_text}")
|
|
|
|
# Score keyword variations (exact and stemmed matching, reduced weight)
|
|
if keyword_variations:
|
|
for variation in keyword_variations:
|
|
variation_lower = variation.lower()
|
|
variation_stem = self._simple_stem(variation_lower)
|
|
|
|
# Exact match for variations in URL path (60% of exact primary)
|
|
if variation_lower in url_lower:
|
|
score += 6.0
|
|
logger.debug(f"Link URL contains exact keyword variation '{variation_lower}': {link_url}")
|
|
# Stemmed match for variations in URL path (40% of exact primary)
|
|
elif variation_stem in url_lower and variation_stem != variation_lower:
|
|
score += 4.0
|
|
logger.debug(f"Link URL contains stemmed keyword variation '{variation_stem}' (from '{variation_lower}'): {link_url}")
|
|
|
|
# Exact match for variations in anchor text (60% of exact primary)
|
|
if variation_lower in anchor_lower:
|
|
score += 3.0
|
|
logger.debug(f"Link anchor contains exact keyword variation '{variation_lower}': {anchor_text}")
|
|
# Stemmed match for variations in anchor text (40% of exact primary)
|
|
elif variation_stem in anchor_lower and variation_stem != variation_lower:
|
|
score += 2.0
|
|
logger.debug(f"Link anchor contains stemmed keyword variation '{variation_stem}' (from '{variation_lower}'): {anchor_text}")
|
|
|
|
return score
|
|
|
|
def _score_page(self, page_data: Dict[str, str], query_keywords: Optional[List[str]] = None) -> float:
|
|
"""
|
|
Score a page based on content quality metrics and optional query relevance.
|
|
Higher scores indicate more valuable content.
|
|
|
|
Scoring factors:
|
|
- Content length (more content = higher score)
|
|
- Title quality (descriptive titles = higher score)
|
|
- URL quality (cleaner URLs = higher score)
|
|
- Content density (unique words = higher score)
|
|
- Query relevance (if keywords provided, matching content scores higher)
|
|
|
|
Args:
|
|
page_data: Dict with 'text', 'title', 'url' keys
|
|
query_keywords: Optional list of keywords from user's query for relevance scoring
|
|
|
|
Returns:
|
|
Float score (0-100+, can exceed 100 with relevance bonus)
|
|
"""
|
|
score = 0.0
|
|
|
|
# Factor 1: Content length (0-30 points - reduced from 40 to make room for relevance)
|
|
# Award points for substantial content, max at 5000 chars
|
|
text_len = len(page_data.get('text', ''))
|
|
score += min(30, text_len / 167) # 5000 chars = 30 points
|
|
|
|
# Factor 2: Title quality (0-15 points - reduced from 20)
|
|
# Descriptive titles with meaningful length get higher scores
|
|
title = page_data.get('title', '')
|
|
if title and title != "Untitled":
|
|
title_len = len(title)
|
|
if 10 <= title_len <= 100: # Sweet spot for titles
|
|
score += 15
|
|
elif 5 <= title_len < 10 or 100 < title_len <= 150:
|
|
score += 8
|
|
else:
|
|
score += 3
|
|
|
|
# Factor 3: URL quality (0-15 points - reduced from 20)
|
|
# Prefer shorter, cleaner URLs over long query-string heavy URLs
|
|
url = page_data.get('url', '')
|
|
if url:
|
|
# Penalize query strings and fragments
|
|
if '?' in url:
|
|
score += 3 # Query strings often mean dynamic/less important content
|
|
elif '#' in url:
|
|
score += 8 # Fragments are slightly better
|
|
else:
|
|
score += 15 # Clean URLs are best
|
|
|
|
# Bonus for human-readable paths
|
|
path_parts = url.split('/')
|
|
if any(len(part) > 3 and part.replace('-', '').replace('_', '').isalnum() for part in path_parts):
|
|
score += 3 # Readable path segments
|
|
|
|
# Factor 4: Content density (0-15 points - reduced from 20)
|
|
# Reward pages with substantial, non-repetitive content
|
|
words = [] # Initialize words for use in Factor 5
|
|
if text_len > 0:
|
|
# Check for unique words (simple heuristic for content quality)
|
|
words = page_data.get('text', '').lower().split()
|
|
unique_words = len(set(words))
|
|
if len(words) > 0:
|
|
uniqueness_ratio = unique_words / len(words)
|
|
score += uniqueness_ratio * 15 # Higher uniqueness = better content
|
|
|
|
# Factor 5: Query relevance (0-40 points - NEW!)
|
|
# If user's query keywords are provided, boost pages that contain them
|
|
if query_keywords and len(query_keywords) > 0:
|
|
text_lower = page_data.get('text', '').lower()
|
|
title_lower = title.lower()
|
|
url_lower = url.lower()
|
|
|
|
keyword_matches = 0
|
|
keyword_density = 0.0
|
|
|
|
for keyword in query_keywords:
|
|
keyword_lower = keyword.lower()
|
|
|
|
# Count occurrences in different sections with different weights
|
|
text_count = text_lower.count(keyword_lower)
|
|
title_count = title_lower.count(keyword_lower)
|
|
url_count = url_lower.count(keyword_lower)
|
|
|
|
if text_count > 0:
|
|
keyword_matches += 1
|
|
keyword_density += text_count
|
|
|
|
# Bonus for keywords in title (very relevant)
|
|
if title_count > 0:
|
|
score += 5 * title_count # Up to 5 points per title match
|
|
|
|
# HIGH BONUS for keywords in URL (strong signal for topic-specific pages)
|
|
# URLs are hand-crafted structure, keywords there mean this page is ABOUT that topic
|
|
if url_count > 0:
|
|
score += 15 * url_count # Up to 15 points per URL match (5x higher than before!)
|
|
logger.debug(f"URL contains keyword '{keyword_lower}' ({url_count}x) - boosting score by {15 * url_count} points")
|
|
|
|
# Award points based on what percentage of query keywords matched
|
|
if len(query_keywords) > 0:
|
|
match_ratio = keyword_matches / len(query_keywords)
|
|
score += match_ratio * 20 # Up to 20 points for matching all keywords
|
|
|
|
# Award points for keyword density (how often keywords appear)
|
|
if len(words) > 0 and keyword_density > 0:
|
|
density_score = min(10, (keyword_density / len(words)) * 1000) # Up to 10 points
|
|
score += density_score
|
|
|
|
return score
|
|
|
|
async def fetch_with_depth(
|
|
self,
|
|
start_url: str,
|
|
depth: int = 2,
|
|
max_pages: int = 10,
|
|
query_keywords: Optional[List[str]] = None,
|
|
keyword_variations: Optional[List[str]] = None,
|
|
progress_callback = None,
|
|
cache_check_callback = None,
|
|
should_continue_callback = None, # Deprecated - kept for compatibility
|
|
user_query: Optional[str] = None, # Deprecated - kept for compatibility
|
|
mode: CrawlMode = CrawlMode.TEXT,
|
|
media_callback = None, # Callback for discovered media: async fn(media_item: Dict) -> None
|
|
page_callback = None, # Callback for page HTML: async fn(url: str, html: str) -> None
|
|
uris_total_callback = None, # Callback for URI total updates: fn(total: int) -> None
|
|
initial_visited: Optional[Set[str]] = None, # Pre-visited URLs for resume support
|
|
) -> List[Dict[str, str]]:
|
|
"""
|
|
Intelligent keyword-driven crawl strategy with domain prioritization.
|
|
|
|
Supports multiple crawl modes:
|
|
- TEXT (default): Extract text content only
|
|
- IMAGES: Collect images from pages
|
|
- VIDEOS: Collect videos from pages
|
|
- MEDIA: Collect all media (images + videos + audio)
|
|
- ALL: Uber crawl - text + all media
|
|
|
|
Crawl strategy:
|
|
1. Always fetch target URI first (the URL user asked about) and extract ALL links
|
|
2. Register links in link_registry with seen_count (boost for repeated links)
|
|
3. Sort links: ALL same-domain before ANY cross-domain
|
|
4. Crawl high-scoring links automatically up to max_pages
|
|
5. Cross-domain links only crawled if score >= 30 (exceptionally high relevance)
|
|
|
|
Example: Searching "dropbox encryption" on https://dropbox.com
|
|
- Target page fetched → extracts links (same-domain and cross-domain)
|
|
- Same-domain links with keyword matches crawled first
|
|
- Cross-domain links only crawled if score >= 30
|
|
- Continues until max_pages or runs out of high-scoring links
|
|
|
|
Args:
|
|
start_url: Starting URL
|
|
depth: How many levels deep to crawl (0 = just this page, 1 = page + links, etc.)
|
|
Use -1 for unlimited depth (full domain crawl)
|
|
max_pages: Maximum number of pages to fetch (default 10, use -1 for unlimited)
|
|
query_keywords: Optional list of primary keywords from user's query (full scoring weight)
|
|
keyword_variations: Optional list of keyword variations/synonyms (60% scoring weight)
|
|
progress_callback: Optional callback for progress updates
|
|
cache_check_callback: Optional callback for cache checking (not currently used)
|
|
should_continue_callback: DEPRECATED - no longer used
|
|
user_query: DEPRECATED - no longer used
|
|
mode: CrawlMode - what type of content to extract (default: TEXT)
|
|
media_callback: Optional async callback for discovered media items.
|
|
Called with dict containing 'url', 'media_type', 'source_page', etc.
|
|
Use this to process/store media as it's discovered.
|
|
|
|
Returns:
|
|
List of dicts with keys: url, title, text, fetched_at, score, depth, links_from_page
|
|
If mode includes media, each page dict also contains 'media' list
|
|
"""
|
|
# Handle unlimited depth/pages
|
|
unlimited_depth = (depth == -1)
|
|
unlimited_pages = (max_pages == -1)
|
|
if unlimited_depth:
|
|
depth = 999999 # Effectively unlimited
|
|
if unlimited_pages:
|
|
max_pages = 999999 # Effectively unlimited
|
|
all_pages = []
|
|
visited = set(initial_visited) if initial_visited else set()
|
|
base_domain = self._get_domain(start_url)
|
|
|
|
# Log resume info
|
|
if initial_visited:
|
|
logger.info(f"Resuming with {len(initial_visited)} previously visited pages")
|
|
|
|
# Track all discovered links: {url: {'anchor_texts': [str], 'seen_count': int, 'total_link_score': float, 'domain': str, 'is_same_domain': bool}}
|
|
link_registry = {}
|
|
|
|
# Log if using query-aware scoring
|
|
if query_keywords or keyword_variations:
|
|
logger.info(f"Using query-aware scoring with keywords: {query_keywords}, variations: {keyword_variations}")
|
|
|
|
logger.info(f"Starting intelligent crawl with depth={depth}, max_pages={max_pages}, primary_domain={base_domain}")
|
|
|
|
# PHASE 1: Always fetch target URI first
|
|
logger.info(f"Phase 1: Fetching target URI: {start_url}")
|
|
html, links = await self.fetch_webpage(
|
|
start_url,
|
|
extract_links=True,
|
|
extract_anchor_text=bool(query_keywords), # Extract anchor text only if we have keywords
|
|
cache_check_callback=cache_check_callback # Pass through SQLite3 cache callback
|
|
)
|
|
|
|
if html is None:
|
|
logger.error(f"Failed to fetch target URI {start_url}")
|
|
return []
|
|
|
|
# TEMPORARY: If HTML is empty but we got it from cache, we can still extract text/title from it
|
|
# Eventually we should store raw HTML in cache for all pages to enable proper virtual crawling
|
|
if not html:
|
|
logger.warning(f"Got empty HTML for {start_url} (likely from cache without raw_html), cannot extract links for virtual crawl")
|
|
# For now, continue with empty HTML - we'll at least process the target page even if we can't crawl links
|
|
# TODO: Store raw_html in cache for all pages to enable full virtual crawling
|
|
|
|
# Detect and skip RSS/XML feeds
|
|
if start_url.endswith(('.xml', '.rss', '.atom', 'feed', 'feeds')) or \
|
|
'/feed' in start_url or '/rss' in start_url or 'atom.xml' in start_url:
|
|
logger.warning(f"Skipping RSS/XML feed: {start_url} (detected feed pattern)")
|
|
return []
|
|
|
|
if html.strip().startswith('<?xml') or '<rss' in html[:500] or '<feed' in html[:500]:
|
|
logger.warning(f"Skipping RSS/XML feed: {start_url} (detected XML content)")
|
|
return []
|
|
|
|
# Extract target page content
|
|
title = self.extract_title(html)
|
|
text = self.extract_text_content(html)
|
|
|
|
target_page = {
|
|
"url": start_url,
|
|
"title": title,
|
|
"text": text,
|
|
"html": html, # Include raw HTML for cache storage (enables virtual crawling)
|
|
"fetched_at": datetime.now(timezone.utc).isoformat(),
|
|
"depth": 0,
|
|
"links_from_page": [l['url'] if isinstance(l, dict) else l for l in links] # Store for CSV
|
|
}
|
|
|
|
# Call page callback to archive raw HTML
|
|
if page_callback and html:
|
|
try:
|
|
await page_callback(start_url, html)
|
|
except Exception as e:
|
|
logger.warning(f"Page callback failed for {start_url}: {e}")
|
|
|
|
# Extract media if mode requires it
|
|
if mode in (CrawlMode.IMAGES, CrawlMode.VIDEOS, CrawlMode.MEDIA, CrawlMode.ALL):
|
|
media_items = extract_media_from_html(html, start_url, mode)
|
|
target_page["media"] = media_items
|
|
logger.info(f"Extracted {len(media_items)} media items from target page")
|
|
|
|
# Call media callback for each discovered item
|
|
if media_callback:
|
|
for item in media_items:
|
|
try:
|
|
await media_callback(item)
|
|
except Exception as e:
|
|
logger.warning(f"Media callback failed for {item.get('url')}: {e}")
|
|
|
|
# Score the target page
|
|
target_page["score"] = self._score_page(target_page, query_keywords=query_keywords)
|
|
all_pages.append(target_page)
|
|
visited.add(start_url)
|
|
|
|
media_count = len(target_page.get("media", []))
|
|
logger.info(f"Crawled target page (depth 0, score {target_page['score']:.1f}): {len(text)} chars, {len(links)} links, {media_count} media extracted")
|
|
|
|
# Update progress callback
|
|
if progress_callback:
|
|
await progress_callback(f"📄 **Fetched page 1**\n{title[:50]}\n<{start_url}>\n~{len(text)//4} tokens (estimated)")
|
|
|
|
# PHASE 2: Register ALL links from target page (both same-domain and cross-domain)
|
|
logger.info(f"Phase 2: Registering {len(links)} links from target page")
|
|
same_domain_count = 0
|
|
cross_domain_count = 0
|
|
|
|
for link in links:
|
|
link_url, anchor_text = normalize_link(link)
|
|
|
|
# Filter: not already visited
|
|
if link_url in visited:
|
|
continue
|
|
|
|
# Track domain
|
|
link_domain = self._get_domain(link_url)
|
|
is_same_domain = (link_domain == base_domain)
|
|
|
|
if is_same_domain:
|
|
same_domain_count += 1
|
|
else:
|
|
cross_domain_count += 1
|
|
|
|
# Register or update link
|
|
if link_url not in link_registry:
|
|
link_score = self._score_link(link_url, anchor_text, query_keywords, keyword_variations) if (query_keywords or keyword_variations) else 0.0
|
|
link_registry[link_url] = {
|
|
'anchor_texts': [anchor_text],
|
|
'seen_count': 1,
|
|
'total_link_score': link_score,
|
|
'domain': link_domain,
|
|
'is_same_domain': is_same_domain
|
|
}
|
|
else:
|
|
# Link seen again - increment count and update score
|
|
link_registry[link_url]['anchor_texts'].append(anchor_text)
|
|
link_registry[link_url]['seen_count'] += 1
|
|
if query_keywords or keyword_variations:
|
|
link_score = self._score_link(link_url, anchor_text, query_keywords, keyword_variations)
|
|
link_registry[link_url]['total_link_score'] += link_score
|
|
|
|
logger.info(f"Registered {len(link_registry)} unique links from target page (same-domain: {same_domain_count}, cross-domain: {cross_domain_count})")
|
|
if uris_total_callback:
|
|
uris_total_callback(len(link_registry) + 1) # +1 for target page
|
|
|
|
# PHASE 3: Decide whether to crawl deeper
|
|
# If depth=0 or we're at max_pages, stop
|
|
if depth == 0 or len(all_pages) >= max_pages:
|
|
logger.info(f"Stopping crawl (depth={depth}, pages={len(all_pages)})")
|
|
return self._finalize_results(all_pages)
|
|
|
|
# If target page has high score (good keyword matches), maybe we're done
|
|
# But if it has low score, we should explore links
|
|
# Threshold: if target page scores < 50, explore links
|
|
if target_page['score'] >= 50 and query_keywords:
|
|
logger.info(f"Target page has high score ({target_page['score']:.1f}), may not need to crawl deeper")
|
|
# Still crawl a few top links if they have strong keyword matches
|
|
else:
|
|
logger.info(f"Target page has low score ({target_page['score']:.1f}), will explore keyword-matching links")
|
|
|
|
# PHASE 4: Sort links by combined score (keyword_score + seen_count_boost)
|
|
# STRATEGY: Prioritize same-domain links, only allow cross-domain if exceptionally high score
|
|
# seen_count_boost = seen_count * 2 (each additional sighting adds 2 points)
|
|
scored_links = []
|
|
for link_url, meta in link_registry.items():
|
|
seen_count_boost = (meta['seen_count'] - 1) * 2 # First sighting doesn't get boost
|
|
combined_score = meta['total_link_score'] + seen_count_boost
|
|
scored_links.append({
|
|
'url': link_url,
|
|
'anchor_texts': meta['anchor_texts'],
|
|
'seen_count': meta['seen_count'],
|
|
'link_score': meta['total_link_score'],
|
|
'combined_score': combined_score,
|
|
'domain': meta['domain'],
|
|
'is_same_domain': meta['is_same_domain']
|
|
})
|
|
|
|
# Sort by: same_domain first (True sorts before False), then by combined_score
|
|
# This ensures ALL same-domain links are processed before ANY cross-domain links
|
|
scored_links.sort(key=lambda x: (not x['is_same_domain'], -x['combined_score']))
|
|
|
|
# Log top scoring links by domain
|
|
same_domain_links = [l for l in scored_links if l['is_same_domain'] and l['combined_score'] > 0][:10]
|
|
cross_domain_links = [l for l in scored_links if not l['is_same_domain'] and l['combined_score'] > 0][:5]
|
|
|
|
if same_domain_links:
|
|
logger.info(f"Top {len(same_domain_links)} same-domain scoring links:")
|
|
for i, link in enumerate(same_domain_links, 1):
|
|
logger.info(f" {i}. [{link['combined_score']:.1f}] {link['url']} (keyword={link['link_score']:.1f}, seen={link['seen_count']}x)")
|
|
|
|
if cross_domain_links:
|
|
logger.info(f"Top {len(cross_domain_links)} cross-domain scoring links (will only crawl if score > 30):")
|
|
for i, link in enumerate(cross_domain_links, 1):
|
|
logger.info(f" {i}. [{link['combined_score']:.1f}] {link['url']} ({link['domain']}, keyword={link['link_score']:.1f})")
|
|
|
|
# PHASE 5: Crawl links iteratively for each depth level (1 to depth)
|
|
for current_depth in range(1, depth + 1):
|
|
if len(all_pages) >= max_pages:
|
|
break
|
|
|
|
logger.info(f"Phase 5.{current_depth}: Crawling depth-{current_depth} links (max {max_pages - len(all_pages)} more pages)")
|
|
|
|
# Re-score and sort all unvisited links
|
|
candidates = []
|
|
for link_url, meta in link_registry.items():
|
|
if link_url in visited:
|
|
continue
|
|
seen_count_boost = (meta['seen_count'] - 1) * 2
|
|
combined_score = meta['total_link_score'] + seen_count_boost
|
|
candidates.append({
|
|
'url': link_url,
|
|
'anchor_texts': meta['anchor_texts'],
|
|
'seen_count': meta['seen_count'],
|
|
'link_score': meta['total_link_score'],
|
|
'combined_score': combined_score,
|
|
'domain': meta['domain'],
|
|
'is_same_domain': meta['is_same_domain']
|
|
})
|
|
|
|
if not candidates:
|
|
logger.info(f"No more unvisited links at depth {current_depth}")
|
|
break
|
|
|
|
# Sort: same_domain first, then by combined_score
|
|
candidates.sort(key=lambda x: (not x['is_same_domain'], -x['combined_score']))
|
|
|
|
# Log top candidates for this depth
|
|
top_same = [l for l in candidates if l['is_same_domain'] and l['combined_score'] > 0][:10]
|
|
if top_same:
|
|
logger.info(f"Top {len(top_same)} depth-{current_depth} same-domain candidates:")
|
|
for i, link in enumerate(top_same, 1):
|
|
logger.info(f" {i}. [{link['combined_score']:.1f}] {link['url']} (keyword={link['link_score']:.1f}, seen={link['seen_count']}x)")
|
|
|
|
# Track pages added at this depth level
|
|
pages_at_this_depth = 0
|
|
|
|
for link_data in candidates:
|
|
if len(all_pages) >= max_pages:
|
|
break
|
|
|
|
# Apply score thresholds
|
|
# When no keywords provided, allow all same-domain links
|
|
if link_data['is_same_domain']:
|
|
if query_keywords and link_data['link_score'] <= 0:
|
|
logger.debug(f"Skipping {link_data['url']} - no keyword matches")
|
|
continue
|
|
else:
|
|
if link_data['combined_score'] < 30:
|
|
logger.info(f"Skipping cross-domain {link_data['url']} ({link_data['domain']}) - score {link_data['combined_score']:.1f} < 30")
|
|
continue
|
|
else:
|
|
logger.info(f"Including cross-domain {link_data['url']} - high score {link_data['combined_score']:.1f}")
|
|
|
|
link_url = link_data['url']
|
|
if link_url in visited:
|
|
continue
|
|
|
|
# Fetch the page
|
|
logger.info(f"Crawling depth-{current_depth}: {link_url} (score={link_data['combined_score']:.1f})")
|
|
html, child_links = await self.fetch_webpage(
|
|
link_url,
|
|
extract_links=True,
|
|
extract_anchor_text=bool(query_keywords),
|
|
cache_check_callback=cache_check_callback
|
|
)
|
|
|
|
if not html:
|
|
logger.warning(f"Skipping {link_url} (fetch failed)")
|
|
continue
|
|
|
|
# Skip RSS/XML feeds
|
|
if link_url.endswith(('.xml', '.rss', '.atom', 'feed', 'feeds')) or \
|
|
'/feed' in link_url or '/rss' in link_url or 'atom.xml' in link_url:
|
|
logger.warning(f"Skipping RSS/XML feed: {link_url}")
|
|
continue
|
|
if html.strip().startswith('<?xml') or '<rss' in html[:500] or '<feed' in html[:500]:
|
|
logger.warning(f"Skipping RSS/XML feed: {link_url}")
|
|
continue
|
|
|
|
# Extract content
|
|
title = self.extract_title(html)
|
|
text = self.extract_text_content(html)
|
|
|
|
page_data = {
|
|
"url": link_url,
|
|
"title": title,
|
|
"text": text,
|
|
"html": html,
|
|
"fetched_at": datetime.now(timezone.utc).isoformat(),
|
|
"depth": current_depth,
|
|
"links_from_page": [l['url'] if isinstance(l, dict) else l for l in child_links]
|
|
}
|
|
|
|
# Call page callback to archive raw HTML
|
|
if page_callback and html:
|
|
try:
|
|
await page_callback(link_url, html)
|
|
except Exception as e:
|
|
logger.warning(f"Page callback failed for {link_url}: {e}")
|
|
|
|
# Extract media if mode requires it
|
|
if mode in (CrawlMode.IMAGES, CrawlMode.VIDEOS, CrawlMode.MEDIA, CrawlMode.ALL):
|
|
media_items = extract_media_from_html(html, link_url, mode)
|
|
page_data["media"] = media_items
|
|
|
|
# Call media callback for each discovered item
|
|
if media_callback:
|
|
for item in media_items:
|
|
try:
|
|
await media_callback(item)
|
|
except Exception as e:
|
|
logger.warning(f"Media callback failed for {item.get('url')}: {e}")
|
|
|
|
page_data["score"] = self._score_page(page_data, query_keywords=query_keywords)
|
|
all_pages.append(page_data)
|
|
visited.add(link_url)
|
|
pages_at_this_depth += 1
|
|
|
|
media_count = len(page_data.get("media", []))
|
|
logger.info(f"Crawled {link_url} (depth {current_depth}, score {page_data['score']:.1f}): {len(text)} chars, {len(child_links)} links, {media_count} media")
|
|
|
|
if progress_callback:
|
|
await progress_callback(f"📄 **Fetched page {len(all_pages)}** (depth {current_depth})\n{title[:50]}\n<{link_url}>")
|
|
|
|
# Register newly discovered links for next depth level
|
|
for link in child_links:
|
|
child_url, anchor_text = normalize_link(link)
|
|
|
|
if child_url in visited:
|
|
continue
|
|
|
|
child_domain = self._get_domain(child_url)
|
|
is_same_domain = (child_domain == base_domain)
|
|
|
|
if child_url not in link_registry:
|
|
link_score = self._score_link(child_url, anchor_text, query_keywords, keyword_variations) if (query_keywords or keyword_variations) else 0.0
|
|
link_registry[child_url] = {
|
|
'anchor_texts': [anchor_text],
|
|
'seen_count': 1,
|
|
'total_link_score': link_score,
|
|
'domain': child_domain,
|
|
'is_same_domain': is_same_domain
|
|
}
|
|
else:
|
|
link_registry[child_url]['anchor_texts'].append(anchor_text)
|
|
link_registry[child_url]['seen_count'] += 1
|
|
if query_keywords or keyword_variations:
|
|
link_score = self._score_link(child_url, anchor_text, query_keywords, keyword_variations)
|
|
link_registry[child_url]['total_link_score'] += link_score
|
|
|
|
# Update total after processing child links
|
|
if uris_total_callback:
|
|
uris_total_callback(len(link_registry) + 1)
|
|
|
|
logger.info(f"Depth {current_depth} complete: crawled {pages_at_this_depth} pages")
|
|
|
|
# Stop if no pages were crawled at this depth (all links exhausted or skipped)
|
|
if pages_at_this_depth == 0:
|
|
logger.info("No pages crawled at this depth - stopping crawl")
|
|
break
|
|
|
|
return self._finalize_results(all_pages)
|
|
|
|
def _finalize_results(self, all_pages: List[Dict]) -> List[Dict]:
|
|
"""Sort and finalize crawl results"""
|
|
# Sort by depth first (main page first), then by score within each depth
|
|
all_pages.sort(key=lambda p: (p.get("depth", 0), -p["score"]))
|
|
|
|
logger.info(f"Crawl complete: fetched {len(all_pages)} pages")
|
|
if len(all_pages) > 1:
|
|
logger.info(f"Score range: {all_pages[-1]['score']:.1f} to {all_pages[0]['score']:.1f}")
|
|
|
|
return all_pages
|
|
|
|
def select_pages_by_token_budget(
|
|
self,
|
|
pages: list,
|
|
token_budget: int,
|
|
reserve_tokens: int = 1000
|
|
) -> list:
|
|
"""
|
|
Select highest-value pages that fit within token budget using greedy knapsack algorithm.
|
|
|
|
Args:
|
|
pages: List of page dicts with 'score', 'tokens', 'url', 'title', 'text' keys
|
|
token_budget: Maximum tokens available for page content
|
|
reserve_tokens: Tokens to reserve for output and system prompts
|
|
|
|
Returns:
|
|
List of selected pages (sorted by score descending) that fit in budget
|
|
"""
|
|
if not pages:
|
|
return []
|
|
|
|
# Adjust budget for reserved tokens
|
|
effective_budget = max(0, token_budget - reserve_tokens)
|
|
|
|
if effective_budget <= 0:
|
|
logger.warning(f"Token budget too low after reserving {reserve_tokens} tokens")
|
|
return []
|
|
|
|
# Calculate value/token ratio for each page (efficiency metric)
|
|
pages_with_ratio = []
|
|
for page in pages:
|
|
tokens = page.get('tokens', len(page.get('text', '')) // 4) # Fallback estimate
|
|
score = page.get('score', 0)
|
|
if tokens > 0:
|
|
ratio = score / tokens
|
|
pages_with_ratio.append({
|
|
**page,
|
|
'tokens': tokens,
|
|
'value_per_token': ratio
|
|
})
|
|
|
|
if not pages_with_ratio:
|
|
return []
|
|
|
|
# Greedy algorithm: Sort by value/token ratio (descending)
|
|
# This maximizes value while minimizing token usage
|
|
pages_with_ratio.sort(key=lambda p: p['value_per_token'], reverse=True)
|
|
|
|
# Select pages greedily until budget exhausted
|
|
selected = []
|
|
total_tokens = 0
|
|
|
|
for page in pages_with_ratio:
|
|
page_tokens = page['tokens']
|
|
if total_tokens + page_tokens <= effective_budget:
|
|
selected.append(page)
|
|
total_tokens += page_tokens
|
|
logger.debug(f"Selected page: {page['title']} (score={page['score']:.1f}, tokens={page_tokens}, ratio={page['value_per_token']:.3f})")
|
|
else:
|
|
logger.debug(f"Skipped page: {page['title']} (would exceed budget: {total_tokens + page_tokens} > {effective_budget})")
|
|
|
|
# Sort selected pages by score (highest first) for presentation
|
|
selected.sort(key=lambda p: p['score'], reverse=True)
|
|
|
|
logger.info(f"Selected {len(selected)}/{len(pages)} pages using {total_tokens}/{effective_budget} tokens (budget utilization: {total_tokens/effective_budget*100:.1f}%)")
|
|
|
|
return selected
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Test the async web fetcher
|
|
async def test():
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
fetcher = AsyncWebFetcher()
|
|
|
|
# Test single page fetch
|
|
print("Testing single page fetch...")
|
|
results = await fetcher.fetch_with_depth("https://example.com", depth=0)
|
|
|
|
if results:
|
|
print(f"\nFetched: {results[0]['title']}")
|
|
print(f"Content length: {len(results[0]['text'])} chars")
|
|
print(f"Preview: {results[0]['text'][:200]}...")
|
|
else:
|
|
print("Failed to fetch")
|
|
|
|
asyncio.run(test())
|