#!/usr/bin/env python3
# This is free software for the public good of a permacomputer hosted at
# permacomputer.com, an always-on computer by the people, for the people.
# One which is durable, easy to repair, & distributed like tap water
# for machine learning intelligence.
#
# The permacomputer is community-owned infrastructure optimized around
# four values:
#
#   TRUTH      First principles, math & science, open source code freely distributed
#   FREEDOM    Voluntary partnerships, freedom from tyranny & corporate control
#   HARMONY    Minimal waste, self-renewing systems with diverse thriving connections
#   LOVE       Be yourself without hurting others, cooperation through natural law
#
# This software contributes to that vision by archiving the web, preserving digital knowledge before it disappears.
# Code is seeds to sprout on any abandoned technology.

"""
Async Web Fetcher for Discord Bot
Ethical web scraping with robots.txt compliance, async/await compatible

Supports multiple crawl modes:
- text: Extract text content (default, current behavior)
- images: Collect images only
- videos: Collect videos only
- media: Collect all media (images + videos + audio)
- all: Uber crawl - text + all media, full domain slurp

# inputs > process > outputs
# The universe is logical. - The Sign Maker
"""

import os
import io
import hashlib
import logging
import aiohttp
import asyncio
import re
from dataclasses import dataclass, field
from datetime import datetime, timezone
from enum import Enum
from typing import Optional, Dict, List, Tuple, Set, Any
from urllib.parse import urljoin
from urllib.robotparser import RobotFileParser
from bs4 import BeautifulSoup
import time

from miniuri import Uri


class CrawlMode(Enum):
    """Crawl modes for different content types."""
    TEXT = "text"       # Extract text content (default)
    IMAGES = "images"   # Collect images only
    VIDEOS = "videos"   # Collect videos only
    MEDIA = "media"     # All media (images + videos + audio)
    ALL = "all"         # Uber crawl: text + all media


# File extensions by media type
IMAGE_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.bmp', '.ico', '.tiff', '.tif', '.avif'}
VIDEO_EXTENSIONS = {'.mp4', '.webm', '.mov', '.avi', '.mkv', '.m4v', '.ogv', '.flv', '.wmv'}
AUDIO_EXTENSIONS = {'.mp3', '.wav', '.ogg', '.m4a', '.flac', '.aac', '.wma'}

# Code file extensions
CODE_EXTENSIONS = {
    '.py', '.js', '.ts', '.jsx', '.tsx', '.mjs', '.cjs',  # Python, JavaScript, TypeScript
    '.rs', '.go', '.rb', '.php', '.pl', '.pm',  # Rust, Go, Ruby, PHP, Perl
    '.java', '.kt', '.kts', '.scala', '.groovy',  # JVM languages
    '.c', '.h', '.cpp', '.hpp', '.cc', '.cxx', '.hxx',  # C/C++
    '.cs', '.fs', '.fsx',  # .NET
    '.swift', '.m', '.mm',  # Apple
    '.lua', '.r', '.R', '.jl',  # Lua, R, Julia
    '.sh', '.bash', '.zsh', '.fish', '.ps1',  # Shell
    '.sql', '.graphql', '.gql',  # Query languages
    '.yaml', '.yml', '.toml', '.json', '.xml',  # Config
    '.md', '.rst', '.txt',  # Docs
    '.zig', '.nim', '.d', '.v',  # Modern systems langs
    '.ex', '.exs', '.erl', '.hrl',  # Erlang/Elixir
    '.clj', '.cljs', '.cljc', '.edn',  # Clojure
    '.hs', '.lhs',  # Haskell
    '.ml', '.mli', '.re', '.rei',  # OCaml/ReasonML
    '.lisp', '.cl', '.el', '.scm', '.rkt',  # Lisps
    '.f90', '.f95', '.f03', '.for',  # Fortran
    '.asm', '.s',  # Assembly
    '.cob', '.cbl',  # COBOL
    '.pro',  # Prolog
    '.tcl',  # Tcl
    '.dart',  # Dart
    '.raku', '.p6',  # Raku
    '.cr',  # Crystal
    '.vue', '.svelte',  # Frontend frameworks
    '.tf', '.hcl',  # Terraform
    '.dockerfile', '.makefile',  # Build files
}

# Font file extensions
FONT_EXTENSIONS = {'.woff', '.woff2', '.ttf', '.otf', '.eot', '.sfnt'}

# Style file extensions
STYLE_EXTENSIONS = {'.css', '.scss', '.sass', '.less', '.styl'}

# MIME types by media type
IMAGE_MIME_PREFIXES = ('image/',)
VIDEO_MIME_PREFIXES = ('video/',)
AUDIO_MIME_PREFIXES = ('audio/',)
CODE_MIME_TYPES = {
    'text/x-python', 'application/x-python', 'text/x-python-script',
    'text/javascript', 'application/javascript', 'application/x-javascript',
    'text/typescript', 'application/typescript',
    'text/x-rust', 'text/x-go', 'text/x-ruby', 'application/x-ruby',
    'text/x-java-source', 'text/x-kotlin', 'text/x-scala',
    'text/x-c', 'text/x-c++', 'text/x-csrc', 'text/x-c++src',
    'text/x-csharp', 'text/x-fsharp',
    'text/x-swift', 'text/x-objective-c',
    'text/x-lua', 'text/x-r', 'text/x-julia',
    'text/x-shellscript', 'application/x-sh', 'text/x-bash',
    'application/sql', 'application/graphql',
    'application/json', 'application/xml', 'text/xml',
    'text/yaml', 'application/x-yaml', 'text/x-yaml',
    'text/markdown', 'text/x-markdown',
    'text/plain',  # Often used for code
}
FONT_MIME_TYPES = {
    'font/woff', 'font/woff2', 'font/ttf', 'font/otf', 'font/sfnt',
    'application/font-woff', 'application/font-woff2',
    'application/x-font-ttf', 'application/x-font-otf',
    'application/vnd.ms-fontobject',
}
STYLE_MIME_TYPES = {'text/css', 'text/x-scss', 'text/x-sass', 'text/x-less'}


@dataclass
class MediaItem:
    """Represents a discovered media item."""
    url: str
    source_page: str
    media_type: str  # 'image', 'video', 'audio'
    mime_type: Optional[str] = None
    md5_hash: Optional[str] = None
    file_size: Optional[int] = None
    alt_text: Optional[str] = None
    title: Optional[str] = None
    width: Optional[int] = None
    height: Optional[int] = None
    discovered_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())


class MediaMetadata:
    """
    Accumulates metadata from all sources without clobbering.

    "Never clobber, always append" - collects ALL metadata from:
    - img.alt, img.title
    - a.title, a.text (link text)
    - figcaption
    - nearby headings
    - page title/h1

    Produces a combined searchable_text for full-text search.
    """

    def __init__(self):
        self.titles: List[str] = []
        self.alt_texts: List[str] = []
        self.descriptions: List[str] = []
        self.captions: List[str] = []
        self.headings: List[str] = []
        self.link_texts: List[str] = []
        self.link_titles: List[str] = []

    def _add_unique(self, lst: List[str], value: str, max_len: int = 500) -> None:
        """Add value to list if non-empty and not duplicate."""
        if value and value.strip():
            clean = value.strip()[:max_len]
            if clean not in lst:
                lst.append(clean)

    def add_img_alt(self, alt: str) -> None:
        """Add img alt attribute."""
        self._add_unique(self.alt_texts, alt)

    def add_img_title(self, title: str) -> None:
        """Add img title attribute."""
        self._add_unique(self.titles, title)

    def add_link_title(self, title: str) -> None:
        """Add <a title="..."> attribute."""
        self._add_unique(self.link_titles, title)

    def add_link_text(self, text: str) -> None:
        """Add <a> inner text."""
        self._add_unique(self.link_texts, text)

    def add_figcaption(self, caption: str) -> None:
        """Add figcaption text."""
        self._add_unique(self.captions, caption)

    def add_heading(self, heading: str) -> None:
        """Add nearby heading (h1-h6)."""
        self._add_unique(self.headings, heading)

    def add_description(self, desc: str) -> None:
        """Add description (og:description, meta description, etc.)."""
        self._add_unique(self.descriptions, desc)

    def add_page_title(self, title: str) -> None:
        """Add page title."""
        self._add_unique(self.titles, title)

    def get_best_title(self) -> Optional[str]:
        """Get best title for display (first non-empty)."""
        for lst in [self.titles, self.alt_texts, self.link_titles,
                    self.captions, self.link_texts, self.headings]:
            if lst:
                return lst[0]
        return None

    def get_best_alt(self) -> Optional[str]:
        """Get best alt text for accessibility."""
        if self.alt_texts:
            return self.alt_texts[0]
        return self.get_best_title()

    def to_searchable_text(self) -> str:
        """
        Combine ALL collected metadata into searchable text.

        This enables finding images by ANY associated text:
        - "find images of cats" matches img alt="cute cat"
        - "find images from blog post about python" matches page content
        """
        all_parts = []
        # Dedupe while preserving order
        seen = set()
        for lst in [self.titles, self.alt_texts, self.descriptions,
                    self.captions, self.headings, self.link_texts, self.link_titles]:
            for item in lst:
                if item and item not in seen:
                    all_parts.append(item)
                    seen.add(item)
        return ' | '.join(all_parts)

    def to_dict(self) -> Dict[str, Any]:
        """Export all collected metadata as dict."""
        return {
            'titles': self.titles,
            'alt_texts': self.alt_texts,
            'descriptions': self.descriptions,
            'captions': self.captions,
            'headings': self.headings,
            'link_texts': self.link_texts,
            'link_titles': self.link_titles,
            'searchable_text': self.to_searchable_text(),
            'best_title': self.get_best_title(),
            'best_alt': self.get_best_alt(),
        }


# PDF text extraction
try:
    from pypdf import PdfReader
    PDF_SUPPORT = True
except ImportError:
    PDF_SUPPORT = False

logger = logging.getLogger(__name__)

# Default crawl delay
DEFAULT_CRAWL_DELAY = 2.0

# Global last error storage (for error reporting without log parsing)
# Format: {'type': 'robots_txt'|'http_error'|'timeout'|'dns'|'ssl'|'unknown', 'details': str, 'url': str}
LAST_FETCH_ERROR = None


def get_last_fetch_error() -> Optional[Dict]:
    """Get the last fetch error, if any."""
    return LAST_FETCH_ERROR


def strip_uri_fragment(uri_str: str) -> str:
    """
    Strip the fragment (#anchor) from a URI using miniuri.

    Args:
        uri_str: URI that may contain a fragment

    Returns:
        URI without fragment
    """
    if not uri_str:
        return uri_str

    uri = Uri(uri_str)
    # Reconstruct URI without fragment
    result = f"{uri.scheme}://{uri.authority}{uri.path or ''}"
    if uri.query:
        result += f"?{uri.query}"
    return result


def normalize_link(link) -> Tuple[str, str]:
    """
    Extract URI and anchor text from a link, stripping fragments.

    Args:
        link: Either a dict with 'url' and optional 'anchor_text', or a string URI

    Returns:
        Tuple of (normalized_uri, anchor_text)
    """
    if isinstance(link, dict):
        uri = strip_uri_fragment(link.get('url', ''))
        anchor_text = link.get('anchor_text', '')
    else:
        uri = strip_uri_fragment(link)
        anchor_text = ''
    return uri, anchor_text


# Alias for backwards compatibility
strip_url_fragment = strip_uri_fragment


def extract_text_from_pdf(pdf_bytes: bytes) -> Optional[str]:
    """
    Extract text from PDF binary data.

    Args:
        pdf_bytes: Raw PDF file bytes

    Returns:
        Extracted text or None if extraction fails
    """
    if not PDF_SUPPORT:
        logger.warning("PDF support not available (pypdf not installed)")
        return None

    try:
        reader = PdfReader(io.BytesIO(pdf_bytes))
        text_parts = []
        for page in reader.pages:
            page_text = page.extract_text()
            if page_text:
                text_parts.append(page_text)
        if text_parts:
            return "\n\n".join(text_parts)
        return None
    except Exception as e:
        logger.error(f"Failed to extract text from PDF: {e}")
        return None


def clear_last_fetch_error():
    """Clear the last fetch error."""
    global LAST_FETCH_ERROR
    LAST_FETCH_ERROR = None


async def fetch_robots_txt(url: str, user_agent: str = "uncloseai.com/1.42") -> Optional[str]:
    """
    Fetch raw robots.txt content for analysis and error reporting.

    Args:
        url: Any URL on the domain (robots.txt will be fetched from root)
        user_agent: User agent string

    Returns:
        Raw robots.txt content or None if not available
    """
    parsed = Uri(url)
    robots_url = f"{parsed.scheme}://{parsed.hostname}/robots.txt"

    try:
        async with aiohttp.ClientSession() as session:
            async with session.get(
                robots_url,
                headers={"User-Agent": user_agent},
                timeout=aiohttp.ClientTimeout(total=5)
            ) as response:
                if response.status == 200:
                    return await response.text()
    except Exception as e:
        logger.warning(f"Could not fetch robots.txt for error analysis: {e}")

    return None


def analyze_robots_txt_blocking(robots_txt: str, blocked_url: str, user_agent: str = "uncloseai.com/1.42") -> str:
    """
    Analyze robots.txt to explain why a URL is blocked.

    Args:
        robots_txt: Raw robots.txt content
        blocked_url: The URL that was blocked
        user_agent: User agent to check against

    Returns:
        Human-readable explanation of the blocking rule
    """
    parsed = Uri(blocked_url)
    path = parsed.path or "/"

    # Parse robots.txt manually for detailed analysis
    lines = robots_txt.strip().split('\n')
    current_agent = None
    blocking_rule = None
    all_rules = []

    for line in lines:
        line = line.strip()
        if not line or line.startswith('#'):
            continue

        if ':' in line:
            key, value = line.split(':', 1)
            key = key.strip().lower()
            value = value.strip()

            if key == 'user-agent':
                current_agent = value
            elif key == 'disallow' and current_agent:
                # Check if this rule applies to us
                agent_matches = (
                    current_agent == '*' or
                    user_agent.lower().startswith(current_agent.lower()) or
                    current_agent.lower() in user_agent.lower()
                )
                if agent_matches:
                    all_rules.append(f"Disallow: {value}")
                    # Check if this rule blocks our path
                    if value and path.startswith(value):
                        blocking_rule = f"Disallow: {value}"

    if blocking_rule:
        result = f"**Blocked by rule:** `{blocking_rule}`\n"
        result += f"**Path requested:** `{path}`\n"
        result += f"**User-Agent:** `{user_agent}`\n\n"

        # Show relevant rules
        if all_rules:
            result += "**Applicable rules:**\n"
            for rule in all_rules[:10]:  # Limit to first 10
                result += f"  • `{rule}`\n"

        return result
    elif all_rules:
        return f"Blocked by robots.txt (exact rule unclear). Rules found:\n" + "\n".join(f"  • `{r}`" for r in all_rules[:10])
    else:
        return "Blocked by robots.txt (no matching disallow rule found - may be a catch-all)"


def get_media_type_from_extension(url: str) -> Optional[str]:
    """
    Determine media type from URL extension.

    Returns:
        'image', 'video', 'audio', 'code', 'font', 'style', or None
    """
    parsed = Uri(url)
    if not parsed.path:
        return None
    path = parsed.path.lower()

    # Check for extension match
    for ext in IMAGE_EXTENSIONS:
        if path.endswith(ext):
            return 'image'
    for ext in VIDEO_EXTENSIONS:
        if path.endswith(ext):
            return 'video'
    for ext in AUDIO_EXTENSIONS:
        if path.endswith(ext):
            return 'audio'
    for ext in CODE_EXTENSIONS:
        if path.endswith(ext):
            return 'code'
    for ext in FONT_EXTENSIONS:
        if path.endswith(ext):
            return 'font'
    for ext in STYLE_EXTENSIONS:
        if path.endswith(ext):
            return 'style'
    return None


def get_media_type_from_mime(mime_type: str) -> Optional[str]:
    """
    Determine media type from MIME type.

    Returns:
        'image', 'video', 'audio', 'code', 'font', 'style', or None
    """
    if not mime_type:
        return None

    mime_lower = mime_type.lower()
    # Check prefixes first
    if mime_lower.startswith(IMAGE_MIME_PREFIXES):
        return 'image'
    if mime_lower.startswith(VIDEO_MIME_PREFIXES):
        return 'video'
    if mime_lower.startswith(AUDIO_MIME_PREFIXES):
        return 'audio'
    # Check exact matches for code/font/style
    if mime_lower in CODE_MIME_TYPES:
        return 'code'
    if mime_lower in FONT_MIME_TYPES:
        return 'font'
    if mime_lower in STYLE_MIME_TYPES:
        return 'style'
    return None


def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMode.MEDIA) -> List[Dict[str, Any]]:
    """
    Extract media URLs from HTML content.

    Extracts from:
    - <img src="..."> and <img srcset="...">
    - <picture><source srcset="..."></picture>
    - <video src="..."> and <video poster="...">
    - <source src="..."> (within video/audio)
    - <audio src="...">
    - CSS background-image: url(...)
    - <a href="..."> pointing to media files
    - data-src, data-lazy-src (lazy loading)
    - Open Graph and Twitter card meta tags

    Args:
        html: HTML content
        base_url: Base URL for resolving relative URLs
        mode: CrawlMode to filter what types to extract

    Returns:
        List of dicts with 'url', 'media_type', 'alt_text', 'title', etc.
    """
    media_items = []
    seen_urls: Set[str] = set()

    # Determine which types to collect based on mode
    collect_images = mode in (CrawlMode.IMAGES, CrawlMode.MEDIA, CrawlMode.ALL)
    collect_videos = mode in (CrawlMode.VIDEOS, CrawlMode.MEDIA, CrawlMode.ALL)
    collect_audio = mode in (CrawlMode.MEDIA, CrawlMode.ALL)

    soup = BeautifulSoup(html, "html.parser")

    # Extract page-level metadata for inference
    page_title = ''
    title_tag = soup.find('title')
    if title_tag:
        page_title = title_tag.get_text(strip=True)

    # Get h1 as fallback title
    h1_tag = soup.find('h1')
    page_h1 = h1_tag.get_text(strip=True) if h1_tag else ''

    # Extract meta description and keywords for searchability
    page_description = ''
    page_keywords = ''
    for meta in soup.find_all('meta'):
        name = (meta.get('name') or meta.get('property') or '').lower()
        content = meta.get('content', '')
        if name == 'description' or name == 'og:description':
            page_description = content[:500]  # Limit length
        elif name == 'keywords':
            page_keywords = content[:500]

    # Extract page content (body text) for full-text search
    # This enables blog images to be searchable by post content
    # IMPORTANT: Parse a fresh copy so we don't remove header/nav images from the soup
    # that will be used for media extraction below
    page_content = ''
    body = soup.find('body')
    if body:
        body_soup = BeautifulSoup(str(body), 'html.parser')
        # Remove script, style, nav, footer elements from the COPY only
        for tag in body_soup.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside']):
            tag.decompose()
        page_content = body_soup.get_text(separator=' ', strip=True)[:10000]  # Limit to 10k chars

    def get_context_for_element(element) -> dict:
        """Extract contextual metadata from surrounding HTML elements."""
        context = {
            'page_title': page_title,
            'page_h1': page_h1,
            'figure_caption': None,
            'nearby_heading': None,
            'link_text': None,
            'link_title': None,  # <a title="..."> attribute
        }

        # Check if inside a <figure> with <figcaption>
        figure = element.find_parent('figure')
        if figure:
            figcaption = figure.find('figcaption')
            if figcaption:
                context['figure_caption'] = figcaption.get_text(strip=True)[:200]

        # Check if inside an <a> tag with text and/or title
        link = element.find_parent('a')
        if link:
            link_text = link.get_text(strip=True)
            if link_text and link_text != element.get('alt', ''):
                context['link_text'] = link_text[:200]
            # Also extract title attribute from <a> tag (tooltip text)
            link_title = link.get('title', '').strip()
            if link_title:
                context['link_title'] = link_title[:200]

        # Find nearest heading (h1-h6) before this element
        for heading_tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
            # Look for preceding siblings and parents
            for prev in element.find_all_previous(heading_tag, limit=3):
                heading_text = prev.get_text(strip=True)
                if heading_text:
                    context['nearby_heading'] = heading_text[:200]
                    break
            if context['nearby_heading']:
                break

        return context

    def infer_title(alt_text: str, title: str, context: dict) -> str:
        """Infer best title from available metadata."""
        # Priority: explicit title > alt text > link title > figure caption > link text > nearby heading > page h1 > page title
        if title and title.strip():
            return title.strip()
        if alt_text and alt_text.strip() and len(alt_text) > 3:
            return alt_text.strip()
        if context.get('link_title'):  # <a title="..."> attribute
            return context['link_title']
        if context.get('figure_caption'):
            return context['figure_caption']
        if context.get('link_text'):
            return context['link_text']
        if context.get('nearby_heading'):
            return context['nearby_heading']
        if context.get('page_h1'):
            return context['page_h1']
        if context.get('page_title'):
            return context['page_title']
        return None

    def add_media(url: str, media_type: str, alt_text: str = None, title: str = None, width: int = None, height: int = None, element=None, detail_page_url: str = None):
        """Helper to add media item if not already seen."""
        if not url or url in seen_urls:
            return
        if url.startswith('data:'):  # Skip data URIs for now
            return

        # Resolve relative URLs
        absolute_url = urljoin(base_url, url)
        if absolute_url in seen_urls:
            return

        seen_urls.add(absolute_url)

        # Get context from surrounding elements
        context = get_context_for_element(element) if element else {'page_title': page_title, 'page_h1': page_h1}

        # Build metadata accumulator - "never clobber, always append"
        metadata = MediaMetadata()

        # Add from img element
        if alt_text:
            metadata.add_img_alt(alt_text)
        if title:
            metadata.add_img_title(title)

        # Add from context
        if context.get('figure_caption'):
            metadata.add_figcaption(context['figure_caption'])
        if context.get('link_text'):
            metadata.add_link_text(context['link_text'])
        if context.get('link_title'):
            metadata.add_link_title(context['link_title'])
        if context.get('nearby_heading'):
            metadata.add_heading(context['nearby_heading'])
        if context.get('page_title'):
            metadata.add_page_title(context['page_title'])
        if context.get('page_h1'):
            metadata.add_heading(context['page_h1'])

        # Add page-level metadata
        if page_description:
            metadata.add_description(page_description)

        # Get best values for backward compatibility
        best_title = metadata.get_best_title()
        best_alt = metadata.get_best_alt()

        # Check if this image is wrapped in an <a> tag pointing to a detail page
        # (Pinterest-style galleries where thumbnail links to detail page with canonical image)
        resolved_detail_url = None
        if element and not detail_page_url:
            parent_link = element.find_parent('a', href=True)
            if parent_link:
                href = parent_link.get('href', '')
                # Only consider internal links (not direct image links)
                if href and not get_media_type_from_extension(href):
                    resolved_detail_url = urljoin(base_url, href)
                    # Only track same-domain detail pages
                    base_domain = Uri(base_url).hostname
                    detail_domain = Uri(resolved_detail_url).hostname
                    if base_domain != detail_domain:
                        resolved_detail_url = None
        elif detail_page_url:
            resolved_detail_url = urljoin(base_url, detail_page_url)

        media_items.append({
            'url': absolute_url,
            'media_type': media_type,
            'alt_text': best_alt,
            'title': best_title,
            'width': width,
            'height': height,
            'source_page': base_url,
            'page_title': page_title,
            'page_description': page_description,
            'page_keywords': page_keywords,
            'page_content': page_content,  # Full text for blog post searchability
            'figure_caption': context.get('figure_caption'),
            'nearby_heading': context.get('nearby_heading'),
            'link_text': context.get('link_text'),
            'link_title': context.get('link_title'),  # <a title="..."> attribute
            'detail_page_url': resolved_detail_url,  # URL to fetch for canonical image + richer metadata
            # Accumulated metadata for full-text search
            'searchable_text': metadata.to_searchable_text(),
            'metadata': metadata.to_dict(),  # Full breakdown for debugging/analysis
        })

    # Extract from <img> tags
    if collect_images:
        for img in soup.find_all('img'):
            src = img.get('src')
            alt = img.get('alt', '')
            title = img.get('title', '')
            width = img.get('width')
            height = img.get('height')

            if src:
                add_media(src, 'image', alt, title,
                         int(width) if width and width.isdigit() else None,
                         int(height) if height and height.isdigit() else None,
                         element=img)

            # Handle srcset
            srcset = img.get('srcset', '')
            for src_entry in srcset.split(','):
                src_entry = src_entry.strip()
                if src_entry:
                    src_url = src_entry.split()[0]  # Get URL part, ignore size descriptor
                    add_media(src_url, 'image', alt, title, element=img)

            # Lazy loading attributes
            for attr in ['data-src', 'data-lazy-src', 'data-original', 'data-srcset']:
                lazy_src = img.get(attr)
                if lazy_src:
                    if attr.endswith('srcset'):
                        for src_entry in lazy_src.split(','):
                            src_entry = src_entry.strip()
                            if src_entry:
                                add_media(src_entry.split()[0], 'image', alt, title, element=img)
                    else:
                        add_media(lazy_src, 'image', alt, title, element=img)

    # Extract from <picture><source> tags
    if collect_images:
        for picture in soup.find_all('picture'):
            for source in picture.find_all('source'):
                srcset = source.get('srcset', '')
                for src_entry in srcset.split(','):
                    src_entry = src_entry.strip()
                    if src_entry:
                        add_media(src_entry.split()[0], 'image')

    # Extract from <video> tags
    if collect_videos:
        for video in soup.find_all('video'):
            src = video.get('src')
            poster = video.get('poster')
            title = video.get('title', '')

            if src:
                add_media(src, 'video', title=title, element=video)
            if poster and collect_images:
                add_media(poster, 'image', title=f"Video poster: {title}", element=video)

            # Sources within video
            for source in video.find_all('source'):
                src = source.get('src')
                if src:
                    add_media(src, 'video', title=title)

    # Extract from <audio> tags
    if collect_audio:
        for audio in soup.find_all('audio'):
            src = audio.get('src')
            title = audio.get('title', '')

            if src:
                add_media(src, 'audio', title=title)

            for source in audio.find_all('source'):
                src = source.get('src')
                if src:
                    add_media(src, 'audio', title=title)

    # Extract from <a href="..."> pointing to media/code/font/style files
    for a in soup.find_all('a', href=True):
        href = a['href']
        media_type = get_media_type_from_extension(href)
        if media_type:
            if (media_type == 'image' and collect_images) or \
               (media_type == 'video' and collect_videos) or \
               (media_type == 'audio' and collect_audio) or \
               media_type in ('code', 'font', 'style'):  # Always collect code/font/style
                add_media(href, media_type, alt_text=a.get_text(strip=True)[:100])

    # Extract from <link> tags for stylesheets and fonts
    for link in soup.find_all('link', href=True):
        href = link.get('href')
        rel = link.get('rel', [])
        as_attr = link.get('as', '')

        if 'stylesheet' in rel:
            add_media(href, 'style', alt_text='stylesheet')
        elif 'preload' in rel and as_attr == 'font':
            add_media(href, 'font', alt_text='preload font')
        elif 'preload' in rel and as_attr == 'style':
            add_media(href, 'style', alt_text='preload style')
        else:
            # Check by extension
            media_type = get_media_type_from_extension(href)
            if media_type in ('font', 'style'):
                add_media(href, media_type, alt_text=f'link {media_type}')

    # Extract from <script src="..."> for JavaScript files
    for script in soup.find_all('script', src=True):
        src = script.get('src')
        if src:
            add_media(src, 'code', alt_text='script')

    # Extract font URLs from @font-face in <style> blocks
    font_face_pattern = re.compile(r'@font-face\s*\{[^}]*url\(["\']?([^"\')\s]+)["\']?\)', re.IGNORECASE | re.DOTALL)
    for style_tag in soup.find_all('style'):
        if style_tag.string:
            for match in font_face_pattern.findall(style_tag.string):
                add_media(match, 'font', alt_text='font-face')

    # Extract from CSS background-image: url(...)
    if collect_images:
        bg_pattern = re.compile(r'background(?:-image)?\s*:\s*url\(["\']?([^"\')\s]+)["\']?\)', re.IGNORECASE)

        # Inline styles
        for elem in soup.find_all(style=True):
            style = elem.get('style', '')
            for match in bg_pattern.findall(style):
                add_media(match, 'image')

        # <style> blocks
        for style_tag in soup.find_all('style'):
            if style_tag.string:
                for match in bg_pattern.findall(style_tag.string):
                    add_media(match, 'image')

    # Extract from Open Graph and Twitter meta tags
    # Also look for og:title and og:image:alt for metadata
    og_title = None
    og_image_alt = None
    media_filename = None
    media_url = None
    media_type_meta = None

    for meta in soup.find_all('meta'):
        prop = meta.get('property', '') or meta.get('name', '')
        content = meta.get('content', '')
        prop_lower = prop.lower()

        # Extract metadata for richer media info
        if prop_lower == 'og:title':
            og_title = content
        elif prop_lower == 'og:image:alt':
            og_image_alt = content
        elif prop_lower == 'media:filename':
            media_filename = content
        elif prop_lower == 'media:url':
            media_url = content
        elif prop_lower == 'media:type':
            media_type_meta = content

    # Process OG/Twitter image tags with enriched metadata
    if collect_images:
        for meta in soup.find_all('meta'):
            prop = meta.get('property', '') or meta.get('name', '')
            content = meta.get('content', '')

            if content and prop.lower() in ('og:image', 'og:image:url', 'twitter:image', 'twitter:image:src'):
                title = og_title or media_filename or 'Open Graph/Twitter image'
                alt = og_image_alt or og_title or media_filename
                add_media(content, 'image', alt_text=alt, title=title)

    if collect_videos:
        for meta in soup.find_all('meta'):
            prop = meta.get('property', '') or meta.get('name', '')
            content = meta.get('content', '')

            if content and prop.lower() in ('og:video', 'og:video:url', 'twitter:player'):
                title = og_title or media_filename or 'Open Graph/Twitter video'
                add_media(content, 'video', title=title)

    # Also check custom media:url meta tag (PyraFiles specific)
    if media_url:
        detected_type = media_type_meta or 'image'  # Default to image
        if detected_type in ('image', 'video', 'audio'):
            should_add = (
                (detected_type == 'image' and collect_images) or
                (detected_type == 'video' and collect_videos) or
                (detected_type == 'audio' and collect_audio)
            )
            if should_add:
                title = og_title or media_filename
                add_media(media_url, detected_type, alt_text=og_title, title=title)

    logger.debug(f"Extracted {len(media_items)} media items from {base_url}")
    return media_items


# ============================================================================
# Hydra Mode: Feed/Sitemap Discovery and Parsing
# ============================================================================

@dataclass
class FeedItem:
    """Represents an item discovered from RSS/Atom/Sitemap."""
    url: str
    title: Optional[str] = None
    published: Optional[str] = None  # ISO date string
    updated: Optional[str] = None
    priority: float = 0.5  # Sitemap priority (0.0-1.0)


def discover_feeds(html: str, base_url: str) -> List[str]:
    """
    Discover RSS/Atom feed and sitemap URLs from HTML.

    Looks for:
    - <link rel="alternate" type="application/rss+xml">
    - <link rel="alternate" type="application/atom+xml">
    - <link rel="sitemap" href="...">
    - Common sitemap locations: /sitemap.xml, /sitemap_index.xml
    """
    feeds = []
    soup = BeautifulSoup(html, 'html.parser')

    # RSS/Atom feeds from link tags
    for link in soup.find_all('link', rel='alternate'):
        link_type = link.get('type', '')
        href = link.get('href', '')
        if href and ('rss' in link_type or 'atom' in link_type):
            feeds.append(urljoin(base_url, href))

    # Sitemap from link tag
    for link in soup.find_all('link', rel='sitemap'):
        href = link.get('href', '')
        if href:
            feeds.append(urljoin(base_url, href))

    # Common sitemap locations
    parsed = Uri(base_url)
    base = f"{parsed.scheme}://{parsed.hostname}"
    if parsed.port and parsed.port not in (80, 443):
        base += f":{parsed.port}"

    common_sitemaps = [
        '/sitemap.xml',
        '/sitemap_index.xml',
        '/sitemap-index.xml',
        '/sitemaps.xml',
    ]
    for path in common_sitemaps:
        sitemap_url = base + path
        if sitemap_url not in feeds:
            feeds.append(sitemap_url)

    return feeds


def parse_rss(xml_content: str, base_url: str) -> List[FeedItem]:
    """Parse RSS feed and extract items."""
    items = []
    try:
        soup = BeautifulSoup(xml_content, 'xml')
        for item in soup.find_all('item'):
            link = item.find('link')
            title = item.find('title')
            pub_date = item.find('pubDate')

            if link:
                url = link.get_text().strip() if link.get_text() else link.get('href', '')
                if url:
                    items.append(FeedItem(
                        url=urljoin(base_url, url),
                        title=title.get_text().strip() if title else None,
                        published=pub_date.get_text().strip() if pub_date else None,
                    ))
    except Exception as e:
        logger.debug(f"RSS parse error: {e}")
    return items


def parse_atom(xml_content: str, base_url: str) -> List[FeedItem]:
    """Parse Atom feed and extract entries."""
    items = []
    try:
        soup = BeautifulSoup(xml_content, 'xml')
        for entry in soup.find_all('entry'):
            # Atom uses <link href="..."> or <link>url</link>
            link = entry.find('link')
            title = entry.find('title')
            published = entry.find('published')
            updated = entry.find('updated')

            url = None
            if link:
                url = link.get('href') or link.get_text().strip()

            if url:
                items.append(FeedItem(
                    url=urljoin(base_url, url),
                    title=title.get_text().strip() if title else None,
                    published=published.get_text().strip() if published else None,
                    updated=updated.get_text().strip() if updated else None,
                ))
    except Exception as e:
        logger.debug(f"Atom parse error: {e}")
    return items


def parse_sitemap(xml_content: str, base_url: str) -> Tuple[List[FeedItem], List[str]]:
    """
    Parse sitemap and extract URLs.

    Returns:
        Tuple of (items, sub_sitemaps) - items are page URLs, sub_sitemaps are nested sitemap URLs
    """
    items = []
    sub_sitemaps = []
    try:
        soup = BeautifulSoup(xml_content, 'xml')

        # Check for sitemap index (contains other sitemaps)
        for sitemap in soup.find_all('sitemap'):
            loc = sitemap.find('loc')
            if loc:
                sub_sitemaps.append(loc.get_text().strip())

        # Regular sitemap URLs
        for url_tag in soup.find_all('url'):
            loc = url_tag.find('loc')
            lastmod = url_tag.find('lastmod')
            priority = url_tag.find('priority')

            if loc:
                url = loc.get_text().strip()
                items.append(FeedItem(
                    url=urljoin(base_url, url),
                    updated=lastmod.get_text().strip() if lastmod else None,
                    priority=float(priority.get_text().strip()) if priority else 0.5,
                ))
    except Exception as e:
        logger.debug(f"Sitemap parse error: {e}")
    return items, sub_sitemaps


def parse_feed(content: str, url: str) -> Tuple[List[FeedItem], List[str]]:
    """
    Auto-detect feed type and parse.

    Returns:
        Tuple of (items, sub_feeds) - sub_feeds for sitemap indexes
    """
    content_lower = content[:1000].lower()

    if '<urlset' in content_lower or '<sitemapindex' in content_lower:
        return parse_sitemap(content, url)
    elif '<feed' in content_lower:
        return parse_atom(content, url), []
    elif '<rss' in content_lower or '<channel' in content_lower:
        return parse_rss(content, url), []
    else:
        logger.debug(f"Unknown feed format: {url}")
        return [], []


class AsyncWebFetcher:
    """
    Async web fetcher that respects robots.txt and crawl delays.
    Designed for use with Discord bot's async event loop.
    """

    def __init__(
        self,
        user_agent: str = "uncloseai.com/1.42 (ethical web crawler; +https://uncloseai.com)",
        default_crawl_delay: float = DEFAULT_CRAWL_DELAY,
        fast_mode: bool = False
    ):
        self.user_agent = user_agent
        self.default_crawl_delay = default_crawl_delay
        self.fast_mode = fast_mode

        # Timeouts: 5s in fast mode, 60s normally
        self.media_timeout = 5 if fast_mode else 60
        self.page_timeout = 5 if fast_mode else 15

        # Parallel workers for page fetching: 3x CPU in fast mode, 1 otherwise
        import multiprocessing
        self.max_page_workers = multiprocessing.cpu_count() * 3 if fast_mode else 1

        # Caches for robots.txt and crawl delays per domain
        self.robot_parsers: Dict[str, Optional[RobotFileParser]] = {}
        self.domain_crawl_delays: Dict[str, float] = {}
        self.domain_last_fetched: Dict[str, float] = {}

        # Page cache: {url: (html, links, timestamp)}
        self.page_cache: Dict[str, Tuple[str, List, float]] = {}

        # Domain skip list: domains with too many consecutive timeouts
        self.skip_domains: Set[str] = set()
        self.domain_timeout_counts: Dict[str, int] = {}
        self.MAX_CONSECUTIVE_TIMEOUTS = 5

        logger.info(f"AsyncWebFetcher initialized with user-agent: {self.user_agent}")

    def _get_domain(self, url: str) -> str:
        """Extract domain from URL"""
        parsed = Uri(url)
        return parsed.hostname

    async def _fetch_robots_txt(self, domain: str) -> Optional[RobotFileParser]:
        """
        Fetch and parse robots.txt for a domain.
        Returns RobotFileParser or None if not available.
        """
        if domain in self.robot_parsers:
            logger.debug(f"Using cached robots.txt for {domain}")
            return self.robot_parsers[domain]

        logger.info(f"Fetching robots.txt for {domain}")
        robots_url = f"https://{domain}/robots.txt"
        parser = RobotFileParser()
        parser.set_url(robots_url)

        try:
            async with aiohttp.ClientSession() as session:
                async with session.get(
                    robots_url,
                    headers={"User-Agent": self.user_agent},
                    timeout=aiohttp.ClientTimeout(total=5)
                ) as response:
                    if response.status == 200:
                        text = await response.text()
                        parser.parse(text.splitlines())
                        self.robot_parsers[domain] = parser

                        # Extract crawl delay (ignored in fast mode)
                        if self.fast_mode:
                            self.domain_crawl_delays[domain] = 0.0
                        else:
                            delay = parser.crawl_delay(self.user_agent)
                            self.domain_crawl_delays[domain] = (
                                delay if delay is not None else self.default_crawl_delay
                            )

                        logger.info(f"Parsed robots.txt for {domain}, crawl delay: {self.domain_crawl_delays[domain]}s")
                        return parser
                    else:
                        logger.info(f"No robots.txt for {domain} (status {response.status})")
                        self.robot_parsers[domain] = None
                        self.domain_crawl_delays[domain] = self.default_crawl_delay
                        return None
        except Exception as e:
            logger.warning(f"Could not fetch robots.txt for {domain}: {e}")
            self.robot_parsers[domain] = None
            self.domain_crawl_delays[domain] = self.default_crawl_delay
            return None

    async def _can_fetch(self, url: str) -> bool:
        """Check if URL can be fetched according to robots.txt"""
        domain = self._get_domain(url)
        parser = await self._fetch_robots_txt(domain)

        if parser is None:
            # No robots.txt means we can fetch
            return True

        can_fetch = parser.can_fetch(self.user_agent, url)
        if not can_fetch:
            logger.warning(f"Blocked by robots.txt: {url}")

        return can_fetch

    async def _enforce_crawl_delay(self, domain: str):
        """Enforce crawl delay for a domain.

        fast_mode's contract is "ignore crawl-delay" — so short-circuit
        here, NOT just on the robots-200 path. The per-domain delay is
        only zeroed in _fetch_robots_txt when robots.txt returns 200; on
        a 404/error it falls back to default_crawl_delay (2s), which
        would otherwise make every concurrent fetch wave sleep ~2s even
        under fast_mode (the dominant cost on a robots-less site).
        """
        if self.fast_mode:
            return
        delay = self.domain_crawl_delays.get(domain, self.default_crawl_delay)
        last_fetched = self.domain_last_fetched.get(domain, 0)
        elapsed = time.time() - last_fetched

        if elapsed < delay:
            sleep_time = delay - elapsed
            logger.info(f"Enforcing crawl delay for {domain}: sleeping {sleep_time:.2f}s")
            await asyncio.sleep(sleep_time)

        self.domain_last_fetched[domain] = time.time()

    async def check_media_url(
        self,
        url: str,
        session: Optional[aiohttp.ClientSession] = None
    ) -> Optional[Dict[str, Any]]:
        """
        Check if a URL points to media content using HEAD request.

        Useful for extensionless URLs where we can't determine media type from path.

        Args:
            url: URL to check
            session: Optional aiohttp session to reuse

        Returns:
            Dict with 'media_type', 'mime_type', 'content_length' if media, None otherwise
        """
        try:
            close_session = session is None
            if session is None:
                session = aiohttp.ClientSession()

            try:
                async with session.head(
                    url,
                    headers={"User-Agent": self.user_agent},
                    timeout=aiohttp.ClientTimeout(total=10),
                    allow_redirects=True
                ) as response:
                    if response.status != 200:
                        return None

                    content_type = response.headers.get('Content-Type', '').lower()
                    mime_type = content_type.split(';')[0].strip()
                    media_type = get_media_type_from_mime(mime_type)

                    if media_type:
                        content_length = response.headers.get('Content-Length')
                        return {
                            'media_type': media_type,
                            'mime_type': mime_type,
                            'content_length': int(content_length) if content_length else None,
                            'url': str(response.url)  # Final URL after redirects
                        }
                    return None
            finally:
                if close_session:
                    await session.close()

        except Exception as e:
            logger.debug(f"HEAD check failed for {url}: {e}")
            return None

    async def resolve_canonical_image(
        self,
        detail_page_url: str,
        thumbnail_url: str,
        embedding_title: Optional[str] = None,
        session: Optional[aiohttp.ClientSession] = None
    ) -> Optional[Dict[str, Any]]:
        """
        Fetch a detail page and extract the canonical (full-res) image URL.

        Universal Algorithm - no hardcoded strings, purely structural detection:

        1. REST pattern: If an <img> src matches detail_page_url minus last path segment
        2. Wrapped links: <a> tags wrapping <img> where href has query params (versioned)
        3. Download links: <a download> pointing to media
        4. Path similarity: <img> src sharing path structure with detail page
        5. og:image fallback

        Title: Prefers embedding_title (from listing page where image was found).
               Falls back to detail page metadata only if embedding_title not provided.

        Args:
            detail_page_url: URL of the detail page to fetch
            thumbnail_url: Original thumbnail URL (to avoid returning same URL)
            embedding_title: Title from the page that linked here (preferred)
            session: Optional aiohttp session to reuse

        Returns:
            Dict with 'canonical_url', 'title', 'description', 'og_image' or None
        """
        try:
            close_session = session is None
            if session is None:
                session = aiohttp.ClientSession()

            try:
                # Check robots.txt
                if not await self._can_fetch(detail_page_url):
                    return None

                # Enforce crawl delay
                domain = self._get_domain(detail_page_url)
                await self._enforce_crawl_delay(domain)

                async with session.get(
                    detail_page_url,
                    headers={"User-Agent": self.user_agent},
                    timeout=aiohttp.ClientTimeout(total=15),
                    allow_redirects=True
                ) as response:
                    if response.status != 200:
                        return None

                    html = await response.text()

                    # Parse detail page in thread to not block event loop
                    def _parse_detail_page():
                        soup = BeautifulSoup(html, "html.parser")

                        result = {
                            'detail_page_url': detail_page_url,
                            'canonical_url': None,
                            'embedding_title': embedding_title,  # from listing page
                            'detail_title': None,                # from detail page
                            'detail_content': None,              # body text from detail page
                            'title': None,                       # best available
                            'description': None,
                            'og_image': None,
                        }

                        # ========================================
                        # COLLECT BOTH TITLES (skeleton key approach)
                        # ========================================

                        # Always extract detail page title
                        detail_title = None

                        # 1. og:title
                        og_title = soup.find('meta', property='og:title')
                        if og_title:
                            detail_title = og_title.get('content', '').strip()

                        # 2. First image alt text
                        if not detail_title:
                            for img in soup.find_all('img', alt=True):
                                alt = img.get('alt', '').strip()
                                if alt and len(alt) > 2:
                                    detail_title = alt
                                    break

                        # 3. h1 tag
                        if not detail_title:
                            h1_tag = soup.find('h1')
                            if h1_tag:
                                detail_title = h1_tag.get_text(strip=True)

                        # 4. title tag
                        if not detail_title:
                            title_tag = soup.find('title')
                            if title_tag:
                                detail_title = title_tag.get_text(strip=True)

                        result['detail_title'] = detail_title
                        # Primary title: prefer embedding, fallback to detail
                        result['title'] = embedding_title or detail_title

                        # ========================================
                        # DESCRIPTION EXTRACTION
                        # ========================================
                        og_desc = soup.find('meta', property='og:description')
                        if og_desc:
                            result['description'] = og_desc.get('content', '')
                        if not result['description']:
                            meta_desc = soup.find('meta', attrs={'name': 'description'})
                            if meta_desc:
                                result['description'] = meta_desc.get('content', '')

                        # Extract og:image
                        og_image = soup.find('meta', property='og:image')
                        if og_image:
                            result['og_image'] = og_image.get('content', '')

                        # ========================================
                        # DETAIL CONTENT EXTRACTION (body text)
                        # ========================================
                        # Extract body text from detail page for full-text searchability
                        # This enables Pinterest-style galleries where detail pages
                        # have richer descriptions than thumbnails on listing pages
                        body = soup.find('body')
                        if body:
                            # Remove non-content elements
                            for tag in body.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside']):
                                tag.decompose()
                            detail_content = body.get_text(separator=' ', strip=True)[:10000]
                            result['detail_content'] = detail_content

                        # ========================================
                        # CANONICAL URL DETECTION (universal patterns)
                        # ========================================
                        canonical_url = None
                        detail_parsed = Uri(detail_page_url)

                        # Pattern 1: REST-style - check if img src matches URL minus last segment
                        # e.g., /media/ID/details has img pointing to /media/ID
                        path_segments = detail_parsed.path.rstrip('/').split('/')
                        if len(path_segments) > 1:
                            parent_path = '/'.join(path_segments[:-1])
                            parent_url = f"{detail_parsed.scheme}://{detail_parsed.hostname}{parent_path}"
                            for img in soup.find_all('img', src=True):
                                src = img.get('src', '')
                                full_src = urljoin(detail_page_url, src)
                                if full_src == parent_url or full_src.rstrip('/') == parent_url:
                                    if full_src != thumbnail_url:
                                        canonical_url = full_src
                                        break

                        # Pattern 2: Find <a> tags wrapping images with query params (versioned URLs)
                        # The href with ?param=value suggests a cache-busted/versioned canonical
                        if not canonical_url:
                            for a_tag in soup.find_all('a', href=True):
                                href = a_tag.get('href', '')
                                full_href = urljoin(detail_page_url, href)
                                # Must have query params (indicates versioned/timestamped)
                                if '?' not in full_href:
                                    continue
                                # Must wrap or be near an image
                                img_inside = a_tag.find('img')
                                if not img_inside:
                                    continue
                                # Check it's a media URL or same-domain endpoint
                                href_parsed = Uri(full_href)
                                media_type = get_media_type_from_extension(full_href)
                                if media_type == 'image':
                                    canonical_url = full_href
                                    break
                                # Same domain with query params - likely image endpoint
                                if href_parsed.hostname == detail_parsed.hostname or not href_parsed.hostname:
                                    canonical_url = full_href
                                    break

                        # Pattern 3: <a download> attribute - semantic HTML for downloadable content
                        if not canonical_url:
                            for a_tag in soup.find_all('a', href=True, download=True):
                                href = a_tag.get('href', '')
                                if href:
                                    full_href = urljoin(detail_page_url, href)
                                    media_type = get_media_type_from_extension(full_href)
                                    if media_type == 'image':
                                        canonical_url = full_href
                                        break

                        # Pattern 4: First image sharing path structure with detail page
                        if not canonical_url:
                            for img in soup.find_all('img', src=True):
                                src = img.get('src', '')
                                full_src = urljoin(detail_page_url, src)
                                if full_src == thumbnail_url:
                                    continue
                                img_parsed = Uri(full_src)
                                # Same host
                                if img_parsed.hostname == detail_parsed.hostname:
                                    # Count shared path segments
                                    detail_parts = detail_parsed.path.rstrip('/').split('/')
                                    img_parts = img_parsed.path.rstrip('/').split('/')
                                    common = sum(1 for d, i in zip(detail_parts, img_parts) if d == i)
                                    # At least 2 shared segments suggests same resource
                                    if common >= 2:
                                        canonical_url = full_src
                                        break

                        # Pattern 5: og:image fallback (if different from thumbnail)
                        if not canonical_url and result['og_image']:
                            if result['og_image'] != thumbnail_url:
                                canonical_url = result['og_image']

                        result['canonical_url'] = canonical_url
                        return result

                    result = await asyncio.to_thread(_parse_detail_page)
                    logger.info(f"Resolved canonical image from {detail_page_url}: {result['canonical_url']}, title: {result['title'][:50] if result['title'] else 'None'}")
                    return result

            finally:
                if close_session:
                    await session.close()

        except Exception as e:
            logger.debug(f"Failed to resolve canonical image from {detail_page_url}: {e}")
            return None

    def _record_timeout(self, domain: str):
        """Record a timeout for a domain. After MAX_CONSECUTIVE_TIMEOUTS, add to skip list."""
        self.domain_timeout_counts[domain] = self.domain_timeout_counts.get(domain, 0) + 1
        if self.domain_timeout_counts[domain] >= self.MAX_CONSECUTIVE_TIMEOUTS:
            if domain not in self.skip_domains:
                self.skip_domains.add(domain)
                logger.warning(f"Skipping domain {domain} after {self.MAX_CONSECUTIVE_TIMEOUTS} consecutive timeouts")

    def _record_success(self, domain: str):
        """Record a successful fetch, resetting timeout count."""
        self.domain_timeout_counts[domain] = 0

    async def fetch_media(
        self,
        url: str,
        session: Optional[aiohttp.ClientSession] = None,
        max_size: int = 100 * 1024 * 1024  # 100MB default limit
    ) -> Optional[Dict[str, Any]]:
        """
        Fetch media content and compute MD5 hash.

        Args:
            url: URL of media to fetch
            session: Optional aiohttp session to reuse
            max_size: Maximum file size in bytes (default 100MB)

        Returns:
            Dict with 'data', 'md5_hash', 'mime_type', 'size', 'url' or None on failure
        """
        global LAST_FETCH_ERROR

        # Check if domain is in skip list
        domain = self._get_domain(url)
        if domain in self.skip_domains:
            LAST_FETCH_ERROR = {'type': 'skip_domain', 'details': f'Domain {domain} skipped (too many timeouts)', 'url': url}
            logger.debug(f"Skipping {url}: domain {domain} in skip list")
            return None

        # Check robots.txt
        if not await self._can_fetch(url):
            LAST_FETCH_ERROR = {'type': 'robots_txt', 'details': 'Blocked by robots.txt', 'url': url}
            return None

        # Enforce crawl delay
        await self._enforce_crawl_delay(domain)

        try:
            close_session = session is None
            if session is None:
                session = aiohttp.ClientSession()

            try:
                async with session.get(
                    url,
                    headers={"User-Agent": self.user_agent},
                    timeout=aiohttp.ClientTimeout(total=self.media_timeout),
                    allow_redirects=True
                ) as response:
                    if response.status != 200:
                        LAST_FETCH_ERROR = {'type': 'http_error', 'details': f'HTTP {response.status}', 'url': url}
                        return None

                    # Check content length before downloading
                    content_length = response.headers.get('Content-Length')
                    if content_length and int(content_length) > max_size:
                        LAST_FETCH_ERROR = {'type': 'too_large', 'details': f'File too large: {int(content_length)} bytes', 'url': url}
                        logger.warning(f"Skipping {url}: file too large ({int(content_length)} bytes)")
                        return None

                    content_type = response.headers.get('Content-Type', '').lower()
                    mime_type = content_type.split(';')[0].strip()

                    # Download content
                    data = await response.read()

                    if len(data) > max_size:
                        LAST_FETCH_ERROR = {'type': 'too_large', 'details': f'File too large: {len(data)} bytes', 'url': url}
                        return None

                    # Compute MD5 hash
                    md5_hash = hashlib.md5(data).hexdigest()

                    # Determine media type
                    media_type = get_media_type_from_mime(mime_type) or get_media_type_from_extension(url)

                    # Success - reset timeout count
                    self._record_success(domain)

                    logger.info(f"Fetched media {url}: {len(data)} bytes, MD5: {md5_hash}, type: {media_type}")

                    return {
                        'data': data,
                        'md5_hash': md5_hash,
                        'mime_type': mime_type,
                        'media_type': media_type,
                        'size': len(data),
                        'url': str(response.url)  # Final URL after redirects
                    }

            finally:
                if close_session:
                    await session.close()

        except asyncio.TimeoutError:
            self._record_timeout(domain)
            LAST_FETCH_ERROR = {'type': 'timeout', 'details': f'Download timeout ({self.media_timeout}s)', 'url': url}
            logger.error(f"Timeout fetching media {url} ({self.media_timeout}s)")
            return None
        except Exception as e:
            LAST_FETCH_ERROR = {'type': 'unknown', 'details': str(e), 'url': url}
            logger.error(f"Error fetching media {url}: {e}")
            return None

    async def fetch_webpage(
        self,
        url: str,
        extract_links: bool = False,
        extract_anchor_text: bool = False,
        cache_check_callback = None
    ) -> Tuple[Optional[str], List]:
        """
        Fetch a webpage, respecting robots.txt and crawl delays.
        Uses cache to avoid redundant fetches and delays.

        Args:
            url: URL to fetch
            extract_links: Whether to extract links from the page
            extract_anchor_text: Whether to extract anchor text with links
            cache_check_callback: Optional async callback to check external cache (e.g., SQLite3)
                                  Should return (html, links) tuple if cached, None otherwise

        Returns:
            If extract_anchor_text=True: Tuple of (html_content, list of dicts with 'url' and 'anchor_text')
            If extract_anchor_text=False: Tuple of (html_content, list of URLs)
            Returns (None, []) if fetch fails
        """
        global LAST_FETCH_ERROR

        # Validate URL
        if not url.startswith(("http://", "https://")):
            logger.error(f"Invalid URL scheme: {url}")
            return None, []

        # Check external cache first (e.g., SQLite3 via WebCacheManager)
        if cache_check_callback:
            try:
                cached_result = await cache_check_callback(url)
                if cached_result:
                    html, links = cached_result
                    # If we need links but cache has none, re-extract from HTML (in thread to not block)
                    if extract_links and not links and html:
                        logger.info(f"💾 Cache hit but no links cached, re-extracting from HTML: {url}")
                        def _reextract_links():
                            soup = BeautifulSoup(html, "html.parser")
                            result = []
                            for a_tag in soup.find_all("a", href=True):
                                href = a_tag["href"]
                                if href.startswith(("javascript:", "#")):
                                    continue
                                rel = a_tag.get("rel", [])
                                if isinstance(rel, str):
                                    rel = rel.split()
                                if "nofollow" in rel:
                                    continue
                                absolute_url = strip_url_fragment(urljoin(url, href))
                                parsed = Uri(absolute_url)
                                if parsed.scheme in ("http", "https"):
                                    if extract_anchor_text:
                                        anchor_text = a_tag.get_text(strip=True)
                                        result.append({"url": absolute_url, "anchor_text": anchor_text})
                                    else:
                                        result.append(absolute_url)
                            return result
                        links = await asyncio.to_thread(_reextract_links)
                        logger.info(f"Re-extracted {len(links)} links from cached HTML")
                    else:
                        logger.info(f"💾 Using SQLite3 cached page: {url} (skipping robots.txt + crawl delay)")
                    return html, links
            except Exception as e:
                logger.warning(f"Cache check callback failed for {url}: {e}")

        # Check in-memory cache (fallback for when no external cache available)
        if url in self.page_cache:
            html, cached_links, cached_time = self.page_cache[url]
            # If we need links but cache has none, re-extract from HTML (in thread to not block)
            if extract_links and not cached_links and html:
                logger.info(f"💾 In-memory cache hit but no links, re-extracting from HTML: {url}")
                def _reextract_cached_links():
                    soup = BeautifulSoup(html, "html.parser")
                    result = []
                    for a_tag in soup.find_all("a", href=True):
                        href = a_tag["href"]
                        if href.startswith(("javascript:", "#")):
                            continue
                        rel = a_tag.get("rel", [])
                        if isinstance(rel, str):
                            rel = rel.split()
                        if "nofollow" in rel:
                            continue
                        absolute_url = strip_url_fragment(urljoin(url, href))
                        parsed_link = Uri(absolute_url)
                        if parsed_link.scheme in ("http", "https"):
                            if extract_anchor_text:
                                anchor_text = a_tag.get_text(strip=True)
                                result.append({"url": absolute_url, "anchor_text": anchor_text})
                            else:
                                result.append(absolute_url)
                    return result
                cached_links = await asyncio.to_thread(_reextract_cached_links)
                logger.info(f"Re-extracted {len(cached_links)} links from in-memory cached HTML")
            else:
                logger.info(f"💾 Using in-memory cached page: {url} (cached {time.time() - cached_time:.0f}s ago, skipping crawl delay)")
            return html, cached_links

        # Check robots.txt
        if not await self._can_fetch(url):
            LAST_FETCH_ERROR = {'type': 'robots_txt', 'details': 'Blocked by robots.txt', 'url': url}
            return None, []

        # Enforce crawl delay (only for fresh fetches)
        domain = self._get_domain(url)
        await self._enforce_crawl_delay(domain)

        # Fetch the page
        try:
            async with aiohttp.ClientSession() as session:
                # HEAD request first to check content type without downloading
                try:
                    async with session.head(
                        url,
                        headers={"User-Agent": self.user_agent},
                        timeout=aiohttp.ClientTimeout(total=10),
                        allow_redirects=True
                    ) as head_response:
                        head_content_type = head_response.headers.get('Content-Type', '').lower()
                        mime_type = head_content_type.split(';')[0].strip()
                        logger.info(f"HEAD {url} -> {head_response.status}, Content-Type: {mime_type}")

                        # Skip images, video, audio - these need vision mode, not text fetch
                        if mime_type.startswith(('image/', 'video/', 'audio/')):
                            LAST_FETCH_ERROR = {
                                'type': 'binary_content',
                                'details': f'Content is {mime_type} - requires vision mode for images',
                                'url': url,
                                'content_type': mime_type
                            }
                            logger.info(f"Skipping {mime_type} content (not text-extractable): {url}")
                            return None, []
                except Exception as e:
                    # HEAD failed, continue with GET anyway
                    logger.debug(f"HEAD request failed for {url}: {e}, continuing with GET")

                async with session.get(
                    url,
                    headers={"User-Agent": self.user_agent},
                    timeout=aiohttp.ClientTimeout(total=15),
                    allow_redirects=True,
                    max_redirects=3
                ) as response:
                    if response.status != 200:
                        LAST_FETCH_ERROR = {'type': 'http_error', 'details': f'HTTP {response.status}', 'url': url, 'status_code': response.status}
                        logger.warning(f"Failed to fetch {url}: HTTP {response.status}")
                        return None, []

                    # Check content type for binary files (PDF, etc.)
                    content_type = response.headers.get('Content-Type', '').lower()
                    mime_type = content_type.split(';')[0].strip()

                    # Double-check for binary content that HEAD might have missed
                    if mime_type.startswith(('image/', 'video/', 'audio/')):
                        LAST_FETCH_ERROR = {
                            'type': 'binary_content',
                            'details': f'Content is {mime_type} - requires vision mode for images',
                            'url': url,
                            'content_type': mime_type
                        }
                        logger.info(f"Skipping {mime_type} content (not text-extractable): {url}")
                        return None, []

                    is_pdf = 'application/pdf' in content_type or url.lower().endswith('.pdf')

                    if is_pdf:
                        # Handle PDF: read as binary and extract text
                        pdf_bytes = await response.read()
                        logger.info(f"Fetched PDF {url} ({len(pdf_bytes)} bytes)")
                        pdf_text = extract_text_from_pdf(pdf_bytes)
                        if pdf_text:
                            # Wrap extracted text in minimal HTML for consistent processing
                            # Extract filename from URL for title
                            pdf_filename = url.split('/')[-1].split('?')[0] or "PDF Document"
                            html = f"<html><head><title>{pdf_filename}</title></head><body><pre>{pdf_text}</pre></body></html>"
                            logger.info(f"Extracted {len(pdf_text)} chars of text from PDF")
                        else:
                            LAST_FETCH_ERROR = {'type': 'pdf_extraction_failed', 'details': 'Could not extract text from PDF', 'url': url}
                            logger.warning(f"Failed to extract text from PDF: {url}")
                            return None, []
                    else:
                        html = await response.text()
                    logger.info(f"Successfully fetched {url} ({len(html)} bytes)")

                    # Extract links if requested (run in thread to not block event loop)
                    links = []
                    if extract_links:
                        def _extract_links():
                            result = []
                            soup = BeautifulSoup(html, "html.parser")
                            for a_tag in soup.find_all("a", href=True):
                                href = a_tag["href"]
                                if href.startswith(("javascript:", "#")):
                                    continue
                                rel = a_tag.get("rel", [])
                                if isinstance(rel, str):
                                    rel = rel.split()
                                if "nofollow" in rel:
                                    continue
                                absolute_url = strip_url_fragment(urljoin(url, href))
                                parsed = Uri(absolute_url)
                                if parsed.scheme in ("http", "https"):
                                    if extract_anchor_text:
                                        anchor_text = a_tag.get_text(strip=True)
                                        result.append({"url": absolute_url, "anchor_text": anchor_text})
                                    else:
                                        result.append(absolute_url)
                            return result

                        links = await asyncio.to_thread(_extract_links)
                        logger.info(f"Extracted {len(links)} links from {url}")

                    # Cache in memory for future fetches in this session
                    self.page_cache[url] = (html, links, time.time())

                    return html, links

        except asyncio.TimeoutError:
            LAST_FETCH_ERROR = {'type': 'timeout', 'details': 'Connection timeout (>15 seconds)', 'url': url}
            logger.error(f"Timeout fetching {url}")
            return None, []
        except Exception as e:
            error_str = str(e)
            if 'SSL' in error_str or 'certificate' in error_str.lower():
                LAST_FETCH_ERROR = {'type': 'ssl', 'details': str(e), 'url': url}
            elif 'Name or service not known' in error_str or 'DNS' in error_str:
                LAST_FETCH_ERROR = {'type': 'dns', 'details': 'DNS resolution failed', 'url': url}
            else:
                LAST_FETCH_ERROR = {'type': 'unknown', 'details': str(e), 'url': url}
            logger.error(f"Error fetching {url}: {e}")
            return None, []

    def extract_text_content(self, html: str) -> str:
        """
        Extract main text content from HTML using BeautifulSoup.

        Args:
            html: HTML content

        Returns:
            Extracted text
        """
        soup = BeautifulSoup(html, "html.parser")

        # Remove script and style elements
        for script in soup(["script", "style"]):
            script.extract()

        # Block-level elements that should create paragraph breaks
        import re
        block_tags = {'p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
                      'section', 'article', 'header', 'footer', 'main',
                      'li', 'tr', 'blockquote', 'pre', 'br', 'hr'}

        # Insert markers before/after block elements
        for tag in soup.find_all(block_tags):
            tag.insert_before('\n\n')
            tag.insert_after('\n\n')

        # Get text with spaces between inline elements
        text = soup.get_text(separator=' ')

        # Clean up: split on newlines, strip lines, filter empties
        lines = []
        for line in text.split('\n'):
            # Collapse multiple spaces to single space
            cleaned = re.sub(r' +', ' ', line.strip())
            if cleaned:
                lines.append(cleaned)

        # Join with single newlines, then collapse 2+ to paragraph breaks
        text = '\n'.join(lines)
        text = re.sub(r'\n{2,}', '\n\n', text)

        return text

    def extract_title(self, html: str) -> str:
        """Extract title from HTML"""
        soup = BeautifulSoup(html, "html.parser")
        if soup.title and soup.title.string:
            return soup.title.string.strip()
        return "Untitled"

    def _simple_stem(self, word: str) -> str:
        """
        Simple suffix-stripping stemmer for common English suffixes.

        Args:
            word: Word to stem

        Returns:
            Stemmed word
        """
        word = word.lower()
        # Common suffixes in order of longest first
        suffixes = ['ings', 'ing', 'ed', 'es', 's', 'ly', 'er', 'est', 'tion', 'sion', 'ness', 'ment', 'able', 'ible']
        for suffix in suffixes:
            if word.endswith(suffix) and len(word) > len(suffix) + 2:  # Keep at least 3 chars in stem
                return word[:-len(suffix)]
        return word

    def _score_link(self, link_url: str, anchor_text: str, query_keywords: Optional[List[str]] = None, keyword_variations: Optional[List[str]] = None) -> float:
        """
        Score a link based on URL and anchor text keyword matches.
        Uses both exact and stemmed matching with different scoring weights.

        Scoring weights:
        - Exact primary keyword match: 10.0 (URL) / 5.0 (anchor)
        - Stemmed primary keyword match: 7.0 (URL) / 3.5 (anchor) [70% of exact]
        - Exact variation match: 6.0 (URL) / 3.0 (anchor) [60% of exact primary]
        - Stemmed variation match: 4.0 (URL) / 2.0 (anchor) [40% of exact primary]

        Args:
            link_url: URL of the link
            anchor_text: Anchor text of the link
            query_keywords: Primary keywords to match against (full score)
            keyword_variations: Keyword variations/synonyms (reduced score)

        Returns:
            Float score (higher = more relevant to follow)
        """
        if not query_keywords and not keyword_variations:
            return 0.0

        score = 0.0
        url_lower = link_url.lower()
        anchor_lower = anchor_text.lower()

        # Score primary keywords (exact and stemmed matching)
        if query_keywords:
            for keyword in query_keywords:
                keyword_lower = keyword.lower()
                keyword_stem = self._simple_stem(keyword_lower)

                # Exact match for keywords in URL path
                if keyword_lower in url_lower:
                    score += 10.0
                    logger.debug(f"Link URL contains exact primary keyword '{keyword_lower}': {link_url}")
                # Stemmed match for keywords in URL path (70% of exact)
                elif keyword_stem in url_lower and keyword_stem != keyword_lower:
                    score += 7.0
                    logger.debug(f"Link URL contains stemmed primary keyword '{keyword_stem}' (from '{keyword_lower}'): {link_url}")

                # Exact match for keywords in anchor text
                if keyword_lower in anchor_lower:
                    score += 5.0
                    logger.debug(f"Link anchor contains exact primary keyword '{keyword_lower}': {anchor_text}")
                # Stemmed match for keywords in anchor text (70% of exact)
                elif keyword_stem in anchor_lower and keyword_stem != keyword_lower:
                    score += 3.5
                    logger.debug(f"Link anchor contains stemmed primary keyword '{keyword_stem}' (from '{keyword_lower}'): {anchor_text}")

        # Score keyword variations (exact and stemmed matching, reduced weight)
        if keyword_variations:
            for variation in keyword_variations:
                variation_lower = variation.lower()
                variation_stem = self._simple_stem(variation_lower)

                # Exact match for variations in URL path (60% of exact primary)
                if variation_lower in url_lower:
                    score += 6.0
                    logger.debug(f"Link URL contains exact keyword variation '{variation_lower}': {link_url}")
                # Stemmed match for variations in URL path (40% of exact primary)
                elif variation_stem in url_lower and variation_stem != variation_lower:
                    score += 4.0
                    logger.debug(f"Link URL contains stemmed keyword variation '{variation_stem}' (from '{variation_lower}'): {link_url}")

                # Exact match for variations in anchor text (60% of exact primary)
                if variation_lower in anchor_lower:
                    score += 3.0
                    logger.debug(f"Link anchor contains exact keyword variation '{variation_lower}': {anchor_text}")
                # Stemmed match for variations in anchor text (40% of exact primary)
                elif variation_stem in anchor_lower and variation_stem != variation_lower:
                    score += 2.0
                    logger.debug(f"Link anchor contains stemmed keyword variation '{variation_stem}' (from '{variation_lower}'): {anchor_text}")

        return score

    def _score_page(self, page_data: Dict[str, str], query_keywords: Optional[List[str]] = None) -> float:
        """
        Score a page based on content quality metrics and optional query relevance.
        Higher scores indicate more valuable content.

        Scoring factors:
        - Content length (more content = higher score)
        - Title quality (descriptive titles = higher score)
        - URL quality (cleaner URLs = higher score)
        - Content density (unique words = higher score)
        - Query relevance (if keywords provided, matching content scores higher)

        Args:
            page_data: Dict with 'text', 'title', 'url' keys
            query_keywords: Optional list of keywords from user's query for relevance scoring

        Returns:
            Float score (0-100+, can exceed 100 with relevance bonus)
        """
        score = 0.0

        # Factor 1: Content length (0-30 points - reduced from 40 to make room for relevance)
        # Award points for substantial content, max at 5000 chars
        text_len = len(page_data.get('text', ''))
        score += min(30, text_len / 167)  # 5000 chars = 30 points

        # Factor 2: Title quality (0-15 points - reduced from 20)
        # Descriptive titles with meaningful length get higher scores
        title = page_data.get('title', '')
        if title and title != "Untitled":
            title_len = len(title)
            if 10 <= title_len <= 100:  # Sweet spot for titles
                score += 15
            elif 5 <= title_len < 10 or 100 < title_len <= 150:
                score += 8
            else:
                score += 3

        # Factor 3: URL quality (0-15 points - reduced from 20)
        # Prefer shorter, cleaner URLs over long query-string heavy URLs
        url = page_data.get('url', '')
        if url:
            # Penalize query strings and fragments
            if '?' in url:
                score += 3  # Query strings often mean dynamic/less important content
            elif '#' in url:
                score += 8  # Fragments are slightly better
            else:
                score += 15  # Clean URLs are best

            # Bonus for human-readable paths
            path_parts = url.split('/')
            if any(len(part) > 3 and part.replace('-', '').replace('_', '').isalnum() for part in path_parts):
                score += 3  # Readable path segments

        # Factor 4: Content density (0-15 points - reduced from 20)
        # Reward pages with substantial, non-repetitive content
        words = []  # Initialize words for use in Factor 5
        if text_len > 0:
            # Check for unique words (simple heuristic for content quality)
            words = page_data.get('text', '').lower().split()
            unique_words = len(set(words))
            if len(words) > 0:
                uniqueness_ratio = unique_words / len(words)
                score += uniqueness_ratio * 15  # Higher uniqueness = better content

        # Factor 5: Query relevance (0-40 points - NEW!)
        # If user's query keywords are provided, boost pages that contain them
        if query_keywords and len(query_keywords) > 0:
            text_lower = page_data.get('text', '').lower()
            title_lower = title.lower()
            url_lower = url.lower()

            keyword_matches = 0
            keyword_density = 0.0

            for keyword in query_keywords:
                keyword_lower = keyword.lower()

                # Count occurrences in different sections with different weights
                text_count = text_lower.count(keyword_lower)
                title_count = title_lower.count(keyword_lower)
                url_count = url_lower.count(keyword_lower)

                if text_count > 0:
                    keyword_matches += 1
                    keyword_density += text_count

                # Bonus for keywords in title (very relevant)
                if title_count > 0:
                    score += 5 * title_count  # Up to 5 points per title match

                # HIGH BONUS for keywords in URL (strong signal for topic-specific pages)
                # URLs are hand-crafted structure, keywords there mean this page is ABOUT that topic
                if url_count > 0:
                    score += 15 * url_count  # Up to 15 points per URL match (5x higher than before!)
                    logger.debug(f"URL contains keyword '{keyword_lower}' ({url_count}x) - boosting score by {15 * url_count} points")

            # Award points based on what percentage of query keywords matched
            if len(query_keywords) > 0:
                match_ratio = keyword_matches / len(query_keywords)
                score += match_ratio * 20  # Up to 20 points for matching all keywords

            # Award points for keyword density (how often keywords appear)
            if len(words) > 0 and keyword_density > 0:
                density_score = min(10, (keyword_density / len(words)) * 1000)  # Up to 10 points
                score += density_score

        return score

    async def fetch_feeds(
        self,
        base_url: str,
        html: str = None,
        max_sitemaps: int = 10,
        feed_urls: List[str] = None,
    ) -> List[FeedItem]:
        """
        Hydra mode: Fetch RSS/Atom/Sitemap feeds.

        Args:
            base_url: Base URL to discover feeds from
            html: Optional HTML content (if already fetched)
            max_sitemaps: Maximum nested sitemaps to follow
            feed_urls: Optional list of known feed URLs to fetch directly

        Returns:
            List of FeedItems with discovered URLs
        """
        all_items = []
        processed_feeds = set()

        # Use provided feed URLs or discover them
        if feed_urls:
            # Use known feeds directly
            feeds_to_check = list(feed_urls)
            self._last_discovered_feeds = []  # No new discovery
        elif html:
            feeds_to_check = discover_feeds(html, base_url)
            self._last_discovered_feeds = list(feeds_to_check)  # Track discovered
        else:
            # Try common sitemap locations
            parsed = Uri(base_url)
            base = f"{parsed.scheme}://{parsed.hostname}"
            if parsed.port and parsed.port not in (80, 443):
                base += f":{parsed.port}"
            feeds_to_check = [
                base + '/sitemap.xml',
                base + '/sitemap_index.xml',
                base + '/feed/',
                base + '/rss/',
                base + '/atom.xml',
            ]
            self._last_discovered_feeds = []  # Will track which ones actually work

        # Queue for processing (includes nested sitemaps)
        to_process = list(feeds_to_check)
        sitemap_count = 0

        while to_process and sitemap_count < max_sitemaps:
            feed_url = to_process.pop(0)
            if feed_url in processed_feeds:
                continue
            processed_feeds.add(feed_url)

            try:
                # Fetch feed content
                async with self.session.get(
                    feed_url,
                    headers={'User-Agent': self.user_agent},
                    timeout=aiohttp.ClientTimeout(total=30),
                    allow_redirects=True,
                ) as response:
                    if response.status != 200:
                        logger.debug(f"Feed {feed_url} returned {response.status}")
                        continue

                    content = await response.text()

                    # Parse feed
                    items, sub_feeds = parse_feed(content, feed_url)
                    all_items.extend(items)

                    # Add nested sitemaps to queue
                    for sub_feed in sub_feeds:
                        if sub_feed not in processed_feeds:
                            to_process.append(sub_feed)

                    if items:
                        logger.info(f"Hydra: Found {len(items)} URLs in {feed_url}")
                        # Track working feeds for persistence
                        if not feed_urls and feed_url not in self._last_discovered_feeds:
                            self._last_discovered_feeds.append(feed_url)

                    sitemap_count += 1

            except asyncio.TimeoutError:
                logger.debug(f"Timeout fetching feed: {feed_url}")
            except Exception as e:
                logger.debug(f"Error fetching feed {feed_url}: {e}")

        logger.info(f"Hydra mode: Discovered {len(all_items)} URLs from {len(processed_feeds)} feeds")
        return all_items

    async def fetch_with_depth(
        self,
        start_url: str,
        depth: int = 2,
        max_pages: int = 10,
        query_keywords: Optional[List[str]] = None,
        keyword_variations: Optional[List[str]] = None,
        progress_callback = None,
        cache_check_callback = None,
        should_continue_callback = None,  # Deprecated - kept for compatibility
        user_query: Optional[str] = None,  # Deprecated - kept for compatibility
        mode: CrawlMode = CrawlMode.TEXT,
        media_callback = None,  # Callback for discovered media: async fn(media_item: Dict) -> None
        page_callback = None,  # Callback for page HTML: async fn(url: str, html: str) -> None
        uris_total_callback = None,  # Callback for URI total updates: fn(total: int) -> None
        initial_visited: Optional[Set[str]] = None,  # Pre-visited URLs for resume support
        hydra_urls: Optional[List[str]] = None,  # Priority URLs from feeds/sitemaps (hydra mode)
    ) -> List[Dict[str, str]]:
        """
        Intelligent keyword-driven crawl strategy with domain prioritization.

        Supports multiple crawl modes:
        - TEXT (default): Extract text content only
        - IMAGES: Collect images from pages
        - VIDEOS: Collect videos from pages
        - MEDIA: Collect all media (images + videos + audio)
        - ALL: Uber crawl - text + all media

        Crawl strategy:
        1. Always fetch target URI first (the URL user asked about) and extract ALL links
        2. Register links in link_registry with seen_count (boost for repeated links)
        3. Sort links: ALL same-domain before ANY cross-domain
        4. Crawl high-scoring links automatically up to max_pages
        5. Cross-domain links only crawled if score >= 30 (exceptionally high relevance)

        Example: Searching "dropbox encryption" on https://dropbox.com
        - Target page fetched → extracts links (same-domain and cross-domain)
        - Same-domain links with keyword matches crawled first
        - Cross-domain links only crawled if score >= 30
        - Continues until max_pages or runs out of high-scoring links

        Args:
            start_url: Starting URL
            depth: How many levels deep to crawl (0 = just this page, 1 = page + links, etc.)
                   Use -1 for unlimited depth (full domain crawl)
            max_pages: Maximum number of pages to fetch (default 10, use -1 for unlimited)
            query_keywords: Optional list of primary keywords from user's query (full scoring weight)
            keyword_variations: Optional list of keyword variations/synonyms (60% scoring weight)
            progress_callback: Optional callback for progress updates
            cache_check_callback: Optional callback for cache checking (not currently used)
            should_continue_callback: DEPRECATED - no longer used
            user_query: DEPRECATED - no longer used
            mode: CrawlMode - what type of content to extract (default: TEXT)
            media_callback: Optional async callback for discovered media items.
                           Called with dict containing 'url', 'media_type', 'source_page', etc.
                           Use this to process/store media as it's discovered.

        Returns:
            List of dicts with keys: url, title, text, fetched_at, score, depth, links_from_page
            If mode includes media, each page dict also contains 'media' list
        """
        # Handle unlimited depth/pages
        unlimited_depth = (depth == -1)
        unlimited_pages = (max_pages == -1)
        if unlimited_depth:
            depth = 999999  # Effectively unlimited
        if unlimited_pages:
            max_pages = 999999  # Effectively unlimited
        all_pages = []
        visited = set(initial_visited) if initial_visited else set()
        base_domain = self._get_domain(start_url)

        # Log resume info
        if initial_visited:
            logger.info(f"Resuming with {len(initial_visited)} previously visited pages")

        # Track all discovered links: {url: {'anchor_texts': [str], 'seen_count': int, 'total_link_score': float, 'domain': str, 'is_same_domain': bool}}
        link_registry = {}

        # Hydra mode: Inject priority URLs from feeds/sitemaps
        if hydra_urls:
            logger.info(f"Hydra mode: Injecting {len(hydra_urls)} priority URLs from feeds/sitemaps")
            for url in hydra_urls:
                if url not in visited:
                    domain = self._get_domain(url)
                    is_same = (domain == base_domain)
                    link_registry[url] = {
                        'anchor_texts': ['[hydra]'],
                        'seen_count': 100,  # High priority
                        'total_link_score': 100.0,  # High score to ensure they're crawled first
                        'domain': domain,
                        'is_same_domain': is_same,
                        'from_hydra': True,
                    }

        # Log if using query-aware scoring
        if query_keywords or keyword_variations:
            logger.info(f"Using query-aware scoring with keywords: {query_keywords}, variations: {keyword_variations}")

        logger.info(f"Starting intelligent crawl with depth={depth}, max_pages={max_pages}, primary_domain={base_domain}")

        # PHASE 1: Always fetch target URI first
        logger.info(f"Phase 1: Fetching target URI: {start_url}")
        html, links = await self.fetch_webpage(
            start_url,
            extract_links=True,
            extract_anchor_text=bool(query_keywords),  # Extract anchor text only if we have keywords
            cache_check_callback=cache_check_callback  # Pass through SQLite3 cache callback
        )

        if html is None:
            logger.error(f"Failed to fetch target URI {start_url}")
            return []

        # TEMPORARY: If HTML is empty but we got it from cache, we can still extract text/title from it
        # Eventually we should store raw HTML in cache for all pages to enable proper virtual crawling
        if not html:
            logger.warning(f"Got empty HTML for {start_url} (likely from cache without raw_html), cannot extract links for virtual crawl")
            # For now, continue with empty HTML - we'll at least process the target page even if we can't crawl links
            # TODO: Store raw_html in cache for all pages to enable full virtual crawling

        # Detect and skip RSS/XML feeds
        if start_url.endswith(('.xml', '.rss', '.atom', 'feed', 'feeds')) or \
           '/feed' in start_url or '/rss' in start_url or 'atom.xml' in start_url:
            logger.warning(f"Skipping RSS/XML feed: {start_url} (detected feed pattern)")
            return []

        if html.strip().startswith('<?xml') or '<rss' in html[:500] or '<feed' in html[:500]:
            logger.warning(f"Skipping RSS/XML feed: {start_url} (detected XML content)")
            return []

        # Extract target page content (in thread to not block event loop)
        def _extract_content():
            return self.extract_title(html), self.extract_text_content(html)
        title, text = await asyncio.to_thread(_extract_content)

        target_page = {
            "url": start_url,
            "title": title,
            "text": text,
            "html": html,  # Include raw HTML for cache storage (enables virtual crawling)
            "fetched_at": datetime.now(timezone.utc).isoformat(),
            "depth": 0,
            "links_from_page": [l['url'] if isinstance(l, dict) else l for l in links]  # Store for CSV
        }

        # Call page callback to archive raw HTML
        if page_callback and html:
            try:
                await page_callback(start_url, html)
            except Exception as e:
                logger.warning(f"Page callback failed for {start_url}: {e}")

        # Extract media if mode requires it
        if mode in (CrawlMode.IMAGES, CrawlMode.VIDEOS, CrawlMode.MEDIA, CrawlMode.ALL):
            media_items = await asyncio.to_thread(extract_media_from_html, html, start_url, mode)
            target_page["media"] = media_items
            logger.info(f"Extracted {len(media_items)} media items from target page")

            # Call media callbacks concurrently (don't block on each one)
            if media_callback:
                await asyncio.gather(*[media_callback(item) for item in media_items], return_exceptions=True)

        # Score the target page
        target_page["score"] = self._score_page(target_page, query_keywords=query_keywords)
        all_pages.append(target_page)
        visited.add(start_url)

        media_count = len(target_page.get("media", []))
        logger.info(f"Crawled target page (depth 0, score {target_page['score']:.1f}): {len(text)} chars, {len(links)} links, {media_count} media extracted")

        # Update progress callback
        if progress_callback:
            await progress_callback(f"📄 **Fetched page 1**\n{title[:50]}\n<{start_url}>\n~{len(text)//4} tokens (estimated)")

        # PHASE 2: Register ALL links from target page (both same-domain and cross-domain)
        logger.info(f"Phase 2: Registering {len(links)} links from target page")
        same_domain_count = 0
        cross_domain_count = 0

        for link in links:
            link_url, anchor_text = normalize_link(link)

            # Filter: not already visited
            if link_url in visited:
                continue

            # Track domain
            link_domain = self._get_domain(link_url)
            is_same_domain = (link_domain == base_domain)

            if is_same_domain:
                same_domain_count += 1
            else:
                cross_domain_count += 1

            # Register or update link
            if link_url not in link_registry:
                link_score = self._score_link(link_url, anchor_text, query_keywords, keyword_variations) if (query_keywords or keyword_variations) else 0.0
                link_registry[link_url] = {
                    'anchor_texts': [anchor_text],
                    'seen_count': 1,
                    'total_link_score': link_score,
                    'domain': link_domain,
                    'is_same_domain': is_same_domain
                }
            else:
                # Link seen again - increment count and update score
                link_registry[link_url]['anchor_texts'].append(anchor_text)
                link_registry[link_url]['seen_count'] += 1
                if query_keywords or keyword_variations:
                    link_score = self._score_link(link_url, anchor_text, query_keywords, keyword_variations)
                    link_registry[link_url]['total_link_score'] += link_score

        logger.info(f"Registered {len(link_registry)} unique links from target page (same-domain: {same_domain_count}, cross-domain: {cross_domain_count})")
        if uris_total_callback:
            uris_total_callback(len(link_registry) + 1)  # +1 for target page

        # PHASE 3: Decide whether to crawl deeper
        # If depth=0 or we're at max_pages, stop
        if depth == 0 or (max_pages > 0 and len(all_pages) >= max_pages):
            logger.info(f"Stopping crawl (depth={depth}, pages={len(all_pages)})")
            return self._finalize_results(all_pages)

        # If target page has high score (good keyword matches), maybe we're done
        # But if it has low score, we should explore links
        # Threshold: if target page scores < 50, explore links
        if target_page['score'] >= 50 and query_keywords:
            logger.info(f"Target page has high score ({target_page['score']:.1f}), may not need to crawl deeper")
            # Still crawl a few top links if they have strong keyword matches
        else:
            logger.info(f"Target page has low score ({target_page['score']:.1f}), will explore keyword-matching links")

        # PHASE 4: Sort links by combined score (keyword_score + seen_count_boost)
        # STRATEGY: Prioritize same-domain links, only allow cross-domain if exceptionally high score
        # seen_count_boost = seen_count * 2 (each additional sighting adds 2 points)
        scored_links = []
        for link_url, meta in link_registry.items():
            seen_count_boost = (meta['seen_count'] - 1) * 2  # First sighting doesn't get boost
            combined_score = meta['total_link_score'] + seen_count_boost
            scored_links.append({
                'url': link_url,
                'anchor_texts': meta['anchor_texts'],
                'seen_count': meta['seen_count'],
                'link_score': meta['total_link_score'],
                'combined_score': combined_score,
                'domain': meta['domain'],
                'is_same_domain': meta['is_same_domain']
            })

        # Sort by: same_domain first (True sorts before False), then by combined_score
        # This ensures ALL same-domain links are processed before ANY cross-domain links
        scored_links.sort(key=lambda x: (not x['is_same_domain'], -x['combined_score']))

        # Log top scoring links by domain
        same_domain_links = [l for l in scored_links if l['is_same_domain'] and l['combined_score'] > 0][:10]
        cross_domain_links = [l for l in scored_links if not l['is_same_domain'] and l['combined_score'] > 0][:5]

        if same_domain_links:
            logger.info(f"Top {len(same_domain_links)} same-domain scoring links:")
            for i, link in enumerate(same_domain_links, 1):
                logger.info(f"  {i}. [{link['combined_score']:.1f}] {link['url']} (keyword={link['link_score']:.1f}, seen={link['seen_count']}x)")

        if cross_domain_links:
            logger.info(f"Top {len(cross_domain_links)} cross-domain scoring links (will only crawl if score > 30):")
            for i, link in enumerate(cross_domain_links, 1):
                logger.info(f"  {i}. [{link['combined_score']:.1f}] {link['url']} ({link['domain']}, keyword={link['link_score']:.1f})")

        # PHASE 5: Crawl links iteratively for each depth level (1 to depth)
        # Handle unlimited depth (-1) by using large upper bound
        max_depth = depth if depth > 0 else 10000
        for current_depth in range(1, max_depth + 1):
            if max_pages > 0 and len(all_pages) >= max_pages:
                break

            remaining = "unlimited" if max_pages < 0 else str(max_pages - len(all_pages))
            logger.info(f"Phase 5.{current_depth}: Crawling depth-{current_depth} links (max {remaining} more pages)")

            # Re-score and sort all unvisited links
            candidates = []
            for link_url, meta in link_registry.items():
                if link_url in visited:
                    continue
                seen_count_boost = (meta['seen_count'] - 1) * 2
                combined_score = meta['total_link_score'] + seen_count_boost
                candidates.append({
                    'url': link_url,
                    'anchor_texts': meta['anchor_texts'],
                    'seen_count': meta['seen_count'],
                    'link_score': meta['total_link_score'],
                    'combined_score': combined_score,
                    'domain': meta['domain'],
                    'is_same_domain': meta['is_same_domain']
                })

            if not candidates:
                logger.info(f"No more unvisited links at depth {current_depth}")
                break

            # Sort: same_domain first, then by combined_score
            candidates.sort(key=lambda x: (not x['is_same_domain'], -x['combined_score']))

            # Log top candidates for this depth
            top_same = [l for l in candidates if l['is_same_domain'] and l['combined_score'] > 0][:10]
            if top_same:
                logger.info(f"Top {len(top_same)} depth-{current_depth} same-domain candidates:")
                for i, link in enumerate(top_same, 1):
                    logger.info(f"  {i}. [{link['combined_score']:.1f}] {link['url']} (keyword={link['link_score']:.1f}, seen={link['seen_count']}x)")

            # Track pages added at this depth level
            pages_at_this_depth = 0

            # Pre-filter candidates to get valid URLs
            valid_candidates = []
            for link_data in candidates:
                if max_pages > 0 and len(all_pages) + len(valid_candidates) >= max_pages:
                    break
                # Apply score thresholds
                if link_data['is_same_domain']:
                    if query_keywords and link_data['link_score'] <= 0:
                        continue
                else:
                    if link_data['combined_score'] < 30:
                        continue
                link_url = link_data['url']
                if link_url in visited:
                    continue
                valid_candidates.append(link_data)
                visited.add(link_url)  # Mark as visited to prevent duplicates in batch

            if not valid_candidates:
                logger.info(f"No valid candidates at depth {current_depth}")
                break

            # Parallel fetching with semaphore (fast_mode uses multiple workers)
            semaphore = asyncio.Semaphore(self.max_page_workers)
            if self.fast_mode and self.max_page_workers > 1:
                logger.info(f"Parallel fetch: {len(valid_candidates)} pages with {self.max_page_workers} workers")

            async def fetch_one(link_data):
                """Fetch a single page with semaphore control."""
                async with semaphore:
                    link_url = link_data['url']
                    try:
                        html, child_links = await self.fetch_webpage(
                            link_url,
                            extract_links=True,
                            extract_anchor_text=bool(query_keywords),
                            cache_check_callback=cache_check_callback
                        )
                        return {'url': link_url, 'html': html, 'child_links': child_links, 'link_data': link_data, 'error': None}
                    except Exception as e:
                        return {'url': link_url, 'html': None, 'child_links': [], 'link_data': link_data, 'error': str(e)}

            # Fetch pages in parallel, process each as it completes (fractal/streaming)
            fetch_tasks = [asyncio.create_task(fetch_one(c)) for c in valid_candidates]
            logger.info(f"Streaming {len(fetch_tasks)} fetches (processing as each completes)")

            # Process results as they arrive (not waiting for all to complete)
            processed_count = 0
            for coro in asyncio.as_completed(fetch_tasks):
                result = await coro
                if max_pages > 0 and len(all_pages) >= max_pages:
                    logger.info(f"Reached max_pages limit ({max_pages}), stopping")
                    break

                link_url = result['url']
                html = result['html']
                child_links = result['child_links']
                link_data = result['link_data']
                processed_count += 1

                if not html:
                    if result['error']:
                        logger.warning(f"Skipping {link_url} (fetch failed: {result['error']})")
                    else:
                        logger.warning(f"Skipping {link_url} (fetch failed)")
                    continue

                # Skip RSS/XML feeds
                if link_url.endswith(('.xml', '.rss', '.atom', 'feed', 'feeds')) or \
                   '/feed' in link_url or '/rss' in link_url or 'atom.xml' in link_url:
                    logger.warning(f"Skipping RSS/XML feed: {link_url}")
                    continue
                if html.strip().startswith('<?xml') or '<rss' in html[:500] or '<feed' in html[:500]:
                    logger.warning(f"Skipping RSS/XML feed: {link_url}")
                    continue

                # Extract content (in thread to not block event loop)
                def _extract_page_content():
                    return self.extract_title(html), self.extract_text_content(html)
                title, text = await asyncio.to_thread(_extract_page_content)

                page_data = {
                    "url": link_url,
                    "title": title,
                    "text": text,
                    "html": html,
                    "fetched_at": datetime.now(timezone.utc).isoformat(),
                    "depth": current_depth,
                    "links_from_page": [l['url'] if isinstance(l, dict) else l for l in child_links]
                }

                # Call page callback to archive raw HTML
                if page_callback and html:
                    try:
                        await page_callback(link_url, html)
                    except Exception as e:
                        logger.warning(f"Page callback failed for {link_url}: {e}")

                # Extract media if mode requires it
                if mode in (CrawlMode.IMAGES, CrawlMode.VIDEOS, CrawlMode.MEDIA, CrawlMode.ALL):
                    media_items = await asyncio.to_thread(extract_media_from_html, html, link_url, mode)
                    page_data["media"] = media_items

                    # Call media callbacks concurrently (don't block on each one)
                    if media_callback:
                        await asyncio.gather(*[media_callback(item) for item in media_items], return_exceptions=True)

                page_data["score"] = self._score_page(page_data, query_keywords=query_keywords)
                all_pages.append(page_data)
                pages_at_this_depth += 1

                media_count = len(page_data.get("media", []))
                logger.info(f"Crawled {link_url} (depth {current_depth}, score {page_data['score']:.1f}): {len(text)} chars, {len(child_links)} links, {media_count} media")

                if progress_callback:
                    await progress_callback(f"📄 **Fetched page {len(all_pages)}** (depth {current_depth})\n{title[:50]}\n<{link_url}>")

                # Register newly discovered links for next depth level
                for link in child_links:
                    child_url, anchor_text = normalize_link(link)

                    if child_url in visited:
                        continue

                    child_domain = self._get_domain(child_url)
                    is_same_domain = (child_domain == base_domain)

                    if child_url not in link_registry:
                        link_score = self._score_link(child_url, anchor_text, query_keywords, keyword_variations) if (query_keywords or keyword_variations) else 0.0
                        link_registry[child_url] = {
                            'anchor_texts': [anchor_text],
                            'seen_count': 1,
                            'total_link_score': link_score,
                            'domain': child_domain,
                            'is_same_domain': is_same_domain
                        }
                    else:
                        link_registry[child_url]['anchor_texts'].append(anchor_text)
                        link_registry[child_url]['seen_count'] += 1
                        if query_keywords or keyword_variations:
                            link_score = self._score_link(child_url, anchor_text, query_keywords, keyword_variations)
                            link_registry[child_url]['total_link_score'] += link_score

                # Update total after processing child links
                if uris_total_callback:
                    uris_total_callback(len(link_registry) + 1)

            logger.info(f"Depth {current_depth} complete: processed {processed_count}/{len(fetch_tasks)} results, crawled {pages_at_this_depth} pages")

            # Stop if no pages were crawled at this depth (all links exhausted or skipped)
            if pages_at_this_depth == 0:
                logger.info("No pages crawled at this depth - stopping crawl")
                break

        return self._finalize_results(all_pages)

    def _finalize_results(self, all_pages: List[Dict]) -> List[Dict]:
        """Sort and finalize crawl results"""
        # Sort by depth first (main page first), then by score within each depth
        all_pages.sort(key=lambda p: (p.get("depth", 0), -p["score"]))

        logger.info(f"Crawl complete: fetched {len(all_pages)} pages")
        if len(all_pages) > 1:
            logger.info(f"Score range: {all_pages[-1]['score']:.1f} to {all_pages[0]['score']:.1f}")

        return all_pages

    def select_pages_by_token_budget(
        self,
        pages: list,
        token_budget: int,
        reserve_tokens: int = 1000
    ) -> list:
        """
        Select highest-value pages that fit within token budget using greedy knapsack algorithm.

        Args:
            pages: List of page dicts with 'score', 'tokens', 'url', 'title', 'text' keys
            token_budget: Maximum tokens available for page content
            reserve_tokens: Tokens to reserve for output and system prompts

        Returns:
            List of selected pages (sorted by score descending) that fit in budget
        """
        if not pages:
            return []

        # Adjust budget for reserved tokens
        effective_budget = max(0, token_budget - reserve_tokens)

        if effective_budget <= 0:
            logger.warning(f"Token budget too low after reserving {reserve_tokens} tokens")
            return []

        # Calculate value/token ratio for each page (efficiency metric)
        pages_with_ratio = []
        for page in pages:
            tokens = page.get('tokens', len(page.get('text', '')) // 4)  # Fallback estimate
            score = page.get('score', 0)
            if tokens > 0:
                ratio = score / tokens
                pages_with_ratio.append({
                    **page,
                    'tokens': tokens,
                    'value_per_token': ratio
                })

        if not pages_with_ratio:
            return []

        # Greedy algorithm: Sort by value/token ratio (descending)
        # This maximizes value while minimizing token usage
        pages_with_ratio.sort(key=lambda p: p['value_per_token'], reverse=True)

        # Select pages greedily until budget exhausted
        selected = []
        total_tokens = 0

        for page in pages_with_ratio:
            page_tokens = page['tokens']
            if total_tokens + page_tokens <= effective_budget:
                selected.append(page)
                total_tokens += page_tokens
                logger.debug(f"Selected page: {page['title']} (score={page['score']:.1f}, tokens={page_tokens}, ratio={page['value_per_token']:.3f})")
            else:
                logger.debug(f"Skipped page: {page['title']} (would exceed budget: {total_tokens + page_tokens} > {effective_budget})")

        # Sort selected pages by score (highest first) for presentation
        selected.sort(key=lambda p: p['score'], reverse=True)

        logger.info(f"Selected {len(selected)}/{len(pages)} pages using {total_tokens}/{effective_budget} tokens (budget utilization: {total_tokens/effective_budget*100:.1f}%)")

        return selected


if __name__ == "__main__":
    # Test the async web fetcher
    async def test():
        logging.basicConfig(level=logging.INFO)

        fetcher = AsyncWebFetcher()

        # Test single page fetch
        print("Testing single page fetch...")
        results = await fetcher.fetch_with_depth("https://example.com", depth=0)

        if results:
            print(f"\nFetched: {results[0]['title']}")
            print(f"Content length: {len(results[0]['text'])} chars")
            print(f"Preview: {results[0]['text'][:200]}...")
        else:
            print("Failed to fetch")

    asyncio.run(test())