modified: async_web_fetcher.py

modified:   database.py
	modified:   neopig.py

async_web_fetcher.py

@@ -65,6 +65,118 @@ class MediaItem:
     height: Optional[int] = None
     discovered_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
 
+
+class MediaMetadata:
+    """
+    Accumulates metadata from all sources without clobbering.
+
+    "Never clobber, always append" - collects ALL metadata from:
+    - img.alt, img.title
+    - a.title, a.text (link text)
+    - figcaption
+    - nearby headings
+    - page title/h1
+
+    Produces a combined searchable_text for full-text search.
+    """
+
+    def __init__(self):
+        self.titles: List[str] = []
+        self.alt_texts: List[str] = []
+        self.descriptions: List[str] = []
+        self.captions: List[str] = []
+        self.headings: List[str] = []
+        self.link_texts: List[str] = []
+        self.link_titles: List[str] = []
+
+    def _add_unique(self, lst: List[str], value: str, max_len: int = 500) -> None:
+        """Add value to list if non-empty and not duplicate."""
+        if value and value.strip():
+            clean = value.strip()[:max_len]
+            if clean not in lst:
+                lst.append(clean)
+
+    def add_img_alt(self, alt: str) -> None:
+        """Add img alt attribute."""
+        self._add_unique(self.alt_texts, alt)
+
+    def add_img_title(self, title: str) -> None:
+        """Add img title attribute."""
+        self._add_unique(self.titles, title)
+
+    def add_link_title(self, title: str) -> None:
+        """Add <a title="..."> attribute."""
+        self._add_unique(self.link_titles, title)
+
+    def add_link_text(self, text: str) -> None:
+        """Add <a> inner text."""
+        self._add_unique(self.link_texts, text)
+
+    def add_figcaption(self, caption: str) -> None:
+        """Add figcaption text."""
+        self._add_unique(self.captions, caption)
+
+    def add_heading(self, heading: str) -> None:
+        """Add nearby heading (h1-h6)."""
+        self._add_unique(self.headings, heading)
+
+    def add_description(self, desc: str) -> None:
+        """Add description (og:description, meta description, etc.)."""
+        self._add_unique(self.descriptions, desc)
+
+    def add_page_title(self, title: str) -> None:
+        """Add page title."""
+        self._add_unique(self.titles, title)
+
+    def get_best_title(self) -> Optional[str]:
+        """Get best title for display (first non-empty)."""
+        for lst in [self.titles, self.alt_texts, self.link_titles,
+                    self.captions, self.link_texts, self.headings]:
+            if lst:
+                return lst[0]
+        return None
+
+    def get_best_alt(self) -> Optional[str]:
+        """Get best alt text for accessibility."""
+        if self.alt_texts:
+            return self.alt_texts[0]
+        return self.get_best_title()
+
+    def to_searchable_text(self) -> str:
+        """
+        Combine ALL collected metadata into searchable text.
+
+        This enables finding images by ANY associated text:
+        - "find images of cats" matches img alt="cute cat"
+        - "find images from blog post about python" matches page content
+        """
+        all_parts = []
+        # Dedupe while preserving order
+        seen = set()
+        for lst in [self.titles, self.alt_texts, self.descriptions,
+                    self.captions, self.headings, self.link_texts, self.link_titles]:
+            for item in lst:
+                if item and item not in seen:
+                    all_parts.append(item)
+                    seen.add(item)
+        return ' | '.join(all_parts)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Export all collected metadata as dict."""
+        return {
+            'titles': self.titles,
+            'alt_texts': self.alt_texts,
+            'descriptions': self.descriptions,
+            'captions': self.captions,
+            'headings': self.headings,
+            'link_texts': self.link_texts,
+            'link_titles': self.link_titles,
+            'searchable_text': self.to_searchable_text(),
+            'best_title': self.get_best_title(),
+            'best_alt': self.get_best_alt(),
+        }
+
+
 # PDF text extraction
 try:
     from pypdf import PdfReader
@@ -355,6 +467,16 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
         elif name == 'keywords':
             page_keywords = content[:500]
 
+    # Extract page content (body text) for full-text search
+    # This enables blog images to be searchable by post content
+    page_content = ''
+    body = soup.find('body')
+    if body:
+        # Remove script, style, nav, footer elements
+        for tag in body.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside']):
+            tag.decompose()
+        page_content = body.get_text(separator=' ', strip=True)[:10000]  # Limit to 10k chars
+
     def get_context_for_element(element) -> dict:
         """Extract contextual metadata from surrounding HTML elements."""
         context = {
@@ -363,6 +485,7 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
             'figure_caption': None,
             'nearby_heading': None,
             'link_text': None,
+            'link_title': None,  # <a title="..."> attribute
         }
 
         # Check if inside a <figure> with <figcaption>
@@ -372,12 +495,16 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
             if figcaption:
                 context['figure_caption'] = figcaption.get_text(strip=True)[:200]
 
-        # Check if inside an <a> tag with text
+        # Check if inside an <a> tag with text and/or title
         link = element.find_parent('a')
         if link:
             link_text = link.get_text(strip=True)
             if link_text and link_text != element.get('alt', ''):
                 context['link_text'] = link_text[:200]
+            # Also extract title attribute from <a> tag (tooltip text)
+            link_title = link.get('title', '').strip()
+            if link_title:
+                context['link_title'] = link_title[:200]
 
         # Find nearest heading (h1-h6) before this element
         for heading_tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
@@ -394,11 +521,13 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
 
     def infer_title(alt_text: str, title: str, context: dict) -> str:
         """Infer best title from available metadata."""
-        # Priority: explicit title > alt text > figure caption > link text > nearby heading > page h1 > page title
+        # Priority: explicit title > alt text > link title > figure caption > link text > nearby heading > page h1 > page title
         if title and title.strip():
             return title.strip()
         if alt_text and alt_text.strip() and len(alt_text) > 3:
             return alt_text.strip()
+        if context.get('link_title'):  # <a title="..."> attribute
+            return context['link_title']
         if context.get('figure_caption'):
             return context['figure_caption']
         if context.get('link_text'):
@@ -411,7 +540,7 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
             return context['page_title']
         return None
 
-    def add_media(url: str, media_type: str, alt_text: str = None, title: str = None, width: int = None, height: int = None, element=None):
+    def add_media(url: str, media_type: str, alt_text: str = None, title: str = None, width: int = None, height: int = None, element=None, detail_page_url: str = None):
         """Helper to add media item if not already seen."""
         if not url or url in seen_urls:
             return
@@ -428,23 +557,75 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
         # Get context from surrounding elements
         context = get_context_for_element(element) if element else {'page_title': page_title, 'page_h1': page_h1}
 
-        # Infer title if not provided
-        inferred_title = infer_title(alt_text, title, context)
+        # Build metadata accumulator - "never clobber, always append"
+        metadata = MediaMetadata()
+
+        # Add from img element
+        if alt_text:
+            metadata.add_img_alt(alt_text)
+        if title:
+            metadata.add_img_title(title)
+
+        # Add from context
+        if context.get('figure_caption'):
+            metadata.add_figcaption(context['figure_caption'])
+        if context.get('link_text'):
+            metadata.add_link_text(context['link_text'])
+        if context.get('link_title'):
+            metadata.add_link_title(context['link_title'])
+        if context.get('nearby_heading'):
+            metadata.add_heading(context['nearby_heading'])
+        if context.get('page_title'):
+            metadata.add_page_title(context['page_title'])
+        if context.get('page_h1'):
+            metadata.add_heading(context['page_h1'])
+
+        # Add page-level metadata
+        if page_description:
+            metadata.add_description(page_description)
+
+        # Get best values for backward compatibility
+        best_title = metadata.get_best_title()
+        best_alt = metadata.get_best_alt()
+
+        # Check if this image is wrapped in an <a> tag pointing to a detail page
+        # (Pinterest-style galleries where thumbnail links to detail page with canonical image)
+        resolved_detail_url = None
+        if element and not detail_page_url:
+            parent_link = element.find_parent('a', href=True)
+            if parent_link:
+                href = parent_link.get('href', '')
+                # Only consider internal links (not direct image links)
+                if href and not get_media_type_from_extension(href):
+                    resolved_detail_url = urljoin(base_url, href)
+                    # Only track same-domain detail pages
+                    base_domain = urlparse(base_url).netloc
+                    detail_domain = urlparse(resolved_detail_url).netloc
+                    if base_domain != detail_domain:
+                        resolved_detail_url = None
+        elif detail_page_url:
+            resolved_detail_url = urljoin(base_url, detail_page_url)
 
         media_items.append({
             'url': absolute_url,
             'media_type': media_type,
-            'alt_text': alt_text or inferred_title,  # Use inferred as alt if no alt
-            'title': inferred_title,
+            'alt_text': best_alt,
+            'title': best_title,
             'width': width,
             'height': height,
             'source_page': base_url,
             'page_title': page_title,
             'page_description': page_description,
             'page_keywords': page_keywords,
+            'page_content': page_content,  # Full text for blog post searchability
             'figure_caption': context.get('figure_caption'),
             'nearby_heading': context.get('nearby_heading'),
             'link_text': context.get('link_text'),
+            'link_title': context.get('link_title'),  # <a title="..."> attribute
+            'detail_page_url': resolved_detail_url,  # URL to fetch for canonical image + richer metadata
+            # Accumulated metadata for full-text search
+            'searchable_text': metadata.to_searchable_text(),
+            'metadata': metadata.to_dict(),  # Full breakdown for debugging/analysis
         })
 
     # Extract from <img> tags
@@ -767,6 +948,231 @@ class AsyncWebFetcher:
             logger.debug(f"HEAD check failed for {url}: {e}")
             return None
 
+    async def resolve_canonical_image(
+        self,
+        detail_page_url: str,
+        thumbnail_url: str,
+        embedding_title: Optional[str] = None,
+        session: Optional[aiohttp.ClientSession] = None
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Fetch a detail page and extract the canonical (full-res) image URL.
+
+        Universal Algorithm - no hardcoded strings, purely structural detection:
+
+        1. REST pattern: If an <img> src matches detail_page_url minus last path segment
+        2. Wrapped links: <a> tags wrapping <img> where href has query params (versioned)
+        3. Download links: <a download> pointing to media
+        4. Path similarity: <img> src sharing path structure with detail page
+        5. og:image fallback
+
+        Title: Prefers embedding_title (from listing page where image was found).
+               Falls back to detail page metadata only if embedding_title not provided.
+
+        Args:
+            detail_page_url: URL of the detail page to fetch
+            thumbnail_url: Original thumbnail URL (to avoid returning same URL)
+            embedding_title: Title from the page that linked here (preferred)
+            session: Optional aiohttp session to reuse
+
+        Returns:
+            Dict with 'canonical_url', 'title', 'description', 'og_image' or None
+        """
+        try:
+            close_session = session is None
+            if session is None:
+                session = aiohttp.ClientSession()
+
+            try:
+                # Check robots.txt
+                if not await self._can_fetch(detail_page_url):
+                    return None
+
+                # Enforce crawl delay
+                domain = self._get_domain(detail_page_url)
+                await self._enforce_crawl_delay(domain)
+
+                async with session.get(
+                    detail_page_url,
+                    headers={"User-Agent": self.user_agent},
+                    timeout=aiohttp.ClientTimeout(total=15),
+                    allow_redirects=True
+                ) as response:
+                    if response.status != 200:
+                        return None
+
+                    html = await response.text()
+                    soup = BeautifulSoup(html, "html.parser")
+
+                    result = {
+                        'detail_page_url': detail_page_url,
+                        'canonical_url': None,
+                        'embedding_title': embedding_title,  # from listing page
+                        'detail_title': None,                # from detail page
+                        'detail_content': None,              # body text from detail page
+                        'title': None,                       # best available
+                        'description': None,
+                        'og_image': None,
+                    }
+
+                    # ========================================
+                    # COLLECT BOTH TITLES (skeleton key approach)
+                    # ========================================
+
+                    # Always extract detail page title
+                    detail_title = None
+
+                    # 1. og:title
+                    og_title = soup.find('meta', property='og:title')
+                    if og_title:
+                        detail_title = og_title.get('content', '').strip()
+
+                    # 2. First image alt text
+                    if not detail_title:
+                        for img in soup.find_all('img', alt=True):
+                            alt = img.get('alt', '').strip()
+                            if alt and len(alt) > 2:
+                                detail_title = alt
+                                break
+
+                    # 3. h1 tag
+                    if not detail_title:
+                        h1_tag = soup.find('h1')
+                        if h1_tag:
+                            detail_title = h1_tag.get_text(strip=True)
+
+                    # 4. title tag
+                    if not detail_title:
+                        title_tag = soup.find('title')
+                        if title_tag:
+                            detail_title = title_tag.get_text(strip=True)
+
+                    result['detail_title'] = detail_title
+                    # Primary title: prefer embedding, fallback to detail
+                    result['title'] = embedding_title or detail_title
+
+                    # ========================================
+                    # DESCRIPTION EXTRACTION
+                    # ========================================
+                    og_desc = soup.find('meta', property='og:description')
+                    if og_desc:
+                        result['description'] = og_desc.get('content', '')
+                    if not result['description']:
+                        meta_desc = soup.find('meta', attrs={'name': 'description'})
+                        if meta_desc:
+                            result['description'] = meta_desc.get('content', '')
+
+                    # Extract og:image
+                    og_image = soup.find('meta', property='og:image')
+                    if og_image:
+                        result['og_image'] = og_image.get('content', '')
+
+                    # ========================================
+                    # DETAIL CONTENT EXTRACTION (body text)
+                    # ========================================
+                    # Extract body text from detail page for full-text searchability
+                    # This enables Pinterest-style galleries where detail pages
+                    # have richer descriptions than thumbnails on listing pages
+                    body = soup.find('body')
+                    if body:
+                        # Remove non-content elements
+                        for tag in body.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside']):
+                            tag.decompose()
+                        detail_content = body.get_text(separator=' ', strip=True)[:10000]
+                        result['detail_content'] = detail_content
+
+                    # ========================================
+                    # CANONICAL URL DETECTION (universal patterns)
+                    # ========================================
+                    canonical_url = None
+                    detail_parsed = urlparse(detail_page_url)
+
+                    # Pattern 1: REST-style - check if img src matches URL minus last segment
+                    # e.g., /media/ID/details has img pointing to /media/ID
+                    path_segments = detail_parsed.path.rstrip('/').split('/')
+                    if len(path_segments) > 1:
+                        parent_path = '/'.join(path_segments[:-1])
+                        parent_url = f"{detail_parsed.scheme}://{detail_parsed.netloc}{parent_path}"
+                        for img in soup.find_all('img', src=True):
+                            src = img.get('src', '')
+                            full_src = urljoin(detail_page_url, src)
+                            if full_src == parent_url or full_src.rstrip('/') == parent_url:
+                                if full_src != thumbnail_url:
+                                    canonical_url = full_src
+                                    break
+
+                    # Pattern 2: Find <a> tags wrapping images with query params (versioned URLs)
+                    # The href with ?param=value suggests a cache-busted/versioned canonical
+                    if not canonical_url:
+                        for a_tag in soup.find_all('a', href=True):
+                            href = a_tag.get('href', '')
+                            full_href = urljoin(detail_page_url, href)
+                            # Must have query params (indicates versioned/timestamped)
+                            if '?' not in full_href:
+                                continue
+                            # Must wrap or be near an image
+                            img_inside = a_tag.find('img')
+                            if not img_inside:
+                                continue
+                            # Check it's a media URL or same-domain endpoint
+                            href_parsed = urlparse(full_href)
+                            media_type = get_media_type_from_extension(full_href)
+                            if media_type == 'image':
+                                canonical_url = full_href
+                                break
+                            # Same domain with query params - likely image endpoint
+                            if href_parsed.netloc == detail_parsed.netloc or not href_parsed.netloc:
+                                canonical_url = full_href
+                                break
+
+                    # Pattern 3: <a download> attribute - semantic HTML for downloadable content
+                    if not canonical_url:
+                        for a_tag in soup.find_all('a', href=True, download=True):
+                            href = a_tag.get('href', '')
+                            if href:
+                                full_href = urljoin(detail_page_url, href)
+                                media_type = get_media_type_from_extension(full_href)
+                                if media_type == 'image':
+                                    canonical_url = full_href
+                                    break
+
+                    # Pattern 4: First image sharing path structure with detail page
+                    if not canonical_url:
+                        for img in soup.find_all('img', src=True):
+                            src = img.get('src', '')
+                            full_src = urljoin(detail_page_url, src)
+                            if full_src == thumbnail_url:
+                                continue
+                            img_parsed = urlparse(full_src)
+                            # Same host
+                            if img_parsed.netloc == detail_parsed.netloc:
+                                # Count shared path segments
+                                detail_parts = detail_parsed.path.rstrip('/').split('/')
+                                img_parts = img_parsed.path.rstrip('/').split('/')
+                                common = sum(1 for d, i in zip(detail_parts, img_parts) if d == i)
+                                # At least 2 shared segments suggests same resource
+                                if common >= 2:
+                                    canonical_url = full_src
+                                    break
+
+                    # Pattern 5: og:image fallback (if different from thumbnail)
+                    if not canonical_url and result['og_image']:
+                        if result['og_image'] != thumbnail_url:
+                            canonical_url = result['og_image']
+
+                    result['canonical_url'] = canonical_url
+
+                    logger.info(f"Resolved canonical image from {detail_page_url}: {canonical_url}, title: {result['title'][:50] if result['title'] else 'None'}")
+                    return result
+
+            finally:
+                if close_session:
+                    await session.close()
+
+        except Exception as e:
+            logger.debug(f"Failed to resolve canonical image from {detail_page_url}: {e}")
+            return None
+
     async def fetch_media(
         self,
         url: str,

database.py

@@ -73,8 +73,13 @@ class Database:
                     page_title TEXT,
                     page_description TEXT,
                     page_keywords TEXT,
+                    page_content TEXT,
                     alt_text TEXT,
                     link_text TEXT,
+                    detail_page_uri TEXT,
+                    detail_title TEXT,
+                    detail_content TEXT,
+                    searchable_text TEXT,  -- Combined metadata for full-text search
                     crawl_job_id INTEGER,
                     discovered_at TEXT NOT NULL,
                     FOREIGN KEY (md5_hash) REFERENCES media(md5_hash),
@@ -148,10 +153,22 @@ class Database:
         page_title: str = "",
         page_description: str = "",
         page_keywords: str = "",
+        page_content: str = "",
         alt_text: str = "",
         link_text: str = "",
+        detail_page_uri: str = "",
+        detail_title: str = "",
+        detail_content: str = "",
+        searchable_text: str = "",
     ) -> None:
-        """Create a new media record and add source context."""
+        """Create a new media record and add source context.
+
+        Skeleton key approach: stores both embedding context (page_title, page_content from listing)
+        and detail context (detail_title, detail_content from detail page) for maximum searchability.
+
+        For blogs: page_content contains the post text so images are searchable by post content.
+        For galleries: detail_content contains the detail page text for richer metadata.
+        """
         now = datetime.now(timezone.utc).isoformat()
 
         async with aiosqlite.connect(self.db_path) as db:
@@ -169,10 +186,10 @@ class Database:
             await db.execute(
                 """
                 INSERT OR IGNORE INTO media_sources
-                (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, alt_text, link_text, crawl_job_id, discovered_at)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, page_content, alt_text, link_text, detail_page_uri, detail_title, detail_content, searchable_text, crawl_job_id, discovered_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                 """,
-                (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, alt_text, link_text, crawl_job_id, now)
+                (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, page_content, alt_text, link_text, detail_page_uri, detail_title, detail_content, searchable_text, crawl_job_id, now)
             )
 
             await db.commit()
@@ -185,19 +202,31 @@ class Database:
         page_title: str = "",
         page_description: str = "",
         page_keywords: str = "",
+        page_content: str = "",
         alt_text: str = "",
         link_text: str = "",
+        detail_page_uri: str = "",
+        detail_title: str = "",
+        detail_content: str = "",
+        searchable_text: str = "",
         crawl_job_id: int = None
     ) -> None:
-        """Add another source context for an existing media hash."""
+        """Add another source context for an existing media hash.
+
+        Skeleton key approach: stores both embedding context (page_title, page_content from listing)
+        and detail context (detail_title, detail_content from detail page) for maximum searchability.
+
+        For blogs: page_content contains the post text so images are searchable by post content.
+        For galleries: detail_content contains the detail page text for richer metadata.
+        """
         async with aiosqlite.connect(self.db_path) as db:
             await db.execute(
                 """
                 INSERT OR IGNORE INTO media_sources
-                (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, alt_text, link_text, crawl_job_id, discovered_at)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, page_content, alt_text, link_text, detail_page_uri, detail_title, detail_content, searchable_text, crawl_job_id, discovered_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                 """,
-                (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, alt_text, link_text, crawl_job_id, datetime.now(timezone.utc).isoformat())
+                (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, page_content, alt_text, link_text, detail_page_uri, detail_title, detail_content, searchable_text, crawl_job_id, datetime.now(timezone.utc).isoformat())
             )
             await db.commit()
 

neopig.py

@@ -316,19 +316,51 @@ class NeoPig:
         job_id: int,
         keywords: List[str]
     ):
-        """Download and store a media item with page context."""
+        """Download and store a media item with page context.
+
+        Implements the skeleton key approach:
+        - If detail_page_url is set, resolves canonical image URL
+        - Collects both embedding_title (page_title) and detail_title
+        - Prefers canonical URL for download, falls back to original
+        """
         media_uri = item['url']
         page_uri = item.get('source_page', '')
         media_type = item.get('media_type', 'unknown')
+        detail_page_url = item.get('detail_page_url')
 
-        # Extract page context for searchability
+        # Extract page context for searchability (embedding context)
         page_title = item.get('page_title', '')
         page_description = item.get('page_description', '')
         page_keywords = item.get('page_keywords', '')
+        page_content = item.get('page_content', '')  # Blog post text for full-text search
         alt_text = item.get('alt_text', '')
         link_text = item.get('link_text', '')
+        searchable_text = item.get('searchable_text', '')  # Combined metadata from accumulator
+
+        # Skeleton key: detail context (from detail page if Pinterest-style gallery)
+        detail_page_uri = ''
+        detail_title = ''
+        detail_content = ''
 
         try:
+            # Crystal algorithm: resolve canonical image if this looks like a gallery thumbnail
+            if detail_page_url:
+                canonical_result = await self.fetcher.resolve_canonical_image(
+                    detail_page_url=detail_page_url,
+                    thumbnail_url=media_uri,
+                    embedding_title=page_title,  # Pass listing page title
+                )
+                if canonical_result:
+                    detail_page_uri = detail_page_url
+                    detail_title = canonical_result.get('detail_title', '')
+                    detail_content = canonical_result.get('detail_content', '')
+                    # Use canonical URL if found, otherwise keep thumbnail
+                    if canonical_result.get('canonical_url'):
+                        logger.debug(f"Canonical resolution: {media_uri} -> {canonical_result['canonical_url']}")
+                        media_uri = canonical_result['canonical_url']
+                    # Enrich metadata from detail page
+                    if not alt_text and canonical_result.get('description'):
+                        alt_text = canonical_result['description']
             # Check if this exact media+page combo was already crawled
             existing_hash = await self.db.check_media_uri_exists(media_uri, page_uri)
             if existing_hash:
@@ -353,7 +385,7 @@ class NeoPig:
 
             # Check if content already in vault
             if await self.vault.exists(md5_hash):
-                # Content exists, but add this new page context
+                # Content exists, but add this new page context with skeleton key
                 await self.db.add_media_source(
                     md5_hash=md5_hash,
                     media_uri=media_uri,
@@ -361,8 +393,13 @@ class NeoPig:
                     page_title=page_title,
                     page_description=page_description,
                     page_keywords=page_keywords,
+                    page_content=page_content,
                     alt_text=alt_text,
                     link_text=link_text,
+                    detail_page_uri=detail_page_uri,
+                    detail_title=detail_title,
+                    detail_content=detail_content,
+                    searchable_text=searchable_text,
                     crawl_job_id=job_id,
                 )
                 self.stats['duplicates_skipped'] += 1
@@ -376,7 +413,7 @@ class NeoPig:
             # Archive to domain media vault (git-tracked)
             await self._archive_media_to_vault(media_uri, result['data'], page_uri)
 
-            # Record in database with full context
+            # Record in database with full context and skeleton key
             await self.db.create_media_record(
                 md5_hash=md5_hash,
                 media_uri=media_uri,
@@ -388,8 +425,13 @@ class NeoPig:
                 page_title=page_title,
                 page_description=page_description,
                 page_keywords=page_keywords,
+                page_content=page_content,
                 alt_text=alt_text,
                 link_text=link_text,
+                detail_page_uri=detail_page_uri,
+                detail_title=detail_title,
+                detail_content=detail_content,
+                searchable_text=searchable_text,
             )
 
             self.stats['media_downloaded'] += 1