diff --git a/async_web_fetcher.py b/async_web_fetcher.py
index 965169e..5740363 100644
--- a/async_web_fetcher.py
+++ b/async_web_fetcher.py
@@ -65,6 +65,118 @@ class MediaItem:
height: Optional[int] = None
discovered_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+
+class MediaMetadata:
+ """
+ Accumulates metadata from all sources without clobbering.
+
+ "Never clobber, always append" - collects ALL metadata from:
+ - img.alt, img.title
+ - a.title, a.text (link text)
+ - figcaption
+ - nearby headings
+ - page title/h1
+
+ Produces a combined searchable_text for full-text search.
+ """
+
+ def __init__(self):
+ self.titles: List[str] = []
+ self.alt_texts: List[str] = []
+ self.descriptions: List[str] = []
+ self.captions: List[str] = []
+ self.headings: List[str] = []
+ self.link_texts: List[str] = []
+ self.link_titles: List[str] = []
+
+ def _add_unique(self, lst: List[str], value: str, max_len: int = 500) -> None:
+ """Add value to list if non-empty and not duplicate."""
+ if value and value.strip():
+ clean = value.strip()[:max_len]
+ if clean not in lst:
+ lst.append(clean)
+
+ def add_img_alt(self, alt: str) -> None:
+ """Add img alt attribute."""
+ self._add_unique(self.alt_texts, alt)
+
+ def add_img_title(self, title: str) -> None:
+ """Add img title attribute."""
+ self._add_unique(self.titles, title)
+
+ def add_link_title(self, title: str) -> None:
+ """Add attribute."""
+ self._add_unique(self.link_titles, title)
+
+ def add_link_text(self, text: str) -> None:
+ """Add inner text."""
+ self._add_unique(self.link_texts, text)
+
+ def add_figcaption(self, caption: str) -> None:
+ """Add figcaption text."""
+ self._add_unique(self.captions, caption)
+
+ def add_heading(self, heading: str) -> None:
+ """Add nearby heading (h1-h6)."""
+ self._add_unique(self.headings, heading)
+
+ def add_description(self, desc: str) -> None:
+ """Add description (og:description, meta description, etc.)."""
+ self._add_unique(self.descriptions, desc)
+
+ def add_page_title(self, title: str) -> None:
+ """Add page title."""
+ self._add_unique(self.titles, title)
+
+ def get_best_title(self) -> Optional[str]:
+ """Get best title for display (first non-empty)."""
+ for lst in [self.titles, self.alt_texts, self.link_titles,
+ self.captions, self.link_texts, self.headings]:
+ if lst:
+ return lst[0]
+ return None
+
+ def get_best_alt(self) -> Optional[str]:
+ """Get best alt text for accessibility."""
+ if self.alt_texts:
+ return self.alt_texts[0]
+ return self.get_best_title()
+
+ def to_searchable_text(self) -> str:
+ """
+ Combine ALL collected metadata into searchable text.
+
+ This enables finding images by ANY associated text:
+ - "find images of cats" matches img alt="cute cat"
+ - "find images from blog post about python" matches page content
+ """
+ all_parts = []
+ # Dedupe while preserving order
+ seen = set()
+ for lst in [self.titles, self.alt_texts, self.descriptions,
+ self.captions, self.headings, self.link_texts, self.link_titles]:
+ for item in lst:
+ if item and item not in seen:
+ all_parts.append(item)
+ seen.add(item)
+ return ' | '.join(all_parts)
+
+ def to_dict(self) -> Dict[str, Any]:
+ """Export all collected metadata as dict."""
+ return {
+ 'titles': self.titles,
+ 'alt_texts': self.alt_texts,
+ 'descriptions': self.descriptions,
+ 'captions': self.captions,
+ 'headings': self.headings,
+ 'link_texts': self.link_texts,
+ 'link_titles': self.link_titles,
+ 'searchable_text': self.to_searchable_text(),
+ 'best_title': self.get_best_title(),
+ 'best_alt': self.get_best_alt(),
+ }
+
+
# PDF text extraction
try:
from pypdf import PdfReader
@@ -355,6 +467,16 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
elif name == 'keywords':
page_keywords = content[:500]
+ # Extract page content (body text) for full-text search
+ # This enables blog images to be searchable by post content
+ page_content = ''
+ body = soup.find('body')
+ if body:
+ # Remove script, style, nav, footer elements
+ for tag in body.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside']):
+ tag.decompose()
+ page_content = body.get_text(separator=' ', strip=True)[:10000] # Limit to 10k chars
+
def get_context_for_element(element) -> dict:
"""Extract contextual metadata from surrounding HTML elements."""
context = {
@@ -363,6 +485,7 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
'figure_caption': None,
'nearby_heading': None,
'link_text': None,
+ 'link_title': None, # attribute
}
# Check if inside a with
@@ -372,12 +495,16 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
if figcaption:
context['figure_caption'] = figcaption.get_text(strip=True)[:200]
- # Check if inside an tag with text
+ # Check if inside an tag with text and/or title
link = element.find_parent('a')
if link:
link_text = link.get_text(strip=True)
if link_text and link_text != element.get('alt', ''):
context['link_text'] = link_text[:200]
+ # Also extract title attribute from tag (tooltip text)
+ link_title = link.get('title', '').strip()
+ if link_title:
+ context['link_title'] = link_title[:200]
# Find nearest heading (h1-h6) before this element
for heading_tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
@@ -394,11 +521,13 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
def infer_title(alt_text: str, title: str, context: dict) -> str:
"""Infer best title from available metadata."""
- # Priority: explicit title > alt text > figure caption > link text > nearby heading > page h1 > page title
+ # Priority: explicit title > alt text > link title > figure caption > link text > nearby heading > page h1 > page title
if title and title.strip():
return title.strip()
if alt_text and alt_text.strip() and len(alt_text) > 3:
return alt_text.strip()
+ if context.get('link_title'): # attribute
+ return context['link_title']
if context.get('figure_caption'):
return context['figure_caption']
if context.get('link_text'):
@@ -411,7 +540,7 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
return context['page_title']
return None
- def add_media(url: str, media_type: str, alt_text: str = None, title: str = None, width: int = None, height: int = None, element=None):
+ def add_media(url: str, media_type: str, alt_text: str = None, title: str = None, width: int = None, height: int = None, element=None, detail_page_url: str = None):
"""Helper to add media item if not already seen."""
if not url or url in seen_urls:
return
@@ -428,23 +557,75 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
# Get context from surrounding elements
context = get_context_for_element(element) if element else {'page_title': page_title, 'page_h1': page_h1}
- # Infer title if not provided
- inferred_title = infer_title(alt_text, title, context)
+ # Build metadata accumulator - "never clobber, always append"
+ metadata = MediaMetadata()
+
+ # Add from img element
+ if alt_text:
+ metadata.add_img_alt(alt_text)
+ if title:
+ metadata.add_img_title(title)
+
+ # Add from context
+ if context.get('figure_caption'):
+ metadata.add_figcaption(context['figure_caption'])
+ if context.get('link_text'):
+ metadata.add_link_text(context['link_text'])
+ if context.get('link_title'):
+ metadata.add_link_title(context['link_title'])
+ if context.get('nearby_heading'):
+ metadata.add_heading(context['nearby_heading'])
+ if context.get('page_title'):
+ metadata.add_page_title(context['page_title'])
+ if context.get('page_h1'):
+ metadata.add_heading(context['page_h1'])
+
+ # Add page-level metadata
+ if page_description:
+ metadata.add_description(page_description)
+
+ # Get best values for backward compatibility
+ best_title = metadata.get_best_title()
+ best_alt = metadata.get_best_alt()
+
+ # Check if this image is wrapped in an tag pointing to a detail page
+ # (Pinterest-style galleries where thumbnail links to detail page with canonical image)
+ resolved_detail_url = None
+ if element and not detail_page_url:
+ parent_link = element.find_parent('a', href=True)
+ if parent_link:
+ href = parent_link.get('href', '')
+ # Only consider internal links (not direct image links)
+ if href and not get_media_type_from_extension(href):
+ resolved_detail_url = urljoin(base_url, href)
+ # Only track same-domain detail pages
+ base_domain = urlparse(base_url).netloc
+ detail_domain = urlparse(resolved_detail_url).netloc
+ if base_domain != detail_domain:
+ resolved_detail_url = None
+ elif detail_page_url:
+ resolved_detail_url = urljoin(base_url, detail_page_url)
media_items.append({
'url': absolute_url,
'media_type': media_type,
- 'alt_text': alt_text or inferred_title, # Use inferred as alt if no alt
- 'title': inferred_title,
+ 'alt_text': best_alt,
+ 'title': best_title,
'width': width,
'height': height,
'source_page': base_url,
'page_title': page_title,
'page_description': page_description,
'page_keywords': page_keywords,
+ 'page_content': page_content, # Full text for blog post searchability
'figure_caption': context.get('figure_caption'),
'nearby_heading': context.get('nearby_heading'),
'link_text': context.get('link_text'),
+ 'link_title': context.get('link_title'), # attribute
+ 'detail_page_url': resolved_detail_url, # URL to fetch for canonical image + richer metadata
+ # Accumulated metadata for full-text search
+ 'searchable_text': metadata.to_searchable_text(),
+ 'metadata': metadata.to_dict(), # Full breakdown for debugging/analysis
})
# Extract from ![]()
tags
@@ -767,6 +948,231 @@ class AsyncWebFetcher:
logger.debug(f"HEAD check failed for {url}: {e}")
return None
+ async def resolve_canonical_image(
+ self,
+ detail_page_url: str,
+ thumbnail_url: str,
+ embedding_title: Optional[str] = None,
+ session: Optional[aiohttp.ClientSession] = None
+ ) -> Optional[Dict[str, Any]]:
+ """
+ Fetch a detail page and extract the canonical (full-res) image URL.
+
+ Universal Algorithm - no hardcoded strings, purely structural detection:
+
+ 1. REST pattern: If an ![]()
src matches detail_page_url minus last path segment
+ 2. Wrapped links: tags wrapping ![]()
where href has query params (versioned)
+ 3. Download links: pointing to media
+ 4. Path similarity: ![]()
src sharing path structure with detail page
+ 5. og:image fallback
+
+ Title: Prefers embedding_title (from listing page where image was found).
+ Falls back to detail page metadata only if embedding_title not provided.
+
+ Args:
+ detail_page_url: URL of the detail page to fetch
+ thumbnail_url: Original thumbnail URL (to avoid returning same URL)
+ embedding_title: Title from the page that linked here (preferred)
+ session: Optional aiohttp session to reuse
+
+ Returns:
+ Dict with 'canonical_url', 'title', 'description', 'og_image' or None
+ """
+ try:
+ close_session = session is None
+ if session is None:
+ session = aiohttp.ClientSession()
+
+ try:
+ # Check robots.txt
+ if not await self._can_fetch(detail_page_url):
+ return None
+
+ # Enforce crawl delay
+ domain = self._get_domain(detail_page_url)
+ await self._enforce_crawl_delay(domain)
+
+ async with session.get(
+ detail_page_url,
+ headers={"User-Agent": self.user_agent},
+ timeout=aiohttp.ClientTimeout(total=15),
+ allow_redirects=True
+ ) as response:
+ if response.status != 200:
+ return None
+
+ html = await response.text()
+ soup = BeautifulSoup(html, "html.parser")
+
+ result = {
+ 'detail_page_url': detail_page_url,
+ 'canonical_url': None,
+ 'embedding_title': embedding_title, # from listing page
+ 'detail_title': None, # from detail page
+ 'detail_content': None, # body text from detail page
+ 'title': None, # best available
+ 'description': None,
+ 'og_image': None,
+ }
+
+ # ========================================
+ # COLLECT BOTH TITLES (skeleton key approach)
+ # ========================================
+
+ # Always extract detail page title
+ detail_title = None
+
+ # 1. og:title
+ og_title = soup.find('meta', property='og:title')
+ if og_title:
+ detail_title = og_title.get('content', '').strip()
+
+ # 2. First image alt text
+ if not detail_title:
+ for img in soup.find_all('img', alt=True):
+ alt = img.get('alt', '').strip()
+ if alt and len(alt) > 2:
+ detail_title = alt
+ break
+
+ # 3. h1 tag
+ if not detail_title:
+ h1_tag = soup.find('h1')
+ if h1_tag:
+ detail_title = h1_tag.get_text(strip=True)
+
+ # 4. title tag
+ if not detail_title:
+ title_tag = soup.find('title')
+ if title_tag:
+ detail_title = title_tag.get_text(strip=True)
+
+ result['detail_title'] = detail_title
+ # Primary title: prefer embedding, fallback to detail
+ result['title'] = embedding_title or detail_title
+
+ # ========================================
+ # DESCRIPTION EXTRACTION
+ # ========================================
+ og_desc = soup.find('meta', property='og:description')
+ if og_desc:
+ result['description'] = og_desc.get('content', '')
+ if not result['description']:
+ meta_desc = soup.find('meta', attrs={'name': 'description'})
+ if meta_desc:
+ result['description'] = meta_desc.get('content', '')
+
+ # Extract og:image
+ og_image = soup.find('meta', property='og:image')
+ if og_image:
+ result['og_image'] = og_image.get('content', '')
+
+ # ========================================
+ # DETAIL CONTENT EXTRACTION (body text)
+ # ========================================
+ # Extract body text from detail page for full-text searchability
+ # This enables Pinterest-style galleries where detail pages
+ # have richer descriptions than thumbnails on listing pages
+ body = soup.find('body')
+ if body:
+ # Remove non-content elements
+ for tag in body.find_all(['script', 'style', 'nav', 'footer', 'header', 'aside']):
+ tag.decompose()
+ detail_content = body.get_text(separator=' ', strip=True)[:10000]
+ result['detail_content'] = detail_content
+
+ # ========================================
+ # CANONICAL URL DETECTION (universal patterns)
+ # ========================================
+ canonical_url = None
+ detail_parsed = urlparse(detail_page_url)
+
+ # Pattern 1: REST-style - check if img src matches URL minus last segment
+ # e.g., /media/ID/details has img pointing to /media/ID
+ path_segments = detail_parsed.path.rstrip('/').split('/')
+ if len(path_segments) > 1:
+ parent_path = '/'.join(path_segments[:-1])
+ parent_url = f"{detail_parsed.scheme}://{detail_parsed.netloc}{parent_path}"
+ for img in soup.find_all('img', src=True):
+ src = img.get('src', '')
+ full_src = urljoin(detail_page_url, src)
+ if full_src == parent_url or full_src.rstrip('/') == parent_url:
+ if full_src != thumbnail_url:
+ canonical_url = full_src
+ break
+
+ # Pattern 2: Find tags wrapping images with query params (versioned URLs)
+ # The href with ?param=value suggests a cache-busted/versioned canonical
+ if not canonical_url:
+ for a_tag in soup.find_all('a', href=True):
+ href = a_tag.get('href', '')
+ full_href = urljoin(detail_page_url, href)
+ # Must have query params (indicates versioned/timestamped)
+ if '?' not in full_href:
+ continue
+ # Must wrap or be near an image
+ img_inside = a_tag.find('img')
+ if not img_inside:
+ continue
+ # Check it's a media URL or same-domain endpoint
+ href_parsed = urlparse(full_href)
+ media_type = get_media_type_from_extension(full_href)
+ if media_type == 'image':
+ canonical_url = full_href
+ break
+ # Same domain with query params - likely image endpoint
+ if href_parsed.netloc == detail_parsed.netloc or not href_parsed.netloc:
+ canonical_url = full_href
+ break
+
+ # Pattern 3: attribute - semantic HTML for downloadable content
+ if not canonical_url:
+ for a_tag in soup.find_all('a', href=True, download=True):
+ href = a_tag.get('href', '')
+ if href:
+ full_href = urljoin(detail_page_url, href)
+ media_type = get_media_type_from_extension(full_href)
+ if media_type == 'image':
+ canonical_url = full_href
+ break
+
+ # Pattern 4: First image sharing path structure with detail page
+ if not canonical_url:
+ for img in soup.find_all('img', src=True):
+ src = img.get('src', '')
+ full_src = urljoin(detail_page_url, src)
+ if full_src == thumbnail_url:
+ continue
+ img_parsed = urlparse(full_src)
+ # Same host
+ if img_parsed.netloc == detail_parsed.netloc:
+ # Count shared path segments
+ detail_parts = detail_parsed.path.rstrip('/').split('/')
+ img_parts = img_parsed.path.rstrip('/').split('/')
+ common = sum(1 for d, i in zip(detail_parts, img_parts) if d == i)
+ # At least 2 shared segments suggests same resource
+ if common >= 2:
+ canonical_url = full_src
+ break
+
+ # Pattern 5: og:image fallback (if different from thumbnail)
+ if not canonical_url and result['og_image']:
+ if result['og_image'] != thumbnail_url:
+ canonical_url = result['og_image']
+
+ result['canonical_url'] = canonical_url
+
+ logger.info(f"Resolved canonical image from {detail_page_url}: {canonical_url}, title: {result['title'][:50] if result['title'] else 'None'}")
+ return result
+
+ finally:
+ if close_session:
+ await session.close()
+
+ except Exception as e:
+ logger.debug(f"Failed to resolve canonical image from {detail_page_url}: {e}")
+ return None
+
async def fetch_media(
self,
url: str,
diff --git a/database.py b/database.py
index b79d8ba..c912391 100644
--- a/database.py
+++ b/database.py
@@ -73,8 +73,13 @@ class Database:
page_title TEXT,
page_description TEXT,
page_keywords TEXT,
+ page_content TEXT,
alt_text TEXT,
link_text TEXT,
+ detail_page_uri TEXT,
+ detail_title TEXT,
+ detail_content TEXT,
+ searchable_text TEXT, -- Combined metadata for full-text search
crawl_job_id INTEGER,
discovered_at TEXT NOT NULL,
FOREIGN KEY (md5_hash) REFERENCES media(md5_hash),
@@ -148,10 +153,22 @@ class Database:
page_title: str = "",
page_description: str = "",
page_keywords: str = "",
+ page_content: str = "",
alt_text: str = "",
link_text: str = "",
+ detail_page_uri: str = "",
+ detail_title: str = "",
+ detail_content: str = "",
+ searchable_text: str = "",
) -> None:
- """Create a new media record and add source context."""
+ """Create a new media record and add source context.
+
+ Skeleton key approach: stores both embedding context (page_title, page_content from listing)
+ and detail context (detail_title, detail_content from detail page) for maximum searchability.
+
+ For blogs: page_content contains the post text so images are searchable by post content.
+ For galleries: detail_content contains the detail page text for richer metadata.
+ """
now = datetime.now(timezone.utc).isoformat()
async with aiosqlite.connect(self.db_path) as db:
@@ -169,10 +186,10 @@ class Database:
await db.execute(
"""
INSERT OR IGNORE INTO media_sources
- (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, alt_text, link_text, crawl_job_id, discovered_at)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, page_content, alt_text, link_text, detail_page_uri, detail_title, detail_content, searchable_text, crawl_job_id, discovered_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
- (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, alt_text, link_text, crawl_job_id, now)
+ (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, page_content, alt_text, link_text, detail_page_uri, detail_title, detail_content, searchable_text, crawl_job_id, now)
)
await db.commit()
@@ -185,19 +202,31 @@ class Database:
page_title: str = "",
page_description: str = "",
page_keywords: str = "",
+ page_content: str = "",
alt_text: str = "",
link_text: str = "",
+ detail_page_uri: str = "",
+ detail_title: str = "",
+ detail_content: str = "",
+ searchable_text: str = "",
crawl_job_id: int = None
) -> None:
- """Add another source context for an existing media hash."""
+ """Add another source context for an existing media hash.
+
+ Skeleton key approach: stores both embedding context (page_title, page_content from listing)
+ and detail context (detail_title, detail_content from detail page) for maximum searchability.
+
+ For blogs: page_content contains the post text so images are searchable by post content.
+ For galleries: detail_content contains the detail page text for richer metadata.
+ """
async with aiosqlite.connect(self.db_path) as db:
await db.execute(
"""
INSERT OR IGNORE INTO media_sources
- (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, alt_text, link_text, crawl_job_id, discovered_at)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, page_content, alt_text, link_text, detail_page_uri, detail_title, detail_content, searchable_text, crawl_job_id, discovered_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
- (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, alt_text, link_text, crawl_job_id, datetime.now(timezone.utc).isoformat())
+ (md5_hash, media_uri, page_uri, page_title, page_description, page_keywords, page_content, alt_text, link_text, detail_page_uri, detail_title, detail_content, searchable_text, crawl_job_id, datetime.now(timezone.utc).isoformat())
)
await db.commit()
diff --git a/neopig.py b/neopig.py
index 2ee16ec..18fadf4 100644
--- a/neopig.py
+++ b/neopig.py
@@ -316,19 +316,51 @@ class NeoPig:
job_id: int,
keywords: List[str]
):
- """Download and store a media item with page context."""
+ """Download and store a media item with page context.
+
+ Implements the skeleton key approach:
+ - If detail_page_url is set, resolves canonical image URL
+ - Collects both embedding_title (page_title) and detail_title
+ - Prefers canonical URL for download, falls back to original
+ """
media_uri = item['url']
page_uri = item.get('source_page', '')
media_type = item.get('media_type', 'unknown')
+ detail_page_url = item.get('detail_page_url')
- # Extract page context for searchability
+ # Extract page context for searchability (embedding context)
page_title = item.get('page_title', '')
page_description = item.get('page_description', '')
page_keywords = item.get('page_keywords', '')
+ page_content = item.get('page_content', '') # Blog post text for full-text search
alt_text = item.get('alt_text', '')
link_text = item.get('link_text', '')
+ searchable_text = item.get('searchable_text', '') # Combined metadata from accumulator
+
+ # Skeleton key: detail context (from detail page if Pinterest-style gallery)
+ detail_page_uri = ''
+ detail_title = ''
+ detail_content = ''
try:
+ # Crystal algorithm: resolve canonical image if this looks like a gallery thumbnail
+ if detail_page_url:
+ canonical_result = await self.fetcher.resolve_canonical_image(
+ detail_page_url=detail_page_url,
+ thumbnail_url=media_uri,
+ embedding_title=page_title, # Pass listing page title
+ )
+ if canonical_result:
+ detail_page_uri = detail_page_url
+ detail_title = canonical_result.get('detail_title', '')
+ detail_content = canonical_result.get('detail_content', '')
+ # Use canonical URL if found, otherwise keep thumbnail
+ if canonical_result.get('canonical_url'):
+ logger.debug(f"Canonical resolution: {media_uri} -> {canonical_result['canonical_url']}")
+ media_uri = canonical_result['canonical_url']
+ # Enrich metadata from detail page
+ if not alt_text and canonical_result.get('description'):
+ alt_text = canonical_result['description']
# Check if this exact media+page combo was already crawled
existing_hash = await self.db.check_media_uri_exists(media_uri, page_uri)
if existing_hash:
@@ -353,7 +385,7 @@ class NeoPig:
# Check if content already in vault
if await self.vault.exists(md5_hash):
- # Content exists, but add this new page context
+ # Content exists, but add this new page context with skeleton key
await self.db.add_media_source(
md5_hash=md5_hash,
media_uri=media_uri,
@@ -361,8 +393,13 @@ class NeoPig:
page_title=page_title,
page_description=page_description,
page_keywords=page_keywords,
+ page_content=page_content,
alt_text=alt_text,
link_text=link_text,
+ detail_page_uri=detail_page_uri,
+ detail_title=detail_title,
+ detail_content=detail_content,
+ searchable_text=searchable_text,
crawl_job_id=job_id,
)
self.stats['duplicates_skipped'] += 1
@@ -376,7 +413,7 @@ class NeoPig:
# Archive to domain media vault (git-tracked)
await self._archive_media_to_vault(media_uri, result['data'], page_uri)
- # Record in database with full context
+ # Record in database with full context and skeleton key
await self.db.create_media_record(
md5_hash=md5_hash,
media_uri=media_uri,
@@ -388,8 +425,13 @@ class NeoPig:
page_title=page_title,
page_description=page_description,
page_keywords=page_keywords,
+ page_content=page_content,
alt_text=alt_text,
link_text=link_text,
+ detail_page_uri=detail_page_uri,
+ detail_title=detail_title,
+ detail_content=detail_content,
+ searchable_text=searchable_text,
)
self.stats['media_downloaded'] += 1