From 3952978ba45e65430dd555cac1bead9f5b8fb8ba Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Tue, 30 Dec 2025 11:05:10 -0500 Subject: [PATCH] Screenshot backfill improvements and oversized image handling - Filter completed URIs at query level for faster backfill resume - Split oversized screenshots (>65000px) into multiple JPEG chunks - Progress bar shows total including skipped, advances on errors - Suppress PIL DecompressionBombWarning for large screenshots - Cleaner error logging (truncate verbose wkhtmltoimage output) - Per-page capture logs moved to debug level --- archive.py | 8 +- async_web_fetcher.py | 36 +-- domain_vault.py | 23 +- neopig.py | 191 ++++++++------- screenshot.py | 39 +++- serp.py | 540 +++++++++---------------------------------- 6 files changed, 289 insertions(+), 548 deletions(-) diff --git a/archive.py b/archive.py index 3b8e6e7..ebb0a5f 100644 --- a/archive.py +++ b/archive.py @@ -36,7 +36,7 @@ import tempfile from datetime import datetime, timezone from pathlib import Path from typing import Dict, Any, List, Optional -from urllib.parse import urlparse +from miniuri import Uri import aiofiles import aiofiles.os @@ -67,7 +67,7 @@ def sanitize_filename(name: str) -> str: def url_to_path(url: str) -> str: """Convert URL to filesystem path.""" - parsed = urlparse(url) + parsed = Uri(url) path = parsed.path.strip('/') if not path: return 'index.html' @@ -134,8 +134,8 @@ class SiteArchiver: """ Archive a site using neopig and package into tar.gz. """ - parsed = urlparse(target_url) - domain = parsed.netloc.lower() + parsed = Uri(target_url) + domain = parsed.hostname.lower() date_str = datetime.now().strftime('%Y%m%d') archive_name = f"{sanitize_filename(domain)}-{date_str}" diff --git a/async_web_fetcher.py b/async_web_fetcher.py index edbcc70..2ab7edd 100644 --- a/async_web_fetcher.py +++ b/async_web_fetcher.py @@ -22,7 +22,7 @@ from dataclasses import dataclass, field from datetime import datetime, timezone from enum import Enum from typing import Optional, Dict, List, Tuple, Set, Any -from urllib.parse import urlparse, urljoin +from urllib.parse import urljoin from urllib.robotparser import RobotFileParser from bs4 import BeautifulSoup import time @@ -289,8 +289,8 @@ async def fetch_robots_txt(url: str, user_agent: str = "uncloseai.com/1.42") -> Returns: Raw robots.txt content or None if not available """ - parsed = urlparse(url) - robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt" + parsed = Uri(url) + robots_url = f"{parsed.scheme}://{parsed.hostname}/robots.txt" try: async with aiohttp.ClientSession() as session: @@ -319,7 +319,7 @@ def analyze_robots_txt_blocking(robots_txt: str, blocked_url: str, user_agent: s Returns: Human-readable explanation of the blocking rule """ - parsed = urlparse(blocked_url) + parsed = Uri(blocked_url) path = parsed.path or "/" # Parse robots.txt manually for detailed analysis @@ -378,7 +378,7 @@ def get_media_type_from_extension(url: str) -> Optional[str]: Returns: 'image', 'video', 'audio', or None """ - parsed = urlparse(url) + parsed = Uri(url) path = parsed.path.lower() for ext in IMAGE_EXTENSIONS: @@ -599,8 +599,8 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod if href and not get_media_type_from_extension(href): resolved_detail_url = urljoin(base_url, href) # Only track same-domain detail pages - base_domain = urlparse(base_url).netloc - detail_domain = urlparse(resolved_detail_url).netloc + base_domain = Uri(base_url).hostname + detail_domain = Uri(resolved_detail_url).hostname if base_domain != detail_domain: resolved_detail_url = None elif detail_page_url: @@ -830,8 +830,8 @@ class AsyncWebFetcher: def _get_domain(self, url: str) -> str: """Extract domain from URL""" - parsed = urlparse(url) - return parsed.netloc + parsed = Uri(url) + return parsed.hostname async def _fetch_robots_txt(self, domain: str) -> Optional[RobotFileParser]: """ @@ -1096,14 +1096,14 @@ class AsyncWebFetcher: # CANONICAL URL DETECTION (universal patterns) # ======================================== canonical_url = None - detail_parsed = urlparse(detail_page_url) + detail_parsed = Uri(detail_page_url) # Pattern 1: REST-style - check if img src matches URL minus last segment # e.g., /media/ID/details has img pointing to /media/ID path_segments = detail_parsed.path.rstrip('/').split('/') if len(path_segments) > 1: parent_path = '/'.join(path_segments[:-1]) - parent_url = f"{detail_parsed.scheme}://{detail_parsed.netloc}{parent_path}" + parent_url = f"{detail_parsed.scheme}://{detail_parsed.hostname}{parent_path}" for img in soup.find_all('img', src=True): src = img.get('src', '') full_src = urljoin(detail_page_url, src) @@ -1126,13 +1126,13 @@ class AsyncWebFetcher: if not img_inside: continue # Check it's a media URL or same-domain endpoint - href_parsed = urlparse(full_href) + href_parsed = Uri(full_href) media_type = get_media_type_from_extension(full_href) if media_type == 'image': canonical_url = full_href break # Same domain with query params - likely image endpoint - if href_parsed.netloc == detail_parsed.netloc or not href_parsed.netloc: + if href_parsed.hostname == detail_parsed.hostname or not href_parsed.hostname: canonical_url = full_href break @@ -1154,9 +1154,9 @@ class AsyncWebFetcher: full_src = urljoin(detail_page_url, src) if full_src == thumbnail_url: continue - img_parsed = urlparse(full_src) + img_parsed = Uri(full_src) # Same host - if img_parsed.netloc == detail_parsed.netloc: + if img_parsed.hostname == detail_parsed.hostname: # Count shared path segments detail_parts = detail_parsed.path.rstrip('/').split('/') img_parts = img_parsed.path.rstrip('/').split('/') @@ -1348,7 +1348,7 @@ class AsyncWebFetcher: if "nofollow" in rel: continue absolute_url = strip_url_fragment(urljoin(url, href)) - parsed = urlparse(absolute_url) + parsed = Uri(absolute_url) if parsed.scheme in ("http", "https"): if extract_anchor_text: anchor_text = a_tag.get_text(strip=True) @@ -1380,7 +1380,7 @@ class AsyncWebFetcher: if "nofollow" in rel: continue absolute_url = strip_url_fragment(urljoin(url, href)) - parsed_link = urlparse(absolute_url) + parsed_link = Uri(absolute_url) if parsed_link.scheme in ("http", "https"): if extract_anchor_text: anchor_text = a_tag.get_text(strip=True) @@ -1494,7 +1494,7 @@ class AsyncWebFetcher: logger.debug(f"Skipping nofollow link: {href}") continue absolute_url = strip_url_fragment(urljoin(url, href)) - parsed = urlparse(absolute_url) + parsed = Uri(absolute_url) if parsed.scheme in ("http", "https"): if extract_anchor_text: # Extract anchor text for keyword matching diff --git a/domain_vault.py b/domain_vault.py index cdea853..6b5003f 100644 --- a/domain_vault.py +++ b/domain_vault.py @@ -48,7 +48,8 @@ import re from datetime import datetime, timezone from pathlib import Path from typing import Dict, Any, Optional, List, Tuple -from urllib.parse import urlparse, urljoin +from urllib.parse import urljoin +from miniuri import Uri import aiofiles logger = logging.getLogger(__name__) @@ -108,7 +109,7 @@ def url_to_filepath(url: str) -> str: https://example.com/blog/post.html -> blog/post.html https://example.com/images/logo.png -> images/logo.png """ - parsed = urlparse(url) + parsed = Uri(url) path = parsed.path.strip('/') if not path: @@ -288,8 +289,8 @@ class DomainHtmlVault: Example: https://example.com/images/logo.png -> /media/{9-layers}/{salted_hash}/images/logo.png """ - parsed = urlparse(media_url) - media_domain = parsed.netloc.lower() + parsed = Uri(media_url) + media_domain = parsed.hostname.lower() media_path = parsed.path.strip('/') # Get salted hash for media domain (same across all vaults) @@ -454,7 +455,7 @@ class DomainMediaVault: await self.init() # Use URL path as file path - parsed = urlparse(url) + parsed = Uri(url) filepath = parsed.path.strip('/') if not filepath: # Fallback to content hash @@ -491,7 +492,7 @@ class DomainMediaVault: async def get_media(self, url: str) -> Optional[Tuple[bytes, Dict[str, Any]]]: """Get archived media content and metadata.""" - parsed = urlparse(url) + parsed = Uri(url) filepath = parsed.path.strip('/') file_path = self.path / filepath @@ -572,7 +573,7 @@ class DomainLinkpeekVault: https://example.com/about -> about/index.png https://example.com/blog/post.html -> blog/post.png """ - parsed = urlparse(url) + parsed = Uri(url) path = parsed.path.strip('/') if not path: @@ -712,8 +713,8 @@ class VaultManager: def get_vaults_for_url(self, url: str) -> Tuple[DomainHtmlVault, DomainMediaVault, DomainLinkpeekVault]: """Get HTML, media, and linkpeek vaults for URL's domain.""" - parsed = urlparse(url) - domain = parsed.netloc or parsed.path.split('/')[0] + parsed = Uri(url) + domain = parsed.hostname or parsed.path.split('/')[0] return self.get_html_vault(domain), self.get_media_vault(domain), self.get_linkpeek_vault(domain) def get_linkpeek_url(self, url: str) -> str: @@ -723,8 +724,8 @@ class VaultManager: Example: https://example.com/about -> /linkpeek/{9-layers}/{salted_hash}/about/index.png """ - parsed = urlparse(url) - domain = parsed.netloc.lower() + parsed = Uri(url) + domain = parsed.hostname.lower() # Get screenshot path vault = self.get_linkpeek_vault(domain) diff --git a/neopig.py b/neopig.py index 36cc8d8..a91a744 100644 --- a/neopig.py +++ b/neopig.py @@ -29,7 +29,7 @@ from datetime import datetime, timezone, timezone from pathlib import Path from typing import List, Dict, Any, Optional, Set -from urllib.parse import urlparse +from miniuri import Uri from async_web_fetcher import ( AsyncWebFetcher, @@ -186,8 +186,8 @@ class NeoPig: def _get_state_file(self, target_url: str) -> Path: """Get unified state file path for domain.""" - parsed = urlparse(target_url) - domain = parsed.netloc.lower() + parsed = Uri(target_url) + domain = parsed.hostname.lower() return get_state_file_path(domain) def _save_state(self, target_url: str): @@ -270,8 +270,8 @@ class NeoPig: def _get_domain(self, url: str) -> str: """Extract domain from URL.""" - parsed = urlparse(url) - return parsed.netloc.lower() + parsed = Uri(url) + return parsed.hostname.lower() def _track_domain_stat(self, domain: str, stat: str, increment: int = 1): """Track per-domain stats for vault commits.""" @@ -349,7 +349,7 @@ class NeoPig: # Store page content for full-text search and phantom site recreation title, content, markdown = self._extract_text_from_html(html, base_url=uri) - parsed = urlparse(uri) + parsed = Uri(uri) path = parsed.path or '/' await self.db.store_page( uri=uri, @@ -371,7 +371,7 @@ class NeoPig: """Create symlink in domain media vault pointing to hash vault.""" domain = self._get_domain(url) # Domain media path: vault/media_vault/{domain}/{url_path} - parsed = urlparse(url) + parsed = Uri(url) url_path = parsed.path.lstrip('/') or 'index' if not url_path.endswith(ext): url_path = f"{url_path}{ext}" @@ -398,13 +398,14 @@ class NeoPig: url: str, md5_hash: str, ext: str = 'jpg', + suffix: str = '', ): """Create symlink in domain linkpeek vault pointing to hash vault.""" domain = self._get_domain(url) - # Domain screenshot path: vault/linkpeek_vault/{domain}/{url_path}.{ext} - parsed = urlparse(url) + # Domain screenshot path: vault/linkpeek_vault/{domain}/{url_path}{suffix}.{ext} + parsed = Uri(url) url_path = parsed.path.lstrip('/') or 'index' - url_path = url_path.replace('/', '_') + f'.{ext}' + url_path = url_path.replace('/', '_') + f'{suffix}.{ext}' domain_ss_dir = Path(self.vault_path) / 'linkpeek_vault' / domain domain_ss_path = domain_ss_dir / url_path @@ -804,52 +805,57 @@ class NeoPig: if not result: return - md5_hash = result['md5_hash'] - screenshot_data = result['data'] - screenshot_size = len(screenshot_data) - screenshot_ext = result.get('format', 'png') - screenshot_mime = result.get('mime_type', 'image/png') + # Normalize to list (oversized images return multiple chunks) + chunks = result if isinstance(result, list) else [result] + total_size = 0 + + for i, chunk in enumerate(chunks): + md5_hash = chunk['md5_hash'] + screenshot_data = chunk['data'] + screenshot_size = len(screenshot_data) + screenshot_ext = chunk.get('format', 'png') + screenshot_mime = chunk.get('mime_type', 'image/png') + total_size += screenshot_size + + # Store in MD5 vault (for deduplication) + if not await self.vault.exists(md5_hash): + await self.vault.store(md5_hash, screenshot_data, screenshot_ext) + self.stats['bytes_stored'] += screenshot_size + + # Create symlink in linkpeek vault (with suffix for chunks) + suffix = f'_{i}' if len(chunks) > 1 else '' + await self._archive_screenshot_to_vault(page_uri, md5_hash, screenshot_ext, suffix=suffix) + + # Record in database (all chunks, with chunk index in alt_text) + chunk_label = f" (part {i+1}/{len(chunks)})" if len(chunks) > 1 else "" + await self.db.create_media_record( + md5_hash=md5_hash, + media_uri=f"screenshot:{page_uri}{suffix}", + page_uri=page_uri, + crawl_job_id=job_id, + media_type='screenshot', + mime_type=screenshot_mime, + file_size=screenshot_size, + page_title=page_title, + page_description='', + page_keywords='', + alt_text=f"Screenshot of {page_uri}{chunk_label}", + link_text='', + ) # Screenshots are fetched by headless browser (network traffic) - self.stats['bytes_downloaded'] += screenshot_size - - # Store in MD5 vault (for deduplication) - if not await self.vault.exists(md5_hash): - await self.vault.store(md5_hash, screenshot_data, screenshot_ext) - self.stats['bytes_stored'] += screenshot_size - - # Create symlink in linkpeek vault pointing to hash vault - await self._archive_screenshot_to_vault(page_uri, md5_hash, screenshot_ext) - - # Record in database as screenshot type - await self.db.create_media_record( - md5_hash=md5_hash, - media_uri=f"screenshot:{page_uri}", - page_uri=page_uri, - crawl_job_id=job_id, - media_type='screenshot', - mime_type=screenshot_mime, - file_size=result.get('size', 0), - page_title=page_title, - page_description='', - page_keywords='', - alt_text=f"Screenshot of {page_uri}", - link_text='', - ) - + self.stats['bytes_downloaded'] += total_size self.stats['screenshots_taken'] += 1 self.seen_screenshots.add(page_uri) # Only mark seen after success - logger.debug(f"Screenshot captured: {page_uri} -> {md5_hash}") + logger.debug(f"Screenshot captured: {page_uri} -> {chunks[0]['md5_hash']} ({len(chunks)} chunk(s))") except Exception as e: logger.warning(f"Screenshot failed for {page_uri}: {e}") def _get_extension(self, url: str, mime_type: str) -> str: """Determine file extension from URL or MIME type.""" - from urllib.parse import urlparse - # Try from URL path - path = urlparse(url).path.lower() + path = Uri(url).path.lower() for ext in ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.bmp', '.mp4', '.webm', '.mov', '.avi', '.mkv']: if path.endswith(ext): @@ -1013,7 +1019,7 @@ async def backfill_screenshots( await db.init() async with db.session() as session: - # Find all screenshot records using ORM + # Find all screenshot records using ORM, excluding already-completed URIs if domain_filter: pattern = f"%://{domain_filter}%" if '://' not in domain_filter else f"%{domain_filter}%" stmt = ( @@ -1021,6 +1027,7 @@ async def backfill_screenshots( .join(Media, MediaSource.md5_hash == Media.md5_hash) .where(Media.media_type == 'screenshot') .where(MediaSource.page_uri.like(pattern)) + .where(MediaSource.page_uri.notin_(completed_uris) if completed_uris else True) .distinct() ) logger.info(f"Backfilling screenshots for domain: {domain_filter}") @@ -1029,35 +1036,32 @@ async def backfill_screenshots( select(MediaSource.page_uri, Media.md5_hash) .join(Media, MediaSource.md5_hash == Media.md5_hash) .where(Media.media_type == 'screenshot') + .where(MediaSource.page_uri.notin_(completed_uris) if completed_uris else True) .distinct() ) logger.info("Backfilling screenshots for ALL pages") result = await session.execute(stmt) rows = result.fetchall() - total = len(rows) - logger.info(f"Found {total} screenshots to re-capture...") + to_process = len(rows) + skipped_count = len(completed_uris) + total = to_process + skipped_count + logger.info(f"Found {to_process} to process ({skipped_count} already completed)") captured = 0 - skipped = 0 failed = 0 bytes_saved = 0 domain_last_fetched = {} # Track last fetch time per domain crawl_delay = 2.0 # Default crawl delay in seconds - pbar = tqdm(rows, desc="Screenshots", unit="pages") - for row in pbar: + pbar = tqdm(total=total, initial=skipped_count, desc="Screenshots", unit="pages") + for row in rows: page_uri = row.page_uri old_hash = row.md5_hash - domain = urlparse(page_uri).netloc.lower() + domain = Uri(page_uri).hostname.lower() - # Skip if already completed (from state file) - if page_uri in completed_uris: - skipped += 1 - continue - - # Log current page - logger.info(f"Capturing: {page_uri}") + # Log current page (debug level to reduce noise) + logger.debug(f"Capturing: {page_uri}") try: # Enforce crawl delay (skip in fast mode) @@ -1079,43 +1083,56 @@ async def backfill_screenshots( result = await capture.capture(page_uri, content_length=content_length) if not result: failed += 1 + pbar.update(1) continue - new_hash = result['md5_hash'] - new_data = result['data'] - new_ext = result.get('format', 'jpg') - new_mime = result.get('mime_type', 'image/jpeg') - new_size = len(new_data) + # Normalize to list (oversized images return multiple chunks) + chunks = result if isinstance(result, list) else [result] # Get old file size for comparison old_path = Path(vault_path) / old_hash[:2] / f"{old_hash}.png" old_size = old_path.stat().st_size if old_path.exists() else 0 - # Store new screenshot - if not await vault.exists(new_hash): - await vault.store(new_hash, new_data, new_ext) + # Store all chunks + total_new_size = 0 + for i, chunk in enumerate(chunks): + new_hash = chunk['md5_hash'] + new_data = chunk['data'] + new_ext = chunk.get('format', 'jpg') + new_mime = chunk.get('mime_type', 'image/jpeg') + total_new_size += len(new_data) + + if not await vault.exists(new_hash): + await vault.store(new_hash, new_data, new_ext) + + # Create symlink for each chunk + parsed = Uri(page_uri) + url_path = parsed.path.lstrip('/') or 'index' + suffix = f'_{i}' if len(chunks) > 1 else '' + new_symlink = Path(vault_path) / 'linkpeek_vault' / domain / (url_path.replace('/', '_') + f'{suffix}.{new_ext}') + new_symlink.parent.mkdir(parents=True, exist_ok=True) + hash_vault_path = Path(vault_path) / new_hash[:2] / f"{new_hash}.{new_ext}" + rel_path = os.path.relpath(hash_vault_path, new_symlink.parent) + if not new_symlink.exists(): + new_symlink.symlink_to(rel_path) # Delete old PNG file FIRST (before metadata update) - if delete_old and old_hash != new_hash and old_path.exists(): + if delete_old and old_path.exists(): old_path.unlink() - bytes_saved += old_size - new_size - - # Update linkpeek symlink - parsed = urlparse(page_uri) - url_path = parsed.path.lstrip('/') or 'index' + bytes_saved += old_size - total_new_size # Remove old symlink + parsed = Uri(page_uri) + url_path = parsed.path.lstrip('/') or 'index' old_symlink = Path(vault_path) / 'linkpeek_vault' / domain / (url_path.replace('/', '_') + '.png') if old_symlink.exists(): old_symlink.unlink() - # Create new symlink - new_symlink = Path(vault_path) / 'linkpeek_vault' / domain / (url_path.replace('/', '_') + f'.{new_ext}') - new_symlink.parent.mkdir(parents=True, exist_ok=True) - hash_vault_path = Path(vault_path) / new_hash[:2] / f"{new_hash}.{new_ext}" - rel_path = os.path.relpath(hash_vault_path, new_symlink.parent) - if not new_symlink.exists(): - new_symlink.symlink_to(rel_path) + # Use first chunk for database record + new_hash = chunks[0]['md5_hash'] + new_ext = chunks[0].get('format', 'jpg') + new_mime = chunks[0].get('mime_type', 'image/jpeg') + new_size = total_new_size # Update database using ORM # md5_hash is PRIMARY KEY, so we need to: insert new -> update refs -> delete old @@ -1175,12 +1192,18 @@ async def backfill_screenshots( full_state['backfill_screenshots'] = list(completed_uris) state_path.write_text(json.dumps(full_state, indent=2)) - except Exception as e: - logger.warning(f"Error re-capturing {page_uri}: {e}") - failed += 1 + pbar.update(1) + except Exception as e: + # Truncate error message for cleaner logs + err_msg = str(e).split('\n')[0][:60] + logger.warning(f"Failed: {page_uri} - {err_msg}") + failed += 1 + pbar.update(1) + + pbar.close() await session.commit() - logger.info(f"Backfill complete: {captured} captured, {skipped} skipped, {failed} failed") + logger.info(f"Backfill complete: {captured} captured, {skipped_count} skipped, {failed} failed") if bytes_saved > 0: logger.info(f"Space saved: {bytes_saved / 1024 / 1024:.1f} MB") diff --git a/screenshot.py b/screenshot.py index 5b4e980..ce47ec2 100644 --- a/screenshot.py +++ b/screenshot.py @@ -22,6 +22,7 @@ import hashlib import io import logging import shutil +import warnings from dataclasses import dataclass, field from pathlib import Path from typing import Optional, List, Dict, Any @@ -29,6 +30,9 @@ import tempfile from PIL import Image +# Suppress PIL decompression bomb warnings for large screenshots +warnings.filterwarnings('ignore', category=Image.DecompressionBombWarning) + logger = logging.getLogger(__name__) # Engine preference order - lightest/fastest first @@ -258,7 +262,9 @@ class ScreenshotCapture: ) if not result.success: - logger.warning(f"Screenshot failed: {uri} - {result.error}") + # Extract just the first line of error, skip wkhtmltoimage verbosity + error_msg = (result.error or 'unknown error').split('\n')[0][:80] + logger.warning(f"Screenshot failed: {uri} - {error_msg}") return None # Read bytes from output file @@ -270,9 +276,40 @@ class ScreenshotCapture: # Convert to JPEG if configured if self.config.format == 'jpeg': img = Image.open(io.BytesIO(png_data)) + # Convert RGBA to RGB (JPEG doesn't support alpha) if img.mode in ('RGBA', 'LA', 'P'): img = img.convert('RGB') + + # Split oversized images (JPEG max is 65500px) + max_dim = 65000 + if img.height > max_dim: + logger.debug(f"Splitting oversized image: {img.height}px into chunks") + results = [] + chunk_idx = 0 + y = 0 + while y < img.height: + chunk_height = min(max_dim, img.height - y) + chunk = img.crop((0, y, img.width, y + chunk_height)) + output = io.BytesIO() + chunk.save(output, format='JPEG', quality=self.config.quality, optimize=True) + chunk_data = output.getvalue() + chunk_hash = hashlib.md5(chunk_data).hexdigest() + results.append({ + 'data': chunk_data, + 'md5_hash': chunk_hash, + 'mime_type': 'image/jpeg', + 'size': len(chunk_data), + 'source_uri': uri, + 'engine': self._engine_name, + 'format': 'jpg', + 'chunk': chunk_idx, + 'total_chunks': (img.height + max_dim - 1) // max_dim, + }) + y += max_dim + chunk_idx += 1 + return results # Return list for oversized images + output = io.BytesIO() img.save(output, format='JPEG', quality=self.config.quality, optimize=True) data = output.getvalue() diff --git a/serp.py b/serp.py index d4c5129..667a0b3 100644 --- a/serp.py +++ b/serp.py @@ -32,6 +32,7 @@ from sqlalchemy import text import uvicorn from database import Database +from miniuri import Uri logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -65,6 +66,84 @@ ARCHIVE_ROOT: str = None # e.g., "example.com-20251230" TEMP_DB_PATH: str = None # Extracted database (SQLite needs real file) +# Shared CSS for view pages +VIEW_CSS = """ +* { box-sizing: border-box; } +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + margin: 0; padding: 0; + background: #0a0a0a; color: #e0e0e0; +} +.nav { + background: #1a1a1a; padding: 10px 20px; + display: flex; gap: 20px; align-items: center; + border-bottom: 1px solid #333; +} +.nav a { color: #ff6b6b; text-decoration: none; } +.nav a:hover { text-decoration: underline; } +.nav .brand { font-weight: bold; font-size: 18px; } +.container { padding: 20px; max-width: 1200px; margin: 0 auto; } +h1 { color: #ff6b6b; font-size: 20px; margin: 0 0 10px 0; } +h3 { color: #ff6b6b; font-size: 16px; margin: 30px 0 15px 0; border-bottom: 1px solid #333; padding-bottom: 8px; } +a { color: #ff6b6b; } +.meta { background: #1a1a1a; padding: 15px; border-radius: 8px; margin: 15px 0; } +.meta-row { display: flex; margin: 8px 0; } +.meta-label { width: 100px; color: #888; font-size: 13px; } +.meta-value { flex: 1; word-break: break-all; font-size: 13px; } +.meta-value a { color: #4ade80; } +.hero { text-align: center; margin-bottom: 20px; } +.hero img, .hero video { max-width: 100%; max-height: 60vh; border-radius: 8px; } +.content-rendered { + background: #1a1a1a; padding: 20px; border-radius: 8px; + line-height: 1.7; font-size: 14px; color: #ccc; +} +.content-rendered img { max-width: 100%; height: auto; margin: 10px 0; } +.content-rendered img.avatar { + display: inline-block; vertical-align: middle; + width: 40px; height: 40px; border-radius: 50%; + margin: 0 10px 0 0; object-fit: cover; +} +.content-rendered img.emoji { + display: inline; width: 20px; height: 20px; + margin: 0 2px; vertical-align: text-bottom; +} +.content-rendered a { color: #ff6b6b; } +.content-rendered pre, .content-rendered code { + background: #252525; padding: 2px 6px; + border-radius: 4px; font-family: monospace; font-size: 13px; +} +.content-rendered pre { + padding: 15px; display: block; + white-space: pre-wrap; overflow-x: auto; +} +.content-rendered blockquote { + background: #151520; border-left: 3px solid #4a9eff; + padding: 12px 16px; margin: 16px 0; + border-radius: 0 6px 6px 0; color: #aaa; font-style: italic; +} +.content-rendered hr { border: none; border-top: 1px solid #333; margin: 24px 0; } +.content-rendered p { margin: 0 0 16px 0; line-height: 1.7; } +.content-rendered h1, .content-rendered h2, .content-rendered h3 { + color: #ff6b6b; margin-top: 28px; margin-bottom: 12px; +} +.media-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(150px, 1fr)); + gap: 10px; margin-top: 15px; +} +.media-card { + background: #1a1a1a; border-radius: 8px; + overflow: hidden; display: block; transition: transform 0.2s; +} +.media-card:hover { transform: scale(1.02); } +.media-card img, .media-card video { + width: 100%; height: 120px; + object-fit: contain; background: #0a0a0a; +} +.tag { background: #333; padding: 2px 8px; border-radius: 4px; font-size: 12px; margin-right: 5px; } +""" + + def read_from_tarball(path: str) -> bytes: """Read a file from the tarball. Path is relative to archive root.""" if not TAR_FILE or not ARCHIVE_ROOT: @@ -1231,6 +1310,10 @@ async def view_media_page(md5_hash: str): # Get page URI for content lookup page_uri = sorted_sources[0]["page_uri"] if sorted_sources else None + # Extract domain for AI system prompt + source_uri = page_uri or (sorted_sources[0]["media_uri"] if sorted_sources else "unknown") + source_domain = Uri(source_uri).hostname if source_uri else "unknown" + # Get page content - prefer markdown, fallback to raw HTML or text page_content_html = "" @@ -1756,6 +1839,9 @@ async def view_media_page(md5_hash: str): // Run after DOM ready (avatars may be empty placeholders) restructureForumPosts(); + @@ -1771,6 +1857,8 @@ async def view_page( import html as html_module import re + source_domain = Uri(uri).hostname + page = await db.get_page_by_uri(uri) if not page: raise HTTPException(status_code=404, detail="Page not found") @@ -1854,258 +1942,13 @@ async def view_page(
{''.join(media_cards)}
''' - return f""" - + return f""" {html_module.escape(page_title)} - neopig - +
@@ -2113,190 +1956,27 @@ async def view_page( Search Live Random - Crawl - Phantom
-
- - - -
-
-
- {screenshot_html or '
No screenshot available
'} -

{html_module.escape(page_title)}

-
-
-
-
URL:{uri}
-
Media:{len(media_items)} items
-
-
{content_html or '

No content available

'}
-
+
+ {screenshot_html or ''} +
+

{html_module.escape(page_title)}

+
+
URL:{uri}
+
Media:{len(media_items)} items
{media_grid} +

Page Content

+
{content_html or '

No content available

'}
- + {'' if noai else f''' - {'' if noai else ''} + '''} - -""" +""" @app.get("/phantom/export") @@ -2309,7 +1989,7 @@ async def phantom_export(domain: str = Query(None, description="Filter by domain import io import re import zipfile - from urllib.parse import urlparse, urljoin + from urllib.parse import urljoin from fastapi.responses import StreamingResponse # Get all pages with raw_html @@ -2335,8 +2015,8 @@ async def phantom_export(domain: str = Query(None, description="Filter by domain continue # Parse URI to get path - parsed = urlparse(uri) - site_domain = parsed.netloc + parsed = Uri(uri) + site_domain = parsed.hostname path = parsed.path.strip('/') or 'index' if not path.endswith('.html') and '.' not in path.split('/')[-1]: path = f"{path}/index.html" if path else "index.html" @@ -2412,7 +2092,7 @@ async def phantom_export(domain: str = Query(None, description="Filter by domain