Screenshot backfill improvements and oversized image handling
- Filter completed URIs at query level for faster backfill resume - Split oversized screenshots (>65000px) into multiple JPEG chunks - Progress bar shows total including skipped, advances on errors - Suppress PIL DecompressionBombWarning for large screenshots - Cleaner error logging (truncate verbose wkhtmltoimage output) - Per-page capture logs moved to debug level
This commit is contained in:
parent
7b4c8b0241
commit
3952978ba4
6 changed files with 289 additions and 548 deletions
|
|
@ -36,7 +36,7 @@ import tempfile
|
|||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
from miniuri import Uri
|
||||
|
||||
import aiofiles
|
||||
import aiofiles.os
|
||||
|
|
@ -67,7 +67,7 @@ def sanitize_filename(name: str) -> str:
|
|||
|
||||
def url_to_path(url: str) -> str:
|
||||
"""Convert URL to filesystem path."""
|
||||
parsed = urlparse(url)
|
||||
parsed = Uri(url)
|
||||
path = parsed.path.strip('/')
|
||||
if not path:
|
||||
return 'index.html'
|
||||
|
|
@ -134,8 +134,8 @@ class SiteArchiver:
|
|||
"""
|
||||
Archive a site using neopig and package into tar.gz.
|
||||
"""
|
||||
parsed = urlparse(target_url)
|
||||
domain = parsed.netloc.lower()
|
||||
parsed = Uri(target_url)
|
||||
domain = parsed.hostname.lower()
|
||||
date_str = datetime.now().strftime('%Y%m%d')
|
||||
archive_name = f"{sanitize_filename(domain)}-{date_str}"
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from dataclasses import dataclass, field
|
|||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Optional, Dict, List, Tuple, Set, Any
|
||||
from urllib.parse import urlparse, urljoin
|
||||
from urllib.parse import urljoin
|
||||
from urllib.robotparser import RobotFileParser
|
||||
from bs4 import BeautifulSoup
|
||||
import time
|
||||
|
|
@ -289,8 +289,8 @@ async def fetch_robots_txt(url: str, user_agent: str = "uncloseai.com/1.42") ->
|
|||
Returns:
|
||||
Raw robots.txt content or None if not available
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
|
||||
parsed = Uri(url)
|
||||
robots_url = f"{parsed.scheme}://{parsed.hostname}/robots.txt"
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
|
|
@ -319,7 +319,7 @@ def analyze_robots_txt_blocking(robots_txt: str, blocked_url: str, user_agent: s
|
|||
Returns:
|
||||
Human-readable explanation of the blocking rule
|
||||
"""
|
||||
parsed = urlparse(blocked_url)
|
||||
parsed = Uri(blocked_url)
|
||||
path = parsed.path or "/"
|
||||
|
||||
# Parse robots.txt manually for detailed analysis
|
||||
|
|
@ -378,7 +378,7 @@ def get_media_type_from_extension(url: str) -> Optional[str]:
|
|||
Returns:
|
||||
'image', 'video', 'audio', or None
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
parsed = Uri(url)
|
||||
path = parsed.path.lower()
|
||||
|
||||
for ext in IMAGE_EXTENSIONS:
|
||||
|
|
@ -599,8 +599,8 @@ def extract_media_from_html(html: str, base_url: str, mode: CrawlMode = CrawlMod
|
|||
if href and not get_media_type_from_extension(href):
|
||||
resolved_detail_url = urljoin(base_url, href)
|
||||
# Only track same-domain detail pages
|
||||
base_domain = urlparse(base_url).netloc
|
||||
detail_domain = urlparse(resolved_detail_url).netloc
|
||||
base_domain = Uri(base_url).hostname
|
||||
detail_domain = Uri(resolved_detail_url).hostname
|
||||
if base_domain != detail_domain:
|
||||
resolved_detail_url = None
|
||||
elif detail_page_url:
|
||||
|
|
@ -830,8 +830,8 @@ class AsyncWebFetcher:
|
|||
|
||||
def _get_domain(self, url: str) -> str:
|
||||
"""Extract domain from URL"""
|
||||
parsed = urlparse(url)
|
||||
return parsed.netloc
|
||||
parsed = Uri(url)
|
||||
return parsed.hostname
|
||||
|
||||
async def _fetch_robots_txt(self, domain: str) -> Optional[RobotFileParser]:
|
||||
"""
|
||||
|
|
@ -1096,14 +1096,14 @@ class AsyncWebFetcher:
|
|||
# CANONICAL URL DETECTION (universal patterns)
|
||||
# ========================================
|
||||
canonical_url = None
|
||||
detail_parsed = urlparse(detail_page_url)
|
||||
detail_parsed = Uri(detail_page_url)
|
||||
|
||||
# Pattern 1: REST-style - check if img src matches URL minus last segment
|
||||
# e.g., /media/ID/details has img pointing to /media/ID
|
||||
path_segments = detail_parsed.path.rstrip('/').split('/')
|
||||
if len(path_segments) > 1:
|
||||
parent_path = '/'.join(path_segments[:-1])
|
||||
parent_url = f"{detail_parsed.scheme}://{detail_parsed.netloc}{parent_path}"
|
||||
parent_url = f"{detail_parsed.scheme}://{detail_parsed.hostname}{parent_path}"
|
||||
for img in soup.find_all('img', src=True):
|
||||
src = img.get('src', '')
|
||||
full_src = urljoin(detail_page_url, src)
|
||||
|
|
@ -1126,13 +1126,13 @@ class AsyncWebFetcher:
|
|||
if not img_inside:
|
||||
continue
|
||||
# Check it's a media URL or same-domain endpoint
|
||||
href_parsed = urlparse(full_href)
|
||||
href_parsed = Uri(full_href)
|
||||
media_type = get_media_type_from_extension(full_href)
|
||||
if media_type == 'image':
|
||||
canonical_url = full_href
|
||||
break
|
||||
# Same domain with query params - likely image endpoint
|
||||
if href_parsed.netloc == detail_parsed.netloc or not href_parsed.netloc:
|
||||
if href_parsed.hostname == detail_parsed.hostname or not href_parsed.hostname:
|
||||
canonical_url = full_href
|
||||
break
|
||||
|
||||
|
|
@ -1154,9 +1154,9 @@ class AsyncWebFetcher:
|
|||
full_src = urljoin(detail_page_url, src)
|
||||
if full_src == thumbnail_url:
|
||||
continue
|
||||
img_parsed = urlparse(full_src)
|
||||
img_parsed = Uri(full_src)
|
||||
# Same host
|
||||
if img_parsed.netloc == detail_parsed.netloc:
|
||||
if img_parsed.hostname == detail_parsed.hostname:
|
||||
# Count shared path segments
|
||||
detail_parts = detail_parsed.path.rstrip('/').split('/')
|
||||
img_parts = img_parsed.path.rstrip('/').split('/')
|
||||
|
|
@ -1348,7 +1348,7 @@ class AsyncWebFetcher:
|
|||
if "nofollow" in rel:
|
||||
continue
|
||||
absolute_url = strip_url_fragment(urljoin(url, href))
|
||||
parsed = urlparse(absolute_url)
|
||||
parsed = Uri(absolute_url)
|
||||
if parsed.scheme in ("http", "https"):
|
||||
if extract_anchor_text:
|
||||
anchor_text = a_tag.get_text(strip=True)
|
||||
|
|
@ -1380,7 +1380,7 @@ class AsyncWebFetcher:
|
|||
if "nofollow" in rel:
|
||||
continue
|
||||
absolute_url = strip_url_fragment(urljoin(url, href))
|
||||
parsed_link = urlparse(absolute_url)
|
||||
parsed_link = Uri(absolute_url)
|
||||
if parsed_link.scheme in ("http", "https"):
|
||||
if extract_anchor_text:
|
||||
anchor_text = a_tag.get_text(strip=True)
|
||||
|
|
@ -1494,7 +1494,7 @@ class AsyncWebFetcher:
|
|||
logger.debug(f"Skipping nofollow link: {href}")
|
||||
continue
|
||||
absolute_url = strip_url_fragment(urljoin(url, href))
|
||||
parsed = urlparse(absolute_url)
|
||||
parsed = Uri(absolute_url)
|
||||
if parsed.scheme in ("http", "https"):
|
||||
if extract_anchor_text:
|
||||
# Extract anchor text for keyword matching
|
||||
|
|
|
|||
|
|
@ -48,7 +48,8 @@ import re
|
|||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional, List, Tuple
|
||||
from urllib.parse import urlparse, urljoin
|
||||
from urllib.parse import urljoin
|
||||
from miniuri import Uri
|
||||
import aiofiles
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -108,7 +109,7 @@ def url_to_filepath(url: str) -> str:
|
|||
https://example.com/blog/post.html -> blog/post.html
|
||||
https://example.com/images/logo.png -> images/logo.png
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
parsed = Uri(url)
|
||||
path = parsed.path.strip('/')
|
||||
|
||||
if not path:
|
||||
|
|
@ -288,8 +289,8 @@ class DomainHtmlVault:
|
|||
Example: https://example.com/images/logo.png
|
||||
-> /media/{9-layers}/{salted_hash}/images/logo.png
|
||||
"""
|
||||
parsed = urlparse(media_url)
|
||||
media_domain = parsed.netloc.lower()
|
||||
parsed = Uri(media_url)
|
||||
media_domain = parsed.hostname.lower()
|
||||
media_path = parsed.path.strip('/')
|
||||
|
||||
# Get salted hash for media domain (same across all vaults)
|
||||
|
|
@ -454,7 +455,7 @@ class DomainMediaVault:
|
|||
await self.init()
|
||||
|
||||
# Use URL path as file path
|
||||
parsed = urlparse(url)
|
||||
parsed = Uri(url)
|
||||
filepath = parsed.path.strip('/')
|
||||
if not filepath:
|
||||
# Fallback to content hash
|
||||
|
|
@ -491,7 +492,7 @@ class DomainMediaVault:
|
|||
|
||||
async def get_media(self, url: str) -> Optional[Tuple[bytes, Dict[str, Any]]]:
|
||||
"""Get archived media content and metadata."""
|
||||
parsed = urlparse(url)
|
||||
parsed = Uri(url)
|
||||
filepath = parsed.path.strip('/')
|
||||
file_path = self.path / filepath
|
||||
|
||||
|
|
@ -572,7 +573,7 @@ class DomainLinkpeekVault:
|
|||
https://example.com/about -> about/index.png
|
||||
https://example.com/blog/post.html -> blog/post.png
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
parsed = Uri(url)
|
||||
path = parsed.path.strip('/')
|
||||
|
||||
if not path:
|
||||
|
|
@ -712,8 +713,8 @@ class VaultManager:
|
|||
|
||||
def get_vaults_for_url(self, url: str) -> Tuple[DomainHtmlVault, DomainMediaVault, DomainLinkpeekVault]:
|
||||
"""Get HTML, media, and linkpeek vaults for URL's domain."""
|
||||
parsed = urlparse(url)
|
||||
domain = parsed.netloc or parsed.path.split('/')[0]
|
||||
parsed = Uri(url)
|
||||
domain = parsed.hostname or parsed.path.split('/')[0]
|
||||
return self.get_html_vault(domain), self.get_media_vault(domain), self.get_linkpeek_vault(domain)
|
||||
|
||||
def get_linkpeek_url(self, url: str) -> str:
|
||||
|
|
@ -723,8 +724,8 @@ class VaultManager:
|
|||
Example: https://example.com/about
|
||||
-> /linkpeek/{9-layers}/{salted_hash}/about/index.png
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
domain = parsed.netloc.lower()
|
||||
parsed = Uri(url)
|
||||
domain = parsed.hostname.lower()
|
||||
|
||||
# Get screenshot path
|
||||
vault = self.get_linkpeek_vault(domain)
|
||||
|
|
|
|||
191
neopig.py
191
neopig.py
|
|
@ -29,7 +29,7 @@ from datetime import datetime, timezone, timezone
|
|||
from pathlib import Path
|
||||
from typing import List, Dict, Any, Optional, Set
|
||||
|
||||
from urllib.parse import urlparse
|
||||
from miniuri import Uri
|
||||
|
||||
from async_web_fetcher import (
|
||||
AsyncWebFetcher,
|
||||
|
|
@ -186,8 +186,8 @@ class NeoPig:
|
|||
|
||||
def _get_state_file(self, target_url: str) -> Path:
|
||||
"""Get unified state file path for domain."""
|
||||
parsed = urlparse(target_url)
|
||||
domain = parsed.netloc.lower()
|
||||
parsed = Uri(target_url)
|
||||
domain = parsed.hostname.lower()
|
||||
return get_state_file_path(domain)
|
||||
|
||||
def _save_state(self, target_url: str):
|
||||
|
|
@ -270,8 +270,8 @@ class NeoPig:
|
|||
|
||||
def _get_domain(self, url: str) -> str:
|
||||
"""Extract domain from URL."""
|
||||
parsed = urlparse(url)
|
||||
return parsed.netloc.lower()
|
||||
parsed = Uri(url)
|
||||
return parsed.hostname.lower()
|
||||
|
||||
def _track_domain_stat(self, domain: str, stat: str, increment: int = 1):
|
||||
"""Track per-domain stats for vault commits."""
|
||||
|
|
@ -349,7 +349,7 @@ class NeoPig:
|
|||
|
||||
# Store page content for full-text search and phantom site recreation
|
||||
title, content, markdown = self._extract_text_from_html(html, base_url=uri)
|
||||
parsed = urlparse(uri)
|
||||
parsed = Uri(uri)
|
||||
path = parsed.path or '/'
|
||||
await self.db.store_page(
|
||||
uri=uri,
|
||||
|
|
@ -371,7 +371,7 @@ class NeoPig:
|
|||
"""Create symlink in domain media vault pointing to hash vault."""
|
||||
domain = self._get_domain(url)
|
||||
# Domain media path: vault/media_vault/{domain}/{url_path}
|
||||
parsed = urlparse(url)
|
||||
parsed = Uri(url)
|
||||
url_path = parsed.path.lstrip('/') or 'index'
|
||||
if not url_path.endswith(ext):
|
||||
url_path = f"{url_path}{ext}"
|
||||
|
|
@ -398,13 +398,14 @@ class NeoPig:
|
|||
url: str,
|
||||
md5_hash: str,
|
||||
ext: str = 'jpg',
|
||||
suffix: str = '',
|
||||
):
|
||||
"""Create symlink in domain linkpeek vault pointing to hash vault."""
|
||||
domain = self._get_domain(url)
|
||||
# Domain screenshot path: vault/linkpeek_vault/{domain}/{url_path}.{ext}
|
||||
parsed = urlparse(url)
|
||||
# Domain screenshot path: vault/linkpeek_vault/{domain}/{url_path}{suffix}.{ext}
|
||||
parsed = Uri(url)
|
||||
url_path = parsed.path.lstrip('/') or 'index'
|
||||
url_path = url_path.replace('/', '_') + f'.{ext}'
|
||||
url_path = url_path.replace('/', '_') + f'{suffix}.{ext}'
|
||||
domain_ss_dir = Path(self.vault_path) / 'linkpeek_vault' / domain
|
||||
domain_ss_path = domain_ss_dir / url_path
|
||||
|
||||
|
|
@ -804,52 +805,57 @@ class NeoPig:
|
|||
if not result:
|
||||
return
|
||||
|
||||
md5_hash = result['md5_hash']
|
||||
screenshot_data = result['data']
|
||||
screenshot_size = len(screenshot_data)
|
||||
screenshot_ext = result.get('format', 'png')
|
||||
screenshot_mime = result.get('mime_type', 'image/png')
|
||||
# Normalize to list (oversized images return multiple chunks)
|
||||
chunks = result if isinstance(result, list) else [result]
|
||||
total_size = 0
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
md5_hash = chunk['md5_hash']
|
||||
screenshot_data = chunk['data']
|
||||
screenshot_size = len(screenshot_data)
|
||||
screenshot_ext = chunk.get('format', 'png')
|
||||
screenshot_mime = chunk.get('mime_type', 'image/png')
|
||||
total_size += screenshot_size
|
||||
|
||||
# Store in MD5 vault (for deduplication)
|
||||
if not await self.vault.exists(md5_hash):
|
||||
await self.vault.store(md5_hash, screenshot_data, screenshot_ext)
|
||||
self.stats['bytes_stored'] += screenshot_size
|
||||
|
||||
# Create symlink in linkpeek vault (with suffix for chunks)
|
||||
suffix = f'_{i}' if len(chunks) > 1 else ''
|
||||
await self._archive_screenshot_to_vault(page_uri, md5_hash, screenshot_ext, suffix=suffix)
|
||||
|
||||
# Record in database (all chunks, with chunk index in alt_text)
|
||||
chunk_label = f" (part {i+1}/{len(chunks)})" if len(chunks) > 1 else ""
|
||||
await self.db.create_media_record(
|
||||
md5_hash=md5_hash,
|
||||
media_uri=f"screenshot:{page_uri}{suffix}",
|
||||
page_uri=page_uri,
|
||||
crawl_job_id=job_id,
|
||||
media_type='screenshot',
|
||||
mime_type=screenshot_mime,
|
||||
file_size=screenshot_size,
|
||||
page_title=page_title,
|
||||
page_description='',
|
||||
page_keywords='',
|
||||
alt_text=f"Screenshot of {page_uri}{chunk_label}",
|
||||
link_text='',
|
||||
)
|
||||
|
||||
# Screenshots are fetched by headless browser (network traffic)
|
||||
self.stats['bytes_downloaded'] += screenshot_size
|
||||
|
||||
# Store in MD5 vault (for deduplication)
|
||||
if not await self.vault.exists(md5_hash):
|
||||
await self.vault.store(md5_hash, screenshot_data, screenshot_ext)
|
||||
self.stats['bytes_stored'] += screenshot_size
|
||||
|
||||
# Create symlink in linkpeek vault pointing to hash vault
|
||||
await self._archive_screenshot_to_vault(page_uri, md5_hash, screenshot_ext)
|
||||
|
||||
# Record in database as screenshot type
|
||||
await self.db.create_media_record(
|
||||
md5_hash=md5_hash,
|
||||
media_uri=f"screenshot:{page_uri}",
|
||||
page_uri=page_uri,
|
||||
crawl_job_id=job_id,
|
||||
media_type='screenshot',
|
||||
mime_type=screenshot_mime,
|
||||
file_size=result.get('size', 0),
|
||||
page_title=page_title,
|
||||
page_description='',
|
||||
page_keywords='',
|
||||
alt_text=f"Screenshot of {page_uri}",
|
||||
link_text='',
|
||||
)
|
||||
|
||||
self.stats['bytes_downloaded'] += total_size
|
||||
self.stats['screenshots_taken'] += 1
|
||||
self.seen_screenshots.add(page_uri) # Only mark seen after success
|
||||
logger.debug(f"Screenshot captured: {page_uri} -> {md5_hash}")
|
||||
logger.debug(f"Screenshot captured: {page_uri} -> {chunks[0]['md5_hash']} ({len(chunks)} chunk(s))")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Screenshot failed for {page_uri}: {e}")
|
||||
|
||||
def _get_extension(self, url: str, mime_type: str) -> str:
|
||||
"""Determine file extension from URL or MIME type."""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# Try from URL path
|
||||
path = urlparse(url).path.lower()
|
||||
path = Uri(url).path.lower()
|
||||
for ext in ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.bmp',
|
||||
'.mp4', '.webm', '.mov', '.avi', '.mkv']:
|
||||
if path.endswith(ext):
|
||||
|
|
@ -1013,7 +1019,7 @@ async def backfill_screenshots(
|
|||
await db.init()
|
||||
|
||||
async with db.session() as session:
|
||||
# Find all screenshot records using ORM
|
||||
# Find all screenshot records using ORM, excluding already-completed URIs
|
||||
if domain_filter:
|
||||
pattern = f"%://{domain_filter}%" if '://' not in domain_filter else f"%{domain_filter}%"
|
||||
stmt = (
|
||||
|
|
@ -1021,6 +1027,7 @@ async def backfill_screenshots(
|
|||
.join(Media, MediaSource.md5_hash == Media.md5_hash)
|
||||
.where(Media.media_type == 'screenshot')
|
||||
.where(MediaSource.page_uri.like(pattern))
|
||||
.where(MediaSource.page_uri.notin_(completed_uris) if completed_uris else True)
|
||||
.distinct()
|
||||
)
|
||||
logger.info(f"Backfilling screenshots for domain: {domain_filter}")
|
||||
|
|
@ -1029,35 +1036,32 @@ async def backfill_screenshots(
|
|||
select(MediaSource.page_uri, Media.md5_hash)
|
||||
.join(Media, MediaSource.md5_hash == Media.md5_hash)
|
||||
.where(Media.media_type == 'screenshot')
|
||||
.where(MediaSource.page_uri.notin_(completed_uris) if completed_uris else True)
|
||||
.distinct()
|
||||
)
|
||||
logger.info("Backfilling screenshots for ALL pages")
|
||||
|
||||
result = await session.execute(stmt)
|
||||
rows = result.fetchall()
|
||||
total = len(rows)
|
||||
logger.info(f"Found {total} screenshots to re-capture...")
|
||||
to_process = len(rows)
|
||||
skipped_count = len(completed_uris)
|
||||
total = to_process + skipped_count
|
||||
logger.info(f"Found {to_process} to process ({skipped_count} already completed)")
|
||||
|
||||
captured = 0
|
||||
skipped = 0
|
||||
failed = 0
|
||||
bytes_saved = 0
|
||||
domain_last_fetched = {} # Track last fetch time per domain
|
||||
crawl_delay = 2.0 # Default crawl delay in seconds
|
||||
|
||||
pbar = tqdm(rows, desc="Screenshots", unit="pages")
|
||||
for row in pbar:
|
||||
pbar = tqdm(total=total, initial=skipped_count, desc="Screenshots", unit="pages")
|
||||
for row in rows:
|
||||
page_uri = row.page_uri
|
||||
old_hash = row.md5_hash
|
||||
domain = urlparse(page_uri).netloc.lower()
|
||||
domain = Uri(page_uri).hostname.lower()
|
||||
|
||||
# Skip if already completed (from state file)
|
||||
if page_uri in completed_uris:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# Log current page
|
||||
logger.info(f"Capturing: {page_uri}")
|
||||
# Log current page (debug level to reduce noise)
|
||||
logger.debug(f"Capturing: {page_uri}")
|
||||
|
||||
try:
|
||||
# Enforce crawl delay (skip in fast mode)
|
||||
|
|
@ -1079,43 +1083,56 @@ async def backfill_screenshots(
|
|||
result = await capture.capture(page_uri, content_length=content_length)
|
||||
if not result:
|
||||
failed += 1
|
||||
pbar.update(1)
|
||||
continue
|
||||
|
||||
new_hash = result['md5_hash']
|
||||
new_data = result['data']
|
||||
new_ext = result.get('format', 'jpg')
|
||||
new_mime = result.get('mime_type', 'image/jpeg')
|
||||
new_size = len(new_data)
|
||||
# Normalize to list (oversized images return multiple chunks)
|
||||
chunks = result if isinstance(result, list) else [result]
|
||||
|
||||
# Get old file size for comparison
|
||||
old_path = Path(vault_path) / old_hash[:2] / f"{old_hash}.png"
|
||||
old_size = old_path.stat().st_size if old_path.exists() else 0
|
||||
|
||||
# Store new screenshot
|
||||
if not await vault.exists(new_hash):
|
||||
await vault.store(new_hash, new_data, new_ext)
|
||||
# Store all chunks
|
||||
total_new_size = 0
|
||||
for i, chunk in enumerate(chunks):
|
||||
new_hash = chunk['md5_hash']
|
||||
new_data = chunk['data']
|
||||
new_ext = chunk.get('format', 'jpg')
|
||||
new_mime = chunk.get('mime_type', 'image/jpeg')
|
||||
total_new_size += len(new_data)
|
||||
|
||||
if not await vault.exists(new_hash):
|
||||
await vault.store(new_hash, new_data, new_ext)
|
||||
|
||||
# Create symlink for each chunk
|
||||
parsed = Uri(page_uri)
|
||||
url_path = parsed.path.lstrip('/') or 'index'
|
||||
suffix = f'_{i}' if len(chunks) > 1 else ''
|
||||
new_symlink = Path(vault_path) / 'linkpeek_vault' / domain / (url_path.replace('/', '_') + f'{suffix}.{new_ext}')
|
||||
new_symlink.parent.mkdir(parents=True, exist_ok=True)
|
||||
hash_vault_path = Path(vault_path) / new_hash[:2] / f"{new_hash}.{new_ext}"
|
||||
rel_path = os.path.relpath(hash_vault_path, new_symlink.parent)
|
||||
if not new_symlink.exists():
|
||||
new_symlink.symlink_to(rel_path)
|
||||
|
||||
# Delete old PNG file FIRST (before metadata update)
|
||||
if delete_old and old_hash != new_hash and old_path.exists():
|
||||
if delete_old and old_path.exists():
|
||||
old_path.unlink()
|
||||
bytes_saved += old_size - new_size
|
||||
|
||||
# Update linkpeek symlink
|
||||
parsed = urlparse(page_uri)
|
||||
url_path = parsed.path.lstrip('/') or 'index'
|
||||
bytes_saved += old_size - total_new_size
|
||||
|
||||
# Remove old symlink
|
||||
parsed = Uri(page_uri)
|
||||
url_path = parsed.path.lstrip('/') or 'index'
|
||||
old_symlink = Path(vault_path) / 'linkpeek_vault' / domain / (url_path.replace('/', '_') + '.png')
|
||||
if old_symlink.exists():
|
||||
old_symlink.unlink()
|
||||
|
||||
# Create new symlink
|
||||
new_symlink = Path(vault_path) / 'linkpeek_vault' / domain / (url_path.replace('/', '_') + f'.{new_ext}')
|
||||
new_symlink.parent.mkdir(parents=True, exist_ok=True)
|
||||
hash_vault_path = Path(vault_path) / new_hash[:2] / f"{new_hash}.{new_ext}"
|
||||
rel_path = os.path.relpath(hash_vault_path, new_symlink.parent)
|
||||
if not new_symlink.exists():
|
||||
new_symlink.symlink_to(rel_path)
|
||||
# Use first chunk for database record
|
||||
new_hash = chunks[0]['md5_hash']
|
||||
new_ext = chunks[0].get('format', 'jpg')
|
||||
new_mime = chunks[0].get('mime_type', 'image/jpeg')
|
||||
new_size = total_new_size
|
||||
|
||||
# Update database using ORM
|
||||
# md5_hash is PRIMARY KEY, so we need to: insert new -> update refs -> delete old
|
||||
|
|
@ -1175,12 +1192,18 @@ async def backfill_screenshots(
|
|||
full_state['backfill_screenshots'] = list(completed_uris)
|
||||
state_path.write_text(json.dumps(full_state, indent=2))
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Error re-capturing {page_uri}: {e}")
|
||||
failed += 1
|
||||
pbar.update(1)
|
||||
|
||||
except Exception as e:
|
||||
# Truncate error message for cleaner logs
|
||||
err_msg = str(e).split('\n')[0][:60]
|
||||
logger.warning(f"Failed: {page_uri} - {err_msg}")
|
||||
failed += 1
|
||||
pbar.update(1)
|
||||
|
||||
pbar.close()
|
||||
await session.commit()
|
||||
logger.info(f"Backfill complete: {captured} captured, {skipped} skipped, {failed} failed")
|
||||
logger.info(f"Backfill complete: {captured} captured, {skipped_count} skipped, {failed} failed")
|
||||
if bytes_saved > 0:
|
||||
logger.info(f"Space saved: {bytes_saved / 1024 / 1024:.1f} MB")
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import hashlib
|
|||
import io
|
||||
import logging
|
||||
import shutil
|
||||
import warnings
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
|
@ -29,6 +30,9 @@ import tempfile
|
|||
|
||||
from PIL import Image
|
||||
|
||||
# Suppress PIL decompression bomb warnings for large screenshots
|
||||
warnings.filterwarnings('ignore', category=Image.DecompressionBombWarning)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Engine preference order - lightest/fastest first
|
||||
|
|
@ -258,7 +262,9 @@ class ScreenshotCapture:
|
|||
)
|
||||
|
||||
if not result.success:
|
||||
logger.warning(f"Screenshot failed: {uri} - {result.error}")
|
||||
# Extract just the first line of error, skip wkhtmltoimage verbosity
|
||||
error_msg = (result.error or 'unknown error').split('\n')[0][:80]
|
||||
logger.warning(f"Screenshot failed: {uri} - {error_msg}")
|
||||
return None
|
||||
|
||||
# Read bytes from output file
|
||||
|
|
@ -270,9 +276,40 @@ class ScreenshotCapture:
|
|||
# Convert to JPEG if configured
|
||||
if self.config.format == 'jpeg':
|
||||
img = Image.open(io.BytesIO(png_data))
|
||||
|
||||
# Convert RGBA to RGB (JPEG doesn't support alpha)
|
||||
if img.mode in ('RGBA', 'LA', 'P'):
|
||||
img = img.convert('RGB')
|
||||
|
||||
# Split oversized images (JPEG max is 65500px)
|
||||
max_dim = 65000
|
||||
if img.height > max_dim:
|
||||
logger.debug(f"Splitting oversized image: {img.height}px into chunks")
|
||||
results = []
|
||||
chunk_idx = 0
|
||||
y = 0
|
||||
while y < img.height:
|
||||
chunk_height = min(max_dim, img.height - y)
|
||||
chunk = img.crop((0, y, img.width, y + chunk_height))
|
||||
output = io.BytesIO()
|
||||
chunk.save(output, format='JPEG', quality=self.config.quality, optimize=True)
|
||||
chunk_data = output.getvalue()
|
||||
chunk_hash = hashlib.md5(chunk_data).hexdigest()
|
||||
results.append({
|
||||
'data': chunk_data,
|
||||
'md5_hash': chunk_hash,
|
||||
'mime_type': 'image/jpeg',
|
||||
'size': len(chunk_data),
|
||||
'source_uri': uri,
|
||||
'engine': self._engine_name,
|
||||
'format': 'jpg',
|
||||
'chunk': chunk_idx,
|
||||
'total_chunks': (img.height + max_dim - 1) // max_dim,
|
||||
})
|
||||
y += max_dim
|
||||
chunk_idx += 1
|
||||
return results # Return list for oversized images
|
||||
|
||||
output = io.BytesIO()
|
||||
img.save(output, format='JPEG', quality=self.config.quality, optimize=True)
|
||||
data = output.getvalue()
|
||||
|
|
|
|||
540
serp.py
540
serp.py
|
|
@ -32,6 +32,7 @@ from sqlalchemy import text
|
|||
import uvicorn
|
||||
|
||||
from database import Database
|
||||
from miniuri import Uri
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -65,6 +66,84 @@ ARCHIVE_ROOT: str = None # e.g., "example.com-20251230"
|
|||
TEMP_DB_PATH: str = None # Extracted database (SQLite needs real file)
|
||||
|
||||
|
||||
# Shared CSS for view pages
|
||||
VIEW_CSS = """
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
margin: 0; padding: 0;
|
||||
background: #0a0a0a; color: #e0e0e0;
|
||||
}
|
||||
.nav {
|
||||
background: #1a1a1a; padding: 10px 20px;
|
||||
display: flex; gap: 20px; align-items: center;
|
||||
border-bottom: 1px solid #333;
|
||||
}
|
||||
.nav a { color: #ff6b6b; text-decoration: none; }
|
||||
.nav a:hover { text-decoration: underline; }
|
||||
.nav .brand { font-weight: bold; font-size: 18px; }
|
||||
.container { padding: 20px; max-width: 1200px; margin: 0 auto; }
|
||||
h1 { color: #ff6b6b; font-size: 20px; margin: 0 0 10px 0; }
|
||||
h3 { color: #ff6b6b; font-size: 16px; margin: 30px 0 15px 0; border-bottom: 1px solid #333; padding-bottom: 8px; }
|
||||
a { color: #ff6b6b; }
|
||||
.meta { background: #1a1a1a; padding: 15px; border-radius: 8px; margin: 15px 0; }
|
||||
.meta-row { display: flex; margin: 8px 0; }
|
||||
.meta-label { width: 100px; color: #888; font-size: 13px; }
|
||||
.meta-value { flex: 1; word-break: break-all; font-size: 13px; }
|
||||
.meta-value a { color: #4ade80; }
|
||||
.hero { text-align: center; margin-bottom: 20px; }
|
||||
.hero img, .hero video { max-width: 100%; max-height: 60vh; border-radius: 8px; }
|
||||
.content-rendered {
|
||||
background: #1a1a1a; padding: 20px; border-radius: 8px;
|
||||
line-height: 1.7; font-size: 14px; color: #ccc;
|
||||
}
|
||||
.content-rendered img { max-width: 100%; height: auto; margin: 10px 0; }
|
||||
.content-rendered img.avatar {
|
||||
display: inline-block; vertical-align: middle;
|
||||
width: 40px; height: 40px; border-radius: 50%;
|
||||
margin: 0 10px 0 0; object-fit: cover;
|
||||
}
|
||||
.content-rendered img.emoji {
|
||||
display: inline; width: 20px; height: 20px;
|
||||
margin: 0 2px; vertical-align: text-bottom;
|
||||
}
|
||||
.content-rendered a { color: #ff6b6b; }
|
||||
.content-rendered pre, .content-rendered code {
|
||||
background: #252525; padding: 2px 6px;
|
||||
border-radius: 4px; font-family: monospace; font-size: 13px;
|
||||
}
|
||||
.content-rendered pre {
|
||||
padding: 15px; display: block;
|
||||
white-space: pre-wrap; overflow-x: auto;
|
||||
}
|
||||
.content-rendered blockquote {
|
||||
background: #151520; border-left: 3px solid #4a9eff;
|
||||
padding: 12px 16px; margin: 16px 0;
|
||||
border-radius: 0 6px 6px 0; color: #aaa; font-style: italic;
|
||||
}
|
||||
.content-rendered hr { border: none; border-top: 1px solid #333; margin: 24px 0; }
|
||||
.content-rendered p { margin: 0 0 16px 0; line-height: 1.7; }
|
||||
.content-rendered h1, .content-rendered h2, .content-rendered h3 {
|
||||
color: #ff6b6b; margin-top: 28px; margin-bottom: 12px;
|
||||
}
|
||||
.media-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(150px, 1fr));
|
||||
gap: 10px; margin-top: 15px;
|
||||
}
|
||||
.media-card {
|
||||
background: #1a1a1a; border-radius: 8px;
|
||||
overflow: hidden; display: block; transition: transform 0.2s;
|
||||
}
|
||||
.media-card:hover { transform: scale(1.02); }
|
||||
.media-card img, .media-card video {
|
||||
width: 100%; height: 120px;
|
||||
object-fit: contain; background: #0a0a0a;
|
||||
}
|
||||
.tag { background: #333; padding: 2px 8px; border-radius: 4px; font-size: 12px; margin-right: 5px; }
|
||||
"""
|
||||
|
||||
|
||||
def read_from_tarball(path: str) -> bytes:
|
||||
"""Read a file from the tarball. Path is relative to archive root."""
|
||||
if not TAR_FILE or not ARCHIVE_ROOT:
|
||||
|
|
@ -1231,6 +1310,10 @@ async def view_media_page(md5_hash: str):
|
|||
# Get page URI for content lookup
|
||||
page_uri = sorted_sources[0]["page_uri"] if sorted_sources else None
|
||||
|
||||
# Extract domain for AI system prompt
|
||||
source_uri = page_uri or (sorted_sources[0]["media_uri"] if sorted_sources else "unknown")
|
||||
source_domain = Uri(source_uri).hostname if source_uri else "unknown"
|
||||
|
||||
# Get page content - prefer markdown, fallback to raw HTML or text
|
||||
page_content_html = ""
|
||||
|
||||
|
|
@ -1756,6 +1839,9 @@ async def view_media_page(md5_hash: str):
|
|||
// Run after DOM ready (avatars may be empty placeholders)
|
||||
restructureForumPosts();
|
||||
</script>
|
||||
<script>
|
||||
window.UNCLOSEAI_SYSTEM_PROMPT = "This is an archived copy of {source_domain} preserved by neopig. The original site may no longer exist. You are viewing: {display_title} ({source_uri}). Help users explore the preserved discussions, media, and pages.";
|
||||
</script>
|
||||
<script src="https://uncloseai.com/uncloseai.js" type="module"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1771,6 +1857,8 @@ async def view_page(
|
|||
import html as html_module
|
||||
import re
|
||||
|
||||
source_domain = Uri(uri).hostname
|
||||
|
||||
page = await db.get_page_by_uri(uri)
|
||||
if not page:
|
||||
raise HTTPException(status_code=404, detail="Page not found")
|
||||
|
|
@ -1854,258 +1942,13 @@ async def view_page(
|
|||
<div class="media-grid">{''.join(media_cards)}</div>
|
||||
</div>'''
|
||||
|
||||
return f"""
|
||||
<!DOCTYPE html>
|
||||
return f"""<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>{html_module.escape(page_title)} - neopig</title>
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github-dark.min.css">
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
|
||||
<style>
|
||||
* {{ box-sizing: border-box; }}
|
||||
body {{
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
background: #0a0a0a;
|
||||
color: #e0e0e0;
|
||||
}}
|
||||
.nav {{
|
||||
background: #1a1a1a;
|
||||
padding: 10px 20px;
|
||||
display: flex;
|
||||
gap: 20px;
|
||||
align-items: center;
|
||||
border-bottom: 1px solid #333;
|
||||
}}
|
||||
.nav a {{ color: #ff6b6b; text-decoration: none; }}
|
||||
.nav a:hover {{ text-decoration: underline; }}
|
||||
.nav .brand {{ font-weight: bold; font-size: 18px; }}
|
||||
.container {{ padding: 20px; }}
|
||||
h1 {{ color: #ff6b6b; font-size: 22px; margin-bottom: 5px; }}
|
||||
.page-uri {{ color: #4ade80; font-size: 12px; font-family: monospace; margin-bottom: 20px; }}
|
||||
.page-uri a {{ color: #4ade80; }}
|
||||
.hero-grid {{
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 20px;
|
||||
margin-bottom: 30px;
|
||||
}}
|
||||
.hero-left {{
|
||||
background: #111;
|
||||
border-radius: 12px;
|
||||
padding: 20px;
|
||||
}}
|
||||
.hero-right {{
|
||||
background: #151515;
|
||||
border-radius: 12px;
|
||||
padding: 20px;
|
||||
}}
|
||||
.meta {{
|
||||
background: #1a1a1a;
|
||||
padding: 15px;
|
||||
border-radius: 8px;
|
||||
margin-bottom: 15px;
|
||||
}}
|
||||
.meta-row {{
|
||||
display: flex;
|
||||
margin: 8px 0;
|
||||
}}
|
||||
.meta-label {{
|
||||
width: 100px;
|
||||
color: #888;
|
||||
font-size: 13px;
|
||||
}}
|
||||
.meta-value {{
|
||||
flex: 1;
|
||||
word-break: break-all;
|
||||
font-size: 13px;
|
||||
}}
|
||||
.meta-value a {{
|
||||
color: #4ade80;
|
||||
}}
|
||||
.content-rendered {{
|
||||
background: #1a1a1a;
|
||||
padding: 20px;
|
||||
border-radius: 8px;
|
||||
line-height: 1.7;
|
||||
font-size: 14px;
|
||||
color: #ccc;
|
||||
}}
|
||||
.content-rendered img {{ max-width: 100%; height: auto; margin: 10px 0; }}
|
||||
.content-rendered img.avatar, .content-rendered img.small-img {{
|
||||
display: inline-block;
|
||||
vertical-align: middle;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
border-radius: 50%;
|
||||
margin: 0 8px 0 0;
|
||||
object-fit: cover;
|
||||
}}
|
||||
.content-rendered img.emoji {{
|
||||
display: inline;
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
margin: 0 2px;
|
||||
vertical-align: text-bottom;
|
||||
}}
|
||||
/* Forum-style post layout */
|
||||
.content-rendered .post-block {{
|
||||
display: grid;
|
||||
grid-template-columns: 48px 1fr;
|
||||
gap: 12px;
|
||||
padding: 16px 0;
|
||||
border-bottom: 1px solid #252530;
|
||||
}}
|
||||
.content-rendered .post-block:last-child {{
|
||||
border-bottom: none;
|
||||
}}
|
||||
.content-rendered .post-avatar-col {{
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
}}
|
||||
.content-rendered .post-avatar {{
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
border-radius: 50%;
|
||||
object-fit: cover;
|
||||
}}
|
||||
.content-rendered .post-avatar-placeholder {{
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
border-radius: 50%;
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: #fff;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-weight: bold;
|
||||
font-size: 18px;
|
||||
}}
|
||||
.content-rendered .post-content {{
|
||||
min-width: 0;
|
||||
}}
|
||||
.content-rendered .post-content p:first-child {{
|
||||
margin-top: 0;
|
||||
}}
|
||||
.content-rendered a {{ color: #ff6b6b; }}
|
||||
.content-rendered pre, .content-rendered code {{
|
||||
background: #252525;
|
||||
padding: 2px 6px;
|
||||
border-radius: 4px;
|
||||
font-family: monospace;
|
||||
font-size: 13px;
|
||||
}}
|
||||
.content-rendered pre {{
|
||||
padding: 15px;
|
||||
display: block;
|
||||
white-space: pre-wrap;
|
||||
overflow-x: auto;
|
||||
position: relative;
|
||||
}}
|
||||
.content-rendered pre .code-actions {{
|
||||
position: absolute;
|
||||
top: 5px;
|
||||
right: 5px;
|
||||
display: flex;
|
||||
gap: 5px;
|
||||
opacity: 0;
|
||||
transition: opacity 0.2s;
|
||||
}}
|
||||
.content-rendered pre:hover .code-actions {{ opacity: 1; }}
|
||||
.content-rendered pre .code-actions button {{
|
||||
background: #444;
|
||||
border: none;
|
||||
color: #ccc;
|
||||
padding: 4px 8px;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
font-size: 11px;
|
||||
}}
|
||||
.content-rendered pre .code-actions button:hover {{ background: #555; }}
|
||||
.content-rendered pre .code-actions button.copied {{ background: #4ade80; color: #000; }}
|
||||
.content-rendered blockquote {{
|
||||
background: #151520;
|
||||
border-left: 3px solid #4a9eff;
|
||||
padding: 12px 16px;
|
||||
margin: 16px 0;
|
||||
border-radius: 0 6px 6px 0;
|
||||
color: #aaa;
|
||||
font-style: italic;
|
||||
}}
|
||||
.content-rendered blockquote p {{
|
||||
margin: 0 0 8px 0;
|
||||
}}
|
||||
.content-rendered blockquote p:last-child {{
|
||||
margin-bottom: 0;
|
||||
}}
|
||||
/* Nested quotes */
|
||||
.content-rendered blockquote blockquote {{
|
||||
background: #1a1a25;
|
||||
border-left-color: #666;
|
||||
margin: 12px 0;
|
||||
}}
|
||||
/* Post/comment spacing */
|
||||
.content-rendered hr {{
|
||||
border: none;
|
||||
border-top: 1px solid #333;
|
||||
margin: 24px 0;
|
||||
}}
|
||||
.content-rendered p {{
|
||||
margin: 0 0 16px 0;
|
||||
line-height: 1.7;
|
||||
}}
|
||||
.content-rendered h1, .content-rendered h2, .content-rendered h3 {{
|
||||
color: #ff6b6b;
|
||||
margin-top: 28px;
|
||||
margin-bottom: 12px;
|
||||
padding-top: 16px;
|
||||
border-top: 1px solid #252530;
|
||||
}}
|
||||
.content-rendered h1:first-child, .content-rendered h2:first-child, .content-rendered h3:first-child {{
|
||||
margin-top: 0;
|
||||
padding-top: 0;
|
||||
border-top: none;
|
||||
}}
|
||||
.page-screenshot {{
|
||||
position: sticky;
|
||||
top: 20px;
|
||||
}}
|
||||
.page-screenshot img {{
|
||||
width: 100%;
|
||||
border-radius: 8px;
|
||||
border: 1px solid #333;
|
||||
}}
|
||||
.page-media {{ margin-top: 30px; }}
|
||||
.page-media h3 {{ color: #ff6b6b; font-size: 16px; border-bottom: 1px solid #333; padding-bottom: 8px; margin: 0 0 15px 0; }}
|
||||
.media-grid {{
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(150px, 1fr));
|
||||
gap: 10px;
|
||||
}}
|
||||
.media-card {{
|
||||
background: #1a1a1a;
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
display: block;
|
||||
transition: transform 0.2s;
|
||||
}}
|
||||
.media-card:hover {{
|
||||
transform: scale(1.02);
|
||||
}}
|
||||
.media-card img, .media-card video {{
|
||||
width: 100%;
|
||||
height: 120px;
|
||||
object-fit: contain;
|
||||
background: #0a0a0a;
|
||||
}}
|
||||
@media (max-width: 900px) {{
|
||||
.hero-grid {{
|
||||
grid-template-columns: 1fr;
|
||||
}}
|
||||
}}
|
||||
</style>
|
||||
<style>{VIEW_CSS}</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="nav">
|
||||
|
|
@ -2113,190 +1956,27 @@ async def view_page(
|
|||
<a href="/">Search</a>
|
||||
<a href="/live">Live</a>
|
||||
<a href="/random">Random</a>
|
||||
<a href="/crawl">Crawl</a>
|
||||
<a href="/phantom">Phantom</a>
|
||||
</div>
|
||||
<div class="container">
|
||||
<form class="search-box" action="/" method="get" style="display:flex;gap:10px;margin-bottom:20px;">
|
||||
<input type="text" name="q" placeholder="Search keywords, alt text, page content..." style="flex:1;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
||||
<select name="type" style="flex:0 0 auto;padding:12px 16px;font-size:16px;border:2px solid #333;border-radius:8px;background:#1a1a1a;color:#fff;width:auto;">
|
||||
<option value="">All types</option>
|
||||
<option value="image">Images</option>
|
||||
<option value="video">Videos</option>
|
||||
<option value="audio">Audio</option>
|
||||
</select>
|
||||
<button type="submit" style="flex:0 0 auto;padding:12px 24px;font-size:16px;background:#ff6b6b;color:#fff;border:none;border-radius:8px;cursor:pointer;">Search</button>
|
||||
</form>
|
||||
<div class="hero-grid">
|
||||
<div class="hero-left">
|
||||
{screenshot_html or '<div style="color:#666;text-align:center;padding:40px;">No screenshot available</div>'}
|
||||
<h1 style="margin-top:15px;font-size:1.3em;">{html_module.escape(page_title)}</h1>
|
||||
</div>
|
||||
<div class="hero-right">
|
||||
<div class="meta">
|
||||
<div class="meta-row"><span class="meta-label">URL:</span><span class="meta-value"><a href="{uri}" target="_blank">{uri}</a></span></div>
|
||||
<div class="meta-row"><span class="meta-label">Media:</span><span class="meta-value">{len(media_items)} items</span></div>
|
||||
</div>
|
||||
<div class="content-rendered" style="max-height:400px;overflow-y:auto;">{content_html or '<p>No content available</p>'}</div>
|
||||
</div>
|
||||
<div class="hero">
|
||||
{screenshot_html or ''}
|
||||
</div>
|
||||
<h1>{html_module.escape(page_title)}</h1>
|
||||
<div class="meta">
|
||||
<div class="meta-row"><span class="meta-label">URL:</span><span class="meta-value"><a href="{uri}" target="_blank">{uri}</a></span></div>
|
||||
<div class="meta-row"><span class="meta-label">Media:</span><span class="meta-value">{len(media_items)} items</span></div>
|
||||
</div>
|
||||
{media_grid}
|
||||
<h3>Page Content</h3>
|
||||
<div class="content-rendered">{content_html or '<p>No content available</p>'}</div>
|
||||
</div>
|
||||
<script>
|
||||
// Syntax highlighting
|
||||
hljs.highlightAll();
|
||||
|
||||
// Auto-detect small images as emojis/avatars
|
||||
document.querySelectorAll('.content-rendered img').forEach(img => {{
|
||||
const checkSize = () => {{
|
||||
const w = img.naturalWidth || img.width;
|
||||
const h = img.naturalHeight || img.height;
|
||||
if (w > 0 && h > 0) {{
|
||||
if (w <= 24 && h <= 24) {{
|
||||
img.classList.add('emoji');
|
||||
}} else if (w <= 60 && h <= 60) {{
|
||||
img.classList.add('avatar');
|
||||
}}
|
||||
}}
|
||||
}};
|
||||
if (img.complete) checkSize();
|
||||
else img.onload = checkSize;
|
||||
}});
|
||||
|
||||
// Add copy/download buttons to all code blocks
|
||||
const langExtMap = {{
|
||||
'python': 'py', 'py': 'py', 'javascript': 'js', 'js': 'js', 'typescript': 'ts', 'ts': 'ts',
|
||||
'cpp': 'cpp', 'c++': 'cpp', 'c': 'c', 'java': 'java', 'rust': 'rs', 'go': 'go',
|
||||
'ruby': 'rb', 'php': 'php', 'swift': 'swift', 'kotlin': 'kt', 'scala': 'scala',
|
||||
'html': 'html', 'css': 'css', 'scss': 'scss', 'json': 'json', 'yaml': 'yaml', 'yml': 'yml',
|
||||
'xml': 'xml', 'sql': 'sql', 'bash': 'sh', 'sh': 'sh', 'shell': 'sh', 'powershell': 'ps1',
|
||||
'markdown': 'md', 'md': 'md', 'lua': 'lua', 'perl': 'pl', 'r': 'r'
|
||||
}};
|
||||
document.querySelectorAll('.content-rendered pre').forEach((pre, idx) => {{
|
||||
const code = pre.querySelector('code') || pre;
|
||||
const text = code.textContent;
|
||||
|
||||
// Detect language from class
|
||||
let ext = 'txt';
|
||||
const classes = (code.className || '').split(/\\s+/);
|
||||
for (const cls of classes) {{
|
||||
const match = cls.match(/^(?:language-|lang-)?(.+)$/);
|
||||
if (match && langExtMap[match[1].toLowerCase()]) {{
|
||||
ext = langExtMap[match[1].toLowerCase()];
|
||||
break;
|
||||
}}
|
||||
}}
|
||||
|
||||
const actions = document.createElement('div');
|
||||
actions.className = 'code-actions';
|
||||
|
||||
const copyBtn = document.createElement('button');
|
||||
copyBtn.textContent = 'Copy';
|
||||
copyBtn.onclick = async () => {{
|
||||
await navigator.clipboard.writeText(text);
|
||||
copyBtn.textContent = 'Copied!';
|
||||
copyBtn.classList.add('copied');
|
||||
setTimeout(() => {{ copyBtn.textContent = 'Copy'; copyBtn.classList.remove('copied'); }}, 2000);
|
||||
}};
|
||||
|
||||
const dlBtn = document.createElement('button');
|
||||
dlBtn.textContent = 'Download';
|
||||
dlBtn.onclick = () => {{
|
||||
const blob = new Blob([text], {{ type: 'text/plain' }});
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = `code-${{idx + 1}}.${{ext}}`;
|
||||
a.click();
|
||||
URL.revokeObjectURL(url);
|
||||
}};
|
||||
|
||||
actions.appendChild(copyBtn);
|
||||
actions.appendChild(dlBtn);
|
||||
pre.appendChild(actions);
|
||||
}});
|
||||
// Restructure forum posts into two-column layout
|
||||
// Looks for pattern: <p> <strong>username</strong></p> followed by content until <hr>
|
||||
function restructureForumPosts() {{
|
||||
const container = document.querySelector('.content-rendered');
|
||||
if (!container) return;
|
||||
|
||||
// Find paragraphs that start with an avatar image followed by bold username
|
||||
const postStarts = [];
|
||||
container.querySelectorAll('p').forEach(p => {{
|
||||
const img = p.querySelector('img[alt="avatar"]');
|
||||
const strong = p.querySelector('strong');
|
||||
if (img && strong) {{
|
||||
postStarts.push({{ p, img, username: strong.textContent }});
|
||||
}}
|
||||
}});
|
||||
|
||||
if (postStarts.length === 0) return;
|
||||
|
||||
// For each post header, create a post-block
|
||||
postStarts.forEach(({{ p, img, username }}, idx) => {{
|
||||
if (p.closest('.post-block')) return;
|
||||
|
||||
const postBlock = document.createElement('div');
|
||||
postBlock.className = 'post-block';
|
||||
|
||||
// Avatar column - use placeholder if no src
|
||||
const avatarCol = document.createElement('div');
|
||||
avatarCol.className = 'post-avatar-col';
|
||||
|
||||
const hasAvatar = img.src && !img.src.endsWith('#') && img.getAttribute('src') !== '#';
|
||||
if (hasAvatar) {{
|
||||
const avatarImg = img.cloneNode(true);
|
||||
avatarImg.classList.add('post-avatar');
|
||||
avatarCol.appendChild(avatarImg);
|
||||
}} else {{
|
||||
// Placeholder for posts without avatar
|
||||
const placeholder = document.createElement('div');
|
||||
placeholder.className = 'post-avatar-placeholder';
|
||||
placeholder.textContent = username.charAt(0).toUpperCase();
|
||||
avatarCol.appendChild(placeholder);
|
||||
}}
|
||||
|
||||
// Content column
|
||||
const contentCol = document.createElement('div');
|
||||
contentCol.className = 'post-content';
|
||||
|
||||
// Add username header
|
||||
const header = document.createElement('div');
|
||||
header.className = 'post-header';
|
||||
header.innerHTML = `<strong>${{username}}</strong>`;
|
||||
contentCol.appendChild(header);
|
||||
|
||||
// Collect siblings until next post or HR
|
||||
let sibling = p.nextElementSibling;
|
||||
const nextP = idx < postStarts.length - 1 ? postStarts[idx + 1].p : null;
|
||||
|
||||
while (sibling && sibling !== nextP && sibling.tagName !== 'HR') {{
|
||||
contentCol.appendChild(sibling.cloneNode(true));
|
||||
const toRemove = sibling;
|
||||
sibling = sibling.nextElementSibling;
|
||||
toRemove.remove();
|
||||
}}
|
||||
|
||||
// Remove the HR separator if present
|
||||
if (sibling && sibling.tagName === 'HR') {{
|
||||
sibling.remove();
|
||||
}}
|
||||
|
||||
postBlock.appendChild(avatarCol);
|
||||
postBlock.appendChild(contentCol);
|
||||
p.parentNode.insertBefore(postBlock, p);
|
||||
p.remove();
|
||||
}});
|
||||
}}
|
||||
|
||||
// Run after DOM ready (avatars may be empty placeholders)
|
||||
restructureForumPosts();
|
||||
<script>hljs.highlightAll();</script>
|
||||
{'' if noai else f'''<script>
|
||||
window.UNCLOSEAI_SYSTEM_PROMPT = "This is an archived copy of {source_domain} preserved by neopig. The original site may no longer exist. You are viewing: {page_title} ({uri}). Help users explore the preserved discussions, media, and pages.";
|
||||
</script>
|
||||
{'' if noai else '<script src="https://uncloseai.com/uncloseai.js" type="module"></script>'}
|
||||
<script src="https://uncloseai.com/uncloseai.js" type="module"></script>'''}
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
</html>"""
|
||||
|
||||
|
||||
@app.get("/phantom/export")
|
||||
|
|
@ -2309,7 +1989,7 @@ async def phantom_export(domain: str = Query(None, description="Filter by domain
|
|||
import io
|
||||
import re
|
||||
import zipfile
|
||||
from urllib.parse import urlparse, urljoin
|
||||
from urllib.parse import urljoin
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
# Get all pages with raw_html
|
||||
|
|
@ -2335,8 +2015,8 @@ async def phantom_export(domain: str = Query(None, description="Filter by domain
|
|||
continue
|
||||
|
||||
# Parse URI to get path
|
||||
parsed = urlparse(uri)
|
||||
site_domain = parsed.netloc
|
||||
parsed = Uri(uri)
|
||||
site_domain = parsed.hostname
|
||||
path = parsed.path.strip('/') or 'index'
|
||||
if not path.endswith('.html') and '.' not in path.split('/')[-1]:
|
||||
path = f"{path}/index.html" if path else "index.html"
|
||||
|
|
@ -2412,7 +2092,7 @@ async def phantom_export(domain: str = Query(None, description="Filter by domain
|
|||
<ul>
|
||||
"""
|
||||
for page in pages[:100]:
|
||||
parsed = urlparse(page['uri'])
|
||||
parsed = Uri(page['uri'])
|
||||
path = parsed.path.strip('/') or 'index'
|
||||
if not path.endswith('.html') and '.' not in path.split('/')[-1]:
|
||||
path = f"{path}/index.html" if path else "index.html"
|
||||
|
|
@ -2733,8 +2413,8 @@ async def serve_media(md5_hash: str, download: bool = False):
|
|||
filename = f"{slug}{ext}"
|
||||
# Fallback: original filename from URL
|
||||
if not filename and row2.get("media_uri"):
|
||||
from urllib.parse import urlparse, unquote
|
||||
parsed = urlparse(row2["media_uri"])
|
||||
from urllib.parse import unquote
|
||||
parsed = Uri(row2["media_uri"])
|
||||
orig_name = Path(unquote(parsed.path)).name
|
||||
if orig_name and '.' in orig_name:
|
||||
filename = orig_name
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue