Russell Ballestrini
623213f5b6
- Removed embedded serve.py (872 lines) from archive.py - bootstrap.c extracts neopig/*.py to /tmp and runs serp.py - serp.py: tarball mode serves media directly from tar.gz - OffsetFile wrapper for reading .run files at correct offset - ArchiveDB: simple SQLite wrapper for archive search (no async deps) - Archives bundle all neopig source files for self-contained operation - --upgrade-neopig flag with progress logging
711 lines
28 KiB
Python
711 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
archive.py - Site Archiver for Sunset Sites
|
|
|
|
Thin wrapper around neopig that packages crawl results into a distributable
|
|
tar.gz archive. Uses neopig for the actual crawling.
|
|
|
|
Output structure:
|
|
{domain}-{date}/
|
|
html/ # Original HTML pages
|
|
markdown/ # Converted markdown (optional)
|
|
media/ # Images, videos, audio
|
|
screenshots/ # Page screenshots (optional)
|
|
archive.db # SQLite search database
|
|
serve.py # Embedded search server
|
|
metadata.json # Crawl metadata
|
|
|
|
Usage:
|
|
python archive.py https://discourse-urho3d.github.io/
|
|
python archive.py https://example.com --no-screenshots --no-markdown
|
|
"""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import signal
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Dict, Any, List, Optional
|
|
from urllib.parse import urlparse
|
|
|
|
import aiofiles
|
|
import aiofiles.os
|
|
from bs4 import BeautifulSoup
|
|
from tqdm import tqdm
|
|
|
|
from neopig import NeoPig, setup_logging
|
|
from async_web_fetcher import CrawlMode
|
|
from screenshot import ScreenshotConfig
|
|
|
|
# Optional markdown conversion
|
|
try:
|
|
import html2text
|
|
HAS_HTML2TEXT = True
|
|
except ImportError:
|
|
HAS_HTML2TEXT = False
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def sanitize_filename(name: str) -> str:
|
|
"""Sanitize a string for use as filename. Uses - as separator."""
|
|
name = re.sub(r'[<>:"/\\|?*.]', '-', name)
|
|
name = re.sub(r'-+', '-', name) # collapse multiple dashes
|
|
name = name.strip('- ')
|
|
return name[:200] if name else 'unnamed'
|
|
|
|
|
|
def url_to_path(url: str) -> str:
|
|
"""Convert URL to filesystem path."""
|
|
parsed = urlparse(url)
|
|
path = parsed.path.strip('/')
|
|
if not path:
|
|
return 'index.html'
|
|
if path.endswith('.html') or path.endswith('.htm'):
|
|
return path
|
|
if '.' in path.split('/')[-1]:
|
|
return path
|
|
return f"{path}/index.html"
|
|
|
|
|
|
def html_to_markdown(html: str, base_url: str = '', trim_wrapper: bool = False) -> str:
|
|
"""Convert HTML to markdown."""
|
|
if not HAS_HTML2TEXT:
|
|
return html
|
|
# Optionally strip nav/header/footer/logo before conversion
|
|
if trim_wrapper:
|
|
from neopig import trim_html_wrapper
|
|
html = trim_html_wrapper(html)
|
|
h = html2text.HTML2Text()
|
|
h.ignore_links = False
|
|
h.ignore_images = False
|
|
h.body_width = 0
|
|
h.unicode_snob = True
|
|
if base_url:
|
|
h.baseurl = base_url
|
|
return h.handle(html)
|
|
|
|
|
|
class SiteArchiver:
|
|
"""
|
|
Packages neopig crawl results into a distributable tar.gz archive.
|
|
|
|
Uses neopig for crawling, then reads from its vaults to build the archive.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
output_dir: str = '.',
|
|
include_screenshots: bool = True,
|
|
include_markdown: bool = True,
|
|
screenshot_config: ScreenshotConfig = None,
|
|
fast_mode: bool = False,
|
|
trim_wrapper: bool = False,
|
|
):
|
|
self.output_dir = Path(output_dir)
|
|
self.include_screenshots = include_screenshots
|
|
self.include_markdown = include_markdown and HAS_HTML2TEXT
|
|
self.screenshot_config = screenshot_config or ScreenshotConfig(enabled=include_screenshots)
|
|
self.fast_mode = fast_mode
|
|
self.trim_wrapper = trim_wrapper
|
|
|
|
async def archive(
|
|
self,
|
|
target_url: str,
|
|
depth: int = -1,
|
|
max_pages: int = -1,
|
|
db_path: str = None,
|
|
vault_path: str = None,
|
|
) -> Path:
|
|
"""
|
|
Archive a site using neopig and package into tar.gz.
|
|
"""
|
|
parsed = urlparse(target_url)
|
|
domain = parsed.netloc.lower()
|
|
date_str = datetime.now().strftime('%Y%m%d')
|
|
archive_name = f"{sanitize_filename(domain)}-{date_str}"
|
|
|
|
# Use standard neopig data paths - one vault, multiple domains
|
|
if not db_path:
|
|
db_path = "data/neopig.db"
|
|
if not vault_path:
|
|
vault_path = "data/vault"
|
|
|
|
# Ensure data directory exists
|
|
Path("data").mkdir(exist_ok=True)
|
|
|
|
logger.info(f"Starting archive of {target_url}")
|
|
logger.info(f"Archive name: {archive_name}")
|
|
|
|
# Create neopig instance and crawl
|
|
pig = NeoPig(
|
|
db_path=db_path,
|
|
vault_path=vault_path,
|
|
screenshot_config=self.screenshot_config,
|
|
fast_mode=self.fast_mode,
|
|
trim_wrapper=self.trim_wrapper,
|
|
)
|
|
await pig.init()
|
|
|
|
# Load previously crawled media/screenshots from DB (source of truth for resume)
|
|
crawled_media = await pig.db.get_crawled_media_uris()
|
|
crawled_screenshots = await pig.db.get_crawled_screenshot_uris()
|
|
logger.info(f"Resume state: {len(crawled_media)} media URIs, {len(crawled_screenshots)} screenshotted pages in DB")
|
|
if crawled_media:
|
|
pig.seen_media = crawled_media
|
|
if crawled_screenshots:
|
|
pig.seen_screenshots = crawled_screenshots
|
|
logger.info(f"Screenshots enabled: {pig.screenshot_config.enabled}")
|
|
|
|
# Run the crawl
|
|
stats = await pig.crawl(
|
|
target_uri=target_url,
|
|
mode=CrawlMode.ALL,
|
|
depth=depth,
|
|
max_pages=max_pages,
|
|
download_media=True,
|
|
)
|
|
|
|
# Now package the results
|
|
logger.info("Packaging archive (streaming mode)...")
|
|
|
|
tar_path = self.output_dir / f"{archive_name}.tar.gz"
|
|
html_vault_path = Path(vault_path) / 'html_vault' / domain
|
|
media_vault_path = Path(vault_path) / 'media_vault' / domain
|
|
linkpeek_vault_path = Path(vault_path) / 'linkpeek_vault' / domain
|
|
|
|
# Get URL-to-hash mapping for rewriting external URLs to local copies
|
|
logger.info("Loading media URL mappings...")
|
|
url_to_hash = await pig.db.get_all_media_uri_mappings()
|
|
logger.info(f"Loaded {len(url_to_hash)} URL mappings for rewriting")
|
|
|
|
def rewrite_urls(html_content: str) -> str:
|
|
"""Rewrite external image/media URLs to local archive paths."""
|
|
import re
|
|
def replace_url(match):
|
|
url = match.group(1)
|
|
if url in url_to_hash:
|
|
md5 = url_to_hash[url]
|
|
# Get extension from original URL
|
|
ext = Path(url.split('?')[0]).suffix or '.bin'
|
|
return match.group(0).replace(url, f'../media/{md5}{ext}')
|
|
return match.group(0)
|
|
|
|
# Replace src="url" and href="url" patterns
|
|
html_content = re.sub(r'src=["\']([^"\']+)["\']', replace_url, html_content)
|
|
html_content = re.sub(r'href=["\']([^"\']+\.(jpg|jpeg|png|gif|webp|mp4|webm|svg))["\']',
|
|
replace_url, html_content, flags=re.IGNORECASE)
|
|
return html_content
|
|
|
|
# Step 1: Scan HTML files to build sitemap and search index (no copying)
|
|
logger.info("Building search index...")
|
|
sitemap = []
|
|
html_contents = {}
|
|
|
|
if html_vault_path.exists():
|
|
for html_file in html_vault_path.rglob('*.html'):
|
|
rel_path = html_file.relative_to(html_vault_path)
|
|
try:
|
|
content = html_file.read_text(encoding='utf-8', errors='replace')
|
|
# Rewrite external URLs to local copies
|
|
content = rewrite_urls(content)
|
|
title = self._extract_title(content) or str(rel_path)
|
|
sitemap.append({'path': f'html/{rel_path}', 'title': title})
|
|
html_contents[str(rel_path)] = content
|
|
except Exception:
|
|
pass
|
|
|
|
# Step 2: Create generated files in small temp dir
|
|
local_tmpdir = self.output_dir / '.tmp'
|
|
local_tmpdir.mkdir(parents=True, exist_ok=True)
|
|
|
|
with tempfile.TemporaryDirectory(dir=local_tmpdir) as tmpdir:
|
|
tmpdir_path = Path(tmpdir)
|
|
|
|
# Create search database
|
|
await self._create_search_database_streaming(tmpdir_path, sitemap, domain, html_contents)
|
|
|
|
# Write metadata
|
|
metadata = {
|
|
'domain': domain,
|
|
'target_url': target_url,
|
|
'created': datetime.now(timezone.utc).isoformat(),
|
|
'stats': stats,
|
|
'include_screenshots': self.include_screenshots,
|
|
'include_markdown': self.include_markdown,
|
|
}
|
|
(tmpdir_path / 'metadata.json').write_text(json.dumps(metadata, indent=2))
|
|
|
|
# Copy state file
|
|
state_domain = domain.replace('.', '-').replace(':', '-')
|
|
state_file = Path("data") / f"crawl-state-{state_domain}.json"
|
|
if state_file.exists():
|
|
shutil.copy(state_file, tmpdir_path / 'crawl_state.json')
|
|
logger.info(f"Included crawl state for future delta crawls")
|
|
|
|
# Write index.html
|
|
self._write_index_html(tmpdir_path, sitemap, domain)
|
|
|
|
# Step 3: Stream everything to tar.gz in one pass
|
|
logger.info("Streaming to archive...")
|
|
|
|
def stream_to_tar():
|
|
with tarfile.open(tar_path, 'w:gz') as tar:
|
|
# Add generated files first (from temp)
|
|
for f in tmpdir_path.iterdir():
|
|
tar.add(f, arcname=f"{archive_name}/{f.name}")
|
|
|
|
# Stream HTML files (with rewritten URLs)
|
|
for rel_path_str, content in html_contents.items():
|
|
try:
|
|
html_bytes = content.encode('utf-8')
|
|
arcname = f"{archive_name}/html/{rel_path_str}"
|
|
info = tarfile.TarInfo(name=arcname)
|
|
info.size = len(html_bytes)
|
|
tar.addfile(info, io.BytesIO(html_bytes))
|
|
|
|
# Generate markdown on the fly
|
|
if self.include_markdown:
|
|
md_content = html_to_markdown(content, trim_wrapper=self.trim_wrapper)
|
|
md_bytes = md_content.encode('utf-8')
|
|
md_rel = rel_path_str.replace('.html', '.md')
|
|
md_arcname = f"{archive_name}/markdown/{md_rel}"
|
|
md_info = tarfile.TarInfo(name=md_arcname)
|
|
md_info.size = len(md_bytes)
|
|
tar.addfile(md_info, io.BytesIO(md_bytes))
|
|
except Exception as e:
|
|
logger.debug(f"Error adding HTML {rel_path_str}: {e}")
|
|
|
|
# Stream media files by hash (matching rewritten URLs)
|
|
# Build hash -> file path mapping from hash vault
|
|
hash_vault = Path(vault_path)
|
|
added_hashes = set()
|
|
for url, md5 in url_to_hash.items():
|
|
if md5 in added_hashes:
|
|
continue
|
|
# Find file in hash vault: vault/xx/hash.ext
|
|
bucket = md5[:2]
|
|
bucket_dir = hash_vault / bucket
|
|
if bucket_dir.exists():
|
|
for f in bucket_dir.iterdir():
|
|
if f.stem == md5:
|
|
try:
|
|
arcname = f"{archive_name}/media/{f.name}"
|
|
tar.add(f, arcname=arcname)
|
|
added_hashes.add(md5)
|
|
except Exception as e:
|
|
logger.debug(f"Error adding media {f}: {e}")
|
|
break
|
|
|
|
# Stream screenshots (follow symlinks)
|
|
if self.include_screenshots and linkpeek_vault_path.exists():
|
|
for screenshot_file in linkpeek_vault_path.rglob('*.png'):
|
|
try:
|
|
rel_path = screenshot_file.relative_to(linkpeek_vault_path)
|
|
arcname = f"{archive_name}/screenshots/{rel_path}"
|
|
if screenshot_file.is_symlink():
|
|
target = screenshot_file.resolve()
|
|
if target.exists():
|
|
tar.add(target, arcname=arcname)
|
|
else:
|
|
tar.add(screenshot_file, arcname=arcname)
|
|
except Exception as e:
|
|
logger.debug(f"Error adding screenshot {screenshot_file}: {e}")
|
|
|
|
# Bundle neopig source files for self-contained crawling
|
|
neopig_src = Path(__file__).parent
|
|
neopig_files = [
|
|
'neopig.py', 'database.py', 'async_web_fetcher.py',
|
|
'storage.py', 'domain_vault.py', 'screenshot.py',
|
|
'html2md.py', 'serp.py', 'filevault.py', 'async_filevault.py',
|
|
]
|
|
for pyfile in neopig_files:
|
|
src_path = neopig_src / pyfile
|
|
if src_path.exists():
|
|
tar.add(src_path, arcname=f"{archive_name}/neopig/{pyfile}")
|
|
|
|
# Add requirements.txt for neopig dependencies
|
|
req_path = neopig_src / 'requirements.txt'
|
|
if req_path.exists():
|
|
tar.add(req_path, arcname=f"{archive_name}/requirements.txt")
|
|
|
|
await asyncio.to_thread(stream_to_tar)
|
|
|
|
# Clear html_contents to free memory
|
|
html_contents.clear()
|
|
|
|
final_size = tar_path.stat().st_size
|
|
logger.info(f"Archive complete: {tar_path} ({final_size / 1024 / 1024:.1f} MB)")
|
|
|
|
return tar_path
|
|
|
|
async def _create_search_database_streaming(self, tmpdir: Path, sitemap: list, domain: str, html_contents: dict):
|
|
"""Create FTS5 search database from collected html contents."""
|
|
import sqlite3
|
|
|
|
db_path = tmpdir / 'archive.db'
|
|
|
|
def create_db():
|
|
conn = sqlite3.connect(db_path)
|
|
c = conn.cursor()
|
|
|
|
c.execute('''
|
|
CREATE TABLE pages (
|
|
id INTEGER PRIMARY KEY,
|
|
path TEXT UNIQUE,
|
|
title TEXT,
|
|
content TEXT
|
|
)
|
|
''')
|
|
c.execute('''
|
|
CREATE VIRTUAL TABLE pages_fts USING fts5(
|
|
title, content, path,
|
|
content='pages',
|
|
content_rowid='id'
|
|
)
|
|
''')
|
|
|
|
for item in sitemap:
|
|
html_path = item['path'].replace('html/', '', 1)
|
|
content = html_contents.get(html_path, '')
|
|
if content:
|
|
try:
|
|
soup = BeautifulSoup(content, 'html.parser')
|
|
for tag in soup(['script', 'style', 'nav', 'header', 'footer']):
|
|
tag.decompose()
|
|
text = soup.get_text(separator=' ', strip=True)[:50000]
|
|
except Exception:
|
|
text = ''
|
|
else:
|
|
text = ''
|
|
|
|
try:
|
|
c.execute('INSERT INTO pages (path, title, content) VALUES (?, ?, ?)',
|
|
(item['path'], item['title'], text))
|
|
except Exception:
|
|
pass
|
|
|
|
c.execute('''
|
|
INSERT INTO pages_fts(rowid, title, content, path)
|
|
SELECT id, title, content, path FROM pages
|
|
''')
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
await asyncio.to_thread(create_db)
|
|
|
|
def _write_index_html(self, tmpdir: Path, sitemap: list, domain: str):
|
|
"""Write index.html with sitemap."""
|
|
html = f'''<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>{domain} Archive</title>
|
|
<style>
|
|
body {{ font-family: sans-serif; max-width: 800px; margin: 40px auto; padding: 0 20px; }}
|
|
h1 {{ color: #333; }}
|
|
ul {{ list-style: none; padding: 0; }}
|
|
li {{ padding: 8px 0; border-bottom: 1px solid #eee; }}
|
|
a {{ color: #0066cc; text-decoration: none; }}
|
|
a:hover {{ text-decoration: underline; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>{domain} Archive</h1>
|
|
<p>{len(sitemap)} pages archived</p>
|
|
<ul>
|
|
'''
|
|
for item in sitemap[:1000]: # Limit to 1000 in index
|
|
html += f' <li><a href="{item["path"]}">{item["title"]}</a></li>\n'
|
|
if len(sitemap) > 1000:
|
|
html += f' <li>... and {len(sitemap) - 1000} more pages</li>\n'
|
|
html += ''' </ul>
|
|
</body>
|
|
</html>'''
|
|
(tmpdir / 'index.html').write_text(html)
|
|
|
|
def _extract_title(self, html: str) -> Optional[str]:
|
|
"""Extract title from HTML."""
|
|
try:
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
if soup.title and soup.title.string:
|
|
return soup.title.string.strip()
|
|
h1 = soup.find('h1')
|
|
if h1:
|
|
return h1.get_text().strip()
|
|
except Exception:
|
|
pass
|
|
return None
|
|
|
|
def _extract_text(self, html: str) -> str:
|
|
"""Extract readable text from HTML for search indexing."""
|
|
try:
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
for tag in soup(['script', 'style', 'nav', 'footer', 'header']):
|
|
tag.decompose()
|
|
text = soup.get_text(separator=' ', strip=True)
|
|
text = re.sub(r'\s+', ' ', text)
|
|
return text[:50000]
|
|
except Exception:
|
|
return ''
|
|
|
|
async def _create_search_database(
|
|
self,
|
|
archive_root: Path,
|
|
sitemap: List[Dict[str, str]],
|
|
domain: str,
|
|
html_dir: Path,
|
|
):
|
|
"""Create SQLite database with searchable page content."""
|
|
db_path = archive_root / 'archive.db'
|
|
|
|
def create_db():
|
|
conn = sqlite3.connect(db_path)
|
|
c = conn.cursor()
|
|
|
|
c.execute('''
|
|
CREATE TABLE pages (
|
|
id INTEGER PRIMARY KEY,
|
|
url TEXT NOT NULL,
|
|
path TEXT NOT NULL,
|
|
title TEXT,
|
|
content TEXT
|
|
)
|
|
''')
|
|
|
|
c.execute('''
|
|
CREATE VIRTUAL TABLE pages_fts USING fts5(
|
|
title, content, url, path,
|
|
content='pages',
|
|
content_rowid='id'
|
|
)
|
|
''')
|
|
|
|
c.execute('''
|
|
CREATE TRIGGER pages_ai AFTER INSERT ON pages BEGIN
|
|
INSERT INTO pages_fts(rowid, title, content, url, path)
|
|
VALUES (new.id, new.title, new.content, new.url, new.path);
|
|
END
|
|
''')
|
|
|
|
for item in sitemap:
|
|
path = item['path']
|
|
title = item['title']
|
|
|
|
html_path = archive_root / path
|
|
if html_path.exists():
|
|
html_content = html_path.read_text(encoding='utf-8', errors='replace')
|
|
text_content = self._extract_text(html_content)
|
|
else:
|
|
text_content = ''
|
|
|
|
c.execute(
|
|
'INSERT INTO pages (url, path, title, content) VALUES (?, ?, ?, ?)',
|
|
(path, path, title, text_content)
|
|
)
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
await asyncio.to_thread(create_db)
|
|
logger.info(f"Created search database: {db_path}")
|
|
|
|
|
|
# NOTE: Embedded serve.py removed - archives now use serp.py from bundled neopig/
|
|
# The bootstrap.c extracts neopig/*.py and runs serp.py with the tarball as argument.
|
|
# See: make run TARBALL=archive.tar.gz
|
|
|
|
|
|
def upgrade_neopig_in_archive(archive_path: Path) -> Path:
|
|
"""Replace neopig/ directory in existing archive with current source."""
|
|
import shutil
|
|
|
|
if not archive_path.exists():
|
|
raise FileNotFoundError(f"Archive not found: {archive_path}")
|
|
|
|
neopig_src = Path(__file__).parent
|
|
neopig_files = [
|
|
'neopig.py', 'database.py', 'async_web_fetcher.py',
|
|
'storage.py', 'domain_vault.py', 'screenshot.py',
|
|
'html2md.py', 'serp.py', 'filevault.py', 'async_filevault.py',
|
|
]
|
|
|
|
# Create new archive with updated neopig
|
|
output_path = archive_path.with_suffix('.upgraded.tar.gz')
|
|
|
|
logger.info(f"Reading archive: {archive_path}")
|
|
with tarfile.open(archive_path, 'r:gz') as old_tar:
|
|
# Get members list (this reads the whole index)
|
|
members = old_tar.getmembers()
|
|
total = len(members)
|
|
archive_name = members[0].name.split('/')[0]
|
|
logger.info(f"Found {total} members in {archive_name}")
|
|
|
|
with tarfile.open(output_path, 'w:gz') as new_tar:
|
|
# Copy all members except neopig/ and requirements.txt
|
|
copied = 0
|
|
skipped = 0
|
|
for i, member in enumerate(members):
|
|
parts = member.name.split('/')
|
|
if len(parts) > 1 and parts[1] == 'neopig':
|
|
skipped += 1
|
|
continue # Skip old neopig files
|
|
if len(parts) > 1 and parts[1] == 'requirements.txt':
|
|
skipped += 1
|
|
continue # Skip old requirements
|
|
if member.isfile():
|
|
f = old_tar.extractfile(member)
|
|
if f:
|
|
new_tar.addfile(member, f)
|
|
copied += 1
|
|
else:
|
|
new_tar.addfile(member)
|
|
copied += 1
|
|
|
|
if (i + 1) % 1000 == 0:
|
|
logger.info(f" Progress: {i+1}/{total} ({copied} copied, {skipped} skipped)")
|
|
|
|
logger.info(f"Copied {copied} members, skipped {skipped}")
|
|
|
|
# Add new neopig files
|
|
logger.info("Adding neopig source files...")
|
|
for pyfile in neopig_files:
|
|
src_path = neopig_src / pyfile
|
|
if src_path.exists():
|
|
new_tar.add(src_path, arcname=f"{archive_name}/neopig/{pyfile}")
|
|
logger.info(f" Added: neopig/{pyfile}")
|
|
|
|
# Add requirements.txt
|
|
req_path = neopig_src / 'requirements.txt'
|
|
if req_path.exists():
|
|
new_tar.add(req_path, arcname=f"{archive_name}/requirements.txt")
|
|
logger.info(" Added: requirements.txt")
|
|
|
|
# Replace original with upgraded
|
|
logger.info(f"Replacing original archive...")
|
|
shutil.move(output_path, archive_path)
|
|
return archive_path
|
|
|
|
|
|
async def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Archive a website for preservation (uses neopig for crawling)",
|
|
epilog="Example: python archive.py https://discourse-urho3d.github.io/"
|
|
)
|
|
|
|
parser.add_argument("url", nargs='?', help="URL of the site to archive")
|
|
parser.add_argument("-o", "--output", default=".", help="Output directory for tar.gz")
|
|
parser.add_argument("-d", "--depth", type=int, default=-1, help="Crawl depth (-1 = unlimited)")
|
|
parser.add_argument("-p", "--max-pages", type=int, default=-1, help="Max pages (-1 = unlimited)")
|
|
parser.add_argument("--no-screenshots", action="store_true", help="Disable screenshots")
|
|
parser.add_argument("--no-markdown", action="store_true", help="Disable markdown conversion")
|
|
parser.add_argument("--screenshot-width", type=int, default=1280, help="Screenshot width")
|
|
parser.add_argument("--screenshot-height", type=int, default=1024, help="Screenshot height")
|
|
parser.add_argument("--screenshot-engine", type=str, default=None, help="Screenshot engine")
|
|
parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
|
|
parser.add_argument("--fresh", action="store_true", help="Start fresh, ignore resume state")
|
|
parser.add_argument("--fast", action="store_true", help="Fast mode: no crawl delay (for sites without robots.txt)")
|
|
parser.add_argument("--serve", action="store_true", help="Start SERP server to watch crawl live")
|
|
parser.add_argument("--port", type=int, default=8000, help="Port for SERP server (default: 8000)")
|
|
parser.add_argument("--backfill-markdown", metavar="DOMAIN", help="Re-process stored HTML for DOMAIN to regenerate markdown with absolute URLs")
|
|
parser.add_argument("--trim-wrapper", action="store_true", help="With --backfill-markdown: strip nav/header/footer/logo before conversion")
|
|
parser.add_argument("--db", default="data/neopig.db", help="Database path (for --backfill-markdown)")
|
|
parser.add_argument("--upgrade-neopig", metavar="TARBALL", help="Upgrade neopig inside an existing archive")
|
|
|
|
args = parser.parse_args()
|
|
|
|
setup_logging(level=logging.DEBUG if args.verbose else logging.INFO)
|
|
|
|
# Handle --upgrade-neopig
|
|
if args.upgrade_neopig:
|
|
archive_path = Path(args.upgrade_neopig)
|
|
logger.info(f"Upgrading neopig in {archive_path}...")
|
|
upgrade_neopig_in_archive(archive_path)
|
|
logger.info(f"Done! Archive updated: {archive_path}")
|
|
return
|
|
|
|
# Handle --backfill-markdown
|
|
if args.backfill_markdown:
|
|
from neopig import backfill_markdown
|
|
await backfill_markdown(args.db, domain_filter=args.backfill_markdown, trim_wrapper=args.trim_wrapper)
|
|
return
|
|
|
|
# URL required for archiving
|
|
if not args.url:
|
|
parser.error("URL required (use --upgrade-neopig or --backfill-markdown for other operations)")
|
|
|
|
if args.no_markdown and not HAS_HTML2TEXT:
|
|
pass
|
|
elif not HAS_HTML2TEXT:
|
|
logger.warning("html2text not installed - markdown disabled")
|
|
|
|
screenshot_config = ScreenshotConfig(
|
|
enabled=not args.no_screenshots,
|
|
width=args.screenshot_width,
|
|
height=args.screenshot_height,
|
|
engine=args.screenshot_engine,
|
|
full_page=True, # Archive captures full page by default
|
|
)
|
|
|
|
archiver = SiteArchiver(
|
|
output_dir=args.output,
|
|
include_screenshots=not args.no_screenshots,
|
|
include_markdown=not args.no_markdown,
|
|
screenshot_config=screenshot_config,
|
|
fast_mode=args.fast,
|
|
trim_wrapper=args.trim_wrapper,
|
|
)
|
|
|
|
# Start SERP server if requested
|
|
serp_process = None
|
|
if args.serve:
|
|
serp_script = Path(__file__).parent / 'serp.py'
|
|
if serp_script.exists():
|
|
serp_cmd = [
|
|
sys.executable, str(serp_script),
|
|
'--port', str(args.port),
|
|
'--db', 'data/neopig.db',
|
|
'--vault', 'data/vault',
|
|
]
|
|
serp_process = subprocess.Popen(
|
|
serp_cmd,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
logger.info(f"SERP server started at http://localhost:{args.port}")
|
|
logger.info("Watch the crawl live - pages appear as they're indexed!")
|
|
else:
|
|
logger.warning("serp.py not found - --serve disabled")
|
|
|
|
try:
|
|
archive_path = await archiver.archive(
|
|
target_url=args.url,
|
|
depth=args.depth,
|
|
max_pages=args.max_pages,
|
|
)
|
|
|
|
print(f"\nArchive created: {archive_path}")
|
|
print(f"Extract with: tar -xzf {archive_path.name}")
|
|
finally:
|
|
# Cleanup SERP server
|
|
if serp_process:
|
|
logger.info("Stopping SERP server...")
|
|
serp_process.terminate()
|
|
try:
|
|
serp_process.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
serp_process.kill()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|