Russell Ballestrini
6193ce2724
- Add _initialized check to finish_crawl() and get_stats() in all 3 vault classes - Prevents FileNotFoundError when git commands run on non-existent directories - Update test_crawl.py to use pytest tmp_path fixture - Add network marker to skip functional tests in CI with '-m "not network"'
812 lines
28 KiB
Python
812 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
# Side quest 12/21: Three vaults, three truths, one domain.
|
|
"""
|
|
domain_vault.py - Triple filevault system for web archival
|
|
|
|
Three separate 9-layer deep filevaults, each with git repos per domain.
|
|
Directory names are salted hashes (NEOPIG_VAULT_SALT env var) for privacy.
|
|
The same domain + salt produces identical hashes across all three vaults.
|
|
|
|
1. HTML Vault (html_vault/):
|
|
html_vault/{9-layers}/{salted_hash}/
|
|
.git/ # Git repository (SSH cloneable)
|
|
{url_path}/index.html # Rewritten HTML with neopig media paths
|
|
{url_path}/index.html.og # Original HTML with original URIs
|
|
crawl_log.json # Crawl history
|
|
|
|
2. Media Vault (media_vault/):
|
|
media_vault/{9-layers}/{salted_hash}/
|
|
.git/ # Git repository with LFS (SSH cloneable)
|
|
{url_path}/image.png # Media files mirroring original paths
|
|
index.json # Media index
|
|
|
|
3. Linkpeek Vault (linkpeek_vault/):
|
|
linkpeek_vault/{9-layers}/{salted_hash}/
|
|
.git/ # Git repository with LFS (SSH cloneable)
|
|
{url_path}/index.png # Page screenshots mirroring URL paths
|
|
index.json # Screenshot index
|
|
|
|
Environment Variables:
|
|
NEOPIG_VAULT_SALT: Secret salt for domain hashing (required for privacy)
|
|
|
|
Features:
|
|
- 9-layer deep hash paths for filesystem distribution
|
|
- Salted hash directory names (domain name not exposed)
|
|
- Git versioning per domain (all vaults)
|
|
- Git LFS for media and linkpeek vaults (large files)
|
|
- SSH cloneable repos
|
|
- Idempotent: only commits when content changes
|
|
- .html.og preserves original, .html has rewritten media URIs
|
|
- Tree structure mirrors original domain URL paths
|
|
"""
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Dict, Any, Optional, List, Tuple
|
|
from urllib.parse import urljoin
|
|
from miniuri import Uri
|
|
import aiofiles
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def get_vault_salt() -> str:
|
|
"""Get the vault salt from environment variable."""
|
|
return os.environ.get('NEOPIG_VAULT_SALT', '')
|
|
|
|
|
|
def domain_hash(domain: str, salted: bool = False) -> str:
|
|
"""
|
|
Generate MD5 hash of domain.
|
|
|
|
Args:
|
|
domain: The domain name
|
|
salted: If True, include NEOPIG_VAULT_SALT for directory naming
|
|
"""
|
|
data = domain.lower()
|
|
if salted:
|
|
salt = get_vault_salt()
|
|
data = f"{salt}:{data}"
|
|
return hashlib.md5(data.encode()).hexdigest()
|
|
|
|
|
|
def content_hash(content: bytes) -> str:
|
|
"""Generate MD5 hash of content for deduplication."""
|
|
return hashlib.md5(content).hexdigest()
|
|
|
|
|
|
def get_filevault_path(base_path: Path, domain: str) -> Path:
|
|
"""
|
|
Generate 9-layer deep path for filevault.
|
|
|
|
Uses salted hash for both path layers AND final directory name.
|
|
The same domain + salt produces the same hash across all three vaults.
|
|
|
|
Example: example.com -> base/ab/cd/ef/12/34/56/78/9a/bc/abcdef1234567890abcdef1234567890/
|
|
"""
|
|
h = domain_hash(domain, salted=True)
|
|
# 9 layers of 2-char directories
|
|
layers = [h[i:i+2] for i in range(0, 18, 2)] # 9 pairs (18 chars)
|
|
path = base_path
|
|
for layer in layers:
|
|
path = path / layer
|
|
# Final directory is the full salted hash, not the domain name
|
|
return path / h
|
|
|
|
|
|
def url_to_filepath(url: str) -> str:
|
|
"""
|
|
Convert URL to filesystem path that mirrors the original structure.
|
|
|
|
Examples:
|
|
https://example.com/ -> index.html
|
|
https://example.com/about -> about/index.html
|
|
https://example.com/blog/post.html -> blog/post.html
|
|
https://example.com/images/logo.png -> images/logo.png
|
|
"""
|
|
parsed = Uri(url)
|
|
path = (parsed.path or '').strip('/')
|
|
|
|
if not path:
|
|
return 'index.html'
|
|
|
|
# Check if path has an extension
|
|
if '.' in path.split('/')[-1]:
|
|
return path
|
|
else:
|
|
# Directory-style URL, add index.html
|
|
return f"{path}/index.html"
|
|
|
|
|
|
def extract_media_urls(html: str, base_url: str) -> List[Tuple[str, str]]:
|
|
"""
|
|
Extract media URLs from HTML.
|
|
|
|
Returns list of (absolute_url, original_reference) tuples.
|
|
"""
|
|
media_patterns = [
|
|
# Images
|
|
r'<img[^>]+src=["\']([^"\']+)["\']',
|
|
r'<source[^>]+src=["\']([^"\']+)["\']',
|
|
# CSS backgrounds
|
|
r'url\(["\']?([^"\')\s]+)["\']?\)',
|
|
# Video/Audio
|
|
r'<video[^>]+src=["\']([^"\']+)["\']',
|
|
r'<audio[^>]+src=["\']([^"\']+)["\']',
|
|
# Links to media
|
|
r'<link[^>]+href=["\']([^"\']+\.(?:css|ico|png|jpg|jpeg|gif|svg|woff2?|ttf|eot))["\']',
|
|
]
|
|
|
|
found = []
|
|
for pattern in media_patterns:
|
|
for match in re.finditer(pattern, html, re.IGNORECASE):
|
|
original = match.group(1)
|
|
# Skip data URIs and anchors
|
|
if original.startswith('data:') or original.startswith('#'):
|
|
continue
|
|
absolute = urljoin(base_url, original)
|
|
found.append((absolute, original))
|
|
|
|
return found
|
|
|
|
|
|
class GitRepo:
|
|
"""Async git operations helper."""
|
|
|
|
def __init__(self, path: Path, use_lfs: bool = False):
|
|
self.path = path
|
|
self.use_lfs = use_lfs
|
|
|
|
async def init(self) -> bool:
|
|
"""Initialize git repo. Returns True if newly created."""
|
|
git_path = self.path / '.git'
|
|
if await asyncio.to_thread(git_path.exists):
|
|
return False
|
|
|
|
await asyncio.to_thread(self.path.mkdir, parents=True, exist_ok=True)
|
|
|
|
await self._run('git', 'init')
|
|
await self._run('git', 'config', 'user.email', 'neopig@localhost')
|
|
await self._run('git', 'config', 'user.name', 'neopig')
|
|
|
|
if self.use_lfs:
|
|
await self._init_lfs()
|
|
|
|
logger.info(f"Initialized git repo at {self.path}")
|
|
return True
|
|
|
|
async def _init_lfs(self):
|
|
"""Initialize git LFS for media files."""
|
|
await self._run('git', 'lfs', 'install', '--local')
|
|
# Track common media extensions
|
|
extensions = [
|
|
'*.png', '*.jpg', '*.jpeg', '*.gif', '*.webp', '*.svg',
|
|
'*.mp4', '*.webm', '*.mov', '*.avi',
|
|
'*.mp3', '*.wav', '*.ogg', '*.flac',
|
|
'*.pdf', '*.zip', '*.tar', '*.gz',
|
|
'*.woff', '*.woff2', '*.ttf', '*.eot',
|
|
]
|
|
for ext in extensions:
|
|
await self._run('git', 'lfs', 'track', ext)
|
|
await self._run('git', 'add', '.gitattributes')
|
|
logger.info(f"Initialized git LFS at {self.path}")
|
|
|
|
async def _run(self, *args) -> Tuple[bytes, bytes]:
|
|
"""Run git command."""
|
|
proc = await asyncio.create_subprocess_exec(
|
|
*args,
|
|
cwd=str(self.path),
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
return await proc.communicate()
|
|
|
|
async def has_changes(self) -> bool:
|
|
"""Check if there are uncommitted changes."""
|
|
stdout, _ = await self._run('git', 'status', '--porcelain')
|
|
return len(stdout.strip()) > 0
|
|
|
|
async def add_all(self):
|
|
"""Stage all changes."""
|
|
await self._run('git', 'add', '-A')
|
|
|
|
async def commit(self, message: str) -> Optional[str]:
|
|
"""Commit if changes exist. Returns commit hash or None."""
|
|
await self.add_all()
|
|
if not await self.has_changes():
|
|
return None
|
|
|
|
stdout, stderr = await self._run('git', 'commit', '-m', message)
|
|
stdout, _ = await self._run('git', 'rev-parse', 'HEAD')
|
|
commit_hash = stdout.decode().strip()
|
|
logger.info(f"Committed {commit_hash[:8]}: {message}")
|
|
return commit_hash
|
|
|
|
async def log(self, limit: int = 10) -> List[Dict[str, str]]:
|
|
"""Get recent commit history."""
|
|
stdout, _ = await self._run('git', 'log', f'-{limit}', '--format=%H|%s|%ci')
|
|
commits = []
|
|
for line in stdout.decode().strip().split('\n'):
|
|
if '|' in line:
|
|
parts = line.split('|', 2)
|
|
if len(parts) == 3:
|
|
commits.append({
|
|
'hash': parts[0],
|
|
'message': parts[1],
|
|
'date': parts[2],
|
|
})
|
|
return commits
|
|
|
|
|
|
class DomainHtmlVault:
|
|
"""
|
|
Git-tracked HTML vault for a single domain.
|
|
|
|
Stores pages in a tree structure mirroring the original domain:
|
|
- {path}/index.html.og = Original HTML
|
|
- {path}/index.html = Rewritten with neopig media paths
|
|
"""
|
|
|
|
def __init__(self, vault_base: str, domain: str, media_base_url: str = '/media'):
|
|
self.domain = domain.lower()
|
|
self.base_path = Path(vault_base)
|
|
self.path = get_filevault_path(self.base_path, self.domain)
|
|
self.crawl_log_path = self.path / 'crawl_log.json'
|
|
self.media_base_url = media_base_url.rstrip('/')
|
|
self._git = GitRepo(self.path, use_lfs=False)
|
|
self._initialized = False
|
|
|
|
async def init(self) -> bool:
|
|
"""Initialize vault. Returns True if newly created."""
|
|
is_new = await self._git.init()
|
|
|
|
if not await asyncio.to_thread(self.crawl_log_path.exists):
|
|
await self._save_crawl_log({'crawls': []})
|
|
is_new = True
|
|
|
|
self._initialized = True
|
|
return is_new
|
|
|
|
async def _load_crawl_log(self) -> Dict[str, Any]:
|
|
if not await asyncio.to_thread(self.crawl_log_path.exists):
|
|
return {'crawls': []}
|
|
async with aiofiles.open(self.crawl_log_path, 'r') as f:
|
|
return json.loads(await f.read())
|
|
|
|
async def _save_crawl_log(self, log: Dict[str, Any]):
|
|
async with aiofiles.open(self.crawl_log_path, 'w') as f:
|
|
await f.write(json.dumps(log, indent=2))
|
|
|
|
def get_media_vault_url(self, media_url: str) -> str:
|
|
"""
|
|
Convert original media URL to neopig media vault path.
|
|
|
|
Example: https://example.com/images/logo.png
|
|
-> /media/{9-layers}/{salted_hash}/images/logo.png
|
|
"""
|
|
parsed = Uri(media_url)
|
|
media_domain = parsed.hostname.lower()
|
|
media_path = parsed.path.strip('/')
|
|
|
|
# Get salted hash for media domain (same across all vaults)
|
|
h = domain_hash(media_domain, salted=True)
|
|
layers = '/'.join([h[i:i+2] for i in range(0, 18, 2)])
|
|
|
|
return f"{self.media_base_url}/{layers}/{h}/{media_path}"
|
|
|
|
def rewrite_media_urls(self, html: str, base_url: str, media_mappings: Dict[str, str]) -> str:
|
|
"""
|
|
Rewrite media URLs in HTML to point to neopig media vault.
|
|
|
|
media_mappings: {original_url: neopig_vault_path}
|
|
"""
|
|
result = html
|
|
for original, neopig_path in media_mappings.items():
|
|
# Escape for regex
|
|
escaped = re.escape(original)
|
|
result = re.sub(escaped, neopig_path, result)
|
|
return result
|
|
|
|
async def archive_page(
|
|
self,
|
|
url: str,
|
|
html: str,
|
|
media_mappings: Dict[str, str] = None,
|
|
) -> Tuple[bool, str]:
|
|
"""
|
|
Archive a page's HTML.
|
|
|
|
Saves both original (.html.og) and rewritten (.html) versions.
|
|
Returns (is_changed, content_hash).
|
|
"""
|
|
if not self._initialized:
|
|
await self.init()
|
|
|
|
filepath = url_to_filepath(url)
|
|
og_path = self.path / f"{filepath}.og"
|
|
html_path = self.path / filepath
|
|
|
|
# Create directories
|
|
await asyncio.to_thread(og_path.parent.mkdir, parents=True, exist_ok=True)
|
|
|
|
# Check if content changed
|
|
chash = content_hash(html.encode('utf-8'))
|
|
is_changed = True
|
|
if await asyncio.to_thread(og_path.exists):
|
|
async with aiofiles.open(og_path, 'r', encoding='utf-8') as f:
|
|
existing = await f.read()
|
|
if content_hash(existing.encode('utf-8')) == chash:
|
|
is_changed = False
|
|
|
|
# Save original HTML
|
|
async with aiofiles.open(og_path, 'w', encoding='utf-8') as f:
|
|
await f.write(html)
|
|
|
|
# Save rewritten HTML
|
|
rewritten = html
|
|
if media_mappings:
|
|
rewritten = self.rewrite_media_urls(html, url, media_mappings)
|
|
async with aiofiles.open(html_path, 'w', encoding='utf-8') as f:
|
|
await f.write(rewritten)
|
|
|
|
return is_changed, chash
|
|
|
|
async def get_page(self, url: str, original: bool = False) -> Optional[str]:
|
|
"""Get archived page HTML. Set original=True for .html.og version."""
|
|
filepath = url_to_filepath(url)
|
|
if original:
|
|
filepath = f"{filepath}.og"
|
|
full_path = self.path / filepath
|
|
|
|
if not await asyncio.to_thread(full_path.exists):
|
|
return None
|
|
|
|
async with aiofiles.open(full_path, 'r', encoding='utf-8') as f:
|
|
return await f.read()
|
|
|
|
async def finish_crawl(self, stats: Dict[str, Any]) -> Optional[str]:
|
|
"""Finish crawl, log it, and commit if changes."""
|
|
if not self._initialized:
|
|
await self.init()
|
|
|
|
log = await self._load_crawl_log()
|
|
log['crawls'].append({
|
|
'timestamp': datetime.now(timezone.utc).isoformat(),
|
|
'stats': stats,
|
|
})
|
|
await self._save_crawl_log(log)
|
|
|
|
pages_changed = stats.get('pages_changed', 0)
|
|
if pages_changed > 0:
|
|
return await self._git.commit(f"Crawl: {pages_changed} pages changed")
|
|
return None
|
|
|
|
async def get_stats(self) -> Dict[str, Any]:
|
|
"""Get vault statistics."""
|
|
if not self._initialized:
|
|
await self.init()
|
|
|
|
log = await self._load_crawl_log()
|
|
commits = await self._git.log(5)
|
|
|
|
# Count HTML files (run sync rglob in thread pool)
|
|
def _count_pages():
|
|
return sum(1 for _ in self.path.rglob('*.html') if not str(_).endswith('.og'))
|
|
|
|
page_count = await asyncio.to_thread(_count_pages)
|
|
|
|
return {
|
|
'domain': self.domain,
|
|
'path': str(self.path),
|
|
'pages_count': page_count,
|
|
'crawls_count': len(log.get('crawls', [])),
|
|
'recent_commits': commits,
|
|
'last_crawl': log['crawls'][-1] if log.get('crawls') else None,
|
|
}
|
|
|
|
|
|
class DomainMediaVault:
|
|
"""
|
|
Git LFS-tracked media vault for a single domain.
|
|
|
|
Stores media files in a tree structure mirroring the original domain.
|
|
"""
|
|
|
|
def __init__(self, vault_base: str, domain: str):
|
|
self.domain = domain.lower()
|
|
self.base_path = Path(vault_base)
|
|
self.path = get_filevault_path(self.base_path, self.domain)
|
|
self.index_path = self.path / 'index.json'
|
|
self._git = GitRepo(self.path, use_lfs=True)
|
|
self._initialized = False
|
|
|
|
async def init(self) -> bool:
|
|
"""Initialize vault. Returns True if newly created."""
|
|
is_new = await self._git.init()
|
|
|
|
if not await asyncio.to_thread(self.index_path.exists):
|
|
await self._save_index({'media': {}, 'created': datetime.now(timezone.utc).isoformat()})
|
|
is_new = True
|
|
|
|
self._initialized = True
|
|
return is_new
|
|
|
|
async def _load_index(self) -> Dict[str, Any]:
|
|
if not await asyncio.to_thread(self.index_path.exists):
|
|
return {'media': {}}
|
|
async with aiofiles.open(self.index_path, 'r') as f:
|
|
return json.loads(await f.read())
|
|
|
|
async def _save_index(self, index: Dict[str, Any]):
|
|
async with aiofiles.open(self.index_path, 'w') as f:
|
|
await f.write(json.dumps(index, indent=2))
|
|
|
|
async def archive_media(
|
|
self,
|
|
url: str,
|
|
content: bytes,
|
|
page_url: str = '',
|
|
) -> Tuple[bool, str, Path]:
|
|
"""
|
|
Archive media content.
|
|
|
|
Returns (is_new, content_hash, file_path).
|
|
"""
|
|
if not self._initialized:
|
|
await self.init()
|
|
|
|
# Use URL path as file path
|
|
parsed = Uri(url)
|
|
filepath = parsed.path.strip('/')
|
|
if not filepath:
|
|
# Fallback to content hash
|
|
filepath = f"{content_hash(content)}.bin"
|
|
|
|
file_path = self.path / filepath
|
|
await asyncio.to_thread(file_path.parent.mkdir, parents=True, exist_ok=True)
|
|
|
|
# Check if new or changed
|
|
chash = content_hash(content)
|
|
exists = await asyncio.to_thread(file_path.exists)
|
|
is_new = not exists
|
|
if exists:
|
|
async with aiofiles.open(file_path, 'rb') as f:
|
|
existing_hash = content_hash(await f.read())
|
|
is_new = existing_hash != chash
|
|
|
|
# Save media
|
|
async with aiofiles.open(file_path, 'wb') as f:
|
|
await f.write(content)
|
|
|
|
# Update index
|
|
index = await self._load_index()
|
|
index['media'][filepath] = {
|
|
'url': url,
|
|
'hash': chash,
|
|
'size': len(content),
|
|
'page_url': page_url,
|
|
'updated': datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
await self._save_index(index)
|
|
|
|
return is_new, chash, file_path
|
|
|
|
async def get_media(self, url: str) -> Optional[Tuple[bytes, Dict[str, Any]]]:
|
|
"""Get archived media content and metadata."""
|
|
parsed = Uri(url)
|
|
filepath = parsed.path.strip('/')
|
|
file_path = self.path / filepath
|
|
|
|
if not await asyncio.to_thread(file_path.exists):
|
|
return None
|
|
|
|
async with aiofiles.open(file_path, 'rb') as f:
|
|
content = await f.read()
|
|
|
|
index = await self._load_index()
|
|
meta = index.get('media', {}).get(filepath, {})
|
|
|
|
return content, meta
|
|
|
|
async def finish_crawl(self, stats: Dict[str, Any]) -> Optional[str]:
|
|
"""Finish crawl and commit if changes."""
|
|
if not self._initialized:
|
|
await self.init()
|
|
|
|
media_new = stats.get('media_new', 0)
|
|
if media_new > 0:
|
|
return await self._git.commit(f"Crawl: {media_new} media files")
|
|
return None
|
|
|
|
async def get_stats(self) -> Dict[str, Any]:
|
|
"""Get vault statistics."""
|
|
if not self._initialized:
|
|
await self.init()
|
|
|
|
index = await self._load_index()
|
|
commits = await self._git.log(5)
|
|
|
|
return {
|
|
'domain': self.domain,
|
|
'path': str(self.path),
|
|
'media_count': len(index.get('media', {})),
|
|
'recent_commits': commits,
|
|
}
|
|
|
|
|
|
class DomainLinkpeekVault:
|
|
"""
|
|
Git LFS-tracked screenshot vault for a single domain.
|
|
|
|
Stores page screenshots in a tree structure mirroring the original domain.
|
|
One screenshot per page, named after the URL path.
|
|
"""
|
|
|
|
def __init__(self, vault_base: str, domain: str):
|
|
self.domain = domain.lower()
|
|
self.base_path = Path(vault_base)
|
|
self.path = get_filevault_path(self.base_path, self.domain)
|
|
self.index_path = self.path / 'index.json'
|
|
self._git = GitRepo(self.path, use_lfs=True)
|
|
self._initialized = False
|
|
|
|
async def init(self) -> bool:
|
|
"""Initialize vault. Returns True if newly created."""
|
|
is_new = await self._git.init()
|
|
|
|
if not await asyncio.to_thread(self.index_path.exists):
|
|
await self._save_index({'screenshots': {}, 'created': datetime.now(timezone.utc).isoformat()})
|
|
is_new = True
|
|
|
|
self._initialized = True
|
|
return is_new
|
|
|
|
async def _load_index(self) -> Dict[str, Any]:
|
|
if not await asyncio.to_thread(self.index_path.exists):
|
|
return {'screenshots': {}}
|
|
async with aiofiles.open(self.index_path, 'r') as f:
|
|
return json.loads(await f.read())
|
|
|
|
async def _save_index(self, index: Dict[str, Any]):
|
|
async with aiofiles.open(self.index_path, 'w') as f:
|
|
await f.write(json.dumps(index, indent=2))
|
|
|
|
def url_to_screenshot_path(self, url: str) -> str:
|
|
"""
|
|
Convert URL to screenshot filepath.
|
|
|
|
Examples:
|
|
https://example.com/ -> index.png
|
|
https://example.com/about -> about/index.png
|
|
https://example.com/blog/post.html -> blog/post.png
|
|
"""
|
|
parsed = Uri(url)
|
|
path = parsed.path.strip('/')
|
|
|
|
if not path:
|
|
return 'index.png'
|
|
|
|
# Replace extension with .png or add /index.png for directories
|
|
if '.' in path.split('/')[-1]:
|
|
# Has extension, replace with .png
|
|
base = path.rsplit('.', 1)[0]
|
|
return f"{base}.png"
|
|
else:
|
|
# Directory-style URL
|
|
return f"{path}/index.png"
|
|
|
|
async def archive_screenshot(
|
|
self,
|
|
url: str,
|
|
screenshot_data: bytes,
|
|
) -> Tuple[bool, str, Path]:
|
|
"""
|
|
Archive a page screenshot.
|
|
|
|
Returns (is_new, content_hash, file_path).
|
|
"""
|
|
if not self._initialized:
|
|
await self.init()
|
|
|
|
filepath = self.url_to_screenshot_path(url)
|
|
file_path = self.path / filepath
|
|
await asyncio.to_thread(file_path.parent.mkdir, parents=True, exist_ok=True)
|
|
|
|
# Check if new or changed
|
|
chash = content_hash(screenshot_data)
|
|
exists = await asyncio.to_thread(file_path.exists)
|
|
is_new = not exists
|
|
if exists:
|
|
async with aiofiles.open(file_path, 'rb') as f:
|
|
existing_hash = content_hash(await f.read())
|
|
is_new = existing_hash != chash
|
|
|
|
# Save screenshot
|
|
async with aiofiles.open(file_path, 'wb') as f:
|
|
await f.write(screenshot_data)
|
|
|
|
# Update index
|
|
index = await self._load_index()
|
|
index['screenshots'][filepath] = {
|
|
'url': url,
|
|
'hash': chash,
|
|
'size': len(screenshot_data),
|
|
'updated': datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
await self._save_index(index)
|
|
|
|
return is_new, chash, file_path
|
|
|
|
async def get_screenshot(self, url: str) -> Optional[Tuple[bytes, Dict[str, Any]]]:
|
|
"""Get archived screenshot and metadata."""
|
|
filepath = self.url_to_screenshot_path(url)
|
|
file_path = self.path / filepath
|
|
|
|
if not await asyncio.to_thread(file_path.exists):
|
|
return None
|
|
|
|
async with aiofiles.open(file_path, 'rb') as f:
|
|
content = await f.read()
|
|
|
|
index = await self._load_index()
|
|
meta = index.get('screenshots', {}).get(filepath, {})
|
|
|
|
return content, meta
|
|
|
|
async def finish_crawl(self, stats: Dict[str, Any]) -> Optional[str]:
|
|
"""Finish crawl and commit if changes."""
|
|
if not self._initialized:
|
|
await self.init()
|
|
|
|
screenshots_new = stats.get('screenshots_new', 0)
|
|
if screenshots_new > 0:
|
|
return await self._git.commit(f"Crawl: {screenshots_new} screenshots")
|
|
return None
|
|
|
|
async def get_stats(self) -> Dict[str, Any]:
|
|
"""Get vault statistics."""
|
|
if not self._initialized:
|
|
await self.init()
|
|
|
|
index = await self._load_index()
|
|
commits = await self._git.log(5)
|
|
|
|
return {
|
|
'domain': self.domain,
|
|
'path': str(self.path),
|
|
'screenshots_count': len(index.get('screenshots', {})),
|
|
'recent_commits': commits,
|
|
}
|
|
|
|
|
|
class VaultManager:
|
|
"""
|
|
Manages HTML, Media, and Linkpeek (screenshot) vaults for multiple domains.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
html_vault_base: str = 'html_vault',
|
|
media_vault_base: str = 'media_vault',
|
|
linkpeek_vault_base: str = 'linkpeek_vault',
|
|
media_base_url: str = '/media',
|
|
linkpeek_base_url: str = '/linkpeek',
|
|
):
|
|
self.html_base = Path(html_vault_base)
|
|
self.media_base = Path(media_vault_base)
|
|
self.linkpeek_base = Path(linkpeek_vault_base)
|
|
self.media_base_url = media_base_url
|
|
self.linkpeek_base_url = linkpeek_base_url
|
|
self._html_vaults: Dict[str, DomainHtmlVault] = {}
|
|
self._media_vaults: Dict[str, DomainMediaVault] = {}
|
|
self._linkpeek_vaults: Dict[str, DomainLinkpeekVault] = {}
|
|
|
|
def get_html_vault(self, domain: str) -> DomainHtmlVault:
|
|
"""Get or create HTML vault for domain."""
|
|
domain = domain.lower()
|
|
if domain not in self._html_vaults:
|
|
self._html_vaults[domain] = DomainHtmlVault(
|
|
str(self.html_base), domain, self.media_base_url
|
|
)
|
|
return self._html_vaults[domain]
|
|
|
|
def get_media_vault(self, domain: str) -> DomainMediaVault:
|
|
"""Get or create media vault for domain."""
|
|
domain = domain.lower()
|
|
if domain not in self._media_vaults:
|
|
self._media_vaults[domain] = DomainMediaVault(str(self.media_base), domain)
|
|
return self._media_vaults[domain]
|
|
|
|
def get_linkpeek_vault(self, domain: str) -> DomainLinkpeekVault:
|
|
"""Get or create linkpeek (screenshot) vault for domain."""
|
|
domain = domain.lower()
|
|
if domain not in self._linkpeek_vaults:
|
|
self._linkpeek_vaults[domain] = DomainLinkpeekVault(str(self.linkpeek_base), domain)
|
|
return self._linkpeek_vaults[domain]
|
|
|
|
def get_vaults_for_url(self, url: str) -> Tuple[DomainHtmlVault, DomainMediaVault, DomainLinkpeekVault]:
|
|
"""Get HTML, media, and linkpeek vaults for URL's domain."""
|
|
parsed = Uri(url)
|
|
domain = parsed.hostname or parsed.path.split('/')[0]
|
|
return self.get_html_vault(domain), self.get_media_vault(domain), self.get_linkpeek_vault(domain)
|
|
|
|
def get_linkpeek_url(self, url: str) -> str:
|
|
"""
|
|
Get linkpeek URL for a page screenshot.
|
|
|
|
Example: https://example.com/about
|
|
-> /linkpeek/{9-layers}/{salted_hash}/about/index.png
|
|
"""
|
|
parsed = Uri(url)
|
|
domain = parsed.hostname.lower()
|
|
|
|
# Get screenshot path
|
|
vault = self.get_linkpeek_vault(domain)
|
|
screenshot_path = vault.url_to_screenshot_path(url)
|
|
|
|
# Get salted hash for domain (same across all vaults)
|
|
h = domain_hash(domain, salted=True)
|
|
layers = '/'.join([h[i:i+2] for i in range(0, 18, 2)])
|
|
|
|
return f"{self.linkpeek_base_url}/{layers}/{h}/{screenshot_path}"
|
|
|
|
async def list_domains(self, vault_type: str = 'html') -> List[str]:
|
|
"""List all domains with vaults."""
|
|
if vault_type == 'html':
|
|
base = self.html_base
|
|
elif vault_type == 'media':
|
|
base = self.media_base
|
|
else: # linkpeek
|
|
base = self.linkpeek_base
|
|
|
|
def _walk_domains():
|
|
domains = []
|
|
if not base.exists():
|
|
return domains
|
|
|
|
for root, dirs, files in os.walk(base):
|
|
if '.git' in dirs:
|
|
rel_path = Path(root).relative_to(base)
|
|
parts = list(rel_path.parts)
|
|
if len(parts) >= 10: # 9 hash layers + domain
|
|
domains.append(parts[-1])
|
|
|
|
return sorted(set(domains))
|
|
|
|
return await asyncio.to_thread(_walk_domains)
|
|
|
|
async def get_all_stats(self) -> Dict[str, List[Dict[str, Any]]]:
|
|
"""Get stats for all domains."""
|
|
html_domains = await self.list_domains('html')
|
|
media_domains = await self.list_domains('media')
|
|
linkpeek_domains = await self.list_domains('linkpeek')
|
|
|
|
html_stats = []
|
|
for domain in html_domains:
|
|
vault = self.get_html_vault(domain)
|
|
await vault.init()
|
|
html_stats.append(await vault.get_stats())
|
|
|
|
media_stats = []
|
|
for domain in media_domains:
|
|
vault = self.get_media_vault(domain)
|
|
await vault.init()
|
|
media_stats.append(await vault.get_stats())
|
|
|
|
linkpeek_stats = []
|
|
for domain in linkpeek_domains:
|
|
vault = self.get_linkpeek_vault(domain)
|
|
await vault.init()
|
|
linkpeek_stats.append(await vault.get_stats())
|
|
|
|
return {
|
|
'html_vaults': html_stats,
|
|
'media_vaults': media_stats,
|
|
'linkpeek_vaults': linkpeek_stats,
|
|
}
|