#!/usr/bin/env python3
# This is free software for the public good of a permacomputer hosted at
# permacomputer.com, an always-on computer by the people, for the people.
# One which is durable, easy to repair, & distributed like tap water
# for machine learning intelligence.
#
# The permacomputer is community-owned infrastructure optimized around
# four values:
#
#   TRUTH      First principles, math & science, open source code freely distributed
#   FREEDOM    Voluntary partnerships, freedom from tyranny & corporate control
#   HARMONY    Minimal waste, self-renewing systems with diverse thriving connections
#   LOVE       Be yourself without hurting others, cooperation through natural law
#
# This software contributes to that vision by archiving the web, preserving digital knowledge before it disappears.
# Code is seeds to sprout on any abandoned technology.

"""
domain_vault.py - Triple filevault system for web archival

Three separate 9-layer deep filevaults, each with git repos per domain.
Directory names are salted hashes (NEOPIG_VAULT_SALT env var) for privacy.
The same domain + salt produces identical hashes across all three vaults.

1. HTML Vault (html_vault/):
   html_vault/{9-layers}/{salted_hash}/
       .git/                           # Git repository (SSH cloneable)
       {url_path}/index.html           # Rewritten HTML with neopig media paths
       {url_path}/index.html.og        # Original HTML with original URIs
       crawl_log.json                  # Crawl history

2. Media Vault (media_vault/):
   media_vault/{9-layers}/{salted_hash}/
       .git/                           # Git repository with LFS (SSH cloneable)
       {url_path}/image.png            # Media files mirroring original paths
       index.json                      # Media index

3. Linkpeek Vault (linkpeek_vault/):
   linkpeek_vault/{9-layers}/{salted_hash}/
       .git/                           # Git repository with LFS (SSH cloneable)
       {url_path}/index.png            # Page screenshots mirroring URL paths
       index.json                      # Screenshot index

Environment Variables:
    NEOPIG_VAULT_SALT: Secret salt for domain hashing (required for privacy)

Features:
    - 9-layer deep hash paths for filesystem distribution
    - Salted hash directory names (domain name not exposed)
    - Git versioning per domain (all vaults)
    - Git LFS for media and linkpeek vaults (large files)
    - SSH cloneable repos
    - Idempotent: only commits when content changes
    - .html.og preserves original, .html has rewritten media URIs
    - Tree structure mirrors original domain URL paths
"""

import asyncio
import hashlib
import json
import logging
import os
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Dict, Any, Optional, List, Tuple
from urllib.parse import urljoin
from miniuri import Uri
import aiofiles

logger = logging.getLogger(__name__)


def get_vault_salt() -> str:
    """Get the vault salt from environment variable."""
    return os.environ.get('NEOPIG_VAULT_SALT', '')


def domain_hash(domain: str, salted: bool = False) -> str:
    """
    Generate MD5 hash of domain.

    Args:
        domain: The domain name
        salted: If True, include NEOPIG_VAULT_SALT for directory naming
    """
    data = domain.lower()
    if salted:
        salt = get_vault_salt()
        data = f"{salt}:{data}"
    return hashlib.md5(data.encode()).hexdigest()


def content_hash(content: bytes) -> str:
    """Generate MD5 hash of content for deduplication."""
    return hashlib.md5(content).hexdigest()


def get_filevault_path(base_path: Path, domain: str) -> Path:
    """
    Generate 9-layer deep path for filevault.

    Uses salted hash for both path layers AND final directory name.
    The same domain + salt produces the same hash across all three vaults.

    Example: example.com -> base/ab/cd/ef/12/34/56/78/9a/bc/abcdef1234567890abcdef1234567890/
    """
    h = domain_hash(domain, salted=True)
    # 9 layers of 2-char directories
    layers = [h[i:i+2] for i in range(0, 18, 2)]  # 9 pairs (18 chars)
    path = base_path
    for layer in layers:
        path = path / layer
    # Final directory is the full salted hash, not the domain name
    return path / h


def url_to_filepath(url: str) -> str:
    """
    Convert URL to filesystem path that mirrors the original structure.

    Examples:
        https://example.com/ -> index.html
        https://example.com/about -> about/index.html
        https://example.com/blog/post.html -> blog/post.html
        https://example.com/images/logo.png -> images/logo.png
    """
    parsed = Uri(url)
    path = (parsed.path or '').strip('/')

    if not path:
        return 'index.html'

    # Check if path has an extension
    if '.' in path.split('/')[-1]:
        return path
    else:
        # Directory-style URL, add index.html
        return f"{path}/index.html"


def extract_media_urls(html: str, base_url: str) -> List[Tuple[str, str]]:
    """
    Extract media URLs from HTML.

    Returns list of (absolute_url, original_reference) tuples.
    """
    media_patterns = [
        # Images
        r'<img[^>]+src=["\']([^"\']+)["\']',
        r'<source[^>]+src=["\']([^"\']+)["\']',
        # CSS backgrounds
        r'url\(["\']?([^"\')\s]+)["\']?\)',
        # Video/Audio
        r'<video[^>]+src=["\']([^"\']+)["\']',
        r'<audio[^>]+src=["\']([^"\']+)["\']',
        # Links to media
        r'<link[^>]+href=["\']([^"\']+\.(?:css|ico|png|jpg|jpeg|gif|svg|woff2?|ttf|eot))["\']',
    ]

    found = []
    for pattern in media_patterns:
        for match in re.finditer(pattern, html, re.IGNORECASE):
            original = match.group(1)
            # Skip data URIs and anchors
            if original.startswith('data:') or original.startswith('#'):
                continue
            absolute = urljoin(base_url, original)
            found.append((absolute, original))

    return found


class GitRepo:
    """Async git operations helper."""

    def __init__(self, path: Path, use_lfs: bool = False):
        self.path = path
        self.use_lfs = use_lfs

    async def init(self) -> bool:
        """Initialize git repo. Returns True if newly created."""
        git_path = self.path / '.git'
        if await asyncio.to_thread(git_path.exists):
            return False

        await asyncio.to_thread(self.path.mkdir, parents=True, exist_ok=True)

        await self._run('git', 'init')
        await self._run('git', 'config', 'user.email', 'neopig@localhost')
        await self._run('git', 'config', 'user.name', 'neopig')

        if self.use_lfs:
            await self._init_lfs()

        logger.info(f"Initialized git repo at {self.path}")
        return True

    async def _init_lfs(self):
        """Initialize git LFS for media files."""
        await self._run('git', 'lfs', 'install', '--local')
        # Track common media extensions
        extensions = [
            '*.png', '*.jpg', '*.jpeg', '*.gif', '*.webp', '*.svg',
            '*.mp4', '*.webm', '*.mov', '*.avi',
            '*.mp3', '*.wav', '*.ogg', '*.flac',
            '*.pdf', '*.zip', '*.tar', '*.gz',
            '*.woff', '*.woff2', '*.ttf', '*.eot',
        ]
        for ext in extensions:
            await self._run('git', 'lfs', 'track', ext)
        await self._run('git', 'add', '.gitattributes')
        logger.info(f"Initialized git LFS at {self.path}")

    async def _run(self, *args) -> Tuple[bytes, bytes]:
        """Run git command."""
        proc = await asyncio.create_subprocess_exec(
            *args,
            cwd=str(self.path),
            stdout=asyncio.subprocess.PIPE,
            stderr=asyncio.subprocess.PIPE,
        )
        return await proc.communicate()

    async def has_changes(self) -> bool:
        """Check if there are uncommitted changes."""
        stdout, _ = await self._run('git', 'status', '--porcelain')
        return len(stdout.strip()) > 0

    async def add_all(self):
        """Stage all changes."""
        await self._run('git', 'add', '-A')

    async def commit(self, message: str) -> Optional[str]:
        """Commit if changes exist. Returns commit hash or None."""
        await self.add_all()
        if not await self.has_changes():
            return None

        stdout, stderr = await self._run('git', 'commit', '-m', message)
        stdout, _ = await self._run('git', 'rev-parse', 'HEAD')
        commit_hash = stdout.decode().strip()
        logger.info(f"Committed {commit_hash[:8]}: {message}")
        return commit_hash

    async def log(self, limit: int = 10) -> List[Dict[str, str]]:
        """Get recent commit history."""
        stdout, _ = await self._run('git', 'log', f'-{limit}', '--format=%H|%s|%ci')
        commits = []
        for line in stdout.decode().strip().split('\n'):
            if '|' in line:
                parts = line.split('|', 2)
                if len(parts) == 3:
                    commits.append({
                        'hash': parts[0],
                        'message': parts[1],
                        'date': parts[2],
                    })
        return commits


class DomainHtmlVault:
    """
    Git-tracked HTML vault for a single domain.

    Stores pages in a tree structure mirroring the original domain:
    - {path}/index.html.og  = Original HTML
    - {path}/index.html     = Rewritten with neopig media paths
    """

    def __init__(self, vault_base: str, domain: str, media_base_url: str = '/media'):
        self.domain = domain.lower()
        self.base_path = Path(vault_base)
        self.path = get_filevault_path(self.base_path, self.domain)
        self.crawl_log_path = self.path / 'crawl_log.json'
        self.media_base_url = media_base_url.rstrip('/')
        self._git = GitRepo(self.path, use_lfs=False)
        self._initialized = False

    async def init(self) -> bool:
        """Initialize vault. Returns True if newly created."""
        is_new = await self._git.init()

        if not await asyncio.to_thread(self.crawl_log_path.exists):
            await self._save_crawl_log({'crawls': []})
            is_new = True

        self._initialized = True
        return is_new

    async def _load_crawl_log(self) -> Dict[str, Any]:
        if not await asyncio.to_thread(self.crawl_log_path.exists):
            return {'crawls': []}
        async with aiofiles.open(self.crawl_log_path, 'r') as f:
            return json.loads(await f.read())

    async def _save_crawl_log(self, log: Dict[str, Any]):
        async with aiofiles.open(self.crawl_log_path, 'w') as f:
            await f.write(json.dumps(log, indent=2))

    def get_media_vault_url(self, media_url: str) -> str:
        """
        Convert original media URL to neopig media vault path.

        Example: https://example.com/images/logo.png
              -> /media/{9-layers}/{salted_hash}/images/logo.png
        """
        parsed = Uri(media_url)
        media_domain = parsed.hostname.lower()
        media_path = parsed.path.strip('/')

        # Get salted hash for media domain (same across all vaults)
        h = domain_hash(media_domain, salted=True)
        layers = '/'.join([h[i:i+2] for i in range(0, 18, 2)])

        return f"{self.media_base_url}/{layers}/{h}/{media_path}"

    def rewrite_media_urls(self, html: str, base_url: str, media_mappings: Dict[str, str]) -> str:
        """
        Rewrite media URLs in HTML to point to neopig media vault.

        media_mappings: {original_url: neopig_vault_path}
        """
        result = html
        for original, neopig_path in media_mappings.items():
            # Escape for regex
            escaped = re.escape(original)
            result = re.sub(escaped, neopig_path, result)
        return result

    async def archive_page(
        self,
        url: str,
        html: str,
        media_mappings: Dict[str, str] = None,
    ) -> Tuple[bool, str]:
        """
        Archive a page's HTML.

        Saves both original (.html.og) and rewritten (.html) versions.
        Returns (is_changed, content_hash).
        """
        if not self._initialized:
            await self.init()

        filepath = url_to_filepath(url)
        og_path = self.path / f"{filepath}.og"
        html_path = self.path / filepath

        # Create directories
        await asyncio.to_thread(og_path.parent.mkdir, parents=True, exist_ok=True)

        # Check if content changed
        chash = content_hash(html.encode('utf-8'))
        is_changed = True
        if await asyncio.to_thread(og_path.exists):
            async with aiofiles.open(og_path, 'r', encoding='utf-8') as f:
                existing = await f.read()
            if content_hash(existing.encode('utf-8')) == chash:
                is_changed = False

        # Save original HTML
        async with aiofiles.open(og_path, 'w', encoding='utf-8') as f:
            await f.write(html)

        # Save rewritten HTML
        rewritten = html
        if media_mappings:
            rewritten = self.rewrite_media_urls(html, url, media_mappings)
        async with aiofiles.open(html_path, 'w', encoding='utf-8') as f:
            await f.write(rewritten)

        return is_changed, chash

    async def get_page(self, url: str, original: bool = False) -> Optional[str]:
        """Get archived page HTML. Set original=True for .html.og version."""
        filepath = url_to_filepath(url)
        if original:
            filepath = f"{filepath}.og"
        full_path = self.path / filepath

        if not await asyncio.to_thread(full_path.exists):
            return None

        async with aiofiles.open(full_path, 'r', encoding='utf-8') as f:
            return await f.read()

    async def finish_crawl(self, stats: Dict[str, Any]) -> Optional[str]:
        """Finish crawl, log it, and commit if changes."""
        if not self._initialized:
            await self.init()

        log = await self._load_crawl_log()
        log['crawls'].append({
            'timestamp': datetime.now(timezone.utc).isoformat(),
            'stats': stats,
        })
        await self._save_crawl_log(log)

        pages_changed = stats.get('pages_changed', 0)
        if pages_changed > 0:
            return await self._git.commit(f"Crawl: {pages_changed} pages changed")
        return None

    async def get_stats(self) -> Dict[str, Any]:
        """Get vault statistics."""
        if not self._initialized:
            await self.init()

        log = await self._load_crawl_log()
        commits = await self._git.log(5)

        # Count HTML files (run sync rglob in thread pool)
        def _count_pages():
            return sum(1 for _ in self.path.rglob('*.html') if not str(_).endswith('.og'))

        page_count = await asyncio.to_thread(_count_pages)

        return {
            'domain': self.domain,
            'path': str(self.path),
            'pages_count': page_count,
            'crawls_count': len(log.get('crawls', [])),
            'recent_commits': commits,
            'last_crawl': log['crawls'][-1] if log.get('crawls') else None,
        }


class DomainMediaVault:
    """
    Git LFS-tracked media vault for a single domain.

    Stores media files in a tree structure mirroring the original domain.
    """

    def __init__(self, vault_base: str, domain: str):
        self.domain = domain.lower()
        self.base_path = Path(vault_base)
        self.path = get_filevault_path(self.base_path, self.domain)
        self.index_path = self.path / 'index.json'
        self._git = GitRepo(self.path, use_lfs=True)
        self._initialized = False

    async def init(self) -> bool:
        """Initialize vault. Returns True if newly created."""
        is_new = await self._git.init()

        if not await asyncio.to_thread(self.index_path.exists):
            await self._save_index({'media': {}, 'created': datetime.now(timezone.utc).isoformat()})
            is_new = True

        self._initialized = True
        return is_new

    async def _load_index(self) -> Dict[str, Any]:
        if not await asyncio.to_thread(self.index_path.exists):
            return {'media': {}}
        async with aiofiles.open(self.index_path, 'r') as f:
            return json.loads(await f.read())

    async def _save_index(self, index: Dict[str, Any]):
        async with aiofiles.open(self.index_path, 'w') as f:
            await f.write(json.dumps(index, indent=2))

    async def archive_media(
        self,
        url: str,
        content: bytes,
        page_url: str = '',
    ) -> Tuple[bool, str, Path]:
        """
        Archive media content.

        Returns (is_new, content_hash, file_path).
        """
        if not self._initialized:
            await self.init()

        # Use URL path as file path
        parsed = Uri(url)
        filepath = parsed.path.strip('/')
        if not filepath:
            # Fallback to content hash
            filepath = f"{content_hash(content)}.bin"

        file_path = self.path / filepath
        await asyncio.to_thread(file_path.parent.mkdir, parents=True, exist_ok=True)

        # Check if new or changed
        chash = content_hash(content)
        exists = await asyncio.to_thread(file_path.exists)
        is_new = not exists
        if exists:
            async with aiofiles.open(file_path, 'rb') as f:
                existing_hash = content_hash(await f.read())
            is_new = existing_hash != chash

        # Save media
        async with aiofiles.open(file_path, 'wb') as f:
            await f.write(content)

        # Update index
        index = await self._load_index()
        index['media'][filepath] = {
            'url': url,
            'hash': chash,
            'size': len(content),
            'page_url': page_url,
            'updated': datetime.now(timezone.utc).isoformat(),
        }
        await self._save_index(index)

        return is_new, chash, file_path

    async def get_media(self, url: str) -> Optional[Tuple[bytes, Dict[str, Any]]]:
        """Get archived media content and metadata."""
        parsed = Uri(url)
        filepath = parsed.path.strip('/')
        file_path = self.path / filepath

        if not await asyncio.to_thread(file_path.exists):
            return None

        async with aiofiles.open(file_path, 'rb') as f:
            content = await f.read()

        index = await self._load_index()
        meta = index.get('media', {}).get(filepath, {})

        return content, meta

    async def finish_crawl(self, stats: Dict[str, Any]) -> Optional[str]:
        """Finish crawl and commit if changes."""
        if not self._initialized:
            await self.init()

        media_new = stats.get('media_new', 0)
        if media_new > 0:
            return await self._git.commit(f"Crawl: {media_new} media files")
        return None

    async def get_stats(self) -> Dict[str, Any]:
        """Get vault statistics."""
        if not self._initialized:
            await self.init()

        index = await self._load_index()
        commits = await self._git.log(5)

        return {
            'domain': self.domain,
            'path': str(self.path),
            'media_count': len(index.get('media', {})),
            'recent_commits': commits,
        }


class DomainLinkpeekVault:
    """
    Git LFS-tracked screenshot vault for a single domain.

    Stores page screenshots in a tree structure mirroring the original domain.
    One screenshot per page, named after the URL path.
    """

    def __init__(self, vault_base: str, domain: str):
        self.domain = domain.lower()
        self.base_path = Path(vault_base)
        self.path = get_filevault_path(self.base_path, self.domain)
        self.index_path = self.path / 'index.json'
        self._git = GitRepo(self.path, use_lfs=True)
        self._initialized = False

    async def init(self) -> bool:
        """Initialize vault. Returns True if newly created."""
        is_new = await self._git.init()

        if not await asyncio.to_thread(self.index_path.exists):
            await self._save_index({'screenshots': {}, 'created': datetime.now(timezone.utc).isoformat()})
            is_new = True

        self._initialized = True
        return is_new

    async def _load_index(self) -> Dict[str, Any]:
        if not await asyncio.to_thread(self.index_path.exists):
            return {'screenshots': {}}
        async with aiofiles.open(self.index_path, 'r') as f:
            return json.loads(await f.read())

    async def _save_index(self, index: Dict[str, Any]):
        async with aiofiles.open(self.index_path, 'w') as f:
            await f.write(json.dumps(index, indent=2))

    def url_to_screenshot_path(self, url: str) -> str:
        """
        Convert URL to screenshot filepath.

        Examples:
            https://example.com/ -> index.png
            https://example.com/about -> about/index.png
            https://example.com/blog/post.html -> blog/post.png
        """
        parsed = Uri(url)
        path = parsed.path.strip('/')

        if not path:
            return 'index.png'

        # Replace extension with .png or add /index.png for directories
        if '.' in path.split('/')[-1]:
            # Has extension, replace with .png
            base = path.rsplit('.', 1)[0]
            return f"{base}.png"
        else:
            # Directory-style URL
            return f"{path}/index.png"

    async def archive_screenshot(
        self,
        url: str,
        screenshot_data: bytes,
    ) -> Tuple[bool, str, Path]:
        """
        Archive a page screenshot.

        Returns (is_new, content_hash, file_path).
        """
        if not self._initialized:
            await self.init()

        filepath = self.url_to_screenshot_path(url)
        file_path = self.path / filepath
        await asyncio.to_thread(file_path.parent.mkdir, parents=True, exist_ok=True)

        # Check if new or changed
        chash = content_hash(screenshot_data)
        exists = await asyncio.to_thread(file_path.exists)
        is_new = not exists
        if exists:
            async with aiofiles.open(file_path, 'rb') as f:
                existing_hash = content_hash(await f.read())
            is_new = existing_hash != chash

        # Save screenshot
        async with aiofiles.open(file_path, 'wb') as f:
            await f.write(screenshot_data)

        # Update index
        index = await self._load_index()
        index['screenshots'][filepath] = {
            'url': url,
            'hash': chash,
            'size': len(screenshot_data),
            'updated': datetime.now(timezone.utc).isoformat(),
        }
        await self._save_index(index)

        return is_new, chash, file_path

    async def get_screenshot(self, url: str) -> Optional[Tuple[bytes, Dict[str, Any]]]:
        """Get archived screenshot and metadata."""
        filepath = self.url_to_screenshot_path(url)
        file_path = self.path / filepath

        if not await asyncio.to_thread(file_path.exists):
            return None

        async with aiofiles.open(file_path, 'rb') as f:
            content = await f.read()

        index = await self._load_index()
        meta = index.get('screenshots', {}).get(filepath, {})

        return content, meta

    async def finish_crawl(self, stats: Dict[str, Any]) -> Optional[str]:
        """Finish crawl and commit if changes."""
        if not self._initialized:
            await self.init()

        screenshots_new = stats.get('screenshots_new', 0)
        if screenshots_new > 0:
            return await self._git.commit(f"Crawl: {screenshots_new} screenshots")
        return None

    async def get_stats(self) -> Dict[str, Any]:
        """Get vault statistics."""
        if not self._initialized:
            await self.init()

        index = await self._load_index()
        commits = await self._git.log(5)

        return {
            'domain': self.domain,
            'path': str(self.path),
            'screenshots_count': len(index.get('screenshots', {})),
            'recent_commits': commits,
        }


class VaultManager:
    """
    Manages HTML, Media, and Linkpeek (screenshot) vaults for multiple domains.
    """

    def __init__(
        self,
        html_vault_base: str = 'html_vault',
        media_vault_base: str = 'media_vault',
        linkpeek_vault_base: str = 'linkpeek_vault',
        media_base_url: str = '/media',
        linkpeek_base_url: str = '/linkpeek',
    ):
        self.html_base = Path(html_vault_base)
        self.media_base = Path(media_vault_base)
        self.linkpeek_base = Path(linkpeek_vault_base)
        self.media_base_url = media_base_url
        self.linkpeek_base_url = linkpeek_base_url
        self._html_vaults: Dict[str, DomainHtmlVault] = {}
        self._media_vaults: Dict[str, DomainMediaVault] = {}
        self._linkpeek_vaults: Dict[str, DomainLinkpeekVault] = {}

    def get_html_vault(self, domain: str) -> DomainHtmlVault:
        """Get or create HTML vault for domain."""
        domain = domain.lower()
        if domain not in self._html_vaults:
            self._html_vaults[domain] = DomainHtmlVault(
                str(self.html_base), domain, self.media_base_url
            )
        return self._html_vaults[domain]

    def get_media_vault(self, domain: str) -> DomainMediaVault:
        """Get or create media vault for domain."""
        domain = domain.lower()
        if domain not in self._media_vaults:
            self._media_vaults[domain] = DomainMediaVault(str(self.media_base), domain)
        return self._media_vaults[domain]

    def get_linkpeek_vault(self, domain: str) -> DomainLinkpeekVault:
        """Get or create linkpeek (screenshot) vault for domain."""
        domain = domain.lower()
        if domain not in self._linkpeek_vaults:
            self._linkpeek_vaults[domain] = DomainLinkpeekVault(str(self.linkpeek_base), domain)
        return self._linkpeek_vaults[domain]

    def get_vaults_for_url(self, url: str) -> Tuple[DomainHtmlVault, DomainMediaVault, DomainLinkpeekVault]:
        """Get HTML, media, and linkpeek vaults for URL's domain."""
        parsed = Uri(url)
        domain = parsed.hostname or parsed.path.split('/')[0]
        return self.get_html_vault(domain), self.get_media_vault(domain), self.get_linkpeek_vault(domain)

    def get_linkpeek_url(self, url: str) -> str:
        """
        Get linkpeek URL for a page screenshot.

        Example: https://example.com/about
              -> /linkpeek/{9-layers}/{salted_hash}/about/index.png
        """
        parsed = Uri(url)
        domain = parsed.hostname.lower()

        # Get screenshot path
        vault = self.get_linkpeek_vault(domain)
        screenshot_path = vault.url_to_screenshot_path(url)

        # Get salted hash for domain (same across all vaults)
        h = domain_hash(domain, salted=True)
        layers = '/'.join([h[i:i+2] for i in range(0, 18, 2)])

        return f"{self.linkpeek_base_url}/{layers}/{h}/{screenshot_path}"

    async def list_domains(self, vault_type: str = 'html') -> List[str]:
        """List all domains with vaults."""
        if vault_type == 'html':
            base = self.html_base
        elif vault_type == 'media':
            base = self.media_base
        else:  # linkpeek
            base = self.linkpeek_base

        def _walk_domains():
            domains = []
            if not base.exists():
                return domains

            for root, dirs, files in os.walk(base):
                if '.git' in dirs:
                    rel_path = Path(root).relative_to(base)
                    parts = list(rel_path.parts)
                    if len(parts) >= 10:  # 9 hash layers + domain
                        domains.append(parts[-1])

            return sorted(set(domains))

        return await asyncio.to_thread(_walk_domains)

    async def get_all_stats(self) -> Dict[str, List[Dict[str, Any]]]:
        """Get stats for all domains."""
        html_domains = await self.list_domains('html')
        media_domains = await self.list_domains('media')
        linkpeek_domains = await self.list_domains('linkpeek')

        html_stats = []
        for domain in html_domains:
            vault = self.get_html_vault(domain)
            await vault.init()
            html_stats.append(await vault.get_stats())

        media_stats = []
        for domain in media_domains:
            vault = self.get_media_vault(domain)
            await vault.init()
            media_stats.append(await vault.get_stats())

        linkpeek_stats = []
        for domain in linkpeek_domains:
            vault = self.get_linkpeek_vault(domain)
            await vault.init()
            linkpeek_stats.append(await vault.get_stats())

        return {
            'html_vaults': html_stats,
            'media_vaults': media_stats,
            'linkpeek_vaults': linkpeek_stats,
        }
