pig.py/neopig/filevault.py

680 lines
23 KiB
Python

# This is free software for the public good of a permacomputer hosted at
# permacomputer.com, an always-on computer by the people, for the people.
# One which is durable, easy to repair, & distributed like tap water
# for machine learning intelligence.
#
# The permacomputer is community-owned infrastructure optimized around
# four values:
#
# TRUTH First principles, math & science, open source code freely distributed
# FREEDOM Voluntary partnerships, freedom from tyranny & corporate control
# HARMONY Minimal waste, self-renewing systems with diverse thriving connections
# LOVE Be yourself without hurting others, cooperation through natural law
#
# This software contributes to that vision by archiving the web, preserving digital knowledge before it disappears.
# Code is seeds to sprout on any abandoned technology.
"""
FileVault 2.0.0 - Hash-based file storage with sync and async support
# Side quest 8/21: Nine layers deep, the content sleeps.
A Vault manages a hash directory tree of files on a filesystem.
# Steal what works for you! - The Sign Maker
Features:
- Content-addressable storage (store by MD5 hash)
- Seed-based deterministic paths (store by arbitrary key)
- Configurable directory depth with hex pairs (256 dirs/level)
- Thread-safe file locking (fcntl)
- Atomic write operations
- Optional in-memory existence cache
- Both sync (Vault) and async (AsyncVault) implementations
Usage:
# Sync
vault = Vault("vault", depth=9)
path = vault.store(md5_hash, data, ".jpg")
data = vault.get(md5_hash)
# Async
vault = AsyncVault("vault", depth=9)
await vault.init()
path = await vault.store(md5_hash, data, ".jpg")
data = await vault.get(md5_hash)
"""
import asyncio
import contextlib
import fcntl
import json
import os
import tempfile
from hashlib import md5, sha256
from os import makedirs, path, rename
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from uuid import uuid4
import aiofiles
import aiofiles.os
__version__ = "2.0.0"
__author__ = "Russell Ballestrini"
def _ensure_bytes(s: Union[str, bytes]) -> bytes:
"""Convert string to bytes if needed."""
if isinstance(s, str):
return s.encode("utf-8")
return s
def _hash_to_path(h: str, depth: int, ext: str = "") -> Path:
"""
Convert hash to directory path using hex pairs.
Example (depth=9): abcdef123456... -> ab/cd/ef/12/34/56/78/9a/bc/abcdef123456...ext
"""
parts = [h[i:i+2] for i in range(0, depth * 2, 2)]
if ext and not ext.startswith('.'):
ext = '.' + ext
return Path(*parts) / f"{h}{ext}"
class Vault:
"""
Synchronous hash-based file storage.
Supports two modes:
1. Content-addressable: store(hash, data) - use content's MD5 as key
2. Seed-based: create_filename(seed) - deterministic path from arbitrary seed
Directory structure uses hex pairs for 256 subdirs per level:
vault/ab/cd/ef/.../hash.ext
"""
def __init__(
self,
vaultpath: str = "vault",
depth: int = 9,
salt: str = "neopig",
enable_memory_cache: bool = False,
):
"""
Initialize vault.
Args:
vaultpath: Base directory for storage
depth: Directory tree depth (default 9 = 18 hex chars used)
salt: Salt for seed-based hashing
enable_memory_cache: Cache file existence checks in memory
"""
self.vaultpath = Path(vaultpath)
self.depth = depth
self.salt = _ensure_bytes(salt)
self.enable_memory_cache = enable_memory_cache
self._cache: Optional[Dict[str, bool]] = {} if enable_memory_cache else None
def init(self) -> None:
"""Create vault base directory."""
self.vaultpath.mkdir(parents=True, exist_ok=True)
# Backwards compatibility
init_vault = init
# -------------------------------------------------------------------------
# Content-Addressable Storage (by hash)
# -------------------------------------------------------------------------
def _get_hash_path(self, h: str, ext: str = "") -> Path:
"""Get full path for a content hash."""
return self.vaultpath / _hash_to_path(h, self.depth, ext)
def _find_by_hash(self, h: str) -> Optional[Path]:
"""Find file by hash prefix (any extension)."""
subdir = self._get_hash_path(h).parent
if not subdir.exists():
return None
for f in subdir.iterdir():
if f.name.startswith(h):
return f
return None
def store(self, content_hash: str, data: bytes, ext: str = "") -> Path:
"""
Store data by content hash.
Args:
content_hash: MD5 hash of the data (32 hex chars)
data: Raw bytes to store
ext: File extension (e.g., ".jpg" or "jpg")
Returns:
Path to stored file
"""
filepath = self._get_hash_path(content_hash, ext)
filepath.parent.mkdir(parents=True, exist_ok=True)
with open(filepath, 'wb') as f:
fcntl.flock(f.fileno(), fcntl.LOCK_EX)
f.write(data)
f.flush()
os.fsync(f.fileno())
self._cache_set(str(filepath), True)
return filepath
def exists(self, content_hash: str) -> bool:
"""Check if content exists by hash."""
return self._find_by_hash(content_hash) is not None
def get(self, content_hash: str) -> Optional[bytes]:
"""Retrieve data by content hash."""
filepath = self._find_by_hash(content_hash)
if filepath is None:
return None
with open(filepath, 'rb') as f:
fcntl.flock(f.fileno(), fcntl.LOCK_SH)
return f.read()
def get_path(self, content_hash: str) -> Optional[Path]:
"""Get filesystem path for stored content."""
return self._find_by_hash(content_hash)
def delete(self, content_hash: str) -> bool:
"""Delete content by hash. Returns True if deleted."""
filepath = self._find_by_hash(content_hash)
if filepath is None:
return False
filepath.unlink()
self._cache_set(str(filepath), False)
return True
# -------------------------------------------------------------------------
# Seed-Based Storage (deterministic paths from arbitrary keys)
# -------------------------------------------------------------------------
def create_filename(self, seed: str, ext: str = "", absolute: bool = True) -> str:
"""
Create deterministic path from seed string.
Args:
seed: Arbitrary string to hash
ext: File extension
absolute: Include vault base path
Returns:
Path string
"""
h = sha256(_ensure_bytes(seed) + self.salt).hexdigest()
rel_path = _hash_to_path(h, self.depth, ext)
if absolute:
return str(self.vaultpath / rel_path)
return str(rel_path)
def create_random_filename(self, ext: str = "", absolute: bool = True) -> str:
"""Create random vault path."""
return self.create_filename(str(uuid4()), ext, absolute)
# -------------------------------------------------------------------------
# File Operations with Locking
# -------------------------------------------------------------------------
@contextlib.contextmanager
def file_lock(self, file_path: str, mode: str = "r"):
"""
Context manager for locked file access.
Args:
file_path: Path to file
mode: Open mode ('r', 'w', 'a', etc.)
Yields:
Locked file object
"""
p = Path(file_path)
if "w" in mode or "a" in mode:
p.parent.mkdir(parents=True, exist_ok=True)
lock_type = fcntl.LOCK_EX if ("w" in mode or "a" in mode) else fcntl.LOCK_SH
with open(file_path, mode) as f:
try:
fcntl.flock(f.fileno(), lock_type)
yield f
finally:
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
def write_bytes(self, file_path: str, data: bytes) -> None:
"""Write bytes with locking."""
p = Path(file_path)
p.parent.mkdir(parents=True, exist_ok=True)
with open(file_path, 'wb') as f:
fcntl.flock(f.fileno(), fcntl.LOCK_EX)
f.write(data)
f.flush()
os.fsync(f.fileno())
self._cache_set(file_path, True)
def read_bytes(self, file_path: str) -> Optional[bytes]:
"""Read bytes with locking."""
if not path.exists(file_path):
return None
with open(file_path, 'rb') as f:
fcntl.flock(f.fileno(), fcntl.LOCK_SH)
return f.read()
def write_text(self, file_path: str, content: str) -> None:
"""Write text with locking."""
p = Path(file_path)
p.parent.mkdir(parents=True, exist_ok=True)
with open(file_path, 'w') as f:
fcntl.flock(f.fileno(), fcntl.LOCK_EX)
f.write(content)
f.flush()
os.fsync(f.fileno())
self._cache_set(file_path, True)
def read_text(self, file_path: str, default: str = None) -> Optional[str]:
"""Read text with locking."""
if not path.exists(file_path):
return default
with open(file_path, 'r') as f:
fcntl.flock(f.fileno(), fcntl.LOCK_SH)
return f.read()
def write_json(self, file_path: str, data: Any, indent: int = 2) -> None:
"""Atomically write JSON with temp file + rename."""
p = Path(file_path)
p.parent.mkdir(parents=True, exist_ok=True)
fd, temp_path = tempfile.mkstemp(
dir=str(p.parent),
prefix=f".tmp_{p.name}_",
suffix=".json"
)
try:
with open(temp_path, 'w') as f:
fcntl.flock(f.fileno(), fcntl.LOCK_EX)
json.dump(data, f, indent=indent)
f.flush()
os.fsync(f.fileno())
rename(temp_path, file_path)
self._cache_set(file_path, True)
except Exception:
try:
os.unlink(temp_path)
except OSError:
pass
raise
finally:
try:
os.close(fd)
except OSError:
pass
def read_json(self, file_path: str, default: Any = None) -> Any:
"""Read JSON with locking."""
if not path.exists(file_path):
return default
try:
with self.file_lock(file_path, 'r') as f:
return json.load(f)
except (json.JSONDecodeError, IOError):
return default
def remove(self, file_path: str) -> bool:
"""Remove file. Returns True if removed."""
if not path.exists(file_path):
self._cache_set(file_path, False)
return False
os.remove(file_path)
self._cache_set(file_path, False)
return True
# -------------------------------------------------------------------------
# Existence Cache
# -------------------------------------------------------------------------
def _cache_set(self, file_path: str, exists: bool) -> None:
"""Update existence cache."""
if self._cache is not None:
self._cache[file_path] = exists
def _cache_get(self, file_path: str) -> Optional[bool]:
"""Check existence cache."""
if self._cache is not None:
return self._cache.get(file_path)
return None
def file_exists(self, file_path: str) -> bool:
"""Check if file exists (uses cache if enabled)."""
cached = self._cache_get(file_path)
if cached is not None:
return cached
exists = path.exists(file_path)
self._cache_set(file_path, exists)
return exists
def clear_cache(self) -> None:
"""Clear existence cache."""
if self._cache is not None:
self._cache.clear()
# -------------------------------------------------------------------------
# Statistics
# -------------------------------------------------------------------------
def stats(self) -> Dict[str, Any]:
"""Get vault statistics."""
count = 0
total_size = 0
for f in self.vaultpath.rglob('*'):
if f.is_file():
count += 1
total_size += f.stat().st_size
return {
'path': str(self.vaultpath),
'depth': self.depth,
'count': count,
'total_size': total_size,
'cache_enabled': self.enable_memory_cache,
'cache_entries': len(self._cache) if self._cache else 0,
}
# Backwards compatibility aliases
atomic_write_json = write_json
safe_read_json = read_json
write_text_file = write_text
read_text_file = read_text
remove_file = remove
mark_file_exists = _cache_set
class AsyncVault:
"""
Asynchronous hash-based file storage.
Same API as Vault but with async/await. Uses aiofiles for I/O
and asyncio.to_thread for CPU-bound operations.
"""
def __init__(
self,
vaultpath: str = "vault",
depth: int = 9,
salt: str = "neopig",
enable_memory_cache: bool = False,
):
"""
Initialize async vault.
Args:
vaultpath: Base directory for storage
depth: Directory tree depth (default 9)
salt: Salt for seed-based hashing
enable_memory_cache: Cache file existence checks in memory
"""
self.vaultpath = Path(vaultpath)
self.depth = depth
self.salt = _ensure_bytes(salt)
self.enable_memory_cache = enable_memory_cache
self._cache: Optional[Dict[str, bool]] = {} if enable_memory_cache else None
self._initialized = False
async def init(self) -> None:
"""Create vault base directory."""
if self._initialized:
return
await aiofiles.os.makedirs(self.vaultpath, exist_ok=True)
self._initialized = True
# -------------------------------------------------------------------------
# Content-Addressable Storage (by hash)
# -------------------------------------------------------------------------
def _get_hash_path(self, h: str, ext: str = "") -> Path:
"""Get full path for a content hash."""
return self.vaultpath / _hash_to_path(h, self.depth, ext)
async def _find_by_hash(self, h: str) -> Optional[Path]:
"""Find file by hash prefix (any extension)."""
subdir = self._get_hash_path(h).parent
def _find():
if not subdir.exists():
return None
for f in subdir.iterdir():
if f.name.startswith(h):
return f
return None
return await asyncio.to_thread(_find)
async def store(self, content_hash: str, data: bytes, ext: str = "") -> Path:
"""
Store data by content hash.
Args:
content_hash: MD5 hash of the data (32 hex chars)
data: Raw bytes to store
ext: File extension (e.g., ".jpg" or "jpg")
Returns:
Path to stored file
"""
filepath = self._get_hash_path(content_hash, ext)
await aiofiles.os.makedirs(filepath.parent, exist_ok=True)
async with aiofiles.open(filepath, 'wb') as f:
await f.write(data)
self._cache_set(str(filepath), True)
return filepath
async def exists(self, content_hash: str) -> bool:
"""Check if content exists by hash."""
return await self._find_by_hash(content_hash) is not None
async def get(self, content_hash: str) -> Optional[bytes]:
"""Retrieve data by content hash."""
filepath = await self._find_by_hash(content_hash)
if filepath is None:
return None
async with aiofiles.open(filepath, 'rb') as f:
return await f.read()
async def get_path(self, content_hash: str) -> Optional[Path]:
"""Get filesystem path for stored content."""
return await self._find_by_hash(content_hash)
async def delete(self, content_hash: str) -> bool:
"""Delete content by hash. Returns True if deleted."""
filepath = await self._find_by_hash(content_hash)
if filepath is None:
return False
await aiofiles.os.remove(filepath)
self._cache_set(str(filepath), False)
return True
# -------------------------------------------------------------------------
# Seed-Based Storage
# -------------------------------------------------------------------------
def create_filename(self, seed: str, ext: str = "", absolute: bool = True) -> str:
"""Create deterministic path from seed string."""
h = sha256(_ensure_bytes(seed) + self.salt).hexdigest()
rel_path = _hash_to_path(h, self.depth, ext)
if absolute:
return str(self.vaultpath / rel_path)
return str(rel_path)
def create_random_filename(self, ext: str = "", absolute: bool = True) -> str:
"""Create random vault path."""
return self.create_filename(str(uuid4()), ext, absolute)
# -------------------------------------------------------------------------
# File Operations
# -------------------------------------------------------------------------
async def write_bytes(self, file_path: str, data: bytes) -> None:
"""Write bytes."""
p = Path(file_path)
await aiofiles.os.makedirs(p.parent, exist_ok=True)
async with aiofiles.open(file_path, 'wb') as f:
await f.write(data)
self._cache_set(file_path, True)
async def read_bytes(self, file_path: str) -> Optional[bytes]:
"""Read bytes."""
if not await aiofiles.os.path.exists(file_path):
return None
async with aiofiles.open(file_path, 'rb') as f:
return await f.read()
async def write_text(self, file_path: str, content: str) -> None:
"""Write text."""
p = Path(file_path)
await aiofiles.os.makedirs(p.parent, exist_ok=True)
async with aiofiles.open(file_path, 'w') as f:
await f.write(content)
self._cache_set(file_path, True)
async def read_text(self, file_path: str, default: str = None) -> Optional[str]:
"""Read text."""
if not await aiofiles.os.path.exists(file_path):
return default
async with aiofiles.open(file_path, 'r') as f:
return await f.read()
async def write_json(self, file_path: str, data: Any, indent: int = 2) -> None:
"""Atomically write JSON."""
p = Path(file_path)
await aiofiles.os.makedirs(p.parent, exist_ok=True)
# Create temp file
fd, temp_path = await asyncio.to_thread(
tempfile.mkstemp,
dir=str(p.parent),
prefix=f".tmp_{p.name}_",
suffix=".json"
)
try:
async with aiofiles.open(temp_path, 'w') as f:
await f.write(json.dumps(data, indent=indent))
await asyncio.to_thread(rename, temp_path, file_path)
self._cache_set(file_path, True)
except Exception:
try:
await aiofiles.os.remove(temp_path)
except OSError:
pass
raise
finally:
try:
os.close(fd)
except OSError:
pass
async def read_json(self, file_path: str, default: Any = None) -> Any:
"""Read JSON."""
if not await aiofiles.os.path.exists(file_path):
return default
try:
async with aiofiles.open(file_path, 'r') as f:
return json.loads(await f.read())
except (json.JSONDecodeError, IOError):
return default
async def remove(self, file_path: str) -> bool:
"""Remove file. Returns True if removed."""
if not await aiofiles.os.path.exists(file_path):
self._cache_set(file_path, False)
return False
await aiofiles.os.remove(file_path)
self._cache_set(file_path, False)
return True
# -------------------------------------------------------------------------
# Existence Cache
# -------------------------------------------------------------------------
def _cache_set(self, file_path: str, exists: bool) -> None:
"""Update existence cache."""
if self._cache is not None:
self._cache[file_path] = exists
def _cache_get(self, file_path: str) -> Optional[bool]:
"""Check existence cache."""
if self._cache is not None:
return self._cache.get(file_path)
return None
async def file_exists(self, file_path: str) -> bool:
"""Check if file exists (uses cache if enabled)."""
cached = self._cache_get(file_path)
if cached is not None:
return cached
exists = await aiofiles.os.path.exists(file_path)
self._cache_set(file_path, exists)
return exists
def clear_cache(self) -> None:
"""Clear existence cache."""
if self._cache is not None:
self._cache.clear()
# -------------------------------------------------------------------------
# Statistics
# -------------------------------------------------------------------------
async def stats(self) -> Dict[str, Any]:
"""Get vault statistics."""
def _count():
count = 0
total_size = 0
for f in self.vaultpath.rglob('*'):
if f.is_file():
count += 1
total_size += f.stat().st_size
return count, total_size
count, total_size = await asyncio.to_thread(_count)
return {
'path': str(self.vaultpath),
'depth': self.depth,
'count': count,
'total_size': total_size,
'cache_enabled': self.enable_memory_cache,
'cache_entries': len(self._cache) if self._cache else 0,
}
def content_hash(data: bytes) -> str:
"""Generate MD5 hash of bytes for content-addressable storage."""
return md5(data).hexdigest()
def hash_to_path(h: str, depth: int = 9, ext: str = "") -> Path:
"""
Convert hash to directory path using hex pairs.
Example (depth=9): abcdef123456... -> ab/cd/ef/12/34/56/78/9a/bc/abcdef123456...ext
"""
return _hash_to_path(h, depth, ext)
# Backward compatibility exports
ensure_bytes = _ensure_bytes
def create_vault(*args, **kwargs) -> Vault:
"""Factory function for Vault (backward compatibility)."""
return Vault(*args, **kwargs)
def create_async_vault(*args, **kwargs) -> AsyncVault:
"""Factory function for AsyncVault (backward compatibility)."""
return AsyncVault(*args, **kwargs)