Add archive.py with --serve flag for live crawl watching

- archive.py: New site archiver wrapper with embedded serve.py
- --serve flag starts SERP server alongside crawl for live viewing
- --fast mode for sites without robots.txt (no crawl delay)
- Symlink-based storage: domain views link to hash vault
- Tarball resolves symlinks to include only domain content
- Fixed bytes_downloaded/bytes_stored accounting for screenshots
- Data directory (data/) for databases and state files
- Standardized on - separator in filenames

Makefile

@@ -1,4 +1,4 @@
-.PHONY: venv install test crawl serp clean vendor-uri2png vendor-install
+.PHONY: venv install test crawl serp clean vendor-uri2png vendor-install archive bootstrap
 
 VENV := .venv
 PYTHON := $(VENV)/bin/python
@@ -17,12 +17,35 @@ test: install
 crawl: install
 	$(PYTHON) neopig.py $(ARGS)
 
+archive: install
+	$(PYTHON) archive.py $(ARGS)
+
+# Build bootstrap binary for self-extracting archives
+bootstrap: bootstrap.c
+	gcc -O2 -Wall -o bootstrap bootstrap.c -lz
+	@echo "Built: bootstrap ($$(stat -c%s bootstrap 2>/dev/null || stat -f%z bootstrap) bytes)"
+
+# Create self-extracting .run from a tarball
+# Usage: make run TARBALL=example.tar.gz
+run: bootstrap
+ifndef TARBALL
+	$(error TARBALL not set. Usage: make run TARBALL=path/to/archive.tar.gz)
+endif
+	@OUTNAME=$$(basename "$(TARBALL)" .tar.gz).run; \
+	BOOTSTRAP_SIZE=$$(stat -c%s bootstrap 2>/dev/null || stat -f%z bootstrap); \
+	cat bootstrap "$(TARBALL)" > "$$OUTNAME"; \
+	echo -n "NEOPIG" >> "$$OUTNAME"; \
+	printf '%016x' "$$BOOTSTRAP_SIZE" >> "$$OUTNAME"; \
+	chmod +x "$$OUTNAME"; \
+	echo "Created: $$OUTNAME ($$(stat -c%s $$OUTNAME 2>/dev/null || stat -f%z $$OUTNAME) bytes)"
+
 serp: install
 	$(PYTHON) serp.py --host 0.0.0.0 --port 8000
 
 clean:
 	rm -rf $(VENV) __pycache__ *.pyc
 	rm -rf test_vault test_neopig.db
+	rm -f bootstrap *.run
 
 # Vendor dependencies
 vendor-uri2png:
@@ -48,6 +71,13 @@ server: vendor-install
 # make serp             - start basic SERP server
 # make server           - start combined server (SERP + screenshot)
 # make crawl ARGS="https://example.com rick morty --mode images"
+# make archive ARGS="https://discourse-urho3d.github.io/"
+# make bootstrap        - build C bootstrap for self-extracting archives
 # make clean            - remove venv and test artifacts
 # make vendor-uri2png   - fetch uri2png into vendor/
 # make vendor-install   - install uri2png Python package with screenshot support
+#
+# Self-extracting archive:
+#   make archive ARGS="https://example.com"
+#   make run TARBALL=example.com-20251229.tar.gz
+#   ./example.com-20251229.run