Russell Ballestrini
21e344f675
- archive.py: New site archiver wrapper with embedded serve.py - --serve flag starts SERP server alongside crawl for live viewing - --fast mode for sites without robots.txt (no crawl delay) - Symlink-based storage: domain views link to hash vault - Tarball resolves symlinks to include only domain content - Fixed bytes_downloaded/bytes_stored accounting for screenshots - Data directory (data/) for databases and state files - Standardized on - separator in filenames
83 lines
2.6 KiB
Makefile
83 lines
2.6 KiB
Makefile
.PHONY: venv install test crawl serp clean vendor-uri2png vendor-install archive bootstrap
|
|
|
|
VENV := .venv
|
|
PYTHON := $(VENV)/bin/python
|
|
PIP := $(VENV)/bin/pip
|
|
|
|
venv:
|
|
python3 -m venv $(VENV)
|
|
$(PIP) install --upgrade pip
|
|
|
|
install: venv
|
|
$(PIP) install -r requirements.txt
|
|
|
|
test: install
|
|
$(PYTHON) test_crawl.py
|
|
|
|
crawl: install
|
|
$(PYTHON) neopig.py $(ARGS)
|
|
|
|
archive: install
|
|
$(PYTHON) archive.py $(ARGS)
|
|
|
|
# Build bootstrap binary for self-extracting archives
|
|
bootstrap: bootstrap.c
|
|
gcc -O2 -Wall -o bootstrap bootstrap.c -lz
|
|
@echo "Built: bootstrap ($$(stat -c%s bootstrap 2>/dev/null || stat -f%z bootstrap) bytes)"
|
|
|
|
# Create self-extracting .run from a tarball
|
|
# Usage: make run TARBALL=example.tar.gz
|
|
run: bootstrap
|
|
ifndef TARBALL
|
|
$(error TARBALL not set. Usage: make run TARBALL=path/to/archive.tar.gz)
|
|
endif
|
|
@OUTNAME=$$(basename "$(TARBALL)" .tar.gz).run; \
|
|
BOOTSTRAP_SIZE=$$(stat -c%s bootstrap 2>/dev/null || stat -f%z bootstrap); \
|
|
cat bootstrap "$(TARBALL)" > "$$OUTNAME"; \
|
|
echo -n "NEOPIG" >> "$$OUTNAME"; \
|
|
printf '%016x' "$$BOOTSTRAP_SIZE" >> "$$OUTNAME"; \
|
|
chmod +x "$$OUTNAME"; \
|
|
echo "Created: $$OUTNAME ($$(stat -c%s $$OUTNAME 2>/dev/null || stat -f%z $$OUTNAME) bytes)"
|
|
|
|
serp: install
|
|
$(PYTHON) serp.py --host 0.0.0.0 --port 8000
|
|
|
|
clean:
|
|
rm -rf $(VENV) __pycache__ *.pyc
|
|
rm -rf test_vault test_neopig.db
|
|
rm -f bootstrap *.run
|
|
|
|
# Vendor dependencies
|
|
vendor-uri2png:
|
|
@echo "Fetching uri2png..."
|
|
rm -rf vendor/uri2png
|
|
mkdir -p vendor
|
|
git clone --depth 1 https://github.com/russellballestrini/uri2png vendor/uri2png
|
|
rm -rf vendor/uri2png/.git
|
|
@echo "uri2png vendored at vendor/uri2png"
|
|
|
|
vendor-install: vendor-uri2png venv
|
|
$(PIP) install -e vendor/uri2png/python
|
|
$(PYTHON) -m playwright install
|
|
@echo "uri2png Python package installed"
|
|
|
|
# Combined server (SERP + screenshot)
|
|
server: vendor-install
|
|
$(PYTHON) serp.py --host 0.0.0.0 --port 8000
|
|
|
|
# Examples:
|
|
# make install - create venv and install deps
|
|
# make test - run functional test
|
|
# make serp - start basic SERP server
|
|
# make server - start combined server (SERP + screenshot)
|
|
# make crawl ARGS="https://example.com rick morty --mode images"
|
|
# make archive ARGS="https://discourse-urho3d.github.io/"
|
|
# make bootstrap - build C bootstrap for self-extracting archives
|
|
# make clean - remove venv and test artifacts
|
|
# make vendor-uri2png - fetch uri2png into vendor/
|
|
# make vendor-install - install uri2png Python package with screenshot support
|
|
#
|
|
# Self-extracting archive:
|
|
# make archive ARGS="https://example.com"
|
|
# make run TARBALL=example.com-20251229.tar.gz
|
|
# ./example.com-20251229.run
|