russell@unturf.com
05b39054b8
requirements.txt used >= floors and installed with no hashes, so every CI run and install re-resolved to whatever PyPI served. A poisoned release of any dep (fastapi, aiohttp, lxml, pillow...) would land unverified. - requirements.lock: 42 pkgs pinned to exact versions + SHA256 (1258 hashes), uv (targeted py3.11 to match CI image) - Makefile install + CI: pip install --require-hashes -r requirements.lock - requirements.txt stays the loose source; make pins-lock regenerates Validated: lock installs under --require-hashes, 460 tests pass under resolved versions.
106 lines
3.6 KiB
Makefile
106 lines
3.6 KiB
Makefile
.PHONY: venv install test crawl serp clean vendor-uri2png vendor-install archive bootstrap test-alpha
|
|
|
|
VENV := .venv
|
|
PYTHON := $(VENV)/bin/python
|
|
PIP := $(VENV)/bin/pip
|
|
|
|
venv:
|
|
python3 -m venv $(VENV)
|
|
$(PIP) install --upgrade pip
|
|
|
|
# Supply-chain: install from requirements.lock (exact versions + SHA256,
|
|
# --require-hashes). requirements.txt stays the loose source; regenerate the lock
|
|
# with: make pins-lock
|
|
install: venv
|
|
$(PIP) install --require-hashes -r requirements.lock
|
|
|
|
# Regenerate requirements.lock from requirements.txt (latest compatible).
|
|
pins-lock:
|
|
uv pip compile --generate-hashes --upgrade --python-version 3.11 \
|
|
-o requirements.lock requirements.txt
|
|
|
|
test: install
|
|
$(PYTHON) -m pytest tests/ -v --tb=short
|
|
|
|
crawl: install
|
|
$(PYTHON) neopig.py $(ARGS)
|
|
|
|
archive: install
|
|
$(PYTHON) archive.py $(ARGS)
|
|
|
|
# Build bootstrap binary for self-extracting archives
|
|
bootstrap: scripts/bootstrap.c
|
|
gcc -O2 -Wall -o bootstrap scripts/bootstrap.c -lz
|
|
@echo "Built: bootstrap ($$(stat -c%s bootstrap 2>/dev/null || stat -f%z bootstrap) bytes)"
|
|
|
|
# Create self-extracting .run from a tarball
|
|
# Usage: make run TARBALL=example.tar.gz
|
|
run: bootstrap
|
|
ifndef TARBALL
|
|
$(error TARBALL not set. Usage: make run TARBALL=path/to/archive.tar.gz)
|
|
endif
|
|
@OUTNAME=$$(basename "$(TARBALL)" .tar.gz).run; \
|
|
BOOTSTRAP_SIZE=$$(stat -c%s bootstrap 2>/dev/null || stat -f%z bootstrap); \
|
|
cat bootstrap "$(TARBALL)" > "$$OUTNAME"; \
|
|
echo -n "NEOPIG" >> "$$OUTNAME"; \
|
|
printf '%016x' "$$BOOTSTRAP_SIZE" >> "$$OUTNAME"; \
|
|
chmod +x "$$OUTNAME"; \
|
|
echo "Created: $$OUTNAME ($$(stat -c%s $$OUTNAME 2>/dev/null || stat -f%z $$OUTNAME) bytes)"
|
|
|
|
serp: install
|
|
$(PYTHON) serp.py --host 0.0.0.0 --port 31337
|
|
|
|
clean:
|
|
rm -rf $(VENV) __pycache__ *.pyc
|
|
rm -rf test_vault test_neopig.db
|
|
rm -f bootstrap *.run
|
|
|
|
# Vendor dependencies
|
|
vendor-uri2png:
|
|
@echo "Fetching uri2png..."
|
|
rm -rf vendor/uri2png
|
|
mkdir -p vendor
|
|
git clone --depth 1 https://github.com/russellballestrini/uri2png vendor/uri2png
|
|
rm -rf vendor/uri2png/.git
|
|
@echo "uri2png vendored at vendor/uri2png"
|
|
|
|
vendor-install: vendor-uri2png venv
|
|
$(PIP) install -e vendor/uri2png/python
|
|
$(PYTHON) -m playwright install
|
|
@echo "uri2png Python package installed"
|
|
|
|
# Combined server (SERP + screenshot)
|
|
server: vendor-install
|
|
$(PYTHON) serp.py --host 0.0.0.0 --port 31337
|
|
|
|
# Test tarball extraction and run embedded neopig server
|
|
# Usage: make test-alpha TARBALL=example.tar.gz
|
|
test-alpha:
|
|
ifndef TARBALL
|
|
$(error TARBALL not set. Usage: make test-alpha TARBALL=path/to/archive.tar.gz)
|
|
endif
|
|
rm -rf test-alpha
|
|
mkdir -p test-alpha
|
|
tar -xzf "$(TARBALL)" -C test-alpha --strip-components=1
|
|
@echo "Extracted to test-alpha/"
|
|
@echo "Vault structure:"
|
|
@find test-alpha/vault -type f 2>/dev/null | head -5
|
|
@echo "Starting neopig server..."
|
|
cd test-alpha && PYTHONPATH=neopig ../$(PYTHON) neopig/serp.py --db neopig.db --vault vault --host 0.0.0.0 --port 8001
|
|
|
|
# Examples:
|
|
# make install - create venv and install deps
|
|
# make test - run functional test
|
|
# make serp - start basic SERP server
|
|
# make server - start combined server (SERP + screenshot)
|
|
# make crawl ARGS="https://example.com rick morty --mode images"
|
|
# make archive ARGS="https://discourse-urho3d.github.io/"
|
|
# make bootstrap - build C bootstrap for self-extracting archives
|
|
# make clean - remove venv and test artifacts
|
|
# make vendor-uri2png - fetch uri2png into vendor/
|
|
# make vendor-install - install uri2png Python package with screenshot support
|
|
#
|
|
# Self-extracting archive:
|
|
# make archive ARGS="https://example.com"
|
|
# make run TARBALL=example.com-20251229.tar.gz
|
|
# ./example.com-20251229.run
|