Russell Ballestrini
bc11c516b6
New files: - filevault.py: Hash-based file storage with use_pairs option (v1.1.0) - async_filevault.py: Async wrapper using asyncio.to_thread() - domain_vault.py: Triple vault system for web archival (HTML, Media, Linkpeek) - screenshot.py: Async screenshot capture using uri2png - tests/unit/: Comprehensive test suite (85 tests) Sync-to-async conversions: - storage.py: Wrap Path operations in asyncio.to_thread() - domain_vault.py: Wrap exists(), mkdir(), rglob(), os.walk() in asyncio.to_thread() - screenshot.py: Wrap read_bytes(), write_bytes(), unlink() in asyncio.to_thread() All sync filesystem operations now run in thread pool to avoid blocking async loop.
53 lines
1.5 KiB
Makefile
53 lines
1.5 KiB
Makefile
.PHONY: venv install test crawl serp clean vendor-uri2png vendor-install
|
|
|
|
VENV := .venv
|
|
PYTHON := $(VENV)/bin/python
|
|
PIP := $(VENV)/bin/pip
|
|
|
|
venv:
|
|
python3 -m venv $(VENV)
|
|
$(PIP) install --upgrade pip
|
|
|
|
install: venv
|
|
$(PIP) install -r requirements.txt
|
|
|
|
test: install
|
|
$(PYTHON) test_crawl.py
|
|
|
|
crawl: install
|
|
$(PYTHON) neopig.py $(ARGS)
|
|
|
|
serp: install
|
|
$(PYTHON) serp.py --host 0.0.0.0 --port 8000
|
|
|
|
clean:
|
|
rm -rf $(VENV) __pycache__ *.pyc
|
|
rm -rf test_vault test_neopig.db
|
|
|
|
# Vendor dependencies
|
|
vendor-uri2png:
|
|
@echo "Fetching uri2png..."
|
|
rm -rf vendor/uri2png
|
|
mkdir -p vendor
|
|
git clone --depth 1 https://github.com/russellballestrini/uri2png vendor/uri2png
|
|
rm -rf vendor/uri2png/.git
|
|
@echo "uri2png vendored at vendor/uri2png"
|
|
|
|
vendor-install: vendor-uri2png venv
|
|
$(PIP) install -e vendor/uri2png/python
|
|
$(PYTHON) -m playwright install
|
|
@echo "uri2png Python package installed"
|
|
|
|
# Combined server (SERP + screenshot)
|
|
server: vendor-install
|
|
$(PYTHON) serp.py --host 0.0.0.0 --port 8000
|
|
|
|
# Examples:
|
|
# make install - create venv and install deps
|
|
# make test - run functional test
|
|
# make serp - start basic SERP server
|
|
# make server - start combined server (SERP + screenshot)
|
|
# make crawl ARGS="https://example.com rick morty --mode images"
|
|
# make clean - remove venv and test artifacts
|
|
# make vendor-uri2png - fetch uri2png into vendor/
|
|
# make vendor-install - install uri2png Python package with screenshot support
|