diff --git a/Makefile b/Makefile index caa5f7a..bf48c62 100644 --- a/Makefile +++ b/Makefile @@ -327,16 +327,37 @@ test-crawler: bootstrap-crawler ## run only the lifted crawler tests # HEAD requests later. URL is required; DEPTH and MAX have safe defaults. CRAWL_DEPTH ?= 2 CRAWL_MAX ?= 0 -crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1] and ingest into $(DB) - @if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0 FAST=1]" >&2; exit 2; fi - $(ABORIST) --db $(DB) crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) $(if $(FAST),--fast,) --ingest +# Per-domain shard so `make query` (which reads $(SHARDS_DIR)) sees the +# crawled content. Shard filename derived from the seed URL's hostname: +# https://russell.ballestrini.net -> $(SHARDS_DIR)/crawl_russell_ballestrini_net.db +# Override with CRAWL_SHARD=... when you want a custom path. +crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1] into $(SHARDS_DIR)/crawl_.db + @if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0 FAST=1 CRAWL_SHARD=path]" >&2; exit 2; fi + @mkdir -p $(SHARDS_DIR) + @shard="$(CRAWL_SHARD)"; \ + if [ -z "$$shard" ]; then \ + domain=$$(echo "$(URL)" | sed -E 's,^https?://([^/]+).*$$,\1,' | tr '.' '_'); \ + shard="$(SHARDS_DIR)/crawl_$${domain}.db"; \ + fi; \ + echo " shard: $$shard" >&2; \ + $(ABORIST) --db "$$shard" crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) $(if $(FAST),--fast,) --ingest # Fast freshness probe: conditional HEAD per doc, classify fresh/stale/gone. # Send only If-None-Match + If-Modified-Since headers — server returns 304 # with no body when content unchanged. RECRAWL_LIMIT ?= 100 -recrawl-check: bootstrap-crawler ## conditional HEAD on ingested docs [DOMAIN=foo.com LIMIT=100] - $(ABORIST) --db $(DB) crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT) +# By default check every shard under $(SHARDS_DIR). Pass CRAWL_SHARD=path +# to scope to one. DOMAIN= filters to URLs containing the substring. +recrawl-check: bootstrap-crawler ## conditional HEAD per ingested doc [DOMAIN=x.com LIMIT=100 CRAWL_SHARD=path] + @if [ -n "$(CRAWL_SHARD)" ]; then \ + $(ABORIST) --db $(CRAWL_SHARD) crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \ + else \ + for db in $(SHARDS_DIR)/*.db; do \ + case "$$(basename $$db)" in qa.db|snapshots.db) continue;; esac; \ + echo " shard: $$db" >&2; \ + $(ABORIST) --db "$$db" crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \ + done; \ + fi DOT_SRCS := $(wildcard docs/diagrams/*.dot) DOT_PNGS := $(DOT_SRCS:.dot=.png)