engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

crawler: write to a per-domain shard under SHARDS_DIR (was: aborist.db)

Browse source 
Defect found by fox: `make crawl-ingest URL=...` wrote to
~/.aborist/aborist.db (the legacy single-DB path) but `make query Q=...`
searches ~/.aborist/shards/ — two different corpora, query never saw
the crawled content.

Fix mirrors the existing shard-aware ingest pattern (ingest-self,
ingest-git, ingest-grok-attached): derive a shard filename from the
seed URL's hostname and write into $(SHARDS_DIR). Override with
CRAWL_SHARD=path for custom layout.

  https://russell.ballestrini.net
  -> $(SHARDS_DIR)/crawl_russell_ballestrini_net.db

After this, `make query Q=...` (cross-shard search) automatically
picks up newly-crawled domains. Each domain stays in its own shard
file so falsifications, evictions, and deletions are scoped naturally.

recrawl-check gets the same treatment: defaults to walking every shard
under SHARDS_DIR (skipping qa.db and snapshots.db); CRAWL_SHARD=path
scopes to one when you want.

Note for fox: existing pages already at ~/.aborist/aborist.db from
the prior crawl are still there (harmless). To get them under query,
either re-crawl into the new path or migrate via sqlite ATTACH +
INSERT — a follow-up if you want it.
This commit is contained in:
russell@unturf.com 2026-04-28 21:18:48 -04:00
parent dfe39a0bf4
commit cf103fad78
No known key found for this signature in database
1 changed files with 26 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

31
Makefile
View file

@ -327,16 +327,37 @@ test-crawler: bootstrap-crawler ## run only the lifted crawler tests
# HEAD requests later. URL is required; DEPTH and MAX have safe defaults.
CRAWL_DEPTH ?= 2
CRAWL_MAX ?= 0
crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1] and ingest into $(DB)
@if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0 FAST=1]" >&2; exit 2; fi
$(ABORIST) --db $(DB) crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) $(if $(FAST),--fast,) --ingest
# Per-domain shard so `make query` (which reads $(SHARDS_DIR)) sees the
# crawled content. Shard filename derived from the seed URL's hostname:
# https://russell.ballestrini.net -> $(SHARDS_DIR)/crawl_russell_ballestrini_net.db
# Override with CRAWL_SHARD=... when you want a custom path.
crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1] into $(SHARDS_DIR)/crawl_<domain>.db
@if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0 FAST=1 CRAWL_SHARD=path]" >&2; exit 2; fi
@mkdir -p $(SHARDS_DIR)
@shard="$(CRAWL_SHARD)"; \
if [ -z "$$shard" ]; then \
domain=$$(echo "$(URL)" | sed -E 's,^https?://([^/]+).*$$,\1,' | tr '.' '_'); \
shard="$(SHARDS_DIR)/crawl_$${domain}.db"; \
fi; \
echo " shard: $$shard" >&2; \
$(ABORIST) --db "$$shard" crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) $(if $(FAST),--fast,) --ingest
# Fast freshness probe: conditional HEAD per doc, classify fresh/stale/gone.
# Send only If-None-Match + If-Modified-Since headers — server returns 304
# with no body when content unchanged.
RECRAWL_LIMIT ?= 100
recrawl-check: bootstrap-crawler ## conditional HEAD on ingested docs [DOMAIN=foo.com LIMIT=100]
$(ABORIST) --db $(DB) crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT)
# By default check every shard under $(SHARDS_DIR). Pass CRAWL_SHARD=path
# to scope to one. DOMAIN= filters to URLs containing the substring.
recrawl-check: bootstrap-crawler ## conditional HEAD per ingested doc [DOMAIN=x.com LIMIT=100 CRAWL_SHARD=path]
@if [ -n "$(CRAWL_SHARD)" ]; then \
$(ABORIST) --db $(CRAWL_SHARD) crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \
else \
for db in $(SHARDS_DIR)/*.db; do \
case "$$(basename $$db)" in qa.db|snapshots.db) continue;; esac; \
echo " shard: $$db" >&2; \
$(ABORIST) --db "$$db" crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \
done; \
fi
DOT_SRCS := $(wildcard docs/diagrams/*.dot)
DOT_PNGS := $(DOT_SRCS:.dot=.png)