From cf103fad780be7297f931238ecccc53478b39f20 Mon Sep 17 00:00:00 2001
From: "russell@unturf.com" <russell@unturf.com>
Date: Tue, 28 Apr 2026 21:18:48 -0400
Subject: [PATCH] crawler: write to a per-domain shard under SHARDS_DIR (was:
 aborist.db)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Defect found by fox: `make crawl-ingest URL=...` wrote to
~/.aborist/aborist.db (the legacy single-DB path) but `make query Q=...`
searches ~/.aborist/shards/ — two different corpora, query never saw
the crawled content.

Fix mirrors the existing shard-aware ingest pattern (ingest-self,
ingest-git, ingest-grok-attached): derive a shard filename from the
seed URL's hostname and write into $(SHARDS_DIR). Override with
CRAWL_SHARD=path for custom layout.

  https://russell.ballestrini.net
  -> $(SHARDS_DIR)/crawl_russell_ballestrini_net.db

After this, `make query Q=...` (cross-shard search) automatically
picks up newly-crawled domains. Each domain stays in its own shard
file so falsifications, evictions, and deletions are scoped naturally.

recrawl-check gets the same treatment: defaults to walking every shard
under SHARDS_DIR (skipping qa.db and snapshots.db); CRAWL_SHARD=path
scopes to one when you want.

Note for fox: existing pages already at ~/.aborist/aborist.db from
the prior crawl are still there (harmless). To get them under query,
either re-crawl into the new path or migrate via sqlite ATTACH +
INSERT — a follow-up if you want it.
---
 Makefile | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index caa5f7a..bf48c62 100644
--- a/Makefile
+++ b/Makefile
@@ -327,16 +327,37 @@ test-crawler: bootstrap-crawler ## run only the lifted crawler tests
 # HEAD requests later. URL is required; DEPTH and MAX have safe defaults.
 CRAWL_DEPTH ?= 2
 CRAWL_MAX   ?= 0
-crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1] and ingest into $(DB)
-	@if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0 FAST=1]" >&2; exit 2; fi
-	$(ABORIST) --db $(DB) crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) $(if $(FAST),--fast,) --ingest
+# Per-domain shard so `make query` (which reads $(SHARDS_DIR)) sees the
+# crawled content. Shard filename derived from the seed URL's hostname:
+#   https://russell.ballestrini.net  ->  $(SHARDS_DIR)/crawl_russell_ballestrini_net.db
+# Override with CRAWL_SHARD=... when you want a custom path.
+crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1] into $(SHARDS_DIR)/crawl_<domain>.db
+	@if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0 FAST=1 CRAWL_SHARD=path]" >&2; exit 2; fi
+	@mkdir -p $(SHARDS_DIR)
+	@shard="$(CRAWL_SHARD)"; \
+	if [ -z "$$shard" ]; then \
+	  domain=$$(echo "$(URL)" | sed -E 's,^https?://([^/]+).*$$,\1,' | tr '.' '_'); \
+	  shard="$(SHARDS_DIR)/crawl_$${domain}.db"; \
+	fi; \
+	echo "  shard: $$shard" >&2; \
+	$(ABORIST) --db "$$shard" crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) $(if $(FAST),--fast,) --ingest
 
 # Fast freshness probe: conditional HEAD per doc, classify fresh/stale/gone.
 # Send only If-None-Match + If-Modified-Since headers — server returns 304
 # with no body when content unchanged.
 RECRAWL_LIMIT ?= 100
-recrawl-check: bootstrap-crawler ## conditional HEAD on ingested docs [DOMAIN=foo.com LIMIT=100]
-	$(ABORIST) --db $(DB) crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT)
+# By default check every shard under $(SHARDS_DIR). Pass CRAWL_SHARD=path
+# to scope to one. DOMAIN= filters to URLs containing the substring.
+recrawl-check: bootstrap-crawler ## conditional HEAD per ingested doc [DOMAIN=x.com LIMIT=100 CRAWL_SHARD=path]
+	@if [ -n "$(CRAWL_SHARD)" ]; then \
+	  $(ABORIST) --db $(CRAWL_SHARD) crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \
+	else \
+	  for db in $(SHARDS_DIR)/*.db; do \
+	    case "$$(basename $$db)" in qa.db|snapshots.db) continue;; esac; \
+	    echo "  shard: $$db" >&2; \
+	    $(ABORIST) --db "$$db" crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \
+	  done; \
+	fi
 
 DOT_SRCS := $(wildcard docs/diagrams/*.dot)
 DOT_PNGS := $(DOT_SRCS:.dot=.png)