diff --git a/Makefile b/Makefile index e70af52..c34468b 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ VERIFY_N ?= 10 SEARCH_Q ?= computer .PHONY: all bootstrap fetch fetch-cur fetch-old ingest ingest-cur ingest-old \ + ingest-grok ingest-grok-media \ verify search stats test clean clean-db clean-data help all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats @@ -152,6 +153,29 @@ distill-shards-parallel: bootstrap ## one distill process per shard (parallel) ingest: ingest-cur ## default ingest = cur (use ingest-old or *-parallel for full) +# Grok account-export ETL. +# Point GROK_EXPORT at the directory xAI delivered (the one containing +# `ttl/30d/export_data//prod-grok-backend.json`). The source class +# auto-walks down to find the JSON. +GROK_EXPORT ?= $(HOME)/Downloads/ab8ef1f0-0d08-4f87-89c2-d4509e18115b +ingest-grok: bootstrap ## ingest Grok conversations into $(DB) (single-DB mode) + $(ABORIST) --db $(DB) ingest --source grok_export --path $(GROK_EXPORT) + +ingest-grok-media: bootstrap ## ingest Grok media-generation prompts into $(DB) + $(ABORIST) --db $(DB) ingest --source grok_media --path $(GROK_EXPORT) + +# Grok lives in its own shard inside the attach-forever cluster so cross- +# shard queries (`make query`) see it alongside the Wikipedia 2003 corpus. +# Single shard (rank 0/1) — Grok conversations are private and small. +GROK_SHARD := $(SHARDS_DIR)/grok.db +ingest-grok-attached: bootstrap ## ingest Grok conversations into $(GROK_SHARD) + @mkdir -p $(SHARDS_DIR) + $(ABORIST) --db $(GROK_SHARD) ingest --source grok_export --path $(GROK_EXPORT) --resume + +ingest-grok-media-attached: bootstrap ## ingest Grok media prompts into $(GROK_SHARD) + @mkdir -p $(SHARDS_DIR) + $(ABORIST) --db $(GROK_SHARD) ingest --source grok_media --path $(GROK_EXPORT) --resume + verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents $(ABORIST) --db $(DB) verify -n $(VERIFY_N) diff --git a/aborist/qa/query.py b/aborist/qa/query.py index cd90706..acaafb4 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -62,12 +62,18 @@ from aborist.store import ( _TITLE_TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9]*") +# Kept in sync with FTS5 stopwords in aborist.search.fts5 — both filter +# the same set of question-shaping words. "tell" leaking into title-LIKE +# search caused "tell me about permacomputer" to pull Tell_(poker), the +# Tell-Tale_Heart movie, Tell_City Indiana, etc. _TITLE_STOPWORDS = frozenset( """ the a an is are was were be been being of to in on at for with by from - as about into through during and or but not no nor so yet what who where - when why how which this that these those such i you he she it we they - me him her us them do does did have has had can could should would will + as about into through during and or but not no nor so yet too very also just + what who where when why how which this that these those such i you he she + it we they me him her us them do does did have has had can could should + would will may might + tell show describe explain summarize say give list find make please """.split() )