rag: rank cross-page context by BM25 relevance, drop unrelated pages

public/src/storage.js

@@ -373,9 +373,40 @@ export function loadPageSummaries() {
 	return Object.values(knowledge);
 }
 
+// Lightweight English stopwords: dropped before BM25 scoring so common
+// words don't inflate relevance. Not exhaustive on purpose: just the
+// high-frequency noise that would otherwise match every page.
+const BM25_STOPWORDS = new Set([
+	"the", "a", "an", "and", "or", "but", "of", "to", "in", "on", "at",
+	"for", "with", "by", "from", "as", "is", "are", "was", "were", "be",
+	"been", "being", "this", "that", "these", "those", "it", "its", "we",
+	"you", "your", "our", "i", "he", "she", "they", "them", "his", "her",
+	"will", "would", "can", "could", "should", "may", "might", "do", "does",
+	"did", "has", "have", "had", "not", "no", "if", "then", "than", "so",
+	"about", "into", "over", "more", "most", "some", "such", "page", "site",
+]);
+
+// Tokenize text into lowercased alphanumeric terms, minus stopwords and
+// single characters. Shared by both the query and candidate documents.
+function bm25Tokenize(text) {
+	return String(text || "")
+		.toLowerCase()
+		.split(/[^a-z0-9]+/)
+		.filter((t) => t.length > 1 && !BM25_STOPWORDS.has(t));
+}
+
 // Get relevant page summaries that fit within a token budget.
-// Prioritizes pages by keyword overlap with current page, then recency.
-export function getRelevantPageSummaries(currentUrl, budget) {
+// When queryText (the current page's title/topics/content) is supplied,
+// candidates are ranked by BM25 relevance to that query and only pages
+// scoring at or above `threshold` are kept: this avoids prefilling the
+// prompt with recently-visited-but-unrelated pages. Without queryText we
+// fall back to recency ordering for backward compatibility.
+export function getRelevantPageSummaries(
+	currentUrl,
+	budget,
+	queryText = "",
+	threshold = 2.0,
+) {
 	const summaries = loadPageSummaries();
 	if (summaries.length === 0 || budget <= 0) return { text: "", count: 0 };
 
@@ -383,15 +414,69 @@ export function getRelevantPageSummaries(currentUrl, budget) {
 	const candidates = summaries.filter((s) => s.url !== currentUrl);
 	if (candidates.length === 0) return { text: "", count: 0 };
 
-	// Sort by recency (most recent first)
-	candidates.sort((a, b) => (b.indexedAt || 0) - (a.indexedAt || 0));
+	const queryTerms = bm25Tokenize(queryText);
 
-	// Pack summaries within budget (~4 chars per token)
+	if (queryTerms.length > 0) {
+		// Build the candidate corpus: each doc is title + topics + summary.
+		const docs = candidates.map((page) => {
+			const topics = Array.isArray(page.topics) ? page.topics.join(" ") : "";
+			const terms = bm25Tokenize(`${page.title} ${topics} ${page.summary}`);
+			const freq = new Map();
+			for (const t of terms) freq.set(t, (freq.get(t) || 0) + 1);
+			return { page, terms, freq, len: terms.length };
+		});
+
+		const N = docs.length;
+		const avgdl =
+			docs.reduce((sum, d) => sum + d.len, 0) / Math.max(1, N);
+
+		// Document frequency per query term (over the candidate corpus).
+		const df = new Map();
+		for (const term of new Set(queryTerms)) {
+			let n = 0;
+			for (const d of docs) if (d.freq.has(term)) n++;
+			df.set(term, n);
+		}
+
+		// BM25 score per doc against the query.
+		const k1 = 1.5;
+		const b = 0.75;
+		for (const d of docs) {
+			let score = 0;
+			for (const term of new Set(queryTerms)) {
+				const f = d.freq.get(term) || 0;
+				if (f === 0) continue;
+				const n = df.get(term) || 0;
+				const idf = Math.log(1 + (N - n + 0.5) / (n + 0.5));
+				const denom = f + k1 * (1 - b + (b * d.len) / avgdl);
+				score += idf * ((f * (k1 + 1)) / denom);
+			}
+			d.score = score;
+		}
+
+		// Keep only relevant docs, best first.
+		const ranked = docs
+			.filter((d) => d.score >= threshold)
+			.sort((a, b) => b.score - a.score)
+			.map((d) => d.page);
+
+		if (ranked.length === 0) return { text: "", count: 0 };
+		return packSummaries(ranked, budget);
+	}
+
+	// No query: recency ordering (backward compatible fallback).
+	candidates.sort((a, b) => (b.indexedAt || 0) - (a.indexedAt || 0));
+	return packSummaries(candidates, budget);
+}
+
+// Pack ranked page summaries into a labeled block within a token budget
+// (~4 chars per token). Stops once the next entry would overflow.
+function packSummaries(pages, budget) {
 	let text = "";
 	let count = 0;
 	const charBudget = budget * 4;
 
-	for (const page of candidates) {
+	for (const page of pages) {
 		const entry = `\n- "${page.title}" (${page.url}): ${page.summary}`;
 		if (text.length + entry.length > charBudget) break;
 		text += entry;

public/src/uncloseai-embed-modal.js

@@ -2595,10 +2595,13 @@ You have complete knowledge of this page content and can reference any details,
 					const usedEstimate = Math.ceil((pageContent.length + computedContext.length + analysisContext.length) / 4) + 2000;
 					const spareBudget = Math.max(0, Math.floor((maxTok - usedEstimate) * 0.3));
 					if (spareBudget > 500) {
-						const backfill = getRelevantPageSummaries(window.location.href, spareBudget);
+						// Query = current page so BM25 keeps only related pages,
+						// not whatever was merely visited most recently.
+						const ragQuery = `${document.title} ${pageContent.slice(0, 2000)}`;
+						const backfill = getRelevantPageSummaries(window.location.href, spareBudget, ragQuery);
 						if (backfill.count > 0) {
 							indexedKnowledgeContext = backfill.text;
-							console.log(`Reverse RAG: backfilled ${backfill.count} page summaries (${spareBudget} token budget)`);
+							console.log(`Reverse RAG: backfilled ${backfill.count} relevant page summaries (BM25, ${spareBudget} token budget)`);
 						}
 					}
 				}