From 2834e990bdf15f0e10d3ae611616d1c9a0c3ba5c Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Fri, 22 May 2026 06:31:41 -0400 Subject: [PATCH] rag: rank cross-page context by BM25 relevance, drop unrelated pages --- public/src/storage.js | 97 +++++++++++++++++++++++++++-- public/src/uncloseai-embed-modal.js | 7 ++- 2 files changed, 96 insertions(+), 8 deletions(-) diff --git a/public/src/storage.js b/public/src/storage.js index dbfef8a..9aaf641 100644 --- a/public/src/storage.js +++ b/public/src/storage.js @@ -373,9 +373,40 @@ export function loadPageSummaries() { return Object.values(knowledge); } +// Lightweight English stopwords: dropped before BM25 scoring so common +// words don't inflate relevance. Not exhaustive on purpose: just the +// high-frequency noise that would otherwise match every page. +const BM25_STOPWORDS = new Set([ + "the", "a", "an", "and", "or", "but", "of", "to", "in", "on", "at", + "for", "with", "by", "from", "as", "is", "are", "was", "were", "be", + "been", "being", "this", "that", "these", "those", "it", "its", "we", + "you", "your", "our", "i", "he", "she", "they", "them", "his", "her", + "will", "would", "can", "could", "should", "may", "might", "do", "does", + "did", "has", "have", "had", "not", "no", "if", "then", "than", "so", + "about", "into", "over", "more", "most", "some", "such", "page", "site", +]); + +// Tokenize text into lowercased alphanumeric terms, minus stopwords and +// single characters. Shared by both the query and candidate documents. +function bm25Tokenize(text) { + return String(text || "") + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((t) => t.length > 1 && !BM25_STOPWORDS.has(t)); +} + // Get relevant page summaries that fit within a token budget. -// Prioritizes pages by keyword overlap with current page, then recency. -export function getRelevantPageSummaries(currentUrl, budget) { +// When queryText (the current page's title/topics/content) is supplied, +// candidates are ranked by BM25 relevance to that query and only pages +// scoring at or above `threshold` are kept: this avoids prefilling the +// prompt with recently-visited-but-unrelated pages. Without queryText we +// fall back to recency ordering for backward compatibility. +export function getRelevantPageSummaries( + currentUrl, + budget, + queryText = "", + threshold = 2.0, +) { const summaries = loadPageSummaries(); if (summaries.length === 0 || budget <= 0) return { text: "", count: 0 }; @@ -383,15 +414,69 @@ export function getRelevantPageSummaries(currentUrl, budget) { const candidates = summaries.filter((s) => s.url !== currentUrl); if (candidates.length === 0) return { text: "", count: 0 }; - // Sort by recency (most recent first) - candidates.sort((a, b) => (b.indexedAt || 0) - (a.indexedAt || 0)); + const queryTerms = bm25Tokenize(queryText); - // Pack summaries within budget (~4 chars per token) + if (queryTerms.length > 0) { + // Build the candidate corpus: each doc is title + topics + summary. + const docs = candidates.map((page) => { + const topics = Array.isArray(page.topics) ? page.topics.join(" ") : ""; + const terms = bm25Tokenize(`${page.title} ${topics} ${page.summary}`); + const freq = new Map(); + for (const t of terms) freq.set(t, (freq.get(t) || 0) + 1); + return { page, terms, freq, len: terms.length }; + }); + + const N = docs.length; + const avgdl = + docs.reduce((sum, d) => sum + d.len, 0) / Math.max(1, N); + + // Document frequency per query term (over the candidate corpus). + const df = new Map(); + for (const term of new Set(queryTerms)) { + let n = 0; + for (const d of docs) if (d.freq.has(term)) n++; + df.set(term, n); + } + + // BM25 score per doc against the query. + const k1 = 1.5; + const b = 0.75; + for (const d of docs) { + let score = 0; + for (const term of new Set(queryTerms)) { + const f = d.freq.get(term) || 0; + if (f === 0) continue; + const n = df.get(term) || 0; + const idf = Math.log(1 + (N - n + 0.5) / (n + 0.5)); + const denom = f + k1 * (1 - b + (b * d.len) / avgdl); + score += idf * ((f * (k1 + 1)) / denom); + } + d.score = score; + } + + // Keep only relevant docs, best first. + const ranked = docs + .filter((d) => d.score >= threshold) + .sort((a, b) => b.score - a.score) + .map((d) => d.page); + + if (ranked.length === 0) return { text: "", count: 0 }; + return packSummaries(ranked, budget); + } + + // No query: recency ordering (backward compatible fallback). + candidates.sort((a, b) => (b.indexedAt || 0) - (a.indexedAt || 0)); + return packSummaries(candidates, budget); +} + +// Pack ranked page summaries into a labeled block within a token budget +// (~4 chars per token). Stops once the next entry would overflow. +function packSummaries(pages, budget) { let text = ""; let count = 0; const charBudget = budget * 4; - for (const page of candidates) { + for (const page of pages) { const entry = `\n- "${page.title}" (${page.url}): ${page.summary}`; if (text.length + entry.length > charBudget) break; text += entry; diff --git a/public/src/uncloseai-embed-modal.js b/public/src/uncloseai-embed-modal.js index 5d329f0..b7bc103 100644 --- a/public/src/uncloseai-embed-modal.js +++ b/public/src/uncloseai-embed-modal.js @@ -2595,10 +2595,13 @@ You have complete knowledge of this page content and can reference any details, const usedEstimate = Math.ceil((pageContent.length + computedContext.length + analysisContext.length) / 4) + 2000; const spareBudget = Math.max(0, Math.floor((maxTok - usedEstimate) * 0.3)); if (spareBudget > 500) { - const backfill = getRelevantPageSummaries(window.location.href, spareBudget); + // Query = current page so BM25 keeps only related pages, + // not whatever was merely visited most recently. + const ragQuery = `${document.title} ${pageContent.slice(0, 2000)}`; + const backfill = getRelevantPageSummaries(window.location.href, spareBudget, ragQuery); if (backfill.count > 0) { indexedKnowledgeContext = backfill.text; - console.log(`Reverse RAG: backfilled ${backfill.count} page summaries (${spareBudget} token budget)`); + console.log(`Reverse RAG: backfilled ${backfill.count} relevant page summaries (BM25, ${spareBudget} token budget)`); } } }