implement reverse RAG, journey-aware greetings, vault viewer, crawl mode, unfirehose sharing (#004-#012)
This commit is contained in:
parent
fa0ad2782d
commit
a0a2327af6
14 changed files with 1363 additions and 29 deletions
|
|
@ -115,13 +115,37 @@ export const en = {
|
|||
translationModal: "🌐 Translation Modal",
|
||||
translationModalHeading: "Translation Modal",
|
||||
|
||||
// System prompt components
|
||||
// System prompt components: first visit (depth 0)
|
||||
systemPromptIntro: "You are Hermes AI, powered by Nous Research, embedded on a webpage via uncloseai.com. Your task is to generate a contextually intelligent 3-paragraph introduction that demonstrates deep understanding of the specific page content, its purpose, and its audience.",
|
||||
systemPromptTask1: "1. Open naturally by identifying what this page is about - be specific: mention the page title, main topic, author name if present, or key purpose. Show immediate understanding of the content type (article, documentation, product page, etc.)",
|
||||
systemPromptTask2: "2. Reference 2-3 highly specific details from the page - quote exact phrases, cite specific data points, mention particular sections or features you notice. Demonstrate that you've analyzed the actual content, not just skimmed it.",
|
||||
systemPromptTask3: "3. Offer contextually relevant assistance based on the page type - for technical docs, offer to explain concepts; for articles, offer to discuss ideas; for products, answer about features. Be specific about HOW you can help with THIS particular content.",
|
||||
systemPromptInstructions: "Be conversational yet knowledgeable. Ground your introduction in the actual page content. Avoid generic statements. Write naturally in",
|
||||
|
||||
// System prompt: returning visitor (depth 1-2, exploring the site)
|
||||
systemPromptReturning: "You are Hermes AI on uncloseai.com. The user has been browsing this site and just navigated to a new page. Do NOT re-introduce yourself or welcome them. They already know who you are. Instead, act like a knowledgeable companion who moved to this page with them.",
|
||||
systemPromptReturningTask1: "1. Note the transition naturally: what page they're on now and how it relates to where they were before. If the previous pages are listed, reference one briefly (e.g. 'Moving on from the homepage...' or 'From that article to this one...').",
|
||||
systemPromptReturningTask2: "2. Highlight what's interesting or notable about THIS page: key content, data points, or features that stand out. Be specific, reference actual text from the page.",
|
||||
systemPromptReturningTask3: "3. Briefly mention what you can help with here, specific to this page's content.",
|
||||
systemPromptReturningInstructions: "Write exactly 3 short paragraphs. No greeting, no self-introduction. You're already mid-conversation. Be natural, like a friend pointing things out as you browse together. Write in",
|
||||
|
||||
// System prompt: familiar visitor (depth 3-4, getting comfortable)
|
||||
systemPromptFamiliar: "You are Hermes AI on uncloseai.com. The user has been exploring this site for a while now and just landed on another page. You're deep in the browsing session together. No introductions, no formalities.",
|
||||
systemPromptFamiliarTask1: "1. Jump straight into what this page is about with a single observation that shows you understand the content.",
|
||||
systemPromptFamiliarTask2: "2. Connect this page to the user's journey: note a pattern, theme, or contrast with previous pages they've visited on the site.",
|
||||
systemPromptFamiliarInstructions: "Write exactly 2 short paragraphs. Casual, warm, like you're both deep in a research session together. No greeting. No 'welcome back'. You're just... here, browsing together. Write in",
|
||||
|
||||
// System prompt: deep dive (depth 5+, old friends exploring)
|
||||
systemPromptDeepDive: "You are Hermes AI on uncloseai.com. You and the user have been browsing this site together for a while. You're at page {depth} now. You know this site well at this point. Be an expert companion.",
|
||||
systemPromptDeepDiveTask1: "1. One punchy paragraph about this page: what's on it, how it fits into the larger site, and anything surprising or noteworthy. Reference specifics from the content.",
|
||||
systemPromptDeepDiveInstructions: "Write exactly 1 paragraph. Brief, insightful, like a friend who's been reading over your shoulder the whole time. You're not a tour guide anymore, you're a co-explorer. Write in",
|
||||
|
||||
// Loading messages per journey depth
|
||||
loadingFirstVisit: "Analyzing page and generating welcome message...",
|
||||
loadingReturning: "Reading page...",
|
||||
loadingFamiliar: "Checking this page out...",
|
||||
loadingDeepDive: "Got it...",
|
||||
|
||||
// More alerts and messages
|
||||
pleaseEnterText: "Please enter some text first!",
|
||||
pleaseSelectFile: "Please select a file first!",
|
||||
|
|
|
|||
|
|
@ -251,6 +251,31 @@ export function analyzeLinkTopology() {
|
|||
return result;
|
||||
}
|
||||
|
||||
// --- INTERNAL LINK URLS ---
|
||||
// Extract unique internal link URLs for knowledge graph (#008)
|
||||
export function extractInternalLinks() {
|
||||
const currentHost = window.location.hostname;
|
||||
const links = new Set();
|
||||
|
||||
for (const link of document.querySelectorAll("a[href]")) {
|
||||
try {
|
||||
const uri = new URL(link.href, window.location.origin);
|
||||
if (
|
||||
(uri.hostname === currentHost || uri.hostname === "") &&
|
||||
uri.pathname !== window.location.pathname &&
|
||||
!uri.hash &&
|
||||
!uri.pathname.match(/\.(jpg|jpeg|png|gif|svg|pdf|zip|mp3|mp4)$/i)
|
||||
) {
|
||||
links.add(uri.origin + uri.pathname);
|
||||
}
|
||||
} catch (e) {
|
||||
// skip malformed URLs
|
||||
}
|
||||
}
|
||||
|
||||
return [...links].slice(0, 100); // cap at 100 links per page
|
||||
}
|
||||
|
||||
// --- USER CONTEXT ---
|
||||
// Timezone, locale, local time, referrer, device type
|
||||
export function captureUserContext() {
|
||||
|
|
|
|||
|
|
@ -164,6 +164,386 @@ export async function initializeChatHistory() {
|
|||
];
|
||||
}
|
||||
|
||||
// Site journey tracking: per-domain navigation depth
|
||||
// Tracks how many pages a user has visited on a domain so the greeting
|
||||
// evolves from a full welcome (first page) to a brief page intro (later pages).
|
||||
// Journey data is domain-scoped, unlike chat history which is URL-scoped.
|
||||
|
||||
function getSiteDomain() {
|
||||
try {
|
||||
return window.location.hostname;
|
||||
} catch (e) {
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
function getJourneyKey() {
|
||||
const domain = getSiteDomain().replace(/[^a-zA-Z0-9]/g, "_");
|
||||
return `uncloseai-journey-${domain}`;
|
||||
}
|
||||
|
||||
export function loadSiteJourney() {
|
||||
const key = getJourneyKey();
|
||||
const raw = localStorage.getItem(key);
|
||||
const empty = { depth: 0, pages: [], firstVisit: null };
|
||||
if (!raw) return empty;
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (!parsed || typeof parsed !== "object") return empty;
|
||||
|
||||
// Encrypted wrapper
|
||||
if (parsed._encrypted === true) {
|
||||
const vault = getVault();
|
||||
if (!vault) return empty;
|
||||
const decrypted = vault.decrypt(parsed._data, vault._password);
|
||||
if (decrypted && typeof decrypted === "object" && !Array.isArray(decrypted)) {
|
||||
return decrypted;
|
||||
}
|
||||
return empty;
|
||||
}
|
||||
|
||||
// Plain object (not array, not encrypted)
|
||||
if (!Array.isArray(parsed) && typeof parsed.depth === "number") {
|
||||
return parsed;
|
||||
}
|
||||
|
||||
return empty;
|
||||
} catch (e) {
|
||||
return empty;
|
||||
}
|
||||
}
|
||||
|
||||
function saveSiteJourney(journey) {
|
||||
const key = getJourneyKey();
|
||||
|
||||
const vault = getVault();
|
||||
if (vault) {
|
||||
const encrypted = vault.encrypt(journey, vault._password);
|
||||
if (encrypted) {
|
||||
const wrapper = {
|
||||
_encrypted: true,
|
||||
_v: ENCRYPTED_STORAGE_VERSION,
|
||||
_data: encrypted,
|
||||
};
|
||||
localStorage.setItem(key, JSON.stringify(wrapper));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Journey metadata (titles/URLs) is low-sensitivity, save as plaintext
|
||||
// when vault is unavailable so greeting still evolves without a password
|
||||
localStorage.setItem(key, JSON.stringify(journey));
|
||||
}
|
||||
|
||||
// Record that the user visited the current page. Call after generating intro.
|
||||
// Keeps the last 20 pages per domain to avoid unbounded growth.
|
||||
export function recordPageVisit() {
|
||||
const journey = loadSiteJourney();
|
||||
const now = Date.now();
|
||||
const url = window.location.href;
|
||||
const title = document.title || window.location.pathname;
|
||||
|
||||
// Don't double-count the same URL
|
||||
if (journey.pages.some((p) => p.url === url)) {
|
||||
return journey;
|
||||
}
|
||||
|
||||
journey.depth += 1;
|
||||
journey.pages.push({ url, title, timestamp: now });
|
||||
if (!journey.firstVisit) journey.firstVisit = now;
|
||||
|
||||
// Keep only the last 20 pages to avoid unbounded growth
|
||||
if (journey.pages.length > 20) {
|
||||
journey.pages = journey.pages.slice(-20);
|
||||
}
|
||||
|
||||
saveSiteJourney(journey);
|
||||
return journey;
|
||||
}
|
||||
|
||||
// Format previous pages for inclusion in system prompt context
|
||||
export function formatJourneyContext(journey) {
|
||||
if (!journey || journey.depth === 0 || journey.pages.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const recentPages = journey.pages.slice(-5);
|
||||
const pageList = recentPages
|
||||
.map((p) => ` - "${p.title}" (${p.url})`)
|
||||
.join("\n");
|
||||
|
||||
return `\nSITE JOURNEY (${journey.depth} pages visited on ${getSiteDomain()}):\nPrevious pages the user has visited on this site:\n${pageList}\n`;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// REVERSE RAG: Page Knowledge Store (#006)
|
||||
// Stores page summaries per-domain for cross-page context.
|
||||
// Each summary is compact (~100-200 tokens) so many can fit
|
||||
// in spare context budget.
|
||||
// ============================================================
|
||||
|
||||
function getPageKnowledgeKey() {
|
||||
const domain = getSiteDomain().replace(/[^a-zA-Z0-9]/g, "_");
|
||||
return `uncloseai-knowledge-${domain}`;
|
||||
}
|
||||
|
||||
function loadPageKnowledgeRaw() {
|
||||
const key = getPageKnowledgeKey();
|
||||
const raw = localStorage.getItem(key);
|
||||
if (!raw) return {};
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (!parsed || typeof parsed !== "object") return {};
|
||||
|
||||
if (parsed._encrypted === true) {
|
||||
const vault = getVault();
|
||||
if (!vault) return {};
|
||||
const decrypted = vault.decrypt(parsed._data, vault._password);
|
||||
if (decrypted && typeof decrypted === "object" && !Array.isArray(decrypted)) {
|
||||
return decrypted;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
if (!Array.isArray(parsed)) return parsed;
|
||||
return {};
|
||||
} catch (e) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function savePageKnowledgeRaw(knowledge) {
|
||||
const key = getPageKnowledgeKey();
|
||||
|
||||
const vault = getVault();
|
||||
if (vault) {
|
||||
const encrypted = vault.encrypt(knowledge, vault._password);
|
||||
if (encrypted) {
|
||||
const wrapper = {
|
||||
_encrypted: true,
|
||||
_v: ENCRYPTED_STORAGE_VERSION,
|
||||
_data: encrypted,
|
||||
};
|
||||
localStorage.setItem(key, JSON.stringify(wrapper));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Page knowledge (titles, topics, summaries) is moderate-sensitivity.
|
||||
// Save plaintext if vault unavailable so reverse RAG still works.
|
||||
localStorage.setItem(key, JSON.stringify(knowledge));
|
||||
}
|
||||
|
||||
// Save a page summary after generating the intro message.
|
||||
// The intro response IS the summary: no extra LLM call needed.
|
||||
export function savePageSummary({ url, title, summary, topics, wordCount, internalLinks }) {
|
||||
const knowledge = loadPageKnowledgeRaw();
|
||||
const urlKey = url.replace(/[^a-zA-Z0-9]/g, "_");
|
||||
|
||||
knowledge[urlKey] = {
|
||||
url,
|
||||
title,
|
||||
summary: (summary || "").slice(0, 500), // cap summary text
|
||||
topics: (topics || []).slice(0, 10),
|
||||
wordCount: wordCount || 0,
|
||||
internalLinks: (internalLinks || []).slice(0, 50),
|
||||
indexedAt: Date.now(),
|
||||
};
|
||||
|
||||
// Cap at 50 pages per domain to avoid unbounded growth
|
||||
const keys = Object.keys(knowledge);
|
||||
if (keys.length > 50) {
|
||||
// Remove oldest entries
|
||||
const sorted = keys
|
||||
.map((k) => ({ k, t: knowledge[k].indexedAt || 0 }))
|
||||
.sort((a, b) => a.t - b.t);
|
||||
for (const entry of sorted.slice(0, keys.length - 50)) {
|
||||
delete knowledge[entry.k];
|
||||
}
|
||||
}
|
||||
|
||||
savePageKnowledgeRaw(knowledge);
|
||||
}
|
||||
|
||||
// Load all page summaries for this domain
|
||||
export function loadPageSummaries() {
|
||||
const knowledge = loadPageKnowledgeRaw();
|
||||
return Object.values(knowledge);
|
||||
}
|
||||
|
||||
// Get relevant page summaries that fit within a token budget.
|
||||
// Prioritizes pages by keyword overlap with current page, then recency.
|
||||
export function getRelevantPageSummaries(currentUrl, budget) {
|
||||
const summaries = loadPageSummaries();
|
||||
if (summaries.length === 0 || budget <= 0) return { text: "", count: 0 };
|
||||
|
||||
// Exclude current page
|
||||
const candidates = summaries.filter((s) => s.url !== currentUrl);
|
||||
if (candidates.length === 0) return { text: "", count: 0 };
|
||||
|
||||
// Sort by recency (most recent first)
|
||||
candidates.sort((a, b) => (b.indexedAt || 0) - (a.indexedAt || 0));
|
||||
|
||||
// Pack summaries within budget (~4 chars per token)
|
||||
let text = "";
|
||||
let count = 0;
|
||||
const charBudget = budget * 4;
|
||||
|
||||
for (const page of candidates) {
|
||||
const entry = `\n- "${page.title}" (${page.url}): ${page.summary}`;
|
||||
if (text.length + entry.length > charBudget) break;
|
||||
text += entry;
|
||||
count++;
|
||||
}
|
||||
|
||||
if (count === 0) return { text: "", count: 0 };
|
||||
|
||||
return {
|
||||
text: `\nINDEXED SITE KNOWLEDGE (${count} previously visited pages on ${getSiteDomain()}):\n${text}\n`,
|
||||
count,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// CONVERSATION SUMMARIES (#007)
|
||||
// Store conversation context per-page in the journey data.
|
||||
// Uses the intro message as initial summary. Updated as the
|
||||
// user has more exchanges on the page.
|
||||
// ============================================================
|
||||
|
||||
export function saveConversationSummary(url, summary) {
|
||||
const journey = loadSiteJourney();
|
||||
const page = journey.pages.find((p) => p.url === url);
|
||||
if (page) {
|
||||
page.conversationSummary = (summary || "").slice(0, 300);
|
||||
saveSiteJourney(journey);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// SITE LINK GRAPH (#008)
|
||||
// Stores which pages link to which on this domain.
|
||||
// Built from extractInternalLinks() in page-intelligence.js.
|
||||
// ============================================================
|
||||
|
||||
function getLinkGraphKey() {
|
||||
const domain = getSiteDomain().replace(/[^a-zA-Z0-9]/g, "_");
|
||||
return `uncloseai-linkgraph-${domain}`;
|
||||
}
|
||||
|
||||
export function saveSiteLinks(pageUrl, internalLinks) {
|
||||
const key = getLinkGraphKey();
|
||||
let graph = {};
|
||||
|
||||
const raw = localStorage.getItem(key);
|
||||
if (raw) {
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (parsed._encrypted === true) {
|
||||
const vault = getVault();
|
||||
if (vault) {
|
||||
const decrypted = vault.decrypt(parsed._data, vault._password);
|
||||
if (decrypted && typeof decrypted === "object") graph = decrypted;
|
||||
}
|
||||
} else if (typeof parsed === "object" && !Array.isArray(parsed)) {
|
||||
graph = parsed;
|
||||
}
|
||||
} catch (e) {
|
||||
graph = {};
|
||||
}
|
||||
}
|
||||
|
||||
graph[pageUrl] = {
|
||||
links: (internalLinks || []).slice(0, 50),
|
||||
scannedAt: Date.now(),
|
||||
};
|
||||
|
||||
// Cap at 100 pages
|
||||
const keys = Object.keys(graph);
|
||||
if (keys.length > 100) {
|
||||
const sorted = keys
|
||||
.map((k) => ({ k, t: graph[k].scannedAt || 0 }))
|
||||
.sort((a, b) => a.t - b.t);
|
||||
for (const entry of sorted.slice(0, keys.length - 100)) {
|
||||
delete graph[entry.k];
|
||||
}
|
||||
}
|
||||
|
||||
const vault = getVault();
|
||||
if (vault) {
|
||||
const encrypted = vault.encrypt(graph, vault._password);
|
||||
if (encrypted) {
|
||||
localStorage.setItem(key, JSON.stringify({
|
||||
_encrypted: true,
|
||||
_v: ENCRYPTED_STORAGE_VERSION,
|
||||
_data: encrypted,
|
||||
}));
|
||||
return;
|
||||
}
|
||||
}
|
||||
localStorage.setItem(key, JSON.stringify(graph));
|
||||
}
|
||||
|
||||
export function loadSiteLinks() {
|
||||
const key = getLinkGraphKey();
|
||||
const raw = localStorage.getItem(key);
|
||||
if (!raw) return {};
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (parsed._encrypted === true) {
|
||||
const vault = getVault();
|
||||
if (!vault) return {};
|
||||
const decrypted = vault.decrypt(parsed._data, vault._password);
|
||||
if (decrypted && typeof decrypted === "object") return decrypted;
|
||||
return {};
|
||||
}
|
||||
if (typeof parsed === "object" && !Array.isArray(parsed)) return parsed;
|
||||
return {};
|
||||
} catch (e) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
// Get site coverage stats for the vault viewer (#011)
|
||||
export function getSiteCoverageStats() {
|
||||
const journey = loadSiteJourney();
|
||||
const summaries = loadPageSummaries();
|
||||
const linkGraph = loadSiteLinks();
|
||||
|
||||
// Count unique discovered links (pages we know about but haven't visited)
|
||||
const visitedUrls = new Set(journey.pages.map((p) => p.url));
|
||||
const discoveredUrls = new Set();
|
||||
for (const page of Object.values(linkGraph)) {
|
||||
for (const link of page.links || []) {
|
||||
if (!visitedUrls.has(link)) discoveredUrls.add(link);
|
||||
}
|
||||
}
|
||||
|
||||
// Aggregate topics
|
||||
const topicCounts = {};
|
||||
for (const s of summaries) {
|
||||
for (const t of s.topics || []) {
|
||||
topicCounts[t] = (topicCounts[t] || 0) + 1;
|
||||
}
|
||||
}
|
||||
|
||||
const totalWords = summaries.reduce((sum, s) => sum + (s.wordCount || 0), 0);
|
||||
|
||||
return {
|
||||
domain: getSiteDomain(),
|
||||
pagesVisited: journey.depth,
|
||||
pagesIndexed: summaries.length,
|
||||
pagesDiscovered: discoveredUrls.size,
|
||||
totalWordsRead: totalWords,
|
||||
topicsDistribution: topicCounts,
|
||||
firstVisit: journey.firstVisit,
|
||||
recentPages: journey.pages.slice(-10),
|
||||
};
|
||||
}
|
||||
|
||||
// Auto-migrate plaintext conversations when vault is unlocked
|
||||
if (typeof window !== "undefined") {
|
||||
window.addEventListener("uncloseai-vault-unlocked", () => {
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ import hljs from "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.10.0/es
|
|||
import { setSystemMessageAppend, isTTSEnabled } from "./config.js";
|
||||
import { extractWebpageContent, fitPageContent, buildPageAnalysisPrompt, parsePageAnalysis, buildAnalysisContext } from "./content.js";
|
||||
import { computePageIntelligence, formatPageIntelligence } from "./page-intelligence.js";
|
||||
import { loadSiteJourney, recordPageVisit, formatJourneyContext, savePageSummary, getRelevantPageSummaries, saveConversationSummary, saveSiteLinks } from "./storage.js";
|
||||
import { extractInternalLinks } from "./page-intelligence.js";
|
||||
import {
|
||||
detectCurrentTheme,
|
||||
getThemeColors,
|
||||
|
|
@ -1054,10 +1056,19 @@ async function openUncloseaiEmbeddedModalNew() {
|
|||
actionsSection.className = "uncloseai-section uncloseai-actions-full-width";
|
||||
actionsSection.style.cssText = "grid-column: 1 / -1; width: 100%;";
|
||||
|
||||
// Site intelligence dashboard (#011), crawl mode (#010), share (#012)
|
||||
const { buildVaultViewerSection, buildCrawlSection, buildShareSection } = await import("./vault-viewer.js");
|
||||
const vaultViewerSection = buildVaultViewerSection();
|
||||
const crawlSection = buildCrawlSection(getVaultOrStorage);
|
||||
const shareSection = buildShareSection(getVaultOrStorage);
|
||||
|
||||
// Wrap settings in settingsContent container (shown/hidden based on vault state)
|
||||
settingsContent.appendChild(actionsSection);
|
||||
settingsContent.appendChild(leftColumn);
|
||||
settingsContent.appendChild(rightColumn);
|
||||
settingsContent.appendChild(vaultViewerSection);
|
||||
settingsContent.appendChild(crawlSection);
|
||||
settingsContent.appendChild(shareSection);
|
||||
// Two-column grid layout, display controlled by buildVaultUI
|
||||
settingsContent.style.gridTemplateColumns = "1fr 1fr";
|
||||
settingsContent.style.gap = "12px";
|
||||
|
|
@ -1602,6 +1613,10 @@ async function openUncloseaiEmbeddedModalNew() {
|
|||
|
||||
chatArea.scrollTop = chatArea.scrollHeight;
|
||||
|
||||
// Working indicator: pulse the send button while processing (#004)
|
||||
sendBtn.classList.add("working");
|
||||
sendBtn.disabled = true;
|
||||
|
||||
try {
|
||||
let response = "";
|
||||
for await (const chunk of sendMessage(message)) {
|
||||
|
|
@ -1828,6 +1843,10 @@ async function openUncloseaiEmbeddedModalNew() {
|
|||
aiMsg.appendChild(messageActions);
|
||||
} catch (error) {
|
||||
aiMsg.innerHTML = `<span style="color: #dc3545;">Error: ${error.message}</span>`;
|
||||
} finally {
|
||||
// Remove working indicator (#004)
|
||||
sendBtn.classList.remove("working");
|
||||
sendBtn.disabled = false;
|
||||
}
|
||||
|
||||
chatArea.scrollTop = chatArea.scrollHeight;
|
||||
|
|
@ -2485,9 +2504,18 @@ You have complete knowledge of this page content and can reference any details,
|
|||
const introMsg = document.createElement("div");
|
||||
introMsg.className = "ai-message";
|
||||
|
||||
// Show loading message first
|
||||
// Load journey to determine greeting depth
|
||||
const journey = loadSiteJourney();
|
||||
const depth = journey.depth;
|
||||
const journeyContext = formatJourneyContext(journey);
|
||||
|
||||
// Loading message evolves with journey depth
|
||||
const loadingKey = depth === 0 ? "loadingFirstVisit"
|
||||
: depth <= 2 ? "loadingReturning"
|
||||
: depth <= 4 ? "loadingFamiliar"
|
||||
: "loadingDeepDive";
|
||||
introMsg.innerHTML =
|
||||
'<em style="color: #6c757d;">Analyzing page and generating welcome message...</em>';
|
||||
`<em style="color: #6c757d;">${getUIText(loadingKey)}</em>`;
|
||||
chatBox.appendChild(introMsg);
|
||||
|
||||
try {
|
||||
|
|
@ -2511,7 +2539,7 @@ You have complete knowledge of this page content and can reference any details,
|
|||
|
||||
// LLM CLASSIFICATION: async pre-inference for subjective analysis
|
||||
introMsg.innerHTML =
|
||||
'<em style="color: #6c757d;">Reading page...</em>';
|
||||
`<em style="color: #6c757d;">${getUIText(loadingKey)}</em>`;
|
||||
let analysisContext = "";
|
||||
try {
|
||||
const analysisPrompt = buildPageAnalysisPrompt(
|
||||
|
|
@ -2538,11 +2566,7 @@ You have complete knowledge of this page content and can reference any details,
|
|||
console.warn("Page pre-analysis failed, proceeding without:", e);
|
||||
}
|
||||
|
||||
introMsg.innerHTML =
|
||||
'<em style="color: #6c757d;">Generating welcome message...</em>';
|
||||
|
||||
// Generate intro message with specialized intro system prompt
|
||||
// Get user's preferred language and localize the system prompt
|
||||
// Generate intro message with journey-aware system prompt
|
||||
const userLang = getUserLanguagePreference();
|
||||
const languageName = NATIVE_LANGUAGE_NAMES[userLang] || "English";
|
||||
|
||||
|
|
@ -2558,7 +2582,38 @@ You have complete knowledge of this page content and can reference any details,
|
|||
? `\nUse the PAGE INTELLIGENCE and COMPUTED PAGE METRICS to shape your greeting style. Reference specific numbers (word count, reading time, code languages) naturally. For lyrics, discuss the music and emotions. For recipes, mention ingredients and techniques. For news, note how recent or dated the content is. For code or CI logs, reference the technology and build status. For manifestos or philosophy, engage with the ideas. For wikis, acknowledge the knowledge domain. For emails, understand the communication context. Match your tone to what the page actually is. Never mention "page intelligence", "computed metrics", or "pre-analysis" to the user.\n`
|
||||
: "";
|
||||
|
||||
const introSystemPrompt = `${getUIText("systemPromptIntro")}
|
||||
// REVERSE RAG: backfill spare context with indexed site knowledge (#009)
|
||||
let indexedKnowledgeContext = "";
|
||||
if (depth > 0) {
|
||||
const { getSelectedModelMaxTokens: getMaxTok } = await import("./models.js");
|
||||
const maxTok = getMaxTok();
|
||||
// Estimate tokens used so far: page content + computed + analysis + overhead
|
||||
const usedEstimate = Math.ceil((pageContent.length + computedContext.length + analysisContext.length) / 4) + 2000;
|
||||
const spareBudget = Math.max(0, Math.floor((maxTok - usedEstimate) * 0.3));
|
||||
if (spareBudget > 500) {
|
||||
const backfill = getRelevantPageSummaries(window.location.href, spareBudget);
|
||||
if (backfill.count > 0) {
|
||||
indexedKnowledgeContext = backfill.text;
|
||||
console.log(`Reverse RAG: backfilled ${backfill.count} page summaries (${spareBudget} token budget)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract internal links for knowledge graph (#008)
|
||||
let internalLinks = [];
|
||||
try {
|
||||
internalLinks = extractInternalLinks();
|
||||
} catch (e) {
|
||||
console.warn("Internal link extraction failed:", e);
|
||||
}
|
||||
|
||||
// Build journey-aware system prompt based on depth
|
||||
let introSystemPrompt;
|
||||
let introPromptUser;
|
||||
|
||||
if (depth === 0) {
|
||||
// FIRST VISIT: full 3-paragraph welcome (existing behavior)
|
||||
introSystemPrompt = `${getUIText("systemPromptIntro")}
|
||||
${computedContext}${analysisContext}${typeGuidance}
|
||||
PAGE INFORMATION:
|
||||
Title: "${pageTitle}"
|
||||
|
|
@ -2574,15 +2629,84 @@ ${getUIText("systemPromptTask3")}
|
|||
|
||||
${getUIText("systemPromptInstructions")} ${languageName}.${langInstruction}`;
|
||||
|
||||
const introPrompt =
|
||||
userLang !== "en"
|
||||
? `Generate the welcoming introduction message now. Remember: Write ONLY in ${languageName}, not English.`
|
||||
: "Generate the welcoming introduction message now.";
|
||||
introPromptUser =
|
||||
userLang !== "en"
|
||||
? `Generate the welcoming introduction message now. Remember: Write ONLY in ${languageName}, not English.`
|
||||
: "Generate the welcoming introduction message now.";
|
||||
|
||||
} else if (depth <= 2) {
|
||||
// RETURNING: 3 short paragraphs, no welcome, page-focused
|
||||
introSystemPrompt = `${getUIText("systemPromptReturning")}
|
||||
${journeyContext}${indexedKnowledgeContext}${computedContext}${analysisContext}${typeGuidance}
|
||||
PAGE INFORMATION:
|
||||
Title: "${pageTitle}"
|
||||
URI: ${window.location.href}
|
||||
|
||||
FULL PAGE CONTENT:
|
||||
${pageContent}
|
||||
|
||||
Generate a 3-paragraph page introduction:
|
||||
${getUIText("systemPromptReturningTask1")}
|
||||
${getUIText("systemPromptReturningTask2")}
|
||||
${getUIText("systemPromptReturningTask3")}
|
||||
|
||||
${getUIText("systemPromptReturningInstructions")} ${languageName}.${langInstruction}`;
|
||||
|
||||
introPromptUser =
|
||||
userLang !== "en"
|
||||
? `Introduce this page now. No greeting, no self-intro. Write ONLY in ${languageName}.`
|
||||
: "Introduce this page now. No greeting, no self-introduction.";
|
||||
|
||||
} else if (depth <= 4) {
|
||||
// FAMILIAR: 2 paragraphs, casual companion
|
||||
introSystemPrompt = `${getUIText("systemPromptFamiliar")}
|
||||
${journeyContext}${indexedKnowledgeContext}${computedContext}${analysisContext}${typeGuidance}
|
||||
PAGE INFORMATION:
|
||||
Title: "${pageTitle}"
|
||||
URI: ${window.location.href}
|
||||
|
||||
FULL PAGE CONTENT:
|
||||
${pageContent}
|
||||
|
||||
Generate a 2-paragraph page note:
|
||||
${getUIText("systemPromptFamiliarTask1")}
|
||||
${getUIText("systemPromptFamiliarTask2")}
|
||||
|
||||
${getUIText("systemPromptFamiliarInstructions")} ${languageName}.${langInstruction}`;
|
||||
|
||||
introPromptUser =
|
||||
userLang !== "en"
|
||||
? `What's on this page? Write ONLY in ${languageName}.`
|
||||
: "What's on this page?";
|
||||
|
||||
} else {
|
||||
// DEEP DIVE: 1 paragraph, expert companion
|
||||
const deepDiveIntro = getUIText("systemPromptDeepDive").replace("{depth}", String(depth + 1));
|
||||
introSystemPrompt = `${deepDiveIntro}
|
||||
${journeyContext}${indexedKnowledgeContext}${computedContext}${analysisContext}${typeGuidance}
|
||||
PAGE INFORMATION:
|
||||
Title: "${pageTitle}"
|
||||
URI: ${window.location.href}
|
||||
|
||||
FULL PAGE CONTENT:
|
||||
${pageContent}
|
||||
|
||||
${getUIText("systemPromptDeepDiveTask1")}
|
||||
|
||||
${getUIText("systemPromptDeepDiveInstructions")} ${languageName}.${langInstruction}`;
|
||||
|
||||
introPromptUser =
|
||||
userLang !== "en"
|
||||
? `This page. Write ONLY in ${languageName}.`
|
||||
: "This page.";
|
||||
}
|
||||
|
||||
console.log(`Journey depth: ${depth}, prompt tier: ${depth === 0 ? "first-visit" : depth <= 2 ? "returning" : depth <= 4 ? "familiar" : "deep-dive"}`);
|
||||
|
||||
// Create temporary chat history for intro generation
|
||||
const introHistory = [
|
||||
{ role: "system", content: introSystemPrompt },
|
||||
{ role: "user", content: introPrompt },
|
||||
{ role: "user", content: introPromptUser },
|
||||
];
|
||||
|
||||
// Generate intro with specialized system prompt
|
||||
|
|
@ -2779,10 +2903,41 @@ You have complete knowledge of this page content and can reference any details,
|
|||
|
||||
// Save this intro as part of the conversation
|
||||
await saveConversationHistory(await getChatHistory());
|
||||
|
||||
// Record this page visit so the next page gets a different greeting
|
||||
recordPageVisit();
|
||||
|
||||
// REVERSE RAG: store page summary for cross-page context (#006)
|
||||
// The intro response IS the summary: no extra LLM call needed
|
||||
const pageAnalysisTopics = analysisContext
|
||||
? (analysisContext.match(/Topics?:\s*(.+)/i) || [])[1]?.split(/[,;]/).map((t) => t.trim()).filter(Boolean) || []
|
||||
: [];
|
||||
savePageSummary({
|
||||
url: window.location.href,
|
||||
title: pageTitle,
|
||||
summary: response.slice(0, 500),
|
||||
topics: pageAnalysisTopics,
|
||||
wordCount: computedContext
|
||||
? Number.parseInt((computedContext.match(/(\d[\d,]*)\s*words/i) || [])[1]?.replace(/,/g, "") || "0", 10)
|
||||
: 0,
|
||||
internalLinks,
|
||||
});
|
||||
|
||||
// Store conversation summary (#007)
|
||||
saveConversationSummary(window.location.href, response.slice(0, 300));
|
||||
|
||||
// Store link graph (#008)
|
||||
if (internalLinks.length > 0) {
|
||||
saveSiteLinks(window.location.href, internalLinks);
|
||||
}
|
||||
|
||||
console.log(`Recorded page visit, journey depth now: ${depth + 1}. Indexed: summary, ${internalLinks.length} links`);
|
||||
} catch (error) {
|
||||
console.error("Failed to generate contextual intro:", error);
|
||||
// Fallback to basic intro
|
||||
introMsg.innerHTML = `👋 Hi! I'm Hermes AI. I can help you understand this page, answer questions, or assist with various tasks. How can I help you today?`;
|
||||
// Still record the visit even on fallback
|
||||
recordPageVisit();
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
448
public/src/vault-viewer.js
Normal file
448
public/src/vault-viewer.js
Normal file
|
|
@ -0,0 +1,448 @@
|
|||
// This is free software for the public good of a permacomputer hosted at
|
||||
// permacomputer.com, an always-on computer by the people, for the people.
|
||||
// One which is durable, easy to repair, & distributed like tap water
|
||||
// for machine learning intelligence.
|
||||
//
|
||||
// The permacomputer is community-owned infrastructure optimized around
|
||||
// four values:
|
||||
//
|
||||
// TRUTH First principles, math & science, open source code freely distributed
|
||||
// FREEDOM Voluntary partnerships, freedom from tyranny & corporate control
|
||||
// HARMONY Minimal waste, self-renewing systems with diverse thriving connections
|
||||
// LOVE Be yourself without hurting others, cooperation through natural law
|
||||
//
|
||||
// This software contributes to that vision by making machine learning
|
||||
// accessible to everyone through a free, open, embeddable chat interface.
|
||||
// Code is seeds to sprout on any abandoned technology.
|
||||
|
||||
// Vault viewer panel (#011): shows indexed pages, coverage metrics, and charts.
|
||||
// Crawl mode trigger (#010): ethical site indexing with robots.txt compliance.
|
||||
// Share button (#012): opt-in conversation sharing via unfirehose.org.
|
||||
|
||||
import {
|
||||
getSiteCoverageStats,
|
||||
loadPageSummaries,
|
||||
loadSiteLinks,
|
||||
loadSiteJourney,
|
||||
savePageSummary,
|
||||
saveSiteLinks,
|
||||
} from "./storage.js";
|
||||
import { extractInternalLinks } from "./page-intelligence.js";
|
||||
|
||||
// ============================================================
|
||||
// VAULT VIEWER: Site Intelligence Dashboard (#011)
|
||||
// ============================================================
|
||||
|
||||
export function buildVaultViewerSection() {
|
||||
const section = document.createElement("div");
|
||||
section.className = "uncloseai-section";
|
||||
section.style.cssText = "grid-column: 1 / -1; width: 100%;";
|
||||
|
||||
const heading = document.createElement("h4");
|
||||
heading.textContent = "📊 Site Intelligence";
|
||||
heading.style.cssText = "margin: 0 0 8px 0; font-size: 13px;";
|
||||
section.appendChild(heading);
|
||||
|
||||
const statsContainer = document.createElement("div");
|
||||
statsContainer.style.cssText = "display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 8px; margin-bottom: 12px;";
|
||||
section.appendChild(statsContainer);
|
||||
|
||||
const pageList = document.createElement("div");
|
||||
pageList.style.cssText = "max-height: 200px; overflow-y: auto; font-size: 12px;";
|
||||
section.appendChild(pageList);
|
||||
|
||||
function refresh() {
|
||||
const stats = getSiteCoverageStats();
|
||||
statsContainer.innerHTML = "";
|
||||
|
||||
const metrics = [
|
||||
{ label: "Pages Visited", value: stats.pagesVisited },
|
||||
{ label: "Pages Indexed", value: stats.pagesIndexed },
|
||||
{ label: "Discovered", value: stats.pagesDiscovered },
|
||||
];
|
||||
|
||||
for (const m of metrics) {
|
||||
const card = document.createElement("div");
|
||||
card.style.cssText = "text-align: center; padding: 8px; border-radius: 6px; border: 1px solid var(--uncloseai-border-color, #ccc);";
|
||||
card.innerHTML = `<div style="font-size: 20px; font-weight: bold;">${m.value}</div><div style="font-size: 11px; opacity: 0.7;">${m.label}</div>`;
|
||||
statsContainer.appendChild(card);
|
||||
}
|
||||
|
||||
// Topic distribution bar
|
||||
const topics = Object.entries(stats.topicsDistribution);
|
||||
if (topics.length > 0) {
|
||||
const topicBar = document.createElement("div");
|
||||
topicBar.style.cssText = "display: grid; grid-template-columns: repeat(auto-fill, minmax(80px, 1fr)); gap: 4px; margin-bottom: 8px;";
|
||||
for (const [topic, count] of topics.sort((a, b) => b[1] - a[1]).slice(0, 8)) {
|
||||
const chip = document.createElement("span");
|
||||
chip.style.cssText = "font-size: 11px; padding: 2px 6px; border-radius: 4px; background: var(--uncloseai-border-color, #eee); text-overflow: ellipsis; overflow: hidden; white-space: nowrap;";
|
||||
chip.textContent = `${topic} (${count})`;
|
||||
topicBar.appendChild(chip);
|
||||
}
|
||||
statsContainer.parentNode.insertBefore(topicBar, pageList);
|
||||
}
|
||||
|
||||
// Words read
|
||||
if (stats.totalWordsRead > 0) {
|
||||
const readTime = Math.ceil(stats.totalWordsRead / 250);
|
||||
const wordsLine = document.createElement("div");
|
||||
wordsLine.style.cssText = "font-size: 11px; opacity: 0.7; margin-bottom: 8px;";
|
||||
wordsLine.textContent = `${stats.totalWordsRead.toLocaleString()} words read (~${readTime} min reading time)`;
|
||||
statsContainer.parentNode.insertBefore(wordsLine, pageList);
|
||||
}
|
||||
|
||||
// Indexed pages list
|
||||
const summaries = loadPageSummaries();
|
||||
pageList.innerHTML = "";
|
||||
|
||||
if (summaries.length === 0) {
|
||||
pageList.innerHTML = '<div style="opacity: 0.5; padding: 8px;">No pages indexed yet. Browse the site with the chatbot open to build your knowledge base.</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
for (const page of summaries.sort((a, b) => (b.indexedAt || 0) - (a.indexedAt || 0))) {
|
||||
const row = document.createElement("div");
|
||||
row.style.cssText = "padding: 4px 0; border-bottom: 1px solid var(--uncloseai-border-color, #eee);";
|
||||
const title = document.createElement("a");
|
||||
title.href = page.url;
|
||||
title.textContent = page.title || page.url;
|
||||
title.target = "_blank";
|
||||
title.style.cssText = "font-size: 12px; color: inherit; text-decoration: underline;";
|
||||
row.appendChild(title);
|
||||
|
||||
if (page.topics && page.topics.length > 0) {
|
||||
const tags = document.createElement("span");
|
||||
tags.style.cssText = "font-size: 10px; opacity: 0.6; margin-left: 8px;";
|
||||
tags.textContent = page.topics.slice(0, 3).join(", ");
|
||||
row.appendChild(tags);
|
||||
}
|
||||
|
||||
pageList.appendChild(row);
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh on mount
|
||||
refresh();
|
||||
|
||||
// Expose refresh for external calls
|
||||
section._refresh = refresh;
|
||||
|
||||
return section;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// CRAWL MODE: Ethical Site Indexing (#010)
|
||||
// ============================================================
|
||||
|
||||
export function buildCrawlSection(getVaultOrStorage) {
|
||||
const section = document.createElement("div");
|
||||
section.className = "uncloseai-section";
|
||||
section.style.cssText = "grid-column: 1 / -1; width: 100%;";
|
||||
|
||||
const heading = document.createElement("h4");
|
||||
heading.textContent = "🕷️ Index Site";
|
||||
heading.style.cssText = "margin: 0 0 8px 0; font-size: 13px;";
|
||||
section.appendChild(heading);
|
||||
|
||||
const info = document.createElement("div");
|
||||
info.style.cssText = "font-size: 12px; opacity: 0.7; margin-bottom: 8px;";
|
||||
info.textContent = "Crawl this site to index pages for cross-page search. Follows robots.txt, 1 page/sec.";
|
||||
section.appendChild(info);
|
||||
|
||||
const controls = document.createElement("div");
|
||||
controls.style.cssText = "display: grid; grid-template-columns: 1fr auto; gap: 8px; align-items: center;";
|
||||
|
||||
const depthSelect = document.createElement("select");
|
||||
depthSelect.style.cssText = "font-size: 12px; padding: 4px 8px; border-radius: 4px; border: 1px solid var(--uncloseai-border-color, #ccc);";
|
||||
for (const [label, val] of [["10 pages", 10], ["25 pages", 25], ["50 pages", 50], ["100 pages", 100]]) {
|
||||
const opt = document.createElement("option");
|
||||
opt.value = val;
|
||||
opt.textContent = label;
|
||||
depthSelect.appendChild(opt);
|
||||
}
|
||||
controls.appendChild(depthSelect);
|
||||
|
||||
const crawlBtn = document.createElement("button");
|
||||
crawlBtn.textContent = "Start Crawl";
|
||||
crawlBtn.className = "uncloseai-btn-small";
|
||||
crawlBtn.style.cssText = "padding: 4px 12px; font-size: 12px;";
|
||||
controls.appendChild(crawlBtn);
|
||||
|
||||
section.appendChild(controls);
|
||||
|
||||
const progress = document.createElement("div");
|
||||
progress.style.cssText = "font-size: 11px; margin-top: 8px; display: none;";
|
||||
section.appendChild(progress);
|
||||
|
||||
let crawling = false;
|
||||
|
||||
crawlBtn.onclick = async () => {
|
||||
if (crawling) return;
|
||||
|
||||
// Check for unsandbox API key (required for crawl mode)
|
||||
const apiKey = getVaultOrStorage("unsandboxPublicKey", "") ||
|
||||
getVaultOrStorage("customAPIKey", "");
|
||||
|
||||
if (!apiKey) {
|
||||
progress.style.display = "block";
|
||||
progress.textContent = "Requires an API key in settings to enable crawl mode.";
|
||||
return;
|
||||
}
|
||||
|
||||
crawling = true;
|
||||
crawlBtn.textContent = "Crawling...";
|
||||
crawlBtn.classList.add("working");
|
||||
progress.style.display = "block";
|
||||
|
||||
const maxPages = Number.parseInt(depthSelect.value, 10);
|
||||
|
||||
try {
|
||||
await crawlSite(maxPages, (status) => {
|
||||
progress.textContent = status;
|
||||
});
|
||||
} catch (e) {
|
||||
progress.textContent = `Crawl failed: ${e.message}`;
|
||||
} finally {
|
||||
crawling = false;
|
||||
crawlBtn.textContent = "Start Crawl";
|
||||
crawlBtn.classList.remove("working");
|
||||
}
|
||||
};
|
||||
|
||||
return section;
|
||||
}
|
||||
|
||||
// Ethical site crawler: follows robots.txt, rate-limited, stores in vault
|
||||
async function crawlSite(maxPages, onProgress) {
|
||||
const origin = window.location.origin;
|
||||
const domain = window.location.hostname;
|
||||
|
||||
// Fetch and parse robots.txt
|
||||
onProgress("Fetching robots.txt...");
|
||||
let disallowed = [];
|
||||
try {
|
||||
const resp = await fetch(`${origin}/robots.txt`);
|
||||
if (resp.ok) {
|
||||
const text = await resp.text();
|
||||
disallowed = parseRobotsTxt(text);
|
||||
}
|
||||
} catch (e) {
|
||||
// No robots.txt = everything allowed
|
||||
}
|
||||
|
||||
function isAllowed(path) {
|
||||
for (const rule of disallowed) {
|
||||
if (path.startsWith(rule)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// BFS crawl from current page
|
||||
const visited = new Set();
|
||||
const queue = [window.location.pathname];
|
||||
let indexed = 0;
|
||||
|
||||
while (queue.length > 0 && indexed < maxPages) {
|
||||
const path = queue.shift();
|
||||
if (visited.has(path)) continue;
|
||||
visited.add(path);
|
||||
|
||||
if (!isAllowed(path)) {
|
||||
onProgress(`Skipped (robots.txt): ${path}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
onProgress(`Indexing ${indexed + 1}/${maxPages}: ${path}`);
|
||||
|
||||
try {
|
||||
const resp = await fetch(origin + path);
|
||||
if (!resp.ok) continue;
|
||||
const html = await resp.text();
|
||||
const doc = new DOMParser().parseFromString(html, "text/html");
|
||||
|
||||
// Extract content
|
||||
const title = doc.title || path;
|
||||
const bodyText = (doc.body?.innerText || "").slice(0, 5000);
|
||||
const wordCount = bodyText.split(/\s+/).filter(Boolean).length;
|
||||
|
||||
// Extract internal links
|
||||
const links = [];
|
||||
for (const a of doc.querySelectorAll("a[href]")) {
|
||||
try {
|
||||
const uri = new URL(a.href, origin);
|
||||
if (uri.hostname === domain && uri.pathname !== path && !uri.hash) {
|
||||
const clean = uri.pathname;
|
||||
if (!visited.has(clean) && !queue.includes(clean)) {
|
||||
links.push(uri.origin + clean);
|
||||
queue.push(clean);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// skip
|
||||
}
|
||||
}
|
||||
|
||||
// Store page summary
|
||||
savePageSummary({
|
||||
url: origin + path,
|
||||
title,
|
||||
summary: bodyText.slice(0, 500),
|
||||
topics: [],
|
||||
wordCount,
|
||||
internalLinks: links.slice(0, 50),
|
||||
});
|
||||
|
||||
// Store link graph
|
||||
saveSiteLinks(origin + path, links);
|
||||
indexed++;
|
||||
} catch (e) {
|
||||
onProgress(`Failed: ${path} (${e.message})`);
|
||||
}
|
||||
|
||||
// Rate limit: 1 page per second
|
||||
await new Promise((r) => setTimeout(r, 1000));
|
||||
}
|
||||
|
||||
onProgress(`Done. Indexed ${indexed} pages on ${domain}.`);
|
||||
}
|
||||
|
||||
function parseRobotsTxt(text) {
|
||||
const disallowed = [];
|
||||
let inUserAgent = false;
|
||||
|
||||
for (const line of text.split("\n")) {
|
||||
const trimmed = line.trim().toLowerCase();
|
||||
if (trimmed.startsWith("user-agent:")) {
|
||||
const agent = trimmed.slice(11).trim();
|
||||
inUserAgent = agent === "*" || agent === "uncloseai";
|
||||
} else if (inUserAgent && trimmed.startsWith("disallow:")) {
|
||||
const path = line.trim().slice(9).trim();
|
||||
if (path) disallowed.push(path);
|
||||
}
|
||||
}
|
||||
|
||||
return disallowed;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// SHARE BUTTON: Opt-in Unfirehose Sharing (#012)
|
||||
// ============================================================
|
||||
|
||||
export function buildShareSection(getVaultOrStorage) {
|
||||
const section = document.createElement("div");
|
||||
section.className = "uncloseai-section";
|
||||
section.style.cssText = "grid-column: 1 / -1; width: 100%;";
|
||||
|
||||
const heading = document.createElement("h4");
|
||||
heading.textContent = "🔗 Share Conversations";
|
||||
heading.style.cssText = "margin: 0 0 8px 0; font-size: 13px;";
|
||||
section.appendChild(heading);
|
||||
|
||||
const info = document.createElement("div");
|
||||
info.style.cssText = "font-size: 12px; opacity: 0.7; margin-bottom: 8px;";
|
||||
info.textContent = "All conversations are private by default. Opt-in to share specific conversations publicly via unfirehose.org for machine learning training.";
|
||||
section.appendChild(info);
|
||||
|
||||
const controls = document.createElement("div");
|
||||
controls.style.cssText = "display: grid; grid-template-columns: 1fr auto; gap: 8px; align-items: center;";
|
||||
|
||||
const accessSelect = document.createElement("select");
|
||||
accessSelect.style.cssText = "font-size: 12px; padding: 4px 8px; border-radius: 4px; border: 1px solid var(--uncloseai-border-color, #ccc);";
|
||||
for (const [label, val] of [["Public (free)", "public"], ["Unlisted (link only)", "unlisted"]]) {
|
||||
const opt = document.createElement("option");
|
||||
opt.value = val;
|
||||
opt.textContent = label;
|
||||
accessSelect.appendChild(opt);
|
||||
}
|
||||
controls.appendChild(accessSelect);
|
||||
|
||||
const shareBtn = document.createElement("button");
|
||||
shareBtn.textContent = "Share This Chat";
|
||||
shareBtn.className = "uncloseai-btn-small";
|
||||
shareBtn.style.cssText = "padding: 4px 12px; font-size: 12px;";
|
||||
controls.appendChild(shareBtn);
|
||||
|
||||
section.appendChild(controls);
|
||||
|
||||
const status = document.createElement("div");
|
||||
status.style.cssText = "font-size: 11px; margin-top: 8px; display: none;";
|
||||
section.appendChild(status);
|
||||
|
||||
shareBtn.onclick = async () => {
|
||||
// Check for API key
|
||||
const apiKey = getVaultOrStorage("unsandboxPublicKey", "") ||
|
||||
getVaultOrStorage("unsandboxSecretKey", "");
|
||||
|
||||
if (!apiKey) {
|
||||
status.style.display = "block";
|
||||
status.textContent = "Requires an unsandbox.com API key. Add one in settings above.";
|
||||
return;
|
||||
}
|
||||
|
||||
// Load current conversation
|
||||
const { loadConversationHistory } = await import("./storage.js");
|
||||
const history = loadConversationHistory();
|
||||
|
||||
if (!history || history.length === 0) {
|
||||
status.style.display = "block";
|
||||
status.textContent = "No conversation to share on this page.";
|
||||
return;
|
||||
}
|
||||
|
||||
// Confirm with user
|
||||
const messageCount = history.length;
|
||||
const confirmed = confirm(
|
||||
`Share ${messageCount} messages from this conversation as "${accessSelect.value}"?\n\nThis will upload to unfirehose.org. Your API key will be used for authentication.`,
|
||||
);
|
||||
if (!confirmed) return;
|
||||
|
||||
shareBtn.textContent = "Sharing...";
|
||||
shareBtn.classList.add("working");
|
||||
shareBtn.disabled = true;
|
||||
status.style.display = "block";
|
||||
|
||||
try {
|
||||
// Format as JSONL
|
||||
const sessionId = `uncloseai-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
const jsonl = history
|
||||
.map((msg) =>
|
||||
JSON.stringify({
|
||||
role: msg.role,
|
||||
content: msg.content,
|
||||
timestamp: Date.now(),
|
||||
session_id: sessionId,
|
||||
source_url: window.location.href,
|
||||
source_title: document.title,
|
||||
}),
|
||||
)
|
||||
.join("\n");
|
||||
|
||||
const resp = await fetch("https://api.unfirehose.org/v1/ingest", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"Content-Type": "application/x-jsonl",
|
||||
"X-Source": "uncloseai",
|
||||
"X-Session-Id": sessionId,
|
||||
"X-Access-Level": accessSelect.value,
|
||||
},
|
||||
body: jsonl,
|
||||
});
|
||||
|
||||
if (resp.ok) {
|
||||
status.textContent = `Shared successfully. Session: ${sessionId}`;
|
||||
} else {
|
||||
const err = await resp.text();
|
||||
status.textContent = `Share failed: ${resp.status} ${err.slice(0, 100)}`;
|
||||
}
|
||||
} catch (e) {
|
||||
status.textContent = `Share failed: ${e.message}`;
|
||||
} finally {
|
||||
shareBtn.textContent = "Share This Chat";
|
||||
shareBtn.classList.remove("working");
|
||||
shareBtn.disabled = false;
|
||||
}
|
||||
};
|
||||
|
||||
return section;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue