uncloseai.com/public/src/content.js

// This is free software for the public good of a permacomputer hosted at
// permacomputer.com, an always-on computer by the people, for the people.
// One which is durable, easy to repair, & distributed like tap water
// for machine learning intelligence.
//
// The permacomputer is community-owned infrastructure optimized around
// four values:
//
//   TRUTH      First principles, math & science, open source code freely distributed
//   FREEDOM    Voluntary partnerships, freedom from tyranny & corporate control
//   HARMONY    Minimal waste, self-renewing systems with diverse thriving connections
//   LOVE       Be yourself without hurting others, cooperation through natural law
//
// This software contributes to that vision by making machine learning
// accessible to everyone through a free, open, embeddable chat interface.
// Code is seeds to sprout on any abandoned technology.

import { human, delta2dict } from "./ago.js";

// URI translation rules — map dynamic page patterns to raw/plain text equivalents.
// Each rule: { pattern: RegExp matching the full URI, translate: (match) => rawURI }
const URI_TRANSLATIONS = [
	{
		// GitLab CI job pages → raw log output
		// e.g. https://git.example.com/group/project/-/jobs/12345 → .../jobs/12345/raw
		pattern: /^(https?:\/\/[^/]+\/.*\/-\/jobs\/\d+)\/?$/,
		translate: (match) => `${match[1]}/raw`,
	},
];

// Try to fetch raw content via URI translation (same-origin, cookies included).
// Returns { raw: string } on success, { matched: true } if pattern matched but
// fetch failed (signals a dynamic page worth waiting for), or null if no match.
async function fetchTranslatedContent(uri) {
	for (const rule of URI_TRANSLATIONS) {
		const match = uri.match(rule.pattern);
		if (match) {
			try {
				const rawUri = rule.translate(match);
				const response = await fetch(rawUri, { credentials: "same-origin" });
				if (response.ok) {
					const text = await response.text();
					if (text && text.length > 0) {
						return { raw: text };
					}
				}
			} catch (e) {
				console.warn("URI translation fetch failed, falling back to DOM:", e);
			}
			// Pattern matched but fetch failed — caller should wait for DOM to settle
			return { matched: true };
		}
	}
	return null;
}

// Wait for DOM mutations to settle (content stops streaming in)
function waitForDOMSettle(timeout = 3000, quiet = 500) {
	return new Promise((resolve) => {
		let timer;
		const observer = new MutationObserver(() => {
			clearTimeout(timer);
			timer = setTimeout(() => {
				observer.disconnect();
				resolve();
			}, quiet);
		});
		observer.observe(document.body, {
			childList: true,
			subtree: true,
			characterData: true,
		});
		// Start the quiet timer immediately in case DOM is already settled
		timer = setTimeout(() => {
			observer.disconnect();
			resolve();
		}, quiet);
		// Hard ceiling so we don't wait forever
		setTimeout(() => {
			observer.disconnect();
			resolve();
		}, timeout);
	});
}

// Extract DOM content — title, metadata, text with links
function extractDOMContent() {
	let content = "";

	// Extract title
	const title = document.title;
	if (title) {
		content += `**Page Title**: ${title}\n\n`;
	}

	// Extract meta description
	const metaDescription = document.querySelector('meta[name="description"]');
	if (metaDescription) {
		content += `**Meta Description**: ${metaDescription.content}\n\n`;
	}

	// Extract other metadata (if needed)
	const metaKeywords = document.querySelector('meta[name="keywords"]');
	if (metaKeywords) {
		content += `**Meta Keywords**: ${metaKeywords.content}\n\n`;
	}

	// Tags that never contain translatable/readable content
	const SKIP_TAGS = new Set([
		"script", "style", "noscript", "template", "svg",
		"iframe", "object", "embed", "canvas", "video", "audio",
	]);

	// Recursively extract text and links from the body content
	function getTextWithLinks(element) {
		if (element.nodeType === Node.TEXT_NODE) {
			content += `${element.textContent} `;
		} else if (element.nodeType === Node.ELEMENT_NODE) {
			const tag = element.tagName.toLowerCase();

			// Skip non-content elements
			if (SKIP_TAGS.has(tag)) return;
			if (element.hidden || element.getAttribute("aria-hidden") === "true") return;

			if (tag === "a") {
				// If it's a link, append the text and the href
				content += `[${element.textContent}](${element.href}) `;
			} else {
				// Recursively process child nodes
				element.childNodes.forEach(getTextWithLinks);
			}
		}
	}

	getTextWithLinks(document.body);
	return content.trim();
}

// Extract text content along with links and metadata from the webpage.
// Tries URI translation first (raw/plain text for known dynamic pages),
// falls back to DOM extraction.
export async function extractWebpageContent() {
	const result = await fetchTranslatedContent(window.location.href);

	// Raw content fetched successfully — use it directly
	if (result?.raw) {
		let content = "";
		const title = document.title;
		if (title) {
			content += `**Page Title**: ${title}\n\n`;
		}
		content += `**Source URI**: ${window.location.href}\n\n`;
		content += result.raw;
		return content.trim();
	}

	// Pattern matched but raw fetch failed (auth, CORS, etc) —
	// this is a known dynamic page, wait for DOM to settle before scraping
	if (result?.matched) {
		await waitForDOMSettle();
	}

	return extractDOMContent();
}

// Estimate tokens: ~4 chars per token for English prose
function estimateTokens(text) {
	return Math.ceil(text.length / 4);
}

// Collapse navigation link clusters: 5+ consecutive links become a single line
function collapseNavLinks(text) {
	// Match runs of lines that are just markdown links, possibly with separators
	return text.replace(
		/(\[([^\]]*)\]\([^)]*\)\s*){5,}/g,
		(match) => {
			const links = match.match(/\[([^\]]*)\]\([^)]*\)/g);
			if (!links) return match;
			const first3 = links.slice(0, 3).map(l => l.match(/\[([^\]]*)\]/)?.[1]).filter(Boolean);
			return `[Navigation: ${links.length} links including ${first3.join(", ")}]\n`;
		},
	);
}

// Collapse long lists: keep first 5 items, note the rest
function collapseLists(text) {
	const lines = text.split("\n");
	const result = [];
	let listRun = [];
	let inList = false;

	const flushList = () => {
		if (listRun.length <= 7) {
			result.push(...listRun);
		} else {
			result.push(...listRun.slice(0, 5));
			result.push(`  [... ${listRun.length - 5} more items]`);
		}
		listRun = [];
		inList = false;
	};

	for (const line of lines) {
		const isList = /^\s*[-*•]\s/.test(line) || /^\s*\d+[.)]\s/.test(line);
		if (isList) {
			inList = true;
			listRun.push(line);
		} else {
			if (inList) flushList();
			result.push(line);
		}
	}
	if (inList) flushList();
	return result.join("\n");
}

// Collapse code blocks: keep first 10 lines, note the rest
function collapseCodeBlocks(text) {
	return text.replace(/```[\w]*\n([\s\S]*?)```/g, (match, code) => {
		const lines = code.split("\n");
		if (lines.length <= 12) return match;
		const lang = match.match(/```(\w*)/)?.[1] || "";
		return `\`\`\`${lang}\n${lines.slice(0, 10).join("\n")}\n[... ${lines.length - 10} more lines]\n\`\`\``;
	});
}

// Remove exact duplicate paragraphs (common in scraped DOM content)
function deduplicateParagraphs(text) {
	const paragraphs = text.split("\n\n");
	const seen = new Set();
	const result = [];
	for (const p of paragraphs) {
		const trimmed = p.trim();
		if (!trimmed) continue;
		// Normalize whitespace for comparison
		const key = trimmed.replace(/\s+/g, " ");
		if (key.length > 20 && seen.has(key)) continue;
		seen.add(key);
		result.push(p);
	}
	return result.join("\n\n");
}

// Collapse reference/citation sections (Wikipedia pattern: dense URL clusters)
function collapseReferences(text) {
	const lines = text.split("\n");
	const result = [];
	let refRun = [];
	let inRefs = false;

	const flushRefs = () => {
		if (refRun.length <= 5) {
			result.push(...refRun);
		} else {
			result.push(refRun[0]);
			result.push(`[... ${refRun.length - 1} references collapsed]`);
		}
		refRun = [];
		inRefs = false;
	};

	for (const line of lines) {
		// Lines that are mostly URLs or citation markers
		const urlCount = (line.match(/https?:\/\//g) || []).length;
		const isRef = urlCount >= 2 || (urlCount >= 1 && line.length < 200 && /\[\d+\]|\^/.test(line));
		if (isRef) {
			inRefs = true;
			refRun.push(line);
		} else {
			if (inRefs) flushRefs();
			result.push(line);
		}
	}
	if (inRefs) flushRefs();
	return result.join("\n");
}

// Collapse consecutive short lines (sidebar fragments, metadata debris)
function collapseSidebarDebris(text) {
	const lines = text.split("\n");
	const result = [];
	let shortRun = [];

	const flushShort = () => {
		if (shortRun.length <= 6) {
			result.push(...shortRun);
		} else {
			result.push(...shortRun.slice(0, 3));
			result.push(`[... ${shortRun.length - 3} short fragments collapsed]`);
		}
		shortRun = [];
	};

	for (const line of lines) {
		// Short lines with little content (under 30 chars, not headings or metadata)
		const trimmed = line.trim();
		if (trimmed.length > 0 && trimmed.length < 30 && !/^\*\*|^#|^---/.test(trimmed)) {
			shortRun.push(line);
		} else {
			if (shortRun.length > 6) flushShort();
			else if (shortRun.length > 0) { result.push(...shortRun); shortRun = []; }
			result.push(line);
		}
	}
	if (shortRun.length > 6) flushShort();
	else result.push(...shortRun);
	return result.join("\n");
}

// Fit page content to model context using progressive structural collapsing.
// No summarization: content is real, just structurally compressed.
// Stages run in order of aggressiveness, stopping when content fits.
export function fitPageContent(pageContent, maxModelTokens) {
	// Reserve 20% of context for overhead (system prompt, chat history, output)
	// but never more than 15k and always leave at least 60% for page content
	const overhead = Math.min(15000, Math.floor(maxModelTokens * 0.2));
	const maxPageTokens = Math.max(Math.floor(maxModelTokens * 0.6), maxModelTokens - overhead);
	const originalTokens = estimateTokens(pageContent);

	if (originalTokens <= maxPageTokens) {
		return { content: pageContent, truncated: false, originalTokens, finalTokens: originalTokens };
	}

	console.log(`Page content (~${originalTokens.toLocaleString()} tokens) exceeds budget (~${maxPageTokens.toLocaleString()} tokens), collapsing...`);

	// Progressive collapse stages, least aggressive first
	const stages = [
		{ name: "dedup paragraphs", fn: deduplicateParagraphs },
		{ name: "collapse nav links", fn: collapseNavLinks },
		{ name: "collapse references", fn: collapseReferences },
		{ name: "collapse lists", fn: collapseLists },
		{ name: "collapse code blocks", fn: collapseCodeBlocks },
		{ name: "collapse sidebar debris", fn: collapseSidebarDebris },
	];

	let content = pageContent;
	let appliedStages = [];

	for (const stage of stages) {
		content = stage.fn(content);
		const tokens = estimateTokens(content);
		appliedStages.push(stage.name);
		console.log(`  ${stage.name}: ~${tokens.toLocaleString()} tokens`);
		if (tokens <= maxPageTokens) {
			const finalTokens = estimateTokens(content);
			console.log(`Content fits after ${appliedStages.length} collapse stages: ~${originalTokens.toLocaleString()} → ~${finalTokens.toLocaleString()} tokens`);
			return { content, truncated: true, originalTokens, finalTokens, stages: appliedStages };
		}
	}

	// All stages applied, still too large: hard truncate at paragraph boundary
	const maxChars = maxPageTokens * 4;
	const slice = content.substring(0, maxChars);
	const lastParagraph = slice.lastIndexOf("\n\n");
	const lastSentence = slice.lastIndexOf(". ");
	const breakPoint =
		lastParagraph > maxChars * 0.8 ? lastParagraph :
		lastSentence > maxChars * 0.8 ? lastSentence + 1 :
		maxChars;

	const finalContent = content.substring(0, breakPoint);
	const finalTokens = estimateTokens(finalContent);

	console.warn(`Page content hard truncated after all collapse stages: ~${originalTokens.toLocaleString()} → ~${finalTokens.toLocaleString()} tokens`);

	return {
		content: `${finalContent}\n\n[Content collapsed and truncated: ~${originalTokens.toLocaleString()} tokens → ~${finalTokens.toLocaleString()} tokens]`,
		truncated: true,
		originalTokens,
		finalTokens,
		stages: [...appliedStages, "hard truncate"],
	};
}

// --- Page Analysis (pre-inference classification) ---
// Runs before the greeting to extract structured intelligence about the page.
// The greeting uses these keywords instead of guessing from raw text.

// Build the classification prompt for a page
export function buildPageAnalysisPrompt(pageContent, pageTitle, pageUrl) {
	const currentDate = new Date().toISOString().split("T")[0];
	// Trim content to keep classification fast
	const contentPreview = pageContent.substring(0, 4000);

	return {
		system:
			"You are a page classification engine. Analyze web pages and extract structured metadata. Respond with ONLY a valid JSON object. No markdown fences, no explanation, no commentary. Just the JSON object.",
		user: `Classify this webpage. Current date: ${currentDate}

TITLE: ${pageTitle}
URI: ${pageUrl}

CONTENT:
${contentPreview}

Respond with this exact JSON structure (use null for unknown fields):
{
  "type": "<lyrics|recipe|blog|news|wiki|email|documentation|product|landing|portfolio|social|academic|code|tutorial|legal|ecommerce|video|podcast|event|job|review|qa|forum|gallery|ci-log|error-page|manifesto|personal|other>",
  "author": "<author name or null>",
  "publishDate": "<YYYY-MM-DD or null>",
  "topics": ["<primary>", "<secondary>"],
  "entities": ["<key names, orgs, technologies>"],
  "tone": "<technical|casual|formal|creative|academic|promotional|instructional|journalistic|poetic|humorous|philosophical>",
  "domain": "<technology|science|cooking|music|art|politics|business|health|education|entertainment|sports|finance|law|philosophy|nature|travel|other>",
  "audience": "<one phrase>",
  "keyPhrases": ["<3-5 notable verbatim phrases from the content>"],
  "summary": "<one sentence>",
  "contentLanguage": "<ISO 639-1>",
  "freshness": "<evergreen|dated|breaking|historical|timeless>"
}`,
	};
}

// Parse the JSON classification from LLM response
export function parsePageAnalysis(response) {
	try {
		const jsonMatch = response.match(/\{[\s\S]*\}/);
		if (jsonMatch) {
			return JSON.parse(jsonMatch[0]);
		}
	} catch (e) {
		console.warn("Page analysis parsing failed:", e);
	}
	return null;
}

// Build enriched context string from analysis for the greeting prompt
export function buildAnalysisContext(analysis) {
	if (!analysis) return "";

	const currentDate = new Date().toISOString().split("T")[0];
	const lines = [`\nPAGE INTELLIGENCE (pre-analyzed):`, `Current Date: ${currentDate}`, `Page Type: ${analysis.type}`];

	if (analysis.author) lines.push(`Author: ${analysis.author}`);
	if (analysis.publishDate) {
		lines.push(`Published: ${analysis.publishDate}`);
		// Use russell ballestrini's ago algorithm for human-readable time distance
		try {
			const publishedDate = new Date(analysis.publishDate);
			if (!Number.isNaN(publishedDate.getTime())) {
				const age = human(publishedDate, 2);
				lines.push(`Age: ${age}`);
			}
		} catch (e) {
			// LLM returned unparseable date, skip age
		}
	}

	if (analysis.topics?.length) lines.push(`Topics: ${analysis.topics.join(", ")}`);
	if (analysis.entities?.length) lines.push(`Key Entities: ${analysis.entities.join(", ")}`);
	if (analysis.tone) lines.push(`Tone: ${analysis.tone}`);
	if (analysis.domain) lines.push(`Domain: ${analysis.domain}`);
	if (analysis.audience) lines.push(`Audience: ${analysis.audience}`);
	if (analysis.keyPhrases?.length) lines.push(`Notable Phrases: "${analysis.keyPhrases.join('", "')}"`);
	if (analysis.summary) lines.push(`Summary: ${analysis.summary}`);
	if (analysis.contentLanguage) lines.push(`Content Language: ${analysis.contentLanguage}`);
	if (analysis.freshness) lines.push(`Freshness: ${analysis.freshness}`);

	return `${lines.join("\n")}\n`;
}