From beab5d01a8f871acd1beefc451a6c2054cc721cf Mon Sep 17 00:00:00 2001
From: Russell Ballestrini <russell.ballestrini@gmail.com>
Date: Thu, 3 Jul 2025 20:28:18 -0400
Subject: [PATCH] feat: reuse markdown extraction logic for TTS spoken tokens

- extractSpokenTokens now detects if input is already markdown format
- Updated prompt to handle both markdown and HTML input
- Added markdown to plain text conversion in prompt (bold, links)
- processContentWithHermes now delegates to extractSpokenTokens
- Consistent processing pipeline whether from Batman button or Read Page
- Better reuse of extractWebpageContent markdown format
---
 src/tts.js | 79 +++++++++---------------------------------------------
 1 file changed, 12 insertions(+), 67 deletions(-)

diff --git a/src/tts.js b/src/tts.js
index 2d8d409..e69a2b5 100644
--- a/src/tts.js
+++ b/src/tts.js
@@ -100,75 +100,19 @@ export async function speakTextDirect(text, voice = "alloy", rate = 0.9) {
 	}
 }
 
-// Process page content using Hermes
+// Process page content using Hermes (deprecated - use extractSpokenTokens instead)
 export async function processContentWithHermes(content) {
-	const payload = {
-		model: getSelectedModel(),
-		messages: [
-			{
-				role: "system",
-				content: `Extract the exact main content of the article.
-
-Avoid reading ads. 
-
-Do not change or summarize content. Return the same words verbatim.
-
-When you encounter a list or bullets, add . . . to make the TTS pause between items.
-
-Try to stay as close to the truth of the original version as possible. Start with the title of the post and then jump into it.
-
-Do not mention the TTS stream just do the work!
-
-Remember your only goal is to extract the entire & exact main content of the article.
-
-Ready? Breath and then return a stream of tokens to be used in a TTS system.
-`,
-			},
-			{
-				role: "user",
-				content: content,
-			},
-		],
-		temperature: 0,
-		max_tokens: 76000,
-	};
-
-	console.log(
-		"Sending payload to LLM for spoken tokens:",
-		JSON.stringify(payload, null, 2),
-	);
-
-	try {
-		const response = await fetch(
-			`${getSelectedModelEndpoint()}/chat/completions`,
-			{
-				method: "POST",
-				headers: {
-					"Content-Type": "application/json",
-					Authorization: `Bearer ${API_KEY}`,
-				},
-				body: JSON.stringify(payload),
-			},
-		);
-
-		if (!response.ok) {
-			const errorText = await response.text();
-			console.error(`HTTP error! status: ${response.status}`, errorText);
-			throw new Error(`HTTP error! status: ${response.status}`);
-		}
-
-		const data = await response.json();
-		return data.choices[0].message.content;
-	} catch (error) {
-		console.error("Error in processContentWithHermes:", error);
-		throw error;
-	}
+	// Just call extractSpokenTokens for consistency
+	return await extractSpokenTokens(content);
 }
 
 // Extract spoken tokens using Hermes - for Read Page functionality
 export async function extractSpokenTokens(content) {
-	// Pre-clean content with JavaScript to help Hermes
-	const preCleanedContent = content
+	// If content is already markdown from extractWebpageContent, use it directly
+	// Otherwise, pre-clean HTML content
+	const isMarkdown = content.includes('**Page Title**:') || content.includes('[') && content.includes('](');
+	
+	const preCleanedContent = isMarkdown ? content : content
 		// Remove script and style tags completely
 		.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
 		.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
@@ -190,15 +134,16 @@ export async function extractSpokenTokens(content) {
 		messages: [
 			{
 				role: "system",
-				content: `You are extracting the main article content for text-to-speech. The input has been pre-cleaned of scripts and ads.
+				content: `You are extracting the main article content for text-to-speech. The input may be markdown or pre-cleaned HTML.
 
 Your task:
 1. Find and extract the main article content (title, subtitle, body text)
 2. Preserve the exact wording - do not summarize or paraphrase
 3. Skip navigation menus, sidebars, footers, and advertisements
 4. For lists or bullet points, add "..." between items for natural pauses
-5. Remove any remaining HTML tags but keep the text inside them
-6. Start with the article title, then the main content
+5. Convert markdown to plain text: **bold** → bold, [text](url) → text
+6. Remove any remaining HTML tags but keep the text inside them
+7. Start with the article title, then the main content
 
 Return only the article text ready for TTS, nothing else.`,
 			},