From 4dc9568079310b30c6ab0343844ddbcd283e02b6 Mon Sep 17 00:00:00 2001
From: Russell Ballestrini <russell.ballestrini@gmail.com>
Date: Thu, 3 Jul 2025 20:23:52 -0400
Subject: [PATCH] feat: improve extractSpokenTokens with JavaScript pre and
 post-processing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Pre-clean HTML before sending to Hermes (remove scripts, styles, ads, iframes)
- Decode HTML entities (&nbsp; &amp; etc.) before Hermes processing
- Simplified, clearer system prompt focused on extraction tasks
- Post-process Hermes output with cleanTextForTTS() for final cleanup
- Reduces Hermes workload and improves extraction quality
- Three-stage pipeline: JS pre-clean → Hermes extraction → JS post-clean
---
 src/tts.js | 37 ++++++++++++++++++++++++++++++++++---
 1 file changed, 34 insertions(+), 3 deletions(-)

diff --git a/src/tts.js b/src/tts.js
index 7823279..2d8d409 100644
--- a/src/tts.js
+++ b/src/tts.js
@@ -167,16 +167,44 @@ Ready? Breath and then return a stream of tokens to be used in a TTS system.
 
 // Extract spoken tokens using Hermes - for Read Page functionality
 export async function extractSpokenTokens(content) {
+	// Pre-clean content with JavaScript to help Hermes
+	const preCleanedContent = content
+		// Remove script and style tags completely
+		.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+		.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+		// Remove common ad/tracking elements
+		.replace(/<(ins|iframe|object|embed)[^>]*>[\s\S]*?<\/\1>/gi, '')
+		// Clean up HTML entities
+		.replace(/&nbsp;/g, ' ')
+		.replace(/&amp;/g, '&')
+		.replace(/&lt;/g, '<')
+		.replace(/&gt;/g, '>')
+		.replace(/&quot;/g, '"')
+		.replace(/&#39;/g, "'")
+		// Remove excessive whitespace
+		.replace(/\s+/g, ' ')
+		.trim();
+
 	const payload = {
 		model: getSelectedModel(),
 		messages: [
 			{
 				role: "system",
-				content: `Extract the exact main content of the article. Avoid reading ads. Do not change or summarize content. Return the same words verbatim. When you encounter a list or bullets, add . . . to make the TTS pause between items. Try to stay as close to the truth of the original version as possible. Start with the title of the post and then jump into it. Do not mention the TTS stream just do the work! Remember your only goal is to extract the entire & exact main content of the article. Ready? Breath and then return a stream of tokens to be used in a TTS system.`,
+				content: `You are extracting the main article content for text-to-speech. The input has been pre-cleaned of scripts and ads.
+
+Your task:
+1. Find and extract the main article content (title, subtitle, body text)
+2. Preserve the exact wording - do not summarize or paraphrase
+3. Skip navigation menus, sidebars, footers, and advertisements
+4. For lists or bullet points, add "..." between items for natural pauses
+5. Remove any remaining HTML tags but keep the text inside them
+6. Start with the article title, then the main content
+
+Return only the article text ready for TTS, nothing else.`,
 			},
 			{
 				role: "user",
-				content: content,
+				content: preCleanedContent,
 			},
 		],
 		temperature: 0,
@@ -205,7 +233,10 @@ export async function extractSpokenTokens(content) {
 		}
 
 		const data = await response.json();
-		return data.choices[0].message.content;
+		
+		// Post-process Hermes output with JavaScript cleaning
+		const finalContent = cleanTextForTTS(data.choices[0].message.content);
+		return finalContent;
 	} catch (error) {
 		console.error("Error in extractSpokenTokens:", error);
 		throw error;