From 47bf62ef666a6703db2d75f571cc719623253244 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 28 Jan 2026 07:53:11 -0500 Subject: [PATCH] Add streaming TTS using MediaSource API for instant playback - Audio starts playing as soon as ~1KB arrives instead of waiting for full download - Browser-aware format detection (webm+opus for Firefox, mp3 for Chrome) - Falls back to buffered download if MediaSource not supported - Updated speakText() and speakTextQueued() to use streaming - Cache blobs instead of Audio objects for cleaner replay --- templates/chat.html | 320 +++++++++++++++++++++++++++++++++----------- 1 file changed, 244 insertions(+), 76 deletions(-) diff --git a/templates/chat.html b/templates/chat.html index d20318c..cd76d7c 100644 --- a/templates/chat.html +++ b/templates/chat.html @@ -101,6 +101,20 @@ const VOICES_API_URL = "https://speech.ai.unturf.com/v1/voices"; // Code execution API (proxied through backend to keep API key secure) const room_name = "{{ room_name }}"; +// TTS Streaming: Detect best audio format for this browser +// Chrome/Edge: mp3 works, MediaSource supports audio/mpeg +// Firefox: mp3 often broken on Linux; use webm+opus +function detectAudioFormat() { + const isFirefox = navigator.userAgent.includes('Firefox'); + if (isFirefox) { + console.log("Firefox detected, using webm+opus format for TTS"); + return { format: 'webm', mime: 'audio/webm', mseMime: 'audio/webm;codecs=opus' }; + } + // Chromium-based browsers: mp3 works, MSE supports audio/mpeg + return { format: 'mp3', mime: 'audio/mpeg', mseMime: 'audio/mpeg' }; +} +const AUDIO_FORMAT = detectAudioFormat(); + // Get username from server (authenticated user's display name or None) let username = {% if username %}"{{ username }}"{% else %}null{% endif %}; @@ -775,13 +789,163 @@ function enableDownloadButton(messageId, playButton, audioUrl, voice) { downloadButton.onclick = () => { const link = document.createElement('a'); link.href = audioUrl; - link.download = `tts-${messageId}-${voice}.mp3`; + link.download = `tts-${messageId}-${voice}.${AUDIO_FORMAT.format}`; link.click(); }; } } -// Function to read text using TTS (for manual button clicks) +// Streaming TTS function - plays audio as chunks arrive using MediaSource API +// Returns a promise that resolves with { audio, blob, blobUrl, streamed } +async function fetchTTSStreaming(cleanText, model, voice) { + const mseMime = AUDIO_FORMAT.mseMime; + const canStream = mseMime && window.MediaSource && MediaSource.isTypeSupported(mseMime); + const requestFormat = AUDIO_FORMAT.format; + + console.log(`TTS streaming: canStream=${canStream}, format=${requestFormat}, mseMime=${mseMime}`); + + const response = await fetch(TTS_API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${API_KEY}` + }, + body: JSON.stringify({ + model: model, + voice: voice, + input: cleanText, + response_format: requestFormat + }) + }); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + // If MSE isn't available, fall back to full buffered download + if (!canStream) { + console.warn(`No streaming support for ${requestFormat}, using full buffer`); + const audioBlob = await response.blob(); + const audioUrl = URL.createObjectURL(audioBlob); + const audio = new Audio(audioUrl); + audio.playbackRate = 0.9; + return { audio, blob: audioBlob, blobUrl: audioUrl, streamed: false }; + } + + // Use MediaSource API for true streaming + const mediaSource = new MediaSource(); + const audioUrl = URL.createObjectURL(mediaSource); + const audio = new Audio(audioUrl); + audio.playbackRate = 0.9; + console.log(`TTS streaming: MediaSource created, readyState=${mediaSource.readyState}`); + + // Collect all chunks for download later + const chunks = []; + + // Create a promise that resolves when streaming is complete + const streamingComplete = new Promise((resolve, reject) => { + mediaSource.addEventListener('sourceopen', async () => { + console.log("TTS streaming: sourceopen fired"); + let sourceBuffer; + try { + sourceBuffer = mediaSource.addSourceBuffer(mseMime); + sourceBuffer.mode = 'sequence'; + } catch (e) { + console.error("Failed to create SourceBuffer:", e); + reject(e); + return; + } + + const reader = response.body.getReader(); + let totalBytes = 0; + + // Queue for appending buffers (SourceBuffer can only append one at a time) + const appendQueue = []; + let appending = false; + + function processQueue() { + if (appending || appendQueue.length === 0) return; + appending = true; + const chunk = appendQueue.shift(); + try { + sourceBuffer.appendBuffer(chunk); + } catch (e) { + console.error("appendBuffer error:", e); + appending = false; + } + } + + sourceBuffer.addEventListener('updateend', () => { + appending = false; + processQueue(); + }); + + try { + while (true) { + const { done: readerDone, value } = await reader.read(); + if (readerDone) break; + + chunks.push(value); + totalBytes += value.byteLength; + + // Queue the chunk for appending (use slice to avoid shared ArrayBuffer issues) + appendQueue.push(value.slice().buffer); + processQueue(); + + // Auto-play once we have some data (~1KB) + if (totalBytes > 1024 && audio.paused) { + console.log("TTS streaming: starting playback at", totalBytes, "bytes"); + audio.play().catch(e => console.warn("Auto-play blocked:", e.message)); + } + } + + // Wait for all queued appends to finish + await new Promise((res) => { + const check = () => { + if (!appending && appendQueue.length === 0) { + res(); + } else { + setTimeout(check, 50); + } + }; + check(); + }); + + if (mediaSource.readyState === 'open') { + mediaSource.endOfStream(); + } + + console.log(`TTS streaming complete: ${totalBytes} bytes`); + + // Build blob for download/caching + const audioBlob = new Blob(chunks, { type: mseMime }); + resolve({ blob: audioBlob }); + + } catch (error) { + console.error("Streaming read error:", error); + if (mediaSource.readyState === 'open') { + mediaSource.endOfStream('network'); + } + reject(error); + } + }); + + mediaSource.addEventListener('error', (e) => { + console.error("MediaSource error:", e); + reject(new Error("MediaSource error")); + }); + }); + + // Return immediately with audio element - streaming happens in background + return { + audio, + blobUrl: audioUrl, + streamed: true, + streamingComplete // Promise that resolves with { blob } when done + }; +} + +// Function to read text using TTS (for manual button clicks) - now with streaming async function speakText(text, playButton, messageId) { console.log("speakText called with:", {text: text, messageId, autoPlay: autoPlayTTS}); const voiceSelectValue = document.getElementById("voice-select").value; @@ -796,47 +960,42 @@ async function speakText(text, playButton, messageId) { try { // Check if the audio is already cached if (audioCache[cacheKey]) { - const audio = audioCache[cacheKey]; - enableDownloadButton(messageId, playButton, audio.src, voice); + const cachedData = audioCache[cacheKey]; + // Create new audio from cached blob for replay + const audioUrl = URL.createObjectURL(cachedData.blob); + const audio = new Audio(audioUrl); + audio.playbackRate = 0.9; + enableDownloadButton(messageId, playButton, audioUrl, voice); toggleAudioPlayback(audio, playButton); return; } - // Set button to processing state - playButton.textContent = "Processing..."; + // Set button to streaming state + playButton.textContent = "Streaming..."; playButton.disabled = true; - const response = await fetch(TTS_API_URL, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${API_KEY}` - }, - body: JSON.stringify({ - model: model, - voice: voice, - input: cleanText // Use the cleaned text - }) - }); + // Use streaming TTS + const result = await fetchTTSStreaming(cleanText, model, voice); + const audio = result.audio; - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - - const audioBlob = await response.blob(); - const audioUrl = URL.createObjectURL(audioBlob); - const audio = new Audio(audioUrl); - audio.playbackRate = 0.9; - - // Cache the audio only after it is successfully created - audioCache[cacheKey] = audio; - - // Enable and show download button - enableDownloadButton(messageId, playButton, audioUrl, voice); - - // Enable button and change text to "Pause" + // Enable button immediately and start playback playButton.disabled = false; toggleAudioPlayback(audio, playButton); + + // Handle streaming completion for caching and download button + if (result.streamed && result.streamingComplete) { + result.streamingComplete.then(({ blob }) => { + // Cache the blob for replay + audioCache[cacheKey] = { blob }; + // Create downloadable URL from blob + const downloadUrl = URL.createObjectURL(blob); + enableDownloadButton(messageId, playButton, downloadUrl, voice); + }).catch(e => console.error("Streaming completion error:", e)); + } else { + // Non-streamed fallback - cache immediately + audioCache[cacheKey] = { blob: result.blob }; + enableDownloadButton(messageId, playButton, result.blobUrl, voice); + } } catch (error) { console.error('Error in TTS:', error); playButton.textContent = "Play"; // Reset button text on error @@ -844,9 +1003,9 @@ async function speakText(text, playButton, messageId) { } } -// Function to read text using TTS (for queued auto-play) +// Function to read text using TTS (for queued auto-play) - now with streaming async function speakTextQueued(text, playButton, messageId) { - return new Promise((resolve, reject) => { + return new Promise(async (resolve, reject) => { const voiceSelectValue = document.getElementById("voice-select").value; // Parse model and voice from the dropdown value (format: "model:voice") @@ -854,60 +1013,69 @@ async function speakTextQueued(text, playButton, messageId) { const cacheKey = `${messageId}-${voiceSelectValue}`; const cleanText = text.replace(/[^a-zA-Z0-9\s.,!?]/g, ''); - const playAudio = (audio) => { - currentQueuedAudio = audio; // Track the currently playing queued audio + // Check if audio is cached + if (audioCache[cacheKey]) { + const cachedData = audioCache[cacheKey]; + // Create new audio from cached blob for replay + const audioUrl = URL.createObjectURL(cachedData.blob); + const audio = new Audio(audioUrl); + audio.playbackRate = 0.9; + enableDownloadButton(messageId, playButton, audioUrl, voice); + + currentQueuedAudio = audio; audio.onended = () => { console.log("TTS finished for:", messageId); - currentQueuedAudio = null; // Clear when finished + currentQueuedAudio = null; resolve(); }; audio.onerror = () => { console.error("TTS audio error for:", messageId); - currentQueuedAudio = null; // Clear on error + currentQueuedAudio = null; reject(new Error("Audio playback failed")); }; audio.play().catch(reject); - }; - - // Check if audio is cached - if (audioCache[cacheKey]) { - const audio = audioCache[cacheKey]; - enableDownloadButton(messageId, playButton, audio.src, voice); - playAudio(audio); return; } - // Fetch new audio - fetch(TTS_API_URL, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${API_KEY}` - }, - body: JSON.stringify({ - model: model, - voice: voice, - input: cleanText - }) - }) - .then(response => { - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); + try { + // Use streaming TTS for faster playback start + const result = await fetchTTSStreaming(cleanText, model, voice); + const audio = result.audio; + + currentQueuedAudio = audio; + audio.onended = () => { + console.log("TTS finished for:", messageId); + currentQueuedAudio = null; + resolve(); + }; + audio.onerror = () => { + console.error("TTS audio error for:", messageId); + currentQueuedAudio = null; + reject(new Error("Audio playback failed")); + }; + + // Handle streaming completion for caching + if (result.streamed && result.streamingComplete) { + result.streamingComplete.then(({ blob }) => { + // Cache the blob for replay + audioCache[cacheKey] = { blob }; + // Create downloadable URL from blob + const downloadUrl = URL.createObjectURL(blob); + enableDownloadButton(messageId, playButton, downloadUrl, voice); + }).catch(e => console.error("Streaming completion error:", e)); + } else { + // Non-streamed fallback - cache immediately + audioCache[cacheKey] = { blob: result.blob }; + enableDownloadButton(messageId, playButton, result.blobUrl, voice); } - return response.blob(); - }) - .then(audioBlob => { - const audioUrl = URL.createObjectURL(audioBlob); - const audio = new Audio(audioUrl); - audio.playbackRate = 0.9; - audioCache[cacheKey] = audio; - // Enable and show download button - enableDownloadButton(messageId, playButton, audioUrl, voice); - - playAudio(audio); - }) - .catch(reject); + // Audio should auto-play from streaming, but ensure it starts + if (audio.paused) { + audio.play().catch(reject); + } + } catch (error) { + reject(error); + } }); }