Fix streaming TTS to return audio immediately, collect stream in background
- speakTextStreaming now returns immediately with the audio element already playing, plus a done promise for when the stream completes - Modal wires up controls immediately using the live audio element - Pause/play works during streaming (stream continues in background) - Button shows "Streaming..." while receiving, switches to "Pause" when done - Download button enables only after stream completes - All TTS functions now return consistent done promise interface
This commit is contained in:
parent
81dbae3c7a
commit
519e281623
2 changed files with 190 additions and 137 deletions
|
|
@ -347,164 +347,206 @@ export function openTTSModal() {
|
|||
playButton.disabled = true;
|
||||
|
||||
const result = await handleTTS(text, voice, speed, model);
|
||||
if (result.streamed) {
|
||||
console.log("TTS played via streaming mode");
|
||||
}
|
||||
if (result.audio) {
|
||||
resultDiv.classList.add("show");
|
||||
resultDiv.innerHTML = "";
|
||||
if (!result.audio) return;
|
||||
|
||||
// Create hidden audio element (no controls)
|
||||
const audio = new Audio(result.blobUrl);
|
||||
const isStreaming = result.streamed === true;
|
||||
console.log(`TTS result: streamed=${isStreaming}`);
|
||||
|
||||
resultDiv.classList.add("show");
|
||||
resultDiv.innerHTML = "";
|
||||
|
||||
// Use the audio element from the result directly (critical for streaming -
|
||||
// it's already playing via MediaSource, creating a new Audio would lose the stream)
|
||||
const audio = result.audio;
|
||||
|
||||
// For buffered mode, the audio isn't playing yet, so set it up
|
||||
if (!isStreaming) {
|
||||
audio.preload = "auto";
|
||||
audio.playbackRate = speed; // Apply the speed setting!
|
||||
|
||||
// Create custom control container
|
||||
const controlsContainer = document.createElement("div");
|
||||
controlsContainer.className = "tts-audio-controls";
|
||||
}
|
||||
|
||||
// Create play/pause button
|
||||
const playPauseBtn = document.createElement("button");
|
||||
playPauseBtn.textContent = "⏸️ Pause";
|
||||
// Create custom control container
|
||||
const controlsContainer = document.createElement("div");
|
||||
controlsContainer.className = "tts-audio-controls";
|
||||
|
||||
// Create progress bar
|
||||
const progressContainer = document.createElement("div");
|
||||
progressContainer.className = "tts-progress-container";
|
||||
// Create play/pause button
|
||||
const playPauseBtn = document.createElement("button");
|
||||
playPauseBtn.textContent = isStreaming ? "⏸️ Streaming..." : "⏸️ Pause";
|
||||
|
||||
const progressBar = document.createElement("div");
|
||||
progressBar.className = "tts-progress-bar";
|
||||
progressContainer.appendChild(progressBar);
|
||||
// Create progress bar
|
||||
const progressContainer = document.createElement("div");
|
||||
progressContainer.className = "tts-progress-container";
|
||||
|
||||
// Create time display
|
||||
const timeDisplay = document.createElement("span");
|
||||
timeDisplay.textContent = "0:00 / 0:00";
|
||||
timeDisplay.className = "tts-time-display";
|
||||
const progressBar = document.createElement("div");
|
||||
progressBar.className = "tts-progress-bar";
|
||||
progressContainer.appendChild(progressBar);
|
||||
|
||||
// Create download button
|
||||
const downloadBtn = document.createElement("button");
|
||||
// Create time display
|
||||
const timeDisplay = document.createElement("span");
|
||||
timeDisplay.textContent = "0:00 / 0:00";
|
||||
timeDisplay.className = "tts-time-display";
|
||||
|
||||
// Create download button (disabled until stream completes)
|
||||
const downloadBtn = document.createElement("button");
|
||||
downloadBtn.className = "download-btn";
|
||||
let downloadBlobUrl = result.blobUrl;
|
||||
|
||||
if (isStreaming && !result.blob) {
|
||||
downloadBtn.textContent = "💾 Loading...";
|
||||
downloadBtn.disabled = true;
|
||||
} else {
|
||||
downloadBtn.textContent = "💾 Download";
|
||||
downloadBtn.className = "download-btn";
|
||||
downloadBtn.onclick = async () => {
|
||||
try {
|
||||
// Generate filename based on textarea content
|
||||
const filename = await generateTitleForTTS(text);
|
||||
downloadAudio(result.blobUrl, `${filename}.mp3`);
|
||||
} catch (error) {
|
||||
console.error("Error generating filename:", error);
|
||||
// Fallback to default filename
|
||||
downloadAudio(result.blobUrl, "tts-audio.mp3");
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Assemble controls
|
||||
controlsContainer.appendChild(playPauseBtn);
|
||||
controlsContainer.appendChild(progressContainer);
|
||||
controlsContainer.appendChild(timeDisplay);
|
||||
controlsContainer.appendChild(downloadBtn);
|
||||
resultDiv.appendChild(controlsContainer);
|
||||
downloadBtn.onclick = async () => {
|
||||
if (!downloadBlobUrl) return;
|
||||
try {
|
||||
const filename = await generateTitleForTTS(text);
|
||||
downloadAudio(downloadBlobUrl, `${filename}.mp3`);
|
||||
} catch (error) {
|
||||
console.error("Error generating filename:", error);
|
||||
downloadAudio(downloadBlobUrl, "tts-audio.mp3");
|
||||
}
|
||||
};
|
||||
|
||||
// Audio event handlers
|
||||
let isPlaying = false;
|
||||
// Assemble controls
|
||||
controlsContainer.appendChild(playPauseBtn);
|
||||
controlsContainer.appendChild(progressContainer);
|
||||
controlsContainer.appendChild(timeDisplay);
|
||||
controlsContainer.appendChild(downloadBtn);
|
||||
resultDiv.appendChild(controlsContainer);
|
||||
|
||||
const formatTime = (seconds) => {
|
||||
const mins = Math.floor(seconds / 60);
|
||||
const secs = Math.floor(seconds % 60);
|
||||
return `${mins}:${secs.toString().padStart(2, '0')}`;
|
||||
};
|
||||
// Track state
|
||||
let isPlaying = !audio.paused;
|
||||
let streamDone = !isStreaming; // buffered mode is already "done"
|
||||
|
||||
const updateProgress = () => {
|
||||
if (audio.duration) {
|
||||
const progress = (audio.currentTime / audio.duration) * 100;
|
||||
progressBar.style.width = `${progress}%`;
|
||||
timeDisplay.textContent = `${formatTime(audio.currentTime)} / ${formatTime(audio.duration)}`;
|
||||
}
|
||||
};
|
||||
const formatTime = (seconds) => {
|
||||
if (!isFinite(seconds)) return "?:??";
|
||||
const mins = Math.floor(seconds / 60);
|
||||
const secs = Math.floor(seconds % 60);
|
||||
return `${mins}:${secs.toString().padStart(2, '0')}`;
|
||||
};
|
||||
|
||||
playPauseBtn.onclick = () => {
|
||||
if (isPlaying) {
|
||||
audio.pause();
|
||||
playPauseBtn.textContent = "▶️ Play";
|
||||
isPlaying = false;
|
||||
} else {
|
||||
audio.play();
|
||||
playPauseBtn.textContent = "⏸️ Pause";
|
||||
isPlaying = true;
|
||||
}
|
||||
};
|
||||
const updateProgress = () => {
|
||||
if (audio.duration && isFinite(audio.duration)) {
|
||||
const progress = (audio.currentTime / audio.duration) * 100;
|
||||
progressBar.style.width = `${progress}%`;
|
||||
timeDisplay.textContent = `${formatTime(audio.currentTime)} / ${formatTime(audio.duration)}`;
|
||||
} else if (isStreaming) {
|
||||
// During streaming, duration may not be known yet
|
||||
timeDisplay.textContent = `${formatTime(audio.currentTime)} / streaming...`;
|
||||
}
|
||||
};
|
||||
|
||||
// Progress bar click handler
|
||||
progressContainer.onclick = (e) => {
|
||||
if (audio.duration) {
|
||||
const rect = progressContainer.getBoundingClientRect();
|
||||
const clickX = e.clientX - rect.left;
|
||||
const clickRatio = clickX / rect.width;
|
||||
audio.currentTime = clickRatio * audio.duration;
|
||||
}
|
||||
};
|
||||
const updatePlayPauseBtn = () => {
|
||||
if (isPlaying) {
|
||||
playPauseBtn.textContent = streamDone ? "⏸️ Pause" : "⏸️ Streaming...";
|
||||
} else {
|
||||
playPauseBtn.textContent = "▶️ Play";
|
||||
}
|
||||
};
|
||||
|
||||
// Audio event listeners
|
||||
audio.addEventListener('loadedmetadata', () => {
|
||||
timeDisplay.textContent = `0:00 / ${formatTime(audio.duration)}`;
|
||||
});
|
||||
|
||||
audio.addEventListener('timeupdate', updateProgress);
|
||||
|
||||
audio.addEventListener('ended', () => {
|
||||
playPauseBtn.textContent = "▶️ Replay";
|
||||
playPauseBtn.onclick = () => {
|
||||
if (isPlaying) {
|
||||
audio.pause();
|
||||
isPlaying = false;
|
||||
progressBar.style.width = '0%';
|
||||
audio.currentTime = 0;
|
||||
});
|
||||
} else {
|
||||
audio.play();
|
||||
isPlaying = true;
|
||||
}
|
||||
updatePlayPauseBtn();
|
||||
};
|
||||
|
||||
audio.addEventListener('error', (e) => {
|
||||
console.error('Audio playback error:', e);
|
||||
playPauseBtn.textContent = "❌ Error";
|
||||
playPauseBtn.disabled = true;
|
||||
|
||||
// Show user-friendly error message
|
||||
const errorMsg = document.createElement('div');
|
||||
errorMsg.className = "tts-error-message";
|
||||
errorMsg.textContent = 'Audio failed to load. Please try again or check your connection.';
|
||||
controlsContainer.appendChild(errorMsg);
|
||||
});
|
||||
// Progress bar click handler
|
||||
progressContainer.onclick = (e) => {
|
||||
if (audio.duration && isFinite(audio.duration)) {
|
||||
const rect = progressContainer.getBoundingClientRect();
|
||||
const clickX = e.clientX - rect.left;
|
||||
const clickRatio = clickX / rect.width;
|
||||
audio.currentTime = clickRatio * audio.duration;
|
||||
}
|
||||
};
|
||||
|
||||
audio.addEventListener('abort', (e) => {
|
||||
console.warn('Audio loading was aborted:', e);
|
||||
if (isPlaying) {
|
||||
playPauseBtn.textContent = "▶️ Retry";
|
||||
isPlaying = false;
|
||||
}
|
||||
});
|
||||
// Audio event listeners
|
||||
audio.addEventListener('loadedmetadata', () => {
|
||||
if (isFinite(audio.duration)) {
|
||||
timeDisplay.textContent = `0:00 / ${formatTime(audio.duration)}`;
|
||||
}
|
||||
});
|
||||
|
||||
audio.addEventListener('stalled', (e) => {
|
||||
console.warn('Audio loading stalled:', e);
|
||||
playPauseBtn.textContent = "⏳ Loading...";
|
||||
});
|
||||
audio.addEventListener('timeupdate', updateProgress);
|
||||
|
||||
// Auto-play and set initial state with improved error handling
|
||||
audio.addEventListener('ended', () => {
|
||||
playPauseBtn.textContent = "▶️ Replay";
|
||||
isPlaying = false;
|
||||
progressBar.style.width = '0%';
|
||||
audio.currentTime = 0;
|
||||
});
|
||||
|
||||
audio.addEventListener('error', (e) => {
|
||||
console.error('Audio playback error:', e);
|
||||
playPauseBtn.textContent = "❌ Error";
|
||||
playPauseBtn.disabled = true;
|
||||
const errorMsg = document.createElement('div');
|
||||
errorMsg.className = "tts-error-message";
|
||||
errorMsg.textContent = 'Audio failed to load. Please try again or check your connection.';
|
||||
controlsContainer.appendChild(errorMsg);
|
||||
});
|
||||
|
||||
audio.addEventListener('abort', (e) => {
|
||||
console.warn('Audio loading was aborted:', e);
|
||||
if (isPlaying) {
|
||||
playPauseBtn.textContent = "▶️ Retry";
|
||||
isPlaying = false;
|
||||
}
|
||||
});
|
||||
|
||||
audio.addEventListener('stalled', (e) => {
|
||||
console.warn('Audio loading stalled:', e);
|
||||
if (!streamDone) {
|
||||
playPauseBtn.textContent = "⏳ Buffering...";
|
||||
}
|
||||
});
|
||||
|
||||
// Wait for stream to complete in background (doesn't block UI)
|
||||
result.done.then(({ blob, blobUrl }) => {
|
||||
streamDone = true;
|
||||
downloadBlobUrl = blobUrl;
|
||||
downloadBtn.textContent = "💾 Download";
|
||||
downloadBtn.disabled = false;
|
||||
console.log("Stream done, download ready");
|
||||
// Update button text now that stream is complete
|
||||
if (isPlaying) {
|
||||
playPauseBtn.textContent = "⏸️ Pause";
|
||||
}
|
||||
}).catch((err) => {
|
||||
console.error("Stream failed:", err);
|
||||
downloadBtn.textContent = "💾 Failed";
|
||||
});
|
||||
|
||||
// For buffered mode, auto-play; for streaming, it's already playing
|
||||
if (!isStreaming) {
|
||||
audio.play().then(() => {
|
||||
isPlaying = true;
|
||||
playPauseBtn.textContent = "⏸️ Pause";
|
||||
updatePlayPauseBtn();
|
||||
}).catch((e) => {
|
||||
console.warn('Auto-play failed, user interaction required:', e.name, e.message);
|
||||
console.warn('Auto-play failed:', e.name, e.message);
|
||||
isPlaying = false;
|
||||
playPauseBtn.textContent = "▶️ Play";
|
||||
|
||||
// Handle specific error types
|
||||
if (e.name === 'NotAllowedError') {
|
||||
console.log('Auto-play blocked by browser - user interaction required');
|
||||
} else if (e.name === 'AbortError') {
|
||||
console.log('Audio loading was aborted');
|
||||
} else if (e.name === 'NotSupportedError') {
|
||||
console.error('Audio format not supported');
|
||||
if (e.name === 'NotSupportedError') {
|
||||
playPauseBtn.textContent = "❌ Unsupported";
|
||||
playPauseBtn.disabled = true;
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// Streaming mode - audio is already playing
|
||||
updatePlayPauseBtn();
|
||||
}
|
||||
|
||||
// Re-enable the generate button now that controls are shown
|
||||
playButton.textContent = getUIText("playText");
|
||||
playButton.disabled = false;
|
||||
} catch (error) {
|
||||
alert(getUIText("ttsFailed", { error: error.message }));
|
||||
} finally {
|
||||
playButton.textContent = getUIText("playText");
|
||||
playButton.disabled = false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -70,7 +70,9 @@ export async function speakText(text, voice = "alloy", rate = 0.9, model = "tts-
|
|||
// Store the last TTS result
|
||||
setLastTTS(text, { audio, blob: audioBlob });
|
||||
|
||||
return { audio, blob: audioBlob, blobUrl: audioUrl };
|
||||
const result = { audio, blob: audioBlob, blobUrl: audioUrl };
|
||||
result.done = Promise.resolve({ blob: audioBlob, blobUrl: audioUrl });
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.error("Error in TTS:", error);
|
||||
throw error;
|
||||
|
|
@ -110,7 +112,9 @@ export async function speakTextDirect(text, voice = "alloy", rate = 0.9, model =
|
|||
// Store the last TTS result
|
||||
setLastTTS(text, { audio, blob: audioBlob });
|
||||
|
||||
return { audio, blob: audioBlob, blobUrl: audioUrl };
|
||||
const result = { audio, blob: audioBlob, blobUrl: audioUrl };
|
||||
result.done = Promise.resolve({ blob: audioBlob, blobUrl: audioUrl });
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.error("Error in TTS:", error);
|
||||
throw error;
|
||||
|
|
@ -118,6 +122,7 @@ export async function speakTextDirect(text, voice = "alloy", rate = 0.9, model =
|
|||
}
|
||||
|
||||
// Streaming TTS function - plays audio as chunks arrive using MediaSource API
|
||||
// Returns immediately with audio element; done promise resolves when stream completes
|
||||
export async function speakTextStreaming(text, voice = "alloy", rate = 0.9, model = "tts-1-qwen") {
|
||||
// Clean text with simple JavaScript instead of Hermes
|
||||
const cleanedText = cleanTextForTTS(text);
|
||||
|
|
@ -143,13 +148,12 @@ export async function speakTextStreaming(text, voice = "alloy", rate = 0.9, mode
|
|||
// Check if MediaSource is supported and can handle audio/mpeg
|
||||
if (!window.MediaSource || !MediaSource.isTypeSupported('audio/mpeg')) {
|
||||
console.warn("MediaSource API doesn't support audio/mpeg, falling back to buffered mode");
|
||||
// Fallback: buffer entire response but at least we tried
|
||||
const audioBlob = await response.blob();
|
||||
const audioUrl = URL.createObjectURL(audioBlob);
|
||||
const audio = new Audio(audioUrl);
|
||||
audio.playbackRate = rate;
|
||||
setLastTTS(text, { audio, blob: audioBlob });
|
||||
return { audio, blob: audioBlob, blobUrl: audioUrl, streamed: false };
|
||||
return { audio, blob: audioBlob, blobUrl: audioUrl, streamed: false, done: Promise.resolve({ blob: audioBlob, blobUrl: audioUrl }) };
|
||||
}
|
||||
|
||||
const mediaSource = new MediaSource();
|
||||
|
|
@ -160,7 +164,8 @@ export async function speakTextStreaming(text, voice = "alloy", rate = 0.9, mode
|
|||
// Collect all chunks for download later
|
||||
const chunks = [];
|
||||
|
||||
const streamPromise = new Promise((resolve, reject) => {
|
||||
// done resolves when the full stream has been received and appended
|
||||
const done = new Promise((resolve, reject) => {
|
||||
mediaSource.addEventListener('sourceopen', async () => {
|
||||
let sourceBuffer;
|
||||
try {
|
||||
|
|
@ -197,8 +202,8 @@ export async function speakTextStreaming(text, voice = "alloy", rate = 0.9, mode
|
|||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
const { done: readerDone, value } = await reader.read();
|
||||
if (readerDone) break;
|
||||
|
||||
chunks.push(value);
|
||||
totalBytes += value.byteLength;
|
||||
|
|
@ -234,7 +239,7 @@ export async function speakTextStreaming(text, voice = "alloy", rate = 0.9, mode
|
|||
// Build blob for download/storage
|
||||
const audioBlob = new Blob(chunks, { type: 'audio/mpeg' });
|
||||
setLastTTS(text, { audio, blob: audioBlob });
|
||||
resolve({ audio, blob: audioBlob, blobUrl: audioUrl, streamed: true });
|
||||
resolve({ blob: audioBlob, blobUrl: audioUrl });
|
||||
|
||||
} catch (error) {
|
||||
console.error("Streaming read error:", error);
|
||||
|
|
@ -254,7 +259,8 @@ export async function speakTextStreaming(text, voice = "alloy", rate = 0.9, mode
|
|||
// Start playing immediately (will buffer internally)
|
||||
audio.play().catch(e => console.warn("Initial play blocked:", e.message));
|
||||
|
||||
return streamPromise;
|
||||
// Return immediately with audio element - caller uses done promise to know when stream finishes
|
||||
return { audio, blob: null, blobUrl: audioUrl, streamed: true, done };
|
||||
}
|
||||
|
||||
// Process page content using Hermes (deprecated - use extractSpokenTokens instead)
|
||||
|
|
@ -407,7 +413,9 @@ export async function generateTitleForTTS(text) {
|
|||
.substring(0, 50); // Limit length for filesystem compatibility
|
||||
}
|
||||
|
||||
// Handle TTS from element (for modal usage - respects streaming mode setting)
|
||||
// Handle TTS - respects streaming mode setting
|
||||
// In streaming mode, returns immediately with audio playing and a `done` promise
|
||||
// In buffered mode, waits for full download then returns
|
||||
export async function handleTTS(text, voice = "alloy", rate = 0.9, model = "tts-1-qwen") {
|
||||
try {
|
||||
const mode = getTTSMode();
|
||||
|
|
@ -415,7 +423,10 @@ export async function handleTTS(text, voice = "alloy", rate = 0.9, model = "tts-
|
|||
if (mode === "streaming") {
|
||||
return await speakTextStreaming(text, voice, rate, model);
|
||||
} else {
|
||||
return await speakTextDirect(text, voice, rate, model);
|
||||
const result = await speakTextDirect(text, voice, rate, model);
|
||||
// Add done: already-resolved promise for consistent interface
|
||||
result.done = Promise.resolve({ blob: result.blob, blobUrl: result.blobUrl });
|
||||
return result;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error in handleTTS:", error);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue