From b503c4d5d6f56c2fb734a66218af18b12838fe07 Mon Sep 17 00:00:00 2001
From: "russell@unturf.com" <russell@unturf.com>
Date: Fri, 29 May 2026 12:29:18 -0400
Subject: [PATCH] F5-TTS: add opt-in sentence timestamps mode
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add `timestamps: bool` to GenerateSpeechRequest. When set for tts-1-f5,
generate the whole clip up front and return JSON {audio (base64), format,
sample_rate, duration_ms, sentences:[{index,text,start_ms,end_ms}]} instead
of streaming audio. F5 already synthesizes one PCM chunk per sentence, so
each chunk's sample count is its exact duration — no forced aligner needed.
Lets clients highlight each sentence as it is spoken. Non-F5 models with
timestamps=true get a clear 400. Default off preserves streaming behavior.
---
 speech.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 51 insertions(+), 1 deletion(-)

diff --git a/speech.py b/speech.py
index 69360e5..df67492 100755
--- a/speech.py
+++ b/speech.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import argparse
 import asyncio
+import base64
 import contextlib
 import gc
 import os
@@ -13,7 +14,7 @@ import time
 import yaml
 import json
 
-from fastapi.responses import StreamingResponse
+from fastapi.responses import StreamingResponse, JSONResponse
 from loguru import logger
 from openedai import OpenAIStub, BadRequestError, ServiceUnavailableError
 from pydantic import BaseModel
@@ -719,6 +720,7 @@ class GenerateSpeechRequest(BaseModel):
     voice: str = "alloy"  # alloy, echo, fable, onyx, nova, and shimmer
     response_format: str = "mp3" # mp3, opus, aac, flac
     speed: float = 1.0 # 0.25 - 4.0
+    timestamps: bool = False  # F5 only: return JSON {audio, sentences:[{start_ms,end_ms}]} instead of streaming audio
 
 def build_ffmpeg_args(response_format, input_format, sample_rate):
     # Convert the output to the desired format using ffmpeg
@@ -855,6 +857,9 @@ async def generate_speech(request: GenerateSpeechRequest):
         else:
             raise BadRequestError(f"Voice '{voice}' not found in any model. Please specify a model.", param='voice')
 
+    if request.timestamps and model != 'tts-1-f5':
+        raise BadRequestError("timestamps mode is currently supported only for model 'tts-1-f5'", param='timestamps')
+
     # Set the Content-Type header based on the requested format
     if response_format == "mp3":
         media_type = "audio/mpeg"
@@ -1320,6 +1325,51 @@ async def generate_speech(request: GenerateSpeechRequest):
         sentences = simple_sentence_split(input_text, max_length=500)
         logger.info(f"F5-TTS streaming {len(sentences)} sentence chunks from {len(input_text)} chars")
 
+        # Opt-in: generate the whole clip up front and return JSON carrying exact
+        # per-sentence timing. F5 already synthesizes one PCM chunk per sentence,
+        # so each chunk's sample count IS its duration — no forced aligner needed.
+        # Trades streaming for exact sync (used by clients that highlight as they read).
+        if request.timestamps:
+            pcm_parts = []
+            sentence_meta = []
+            cursor_ms = 0.0
+            for idx, sentence in enumerate(sentences):
+                audio_bytes = await asyncio.to_thread(
+                    f5_model.tts,
+                    text=sentence,
+                    ref_audio=ref_audio,
+                    ref_text=ref_text,
+                    speed=speed,
+                )
+                # f5_wrapper.tts() returns float32 PCM @ 24kHz (4 bytes/sample).
+                dur_ms = (len(audio_bytes) // 4) / 24000.0 * 1000.0
+                sentence_meta.append({
+                    "index": idx,
+                    "text": sentence,
+                    "start_ms": round(cursor_ms),
+                    "end_ms": round(cursor_ms + dur_ms),
+                })
+                cursor_ms += dur_ms
+                pcm_parts.append(audio_bytes)
+
+            pcm = b"".join(pcm_parts)
+            ff = build_ffmpeg_args(response_format, input_format="f32le", sample_rate="24000")
+            ff.extend(["-"])
+            ff_proc = subprocess.Popen(ff, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            out_audio, ff_err = await asyncio.to_thread(ff_proc.communicate, pcm)
+            if ff_proc.returncode != 0:
+                logger.error(f"F5-TTS timestamps ffmpeg failed: {ff_err.decode('utf-8', 'replace')}")
+                raise ServiceUnavailableError("Audio encoding failed")
+
+            logger.info(f"F5-TTS timestamps: {len(sentences)} sentences, {round(cursor_ms)}ms total")
+            return JSONResponse({
+                "audio": base64.b64encode(out_audio).decode("ascii"),
+                "format": response_format,
+                "sample_rate": 24000,
+                "duration_ms": round(cursor_ms),
+                "sentences": sentence_meta,
+            })
+
         # F5-TTS outputs float32 PCM at 24kHz
         ffmpeg_args = build_ffmpeg_args(response_format, input_format="f32le", sample_rate="24000")
         ffmpeg_args.extend(["-"])