F5-TTS: per-chunk silence trim + fade (port from VoiceClone) + restore per-sentence streaming

speech.py

@@ -511,6 +511,38 @@ class qwen3_wrapper():
             logger.error(f"Qwen3-TTS generation failed: {e}")
             raise
 
+def _f5_trim_audio(audio_np, sample_rate, silence_db=-40):
+    """Trim leading/trailing silence + fade + normalize for F5-TTS output.
+
+    Adapted from MonumentalSystems/VoiceClone web_tts_server.py (MIT).
+    Lets per-sentence chunking be artifact-free: silence-trim at -40 dB
+    cuts the brief ref-bleed at chunk start; small fade kills click.
+    """
+    import numpy as np
+    if len(audio_np) == 0:
+        return audio_np
+    abs_thresh = 10 ** (silence_db / 20)
+    above = np.where(np.abs(audio_np) > abs_thresh)[0]
+    if len(above) == 0:
+        return audio_np[:1]
+    lead_in = int(sample_rate * 0.01)   # keep 10 ms before first speech
+    trail_out = int(sample_rate * 0.03) # keep 30 ms after last speech
+    start_idx = max(0, above[0] - lead_in)
+    end_idx = min(len(audio_np), above[-1] + trail_out)
+    audio_np = audio_np[start_idx:end_idx].copy()
+    if len(audio_np) == 0:
+        return audio_np
+    fi = min(int(sample_rate * 0.015), len(audio_np))  # 15 ms fade in
+    fo = min(int(sample_rate * 0.005), len(audio_np))  # 5 ms fade out
+    if fi > 0:
+        audio_np[:fi] *= np.linspace(0.0, 1.0, fi, dtype=audio_np.dtype)
+    if fo > 0:
+        audio_np[-fo:] *= np.linspace(1.0, 0.0, fo, dtype=audio_np.dtype)
+    peak = float(np.max(np.abs(audio_np)))
+    if peak > 0.01:
+        audio_np = audio_np * (10 ** (-1.0 / 20) / peak)  # normalize -1 dB
+    return audio_np
+
 class f5_wrapper():
     """Wrapper for F5-TTS model (SWivid/F5-TTS, MIT)
 
@@ -569,9 +601,11 @@ class f5_wrapper():
             audio = np.asarray(wav, dtype=np.float32).flatten()
 
             self.sample_rate = sr
-            logger.info(f"F5-TTS generated {len(audio)} samples at {sr}Hz")
+            raw_len = len(audio)
+            audio = _f5_trim_audio(audio, sr, silence_db=-40)
+            logger.info(f"F5-TTS generated {raw_len} samples at {sr}Hz, trimmed to {len(audio)}")
 
-            return audio.tobytes()
+            return audio.astype(np.float32).tobytes()
 
         except Exception as e:
             logger.error(f"F5-TTS generation failed: {e}")
@@ -1295,10 +1329,11 @@ async def generate_speech(request: GenerateSpeechRequest):
                     logger.info(f"Loading F5-TTS model on device '{device}'")
                     f5_model = await asyncio.to_thread(f5_wrapper, device=device)
 
-        # F5-TTS handles long text via internal chunking. Per-sentence splitting
-        # produces a ref-to-gen transition artifact at the start of EACH chunk
-        # (audible as gibberish/tinny prefix). Pass full text as one inference.
-        logger.info(f"F5-TTS processing {len(input_text)} chars as single inference")
+        # Split into sentences so first audio bytes stream early (~2 s for long texts).
+        # f5_wrapper.tts() applies _f5_trim_audio per chunk to scrub the brief
+        # ref-bleed at chunk start so per-sentence boundaries stay clean.
+        sentences = simple_sentence_split(input_text, max_length=500)
+        logger.info(f"F5-TTS streaming {len(sentences)} sentence chunks from {len(input_text)} chars")
 
         # F5-TTS outputs float32 PCM at 24kHz
         ffmpeg_args = build_ffmpeg_args(response_format, input_format="f32le", sample_rate="24000")
@@ -1308,20 +1343,22 @@ async def generate_speech(request: GenerateSpeechRequest):
         in_q = queue.Queue()
 
         def generator():
-            """Run F5-TTS once for the full input text"""
+            """Process sentences sequentially and feed to queue"""
             try:
-                audio_bytes = f5_model.tts(
-                    text=input_text,
-                    ref_audio=ref_audio,
-                    ref_text=ref_text,
-                    speed=speed,
-                )
-                in_q.put(audio_bytes)
+                for idx, sentence in enumerate(sentences):
+                    logger.debug(f"F5-TTS chunk {idx+1}/{len(sentences)}: {len(sentence)} chars")
+                    audio_bytes = f5_model.tts(
+                        text=sentence,
+                        ref_audio=ref_audio,
+                        ref_text=ref_text,
+                        speed=speed,
+                    )
+                    in_q.put(audio_bytes)
             except Exception as e:
-                logger.error(f"F5-TTS generation error: {e}")
+                logger.error(f"F5-TTS streaming error: {e}")
             finally:
                 in_q.put(None)
-                logger.info("F5-TTS generation complete")
+                logger.info(f"F5-TTS streaming complete: {len(sentences)} chunks")
 
         def out_writer():
             """Write audio from queue to ffmpeg stdin"""