From 831b937fd12df77c57b0b0de9d0db35c65afef29 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sun, 24 May 2026 11:39:49 -0400 Subject: [PATCH] F5-TTS: per-chunk silence trim + fade (port from VoiceClone) + restore per-sentence streaming --- speech.py | 69 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 16 deletions(-) diff --git a/speech.py b/speech.py index 7878746..2747069 100755 --- a/speech.py +++ b/speech.py @@ -511,6 +511,38 @@ class qwen3_wrapper(): logger.error(f"Qwen3-TTS generation failed: {e}") raise +def _f5_trim_audio(audio_np, sample_rate, silence_db=-40): + """Trim leading/trailing silence + fade + normalize for F5-TTS output. + + Adapted from MonumentalSystems/VoiceClone web_tts_server.py (MIT). + Lets per-sentence chunking be artifact-free: silence-trim at -40 dB + cuts the brief ref-bleed at chunk start; small fade kills click. + """ + import numpy as np + if len(audio_np) == 0: + return audio_np + abs_thresh = 10 ** (silence_db / 20) + above = np.where(np.abs(audio_np) > abs_thresh)[0] + if len(above) == 0: + return audio_np[:1] + lead_in = int(sample_rate * 0.01) # keep 10 ms before first speech + trail_out = int(sample_rate * 0.03) # keep 30 ms after last speech + start_idx = max(0, above[0] - lead_in) + end_idx = min(len(audio_np), above[-1] + trail_out) + audio_np = audio_np[start_idx:end_idx].copy() + if len(audio_np) == 0: + return audio_np + fi = min(int(sample_rate * 0.015), len(audio_np)) # 15 ms fade in + fo = min(int(sample_rate * 0.005), len(audio_np)) # 5 ms fade out + if fi > 0: + audio_np[:fi] *= np.linspace(0.0, 1.0, fi, dtype=audio_np.dtype) + if fo > 0: + audio_np[-fo:] *= np.linspace(1.0, 0.0, fo, dtype=audio_np.dtype) + peak = float(np.max(np.abs(audio_np))) + if peak > 0.01: + audio_np = audio_np * (10 ** (-1.0 / 20) / peak) # normalize -1 dB + return audio_np + class f5_wrapper(): """Wrapper for F5-TTS model (SWivid/F5-TTS, MIT) @@ -569,9 +601,11 @@ class f5_wrapper(): audio = np.asarray(wav, dtype=np.float32).flatten() self.sample_rate = sr - logger.info(f"F5-TTS generated {len(audio)} samples at {sr}Hz") + raw_len = len(audio) + audio = _f5_trim_audio(audio, sr, silence_db=-40) + logger.info(f"F5-TTS generated {raw_len} samples at {sr}Hz, trimmed to {len(audio)}") - return audio.tobytes() + return audio.astype(np.float32).tobytes() except Exception as e: logger.error(f"F5-TTS generation failed: {e}") @@ -1295,10 +1329,11 @@ async def generate_speech(request: GenerateSpeechRequest): logger.info(f"Loading F5-TTS model on device '{device}'") f5_model = await asyncio.to_thread(f5_wrapper, device=device) - # F5-TTS handles long text via internal chunking. Per-sentence splitting - # produces a ref-to-gen transition artifact at the start of EACH chunk - # (audible as gibberish/tinny prefix). Pass full text as one inference. - logger.info(f"F5-TTS processing {len(input_text)} chars as single inference") + # Split into sentences so first audio bytes stream early (~2 s for long texts). + # f5_wrapper.tts() applies _f5_trim_audio per chunk to scrub the brief + # ref-bleed at chunk start so per-sentence boundaries stay clean. + sentences = simple_sentence_split(input_text, max_length=500) + logger.info(f"F5-TTS streaming {len(sentences)} sentence chunks from {len(input_text)} chars") # F5-TTS outputs float32 PCM at 24kHz ffmpeg_args = build_ffmpeg_args(response_format, input_format="f32le", sample_rate="24000") @@ -1308,20 +1343,22 @@ async def generate_speech(request: GenerateSpeechRequest): in_q = queue.Queue() def generator(): - """Run F5-TTS once for the full input text""" + """Process sentences sequentially and feed to queue""" try: - audio_bytes = f5_model.tts( - text=input_text, - ref_audio=ref_audio, - ref_text=ref_text, - speed=speed, - ) - in_q.put(audio_bytes) + for idx, sentence in enumerate(sentences): + logger.debug(f"F5-TTS chunk {idx+1}/{len(sentences)}: {len(sentence)} chars") + audio_bytes = f5_model.tts( + text=sentence, + ref_audio=ref_audio, + ref_text=ref_text, + speed=speed, + ) + in_q.put(audio_bytes) except Exception as e: - logger.error(f"F5-TTS generation error: {e}") + logger.error(f"F5-TTS streaming error: {e}") finally: in_q.put(None) - logger.info("F5-TTS generation complete") + logger.info(f"F5-TTS streaming complete: {len(sentences)} chunks") def out_writer(): """Write audio from queue to ffmpeg stdin"""