From 650ae49f65f8b78b7eaf85cd48142f70426de722 Mon Sep 17 00:00:00 2001
From: Russell Ballestrini <russell@unturf.com>
Date: Sun, 9 Nov 2025 16:05:12 -0500
Subject: [PATCH] =?UTF-8?q?=F0=9F=A6=9D=20Fix=20blocking=20model=20loads?=
 =?UTF-8?q?=20-=20enable=20concurrent=20TTS=20requests?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Problem:
- Silero and Kokoro model initialization was blocking the FastAPI event loop
- First request to Silero downloads 54.5MB synchronously, blocking ALL requests
- No concurrent request handling - server frozen during model loads

Solution:
- Added asyncio import
- Wrapped blocking operations in asyncio.to_thread():
  * silero_wrapper() initialization (torch.hub.load download)
  * kokoro_wrapper() initialization
  * silero_model.tts() generation
  * kokoro_pipeline.tts() generation

Impact:
- Concurrent requests now work - fast models don't wait for slow ones
- Model loading runs in thread pool, freeing event loop
- Multiple users can make requests simultaneously
- First Silero request still takes time, but doesn't block other engines

Related to: User reported timeout issues with deployed TTS service
Raccoon Mission: Production-ready concurrent TTS serving
---
 speech.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/speech.py b/speech.py
index f6b67a5..85c004d 100755
--- a/speech.py
+++ b/speech.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import argparse
+import asyncio
 import contextlib
 import gc
 import os
@@ -658,11 +659,12 @@ async def generate_speech(request: GenerateSpeechRequest):
         # Load Silero model if not already loaded or if language/speaker changed
         if silero_model is None or silero_speakers.get('current') != model_key:
             logger.info(f"Loading/switching Silero model to {language}/{silero_speaker_key}")
-            silero_model = silero_wrapper(language=language, speaker=silero_speaker_key, device='cpu')
+            # Run blocking model initialization in thread pool to avoid blocking event loop
+            silero_model = await asyncio.to_thread(silero_wrapper, language=language, speaker=silero_speaker_key, device='cpu')
             silero_speakers['current'] = model_key
 
-        # Generate audio
-        audio_data = silero_model.tts(input_text, speaker_id=speaker_id)
+        # Generate audio (also blocking, so run in thread pool)
+        audio_data = await asyncio.to_thread(silero_model.tts, input_text, speaker_id=speaker_id)
 
         # Silero outputs float32 PCM at 48000 Hz
         ffmpeg_args = build_ffmpeg_args(response_format, input_format="f32le", sample_rate="48000")
@@ -688,11 +690,12 @@ async def generate_speech(request: GenerateSpeechRequest):
         # Load Kokoro pipeline if not already loaded or if language changed
         if kokoro_pipeline is None or kokoro_lang != lang_code:
             logger.info(f"Loading/switching Kokoro pipeline to language '{lang_code}'")
-            kokoro_pipeline = kokoro_wrapper(lang_code=lang_code)
+            # Run blocking model initialization in thread pool to avoid blocking event loop
+            kokoro_pipeline = await asyncio.to_thread(kokoro_wrapper, lang_code=lang_code)
             kokoro_lang = lang_code
 
-        # Generate audio
-        audio_data = kokoro_pipeline.tts(input_text, voice=kokoro_voice, speed=speed)
+        # Generate audio (also blocking, so run in thread pool)
+        audio_data = await asyncio.to_thread(kokoro_pipeline.tts, input_text, voice=kokoro_voice, speed=speed)
 
         # Kokoro outputs float32 PCM at 24000 Hz
         ffmpeg_args = build_ffmpeg_args(response_format, input_format="f32le", sample_rate="24000")