🦝 Fix blocking model loads - enable concurrent TTS requests
Problem: - Silero and Kokoro model initialization was blocking the FastAPI event loop - First request to Silero downloads 54.5MB synchronously, blocking ALL requests - No concurrent request handling - server frozen during model loads Solution: - Added asyncio import - Wrapped blocking operations in asyncio.to_thread(): * silero_wrapper() initialization (torch.hub.load download) * kokoro_wrapper() initialization * silero_model.tts() generation * kokoro_pipeline.tts() generation Impact: - Concurrent requests now work - fast models don't wait for slow ones - Model loading runs in thread pool, freeing event loop - Multiple users can make requests simultaneously - First Silero request still takes time, but doesn't block other engines Related to: User reported timeout issues with deployed TTS service Raccoon Mission: Production-ready concurrent TTS serving
This commit is contained in:
parent
21e8f27519
commit
650ae49f65
1 changed files with 9 additions and 6 deletions
15
speech.py
15
speech.py
|
|
@ -1,5 +1,6 @@
|
|||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import asyncio
|
||||
import contextlib
|
||||
import gc
|
||||
import os
|
||||
|
|
@ -658,11 +659,12 @@ async def generate_speech(request: GenerateSpeechRequest):
|
|||
# Load Silero model if not already loaded or if language/speaker changed
|
||||
if silero_model is None or silero_speakers.get('current') != model_key:
|
||||
logger.info(f"Loading/switching Silero model to {language}/{silero_speaker_key}")
|
||||
silero_model = silero_wrapper(language=language, speaker=silero_speaker_key, device='cpu')
|
||||
# Run blocking model initialization in thread pool to avoid blocking event loop
|
||||
silero_model = await asyncio.to_thread(silero_wrapper, language=language, speaker=silero_speaker_key, device='cpu')
|
||||
silero_speakers['current'] = model_key
|
||||
|
||||
# Generate audio
|
||||
audio_data = silero_model.tts(input_text, speaker_id=speaker_id)
|
||||
# Generate audio (also blocking, so run in thread pool)
|
||||
audio_data = await asyncio.to_thread(silero_model.tts, input_text, speaker_id=speaker_id)
|
||||
|
||||
# Silero outputs float32 PCM at 48000 Hz
|
||||
ffmpeg_args = build_ffmpeg_args(response_format, input_format="f32le", sample_rate="48000")
|
||||
|
|
@ -688,11 +690,12 @@ async def generate_speech(request: GenerateSpeechRequest):
|
|||
# Load Kokoro pipeline if not already loaded or if language changed
|
||||
if kokoro_pipeline is None or kokoro_lang != lang_code:
|
||||
logger.info(f"Loading/switching Kokoro pipeline to language '{lang_code}'")
|
||||
kokoro_pipeline = kokoro_wrapper(lang_code=lang_code)
|
||||
# Run blocking model initialization in thread pool to avoid blocking event loop
|
||||
kokoro_pipeline = await asyncio.to_thread(kokoro_wrapper, lang_code=lang_code)
|
||||
kokoro_lang = lang_code
|
||||
|
||||
# Generate audio
|
||||
audio_data = kokoro_pipeline.tts(input_text, voice=kokoro_voice, speed=speed)
|
||||
# Generate audio (also blocking, so run in thread pool)
|
||||
audio_data = await asyncio.to_thread(kokoro_pipeline.tts, input_text, voice=kokoro_voice, speed=speed)
|
||||
|
||||
# Kokoro outputs float32 PCM at 24000 Hz
|
||||
ffmpeg_args = build_ffmpeg_args(response_format, input_format="f32le", sample_rate="24000")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue