diff --git a/requirements.txt b/requirements.txt index ff2799e..d8de8f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,9 @@ omegaconf # Required by Silero TTS langdetect pyyaml # Kokoro TTS - fast decoder-only architecture -# Install from Hugging Face transformers +# Lightweight decoder-only TTS, 82M params, 24kHz output +kokoro>=0.9.2 +soundfile # Required by Kokoro for audio output transformers>=4.35.0 # Hugging Face Hub for model downloads huggingface-hub[cli] diff --git a/speech.py b/speech.py index 9d0245e..cdd367a 100755 --- a/speech.py +++ b/speech.py @@ -34,6 +34,8 @@ app = OpenAIStub(lifespan=lifespan) xtts = None silero_model = None silero_speakers = {} +kokoro_pipeline = None +kokoro_lang = None args = None def unload_model(): @@ -167,6 +169,58 @@ class silero_wrapper(): # audio is a tensor, convert to numpy float32 return audio.cpu().numpy().tobytes() +class kokoro_wrapper(): + """Wrapper for Kokoro TTS model + + Kokoro is a lightweight decoder-only TTS model (82M params) + Output: 24kHz audio + """ + def __init__(self, lang_code='a', model_path='/app/voices/kokoro'): + self.lang_code = lang_code + self.model_path = model_path + self.sample_rate = 24000 # Kokoro outputs 24kHz + + logger.info(f"Loading Kokoro TTS pipeline for language '{lang_code}'") + + try: + from kokoro import KPipeline + import numpy as np + + self.pipeline = KPipeline(lang_code=lang_code, model_path=model_path) + logger.info(f"Successfully loaded Kokoro pipeline for lang={lang_code}") + except Exception as e: + logger.error(f"Failed to load Kokoro model: {e}") + raise + + def tts(self, text, voice='af_heart', speed=1.0): + """Generate speech from text using Kokoro""" + import numpy as np + + logger.info(f"Kokoro tts() called: text length={len(text)}, voice={voice}, speed={speed}") + + try: + # Generate audio using Kokoro pipeline + generator = self.pipeline(text, voice=voice, speed=speed) + + # Collect all audio chunks + audio_chunks = [] + for _, _, audio in generator: + if audio is not None and len(audio) > 0: + audio_chunks.append(audio) + + # Concatenate all chunks + if len(audio_chunks) > 0: + full_audio = np.concatenate(audio_chunks) + # Convert float32 numpy array to bytes + return full_audio.astype(np.float32).tobytes() + else: + logger.warning("Kokoro generated no audio") + return b'' + + except Exception as e: + logger.error(f"Kokoro TTS generation failed: {e}") + raise + def default_exists(filename: str): if not os.path.exists(filename): fpath, ext = os.path.splitext(filename) @@ -266,6 +320,8 @@ async def generate_speech(request: GenerateSpeechRequest): media_type = "audio/pcm;rate=24000" elif model == 'tts-1-silero': # silero media_type = "audio/pcm;rate=48000" + elif model == 'tts-1-kokoro': # kokoro + media_type = "audio/pcm;rate=24000" else: raise BadRequestError(f"Invalid response_format: '{response_format}'", param='response_format') @@ -501,9 +557,35 @@ async def generate_speech(request: GenerateSpeechRequest): ffmpeg_proc.stdin.write(audio_data) ffmpeg_proc.stdin.close() + return StreamingResponse(content=ffmpeg_proc.stdout, media_type=media_type) + # Use Kokoro for tts-1-kokoro + elif model == 'tts-1-kokoro': + global kokoro_pipeline, kokoro_lang + + voice_map = map_voice_to_speaker(voice, 'tts-1-kokoro') + lang_code = voice_map.get('lang_code', 'a') + kokoro_voice = voice_map.get('kokoro_voice', 'af_heart') + + # Load Kokoro pipeline if not already loaded or if language changed + if kokoro_pipeline is None or kokoro_lang != lang_code: + logger.info(f"Loading/switching Kokoro pipeline to language '{lang_code}'") + kokoro_pipeline = kokoro_wrapper(lang_code=lang_code) + kokoro_lang = lang_code + + # Generate audio + audio_data = kokoro_pipeline.tts(input_text, voice=kokoro_voice, speed=speed) + + # Kokoro outputs float32 PCM at 24000 Hz + ffmpeg_args = build_ffmpeg_args(response_format, input_format="f32le", sample_rate="24000") + + ffmpeg_args.extend(["-"]) + ffmpeg_proc = subprocess.Popen(ffmpeg_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + ffmpeg_proc.stdin.write(audio_data) + ffmpeg_proc.stdin.close() + return StreamingResponse(content=ffmpeg_proc.stdout, media_type=media_type) else: - raise BadRequestError("No such model, must be tts-1, tts-1-hd, or tts-1-silero.", param='model') + raise BadRequestError("No such model, must be tts-1, tts-1-hd, tts-1-silero, or tts-1-kokoro.", param='model') # We return 'mps' but currently XTTS will not work with mps devices as the cuda support is incomplete @@ -551,5 +633,6 @@ if __name__ == "__main__": app.register_model('tts-1') app.register_model('tts-1-hd') app.register_model('tts-1-silero') + app.register_model('tts-1-kokoro') uvicorn.run(app, host=args.host, port=args.port) diff --git a/voice_to_speaker.default.yaml b/voice_to_speaker.default.yaml index 089c95f..20301e3 100644 --- a/voice_to_speaker.default.yaml +++ b/voice_to_speaker.default.yaml @@ -804,4 +804,112 @@ tts-1-silero: fr_random: language: fr speaker: random - silero_speaker: v3_fr \ No newline at end of file + silero_speaker: v3_fr +tts-1-kokoro: + # OpenAI-compatible voice aliases (American English) + alloy: + lang_code: a + kokoro_voice: af_alloy + echo: + lang_code: a + kokoro_voice: am_echo + fable: + lang_code: b + kokoro_voice: bm_fable + onyx: + lang_code: a + kokoro_voice: am_onyx + nova: + lang_code: a + kokoro_voice: af_nova + shimmer: + lang_code: a + kokoro_voice: af_sky + # Female American voices + af_heart: + lang_code: a + kokoro_voice: af_heart + af_bella: + lang_code: a + kokoro_voice: af_bella + af_nicole: + lang_code: a + kokoro_voice: af_nicole + af_aoede: + lang_code: a + kokoro_voice: af_aoede + af_kore: + lang_code: a + kokoro_voice: af_kore + af_sarah: + lang_code: a + kokoro_voice: af_sarah + af_nova: + lang_code: a + kokoro_voice: af_nova + af_sky: + lang_code: a + kokoro_voice: af_sky + af_alloy: + lang_code: a + kokoro_voice: af_alloy + af_jessica: + lang_code: a + kokoro_voice: af_jessica + af_river: + lang_code: a + kokoro_voice: af_river + # Male American voices + am_michael: + lang_code: a + kokoro_voice: am_michael + am_fenrir: + lang_code: a + kokoro_voice: am_fenrir + am_puck: + lang_code: a + kokoro_voice: am_puck + am_echo: + lang_code: a + kokoro_voice: am_echo + am_eric: + lang_code: a + kokoro_voice: am_eric + am_liam: + lang_code: a + kokoro_voice: am_liam + am_onyx: + lang_code: a + kokoro_voice: am_onyx + am_santa: + lang_code: a + kokoro_voice: am_santa + am_adam: + lang_code: a + kokoro_voice: am_adam + # Female British voices + bf_emma: + lang_code: b + kokoro_voice: bf_emma + bf_isabella: + lang_code: b + kokoro_voice: bf_isabella + bf_alice: + lang_code: b + kokoro_voice: bf_alice + bf_lily: + lang_code: b + kokoro_voice: bf_lily + # Male British voices + bm_george: + lang_code: b + kokoro_voice: bm_george + bm_fable: + lang_code: b + kokoro_voice: bm_fable + bm_lewis: + lang_code: b + kokoro_voice: bm_lewis + bm_daniel: + lang_code: b + kokoro_voice: bm_daniel \ No newline at end of file