From 38a479f918e7f273335ff7a067f773924f5e99f7 Mon Sep 17 00:00:00 2001 From: matatonic <73265741+matatonic@users.noreply.github.com> Date: Sat, 29 Jun 2024 18:51:06 -0400 Subject: [PATCH 1/6] Update README.md custom piper voice download example --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 44fdc3a..42b8cfe 100644 --- a/README.md +++ b/README.md @@ -254,7 +254,11 @@ tts-1: ``` 3. New models will be downloaded as needed, of you can download them in advance with `download_voices_tts-1.sh`. For example: ```shell -bash download_voices_tts-1.sh en_US-ryan-high +$ bash download_voices_tts-1.sh en_US-ryan-high +INFO:piper.download:Downloaded voices/en_US-ryan-high.onnx.json (https://huggingface.co/rhasspy/piper-voices/resolve/v1.0.0/en/en_US/ryan/high/en_US-ryan-high.onnx.json) +INFO:piper.download:Downloaded voices/en_US-ryan-high.onnx (https://huggingface.co/rhasspy/piper-voices/resolve/v1.0.0/en/en_US/ryan/high/en_US-ryan-high.onnx) +$ ls voices/en_US-ryan-high.onnx* +voices/en_US-ryan-high.onnx.json voices/en_US-ryan-high.onnx ``` ### Coqui XTTS v2 @@ -358,4 +362,4 @@ tts-1-hd: temperature: 0.75 top_k: 50 top_p: 0.85 -``` \ No newline at end of file +``` From a99b196c4522bb123c21e039b8e09f90be998fbb Mon Sep 17 00:00:00 2001 From: matatonic <73265741+matatonic@users.noreply.github.com> Date: Sat, 29 Jun 2024 18:52:27 -0400 Subject: [PATCH 2/6] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 42b8cfe..e36c297 100644 --- a/README.md +++ b/README.md @@ -252,7 +252,7 @@ tts-1: model: voices/en_US-ryan-high.onnx speaker: # default speaker ``` - 3. New models will be downloaded as needed, of you can download them in advance with `download_voices_tts-1.sh`. For example: + 3. New models will be downloaded as needed, or you can download them in advance with `download_voices_tts-1.sh`. For example: ```shell $ bash download_voices_tts-1.sh en_US-ryan-high INFO:piper.download:Downloaded voices/en_US-ryan-high.onnx.json (https://huggingface.co/rhasspy/piper-voices/resolve/v1.0.0/en/en_US/ryan/high/en_US-ryan-high.onnx.json) From be02887ad97c68d81ca9ca5e86aaf55def06493f Mon Sep 17 00:00:00 2001 From: book3 Date: Mon, 1 Jul 2024 12:33:27 -0300 Subject: [PATCH 3/6] using landetect to automatically set the language of the request in xtts inference --- requirements.txt | 3 ++- speech.py | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5170bd0..9ba5862 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ uvicorn loguru piper-tts coqui-tts[languages] +langdetect # Creating an environment where deepspeed works is complex, for now it will be disabled by default. #deepspeed @@ -12,4 +13,4 @@ torchaudio; sys_platform != "darwin" torch; --index-url https://download.pytorch.org/whl/cpu; sys_platform == "darwin" torchaudio; --index-url https://download.pytorch.org/whl/cpu; sys_platform == "darwin" -# ROCM (Linux only) - use requirements.amd.txt \ No newline at end of file +# ROCM (Linux only) - use requirements.amd.txt diff --git a/speech.py b/speech.py index 6cbeeb5..0cb5cd3 100755 --- a/speech.py +++ b/speech.py @@ -16,7 +16,7 @@ from loguru import logger from openedai import OpenAIStub, BadRequestError, ServiceUnavailableError from pydantic import BaseModel import uvicorn - +from langdetect import detect @contextlib.asynccontextmanager async def lifespan(app): @@ -270,7 +270,21 @@ async def generate_speech(request: GenerateSpeechRequest): # Pipe the output from piper/xtts to the input of ffmpeg ffmpeg_args.extend(["-"]) - language = voice_map.pop('language', 'en') + language = voice_map.pop('language', 'auto') + if language == 'auto': + try: + language = detect(input_text) + if language not in [ + 'en', 'es', 'fr', 'de', 'it', 'pt', 'pl', 'tr', + 'ru', 'nl', 'cs', 'ar', 'zh-cn', 'hu', 'ko', 'ja', 'hi' + ]: + logger.debug(f"Detected language {language} not supported, defaulting to en") + language = 'en' + else: + logger.debug(f"Detected language: {language}") + except: + language = 'en' + logger.debug(f"Failed to detect language, defaulting to en") comment = voice_map.pop('comment', None) # ignored. From 02a6c7eb3e8ce45e45cfe17b1df794168bce4e90 Mon Sep 17 00:00:00 2001 From: matatonic Date: Mon, 1 Jul 2024 13:56:41 -0400 Subject: [PATCH 4/6] 0.16.0 --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index e36c297..8c73f1a 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Details: * You can map your own [piper voices](https://rhasspy.github.io/piper-samples/) via the `voice_to_speaker.yaml` configuration file * Model `tts-1-hd` via [coqui-ai/TTS](https://github.com/coqui-ai/TTS) xtts_v2 voice cloning (fast, but requires around 4GB GPU VRAM) * Custom cloned voices can be used for tts-1-hd, See: [Custom Voices Howto](#custom-voices-howto) - * 🌐 [Multilingual](#multilingual) support with XTTS voices + * 🌐 [Multilingual](#multilingual) support with XTTS voices, the language is automatically detected if not set * [Custom fine-tuned XTTS model support](#custom-fine-tuned-model-support) * Configurable [generation parameters](#generation-parameters) * Streamed output while generating @@ -30,6 +30,10 @@ If you find a better voice match for `tts-1` or `tts-1-hd`, please let me know s ## Recent Changes +Version 0.17.0, 2024-07-01 + +* Automatic language detection, thanks [@RodolfoCastanheira](https://github.com/RodolfoCastanheira) + Version 0.16.0, 2024-06-29 * Multi-client safe version. Audio generation is synchronized in a single process. The estimated 'realtime' factor of XTTS on a GPU is roughly 1/3, this means that multiple streams simultaneously, or `speed` over 2, may experience audio underrun (delays or pauses in playback). This makes multiple clients possible and safe, but in practice 2 or 3 simultaneous streams is the maximum without audio underrun. @@ -58,7 +62,7 @@ Version 0.14.0, 2024-06-26 Version 0.13.0, 2024-06-25 * Added [Custom fine-tuned XTTS model support](#custom-fine-tuned-model-support) -* Initial prebuilt arm64 image support (Apple M-series, Raspberry Pi - MPS is not supported in XTTS/torch), thanks @JakeStevenson, @hchasens +* Initial prebuilt arm64 image support (Apple M-series, Raspberry Pi - MPS is not supported in XTTS/torch), thanks [@JakeStevenson](https://github.com/JakeStevenson), [@hchasens](https://github.com/hchasens) * Initial attempt at AMD GPU (ROCm 5.7) support * Parler-tts support removed * Move the *.default.yaml to the root folder @@ -88,7 +92,7 @@ Version 0.11.0, 2024-05-29 Version: 0.10.1, 2024-05-05 -* Remove `runtime: nvidia` from docker-compose.yml, this assumes nvidia/cuda compatible runtime is available by default. thanks @jmtatsch +* Remove `runtime: nvidia` from docker-compose.yml, this assumes nvidia/cuda compatible runtime is available by default. thanks [@jmtatsch](https://github.com/jmtatsch) Version: 0.10.0, 2024-04-27 @@ -252,13 +256,9 @@ tts-1: model: voices/en_US-ryan-high.onnx speaker: # default speaker ``` - 3. New models will be downloaded as needed, or you can download them in advance with `download_voices_tts-1.sh`. For example: + 3. New models will be downloaded as needed, of you can download them in advance with `download_voices_tts-1.sh`. For example: ```shell -$ bash download_voices_tts-1.sh en_US-ryan-high -INFO:piper.download:Downloaded voices/en_US-ryan-high.onnx.json (https://huggingface.co/rhasspy/piper-voices/resolve/v1.0.0/en/en_US/ryan/high/en_US-ryan-high.onnx.json) -INFO:piper.download:Downloaded voices/en_US-ryan-high.onnx (https://huggingface.co/rhasspy/piper-voices/resolve/v1.0.0/en/en_US/ryan/high/en_US-ryan-high.onnx) -$ ls voices/en_US-ryan-high.onnx* -voices/en_US-ryan-high.onnx.json voices/en_US-ryan-high.onnx +bash download_voices_tts-1.sh en_US-ryan-high ``` ### Coqui XTTS v2 @@ -299,7 +299,7 @@ tts-1-hd: Multilingual cloning support was added in version 0.11.0 and is available only with the XTTS v2 model. To use multilingual voices with piper simply download a language specific voice. -Coqui XTTSv2 has support for 16 languages: English (`en`), Spanish (`es`), French (`fr`), German (`de`), Italian (`it`), Portuguese (`pt`), Polish (`pl`), Turkish (`tr`), Russian (`ru`), Dutch (`nl`), Czech (`cs`), Arabic (`ar`), Chinese (`zh-cn`), Japanese (`ja`), Hungarian (`hu`) and Korean (`ko`). +Coqui XTTSv2 has support for multiple languages: English (`en`), Spanish (`es`), French (`fr`), German (`de`), Italian (`it`), Portuguese (`pt`), Polish (`pl`), Turkish (`tr`), Russian (`ru`), Dutch (`nl`), Czech (`cs`), Arabic (`ar`), Chinese (`zh-cn`), Hungarian (`hu`), Korean (`ko`), Japanese (`ja`), and Hindi (`hi`). When not set, an attempt will be made to automatically detect the language, falling back to English (`en`). Unfortunately the OpenAI API does not support language, but you can create your own custom speaker voice and set the language for that. @@ -362,4 +362,4 @@ tts-1-hd: temperature: 0.75 top_k: 50 top_p: 0.85 -``` +``` \ No newline at end of file From fdd443b10f099e089f09d7cc471279f4609b2751 Mon Sep 17 00:00:00 2001 From: matatonic Date: Mon, 1 Jul 2024 19:43:32 -0400 Subject: [PATCH 5/6] 0.17.1 --- README.md | 5 +++++ add_voice.py | 4 ++-- requirements-rocm.txt | 1 + speech.py | 6 +++++- voice_to_speaker.default.yaml | 3 ++- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8c73f1a..f775da3 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,11 @@ If you find a better voice match for `tts-1` or `tts-1-hd`, please let me know s ## Recent Changes +Version 0.17.1, 2024-07-01 + +* fix ROCm (add langdetect to requirements-rocm.txt) +* Fix zh-cn for xtts + Version 0.17.0, 2024-07-01 * Automatic language detection, thanks [@RodolfoCastanheira](https://github.com/RodolfoCastanheira) diff --git a/add_voice.py b/add_voice.py index bb86ac5..7b2c1f8 100755 --- a/add_voice.py +++ b/add_voice.py @@ -11,8 +11,8 @@ parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFo parser.add_argument('sample', action='store', help="Set the wav sample file") parser.add_argument('-n', '--name', action='store', help="Set the name for the voice (by default will use the WAV file name)") -parser.add_argument('-l', '--language', action='store', default="en", help="Set the language for the voice", - choices=['en', 'es', 'fr', 'de', 'it', 'pt', 'pl', 'tr', 'ru', 'nl', 'cs', 'ar', 'zh-cn', 'ja', 'hu', 'ko']) +parser.add_argument('-l', '--language', action='store', default="auto", help="Set the language for the voice", + choices=['auto', 'en', 'es', 'fr', 'de', 'it', 'pt', 'pl', 'tr', 'ru', 'nl', 'cs', 'ar', 'zh-cn', 'ja', 'hu', 'ko', 'hi']) parser.add_argument('--openai-model', action='store', default="tts-1-hd", help="Set the openai model for the voice") parser.add_argument('--xtts-model', action='store', default="xtts", help="Set the xtts model for the voice (if using a custom model, also set model_path)") parser.add_argument('--model-path', action='store', default=None, help="Set the path for a custom xtts model") diff --git a/requirements-rocm.txt b/requirements-rocm.txt index a47e4e8..2dcb36f 100644 --- a/requirements-rocm.txt +++ b/requirements-rocm.txt @@ -3,6 +3,7 @@ uvicorn loguru piper-tts coqui-tts +langdetect # Creating an environment where deepspeed works is complex, for now it will be disabled by default. #deepspeed torch; --index-url https://download.pytorch.org/whl/rocm5.7; sys_platform == "linux" diff --git a/speech.py b/speech.py index 0cb5cd3..687a8fc 100755 --- a/speech.py +++ b/speech.py @@ -296,7 +296,11 @@ async def generate_speech(request: GenerateSpeechRequest): hf_generate_kwargs['enable_text_splitting'] = hf_generate_kwargs.get('enable_text_splitting', True) # change the default to true if hf_generate_kwargs['enable_text_splitting']: - all_text = split_sentence(input_text, language, xtts.xtts.tokenizer.char_limits[language]) + if language == 'zh-cn': + split_lang = 'zh' + else: + split_lang = language + all_text = split_sentence(input_text, split_lang, xtts.xtts.tokenizer.char_limits[split_lang]) else: all_text = [input_text] diff --git a/voice_to_speaker.default.yaml b/voice_to_speaker.default.yaml index 0604830..afce1a6 100644 --- a/voice_to_speaker.default.yaml +++ b/voice_to_speaker.default.yaml @@ -46,8 +46,9 @@ tts-1-hd: model: xtts speaker: voices/shimmer.wav me: - model: xtts_v2.0.2 # you can specify different xtts version + model: xtts_v2.0.2 # you can specify an older xtts version speaker: voices/me.wav # this could be you + language: auto enable_text_splitting: True length_penalty: 1.0 repetition_penalty: 10 From 43dbf431f833a8cb4e66fa45902ce1a584f3717b Mon Sep 17 00:00:00 2001 From: matatonic Date: Mon, 1 Jul 2024 20:38:44 -0400 Subject: [PATCH 6/6] unbreak -min --- README.md | 4 ++++ speech.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f775da3..419d555 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,10 @@ If you find a better voice match for `tts-1` or `tts-1-hd`, please let me know s ## Recent Changes +Version 0.17.2, 2024-07-01 + +* fix -min image (re: langdetect) + Version 0.17.1, 2024-07-01 * fix ROCm (add langdetect to requirements-rocm.txt) diff --git a/speech.py b/speech.py index 687a8fc..2fd0e3e 100755 --- a/speech.py +++ b/speech.py @@ -16,7 +16,6 @@ from loguru import logger from openedai import OpenAIStub, BadRequestError, ServiceUnavailableError from pydantic import BaseModel import uvicorn -from langdetect import detect @contextlib.asynccontextmanager async def lifespan(app): @@ -405,6 +404,7 @@ if __name__ == "__main__": from TTS.tts.models.xtts import Xtts from TTS.utils.manage import ModelManager from TTS.tts.layers.xtts.tokenizer import split_sentence + from langdetect import detect if args.preload: xtts = xtts_wrapper(args.preload, device=args.xtts_device, unload_timer=args.unload_timer)