From d4ff4e8da41d508f4562e03c226034a0e5ae6d7d Mon Sep 17 00:00:00 2001 From: timehexon Date: Sun, 7 Jun 2026 08:39:49 -0400 Subject: [PATCH] =?UTF-8?q?speech.py:=20disable=20tts-1-qwen=20registratio?= =?UTF-8?q?n=20=E2=80=94=20free=20VRAM=20for=20llama-qwen=20LLM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit qwen3-tts (1.7B params) on speech.py was holding 6-7 GB VRAM that collided with llama-qwen.service (Qwen3.6-27B, 20 GB @ -c 65536) on the same 4090. Result: speech.py OOMed mid-inference, returning HTTP 500 InternalServerError that browsers surfaced as NetworkError. Reverse order made llama-qwen OOM during model load. Drop tts-1-qwen registration so only f5-tts (336M params, ~1.3 GB VRAM) lands on the GPU. Speech requests for tts-1-f5 work; tts-1-qwen returns 500 (model still listed in /v1/models via the loop at line 758 — refine to a proper 4xx in a later pass). To reverse: uncomment line 1497 + reduce llama-qwen context (e.g. -c 32768 frees ~10 GB, enough for both engines). --- speech.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/speech.py b/speech.py index 09c4417..32d3a19 100755 --- a/speech.py +++ b/speech.py @@ -1494,7 +1494,7 @@ if __name__ == "__main__": xtts = xtts_wrapper(args.preload, device=args.xtts_device, unload_timer=args.unload_timer) # Register Qwen + F5 by default (other models disabled) - app.register_model('tts-1-qwen') + # app.register_model('tts-1-qwen') # disabled — frees VRAM for llama-qwen LLM app.register_model('tts-1-f5') # To enable other models, uncomment below: # app.register_model('tts-1')