Fix sentence splitter to split on every sentence boundary

Previous version accumulated sentences until 500 chars, defeating
the purpose of streaming. Now splits on every sentence, only
combining very short sentences (<50 chars) with the next.
This commit is contained in:
russell@unturf.com 2026-01-26 19:41:44 -05:00
parent 802eaf2b29
commit 0a0d023517

View file

@ -517,38 +517,61 @@ def preprocess(raw_input):
return raw_input
def simple_sentence_split(text: str, max_length: int = 500) -> list[str]:
"""Split text into sentences for better TTS processing.
"""Split text into sentences for streaming TTS.
Simple sentence splitter that breaks on sentence boundaries (.!?)
and ensures no sentence exceeds max_length characters.
Splits on sentence boundaries (.!?) for immediate streaming.
Only combines very short sentences (< 50 chars) with the next one.
Long sentences exceeding max_length are split at word boundaries.
"""
# Split on sentence boundaries
sentences = re.split(r'([.!?]+[\s\n]+)', text)
# Split on sentence boundaries, keeping the punctuation
parts = re.split(r'([.!?]+\s*)', text)
# Recombine sentences with their punctuation
result = []
current = ""
for i in range(0, len(sentences), 2):
sentence = sentences[i]
punct = sentences[i+1] if i+1 < len(sentences) else ""
for i in range(0, len(parts), 2):
sentence = parts[i].strip()
punct = parts[i+1] if i+1 < len(parts) else ""
# If adding this sentence would exceed max_length, save current and start new
if current and len(current) + len(sentence) + len(punct) > max_length:
if not sentence:
continue
full_sentence = sentence + punct
# If sentence is very short, combine with current buffer
if len(full_sentence) < 50 and current:
current += " " + full_sentence
elif current:
# Save buffered content and start fresh
result.append(current.strip())
current = sentence + punct
current = full_sentence
else:
current += sentence + punct
current = full_sentence
# Add remaining text
if current.strip():
result.append(current.strip())
# If we got nothing (no sentence boundaries), split on max_length
if not result and text:
result = [text[i:i+max_length] for i in range(0, len(text), max_length)]
# Split any sentences that exceed max_length at word boundaries
final_result = []
for sentence in result:
if len(sentence) <= max_length:
final_result.append(sentence)
else:
# Split at word boundaries
words = sentence.split()
chunk = ""
for word in words:
if len(chunk) + len(word) + 1 > max_length:
if chunk:
final_result.append(chunk.strip())
chunk = word
else:
chunk += " " + word if chunk else word
if chunk:
final_result.append(chunk.strip())
return result if result else [text]
return final_result if final_result else [text]
# Auto-detect which model a voice belongs to (uses cached mapping)
def detect_model_from_voice(voice: str) -> str: