Fix sentence splitter to split on every sentence boundary

Previous version accumulated sentences until 500 chars, defeating
the purpose of streaming. Now splits on every sentence, only
combining very short sentences (<50 chars) with the next.
This commit is contained in:
russell@unturf.com 2026-01-26 19:41:44 -05:00
parent 802eaf2b29
commit 0a0d023517

View file

@ -517,38 +517,61 @@ def preprocess(raw_input):
return raw_input return raw_input
def simple_sentence_split(text: str, max_length: int = 500) -> list[str]: def simple_sentence_split(text: str, max_length: int = 500) -> list[str]:
"""Split text into sentences for better TTS processing. """Split text into sentences for streaming TTS.
Simple sentence splitter that breaks on sentence boundaries (.!?) Splits on sentence boundaries (.!?) for immediate streaming.
and ensures no sentence exceeds max_length characters. Only combines very short sentences (< 50 chars) with the next one.
Long sentences exceeding max_length are split at word boundaries.
""" """
# Split on sentence boundaries # Split on sentence boundaries, keeping the punctuation
sentences = re.split(r'([.!?]+[\s\n]+)', text) parts = re.split(r'([.!?]+\s*)', text)
# Recombine sentences with their punctuation
result = [] result = []
current = "" current = ""
for i in range(0, len(sentences), 2): for i in range(0, len(parts), 2):
sentence = sentences[i] sentence = parts[i].strip()
punct = sentences[i+1] if i+1 < len(sentences) else "" punct = parts[i+1] if i+1 < len(parts) else ""
# If adding this sentence would exceed max_length, save current and start new if not sentence:
if current and len(current) + len(sentence) + len(punct) > max_length: continue
full_sentence = sentence + punct
# If sentence is very short, combine with current buffer
if len(full_sentence) < 50 and current:
current += " " + full_sentence
elif current:
# Save buffered content and start fresh
result.append(current.strip()) result.append(current.strip())
current = sentence + punct current = full_sentence
else: else:
current += sentence + punct current = full_sentence
# Add remaining text # Add remaining text
if current.strip(): if current.strip():
result.append(current.strip()) result.append(current.strip())
# If we got nothing (no sentence boundaries), split on max_length # Split any sentences that exceed max_length at word boundaries
if not result and text: final_result = []
result = [text[i:i+max_length] for i in range(0, len(text), max_length)] for sentence in result:
if len(sentence) <= max_length:
final_result.append(sentence)
else:
# Split at word boundaries
words = sentence.split()
chunk = ""
for word in words:
if len(chunk) + len(word) + 1 > max_length:
if chunk:
final_result.append(chunk.strip())
chunk = word
else:
chunk += " " + word if chunk else word
if chunk:
final_result.append(chunk.strip())
return result if result else [text] return final_result if final_result else [text]
# Auto-detect which model a voice belongs to (uses cached mapping) # Auto-detect which model a voice belongs to (uses cached mapping)
def detect_model_from_voice(voice: str) -> str: def detect_model_from_voice(voice: str) -> str: