change the method of splitting sentences

sentence_api/transcription.py

@@ -239,7 +239,8 @@ def _parse_word_timestamps(raw_words) -> Optional[List[WordTimestamp]]:
     return words or None
 
 
-_PUNCTUATION_BOUNDARY = re.compile(r"[.?]+")
+_PUNCTUATION_BOUNDARY = re.compile(r"[.?]+|,+")
+MIN_WORDS_FOR_COMMA_SPLIT = 50
 _ABBREVIATION_RE = re.compile(
     r"(?i)\b(?:a\.m\.|p\.m\.|mr\.|mrs\.|ms\.|dr\.|prof\.|st\.|vs\.|etc\.|"
     r"e\.g\.|i\.e\.|no\.|approx\.|fig\.|inc\.|ltd\.|jr\.|sr\.|u\.s\.|u\.k\.)"
@@ -250,11 +251,12 @@ _ABBREVIATION_RE = re.compile(
 def split_sentences_at_punctuation(
     segments: List[TranscriptionSegment],
 ) -> List[TranscriptionSegment]:
-    """Merge whisper segments and cut sentences at periods/question marks.
+    """Merge whisper segments and cut sentences at sentence punctuation.
 
     Whisper's own segment breaks often fall mid-sentence, so text is accumulated
-    across segments and a sentence is only closed once the accumulated text
-    reaches a period or a question mark.
+    across segments and a sentence is closed at periods/question marks. Commas
+    are also eligible boundaries, but only when the text before the comma has
+    enough words; otherwise the comma stays with the following text.
     """
     sentences: List[TranscriptionSegment] = []
     pieces: List[str] = []
@@ -283,15 +285,20 @@ def split_sentences_at_punctuation(
             continue
         if not pieces:
             start_seconds = segment.start_seconds
-        seg_pieces: List[Tuple[str, bool]] = []
+        seg_pieces: List[Tuple[str, Optional[str]]] = []
         cursor = 0
         for match in _PUNCTUATION_BOUNDARY.finditer(text):
-            if not _is_sentence_end_run(text, match.start(), match.end()):
+            boundary_type: Optional[str] = None
+            if "," in match.group(0):
+                boundary_type = "comma"
+            elif _is_sentence_end_run(text, match.start(), match.end()):
+                boundary_type = "sentence"
+            if boundary_type is None:
                 continue
             piece = text[cursor:match.end()].strip()
             cursor = match.end()
-            if piece.strip(".?").strip():
-                seg_pieces.append((piece, True))
+            if _has_word_token(piece):
+                seg_pieces.append((piece, boundary_type))
         tail = text[cursor:].strip()
         if tail:
             seg_pieces.append((tail, False))
@@ -306,9 +313,12 @@ def split_sentences_at_punctuation(
         if not ends:
             ends = _proportional_ends(segment, lengths)
         last_segment_end = segment.end_seconds
-        for index, (piece, is_boundary) in enumerate(seg_pieces):
+        for index, (piece, boundary_type) in enumerate(seg_pieces):
             pieces.append(piece)
-            if is_boundary:
+            if boundary_type == "sentence" or (
+                boundary_type == "comma"
+                and _word_count(" ".join(pieces)) >= MIN_WORDS_FOR_COMMA_SPLIT
+            ):
                 end_seconds = ends[index] if index < len(ends) else segment.end_seconds
                 flush(end_seconds)
                 start_seconds = end_seconds
@@ -317,6 +327,14 @@ def split_sentences_at_punctuation(
     return sentences
 
 
+def _word_count(text: str) -> int:
+    return len(text.split())
+
+
+def _has_word_token(text: str) -> bool:
+    return any(char.isalnum() for char in text)
+
+
 def _is_sentence_end_run(text: str, run_start: int, run_end: int) -> bool:
     """Decide whether a "." / "?" run really ends a sentence."""
     punctuation = text[run_start:run_end]