change the method of splitting sentences

sentence_api/processing.py

@@ -19,7 +19,7 @@ from .repository import VideoRepository
 from .transcription import Transcript, Transcriber, split_sentences_at_punctuation
 
 
-MOSS_ALGORITHM_VERSION = "moss-period-v2"
+MOSS_ALGORITHM_VERSION = "moss-punctuation-v3"
 
 
 class VideoProcessor:

sentence_api/tests/test_transcription.py

@@ -287,6 +287,38 @@ def test_split_sentences_splits_at_question_marks():
     assert sentences[2].end_seconds == 10.0
 
 
+def test_split_sentences_splits_at_comma_after_fifty_words():
+    before_comma = " ".join(f"word{index}" for index in range(1, 51))
+    after_comma = "This continuation becomes its own sentence."
+    segment = TranscriptionSegment(
+        start_seconds=0.0,
+        end_seconds=55.0,
+        text=f"{before_comma}, {after_comma}",
+    )
+
+    sentences = split_sentences_at_punctuation([segment])
+
+    assert [s.text for s in sentences] == [
+        f"{before_comma},",
+        after_comma,
+    ]
+    assert sentences[0].start_seconds == 0.0
+    assert sentences[0].end_seconds == sentences[1].start_seconds
+    assert sentences[1].end_seconds == 55.0
+
+
+def test_split_sentences_merges_comma_piece_shorter_than_fifty_words():
+    segment = TranscriptionSegment(
+        start_seconds=0.0,
+        end_seconds=5.0,
+        text="One, two, and three.",
+    )
+
+    sentences = split_sentences_at_punctuation([segment])
+
+    assert [s.text for s in sentences] == ["One, two, and three."]
+
+
 def test_split_sentences_falls_back_to_proportional():
     segments = [
         TranscriptionSegment(

sentence_api/transcription.py

@@ -239,7 +239,8 @@ def _parse_word_timestamps(raw_words) -> Optional[List[WordTimestamp]]:
     return words or None
 
 
-_PUNCTUATION_BOUNDARY = re.compile(r"[.?]+")
+_PUNCTUATION_BOUNDARY = re.compile(r"[.?]+|,+")
+MIN_WORDS_FOR_COMMA_SPLIT = 50
 _ABBREVIATION_RE = re.compile(
     r"(?i)\b(?:a\.m\.|p\.m\.|mr\.|mrs\.|ms\.|dr\.|prof\.|st\.|vs\.|etc\.|"
     r"e\.g\.|i\.e\.|no\.|approx\.|fig\.|inc\.|ltd\.|jr\.|sr\.|u\.s\.|u\.k\.)"
@@ -250,11 +251,12 @@ _ABBREVIATION_RE = re.compile(
 def split_sentences_at_punctuation(
     segments: List[TranscriptionSegment],
 ) -> List[TranscriptionSegment]:
-    """Merge whisper segments and cut sentences at periods/question marks.
+    """Merge whisper segments and cut sentences at sentence punctuation.
 
     Whisper's own segment breaks often fall mid-sentence, so text is accumulated
-    across segments and a sentence is only closed once the accumulated text
+    across segments and a sentence is closed at periods/question marks. Commas
-    reaches a period or a question mark.
+    are also eligible boundaries, but only when the text before the comma has
+    enough words; otherwise the comma stays with the following text.
     """
     sentences: List[TranscriptionSegment] = []
     pieces: List[str] = []
@@ -283,15 +285,20 @@ def split_sentences_at_punctuation(
             continue
         if not pieces:
             start_seconds = segment.start_seconds
-        seg_pieces: List[Tuple[str, bool]] = []
+        seg_pieces: List[Tuple[str, Optional[str]]] = []
         cursor = 0
         for match in _PUNCTUATION_BOUNDARY.finditer(text):
-            if not _is_sentence_end_run(text, match.start(), match.end()):
+            boundary_type: Optional[str] = None
+            if "," in match.group(0):
+                boundary_type = "comma"
+            elif _is_sentence_end_run(text, match.start(), match.end()):
+                boundary_type = "sentence"
+            if boundary_type is None:
                 continue
             piece = text[cursor:match.end()].strip()
             cursor = match.end()
-            if piece.strip(".?").strip():
+            if _has_word_token(piece):
-                seg_pieces.append((piece, True))
+                seg_pieces.append((piece, boundary_type))
         tail = text[cursor:].strip()
         if tail:
             seg_pieces.append((tail, False))
@@ -306,9 +313,12 @@ def split_sentences_at_punctuation(
         if not ends:
             ends = _proportional_ends(segment, lengths)
         last_segment_end = segment.end_seconds
-        for index, (piece, is_boundary) in enumerate(seg_pieces):
+        for index, (piece, boundary_type) in enumerate(seg_pieces):
             pieces.append(piece)
-            if is_boundary:
+            if boundary_type == "sentence" or (
+                boundary_type == "comma"
+                and _word_count(" ".join(pieces)) >= MIN_WORDS_FOR_COMMA_SPLIT
+            ):
                 end_seconds = ends[index] if index < len(ends) else segment.end_seconds
                 flush(end_seconds)
                 start_seconds = end_seconds
@@ -317,6 +327,14 @@ def split_sentences_at_punctuation(
     return sentences
 
 
+def _word_count(text: str) -> int:
+    return len(text.split())
+
+
+def _has_word_token(text: str) -> bool:
+    return any(char.isalnum() for char in text)
+
+
 def _is_sentence_end_run(text: str, run_start: int, run_end: int) -> bool:
     """Decide whether a "." / "?" run really ends a sentence."""
     punctuation = text[run_start:run_end]