From abc052e42dac0b8074c238ac4b676fcffd0ecff8 Mon Sep 17 00:00:00 2001 From: Shuming Liu Date: Sun, 30 Aug 2026 09:06:01 +0800 Subject: [PATCH] change the method of splitting sentences --- sentence_api/processing.py | 2 +- sentence_api/tests/test_transcription.py | 32 ++++++++++++++++++++ sentence_api/transcription.py | 38 +++++++++++++++++------- 3 files changed, 61 insertions(+), 11 deletions(-) diff --git a/sentence_api/processing.py b/sentence_api/processing.py index 807ea99..ec2ae7c 100644 --- a/sentence_api/processing.py +++ b/sentence_api/processing.py @@ -19,7 +19,7 @@ from .repository import VideoRepository from .transcription import Transcript, Transcriber, split_sentences_at_punctuation -MOSS_ALGORITHM_VERSION = "moss-period-v2" +MOSS_ALGORITHM_VERSION = "moss-punctuation-v3" class VideoProcessor: diff --git a/sentence_api/tests/test_transcription.py b/sentence_api/tests/test_transcription.py index 383f61e..2501050 100644 --- a/sentence_api/tests/test_transcription.py +++ b/sentence_api/tests/test_transcription.py @@ -287,6 +287,38 @@ def test_split_sentences_splits_at_question_marks(): assert sentences[2].end_seconds == 10.0 +def test_split_sentences_splits_at_comma_after_fifty_words(): + before_comma = " ".join(f"word{index}" for index in range(1, 51)) + after_comma = "This continuation becomes its own sentence." + segment = TranscriptionSegment( + start_seconds=0.0, + end_seconds=55.0, + text=f"{before_comma}, {after_comma}", + ) + + sentences = split_sentences_at_punctuation([segment]) + + assert [s.text for s in sentences] == [ + f"{before_comma},", + after_comma, + ] + assert sentences[0].start_seconds == 0.0 + assert sentences[0].end_seconds == sentences[1].start_seconds + assert sentences[1].end_seconds == 55.0 + + +def test_split_sentences_merges_comma_piece_shorter_than_fifty_words(): + segment = TranscriptionSegment( + start_seconds=0.0, + end_seconds=5.0, + text="One, two, and three.", + ) + + sentences = split_sentences_at_punctuation([segment]) + + assert [s.text for s in sentences] == ["One, two, and three."] + + def test_split_sentences_falls_back_to_proportional(): segments = [ TranscriptionSegment( diff --git a/sentence_api/transcription.py b/sentence_api/transcription.py index fc5e429..1969e70 100644 --- a/sentence_api/transcription.py +++ b/sentence_api/transcription.py @@ -239,7 +239,8 @@ def _parse_word_timestamps(raw_words) -> Optional[List[WordTimestamp]]: return words or None -_PUNCTUATION_BOUNDARY = re.compile(r"[.?]+") +_PUNCTUATION_BOUNDARY = re.compile(r"[.?]+|,+") +MIN_WORDS_FOR_COMMA_SPLIT = 50 _ABBREVIATION_RE = re.compile( r"(?i)\b(?:a\.m\.|p\.m\.|mr\.|mrs\.|ms\.|dr\.|prof\.|st\.|vs\.|etc\.|" r"e\.g\.|i\.e\.|no\.|approx\.|fig\.|inc\.|ltd\.|jr\.|sr\.|u\.s\.|u\.k\.)" @@ -250,11 +251,12 @@ _ABBREVIATION_RE = re.compile( def split_sentences_at_punctuation( segments: List[TranscriptionSegment], ) -> List[TranscriptionSegment]: - """Merge whisper segments and cut sentences at periods/question marks. + """Merge whisper segments and cut sentences at sentence punctuation. Whisper's own segment breaks often fall mid-sentence, so text is accumulated - across segments and a sentence is only closed once the accumulated text - reaches a period or a question mark. + across segments and a sentence is closed at periods/question marks. Commas + are also eligible boundaries, but only when the text before the comma has + enough words; otherwise the comma stays with the following text. """ sentences: List[TranscriptionSegment] = [] pieces: List[str] = [] @@ -283,15 +285,20 @@ def split_sentences_at_punctuation( continue if not pieces: start_seconds = segment.start_seconds - seg_pieces: List[Tuple[str, bool]] = [] + seg_pieces: List[Tuple[str, Optional[str]]] = [] cursor = 0 for match in _PUNCTUATION_BOUNDARY.finditer(text): - if not _is_sentence_end_run(text, match.start(), match.end()): + boundary_type: Optional[str] = None + if "," in match.group(0): + boundary_type = "comma" + elif _is_sentence_end_run(text, match.start(), match.end()): + boundary_type = "sentence" + if boundary_type is None: continue piece = text[cursor:match.end()].strip() cursor = match.end() - if piece.strip(".?").strip(): - seg_pieces.append((piece, True)) + if _has_word_token(piece): + seg_pieces.append((piece, boundary_type)) tail = text[cursor:].strip() if tail: seg_pieces.append((tail, False)) @@ -306,9 +313,12 @@ def split_sentences_at_punctuation( if not ends: ends = _proportional_ends(segment, lengths) last_segment_end = segment.end_seconds - for index, (piece, is_boundary) in enumerate(seg_pieces): + for index, (piece, boundary_type) in enumerate(seg_pieces): pieces.append(piece) - if is_boundary: + if boundary_type == "sentence" or ( + boundary_type == "comma" + and _word_count(" ".join(pieces)) >= MIN_WORDS_FOR_COMMA_SPLIT + ): end_seconds = ends[index] if index < len(ends) else segment.end_seconds flush(end_seconds) start_seconds = end_seconds @@ -317,6 +327,14 @@ def split_sentences_at_punctuation( return sentences +def _word_count(text: str) -> int: + return len(text.split()) + + +def _has_word_token(text: str) -> bool: + return any(char.isalnum() for char in text) + + def _is_sentence_end_run(text: str, run_start: int, run_end: int) -> bool: """Decide whether a "." / "?" run really ends a sentence.""" punctuation = text[run_start:run_end]