diff --git a/sentence_api/DEPLOYMENT.md b/sentence_api/DEPLOYMENT.md index 0a757b4..ae68726 100644 --- a/sentence_api/DEPLOYMENT.md +++ b/sentence_api/DEPLOYMENT.md @@ -152,10 +152,11 @@ curl -X POST http://127.0.0.1:8001/v1/audio/transcriptions \ ### 断句规则 -`sentence_api` 不以 Whisper 返回的 segment 直接作为句子,而是把每个 segment 的文本 -按句号(`.`)拆成一句句,句号即句子结束。若转写服务支持词级时间戳 +`sentence_api` 不以 Whisper 返回的 segment 直接作为句子:文本会跨 segment 累积, +只有碰到句号(`.`)或问号(`?`)才算一句结束;Whisper 在没有标点处断开的 +segment 会自动合并成同一句。若转写服务支持词级时间戳 (`timestamp_granularities[]=word`,Whisper/Speaches 支持),句子的结束时间用 -句号所在单词的时间戳;不支持时自动按文字长度在 segment 内估算,请求失败会回退为 +标点所在单词的时间戳;不支持时自动按文字长度在 segment 内估算,请求失败会回退为 不带词级时间戳的普通请求。MOSS 不支持该参数时同样自动回退,不影响转写。 ## 3. 部署 API diff --git a/sentence_api/processing.py b/sentence_api/processing.py index b92f57f..7ebfe9c 100644 --- a/sentence_api/processing.py +++ b/sentence_api/processing.py @@ -9,7 +9,7 @@ from .config import Settings from .generate_boundaries import ALGORITHM_VERSION, make_entry from .models import SentenceBoundary, SentenceBoundaryDocument from .repository import VideoRepository -from .transcription import Transcript, Transcriber, split_segment_by_periods +from .transcription import Transcript, Transcriber, split_sentences_at_punctuation MOSS_ALGORITHM_VERSION = "moss-period-v2" @@ -107,30 +107,29 @@ def document_from_transcript( samples, sample_rate = decode_audio_mono(audio_path) sentences: List[SentenceBoundary] = [] previous_end = 0 - for segment in sorted(transcript.segments, key=lambda item: (item.start_seconds, item.end_seconds)): - for sentence_segment in split_segment_by_periods(segment): - start_ms = max(previous_end, int(round(sentence_segment.start_seconds * 1000))) - end_ms = min(duration_ms, int(round(sentence_segment.end_seconds * 1000))) - if not sentence_segment.text.strip() or end_ms <= start_ms: - continue - start_sample = max(0, int(start_ms / 1000 * sample_rate)) - end_sample = min(samples.size, int(end_ms / 1000 * sample_rate)) - try: - metrics = analyze_samples(samples[start_sample:end_sample], sample_rate) - speech_duration_ms = metrics.speech_duration_ms - except AudioAnalysisError: - speech_duration_ms = end_ms - start_ms - sentences.append( - SentenceBoundary( - index=len(sentences), - start_ms=start_ms, - end_ms=end_ms, - text=sentence_segment.text.strip(), - language=language, - reference_speech_duration_ms=max(1, speech_duration_ms), - ) + for sentence_segment in split_sentences_at_punctuation(transcript.segments): + start_ms = max(previous_end, int(round(sentence_segment.start_seconds * 1000))) + end_ms = min(duration_ms, int(round(sentence_segment.end_seconds * 1000))) + if not sentence_segment.text.strip() or end_ms <= start_ms: + continue + start_sample = max(0, int(start_ms / 1000 * sample_rate)) + end_sample = min(samples.size, int(end_ms / 1000 * sample_rate)) + try: + metrics = analyze_samples(samples[start_sample:end_sample], sample_rate) + speech_duration_ms = metrics.speech_duration_ms + except AudioAnalysisError: + speech_duration_ms = end_ms - start_ms + sentences.append( + SentenceBoundary( + index=len(sentences), + start_ms=start_ms, + end_ms=end_ms, + text=sentence_segment.text.strip(), + language=language, + reference_speech_duration_ms=max(1, speech_duration_ms), ) - previous_end = end_ms + ) + previous_end = end_ms return SentenceBoundaryDocument( video_hash=video_hash, duration_ms=duration_ms, diff --git a/sentence_api/tests/test_processing.py b/sentence_api/tests/test_processing.py index 01aadb2..908e8aa 100644 --- a/sentence_api/tests/test_processing.py +++ b/sentence_api/tests/test_processing.py @@ -22,48 +22,61 @@ def _silence(seconds: float) -> np.ndarray: return np.zeros(int(seconds * SAMPLE_RATE)) -def test_document_from_transcript_splits_sentences_at_periods(tmp_path): +def test_document_from_transcript_splits_sentences_at_punctuation(tmp_path): samples = np.concatenate( [ - _speech(2.0), - _silence(0.2), - _speech(2.0), - _silence(0.2), - _speech(1.6), + _speech(8.0), + _speech(1.5), + _speech(10.5), + _speech(4.0), ] ).astype(np.float32) wav = tmp_path / "audio.wav" wav.write_bytes(_wav_bytes(samples, SAMPLE_RATE)) transcript = Transcript( - text="Hello world. Good day everyone. Nice to meet you.", + text=( + "So I've been single for about four years now and I find it hard to meet " + "a guy especially in California. I've tried dating apps, I've met friends " + "through friends, I've done online dating and I just feel like most of the " + "guys that I meet aren't serious about a relationship." + ), segments=[ TranscriptionSegment( start_seconds=0.0, - end_seconds=2.0, - text="Hello world.", - words=[ - WordTimestamp(0.0, 0.6, "Hello"), - WordTimestamp(0.7, 1.5, "world."), - ], + end_seconds=8.0, + text=( + "So I've been single for about four years now and I find it hard " + "to meet a guy especially in" + ), ), TranscriptionSegment( - start_seconds=2.2, - end_seconds=5.5, - text="Good day everyone. Nice to meet you.", - words=[ - WordTimestamp(2.2, 2.8, "Good"), - WordTimestamp(2.9, 3.5, "day"), - WordTimestamp(3.6, 4.2, "everyone."), - WordTimestamp(4.4, 4.9, "Nice"), - WordTimestamp(5.0, 5.5, "you."), - ], + start_seconds=8.0, + end_seconds=9.5, + text="California.", + words=[WordTimestamp(8.0, 9.5, "California.")], + ), + TranscriptionSegment( + start_seconds=9.5, + end_seconds=20.0, + text=( + "I've tried dating apps, I've met friends through friends, " + "I've done online dating" + ), + ), + TranscriptionSegment( + start_seconds=20.0, + end_seconds=24.0, + text=( + "and I just feel like most of the guys that I meet aren't serious " + "about a relationship." + ), ), ], ) document = document_from_transcript( video_hash=VIDEO_HASH, - duration_ms=6000, + duration_ms=24000, transcript=transcript, language="en", audio_path=wav, @@ -71,14 +84,18 @@ def test_document_from_transcript_splits_sentences_at_periods(tmp_path): assert document.algorithm_version == MOSS_ALGORITHM_VERSION assert [sentence.text for sentence in document.sentences] == [ - "Hello world.", - "Good day everyone.", - "Nice to meet you.", + ( + "So I've been single for about four years now and I find it hard to meet " + "a guy especially in California." + ), + ( + "I've tried dating apps, I've met friends through friends, I've done " + "online dating and I just feel like most of the guys that I meet aren't " + "serious about a relationship." + ), ] assert document.sentences[0].start_ms == 0 - assert document.sentences[0].end_ms == 1500 - assert document.sentences[1].start_ms == 2200 - assert document.sentences[1].end_ms == 4200 - assert document.sentences[2].start_ms == 4200 - assert document.sentences[2].end_ms == 5500 + assert document.sentences[0].end_ms == 9500 + assert document.sentences[1].start_ms == 9500 + assert document.sentences[1].end_ms == 24000 assert all(sentence.reference_speech_duration_ms > 0 for sentence in document.sentences) diff --git a/sentence_api/tests/test_transcription.py b/sentence_api/tests/test_transcription.py index ce1270d..7ea0227 100644 --- a/sentence_api/tests/test_transcription.py +++ b/sentence_api/tests/test_transcription.py @@ -11,7 +11,7 @@ from sentence_api.transcription import ( _parse_json_segments, _plan_chunks, _parse_word_timestamps, - split_segment_by_periods, + split_sentences_at_punctuation, _wav_bytes, _wav_duration_seconds, ) @@ -195,7 +195,59 @@ def test_parse_word_timestamps_ignores_invalid_entries(): assert _parse_word_timestamps("nope") is None -def test_split_segment_by_periods_uses_word_timestamps(): +def test_split_sentences_merges_segments_without_punctuation(): + segments = [ + TranscriptionSegment( + start_seconds=0.0, + end_seconds=8.0, + text=( + "So I've been single for about four years now and I find it hard " + "to meet a guy especially in" + ), + ), + TranscriptionSegment( + start_seconds=8.0, + end_seconds=9.5, + text="California.", + words=[WordTimestamp(8.0, 9.5, "California.")], + ), + TranscriptionSegment( + start_seconds=9.5, + end_seconds=20.0, + text=( + "I've tried dating apps, I've met friends through friends, " + "I've done online dating" + ), + ), + TranscriptionSegment( + start_seconds=20.0, + end_seconds=24.0, + text=( + "and I just feel like most of the guys that I meet aren't serious " + "about a relationship." + ), + words=[WordTimestamp(20.0, 24.0, "relationship.")], + ), + ] + sentences = split_sentences_at_punctuation(segments) + assert [(s.text, s.start_seconds, s.end_seconds) for s in sentences] == [ + ( + "So I've been single for about four years now and I find it hard to meet " + "a guy especially in California.", + 0.0, + 9.5, + ), + ( + "I've tried dating apps, I've met friends through friends, I've done " + "online dating and I just feel like most of the guys that I meet aren't " + "serious about a relationship.", + 9.5, + 24.0, + ), + ] + + +def test_split_sentences_uses_word_timestamps_within_segment(): segment = TranscriptionSegment( start_seconds=0.0, end_seconds=4.0, @@ -209,7 +261,7 @@ def test_split_segment_by_periods_uses_word_timestamps(): WordTimestamp(3.9, 4.0, "you."), ], ) - sentences = split_segment_by_periods(segment) + sentences = split_sentences_at_punctuation([segment]) assert [(s.text, s.start_seconds, s.end_seconds) for s in sentences] == [ ("Hello world.", 0.0, 1.5), ("Good day.", 1.5, 3.2), @@ -217,27 +269,46 @@ def test_split_segment_by_periods_uses_word_timestamps(): ] -def test_split_segment_by_periods_falls_back_to_proportional(): +def test_split_sentences_splits_at_question_marks(): segment = TranscriptionSegment( - start_seconds=10.0, - end_seconds=20.0, - text="First sentence. Second sentence. Third.", + start_seconds=0.0, + end_seconds=10.0, + text="Really? Wait a minute. What about now?", ) - sentences = split_segment_by_periods(segment) + sentences = split_sentences_at_punctuation([segment]) assert [s.text for s in sentences] == [ - "First sentence.", - "Second sentence.", - "Third.", + "Really?", + "Wait a minute.", + "What about now?", + ] + assert sentences[0].start_seconds == 0.0 + assert sentences[0].end_seconds == sentences[1].start_seconds + assert sentences[1].end_seconds == sentences[2].start_seconds + assert sentences[2].end_seconds == 10.0 + + +def test_split_sentences_falls_back_to_proportional(): + segments = [ + TranscriptionSegment( + start_seconds=10.0, end_seconds=14.0, text="This is the first" + ), + TranscriptionSegment( + start_seconds=14.0, + end_seconds=18.0, + text="sentence. And this is the second one.", + ), + ] + sentences = split_sentences_at_punctuation(segments) + assert [s.text for s in sentences] == [ + "This is the first sentence.", + "And this is the second one.", ] assert sentences[0].start_seconds == 10.0 - assert sentences[1].start_seconds == sentences[0].end_seconds - assert sentences[2].end_seconds == 20.0 - assert sentences[0].end_seconds > 10.0 - assert sentences[1].end_seconds < 20.0 - assert sentences[0].end_seconds < sentences[1].end_seconds + assert sentences[0].end_seconds == sentences[1].start_seconds + assert sentences[1].end_seconds == 18.0 -def test_split_segment_by_periods_falls_back_when_word_ends_are_invalid(): +def test_split_sentences_falls_back_when_word_ends_are_invalid(): segment = TranscriptionSegment( start_seconds=0.0, end_seconds=2.0, @@ -247,30 +318,37 @@ def test_split_segment_by_periods_falls_back_when_word_ends_are_invalid(): WordTimestamp(2.6, 3.0, "Two."), ], ) - sentences = split_segment_by_periods(segment) + sentences = split_sentences_at_punctuation([segment]) assert [s.text for s in sentences] == ["One.", "Two."] assert sentences[0].end_seconds == sentences[1].start_seconds assert sentences[1].end_seconds == 2.0 -def test_split_segment_by_periods_keeps_segment_without_period(): +def test_split_sentences_keeps_incomplete_tail_as_sentence(): segment = TranscriptionSegment( start_seconds=1.0, end_seconds=2.0, text="no period here" ) - assert split_segment_by_periods(segment) == [segment] + sentences = split_sentences_at_punctuation([segment]) + assert [(s.text, s.start_seconds, s.end_seconds) for s in sentences] == [ + ("no period here", 1.0, 2.0) + ] + assert split_sentences_at_punctuation([]) == [] -def test_split_segment_by_periods_handles_ellipsis_and_dots_only(): +def test_split_sentences_handles_ellipsis_and_dots_only(): segment = TranscriptionSegment( start_seconds=0.0, end_seconds=2.0, text="Wait... What? ...", ) - sentences = split_segment_by_periods(segment) - assert [s.text for s in sentences] == ["Wait...", "What? ..."] + sentences = split_sentences_at_punctuation([segment]) + assert [s.text for s in sentences] == ["Wait...", "What?"] assert sentences[0].start_seconds == 0.0 assert sentences[0].end_seconds == sentences[1].start_seconds assert sentences[1].end_seconds == 2.0 + assert split_sentences_at_punctuation( + [TranscriptionSegment(0.0, 1.0, "...")] + ) == [] def test_post_audio_requests_word_timestamps_and_falls_back(monkeypatch): diff --git a/sentence_api/transcription.py b/sentence_api/transcription.py index 642205c..b1c0469 100644 --- a/sentence_api/transcription.py +++ b/sentence_api/transcription.py @@ -239,55 +239,75 @@ def _parse_word_timestamps(raw_words) -> Optional[List[WordTimestamp]]: return words or None -_PERIOD_BOUNDARY = re.compile(r"\.+") +_PUNCTUATION_BOUNDARY = re.compile(r"[.?]+") -def split_segment_by_periods( - segment: TranscriptionSegment, +def split_sentences_at_punctuation( + segments: List[TranscriptionSegment], ) -> List[TranscriptionSegment]: - """Split one whisper segment into sentences at periods (".").""" - text = segment.text.strip() - if not text: - return [] + """Merge whisper segments and cut sentences at periods/question marks. + + Whisper's own segment breaks often fall mid-sentence, so text is accumulated + across segments and a sentence is only closed once the accumulated text + reaches a period or a question mark. + """ + sentences: List[TranscriptionSegment] = [] pieces: List[str] = [] - cursor = 0 - for match in _PERIOD_BOUNDARY.finditer(text): - piece = text[cursor:match.end()].strip() - if piece.strip(".").strip(): - pieces.append(piece) - cursor = match.end() - tail = text[cursor:].strip() - if tail: - pieces.append(tail) - if len(pieces) <= 1: - return [segment] if pieces else [] - ends = _sentence_end_seconds(segment, pieces) - sub_segments: List[TranscriptionSegment] = [] - start = segment.start_seconds - for piece, end in zip(pieces, ends): - if end > start: - sub_segments.append( + start_seconds: Optional[float] = None + last_segment_end: Optional[float] = None + + def flush(end_seconds: float) -> None: + nonlocal pieces, start_seconds + text = " ".join(piece for piece in pieces if piece).strip() + if text and start_seconds is not None and end_seconds > start_seconds: + sentences.append( TranscriptionSegment( - start_seconds=start, - end_seconds=end, - text=piece, - speaker=segment.speaker, + start_seconds=start_seconds, + end_seconds=end_seconds, + text=text, ) ) - start = end - return sub_segments + pieces = [] + start_seconds = None - -def _sentence_end_seconds( - segment: TranscriptionSegment, pieces: List[str] -) -> List[float]: - lengths = [len(piece) for piece in pieces] - total = sum(lengths) - if segment.words and len(segment.words) >= len(pieces): - word_ends = _word_boundary_ends(segment, lengths) - if _valid_boundaries(segment, word_ends): - return word_ends + [segment.end_seconds] - return _proportional_ends(segment, lengths) + [segment.end_seconds] + for segment in sorted( + segments, key=lambda item: (item.start_seconds, item.end_seconds) + ): + text = segment.text.strip() + if not text: + continue + if not pieces: + start_seconds = segment.start_seconds + seg_pieces: List[str] = [] + cursor = 0 + for match in _PUNCTUATION_BOUNDARY.finditer(text): + piece = text[cursor:match.end()].strip() + cursor = match.end() + if piece.strip(".?").strip(): + seg_pieces.append(piece) + tail = text[cursor:].strip() + if tail: + seg_pieces.append(tail) + if not seg_pieces: + continue + lengths = [len(piece) for piece in seg_pieces] + ends: List[float] = [] + if segment.words and len(segment.words) >= len(seg_pieces): + ends = _word_boundary_ends(segment, lengths) + if not _valid_boundaries(segment, ends): + ends = [] + if not ends: + ends = _proportional_ends(segment, lengths) + last_segment_end = segment.end_seconds + for index, piece in enumerate(seg_pieces): + pieces.append(piece) + if piece[-1] in ".?": + end_seconds = ends[index] if index < len(ends) else segment.end_seconds + flush(end_seconds) + start_seconds = end_seconds + if pieces: + flush(last_segment_end if last_segment_end is not None else start_seconds) + return sentences def _word_boundary_ends(