From 3f86e8f44e4ac8273e1de6794ab7e2f1a1d6e71e Mon Sep 17 00:00:00 2001 From: Shuming Liu Date: Tue, 18 Aug 2026 21:57:25 +0800 Subject: [PATCH] continue fixing --- sentence_api/DEPLOYMENT.md | 6 +- sentence_api/audio_metrics.py | 55 ++++++++++++++++- sentence_api/processing.py | 12 +++- sentence_api/static/admin.js | 12 ++-- sentence_api/tests/test_transcription.py | 77 ++++++++++++++++++++++++ sentence_api/transcription.py | 55 ++++++++++++++--- 6 files changed, 202 insertions(+), 15 deletions(-) diff --git a/sentence_api/DEPLOYMENT.md b/sentence_api/DEPLOYMENT.md index 85d1963..f284cd1 100644 --- a/sentence_api/DEPLOYMENT.md +++ b/sentence_api/DEPLOYMENT.md @@ -158,9 +158,13 @@ segment 会自动合并成同一句。若转写服务支持词级时间戳 （`timestamp_granularities[]=word`，Whisper/Speaches 支持），句子的结束时间用 标点所在单词的时间戳；不支持时自动按文字长度在 segment 内估算，请求失败会回退为 不带词级时间戳的普通请求。MOSS 不支持该参数时同样自动回退，不影响转写。 +小数点和常见缩写（如 `0.4%`、`a.m.`、`p.m.`、`Mr.`、`Dr.`、`U.S.`、`no.`）不会被 +当作句子结束；`No.` 作为否定回答时仍会正常断句。 每句结束时间默认再向后顺延 300ms（`MOSS_END_PADDING_MS`，可在 200–500 之间调整）， 避免句子末尾发音被切掉；顺延不会越过下一句的开头。顺延后还会用音频检测句子间的 -停顿，把结束时间拉回到下一句语音真正开始之前，避免偶尔切到下一句的开头。 +停顿，把结束时间拉回到下一句语音真正开始之前，避免偶尔切到下一句的开头。每句的 +开始时间也会用同样的音频检测对齐到真实语音起点，避免 Whisper 起始时间戳偏早时 +上一句的尾部语音混进下一句开头，保证相邻两句不共享音频内容。 ## 3. 部署 API diff --git a/sentence_api/audio_metrics.py b/sentence_api/audio_metrics.py index f2cf9f2..e9ebc5d 100644 --- a/sentence_api/audio_metrics.py +++ b/sentence_api/audio_metrics.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from pathlib import Path -from typing import Tuple +from typing import Optional, Tuple import numpy as np @@ -11,6 +11,8 @@ FRAME_SAMPLES = 480 # speech_frame_mask bridges gaps of up to 5 frames, so use 6 frames (~180 ms). MIN_PAUSE_FRAMES = 6 ONSET_MARGIN_MS = 50 +START_LOOKBACK_MS = 300 +START_LOOKAHEAD_MS = 400 class AudioAnalysisError(RuntimeError): @@ -150,6 +152,57 @@ def refine_sentence_end_ms( return padded_end_ms +def refine_sentence_start_ms( + samples: np.ndarray, + sample_rate: int = SAMPLE_RATE, + *, + raw_start_ms: int, + previous_end_ms: int, +) -> int: + """Pull a sentence start forward to the actual speech onset. + + Whisper's segment start can be earlier than the real speech onset (the tail + of the previous sentence is still audible at the beginning of the next + sentence). When the raw start lands inside continuous speech, this looks for + the last pause of at least MIN_PAUSE_FRAMES before the next speech run and + moves the start to that onset, so adjacent sentences share no audio. + """ + if raw_start_ms <= previous_end_ms: + return max(previous_end_ms, raw_start_ms) + frame_ms = 1000 * max(1, int(round(sample_rate * 0.03))) / sample_rate + total_ms = samples.size / sample_rate * 1000 + window_start_ms = max(0, raw_start_ms - START_LOOKBACK_MS) + window_end_ms = min(int(total_ms), raw_start_ms + START_LOOKAHEAD_MS) + start_sample = int(window_start_ms / 1000 * sample_rate) + end_sample = min(samples.size, int(window_end_ms / 1000 * sample_rate)) + if end_sample - start_sample < max(1, int(round(sample_rate * 0.03))): + return raw_start_ms + try: + speech = speech_frame_mask(samples[start_sample:end_sample], sample_rate) + except AudioAnalysisError: + return raw_start_ms + onset_frame = int(round((raw_start_ms - window_start_ms) / frame_ms)) + if ( + 0 <= onset_frame < len(speech) + and speech[onset_frame] + and (onset_frame == 0 or not speech[onset_frame - 1]) + ): + return raw_start_ms # already at a speech onset + boundary_ms: Optional[float] = None + silence_frames = 0 + for index, is_speech in enumerate(speech): + if not is_speech: + silence_frames += 1 + continue + if silence_frames >= MIN_PAUSE_FRAMES: + boundary_ms = window_start_ms + index * frame_ms + silence_frames = 0 + if boundary_ms is None: + return raw_start_ms + boundary_ms = min(boundary_ms, raw_start_ms + START_LOOKAHEAD_MS) + return max(previous_end_ms, int(boundary_ms)) + + def _bridge_false_runs(values: np.ndarray, max_frames: int) -> None: start = None for index, value in enumerate(values): diff --git a/sentence_api/processing.py b/sentence_api/processing.py index 6c7f7c8..6ba30c7 100644 --- a/sentence_api/processing.py +++ b/sentence_api/processing.py @@ -9,6 +9,7 @@ from .audio_metrics import ( analyze_samples, decode_audio_mono, refine_sentence_end_ms, + refine_sentence_start_ms, ) from .config import Settings from .generate_boundaries import ALGORITHM_VERSION, make_entry @@ -116,7 +117,8 @@ def document_from_transcript( previous_end = 0 sentence_segments = split_sentences_at_punctuation(transcript.segments) for index, sentence_segment in enumerate(sentence_segments): - start_ms = max(previous_end, int(round(sentence_segment.start_seconds * 1000))) + raw_start_ms = int(round(sentence_segment.start_seconds * 1000)) + start_ms = max(previous_end, raw_start_ms) raw_end_ms = int(round(sentence_segment.end_seconds * 1000)) end_ms = min( duration_ms, @@ -130,6 +132,14 @@ def document_from_transcript( end_ms = refine_sentence_end_ms( samples, sample_rate, raw_end_ms=raw_end_ms, padded_end_ms=end_ms ) + refined_start_ms = refine_sentence_start_ms( + samples, + sample_rate, + raw_start_ms=raw_start_ms, + previous_end_ms=previous_end, + ) + if previous_end <= refined_start_ms and end_ms - refined_start_ms >= 100: + start_ms = refined_start_ms if not sentence_segment.text.strip() or end_ms <= start_ms: continue start_sample = max(0, int(start_ms / 1000 * sample_rate)) diff --git a/sentence_api/static/admin.js b/sentence_api/static/admin.js index 90db333..d8f222d 100644 --- a/sentence_api/static/admin.js +++ b/sentence_api/static/admin.js @@ -258,11 +258,13 @@ async function deleteVideo(video) { function formatDuration(ms) { if (!ms) return "-"; - const total = Math.floor(ms / 1000); - const hours = Math.floor(total / 3600); - const minutes = Math.floor(total % 3600 / 60); - const seconds = total % 60; - return hours ? `${hours}:${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}` : `${minutes}:${String(seconds).padStart(2, "0")}`; + const totalTenths = Math.floor(ms / 100); + const hours = Math.floor(totalTenths / 36000); + const minutes = Math.floor(totalTenths % 36000 / 600); + const seconds = Math.floor(totalTenths % 600 / 10); + const tenths = totalTenths % 10; + const base = `${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}.${tenths}`; + return hours ? `${hours}:${base}` : base; } function formatBytes(bytes) { diff --git a/sentence_api/tests/test_transcription.py b/sentence_api/tests/test_transcription.py index 7ea0227..383f61e 100644 --- a/sentence_api/tests/test_transcription.py +++ b/sentence_api/tests/test_transcription.py @@ -351,6 +351,83 @@ def test_split_sentences_handles_ellipsis_and_dots_only(): ) == [] +def test_split_sentences_ignores_decimal_points(): + segment = TranscriptionSegment( + start_seconds=0.0, + end_seconds=6.0, + text="The rate rose by 0.4%. That is a big jump.", + ) + sentences = split_sentences_at_punctuation([segment]) + assert [s.text for s in sentences] == [ + "The rate rose by 0.4%.", + "That is a big jump.", + ] + + +def test_split_sentences_ignores_am_pm_abbreviations(): + segment = TranscriptionSegment( + start_seconds=0.0, + end_seconds=8.0, + text=( + "It was at 9 o'clock p.m. Eastern, President Trump was speaking. " + "The room was quiet." + ), + ) + sentences = split_sentences_at_punctuation([segment]) + assert [s.text for s in sentences] == [ + "It was at 9 o'clock p.m. Eastern, President Trump was speaking.", + "The room was quiet.", + ] + + morning = TranscriptionSegment( + start_seconds=0.0, + end_seconds=4.0, + text="I woke up at 7 a.m. and had coffee. Then I left.", + ) + sentences = split_sentences_at_punctuation([morning]) + assert [s.text for s in sentences] == [ + "I woke up at 7 a.m. and had coffee.", + "Then I left.", + ] + + +def test_split_sentences_merges_segment_ending_with_abbreviation(): + segments = [ + TranscriptionSegment(0.0, 5.0, "The meeting ends at 5 p.m."), + TranscriptionSegment(5.0, 9.0, "Eastern time. Then we go home."), + ] + sentences = split_sentences_at_punctuation(segments) + assert [s.text for s in sentences] == [ + "The meeting ends at 5 p.m. Eastern time.", + "Then we go home.", + ] + + +def test_split_sentences_distinguishes_no_reply_from_number(): + reply = TranscriptionSegment(0.0, 4.0, "No. I don't think so.") + assert [s.text for s in split_sentences_at_punctuation([reply])] == [ + "No.", + "I don't think so.", + ] + numbered = TranscriptionSegment(0.0, 4.0, "See no. 5 on the list.") + assert [s.text for s in split_sentences_at_punctuation([numbered])] == [ + "See no. 5 on the list." + ] + + +def test_split_sentences_ignores_common_abbreviations(): + segment = TranscriptionSegment( + start_seconds=0.0, + end_seconds=4.0, + text="Dr. Smith said the U.S. economy is growing. He was right.", + ) + sentences = split_sentences_at_punctuation([segment]) + assert [s.text for s in sentences] == [ + "Dr. Smith said the U.S. economy is growing.", + "He was right.", + ] + + def test_post_audio_requests_word_timestamps_and_falls_back(monkeypatch): calls = [] diff --git a/sentence_api/transcription.py b/sentence_api/transcription.py index b1c0469..fc5e429 100644 --- a/sentence_api/transcription.py +++ b/sentence_api/transcription.py @@ -4,7 +4,7 @@ import re import wave from dataclasses import dataclass from pathlib import Path -from typing import List, Optional, Protocol +from typing import List, Optional, Protocol, Tuple import httpx import numpy as np @@ -240,6 +240,11 @@ def _parse_word_timestamps(raw_words) -> Optional[List[WordTimestamp]]: _PUNCTUATION_BOUNDARY = re.compile(r"[.?]+") +_ABBREVIATION_RE = re.compile( + r"(?i)\b(?:a\.m\.|p\.m\.|mr\.|mrs\.|ms\.|dr\.|prof\.|st\.|vs\.|etc\.|" + r"e\.g\.|i\.e\.|no\.|approx\.|fig\.|inc\.|ltd\.|jr\.|sr\.|u\.s\.|u\.k\.)" + r"(?![A-Za-z0-9])" +) def split_sentences_at_punctuation( @@ -278,19 +283,21 @@ def split_sentences_at_punctuation( continue if not pieces: start_seconds = segment.start_seconds - seg_pieces: List[str] = [] + seg_pieces: List[Tuple[str, bool]] = [] cursor = 0 for match in _PUNCTUATION_BOUNDARY.finditer(text): + if not _is_sentence_end_run(text, match.start(), match.end()): + continue piece = text[cursor:match.end()].strip() cursor = match.end() if piece.strip(".?").strip(): - seg_pieces.append(piece) + seg_pieces.append((piece, True)) tail = text[cursor:].strip() if tail: - seg_pieces.append(tail) + seg_pieces.append((tail, False)) if not seg_pieces: continue - lengths = [len(piece) for piece in seg_pieces] + lengths = [len(piece) for piece, _ in seg_pieces] ends: List[float] = [] if segment.words and len(segment.words) >= len(seg_pieces): ends = _word_boundary_ends(segment, lengths) @@ -299,9 +306,9 @@ def split_sentences_at_punctuation( if not ends: ends = _proportional_ends(segment, lengths) last_segment_end = segment.end_seconds - for index, piece in enumerate(seg_pieces): + for index, (piece, is_boundary) in enumerate(seg_pieces): pieces.append(piece) - if piece[-1] in ".?": + if is_boundary: end_seconds = ends[index] if index < len(ends) else segment.end_seconds flush(end_seconds) start_seconds = end_seconds @@ -310,6 +317,40 @@ def split_sentences_at_punctuation( return sentences +def _is_sentence_end_run(text: str, run_start: int, run_end: int) -> bool: + """Decide whether a "." / "?" run really ends a sentence.""" + punctuation = text[run_start:run_end] + if "?" in punctuation: + return True + if ( + run_start > 0 + and run_end < len(text) + and text[run_start - 1].isdigit() + and text[run_end].isdigit() + ): + return False # decimal point, e.g. "0.4" + if len(punctuation) > 1: + return True # ellipsis or "!?" etc. + if _is_abbreviation_run(text, run_start, run_end): + return False # e.g. "a.m.", "p.m.", "Mr." + rest = text[run_end:].lstrip() + if rest and rest[0].islower(): + return False # period followed by a lowercase word is an abbreviation + return True + + +def _is_abbreviation_run(text: str, run_start: int, run_end: int) -> bool: + for match in _ABBREVIATION_RE.finditer(text): + if not (match.start() < run_end <= match.end()): + continue + if match.group(0).lower() == "no.": + rest = text[match.end():].lstrip() + if not rest or not rest[0].isdigit(): + return False # "No." as a reply ends the sentence + return True + return False + + def _word_boundary_ends( segment: TranscriptionSegment, lengths: List[int] ) -> List[float]: