diff --git a/ios/OralTrainer.xcodeproj/project.pbxproj b/ios/OralTrainer.xcodeproj/project.pbxproj index 497dd62..7d25a60 100644 --- a/ios/OralTrainer.xcodeproj/project.pbxproj +++ b/ios/OralTrainer.xcodeproj/project.pbxproj @@ -367,6 +367,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 28FY92RBGB; GENERATE_INFOPLIST_FILE = NO; INFOPLIST_FILE = OralTrainer/Resources/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 16.0; @@ -390,6 +391,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 28FY92RBGB; GENERATE_INFOPLIST_FILE = NO; INFOPLIST_FILE = OralTrainer/Resources/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 16.0; diff --git a/ios/OralTrainer.xcodeproj/project.xcworkspace/xcuserdata/liushuming.xcuserdatad/UserInterfaceState.xcuserstate b/ios/OralTrainer.xcodeproj/project.xcworkspace/xcuserdata/liushuming.xcuserdatad/UserInterfaceState.xcuserstate index bfeddf0..9a0e1b3 100644 Binary files a/ios/OralTrainer.xcodeproj/project.xcworkspace/xcuserdata/liushuming.xcuserdatad/UserInterfaceState.xcuserstate and b/ios/OralTrainer.xcodeproj/project.xcworkspace/xcuserdata/liushuming.xcuserdatad/UserInterfaceState.xcuserstate differ diff --git a/sentence_api/DEPLOYMENT.md b/sentence_api/DEPLOYMENT.md index d83b81e..0a757b4 100644 --- a/sentence_api/DEPLOYMENT.md +++ b/sentence_api/DEPLOYMENT.md @@ -150,6 +150,14 @@ curl -X POST http://127.0.0.1:8001/v1/audio/transcriptions \ 响应应包含 `text`；开启 `verbose_json` 后应包含带 `start`、`end`、`text` 的 `segments`。 +### 断句规则 + +`sentence_api` 不以 Whisper 返回的 segment 直接作为句子，而是把每个 segment 的文本 +按句号（`.`）拆成一句句，句号即句子结束。若转写服务支持词级时间戳 +（`timestamp_granularities[]=word`，Whisper/Speaches 支持），句子的结束时间用 +句号所在单词的时间戳；不支持时自动按文字长度在 segment 内估算，请求失败会回退为 +不带词级时间戳的普通请求。MOSS 不支持该参数时同样自动回退，不影响转写。 + ## 3. 部署 API ### Docker Compose 方式 diff --git a/sentence_api/processing.py b/sentence_api/processing.py index 2bfee26..b92f57f 100644 --- a/sentence_api/processing.py +++ b/sentence_api/processing.py @@ -9,10 +9,10 @@ from .config import Settings from .generate_boundaries import ALGORITHM_VERSION, make_entry from .models import SentenceBoundary, SentenceBoundaryDocument from .repository import VideoRepository -from .transcription import Transcript, Transcriber +from .transcription import Transcript, Transcriber, split_segment_by_periods -MOSS_ALGORITHM_VERSION = "moss-timestamp-v1" +MOSS_ALGORITHM_VERSION = "moss-period-v2" class VideoProcessor: @@ -108,28 +108,29 @@ def document_from_transcript( sentences: List[SentenceBoundary] = [] previous_end = 0 for segment in sorted(transcript.segments, key=lambda item: (item.start_seconds, item.end_seconds)): - start_ms = max(previous_end, int(round(segment.start_seconds * 1000))) - end_ms = min(duration_ms, int(round(segment.end_seconds * 1000))) - if not segment.text.strip() or end_ms <= start_ms: - continue - start_sample = max(0, int(start_ms / 1000 * sample_rate)) - end_sample = min(samples.size, int(end_ms / 1000 * sample_rate)) - try: - metrics = analyze_samples(samples[start_sample:end_sample], sample_rate) - speech_duration_ms = metrics.speech_duration_ms - except AudioAnalysisError: - speech_duration_ms = end_ms - start_ms - sentences.append( - SentenceBoundary( - index=len(sentences), - start_ms=start_ms, - end_ms=end_ms, - text=segment.text.strip(), - language=language, - reference_speech_duration_ms=max(1, speech_duration_ms), + for sentence_segment in split_segment_by_periods(segment): + start_ms = max(previous_end, int(round(sentence_segment.start_seconds * 1000))) + end_ms = min(duration_ms, int(round(sentence_segment.end_seconds * 1000))) + if not sentence_segment.text.strip() or end_ms <= start_ms: + continue + start_sample = max(0, int(start_ms / 1000 * sample_rate)) + end_sample = min(samples.size, int(end_ms / 1000 * sample_rate)) + try: + metrics = analyze_samples(samples[start_sample:end_sample], sample_rate) + speech_duration_ms = metrics.speech_duration_ms + except AudioAnalysisError: + speech_duration_ms = end_ms - start_ms + sentences.append( + SentenceBoundary( + index=len(sentences), + start_ms=start_ms, + end_ms=end_ms, + text=sentence_segment.text.strip(), + language=language, + reference_speech_duration_ms=max(1, speech_duration_ms), + ) ) - ) - previous_end = end_ms + previous_end = end_ms return SentenceBoundaryDocument( video_hash=video_hash, duration_ms=duration_ms, diff --git a/sentence_api/tests/test_processing.py b/sentence_api/tests/test_processing.py new file mode 100644 index 0000000..01aadb2 --- /dev/null +++ b/sentence_api/tests/test_processing.py @@ -0,0 +1,84 @@ +import numpy as np + +from sentence_api.processing import MOSS_ALGORITHM_VERSION, document_from_transcript +from sentence_api.transcription import ( + Transcript, + TranscriptionSegment, + WordTimestamp, + _wav_bytes, +) + + +SAMPLE_RATE = 16000 +VIDEO_HASH = "a" * 64 + + +def _speech(seconds: float) -> np.ndarray: + t = np.arange(int(seconds * SAMPLE_RATE)) / SAMPLE_RATE + return 0.25 * np.sin(2 * np.pi * 220 * t) + + +def _silence(seconds: float) -> np.ndarray: + return np.zeros(int(seconds * SAMPLE_RATE)) + + +def test_document_from_transcript_splits_sentences_at_periods(tmp_path): + samples = np.concatenate( + [ + _speech(2.0), + _silence(0.2), + _speech(2.0), + _silence(0.2), + _speech(1.6), + ] + ).astype(np.float32) + wav = tmp_path / "audio.wav" + wav.write_bytes(_wav_bytes(samples, SAMPLE_RATE)) + transcript = Transcript( + text="Hello world. Good day everyone. Nice to meet you.", + segments=[ + TranscriptionSegment( + start_seconds=0.0, + end_seconds=2.0, + text="Hello world.", + words=[ + WordTimestamp(0.0, 0.6, "Hello"), + WordTimestamp(0.7, 1.5, "world."), + ], + ), + TranscriptionSegment( + start_seconds=2.2, + end_seconds=5.5, + text="Good day everyone. Nice to meet you.", + words=[ + WordTimestamp(2.2, 2.8, "Good"), + WordTimestamp(2.9, 3.5, "day"), + WordTimestamp(3.6, 4.2, "everyone."), + WordTimestamp(4.4, 4.9, "Nice"), + WordTimestamp(5.0, 5.5, "you."), + ], + ), + ], + ) + + document = document_from_transcript( + video_hash=VIDEO_HASH, + duration_ms=6000, + transcript=transcript, + language="en", + audio_path=wav, + ) + + assert document.algorithm_version == MOSS_ALGORITHM_VERSION + assert [sentence.text for sentence in document.sentences] == [ + "Hello world.", + "Good day everyone.", + "Nice to meet you.", + ] + assert document.sentences[0].start_ms == 0 + assert document.sentences[0].end_ms == 1500 + assert document.sentences[1].start_ms == 2200 + assert document.sentences[1].end_ms == 4200 + assert document.sentences[2].start_ms == 4200 + assert document.sentences[2].end_ms == 5500 + assert all(sentence.reference_speech_duration_ms > 0 for sentence in document.sentences) diff --git a/sentence_api/tests/test_transcription.py b/sentence_api/tests/test_transcription.py index 7db9cdd..ce1270d 100644 --- a/sentence_api/tests/test_transcription.py +++ b/sentence_api/tests/test_transcription.py @@ -6,8 +6,12 @@ import pytest from sentence_api.transcription import ( MossTranscriber, + TranscriptionSegment, + WordTimestamp, _parse_json_segments, _plan_chunks, + _parse_word_timestamps, + split_segment_by_periods, _wav_bytes, _wav_duration_seconds, ) @@ -107,7 +111,16 @@ def test_chunked_transcribe_offsets_timestamps(monkeypatch): ] elif call_count["n"] == 2: segments = [ - {"start": 1.0, "end": 4.0, "text": "third part", "compression_ratio": 1.4}, + { + "start": 1.0, + "end": 4.0, + "text": "third part", + "compression_ratio": 1.4, + "words": [ + {"word": "third", "start": 1.0, "end": 2.5}, + {"word": " part", "start": 2.6, "end": 4.0}, + ], + }, {"start": 27.0, "end": 30.0, "text": "fourth part", "compression_ratio": 1.5}, ] else: @@ -115,6 +128,8 @@ def test_chunked_transcribe_offsets_timestamps(monkeypatch): assert abs(duration - 30.0) < 1.0 or abs(duration - 5.0) < 1.0 class Response: + status_code = 200 + def raise_for_status(self): pass @@ -134,5 +149,162 @@ def test_chunked_transcribe_offsets_timestamps(monkeypatch): (60.5, 63.5, "tail part"), ] assert result.text == "first part second part third part fourth part tail part" + assert [(w.start_seconds, w.end_seconds, w.text) for w in result.segments[2].words] == [ + (31.0, 32.5, "third"), + (32.6, 34.0, "part"), + ] + assert result.segments[0].words is None assert len(requests) == 3 assert all(request.get("condition_on_previous_text") == "false" for request in requests) + assert all(request.get("timestamp_granularities[]") == "word" for request in requests) + + +def test_parse_json_segments_parses_word_timestamps(): + raw = [ + { + "start": 0.0, + "end": 4.0, + "text": "Hello world. Good day.", + "words": [ + {"word": "Hello", "start": 0.0, "end": 0.6}, + {"word": " world.", "start": 0.7, "end": 1.5}, + {"word": " Good", "start": 1.8, "end": 2.4}, + {"word": " day.", "start": 2.5, "end": 3.2}, + ], + }, + {"start": 4.5, "end": 6.0, "text": "no words"}, + ] + segments = _parse_json_segments(raw) + assert [word.text for word in segments[0].words] == ["Hello", "world.", "Good", "day."] + assert segments[1].words is None + + +def test_parse_word_timestamps_ignores_invalid_entries(): + words = _parse_word_timestamps( + [ + {"word": "ok", "start": 0.0, "end": 0.5}, + {"word": "bad"}, + {"word": "", "start": 1.0, "end": 1.5}, + {"word": "flat", "start": 2.0, "end": 2.0}, + ] + ) + assert [(word.text, word.start_seconds, word.end_seconds) for word in words] == [ + ("ok", 0.0, 0.5) + ] + assert _parse_word_timestamps(None) is None + assert _parse_word_timestamps("nope") is None + + +def test_split_segment_by_periods_uses_word_timestamps(): + segment = TranscriptionSegment( + start_seconds=0.0, + end_seconds=4.0, + text="Hello world. Good day. Nice to meet you.", + words=[ + WordTimestamp(0.0, 0.6, "Hello"), + WordTimestamp(0.7, 1.5, "world."), + WordTimestamp(1.8, 2.4, "Good"), + WordTimestamp(2.5, 3.2, "day."), + WordTimestamp(3.3, 3.8, "Nice"), + WordTimestamp(3.9, 4.0, "you."), + ], + ) + sentences = split_segment_by_periods(segment) + assert [(s.text, s.start_seconds, s.end_seconds) for s in sentences] == [ + ("Hello world.", 0.0, 1.5), + ("Good day.", 1.5, 3.2), + ("Nice to meet you.", 3.2, 4.0), + ] + + +def test_split_segment_by_periods_falls_back_to_proportional(): + segment = TranscriptionSegment( + start_seconds=10.0, + end_seconds=20.0, + text="First sentence. Second sentence. Third.", + ) + sentences = split_segment_by_periods(segment) + assert [s.text for s in sentences] == [ + "First sentence.", + "Second sentence.", + "Third.", + ] + assert sentences[0].start_seconds == 10.0 + assert sentences[1].start_seconds == sentences[0].end_seconds + assert sentences[2].end_seconds == 20.0 + assert sentences[0].end_seconds > 10.0 + assert sentences[1].end_seconds < 20.0 + assert sentences[0].end_seconds < sentences[1].end_seconds + + +def test_split_segment_by_periods_falls_back_when_word_ends_are_invalid(): + segment = TranscriptionSegment( + start_seconds=0.0, + end_seconds=2.0, + text="One. Two.", + words=[ + WordTimestamp(0.0, 2.5, "One."), + WordTimestamp(2.6, 3.0, "Two."), + ], + ) + sentences = split_segment_by_periods(segment) + assert [s.text for s in sentences] == ["One.", "Two."] + assert sentences[0].end_seconds == sentences[1].start_seconds + assert sentences[1].end_seconds == 2.0 + + +def test_split_segment_by_periods_keeps_segment_without_period(): + segment = TranscriptionSegment( + start_seconds=1.0, end_seconds=2.0, text="no period here" + ) + assert split_segment_by_periods(segment) == [segment] + + +def test_split_segment_by_periods_handles_ellipsis_and_dots_only(): + segment = TranscriptionSegment( + start_seconds=0.0, + end_seconds=2.0, + text="Wait... What? ...", + ) + sentences = split_segment_by_periods(segment) + assert [s.text for s in sentences] == ["Wait...", "What? ..."] + assert sentences[0].start_seconds == 0.0 + assert sentences[0].end_seconds == sentences[1].start_seconds + assert sentences[1].end_seconds == 2.0 + + +def test_post_audio_requests_word_timestamps_and_falls_back(monkeypatch): + calls = [] + + class RejectedResponse: + status_code = 400 + + def raise_for_status(self): + pass + + def json(self): + return {"error": "unknown parameter"} + + class OkResponse: + status_code = 200 + + def raise_for_status(self): + pass + + def json(self): + return {"text": "hi.", "segments": []} + + def fake_post(endpoint, data=None, files=None, timeout=None): + calls.append(dict(data)) + if "timestamp_granularities[]" in data: + return RejectedResponse() + return OkResponse() + + monkeypatch.setattr("sentence_api.transcription.httpx.post", fake_post) + transcriber = MossTranscriber(endpoint="http://whisper:9000", model="whisper") + result = transcriber._post_audio( + io.BytesIO(b"fake-audio"), "clip.wav", "audio/wav", "en" + ) + assert result["text"] == "hi." + assert calls[0]["timestamp_granularities[]"] == "word" + assert "timestamp_granularities[]" not in calls[1] diff --git a/sentence_api/transcription.py b/sentence_api/transcription.py index e2801ec..642205c 100644 --- a/sentence_api/transcription.py +++ b/sentence_api/transcription.py @@ -31,6 +31,14 @@ class TranscriptionSegment: end_seconds: float text: str speaker: Optional[str] = None + words: Optional[List["WordTimestamp"]] = None + + +@dataclass(frozen=True) +class WordTimestamp: + start_seconds: float + end_seconds: float + text: str @dataclass(frozen=True) @@ -114,12 +122,23 @@ class MossTranscriber: if not chunk_segments: chunk_segments = _parse_compact_segments(str(payload.get("text") or "").strip()) for segment in chunk_segments: + words = None + if segment.words: + words = [ + WordTimestamp( + start_seconds=word.start_seconds + start_seconds, + end_seconds=word.end_seconds + start_seconds, + text=word.text, + ) + for word in segment.words + ] segments.append( TranscriptionSegment( start_seconds=segment.start_seconds + start_seconds, end_seconds=segment.end_seconds + start_seconds, text=segment.text, speaker=segment.speaker, + words=words, ) ) segments.sort(key=lambda item: (item.start_seconds, item.end_seconds)) @@ -144,17 +163,27 @@ class MossTranscriber: data["language"] = language try: - response = httpx.post( - self.endpoint, - data=data, - files={"file": (filename, audio_file, content_type)}, - timeout=httpx.Timeout(self.timeout_seconds, connect=30), + word_payload = dict(data) + word_payload["timestamp_granularities[]"] = "word" + response = self._post_once( + word_payload, audio_file, filename, content_type ) + if response.status_code in (400, 422): + audio_file.seek(0) + response = self._post_once(data, audio_file, filename, content_type) response.raise_for_status() return response.json() except (httpx.HTTPError, ValueError) as exc: raise RuntimeError(f"MOSS transcription request failed: {exc}") from exc + def _post_once(self, data: dict, audio_file, filename: str, content_type: str): + return httpx.post( + self.endpoint, + data=data, + files={"file": (filename, audio_file, content_type)}, + timeout=httpx.Timeout(self.timeout_seconds, connect=30), + ) + def _parse_json_segments( raw_segments, compression_limit: Optional[float] = None @@ -186,11 +215,126 @@ def _parse_json_segments( end_seconds=end, text=text, speaker=str(speaker) if speaker is not None else None, + words=_parse_word_timestamps(item.get("words")), ) ) return segments +def _parse_word_timestamps(raw_words) -> Optional[List[WordTimestamp]]: + if not isinstance(raw_words, list): + return None + words = [] + for item in raw_words: + if not isinstance(item, dict): + continue + try: + start = float(item["start"]) + end = float(item["end"]) + except (KeyError, TypeError, ValueError): + continue + text = str(item.get("word") or "").strip() + if text and end > start: + words.append(WordTimestamp(start_seconds=start, end_seconds=end, text=text)) + return words or None + + +_PERIOD_BOUNDARY = re.compile(r"\.+") + + +def split_segment_by_periods( + segment: TranscriptionSegment, +) -> List[TranscriptionSegment]: + """Split one whisper segment into sentences at periods (".").""" + text = segment.text.strip() + if not text: + return [] + pieces: List[str] = [] + cursor = 0 + for match in _PERIOD_BOUNDARY.finditer(text): + piece = text[cursor:match.end()].strip() + if piece.strip(".").strip(): + pieces.append(piece) + cursor = match.end() + tail = text[cursor:].strip() + if tail: + pieces.append(tail) + if len(pieces) <= 1: + return [segment] if pieces else [] + ends = _sentence_end_seconds(segment, pieces) + sub_segments: List[TranscriptionSegment] = [] + start = segment.start_seconds + for piece, end in zip(pieces, ends): + if end > start: + sub_segments.append( + TranscriptionSegment( + start_seconds=start, + end_seconds=end, + text=piece, + speaker=segment.speaker, + ) + ) + start = end + return sub_segments + + +def _sentence_end_seconds( + segment: TranscriptionSegment, pieces: List[str] +) -> List[float]: + lengths = [len(piece) for piece in pieces] + total = sum(lengths) + if segment.words and len(segment.words) >= len(pieces): + word_ends = _word_boundary_ends(segment, lengths) + if _valid_boundaries(segment, word_ends): + return word_ends + [segment.end_seconds] + return _proportional_ends(segment, lengths) + [segment.end_seconds] + + +def _word_boundary_ends( + segment: TranscriptionSegment, lengths: List[int] +) -> List[float]: + targets = [sum(lengths[:count]) for count in range(1, len(lengths))] + buffer = "" + word_indexes: List[int] = [] + target_index = 0 + for word_index, word in enumerate(segment.words): + buffer += (" " if buffer else "") + word.text + while target_index < len(targets) and len(buffer) >= targets[target_index]: + word_indexes.append(word_index) + target_index += 1 + if len(word_indexes) != len(targets): + return [] + return [segment.words[index].end_seconds for index in word_indexes] + + +def _valid_boundaries( + segment: TranscriptionSegment, ends: List[float] +) -> bool: + previous = segment.start_seconds + for end in ends: + if not segment.start_seconds < end < segment.end_seconds: + return False + if end <= previous: + return False + previous = end + return True + + +def _proportional_ends( + segment: TranscriptionSegment, lengths: List[int] +) -> List[float]: + total = sum(lengths) + ends = [] + cumulative = 0 + for length in lengths[:-1]: + cumulative += length + ends.append( + segment.start_seconds + + (cumulative / total) * (segment.end_seconds - segment.start_seconds) + ) + return ends + + def _wav_duration_seconds(path: Path) -> Optional[float]: try: with wave.open(str(path), "rb") as wav: