change sentence cut method

ios/OralTrainer.xcodeproj/project.pbxproj

@@ -367,6 +367,7 @@
 				ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
 				CODE_SIGN_STYLE = Automatic;
 				CURRENT_PROJECT_VERSION = 1;
+				DEVELOPMENT_TEAM = 28FY92RBGB;
 				GENERATE_INFOPLIST_FILE = NO;
 				INFOPLIST_FILE = OralTrainer/Resources/Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 16.0;
@@ -390,6 +391,7 @@
 				ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
 				CODE_SIGN_STYLE = Automatic;
 				CURRENT_PROJECT_VERSION = 1;
+				DEVELOPMENT_TEAM = 28FY92RBGB;
 				GENERATE_INFOPLIST_FILE = NO;
 				INFOPLIST_FILE = OralTrainer/Resources/Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 16.0;

sentence_api/DEPLOYMENT.md

@@ -150,6 +150,14 @@ curl -X POST http://127.0.0.1:8001/v1/audio/transcriptions \
 
 响应应包含 `text`；开启 `verbose_json` 后应包含带 `start`、`end`、`text` 的 `segments`。
 
+### 断句规则
+
+`sentence_api` 不以 Whisper 返回的 segment 直接作为句子，而是把每个 segment 的文本
+按句号（`.`）拆成一句句，句号即句子结束。若转写服务支持词级时间戳
+（`timestamp_granularities[]=word`，Whisper/Speaches 支持），句子的结束时间用
+句号所在单词的时间戳；不支持时自动按文字长度在 segment 内估算，请求失败会回退为
+不带词级时间戳的普通请求。MOSS 不支持该参数时同样自动回退，不影响转写。
+
 ## 3. 部署 API
 
 ### Docker Compose 方式

sentence_api/processing.py

@@ -9,10 +9,10 @@ from .config import Settings
 from .generate_boundaries import ALGORITHM_VERSION, make_entry
 from .models import SentenceBoundary, SentenceBoundaryDocument
 from .repository import VideoRepository
-from .transcription import Transcript, Transcriber
+from .transcription import Transcript, Transcriber, split_segment_by_periods
 
 
-MOSS_ALGORITHM_VERSION = "moss-timestamp-v1"
+MOSS_ALGORITHM_VERSION = "moss-period-v2"
 
 
 class VideoProcessor:
@@ -108,28 +108,29 @@ def document_from_transcript(
     sentences: List[SentenceBoundary] = []
     previous_end = 0
     for segment in sorted(transcript.segments, key=lambda item: (item.start_seconds, item.end_seconds)):
-        start_ms = max(previous_end, int(round(segment.start_seconds * 1000)))
-        end_ms = min(duration_ms, int(round(segment.end_seconds * 1000)))
-        if not segment.text.strip() or end_ms <= start_ms:
-            continue
-        start_sample = max(0, int(start_ms / 1000 * sample_rate))
-        end_sample = min(samples.size, int(end_ms / 1000 * sample_rate))
-        try:
-            metrics = analyze_samples(samples[start_sample:end_sample], sample_rate)
-            speech_duration_ms = metrics.speech_duration_ms
-        except AudioAnalysisError:
-            speech_duration_ms = end_ms - start_ms
-        sentences.append(
-            SentenceBoundary(
-                index=len(sentences),
-                start_ms=start_ms,
-                end_ms=end_ms,
-                text=segment.text.strip(),
-                language=language,
-                reference_speech_duration_ms=max(1, speech_duration_ms),
+        for sentence_segment in split_segment_by_periods(segment):
+            start_ms = max(previous_end, int(round(sentence_segment.start_seconds * 1000)))
+            end_ms = min(duration_ms, int(round(sentence_segment.end_seconds * 1000)))
+            if not sentence_segment.text.strip() or end_ms <= start_ms:
+                continue
+            start_sample = max(0, int(start_ms / 1000 * sample_rate))
+            end_sample = min(samples.size, int(end_ms / 1000 * sample_rate))
+            try:
+                metrics = analyze_samples(samples[start_sample:end_sample], sample_rate)
+                speech_duration_ms = metrics.speech_duration_ms
+            except AudioAnalysisError:
+                speech_duration_ms = end_ms - start_ms
+            sentences.append(
+                SentenceBoundary(
+                    index=len(sentences),
+                    start_ms=start_ms,
+                    end_ms=end_ms,
+                    text=sentence_segment.text.strip(),
+                    language=language,
+                    reference_speech_duration_ms=max(1, speech_duration_ms),
+                )
             )
-        )
-        previous_end = end_ms
+            previous_end = end_ms
     return SentenceBoundaryDocument(
         video_hash=video_hash,
         duration_ms=duration_ms,

sentence_api/tests/test_processing.py

@@ -0,0 +1,84 @@
+import numpy as np
+
+from sentence_api.processing import MOSS_ALGORITHM_VERSION, document_from_transcript
+from sentence_api.transcription import (
+    Transcript,
+    TranscriptionSegment,
+    WordTimestamp,
+    _wav_bytes,
+)
+
+
+SAMPLE_RATE = 16000
+VIDEO_HASH = "a" * 64
+
+
+def _speech(seconds: float) -> np.ndarray:
+    t = np.arange(int(seconds * SAMPLE_RATE)) / SAMPLE_RATE
+    return 0.25 * np.sin(2 * np.pi * 220 * t)
+
+
+def _silence(seconds: float) -> np.ndarray:
+    return np.zeros(int(seconds * SAMPLE_RATE))
+
+
+def test_document_from_transcript_splits_sentences_at_periods(tmp_path):
+    samples = np.concatenate(
+        [
+            _speech(2.0),
+            _silence(0.2),
+            _speech(2.0),
+            _silence(0.2),
+            _speech(1.6),
+        ]
+    ).astype(np.float32)
+    wav = tmp_path / "audio.wav"
+    wav.write_bytes(_wav_bytes(samples, SAMPLE_RATE))
+    transcript = Transcript(
+        text="Hello world. Good day everyone. Nice to meet you.",
+        segments=[
+            TranscriptionSegment(
+                start_seconds=0.0,
+                end_seconds=2.0,
+                text="Hello world.",
+                words=[
+                    WordTimestamp(0.0, 0.6, "Hello"),
+                    WordTimestamp(0.7, 1.5, "world."),
+                ],
+            ),
+            TranscriptionSegment(
+                start_seconds=2.2,
+                end_seconds=5.5,
+                text="Good day everyone. Nice to meet you.",
+                words=[
+                    WordTimestamp(2.2, 2.8, "Good"),
+                    WordTimestamp(2.9, 3.5, "day"),
+                    WordTimestamp(3.6, 4.2, "everyone."),
+                    WordTimestamp(4.4, 4.9, "Nice"),
+                    WordTimestamp(5.0, 5.5, "you."),
+                ],
+            ),
+        ],
+    )
+
+    document = document_from_transcript(
+        video_hash=VIDEO_HASH,
+        duration_ms=6000,
+        transcript=transcript,
+        language="en",
+        audio_path=wav,
+    )
+
+    assert document.algorithm_version == MOSS_ALGORITHM_VERSION
+    assert [sentence.text for sentence in document.sentences] == [
+        "Hello world.",
+        "Good day everyone.",
+        "Nice to meet you.",
+    ]
+    assert document.sentences[0].start_ms == 0
+    assert document.sentences[0].end_ms == 1500
+    assert document.sentences[1].start_ms == 2200
+    assert document.sentences[1].end_ms == 4200
+    assert document.sentences[2].start_ms == 4200
+    assert document.sentences[2].end_ms == 5500
+    assert all(sentence.reference_speech_duration_ms > 0 for sentence in document.sentences)

sentence_api/tests/test_transcription.py

@@ -6,8 +6,12 @@ import pytest
 
 from sentence_api.transcription import (
     MossTranscriber,
+    TranscriptionSegment,
+    WordTimestamp,
     _parse_json_segments,
     _plan_chunks,
+    _parse_word_timestamps,
+    split_segment_by_periods,
     _wav_bytes,
     _wav_duration_seconds,
 )
@@ -107,7 +111,16 @@ def test_chunked_transcribe_offsets_timestamps(monkeypatch):
             ]
         elif call_count["n"] == 2:
             segments = [
-                {"start": 1.0, "end": 4.0, "text": "third part", "compression_ratio": 1.4},
+                {
+                    "start": 1.0,
+                    "end": 4.0,
+                    "text": "third part",
+                    "compression_ratio": 1.4,
+                    "words": [
+                        {"word": "third", "start": 1.0, "end": 2.5},
+                        {"word": " part", "start": 2.6, "end": 4.0},
+                    ],
+                },
                 {"start": 27.0, "end": 30.0, "text": "fourth part", "compression_ratio": 1.5},
             ]
         else:
@@ -115,6 +128,8 @@ def test_chunked_transcribe_offsets_timestamps(monkeypatch):
         assert abs(duration - 30.0) < 1.0 or abs(duration - 5.0) < 1.0
 
         class Response:
+            status_code = 200
+
             def raise_for_status(self):
                 pass
 
@@ -134,5 +149,162 @@ def test_chunked_transcribe_offsets_timestamps(monkeypatch):
         (60.5, 63.5, "tail part"),
     ]
     assert result.text == "first part second part third part fourth part tail part"
+    assert [(w.start_seconds, w.end_seconds, w.text) for w in result.segments[2].words] == [
+        (31.0, 32.5, "third"),
+        (32.6, 34.0, "part"),
+    ]
+    assert result.segments[0].words is None
     assert len(requests) == 3
     assert all(request.get("condition_on_previous_text") == "false" for request in requests)
+    assert all(request.get("timestamp_granularities[]") == "word" for request in requests)
+
+
+def test_parse_json_segments_parses_word_timestamps():
+    raw = [
+        {
+            "start": 0.0,
+            "end": 4.0,
+            "text": "Hello world. Good day.",
+            "words": [
+                {"word": "Hello", "start": 0.0, "end": 0.6},
+                {"word": " world.", "start": 0.7, "end": 1.5},
+                {"word": " Good", "start": 1.8, "end": 2.4},
+                {"word": " day.", "start": 2.5, "end": 3.2},
+            ],
+        },
+        {"start": 4.5, "end": 6.0, "text": "no words"},
+    ]
+    segments = _parse_json_segments(raw)
+    assert [word.text for word in segments[0].words] == ["Hello", "world.", "Good", "day."]
+    assert segments[1].words is None
+
+
+def test_parse_word_timestamps_ignores_invalid_entries():
+    words = _parse_word_timestamps(
+        [
+            {"word": "ok", "start": 0.0, "end": 0.5},
+            {"word": "bad"},
+            {"word": "", "start": 1.0, "end": 1.5},
+            {"word": "flat", "start": 2.0, "end": 2.0},
+        ]
+    )
+    assert [(word.text, word.start_seconds, word.end_seconds) for word in words] == [
+        ("ok", 0.0, 0.5)
+    ]
+    assert _parse_word_timestamps(None) is None
+    assert _parse_word_timestamps("nope") is None
+
+
+def test_split_segment_by_periods_uses_word_timestamps():
+    segment = TranscriptionSegment(
+        start_seconds=0.0,
+        end_seconds=4.0,
+        text="Hello world. Good day. Nice to meet you.",
+        words=[
+            WordTimestamp(0.0, 0.6, "Hello"),
+            WordTimestamp(0.7, 1.5, "world."),
+            WordTimestamp(1.8, 2.4, "Good"),
+            WordTimestamp(2.5, 3.2, "day."),
+            WordTimestamp(3.3, 3.8, "Nice"),
+            WordTimestamp(3.9, 4.0, "you."),
+        ],
+    )
+    sentences = split_segment_by_periods(segment)
+    assert [(s.text, s.start_seconds, s.end_seconds) for s in sentences] == [
+        ("Hello world.", 0.0, 1.5),
+        ("Good day.", 1.5, 3.2),
+        ("Nice to meet you.", 3.2, 4.0),
+    ]
+
+
+def test_split_segment_by_periods_falls_back_to_proportional():
+    segment = TranscriptionSegment(
+        start_seconds=10.0,
+        end_seconds=20.0,
+        text="First sentence. Second sentence. Third.",
+    )
+    sentences = split_segment_by_periods(segment)
+    assert [s.text for s in sentences] == [
+        "First sentence.",
+        "Second sentence.",
+        "Third.",
+    ]
+    assert sentences[0].start_seconds == 10.0
+    assert sentences[1].start_seconds == sentences[0].end_seconds
+    assert sentences[2].end_seconds == 20.0
+    assert sentences[0].end_seconds > 10.0
+    assert sentences[1].end_seconds < 20.0
+    assert sentences[0].end_seconds < sentences[1].end_seconds
+
+
+def test_split_segment_by_periods_falls_back_when_word_ends_are_invalid():
+    segment = TranscriptionSegment(
+        start_seconds=0.0,
+        end_seconds=2.0,
+        text="One. Two.",
+        words=[
+            WordTimestamp(0.0, 2.5, "One."),
+            WordTimestamp(2.6, 3.0, "Two."),
+        ],
+    )
+    sentences = split_segment_by_periods(segment)
+    assert [s.text for s in sentences] == ["One.", "Two."]
+    assert sentences[0].end_seconds == sentences[1].start_seconds
+    assert sentences[1].end_seconds == 2.0
+
+
+def test_split_segment_by_periods_keeps_segment_without_period():
+    segment = TranscriptionSegment(
+        start_seconds=1.0, end_seconds=2.0, text="no period here"
+    )
+    assert split_segment_by_periods(segment) == [segment]
+
+
+def test_split_segment_by_periods_handles_ellipsis_and_dots_only():
+    segment = TranscriptionSegment(
+        start_seconds=0.0,
+        end_seconds=2.0,
+        text="Wait... What? ...",
+    )
+    sentences = split_segment_by_periods(segment)
+    assert [s.text for s in sentences] == ["Wait...", "What? ..."]
+    assert sentences[0].start_seconds == 0.0
+    assert sentences[0].end_seconds == sentences[1].start_seconds
+    assert sentences[1].end_seconds == 2.0
+
+
+def test_post_audio_requests_word_timestamps_and_falls_back(monkeypatch):
+    calls = []
+
+    class RejectedResponse:
+        status_code = 400
+
+        def raise_for_status(self):
+            pass
+
+        def json(self):
+            return {"error": "unknown parameter"}
+
+    class OkResponse:
+        status_code = 200
+
+        def raise_for_status(self):
+            pass
+
+        def json(self):
+            return {"text": "hi.", "segments": []}
+
+    def fake_post(endpoint, data=None, files=None, timeout=None):
+        calls.append(dict(data))
+        if "timestamp_granularities[]" in data:
+            return RejectedResponse()
+        return OkResponse()
+
+    monkeypatch.setattr("sentence_api.transcription.httpx.post", fake_post)
+    transcriber = MossTranscriber(endpoint="http://whisper:9000", model="whisper")
+    result = transcriber._post_audio(
+        io.BytesIO(b"fake-audio"), "clip.wav", "audio/wav", "en"
+    )
+    assert result["text"] == "hi."
+    assert calls[0]["timestamp_granularities[]"] == "word"
+    assert "timestamp_granularities[]" not in calls[1]

sentence_api/transcription.py

@@ -31,6 +31,14 @@ class TranscriptionSegment:
     end_seconds: float
     text: str
     speaker: Optional[str] = None
+    words: Optional[List["WordTimestamp"]] = None
+
+
+@dataclass(frozen=True)
+class WordTimestamp:
+    start_seconds: float
+    end_seconds: float
+    text: str
 
 
 @dataclass(frozen=True)
@@ -114,12 +122,23 @@ class MossTranscriber:
             if not chunk_segments:
                 chunk_segments = _parse_compact_segments(str(payload.get("text") or "").strip())
             for segment in chunk_segments:
+                words = None
+                if segment.words:
+                    words = [
+                        WordTimestamp(
+                            start_seconds=word.start_seconds + start_seconds,
+                            end_seconds=word.end_seconds + start_seconds,
+                            text=word.text,
+                        )
+                        for word in segment.words
+                    ]
                 segments.append(
                     TranscriptionSegment(
                         start_seconds=segment.start_seconds + start_seconds,
                         end_seconds=segment.end_seconds + start_seconds,
                         text=segment.text,
                         speaker=segment.speaker,
+                        words=words,
                     )
                 )
         segments.sort(key=lambda item: (item.start_seconds, item.end_seconds))
@@ -144,17 +163,27 @@ class MossTranscriber:
             data["language"] = language
 
         try:
-            response = httpx.post(
-                self.endpoint,
-                data=data,
-                files={"file": (filename, audio_file, content_type)},
-                timeout=httpx.Timeout(self.timeout_seconds, connect=30),
+            word_payload = dict(data)
+            word_payload["timestamp_granularities[]"] = "word"
+            response = self._post_once(
+                word_payload, audio_file, filename, content_type
             )
+            if response.status_code in (400, 422):
+                audio_file.seek(0)
+                response = self._post_once(data, audio_file, filename, content_type)
             response.raise_for_status()
             return response.json()
         except (httpx.HTTPError, ValueError) as exc:
             raise RuntimeError(f"MOSS transcription request failed: {exc}") from exc
 
+    def _post_once(self, data: dict, audio_file, filename: str, content_type: str):
+        return httpx.post(
+            self.endpoint,
+            data=data,
+            files={"file": (filename, audio_file, content_type)},
+            timeout=httpx.Timeout(self.timeout_seconds, connect=30),
+        )
+
 
 def _parse_json_segments(
     raw_segments, compression_limit: Optional[float] = None
@@ -186,11 +215,126 @@ def _parse_json_segments(
                     end_seconds=end,
                     text=text,
                     speaker=str(speaker) if speaker is not None else None,
+                    words=_parse_word_timestamps(item.get("words")),
                 )
             )
     return segments
 
 
+def _parse_word_timestamps(raw_words) -> Optional[List[WordTimestamp]]:
+    if not isinstance(raw_words, list):
+        return None
+    words = []
+    for item in raw_words:
+        if not isinstance(item, dict):
+            continue
+        try:
+            start = float(item["start"])
+            end = float(item["end"])
+        except (KeyError, TypeError, ValueError):
+            continue
+        text = str(item.get("word") or "").strip()
+        if text and end > start:
+            words.append(WordTimestamp(start_seconds=start, end_seconds=end, text=text))
+    return words or None
+
+
+_PERIOD_BOUNDARY = re.compile(r"\.+")
+
+
+def split_segment_by_periods(
+    segment: TranscriptionSegment,
+) -> List[TranscriptionSegment]:
+    """Split one whisper segment into sentences at periods (".")."""
+    text = segment.text.strip()
+    if not text:
+        return []
+    pieces: List[str] = []
+    cursor = 0
+    for match in _PERIOD_BOUNDARY.finditer(text):
+        piece = text[cursor:match.end()].strip()
+        if piece.strip(".").strip():
+            pieces.append(piece)
+        cursor = match.end()
+    tail = text[cursor:].strip()
+    if tail:
+        pieces.append(tail)
+    if len(pieces) <= 1:
+        return [segment] if pieces else []
+    ends = _sentence_end_seconds(segment, pieces)
+    sub_segments: List[TranscriptionSegment] = []
+    start = segment.start_seconds
+    for piece, end in zip(pieces, ends):
+        if end > start:
+            sub_segments.append(
+                TranscriptionSegment(
+                    start_seconds=start,
+                    end_seconds=end,
+                    text=piece,
+                    speaker=segment.speaker,
+                )
+            )
+        start = end
+    return sub_segments
+
+
+def _sentence_end_seconds(
+    segment: TranscriptionSegment, pieces: List[str]
+) -> List[float]:
+    lengths = [len(piece) for piece in pieces]
+    total = sum(lengths)
+    if segment.words and len(segment.words) >= len(pieces):
+        word_ends = _word_boundary_ends(segment, lengths)
+        if _valid_boundaries(segment, word_ends):
+            return word_ends + [segment.end_seconds]
+    return _proportional_ends(segment, lengths) + [segment.end_seconds]
+
+
+def _word_boundary_ends(
+    segment: TranscriptionSegment, lengths: List[int]
+) -> List[float]:
+    targets = [sum(lengths[:count]) for count in range(1, len(lengths))]
+    buffer = ""
+    word_indexes: List[int] = []
+    target_index = 0
+    for word_index, word in enumerate(segment.words):
+        buffer += (" " if buffer else "") + word.text
+        while target_index < len(targets) and len(buffer) >= targets[target_index]:
+            word_indexes.append(word_index)
+            target_index += 1
+    if len(word_indexes) != len(targets):
+        return []
+    return [segment.words[index].end_seconds for index in word_indexes]
+
+
+def _valid_boundaries(
+    segment: TranscriptionSegment, ends: List[float]
+) -> bool:
+    previous = segment.start_seconds
+    for end in ends:
+        if not segment.start_seconds < end < segment.end_seconds:
+            return False
+        if end <= previous:
+            return False
+        previous = end
+    return True
+
+
+def _proportional_ends(
+    segment: TranscriptionSegment, lengths: List[int]
+) -> List[float]:
+    total = sum(lengths)
+    ends = []
+    cumulative = 0
+    for length in lengths[:-1]:
+        cumulative += length
+        ends.append(
+            segment.start_seconds
+            + (cumulative / total) * (segment.end_seconds - segment.start_seconds)
+        )
+    return ends
+
+
 def _wav_duration_seconds(path: Path) -> Optional[float]:
     try:
         with wave.open(str(path), "rb") as wav: