add Moss result to db

sentence_api/processing.py

@@ -4,7 +4,7 @@ import subprocess
 import tempfile
 import uuid
 from pathlib import Path
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
 
 from .audio_metrics import (
     AudioAnalysisError,
@@ -124,7 +124,11 @@ class VideoProcessor:
                 )
                 if not document.sentences:
                     raise RuntimeError("MOSS returned no timestamped speech segments.")
-                self._finish_processing(document, transcript.text)
+                self._finish_processing(
+                    document,
+                    transcript.text,
+                    moss_segments=_serialize_moss_segments(transcript),
+                )
             else:
                 entry, _ = make_entry(media_path, video_hash=video_hash)
                 document = SentenceBoundaryDocument(
@@ -145,13 +149,18 @@ class VideoProcessor:
         self,
         document: SentenceBoundaryDocument,
         transcription: Optional[str],
+        moss_segments: Optional[List[Dict[str, Any]]] = None,
     ) -> None:
         if self.oss_uploader is None or not self.oss_uploader.enabled:
-            self.repository.save_processing_result(document, transcription, ready=True)
+            self.repository.save_processing_result(
+                document, transcription, ready=True, moss_segments=moss_segments
+            )
             return
 
         # Keep the course invisible while the mandatory OSS upload is running.
-        self.repository.save_processing_result(document, transcription, ready=False)
+        self.repository.save_processing_result(
+            document, transcription, ready=False, moss_segments=moss_segments
+        )
         video = self.repository.get_video(document.video_hash)
         if video is None:
             raise ValueError(f"Unknown video: {document.video_hash}")
@@ -539,3 +548,32 @@ def _media_duration_ms(path: Path) -> int:
     from .generate_boundaries import media_duration_ms
 
     return media_duration_ms(path)
+
+
+def _serialize_moss_segments(transcript: Transcript) -> List[Dict[str, Any]]:
+    rows: List[Dict[str, Any]] = []
+    for segment in transcript.segments:
+        text = segment.text.strip()
+        if not text:
+            continue
+        words = None
+        if segment.words:
+            words = [
+                {
+                    "start_seconds": word.start_seconds,
+                    "end_seconds": word.end_seconds,
+                    "text": word.text,
+                }
+                for word in segment.words
+            ]
+        rows.append(
+            {
+                "start_seconds": segment.start_seconds,
+                "end_seconds": segment.end_seconds,
+                "text": text,
+                "speaker": segment.speaker,
+                "words": words,
+            }
+        )
+    return rows
+

sentence_api/repository.py

@@ -120,6 +120,20 @@ class VideoRepository:
                     FOREIGN KEY (share_id) REFERENCES dub_shares(share_id) ON DELETE CASCADE,
                     UNIQUE (share_id, sentence_index)
                 );
+
+                CREATE TABLE IF NOT EXISTS moss_segments (
+                    video_hash TEXT NOT NULL,
+                    segment_index INTEGER NOT NULL,
+                    start_seconds REAL NOT NULL,
+                    end_seconds REAL NOT NULL,
+                    start_ms INTEGER NOT NULL,
+                    end_ms INTEGER NOT NULL,
+                    text TEXT NOT NULL,
+                    speaker TEXT,
+                    words_json TEXT,
+                    PRIMARY KEY (video_hash, segment_index),
+                    FOREIGN KEY (video_hash) REFERENCES videos(video_hash) ON DELETE CASCADE
+                );
                 """
             )
 
@@ -247,12 +261,17 @@ class VideoRepository:
         document: SentenceBoundaryDocument,
         transcription: Optional[str],
         ready: bool = True,
+        moss_segments: Optional[List[Dict[str, Any]]] = None,
     ) -> None:
         with self._connect() as connection:
             connection.execute(
                 "DELETE FROM sentences WHERE video_hash = ?",
                 (document.video_hash,),
             )
+            connection.execute(
+                "DELETE FROM moss_segments WHERE video_hash = ?",
+                (document.video_hash,),
+            )
             connection.executemany(
                 """
                 INSERT INTO sentences (
@@ -288,6 +307,19 @@ class VideoRepository:
                     document.video_hash,
                 ),
             )
+            if moss_segments:
+                connection.executemany(
+                    """
+                    INSERT INTO moss_segments (
+                        video_hash, segment_index, start_seconds, end_seconds,
+                        start_ms, end_ms, text, speaker, words_json
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    """,
+                    [
+                        _moss_segment_row(document.video_hash, index, segment)
+                        for index, segment in enumerate(moss_segments)
+                    ],
+                )
 
     def mark_oss_uploaded(self, video_hash: str, remote_url: str) -> None:
         with self._connect() as connection:
@@ -327,6 +359,32 @@ class VideoRepository:
             ).fetchone()
         return dict(row) if row else None
 
+    def get_moss_segments(self, video_hash: str) -> List[Dict[str, Any]]:
+        with self._connect() as connection:
+            rows = connection.execute(
+                """
+                SELECT * FROM moss_segments
+                WHERE video_hash = ?
+                ORDER BY segment_index
+                """,
+                (normalize_video_hash(video_hash),),
+            ).fetchall()
+        segments = []
+        for row in rows:
+            item = dict(row)
+            raw_words = item.pop("words_json", None)
+            words = None
+            if raw_words:
+                try:
+                    parsed = json.loads(raw_words)
+                except json.JSONDecodeError:
+                    parsed = None
+                if isinstance(parsed, list):
+                    words = parsed
+            item["words"] = words
+            segments.append(item)
+        return segments
+
     def get_document(self, video_hash: str) -> Optional[SentenceBoundaryDocument]:
         video = self.get_video(video_hash)
         if video is None or video["duration_ms"] is None:
@@ -1221,3 +1279,22 @@ def _is_cjk_char(char: str) -> bool:
         or 0xF900 <= code <= 0xFAFF
         or 0x20000 <= code <= 0x2CEAF
     )
+
+
+def _moss_segment_row(video_hash: str, index: int, segment: Dict[str, Any]) -> tuple:
+    start_seconds = float(segment["start_seconds"])
+    end_seconds = float(segment["end_seconds"])
+    words = segment.get("words")
+    words_json = json.dumps(words, ensure_ascii=False) if words else None
+    speaker = segment.get("speaker")
+    return (
+        video_hash,
+        index,
+        start_seconds,
+        end_seconds,
+        int(round(start_seconds * 1000)),
+        int(round(end_seconds * 1000)),
+        str(segment.get("text") or "").strip(),
+        str(speaker) if speaker is not None else None,
+        words_json,
+    )

sentence_api/tests/test_processing.py

@@ -2,10 +2,12 @@ import numpy as np
 
 from sentence_api.processing import (
     MOSS_ALGORITHM_VERSION,
+    VideoProcessor,
     document_from_transcript,
     split_sentence_text,
 )
-from sentence_api.models import SentenceBoundary
+from sentence_api.models import SentenceBoundary, SentenceBoundaryDocument
+from sentence_api.repository import VideoRepository
 from sentence_api.transcription import (
     Transcript,
     TranscriptionSegment,
@@ -158,3 +160,166 @@ def test_split_sentence_text_uses_whisper_word_timestamps(tmp_path):
         ("Alpha beta", 1000, 3000),
         ("Gamma delta", 3000, 5000),
     ]
+
+
+def test_processor_persists_original_moss_segments(tmp_path, monkeypatch):
+    from sentence_api.config import Settings
+    from dataclasses import replace
+
+    settings = replace(Settings.from_env(), data_dir=tmp_path / "data")
+    settings.ensure_directories()
+    media_path = settings.videos_dir / f"{VIDEO_HASH}.mp4"
+    media_path.write_bytes(b"video")
+    repository = VideoRepository(settings.database_path)
+    repository.upsert_upload(
+        video_hash=VIDEO_HASH,
+        title="MOSS lesson",
+        filename="lesson.mp4",
+        stored_filename=f"{VIDEO_HASH}.mp4",
+        content_type="video/mp4",
+        size_bytes=5,
+        language="en",
+    )
+    transcript = Transcript(
+        text="Hello world. Next.",
+        segments=[
+            TranscriptionSegment(
+                start_seconds=0.0,
+                end_seconds=1.2,
+                text="Hello world.",
+                speaker="S01",
+                words=[WordTimestamp(0.0, 0.5, "Hello"), WordTimestamp(0.5, 1.2, "world.")],
+            ),
+            TranscriptionSegment(
+                start_seconds=1.2,
+                end_seconds=2.4,
+                text="Next.",
+                speaker="S01",
+            ),
+        ],
+    )
+
+    class AvailableTranscriber:
+        available = True
+
+        def transcribe(self, audio_path, language=None):
+            return transcript
+
+    monkeypatch.setattr("sentence_api.processing._ensure_webm_cues_front", lambda path: None)
+    monkeypatch.setattr(
+        "sentence_api.processing.extract_audio",
+        lambda src, dst: dst.write_bytes(b"RIFF"),
+    )
+    monkeypatch.setattr("sentence_api.processing._media_duration_ms", lambda path: 2400)
+    monkeypatch.setattr(
+        "sentence_api.processing.document_from_transcript",
+        lambda **kwargs: SentenceBoundaryDocument(
+            video_hash=kwargs["video_hash"],
+            duration_ms=2400,
+            algorithm_version=MOSS_ALGORITHM_VERSION,
+            sentences=[
+                SentenceBoundary(
+                    index=0,
+                    start_ms=0,
+                    end_ms=1200,
+                    text="Hello world.",
+                    language="en",
+                    reference_speech_duration_ms=1100,
+                ),
+                SentenceBoundary(
+                    index=1,
+                    start_ms=1200,
+                    end_ms=2400,
+                    text="Next.",
+                    language="en",
+                    reference_speech_duration_ms=1100,
+                ),
+            ],
+        ),
+    )
+
+    VideoProcessor(settings, repository, AvailableTranscriber()).process(VIDEO_HASH)
+
+    segments = repository.get_moss_segments(VIDEO_HASH)
+    assert [(item["start_ms"], item["end_ms"], item["text"], item["speaker"]) for item in segments] == [
+        (0, 1200, "Hello world.", "S01"),
+        (1200, 2400, "Next.", "S01"),
+    ]
+    assert segments[0]["words"] == [
+        {"start_seconds": 0.0, "end_seconds": 0.5, "text": "Hello"},
+        {"start_seconds": 0.5, "end_seconds": 1.2, "text": "world."},
+    ]
+    assert segments[1]["words"] is None
+
+
+def test_reprocess_without_moss_clears_original_segments(tmp_path, monkeypatch):
+    from sentence_api.config import Settings
+    from dataclasses import replace
+
+    settings = replace(Settings.from_env(), data_dir=tmp_path / "data")
+    settings.ensure_directories()
+    media_path = settings.videos_dir / f"{VIDEO_HASH}.mp4"
+    media_path.write_bytes(b"video")
+    repository = VideoRepository(settings.database_path)
+    repository.upsert_upload(
+        video_hash=VIDEO_HASH,
+        title="MOSS lesson",
+        filename="lesson.mp4",
+        stored_filename=f"{VIDEO_HASH}.mp4",
+        content_type="video/mp4",
+        size_bytes=5,
+        language="en",
+    )
+    repository.save_processing_result(
+        SentenceBoundaryDocument(
+            video_hash=VIDEO_HASH,
+            duration_ms=2400,
+            algorithm_version=MOSS_ALGORITHM_VERSION,
+            sentences=[
+                SentenceBoundary(
+                    index=0,
+                    start_ms=0,
+                    end_ms=2400,
+                    text="Hello.",
+                    language="en",
+                    reference_speech_duration_ms=2000,
+                )
+            ],
+        ),
+        "Hello.",
+        moss_segments=[
+            {
+                "start_seconds": 0.0,
+                "end_seconds": 2.4,
+                "text": "Hello.",
+                "speaker": "S01",
+            }
+        ],
+    )
+    assert repository.get_moss_segments(VIDEO_HASH)
+
+    class UnavailableTranscriber:
+        available = False
+
+    monkeypatch.setattr("sentence_api.processing._ensure_webm_cues_front", lambda path: None)
+    monkeypatch.setattr(
+        "sentence_api.processing.make_entry",
+        lambda *args, **kwargs: (
+            {
+                "duration_ms": 2400,
+                "sentences": [
+                    SentenceBoundary(
+                        index=0,
+                        start_ms=0,
+                        end_ms=2400,
+                        text=None,
+                        language="en",
+                    )
+                ],
+            },
+            {},
+        ),
+    )
+    VideoProcessor(settings, repository, UnavailableTranscriber()).process(VIDEO_HASH)
+    assert repository.get_moss_segments(VIDEO_HASH) == []
+