From ce8b765103e90e8534d55fe8bb2e54ce049b96e5 Mon Sep 17 00:00:00 2001 From: Shuming Liu Date: Sat, 12 Sep 2026 20:50:54 +0800 Subject: [PATCH] add Moss result to db --- sentence_api/processing.py | 46 ++++++- sentence_api/repository.py | 77 ++++++++++++ sentence_api/tests/test_processing.py | 167 +++++++++++++++++++++++++- 3 files changed, 285 insertions(+), 5 deletions(-) diff --git a/sentence_api/processing.py b/sentence_api/processing.py index 3fe7871..f4a717c 100644 --- a/sentence_api/processing.py +++ b/sentence_api/processing.py @@ -4,7 +4,7 @@ import subprocess import tempfile import uuid from pathlib import Path -from typing import List, Optional +from typing import Any, Dict, List, Optional from .audio_metrics import ( AudioAnalysisError, @@ -124,7 +124,11 @@ class VideoProcessor: ) if not document.sentences: raise RuntimeError("MOSS returned no timestamped speech segments.") - self._finish_processing(document, transcript.text) + self._finish_processing( + document, + transcript.text, + moss_segments=_serialize_moss_segments(transcript), + ) else: entry, _ = make_entry(media_path, video_hash=video_hash) document = SentenceBoundaryDocument( @@ -145,13 +149,18 @@ class VideoProcessor: self, document: SentenceBoundaryDocument, transcription: Optional[str], + moss_segments: Optional[List[Dict[str, Any]]] = None, ) -> None: if self.oss_uploader is None or not self.oss_uploader.enabled: - self.repository.save_processing_result(document, transcription, ready=True) + self.repository.save_processing_result( + document, transcription, ready=True, moss_segments=moss_segments + ) return # Keep the course invisible while the mandatory OSS upload is running. - self.repository.save_processing_result(document, transcription, ready=False) + self.repository.save_processing_result( + document, transcription, ready=False, moss_segments=moss_segments + ) video = self.repository.get_video(document.video_hash) if video is None: raise ValueError(f"Unknown video: {document.video_hash}") @@ -539,3 +548,32 @@ def _media_duration_ms(path: Path) -> int: from .generate_boundaries import media_duration_ms return media_duration_ms(path) + + +def _serialize_moss_segments(transcript: Transcript) -> List[Dict[str, Any]]: + rows: List[Dict[str, Any]] = [] + for segment in transcript.segments: + text = segment.text.strip() + if not text: + continue + words = None + if segment.words: + words = [ + { + "start_seconds": word.start_seconds, + "end_seconds": word.end_seconds, + "text": word.text, + } + for word in segment.words + ] + rows.append( + { + "start_seconds": segment.start_seconds, + "end_seconds": segment.end_seconds, + "text": text, + "speaker": segment.speaker, + "words": words, + } + ) + return rows + diff --git a/sentence_api/repository.py b/sentence_api/repository.py index 8bcda5f..bf46da3 100644 --- a/sentence_api/repository.py +++ b/sentence_api/repository.py @@ -120,6 +120,20 @@ class VideoRepository: FOREIGN KEY (share_id) REFERENCES dub_shares(share_id) ON DELETE CASCADE, UNIQUE (share_id, sentence_index) ); + + CREATE TABLE IF NOT EXISTS moss_segments ( + video_hash TEXT NOT NULL, + segment_index INTEGER NOT NULL, + start_seconds REAL NOT NULL, + end_seconds REAL NOT NULL, + start_ms INTEGER NOT NULL, + end_ms INTEGER NOT NULL, + text TEXT NOT NULL, + speaker TEXT, + words_json TEXT, + PRIMARY KEY (video_hash, segment_index), + FOREIGN KEY (video_hash) REFERENCES videos(video_hash) ON DELETE CASCADE + ); """ ) @@ -247,12 +261,17 @@ class VideoRepository: document: SentenceBoundaryDocument, transcription: Optional[str], ready: bool = True, + moss_segments: Optional[List[Dict[str, Any]]] = None, ) -> None: with self._connect() as connection: connection.execute( "DELETE FROM sentences WHERE video_hash = ?", (document.video_hash,), ) + connection.execute( + "DELETE FROM moss_segments WHERE video_hash = ?", + (document.video_hash,), + ) connection.executemany( """ INSERT INTO sentences ( @@ -288,6 +307,19 @@ class VideoRepository: document.video_hash, ), ) + if moss_segments: + connection.executemany( + """ + INSERT INTO moss_segments ( + video_hash, segment_index, start_seconds, end_seconds, + start_ms, end_ms, text, speaker, words_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + [ + _moss_segment_row(document.video_hash, index, segment) + for index, segment in enumerate(moss_segments) + ], + ) def mark_oss_uploaded(self, video_hash: str, remote_url: str) -> None: with self._connect() as connection: @@ -327,6 +359,32 @@ class VideoRepository: ).fetchone() return dict(row) if row else None + def get_moss_segments(self, video_hash: str) -> List[Dict[str, Any]]: + with self._connect() as connection: + rows = connection.execute( + """ + SELECT * FROM moss_segments + WHERE video_hash = ? + ORDER BY segment_index + """, + (normalize_video_hash(video_hash),), + ).fetchall() + segments = [] + for row in rows: + item = dict(row) + raw_words = item.pop("words_json", None) + words = None + if raw_words: + try: + parsed = json.loads(raw_words) + except json.JSONDecodeError: + parsed = None + if isinstance(parsed, list): + words = parsed + item["words"] = words + segments.append(item) + return segments + def get_document(self, video_hash: str) -> Optional[SentenceBoundaryDocument]: video = self.get_video(video_hash) if video is None or video["duration_ms"] is None: @@ -1221,3 +1279,22 @@ def _is_cjk_char(char: str) -> bool: or 0xF900 <= code <= 0xFAFF or 0x20000 <= code <= 0x2CEAF ) + + +def _moss_segment_row(video_hash: str, index: int, segment: Dict[str, Any]) -> tuple: + start_seconds = float(segment["start_seconds"]) + end_seconds = float(segment["end_seconds"]) + words = segment.get("words") + words_json = json.dumps(words, ensure_ascii=False) if words else None + speaker = segment.get("speaker") + return ( + video_hash, + index, + start_seconds, + end_seconds, + int(round(start_seconds * 1000)), + int(round(end_seconds * 1000)), + str(segment.get("text") or "").strip(), + str(speaker) if speaker is not None else None, + words_json, + ) diff --git a/sentence_api/tests/test_processing.py b/sentence_api/tests/test_processing.py index ce632aa..1b6d616 100644 --- a/sentence_api/tests/test_processing.py +++ b/sentence_api/tests/test_processing.py @@ -2,10 +2,12 @@ import numpy as np from sentence_api.processing import ( MOSS_ALGORITHM_VERSION, + VideoProcessor, document_from_transcript, split_sentence_text, ) -from sentence_api.models import SentenceBoundary +from sentence_api.models import SentenceBoundary, SentenceBoundaryDocument +from sentence_api.repository import VideoRepository from sentence_api.transcription import ( Transcript, TranscriptionSegment, @@ -158,3 +160,166 @@ def test_split_sentence_text_uses_whisper_word_timestamps(tmp_path): ("Alpha beta", 1000, 3000), ("Gamma delta", 3000, 5000), ] + + +def test_processor_persists_original_moss_segments(tmp_path, monkeypatch): + from sentence_api.config import Settings + from dataclasses import replace + + settings = replace(Settings.from_env(), data_dir=tmp_path / "data") + settings.ensure_directories() + media_path = settings.videos_dir / f"{VIDEO_HASH}.mp4" + media_path.write_bytes(b"video") + repository = VideoRepository(settings.database_path) + repository.upsert_upload( + video_hash=VIDEO_HASH, + title="MOSS lesson", + filename="lesson.mp4", + stored_filename=f"{VIDEO_HASH}.mp4", + content_type="video/mp4", + size_bytes=5, + language="en", + ) + transcript = Transcript( + text="Hello world. Next.", + segments=[ + TranscriptionSegment( + start_seconds=0.0, + end_seconds=1.2, + text="Hello world.", + speaker="S01", + words=[WordTimestamp(0.0, 0.5, "Hello"), WordTimestamp(0.5, 1.2, "world.")], + ), + TranscriptionSegment( + start_seconds=1.2, + end_seconds=2.4, + text="Next.", + speaker="S01", + ), + ], + ) + + class AvailableTranscriber: + available = True + + def transcribe(self, audio_path, language=None): + return transcript + + monkeypatch.setattr("sentence_api.processing._ensure_webm_cues_front", lambda path: None) + monkeypatch.setattr( + "sentence_api.processing.extract_audio", + lambda src, dst: dst.write_bytes(b"RIFF"), + ) + monkeypatch.setattr("sentence_api.processing._media_duration_ms", lambda path: 2400) + monkeypatch.setattr( + "sentence_api.processing.document_from_transcript", + lambda **kwargs: SentenceBoundaryDocument( + video_hash=kwargs["video_hash"], + duration_ms=2400, + algorithm_version=MOSS_ALGORITHM_VERSION, + sentences=[ + SentenceBoundary( + index=0, + start_ms=0, + end_ms=1200, + text="Hello world.", + language="en", + reference_speech_duration_ms=1100, + ), + SentenceBoundary( + index=1, + start_ms=1200, + end_ms=2400, + text="Next.", + language="en", + reference_speech_duration_ms=1100, + ), + ], + ), + ) + + VideoProcessor(settings, repository, AvailableTranscriber()).process(VIDEO_HASH) + + segments = repository.get_moss_segments(VIDEO_HASH) + assert [(item["start_ms"], item["end_ms"], item["text"], item["speaker"]) for item in segments] == [ + (0, 1200, "Hello world.", "S01"), + (1200, 2400, "Next.", "S01"), + ] + assert segments[0]["words"] == [ + {"start_seconds": 0.0, "end_seconds": 0.5, "text": "Hello"}, + {"start_seconds": 0.5, "end_seconds": 1.2, "text": "world."}, + ] + assert segments[1]["words"] is None + + +def test_reprocess_without_moss_clears_original_segments(tmp_path, monkeypatch): + from sentence_api.config import Settings + from dataclasses import replace + + settings = replace(Settings.from_env(), data_dir=tmp_path / "data") + settings.ensure_directories() + media_path = settings.videos_dir / f"{VIDEO_HASH}.mp4" + media_path.write_bytes(b"video") + repository = VideoRepository(settings.database_path) + repository.upsert_upload( + video_hash=VIDEO_HASH, + title="MOSS lesson", + filename="lesson.mp4", + stored_filename=f"{VIDEO_HASH}.mp4", + content_type="video/mp4", + size_bytes=5, + language="en", + ) + repository.save_processing_result( + SentenceBoundaryDocument( + video_hash=VIDEO_HASH, + duration_ms=2400, + algorithm_version=MOSS_ALGORITHM_VERSION, + sentences=[ + SentenceBoundary( + index=0, + start_ms=0, + end_ms=2400, + text="Hello.", + language="en", + reference_speech_duration_ms=2000, + ) + ], + ), + "Hello.", + moss_segments=[ + { + "start_seconds": 0.0, + "end_seconds": 2.4, + "text": "Hello.", + "speaker": "S01", + } + ], + ) + assert repository.get_moss_segments(VIDEO_HASH) + + class UnavailableTranscriber: + available = False + + monkeypatch.setattr("sentence_api.processing._ensure_webm_cues_front", lambda path: None) + monkeypatch.setattr( + "sentence_api.processing.make_entry", + lambda *args, **kwargs: ( + { + "duration_ms": 2400, + "sentences": [ + SentenceBoundary( + index=0, + start_ms=0, + end_ms=2400, + text=None, + language="en", + ) + ], + }, + {}, + ), + ) + VideoProcessor(settings, repository, UnavailableTranscriber()).process(VIDEO_HASH) + assert repository.get_moss_segments(VIDEO_HASH) == [] +