diff --git a/sentence_api/DEPLOYMENT.md b/sentence_api/DEPLOYMENT.md index f284cd1..3dd2eae 100644 --- a/sentence_api/DEPLOYMENT.md +++ b/sentence_api/DEPLOYMENT.md @@ -153,8 +153,11 @@ curl -X POST http://127.0.0.1:8001/v1/audio/transcriptions \ ### 断句规则 `sentence_api` 不以 Whisper 返回的 segment 直接作为句子：文本会跨 segment 累积， -只有碰到句号（`.`）或问号（`?`）才算一句结束；Whisper 在没有标点处断开的 -segment 会自动合并成同一句。若转写服务支持词级时间戳 +只有碰到句号（`.`）、问号（`?`）或逗号前的单词数达到 50 时才算一句结束； +逗号前内容不足 50 个单词时不在此处断句，会继续与后续内容合并。Whisper 在没有 +标点处断开的 segment 会自动合并成同一句。管理后台编辑句子文本时，换行并保存会把 +该句拆成多句；拆分点优先使用对该句音频再次转写得到的词级时间戳，转写不可用时按 +文本长度估算。若转写服务支持词级时间戳 （`timestamp_granularities[]=word`，Whisper/Speaches 支持），句子的结束时间用 标点所在单词的时间戳；不支持时自动按文字长度在 segment 内估算，请求失败会回退为 不带词级时间戳的普通请求。MOSS 不支持该参数时同样自动回退，不影响转写。 diff --git a/sentence_api/README.md b/sentence_api/README.md index 424a0d7..f103202 100644 --- a/sentence_api/README.md +++ b/sentence_api/README.md @@ -56,6 +56,7 @@ POST /api/v1/admin/videos PUT /api/v1/admin/videos/raw POST /api/v1/admin/videos/{sha256}/process PUT /api/v1/admin/videos/{sha256}/sentences/{index} +PUT /api/v1/admin/videos/{sha256}/sentences/{index}/split DELETE /api/v1/admin/videos/{sha256} ``` diff --git a/sentence_api/main.py b/sentence_api/main.py index e6d78c6..fd70f02 100644 --- a/sentence_api/main.py +++ b/sentence_api/main.py @@ -41,7 +41,7 @@ from .models import ( VideoUploadResponse, ) from .oss import VolcanoOSSUploader -from .processing import VideoProcessor +from .processing import VideoProcessor, split_sentence_text from .repository import VideoRepository from .store import BoundaryStore from .transcription import MossTranscriber, Transcriber @@ -328,6 +328,41 @@ def create_app( raise HTTPException(status_code=404, detail="Sentence was not found.") return sentence + @application.put( + "/api/v1/admin/videos/{video_hash}/sentences/{sentence_index}/split", + response_model=List[SentenceBoundary], + dependencies=[Depends(require_admin)], + ) + def split_sentence( + payload: SentenceTextUpdate, + video_hash: str = SHA256_PATH, + sentence_index: int = ApiPath(ge=0), + ) -> List[SentenceBoundary]: + video = video_repository.get_video(video_hash) + sentence = video_repository.get_sentence(video_hash, sentence_index) + if video is None or sentence is None: + raise HTTPException(status_code=404, detail="Sentence was not found.") + + media_path = service_settings.videos_dir / video["stored_filename"] + replacements = split_sentence_text( + transcriber=moss, + media_path=media_path, + sentence=sentence, + text=payload.text, + language=payload.language if payload.language is not None else sentence.language, + ) + try: + result = video_repository.replace_sentence( + video_hash, + sentence_index, + replacements, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + if result is None: + raise HTTPException(status_code=404, detail="Sentence was not found.") + return result + @application.delete( "/api/v1/admin/videos/{video_hash}", status_code=204, diff --git a/sentence_api/processing.py b/sentence_api/processing.py index ec2ae7c..02a30c4 100644 --- a/sentence_api/processing.py +++ b/sentence_api/processing.py @@ -1,5 +1,7 @@ +import logging import shutil import subprocess +import tempfile import uuid from pathlib import Path from typing import List, Optional @@ -16,9 +18,14 @@ from .generate_boundaries import ALGORITHM_VERSION, make_entry from .models import SentenceBoundary, SentenceBoundaryDocument from .oss import VolcanoOSSUploader from .repository import VideoRepository -from .transcription import Transcript, Transcriber, split_sentences_at_punctuation +from .transcription import ( + Transcript, + Transcriber, + split_sentences_at_punctuation, +) +logger = logging.getLogger(__name__) MOSS_ALGORITHM_VERSION = "moss-punctuation-v3" @@ -128,6 +135,223 @@ def extract_audio(media_path: Path, output_path: Path) -> None: raise RuntimeError(f"Could not extract video audio: {message[-2000:]}") +def extract_audio_segment( + media_path: Path, + output_path: Path, + *, + start_ms: int, + duration_ms: int, +) -> None: + """Extract one mono WAV segment for a focused Whisper request.""" + ffmpeg = shutil.which("ffmpeg") + if ffmpeg is None: + raise RuntimeError("ffmpeg is required for MOSS transcription but was not found.") + output_path.parent.mkdir(parents=True, exist_ok=True) + command = [ + ffmpeg, + "-hide_banner", + "-loglevel", + "error", + "-y", + "-ss", + f"{max(0, start_ms) / 1000:.3f}", + "-t", + f"{max(1, duration_ms) / 1000:.3f}", + "-i", + str(media_path), + "-vn", + "-ac", + "1", + "-ar", + "16000", + "-c:a", + "pcm_s16le", + str(output_path), + ] + completed = subprocess.run(command, capture_output=True, text=True, timeout=1800) + if completed.returncode != 0: + message = completed.stderr.strip() or "unknown ffmpeg error" + raise RuntimeError(f"Could not extract sentence audio: {message[-2000:]}") + + +def split_sentence_text( + *, + transcriber: Transcriber, + media_path: Path, + sentence: SentenceBoundary, + text: str, + language: Optional[str], +) -> List[SentenceBoundary]: + """Split one stored sentence at explicit line breaks. + + When MOSS/Whisper is available, the sentence's own audio is transcribed and + its word timestamps select the new boundary. If transcription is unavailable + or fails, editing remains possible and boundaries fall back to text-length + proportions so the admin UI does not lose the user's manual edit. + """ + lines = [line.strip() for line in text.splitlines() if line.strip()] + if len(lines) < 2: + return [ + sentence.model_copy( + update={ + "text": text.strip(), + "language": language if language is not None else sentence.language, + } + ) + ] + + transcript: Optional[Transcript] = None + if transcriber.available and media_path.is_file(): + try: + with tempfile.TemporaryDirectory(prefix="sentence-split-") as temp_dir: + audio_path = Path(temp_dir) / "sentence.wav" + extract_audio_segment( + media_path, + audio_path, + start_ms=sentence.start_ms, + duration_ms=sentence.end_ms - sentence.start_ms, + ) + transcript = transcriber.transcribe(audio_path, language) + except Exception as exc: + logger.warning("Whisper split-timing failed for sentence %s: %s", sentence.index, exc) + + boundaries = _estimate_split_boundaries_ms(sentence, lines, transcript) + weights = [_text_weight(line) for line in lines] + total_weight = max(1, sum(weights)) + replacements: List[SentenceBoundary] = [] + for boundary, line, weight in zip(boundaries, lines, weights): + reference_duration = sentence.reference_speech_duration_ms or ( + boundary.end_ms - boundary.start_ms + ) + replacements.append( + boundary.model_copy( + update={ + "text": line, + "language": language if language is not None else sentence.language, + "reference_speech_duration_ms": max( + 1, + int(round(reference_duration * weight / total_weight)), + ), + } + ) + ) + return replacements + + +def _estimate_split_boundaries_ms( + sentence: SentenceBoundary, + lines: List[str], + transcript: Optional[Transcript], +) -> List[SentenceBoundary]: + estimates = _whisper_split_boundaries_ms(sentence, lines, transcript) + if len(estimates) != len(lines) - 1: + estimates = _proportional_split_boundaries_ms(sentence, lines) + + # Keep every split strictly inside the source sentence, even if a Whisper + # timestamp is slightly outside or two estimates collapse to one point. + previous = sentence.start_ms + remaining = len(lines) + for index, estimate in enumerate(list(estimates)): + low = previous + 1 + high = sentence.end_ms - (remaining - 1) + if low >= high: + estimates = _proportional_split_boundaries_ms(sentence, lines) + break + estimate = min(max(estimate, low), high) + estimates[index] = estimate + previous = estimate + remaining -= 1 + + starts = [sentence.start_ms, *estimates] + ends = [*estimates, sentence.end_ms] + return [ + SentenceBoundary( + index=index, + start_ms=start, + end_ms=end, + text=lines[index], + language=sentence.language, + reference_speech_duration_ms=max(1, end - start), + ) + for index, (start, end) in enumerate(zip(starts, ends)) + ] + + +def _whisper_split_boundaries_ms( + sentence: SentenceBoundary, + lines: List[str], + transcript: Optional[Transcript], +) -> List[int]: + if transcript is None or not transcript.segments: + return [] + + source_ratios = _cumulative_ratios([_text_weight(line) for line in lines]) + words = [word for segment in transcript.segments for word in (segment.words or [])] + if len(words) >= 2: + span_start = words[0].start_seconds + span_end = max(word.end_seconds for word in words) + if span_end <= span_start: + return [] + total_weight = sum(max(1, len(word.text.strip())) for word in words) + cumulative = 0 + samples: List[tuple[float, int]] = [] + for word in words[:-1]: + cumulative += max(1, len(word.text.strip())) + timestamp_ms = sentence.start_ms + int( + round( + (word.end_seconds - span_start) + / (span_end - span_start) + * (sentence.end_ms - sentence.start_ms) + ) + ) + samples.append((cumulative / total_weight, timestamp_ms)) + return [ + min(samples, key=lambda item: (abs(item[0] - ratio), item[1]))[1] + for ratio in source_ratios + ] + + # Some MOSS configurations omit word timestamps. Segment ends are still + # much better than blind proportional allocation when they exist. + if len(transcript.segments) < 2: + return [] + total_weight = sum(max(1, len(segment.text.strip())) for segment in transcript.segments) + cumulative = 0 + samples: List[tuple[float, int]] = [] + for segment in transcript.segments[:-1]: + cumulative += max(1, len(segment.text.strip())) + timestamp_ms = sentence.start_ms + int(round(segment.end_seconds * 1000)) + samples.append((cumulative / total_weight, timestamp_ms)) + return [ + min(samples, key=lambda item: (abs(item[0] - ratio), item[1]))[1] + for ratio in source_ratios + ] + + +def _proportional_split_boundaries_ms( + sentence: SentenceBoundary, + lines: List[str], +) -> List[int]: + duration = sentence.end_ms - sentence.start_ms + return [ + sentence.start_ms + int(round(ratio * duration)) + for ratio in _cumulative_ratios([_text_weight(line) for line in lines]) + ] + + +def _cumulative_ratios(weights: List[int]) -> List[float]: + total = max(1, sum(weights)) + cumulative = 0 + ratios = [] + for weight in weights[:-1]: + cumulative += weight + ratios.append(cumulative / total) + return ratios + + +def _text_weight(text: str) -> int: + return max(1, sum(1 for char in text if not char.isspace())) + + def document_from_transcript( *, video_hash: str, diff --git a/sentence_api/repository.py b/sentence_api/repository.py index 85dc137..a9d3eb7 100644 --- a/sentence_api/repository.py +++ b/sentence_api/repository.py @@ -331,6 +331,166 @@ class VideoRepository: reference_speech_duration_ms=row["reference_speech_duration_ms"], ) + def get_sentence( + self, + video_hash: str, + sentence_index: int, + ) -> Optional[SentenceBoundary]: + normalized_hash = normalize_video_hash(video_hash) + with self._connect() as connection: + row = connection.execute( + """ + SELECT * FROM sentences + WHERE video_hash = ? AND sentence_index = ? + """, + (normalized_hash, sentence_index), + ).fetchone() + if row is None: + return None + return SentenceBoundary( + index=row["sentence_index"], + start_ms=row["start_ms"], + end_ms=row["end_ms"], + text=row["text"], + language=row["language"], + reference_speech_duration_ms=row["reference_speech_duration_ms"], + ) + + def replace_sentence( + self, + video_hash: str, + sentence_index: int, + replacements: List[SentenceBoundary], + ) -> Optional[List[SentenceBoundary]]: + """Replace one sentence row with one or more contiguous sentence rows.""" + normalized_hash = normalize_video_hash(video_hash) + if not replacements: + raise ValueError("At least one replacement sentence is required.") + + with self._connect() as connection: + source = connection.execute( + """ + SELECT * FROM sentences + WHERE video_hash = ? AND sentence_index = ? + """, + (normalized_hash, sentence_index), + ).fetchone() + if source is None: + return None + + rows = [ + SentenceBoundary( + index=index, + start_ms=replacement.start_ms, + end_ms=replacement.end_ms, + text=replacement.text.strip(), + language=replacement.language, + reference_speech_duration_ms=max( + 1, replacement.reference_speech_duration_ms or 1 + ), + ) + for index, replacement in enumerate(replacements) + ] + previous_end = source["start_ms"] + for replacement in rows: + if ( + replacement.start_ms < previous_end + or replacement.end_ms <= replacement.start_ms + ): + raise ValueError("Replacement sentence times must be contiguous.") + if replacement.end_ms > source["end_ms"]: + raise ValueError( + "Replacement sentences cannot exceed the source sentence." + ) + previous_end = replacement.end_ms + if rows[-1].end_ms != source["end_ms"]: + raise ValueError("Replacement sentences must cover the source sentence.") + + # Shift following rows down first. Descending indexes avoid violating + # the (video_hash, sentence_index) primary key while making room. + added_count = len(rows) - 1 + if added_count: + following = [ + row["sentence_index"] + for row in connection.execute( + """ + SELECT sentence_index FROM sentences + WHERE video_hash = ? AND sentence_index > ? + ORDER BY sentence_index DESC + """, + (normalized_hash, sentence_index), + ).fetchall() + ] + for old_index in following: + connection.execute( + """ + UPDATE sentences SET sentence_index = ? + WHERE video_hash = ? AND sentence_index = ? + """, + (old_index + added_count, normalized_hash, old_index), + ) + + first = rows[0] + connection.execute( + """ + UPDATE sentences + SET start_ms = ?, end_ms = ?, text = ?, language = ?, + reference_speech_duration_ms = ? + WHERE video_hash = ? AND sentence_index = ? + """, + ( + first.start_ms, + first.end_ms, + first.text, + first.language, + first.reference_speech_duration_ms, + normalized_hash, + sentence_index, + ), + ) + for replacement in rows[1:]: + connection.execute( + """ + INSERT INTO sentences ( + video_hash, sentence_index, start_ms, end_ms, text, + language, reference_speech_duration_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + normalized_hash, + replacement.index, + replacement.start_ms, + replacement.end_ms, + replacement.text, + replacement.language, + replacement.reference_speech_duration_ms, + ), + ) + connection.execute( + "UPDATE videos SET updated_at = ? WHERE video_hash = ?", + (utc_now(), normalized_hash), + ) + result_rows = connection.execute( + """ + SELECT * FROM sentences + WHERE video_hash = ? AND sentence_index >= ? AND sentence_index < ? + ORDER BY sentence_index + """, + (normalized_hash, sentence_index, sentence_index + len(rows)), + ).fetchall() + + return [ + SentenceBoundary( + index=row["sentence_index"], + start_ms=row["start_ms"], + end_ms=row["end_ms"], + text=row["text"], + language=row["language"], + reference_speech_duration_ms=row["reference_speech_duration_ms"], + ) + for row in result_rows + ] + def create_dub_share( self, *, diff --git a/sentence_api/static/admin.js b/sentence_api/static/admin.js index d8f222d..046a814 100644 --- a/sentence_api/static/admin.js +++ b/sentence_api/static/admin.js @@ -186,7 +186,7 @@ async function openSentences(video) { elements.sentenceMeta.textContent = `${sentences.length} 句 / ${detail.video.video_hash}`; elements.sentenceRows.replaceChildren(); for (const sentence of sentences) { - elements.sentenceRows.append(sentenceRow(video.video_hash, sentence)); + elements.sentenceRows.append(sentenceRow(video, sentence)); } elements.dialog.showModal(); } catch (error) { @@ -194,7 +194,7 @@ async function openSentences(video) { } } -function sentenceRow(videoHash, sentence) { +function sentenceRow(video, sentence) { const row = document.createElement("tr"); const textArea = document.createElement("textarea"); textArea.value = sentence.text || ""; @@ -214,12 +214,23 @@ function sentenceRow(videoHash, sentence) { const save = button("保存", async () => { save.disabled = true; try { - await api(`/api/v1/admin/videos/${videoHash}/sentences/${sentence.index}`, { - method: "PUT", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ text: textArea.value, language: language.value || null }), - }); - save.textContent = "已保存"; + const body = JSON.stringify({ text: textArea.value, language: language.value || null }); + if (/\r?\n/.test(textArea.value)) { + await api(`/api/v1/admin/videos/${video.video_hash}/sentences/${sentence.index}/split`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body, + }); + save.textContent = "已拆分"; + await openSentences(video); + } else { + await api(`/api/v1/admin/videos/${video.video_hash}/sentences/${sentence.index}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body, + }); + save.textContent = "已保存"; + } } catch (error) { save.textContent = error.message; } finally { diff --git a/sentence_api/tests/test_api.py b/sentence_api/tests/test_api.py index 2b4c3a0..e438cea 100644 --- a/sentence_api/tests/test_api.py +++ b/sentence_api/tests/test_api.py @@ -188,3 +188,74 @@ def test_processor_marks_ready_only_after_oss_upload(tmp_path, monkeypatch): assert uploads[0]["stored_filename"] == f"{VIDEO_HASH}.mp4" assert video["status"] == "ready" assert video["remote_url"] == "https://media.example.com/lesson.mp4" + + +def test_admin_can_split_sentence_and_reindexes_following_sentences(tmp_path): + settings = replace(Settings.from_env(), data_dir=tmp_path / "data") + settings.ensure_directories() + repository = VideoRepository(settings.database_path) + repository.upsert_upload( + video_hash=VIDEO_HASH, + title="Manual split lesson", + filename="lesson.mp4", + stored_filename=f"{VIDEO_HASH}.mp4", + content_type="video/mp4", + size_bytes=5, + language="en", + ) + repository.save_processing_result( + SentenceBoundaryDocument( + video_hash=VIDEO_HASH, + duration_ms=9000, + algorithm_version="test", + sentences=[ + SentenceBoundary( + index=0, + start_ms=0, + end_ms=4000, + text="First part second part", + language="en", + reference_speech_duration_ms=3600, + ), + SentenceBoundary( + index=1, + start_ms=4000, + end_ms=9000, + text="Next sentence.", + language="en", + reference_speech_duration_ms=4600, + ), + ], + ), + None, + ) + + class UnavailableTranscriber: + available = False + + client = TestClient( + create_app( + BoundaryStore(tmp_path / "boundaries.json"), + settings=settings, + repository=repository, + transcriber=UnavailableTranscriber(), + ) + ) + response = client.put( + f"/api/v1/admin/videos/{VIDEO_HASH}/sentences/0/split", + json={"text": "First part\nsecond part", "language": "en"}, + ) + + assert response.status_code == 200 + replacements = response.json() + assert [item["text"] for item in replacements] == ["First part", "second part"] + assert replacements[0]["start_ms"] == 0 + assert replacements[0]["end_ms"] == replacements[1]["start_ms"] + assert replacements[1]["end_ms"] == 4000 + document = repository.get_document(VIDEO_HASH) + assert [item.text for item in document.sentences] == [ + "First part", + "second part", + "Next sentence.", + ] + assert [item.index for item in document.sentences] == [0, 1, 2] diff --git a/sentence_api/tests/test_processing.py b/sentence_api/tests/test_processing.py index 908e8aa..ce632aa 100644 --- a/sentence_api/tests/test_processing.py +++ b/sentence_api/tests/test_processing.py @@ -1,6 +1,11 @@ import numpy as np -from sentence_api.processing import MOSS_ALGORITHM_VERSION, document_from_transcript +from sentence_api.processing import ( + MOSS_ALGORITHM_VERSION, + document_from_transcript, + split_sentence_text, +) +from sentence_api.models import SentenceBoundary from sentence_api.transcription import ( Transcript, TranscriptionSegment, @@ -13,6 +18,18 @@ SAMPLE_RATE = 16000 VIDEO_HASH = "a" * 64 +class WordTimestampTranscriber: + def __init__(self, transcript): + self.transcript = transcript + + @property + def available(self): + return True + + def transcribe(self, audio_path, language=None): + return self.transcript + + def _speech(seconds: float) -> np.ndarray: t = np.arange(int(seconds * SAMPLE_RATE)) / SAMPLE_RATE return 0.25 * np.sin(2 * np.pi * 220 * t) @@ -99,3 +116,45 @@ def test_document_from_transcript_splits_sentences_at_punctuation(tmp_path): assert document.sentences[1].start_ms == 9500 assert document.sentences[1].end_ms == 24000 assert all(sentence.reference_speech_duration_ms > 0 for sentence in document.sentences) + + +def test_split_sentence_text_uses_whisper_word_timestamps(tmp_path): + transcript = Transcript( + text="Alpha beta. Gamma delta.", + segments=[ + TranscriptionSegment( + start_seconds=0.0, + end_seconds=1.0, + text="Alpha beta. Gamma delta.", + words=[ + WordTimestamp(0.0, 0.25, "Alpha"), + WordTimestamp(0.25, 0.5, "beta."), + WordTimestamp(0.5, 0.75, "Gamma"), + WordTimestamp(0.75, 1.0, "delta."), + ], + ) + ], + ) + audio_path = tmp_path / "sentence.wav" + audio_path.write_bytes(_wav_bytes(_speech(1.0), SAMPLE_RATE)) + source = SentenceBoundary( + index=2, + start_ms=1000, + end_ms=5000, + text="Alpha beta Gamma delta", + language="en", + reference_speech_duration_ms=3800, + ) + + replacements = split_sentence_text( + transcriber=WordTimestampTranscriber(transcript), + media_path=audio_path, + sentence=source, + text="Alpha beta\nGamma delta", + language="en", + ) + + assert [(item.text, item.start_ms, item.end_ms) for item in replacements] == [ + ("Alpha beta", 1000, 3000), + ("Gamma delta", 3000, 5000), + ]