1087 lines
41 KiB
Python
1087 lines
41 KiB
Python
import logging
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
|
|
from .audio_metrics import (
|
|
AudioAnalysisError,
|
|
analyze_samples,
|
|
decode_audio_mono,
|
|
FRAME_SAMPLES,
|
|
refine_sentence_end_ms,
|
|
refine_sentence_start_ms,
|
|
speech_frame_mask,
|
|
)
|
|
from .config import Settings
|
|
from .generate_boundaries import ALGORITHM_VERSION, make_entry, media_duration_ms
|
|
from sentence_analysis import detect_sentence_boundaries
|
|
from .models import SentenceBoundary, SentenceBoundaryDocument
|
|
from .oss import VolcanoOSSUploader
|
|
from .repository import VideoRepository
|
|
from .transcription import (
|
|
Transcript,
|
|
Transcriber,
|
|
split_sentences_at_punctuation,
|
|
)
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
MOSS_ALGORITHM_VERSION = "moss-punctuation-v4"
|
|
SILENCE_ALGORITHM_VERSION = "silence-rms-v1"
|
|
GAP_REPAIR_MIN_MS = 8_000
|
|
GAP_REPAIR_INSET_MS = 150
|
|
GAP_REPAIR_MIN_SPEECH_MS = 400
|
|
GAP_REPAIR_MIN_SENTENCE_MS = 100
|
|
GAP_REPAIR_TARGET_CHUNK_MS = 15_000
|
|
GAP_REPAIR_MAX_CHUNK_MS = 20_000
|
|
GAP_REPAIR_MIN_CHUNK_MS = 5_000
|
|
GAP_REPAIR_MIN_SPLIT_SILENCE_MS = 120
|
|
GAP_REPAIR_FALLBACK_LANGUAGE = "en"
|
|
_CUES_ID = b"\x1e\x53\xb6\x80"
|
|
_CLUSTER_ID = b"\x1f\x43\xb6\x75"
|
|
_HEADER_SCAN_SIZE = 64 * 1024
|
|
_WEBM_SUFFIXES = {".webm", ".mkv"}
|
|
|
|
|
|
def _webm_cues_at_front(filepath: Path) -> bool:
|
|
"""Return True if WebM/MKV CUES appear before the first Cluster."""
|
|
try:
|
|
with open(filepath, "rb") as f:
|
|
header = f.read(_HEADER_SCAN_SIZE)
|
|
except OSError:
|
|
return True # can't read → assume OK, don't touch
|
|
cues_pos = header.find(_CUES_ID)
|
|
cluster_pos = header.find(_CLUSTER_ID)
|
|
if cues_pos != -1 and (cluster_pos == -1 or cues_pos < cluster_pos):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _ensure_webm_cues_front(filepath: Path) -> None:
|
|
"""Re-mux a WebM/MKV file (stream copy) if CUES are not at the front.
|
|
|
|
YouTube-dl / yt-dlp writes CUES at the end by default, which slows HTTP
|
|
Range seeking. This runs once during upload processing and takes seconds.
|
|
"""
|
|
if filepath.suffix.lower() not in _WEBM_SUFFIXES:
|
|
return
|
|
if _webm_cues_at_front(filepath):
|
|
return
|
|
ffmpeg = shutil.which("ffmpeg")
|
|
if not ffmpeg:
|
|
logger.warning("ffmpeg not found; skipping WebM CUES optimization.")
|
|
return
|
|
tmp = filepath.with_suffix(".cues_tmp" + filepath.suffix)
|
|
try:
|
|
result = subprocess.run(
|
|
[ffmpeg, "-y", "-loglevel", "error",
|
|
"-i", str(filepath), "-c", "copy",
|
|
"-cues_to_front", "1", str(tmp)],
|
|
capture_output=True, timeout=300,
|
|
)
|
|
if result.returncode == 0 and tmp.is_file() and tmp.stat().st_size > 0:
|
|
tmp.replace(filepath)
|
|
logger.info("WebM CUES moved to front: %s", filepath.name)
|
|
else:
|
|
logger.warning(
|
|
"WebM CUES re-mux failed for %s: rc=%d stderr=%s",
|
|
filepath.name, result.returncode, result.stderr.decode()[-200:],
|
|
)
|
|
except (subprocess.TimeoutExpired, OSError) as exc:
|
|
logger.warning("WebM CUES re-mux error for %s: %s", filepath.name, exc)
|
|
finally:
|
|
tmp.unlink(missing_ok=True)
|
|
|
|
|
|
|
|
class VideoProcessor:
|
|
def __init__(
|
|
self,
|
|
settings: Settings,
|
|
repository: VideoRepository,
|
|
transcriber: Transcriber,
|
|
oss_uploader: Optional[VolcanoOSSUploader] = None,
|
|
):
|
|
self.settings = settings
|
|
self.repository = repository
|
|
self.transcriber = transcriber
|
|
self.oss_uploader = oss_uploader
|
|
|
|
def process(self, video_hash: str) -> None:
|
|
video = self.repository.get_video(video_hash)
|
|
if video is None:
|
|
raise ValueError(f"Unknown video: {video_hash}")
|
|
media_path = self.settings.videos_dir / video["stored_filename"]
|
|
if not media_path.is_file():
|
|
raise FileNotFoundError(f"Stored video is missing: {media_path.name}")
|
|
|
|
self.repository.mark_processing(video_hash)
|
|
work_path: Optional[Path] = None
|
|
try:
|
|
_ensure_webm_cues_front(media_path)
|
|
if self.transcriber.available:
|
|
work_path = self.settings.work_dir / f"{video_hash}-{uuid.uuid4().hex}.wav"
|
|
extract_audio(media_path, work_path)
|
|
transcript = self.transcriber.transcribe(work_path, video.get("language"))
|
|
document = document_from_transcript(
|
|
video_hash=video_hash,
|
|
duration_ms=_media_duration_ms(media_path),
|
|
transcript=transcript,
|
|
language=video.get("language"),
|
|
audio_path=work_path,
|
|
end_padding_ms=self.settings.moss_end_padding_ms,
|
|
)
|
|
if not document.sentences:
|
|
raise RuntimeError("MOSS returned no timestamped speech segments.")
|
|
self._finish_processing(
|
|
document,
|
|
transcript.text,
|
|
moss_segments=_serialize_moss_segments(transcript),
|
|
)
|
|
else:
|
|
entry, _ = make_entry(media_path, video_hash=video_hash)
|
|
document = SentenceBoundaryDocument(
|
|
video_hash=video_hash,
|
|
duration_ms=entry["duration_ms"],
|
|
algorithm_version=ALGORITHM_VERSION,
|
|
sentences=entry["sentences"],
|
|
)
|
|
self._finish_processing(document, None)
|
|
except Exception as exc:
|
|
self.repository.mark_failed(video_hash, str(exc))
|
|
raise
|
|
finally:
|
|
if work_path is not None:
|
|
work_path.unlink(missing_ok=True)
|
|
|
|
def _finish_processing(
|
|
self,
|
|
document: SentenceBoundaryDocument,
|
|
transcription: Optional[str],
|
|
moss_segments: Optional[List[Dict[str, Any]]] = None,
|
|
) -> None:
|
|
if self.oss_uploader is None or not self.oss_uploader.enabled:
|
|
self.repository.save_processing_result(
|
|
document, transcription, ready=True, moss_segments=moss_segments
|
|
)
|
|
return
|
|
|
|
# Keep the course invisible while the mandatory OSS upload is running.
|
|
self.repository.save_processing_result(
|
|
document, transcription, ready=False, moss_segments=moss_segments
|
|
)
|
|
video = self.repository.get_video(document.video_hash)
|
|
if video is None:
|
|
raise ValueError(f"Unknown video: {document.video_hash}")
|
|
remote_url = self.oss_uploader.upload(
|
|
video_hash=document.video_hash,
|
|
stored_filename=video["stored_filename"],
|
|
local_path=self.settings.videos_dir / video["stored_filename"],
|
|
content_type=video.get("content_type"),
|
|
)
|
|
self.repository.mark_oss_uploaded(document.video_hash, remote_url)
|
|
|
|
def repair_gaps(
|
|
self,
|
|
video_hash: str,
|
|
min_gap_ms: int = GAP_REPAIR_MIN_MS,
|
|
) -> Dict[str, int]:
|
|
"""Re-transcribe long unsentenced gaps without replacing existing cues."""
|
|
if min_gap_ms < 1:
|
|
raise ValueError("min_gap_ms must be positive")
|
|
if not self.transcriber.available:
|
|
raise RuntimeError("MOSS transcription is not configured on this server.")
|
|
video = self.repository.get_video(video_hash)
|
|
if video is None:
|
|
raise KeyError(video_hash)
|
|
document = self.repository.get_document(video_hash)
|
|
if document is None or not document.sentences:
|
|
raise ValueError("Sentence boundaries are not available for this video.")
|
|
media_path = self.settings.videos_dir / video["stored_filename"]
|
|
if not media_path.is_file():
|
|
raise FileNotFoundError(f"Stored video is missing: {media_path.name}")
|
|
|
|
gaps = find_sentence_gaps(document, min_gap_ms)
|
|
inserted: List[SentenceBoundary] = []
|
|
transcribed_gaps = 0
|
|
logger.info(
|
|
"Gap repair started: video=%s min_gap_ms=%d language=%s gaps=%d "
|
|
"sentences=%d duration_ms=%d transcriber=%s",
|
|
video_hash,
|
|
min_gap_ms,
|
|
video.get("language") or "auto",
|
|
len(gaps),
|
|
len(document.sentences),
|
|
document.duration_ms,
|
|
type(self.transcriber).__name__,
|
|
)
|
|
if gaps:
|
|
language = video.get("language")
|
|
with tempfile.TemporaryDirectory(prefix="gap-repair-") as temp_dir:
|
|
for gap_index, (raw_start, raw_end) in enumerate(gaps):
|
|
logger.info(
|
|
"Gap %d/%d: raw=%d-%dms duration=%dms",
|
|
gap_index + 1,
|
|
len(gaps),
|
|
raw_start,
|
|
raw_end,
|
|
raw_end - raw_start,
|
|
)
|
|
clipped = _inset_gap(raw_start, raw_end, document.duration_ms)
|
|
if clipped is None:
|
|
logger.warning(
|
|
"Gap %d/%d skipped after inset: raw=%d-%dms",
|
|
gap_index + 1,
|
|
len(gaps),
|
|
raw_start,
|
|
raw_end,
|
|
)
|
|
continue
|
|
start_ms, end_ms = clipped
|
|
chunk_path = Path(temp_dir) / f"gap-{gap_index}-full.wav"
|
|
logger.info(
|
|
"Gap %d/%d extracted range=%d-%dms duration=%dms file=%s",
|
|
gap_index + 1,
|
|
len(gaps),
|
|
start_ms,
|
|
end_ms,
|
|
end_ms - start_ms,
|
|
chunk_path,
|
|
)
|
|
try:
|
|
extract_audio_segment(
|
|
media_path,
|
|
chunk_path,
|
|
start_ms=start_ms,
|
|
duration_ms=end_ms - start_ms,
|
|
)
|
|
samples, sample_rate = decode_audio_mono(chunk_path)
|
|
metrics = analyze_samples(samples, sample_rate)
|
|
except (AudioAnalysisError, RuntimeError) as exc:
|
|
logger.warning(
|
|
"Skipping sentence gap %s-%sms: %s",
|
|
start_ms,
|
|
end_ms,
|
|
exc,
|
|
exc_info=True,
|
|
)
|
|
continue
|
|
logger.info(
|
|
"Gap %d/%d audio metrics: duration_ms=%d speech_duration_ms=%d "
|
|
"internal_silence_ms=%d pause_ratio=%.3f threshold_ms=%d",
|
|
gap_index + 1,
|
|
len(gaps),
|
|
metrics.recording_duration_ms,
|
|
metrics.speech_duration_ms,
|
|
metrics.internal_silence_ms,
|
|
metrics.internal_pause_ratio,
|
|
GAP_REPAIR_MIN_SPEECH_MS,
|
|
)
|
|
if metrics.speech_duration_ms < GAP_REPAIR_MIN_SPEECH_MS:
|
|
logger.warning(
|
|
"Gap %d/%d skipped as silence: speech_duration_ms=%d < %d",
|
|
gap_index + 1,
|
|
len(gaps),
|
|
metrics.speech_duration_ms,
|
|
GAP_REPAIR_MIN_SPEECH_MS,
|
|
)
|
|
continue
|
|
relative_chunks = _plan_gap_chunks(samples, sample_rate, end_ms - start_ms)
|
|
logger.info(
|
|
"Gap %d/%d split into %d transcription chunks: %s",
|
|
gap_index + 1,
|
|
len(gaps),
|
|
len(relative_chunks),
|
|
relative_chunks,
|
|
)
|
|
gap_inserted_count = 0
|
|
for chunk_index, (chunk_offset_start, chunk_offset_end) in enumerate(
|
|
relative_chunks
|
|
):
|
|
chunk_start = start_ms + chunk_offset_start
|
|
chunk_end = start_ms + chunk_offset_end
|
|
chunk_path = (
|
|
Path(temp_dir) / f"gap-{gap_index}-chunk-{chunk_index}.wav"
|
|
)
|
|
try:
|
|
extract_audio_segment(
|
|
media_path,
|
|
chunk_path,
|
|
start_ms=chunk_start,
|
|
duration_ms=chunk_end - chunk_start,
|
|
)
|
|
chunk_samples, chunk_sample_rate = decode_audio_mono(chunk_path)
|
|
chunk_metrics = analyze_samples(
|
|
chunk_samples, chunk_sample_rate
|
|
)
|
|
except (AudioAnalysisError, RuntimeError) as exc:
|
|
logger.warning(
|
|
"Could not prepare gap chunk %d/%d %s-%sms: %s",
|
|
chunk_index + 1,
|
|
len(relative_chunks),
|
|
chunk_start,
|
|
chunk_end,
|
|
exc,
|
|
exc_info=True,
|
|
)
|
|
continue
|
|
if chunk_metrics.speech_duration_ms < GAP_REPAIR_MIN_SPEECH_MS:
|
|
logger.info(
|
|
"Skipping quiet gap chunk %d/%d: range=%d-%dms "
|
|
"speech_duration_ms=%d",
|
|
chunk_index + 1,
|
|
len(relative_chunks),
|
|
chunk_start,
|
|
chunk_end,
|
|
chunk_metrics.speech_duration_ms,
|
|
)
|
|
continue
|
|
|
|
languages_to_try = [language]
|
|
if not language or language.strip().lower() == "auto":
|
|
fallback = GAP_REPAIR_FALLBACK_LANGUAGE
|
|
if fallback not in languages_to_try:
|
|
languages_to_try.append(fallback)
|
|
|
|
transcript = None
|
|
for attempt_index, attempt_language in enumerate(languages_to_try):
|
|
try:
|
|
logger.info(
|
|
"Transcribing gap chunk %d/%d: gap=%d/%d "
|
|
"range=%d-%dms language=%s",
|
|
chunk_index + 1,
|
|
len(relative_chunks),
|
|
gap_index + 1,
|
|
len(gaps),
|
|
chunk_start,
|
|
chunk_end,
|
|
attempt_language or "auto",
|
|
)
|
|
transcript = self.transcriber.transcribe(
|
|
chunk_path, attempt_language
|
|
)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"Gap chunk transcription failed for %s %s-%sms: %s",
|
|
video_hash,
|
|
chunk_start,
|
|
chunk_end,
|
|
exc,
|
|
exc_info=True,
|
|
)
|
|
continue
|
|
logger.info(
|
|
"Gap chunk transcript returned: gap=%d/%d chunk=%d/%d "
|
|
"segments=%d text=%r",
|
|
gap_index + 1,
|
|
len(gaps),
|
|
chunk_index + 1,
|
|
len(relative_chunks),
|
|
len(transcript.segments),
|
|
transcript.text,
|
|
)
|
|
if transcript.segments:
|
|
break
|
|
if attempt_index + 1 < len(languages_to_try):
|
|
logger.warning(
|
|
"Gap chunk returned no segments; retrying with %s: "
|
|
"range=%d-%dms",
|
|
languages_to_try[attempt_index + 1],
|
|
chunk_start,
|
|
chunk_end,
|
|
)
|
|
|
|
if transcript is None or not transcript.segments:
|
|
logger.warning(
|
|
"Gap chunk produced no transcript segments: gap=%d/%d "
|
|
"chunk=%d/%d range=%d-%dms attempts=%s",
|
|
gap_index + 1,
|
|
len(gaps),
|
|
chunk_index + 1,
|
|
len(relative_chunks),
|
|
chunk_start,
|
|
chunk_end,
|
|
[item or "auto" for item in languages_to_try],
|
|
)
|
|
continue
|
|
|
|
gap_document = document_from_transcript(
|
|
video_hash=video_hash,
|
|
duration_ms=chunk_end - chunk_start,
|
|
transcript=transcript,
|
|
language=language,
|
|
audio_path=chunk_path,
|
|
end_padding_ms=self.settings.moss_end_padding_ms,
|
|
)
|
|
shifted = _shift_gap_sentences(
|
|
gap_document.sentences,
|
|
offset_ms=chunk_start,
|
|
limit_end_ms=chunk_end,
|
|
)
|
|
if not shifted:
|
|
logger.warning(
|
|
"Gap chunk produced no insertable sentences: gap=%d/%d "
|
|
"chunk=%d/%d transcript_segments=%d "
|
|
"boundary_sentences=%d range=%d-%dms",
|
|
gap_index + 1,
|
|
len(gaps),
|
|
chunk_index + 1,
|
|
len(relative_chunks),
|
|
len(transcript.segments),
|
|
len(gap_document.sentences),
|
|
chunk_start,
|
|
chunk_end,
|
|
)
|
|
for boundary in gap_document.sentences:
|
|
logger.warning(
|
|
"Candidate boundary before shift: start=%d end=%d text=%r",
|
|
boundary.start_ms,
|
|
boundary.end_ms,
|
|
boundary.text,
|
|
)
|
|
continue
|
|
inserted.extend(shifted)
|
|
gap_inserted_count += len(shifted)
|
|
logger.info(
|
|
"Gap chunk produced %d insertable sentences: gap=%d/%d "
|
|
"chunk=%d/%d range=%d-%dms",
|
|
len(shifted),
|
|
gap_index + 1,
|
|
len(gaps),
|
|
chunk_index + 1,
|
|
len(relative_chunks),
|
|
chunk_start,
|
|
chunk_end,
|
|
)
|
|
|
|
if gap_inserted_count:
|
|
transcribed_gaps += 1
|
|
logger.info(
|
|
"Gap %d/%d produced %d insertable sentences total: range=%d-%dms",
|
|
gap_index + 1,
|
|
len(gaps),
|
|
gap_inserted_count,
|
|
start_ms,
|
|
end_ms,
|
|
)
|
|
else:
|
|
logger.warning(
|
|
"Gap %d/%d produced no insertable sentences after chunking: "
|
|
"range=%d-%dms chunks=%d",
|
|
gap_index + 1,
|
|
len(gaps),
|
|
start_ms,
|
|
end_ms,
|
|
len(relative_chunks),
|
|
)
|
|
|
|
|
|
if not inserted:
|
|
logger.warning(
|
|
"Gap repair finished without inserts: video=%s gaps=%d "
|
|
"transcribed_gaps=%d inserted=0",
|
|
video_hash,
|
|
len(gaps),
|
|
transcribed_gaps,
|
|
)
|
|
return {
|
|
"min_gap_ms": min_gap_ms,
|
|
"gap_count": len(gaps),
|
|
"transcribed_gaps": transcribed_gaps,
|
|
"inserted_count": 0,
|
|
"sentence_count": len(document.sentences),
|
|
}
|
|
|
|
merged = _merge_sentence_lists(document.sentences, inserted)
|
|
updated = document.model_copy(update={"sentences": merged})
|
|
saved = self.repository.replace_sentences(updated)
|
|
logger.info(
|
|
"Gap repair finished: video=%s gaps=%d transcribed_gaps=%d inserted=%d "
|
|
"sentences=%d->%d",
|
|
video_hash,
|
|
len(gaps),
|
|
transcribed_gaps,
|
|
len(saved.sentences) - len(document.sentences),
|
|
len(document.sentences),
|
|
len(saved.sentences),
|
|
)
|
|
return {
|
|
"min_gap_ms": min_gap_ms,
|
|
"gap_count": len(gaps),
|
|
"transcribed_gaps": transcribed_gaps,
|
|
"inserted_count": len(saved.sentences) - len(document.sentences),
|
|
"sentence_count": len(saved.sentences),
|
|
}
|
|
|
|
|
|
def extract_audio(media_path: Path, output_path: Path) -> None:
|
|
ffmpeg = shutil.which("ffmpeg")
|
|
if ffmpeg is None:
|
|
raise RuntimeError("ffmpeg is required for MOSS transcription but was not found.")
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
command = [
|
|
ffmpeg,
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-y",
|
|
"-i",
|
|
str(media_path),
|
|
"-vn",
|
|
"-ac",
|
|
"1",
|
|
"-ar",
|
|
"16000",
|
|
"-c:a",
|
|
"pcm_s16le",
|
|
str(output_path),
|
|
]
|
|
completed = subprocess.run(command, capture_output=True, text=True, timeout=7200)
|
|
if completed.returncode != 0:
|
|
message = completed.stderr.strip() or "unknown ffmpeg error"
|
|
raise RuntimeError(f"Could not extract video audio: {message[-2000:]}")
|
|
|
|
|
|
def extract_audio_segment(
|
|
media_path: Path,
|
|
output_path: Path,
|
|
*,
|
|
start_ms: int,
|
|
duration_ms: int,
|
|
) -> None:
|
|
"""Extract one mono WAV segment for a focused Whisper request."""
|
|
ffmpeg = shutil.which("ffmpeg")
|
|
if ffmpeg is None:
|
|
raise RuntimeError("ffmpeg is required for MOSS transcription but was not found.")
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
command = [
|
|
ffmpeg,
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-y",
|
|
"-ss",
|
|
f"{max(0, start_ms) / 1000:.3f}",
|
|
"-t",
|
|
f"{max(1, duration_ms) / 1000:.3f}",
|
|
"-i",
|
|
str(media_path),
|
|
"-vn",
|
|
"-ac",
|
|
"1",
|
|
"-ar",
|
|
"16000",
|
|
"-c:a",
|
|
"pcm_s16le",
|
|
str(output_path),
|
|
]
|
|
completed = subprocess.run(command, capture_output=True, text=True, timeout=1800)
|
|
if completed.returncode != 0:
|
|
message = completed.stderr.strip() or "unknown ffmpeg error"
|
|
raise RuntimeError(f"Could not extract sentence audio: {message[-2000:]}")
|
|
|
|
|
|
def find_sentence_gaps(
|
|
document: SentenceBoundaryDocument,
|
|
min_gap_ms: int = GAP_REPAIR_MIN_MS,
|
|
) -> List[Tuple[int, int]]:
|
|
"""Return [start_ms, end_ms) ranges that are missing sentence coverage."""
|
|
if min_gap_ms < 1:
|
|
raise ValueError("min_gap_ms must be positive")
|
|
gaps: List[Tuple[int, int]] = []
|
|
cursor = 0
|
|
for sentence in document.sentences:
|
|
if sentence.start_ms - cursor >= min_gap_ms:
|
|
gaps.append((cursor, sentence.start_ms))
|
|
cursor = max(cursor, sentence.end_ms)
|
|
if document.duration_ms - cursor >= min_gap_ms:
|
|
gaps.append((cursor, document.duration_ms))
|
|
return gaps
|
|
|
|
|
|
def _inset_gap(
|
|
start_ms: int,
|
|
end_ms: int,
|
|
duration_ms: int,
|
|
inset_ms: int = GAP_REPAIR_INSET_MS,
|
|
) -> Optional[Tuple[int, int]]:
|
|
start = start_ms + (inset_ms if start_ms > 0 else 0)
|
|
end = end_ms - (inset_ms if end_ms < duration_ms else 0)
|
|
if end - start < GAP_REPAIR_MIN_SPEECH_MS:
|
|
return None
|
|
return start, end
|
|
|
|
|
|
def _plan_gap_chunks(
|
|
samples,
|
|
sample_rate: int,
|
|
duration_ms: int,
|
|
) -> List[Tuple[int, int]]:
|
|
"""Split a long gap near silence so no ASR request exceeds Whisper's window."""
|
|
if duration_ms <= GAP_REPAIR_MAX_CHUNK_MS:
|
|
return [(0, duration_ms)]
|
|
|
|
chunks: List[Tuple[int, int]] = []
|
|
cursor = 0
|
|
while cursor < duration_ms:
|
|
remaining = duration_ms - cursor
|
|
if remaining <= GAP_REPAIR_MAX_CHUNK_MS:
|
|
chunks.append((cursor, duration_ms))
|
|
break
|
|
|
|
preferred = min(cursor + GAP_REPAIR_TARGET_CHUNK_MS, duration_ms)
|
|
lower = min(cursor + GAP_REPAIR_MIN_CHUNK_MS, preferred)
|
|
upper = min(cursor + GAP_REPAIR_MAX_CHUNK_MS, duration_ms)
|
|
cut = _find_silence_cut(
|
|
samples,
|
|
sample_rate,
|
|
lower,
|
|
upper,
|
|
preferred,
|
|
min_silence_ms=GAP_REPAIR_MIN_SPLIT_SILENCE_MS,
|
|
)
|
|
cut = max(cursor + 1, min(duration_ms, cut))
|
|
chunks.append((cursor, cut))
|
|
cursor = cut
|
|
return chunks
|
|
|
|
|
|
def _find_silence_cut(
|
|
samples,
|
|
sample_rate: int,
|
|
lower_ms: int,
|
|
upper_ms: int,
|
|
preferred_ms: int,
|
|
*,
|
|
min_silence_ms: int,
|
|
) -> int:
|
|
"""Choose the silence midpoint nearest ``preferred_ms``."""
|
|
frame_ms = 1000.0 * FRAME_SAMPLES / sample_rate
|
|
min_silence_frames = max(1, int(round(min_silence_ms / frame_ms)))
|
|
start_sample = max(0, int(lower_ms / 1000 * sample_rate))
|
|
end_sample = min(samples.size, int(upper_ms / 1000 * sample_rate) + 1)
|
|
if end_sample <= start_sample:
|
|
return preferred_ms
|
|
|
|
try:
|
|
speech = speech_frame_mask(samples[start_sample:end_sample], sample_rate)
|
|
except AudioAnalysisError:
|
|
return preferred_ms
|
|
|
|
run_start: Optional[int] = None
|
|
candidates: List[Tuple[int, int]] = []
|
|
for index, is_speech in enumerate(speech):
|
|
if not is_speech:
|
|
if run_start is None:
|
|
run_start = index
|
|
continue
|
|
if run_start is not None:
|
|
candidates.append((run_start, index))
|
|
run_start = None
|
|
if run_start is not None:
|
|
candidates.append((run_start, len(speech)))
|
|
|
|
valid = [
|
|
(run_start, run_end)
|
|
for run_start, run_end in candidates
|
|
if run_end - run_start >= min_silence_frames
|
|
]
|
|
if not valid:
|
|
return preferred_ms
|
|
|
|
def distance(run: Tuple[int, int]) -> float:
|
|
midpoint_frame = (run[0] + run[1]) / 2.0
|
|
midpoint_ms = lower_ms + midpoint_frame * frame_ms
|
|
return abs(midpoint_ms - preferred_ms)
|
|
|
|
run_start, run_end = min(valid, key=distance)
|
|
midpoint_ms = lower_ms + ((run_start + run_end) / 2.0) * frame_ms
|
|
return int(round(midpoint_ms))
|
|
|
|
|
|
def _shift_gap_sentences(
|
|
sentences: Sequence[SentenceBoundary],
|
|
*,
|
|
offset_ms: int,
|
|
limit_end_ms: int,
|
|
) -> List[SentenceBoundary]:
|
|
shifted: List[SentenceBoundary] = []
|
|
previous_end = offset_ms
|
|
for sentence in sentences:
|
|
start_ms = max(previous_end, sentence.start_ms + offset_ms)
|
|
end_ms = min(limit_end_ms, sentence.end_ms + offset_ms)
|
|
text = (sentence.text or "").strip()
|
|
if not text or end_ms - start_ms < GAP_REPAIR_MIN_SENTENCE_MS:
|
|
continue
|
|
shifted.append(
|
|
sentence.model_copy(
|
|
update={
|
|
"index": len(shifted),
|
|
"start_ms": start_ms,
|
|
"end_ms": end_ms,
|
|
"text": text,
|
|
"zh_text": None,
|
|
}
|
|
)
|
|
)
|
|
previous_end = end_ms
|
|
return shifted
|
|
|
|
|
|
def _merge_sentence_lists(
|
|
existing: Sequence[SentenceBoundary],
|
|
inserted: Sequence[SentenceBoundary],
|
|
) -> List[SentenceBoundary]:
|
|
combined = [*existing, *inserted]
|
|
combined.sort(key=lambda sentence: (sentence.start_ms, sentence.end_ms))
|
|
merged: List[SentenceBoundary] = []
|
|
previous_end = 0
|
|
for sentence in combined:
|
|
if sentence.start_ms < previous_end:
|
|
continue
|
|
merged.append(sentence.model_copy(update={"index": len(merged)}))
|
|
previous_end = sentence.end_ms
|
|
return merged
|
|
|
|
|
|
def split_sentence_text(
|
|
*,
|
|
transcriber: Transcriber,
|
|
media_path: Path,
|
|
sentence: SentenceBoundary,
|
|
text: str,
|
|
language: Optional[str],
|
|
) -> List[SentenceBoundary]:
|
|
"""Split one stored sentence at explicit line breaks.
|
|
|
|
When MOSS/Whisper is available, the sentence's own audio is transcribed and
|
|
its word timestamps select the new boundary. If transcription is unavailable
|
|
or fails, editing remains possible and boundaries fall back to text-length
|
|
proportions so the admin UI does not lose the user's manual edit.
|
|
"""
|
|
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
|
if len(lines) < 2:
|
|
return [
|
|
sentence.model_copy(
|
|
update={
|
|
"text": text.strip(),
|
|
"language": language if language is not None else sentence.language,
|
|
}
|
|
)
|
|
]
|
|
|
|
transcript: Optional[Transcript] = None
|
|
if transcriber.available and media_path.is_file():
|
|
try:
|
|
with tempfile.TemporaryDirectory(prefix="sentence-split-") as temp_dir:
|
|
audio_path = Path(temp_dir) / "sentence.wav"
|
|
extract_audio_segment(
|
|
media_path,
|
|
audio_path,
|
|
start_ms=sentence.start_ms,
|
|
duration_ms=sentence.end_ms - sentence.start_ms,
|
|
)
|
|
transcript = transcriber.transcribe(audio_path, language)
|
|
except Exception as exc:
|
|
logger.warning("Whisper split-timing failed for sentence %s: %s", sentence.index, exc)
|
|
|
|
boundaries = _estimate_split_boundaries_ms(sentence, lines, transcript)
|
|
weights = [_text_weight(line) for line in lines]
|
|
total_weight = max(1, sum(weights))
|
|
replacements: List[SentenceBoundary] = []
|
|
for boundary, line, weight in zip(boundaries, lines, weights):
|
|
reference_duration = sentence.reference_speech_duration_ms or (
|
|
boundary.end_ms - boundary.start_ms
|
|
)
|
|
replacements.append(
|
|
boundary.model_copy(
|
|
update={
|
|
"text": line,
|
|
"language": language if language is not None else sentence.language,
|
|
"reference_speech_duration_ms": max(
|
|
1,
|
|
int(round(reference_duration * weight / total_weight)),
|
|
),
|
|
}
|
|
)
|
|
)
|
|
return replacements
|
|
|
|
|
|
def _estimate_split_boundaries_ms(
|
|
sentence: SentenceBoundary,
|
|
lines: List[str],
|
|
transcript: Optional[Transcript],
|
|
) -> List[SentenceBoundary]:
|
|
estimates = _whisper_split_boundaries_ms(sentence, lines, transcript)
|
|
if len(estimates) != len(lines) - 1:
|
|
estimates = _proportional_split_boundaries_ms(sentence, lines)
|
|
|
|
# Keep every split strictly inside the source sentence, even if a Whisper
|
|
# timestamp is slightly outside or two estimates collapse to one point.
|
|
previous = sentence.start_ms
|
|
remaining = len(lines)
|
|
for index, estimate in enumerate(list(estimates)):
|
|
low = previous + 1
|
|
high = sentence.end_ms - (remaining - 1)
|
|
if low >= high:
|
|
estimates = _proportional_split_boundaries_ms(sentence, lines)
|
|
break
|
|
estimate = min(max(estimate, low), high)
|
|
estimates[index] = estimate
|
|
previous = estimate
|
|
remaining -= 1
|
|
|
|
starts = [sentence.start_ms, *estimates]
|
|
ends = [*estimates, sentence.end_ms]
|
|
return [
|
|
SentenceBoundary(
|
|
index=index,
|
|
start_ms=start,
|
|
end_ms=end,
|
|
text=lines[index],
|
|
language=sentence.language,
|
|
reference_speech_duration_ms=max(1, end - start),
|
|
)
|
|
for index, (start, end) in enumerate(zip(starts, ends))
|
|
]
|
|
|
|
|
|
def _whisper_split_boundaries_ms(
|
|
sentence: SentenceBoundary,
|
|
lines: List[str],
|
|
transcript: Optional[Transcript],
|
|
) -> List[int]:
|
|
if transcript is None or not transcript.segments:
|
|
return []
|
|
|
|
source_ratios = _cumulative_ratios([_text_weight(line) for line in lines])
|
|
words = [word for segment in transcript.segments for word in (segment.words or [])]
|
|
if len(words) >= 2:
|
|
span_start = words[0].start_seconds
|
|
span_end = max(word.end_seconds for word in words)
|
|
if span_end <= span_start:
|
|
return []
|
|
total_weight = sum(max(1, len(word.text.strip())) for word in words)
|
|
cumulative = 0
|
|
samples: List[tuple[float, int]] = []
|
|
for word in words[:-1]:
|
|
cumulative += max(1, len(word.text.strip()))
|
|
timestamp_ms = sentence.start_ms + int(
|
|
round(
|
|
(word.end_seconds - span_start)
|
|
/ (span_end - span_start)
|
|
* (sentence.end_ms - sentence.start_ms)
|
|
)
|
|
)
|
|
samples.append((cumulative / total_weight, timestamp_ms))
|
|
return [
|
|
min(samples, key=lambda item: (abs(item[0] - ratio), item[1]))[1]
|
|
for ratio in source_ratios
|
|
]
|
|
|
|
# Some MOSS configurations omit word timestamps. Segment ends are still
|
|
# much better than blind proportional allocation when they exist.
|
|
if len(transcript.segments) < 2:
|
|
return []
|
|
total_weight = sum(max(1, len(segment.text.strip())) for segment in transcript.segments)
|
|
cumulative = 0
|
|
samples: List[tuple[float, int]] = []
|
|
for segment in transcript.segments[:-1]:
|
|
cumulative += max(1, len(segment.text.strip()))
|
|
timestamp_ms = sentence.start_ms + int(round(segment.end_seconds * 1000))
|
|
samples.append((cumulative / total_weight, timestamp_ms))
|
|
return [
|
|
min(samples, key=lambda item: (abs(item[0] - ratio), item[1]))[1]
|
|
for ratio in source_ratios
|
|
]
|
|
|
|
|
|
def _proportional_split_boundaries_ms(
|
|
sentence: SentenceBoundary,
|
|
lines: List[str],
|
|
) -> List[int]:
|
|
duration = sentence.end_ms - sentence.start_ms
|
|
return [
|
|
sentence.start_ms + int(round(ratio * duration))
|
|
for ratio in _cumulative_ratios([_text_weight(line) for line in lines])
|
|
]
|
|
|
|
|
|
def _cumulative_ratios(weights: List[int]) -> List[float]:
|
|
total = max(1, sum(weights))
|
|
cumulative = 0
|
|
ratios = []
|
|
for weight in weights[:-1]:
|
|
cumulative += weight
|
|
ratios.append(cumulative / total)
|
|
return ratios
|
|
|
|
|
|
def _text_weight(text: str) -> int:
|
|
return max(1, sum(1 for char in text if not char.isspace()))
|
|
|
|
|
|
def document_from_transcript(
|
|
*,
|
|
video_hash: str,
|
|
duration_ms: int,
|
|
transcript: Transcript,
|
|
language: Optional[str],
|
|
audio_path: Path,
|
|
end_padding_ms: int = 300,
|
|
) -> SentenceBoundaryDocument:
|
|
samples, sample_rate = decode_audio_mono(audio_path)
|
|
sentences: List[SentenceBoundary] = []
|
|
previous_end = 0
|
|
sentence_segments = split_sentences_at_punctuation(transcript.segments)
|
|
for index, sentence_segment in enumerate(sentence_segments):
|
|
raw_start_ms = int(round(sentence_segment.start_seconds * 1000))
|
|
start_ms = max(previous_end, raw_start_ms)
|
|
raw_end_ms = int(round(sentence_segment.end_seconds * 1000))
|
|
end_ms = min(
|
|
duration_ms,
|
|
raw_end_ms + end_padding_ms,
|
|
)
|
|
if index + 1 < len(sentence_segments):
|
|
next_start_ms = int(
|
|
round(sentence_segments[index + 1].start_seconds * 1000)
|
|
)
|
|
end_ms = min(end_ms, next_start_ms)
|
|
end_ms = refine_sentence_end_ms(
|
|
samples, sample_rate, raw_end_ms=raw_end_ms, padded_end_ms=end_ms
|
|
)
|
|
refined_start_ms = refine_sentence_start_ms(
|
|
samples,
|
|
sample_rate,
|
|
raw_start_ms=raw_start_ms,
|
|
previous_end_ms=previous_end,
|
|
)
|
|
if previous_end <= refined_start_ms and end_ms - refined_start_ms >= 100:
|
|
start_ms = refined_start_ms
|
|
if not sentence_segment.text.strip() or end_ms <= start_ms:
|
|
continue
|
|
start_sample = max(0, int(start_ms / 1000 * sample_rate))
|
|
end_sample = min(samples.size, int(end_ms / 1000 * sample_rate))
|
|
try:
|
|
metrics = analyze_samples(samples[start_sample:end_sample], sample_rate)
|
|
speech_duration_ms = metrics.speech_duration_ms
|
|
except AudioAnalysisError:
|
|
speech_duration_ms = end_ms - start_ms
|
|
sentences.append(
|
|
SentenceBoundary(
|
|
index=len(sentences),
|
|
start_ms=start_ms,
|
|
end_ms=end_ms,
|
|
text=sentence_segment.text.strip(),
|
|
language=language,
|
|
reference_speech_duration_ms=max(1, speech_duration_ms),
|
|
)
|
|
)
|
|
previous_end = end_ms
|
|
return SentenceBoundaryDocument(
|
|
video_hash=video_hash,
|
|
duration_ms=duration_ms,
|
|
algorithm_version=MOSS_ALGORITHM_VERSION,
|
|
sentences=sentences,
|
|
)
|
|
|
|
|
|
def generate_audio_boundary_document(
|
|
*,
|
|
video_hash: str,
|
|
audio_path: Path,
|
|
transcriber: Transcriber,
|
|
language: Optional[str] = None,
|
|
) -> SentenceBoundaryDocument:
|
|
"""Generate sentence boundaries for a client-supplied audio file.
|
|
|
|
The caller owns ``audio_path`` and is responsible for deleting it. Nothing
|
|
in this function copies or registers the audio in application storage.
|
|
"""
|
|
duration_ms = media_duration_ms(audio_path)
|
|
transcript = None
|
|
if transcriber.available:
|
|
try:
|
|
transcript = transcriber.transcribe(audio_path, language)
|
|
except Exception as exc:
|
|
logger.warning("Audio boundary transcription failed; using silence fallback: %s", exc)
|
|
|
|
if transcript is not None:
|
|
document = document_from_transcript(
|
|
video_hash=video_hash,
|
|
duration_ms=duration_ms,
|
|
transcript=transcript,
|
|
language=language,
|
|
audio_path=audio_path,
|
|
)
|
|
if document.sentences:
|
|
return document
|
|
|
|
starts = detect_sentence_boundaries(audio_path)
|
|
if not starts:
|
|
raise ValueError("No sentence boundaries could be detected in the audio.")
|
|
|
|
starts_ms = sorted({max(0, min(duration_ms - 1, int(round(start * 1000)))) for start in starts})
|
|
sentences: List[SentenceBoundary] = []
|
|
for index, start_ms in enumerate(starts_ms):
|
|
end_ms = starts_ms[index + 1] if index + 1 < len(starts_ms) else duration_ms
|
|
if end_ms <= start_ms:
|
|
continue
|
|
sentences.append(
|
|
SentenceBoundary(
|
|
index=len(sentences),
|
|
start_ms=start_ms,
|
|
end_ms=end_ms,
|
|
text=None,
|
|
language=language,
|
|
)
|
|
)
|
|
if not sentences:
|
|
raise ValueError("Detected audio boundaries do not form valid sentence ranges.")
|
|
|
|
return SentenceBoundaryDocument(
|
|
video_hash=video_hash,
|
|
duration_ms=duration_ms,
|
|
algorithm_version=SILENCE_ALGORITHM_VERSION,
|
|
sentences=sentences,
|
|
)
|
|
|
|
|
|
def _media_duration_ms(path: Path) -> int:
|
|
from .generate_boundaries import media_duration_ms
|
|
|
|
return media_duration_ms(path)
|
|
|
|
|
|
def _serialize_moss_segments(transcript: Transcript) -> List[Dict[str, Any]]:
|
|
rows: List[Dict[str, Any]] = []
|
|
for segment in transcript.segments:
|
|
text = segment.text.strip()
|
|
if not text:
|
|
continue
|
|
words = None
|
|
if segment.words:
|
|
words = [
|
|
{
|
|
"start_seconds": word.start_seconds,
|
|
"end_seconds": word.end_seconds,
|
|
"text": word.text,
|
|
}
|
|
for word in segment.words
|
|
]
|
|
rows.append(
|
|
{
|
|
"start_seconds": segment.start_seconds,
|
|
"end_seconds": segment.end_seconds,
|
|
"text": text,
|
|
"speaker": segment.speaker,
|
|
"words": words,
|
|
}
|
|
)
|
|
return rows
|
|
|