From fe21617c6345f509960bdbc042e54d7e5a1c02f9 Mon Sep 17 00:00:00 2001 From: Shuming Liu Date: Sun, 25 Oct 2026 11:41:08 +0800 Subject: [PATCH] fixed a bug --- sentence_api/processing.py | 329 +++++++++++++++++++++----- sentence_api/tests/test_processing.py | 17 ++ 2 files changed, 292 insertions(+), 54 deletions(-) diff --git a/sentence_api/processing.py b/sentence_api/processing.py index 4a6fb0e..e9b4815 100644 --- a/sentence_api/processing.py +++ b/sentence_api/processing.py @@ -10,8 +10,10 @@ from .audio_metrics import ( AudioAnalysisError, analyze_samples, decode_audio_mono, + FRAME_SAMPLES, refine_sentence_end_ms, refine_sentence_start_ms, + speech_frame_mask, ) from .config import Settings from .generate_boundaries import ALGORITHM_VERSION, make_entry, media_duration_ms @@ -33,6 +35,11 @@ GAP_REPAIR_MIN_MS = 8_000 GAP_REPAIR_INSET_MS = 150 GAP_REPAIR_MIN_SPEECH_MS = 400 GAP_REPAIR_MIN_SENTENCE_MS = 100 +GAP_REPAIR_TARGET_CHUNK_MS = 15_000 +GAP_REPAIR_MAX_CHUNK_MS = 20_000 +GAP_REPAIR_MIN_CHUNK_MS = 5_000 +GAP_REPAIR_MIN_SPLIT_SILENCE_MS = 120 +GAP_REPAIR_FALLBACK_LANGUAGE = "en" _CUES_ID = b"\x1e\x53\xb6\x80" _CLUSTER_ID = b"\x1f\x43\xb6\x75" _HEADER_SCAN_SIZE = 64 * 1024 @@ -234,7 +241,7 @@ class VideoProcessor: ) continue start_ms, end_ms = clipped - chunk_path = Path(temp_dir) / f"gap-{gap_index}.wav" + chunk_path = Path(temp_dir) / f"gap-{gap_index}-full.wav" logger.info( "Gap %d/%d extracted range=%d-%dms duration=%dms file=%s", gap_index + 1, @@ -282,68 +289,195 @@ class VideoProcessor: GAP_REPAIR_MIN_SPEECH_MS, ) continue - try: - transcript = self.transcriber.transcribe(chunk_path, language) - except Exception as exc: - logger.warning( - "Gap transcription failed for %s %s-%sms: %s", - video_hash, - start_ms, - end_ms, - exc, - exc_info=True, - ) - continue + relative_chunks = _plan_gap_chunks(samples, sample_rate, end_ms - start_ms) logger.info( - "Gap %d/%d transcript returned: segments=%d text=%r", + "Gap %d/%d split into %d transcription chunks: %s", gap_index + 1, len(gaps), - len(transcript.segments), - transcript.text, + len(relative_chunks), + relative_chunks, ) - gap_document = document_from_transcript( - video_hash=video_hash, - duration_ms=end_ms - start_ms, - transcript=transcript, - language=language, - audio_path=chunk_path, - end_padding_ms=self.settings.moss_end_padding_ms, - ) - shifted = _shift_gap_sentences( - gap_document.sentences, - offset_ms=start_ms, - limit_end_ms=end_ms, - ) - if not shifted: - logger.warning( - "Gap %d/%d produced no insertable sentences: " - "transcript_segments=%d boundary_sentences=%d range=%d-%dms", + gap_inserted_count = 0 + for chunk_index, (chunk_offset_start, chunk_offset_end) in enumerate( + relative_chunks + ): + chunk_start = start_ms + chunk_offset_start + chunk_end = start_ms + chunk_offset_end + chunk_path = ( + Path(temp_dir) / f"gap-{gap_index}-chunk-{chunk_index}.wav" + ) + try: + extract_audio_segment( + media_path, + chunk_path, + start_ms=chunk_start, + duration_ms=chunk_end - chunk_start, + ) + chunk_samples, chunk_sample_rate = decode_audio_mono(chunk_path) + chunk_metrics = analyze_samples( + chunk_samples, chunk_sample_rate + ) + except (AudioAnalysisError, RuntimeError) as exc: + logger.warning( + "Could not prepare gap chunk %d/%d %s-%sms: %s", + chunk_index + 1, + len(relative_chunks), + chunk_start, + chunk_end, + exc, + exc_info=True, + ) + continue + if chunk_metrics.speech_duration_ms < GAP_REPAIR_MIN_SPEECH_MS: + logger.info( + "Skipping quiet gap chunk %d/%d: range=%d-%dms " + "speech_duration_ms=%d", + chunk_index + 1, + len(relative_chunks), + chunk_start, + chunk_end, + chunk_metrics.speech_duration_ms, + ) + continue + + languages_to_try = [language] + if not language or language.strip().lower() == "auto": + fallback = GAP_REPAIR_FALLBACK_LANGUAGE + if fallback not in languages_to_try: + languages_to_try.append(fallback) + + transcript = None + for attempt_index, attempt_language in enumerate(languages_to_try): + try: + logger.info( + "Transcribing gap chunk %d/%d: gap=%d/%d " + "range=%d-%dms language=%s", + chunk_index + 1, + len(relative_chunks), + gap_index + 1, + len(gaps), + chunk_start, + chunk_end, + attempt_language or "auto", + ) + transcript = self.transcriber.transcribe( + chunk_path, attempt_language + ) + except Exception as exc: + logger.warning( + "Gap chunk transcription failed for %s %s-%sms: %s", + video_hash, + chunk_start, + chunk_end, + exc, + exc_info=True, + ) + continue + logger.info( + "Gap chunk transcript returned: gap=%d/%d chunk=%d/%d " + "segments=%d text=%r", + gap_index + 1, + len(gaps), + chunk_index + 1, + len(relative_chunks), + len(transcript.segments), + transcript.text, + ) + if transcript.segments: + break + if attempt_index + 1 < len(languages_to_try): + logger.warning( + "Gap chunk returned no segments; retrying with %s: " + "range=%d-%dms", + languages_to_try[attempt_index + 1], + chunk_start, + chunk_end, + ) + + if transcript is None or not transcript.segments: + logger.warning( + "Gap chunk produced no transcript segments: gap=%d/%d " + "chunk=%d/%d range=%d-%dms attempts=%s", + gap_index + 1, + len(gaps), + chunk_index + 1, + len(relative_chunks), + chunk_start, + chunk_end, + [item or "auto" for item in languages_to_try], + ) + continue + + gap_document = document_from_transcript( + video_hash=video_hash, + duration_ms=chunk_end - chunk_start, + transcript=transcript, + language=language, + audio_path=chunk_path, + end_padding_ms=self.settings.moss_end_padding_ms, + ) + shifted = _shift_gap_sentences( + gap_document.sentences, + offset_ms=chunk_start, + limit_end_ms=chunk_end, + ) + if not shifted: + logger.warning( + "Gap chunk produced no insertable sentences: gap=%d/%d " + "chunk=%d/%d transcript_segments=%d " + "boundary_sentences=%d range=%d-%dms", + gap_index + 1, + len(gaps), + chunk_index + 1, + len(relative_chunks), + len(transcript.segments), + len(gap_document.sentences), + chunk_start, + chunk_end, + ) + for boundary in gap_document.sentences: + logger.warning( + "Candidate boundary before shift: start=%d end=%d text=%r", + boundary.start_ms, + boundary.end_ms, + boundary.text, + ) + continue + inserted.extend(shifted) + gap_inserted_count += len(shifted) + logger.info( + "Gap chunk produced %d insertable sentences: gap=%d/%d " + "chunk=%d/%d range=%d-%dms", + len(shifted), gap_index + 1, len(gaps), - len(transcript.segments), - len(gap_document.sentences), + chunk_index + 1, + len(relative_chunks), + chunk_start, + chunk_end, + ) + + if gap_inserted_count: + transcribed_gaps += 1 + logger.info( + "Gap %d/%d produced %d insertable sentences total: range=%d-%dms", + gap_index + 1, + len(gaps), + gap_inserted_count, start_ms, end_ms, ) - for boundary in gap_document.sentences: - logger.warning( - "Candidate boundary before shift: start=%d end=%d text=%r", - boundary.start_ms, - boundary.end_ms, - boundary.text, - ) - if not shifted: - continue - transcribed_gaps += 1 - inserted.extend(shifted) - logger.info( - "Gap %d/%d produced %d insertable sentences: range=%d-%dms", - gap_index + 1, - len(gaps), - len(shifted), - start_ms, - end_ms, - ) + else: + logger.warning( + "Gap %d/%d produced no insertable sentences after chunking: " + "range=%d-%dms chunks=%d", + gap_index + 1, + len(gaps), + start_ms, + end_ms, + len(relative_chunks), + ) + if not inserted: logger.warning( @@ -481,6 +615,93 @@ def _inset_gap( return start, end +def _plan_gap_chunks( + samples, + sample_rate: int, + duration_ms: int, +) -> List[Tuple[int, int]]: + """Split a long gap near silence so no ASR request exceeds Whisper's window.""" + if duration_ms <= GAP_REPAIR_MAX_CHUNK_MS: + return [(0, duration_ms)] + + chunks: List[Tuple[int, int]] = [] + cursor = 0 + while cursor < duration_ms: + remaining = duration_ms - cursor + if remaining <= GAP_REPAIR_MAX_CHUNK_MS: + chunks.append((cursor, duration_ms)) + break + + preferred = min(cursor + GAP_REPAIR_TARGET_CHUNK_MS, duration_ms) + lower = min(cursor + GAP_REPAIR_MIN_CHUNK_MS, preferred) + upper = min(cursor + GAP_REPAIR_MAX_CHUNK_MS, duration_ms) + cut = _find_silence_cut( + samples, + sample_rate, + lower, + upper, + preferred, + min_silence_ms=GAP_REPAIR_MIN_SPLIT_SILENCE_MS, + ) + cut = max(cursor + 1, min(duration_ms, cut)) + chunks.append((cursor, cut)) + cursor = cut + return chunks + + +def _find_silence_cut( + samples, + sample_rate: int, + lower_ms: int, + upper_ms: int, + preferred_ms: int, + *, + min_silence_ms: int, +) -> int: + """Choose the silence midpoint nearest ``preferred_ms``.""" + frame_ms = 1000.0 * FRAME_SAMPLES / sample_rate + min_silence_frames = max(1, int(round(min_silence_ms / frame_ms))) + start_sample = max(0, int(lower_ms / 1000 * sample_rate)) + end_sample = min(samples.size, int(upper_ms / 1000 * sample_rate) + 1) + if end_sample <= start_sample: + return preferred_ms + + try: + speech = speech_frame_mask(samples[start_sample:end_sample], sample_rate) + except AudioAnalysisError: + return preferred_ms + + run_start: Optional[int] = None + candidates: List[Tuple[int, int]] = [] + for index, is_speech in enumerate(speech): + if not is_speech: + if run_start is None: + run_start = index + continue + if run_start is not None: + candidates.append((run_start, index)) + run_start = None + if run_start is not None: + candidates.append((run_start, len(speech))) + + valid = [ + (run_start, run_end) + for run_start, run_end in candidates + if run_end - run_start >= min_silence_frames + ] + if not valid: + return preferred_ms + + def distance(run: Tuple[int, int]) -> float: + midpoint_frame = (run[0] + run[1]) / 2.0 + midpoint_ms = lower_ms + midpoint_frame * frame_ms + return abs(midpoint_ms - preferred_ms) + + run_start, run_end = min(valid, key=distance) + midpoint_ms = lower_ms + ((run_start + run_end) / 2.0) * frame_ms + return int(round(midpoint_ms)) + + def _shift_gap_sentences( sentences: Sequence[SentenceBoundary], *, diff --git a/sentence_api/tests/test_processing.py b/sentence_api/tests/test_processing.py index dcf6ca6..30594e7 100644 --- a/sentence_api/tests/test_processing.py +++ b/sentence_api/tests/test_processing.py @@ -5,6 +5,7 @@ from sentence_api.processing import ( VideoProcessor, document_from_transcript, find_sentence_gaps, + _plan_gap_chunks, split_sentence_text, ) from sentence_api.models import SentenceBoundary, SentenceBoundaryDocument @@ -417,6 +418,22 @@ def test_find_sentence_gaps_ignores_short_pauses(): assert find_sentence_gaps(document) == [] +def test_plan_gap_chunks_splits_long_gap_at_silence(): + samples = np.concatenate( + [_speech(14.0), _silence(0.6), _speech(16.0)] + ).astype(np.float32) + chunks = _plan_gap_chunks(samples, SAMPLE_RATE, int(samples.size / SAMPLE_RATE * 1000)) + + assert len(chunks) == 2 + first_start, first_end = chunks[0] + second_start, second_end = chunks[1] + assert first_start == 0 + assert 13_000 <= first_end <= 17_000 + assert second_start == first_end + assert second_end == int(samples.size / SAMPLE_RATE * 1000) + assert all(end - start <= 20_000 for start, end in chunks) + + def test_repair_gaps_inserts_transcribed_sentences_and_keeps_existing(tmp_path, monkeypatch): from dataclasses import replace