smshine/mediaplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed a bug

Browse Source
This commit is contained in:
Shuming Liu
2026-10-25 11:41:08 +08:00
parent 5428c42ffd
commit fe21617c63
2 changed files with 292 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

329
sentence_api/processing.py
View File

@@ -10,8 +10,10 @@ from .audio_metrics import (
AudioAnalysisError, AudioAnalysisError,
analyze_samples, analyze_samples,
decode_audio_mono, decode_audio_mono,
FRAME_SAMPLES,
refine_sentence_end_ms, refine_sentence_end_ms,
refine_sentence_start_ms, refine_sentence_start_ms,
speech_frame_mask,
) )
from .config import Settings from .config import Settings
from .generate_boundaries import ALGORITHM_VERSION, make_entry, media_duration_ms from .generate_boundaries import ALGORITHM_VERSION, make_entry, media_duration_ms
@@ -33,6 +35,11 @@ GAP_REPAIR_MIN_MS = 8_000
GAP_REPAIR_INSET_MS = 150 GAP_REPAIR_INSET_MS = 150
GAP_REPAIR_MIN_SPEECH_MS = 400 GAP_REPAIR_MIN_SPEECH_MS = 400
GAP_REPAIR_MIN_SENTENCE_MS = 100 GAP_REPAIR_MIN_SENTENCE_MS = 100
GAP_REPAIR_TARGET_CHUNK_MS = 15_000
GAP_REPAIR_MAX_CHUNK_MS = 20_000
GAP_REPAIR_MIN_CHUNK_MS = 5_000
GAP_REPAIR_MIN_SPLIT_SILENCE_MS = 120
GAP_REPAIR_FALLBACK_LANGUAGE = "en"
_CUES_ID = b"\x1e\x53\xb6\x80" _CUES_ID = b"\x1e\x53\xb6\x80"
_CLUSTER_ID = b"\x1f\x43\xb6\x75" _CLUSTER_ID = b"\x1f\x43\xb6\x75"
_HEADER_SCAN_SIZE = 64 * 1024 _HEADER_SCAN_SIZE = 64 * 1024
@@ -234,7 +241,7 @@ class VideoProcessor:
) )
continue continue
start_ms, end_ms = clipped start_ms, end_ms = clipped
chunk_path = Path(temp_dir) / f"gap-{gap_index}.wav" chunk_path = Path(temp_dir) / f"gap-{gap_index}-full.wav"
logger.info( logger.info(
"Gap %d/%d extracted range=%d-%dms duration=%dms file=%s", "Gap %d/%d extracted range=%d-%dms duration=%dms file=%s",
gap_index + 1, gap_index + 1,
@@ -282,68 +289,195 @@ class VideoProcessor:
GAP_REPAIR_MIN_SPEECH_MS, GAP_REPAIR_MIN_SPEECH_MS,
) )
continue continue
try: relative_chunks = _plan_gap_chunks(samples, sample_rate, end_ms - start_ms)
transcript = self.transcriber.transcribe(chunk_path, language)
except Exception as exc:
logger.warning(
"Gap transcription failed for %s %s-%sms: %s",
video_hash,
start_ms,
end_ms,
exc,
exc_info=True,
)
continue
logger.info( logger.info(
"Gap %d/%d transcript returned: segments=%d text=%r", "Gap %d/%d split into %d transcription chunks: %s",
gap_index + 1, gap_index + 1,
len(gaps), len(gaps),
len(transcript.segments), len(relative_chunks),
transcript.text, relative_chunks,
) )
gap_document = document_from_transcript( gap_inserted_count = 0
video_hash=video_hash, for chunk_index, (chunk_offset_start, chunk_offset_end) in enumerate(
duration_ms=end_ms - start_ms, relative_chunks
transcript=transcript, ):
language=language, chunk_start = start_ms + chunk_offset_start
audio_path=chunk_path, chunk_end = start_ms + chunk_offset_end
end_padding_ms=self.settings.moss_end_padding_ms, chunk_path = (
) Path(temp_dir) / f"gap-{gap_index}-chunk-{chunk_index}.wav"
shifted = _shift_gap_sentences( )
gap_document.sentences, try:
offset_ms=start_ms, extract_audio_segment(
limit_end_ms=end_ms, media_path,
) chunk_path,
if not shifted: start_ms=chunk_start,
logger.warning( duration_ms=chunk_end - chunk_start,
"Gap %d/%d produced no insertable sentences: " )
"transcript_segments=%d boundary_sentences=%d range=%d-%dms", chunk_samples, chunk_sample_rate = decode_audio_mono(chunk_path)
chunk_metrics = analyze_samples(
chunk_samples, chunk_sample_rate
)
except (AudioAnalysisError, RuntimeError) as exc:
logger.warning(
"Could not prepare gap chunk %d/%d %s-%sms: %s",
chunk_index + 1,
len(relative_chunks),
chunk_start,
chunk_end,
exc,
exc_info=True,
)
continue
if chunk_metrics.speech_duration_ms < GAP_REPAIR_MIN_SPEECH_MS:
logger.info(
"Skipping quiet gap chunk %d/%d: range=%d-%dms "
"speech_duration_ms=%d",
chunk_index + 1,
len(relative_chunks),
chunk_start,
chunk_end,
chunk_metrics.speech_duration_ms,
)
continue
languages_to_try = [language]
if not language or language.strip().lower() == "auto":
fallback = GAP_REPAIR_FALLBACK_LANGUAGE
if fallback not in languages_to_try:
languages_to_try.append(fallback)
transcript = None
for attempt_index, attempt_language in enumerate(languages_to_try):
try:
logger.info(
"Transcribing gap chunk %d/%d: gap=%d/%d "
"range=%d-%dms language=%s",
chunk_index + 1,
len(relative_chunks),
gap_index + 1,
len(gaps),
chunk_start,
chunk_end,
attempt_language or "auto",
)
transcript = self.transcriber.transcribe(
chunk_path, attempt_language
)
except Exception as exc:
logger.warning(
"Gap chunk transcription failed for %s %s-%sms: %s",
video_hash,
chunk_start,
chunk_end,
exc,
exc_info=True,
)
continue
logger.info(
"Gap chunk transcript returned: gap=%d/%d chunk=%d/%d "
"segments=%d text=%r",
gap_index + 1,
len(gaps),
chunk_index + 1,
len(relative_chunks),
len(transcript.segments),
transcript.text,
)
if transcript.segments:
break
if attempt_index + 1 < len(languages_to_try):
logger.warning(
"Gap chunk returned no segments; retrying with %s: "
"range=%d-%dms",
languages_to_try[attempt_index + 1],
chunk_start,
chunk_end,
)
if transcript is None or not transcript.segments:
logger.warning(
"Gap chunk produced no transcript segments: gap=%d/%d "
"chunk=%d/%d range=%d-%dms attempts=%s",
gap_index + 1,
len(gaps),
chunk_index + 1,
len(relative_chunks),
chunk_start,
chunk_end,
[item or "auto" for item in languages_to_try],
)
continue
gap_document = document_from_transcript(
video_hash=video_hash,
duration_ms=chunk_end - chunk_start,
transcript=transcript,
language=language,
audio_path=chunk_path,
end_padding_ms=self.settings.moss_end_padding_ms,
)
shifted = _shift_gap_sentences(
gap_document.sentences,
offset_ms=chunk_start,
limit_end_ms=chunk_end,
)
if not shifted:
logger.warning(
"Gap chunk produced no insertable sentences: gap=%d/%d "
"chunk=%d/%d transcript_segments=%d "
"boundary_sentences=%d range=%d-%dms",
gap_index + 1,
len(gaps),
chunk_index + 1,
len(relative_chunks),
len(transcript.segments),
len(gap_document.sentences),
chunk_start,
chunk_end,
)
for boundary in gap_document.sentences:
logger.warning(
"Candidate boundary before shift: start=%d end=%d text=%r",
boundary.start_ms,
boundary.end_ms,
boundary.text,
)
continue
inserted.extend(shifted)
gap_inserted_count += len(shifted)
logger.info(
"Gap chunk produced %d insertable sentences: gap=%d/%d "
"chunk=%d/%d range=%d-%dms",
len(shifted),
gap_index + 1, gap_index + 1,
len(gaps), len(gaps),
len(transcript.segments), chunk_index + 1,
len(gap_document.sentences), len(relative_chunks),
chunk_start,
chunk_end,
)
if gap_inserted_count:
transcribed_gaps += 1
logger.info(
"Gap %d/%d produced %d insertable sentences total: range=%d-%dms",
gap_index + 1,
len(gaps),
gap_inserted_count,
start_ms, start_ms,
end_ms, end_ms,
) )
for boundary in gap_document.sentences: else:
logger.warning( logger.warning(
"Candidate boundary before shift: start=%d end=%d text=%r", "Gap %d/%d produced no insertable sentences after chunking: "
boundary.start_ms, "range=%d-%dms chunks=%d",
boundary.end_ms, gap_index + 1,
boundary.text, len(gaps),
) start_ms,
if not shifted: end_ms,
continue len(relative_chunks),
transcribed_gaps += 1 )
inserted.extend(shifted)
logger.info(
"Gap %d/%d produced %d insertable sentences: range=%d-%dms",
gap_index + 1,
len(gaps),
len(shifted),
start_ms,
end_ms,
)
if not inserted: if not inserted:
logger.warning( logger.warning(
@@ -481,6 +615,93 @@ def _inset_gap(
return start, end return start, end
def _plan_gap_chunks(
samples,
sample_rate: int,
duration_ms: int,
) -> List[Tuple[int, int]]:
"""Split a long gap near silence so no ASR request exceeds Whisper's window."""
if duration_ms <= GAP_REPAIR_MAX_CHUNK_MS:
return [(0, duration_ms)]
chunks: List[Tuple[int, int]] = []
cursor = 0
while cursor < duration_ms:
remaining = duration_ms - cursor
if remaining <= GAP_REPAIR_MAX_CHUNK_MS:
chunks.append((cursor, duration_ms))
break
preferred = min(cursor + GAP_REPAIR_TARGET_CHUNK_MS, duration_ms)
lower = min(cursor + GAP_REPAIR_MIN_CHUNK_MS, preferred)
upper = min(cursor + GAP_REPAIR_MAX_CHUNK_MS, duration_ms)
cut = _find_silence_cut(
samples,
sample_rate,
lower,
upper,
preferred,
min_silence_ms=GAP_REPAIR_MIN_SPLIT_SILENCE_MS,
)
cut = max(cursor + 1, min(duration_ms, cut))
chunks.append((cursor, cut))
cursor = cut
return chunks
def _find_silence_cut(
samples,
sample_rate: int,
lower_ms: int,
upper_ms: int,
preferred_ms: int,
*,
min_silence_ms: int,
) -> int:
"""Choose the silence midpoint nearest ``preferred_ms``."""
frame_ms = 1000.0 * FRAME_SAMPLES / sample_rate
min_silence_frames = max(1, int(round(min_silence_ms / frame_ms)))
start_sample = max(0, int(lower_ms / 1000 * sample_rate))
end_sample = min(samples.size, int(upper_ms / 1000 * sample_rate) + 1)
if end_sample <= start_sample:
return preferred_ms
try:
speech = speech_frame_mask(samples[start_sample:end_sample], sample_rate)
except AudioAnalysisError:
return preferred_ms
run_start: Optional[int] = None
candidates: List[Tuple[int, int]] = []
for index, is_speech in enumerate(speech):
if not is_speech:
if run_start is None:
run_start = index
continue
if run_start is not None:
candidates.append((run_start, index))
run_start = None
if run_start is not None:
candidates.append((run_start, len(speech)))
valid = [
(run_start, run_end)
for run_start, run_end in candidates
if run_end - run_start >= min_silence_frames
]
if not valid:
return preferred_ms
def distance(run: Tuple[int, int]) -> float:
midpoint_frame = (run[0] + run[1]) / 2.0
midpoint_ms = lower_ms + midpoint_frame * frame_ms
return abs(midpoint_ms - preferred_ms)
run_start, run_end = min(valid, key=distance)
midpoint_ms = lower_ms + ((run_start + run_end) / 2.0) * frame_ms
return int(round(midpoint_ms))
def _shift_gap_sentences( def _shift_gap_sentences(
sentences: Sequence[SentenceBoundary], sentences: Sequence[SentenceBoundary],
*, *,

17
sentence_api/tests/test_processing.py
View File

@@ -5,6 +5,7 @@ from sentence_api.processing import (
VideoProcessor, VideoProcessor,
document_from_transcript, document_from_transcript,
find_sentence_gaps, find_sentence_gaps,
_plan_gap_chunks,
split_sentence_text, split_sentence_text,
) )
from sentence_api.models import SentenceBoundary, SentenceBoundaryDocument from sentence_api.models import SentenceBoundary, SentenceBoundaryDocument
@@ -417,6 +418,22 @@ def test_find_sentence_gaps_ignores_short_pauses():
assert find_sentence_gaps(document) == [] assert find_sentence_gaps(document) == []
def test_plan_gap_chunks_splits_long_gap_at_silence():
samples = np.concatenate(
[_speech(14.0), _silence(0.6), _speech(16.0)]
).astype(np.float32)
chunks = _plan_gap_chunks(samples, SAMPLE_RATE, int(samples.size / SAMPLE_RATE * 1000))
assert len(chunks) == 2
first_start, first_end = chunks[0]
second_start, second_end = chunks[1]
assert first_start == 0
assert 13_000 <= first_end <= 17_000
assert second_start == first_end
assert second_end == int(samples.size / SAMPLE_RATE * 1000)
assert all(end - start <= 20_000 for start, end in chunks)
def test_repair_gaps_inserts_transcribed_sentences_and_keeps_existing(tmp_path, monkeypatch): def test_repair_gaps_inserts_transcribed_sentences_and_keeps_existing(tmp_path, monkeypatch):
from dataclasses import replace from dataclasses import replace