diff --git a/sentence_api/processing.py b/sentence_api/processing.py index f4a717c..652e06e 100644 --- a/sentence_api/processing.py +++ b/sentence_api/processing.py @@ -27,7 +27,7 @@ from .transcription import ( logger = logging.getLogger(__name__) -MOSS_ALGORITHM_VERSION = "moss-punctuation-v3" +MOSS_ALGORITHM_VERSION = "moss-punctuation-v4" SILENCE_ALGORITHM_VERSION = "silence-rms-v1" _CUES_ID = b"\x1e\x53\xb6\x80" _CLUSTER_ID = b"\x1f\x43\xb6\x75" diff --git a/sentence_api/repository.py b/sentence_api/repository.py index bf46da3..6f9fb04 100644 --- a/sentence_api/repository.py +++ b/sentence_api/repository.py @@ -244,7 +244,24 @@ class VideoRepository: ) def mark_processing(self, video_hash: str) -> None: - self._update_status(video_hash, "processing", None) + normalized_hash = normalize_video_hash(video_hash) + with self._connect() as connection: + connection.execute( + "DELETE FROM sentences WHERE video_hash = ?", + (normalized_hash,), + ) + connection.execute( + "DELETE FROM moss_segments WHERE video_hash = ?", + (normalized_hash,), + ) + connection.execute( + """ + UPDATE videos + SET status = ?, error_message = NULL, transcription = NULL, updated_at = ? + WHERE video_hash = ? + """, + ("processing", utc_now(), normalized_hash), + ) def mark_failed(self, video_hash: str, message: str) -> None: self._update_status(video_hash, "failed", message[:4000]) diff --git a/sentence_api/tests/test_processing.py b/sentence_api/tests/test_processing.py index 1b6d616..21ad3ed 100644 --- a/sentence_api/tests/test_processing.py +++ b/sentence_api/tests/test_processing.py @@ -323,3 +323,61 @@ def test_reprocess_without_moss_clears_original_segments(tmp_path, monkeypatch): VideoProcessor(settings, repository, UnavailableTranscriber()).process(VIDEO_HASH) assert repository.get_moss_segments(VIDEO_HASH) == [] + +def test_mark_processing_clears_previous_sentence_data(tmp_path): + repository = VideoRepository(tmp_path / "oral.sqlite3") + repository.upsert_upload( + video_hash=VIDEO_HASH, + title="MOSS lesson", + filename="lesson.mp4", + stored_filename=f"{VIDEO_HASH}.mp4", + content_type="video/mp4", + size_bytes=5, + language="en", + ) + repository.save_processing_result( + SentenceBoundaryDocument( + video_hash=VIDEO_HASH, + duration_ms=2400, + algorithm_version=MOSS_ALGORITHM_VERSION, + sentences=[ + SentenceBoundary( + index=0, + start_ms=0, + end_ms=1200, + text="Old first.", + language="en", + reference_speech_duration_ms=1000, + ), + SentenceBoundary( + index=1, + start_ms=1200, + end_ms=2400, + text="Old second.", + language="en", + reference_speech_duration_ms=1000, + ), + ], + ), + "Old first. Old second.", + moss_segments=[ + { + "start_seconds": 0.0, + "end_seconds": 1.2, + "text": "Old first.", + "speaker": "S01", + } + ], + ) + + repository.mark_processing(VIDEO_HASH) + + video = repository.get_video(VIDEO_HASH) + document = repository.get_document(VIDEO_HASH) + assert video["status"] == "processing" + assert video["transcription"] is None + assert video["sentence_count"] == 0 + assert document is not None + assert document.sentences == [] + assert repository.get_moss_segments(VIDEO_HASH) == [] + diff --git a/sentence_api/tests/test_transcription.py b/sentence_api/tests/test_transcription.py index 2501050..3d468e8 100644 --- a/sentence_api/tests/test_transcription.py +++ b/sentence_api/tests/test_transcription.py @@ -423,6 +423,36 @@ def test_split_sentences_ignores_am_pm_abbreviations(): ] +def test_split_sentences_ignores_clock_times_with_pm_period(): + compact = TranscriptionSegment( + start_seconds=0.0, + end_seconds=6.0, + text="The briefing is at 3:00 PM. Eastern officials will attend later.", + ) + assert [s.text for s in split_sentences_at_punctuation([compact])] == [ + "The briefing is at 3:00 PM. Eastern officials will attend later." + ] + + spoken = TranscriptionSegment( + start_seconds=0.0, + end_seconds=6.0, + text="We start at 5 pm. Eastern time and then break.", + ) + assert [s.text for s in split_sentences_at_punctuation([spoken])] == [ + "We start at 5 pm. Eastern time and then break." + ] + + morning = TranscriptionSegment( + start_seconds=0.0, + end_seconds=6.0, + text="It begins at 9:30 AM. Please arrive early.", + ) + assert [s.text for s in split_sentences_at_punctuation([morning])] == [ + "It begins at 9:30 AM.", + "Please arrive early.", + ] + + def test_split_sentences_merges_segment_ending_with_abbreviation(): segments = [ TranscriptionSegment(0.0, 5.0, "The meeting ends at 5 p.m."), @@ -460,6 +490,77 @@ def test_split_sentences_ignores_common_abbreviations(): ] +def test_split_sentences_merges_same_speaker_line_wraps(): + segments = [ + TranscriptionSegment( + 11.18, + 15.45, + "[S01]Let's begin with some scheduling announcements for the remainder of President Trump's very", + ), + TranscriptionSegment(15.45, 16.29, "[S01]busy week."), + TranscriptionSegment( + 16.29, + 21.8, + "[S01]Today at 3:00 PM, President Trump will host a Black History Month reception in the East", + ), + TranscriptionSegment(21.8, 23.29, "[S01]Room of the White House."), + ] + sentences = split_sentences_at_punctuation(segments) + assert [s.text for s in sentences] == [ + ( + "Let's begin with some scheduling announcements for the remainder of " + "President Trump's very busy week." + ), + ( + "Today at 3:00 PM, President Trump will host a Black History Month " + "reception in the East Room of the White House." + ), + ] + assert sentences[0].start_seconds == 11.18 + assert sentences[0].end_seconds == 16.29 + assert sentences[1].start_seconds == 16.29 + assert sentences[1].end_seconds == 23.29 + + +def test_split_sentences_splits_when_speaker_changes(): + segments = [ + TranscriptionSegment(4.63, 5.93, "[S01]Hello, everybody."), + TranscriptionSegment(6.0, 6.53, "[S02]Hello."), + TranscriptionSegment(6.56, 7.83, "[S01]Good afternoon."), + ] + sentences = split_sentences_at_punctuation(segments) + assert [s.text for s in sentences] == [ + "Hello, everybody.", + "Hello.", + "Good afternoon.", + ] + + +def test_split_sentences_keeps_same_speaker_sentences_apart(): + segments = [ + TranscriptionSegment(6.56, 7.83, "[S01]Good afternoon."), + TranscriptionSegment(7.93, 9.86, "[S01]Thank you for your patience, as always."), + ] + sentences = split_sentences_at_punctuation(segments) + assert [s.text for s in sentences] == [ + "Good afternoon.", + "Thank you for your patience, as always.", + ] + + +def test_split_sentences_ignores_initials_and_version_numbers(): + segment = TranscriptionSegment( + start_seconds=0.0, + end_seconds=8.0, + text="Donald J. Trump discussed version 1.9 today. Then the meeting ended.", + ) + sentences = split_sentences_at_punctuation([segment]) + assert [s.text for s in sentences] == [ + "Donald J. Trump discussed version 1.9 today.", + "Then the meeting ended.", + ] + + def test_post_audio_requests_word_timestamps_and_falls_back(monkeypatch): calls = [] diff --git a/sentence_api/transcription.py b/sentence_api/transcription.py index ee6875a..ef95a57 100644 --- a/sentence_api/transcription.py +++ b/sentence_api/transcription.py @@ -246,9 +246,18 @@ _PUNCTUATION_BOUNDARY = re.compile(r"[.?]+|,+") MIN_WORDS_FOR_COMMA_SPLIT = 50 _ABBREVIATION_RE = re.compile( r"(?i)\b(?:a\.m\.|p\.m\.|mr\.|mrs\.|ms\.|dr\.|prof\.|st\.|vs\.|etc\.|" - r"e\.g\.|i\.e\.|no\.|approx\.|fig\.|inc\.|ltd\.|jr\.|sr\.|u\.s\.|u\.k\.)" + r"e\.g\.|i\.e\.|no\.|approx\.|fig\.|inc\.|ltd\.|jr\.|sr\.|u\.s\.|u\.k\.|" + r"ph\.d\.|m\.d\.|b\.a\.|m\.a\.|d\.c\.|p\.s\.)" r"(?![A-Za-z0-9])" ) +_TIME_MERIDIEM_RE = re.compile( + r"(?i)\b(?:[01]?\d|2[0-3])(?::[0-5]\d)?\s*(?:a\.m\.|p\.m\.|am\.|pm\.)" + r"(?![A-Za-z0-9])" +) +_TIME_CONTINUATION_RE = re.compile( + r"(?i)^(eastern|western|central|pacific|mountain|time|" + r"et|est|edt|pt|pst|pdt|ct|cst|cdt|mt|mst|mdt|gmt|utc|local)\b" +) _SPEAKER_TAG_RE = re.compile(r"\[(S\d+)\]") @@ -319,8 +328,10 @@ def split_sentences_at_punctuation( are also eligible boundaries, but only when the text before the comma has enough words; otherwise the comma stays with the following text. - Inline ``[S01]`` speaker tags are stripped and each speaker turn always - starts a new sentence, even when no sentence-ending punctuation exists. + Inline ``[S01]`` speaker tags are stripped. A new sentence starts when the + speaker actually changes; same-speaker line wraps stay merged until + sentence punctuation. Decimal points, initials, and common abbreviations + are not treated as sentence ends. """ sentences: List[TranscriptionSegment] = [] pieces: List[str] = [] @@ -354,10 +365,14 @@ def split_sentences_at_punctuation( text = segment.text.strip() if not text: continue - # Each speaker-tagged sub-segment always starts a new sentence, - # regardless of whether the speaker actually changed — MOSS emits - # a tag per phrase/line even for the same speaker. - if pieces and segment.speaker is not None: + # MOSS often tags every phrase/line with [S01] even when the speaker + # did not change. Only a real speaker change starts a new sentence. + if ( + pieces + and segment.speaker is not None + and current_speaker is not None + and segment.speaker != current_speaker + ): flush(segment.start_seconds) if not pieces: start_seconds = segment.start_seconds @@ -418,31 +433,70 @@ def _is_sentence_end_run(text: str, run_start: int, run_end: int) -> bool: punctuation = text[run_start:run_end] if "?" in punctuation: return True - if ( - run_start > 0 - and run_end < len(text) - and text[run_start - 1].isdigit() - and text[run_end].isdigit() - ): - return False # decimal point, e.g. "0.4" + if _is_decimal_point(text, run_start, run_end): + return False if len(punctuation) > 1: return True # ellipsis or "!?" etc. if _is_abbreviation_run(text, run_start, run_end): return False # e.g. "a.m.", "p.m.", "Mr." + if _is_time_meridiem_run(text, run_start, run_end): + return False # e.g. "3:00 PM.", "5 pm." + if _is_single_letter_initial(text, run_start, run_end): + return False # e.g. "Donald J. Trump" rest = text[run_end:].lstrip() if rest and rest[0].islower(): return False # period followed by a lowercase word is an abbreviation return True +def _is_decimal_point(text: str, run_start: int, run_end: int) -> bool: + if run_end - run_start != 1 or text[run_start] != ".": + return False + if run_start == 0 or run_end >= len(text): + return False + return text[run_start - 1].isdigit() and text[run_end].isdigit() + + +def _is_single_letter_initial(text: str, run_start: int, run_end: int) -> bool: + if run_end - run_start != 1 or text[run_start] != ".": + return False + if run_start == 0 or not text[run_start - 1].isalpha(): + return False + return run_start == 1 or not text[run_start - 2].isalpha() + + +def _meridiem_continues(text: str, after: int) -> bool: + rest = text[after:].lstrip() + if not rest: + return True + if rest[0].islower(): + return True + return _TIME_CONTINUATION_RE.match(rest) is not None + + +def _is_time_meridiem_run(text: str, run_start: int, run_end: int) -> bool: + for match in _TIME_MERIDIEM_RE.finditer(text): + if not (match.start() < run_end <= match.end()): + continue + if run_end < match.end(): + return True + return _meridiem_continues(text, match.end()) + return False + + def _is_abbreviation_run(text: str, run_start: int, run_end: int) -> bool: for match in _ABBREVIATION_RE.finditer(text): if not (match.start() < run_end <= match.end()): continue - if match.group(0).lower() == "no.": + token = match.group(0).lower() + if token == "no.": rest = text[match.end():].lstrip() if not rest or not rest[0].isdigit(): return False # "No." as a reply ends the sentence + if token in {"a.m.", "p.m."}: + if run_end < match.end(): + return True + return _meridiem_continues(text, match.end()) return True return False