fixed a bug

sentence_api/processing.py

@@ -27,7 +27,7 @@ from .transcription import (
 
 
 logger = logging.getLogger(__name__)
-MOSS_ALGORITHM_VERSION = "moss-punctuation-v3"
+MOSS_ALGORITHM_VERSION = "moss-punctuation-v4"
 SILENCE_ALGORITHM_VERSION = "silence-rms-v1"
 _CUES_ID = b"\x1e\x53\xb6\x80"
 _CLUSTER_ID = b"\x1f\x43\xb6\x75"

sentence_api/repository.py

@@ -244,7 +244,24 @@ class VideoRepository:
             )
 
     def mark_processing(self, video_hash: str) -> None:
-        self._update_status(video_hash, "processing", None)
+        normalized_hash = normalize_video_hash(video_hash)
+        with self._connect() as connection:
+            connection.execute(
+                "DELETE FROM sentences WHERE video_hash = ?",
+                (normalized_hash,),
+            )
+            connection.execute(
+                "DELETE FROM moss_segments WHERE video_hash = ?",
+                (normalized_hash,),
+            )
+            connection.execute(
+                """
+                UPDATE videos
+                SET status = ?, error_message = NULL, transcription = NULL, updated_at = ?
+                WHERE video_hash = ?
+                """,
+                ("processing", utc_now(), normalized_hash),
+            )
 
     def mark_failed(self, video_hash: str, message: str) -> None:
         self._update_status(video_hash, "failed", message[:4000])

sentence_api/tests/test_processing.py

@@ -323,3 +323,61 @@ def test_reprocess_without_moss_clears_original_segments(tmp_path, monkeypatch):
     VideoProcessor(settings, repository, UnavailableTranscriber()).process(VIDEO_HASH)
     assert repository.get_moss_segments(VIDEO_HASH) == []
 
+
+def test_mark_processing_clears_previous_sentence_data(tmp_path):
+    repository = VideoRepository(tmp_path / "oral.sqlite3")
+    repository.upsert_upload(
+        video_hash=VIDEO_HASH,
+        title="MOSS lesson",
+        filename="lesson.mp4",
+        stored_filename=f"{VIDEO_HASH}.mp4",
+        content_type="video/mp4",
+        size_bytes=5,
+        language="en",
+    )
+    repository.save_processing_result(
+        SentenceBoundaryDocument(
+            video_hash=VIDEO_HASH,
+            duration_ms=2400,
+            algorithm_version=MOSS_ALGORITHM_VERSION,
+            sentences=[
+                SentenceBoundary(
+                    index=0,
+                    start_ms=0,
+                    end_ms=1200,
+                    text="Old first.",
+                    language="en",
+                    reference_speech_duration_ms=1000,
+                ),
+                SentenceBoundary(
+                    index=1,
+                    start_ms=1200,
+                    end_ms=2400,
+                    text="Old second.",
+                    language="en",
+                    reference_speech_duration_ms=1000,
+                ),
+            ],
+        ),
+        "Old first. Old second.",
+        moss_segments=[
+            {
+                "start_seconds": 0.0,
+                "end_seconds": 1.2,
+                "text": "Old first.",
+                "speaker": "S01",
+            }
+        ],
+    )
+
+    repository.mark_processing(VIDEO_HASH)
+
+    video = repository.get_video(VIDEO_HASH)
+    document = repository.get_document(VIDEO_HASH)
+    assert video["status"] == "processing"
+    assert video["transcription"] is None
+    assert video["sentence_count"] == 0
+    assert document is not None
+    assert document.sentences == []
+    assert repository.get_moss_segments(VIDEO_HASH) == []
+

sentence_api/tests/test_transcription.py

@@ -423,6 +423,36 @@ def test_split_sentences_ignores_am_pm_abbreviations():
     ]
 
 
+def test_split_sentences_ignores_clock_times_with_pm_period():
+    compact = TranscriptionSegment(
+        start_seconds=0.0,
+        end_seconds=6.0,
+        text="The briefing is at 3:00 PM. Eastern officials will attend later.",
+    )
+    assert [s.text for s in split_sentences_at_punctuation([compact])] == [
+        "The briefing is at 3:00 PM. Eastern officials will attend later."
+    ]
+
+    spoken = TranscriptionSegment(
+        start_seconds=0.0,
+        end_seconds=6.0,
+        text="We start at 5 pm. Eastern time and then break.",
+    )
+    assert [s.text for s in split_sentences_at_punctuation([spoken])] == [
+        "We start at 5 pm. Eastern time and then break."
+    ]
+
+    morning = TranscriptionSegment(
+        start_seconds=0.0,
+        end_seconds=6.0,
+        text="It begins at 9:30 AM. Please arrive early.",
+    )
+    assert [s.text for s in split_sentences_at_punctuation([morning])] == [
+        "It begins at 9:30 AM.",
+        "Please arrive early.",
+    ]
+
+
 def test_split_sentences_merges_segment_ending_with_abbreviation():
     segments = [
         TranscriptionSegment(0.0, 5.0, "The meeting ends at 5 p.m."),
@@ -460,6 +490,77 @@ def test_split_sentences_ignores_common_abbreviations():
     ]
 
 
+def test_split_sentences_merges_same_speaker_line_wraps():
+    segments = [
+        TranscriptionSegment(
+            11.18,
+            15.45,
+            "[S01]Let's begin with some scheduling announcements for the remainder of President Trump's very",
+        ),
+        TranscriptionSegment(15.45, 16.29, "[S01]busy week."),
+        TranscriptionSegment(
+            16.29,
+            21.8,
+            "[S01]Today at 3:00 PM, President Trump will host a Black History Month reception in the East",
+        ),
+        TranscriptionSegment(21.8, 23.29, "[S01]Room of the White House."),
+    ]
+    sentences = split_sentences_at_punctuation(segments)
+    assert [s.text for s in sentences] == [
+        (
+            "Let's begin with some scheduling announcements for the remainder of "
+            "President Trump's very busy week."
+        ),
+        (
+            "Today at 3:00 PM, President Trump will host a Black History Month "
+            "reception in the East Room of the White House."
+        ),
+    ]
+    assert sentences[0].start_seconds == 11.18
+    assert sentences[0].end_seconds == 16.29
+    assert sentences[1].start_seconds == 16.29
+    assert sentences[1].end_seconds == 23.29
+
+
+def test_split_sentences_splits_when_speaker_changes():
+    segments = [
+        TranscriptionSegment(4.63, 5.93, "[S01]Hello, everybody."),
+        TranscriptionSegment(6.0, 6.53, "[S02]Hello."),
+        TranscriptionSegment(6.56, 7.83, "[S01]Good afternoon."),
+    ]
+    sentences = split_sentences_at_punctuation(segments)
+    assert [s.text for s in sentences] == [
+        "Hello, everybody.",
+        "Hello.",
+        "Good afternoon.",
+    ]
+
+
+def test_split_sentences_keeps_same_speaker_sentences_apart():
+    segments = [
+        TranscriptionSegment(6.56, 7.83, "[S01]Good afternoon."),
+        TranscriptionSegment(7.93, 9.86, "[S01]Thank you for your patience, as always."),
+    ]
+    sentences = split_sentences_at_punctuation(segments)
+    assert [s.text for s in sentences] == [
+        "Good afternoon.",
+        "Thank you for your patience, as always.",
+    ]
+
+
+def test_split_sentences_ignores_initials_and_version_numbers():
+    segment = TranscriptionSegment(
+        start_seconds=0.0,
+        end_seconds=8.0,
+        text="Donald J. Trump discussed version 1.9 today. Then the meeting ended.",
+    )
+    sentences = split_sentences_at_punctuation([segment])
+    assert [s.text for s in sentences] == [
+        "Donald J. Trump discussed version 1.9 today.",
+        "Then the meeting ended.",
+    ]
+
+
 def test_post_audio_requests_word_timestamps_and_falls_back(monkeypatch):
     calls = []
 

sentence_api/transcription.py

@@ -246,9 +246,18 @@ _PUNCTUATION_BOUNDARY = re.compile(r"[.?]+|,+")
 MIN_WORDS_FOR_COMMA_SPLIT = 50
 _ABBREVIATION_RE = re.compile(
     r"(?i)\b(?:a\.m\.|p\.m\.|mr\.|mrs\.|ms\.|dr\.|prof\.|st\.|vs\.|etc\.|"
-    r"e\.g\.|i\.e\.|no\.|approx\.|fig\.|inc\.|ltd\.|jr\.|sr\.|u\.s\.|u\.k\.)"
+    r"e\.g\.|i\.e\.|no\.|approx\.|fig\.|inc\.|ltd\.|jr\.|sr\.|u\.s\.|u\.k\.|"
+    r"ph\.d\.|m\.d\.|b\.a\.|m\.a\.|d\.c\.|p\.s\.)"
     r"(?![A-Za-z0-9])"
 )
+_TIME_MERIDIEM_RE = re.compile(
+    r"(?i)\b(?:[01]?\d|2[0-3])(?::[0-5]\d)?\s*(?:a\.m\.|p\.m\.|am\.|pm\.)"
+    r"(?![A-Za-z0-9])"
+)
+_TIME_CONTINUATION_RE = re.compile(
+    r"(?i)^(eastern|western|central|pacific|mountain|time|"
+    r"et|est|edt|pt|pst|pdt|ct|cst|cdt|mt|mst|mdt|gmt|utc|local)\b"
+)
 _SPEAKER_TAG_RE = re.compile(r"\[(S\d+)\]")
 
 
@@ -319,8 +328,10 @@ def split_sentences_at_punctuation(
     are also eligible boundaries, but only when the text before the comma has
     enough words; otherwise the comma stays with the following text.
 
-    Inline ``[S01]`` speaker tags are stripped and each speaker turn always
+    Inline ``[S01]`` speaker tags are stripped. A new sentence starts when the
-    starts a new sentence, even when no sentence-ending punctuation exists.
+    speaker actually changes; same-speaker line wraps stay merged until
+    sentence punctuation. Decimal points, initials, and common abbreviations
+    are not treated as sentence ends.
     """
     sentences: List[TranscriptionSegment] = []
     pieces: List[str] = []
@@ -354,10 +365,14 @@ def split_sentences_at_punctuation(
         text = segment.text.strip()
         if not text:
             continue
-        # Each speaker-tagged sub-segment always starts a new sentence,
+        # MOSS often tags every phrase/line with [S01] even when the speaker
-        # regardless of whether the speaker actually changed — MOSS emits
+        # did not change. Only a real speaker change starts a new sentence.
-        # a tag per phrase/line even for the same speaker.
+        if (
-        if pieces and segment.speaker is not None:
+            pieces
+            and segment.speaker is not None
+            and current_speaker is not None
+            and segment.speaker != current_speaker
+        ):
             flush(segment.start_seconds)
         if not pieces:
             start_seconds = segment.start_seconds
@@ -418,31 +433,70 @@ def _is_sentence_end_run(text: str, run_start: int, run_end: int) -> bool:
     punctuation = text[run_start:run_end]
     if "?" in punctuation:
         return True
-    if (
+    if _is_decimal_point(text, run_start, run_end):
-        run_start > 0
+        return False
-        and run_end < len(text)
-        and text[run_start - 1].isdigit()
-        and text[run_end].isdigit()
-    ):
-        return False  # decimal point, e.g. "0.4"
     if len(punctuation) > 1:
         return True  # ellipsis or "!?" etc.
     if _is_abbreviation_run(text, run_start, run_end):
         return False  # e.g. "a.m.", "p.m.", "Mr."
+    if _is_time_meridiem_run(text, run_start, run_end):
+        return False  # e.g. "3:00 PM.", "5 pm."
+    if _is_single_letter_initial(text, run_start, run_end):
+        return False  # e.g. "Donald J. Trump"
     rest = text[run_end:].lstrip()
     if rest and rest[0].islower():
         return False  # period followed by a lowercase word is an abbreviation
     return True
 
 
+def _is_decimal_point(text: str, run_start: int, run_end: int) -> bool:
+    if run_end - run_start != 1 or text[run_start] != ".":
+        return False
+    if run_start == 0 or run_end >= len(text):
+        return False
+    return text[run_start - 1].isdigit() and text[run_end].isdigit()
+
+
+def _is_single_letter_initial(text: str, run_start: int, run_end: int) -> bool:
+    if run_end - run_start != 1 or text[run_start] != ".":
+        return False
+    if run_start == 0 or not text[run_start - 1].isalpha():
+        return False
+    return run_start == 1 or not text[run_start - 2].isalpha()
+
+
+def _meridiem_continues(text: str, after: int) -> bool:
+    rest = text[after:].lstrip()
+    if not rest:
+        return True
+    if rest[0].islower():
+        return True
+    return _TIME_CONTINUATION_RE.match(rest) is not None
+
+
+def _is_time_meridiem_run(text: str, run_start: int, run_end: int) -> bool:
+    for match in _TIME_MERIDIEM_RE.finditer(text):
+        if not (match.start() < run_end <= match.end()):
+            continue
+        if run_end < match.end():
+            return True
+        return _meridiem_continues(text, match.end())
+    return False
+
+
 def _is_abbreviation_run(text: str, run_start: int, run_end: int) -> bool:
     for match in _ABBREVIATION_RE.finditer(text):
         if not (match.start() < run_end <= match.end()):
             continue
-        if match.group(0).lower() == "no.":
+        token = match.group(0).lower()
+        if token == "no.":
             rest = text[match.end():].lstrip()
             if not rest or not rest[0].isdigit():
                 return False  # "No." as a reply ends the sentence
+        if token in {"a.m.", "p.m."}:
+            if run_end < match.end():
+                return True
+            return _meridiem_continues(text, match.end())
         return True
     return False