continue fixing

sentence_api/transcription.py

@@ -4,7 +4,7 @@ import re
 import wave
 from dataclasses import dataclass
 from pathlib import Path
-from typing import List, Optional, Protocol
+from typing import List, Optional, Protocol, Tuple
 
 import httpx
 import numpy as np
@@ -240,6 +240,11 @@ def _parse_word_timestamps(raw_words) -> Optional[List[WordTimestamp]]:
 
 
 _PUNCTUATION_BOUNDARY = re.compile(r"[.?]+")
+_ABBREVIATION_RE = re.compile(
+    r"(?i)\b(?:a\.m\.|p\.m\.|mr\.|mrs\.|ms\.|dr\.|prof\.|st\.|vs\.|etc\.|"
+    r"e\.g\.|i\.e\.|no\.|approx\.|fig\.|inc\.|ltd\.|jr\.|sr\.|u\.s\.|u\.k\.)"
+    r"(?![A-Za-z0-9])"
+)
 
 
 def split_sentences_at_punctuation(
@@ -278,19 +283,21 @@ def split_sentences_at_punctuation(
             continue
         if not pieces:
             start_seconds = segment.start_seconds
-        seg_pieces: List[str] = []
+        seg_pieces: List[Tuple[str, bool]] = []
         cursor = 0
         for match in _PUNCTUATION_BOUNDARY.finditer(text):
+            if not _is_sentence_end_run(text, match.start(), match.end()):
+                continue
             piece = text[cursor:match.end()].strip()
             cursor = match.end()
             if piece.strip(".?").strip():
-                seg_pieces.append(piece)
+                seg_pieces.append((piece, True))
         tail = text[cursor:].strip()
         if tail:
-            seg_pieces.append(tail)
+            seg_pieces.append((tail, False))
         if not seg_pieces:
             continue
-        lengths = [len(piece) for piece in seg_pieces]
+        lengths = [len(piece) for piece, _ in seg_pieces]
         ends: List[float] = []
         if segment.words and len(segment.words) >= len(seg_pieces):
             ends = _word_boundary_ends(segment, lengths)
@@ -299,9 +306,9 @@ def split_sentences_at_punctuation(
         if not ends:
             ends = _proportional_ends(segment, lengths)
         last_segment_end = segment.end_seconds
-        for index, piece in enumerate(seg_pieces):
+        for index, (piece, is_boundary) in enumerate(seg_pieces):
             pieces.append(piece)
-            if piece[-1] in ".?":
+            if is_boundary:
                 end_seconds = ends[index] if index < len(ends) else segment.end_seconds
                 flush(end_seconds)
                 start_seconds = end_seconds
@@ -310,6 +317,40 @@ def split_sentences_at_punctuation(
     return sentences
 
 
+def _is_sentence_end_run(text: str, run_start: int, run_end: int) -> bool:
+    """Decide whether a "." / "?" run really ends a sentence."""
+    punctuation = text[run_start:run_end]
+    if "?" in punctuation:
+        return True
+    if (
+        run_start > 0
+        and run_end < len(text)
+        and text[run_start - 1].isdigit()
+        and text[run_end].isdigit()
+    ):
+        return False  # decimal point, e.g. "0.4"
+    if len(punctuation) > 1:
+        return True  # ellipsis or "!?" etc.
+    if _is_abbreviation_run(text, run_start, run_end):
+        return False  # e.g. "a.m.", "p.m.", "Mr."
+    rest = text[run_end:].lstrip()
+    if rest and rest[0].islower():
+        return False  # period followed by a lowercase word is an abbreviation
+    return True
+
+
+def _is_abbreviation_run(text: str, run_start: int, run_end: int) -> bool:
+    for match in _ABBREVIATION_RE.finditer(text):
+        if not (match.start() < run_end <= match.end()):
+            continue
+        if match.group(0).lower() == "no.":
+            rest = text[match.end():].lstrip()
+            if not rest or not rest[0].isdigit():
+                return False  # "No." as a reply ends the sentence
+        return True
+    return False
+
+
 def _word_boundary_ends(
     segment: TranscriptionSegment, lengths: List[int]
 ) -> List[float]: