fixed a bug
This commit is contained in:
@@ -246,9 +246,18 @@ _PUNCTUATION_BOUNDARY = re.compile(r"[.?]+|,+")
|
||||
MIN_WORDS_FOR_COMMA_SPLIT = 50
|
||||
_ABBREVIATION_RE = re.compile(
|
||||
r"(?i)\b(?:a\.m\.|p\.m\.|mr\.|mrs\.|ms\.|dr\.|prof\.|st\.|vs\.|etc\.|"
|
||||
r"e\.g\.|i\.e\.|no\.|approx\.|fig\.|inc\.|ltd\.|jr\.|sr\.|u\.s\.|u\.k\.)"
|
||||
r"e\.g\.|i\.e\.|no\.|approx\.|fig\.|inc\.|ltd\.|jr\.|sr\.|u\.s\.|u\.k\.|"
|
||||
r"ph\.d\.|m\.d\.|b\.a\.|m\.a\.|d\.c\.|p\.s\.)"
|
||||
r"(?![A-Za-z0-9])"
|
||||
)
|
||||
_TIME_MERIDIEM_RE = re.compile(
|
||||
r"(?i)\b(?:[01]?\d|2[0-3])(?::[0-5]\d)?\s*(?:a\.m\.|p\.m\.|am\.|pm\.)"
|
||||
r"(?![A-Za-z0-9])"
|
||||
)
|
||||
_TIME_CONTINUATION_RE = re.compile(
|
||||
r"(?i)^(eastern|western|central|pacific|mountain|time|"
|
||||
r"et|est|edt|pt|pst|pdt|ct|cst|cdt|mt|mst|mdt|gmt|utc|local)\b"
|
||||
)
|
||||
_SPEAKER_TAG_RE = re.compile(r"\[(S\d+)\]")
|
||||
|
||||
|
||||
@@ -319,8 +328,10 @@ def split_sentences_at_punctuation(
|
||||
are also eligible boundaries, but only when the text before the comma has
|
||||
enough words; otherwise the comma stays with the following text.
|
||||
|
||||
Inline ``[S01]`` speaker tags are stripped and each speaker turn always
|
||||
starts a new sentence, even when no sentence-ending punctuation exists.
|
||||
Inline ``[S01]`` speaker tags are stripped. A new sentence starts when the
|
||||
speaker actually changes; same-speaker line wraps stay merged until
|
||||
sentence punctuation. Decimal points, initials, and common abbreviations
|
||||
are not treated as sentence ends.
|
||||
"""
|
||||
sentences: List[TranscriptionSegment] = []
|
||||
pieces: List[str] = []
|
||||
@@ -354,10 +365,14 @@ def split_sentences_at_punctuation(
|
||||
text = segment.text.strip()
|
||||
if not text:
|
||||
continue
|
||||
# Each speaker-tagged sub-segment always starts a new sentence,
|
||||
# regardless of whether the speaker actually changed — MOSS emits
|
||||
# a tag per phrase/line even for the same speaker.
|
||||
if pieces and segment.speaker is not None:
|
||||
# MOSS often tags every phrase/line with [S01] even when the speaker
|
||||
# did not change. Only a real speaker change starts a new sentence.
|
||||
if (
|
||||
pieces
|
||||
and segment.speaker is not None
|
||||
and current_speaker is not None
|
||||
and segment.speaker != current_speaker
|
||||
):
|
||||
flush(segment.start_seconds)
|
||||
if not pieces:
|
||||
start_seconds = segment.start_seconds
|
||||
@@ -418,31 +433,70 @@ def _is_sentence_end_run(text: str, run_start: int, run_end: int) -> bool:
|
||||
punctuation = text[run_start:run_end]
|
||||
if "?" in punctuation:
|
||||
return True
|
||||
if (
|
||||
run_start > 0
|
||||
and run_end < len(text)
|
||||
and text[run_start - 1].isdigit()
|
||||
and text[run_end].isdigit()
|
||||
):
|
||||
return False # decimal point, e.g. "0.4"
|
||||
if _is_decimal_point(text, run_start, run_end):
|
||||
return False
|
||||
if len(punctuation) > 1:
|
||||
return True # ellipsis or "!?" etc.
|
||||
if _is_abbreviation_run(text, run_start, run_end):
|
||||
return False # e.g. "a.m.", "p.m.", "Mr."
|
||||
if _is_time_meridiem_run(text, run_start, run_end):
|
||||
return False # e.g. "3:00 PM.", "5 pm."
|
||||
if _is_single_letter_initial(text, run_start, run_end):
|
||||
return False # e.g. "Donald J. Trump"
|
||||
rest = text[run_end:].lstrip()
|
||||
if rest and rest[0].islower():
|
||||
return False # period followed by a lowercase word is an abbreviation
|
||||
return True
|
||||
|
||||
|
||||
def _is_decimal_point(text: str, run_start: int, run_end: int) -> bool:
|
||||
if run_end - run_start != 1 or text[run_start] != ".":
|
||||
return False
|
||||
if run_start == 0 or run_end >= len(text):
|
||||
return False
|
||||
return text[run_start - 1].isdigit() and text[run_end].isdigit()
|
||||
|
||||
|
||||
def _is_single_letter_initial(text: str, run_start: int, run_end: int) -> bool:
|
||||
if run_end - run_start != 1 or text[run_start] != ".":
|
||||
return False
|
||||
if run_start == 0 or not text[run_start - 1].isalpha():
|
||||
return False
|
||||
return run_start == 1 or not text[run_start - 2].isalpha()
|
||||
|
||||
|
||||
def _meridiem_continues(text: str, after: int) -> bool:
|
||||
rest = text[after:].lstrip()
|
||||
if not rest:
|
||||
return True
|
||||
if rest[0].islower():
|
||||
return True
|
||||
return _TIME_CONTINUATION_RE.match(rest) is not None
|
||||
|
||||
|
||||
def _is_time_meridiem_run(text: str, run_start: int, run_end: int) -> bool:
|
||||
for match in _TIME_MERIDIEM_RE.finditer(text):
|
||||
if not (match.start() < run_end <= match.end()):
|
||||
continue
|
||||
if run_end < match.end():
|
||||
return True
|
||||
return _meridiem_continues(text, match.end())
|
||||
return False
|
||||
|
||||
|
||||
def _is_abbreviation_run(text: str, run_start: int, run_end: int) -> bool:
|
||||
for match in _ABBREVIATION_RE.finditer(text):
|
||||
if not (match.start() < run_end <= match.end()):
|
||||
continue
|
||||
if match.group(0).lower() == "no.":
|
||||
token = match.group(0).lower()
|
||||
if token == "no.":
|
||||
rest = text[match.end():].lstrip()
|
||||
if not rest or not rest[0].isdigit():
|
||||
return False # "No." as a reply ends the sentence
|
||||
if token in {"a.m.", "p.m."}:
|
||||
if run_end < match.end():
|
||||
return True
|
||||
return _meridiem_continues(text, match.end())
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
Reference in New Issue
Block a user