diff --git a/ios/OralTrainer.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/ios/OralTrainer.xcodeproj/project.xcworkspace/contents.xcworkspacedata
new file mode 100644
index 0000000..919434a
--- /dev/null
+++ b/ios/OralTrainer.xcodeproj/project.xcworkspace/contents.xcworkspacedata
@@ -0,0 +1,7 @@
+
+
+
+
+
diff --git a/ios/OralTrainer.xcodeproj/project.xcworkspace/xcuserdata/liushuming.xcuserdatad/UserInterfaceState.xcuserstate b/ios/OralTrainer.xcodeproj/project.xcworkspace/xcuserdata/liushuming.xcuserdatad/UserInterfaceState.xcuserstate
new file mode 100644
index 0000000..bfeddf0
Binary files /dev/null and b/ios/OralTrainer.xcodeproj/project.xcworkspace/xcuserdata/liushuming.xcuserdatad/UserInterfaceState.xcuserstate differ
diff --git a/ios/OralTrainer.xcodeproj/xcuserdata/liushuming.xcuserdatad/xcschemes/xcschememanagement.plist b/ios/OralTrainer.xcodeproj/xcuserdata/liushuming.xcuserdatad/xcschemes/xcschememanagement.plist
new file mode 100644
index 0000000..74fab79
--- /dev/null
+++ b/ios/OralTrainer.xcodeproj/xcuserdata/liushuming.xcuserdatad/xcschemes/xcschememanagement.plist
@@ -0,0 +1,14 @@
+
+
+
+
+ SchemeUserState
+
+ OralTrainer.xcscheme_^#shared#^_
+
+ orderHint
+ 0
+
+
+
+
diff --git a/sentence_api/audio_metrics.py b/sentence_api/audio_metrics.py
index a9b65ca..d9b8883 100644
--- a/sentence_api/audio_metrics.py
+++ b/sentence_api/audio_metrics.py
@@ -57,12 +57,38 @@ def analyze_audio(path: Path) -> AudioMetrics:
def analyze_samples(samples: np.ndarray, sample_rate: int = SAMPLE_RATE) -> AudioMetrics:
+ if samples.ndim != 1:
+ samples = samples.reshape(-1)
+ speech = speech_frame_mask(samples, sample_rate)
+ speech_indexes = np.flatnonzero(speech)
+ if speech_indexes.size == 0:
+ raise AudioAnalysisError("No usable speech was detected in the recording.")
+
+ recording_duration_ms = max(1, int(round(samples.size / sample_rate * 1000)))
+ window = max(1, int(round(sample_rate * 0.03)))
+ frame_ms = window / sample_rate * 1000
+ speech_duration_ms = max(1, int(round(speech.sum() * frame_ms)))
+ first = int(speech_indexes[0])
+ last = int(speech_indexes[-1])
+ internal_frames = max(1, last - first + 1)
+ internal_silence_frames = int((~speech[first : last + 1]).sum())
+ internal_silence_ms = int(round(internal_silence_frames * frame_ms))
+ internal_pause_ratio = internal_silence_frames / internal_frames
+ return AudioMetrics(
+ recording_duration_ms=recording_duration_ms,
+ speech_duration_ms=speech_duration_ms,
+ internal_silence_ms=internal_silence_ms,
+ internal_pause_ratio=round(internal_pause_ratio, 4),
+ )
+
+
+def speech_frame_mask(samples: np.ndarray, sample_rate: int = SAMPLE_RATE) -> np.ndarray:
+ """Per-frame speech mask (30 ms frames) using the same RMS threshold as analyze_samples."""
if samples.ndim != 1:
samples = samples.reshape(-1)
if samples.size == 0 or sample_rate <= 0:
raise AudioAnalysisError("The uploaded audio is empty.")
- recording_duration_ms = max(1, int(round(samples.size / sample_rate * 1000)))
window = max(1, int(round(sample_rate * 0.03)))
complete_frames = int(np.ceil(samples.size / window))
padded = np.pad(samples, (0, complete_frames * window - samples.size))
@@ -79,24 +105,7 @@ def analyze_samples(samples: np.ndarray, sample_rate: int = SAMPLE_RATE) -> Audi
# Treat very short gaps inside a word as speech, then reject short clicks.
_bridge_false_runs(speech, max_frames=5)
_remove_true_runs(speech, max_frames=2)
- speech_indexes = np.flatnonzero(speech)
- if speech_indexes.size == 0:
- raise AudioAnalysisError("No usable speech was detected in the recording.")
-
- frame_ms = window / sample_rate * 1000
- speech_duration_ms = max(1, int(round(speech.sum() * frame_ms)))
- first = int(speech_indexes[0])
- last = int(speech_indexes[-1])
- internal_frames = max(1, last - first + 1)
- internal_silence_frames = int((~speech[first : last + 1]).sum())
- internal_silence_ms = int(round(internal_silence_frames * frame_ms))
- internal_pause_ratio = internal_silence_frames / internal_frames
- return AudioMetrics(
- recording_duration_ms=recording_duration_ms,
- speech_duration_ms=speech_duration_ms,
- internal_silence_ms=internal_silence_ms,
- internal_pause_ratio=round(internal_pause_ratio, 4),
- )
+ return speech
def _bridge_false_runs(values: np.ndarray, max_frames: int) -> None:
diff --git a/sentence_api/tests/test_transcription.py b/sentence_api/tests/test_transcription.py
new file mode 100644
index 0000000..7db9cdd
--- /dev/null
+++ b/sentence_api/tests/test_transcription.py
@@ -0,0 +1,138 @@
+import io
+import wave
+
+import numpy as np
+import pytest
+
+from sentence_api.transcription import (
+ MossTranscriber,
+ _parse_json_segments,
+ _plan_chunks,
+ _wav_bytes,
+ _wav_duration_seconds,
+)
+
+
+SAMPLE_RATE = 16000
+
+
+def _speech(seconds: float, sample_rate: int = SAMPLE_RATE) -> np.ndarray:
+ t = np.arange(int(seconds * sample_rate)) / sample_rate
+ return 0.25 * np.sin(2 * np.pi * 220 * t)
+
+
+def _silence(seconds: float, sample_rate: int = SAMPLE_RATE) -> np.ndarray:
+ return np.zeros(int(seconds * sample_rate))
+
+
+def _audio(*parts: np.ndarray, sample_rate: int = SAMPLE_RATE) -> np.ndarray:
+ return np.concatenate([np.asarray(part) for part in parts]).astype(np.float32)
+
+
+def test_plan_chunks_cuts_at_silence_gaps():
+ samples = _audio(
+ _speech(10),
+ _silence(0.8),
+ _speech(29.2),
+ _silence(1.0),
+ _speech(29.0),
+ _silence(0.6),
+ _speech(24.4),
+ )
+ chunks = _plan_chunks(samples, SAMPLE_RATE)
+ assert chunks[0][0] == pytest.approx(0.0)
+ assert chunks[-1][1] == pytest.approx(95.0)
+ for (start, end), (next_start, _) in zip(chunks, chunks[1:]):
+ assert end == pytest.approx(next_start)
+ assert end - start <= 34.0
+ assert 10.0 <= chunks[0][1] <= 10.8
+ assert 40.0 <= chunks[1][1] <= 41.0
+ assert 70.0 <= chunks[2][1] <= 70.6
+
+
+def test_plan_chunks_falls_back_to_fixed_windows():
+ chunks = _plan_chunks(_speech(95), SAMPLE_RATE)
+ assert chunks == [(0.0, 30.0), (30.0, 60.0), (60.0, 90.0), (90.0, 95.0)]
+
+
+def test_plan_chunks_merges_short_tail():
+ chunks = _plan_chunks(_speech(93), SAMPLE_RATE)
+ assert chunks == [(0.0, 30.0), (30.0, 60.0), (60.0, 93.0)]
+
+
+def test_parse_json_segments_filters_hallucinated_segments():
+ raw = [
+ {"start": 0.0, "end": 3.0, "text": "hello world", "compression_ratio": 1.4},
+ {"start": 3.5, "end": 6.0, "text": "hello world", "compression_ratio": 12.0},
+ {"start": 6.5, "end": 9.0, "text": "no ratio field"},
+ ]
+ filtered = _parse_json_segments(raw, compression_limit=2.4)
+ assert [segment.text for segment in filtered] == ["hello world", "no ratio field"]
+ assert len(_parse_json_segments(raw)) == 3
+
+
+def test_wav_bytes_roundtrip(tmp_path):
+ samples = _speech(2.0)
+ data = _wav_bytes(samples, SAMPLE_RATE)
+ assert data.startswith(b"RIFF")
+ path = tmp_path / "chunk.wav"
+ path.write_bytes(data)
+ assert _wav_duration_seconds(path) == pytest.approx(2.0, abs=0.05)
+ with wave.open(str(path), "rb") as wav:
+ assert wav.getnchannels() == 1
+ assert wav.getsampwidth() == 2
+ assert wav.getframerate() == SAMPLE_RATE
+
+
+def test_wav_duration_returns_none_for_non_wav(tmp_path):
+ path = tmp_path / "note.txt"
+ path.write_text("not audio")
+ assert _wav_duration_seconds(path) is None
+
+
+def test_chunked_transcribe_offsets_timestamps(monkeypatch):
+ requests = []
+ call_count = {"n": 0}
+
+ def fake_post(endpoint, data=None, files=None, timeout=None):
+ call_count["n"] += 1
+ requests.append(data)
+ audio = files["file"][1].read()
+ with wave.open(io.BytesIO(audio), "rb") as wav:
+ duration = wav.getnframes() / wav.getframerate()
+ if call_count["n"] == 1:
+ segments = [
+ {"start": 1.0, "end": 4.0, "text": "first part", "compression_ratio": 1.4},
+ {"start": 27.0, "end": 30.0, "text": "second part", "compression_ratio": 1.5},
+ ]
+ elif call_count["n"] == 2:
+ segments = [
+ {"start": 1.0, "end": 4.0, "text": "third part", "compression_ratio": 1.4},
+ {"start": 27.0, "end": 30.0, "text": "fourth part", "compression_ratio": 1.5},
+ ]
+ else:
+ segments = [{"start": 0.5, "end": 3.5, "text": "tail part", "compression_ratio": 1.3}]
+ assert abs(duration - 30.0) < 1.0 or abs(duration - 5.0) < 1.0
+
+ class Response:
+ def raise_for_status(self):
+ pass
+
+ def json(self):
+ return {"text": "", "segments": segments}
+
+ return Response()
+
+ monkeypatch.setattr("sentence_api.transcription.httpx.post", fake_post)
+ transcriber = MossTranscriber(endpoint="http://whisper:9000", model="whisper")
+ result = transcriber._transcribe_chunked(_speech(65), SAMPLE_RATE, None)
+ assert [(s.start_seconds, s.end_seconds, s.text) for s in result.segments] == [
+ (1.0, 4.0, "first part"),
+ (27.0, 30.0, "second part"),
+ (31.0, 34.0, "third part"),
+ (57.0, 60.0, "fourth part"),
+ (60.5, 63.5, "tail part"),
+ ]
+ assert result.text == "first part second part third part fourth part tail part"
+ assert len(requests) == 3
+ assert all(request.get("condition_on_previous_text") == "false" for request in requests)
diff --git a/sentence_api/transcription.py b/sentence_api/transcription.py
index 5267dea..e2801ec 100644
--- a/sentence_api/transcription.py
+++ b/sentence_api/transcription.py
@@ -1,10 +1,28 @@
+import io
import mimetypes
import re
+import wave
from dataclasses import dataclass
from pathlib import Path
from typing import List, Optional, Protocol
import httpx
+import numpy as np
+
+from .audio_metrics import AudioAnalysisError, decode_audio_mono, speech_frame_mask
+
+
+# Whisper was trained on 30-second clips. Sending longer audio in a single request
+# makes the model enter a repetition loop (every later segment becomes the same
+# sentence while timestamps keep advancing), so long audio is split into short
+# chunks at silence gaps and each chunk is transcribed independently.
+CHUNK_SECONDS = 30.0
+MIN_CHUNK_SECONDS = 5.0
+MIN_SILENCE_SECONDS = 0.5
+MIN_TAIL_SECONDS = 4.0
+# Whisper's own repetition-loop detector: segments that compress this well are
+# almost certainly hallucinated repeated text (healthy speech is ~1.3-1.8).
+COMPRESSION_RATIO_LIMIT = 2.4
@dataclass(frozen=True)
@@ -50,29 +68,25 @@ class MossTranscriber:
def transcribe(self, audio_path: Path, language: Optional[str] = None) -> Transcript:
if not self.available:
raise RuntimeError("MOSS transcription is not configured on this server.")
+ duration = _wav_duration_seconds(audio_path)
+ if duration is None or duration > CHUNK_SECONDS + 2.0:
+ try:
+ samples, sample_rate = decode_audio_mono(audio_path)
+ except AudioAnalysisError:
+ return self._transcribe_single(audio_path, language)
+ if samples.size / sample_rate > CHUNK_SECONDS + 2.0:
+ try:
+ return self._transcribe_chunked(samples, sample_rate, language)
+ except AudioAnalysisError:
+ return self._transcribe_single(audio_path, language)
+ return self._transcribe_single(audio_path, language)
+
+ def _transcribe_single(
+ self, audio_path: Path, language: Optional[str]
+ ) -> Transcript:
content_type = mimetypes.guess_type(audio_path.name)[0] or "application/octet-stream"
- data = {
- "model": self.model,
- "response_format": "verbose_json",
- "temperature": "0",
- "max_new_tokens": str(self.max_new_tokens),
- }
- if language:
- data["language"] = language
-
- try:
- with audio_path.open("rb") as audio_file:
- response = httpx.post(
- self.endpoint,
- data=data,
- files={"file": (audio_path.name, audio_file, content_type)},
- timeout=httpx.Timeout(self.timeout_seconds, connect=30),
- )
- response.raise_for_status()
- payload = response.json()
- except (httpx.HTTPError, ValueError) as exc:
- raise RuntimeError(f"MOSS transcription request failed: {exc}") from exc
-
+ with audio_path.open("rb") as audio_file:
+ payload = self._post_audio(audio_file, audio_path.name, content_type, language)
raw_text = str(payload.get("text") or "").strip()
segments = _parse_json_segments(payload.get("segments"))
if not segments:
@@ -80,8 +94,71 @@ class MossTranscriber:
plain_text = " ".join(segment.text for segment in segments).strip() or raw_text
return Transcript(text=plain_text, segments=segments)
+ def _transcribe_chunked(
+ self, samples: np.ndarray, sample_rate: int, language: Optional[str]
+ ) -> Transcript:
+ segments: List[TranscriptionSegment] = []
+ for start_seconds, end_seconds in _plan_chunks(samples, sample_rate):
+ chunk = samples[
+ int(round(start_seconds * sample_rate)) : int(round(end_seconds * sample_rate))
+ ]
+ payload = self._post_audio(
+ io.BytesIO(_wav_bytes(chunk, sample_rate)),
+ "chunk.wav",
+ "audio/wav",
+ language,
+ )
+ chunk_segments = _parse_json_segments(
+ payload.get("segments"), compression_limit=COMPRESSION_RATIO_LIMIT
+ )
+ if not chunk_segments:
+ chunk_segments = _parse_compact_segments(str(payload.get("text") or "").strip())
+ for segment in chunk_segments:
+ segments.append(
+ TranscriptionSegment(
+ start_seconds=segment.start_seconds + start_seconds,
+ end_seconds=segment.end_seconds + start_seconds,
+ text=segment.text,
+ speaker=segment.speaker,
+ )
+ )
+ segments.sort(key=lambda item: (item.start_seconds, item.end_seconds))
+ plain_text = " ".join(segment.text for segment in segments).strip()
+ return Transcript(text=plain_text, segments=segments)
-def _parse_json_segments(raw_segments) -> List[TranscriptionSegment]:
+ def _post_audio(
+ self,
+ audio_file,
+ filename: str,
+ content_type: str,
+ language: Optional[str],
+ ) -> dict:
+ data = {
+ "model": self.model,
+ "response_format": "verbose_json",
+ "temperature": "0",
+ "condition_on_previous_text": "false",
+ "max_new_tokens": str(self.max_new_tokens),
+ }
+ if language:
+ data["language"] = language
+
+ try:
+ response = httpx.post(
+ self.endpoint,
+ data=data,
+ files={"file": (filename, audio_file, content_type)},
+ timeout=httpx.Timeout(self.timeout_seconds, connect=30),
+ )
+ response.raise_for_status()
+ return response.json()
+ except (httpx.HTTPError, ValueError) as exc:
+ raise RuntimeError(f"MOSS transcription request failed: {exc}") from exc
+
+
+def _parse_json_segments(
+ raw_segments, compression_limit: Optional[float] = None
+) -> List[TranscriptionSegment]:
if not isinstance(raw_segments, list):
return []
segments = []
@@ -93,6 +170,13 @@ def _parse_json_segments(raw_segments) -> List[TranscriptionSegment]:
end = float(item["end"])
except (KeyError, TypeError, ValueError):
continue
+ compression_ratio = item.get("compression_ratio")
+ if (
+ compression_limit is not None
+ and isinstance(compression_ratio, (int, float))
+ and compression_ratio > compression_limit
+ ):
+ continue
text = str(item.get("text") or "").strip()
if text and end > start >= 0:
speaker = item.get("speaker")
@@ -107,6 +191,80 @@ def _parse_json_segments(raw_segments) -> List[TranscriptionSegment]:
return segments
+def _wav_duration_seconds(path: Path) -> Optional[float]:
+ try:
+ with wave.open(str(path), "rb") as wav:
+ if wav.getframerate() > 0:
+ return wav.getnframes() / wav.getframerate()
+ except (wave.Error, OSError):
+ pass
+ return None
+
+
+def _wav_bytes(samples: np.ndarray, sample_rate: int) -> bytes:
+ pcm = np.clip(np.round(samples * 32767), -32768, 32767).astype(" List[tuple]:
+ speech = speech_frame_mask(samples, sample_rate)
+ frame_seconds = 0.03
+ gap_frames = max(1, int(round(MIN_SILENCE_SECONDS / frame_seconds)))
+
+ gaps: List[tuple] = []
+ run_start: Optional[int] = None
+ for index, is_speech in enumerate(speech):
+ if not is_speech and run_start is None:
+ run_start = index
+ elif is_speech and run_start is not None:
+ if index - run_start >= gap_frames:
+ gaps.append((run_start * frame_seconds, index * frame_seconds))
+ run_start = None
+ if run_start is not None and len(speech) - run_start >= gap_frames:
+ gaps.append((run_start * frame_seconds, len(speech) * frame_seconds))
+
+ duration = samples.size / sample_rate
+ cuts: List[float] = []
+ cursor = 0.0
+ while duration - cursor > CHUNK_SECONDS + 1e-6:
+ target = cursor + CHUNK_SECONDS
+ best_gap = None
+ for gap_start, gap_end in gaps:
+ if gap_start < cursor + MIN_CHUNK_SECONDS:
+ continue
+ if gap_start > target:
+ break
+ best_gap = (gap_start, gap_end)
+ if best_gap is None:
+ cut = min(target, duration)
+ else:
+ gap_start, gap_end = best_gap
+ cut = min(target, (gap_start + gap_end) / 2.0)
+ cuts.append(cut)
+ cursor = cut
+
+ boundaries = [0.0] + cuts + [duration]
+ chunks = [
+ (boundaries[index], boundaries[index + 1])
+ for index in range(len(boundaries) - 1)
+ if boundaries[index + 1] > boundaries[index] + 0.05
+ ]
+ if (
+ len(chunks) > 1
+ and chunks[-1][1] - chunks[-1][0] < MIN_TAIL_SECONDS
+ and chunks[-1][1] - chunks[-2][0] <= CHUNK_SECONDS + MIN_TAIL_SECONDS
+ ):
+ chunks[-2] = (chunks[-2][0], chunks[-1][1])
+ chunks.pop()
+ return chunks
+
+
_COMPACT_SEGMENT = re.compile(
r"\[(?P\d+(?:\.\d+)?)\]"
r"\[(?PS\d+)\]"