diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt
index f344370..079ddcb 100644
--- a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt
+++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt
@@ -1208,16 +1208,29 @@ class MainActivity : Activity() {
"{\"sentence_index\":$it}"
}
text("segments", payload)
- val scores = dubSegments.keys.sorted().map { index ->
+ val scores = dubSegments.keys.sorted().mapNotNull { index ->
latestScores[index]
- }.filterNotNull().joinToString(",", "[", "]") { score ->
+ ?.let { score -> index to score }
+ }.joinToString(",", "[", "]") { (index, score) ->
buildString {
append("{\"sentence_index\":")
- append(score.sentenceIndex)
+ append(index)
if (score.overallScore != null) {
append(",\"overall_score\":")
append(score.overallScore)
}
+ listOf(
+ "content_score" to score.contentScore,
+ "fluency_score" to score.fluencyScore,
+ "duration_score" to score.durationScore,
+ "pause_score" to score.pauseScore,
+ "speech_rate_score" to score.speechRateScore,
+ ).forEach { (name, value) ->
+ value?.let {
+ append(",\"$name\":")
+ append(it)
+ }
+ }
score.recognizedText?.let { text ->
append(",\"recognized_text\":\"")
append(text.replace("\\", "\\\\").replace("\"", "\\\""))
diff --git a/sentence_api/main.py b/sentence_api/main.py
index 6d36e46..6c34336 100644
--- a/sentence_api/main.py
+++ b/sentence_api/main.py
@@ -432,9 +432,20 @@ def create_app(
audio_name = f"{uuid.uuid4().hex}{suffix}"
audio_path = service_settings.dub_shares_dir / audio_name
await run_in_threadpool(_persist_upload, upload.file, audio_path, 50 * 1024 * 1024)
- await run_in_threadpool(_trim_leading_silence, audio_path)
+ await run_in_threadpool(_align_share_audio, audio_path, boundary.end_ms / 1000, moss)
saved_files.append(audio_path)
score = scores_by_index.get(sentence_index, {})
+ score_details = {
+ key: float(score[key])
+ for key in (
+ "content_score",
+ "fluency_score",
+ "duration_score",
+ "pause_score",
+ "speech_rate_score",
+ )
+ if score.get(key) is not None
+ }
prepared_segments.append({
"sentence_index": sentence_index,
"start_ms": boundary.start_ms,
@@ -444,6 +455,7 @@ def create_app(
"audio_size_bytes": audio_path.stat().st_size,
"overall_score": float(score.get("overall_score", 0)),
"recognized_text": str(score.get("recognized_text") or ""),
+ "score_details": json.dumps(score_details, ensure_ascii=False),
})
share = video_repository.create_dub_share(
video_hash=video_hash.lower(),
@@ -485,6 +497,7 @@ def create_app(
"text": row["text"],
"overall_score": row["overall_score"],
"recognized_text": row["recognized_text"],
+ "score_details": json.loads(row["score_details"] or "{}"),
"audio_url": f"/api/v1/dub-shares/{share_id}/audio/{row['audio_filename']}",
}
for row in share["segments"]
@@ -540,32 +553,73 @@ def _persist_upload(source: BinaryIO, destination: Path, max_bytes: int) -> tupl
return digest.hexdigest(), size
-def _trim_leading_silence(audio_path: Path) -> None:
+def _align_share_audio(audio_path: Path, target_seconds: float, transcriber: Transcriber) -> None:
ffmpeg = shutil.which("ffmpeg")
if ffmpeg is None:
return
- trimmed_path = audio_path.with_name(f"{audio_path.stem}-trimmed{audio_path.suffix}")
- command = [
- ffmpeg,
- "-hide_banner",
- "-loglevel",
- "error",
- "-y",
- "-i",
- str(audio_path),
- "-af",
- "silenceremove=start_periods=1:start_threshold=-45dB:start_silence=0.08",
- "-c:a",
- "aac",
- str(trimmed_path),
- ]
+ aligned_path = audio_path.with_name(f"{audio_path.stem}-aligned{audio_path.suffix}")
try:
+ transcript = transcriber.transcribe(audio_path)
+ speech_segments = [segment for segment in transcript.segments if segment.text.strip()]
+ if not speech_segments:
+ return
+ start_seconds = max(0.0, min(segment.start_seconds for segment in speech_segments) - 0.05)
+ end_seconds = min(
+ _media_duration_seconds(audio_path),
+ max(segment.end_seconds for segment in speech_segments) + 0.08,
+ )
+ if end_seconds <= start_seconds + 0.1:
+ return
+
+ filters: List[str] = [
+ "silenceremove=start_periods=1:start_threshold=-45dB:start_silence=0.03"
+ ]
+ if target_seconds > 0.2:
+ speed_factor = (end_seconds - start_seconds) / target_seconds
+ if 0.5 <= speed_factor <= 2.0:
+ filters.append(f"atempo={speed_factor:.6f}")
+
+ command = [
+ ffmpeg,
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-y",
+ "-ss",
+ f"{start_seconds:.3f}",
+ "-t",
+ f"{end_seconds - start_seconds:.3f}",
+ "-i",
+ str(audio_path),
+ "-af",
+ ",".join(filters),
+ "-c:a",
+ "aac",
+ str(aligned_path),
+ ]
subprocess.run(command, check=True)
- if trimmed_path.stat().st_size == 0:
- raise RuntimeError("Trimmed audio is empty.")
- trimmed_path.replace(audio_path)
- except (OSError, subprocess.SubprocessError):
- trimmed_path.unlink(missing_ok=True)
+ if aligned_path.stat().st_size > 0:
+ aligned_path.replace(audio_path)
+ except Exception:
+ logger.warning("Could not align share audio %s; keeping original.", audio_path.name, exc_info=True)
+ finally:
+ aligned_path.unlink(missing_ok=True)
+
+
+def _media_duration_seconds(audio_path: Path) -> float:
+ ffprobe = shutil.which("ffprobe")
+ if ffprobe is None:
+ return 0.0
+ try:
+ completed = subprocess.run(
+ [ffprobe, "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", str(audio_path)],
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+ return float(completed.stdout.strip())
+ except (OSError, ValueError, subprocess.SubprocessError):
+ return 0.0
async def _persist_request_stream(
diff --git a/sentence_api/repository.py b/sentence_api/repository.py
index 518a519..7f99fe6 100644
--- a/sentence_api/repository.py
+++ b/sentence_api/repository.py
@@ -88,6 +88,7 @@ class VideoRepository:
audio_size_bytes INTEGER NOT NULL,
overall_score REAL NOT NULL DEFAULT 0,
recognized_text TEXT NOT NULL DEFAULT '',
+ score_details TEXT NOT NULL DEFAULT '{}',
FOREIGN KEY (share_id) REFERENCES dub_shares(share_id) ON DELETE CASCADE,
UNIQUE (share_id, sentence_index)
);
@@ -107,6 +108,10 @@ class VideoRepository:
connection.execute(
"ALTER TABLE dub_share_segments ADD COLUMN recognized_text TEXT NOT NULL DEFAULT ''"
)
+ if "score_details" not in columns:
+ connection.execute(
+ "ALTER TABLE dub_share_segments ADD COLUMN score_details TEXT NOT NULL DEFAULT '{}'"
+ )
def upsert_upload(
self,
@@ -328,8 +333,9 @@ class VideoRepository:
"""
INSERT INTO dub_share_segments (
share_id, sentence_index, start_ms, end_ms, text,
- audio_filename, audio_size_bytes, overall_score, recognized_text
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+ audio_filename, audio_size_bytes, overall_score, recognized_text,
+ score_details
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
[
(
@@ -342,6 +348,7 @@ class VideoRepository:
segment["audio_size_bytes"],
segment.get("overall_score", 0),
segment.get("recognized_text", ""),
+ segment.get("score_details", "{}"),
)
for segment in segments
],
diff --git a/sentence_api/static/dub-share.html b/sentence_api/static/dub-share.html
index 39b5cea..7132bad 100644
--- a/sentence_api/static/dub-share.html
+++ b/sentence_api/static/dub-share.html
@@ -17,7 +17,7 @@ button{flex:1;height:44px;border:0;border-radius:10px;color:#fff;background:#256
.selected-sentence{background:#334155;color:#fff}
.score{margin-top:6px;font-size:12px;color:#94a3b8}
.score.good{color:#14b8a6}.score.low{color:#f97316}
-.footer{margin-top:16px;font-size:13px;color:#94a3b8}
+.footer{margin:16px auto 0;max-width:900px;padding:12px;border-radius:10px;background:#1c2228;font-size:13px;color:#cbd5e1;line-height:1.5}
.footer a{color:#60a5fa;text-decoration:none}
@@ -134,7 +134,17 @@ async function load() {
};
const score = document.createElement('div');
score.className = 'score' + (item.overall_score >= 80 ? ' good' : item.overall_score && item.overall_score < 60 ? ' low' : '');
+ const details = item.score_details || {};
score.textContent = `总分 ${Number(item.overall_score || 0).toFixed(1)}`;
+ [
+ ['内容分', details.content_score],
+ ['流畅度', details.fluency_score],
+ ['时长分', details.duration_score],
+ ['停顿分', details.pause_score],
+ ['语速分', details.speech_rate_score],
+ ].forEach(([label, value]) => {
+ if (value != null) score.textContent += ` · ${label} ${Number(value).toFixed(1)}`;
+ });
if (item.recognized_text) score.textContent += ` · 识别：${item.recognized_text}`;
node.appendChild(score);
sentenceList.appendChild(node);
@@ -151,4 +161,4 @@ async function load() {
}
load();
-
+
diff --git a/sentence_api/tests/test_assessment_api.py b/sentence_api/tests/test_assessment_api.py
index 48cae3b..1d520ea 100644
--- a/sentence_api/tests/test_assessment_api.py
+++ b/sentence_api/tests/test_assessment_api.py
@@ -1,5 +1,8 @@
import io
import json
+from pathlib import Path
+import subprocess
+from unittest import mock
import wave
from dataclasses import replace
@@ -20,6 +23,9 @@ VIDEO_HASH = "c" * 64
class FakeTranscriber:
available = True
+ def __init__(self, segments=None):
+ self.segments = segments or []
+
def transcribe(self, audio_path, language=None):
return Transcript(text="The meeting starts at nine.", segments=[])
@@ -37,7 +43,7 @@ def make_wav() -> bytes:
return output.getvalue()
-def make_client(tmp_path, client_api_key=""):
+def make_client(tmp_path, client_api_key="", transcriber=None):
legacy_path = tmp_path / "legacy.json"
legacy_path.write_text(json.dumps({"videos": {}}), encoding="utf-8")
settings = replace(
@@ -82,7 +88,7 @@ def make_client(tmp_path, client_api_key=""):
BoundaryStore(legacy_path),
settings=settings,
repository=repository,
- transcriber=FakeTranscriber(),
+ transcriber=transcriber or FakeTranscriber(),
)
return TestClient(app)
@@ -148,6 +154,54 @@ def test_get_dub_share_includes_video_hash(tmp_path):
assert response.json()["video_hash"] == VIDEO_HASH
+def test_create_dub_share_aligns_audio_with_whisper_boundaries(tmp_path):
+ from sentence_api.transcription import Transcript, TranscriptionSegment
+ from types import SimpleNamespace
+
+ aligned = tmp_path / "aligned.wav"
+ aligned.write_bytes(b"aligned")
+ calls = []
+
+ class AlignedFakeTranscriber(FakeTranscriber):
+ available = True
+
+ def transcribe(self, audio_path, language=None):
+ calls.append(audio_path)
+ return Transcript(
+ text="The meeting starts at nine.",
+ segments=[
+ TranscriptionSegment(
+ start_seconds=0.4,
+ end_seconds=1.6,
+ text="The meeting starts at nine.",
+ )
+ ],
+ )
+
+ client = make_client(tmp_path, transcriber=AlignedFakeTranscriber())
+ original_run = subprocess.run
+
+ def fake_run(command, **kwargs):
+ Path(command[-1]).write_bytes(b"aligned-audio")
+ return SimpleNamespace(returncode=0)
+
+ with mock.patch("sentence_api.main.subprocess.run", side_effect=fake_run), mock.patch(
+ "sentence_api.main._media_duration_seconds",
+ return_value=2.0,
+ ):
+ response = client.post(
+ "/api/v1/dub-shares",
+ data={
+ "video_hash": VIDEO_HASH,
+ "segments": json.dumps([{"sentence_index": 0}]),
+ },
+ files={"files": ("dub.wav", make_wav(), "audio/wav")},
+ )
+
+ assert response.status_code == 201, response.text
+ assert calls[0].read_bytes() == b"aligned-audio"
+
+
def test_assessment_client_key_is_enforced_when_configured(tmp_path):
client = make_client(tmp_path, client_api_key="tablet-key")
endpoint = f"/api/v1/videos/{VIDEO_HASH}/sentences/0/assessments"