diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt index ffff0d0..f344370 100644 --- a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt @@ -100,6 +100,7 @@ class MainActivity : Activity() { private var mediaRecorder: MediaRecorder? = null private var recordingFile: File? = null private val dubSegments = mutableMapOf() + private val latestScores = mutableMapOf() private var mergedDubFile: File? = null private var dubbingStatusText: TextView? = null private var dubbingPlayback = false @@ -1207,6 +1208,25 @@ class MainActivity : Activity() { "{\"sentence_index\":$it}" } text("segments", payload) + val scores = dubSegments.keys.sorted().map { index -> + latestScores[index] + }.filterNotNull().joinToString(",", "[", "]") { score -> + buildString { + append("{\"sentence_index\":") + append(score.sentenceIndex) + if (score.overallScore != null) { + append(",\"overall_score\":") + append(score.overallScore) + } + score.recognizedText?.let { text -> + append(",\"recognized_text\":\"") + append(text.replace("\\", "\\\\").replace("\"", "\\\"")) + append("\"") + } + append("}") + } + } + text("scores", scores) dubSegments.toSortedMap().forEach { (index, file) -> output.writeBytes("--$boundary\r\n") output.writeBytes( @@ -1260,6 +1280,9 @@ class MainActivity : Activity() { if (::testStatusText.isInitialized) { applyAssessmentResult(result) } + sentence.index.let { index -> + latestScores[index] = result + } } override fun onError(error: Throwable) { diff --git a/sentence_api/main.py b/sentence_api/main.py index b2e23a7..6d36e46 100644 --- a/sentence_api/main.py +++ b/sentence_api/main.py @@ -2,6 +2,8 @@ import hashlib import hmac import logging import mimetypes +import shutil +import subprocess import uuid from pathlib import Path from typing import Any, BinaryIO, Dict, List, Optional @@ -388,6 +390,7 @@ def create_app( video_hash: str = Form(...), title: str = Form(default="我的口语配音"), segments: str = Form(...), + scores: str = Form(default="[]"), files: List[UploadFile] = File(...), ) -> Dict[str, Any]: try: @@ -398,6 +401,17 @@ def create_app( raise HTTPException(status_code=400, detail="At least one dubbing segment is required.") if len(segment_items) != len(files): raise HTTPException(status_code=400, detail="Segment count does not match audio file count.") + try: + score_items = json.loads(scores) + except (TypeError, ValueError) as exc: + raise HTTPException(status_code=400, detail="scores must be valid JSON.") from exc + if not isinstance(score_items, list): + raise HTTPException(status_code=400, detail="scores must be a JSON array.") + scores_by_index = { + int(item["sentence_index"]): item + for item in score_items + if isinstance(item, dict) and "sentence_index" in item + } document = find_document(video_hash.lower()) if document is None: raise HTTPException(status_code=404, detail="Video or sentence boundaries were not found.") @@ -418,7 +432,9 @@ def create_app( audio_name = f"{uuid.uuid4().hex}{suffix}" audio_path = service_settings.dub_shares_dir / audio_name await run_in_threadpool(_persist_upload, upload.file, audio_path, 50 * 1024 * 1024) + await run_in_threadpool(_trim_leading_silence, audio_path) saved_files.append(audio_path) + score = scores_by_index.get(sentence_index, {}) prepared_segments.append({ "sentence_index": sentence_index, "start_ms": boundary.start_ms, @@ -426,6 +442,8 @@ def create_app( "text": boundary.text, "audio_filename": audio_name, "audio_size_bytes": audio_path.stat().st_size, + "overall_score": float(score.get("overall_score", 0)), + "recognized_text": str(score.get("recognized_text") or ""), }) share = video_repository.create_dub_share( video_hash=video_hash.lower(), @@ -458,12 +476,15 @@ def create_app( "video_hash": share["video_hash"], "title": share["title"], "created_at": share["created_at"], + "video_hash": share["video_hash"], "segments": [ { "index": row["sentence_index"], "start_ms": row["start_ms"], "end_ms": row["end_ms"], "text": row["text"], + "overall_score": row["overall_score"], + "recognized_text": row["recognized_text"], "audio_url": f"/api/v1/dub-shares/{share_id}/audio/{row['audio_filename']}", } for row in share["segments"] @@ -519,6 +540,34 @@ def _persist_upload(source: BinaryIO, destination: Path, max_bytes: int) -> tupl return digest.hexdigest(), size +def _trim_leading_silence(audio_path: Path) -> None: + ffmpeg = shutil.which("ffmpeg") + if ffmpeg is None: + return + trimmed_path = audio_path.with_name(f"{audio_path.stem}-trimmed{audio_path.suffix}") + command = [ + ffmpeg, + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + str(audio_path), + "-af", + "silenceremove=start_periods=1:start_threshold=-45dB:start_silence=0.08", + "-c:a", + "aac", + str(trimmed_path), + ] + try: + subprocess.run(command, check=True) + if trimmed_path.stat().st_size == 0: + raise RuntimeError("Trimmed audio is empty.") + trimmed_path.replace(audio_path) + except (OSError, subprocess.SubprocessError): + trimmed_path.unlink(missing_ok=True) + + async def _persist_request_stream( request: Request, destination: Path, diff --git a/sentence_api/repository.py b/sentence_api/repository.py index 01d3a6b..518a519 100644 --- a/sentence_api/repository.py +++ b/sentence_api/repository.py @@ -86,12 +86,28 @@ class VideoRepository: text TEXT, audio_filename TEXT NOT NULL, audio_size_bytes INTEGER NOT NULL, + overall_score REAL NOT NULL DEFAULT 0, + recognized_text TEXT NOT NULL DEFAULT '', FOREIGN KEY (share_id) REFERENCES dub_shares(share_id) ON DELETE CASCADE, UNIQUE (share_id, sentence_index) ); """ ) + with self._connect() as connection: + columns = { + row["name"] + for row in connection.execute("PRAGMA table_info(dub_share_segments)") + } + if "overall_score" not in columns: + connection.execute( + "ALTER TABLE dub_share_segments ADD COLUMN overall_score REAL NOT NULL DEFAULT 0" + ) + if "recognized_text" not in columns: + connection.execute( + "ALTER TABLE dub_share_segments ADD COLUMN recognized_text TEXT NOT NULL DEFAULT ''" + ) + def upsert_upload( self, *, @@ -312,8 +328,8 @@ class VideoRepository: """ INSERT INTO dub_share_segments ( share_id, sentence_index, start_ms, end_ms, text, - audio_filename, audio_size_bytes - ) VALUES (?, ?, ?, ?, ?, ?, ?) + audio_filename, audio_size_bytes, overall_score, recognized_text + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, [ ( @@ -324,6 +340,8 @@ class VideoRepository: segment["text"], segment["audio_filename"], segment["audio_size_bytes"], + segment.get("overall_score", 0), + segment.get("recognized_text", ""), ) for segment in segments ], diff --git a/sentence_api/static/dub-share.html b/sentence_api/static/dub-share.html index ec41d82..39b5cea 100644 --- a/sentence_api/static/dub-share.html +++ b/sentence_api/static/dub-share.html @@ -15,6 +15,10 @@ button{flex:1;height:44px;border:0;border-radius:10px;color:#fff;background:#256 .sentence{padding:12px;border-radius:10px;background:#1c2228;color:#cbd5e1;line-height:1.5} .active-sentence{outline:2px solid #14b8a6;color:#fff} .selected-sentence{background:#334155;color:#fff} +.score{margin-top:6px;font-size:12px;color:#94a3b8} +.score.good{color:#14b8a6}.score.low{color:#f97316} +.footer{margin-top:16px;font-size:13px;color:#94a3b8} +.footer a{color:#60a5fa;text-decoration:none}
@@ -128,6 +132,11 @@ async function load() { }); video.currentTime = item.start_ms / 1000; }; + const score = document.createElement('div'); + score.className = 'score' + (item.overall_score >= 80 ? ' good' : item.overall_score && item.overall_score < 60 ? ' low' : ''); + score.textContent = `总分 ${Number(item.overall_score || 0).toFixed(1)}`; + if (item.recognized_text) score.textContent += ` · 识别：${item.recognized_text}`; + node.appendChild(score); sentenceList.appendChild(node); }); originalButton.onclick = playOriginal; @@ -142,3 +151,4 @@ async function load() { } load(); +

了解更多或下载跟读App：www.pgyer.com/genduchong