From f80da8a5f6238e8d52b41740765af941107ec3ab Mon Sep 17 00:00:00 2001 From: Shuming Liu Date: Wed, 26 Aug 2026 09:47:52 +0800 Subject: [PATCH] added share function --- .../oraltrainer/sample/MainActivity.kt | 125 +++++++++++++++--- sentence_api/config.py | 6 +- sentence_api/main.py | 114 +++++++++++++++- sentence_api/repository.py | 88 ++++++++++++ sentence_api/static/dub-share.html | 120 +++++++++++++++++ 5 files changed, 435 insertions(+), 18 deletions(-) create mode 100644 sentence_api/static/dub-share.html diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt index e9aeb75..ffff0d0 100644 --- a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt @@ -22,8 +22,10 @@ import android.media.MediaMetadataRetriever import android.net.Uri import android.os.Handler import android.os.Looper +import android.os.StrictMode import android.os.Build import android.os.Bundle +import android.util.Log import android.view.Gravity import android.view.View import android.view.ViewGroup @@ -57,8 +59,12 @@ import cn.learningpad.oraltrainer.sdk.TrainingMediaItem import cn.learningpad.oraltrainer.sdk.TrainingVideoSummary import cn.learningpad.oraltrainer.sdk.VideoCatalogCallback import java.io.File +import java.io.DataOutputStream +import java.net.HttpURLConnection +import java.net.URL import java.nio.ByteBuffer import java.util.Locale +import org.json.JSONObject import androidx.core.content.FileProvider import kotlin.math.max import androidx.media3.common.MediaItem @@ -95,6 +101,7 @@ class MainActivity : Activity() { private var recordingFile: File? = null private val dubSegments = mutableMapOf() private var mergedDubFile: File? = null + private var dubbingStatusText: TextView? = null private var dubbingPlayback = false private var activeItemId: String = "" @@ -712,6 +719,12 @@ class MainActivity : Activity() { setLineSpacing(4.dp.toFloat(), 1f) setPadding(0, 8.dp, 0, 0) } + dubbingStatusText = TextView(this).apply { + setTextColor(COLOR_ACCENT_DEEP) + textSize = 13f + setPadding(0, 10.dp, 0, 0) + text = "配音：尚未生成" + } val sentenceButton = { label: String, action: () -> Unit -> Button(this).apply { @@ -840,9 +853,14 @@ class MainActivity : Activity() { }) addView(scoreSummaryText) addView(scoreDetailText) + addView(dubbingStatusText) } } + private fun setDubbingStatus(message: String) { + dubbingStatusText?.text = message + } + private fun refreshTestUi() { if (!::testStatusText.isInitialized) { return @@ -862,6 +880,13 @@ class MainActivity : Activity() { "评测对象：第 ${sentence.index + 1} 句（共 ${item.sentences.size} 句）$segmentLabel" } refreshDubButtons() + if (mergedDubFile == null) { + dubbingStatusText?.text = when (dubSegments.size) { + 0 -> "配音：请先录音" + 1 -> "配音：已保存 1 句，可生成配音" + else -> "配音：已保存 ${dubSegments.size} 句，可生成配音" + } + } } private fun applyTestSubtitleVisibility() { @@ -976,7 +1001,7 @@ class MainActivity : Activity() { private fun refreshDubButtons() { playDubButton.isEnabled = dubSegments.isNotEmpty() && !dubbingPlayback - mergeDubButton.isEnabled = dubSegments.size >= 2 && !dubbingPlayback + mergeDubButton.isEnabled = dubSegments.isNotEmpty() && !dubbingPlayback shareDubButton.isEnabled = mergedDubFile?.exists() == true && !dubbingPlayback val activeAudio = controller.currentSentence()?.let { dubSegments[it.index] } playDubButton.text = when { @@ -1049,16 +1074,19 @@ class MainActivity : Activity() { } private fun mergeDubSegments() { - if (dubSegments.size < 2) { + val segments = dubSegments.toSortedMap().values.toList() + if (segments.isEmpty()) { return } val output = File(cacheDir, "dubbing-${System.currentTimeMillis()}.m4a") + var muxer: MediaMuxer? = null try { - val muxer = MediaMuxer(output.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) + muxer = MediaMuxer(output.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) var muxerAudioTrack = -1 + var muxerStarted = false var presentationTimeUs = 0L - dubSegments.toSortedMap().values.forEach { segment -> + segments.forEach { segment -> val extractor = MediaExtractor().apply { setDataSource(segment.absolutePath) } try { val trackIndex = (0 until extractor.trackCount).firstOrNull { index -> @@ -1069,6 +1097,8 @@ class MainActivity : Activity() { val format = extractor.getTrackFormat(trackIndex) if (muxerAudioTrack == -1) { muxerAudioTrack = muxer.addTrack(format) + muxer.start() + muxerStarted = true } val buffer = ByteBuffer.allocateDirect( if (format.containsKey(MediaFormat.KEY_MAX_INPUT_SIZE)) { @@ -1093,14 +1123,24 @@ class MainActivity : Activity() { extractor.release() } } + require(muxerStarted && muxerAudioTrack != -1) { "录音没有可用音频轨道" } muxer.stop() muxer.release() + muxer = null mergedDubFile = output refreshTestUi() - statusText.text = "已合成 ${dubSegments.size} 段配音：${output.name}" + setDubbingStatus("已合成 ${segments.size} 句配音，可点击“分享配音”") } catch (error: Throwable) { + runCatching { muxer?.stop() } + runCatching { muxer?.release() } output.delete() - statusText.text = "合成失败：${error.message.orEmpty()}" + Log.e(TAG_DUBBING, "merge failed", error) + setDubbingStatus( + buildString { + append("生成失败：${error.javaClass.simpleName}: ${error.message.orEmpty()}") + append(" · 已保存 ${dubSegments.size} 句") + } + ) } } @@ -1120,22 +1160,74 @@ class MainActivity : Activity() { } private fun shareMergedDubbing() { - val file = mergedDubFile?.takeIf { it.exists() } ?: return + val item = controller.currentTrainingItem() + if (item == null || !SHA256_PATTERN.matches(item.id)) { + setDubbingStatus("分享失败：网页分享仅支持云端课程视频") + return + } try { - val uri = FileProvider.getUriForFile( - this, - "${packageName}.fileprovider", - file, - ) + val shareUrl = uploadDubShare(item.id, item.title) val intent = Intent(Intent.ACTION_SEND).apply { - type = "audio/mp4" - putExtra(Intent.EXTRA_STREAM, uri) + type = "text/plain" + putExtra(Intent.EXTRA_TEXT, "我的口语配音：$shareUrl") putExtra(Intent.EXTRA_TITLE, "我的口语配音") - addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION or Intent.FLAG_ACTIVITY_NEW_TASK) + addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) } startActivity(Intent.createChooser(intent, "分享配音")) + setDubbingStatus("已生成分享链接：$shareUrl") } catch (error: Throwable) { - statusText.text = "分享失败：${error.message.orEmpty()}" + Log.e(TAG_DUBBING, "share failed", error) + setDubbingStatus("分享失败：${error.javaClass.simpleName}: ${error.message.orEmpty()}") + } + } + + private fun uploadDubShare(videoHash: String, videoTitle: String): String { + val policy = StrictMode.getThreadPolicy() + StrictMode.setThreadPolicy(StrictMode.ThreadPolicy.Builder().detectAll().penaltyLog().build()) + val boundary = "oral-trainer-${System.currentTimeMillis()}" + var connection: HttpURLConnection? = null + try { + connection = URL(URL(SERVER_BASE_URL), "/api/v1/dub-shares").openConnection() as HttpURLConnection + connection.requestMethod = "POST" + connection.doOutput = true + connection.connectTimeout = 15_000 + connection.readTimeout = 120_000 + connection.setRequestProperty("Content-Type", "multipart/form-data; boundary=$boundary") + + DataOutputStream(connection.outputStream).use { output -> + fun text(name: String, value: String) { + output.writeBytes("--$boundary\r\n") + output.writeBytes("Content-Disposition: form-data; name=\"$name\"\r\n\r\n") + output.writeBytes(value.replace("\n", "\\n") + "\r\n") + } + text("video_hash", videoHash.lowercase(Locale.US)) + text("title", videoTitle.ifBlank { "我的口语配音" }) + // Build explicitly to avoid fragile string replacement. + val payload = dubSegments.keys.sorted().joinToString(",", "[", "]") { + "{\"sentence_index\":$it}" + } + text("segments", payload) + dubSegments.toSortedMap().forEach { (index, file) -> + output.writeBytes("--$boundary\r\n") + output.writeBytes( + "Content-Disposition: form-data; name=\"files\"; filename=\"dub-$index.m4a\"\r\n" + ) + output.writeBytes("Content-Type: audio/mp4\r\n\r\n") + file.inputStream().use { input -> input.copyTo(output) } + output.writeBytes("\r\n") + } + output.writeBytes("--$boundary--\r\n") + } + + val code = connection.responseCode + val body = (if (code in 200..299) connection.inputStream else connection.errorStream) + ?.bufferedReader()?.use { it.readText() }.orEmpty() + require(code in 200..299) { "HTTP $code: $body" } + val response = JSONObject(body).getString("share_id") + return "$SERVER_BASE_URL/dub-shares/$response" + } finally { + connection?.disconnect() + StrictMode.setThreadPolicy(policy) } } @@ -1511,6 +1603,7 @@ class MainActivity : Activity() { const val PICK_VIDEO_REQUEST = 1001 const val RECORD_AUDIO_REQUEST = 1002 const val PROGRESS_MAX = 1000 + private const val TAG_DUBBING = "Dubbing" private val SHA256_PATTERN = Regex("^[0-9a-fA-F]{64}$") diff --git a/sentence_api/config.py b/sentence_api/config.py index 120802d..beb25f8 100644 --- a/sentence_api/config.py +++ b/sentence_api/config.py @@ -68,10 +68,14 @@ class Settings: def attempts_dir(self) -> Path: return self.data_dir / "attempts" + @property + def dub_shares_dir(self) -> Path: + return self.data_dir / "dub_shares" + @property def database_path(self) -> Path: return self.data_dir / "oral_trainer.sqlite3" def ensure_directories(self) -> None: - for path in (self.data_dir, self.videos_dir, self.work_dir, self.attempts_dir): + for path in (self.data_dir, self.videos_dir, self.work_dir, self.attempts_dir, self.dub_shares_dir): path.mkdir(parents=True, exist_ok=True) diff --git a/sentence_api/main.py b/sentence_api/main.py index 2bade54..babade9 100644 --- a/sentence_api/main.py +++ b/sentence_api/main.py @@ -4,7 +4,8 @@ import logging import mimetypes import uuid from pathlib import Path -from typing import Any, BinaryIO, Dict, Optional +from typing import Any, BinaryIO, Dict, List, Optional +import json from fastapi import ( BackgroundTasks, @@ -51,6 +52,7 @@ SHA256_PATH = ApiPath( description="SHA-256 hex digest of the exact uploaded video bytes", ) ALLOWED_VIDEO_EXTENSIONS = {".mp4", ".m4v", ".mov", ".mkv", ".webm"} +ALLOWED_AUDIO_EXTENSIONS = {".m4a", ".mp4", ".wav"} class UploadTooLargeError(ValueError): @@ -380,6 +382,116 @@ def create_app( if not service_settings.keep_attempt_audio or not completed: temporary_path.unlink(missing_ok=True) + @application.post("/api/v1/dub-shares", status_code=201) + async def create_dub_share( + request: Request, + video_hash: str = Form(...), + title: str = Form(default="我的口语配音"), + segments: str = Form(...), + files: List[UploadFile] = File(...), + ) -> Dict[str, Any]: + try: + segment_items = json.loads(segments) + except (TypeError, ValueError) as exc: + raise HTTPException(status_code=400, detail="segments must be valid JSON.") from exc + if not isinstance(segment_items, list) or not segment_items: + raise HTTPException(status_code=400, detail="At least one dubbing segment is required.") + if len(segment_items) != len(files): + raise HTTPException(status_code=400, detail="Segment count does not match audio file count.") + document = find_document(video_hash.lower()) + if document is None: + raise HTTPException(status_code=404, detail="Video or sentence boundaries were not found.") + + prepared_segments: List[Dict[str, Any]] = [] + saved_files: List[Path] = [] + try: + for item, upload in zip(segment_items, files): + sentence_index = int(item["sentence_index"]) + boundary = next((s for s in document.sentences if s.index == sentence_index), None) + if boundary is None: + raise HTTPException(status_code=404, detail=f"Sentence {sentence_index} was not found.") + suffix = Path(upload.filename or "dub.m4a").suffix.lower() + if suffix == ".mp4": + suffix = ".m4a" + if suffix not in ALLOWED_AUDIO_EXTENSIONS: + raise HTTPException(status_code=415, detail=f"Unsupported audio extension: {suffix or '(none)'}") + audio_name = f"{uuid.uuid4().hex}{suffix}" + audio_path = service_settings.dub_shares_dir / audio_name + await run_in_threadpool(_persist_upload, upload.file, audio_path, 50 * 1024 * 1024) + saved_files.append(audio_path) + prepared_segments.append({ + "sentence_index": sentence_index, + "start_ms": boundary.start_ms, + "end_ms": boundary.end_ms, + "text": boundary.text, + "audio_filename": audio_name, + "audio_size_bytes": audio_path.stat().st_size, + }) + share = video_repository.create_dub_share( + video_hash=video_hash.lower(), + title=title, + segments=prepared_segments, + ) + if share is None: + raise HTTPException(status_code=404, detail="Video was not found.") + return { + **share, + "share_url": f"/dub-shares/{share['share_id']}", + } + except UploadTooLargeError as exc: + raise HTTPException(status_code=413, detail=str(exc)) from exc + except HTTPException: + raise + finally: + for upload in files: + await upload.close() + # Files intentionally remain until share deletion because the web page references them. + del saved_files + + @application.get("/api/v1/dub-shares/{share_id}") + def get_dub_share(share_id: str) -> Dict[str, Any]: + share = video_repository.get_dub_share(share_id) + if share is None: + raise HTTPException(status_code=404, detail="Dubbing share was not found.") + return { + "share_id": share["share_id"], + "title": share["title"], + "created_at": share["created_at"], + "segments": [ + { + "index": row["sentence_index"], + "start_ms": row["start_ms"], + "end_ms": row["end_ms"], + "text": row["text"], + "audio_url": f"/api/v1/dub-shares/{share_id}/audio/{row['audio_filename']}", + } + for row in share["segments"] + ], + } + + @application.get("/api/v1/dub-shares/{share_id}/audio/{filename}") + def get_dub_share_audio(share_id: str, filename: str) -> FileResponse: + share = video_repository.get_dub_share(share_id) + if share is None: + raise HTTPException(status_code=404, detail="Dubbing share was not found.") + safe_name = Path(filename).name + row = next((item for item in share["segments"] if item["audio_filename"] == safe_name), None) + if row is None: + raise HTTPException(status_code=404, detail="Audio was not found.") + media_path = service_settings.dub_shares_dir / safe_name + if not media_path.is_file(): + raise HTTPException(status_code=404, detail="Stored audio file is missing.") + return FileResponse(media_path, media_type="audio/mp4", headers={"Cache-Control": "public, max-age=31536000"}) + + @application.get("/dub-shares/{share_id}", include_in_schema=False) + def dub_share_page(share_id: str) -> FileResponse: + if video_repository.get_dub_share(share_id) is None: + raise HTTPException(status_code=404, detail="Dubbing share was not found.") + page = static_dir / "dub-share.html" + if not page.is_file(): + raise HTTPException(status_code=404, detail="Dubbing share UI is not installed.") + return FileResponse(page) + return application diff --git a/sentence_api/repository.py b/sentence_api/repository.py index c41b4e6..01d3a6b 100644 --- a/sentence_api/repository.py +++ b/sentence_api/repository.py @@ -3,6 +3,7 @@ import sqlite3 from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, List, Optional +import uuid from .models import SentenceBoundary, SentenceBoundaryDocument from .store import normalize_video_hash @@ -67,6 +68,27 @@ class VideoRepository: created_at TEXT NOT NULL, FOREIGN KEY (video_hash) REFERENCES videos(video_hash) ON DELETE CASCADE ); + + CREATE TABLE IF NOT EXISTS dub_shares ( + share_id TEXT PRIMARY KEY, + video_hash TEXT NOT NULL, + title TEXT NOT NULL, + created_at TEXT NOT NULL, + FOREIGN KEY (video_hash) REFERENCES videos(video_hash) ON DELETE CASCADE + ); + + CREATE TABLE IF NOT EXISTS dub_share_segments ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + share_id TEXT NOT NULL, + sentence_index INTEGER NOT NULL, + start_ms INTEGER NOT NULL, + end_ms INTEGER NOT NULL, + text TEXT, + audio_filename TEXT NOT NULL, + audio_size_bytes INTEGER NOT NULL, + FOREIGN KEY (share_id) REFERENCES dub_shares(share_id) ON DELETE CASCADE, + UNIQUE (share_id, sentence_index) + ); """ ) @@ -266,6 +288,72 @@ class VideoRepository: reference_speech_duration_ms=row["reference_speech_duration_ms"], ) + def create_dub_share( + self, + *, + video_hash: str, + title: str, + segments: List[Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + normalized_hash = normalize_video_hash(video_hash) + if self.get_video(normalized_hash) is None: + return None + share_id = uuid.uuid4().hex + now = utc_now() + with self._connect() as connection: + connection.execute( + """ + INSERT INTO dub_shares (share_id, video_hash, title, created_at) + VALUES (?, ?, ?, ?) + """, + (share_id, normalized_hash, title[:200], now), + ) + connection.executemany( + """ + INSERT INTO dub_share_segments ( + share_id, sentence_index, start_ms, end_ms, text, + audio_filename, audio_size_bytes + ) VALUES (?, ?, ?, ?, ?, ?, ?) + """, + [ + ( + share_id, + segment["sentence_index"], + segment["start_ms"], + segment["end_ms"], + segment["text"], + segment["audio_filename"], + segment["audio_size_bytes"], + ) + for segment in segments + ], + ) + return { + "share_id": share_id, + "video_hash": normalized_hash, + "title": title[:200], + "created_at": now, + } + + def get_dub_share(self, share_id: str) -> Optional[Dict[str, Any]]: + with self._connect() as connection: + share = connection.execute( + "SELECT * FROM dub_shares WHERE share_id = ?", (share_id,) + ).fetchone() + if share is None: + return None + rows = connection.execute( + """ + SELECT * FROM dub_share_segments + WHERE share_id = ? + ORDER BY sentence_index + """, + (share_id,), + ).fetchall() + result = dict(share) + result["segments"] = [dict(row) for row in rows] + return result + def record_attempt( self, *, diff --git a/sentence_api/static/dub-share.html b/sentence_api/static/dub-share.html new file mode 100644 index 0000000..12e5770 --- /dev/null +++ b/sentence_api/static/dub-share.html @@ -0,0 +1,120 @@ + + + + +口语配音 + + +
+

加载中…

+

正在读取配音数据

+ +
+ + +
+
+
+ +