From 9feb876b7ad77710c8a9067060041e93104433e6 Mon Sep 17 00:00:00 2001 From: Shuming Liu Date: Wed, 26 Aug 2026 08:33:43 +0800 Subject: [PATCH] added saving recorded sound --- android/sample-app/build.gradle.kts | 2 + .../sample-app/src/main/AndroidManifest.xml | 10 + .../oraltrainer/sample/MainActivity.kt | 257 +++++++++++++++++- .../src/main/res/xml/file_paths.xml | 6 + 4 files changed, 261 insertions(+), 14 deletions(-) create mode 100644 android/sample-app/src/main/res/xml/file_paths.xml diff --git a/android/sample-app/build.gradle.kts b/android/sample-app/build.gradle.kts index 6768f7a..384c7c9 100644 --- a/android/sample-app/build.gradle.kts +++ b/android/sample-app/build.gradle.kts @@ -36,5 +36,7 @@ kotlin { dependencies { implementation(project(":oral-trainer-sdk")) + implementation("androidx.core:core-ktx:1.13.1") implementation("androidx.media3:media3-common:1.11.0") + implementation("androidx.media3:media3-exoplayer:1.11.0") } diff --git a/android/sample-app/src/main/AndroidManifest.xml b/android/sample-app/src/main/AndroidManifest.xml index a66466b..c2b6721 100644 --- a/android/sample-app/src/main/AndroidManifest.xml +++ b/android/sample-app/src/main/AndroidManifest.xml @@ -18,5 +18,15 @@ + + + + diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt index 31b9cc4..a301ce4 100644 --- a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt @@ -13,8 +13,15 @@ import android.content.res.Configuration import android.graphics.Color import android.graphics.Typeface import android.graphics.drawable.GradientDrawable +import android.media.MediaCodec +import android.media.MediaFormat +import android.media.MediaExtractor +import android.media.MediaMuxer import android.media.MediaRecorder +import android.media.MediaMetadataRetriever import android.net.Uri +import android.os.Handler +import android.os.Looper import android.os.Build import android.os.Bundle import android.view.Gravity @@ -50,7 +57,13 @@ import cn.learningpad.oraltrainer.sdk.TrainingMediaItem import cn.learningpad.oraltrainer.sdk.TrainingVideoSummary import cn.learningpad.oraltrainer.sdk.VideoCatalogCallback import java.io.File +import java.nio.ByteBuffer import java.util.Locale +import androidx.core.content.FileProvider +import kotlin.math.max +import androidx.media3.common.MediaItem +import androidx.media3.common.Player +import androidx.media3.exoplayer.ExoPlayer class MainActivity : Activity() { private lateinit var sdk: OralTrainerSdk @@ -71,8 +84,18 @@ class MainActivity : Activity() { private lateinit var scoreSummaryText: TextView private lateinit var scoreDetailText: TextView private lateinit var recordButton: Button + private lateinit var playDubButton: Button + private lateinit var mergeDubButton: Button + private lateinit var shareDubButton: Button + private var dubPlayer: ExoPlayer? = null + private var dubPlayerListener: Player.Listener? = null + private var dubPlaybackGeneration = 0 + private val mainHandler = Handler(Looper.getMainLooper()) private var mediaRecorder: MediaRecorder? = null private var recordingFile: File? = null + private val dubSegments = mutableListOf() + private var mergedDubFile: File? = null + private var dubbingPlayback = false private var activeItemId: String = "" private var currentSentenceCount = 0 @@ -85,7 +108,11 @@ class MainActivity : Activity() { private val screenActionReceiver = object : BroadcastReceiver() { override fun onReceive(context: Context?, intent: Intent?) { if (intent?.action == Intent.ACTION_SCREEN_OFF) { - controller.pause() + if (dubbingPlayback) { + stopDubPlayback("已停止配音播放") + } else { + controller.pause() + } } } } @@ -153,7 +180,11 @@ class MainActivity : Activity() { override fun onPause() { super.onPause() - controller.pause() + if (dubbingPlayback) { + stopDubPlayback("已停止配音播放") + } else { + controller.pause() + } } override fun onResume() { @@ -227,7 +258,7 @@ class MainActivity : Activity() { } override fun onDestroy() { - discardRecording() + releaseDubbingResources() controller.release() super.onDestroy() } @@ -301,9 +332,6 @@ class MainActivity : Activity() { if (activeModule == module) { return@setOnClickListener } - if (activeModule == Module.TEST) { - discardRecording() - } activeModule = module controller.setContinuousPlayback( if (activeModule == Module.TRAIN) continuousPlaybackEnabled else true @@ -697,6 +725,23 @@ class MainActivity : Activity() { } } + val dubButton: (String, () -> Unit) -> Button = { label, action -> + Button(this).apply { + text = label + isAllCaps = false + setTextColor(Color.WHITE) + textSize = 13f + typeface = Typeface.DEFAULT_BOLD + minWidth = 0 + minHeight = 0 + minimumWidth = 0 + minimumHeight = 0 + background = rounded(COLOR_BUTTON, 8f) + layoutParams = LinearLayout.LayoutParams(0, 40.dp, 1f).withMargins(0, 8.dp, 8.dp, 0) + setOnClickListener { action() } + } + } + recordButton = Button(this).apply { text = "开始录音" isAllCaps = false @@ -711,6 +756,7 @@ class MainActivity : Activity() { layoutParams = LinearLayout.LayoutParams( ViewGroup.LayoutParams.MATCH_PARENT, 44.dp, + 1f, ) setOnClickListener { if (mediaRecorder == null) { @@ -721,6 +767,13 @@ class MainActivity : Activity() { } } + playDubButton = dubButton("播放配音", {}).apply { isEnabled = false } + mergeDubButton = dubButton("生成配音", {}).apply { isEnabled = false } + shareDubButton = dubButton("分享配音", {}).apply { isEnabled = false } + playDubButton.setOnClickListener { toggleDubPlayback() } + mergeDubButton.setOnClickListener { mergeDubSegments() } + shareDubButton.setOnClickListener { shareMergedDubbing() } + val controlRow = LinearLayout(this).apply { orientation = LinearLayout.HORIZONTAL addView(sentenceButton("上一句") { controller.seekToPreviousSentence() }) @@ -748,6 +801,12 @@ class MainActivity : Activity() { addView(testStatusText) addView(controlRow) addView(recordButton) + addView(LinearLayout(this@MainActivity).apply { + orientation = LinearLayout.HORIZONTAL + addView(playDubButton.apply { layoutParams = (layoutParams as LinearLayout.LayoutParams).withMargins(0, 8.dp, 8.dp, 0) }) + addView(mergeDubButton.apply { layoutParams = (layoutParams as LinearLayout.LayoutParams).withMargins(0, 8.dp, 8.dp, 0) }) + addView(shareDubButton.apply { layoutParams = (layoutParams as LinearLayout.LayoutParams).withMargins(0, 8.dp, 0, 0) }) + }) addView(scoreSummaryText) addView(scoreDetailText) } @@ -758,13 +817,15 @@ class MainActivity : Activity() { return } recordButton.text = if (mediaRecorder == null) "开始录音" else "停止并评测" + val segmentLabel = if (dubSegments.isEmpty()) "" else " · 已保存 ${dubSegments.size} 段" val item = controller.currentTrainingItem() val sentence = controller.currentSentence() testStatusText.text = if (item == null || sentence == null) { - "请先在训练模块选择一个云端课程" + "请先在训练模块选择课程$segmentLabel" } else { - "评测对象：第 ${sentence.index + 1} 句（共 ${item.sentences.size} 句）" + "评测对象：第 ${sentence.index + 1} 句（共 ${item.sentences.size} 句）$segmentLabel" } + refreshDubButtons() } private fun startRecording() { @@ -794,8 +855,8 @@ class MainActivity : Activity() { recorder.start() mediaRecorder = recorder recordingFile = file - recordButton.text = "停止并评测" - testStatusText.text = "正在录音…读完当前句子后点击“停止并评测”" + recordButton.text = "停止并保存" + testStatusText.text = "正在录音…读完当前句子后点击“停止并保存”" } catch (error: Throwable) { testStatusText.text = "录音启动失败：${error.message.orEmpty()}" runCatching { recorder.release() } @@ -806,7 +867,7 @@ class MainActivity : Activity() { val recorder = mediaRecorder ?: return val file = recordingFile ?: return mediaRecorder = null - recordButton.text = "开始录音" + recordButton.text = "重新录音" try { recorder.stop() } catch (error: Throwable) { @@ -817,19 +878,187 @@ class MainActivity : Activity() { } finally { runCatching { recorder.release() } } + dubSegments.add(file) + mergedDubFile = null recordingFile = null submitAssessment(Uri.fromFile(file)) + refreshTestUi() } private fun discardRecording() { - val recorder = mediaRecorder ?: return + mediaRecorder?.let { recorder -> + runCatching { recorder.stop() } + runCatching { recorder.release() } + } mediaRecorder = null - runCatching { recorder.stop() } - runCatching { recorder.release() } recordingFile?.delete() recordingFile = null } + private fun releaseDubbingResources() { + discardRecording() + dubPlayerListener?.let { listener -> dubPlayer?.removeListener(listener) } + dubPlayer?.release() + dubPlayerListener = null + dubPlayer = null + } + + private fun refreshDubButtons() { + playDubButton.isEnabled = dubSegments.isNotEmpty() && !dubbingPlayback + mergeDubButton.isEnabled = dubSegments.size >= 2 && !dubbingPlayback + shareDubButton.isEnabled = mergedDubFile?.exists() == true && !dubbingPlayback + playDubButton.text = when { + dubbingPlayback -> "停止配音" + mergedDubFile?.exists() == true -> "播放配音" + else -> "播放录音（${dubSegments.size}）" + } + } + + private fun toggleDubPlayback() { + if (dubbingPlayback) { + stopDubPlayback("已停止配音播放") + return + } + startDubPlayback(mergedDubFile ?: dubSegments.lastOrNull() ?: return) + } + + private fun startDubPlayback(audioFile: File) { + val sentence = controller.currentSentence() + val player = dubPlayer ?: ExoPlayer.Builder(this).build().also { dubPlayer = it } + dubPlayerListener?.let(player::removeListener) + val playbackGeneration = ++dubPlaybackGeneration + player.setMediaItem(MediaItem.fromUri(Uri.fromFile(audioFile))) + player.prepare() + player.volume = 1f + if (mergedDubFile == null && sentence != null) { + controller.seekTo(sentence.startMs) + } + controller.setVolume(0f) + controller.play() + player.playWhenReady = true + dubbingPlayback = true + dubPlayerListener = object : Player.Listener { + override fun onPlaybackStateChanged(playbackState: Int) { + if (playbackState == Player.STATE_ENDED && playbackGeneration == dubPlaybackGeneration) { + mainHandler.postDelayed({ + if (dubbingPlayback && playbackGeneration == dubPlaybackGeneration) { + stopDubPlayback("配音播放完成") + } + }, 250L) + } + } + } + player.addListener(dubPlayerListener!!) + refreshTestUi() + statusText.text = if (mergedDubFile == null) { + "正在播放当前句配音，原视频画面同步但原声静音" + } else { + "正在播放整段配音，原视频画面同步但原声静音" + } + } + + private fun stopDubPlayback(message: String) { + dubPlayer?.pause() + dubPlayer?.seekTo(0L) + controller.pause() + controller.setVolume(1f) + dubbingPlayback = false + dubPlaybackGeneration++ + refreshTestUi() + statusText.text = message + } + + private fun mergeDubSegments() { + if (dubSegments.size < 2) { + return + } + val output = File(cacheDir, "dubbing-${System.currentTimeMillis()}.m4a") + try { + val muxer = MediaMuxer(output.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) + var muxerAudioTrack = -1 + var presentationTimeUs = 0L + + dubSegments.forEach { segment -> + val extractor = MediaExtractor().apply { setDataSource(segment.absolutePath) } + try { + val trackIndex = (0 until extractor.trackCount).firstOrNull { index -> + extractor.getTrackFormat(index).getString(MediaFormat.KEY_MIME) + ?.startsWith("audio/") == true + } ?: throw IllegalStateException("录音没有音频轨道") + extractor.selectTrack(trackIndex) + val format = extractor.getTrackFormat(trackIndex) + if (muxerAudioTrack == -1) { + muxerAudioTrack = muxer.addTrack(format) + } + val buffer = ByteBuffer.allocateDirect( + if (format.containsKey(MediaFormat.KEY_MAX_INPUT_SIZE)) { + format.getInteger(MediaFormat.KEY_MAX_INPUT_SIZE) + } else { + 64 * 1024 + } + ) + val info = MediaCodec.BufferInfo() + while (true) { + val sampleSize = extractor.readSampleData(buffer, 0) + if (sampleSize < 0) break + info.set(0, sampleSize, presentationTimeUs, extractor.sampleFlags) + muxer.writeSampleData(muxerAudioTrack, buffer, info) + presentationTimeUs += 23_000L + extractor.advance() + } + val durationUs = segmentDurationUs(segment) + require(durationUs > 0L) { "无法读取录音时长" } + presentationTimeUs = max(presentationTimeUs, durationUs) + } finally { + extractor.release() + } + } + muxer.stop() + muxer.release() + mergedDubFile = output + refreshTestUi() + statusText.text = "已合成 ${dubSegments.size} 段配音：${output.name}" + } catch (error: Throwable) { + output.delete() + statusText.text = "合成失败：${error.message.orEmpty()}" + } + } + + private fun segmentDurationUs(segment: File): Long { + val retriever = MediaMetadataRetriever() + return try { + retriever.setDataSource(segment.absolutePath) + val durationMs = retriever.extractMetadata( + MediaMetadataRetriever.METADATA_KEY_DURATION + )?.toLongOrNull() ?: return 0L + durationMs * 1000L + } catch (error: Throwable) { + 0L + } finally { + runCatching { retriever.release() } + } + } + + private fun shareMergedDubbing() { + val file = mergedDubFile?.takeIf { it.exists() } ?: return + try { + val uri = FileProvider.getUriForFile( + this, + "${packageName}.fileprovider", + file, + ) + val intent = Intent(Intent.ACTION_SEND).apply { + type = "audio/mp4" + putExtra(Intent.EXTRA_STREAM, uri) + putExtra(Intent.EXTRA_TITLE, "我的口语配音") + addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION or Intent.FLAG_ACTIVITY_NEW_TASK) + } + startActivity(Intent.createChooser(intent, "分享配音")) + } catch (error: Throwable) { + statusText.text = "分享失败：${error.message.orEmpty()}" + } + } + private fun submitAssessment(recordingUri: Uri) { val item = controller.currentTrainingItem() val sentence = controller.currentSentence() diff --git a/android/sample-app/src/main/res/xml/file_paths.xml b/android/sample-app/src/main/res/xml/file_paths.xml new file mode 100644 index 0000000..d7ac3e9 --- /dev/null +++ b/android/sample-app/src/main/res/xml/file_paths.xml @@ -0,0 +1,6 @@ + + + +