fix a bug

This commit is contained in:
2026-09-05 13:09:39 +08:00
parent f928dd5c68
commit 6d9f9a5468
13 changed files with 805 additions and 107 deletions

View File

@@ -258,9 +258,14 @@ class OralTrainerController internal constructor(
fun playCurrentSentenceAndStop() {
val sentence = currentSentence() ?: return
player.seekTo(sentence.startMs)
playSentenceAndStop(sentence.startMs, sentence.index)
}
/** Seek to a known sentence and stop automatically when that sentence ends. */
fun playSentenceAndStop(startMs: Long, sentenceIndex: Int) {
stopAtSentenceEnd = true
stopAtSentenceEndIndex = sentence.index
stopAtSentenceEndIndex = sentenceIndex
player.seekTo(max(0L, startMs))
player.play()
refreshSentenceBoundaryGuard()
}

View File

@@ -67,6 +67,21 @@ data class UserDubShare(
val createdAt: String,
)
data class DubShareSegmentDetail(
val index: Int,
val overallScore: Float,
val recognizedText: String,
val scoreDetails: Map<String, Float>,
val audioUrl: String,
)
data class DubShareDetail(
val shareId: String,
val videoHash: String,
val title: String,
val segments: List<DubShareSegmentDetail>,
)
class UserApiException(
val statusCode: Int,
responseBody: String,
@@ -150,6 +165,13 @@ class UserApi internal constructor(
onComplete: (Result<List<UserDubShare>>) -> Unit,
): CancellableRequest = submitObject(onComplete, token) { parseDubShares(getBlocking(it, "api/v1/me/dub-shares")) }
fun dubShareDetail(
shareId: String,
onComplete: (Result<DubShareDetail>) -> Unit,
): CancellableRequest = submit(onComplete) {
parseDubShareDetail(JSONObject(request("api/v1/dub-shares/$shareId", method = "GET")))
}
fun release() {
executor.shutdownNow()
}
@@ -252,6 +274,34 @@ class UserApi internal constructor(
)
}
private fun parseDubShareDetail(root: JSONObject): DubShareDetail {
val segmentsJson = root.optJSONArray("segments") ?: JSONArray()
val segments = List(segmentsJson.length()) { index ->
val item = segmentsJson.getJSONObject(index)
val detailsJson = item.optJSONObject("score_details") ?: JSONObject()
val scoreDetails = buildMap<String, Float> {
for (key in detailsJson.keys()) {
if (!detailsJson.isNull(key)) {
put(key, detailsJson.optDouble(key).toFloat())
}
}
}
DubShareSegmentDetail(
index = item.getInt("index"),
overallScore = item.optDouble("overall_score", 0.0).toFloat(),
recognizedText = item.optString("recognized_text"),
scoreDetails = scoreDetails,
audioUrl = item.getString("audio_url"),
)
}
return DubShareDetail(
shareId = root.getString("share_id"),
videoHash = root.getString("video_hash"),
title = root.optString("title"),
segments = segments,
)
}
private fun request(
path: String,
token: String? = null,

View File

@@ -9,5 +9,19 @@ internal class AssessmentState {
scores[sentenceIndex] = result
}
fun get(sentenceIndex: Int): ImitationAssessmentResult? = scores[sentenceIndex]
fun replaceAll(values: Map<Int, ImitationAssessmentResult>) {
scores.clear()
scores.putAll(values)
}
fun clear() {
scores.clear()
}
fun toMap(): Map<Int, ImitationAssessmentResult> = scores.toMap()
val size: Int
get() = scores.size
}

View File

@@ -0,0 +1,244 @@
package cn.learningpad.oraltrainer.sample
import android.content.Context
import cn.learningpad.oraltrainer.sdk.ImitationAssessmentResult
import java.io.File
import java.net.HttpURLConnection
import java.net.URL
import java.util.Locale
import org.json.JSONArray
import org.json.JSONObject
/** Persists per-course dubbing audio and assessment scores under app filesDir. */
internal class DubbingHistoryStore(context: Context) {
private val rootDir = File(context.applicationContext.filesDir, "dubbing-history")
fun loadSnapshot(videoHash: String): Snapshot {
val normalized = normalize(videoHash) ?: return Snapshot()
val dir = videoDir(normalized)
if (!dir.isDirectory) {
return Snapshot()
}
val segments = dir.listFiles()
?.mapNotNull { file ->
val match = SEGMENT_PATTERN.matchEntire(file.name) ?: return@mapNotNull null
if (!file.isFile || file.length() <= 0L) return@mapNotNull null
match.groupValues[1].toInt() to file
}
?.toMap()
.orEmpty()
val merged = File(dir, MERGED_FILE_NAME).takeIf { it.isFile && it.length() > 0L }
val scores = readScores(File(dir, SCORES_FILE_NAME))
return Snapshot(segments = segments, scores = scores, mergedFile = merged)
}
fun saveSegment(videoHash: String, sentenceIndex: Int, source: File): File {
val normalized = requireNotNull(normalize(videoHash)) { "invalid video hash" }
val dest = File(videoDir(normalized), "segment-$sentenceIndex.m4a")
if (source.canonicalFile != dest.canonicalFile) {
source.copyTo(dest, overwrite = true)
}
// Any new recording invalidates the previous merged export.
File(videoDir(normalized), MERGED_FILE_NAME).delete()
return dest
}
fun saveScores(videoHash: String, scores: Map<Int, ImitationAssessmentResult>) {
val normalized = normalize(videoHash) ?: return
val file = File(videoDir(normalized), SCORES_FILE_NAME)
val root = JSONObject()
scores.toSortedMap().forEach { (index, result) ->
root.put(index.toString(), encodeScore(result))
}
file.writeText(root.toString())
}
fun saveMerged(videoHash: String, source: File): File {
val normalized = requireNotNull(normalize(videoHash)) { "invalid video hash" }
val dest = File(videoDir(normalized), MERGED_FILE_NAME)
if (source.canonicalFile != dest.canonicalFile) {
source.copyTo(dest, overwrite = true)
}
return dest
}
fun clear(videoHash: String) {
val normalized = normalize(videoHash) ?: return
videoDir(normalized).deleteRecursively()
}
/**
* Downloads one public share-audio URL into local history.
* Returns the saved file, or null when the download fails.
*/
fun downloadSegmentAudio(
baseUrl: String,
videoHash: String,
sentenceIndex: Int,
audioUrl: String,
): File? {
val normalized = normalize(videoHash) ?: return null
val absolute = when {
audioUrl.startsWith("http://") || audioUrl.startsWith("https://") -> audioUrl
audioUrl.startsWith("/") -> baseUrl.trimEnd('/') + audioUrl
else -> baseUrl.trimEnd('/') + "/" + audioUrl
}
var connection: HttpURLConnection? = null
return try {
connection = URL(absolute).openConnection() as HttpURLConnection
connection.connectTimeout = 15_000
connection.readTimeout = 60_000
connection.requestMethod = "GET"
connection.instanceFollowRedirects = true
val code = connection.responseCode
if (code !in 200..299) {
return null
}
val dest = File(videoDir(normalized), "segment-$sentenceIndex.m4a")
connection.inputStream.use { input ->
dest.outputStream().use { output -> input.copyTo(output) }
}
if (dest.length() <= 0L) {
dest.delete()
null
} else {
dest
}
} catch (_: Throwable) {
null
} finally {
connection?.disconnect()
}
}
private fun videoDir(videoHash: String): File =
File(rootDir, videoHash).also { it.mkdirs() }
private fun readScores(file: File): Map<Int, ImitationAssessmentResult> {
if (!file.isFile) {
return emptyMap()
}
return runCatching {
val root = JSONObject(file.readText())
buildMap {
for (key in root.keys()) {
val index = key.toIntOrNull() ?: continue
val value = root.optJSONObject(key) ?: continue
put(index, decodeScore(value))
}
}
}.getOrDefault(emptyMap())
}
data class Snapshot(
val segments: Map<Int, File> = emptyMap(),
val scores: Map<Int, ImitationAssessmentResult> = emptyMap(),
val mergedFile: File? = null,
)
companion object {
private const val SCORES_FILE_NAME = "scores.json"
private const val MERGED_FILE_NAME = "merged.m4a"
private val SEGMENT_PATTERN = Regex("""segment-(\d+)\.m4a""")
fun normalize(videoHash: String): String? {
val value = videoHash.trim().lowercase(Locale.US)
return value.takeIf { PlaybackFormat.isVideoHash(it) }
}
fun encodeScore(result: ImitationAssessmentResult): JSONObject =
JSONObject().apply {
put("overall_score", result.overallScore.toDouble())
result.pronunciationScore?.let { put("pronunciation_score", it.toDouble()) }
result.fluencyScore?.let { put("fluency_score", it.toDouble()) }
result.completenessScore?.let { put("completeness_score", it.toDouble()) }
result.feedback?.let { put("feedback", it) }
if (result.details.isNotEmpty()) {
put(
"details",
JSONObject().also { details ->
result.details.forEach { (key, value) -> details.put(key, value) }
},
)
}
result.passed?.let { put("passed", it) }
result.contentScore?.let { put("content_score", it.toDouble()) }
result.durationScore?.let { put("duration_score", it.toDouble()) }
result.pauseScore?.let { put("pause_score", it.toDouble()) }
result.speechRateScore?.let { put("speech_rate_score", it.toDouble()) }
result.durationRatio?.let { put("duration_ratio", it.toDouble()) }
result.referenceText?.let { put("reference_text", it) }
result.recognizedText?.let { put("recognized_text", it) }
if (result.missingTokens.isNotEmpty()) {
put("missing_tokens", JSONArray(result.missingTokens))
}
if (result.extraTokens.isNotEmpty()) {
put("extra_tokens", JSONArray(result.extraTokens))
}
if (result.substitutions.isNotEmpty()) {
put(
"substitutions",
JSONArray().also { array ->
result.substitutions.forEach { (expected, actual) ->
array.put(
JSONObject()
.put("expected", expected)
.put("actual", actual),
)
}
},
)
}
}
fun decodeScore(root: JSONObject): ImitationAssessmentResult {
val substitutions = buildList {
val array = root.optJSONArray("substitutions") ?: return@buildList
for (index in 0 until array.length()) {
val item = array.optJSONObject(index) ?: continue
add(item.optString("expected") to item.optString("actual"))
}
}
val details = buildMap {
val objectValue = root.optJSONObject("details") ?: return@buildMap
for (key in objectValue.keys()) {
put(key, objectValue.optString(key))
}
}
return ImitationAssessmentResult(
overallScore = root.optDouble("overall_score", 0.0).toFloat(),
pronunciationScore = root.optNullableFloat("pronunciation_score"),
fluencyScore = root.optNullableFloat("fluency_score"),
completenessScore = root.optNullableFloat("completeness_score"),
feedback = root.optString("feedback").ifBlank { null },
details = details,
passed = if (root.has("passed")) root.optBoolean("passed") else null,
contentScore = root.optNullableFloat("content_score"),
durationScore = root.optNullableFloat("duration_score"),
pauseScore = root.optNullableFloat("pause_score"),
speechRateScore = root.optNullableFloat("speech_rate_score"),
durationRatio = root.optNullableFloat("duration_ratio"),
referenceText = root.optString("reference_text").ifBlank { null },
recognizedText = root.optString("recognized_text").ifBlank { null },
missingTokens = root.optStringList("missing_tokens"),
extraTokens = root.optStringList("extra_tokens"),
substitutions = substitutions,
)
}
private fun JSONObject.optNullableFloat(name: String): Float? {
if (!has(name) || isNull(name)) return null
return optDouble(name).toFloat()
}
private fun JSONObject.optStringList(name: String): List<String> {
val array = optJSONArray(name) ?: return emptyList()
return buildList {
for (index in 0 until array.length()) {
val value = array.optString(index)
if (value.isNotBlank()) add(value)
}
}
}
}
}

View File

@@ -24,8 +24,8 @@ internal class DubbingPlayback(
fun start(
audioFile: File,
controller: OralTrainerController,
syncToSentence: Boolean,
sentenceStartMs: Long?,
sentenceStartMs: Long,
sentenceIndex: Int,
onComplete: () -> Unit,
) {
listener?.let(player::removeListener)
@@ -45,11 +45,9 @@ internal class DubbingPlayback(
}
}.also(player::addListener)
if (syncToSentence && sentenceStartMs != null) {
controller.seekTo(sentenceStartMs)
}
// Mute original audio and drive the video for this sentence only.
controller.setVolume(0f)
controller.play()
controller.playSentenceAndStop(sentenceStartMs, sentenceIndex)
player.setMediaItem(MediaItem.fromUri(Uri.fromFile(audioFile)))
player.prepare()
player.volume = 1f

View File

@@ -10,11 +10,13 @@ import org.json.JSONObject
internal class DubbingShareApi {
fun upload(
baseUrl: String,
token: String,
videoHash: String,
videoTitle: String,
segments: Map<Int, java.io.File>,
scores: Map<Int, cn.learningpad.oraltrainer.sdk.ImitationAssessmentResult>,
): String {
require(token.isNotBlank()) { "Login required." }
val previousPolicy = StrictMode.getThreadPolicy()
StrictMode.setThreadPolicy(StrictMode.ThreadPolicy.Builder().detectAll().penaltyLog().build())
val boundary = "oral-trainer-${System.currentTimeMillis()}"
@@ -25,6 +27,7 @@ internal class DubbingShareApi {
connection.doOutput = true
connection.connectTimeout = 15_000
connection.readTimeout = 120_000
connection.setRequestProperty("Authorization", "Bearer $token")
connection.setRequestProperty("Content-Type", "multipart/form-data; boundary=$boundary")
DataOutputStream(connection.outputStream).use { output ->
@@ -83,7 +86,9 @@ internal class DubbingShareApi {
val code = connection.responseCode
val body = (if (code in 200..299) connection.inputStream else connection.errorStream)
?.bufferedReader()?.use { it.readText() }.orEmpty()
require(code in 200..299) { "HTTP $code: $body" }
if (code !in 200..299) {
throw IllegalStateException(friendlyShareError(code, body))
}
val response = JSONObject(body).getString("share_id")
return "$baseUrl/dub-shares/$response"
} finally {
@@ -91,4 +96,19 @@ internal class DubbingShareApi {
StrictMode.setThreadPolicy(previousPolicy)
}
}
private fun friendlyShareError(code: Int, body: String): String {
val detail = runCatching { JSONObject(body).optString("detail") }.getOrNull()
?.takeIf { it.isNotBlank() }
?: body
return when {
code == 401 ||
detail.contains("Login required", ignoreCase = true) ||
detail.contains("sign in", ignoreCase = true) ||
detail.contains("Session expired", ignoreCase = true) ->
"还未登录或登录已过期,请重新登录后再分享"
detail.isNotBlank() -> "HTTP $code: $detail"
else -> "HTTP $code"
}
}
}

View File

@@ -8,12 +8,29 @@ internal class DubbingState {
private set
fun storeSegment(sentenceIndex: Int, file: File) {
segments[sentenceIndex]?.delete()
segments[sentenceIndex] = file
val previous = segments.put(sentenceIndex, file)
if (previous != null && previous.absolutePath != file.absolutePath) {
previous.delete()
}
mergedFile = null
}
fun setMergedFile(file: File) {
fun setMergedFile(file: File?) {
mergedFile = file
}
fun replaceAll(newSegments: Map<Int, File>, merged: File?) {
val keep = newSegments.values.map { it.absolutePath }.toHashSet()
segments.values
.filter { it.absolutePath !in keep }
.forEach { /* keep history files on disk; only drop memory refs */ }
segments.clear()
segments.putAll(newSegments)
mergedFile = merged?.takeIf { it.exists() }
}
fun clearMemory() {
segments.clear()
mergedFile = null
}
}

View File

@@ -32,6 +32,7 @@ class MainActivity : Activity() {
private val dubbingState = DubbingState()
private val assessmentState = AssessmentState()
private val dubbingShareApi = DubbingShareApi()
private lateinit var dubbingHistoryStore: DubbingHistoryStore
private lateinit var attemptRecorder: AttemptRecorder
private lateinit var dubbingMerger: DubbingMerger
@@ -57,6 +58,7 @@ class MainActivity : Activity() {
attemptRecorder = AttemptRecorder(this)
dubbingMerger = DubbingMerger(cacheDir)
dubbingPlaybackController = DubbingPlayback(this)
dubbingHistoryStore = DubbingHistoryStore(this)
configureWindow()
authSessionStore = AuthSessionStore(this)
@@ -89,6 +91,9 @@ class MainActivity : Activity() {
dubbingShareApi = dubbingShareApi,
dubbingPlayback = dubbingPlaybackController,
assessmentState = assessmentState,
historyStore = dubbingHistoryStore,
userApi = sdk.userApi,
tokenProvider = { authSessionStore.session?.token.orEmpty() },
host = testHost(),
)
@@ -109,6 +114,7 @@ class MainActivity : Activity() {
screenFactory.renderCatalog()
refreshCurrentUi()
if (screenState.activeModule == Module.TEST) {
testSession.ensureHistoryForCurrentCourse()
testSession.refreshTestUi()
}
}
@@ -268,6 +274,9 @@ class MainActivity : Activity() {
}
private fun switchModule(module: Module) {
if (module != Module.TEST) {
testSession.stopDubbingIfNeeded()
}
screenState.activeModule = module
if (module == Module.COURSE || module == Module.MINE) {
catalogController.loadUserData(module) {
@@ -310,9 +319,14 @@ class MainActivity : Activity() {
}
override fun onMediaChanged(item: TrainingMediaItem?) {
if (!bindings.hasPlayerChrome) return
bindings.lessonTitleText.text = item?.title ?: "未选择课程"
screenState.currentSentenceCount = item?.sentences?.size ?: 0
if (bindings.hasPlayerChrome) {
bindings.lessonTitleText.text = item?.title ?: "未选择课程"
screenState.currentSentenceCount = item?.sentences?.size ?: 0
}
testSession.ensureHistoryForCurrentCourse()
if (screenState.activeModule == Module.TEST) {
testSession.refreshTestUi()
}
}
override fun onSentenceChanged(sentence: SentenceBoundary?) {
@@ -408,15 +422,21 @@ class MainActivity : Activity() {
}
override fun onPrevSentence() {
testSession.stopDubbingIfNeeded()
controller.seekToPreviousSentence()
testSession.refreshTestUi()
}
override fun onReplaySentence() {
testSession.stopDubbingIfNeeded()
testSession.refreshTestUi()
controller.playCurrentSentenceAndStop()
}
override fun onNextSentence() {
testSession.stopDubbingIfNeeded()
controller.seekToNextSentence()
testSession.refreshTestUi()
}
override fun onToggleRecording() = testSession.toggleRecording()

View File

@@ -19,5 +19,6 @@ internal object SampleTheme {
val textDark = Color.rgb(18, 24, 31)
val textMuted = Color.rgb(151, 162, 174)
val textSubtle = Color.rgb(88, 98, 108)
val danger = Color.rgb(220, 38, 38)
val textOverlay = Color.rgb(163, 172, 182)
}

View File

@@ -5,13 +5,18 @@ import android.app.Activity
import android.content.Intent
import android.content.pm.PackageManager
import android.net.Uri
import android.os.Handler
import android.os.Looper
import android.util.Log
import android.view.View
import cn.learningpad.oraltrainer.sdk.ImitationAssessmentCallback
import cn.learningpad.oraltrainer.sdk.ImitationAssessmentResult
import cn.learningpad.oraltrainer.sdk.OralTrainerController
import cn.learningpad.oraltrainer.sdk.UserApi
import java.io.File
import java.util.Locale
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/** Recording, assessment, and dubbing flow for the TEST module. */
internal class TestSessionController(
@@ -24,6 +29,9 @@ internal class TestSessionController(
private val dubbingShareApi: DubbingShareApi,
private val dubbingPlayback: DubbingPlayback,
private val assessmentState: AssessmentState,
private val historyStore: DubbingHistoryStore,
private val userApi: UserApi,
private val tokenProvider: () -> String,
private val host: Host,
) {
interface Host {
@@ -33,6 +41,15 @@ internal class TestSessionController(
fun refreshPlayerUi()
}
private val mainHandler = Handler(Looper.getMainLooper())
private val background = Executors.newSingleThreadExecutor()
private val remoteLoadGeneration = AtomicInteger(0)
private var loadedVideoHash: String? = null
private var assessing = false
private var sharing = false
private var historyStatusMessage: String? = null
val isDubbingActive: Boolean
get() = dubbingPlayback.isActive
@@ -47,8 +64,10 @@ internal class TestSessionController(
}
fun release() {
remoteLoadGeneration.incrementAndGet()
attemptRecorder.discard()
dubbingPlayback.release()
background.shutdownNow()
}
fun stopDubbingIfNeeded() {
@@ -57,7 +76,29 @@ internal class TestSessionController(
}
}
/** Loads local (and if needed remote) history for the active cloud course. */
fun ensureHistoryForCurrentCourse(forceRemote: Boolean = false) {
val videoHash = currentVideoHash()
if (videoHash == null) {
if (loadedVideoHash != null) {
loadedVideoHash = null
dubbingState.clearMemory()
assessmentState.clear()
historyStatusMessage = null
}
return
}
if (loadedVideoHash == videoHash && !forceRemote) {
return
}
loadLocalHistory(videoHash)
if (forceRemote || dubbingState.segments.isEmpty()) {
restoreRemoteHistory(videoHash)
}
}
fun refreshTestUi() {
ensureHistoryForCurrentCourse()
val ui = host.bindings
if (!ui.hasTestChrome) {
return
@@ -68,22 +109,40 @@ internal class TestSessionController(
val sentence = controller.currentSentence()
val currentRecording = sentence?.let { dubbingState.segments[it.index] }
val segmentLabel = when {
currentRecording != null -> " · 本句已录音,重录将覆盖"
else -> ""
currentRecording != null -> "本句已有配音,重录将覆盖"
else -> null
}
ui.testStatusText.text = if (item == null || sentence == null) {
"请先在训练模块选择课程$segmentLabel"
} else {
"评测对象:第 ${sentence.index + 1} 句(共 ${item.sentences.size} 句)$segmentLabel"
}
refreshDubButtons()
if (dubbingState.mergedFile == null) {
ui.dubbingStatusText?.text = when (dubbingState.segments.size) {
0 -> "配音:请先录音"
1 -> "配音:已保存 1 句,可生成配音"
else -> "配音:已保存 ${dubbingState.segments.size} 句,可生成配音"
if (!assessing && !attemptRecorder.isRecording) {
val score = sentence?.let { assessmentState.get(it.index) }
if (score != null) {
applyAssessmentResult(score, segmentLabel)
} else {
ui.testStatusText.text = when {
item == null || sentence == null -> "请先在训练模块选择课程"
segmentLabel != null -> "评测对象:第 ${sentence.index + 1} 句(共 ${item.sentences.size} 句) · $segmentLabel"
else -> "评测对象:第 ${sentence.index + 1} 句(共 ${item.sentences.size} 句)"
}
ui.scoreSummaryText.text = "尚未评测"
ui.scoreDetailText.text = ""
}
}
refreshDubButtons()
if (!sharing) {
val historyNote = historyStatusMessage
if (dubbingState.mergedFile?.exists() == true) {
ui.dubbingStatusText?.text = historyNote
?: "已合成 ${dubbingState.segments.size} 句配音,可点击“分享配音”"
} else {
ui.dubbingStatusText?.text = historyNote ?: when (dubbingState.segments.size) {
0 -> "配音:请先录音(进入本页会自动加载历史配音)"
1 -> "配音:已保存 1 句,可生成配音"
else -> "配音:已保存 ${dubbingState.segments.size} 句,可生成配音"
}
}
ui.dubbingStatusText?.setTextColor(SampleTheme.accentDeep)
}
}
fun toggleRecording() {
@@ -95,6 +154,7 @@ internal class TestSessionController(
}
fun startRecording() {
stopDubbingIfNeeded()
controller.pause()
val ui = host.bindings
if (!ui.hasTestChrome) {
@@ -136,18 +196,41 @@ internal class TestSessionController(
host.setMainStatus("已停止配音播放")
return
}
val audio = dubbingState.mergedFile ?: activeAudioForPlayback() ?: return
startDubPlayback(audio)
val sentence = controller.currentSentence()
if (sentence == null) {
host.setMainStatus("请先选择要播放的句子")
return
}
val audio = dubbingState.segments[sentence.index]
if (audio == null || !audio.exists()) {
host.setMainStatus("当前句还没有录音,请先录音后再播放")
refreshTestUi()
return
}
startDubPlayback(audio, sentence.startMs, sentence.index)
}
fun mergeDubSegments() {
stopDubbingIfNeeded()
val videoHash = currentVideoHash()
val segments = dubbingState.segments.toSortedMap().map { (index, file) -> index to file }
if (segments.isEmpty()) {
setDubbingStatus("没有可合成的配音")
return
}
try {
val output = dubbingMerger.merge(segments)
dubbingState.setMergedFile(output)
val persisted = if (videoHash != null) {
historyStore.saveMerged(videoHash, output).also {
if (it.absolutePath != output.absolutePath) {
output.delete()
}
}
} else {
output
}
dubbingState.setMergedFile(persisted)
historyStatusMessage = null
refreshTestUi()
setDubbingStatus("已合成 ${segments.size} 句配音,可点击“分享配音”")
} catch (error: Throwable) {
@@ -160,30 +243,62 @@ internal class TestSessionController(
}
fun shareMergedDubbing() {
stopDubbingIfNeeded()
if (sharing) {
return
}
val item = controller.currentTrainingItem()
if (item == null || !PlaybackFormat.isVideoHash(item.id)) {
setDubbingStatus("分享失败:网页分享仅支持云端课程视频")
return
}
try {
val shareUrl = dubbingShareApi.upload(
baseUrl = SERVER_BASE_URL,
videoHash = item.id,
videoTitle = item.title,
segments = dubbingState.segments,
scores = assessmentState.toMap(),
)
val intent = Intent(Intent.ACTION_SEND).apply {
type = "text/plain"
putExtra(Intent.EXTRA_TEXT, "我的口语配音:$shareUrl")
putExtra(Intent.EXTRA_TITLE, "我的口语配音")
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
val token = tokenProvider().trim()
if (token.isEmpty()) {
setDubbingStatus("分享失败:请先登录后再分享配音")
return
}
val segmentsSnapshot = dubbingState.segments.toMap()
val scoresSnapshot = assessmentState.toMap()
val videoHash = item.id
val videoTitle = item.title
sharing = true
setDubbingStatus("正在生成分享链接,请稍候...", alert = true)
refreshDubButtons()
background.execute {
try {
val shareUrl = dubbingShareApi.upload(
baseUrl = SERVER_BASE_URL,
token = token,
videoHash = videoHash,
videoTitle = videoTitle,
segments = segmentsSnapshot,
scores = scoresSnapshot,
)
mainHandler.post {
sharing = false
if (!host.bindings.hasTestChrome) {
return@post
}
val intent = Intent(Intent.ACTION_SEND).apply {
type = "text/plain"
putExtra(Intent.EXTRA_TEXT, "我的口语配音:$shareUrl")
putExtra(Intent.EXTRA_TITLE, "我的口语配音")
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
}
activity.startActivity(Intent.createChooser(intent, "分享配音"))
setDubbingStatus("已生成分享链接:$shareUrl")
refreshDubButtons()
}
} catch (error: Throwable) {
Log.e(TAG, "share failed", error)
val detail = error.message?.takeIf { it.isNotBlank() }
?: error.javaClass.simpleName
mainHandler.post {
sharing = false
setDubbingStatus("分享失败:$detail", alert = true)
refreshDubButtons()
}
}
activity.startActivity(Intent.createChooser(intent, "分享配音"))
setDubbingStatus("已生成分享链接:$shareUrl")
} catch (error: Throwable) {
Log.e(TAG, "share failed", error)
setDubbingStatus("分享失败:${error.javaClass.simpleName}: ${error.message.orEmpty()}")
}
}
@@ -195,6 +310,160 @@ internal class TestSessionController(
refreshTestUi()
}
private fun loadLocalHistory(videoHash: String) {
val snapshot = historyStore.loadSnapshot(videoHash)
dubbingState.replaceAll(snapshot.segments, snapshot.mergedFile)
assessmentState.replaceAll(snapshot.scores)
loadedVideoHash = videoHash
historyStatusMessage = when {
snapshot.segments.isNotEmpty() && snapshot.scores.isNotEmpty() ->
"已加载历史配音 ${snapshot.segments.size} 句、评分 ${snapshot.scores.size} 句"
snapshot.segments.isNotEmpty() ->
"已加载历史配音 ${snapshot.segments.size} 句"
snapshot.scores.isNotEmpty() ->
"已加载历史评分 ${snapshot.scores.size} 句(暂无本地录音)"
else -> null
}
Log.i(
TAG,
"loaded local history video=$videoHash segments=${snapshot.segments.size} scores=${snapshot.scores.size}",
)
}
private fun restoreRemoteHistory(videoHash: String) {
val token = tokenProvider().trim()
if (token.isEmpty()) {
return
}
val generation = remoteLoadGeneration.incrementAndGet()
historyStatusMessage = when {
dubbingState.segments.isEmpty() && assessmentState.size == 0 -> "正在同步历史配音…"
else -> historyStatusMessage
}
if (host.bindings.hasTestChrome) {
host.bindings.dubbingStatusText?.text = historyStatusMessage
}
// Latest score summary per sentence.
userApi.results(token) { result ->
if (generation != remoteLoadGeneration.get() || loadedVideoHash != videoHash) {
return@results
}
result.onSuccess { rows ->
val latestBySentence = linkedMapOf<Int, cn.learningpad.oraltrainer.sdk.UserResult>()
rows.filter { it.videoHash.equals(videoHash, ignoreCase = true) }
.sortedBy { it.createdAt }
.forEach { latestBySentence[it.sentenceIndex] = it }
var added = 0
latestBySentence.forEach { (index, row) ->
if (assessmentState.get(index) == null) {
assessmentState.store(
index,
ImitationAssessmentResult(
overallScore = row.overallScore,
passed = row.passed,
referenceText = row.sentenceText.takeIf { it.isNotBlank() },
feedback = "来自历史测试记录",
),
)
added += 1
}
}
if (added > 0) {
historyStore.saveScores(videoHash, assessmentState.toMap())
historyStatusMessage = "已同步历史评分 ${assessmentState.size} 句"
refreshTestUi()
}
}
}
// Latest shared dubbing audio for this course.
userApi.dubShares(token) { shareResult ->
if (generation != remoteLoadGeneration.get() || loadedVideoHash != videoHash) {
return@dubShares
}
val latestShare = shareResult.getOrNull()
?.filter { it.videoHash.equals(videoHash, ignoreCase = true) }
?.maxByOrNull { it.createdAt }
if (latestShare == null) {
if (dubbingState.segments.isEmpty() && assessmentState.size == 0) {
historyStatusMessage = "暂无历史配音,请开始录音"
if (host.bindings.hasTestChrome) {
host.bindings.dubbingStatusText?.text = historyStatusMessage
}
}
return@dubShares
}
if (dubbingState.segments.isNotEmpty()) {
return@dubShares
}
userApi.dubShareDetail(latestShare.shareId) { detailResult ->
if (generation != remoteLoadGeneration.get() || loadedVideoHash != videoHash) {
return@dubShareDetail
}
val detail = detailResult.getOrNull() ?: run {
Log.w(TAG, "dub share detail failed: ${detailResult.exceptionOrNull()?.message}")
return@dubShareDetail
}
background.execute {
val downloaded = linkedMapOf<Int, File>()
val remoteScores = linkedMapOf<Int, ImitationAssessmentResult>()
detail.segments.forEach { segment ->
val file = historyStore.downloadSegmentAudio(
baseUrl = SERVER_BASE_URL,
videoHash = videoHash,
sentenceIndex = segment.index,
audioUrl = segment.audioUrl,
)
if (file != null) {
downloaded[segment.index] = file
}
if (assessmentState.get(segment.index) == null) {
remoteScores[segment.index] = ImitationAssessmentResult(
overallScore = segment.overallScore,
contentScore = segment.scoreDetails["content_score"],
fluencyScore = segment.scoreDetails["fluency_score"],
durationScore = segment.scoreDetails["duration_score"],
pauseScore = segment.scoreDetails["pause_score"],
speechRateScore = segment.scoreDetails["speech_rate_score"],
recognizedText = segment.recognizedText.takeIf { it.isNotBlank() },
feedback = "来自历史配音分享",
)
}
}
mainHandler.post {
if (generation != remoteLoadGeneration.get() || loadedVideoHash != videoHash) {
return@post
}
if (downloaded.isNotEmpty()) {
val mergedSegments = dubbingState.segments.toMutableMap()
mergedSegments.putAll(downloaded)
dubbingState.replaceAll(mergedSegments, dubbingState.mergedFile)
}
if (remoteScores.isNotEmpty()) {
val mergedScores = assessmentState.toMap().toMutableMap()
remoteScores.forEach { (index, score) ->
if (mergedScores[index] == null) {
mergedScores[index] = score
}
}
assessmentState.replaceAll(mergedScores)
historyStore.saveScores(videoHash, assessmentState.toMap())
}
historyStatusMessage = when {
downloaded.isNotEmpty() ->
"已从云端恢复配音 ${downloaded.size} 句"
assessmentState.size > 0 ->
"已同步历史评分 ${assessmentState.size} 句"
else -> "暂无可用的历史配音音频"
}
refreshTestUi()
}
}
}
}
}
private fun stopRecordingAndAssess() {
if (!attemptRecorder.isRecording) return
val ui = host.bindings
@@ -215,12 +484,28 @@ internal class TestSessionController(
refreshTestUi()
return
}
dubbingState.storeSegment(sentenceIndex, file)
submitAssessment(Uri.fromFile(file))
val videoHash = currentVideoHash()
val persisted = if (videoHash != null) {
try {
historyStore.saveSegment(videoHash, sentenceIndex, file).also {
if (it.absolutePath != file.absolutePath) {
file.delete()
}
}
} catch (error: Throwable) {
Log.e(TAG, "persist segment failed", error)
file
}
} else {
file
}
dubbingState.storeSegment(sentenceIndex, persisted)
historyStatusMessage = null
submitAssessment(Uri.fromFile(persisted), sentenceIndex)
refreshTestUi()
}
private fun submitAssessment(recordingUri: Uri) {
private fun submitAssessment(recordingUri: Uri, sentenceIndex: Int) {
val ui = host.bindings
val item = controller.currentTrainingItem()
val sentence = controller.currentSentence()
@@ -236,6 +521,7 @@ internal class TestSessionController(
ui.testStatusText.text = "未配置评分密钥:请在 AppConfig 填入服务器 .env 的 CLIENT_API_KEY"
return
}
assessing = true
ui.testStatusText.text = "正在评测第 ${sentence.index + 1} 句,请稍候…"
ui.scoreSummaryText.text = "评测中…"
ui.scoreDetailText.text = ""
@@ -247,13 +533,24 @@ internal class TestSessionController(
metadata = mapOf("video_hash" to item.id),
callback = object : ImitationAssessmentCallback {
override fun onResult(result: ImitationAssessmentResult) {
assessing = false
assessmentState.store(sentenceIndex, result)
currentVideoHash()?.let { hash ->
historyStore.saveScores(hash, assessmentState.toMap())
}
if (host.bindings.hasTestChrome) {
applyAssessmentResult(result)
refreshDubButtons()
host.bindings.dubbingStatusText?.text = when (dubbingState.segments.size) {
0 -> "配音:请先录音"
1 -> "配音:已保存 1 句,可生成配音"
else -> "配音:已保存 ${dubbingState.segments.size} 句,可生成配音"
}
}
assessmentState.store(sentence.index, result)
}
override fun onError(error: Throwable) {
assessing = false
if (host.bindings.hasTestChrome) {
host.bindings.testStatusText.text = "评测失败:${error.message.orEmpty()}"
host.bindings.scoreSummaryText.text = "评测失败"
@@ -263,9 +560,17 @@ internal class TestSessionController(
)
}
private fun applyAssessmentResult(result: ImitationAssessmentResult) {
private fun applyAssessmentResult(
result: ImitationAssessmentResult,
segmentLabel: String? = null,
) {
val ui = host.bindings
ui.testStatusText.text = if (result.passed == true) "已通过" else "未通过"
val passLabel = when (result.passed) {
true -> "已通过"
false -> "未通过"
null -> "已评测"
}
ui.testStatusText.text = listOfNotNull(passLabel, segmentLabel).joinToString(" · ")
ui.scoreSummaryText.text = String.format(Locale.US, "总分 %.1f", result.overallScore)
ui.scoreDetailText.text = buildString {
result.contentScore?.let { appendLine(String.format(Locale.US, "内容分 %.1f", it)) }
@@ -302,49 +607,51 @@ internal class TestSessionController(
private fun refreshDubButtons() {
val ui = host.bindings
if (!ui.hasTestChrome) {
return
}
val dubbingActive = dubbingPlayback.isActive
ui.playDubButton.isEnabled = dubbingState.segments.isNotEmpty() && !dubbingActive
val currentSentenceAudio = controller.currentSentence()?.let { dubbingState.segments[it.index] }
// Keep the play button tappable while active so the user can stop playback.
ui.playDubButton.isEnabled = dubbingActive || currentSentenceAudio != null
ui.mergeDubButton.isEnabled = dubbingState.segments.isNotEmpty() && !dubbingActive
ui.shareDubButton.isEnabled = dubbingState.mergedFile?.exists() == true && !dubbingActive
val activeAudio = controller.currentSentence()?.let { dubbingState.segments[it.index] }
ui.shareDubButton.isEnabled =
dubbingState.mergedFile?.exists() == true && !dubbingActive && !sharing
ui.playDubButton.text = when {
dubbingActive -> "停止配音"
dubbingState.mergedFile?.exists() == true -> "播放配音"
activeAudio != null -> "播放本句配音"
else -> "播放录音(${dubbingState.segments.size})"
dubbingActive -> "停止播放"
currentSentenceAudio != null -> "播放本句录音"
dubbingState.segments.isNotEmpty() -> "播放本句录音"
else -> "播放录音"
}
}
private fun activeAudioForPlayback(): File? {
if (dubbingState.mergedFile != null) {
return dubbingState.mergedFile
}
return controller.currentSentence()?.let { sentence -> dubbingState.segments[sentence.index] }
}
private fun startDubPlayback(audioFile: File) {
val sentence = controller.currentSentence()
private fun startDubPlayback(audioFile: File, sentenceStartMs: Long, sentenceIndex: Int) {
dubbingPlayback.start(
audioFile = audioFile,
controller = controller,
syncToSentence = dubbingState.mergedFile == null,
sentenceStartMs = sentence?.startMs,
sentenceStartMs = sentenceStartMs,
sentenceIndex = sentenceIndex,
) {
refreshTestUi()
host.setMainStatus("配音播放完成")
host.setMainStatus("本句录音播放完成")
}
refreshTestUi()
host.setMainStatus(
if (dubbingState.mergedFile == null) {
"正在播放当前句配音,原视频画面同步但原声静音"
} else {
"正在播放整段配音,原视频画面同步但原声静音"
},
)
host.setMainStatus("正在播放当前句录音,原视频画面同步但原声静音")
}
private fun setDubbingStatus(message: String) {
host.bindings.dubbingStatusText?.text = message
private fun setDubbingStatus(message: String, alert: Boolean = false) {
historyStatusMessage = null
host.bindings.dubbingStatusText?.apply {
text = message
setTextColor(if (alert) SampleTheme.danger else SampleTheme.accentDeep)
}
}
private fun currentVideoHash(): String? {
val id = controller.currentTrainingItem()?.id
?: screenState.currentCourse?.videoHash
?: screenState.activeItemId
return DubbingHistoryStore.normalize(id.orEmpty())
}
private companion object {