add test module

2026-08-16 15:39:52 +08:00
parent d0310620fc
commit 6e4d93cea6
46 changed files with 3880 additions and 206 deletions
--- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/ImitationQualityAssessor.kt
+++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/ImitationQualityAssessor.kt
@@ -18,6 +18,17 @@ data class ImitationAssessmentResult @JvmOverloads constructor(
    val completenessScore: Float? = null,
    val feedback: String? = null,
    val details: Map<String, String> = emptyMap(),
+    val passed: Boolean? = null,
+    val contentScore: Float? = null,
+    val durationScore: Float? = null,
+    val pauseScore: Float? = null,
+    val speechRateScore: Float? = null,
+    val durationRatio: Float? = null,
+    val referenceText: String? = null,
+    val recognizedText: String? = null,
+    val missingTokens: List<String> = emptyList(),
+    val extraTokens: List<String> = emptyList(),
+    val substitutions: List<Pair<String, String>> = emptyList(),
 )

 fun interface CancellableAssessment {
--- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerPlayerView.kt
+++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerPlayerView.kt
@@ -4,6 +4,7 @@ import android.content.Context
 import android.util.AttributeSet
 import android.view.GestureDetector
 import android.view.MotionEvent
+import android.view.ViewConfiguration
 import android.widget.FrameLayout
 import androidx.media3.ui.PlayerView
 import kotlin.math.abs
@@ -21,6 +22,23 @@ class OralTrainerPlayerView @JvmOverloads constructor(
    private var controller: OralTrainerController? = null
    private var gestureControls = GestureControlsConfig()
    private val density = resources.displayMetrics.density
+    private val touchSlop = ViewConfiguration.get(context).scaledTouchSlop
+    private val longPressTimeoutMs = ViewConfiguration.getLongPressTimeout().toLong()
+    private var longPressStartX = 0f
+    private var longPressStartY = 0f
+    private var longPressActive = false
+    private var speedBeforeLongPress: Float? = null
+
+    private val longPressSpeedRunnable = Runnable {
+        val activeController = controller ?: return@Runnable
+        if (!gestureControls.enabled || !gestureControls.longPressAdjustsSpeed || width <= 0) {
+            return@Runnable
+        }
+        longPressActive = true
+        speedBeforeLongPress = activeController.snapshot().playbackSpeed
+        activeController.setPlaybackSpeed(speedForTouchX(longPressStartX))
+        activeController.dispatchGesture(GestureEvent(GestureKind.LONG_PRESS_SPEED))
+    }

    private val gestureDetector = GestureDetector(
        context,
@@ -99,6 +117,72 @@ class OralTrainerPlayerView @JvmOverloads constructor(
    }

    override fun dispatchTouchEvent(ev: MotionEvent): Boolean {
+        if (handleLongPressSpeed(ev)) {
+            return true
+        }
        return gestureDetector.onTouchEvent(ev) || super.dispatchTouchEvent(ev)
    }
+
+    override fun onDetachedFromWindow() {
+        cancelLongPressSpeed(restoreSpeed = true)
+        super.onDetachedFromWindow()
+    }
+
+    private fun handleLongPressSpeed(event: MotionEvent): Boolean {
+        if (!gestureControls.enabled || !gestureControls.longPressAdjustsSpeed) {
+            return false
+        }
+        when (event.actionMasked) {
+            MotionEvent.ACTION_DOWN -> {
+                longPressStartX = event.x
+                longPressStartY = event.y
+                longPressActive = false
+                speedBeforeLongPress = null
+                removeCallbacks(longPressSpeedRunnable)
+                postDelayed(longPressSpeedRunnable, longPressTimeoutMs)
+                return false
+            }
+
+            MotionEvent.ACTION_MOVE -> {
+                if (longPressActive) {
+                    controller?.setPlaybackSpeed(speedForTouchX(event.x))
+                    return true
+                }
+                val dx = abs(event.x - longPressStartX)
+                val dy = abs(event.y - longPressStartY)
+                if (dx > touchSlop || dy > touchSlop) {
+                    removeCallbacks(longPressSpeedRunnable)
+                }
+                return false
+            }
+
+            MotionEvent.ACTION_UP,
+            MotionEvent.ACTION_CANCEL -> {
+                val wasActive = longPressActive
+                cancelLongPressSpeed(restoreSpeed = wasActive)
+                return wasActive
+            }
+        }
+        return false
+    }
+
+    private fun cancelLongPressSpeed(restoreSpeed: Boolean) {
+        removeCallbacks(longPressSpeedRunnable)
+        if (restoreSpeed) {
+            controller?.setPlaybackSpeed(speedBeforeLongPress ?: 1f)
+        }
+        longPressActive = false
+        speedBeforeLongPress = null
+    }
+
+    private fun speedForTouchX(x: Float): Float {
+        val center = width / 2f
+        val distanceRatio = if (center <= 0f) 0f else abs(x - center) / center
+        val farFromCenter = distanceRatio >= 0.5f
+        return if (x < center) {
+            if (farFromCenter) gestureControls.longPressSlowerSpeed else gestureControls.longPressSlowSpeed
+        } else {
+            if (farFromCenter) gestureControls.longPressFasterSpeed else gestureControls.longPressFastSpeed
+        }
+    }
 }
--- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdk.kt
+++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdk.kt
@@ -1,6 +1,7 @@
 package cn.learningpad.oraltrainer.sdk

 import android.content.Context
+import java.util.concurrent.CopyOnWriteArrayList

 class OralTrainerSdk private constructor(
    context: Context,
@@ -10,6 +11,12 @@ class OralTrainerSdk private constructor(

    val cache: OralTrainerCache = OralTrainerCache(appContext, config)
    val sentenceBoundaryApi: SentenceBoundaryApi = SentenceBoundaryApi(appContext, config)
+    val videoCatalogApi: VideoCatalogApi = VideoCatalogApi(appContext, config)
+    private val remoteAssessors = CopyOnWriteArrayList<RemoteImitationQualityAssessor>()
+
+    fun createRemoteImitationQualityAssessor(): RemoteImitationQualityAssessor {
+        return RemoteImitationQualityAssessor(appContext, config).also(remoteAssessors::add)
+    }

    @JvmOverloads
    fun createController(
@@ -26,6 +33,9 @@ class OralTrainerSdk private constructor(

    fun release() {
        sentenceBoundaryApi.release()
+        videoCatalogApi.release()
+        remoteAssessors.forEach { it.release() }
+        remoteAssessors.clear()
        StreamingCache.release()
    }

--- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdkConfig.kt
+++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdkConfig.kt
@@ -9,4 +9,5 @@ data class OralTrainerSdkConfig @JvmOverloads constructor(
    val connectTimeoutMs: Int = 15_000,
    val readTimeoutMs: Int = 30_000,
    val sentenceBoundaryApiBaseUrl: String = "https://video_service.d1kt.cn",
+    val assessmentApiKey: String? = null,
 )
--- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/PlaybackModels.kt
+++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/PlaybackModels.kt
@@ -27,6 +27,7 @@ enum class GestureKind {
    SINGLE_TAP,
    SWIPE_LEFT,
    SWIPE_RIGHT,
+    LONG_PRESS_SPEED,
 }

 interface OralTrainerListener {
--- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/PlayerConfig.kt
+++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/PlayerConfig.kt
@@ -16,6 +16,11 @@ data class GestureControlsConfig @JvmOverloads constructor(
    val rightSwipeAction: SwipeAction = SwipeAction.NEXT_SENTENCE_OR_FORWARD,
    val minSwipeDistanceDp: Float = 48f,
    val minSwipeVelocityDpPerSecond: Float = 160f,
+    val longPressAdjustsSpeed: Boolean = true,
+    val longPressSlowSpeed: Float = 0.75f,
+    val longPressSlowerSpeed: Float = 0.5f,
+    val longPressFastSpeed: Float = 1.5f,
+    val longPressFasterSpeed: Float = 2.0f,
 )

 enum class SwipeAction {
--- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/RemoteImitationQualityAssessor.kt
+++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/RemoteImitationQualityAssessor.kt
@@ -0,0 +1,198 @@
+package cn.learningpad.oraltrainer.sdk
+
+import android.content.ContentResolver
+import android.content.Context
+import android.net.Uri
+import android.os.Handler
+import android.os.Looper
+import org.json.JSONArray
+import org.json.JSONObject
+import java.io.BufferedInputStream
+import java.io.BufferedOutputStream
+import java.io.IOException
+import java.net.HttpURLConnection
+import java.net.URL
+import java.util.UUID
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.Future
+
+/** Uploads one sentence recording to the server-side MOSS assessment endpoint. */
+class RemoteImitationQualityAssessor internal constructor(
+    context: Context,
+    private val config: OralTrainerSdkConfig,
+) : ImitationQualityAssessor {
+    private val appContext = context.applicationContext
+    private val contentResolver: ContentResolver = appContext.contentResolver
+    private val executor: ExecutorService = Executors.newCachedThreadPool()
+    private val mainHandler = Handler(Looper.getMainLooper())
+    private val baseUri = Uri.parse(config.sentenceBoundaryApiBaseUrl.trimEnd('/'))
+
+    init {
+        require(baseUri.scheme == "https" || baseUri.scheme == "http") {
+            "sentenceBoundaryApiBaseUrl must use http or https."
+        }
+        require(!baseUri.host.isNullOrBlank()) {
+            "sentenceBoundaryApiBaseUrl must include a host."
+        }
+    }
+
+    override fun assess(
+        request: ImitationAssessmentRequest,
+        callback: ImitationAssessmentCallback,
+    ): CancellableAssessment {
+        val future = executor.submit {
+            try {
+                val result = assessBlocking(request)
+                if (!Thread.currentThread().isInterrupted) {
+                    mainHandler.post { callback.onResult(result) }
+                }
+            } catch (error: Throwable) {
+                if (!Thread.currentThread().isInterrupted) {
+                    mainHandler.post { callback.onError(error) }
+                }
+            }
+        }
+        return CancellableAssessment { future.cancel(true) }
+    }
+
+    fun release() {
+        executor.shutdownNow()
+    }
+
+    private fun assessBlocking(request: ImitationAssessmentRequest): ImitationAssessmentResult {
+        val videoHash = request.metadata["video_hash"] ?: request.mediaId
+        require(SHA256_PATTERN.matches(videoHash)) {
+            "ImitationAssessmentRequest.mediaId or metadata[video_hash] must be a SHA-256 hash."
+        }
+        val endpoint = baseUri.buildUpon()
+            .appendPath("api")
+            .appendPath("v1")
+            .appendPath("videos")
+            .appendPath(videoHash.lowercase())
+            .appendPath("sentences")
+            .appendPath(request.sentence.index.toString())
+            .appendPath("assessments")
+            .build()
+        val connection = URL(endpoint.toString()).openConnection() as HttpURLConnection
+        val boundary = "----OralTrainer-${UUID.randomUUID()}"
+        return try {
+            connection.requestMethod = "POST"
+            connection.doOutput = true
+            connection.connectTimeout = config.connectTimeoutMs
+            connection.readTimeout = maxOf(config.readTimeoutMs, 180_000)
+            connection.setChunkedStreamingMode(64 * 1024)
+            connection.setRequestProperty("Accept", "application/json")
+            connection.setRequestProperty("User-Agent", config.userAgent)
+            connection.setRequestProperty("Content-Type", "multipart/form-data; boundary=$boundary")
+            config.assessmentApiKey?.takeIf { it.isNotBlank() }?.let {
+                connection.setRequestProperty("X-Client-Key", it)
+            }
+            BufferedOutputStream(connection.outputStream).use { output ->
+                writeFormField(output, boundary, "language", request.locale ?: request.sentence.language.orEmpty())
+                writeFilePart(output, boundary, request.recordingUri)
+                output.write("--$boundary--\r\n".toByteArray(Charsets.UTF_8))
+            }
+            val statusCode = connection.responseCode
+            val stream = if (statusCode in 200..299) connection.inputStream else connection.errorStream
+            val body = stream?.bufferedReader(Charsets.UTF_8)?.use { it.readText() }.orEmpty()
+            if (statusCode !in 200..299) {
+                throw RemoteAssessmentException(statusCode, body)
+            }
+            parseResponse(body)
+        } finally {
+            connection.disconnect()
+        }
+    }
+
+    private fun writeFormField(output: BufferedOutputStream, boundary: String, name: String, value: String) {
+        output.write("--$boundary\r\n".toByteArray(Charsets.UTF_8))
+        output.write("Content-Disposition: form-data; name=\"$name\"\r\n\r\n".toByteArray(Charsets.UTF_8))
+        output.write(value.toByteArray(Charsets.UTF_8))
+        output.write("\r\n".toByteArray(Charsets.UTF_8))
+    }
+
+    private fun writeFilePart(output: BufferedOutputStream, boundary: String, uri: Uri) {
+        val fileName = uri.lastPathSegment?.substringAfterLast('/')
+            ?.replace("\r", "_")
+            ?.replace("\n", "_")
+            ?.replace("\"", "_")
+            ?.ifBlank { "recording.wav" }
+            ?: "recording.wav"
+        val mimeType = contentResolver.getType(uri) ?: "audio/wav"
+        output.write("--$boundary\r\n".toByteArray(Charsets.UTF_8))
+        output.write(
+            "Content-Disposition: form-data; name=\"audio\"; filename=\"$fileName\"\r\n".toByteArray(
+                Charsets.UTF_8
+            )
+        )
+        output.write("Content-Type: $mimeType\r\n\r\n".toByteArray(Charsets.UTF_8))
+        contentResolver.openInputStream(uri)?.use { input ->
+            BufferedInputStream(input).use { buffered ->
+                val buffer = ByteArray(64 * 1024)
+                while (true) {
+                    if (Thread.currentThread().isInterrupted) throw IOException("Assessment request was cancelled.")
+                    val count = buffered.read(buffer)
+                    if (count < 0) break
+                    output.write(buffer, 0, count)
+                }
+            }
+        } ?: throw IOException("Cannot open recording URI: $uri")
+        output.write("\r\n".toByteArray(Charsets.UTF_8))
+    }
+
+    private fun parseResponse(body: String): ImitationAssessmentResult {
+        val root = JSONObject(body)
+        val substitutions = buildList {
+            val array = root.optJSONArray("substitutions") ?: JSONArray()
+            for (index in 0 until array.length()) {
+                val item = array.optJSONObject(index) ?: continue
+                add(item.optString("expected") to item.optString("actual"))
+            }
+        }
+        val details = buildMap {
+            val objectValue = root.optJSONObject("details") ?: return@buildMap
+            for (key in objectValue.keys()) put(key, objectValue.optString(key))
+        }
+        return ImitationAssessmentResult(
+            overallScore = root.getDouble("overall_score").toFloat(),
+            pronunciationScore = root.optNullableDouble("pronunciation_score")?.toFloat(),
+            fluencyScore = root.optNullableDouble("fluency_score")?.toFloat(),
+            completenessScore = root.optNullableDouble("completeness_score")?.toFloat(),
+            feedback = root.optString("feedback").ifBlank { null },
+            details = details,
+            passed = if (root.has("passed")) root.optBoolean("passed") else null,
+            contentScore = root.optNullableDouble("content_score")?.toFloat(),
+            durationScore = root.optNullableDouble("duration_score")?.toFloat(),
+            pauseScore = root.optNullableDouble("pause_score")?.toFloat(),
+            speechRateScore = root.optNullableDouble("speech_rate_score")?.toFloat(),
+            durationRatio = root.optNullableDouble("duration_ratio")?.toFloat(),
+            referenceText = root.optString("reference_text").ifBlank { null },
+            recognizedText = root.optString("recognized_text").ifBlank { null },
+            missingTokens = root.optStringArray("missing_tokens"),
+            extraTokens = root.optStringArray("extra_tokens"),
+            substitutions = substitutions,
+        )
+    }
+
+    companion object {
+        private val SHA256_PATTERN = Regex("^[0-9a-fA-F]{64}$")
+    }
+}
+
+class RemoteAssessmentException(
+    val statusCode: Int,
+    responseBody: String,
+) : IOException("Remote assessment returned HTTP $statusCode: $responseBody")
+
+private fun JSONObject.optNullableDouble(name: String): Double? {
+    if (!has(name) || isNull(name)) return null
+    return optDouble(name).takeUnless { it.isNaN() }
+}
+
+private fun JSONObject.optStringArray(name: String): List<String> {
+    val array = optJSONArray(name) ?: return emptyList()
+    return buildList(array.length()) {
+        for (index in 0 until array.length()) add(array.optString(index))
+    }
+}
--- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/SentenceBoundaryApi.kt
+++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/SentenceBoundaryApi.kt
@@ -147,6 +147,12 @@ class SentenceBoundaryApi internal constructor(
                        startMs = sentence.getLong("start_ms"),
                        endMs = sentence.getLong("end_ms"),
                        text = if (sentence.isNull("text")) null else sentence.getString("text"),
+                        language = if (sentence.isNull("language")) null else sentence.getString("language"),
+                        referenceSpeechDurationMs = if (sentence.isNull("reference_speech_duration_ms")) {
+                            null
+                        } else {
+                            sentence.getLong("reference_speech_duration_ms")
+                        },
                    )
                )
            }
--- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/TrainingModels.kt
+++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/TrainingModels.kt
@@ -8,11 +8,16 @@ data class SentenceBoundary @JvmOverloads constructor(
    val startMs: Long,
    val endMs: Long,
    val text: String? = null,
+    val language: String? = null,
+    val referenceSpeechDurationMs: Long? = null,
 ) {
    init {
        require(index >= 0) { "Sentence index must be non-negative." }
        require(startMs >= 0) { "Sentence startMs must be non-negative." }
        require(endMs > startMs) { "Sentence endMs must be greater than startMs." }
+        require(referenceSpeechDurationMs == null || referenceSpeechDurationMs > 0) {
+            "Sentence referenceSpeechDurationMs must be positive when present."
+        }
    }
 }

--- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/VideoCatalogApi.kt
+++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/VideoCatalogApi.kt
@@ -0,0 +1,116 @@
+package cn.learningpad.oraltrainer.sdk
+
+import android.content.Context
+import android.net.Uri
+import android.os.Handler
+import android.os.Looper
+import org.json.JSONObject
+import java.io.IOException
+import java.net.HttpURLConnection
+import java.net.URL
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+
+data class TrainingVideoSummary(
+    val videoHash: String,
+    val title: String,
+    val streamUri: Uri,
+    val durationMs: Long?,
+    val sizeBytes: Long,
+    val language: String?,
+    val sentenceCount: Int,
+    val status: String,
+) {
+    fun toTrainingMediaItem(sentences: List<SentenceBoundary> = emptyList()): TrainingMediaItem {
+        return TrainingMediaItem(
+            id = videoHash,
+            title = title,
+            uri = streamUri,
+            sentences = sentences,
+            customCacheKey = videoHash,
+        )
+    }
+}
+
+interface VideoCatalogCallback {
+    fun onSuccess(videos: List<TrainingVideoSummary>)
+
+    fun onError(error: Throwable)
+}
+
+class VideoCatalogApi internal constructor(
+    context: Context,
+    private val config: OralTrainerSdkConfig,
+) {
+    private val mainHandler = Handler(Looper.getMainLooper())
+    private val executor: ExecutorService = Executors.newCachedThreadPool()
+    private val baseUrl = config.sentenceBoundaryApiBaseUrl.trimEnd('/') + "/"
+
+    fun fetch(callback: VideoCatalogCallback): CancellableRequest {
+        val future = executor.submit {
+            try {
+                val videos = fetchBlocking()
+                if (!Thread.currentThread().isInterrupted) {
+                    mainHandler.post { callback.onSuccess(videos) }
+                }
+            } catch (error: Throwable) {
+                if (!Thread.currentThread().isInterrupted) {
+                    mainHandler.post { callback.onError(error) }
+                }
+            }
+        }
+        return CancellableRequest(future)
+    }
+
+    fun release() {
+        executor.shutdownNow()
+    }
+
+    private fun fetchBlocking(): List<TrainingVideoSummary> {
+        val endpoint = URL(URL(baseUrl), "api/v1/videos")
+        val connection = endpoint.openConnection() as HttpURLConnection
+        return try {
+            connection.requestMethod = "GET"
+            connection.connectTimeout = config.connectTimeoutMs
+            connection.readTimeout = config.readTimeoutMs
+            connection.setRequestProperty("Accept", "application/json")
+            connection.setRequestProperty("User-Agent", config.userAgent)
+            val statusCode = connection.responseCode
+            val stream = if (statusCode in 200..299) connection.inputStream else connection.errorStream
+            val body = stream?.bufferedReader(Charsets.UTF_8)?.use { it.readText() }.orEmpty()
+            if (statusCode !in 200..299) throw VideoCatalogApiException(statusCode, body)
+            parseResponse(body)
+        } finally {
+            connection.disconnect()
+        }
+    }
+
+    private fun parseResponse(body: String): List<TrainingVideoSummary> {
+        val videos = JSONObject(body).getJSONArray("videos")
+        return buildList(videos.length()) {
+            for (index in 0 until videos.length()) {
+                val item = videos.getJSONObject(index)
+                val status = item.getString("status")
+                if (status != "ready") continue
+                val streamUrl = URL(URL(baseUrl), item.getString("stream_url"))
+                add(
+                    TrainingVideoSummary(
+                        videoHash = item.getString("video_hash"),
+                        title = item.getString("title"),
+                        streamUri = Uri.parse(streamUrl.toString()),
+                        durationMs = if (item.isNull("duration_ms")) null else item.getLong("duration_ms"),
+                        sizeBytes = item.getLong("size_bytes"),
+                        language = if (item.isNull("language")) null else item.getString("language"),
+                        sentenceCount = item.getInt("sentence_count"),
+                        status = status,
+                    )
+                )
+            }
+        }
+    }
+}
+
+class VideoCatalogApiException(
+    val statusCode: Int,
+    responseBody: String,
+) : IOException("Video catalog API returned HTTP $statusCode: $responseBody")