diff --git a/.curlrc-homebrew-speech b/.curlrc-homebrew-speech new file mode 100644 index 0000000..3d0452a --- /dev/null +++ b/.curlrc-homebrew-speech @@ -0,0 +1 @@ +--http1.1 diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..dcbd87a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.git +.gitignore +.DS_Store +__pycache__ +*.pyc +.pytest_cache +.venv* + +android +sentence_api/data/v +sentence_api/data/work +sentence_api/data/attempts +sentence_api/data/oral_trainer.sqlite3* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..98f37cb --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.DS_Store +__pycache__/ +*.py[cod] +.pytest_cache/ + +sentence_api/data/oral_trainer.sqlite3* +sentence_api/data/v/ +sentence_api/data/work/ +sentence_api/data/attempts/ diff --git a/README.md b/README.md index 5337648..90a7e4b 100644 --- a/README.md +++ b/README.md @@ -122,10 +122,11 @@ SDK 基于 AndroidX Media3 / ExoPlayer,支持在线视频流、本地缓存、 详见 [android/README.md](android/README.md)。 -## 句子边界 API +## 口语训练视频服务 -仓库新增了 `sentence_api/` 服务。它复用桌面播放器的静音检测算法,提前为视频 -生成句子边界,并根据视频 SHA-256 哈希向 Android 返回毫秒级的 -`start_ms` / `end_ms` 列表。服务不会在移动端请求时临时分析视频。 +`sentence_api/` 已扩展为完整的视频服务:支持超过 1 GB 的流式上传、MOSS 自动 +转写、句子文本校对、HTTP Range 在线播放、Android 视频目录、学生录音上传,以及 +内容正确度、有效语音时长、停顿和语速评分。原有的 SHA-256 句子边界接口保持兼容。 -详见 [sentence_api/README.md](sentence_api/README.md)。 +本地使用见 [sentence_api/README.md](sentence_api/README.md),生产部署见 +[sentence_api/DEPLOYMENT.md](sentence_api/DEPLOYMENT.md)。 diff --git a/android/README.md b/android/README.md index 27a6473..44ca8f5 100644 --- a/android/README.md +++ b/android/README.md @@ -2,7 +2,7 @@ Android SDK for online oral-training video playback. It provides an embeddable Media3/ExoPlayer player, streaming cache, tablet gestures, sentence navigation, -and a placeholder interface for future imitation-quality scoring. +the server video catalog, and remote imitation-quality assessment. ## Modules @@ -93,11 +93,36 @@ Override the service only when a staging or private deployment is required: OralTrainerSdk.init( context, OralTrainerSdkConfig( - sentenceBoundaryApiBaseUrl = "https://video_service.d1kt.cn" + sentenceBoundaryApiBaseUrl = "https://video_service.d1kt.cn", + assessmentApiKey = BuildConfig.ORAL_TRAINER_CLIENT_KEY, ) ) ``` +## Video Catalog + +The SDK loads ready videos from `GET /api/v1/videos` and resolves both absolute +and relative stream URLs: + +```kotlin +sdk.videoCatalogApi.fetch(object : VideoCatalogCallback { + override fun onSuccess(videos: List) { + val video = videos.first() + sdk.sentenceBoundaryApi.fetch(video.videoHash, object : SentenceBoundaryApiCallback { + override fun onSuccess(result: SentenceBoundaryApiResult) { + controller.loadItem(video.toTrainingMediaItem(result.sentences)) + } + + override fun onError(error: Throwable) { + controller.loadItem(video.toTrainingMediaItem()) + } + }) + } + + override fun onError(error: Throwable) = Unit +}) +``` + ## Local Video Testing The computer path `/Users/...` is not visible to an Android device. For a @@ -110,19 +135,40 @@ service and pass its URL as `TrainingMediaItem.uri`. The SDK streams it and caches downloaded ranges locally. Copying large 4K files to each device is better reserved for explicitly offline courses. -## Future Imitation Scoring +## Remote Imitation Scoring -Provide an implementation of `ImitationQualityAssessor` when the speech -assessment algorithm is ready: +Create the controller with the built-in remote assessor: ```kotlin +val remoteAssessor = sdk.createRemoteImitationQualityAssessor() val controller = sdk.createController( - imitationAssessor = MyImitationQualityAssessor() + imitationAssessor = remoteAssessor ) ``` Then call `assessCurrentSentence(recordingUri, callback)` after the student -records a sentence. +records a sentence. The current `TrainingMediaItem.id` must be the server video +SHA-256, which is already true for items created by `TrainingVideoSummary`: + +```kotlin +controller.assessCurrentSentence( + recordingUri = recordingUri, + locale = "en", + callback = object : ImitationAssessmentCallback { + override fun onResult(result: ImitationAssessmentResult) { + val passed = result.passed == true + val score = result.overallScore + val durationRatio = result.durationRatio + } + + override fun onError(error: Throwable) = Unit + } +) +``` + +The server returns content, completeness, fluency, duration, pause, and speech +rate scores. Phoneme pronunciation and prosody are nullable until the dedicated +models are enabled. ## Build diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/ImitationQualityAssessor.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/ImitationQualityAssessor.kt index c3df927..2b4aefb 100644 --- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/ImitationQualityAssessor.kt +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/ImitationQualityAssessor.kt @@ -18,6 +18,17 @@ data class ImitationAssessmentResult @JvmOverloads constructor( val completenessScore: Float? = null, val feedback: String? = null, val details: Map = emptyMap(), + val passed: Boolean? = null, + val contentScore: Float? = null, + val durationScore: Float? = null, + val pauseScore: Float? = null, + val speechRateScore: Float? = null, + val durationRatio: Float? = null, + val referenceText: String? = null, + val recognizedText: String? = null, + val missingTokens: List = emptyList(), + val extraTokens: List = emptyList(), + val substitutions: List> = emptyList(), ) fun interface CancellableAssessment { diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerPlayerView.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerPlayerView.kt index 43c4a00..ae4b785 100644 --- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerPlayerView.kt +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerPlayerView.kt @@ -4,6 +4,7 @@ import android.content.Context import android.util.AttributeSet import android.view.GestureDetector import android.view.MotionEvent +import android.view.ViewConfiguration import android.widget.FrameLayout import androidx.media3.ui.PlayerView import kotlin.math.abs @@ -21,6 +22,23 @@ class OralTrainerPlayerView @JvmOverloads constructor( private var controller: OralTrainerController? = null private var gestureControls = GestureControlsConfig() private val density = resources.displayMetrics.density + private val touchSlop = ViewConfiguration.get(context).scaledTouchSlop + private val longPressTimeoutMs = ViewConfiguration.getLongPressTimeout().toLong() + private var longPressStartX = 0f + private var longPressStartY = 0f + private var longPressActive = false + private var speedBeforeLongPress: Float? = null + + private val longPressSpeedRunnable = Runnable { + val activeController = controller ?: return@Runnable + if (!gestureControls.enabled || !gestureControls.longPressAdjustsSpeed || width <= 0) { + return@Runnable + } + longPressActive = true + speedBeforeLongPress = activeController.snapshot().playbackSpeed + activeController.setPlaybackSpeed(speedForTouchX(longPressStartX)) + activeController.dispatchGesture(GestureEvent(GestureKind.LONG_PRESS_SPEED)) + } private val gestureDetector = GestureDetector( context, @@ -99,6 +117,72 @@ class OralTrainerPlayerView @JvmOverloads constructor( } override fun dispatchTouchEvent(ev: MotionEvent): Boolean { + if (handleLongPressSpeed(ev)) { + return true + } return gestureDetector.onTouchEvent(ev) || super.dispatchTouchEvent(ev) } + + override fun onDetachedFromWindow() { + cancelLongPressSpeed(restoreSpeed = true) + super.onDetachedFromWindow() + } + + private fun handleLongPressSpeed(event: MotionEvent): Boolean { + if (!gestureControls.enabled || !gestureControls.longPressAdjustsSpeed) { + return false + } + when (event.actionMasked) { + MotionEvent.ACTION_DOWN -> { + longPressStartX = event.x + longPressStartY = event.y + longPressActive = false + speedBeforeLongPress = null + removeCallbacks(longPressSpeedRunnable) + postDelayed(longPressSpeedRunnable, longPressTimeoutMs) + return false + } + + MotionEvent.ACTION_MOVE -> { + if (longPressActive) { + controller?.setPlaybackSpeed(speedForTouchX(event.x)) + return true + } + val dx = abs(event.x - longPressStartX) + val dy = abs(event.y - longPressStartY) + if (dx > touchSlop || dy > touchSlop) { + removeCallbacks(longPressSpeedRunnable) + } + return false + } + + MotionEvent.ACTION_UP, + MotionEvent.ACTION_CANCEL -> { + val wasActive = longPressActive + cancelLongPressSpeed(restoreSpeed = wasActive) + return wasActive + } + } + return false + } + + private fun cancelLongPressSpeed(restoreSpeed: Boolean) { + removeCallbacks(longPressSpeedRunnable) + if (restoreSpeed) { + controller?.setPlaybackSpeed(speedBeforeLongPress ?: 1f) + } + longPressActive = false + speedBeforeLongPress = null + } + + private fun speedForTouchX(x: Float): Float { + val center = width / 2f + val distanceRatio = if (center <= 0f) 0f else abs(x - center) / center + val farFromCenter = distanceRatio >= 0.5f + return if (x < center) { + if (farFromCenter) gestureControls.longPressSlowerSpeed else gestureControls.longPressSlowSpeed + } else { + if (farFromCenter) gestureControls.longPressFasterSpeed else gestureControls.longPressFastSpeed + } + } } diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdk.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdk.kt index a1f78b0..26e4a95 100644 --- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdk.kt +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdk.kt @@ -1,6 +1,7 @@ package cn.learningpad.oraltrainer.sdk import android.content.Context +import java.util.concurrent.CopyOnWriteArrayList class OralTrainerSdk private constructor( context: Context, @@ -10,6 +11,12 @@ class OralTrainerSdk private constructor( val cache: OralTrainerCache = OralTrainerCache(appContext, config) val sentenceBoundaryApi: SentenceBoundaryApi = SentenceBoundaryApi(appContext, config) + val videoCatalogApi: VideoCatalogApi = VideoCatalogApi(appContext, config) + private val remoteAssessors = CopyOnWriteArrayList() + + fun createRemoteImitationQualityAssessor(): RemoteImitationQualityAssessor { + return RemoteImitationQualityAssessor(appContext, config).also(remoteAssessors::add) + } @JvmOverloads fun createController( @@ -26,6 +33,9 @@ class OralTrainerSdk private constructor( fun release() { sentenceBoundaryApi.release() + videoCatalogApi.release() + remoteAssessors.forEach { it.release() } + remoteAssessors.clear() StreamingCache.release() } diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdkConfig.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdkConfig.kt index b033443..92443f1 100644 --- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdkConfig.kt +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdkConfig.kt @@ -9,4 +9,5 @@ data class OralTrainerSdkConfig @JvmOverloads constructor( val connectTimeoutMs: Int = 15_000, val readTimeoutMs: Int = 30_000, val sentenceBoundaryApiBaseUrl: String = "https://video_service.d1kt.cn", + val assessmentApiKey: String? = null, ) diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/PlaybackModels.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/PlaybackModels.kt index f6f300d..fc1ea17 100644 --- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/PlaybackModels.kt +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/PlaybackModels.kt @@ -27,6 +27,7 @@ enum class GestureKind { SINGLE_TAP, SWIPE_LEFT, SWIPE_RIGHT, + LONG_PRESS_SPEED, } interface OralTrainerListener { diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/PlayerConfig.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/PlayerConfig.kt index 6d66981..8ba91bc 100644 --- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/PlayerConfig.kt +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/PlayerConfig.kt @@ -16,6 +16,11 @@ data class GestureControlsConfig @JvmOverloads constructor( val rightSwipeAction: SwipeAction = SwipeAction.NEXT_SENTENCE_OR_FORWARD, val minSwipeDistanceDp: Float = 48f, val minSwipeVelocityDpPerSecond: Float = 160f, + val longPressAdjustsSpeed: Boolean = true, + val longPressSlowSpeed: Float = 0.75f, + val longPressSlowerSpeed: Float = 0.5f, + val longPressFastSpeed: Float = 1.5f, + val longPressFasterSpeed: Float = 2.0f, ) enum class SwipeAction { diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/RemoteImitationQualityAssessor.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/RemoteImitationQualityAssessor.kt new file mode 100644 index 0000000..652e8c6 --- /dev/null +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/RemoteImitationQualityAssessor.kt @@ -0,0 +1,198 @@ +package cn.learningpad.oraltrainer.sdk + +import android.content.ContentResolver +import android.content.Context +import android.net.Uri +import android.os.Handler +import android.os.Looper +import org.json.JSONArray +import org.json.JSONObject +import java.io.BufferedInputStream +import java.io.BufferedOutputStream +import java.io.IOException +import java.net.HttpURLConnection +import java.net.URL +import java.util.UUID +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.Future + +/** Uploads one sentence recording to the server-side MOSS assessment endpoint. */ +class RemoteImitationQualityAssessor internal constructor( + context: Context, + private val config: OralTrainerSdkConfig, +) : ImitationQualityAssessor { + private val appContext = context.applicationContext + private val contentResolver: ContentResolver = appContext.contentResolver + private val executor: ExecutorService = Executors.newCachedThreadPool() + private val mainHandler = Handler(Looper.getMainLooper()) + private val baseUri = Uri.parse(config.sentenceBoundaryApiBaseUrl.trimEnd('/')) + + init { + require(baseUri.scheme == "https" || baseUri.scheme == "http") { + "sentenceBoundaryApiBaseUrl must use http or https." + } + require(!baseUri.host.isNullOrBlank()) { + "sentenceBoundaryApiBaseUrl must include a host." + } + } + + override fun assess( + request: ImitationAssessmentRequest, + callback: ImitationAssessmentCallback, + ): CancellableAssessment { + val future = executor.submit { + try { + val result = assessBlocking(request) + if (!Thread.currentThread().isInterrupted) { + mainHandler.post { callback.onResult(result) } + } + } catch (error: Throwable) { + if (!Thread.currentThread().isInterrupted) { + mainHandler.post { callback.onError(error) } + } + } + } + return CancellableAssessment { future.cancel(true) } + } + + fun release() { + executor.shutdownNow() + } + + private fun assessBlocking(request: ImitationAssessmentRequest): ImitationAssessmentResult { + val videoHash = request.metadata["video_hash"] ?: request.mediaId + require(SHA256_PATTERN.matches(videoHash)) { + "ImitationAssessmentRequest.mediaId or metadata[video_hash] must be a SHA-256 hash." + } + val endpoint = baseUri.buildUpon() + .appendPath("api") + .appendPath("v1") + .appendPath("videos") + .appendPath(videoHash.lowercase()) + .appendPath("sentences") + .appendPath(request.sentence.index.toString()) + .appendPath("assessments") + .build() + val connection = URL(endpoint.toString()).openConnection() as HttpURLConnection + val boundary = "----OralTrainer-${UUID.randomUUID()}" + return try { + connection.requestMethod = "POST" + connection.doOutput = true + connection.connectTimeout = config.connectTimeoutMs + connection.readTimeout = maxOf(config.readTimeoutMs, 180_000) + connection.setChunkedStreamingMode(64 * 1024) + connection.setRequestProperty("Accept", "application/json") + connection.setRequestProperty("User-Agent", config.userAgent) + connection.setRequestProperty("Content-Type", "multipart/form-data; boundary=$boundary") + config.assessmentApiKey?.takeIf { it.isNotBlank() }?.let { + connection.setRequestProperty("X-Client-Key", it) + } + BufferedOutputStream(connection.outputStream).use { output -> + writeFormField(output, boundary, "language", request.locale ?: request.sentence.language.orEmpty()) + writeFilePart(output, boundary, request.recordingUri) + output.write("--$boundary--\r\n".toByteArray(Charsets.UTF_8)) + } + val statusCode = connection.responseCode + val stream = if (statusCode in 200..299) connection.inputStream else connection.errorStream + val body = stream?.bufferedReader(Charsets.UTF_8)?.use { it.readText() }.orEmpty() + if (statusCode !in 200..299) { + throw RemoteAssessmentException(statusCode, body) + } + parseResponse(body) + } finally { + connection.disconnect() + } + } + + private fun writeFormField(output: BufferedOutputStream, boundary: String, name: String, value: String) { + output.write("--$boundary\r\n".toByteArray(Charsets.UTF_8)) + output.write("Content-Disposition: form-data; name=\"$name\"\r\n\r\n".toByteArray(Charsets.UTF_8)) + output.write(value.toByteArray(Charsets.UTF_8)) + output.write("\r\n".toByteArray(Charsets.UTF_8)) + } + + private fun writeFilePart(output: BufferedOutputStream, boundary: String, uri: Uri) { + val fileName = uri.lastPathSegment?.substringAfterLast('/') + ?.replace("\r", "_") + ?.replace("\n", "_") + ?.replace("\"", "_") + ?.ifBlank { "recording.wav" } + ?: "recording.wav" + val mimeType = contentResolver.getType(uri) ?: "audio/wav" + output.write("--$boundary\r\n".toByteArray(Charsets.UTF_8)) + output.write( + "Content-Disposition: form-data; name=\"audio\"; filename=\"$fileName\"\r\n".toByteArray( + Charsets.UTF_8 + ) + ) + output.write("Content-Type: $mimeType\r\n\r\n".toByteArray(Charsets.UTF_8)) + contentResolver.openInputStream(uri)?.use { input -> + BufferedInputStream(input).use { buffered -> + val buffer = ByteArray(64 * 1024) + while (true) { + if (Thread.currentThread().isInterrupted) throw IOException("Assessment request was cancelled.") + val count = buffered.read(buffer) + if (count < 0) break + output.write(buffer, 0, count) + } + } + } ?: throw IOException("Cannot open recording URI: $uri") + output.write("\r\n".toByteArray(Charsets.UTF_8)) + } + + private fun parseResponse(body: String): ImitationAssessmentResult { + val root = JSONObject(body) + val substitutions = buildList { + val array = root.optJSONArray("substitutions") ?: JSONArray() + for (index in 0 until array.length()) { + val item = array.optJSONObject(index) ?: continue + add(item.optString("expected") to item.optString("actual")) + } + } + val details = buildMap { + val objectValue = root.optJSONObject("details") ?: return@buildMap + for (key in objectValue.keys()) put(key, objectValue.optString(key)) + } + return ImitationAssessmentResult( + overallScore = root.getDouble("overall_score").toFloat(), + pronunciationScore = root.optNullableDouble("pronunciation_score")?.toFloat(), + fluencyScore = root.optNullableDouble("fluency_score")?.toFloat(), + completenessScore = root.optNullableDouble("completeness_score")?.toFloat(), + feedback = root.optString("feedback").ifBlank { null }, + details = details, + passed = if (root.has("passed")) root.optBoolean("passed") else null, + contentScore = root.optNullableDouble("content_score")?.toFloat(), + durationScore = root.optNullableDouble("duration_score")?.toFloat(), + pauseScore = root.optNullableDouble("pause_score")?.toFloat(), + speechRateScore = root.optNullableDouble("speech_rate_score")?.toFloat(), + durationRatio = root.optNullableDouble("duration_ratio")?.toFloat(), + referenceText = root.optString("reference_text").ifBlank { null }, + recognizedText = root.optString("recognized_text").ifBlank { null }, + missingTokens = root.optStringArray("missing_tokens"), + extraTokens = root.optStringArray("extra_tokens"), + substitutions = substitutions, + ) + } + + companion object { + private val SHA256_PATTERN = Regex("^[0-9a-fA-F]{64}$") + } +} + +class RemoteAssessmentException( + val statusCode: Int, + responseBody: String, +) : IOException("Remote assessment returned HTTP $statusCode: $responseBody") + +private fun JSONObject.optNullableDouble(name: String): Double? { + if (!has(name) || isNull(name)) return null + return optDouble(name).takeUnless { it.isNaN() } +} + +private fun JSONObject.optStringArray(name: String): List { + val array = optJSONArray(name) ?: return emptyList() + return buildList(array.length()) { + for (index in 0 until array.length()) add(array.optString(index)) + } +} diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/SentenceBoundaryApi.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/SentenceBoundaryApi.kt index 56a232f..0da60e6 100644 --- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/SentenceBoundaryApi.kt +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/SentenceBoundaryApi.kt @@ -147,6 +147,12 @@ class SentenceBoundaryApi internal constructor( startMs = sentence.getLong("start_ms"), endMs = sentence.getLong("end_ms"), text = if (sentence.isNull("text")) null else sentence.getString("text"), + language = if (sentence.isNull("language")) null else sentence.getString("language"), + referenceSpeechDurationMs = if (sentence.isNull("reference_speech_duration_ms")) { + null + } else { + sentence.getLong("reference_speech_duration_ms") + }, ) ) } diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/TrainingModels.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/TrainingModels.kt index 2f5bded..736eb4e 100644 --- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/TrainingModels.kt +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/TrainingModels.kt @@ -8,11 +8,16 @@ data class SentenceBoundary @JvmOverloads constructor( val startMs: Long, val endMs: Long, val text: String? = null, + val language: String? = null, + val referenceSpeechDurationMs: Long? = null, ) { init { require(index >= 0) { "Sentence index must be non-negative." } require(startMs >= 0) { "Sentence startMs must be non-negative." } require(endMs > startMs) { "Sentence endMs must be greater than startMs." } + require(referenceSpeechDurationMs == null || referenceSpeechDurationMs > 0) { + "Sentence referenceSpeechDurationMs must be positive when present." + } } } diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/VideoCatalogApi.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/VideoCatalogApi.kt new file mode 100644 index 0000000..9b58eb3 --- /dev/null +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/VideoCatalogApi.kt @@ -0,0 +1,116 @@ +package cn.learningpad.oraltrainer.sdk + +import android.content.Context +import android.net.Uri +import android.os.Handler +import android.os.Looper +import org.json.JSONObject +import java.io.IOException +import java.net.HttpURLConnection +import java.net.URL +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +data class TrainingVideoSummary( + val videoHash: String, + val title: String, + val streamUri: Uri, + val durationMs: Long?, + val sizeBytes: Long, + val language: String?, + val sentenceCount: Int, + val status: String, +) { + fun toTrainingMediaItem(sentences: List = emptyList()): TrainingMediaItem { + return TrainingMediaItem( + id = videoHash, + title = title, + uri = streamUri, + sentences = sentences, + customCacheKey = videoHash, + ) + } +} + +interface VideoCatalogCallback { + fun onSuccess(videos: List) + + fun onError(error: Throwable) +} + +class VideoCatalogApi internal constructor( + context: Context, + private val config: OralTrainerSdkConfig, +) { + private val mainHandler = Handler(Looper.getMainLooper()) + private val executor: ExecutorService = Executors.newCachedThreadPool() + private val baseUrl = config.sentenceBoundaryApiBaseUrl.trimEnd('/') + "/" + + fun fetch(callback: VideoCatalogCallback): CancellableRequest { + val future = executor.submit { + try { + val videos = fetchBlocking() + if (!Thread.currentThread().isInterrupted) { + mainHandler.post { callback.onSuccess(videos) } + } + } catch (error: Throwable) { + if (!Thread.currentThread().isInterrupted) { + mainHandler.post { callback.onError(error) } + } + } + } + return CancellableRequest(future) + } + + fun release() { + executor.shutdownNow() + } + + private fun fetchBlocking(): List { + val endpoint = URL(URL(baseUrl), "api/v1/videos") + val connection = endpoint.openConnection() as HttpURLConnection + return try { + connection.requestMethod = "GET" + connection.connectTimeout = config.connectTimeoutMs + connection.readTimeout = config.readTimeoutMs + connection.setRequestProperty("Accept", "application/json") + connection.setRequestProperty("User-Agent", config.userAgent) + val statusCode = connection.responseCode + val stream = if (statusCode in 200..299) connection.inputStream else connection.errorStream + val body = stream?.bufferedReader(Charsets.UTF_8)?.use { it.readText() }.orEmpty() + if (statusCode !in 200..299) throw VideoCatalogApiException(statusCode, body) + parseResponse(body) + } finally { + connection.disconnect() + } + } + + private fun parseResponse(body: String): List { + val videos = JSONObject(body).getJSONArray("videos") + return buildList(videos.length()) { + for (index in 0 until videos.length()) { + val item = videos.getJSONObject(index) + val status = item.getString("status") + if (status != "ready") continue + val streamUrl = URL(URL(baseUrl), item.getString("stream_url")) + add( + TrainingVideoSummary( + videoHash = item.getString("video_hash"), + title = item.getString("title"), + streamUri = Uri.parse(streamUrl.toString()), + durationMs = if (item.isNull("duration_ms")) null else item.getLong("duration_ms"), + sizeBytes = item.getLong("size_bytes"), + language = if (item.isNull("language")) null else item.getString("language"), + sentenceCount = item.getInt("sentence_count"), + status = status, + ) + ) + } + } + } +} + +class VideoCatalogApiException( + val statusCode: Int, + responseBody: String, +) : IOException("Video catalog API returned HTTP $statusCode: $responseBody") diff --git a/android/sample-app/src/main/AndroidManifest.xml b/android/sample-app/src/main/AndroidManifest.xml index ab68f66..ee49011 100644 --- a/android/sample-app/src/main/AndroidManifest.xml +++ b/android/sample-app/src/main/AndroidManifest.xml @@ -3,7 +3,7 @@ = emptyList() override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) + configureWindow() sdk = OralTrainerSdk.init(this) controller = sdk.createController( @@ -47,77 +67,10 @@ class MainActivity : Activity() { ) controller.setLoopMode(LoopMode.ALL) - val playerView = OralTrainerPlayerView(this).apply { - bind(controller) - layoutParams = LinearLayout.LayoutParams( - ViewGroup.LayoutParams.MATCH_PARENT, - 0, - 1f, - ) - } - - statusText = TextView(this).apply { - setTextColor(Color.WHITE) - textSize = 14f - setPadding(24, 16, 24, 8) - } - sentenceText = TextView(this).apply { - setTextColor(Color.WHITE) - textSize = 18f - setPadding(24, 4, 24, 16) - } - - val controls = LinearLayout(this).apply { - orientation = LinearLayout.HORIZONTAL - gravity = Gravity.CENTER - setPadding(16, 8, 16, 20) - addView(commandButton("快退") { controller.rewind() }) - addView(commandButton("播放/暂停") { controller.togglePlayPause() }) - addView(commandButton("快进") { controller.fastForward() }) - addView(commandButton("上一句") { controller.previousSentenceOrRewind() }) - addView(commandButton("下一句") { controller.nextSentenceOrForward() }) - } - - val root = LinearLayout(this).apply { - orientation = LinearLayout.VERTICAL - setBackgroundColor(Color.rgb(18, 18, 18)) - addView(playerView) - addView(statusText) - addView(sentenceText) - addView(controls) - addView(Button(this@MainActivity).apply { - text = "选择本地视频" - setOnClickListener { openLocalVideoPicker() } - }) - } - setContentView(root) - - controller.addListener(object : OralTrainerListener { - override fun onPlaybackSnapshot(snapshot: PlaybackSnapshot) { - statusText.text = buildString { - append(if (snapshot.isPlaying) "播放中" else "已暂停") - append(" ") - append(format(snapshot.positionMs)) - append(" / ") - append(format(snapshot.durationMs)) - append(" 速度 ") - append(snapshot.playbackSpeed) - append("x") - } - } - - override fun onSentenceChanged(sentence: SentenceBoundary?) { - sentenceText.text = sentence?.let { - "第 ${it.index + 1} 句:${it.text.orEmpty()}" - } ?: "当前没有句子边界" - } - - override fun onGesture(event: GestureEvent) { - statusText.text = "手势:${event.kind} ${event.action ?: ""}" - } - }) - - controller.loadItem(sampleOnlineLesson()) + setContentView(createContentView()) + bindPlayerEvents() + loadSampleLesson() + loadCatalog() } override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) { @@ -132,21 +85,38 @@ class MainActivity : Activity() { contentResolver.takePersistableUriPermission(uri, persistableFlags) } } + val item = TrainingMediaItem( id = uri.toString(), title = uri.lastPathSegment ?: "本地视频", uri = uri, ) + activeItemId = item.id + currentSentenceCount = 0 controller.loadItem(item) - statusText.text = "正在获取句子边界..." + lessonTitleText.text = item.title + sentenceMetaText.text = "句子边界分析中" + sentenceText.text = "正在匹配服务端句子边界" + statusText.text = "正在获取本地视频的句子边界..." + renderCatalog() + sdk.sentenceBoundaryApi.fetchForUri(uri, contentResolver, object : SentenceBoundaryApiCallback { override fun onSuccess(result: SentenceBoundaryApiResult) { + if (activeItemId != item.id) { + return + } + currentSentenceCount = result.sentences.size controller.loadItem(item.copy(sentences = result.sentences)) statusText.text = "句子边界已加载:${result.sentences.size} 句" } override fun onError(error: Throwable) { - statusText.text = "句子边界获取失败,使用 10 秒跳转:${error.message.orEmpty()}" + if (activeItemId != item.id) { + return + } + statusText.text = "句子边界获取失败,使用默认快退/快进:${error.message.orEmpty()}" + sentenceMetaText.text = "暂无句子边界" + sentenceText.text = item.title } }) } @@ -156,18 +126,381 @@ class MainActivity : Activity() { super.onDestroy() } - private fun commandButton(label: String, action: () -> Unit): Button { + @Suppress("DEPRECATION") + private fun configureWindow() { + window.statusBarColor = COLOR_BACKGROUND + window.navigationBarColor = COLOR_BACKGROUND + } + + private fun createContentView(): View { + return LinearLayout(this).apply { + orientation = LinearLayout.VERTICAL + setBackgroundColor(COLOR_BACKGROUND) + addView(createHeader()) + addView(createPlayerSection()) + addView(createSentenceSection()) + addView(createCatalogSection()) + } + } + + private fun createHeader(): View { + val titleBlock = LinearLayout(this).apply { + orientation = LinearLayout.VERTICAL + gravity = Gravity.CENTER_VERTICAL + layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f) + addView(TextView(this@MainActivity).apply { + text = "口语宝" + setTextColor(Color.WHITE) + textSize = 26f + typeface = Typeface.DEFAULT_BOLD + includeFontPadding = false + }) + addView(TextView(this@MainActivity).apply { + text = "中英跟读训练" + setTextColor(COLOR_TEXT_MUTED) + textSize = 13f + setPadding(1.dp, 4.dp, 0, 0) + }) + } + + return LinearLayout(this).apply { + orientation = LinearLayout.HORIZONTAL + gravity = Gravity.CENTER_VERTICAL + setPadding(18.dp, 16.dp, 18.dp, 12.dp) + addView(titleBlock) + addView(headerButton("导入") { openLocalVideoPicker() }) + addView(headerButton("刷新") { loadCatalog() }) + } + } + + private fun createPlayerSection(): View { + playerView = OralTrainerPlayerView(this).apply { + bind(controller) + layoutParams = FrameLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + ViewGroup.LayoutParams.MATCH_PARENT, + ) + } + + val overlay = LinearLayout(this).apply { + orientation = LinearLayout.HORIZONTAL + gravity = Gravity.CENTER_VERTICAL + setPadding(12.dp, 12.dp, 12.dp, 12.dp) + layoutParams = FrameLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + ViewGroup.LayoutParams.WRAP_CONTENT, + Gravity.TOP, + ) + } + + timeText = overlayPill("--:-- / --:--").apply { + layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f) + } + speedText = overlayPill("1x") + overlay.addView(timeText) + overlay.addView(speedText) + + return FrameLayout(this).apply { + background = rounded(COLOR_SURFACE, 8f, COLOR_BORDER) + clipToOutline = true + layoutParams = LinearLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + 0, + 1f, + ).withMargins(16.dp, 0, 16.dp, 10.dp) + addView(playerView) + addView(overlay) + } + } + + private fun createSentenceSection(): View { + lessonTitleText = TextView(this).apply { + setTextColor(COLOR_TEXT_DARK) + textSize = 17f + typeface = Typeface.DEFAULT_BOLD + maxLines = 1 + } + sentenceMetaText = TextView(this).apply { + setTextColor(COLOR_ACCENT_DEEP) + textSize = 13f + setPadding(0, 8.dp, 0, 0) + } + sentenceText = TextView(this).apply { + setTextColor(COLOR_TEXT_DARK) + textSize = 22f + typeface = Typeface.DEFAULT_BOLD + setLineSpacing(2.dp.toFloat(), 1.05f) + setPadding(0, 10.dp, 0, 10.dp) + } + progressBar = ProgressBar(this, null, android.R.attr.progressBarStyleHorizontal).apply { + max = PROGRESS_MAX + progress = 0 + progressTintList = ColorStateList.valueOf(COLOR_ACCENT) + progressBackgroundTintList = ColorStateList.valueOf(COLOR_PROGRESS_TRACK) + layoutParams = LinearLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + 6.dp, + ) + } + statusText = TextView(this).apply { + setTextColor(COLOR_TEXT_SUBTLE) + textSize = 13f + setPadding(0, 10.dp, 0, 0) + } + + return LinearLayout(this).apply { + orientation = LinearLayout.VERTICAL + background = rounded(Color.WHITE, 8f, COLOR_LIGHT_BORDER) + setPadding(18.dp, 16.dp, 18.dp, 16.dp) + layoutParams = LinearLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + ViewGroup.LayoutParams.WRAP_CONTENT, + ).withMargins(16.dp, 0, 16.dp, 12.dp) + addView(lessonTitleText) + addView(sentenceMetaText) + addView(sentenceText) + addView(progressBar) + addView(statusText) + } + } + + private fun createCatalogSection(): View { + catalogStatusText = TextView(this).apply { + setTextColor(COLOR_TEXT_MUTED) + textSize = 13f + gravity = Gravity.END + text = "正在同步" + layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f) + } + catalogList = LinearLayout(this).apply { + orientation = LinearLayout.HORIZONTAL + setPadding(0, 10.dp, 0, 4.dp) + } + + val header = LinearLayout(this).apply { + orientation = LinearLayout.HORIZONTAL + gravity = Gravity.CENTER_VERTICAL + addView(TextView(this@MainActivity).apply { + text = "课程" + setTextColor(Color.WHITE) + textSize = 16f + typeface = Typeface.DEFAULT_BOLD + }) + addView(catalogStatusText) + } + + val scroller = HorizontalScrollView(this).apply { + isHorizontalScrollBarEnabled = false + addView(catalogList) + } + + return LinearLayout(this).apply { + orientation = LinearLayout.VERTICAL + setPadding(16.dp, 0, 16.dp, 14.dp) + addView(header) + addView(scroller) + } + } + + private fun bindPlayerEvents() { + controller.addListener(object : OralTrainerListener { + override fun onPlaybackSnapshot(snapshot: PlaybackSnapshot) { + timeText.text = "${formatTime(snapshot.positionMs)} / ${formatTime(snapshot.durationMs)}" + speedText.text = formatSpeed(snapshot.playbackSpeed) + statusText.text = playbackStatus(snapshot) + progressBar.progress = playbackProgress(snapshot) + } + + override fun onMediaChanged(item: TrainingMediaItem?) { + lessonTitleText.text = item?.title ?: "未选择课程" + currentSentenceCount = item?.sentences?.size ?: 0 + } + + override fun onSentenceChanged(sentence: SentenceBoundary?) { + if (sentence == null) { + sentenceMetaText.text = "暂无句子边界" + sentenceText.text = lessonTitleText.text + return + } + val countText = if (currentSentenceCount > 0) { + " / $currentSentenceCount" + } else { + "" + } + sentenceMetaText.text = "第 ${sentence.index + 1}$countText 句 ${formatTime(sentence.startMs)}-${formatTime(sentence.endMs)}" + sentenceText.text = sentence.text?.takeIf { it.isNotBlank() } ?: "当前句子" + } + + override fun onGesture(event: GestureEvent) { + statusText.text = when (event.kind) { + GestureKind.SINGLE_TAP -> "播放状态已切换" + GestureKind.SWIPE_LEFT -> "已跳到上一句" + GestureKind.SWIPE_RIGHT -> "已跳到下一句" + GestureKind.LONG_PRESS_SPEED -> "变速播放中" + } + } + + override fun onPlayerError(error: Throwable) { + statusText.text = "播放失败:${error.message.orEmpty()}" + } + }) + } + + private fun loadSampleLesson() { + val item = sampleOnlineLesson() + activeItemId = item.id + currentSentenceCount = item.sentences.size + controller.loadItem(item) + lessonTitleText.text = item.title + renderCatalog() + } + + private fun loadCatalog() { + catalogStatusText.text = "正在同步" + sdk.videoCatalogApi.fetch(object : VideoCatalogCallback { + override fun onSuccess(videos: List) { + catalogVideos = videos + catalogStatusText.text = if (videos.isEmpty()) { + "暂无云端课程" + } else { + "${videos.size} 个云端课程" + } + renderCatalog() + } + + override fun onError(error: Throwable) { + catalogStatusText.text = "云端暂不可用" + renderCatalog() + } + }) + } + + private fun renderCatalog() { + if (!::catalogList.isInitialized) { + return + } + catalogList.removeAllViews() + catalogList.addView( + videoCard( + id = SAMPLE_ID, + title = "示例课程", + meta = "4 句 · ${formatTime(16_000L)}", + ) { + loadSampleLesson() + } + ) + catalogVideos.forEach { video -> + catalogList.addView( + videoCard( + id = video.videoHash, + title = video.title, + meta = "${video.sentenceCount} 句 · ${formatTime(video.durationMs ?: -1L)}", + ) { + loadRemoteVideo(video) + } + ) + } + } + + private fun loadRemoteVideo(video: TrainingVideoSummary) { + activeItemId = video.videoHash + lessonTitleText.text = video.title + sentenceMetaText.text = "句子边界加载中" + sentenceText.text = video.title + statusText.text = "正在加载云端课程..." + renderCatalog() + + val loadingItem = video.toTrainingMediaItem() + controller.loadItem(loadingItem) + currentSentenceCount = video.sentenceCount + sdk.sentenceBoundaryApi.fetch(video.videoHash, object : SentenceBoundaryApiCallback { + override fun onSuccess(result: SentenceBoundaryApiResult) { + if (activeItemId != video.videoHash) { + return + } + currentSentenceCount = result.sentences.size + controller.loadItem(video.toTrainingMediaItem(result.sentences)) + statusText.text = "课程已就绪:${result.sentences.size} 句" + } + + override fun onError(error: Throwable) { + if (activeItemId != video.videoHash) { + return + } + statusText.text = "句子边界加载失败:${error.message.orEmpty()}" + sentenceMetaText.text = "暂无句子边界" + sentenceText.text = video.title + } + }) + } + + private fun videoCard( + id: String, + title: String, + meta: String, + onClick: () -> Unit, + ): View { + val selected = activeItemId == id + val titleColor = if (selected) COLOR_TEXT_DARK else Color.WHITE + val metaColor = if (selected) COLOR_ACCENT_DEEP else COLOR_TEXT_MUTED + return LinearLayout(this).apply { + orientation = LinearLayout.VERTICAL + gravity = Gravity.CENTER_VERTICAL + isClickable = true + background = if (selected) { + rounded(COLOR_SELECTED, 8f, COLOR_ACCENT) + } else { + rounded(COLOR_SURFACE, 8f, COLOR_BORDER) + } + setPadding(14.dp, 12.dp, 14.dp, 12.dp) + layoutParams = LinearLayout.LayoutParams(218.dp, 84.dp).withMargins(0, 0, 10.dp, 0) + setOnClickListener { onClick() } + addView(TextView(this@MainActivity).apply { + text = title + setTextColor(titleColor) + textSize = 15f + typeface = Typeface.DEFAULT_BOLD + maxLines = 2 + }) + addView(TextView(this@MainActivity).apply { + text = meta + setTextColor(metaColor) + textSize = 12f + setPadding(0, 6.dp, 0, 0) + }) + } + } + + private fun headerButton(label: String, action: () -> Unit): Button { return Button(this).apply { text = label - setOnClickListener { action() } + isAllCaps = false + setTextColor(Color.WHITE) + textSize = 14f + typeface = Typeface.DEFAULT_BOLD + minWidth = 0 + minHeight = 0 + minimumWidth = 0 + minimumHeight = 0 + setPadding(14.dp, 0, 14.dp, 0) + background = rounded(COLOR_BUTTON, 8f) layoutParams = LinearLayout.LayoutParams( - 0, ViewGroup.LayoutParams.WRAP_CONTENT, - 1f, - ).apply { - marginStart = 4 - marginEnd = 4 - } + 40.dp, + ).withMargins(8.dp, 0, 0, 0) + setOnClickListener { action() } + } + } + + private fun overlayPill(label: String): TextView { + return TextView(this).apply { + text = label + setTextColor(Color.WHITE) + textSize = 13f + typeface = Typeface.DEFAULT_BOLD + background = rounded(COLOR_OVERLAY, 8f) + setPadding(10.dp, 6.dp, 10.dp, 6.dp) + gravity = Gravity.CENTER } } @@ -185,27 +518,103 @@ class MainActivity : Activity() { private fun sampleOnlineLesson(): TrainingMediaItem { return TrainingMediaItem( - id = "online_sample_01", - title = "Online Sample Lesson", + id = SAMPLE_ID, + title = "口语宝示例课", uri = Uri.parse("https://storage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4"), mimeType = MimeTypes.VIDEO_MP4, - customCacheKey = "online_sample_01", + customCacheKey = SAMPLE_ID, sentences = listOf( - SentenceBoundary(0, 0L, 3_000L, "Listen once, then imitate."), - SentenceBoundary(1, 3_000L, 7_000L, "Swipe left to go back."), - SentenceBoundary(2, 7_000L, 11_000L, "Swipe right to move forward."), - SentenceBoundary(3, 11_000L, 16_000L, "Tap the video area to pause or play."), + SentenceBoundary(0, 0L, 3_000L, "Good morning, everyone."), + SentenceBoundary(1, 3_000L, 7_000L, "Today we will practice listening carefully."), + SentenceBoundary(2, 7_000L, 11_000L, "Please repeat each sentence clearly."), + SentenceBoundary(3, 11_000L, 16_000L, "Small daily practice makes progress visible."), ), ) } - private fun format(ms: Long): String { + private fun playbackStatus(snapshot: PlaybackSnapshot): String { + val state = when { + snapshot.isPlaying -> "播放中" + snapshot.playbackState.name == "BUFFERING" -> "缓冲中" + snapshot.playbackState.name == "ENDED" -> "已结束" + else -> "已暂停" + } + val sentence = snapshot.sentenceIndex?.let { " · 第 ${it + 1} 句" }.orEmpty() + return "$state$sentence" + } + + private fun playbackProgress(snapshot: PlaybackSnapshot): Int { + if (snapshot.durationMs <= 0L) { + return 0 + } + return ((snapshot.positionMs.coerceAtMost(snapshot.durationMs) * PROGRESS_MAX) / snapshot.durationMs).toInt() + } + + private fun formatTime(ms: Long): String { if (ms < 0) { return "--:--" } val totalSeconds = ms / 1000 - val minutes = totalSeconds / 60 + val hours = totalSeconds / 3600 + val minutes = (totalSeconds % 3600) / 60 val seconds = totalSeconds % 60 - return "%02d:%02d".format(minutes, seconds) + return if (hours > 0) { + "%d:%02d:%02d".format(Locale.US, hours, minutes, seconds) + } else { + "%02d:%02d".format(Locale.US, minutes, seconds) + } + } + + private fun formatSpeed(speed: Float): String { + val value = String.format(Locale.US, "%.2f", speed).trimEnd('0').trimEnd('.') + return "${value}x" + } + + private fun rounded( + color: Int, + radiusDp: Float, + strokeColor: Int? = null, + ): GradientDrawable { + return GradientDrawable().apply { + setColor(color) + cornerRadius = radiusDp.dp.toFloat() + strokeColor?.let { setStroke(1.dp, it) } + } + } + + private fun LinearLayout.LayoutParams.withMargins( + left: Int, + top: Int, + right: Int, + bottom: Int, + ): LinearLayout.LayoutParams { + setMargins(left, top, right, bottom) + return this + } + + private val Int.dp: Int + get() = (this * resources.displayMetrics.density).toInt() + + private val Float.dp: Int + get() = (this * resources.displayMetrics.density).toInt() + + private companion object { + const val PICK_VIDEO_REQUEST = 1001 + const val SAMPLE_ID = "online_sample_01" + const val PROGRESS_MAX = 1000 + + val COLOR_BACKGROUND: Int = Color.rgb(12, 15, 18) + val COLOR_SURFACE: Int = Color.rgb(28, 34, 40) + val COLOR_BORDER: Int = Color.rgb(50, 59, 67) + val COLOR_BUTTON: Int = Color.rgb(37, 99, 235) + val COLOR_ACCENT: Int = Color.rgb(21, 184, 132) + val COLOR_ACCENT_DEEP: Int = Color.rgb(7, 118, 86) + val COLOR_SELECTED: Int = Color.rgb(229, 248, 240) + val COLOR_PROGRESS_TRACK: Int = Color.rgb(224, 231, 235) + val COLOR_LIGHT_BORDER: Int = Color.rgb(218, 226, 232) + val COLOR_OVERLAY: Int = Color.argb(178, 9, 12, 16) + val COLOR_TEXT_DARK: Int = Color.rgb(18, 24, 31) + val COLOR_TEXT_MUTED: Int = Color.rgb(151, 162, 174) + val COLOR_TEXT_SUBTLE: Int = Color.rgb(88, 98, 108) } } diff --git a/android/sample-app/src/main/res/values/strings.xml b/android/sample-app/src/main/res/values/strings.xml new file mode 100644 index 0000000..2aedfcc --- /dev/null +++ b/android/sample-app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + 口语宝 + diff --git a/output/karoline_audio.m4a b/output/karoline_audio.m4a new file mode 100644 index 0000000..a16a811 Binary files /dev/null and b/output/karoline_audio.m4a differ diff --git a/output/karoline_sample_30s.moss_0_9b_after_redownload.json b/output/karoline_sample_30s.moss_0_9b_after_redownload.json new file mode 100644 index 0000000..8610ab7 --- /dev/null +++ b/output/karoline_sample_30s.moss_0_9b_after_redownload.json @@ -0,0 +1 @@ +{"error":{"message":"Model 'MOSS-Transcribe-Diarize-0.9B-MLX-5bit' failed to load: No safetensors found in /Users/liushuming/.omlx/models/aufklarer/MOSS-Transcribe-Diarize-0.9B-MLX-5bit. Reload models after fixing the files to retry.","type":"server_error","param":null,"code":null}} \ No newline at end of file diff --git a/output/karoline_sample_30s.transcription.json b/output/karoline_sample_30s.transcription.json new file mode 100644 index 0000000..e97ea1a --- /dev/null +++ b/output/karoline_sample_30s.transcription.json @@ -0,0 +1 @@ +{"error":{"message":"Model type moss_transcribe_diarize not supported.","type":"server_error","param":null,"code":null}} \ No newline at end of file diff --git a/output/karoline_sample_30s.wav b/output/karoline_sample_30s.wav new file mode 100644 index 0000000..d4d67c3 Binary files /dev/null and b/output/karoline_sample_30s.wav differ diff --git a/sentence_api/.env.example b/sentence_api/.env.example new file mode 100644 index 0000000..d9af6fa --- /dev/null +++ b/sentence_api/.env.example @@ -0,0 +1,14 @@ +ORAL_TRAINER_DATA_DIR=/app/sentence_api/data +PUBLIC_BASE_URL=https://video_service.d1kt.cn +ADMIN_API_KEY=replace-with-a-long-random-secret +CLIENT_API_KEY=replace-with-a-separate-client-secret + +MAX_VIDEO_UPLOAD_BYTES=12884901888 +MAX_ATTEMPT_UPLOAD_BYTES=52428800 +KEEP_ATTEMPT_AUDIO=false +ASSESSMENT_PASS_SCORE=70 + +MOSS_TRANSCRIBE_URL=http://127.0.0.1:8001/v1/audio/transcriptions +MOSS_MODEL=OpenMOSS-Team/MOSS-Transcribe-Diarize +MOSS_TIMEOUT_SECONDS=1800 +MOSS_MAX_NEW_TOKENS=65536 diff --git a/sentence_api/DEPLOYMENT.md b/sentence_api/DEPLOYMENT.md new file mode 100644 index 0000000..a8d1738 --- /dev/null +++ b/sentence_api/DEPLOYMENT.md @@ -0,0 +1,314 @@ +# 口语训练视频服务部署指南 + +本文部署以下两个进程: + +- `oral-trainer-api`:视频上传、管理后台、边界查询、Range 视频播放和朗读评分。 +- `MOSS-Transcribe-Diarize`:GPU 转写服务,只在服务器内网监听。 + +推荐 Ubuntu 22.04/24.04、Python 3.12、FFmpeg、NVIDIA GPU 和已安装的 NVIDIA 驱动。MOSS 0.9B 的实际显存占用受推理框架、并发和音频长度影响,生产环境建议从 16 GB 以上显存开始验证。 + +## 1. 目录与端口 + +本文假设代码位于 `/opt/oral-trainer`: + +```text +/opt/oral-trainer/ + sentence_api/ + data/ + v/ 上传的视频 + work/ 临时转码文件 + attempts/ 可选的学生录音 + oral_trainer.sqlite3 +``` + +端口规划: + +| 服务 | 监听地址 | 用途 | +| --- | --- | --- | +| MOSS | `127.0.0.1:8001` | 内部语音转写 | +| FastAPI | `127.0.0.1:8000` | 内部应用服务 | +| Nginx | `0.0.0.0:443` | 对外 HTTPS | + +不要把 MOSS 的 `8001` 端口直接暴露到公网。 + +## 2. 部署 MOSS + +官方模型为 `OpenMOSS-Team/MOSS-Transcribe-Diarize`。CUDA 12 环境可使用官方当前说明中的 vLLM 构建: + +```bash +sudo apt update +sudo apt install -y ffmpeg git curl +curl -LsSf https://astral.sh/uv/install.sh | sh + +sudo mkdir -p /opt/moss-transcribe +sudo chown "$USER":"$USER" /opt/moss-transcribe +cd /opt/moss-transcribe + +uv venv --python 3.12 .venv +. .venv/bin/activate +uv pip install -U vllm \ + --torch-backend=auto \ + --extra-index-url https://wheels.vllm.ai/68b4a1d582818e67adc903bf1b8fc5a5447da2fa/cu129 +``` + +启动模型: + +```bash +. /opt/moss-transcribe/.venv/bin/activate +vllm serve OpenMOSS-Team/MOSS-Transcribe-Diarize \ + --host 127.0.0.1 \ + --port 8001 \ + --trust-remote-code +``` + +生产环境可把它注册为 systemd 服务: + +```ini +# /etc/systemd/system/moss-transcribe.service +[Unit] +Description=MOSS Transcribe Service +After=network-online.target + +[Service] +Type=simple +User=oraltrainer +WorkingDirectory=/opt/moss-transcribe +ExecStart=/opt/moss-transcribe/.venv/bin/vllm serve OpenMOSS-Team/MOSS-Transcribe-Diarize --host 127.0.0.1 --port 8001 --trust-remote-code +Restart=on-failure +RestartSec=10 +TimeoutStartSec=1800 + +[Install] +WantedBy=multi-user.target +``` + +```bash +sudo systemctl daemon-reload +sudo systemctl enable --now moss-transcribe +sudo journalctl -u moss-transcribe -f +``` + +模型第一次启动会下载权重。CUDA 13 服务器也可以按照 MOSS 官方仓库说明改用 SGLang Omni。推理框架的安装地址可能随上游版本更新,正式部署前应对照 [MOSS 官方仓库](https://github.com/OpenMOSS/MOSS-Transcribe-Diarize) 的 Quickstart。 + +验证: + +```bash +curl -X POST http://127.0.0.1:8001/v1/audio/transcriptions \ + -F model=OpenMOSS-Team/MOSS-Transcribe-Diarize \ + -F file=@/path/to/test.wav \ + -F response_format=verbose_json \ + -F temperature=0 +``` + +响应应包含 `text`;开启 `verbose_json` 后应包含带 `start`、`end`、`text` 的 `segments`。 + +## 3. 部署 API + +### Docker Compose 方式 + +```bash +cd /opt/oral-trainer/sentence_api +cp .env.example .env +openssl rand -hex 32 +``` + +分别生成两个随机值,填入 `.env` 的 `ADMIN_API_KEY` 和 `CLIENT_API_KEY`,并确认: + +```dotenv +PUBLIC_BASE_URL=https://video_service.d1kt.cn +MOSS_TRANSCRIBE_URL=http://127.0.0.1:8001/v1/audio/transcriptions +``` + +启动: + +```bash +docker compose up -d --build +docker compose logs -f oral-trainer-api +``` + +Compose 配置使用 Linux 的 host network,使容器可以访问只监听 +`127.0.0.1:8001` 的 MOSS,同时 API 也只监听 `127.0.0.1:8000`。如果在 +Docker Desktop 上做本地测试,可移除 `network_mode: host`,恢复端口映射,并把 +MOSS 地址改为 `host.docker.internal`。 + +检查: + +```bash +curl http://127.0.0.1:8000/healthz +``` + +`moss_configured` 应为 `true`。这里只表示地址已配置,实际连通性会在上传视频或评分时验证。 + +### 不使用 Docker + +```bash +cd /opt/oral-trainer +python3.12 -m venv .venv-api +. .venv-api/bin/activate +python -m pip install -r sentence_api/requirements.txt + +export ORAL_TRAINER_DATA_DIR=/opt/oral-trainer-data +export PUBLIC_BASE_URL=https://video_service.d1kt.cn +export ADMIN_API_KEY='替换为随机密钥' +export MOSS_TRANSCRIBE_URL=http://127.0.0.1:8001/v1/audio/transcriptions + +python -m uvicorn sentence_api.main:app \ + --host 127.0.0.1 --port 8000 --workers 1 \ + --proxy-headers --forwarded-allow-ips='*' +``` + +不使用 Docker 时可创建 API 的 systemd 服务: + +```ini +# /etc/systemd/system/oral-trainer-api.service +[Unit] +Description=Oral Trainer Video API +After=network-online.target moss-transcribe.service + +[Service] +Type=simple +User=oraltrainer +WorkingDirectory=/opt/oral-trainer +EnvironmentFile=/opt/oral-trainer/sentence_api/.env +ExecStart=/opt/oral-trainer/.venv-api/bin/python -m uvicorn sentence_api.main:app --host 127.0.0.1 --port 8000 --workers 1 --proxy-headers --forwarded-allow-ips=* +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +``` + +```bash +sudo systemctl daemon-reload +sudo systemctl enable --now oral-trainer-api +sudo journalctl -u oral-trainer-api -f +``` + +当前上传后的处理任务运行在 API 进程中,因此只能使用一个 Uvicorn worker。需要多机或多 worker 时,应把 `VideoProcessor.process` 迁移到 Celery、RQ 或其他持久化任务队列。 + +## 4. 配置 Nginx 与 HTTPS + +```bash +sudo cp sentence_api/nginx.conf.example /etc/nginx/sites-available/oral-trainer +sudo ln -s /etc/nginx/sites-available/oral-trainer /etc/nginx/sites-enabled/oral-trainer +sudo nginx -t +sudo systemctl reload nginx +``` + +配置文件中的关键项: + +- `client_max_body_size 12g`:允许超过 1 GB 的视频。 +- `proxy_request_buffering off`:上传时直接流向 FastAPI,避免 Nginx 再完整缓存一份。 +- `proxy_force_ranges on`:保留 Android 随机拖动播放所需的 HTTP Range。 +- `proxy_read_timeout 7200s`:允许长视频处理和慢速上传。 + +签发证书: + +```bash +sudo apt install -y certbot python3-certbot-nginx +sudo certbot --nginx -d video_service.d1kt.cn +``` + +证书完成后验证: + +```bash +curl https://video_service.d1kt.cn/healthz +``` + +管理后台地址: + +```text +https://video_service.d1kt.cn/admin +``` + +后台的管理密钥保存在当前浏览器的 `localStorage`,接口请求通过 `X-Admin-Key` 发送。 +`CLIENT_API_KEY` 应通过 Android 构建配置注入 `OralTrainerSdkConfig.assessmentApiKey`, +不要硬编码进公开代码仓库。SDK 会在评分请求中发送 `X-Client-Key`。公开视频列表和播放 +接口仍可交给 CDN 缓存,GPU 评分接口则受到密钥保护。 + +## 5. 上传与处理流程 + +后台上传后,服务会: + +1. 管理后台通过 raw body 流式写入临时文件并同步计算 SHA-256,不会先在系统临时目录保留完整副本。兼容的 multipart 接口仍以 4 MB 分块复制。 +2. 保存到 `data/v/{sha256}.{extension}`。 +3. 使用 FFmpeg 提取 16 kHz 单声道 WAV。 +4. 请求 MOSS 的 `verbose_json` 转写接口。 +5. 以 MOSS 时间戳生成句子边界,并计算每句原音有效语音时长。 +6. 把视频和句子写入 SQLite,状态变为 `ready`。 + +查询处理状态: + +```bash +curl https://video_service.d1kt.cn/api/v1/videos +curl https://video_service.d1kt.cn/api/v1/videos/{sha256} +``` + +如果 MOSS 未配置,上传仍会使用原来的静音检测生成边界,但句子文本为空,无法开始测验。配置好 MOSS 后可在后台点击“重新处理”。 + +## 6. 视频随机播放 + +移动端播放地址为: + +```text +GET /api/v1/videos/{sha256}/content +``` + +服务支持 HTTP Range,Media3/ExoPlayer 可以随机 seek,并继续使用 SDK 现有的本地缓存。第一阶段不必强制改成 HLS。 + +上传前建议把 MP4 处理成 H.264/AAC 并把 `moov` 元数据移动到文件头: + +```bash +ffmpeg -i input.mp4 -c copy -movflags +faststart output.mp4 +``` + +如果原始编码不被 Android 广泛支持,再转码: + +```bash +ffmpeg -i input.mkv \ + -c:v libx264 -preset medium -crf 22 \ + -c:a aac -b:a 128k -movflags +faststart output.mp4 +``` + +当并发量明显增长或网络波动较大时,再增加 HLS/DASH 转码和对象存储/CDN。 + +## 7. 朗读评分接口 + +```bash +curl -X POST \ + https://video_service.d1kt.cn/api/v1/videos/{sha256}/sentences/0/assessments \ + -H 'X-Client-Key: 替换为客户端密钥' \ + -F audio=@student.wav \ + -F language=en +``` + +当前 `asr-fluency-v1` 评分为: + +```text +总分 = 内容分 * 80% + 流畅度 * 20% +流畅度 = 有效语音时长分 * 35% + 停顿分 * 40% + 语速分 * 25% +``` + +总分不低于 `ASSESSMENT_PASS_SCORE`(默认 70)即通过。`pronunciation_score` 和 `prosody_score` 目前返回 `null`,避免把 ASR 内容匹配误报为音素发音质量。 + +服务只向 MOSS 发送语言提示,不发送标准句子作为 prompt 或热词,避免标准答案诱导识别结果。 + +## 8. 运维与备份 + +需要持久化备份: + +```text +sentence_api/data/v/ +sentence_api/data/oral_trainer.sqlite3 +``` + +`work/` 可以清理。`KEEP_ATTEMPT_AUDIO=false` 时学生录音在评分完成后自动删除;设为 `true` 时还需备份和定期清理 `attempts/`,并在产品隐私政策中说明保存期限。 + +SQLite 在线备份示例: + +```bash +sqlite3 sentence_api/data/oral_trainer.sqlite3 \ + ".backup '/backup/oral_trainer-$(date +%F).sqlite3'" +``` + +升级前先备份数据库和 `data/v/`。API 重启期间被中断的视频会保留 `processing` 状态,可从后台执行“重新处理”。 diff --git a/sentence_api/Dockerfile b/sentence_api/Dockerfile new file mode 100644 index 0000000..a01d57a --- /dev/null +++ b/sentence_api/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY sentence_api/requirements.txt /app/sentence_api/requirements.txt +RUN python -m pip install --upgrade pip \ + && python -m pip install -r /app/sentence_api/requirements.txt + +COPY sentence_api /app/sentence_api +COPY sentence_analysis.py /app/sentence_analysis.py + +EXPOSE 8000 +CMD ["python", "-m", "uvicorn", "sentence_api.main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1", "--proxy-headers", "--forwarded-allow-ips", "*"] diff --git a/sentence_api/README.md b/sentence_api/README.md index afbfc60..af76507 100644 --- a/sentence_api/README.md +++ b/sentence_api/README.md @@ -1,92 +1,82 @@ -# Sentence Boundary API +# Oral Trainer Video Service -This service looks up pre-generated sentence boundaries by the SHA-256 hash of -the exact video bytes. It does not analyze media during an API request. +FastAPI service for the Android oral-training SDK. It provides: -## Install +- chunked video upload with SHA-256 calculation; +- files stored under `data/v/` and metadata stored in SQLite; +- background sentence transcription through MOSS-Transcribe-Diarize; +- a browser-based administration page at `/admin`; +- video catalog and HTTP Range playback; +- sentence-boundary lookup compatible with the existing Android SDK; +- student recording upload and `asr-fluency-v1` assessment; +- the legacy JSON boundary index as a read-only fallback. -From the repository root: +## Local Run ```bash -python3 -m venv .venv-sentence-api +python3.12 -m venv .venv-sentence-api . .venv-sentence-api/bin/activate python -m pip install -r sentence_api/requirements.txt -``` -## Generate The Index - -Generate boundaries with the same silence detector used by the desktop player: - -```bash -python -m sentence_api.generate_boundaries \ - "/path/to/lesson.mp4" \ - --index sentence_api/data/sentence_boundaries.json -``` - -The command calculates the SHA-256 hash, detects boundaries, converts seconds -to milliseconds, infers each `end_ms` from the next sentence start, and writes -the result atomically into the JSON index. The last sentence ends at the media -duration. - -For a large course library, run this command in an ingestion worker and store -the same document in a database or object storage instead of committing the -JSON file to the application image. - -## Run - -```bash -SENTENCE_BOUNDARIES_FILE=sentence_api/data/sentence_boundaries.json \ +export ADMIN_API_KEY=local-development-key +export MOSS_TRANSCRIBE_URL=http://127.0.0.1:8001/v1/audio/transcriptions python -m uvicorn sentence_api.main:app --host 0.0.0.0 --port 8000 ``` -The interactive API documentation is available at `/docs`. +Open: -## Request - -```http -GET /api/v1/videos/{sha256}/sentence-boundaries +```text +http://127.0.0.1:8000/admin +http://127.0.0.1:8000/docs ``` -Example using the demo record in the checked-in index: +When `MOSS_TRANSCRIBE_URL` is empty, uploaded videos still receive silence-based +boundaries, but no reference transcript is produced and assessment is disabled. + +## Main Endpoints + +```text +GET /healthz +GET /api/v1/videos +GET /api/v1/videos/{sha256} +GET /api/v1/videos/{sha256}/content +GET /api/v1/videos/{sha256}/sentence-boundaries +POST /api/v1/videos/{sha256}/sentences/{index}/assessments + +POST /api/v1/admin/videos +PUT /api/v1/admin/videos/raw +POST /api/v1/admin/videos/{sha256}/process +PUT /api/v1/admin/videos/{sha256}/sentences/{index} +DELETE /api/v1/admin/videos/{sha256} +``` + +Admin endpoints use `X-Admin-Key` when `ADMIN_API_KEY` is configured. Assessment +requests use `X-Client-Key` when `CLIENT_API_KEY` is configured. + +## Assessment ```bash -curl http://127.0.0.1:8000/api/v1/videos/468a4d064f6ec49942b45e25ab93c500d31870f978c4f28ff8b3b408852326e0/sentence-boundaries +curl -X POST \ + http://127.0.0.1:8000/api/v1/videos/{sha256}/sentences/0/assessments \ + -F audio=@student.wav \ + -F language=en ``` -The MP4 used during development is also indexed. Its hash is -`b6631d5cf48f37fed0ecc623563dd48b7ed660689b4d25d2ebac7fecab807ddc`, and its -generated index contains 244 boundaries. +The current score intentionally measures reading content and fluency: -The response is: - -```json -{ - "video_hash": "...", - "duration_ms": 16000, - "algorithm_version": "silence-rms-v1", - "sentences": [ - { - "index": 0, - "start_ms": 0, - "end_ms": 4230, - "text": null - } - ] -} +```text +overall = content * 80% + fluency * 20% +fluency = duration * 35% + pauses * 40% + speech rate * 25% ``` -Unknown hashes return `404`. A hash must be a 64-character hexadecimal -SHA-256 digest; malformed values return `422`. +`pronunciation_score` and `prosody_score` remain `null` until a phoneme/GOP +model is connected. -When testing from a physical Android phone, replace `127.0.0.1` with the -computer's LAN IP address. `127.0.0.1` on the phone refers to the phone itself. -For production, expose the API over HTTPS. +## Tests -## Android Request Flow +```bash +python -m pytest sentence_api/tests -q +``` -The mobile app should calculate the hash from the selected `content://` URI in -streaming chunks, request the endpoint, map the returned `sentences` to -`SentenceBoundary`, and then call `controller.loadItem`. The hash must be -calculated from the exact bytes of the same video served to the player. For -HTTPS course videos, the course manifest can carry the hash and avoid hashing -the entire remote file on every device. +See [DEPLOYMENT.md](DEPLOYMENT.md) for Docker, MOSS, Nginx, HTTPS, large-file +upload, backup, and production operation instructions. diff --git a/sentence_api/assessment.py b/sentence_api/assessment.py new file mode 100644 index 0000000..2bf3f88 --- /dev/null +++ b/sentence_api/assessment.py @@ -0,0 +1,107 @@ +import uuid +from pathlib import Path +from typing import Optional + +from .audio_metrics import analyze_audio +from .models import AssessmentResult, SentenceBoundaryDocument, TextSubstitution +from .repository import VideoRepository +from .scoring import SCORING_VERSION, score_attempt +from .transcription import Transcriber + + +class AssessmentService: + def __init__( + self, + repository: VideoRepository, + transcriber: Transcriber, + pass_score: float, + ): + self.repository = repository + self.transcriber = transcriber + self.pass_score = pass_score + + def assess( + self, + *, + document: SentenceBoundaryDocument, + sentence_index: int, + audio_path: Path, + language: Optional[str] = None, + retained_audio_filename: Optional[str] = None, + ) -> AssessmentResult: + if not self.transcriber.available: + raise RuntimeError("MOSS transcription is not configured on this server.") + sentence = next( + (item for item in document.sentences if item.index == sentence_index), + None, + ) + if sentence is None: + raise LookupError("Sentence index was not found.") + if not sentence.text or not sentence.text.strip(): + raise ValueError("The sentence has no reviewed reference text yet.") + + student_metrics = analyze_audio(audio_path) + transcript = self.transcriber.transcribe( + audio_path, + language or sentence.language, + ) + recognized_text = transcript.text.strip() + if not recognized_text: + raise ValueError("MOSS did not recognize any speech in the recording.") + + reference_duration_ms = sentence.end_ms - sentence.start_ms + reference_speech_duration_ms = ( + sentence.reference_speech_duration_ms or reference_duration_ms + ) + score = score_attempt( + reference_text=sentence.text, + recognized_text=recognized_text, + reference_speech_duration_ms=reference_speech_duration_ms, + student_metrics=student_metrics, + ) + attempt_id = uuid.uuid4().hex + result = AssessmentResult( + attempt_id=attempt_id, + scoring_version=SCORING_VERSION, + overall_score=score.overall_score, + passed=score.overall_score >= self.pass_score, + pass_score=self.pass_score, + content_score=score.content_score, + completeness_score=score.completeness_score, + fluency_score=score.fluency_score, + pronunciation_score=None, + prosody_score=None, + duration_score=score.duration_score, + pause_score=score.pause_score, + speech_rate_score=score.speech_rate_score, + reference_text=sentence.text, + recognized_text=recognized_text, + reference_duration_ms=reference_duration_ms, + reference_speech_duration_ms=reference_speech_duration_ms, + student_recording_duration_ms=student_metrics.recording_duration_ms, + student_speech_duration_ms=student_metrics.speech_duration_ms, + duration_ratio=score.duration_ratio, + missing_tokens=score.missing_tokens, + extra_tokens=score.extra_tokens, + substitutions=[ + TextSubstitution(expected=expected, actual=actual) + for expected, actual in score.substitutions + ], + feedback=score.feedback, + details={ + "score_kind": "朗读内容与流畅度匹配分", + "content_weight": "0.80", + "fluency_weight": "0.20", + "duration_weight_within_fluency": "0.35", + "phoneme_scoring": "not_enabled", + }, + ) + if self.repository.get_video(document.video_hash) is not None: + self.repository.record_attempt( + attempt_id=attempt_id, + video_hash=document.video_hash, + sentence_index=sentence_index, + result=result.model_dump(mode="json"), + audio_filename=retained_audio_filename, + ) + return result diff --git a/sentence_api/audio_metrics.py b/sentence_api/audio_metrics.py new file mode 100644 index 0000000..a9b65ca --- /dev/null +++ b/sentence_api/audio_metrics.py @@ -0,0 +1,123 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Tuple + +import numpy as np + + +SAMPLE_RATE = 16_000 +FRAME_SAMPLES = 480 + + +class AudioAnalysisError(RuntimeError): + pass + + +@dataclass(frozen=True) +class AudioMetrics: + recording_duration_ms: int + speech_duration_ms: int + internal_silence_ms: int + internal_pause_ratio: float + + +def decode_audio_mono(path: Path) -> Tuple[np.ndarray, int]: + try: + import av + except ImportError as exc: + raise AudioAnalysisError("PyAV is required for audio analysis.") from exc + + container = av.open(str(path)) + chunks = [] + try: + audio_stream = next( + (stream for stream in container.streams if stream.type == "audio"), + None, + ) + if audio_stream is None: + raise AudioAnalysisError("The uploaded file does not contain an audio stream.") + resampler = av.AudioResampler(format="fltp", layout="mono", rate=SAMPLE_RATE) + for packet in container.demux(audio_stream): + for frame in packet.decode(): + for output in resampler.resample(frame): + chunks.append(output.to_ndarray()[0].astype(np.float32, copy=False)) + for output in resampler.resample(None): + chunks.append(output.to_ndarray()[0].astype(np.float32, copy=False)) + finally: + container.close() + + if not chunks: + raise AudioAnalysisError("The uploaded audio is empty.") + return np.concatenate(chunks), SAMPLE_RATE + + +def analyze_audio(path: Path) -> AudioMetrics: + samples, sample_rate = decode_audio_mono(path) + return analyze_samples(samples, sample_rate) + + +def analyze_samples(samples: np.ndarray, sample_rate: int = SAMPLE_RATE) -> AudioMetrics: + if samples.ndim != 1: + samples = samples.reshape(-1) + if samples.size == 0 or sample_rate <= 0: + raise AudioAnalysisError("The uploaded audio is empty.") + + recording_duration_ms = max(1, int(round(samples.size / sample_rate * 1000))) + window = max(1, int(round(sample_rate * 0.03))) + complete_frames = int(np.ceil(samples.size / window)) + padded = np.pad(samples, (0, complete_frames * window - samples.size)) + frames = padded.reshape(complete_frames, window).astype(np.float64, copy=False) + rms = np.sqrt(np.mean(frames * frames, axis=1)) + + signal_level = float(np.percentile(rms, 95)) + if signal_level < 0.002: + raise AudioAnalysisError("No usable speech was detected in the recording.") + noise_floor = float(np.percentile(rms, 10)) + threshold = min(signal_level * 0.45, max(0.006, noise_floor * 2.2)) + speech = rms >= threshold + + # Treat very short gaps inside a word as speech, then reject short clicks. + _bridge_false_runs(speech, max_frames=5) + _remove_true_runs(speech, max_frames=2) + speech_indexes = np.flatnonzero(speech) + if speech_indexes.size == 0: + raise AudioAnalysisError("No usable speech was detected in the recording.") + + frame_ms = window / sample_rate * 1000 + speech_duration_ms = max(1, int(round(speech.sum() * frame_ms))) + first = int(speech_indexes[0]) + last = int(speech_indexes[-1]) + internal_frames = max(1, last - first + 1) + internal_silence_frames = int((~speech[first : last + 1]).sum()) + internal_silence_ms = int(round(internal_silence_frames * frame_ms)) + internal_pause_ratio = internal_silence_frames / internal_frames + return AudioMetrics( + recording_duration_ms=recording_duration_ms, + speech_duration_ms=speech_duration_ms, + internal_silence_ms=internal_silence_ms, + internal_pause_ratio=round(internal_pause_ratio, 4), + ) + + +def _bridge_false_runs(values: np.ndarray, max_frames: int) -> None: + start = None + for index, value in enumerate(values): + if not value and start is None: + start = index + elif value and start is not None: + if start > 0 and index - start <= max_frames: + values[start:index] = True + start = None + + +def _remove_true_runs(values: np.ndarray, max_frames: int) -> None: + start = None + for index, value in enumerate(values): + if value and start is None: + start = index + elif not value and start is not None: + if index - start <= max_frames: + values[start:index] = False + start = None + if start is not None and len(values) - start <= max_frames: + values[start:] = False diff --git a/sentence_api/config.py b/sentence_api/config.py new file mode 100644 index 0000000..6544424 --- /dev/null +++ b/sentence_api/config.py @@ -0,0 +1,75 @@ +import os +from dataclasses import dataclass +from pathlib import Path + + +def _bool_env(name: str, default: bool) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +@dataclass(frozen=True) +class Settings: + data_dir: Path + legacy_boundaries_file: Path + admin_api_key: str + client_api_key: str + public_base_url: str + max_upload_bytes: int + max_attempt_bytes: int + keep_attempt_audio: bool + moss_transcribe_url: str + moss_model: str + moss_timeout_seconds: float + moss_max_new_tokens: int + pass_score: float + + @classmethod + def from_env(cls) -> "Settings": + package_dir = Path(__file__).resolve().parent + data_dir = Path(os.getenv("ORAL_TRAINER_DATA_DIR", str(package_dir / "data"))) + return cls( + data_dir=data_dir, + legacy_boundaries_file=Path( + os.getenv( + "SENTENCE_BOUNDARIES_FILE", + str(package_dir / "data" / "sentence_boundaries.json"), + ) + ), + admin_api_key=os.getenv("ADMIN_API_KEY", ""), + client_api_key=os.getenv("CLIENT_API_KEY", ""), + public_base_url=os.getenv("PUBLIC_BASE_URL", "").rstrip("/"), + max_upload_bytes=int(os.getenv("MAX_VIDEO_UPLOAD_BYTES", str(12 * 1024**3))), + max_attempt_bytes=int(os.getenv("MAX_ATTEMPT_UPLOAD_BYTES", str(50 * 1024**2))), + keep_attempt_audio=_bool_env("KEEP_ATTEMPT_AUDIO", False), + moss_transcribe_url=os.getenv("MOSS_TRANSCRIBE_URL", "").strip(), + moss_model=os.getenv( + "MOSS_MODEL", + "OpenMOSS-Team/MOSS-Transcribe-Diarize", + ), + moss_timeout_seconds=float(os.getenv("MOSS_TIMEOUT_SECONDS", "1800")), + moss_max_new_tokens=int(os.getenv("MOSS_MAX_NEW_TOKENS", "65536")), + pass_score=float(os.getenv("ASSESSMENT_PASS_SCORE", "70")), + ) + + @property + def videos_dir(self) -> Path: + return self.data_dir / "v" + + @property + def work_dir(self) -> Path: + return self.data_dir / "work" + + @property + def attempts_dir(self) -> Path: + return self.data_dir / "attempts" + + @property + def database_path(self) -> Path: + return self.data_dir / "oral_trainer.sqlite3" + + def ensure_directories(self) -> None: + for path in (self.data_dir, self.videos_dir, self.work_dir, self.attempts_dir): + path.mkdir(parents=True, exist_ok=True) diff --git a/sentence_api/docker-compose.yml b/sentence_api/docker-compose.yml new file mode 100644 index 0000000..aef7d55 --- /dev/null +++ b/sentence_api/docker-compose.yml @@ -0,0 +1,17 @@ +services: + oral-trainer-api: + build: + context: .. + dockerfile: sentence_api/Dockerfile + restart: unless-stopped + network_mode: host + env_file: + - .env + command: ["python", "-m", "uvicorn", "sentence_api.main:app", "--host", "127.0.0.1", "--port", "8000", "--workers", "1", "--proxy-headers", "--forwarded-allow-ips", "*"] + volumes: + - ./data:/app/sentence_api/data + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=5)"] + interval: 30s + timeout: 10s + retries: 3 diff --git a/sentence_api/main.py b/sentence_api/main.py index f139172..8899f2a 100644 --- a/sentence_api/main.py +++ b/sentence_api/main.py @@ -1,31 +1,161 @@ -import os +import hashlib +import hmac +import logging +import mimetypes +import uuid from pathlib import Path -from typing import Any, Dict +from typing import Any, BinaryIO, Dict, Optional -from fastapi import Depends, FastAPI, HTTPException, Path as ApiPath +from fastapi import ( + BackgroundTasks, + Depends, + FastAPI, + File, + Form, + Header, + HTTPException, + Path as ApiPath, + Query, + Request, + UploadFile, +) +from fastapi.responses import FileResponse, Response +from fastapi.staticfiles import StaticFiles +from starlette.concurrency import run_in_threadpool -from .models import SentenceBoundaryDocument +from .assessment import AssessmentService +from .audio_metrics import AudioAnalysisError +from .config import Settings +from .models import ( + AssessmentResult, + SentenceBoundary, + SentenceBoundaryDocument, + SentenceTextUpdate, + VideoDetailResponse, + VideoListResponse, + VideoSummary, + VideoUploadResponse, +) +from .processing import VideoProcessor +from .repository import VideoRepository from .store import BoundaryStore +from .transcription import MossTranscriber, Transcriber -DEFAULT_INDEX_PATH = Path(__file__).resolve().parent / "data" / "sentence_boundaries.json" +logger = logging.getLogger(__name__) +SHA256_PATH = ApiPath( + min_length=64, + max_length=64, + pattern=r"^[A-Fa-f0-9]{64}$", + description="SHA-256 hex digest of the exact uploaded video bytes", +) +ALLOWED_VIDEO_EXTENSIONS = {".mp4", ".m4v", ".mov", ".mkv", ".webm"} -def create_app(store: BoundaryStore = None) -> FastAPI: - application = FastAPI( - title="Oral Trainer Sentence Boundary API", - version="1.0.0", - description="Looks up pre-generated sentence boundaries by video SHA-256.", +class UploadTooLargeError(ValueError): + pass + + +def create_app( + store: Optional[BoundaryStore] = None, + *, + settings: Optional[Settings] = None, + repository: Optional[VideoRepository] = None, + transcriber: Optional[Transcriber] = None, +) -> FastAPI: + service_settings = settings or Settings.from_env() + service_settings.ensure_directories() + video_repository = repository or VideoRepository(service_settings.database_path) + legacy_store = store or BoundaryStore(service_settings.legacy_boundaries_file) + moss = transcriber or MossTranscriber( + endpoint=service_settings.moss_transcribe_url, + model=service_settings.moss_model, + timeout_seconds=service_settings.moss_timeout_seconds, + max_new_tokens=service_settings.moss_max_new_tokens, + ) + processor = VideoProcessor(service_settings, video_repository, moss) + assessment_service = AssessmentService( + video_repository, + moss, + service_settings.pass_score, ) - index_path = Path(os.getenv("SENTENCE_BOUNDARIES_FILE", str(DEFAULT_INDEX_PATH))) - application.state.boundary_store = store or BoundaryStore(index_path) - def get_store() -> BoundaryStore: - return application.state.boundary_store + application = FastAPI( + title="Oral Trainer Video Service", + version="2.0.0", + description="Video ingestion, sentence transcription, streaming, and oral-reading assessment.", + ) + application.state.settings = service_settings + application.state.boundary_store = legacy_store + application.state.video_repository = video_repository + application.state.transcriber = moss + application.state.video_processor = processor + application.state.assessment_service = assessment_service + + static_dir = Path(__file__).resolve().parent / "static" + if static_dir.is_dir(): + application.mount("/static", StaticFiles(directory=static_dir), name="static") + + def require_admin(x_admin_key: Optional[str] = Header(default=None)) -> None: + expected = service_settings.admin_api_key + if expected and not hmac.compare_digest(x_admin_key or "", expected): + raise HTTPException(status_code=401, detail="A valid X-Admin-Key header is required.") + + def require_client(x_client_key: Optional[str] = Header(default=None)) -> None: + expected = service_settings.client_api_key + if expected and not hmac.compare_digest(x_client_key or "", expected): + raise HTTPException(status_code=401, detail="A valid X-Client-Key header is required.") + + def find_document(video_hash: str) -> Optional[SentenceBoundaryDocument]: + return video_repository.get_document(video_hash) or legacy_store.get(video_hash) @application.get("/healthz") - def healthz(boundary_store: BoundaryStore = Depends(get_store)) -> Dict[str, Any]: - return {"status": "ok", "video_count": boundary_store.count()} + def healthz() -> Dict[str, Any]: + videos = video_repository.list_videos() + return { + "status": "ok", + "video_count": len(videos) + legacy_store.count(), + "managed_video_count": len(videos), + "moss_configured": moss.available, + "scoring_version": "asr-fluency-v1", + } + + @application.get("/admin", include_in_schema=False) + def admin_page() -> FileResponse: + page = static_dir / "admin.html" + if not page.is_file(): + raise HTTPException(status_code=404, detail="Admin UI is not installed.") + return FileResponse(page) + + @application.get("/api/v1/videos", response_model=VideoListResponse) + def list_videos() -> VideoListResponse: + return VideoListResponse( + videos=[_video_summary(row, service_settings) for row in video_repository.list_videos()] + ) + + @application.get("/api/v1/videos/{video_hash}", response_model=VideoDetailResponse) + def get_video(video_hash: str = SHA256_PATH) -> VideoDetailResponse: + row = video_repository.get_video(video_hash) + if row is None: + raise HTTPException(status_code=404, detail="Video was not found.") + return VideoDetailResponse( + video=_video_summary(row, service_settings), + boundaries=video_repository.get_document(video_hash), + ) + + @application.get("/api/v1/videos/{video_hash}/content") + def stream_video(video_hash: str = SHA256_PATH) -> FileResponse: + row = video_repository.get_video(video_hash) + if row is None: + raise HTTPException(status_code=404, detail="Video was not found.") + media_path = service_settings.videos_dir / row["stored_filename"] + if not media_path.is_file(): + raise HTTPException(status_code=404, detail="Stored video file is missing.") + return FileResponse( + media_path, + media_type=row["content_type"], + headers={"Accept-Ranges": "bytes", "Cache-Control": "public, max-age=3600"}, + ) @application.get( "/api/v1/videos/{video_hash}/sentence-boundaries", @@ -36,16 +166,8 @@ def create_app(store: BoundaryStore = None) -> FastAPI: response_model=SentenceBoundaryDocument, include_in_schema=False, ) - def get_sentence_boundaries( - video_hash: str = ApiPath( - min_length=64, - max_length=64, - pattern=r"^[A-Fa-f0-9]{64}$", - description="SHA-256 hex digest of the exact video bytes", - ), - boundary_store: BoundaryStore = Depends(get_store), - ) -> SentenceBoundaryDocument: - document = boundary_store.get(video_hash) + def get_sentence_boundaries(video_hash: str = SHA256_PATH) -> SentenceBoundaryDocument: + document = find_document(video_hash) if document is None: raise HTTPException( status_code=404, @@ -56,7 +178,321 @@ def create_app(store: BoundaryStore = None) -> FastAPI: ) return document + @application.post( + "/api/v1/admin/videos", + response_model=VideoUploadResponse, + status_code=202, + dependencies=[Depends(require_admin)], + ) + async def upload_video( + background_tasks: BackgroundTasks, + file: UploadFile = File(...), + title: Optional[str] = Form(default=None), + language: Optional[str] = Form(default=None), + ) -> VideoUploadResponse: + original_name = Path(file.filename or "video.mp4").name + suffix = _validate_video_extension(original_name) + temporary_path = service_settings.work_dir / f"upload-{uuid.uuid4().hex}.part" + try: + video_hash, size_bytes = await run_in_threadpool( + _persist_upload, + file.file, + temporary_path, + service_settings.max_upload_bytes, + ) + except UploadTooLargeError as exc: + temporary_path.unlink(missing_ok=True) + raise HTTPException(status_code=413, detail=str(exc)) from exc + except ValueError as exc: + temporary_path.unlink(missing_ok=True) + raise HTTPException(status_code=400, detail=str(exc)) from exc + finally: + await file.close() + + content_type = file.content_type or mimetypes.guess_type(original_name)[0] or "video/mp4" + response = _register_uploaded_video( + settings=service_settings, + repository=video_repository, + temporary_path=temporary_path, + video_hash=video_hash, + size_bytes=size_bytes, + original_name=original_name, + suffix=suffix, + content_type=content_type, + title=title, + language=language, + ) + background_tasks.add_task(_process_safely, processor, video_hash) + return response + + @application.put( + "/api/v1/admin/videos/raw", + response_model=VideoUploadResponse, + status_code=202, + dependencies=[Depends(require_admin)], + ) + async def upload_video_stream( + request: Request, + background_tasks: BackgroundTasks, + filename: str = Query(min_length=1, max_length=500), + title: Optional[str] = Query(default=None, max_length=200), + language: Optional[str] = Query(default=None, max_length=32), + ) -> VideoUploadResponse: + original_name = Path(filename).name + suffix = _validate_video_extension(original_name) + content_length = request.headers.get("content-length") + if content_length: + try: + declared_size = int(content_length) + except ValueError as exc: + raise HTTPException(status_code=400, detail="Invalid Content-Length header.") from exc + if declared_size > service_settings.max_upload_bytes: + raise HTTPException(status_code=413, detail="Upload exceeds the configured limit.") + temporary_path = service_settings.work_dir / f"upload-{uuid.uuid4().hex}.part" + try: + video_hash, size_bytes = await _persist_request_stream( + request, + temporary_path, + service_settings.max_upload_bytes, + ) + except UploadTooLargeError as exc: + raise HTTPException(status_code=413, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + response = _register_uploaded_video( + settings=service_settings, + repository=video_repository, + temporary_path=temporary_path, + video_hash=video_hash, + size_bytes=size_bytes, + original_name=original_name, + suffix=suffix, + content_type=request.headers.get("content-type") or "application/octet-stream", + title=title, + language=language, + ) + background_tasks.add_task(_process_safely, processor, video_hash) + return response + + @application.post( + "/api/v1/admin/videos/{video_hash}/process", + status_code=202, + dependencies=[Depends(require_admin)], + ) + def reprocess_video( + background_tasks: BackgroundTasks, + video_hash: str = SHA256_PATH, + ) -> Dict[str, str]: + if video_repository.get_video(video_hash) is None: + raise HTTPException(status_code=404, detail="Video was not found.") + background_tasks.add_task(_process_safely, processor, video_hash) + return {"video_hash": video_hash.lower(), "status": "processing_queued"} + + @application.put( + "/api/v1/admin/videos/{video_hash}/sentences/{sentence_index}", + response_model=SentenceBoundary, + dependencies=[Depends(require_admin)], + ) + def update_sentence( + payload: SentenceTextUpdate, + video_hash: str = SHA256_PATH, + sentence_index: int = ApiPath(ge=0), + ) -> SentenceBoundary: + sentence = video_repository.update_sentence_text( + video_hash, + sentence_index, + payload.text, + payload.language, + ) + if sentence is None: + raise HTTPException(status_code=404, detail="Sentence was not found.") + return sentence + + @application.delete( + "/api/v1/admin/videos/{video_hash}", + status_code=204, + dependencies=[Depends(require_admin)], + ) + def delete_video(video_hash: str = SHA256_PATH) -> Response: + row = video_repository.delete_video(video_hash) + if row is None: + raise HTTPException(status_code=404, detail="Video was not found.") + (service_settings.videos_dir / row["stored_filename"]).unlink(missing_ok=True) + return Response(status_code=204) + + @application.post( + "/api/v1/videos/{video_hash}/sentences/{sentence_index}/assessments", + response_model=AssessmentResult, + dependencies=[Depends(require_client)], + ) + async def assess_sentence( + video_hash: str = SHA256_PATH, + sentence_index: int = ApiPath(ge=0), + audio: UploadFile = File(...), + language: Optional[str] = Form(default=None), + ) -> AssessmentResult: + document = find_document(video_hash) + if document is None: + raise HTTPException(status_code=404, detail="Video or sentence boundaries were not found.") + if not moss.available: + raise HTTPException(status_code=503, detail="MOSS transcription is not configured.") + + suffix = Path(audio.filename or "attempt.wav").suffix.lower() or ".wav" + temporary_name = f"attempt-{uuid.uuid4().hex}{suffix}" + temporary_path = service_settings.attempts_dir / temporary_name + completed = False + try: + await run_in_threadpool( + _persist_upload, + audio.file, + temporary_path, + service_settings.max_attempt_bytes, + ) + result = await run_in_threadpool( + assessment_service.assess, + document=document, + sentence_index=sentence_index, + audio_path=temporary_path, + language=language, + retained_audio_filename=temporary_name if service_settings.keep_attempt_audio else None, + ) + completed = True + return result + except UploadTooLargeError as exc: + raise HTTPException(status_code=413, detail=str(exc)) from exc + except LookupError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except (ValueError, AudioAnalysisError) as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(status_code=502, detail=str(exc)) from exc + finally: + await audio.close() + if not service_settings.keep_attempt_audio or not completed: + temporary_path.unlink(missing_ok=True) + return application +def _persist_upload(source: BinaryIO, destination: Path, max_bytes: int) -> tuple[str, int]: + digest = hashlib.sha256() + size = 0 + destination.parent.mkdir(parents=True, exist_ok=True) + try: + with destination.open("wb") as output: + while True: + chunk = source.read(4 * 1024 * 1024) + if not chunk: + break + size += len(chunk) + if size > max_bytes: + raise UploadTooLargeError(f"Upload exceeds the {max_bytes}-byte limit.") + digest.update(chunk) + output.write(chunk) + if size == 0: + raise ValueError("Uploaded file is empty.") + except Exception: + destination.unlink(missing_ok=True) + raise + return digest.hexdigest(), size + + +async def _persist_request_stream( + request: Request, + destination: Path, + max_bytes: int, +) -> tuple[str, int]: + digest = hashlib.sha256() + size = 0 + destination.parent.mkdir(parents=True, exist_ok=True) + try: + with destination.open("wb") as output: + async for chunk in request.stream(): + if not chunk: + continue + size += len(chunk) + if size > max_bytes: + raise UploadTooLargeError(f"Upload exceeds the {max_bytes}-byte limit.") + digest.update(chunk) + output.write(chunk) + if size == 0: + raise ValueError("Uploaded file is empty.") + except Exception: + destination.unlink(missing_ok=True) + raise + return digest.hexdigest(), size + + +def _validate_video_extension(filename: str) -> str: + suffix = Path(filename).suffix.lower() + if suffix not in ALLOWED_VIDEO_EXTENSIONS: + raise HTTPException( + status_code=415, + detail=f"Unsupported video extension: {suffix or '(none)'}", + ) + return suffix + + +def _register_uploaded_video( + *, + settings: Settings, + repository: VideoRepository, + temporary_path: Path, + video_hash: str, + size_bytes: int, + original_name: str, + suffix: str, + content_type: str, + title: Optional[str], + language: Optional[str], +) -> VideoUploadResponse: + stored_filename = f"{video_hash}{suffix}" + destination = settings.videos_dir / stored_filename + if destination.exists(): + temporary_path.unlink(missing_ok=True) + else: + temporary_path.replace(destination) + repository.upsert_upload( + video_hash=video_hash, + title=(title or Path(original_name).stem).strip() or video_hash, + filename=original_name, + stored_filename=stored_filename, + content_type=content_type, + size_bytes=size_bytes, + language=language.strip() if language else None, + ) + return VideoUploadResponse( + video_hash=video_hash, + status="uploaded", + detail_url=f"/api/v1/videos/{video_hash}", + ) + + +def _process_safely(processor: VideoProcessor, video_hash: str) -> None: + try: + processor.process(video_hash) + except Exception: + logger.exception("Video processing failed for %s", video_hash) + + +def _video_summary(row: Dict[str, Any], settings: Settings) -> VideoSummary: + relative_stream_url = f"/api/v1/videos/{row['video_hash']}/content" + stream_url = f"{settings.public_base_url}{relative_stream_url}" if settings.public_base_url else relative_stream_url + return VideoSummary( + video_hash=row["video_hash"], + title=row["title"], + filename=row["filename"], + content_type=row["content_type"], + size_bytes=row["size_bytes"], + duration_ms=row["duration_ms"], + language=row["language"], + status=row["status"], + error_message=row["error_message"], + sentence_count=row["sentence_count"], + stream_url=stream_url, + created_at=row["created_at"], + updated_at=row["updated_at"], + ) + + app = create_app() diff --git a/sentence_api/models.py b/sentence_api/models.py index 095e738..0dd7bbc 100644 --- a/sentence_api/models.py +++ b/sentence_api/models.py @@ -1,4 +1,5 @@ -from typing import List, Optional +from datetime import datetime +from typing import Dict, List, Literal, Optional from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator @@ -10,6 +11,8 @@ class SentenceBoundary(BaseModel): start_ms: int = Field(ge=0) end_ms: int = Field(gt=0) text: Optional[str] = None + language: Optional[str] = None + reference_speech_duration_ms: Optional[int] = Field(default=None, gt=0) @model_validator(mode="after") def validate_range(self): @@ -46,3 +49,79 @@ class SentenceBoundaryDocument(BaseModel): raise ValueError("sentence end_ms cannot exceed duration_ms") previous_end = sentence.end_ms return self + + +VideoStatus = Literal["uploaded", "processing", "ready", "failed"] + + +class VideoSummary(BaseModel): + model_config = ConfigDict(extra="forbid") + + video_hash: str + title: str + filename: str + content_type: str + size_bytes: int = Field(ge=0) + duration_ms: Optional[int] = Field(default=None, gt=0) + language: Optional[str] = None + status: VideoStatus + error_message: Optional[str] = None + sentence_count: int = Field(ge=0) + stream_url: str + created_at: datetime + updated_at: datetime + + +class VideoListResponse(BaseModel): + videos: List[VideoSummary] + + +class VideoUploadResponse(BaseModel): + video_hash: str + status: VideoStatus + detail_url: str + + +class VideoDetailResponse(BaseModel): + video: VideoSummary + boundaries: Optional[SentenceBoundaryDocument] = None + + +class SentenceTextUpdate(BaseModel): + text: str = Field(min_length=1, max_length=4000) + language: Optional[str] = Field(default=None, max_length=32) + + +class TextSubstitution(BaseModel): + expected: str + actual: str + + +class AssessmentResult(BaseModel): + model_config = ConfigDict(extra="forbid") + + attempt_id: str + scoring_version: str + overall_score: float = Field(ge=0, le=100) + passed: bool + pass_score: float = Field(ge=0, le=100) + content_score: float = Field(ge=0, le=100) + completeness_score: float = Field(ge=0, le=100) + fluency_score: float = Field(ge=0, le=100) + pronunciation_score: Optional[float] = Field(default=None, ge=0, le=100) + prosody_score: Optional[float] = Field(default=None, ge=0, le=100) + duration_score: float = Field(ge=0, le=100) + pause_score: float = Field(ge=0, le=100) + speech_rate_score: float = Field(ge=0, le=100) + reference_text: str + recognized_text: str + reference_duration_ms: int = Field(gt=0) + reference_speech_duration_ms: int = Field(gt=0) + student_recording_duration_ms: int = Field(gt=0) + student_speech_duration_ms: int = Field(gt=0) + duration_ratio: float = Field(gt=0) + missing_tokens: List[str] + extra_tokens: List[str] + substitutions: List[TextSubstitution] + feedback: str + details: Dict[str, str] = Field(default_factory=dict) diff --git a/sentence_api/nginx.conf.example b/sentence_api/nginx.conf.example new file mode 100644 index 0000000..8f661ea --- /dev/null +++ b/sentence_api/nginx.conf.example @@ -0,0 +1,30 @@ +server { + listen 80; + server_name video_service.d1kt.cn; + + client_max_body_size 12g; + client_body_timeout 7200s; + send_timeout 7200s; + + location / { + proxy_pass http://127.0.0.1:8000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_request_buffering off; + proxy_read_timeout 7200s; + proxy_send_timeout 7200s; + } + + location ~ ^/api/v1/videos/[0-9a-fA-F]{64}/content$ { + proxy_pass http://127.0.0.1:8000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header Range $http_range; + proxy_set_header If-Range $http_if_range; + proxy_force_ranges on; + proxy_buffering off; + } +} diff --git a/sentence_api/processing.py b/sentence_api/processing.py new file mode 100644 index 0000000..2bfee26 --- /dev/null +++ b/sentence_api/processing.py @@ -0,0 +1,144 @@ +import shutil +import subprocess +import uuid +from pathlib import Path +from typing import List, Optional + +from .audio_metrics import AudioAnalysisError, analyze_samples, decode_audio_mono +from .config import Settings +from .generate_boundaries import ALGORITHM_VERSION, make_entry +from .models import SentenceBoundary, SentenceBoundaryDocument +from .repository import VideoRepository +from .transcription import Transcript, Transcriber + + +MOSS_ALGORITHM_VERSION = "moss-timestamp-v1" + + +class VideoProcessor: + def __init__( + self, + settings: Settings, + repository: VideoRepository, + transcriber: Transcriber, + ): + self.settings = settings + self.repository = repository + self.transcriber = transcriber + + def process(self, video_hash: str) -> None: + video = self.repository.get_video(video_hash) + if video is None: + raise ValueError(f"Unknown video: {video_hash}") + media_path = self.settings.videos_dir / video["stored_filename"] + if not media_path.is_file(): + raise FileNotFoundError(f"Stored video is missing: {media_path.name}") + + self.repository.mark_processing(video_hash) + work_path: Optional[Path] = None + try: + if self.transcriber.available: + work_path = self.settings.work_dir / f"{video_hash}-{uuid.uuid4().hex}.wav" + extract_audio(media_path, work_path) + transcript = self.transcriber.transcribe(work_path, video.get("language")) + document = document_from_transcript( + video_hash=video_hash, + duration_ms=_media_duration_ms(media_path), + transcript=transcript, + language=video.get("language"), + audio_path=work_path, + ) + if not document.sentences: + raise RuntimeError("MOSS returned no timestamped speech segments.") + self.repository.save_processing_result(document, transcript.text) + else: + entry, _ = make_entry(media_path, video_hash=video_hash) + document = SentenceBoundaryDocument( + video_hash=video_hash, + duration_ms=entry["duration_ms"], + algorithm_version=ALGORITHM_VERSION, + sentences=entry["sentences"], + ) + self.repository.save_processing_result(document, None) + except Exception as exc: + self.repository.mark_failed(video_hash, str(exc)) + raise + finally: + if work_path is not None: + work_path.unlink(missing_ok=True) + + +def extract_audio(media_path: Path, output_path: Path) -> None: + ffmpeg = shutil.which("ffmpeg") + if ffmpeg is None: + raise RuntimeError("ffmpeg is required for MOSS transcription but was not found.") + output_path.parent.mkdir(parents=True, exist_ok=True) + command = [ + ffmpeg, + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + str(media_path), + "-vn", + "-ac", + "1", + "-ar", + "16000", + "-c:a", + "pcm_s16le", + str(output_path), + ] + completed = subprocess.run(command, capture_output=True, text=True, timeout=7200) + if completed.returncode != 0: + message = completed.stderr.strip() or "unknown ffmpeg error" + raise RuntimeError(f"Could not extract video audio: {message[-2000:]}") + + +def document_from_transcript( + *, + video_hash: str, + duration_ms: int, + transcript: Transcript, + language: Optional[str], + audio_path: Path, +) -> SentenceBoundaryDocument: + samples, sample_rate = decode_audio_mono(audio_path) + sentences: List[SentenceBoundary] = [] + previous_end = 0 + for segment in sorted(transcript.segments, key=lambda item: (item.start_seconds, item.end_seconds)): + start_ms = max(previous_end, int(round(segment.start_seconds * 1000))) + end_ms = min(duration_ms, int(round(segment.end_seconds * 1000))) + if not segment.text.strip() or end_ms <= start_ms: + continue + start_sample = max(0, int(start_ms / 1000 * sample_rate)) + end_sample = min(samples.size, int(end_ms / 1000 * sample_rate)) + try: + metrics = analyze_samples(samples[start_sample:end_sample], sample_rate) + speech_duration_ms = metrics.speech_duration_ms + except AudioAnalysisError: + speech_duration_ms = end_ms - start_ms + sentences.append( + SentenceBoundary( + index=len(sentences), + start_ms=start_ms, + end_ms=end_ms, + text=segment.text.strip(), + language=language, + reference_speech_duration_ms=max(1, speech_duration_ms), + ) + ) + previous_end = end_ms + return SentenceBoundaryDocument( + video_hash=video_hash, + duration_ms=duration_ms, + algorithm_version=MOSS_ALGORITHM_VERSION, + sentences=sentences, + ) + + +def _media_duration_ms(path: Path) -> int: + from .generate_boundaries import media_duration_ms + + return media_duration_ms(path) diff --git a/sentence_api/repository.py b/sentence_api/repository.py new file mode 100644 index 0000000..c41b4e6 --- /dev/null +++ b/sentence_api/repository.py @@ -0,0 +1,305 @@ +import json +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +from .models import SentenceBoundary, SentenceBoundaryDocument +from .store import normalize_video_hash + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +class VideoRepository: + def __init__(self, database_path: Path): + self.database_path = Path(database_path) + self.database_path.parent.mkdir(parents=True, exist_ok=True) + self._initialize() + + def _connect(self) -> sqlite3.Connection: + connection = sqlite3.connect(self.database_path, timeout=30) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + connection.execute("PRAGMA journal_mode = WAL") + return connection + + def _initialize(self) -> None: + with self._connect() as connection: + connection.executescript( + """ + CREATE TABLE IF NOT EXISTS videos ( + video_hash TEXT PRIMARY KEY, + title TEXT NOT NULL, + filename TEXT NOT NULL, + stored_filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + duration_ms INTEGER, + language TEXT, + status TEXT NOT NULL, + error_message TEXT, + algorithm_version TEXT, + transcription TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS sentences ( + video_hash TEXT NOT NULL, + sentence_index INTEGER NOT NULL, + start_ms INTEGER NOT NULL, + end_ms INTEGER NOT NULL, + text TEXT, + language TEXT, + reference_speech_duration_ms INTEGER, + PRIMARY KEY (video_hash, sentence_index), + FOREIGN KEY (video_hash) REFERENCES videos(video_hash) ON DELETE CASCADE + ); + + CREATE TABLE IF NOT EXISTS attempts ( + attempt_id TEXT PRIMARY KEY, + video_hash TEXT NOT NULL, + sentence_index INTEGER NOT NULL, + result_json TEXT NOT NULL, + audio_filename TEXT, + created_at TEXT NOT NULL, + FOREIGN KEY (video_hash) REFERENCES videos(video_hash) ON DELETE CASCADE + ); + """ + ) + + def upsert_upload( + self, + *, + video_hash: str, + title: str, + filename: str, + stored_filename: str, + content_type: str, + size_bytes: int, + language: Optional[str], + ) -> None: + normalized_hash = normalize_video_hash(video_hash) + now = utc_now() + with self._connect() as connection: + existing = connection.execute( + "SELECT created_at FROM videos WHERE video_hash = ?", + (normalized_hash,), + ).fetchone() + created_at = existing["created_at"] if existing else now + connection.execute( + """ + INSERT INTO videos ( + video_hash, title, filename, stored_filename, content_type, + size_bytes, duration_ms, language, status, error_message, + algorithm_version, transcription, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, NULL, ?, 'uploaded', NULL, NULL, NULL, ?, ?) + ON CONFLICT(video_hash) DO UPDATE SET + title = excluded.title, + filename = excluded.filename, + stored_filename = excluded.stored_filename, + content_type = excluded.content_type, + size_bytes = excluded.size_bytes, + language = excluded.language, + status = 'uploaded', + error_message = NULL, + updated_at = excluded.updated_at + """, + ( + normalized_hash, + title, + filename, + stored_filename, + content_type, + size_bytes, + language, + created_at, + now, + ), + ) + + def mark_processing(self, video_hash: str) -> None: + self._update_status(video_hash, "processing", None) + + def mark_failed(self, video_hash: str, message: str) -> None: + self._update_status(video_hash, "failed", message[:4000]) + + def _update_status(self, video_hash: str, status: str, error_message: Optional[str]) -> None: + with self._connect() as connection: + connection.execute( + "UPDATE videos SET status = ?, error_message = ?, updated_at = ? WHERE video_hash = ?", + (status, error_message, utc_now(), normalize_video_hash(video_hash)), + ) + + def save_processing_result( + self, + document: SentenceBoundaryDocument, + transcription: Optional[str], + ) -> None: + with self._connect() as connection: + connection.execute( + "DELETE FROM sentences WHERE video_hash = ?", + (document.video_hash,), + ) + connection.executemany( + """ + INSERT INTO sentences ( + video_hash, sentence_index, start_ms, end_ms, text, + language, reference_speech_duration_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?) + """, + [ + ( + document.video_hash, + sentence.index, + sentence.start_ms, + sentence.end_ms, + sentence.text, + sentence.language, + sentence.reference_speech_duration_ms, + ) + for sentence in document.sentences + ], + ) + connection.execute( + """ + UPDATE videos SET duration_ms = ?, status = 'ready', error_message = NULL, + algorithm_version = ?, transcription = ?, updated_at = ? + WHERE video_hash = ? + """, + ( + document.duration_ms, + document.algorithm_version, + transcription, + utc_now(), + document.video_hash, + ), + ) + + def list_videos(self) -> List[Dict[str, Any]]: + with self._connect() as connection: + rows = connection.execute( + """ + SELECT v.*, COUNT(s.sentence_index) AS sentence_count + FROM videos v + LEFT JOIN sentences s ON s.video_hash = v.video_hash + GROUP BY v.video_hash + ORDER BY v.created_at DESC + """ + ).fetchall() + return [dict(row) for row in rows] + + def get_video(self, video_hash: str) -> Optional[Dict[str, Any]]: + with self._connect() as connection: + row = connection.execute( + """ + SELECT v.*, COUNT(s.sentence_index) AS sentence_count + FROM videos v + LEFT JOIN sentences s ON s.video_hash = v.video_hash + WHERE v.video_hash = ? + GROUP BY v.video_hash + """, + (normalize_video_hash(video_hash),), + ).fetchone() + return dict(row) if row else None + + def get_document(self, video_hash: str) -> Optional[SentenceBoundaryDocument]: + video = self.get_video(video_hash) + if video is None or video["duration_ms"] is None: + return None + with self._connect() as connection: + rows = connection.execute( + "SELECT * FROM sentences WHERE video_hash = ? ORDER BY sentence_index", + (normalize_video_hash(video_hash),), + ).fetchall() + return SentenceBoundaryDocument( + video_hash=video["video_hash"], + duration_ms=video["duration_ms"], + algorithm_version=video["algorithm_version"] or "unknown", + sentences=[ + SentenceBoundary( + index=row["sentence_index"], + start_ms=row["start_ms"], + end_ms=row["end_ms"], + text=row["text"], + language=row["language"], + reference_speech_duration_ms=row["reference_speech_duration_ms"], + ) + for row in rows + ], + ) + + def update_sentence_text( + self, + video_hash: str, + sentence_index: int, + text: str, + language: Optional[str], + ) -> Optional[SentenceBoundary]: + normalized_hash = normalize_video_hash(video_hash) + with self._connect() as connection: + cursor = connection.execute( + """ + UPDATE sentences SET text = ?, language = COALESCE(?, language) + WHERE video_hash = ? AND sentence_index = ? + """, + (text.strip(), language, normalized_hash, sentence_index), + ) + if cursor.rowcount == 0: + return None + connection.execute( + "UPDATE videos SET updated_at = ? WHERE video_hash = ?", + (utc_now(), normalized_hash), + ) + row = connection.execute( + "SELECT * FROM sentences WHERE video_hash = ? AND sentence_index = ?", + (normalized_hash, sentence_index), + ).fetchone() + return SentenceBoundary( + index=row["sentence_index"], + start_ms=row["start_ms"], + end_ms=row["end_ms"], + text=row["text"], + language=row["language"], + reference_speech_duration_ms=row["reference_speech_duration_ms"], + ) + + def record_attempt( + self, + *, + attempt_id: str, + video_hash: str, + sentence_index: int, + result: Dict[str, Any], + audio_filename: Optional[str], + ) -> None: + with self._connect() as connection: + connection.execute( + """ + INSERT INTO attempts ( + attempt_id, video_hash, sentence_index, result_json, + audio_filename, created_at + ) VALUES (?, ?, ?, ?, ?, ?) + """, + ( + attempt_id, + normalize_video_hash(video_hash), + sentence_index, + json.dumps(result, ensure_ascii=False), + audio_filename, + utc_now(), + ), + ) + + def delete_video(self, video_hash: str) -> Optional[Dict[str, Any]]: + video = self.get_video(video_hash) + if video is None: + return None + with self._connect() as connection: + connection.execute( + "DELETE FROM videos WHERE video_hash = ?", + (normalize_video_hash(video_hash),), + ) + return video diff --git a/sentence_api/requirements.txt b/sentence_api/requirements.txt index 6f55850..a310817 100644 --- a/sentence_api/requirements.txt +++ b/sentence_api/requirements.txt @@ -3,5 +3,6 @@ uvicorn[standard]>=0.30,<1 pydantic>=2.7,<3 pytest>=8,<9 httpx>=0.27,<1 +python-multipart>=0.0.9,<1 av>=12.0 numpy>=1.26 diff --git a/sentence_api/scoring.py b/sentence_api/scoring.py new file mode 100644 index 0000000..f9a7423 --- /dev/null +++ b/sentence_api/scoring.py @@ -0,0 +1,241 @@ +import math +import unicodedata +from dataclasses import dataclass +from typing import List, Sequence, Tuple + +from .audio_metrics import AudioMetrics + + +SCORING_VERSION = "asr-fluency-v1" + + +@dataclass(frozen=True) +class TextAlignment: + reference_tokens: List[str] + recognized_tokens: List[str] + matches: int + missing_tokens: List[str] + extra_tokens: List[str] + substitutions: List[Tuple[str, str]] + content_score: float + completeness_score: float + + +@dataclass(frozen=True) +class ScoreBreakdown: + overall_score: float + content_score: float + completeness_score: float + fluency_score: float + duration_score: float + pause_score: float + speech_rate_score: float + duration_ratio: float + missing_tokens: List[str] + extra_tokens: List[str] + substitutions: List[Tuple[str, str]] + feedback: str + + +def tokenize(text: str) -> List[str]: + normalized = unicodedata.normalize("NFKC", text).lower().strip() + tokens: List[str] = [] + word: List[str] = [] + + def flush_word() -> None: + if word: + token = "".join(word).strip("'") + if token: + tokens.append(token) + word.clear() + + for char in normalized: + if _is_cjk(char): + flush_word() + tokens.append(char) + elif char.isalnum() or (char == "'" and word): + word.append(char) + else: + flush_word() + flush_word() + return tokens + + +def align_text(reference_text: str, recognized_text: str) -> TextAlignment: + reference = tokenize(reference_text) + recognized = tokenize(recognized_text) + if not reference: + raise ValueError("Reference text does not contain any scoreable tokens.") + + costs = [[0] * (len(recognized) + 1) for _ in range(len(reference) + 1)] + for row in range(1, len(reference) + 1): + costs[row][0] = row + for column in range(1, len(recognized) + 1): + costs[0][column] = column + for row in range(1, len(reference) + 1): + for column in range(1, len(recognized) + 1): + substitution_cost = 0 if reference[row - 1] == recognized[column - 1] else 1 + costs[row][column] = min( + costs[row - 1][column] + 1, + costs[row][column - 1] + 1, + costs[row - 1][column - 1] + substitution_cost, + ) + + row = len(reference) + column = len(recognized) + matches = 0 + missing: List[str] = [] + extra: List[str] = [] + substitutions: List[Tuple[str, str]] = [] + while row > 0 or column > 0: + if row > 0 and column > 0: + same = reference[row - 1] == recognized[column - 1] + diagonal_cost = costs[row - 1][column - 1] + (0 if same else 1) + if costs[row][column] == diagonal_cost: + if same: + matches += 1 + else: + substitutions.append((reference[row - 1], recognized[column - 1])) + row -= 1 + column -= 1 + continue + if row > 0 and costs[row][column] == costs[row - 1][column] + 1: + missing.append(reference[row - 1]) + row -= 1 + else: + extra.append(recognized[column - 1]) + column -= 1 + + missing.reverse() + extra.reverse() + substitutions.reverse() + distance = len(missing) + len(extra) + len(substitutions) + accuracy = max(0.0, 1.0 - distance / len(reference)) * 100 + attempted_reference_tokens = matches + len(substitutions) + completeness = attempted_reference_tokens / len(reference) * 100 + content = accuracy * 0.7 + completeness * 0.3 + return TextAlignment( + reference_tokens=reference, + recognized_tokens=recognized, + matches=matches, + missing_tokens=missing, + extra_tokens=extra, + substitutions=substitutions, + content_score=_round_score(content), + completeness_score=_round_score(completeness), + ) + + +def score_attempt( + *, + reference_text: str, + recognized_text: str, + reference_speech_duration_ms: int, + student_metrics: AudioMetrics, +) -> ScoreBreakdown: + if reference_speech_duration_ms <= 0: + raise ValueError("reference_speech_duration_ms must be positive") + alignment = align_text(reference_text, recognized_text) + ratio = student_metrics.speech_duration_ms / reference_speech_duration_ms + duration = duration_similarity_score(ratio) + pause = pause_score(student_metrics.internal_pause_ratio) + + reference_rate = len(alignment.reference_tokens) / (reference_speech_duration_ms / 1000) + recognized_units = max(1, len(alignment.recognized_tokens)) + student_rate = recognized_units / (student_metrics.speech_duration_ms / 1000) + rate_ratio = student_rate / reference_rate if reference_rate > 0 else 1.0 + rate = symmetric_rate_score(rate_ratio) + + fluency = duration * 0.35 + pause * 0.40 + rate * 0.25 + overall = alignment.content_score * 0.80 + fluency * 0.20 + feedback = build_feedback(alignment, ratio, student_metrics.internal_pause_ratio) + return ScoreBreakdown( + overall_score=_round_score(overall), + content_score=alignment.content_score, + completeness_score=alignment.completeness_score, + fluency_score=_round_score(fluency), + duration_score=_round_score(duration), + pause_score=_round_score(pause), + speech_rate_score=_round_score(rate), + duration_ratio=round(ratio, 4), + missing_tokens=alignment.missing_tokens, + extra_tokens=alignment.extra_tokens, + substitutions=alignment.substitutions, + feedback=feedback, + ) + + +def duration_similarity_score(ratio: float) -> float: + if ratio <= 0: + return 0.0 + if 0.80 <= ratio <= 1.30: + return 100.0 + if 0.65 <= ratio < 0.80: + return _interpolate(ratio, 0.65, 0.80, 60, 100) + if 1.30 < ratio <= 1.50: + return _interpolate(ratio, 1.30, 1.50, 100, 60) + if 0.50 <= ratio < 0.65: + return _interpolate(ratio, 0.50, 0.65, 20, 60) + if 1.50 < ratio <= 1.80: + return _interpolate(ratio, 1.50, 1.80, 60, 20) + if ratio < 0.50: + return max(0.0, ratio / 0.50 * 20) + return max(0.0, 20 - (ratio - 1.80) / 0.40 * 20) + + +def pause_score(internal_pause_ratio: float) -> float: + if internal_pause_ratio <= 0.15: + return 100.0 + if internal_pause_ratio <= 0.30: + return _interpolate(internal_pause_ratio, 0.15, 0.30, 100, 70) + if internal_pause_ratio <= 0.50: + return _interpolate(internal_pause_ratio, 0.30, 0.50, 70, 20) + return max(0.0, 20 - (internal_pause_ratio - 0.50) / 0.30 * 20) + + +def symmetric_rate_score(ratio: float) -> float: + if ratio <= 0: + return 0.0 + deviation = abs(math.log(ratio)) + free_tolerance = math.log(1.20) + if deviation <= free_tolerance: + return 100.0 + return max(0.0, 100 - (deviation - free_tolerance) / math.log(2.5) * 100) + + +def build_feedback( + alignment: TextAlignment, + duration_ratio: float, + internal_pause_ratio: float, +) -> str: + messages = [] + if alignment.missing_tokens: + messages.append("存在漏读") + if alignment.substitutions: + messages.append("存在错读") + if alignment.extra_tokens: + messages.append("存在多读") + if duration_ratio < 0.65: + messages.append("朗读明显偏快或内容不完整") + elif duration_ratio > 1.50: + messages.append("朗读速度偏慢") + if internal_pause_ratio > 0.30: + messages.append("句内停顿偏多") + return ";".join(messages) if messages else "内容和朗读节奏匹配良好" + + +def _is_cjk(char: str) -> bool: + codepoint = ord(char) + return ( + 0x3400 <= codepoint <= 0x4DBF + or 0x4E00 <= codepoint <= 0x9FFF + or 0xF900 <= codepoint <= 0xFAFF + ) + + +def _interpolate(value: float, start: float, end: float, start_score: float, end_score: float) -> float: + return start_score + (value - start) / (end - start) * (end_score - start_score) + + +def _round_score(value: float) -> float: + return round(min(100.0, max(0.0, value)), 1) diff --git a/sentence_api/static/admin.css b/sentence_api/static/admin.css new file mode 100644 index 0000000..648e737 --- /dev/null +++ b/sentence_api/static/admin.css @@ -0,0 +1,117 @@ +:root { + color-scheme: light; + font-family: Inter, "PingFang SC", "Microsoft YaHei", sans-serif; + color: #17202a; + background: #f4f6f7; + letter-spacing: 0; +} + +* { box-sizing: border-box; } +body { margin: 0; min-width: 320px; } +button, input, select { font: inherit; letter-spacing: 0; } +button { cursor: pointer; } + +.topbar { + min-height: 72px; + padding: 14px 28px; + display: flex; + align-items: center; + justify-content: space-between; + gap: 24px; + color: #fff; + background: #20262c; + border-bottom: 3px solid #1f8a70; +} + +h1, h2 { margin: 0; letter-spacing: 0; } +h1 { font-size: 21px; line-height: 1.35; } +h2 { font-size: 17px; } +.service-state, .section-heading span { font-size: 13px; color: #66727d; } +.topbar .service-state { color: #c8d0d7; } +.admin-auth { display: flex; align-items: center; gap: 8px; } +.admin-auth label { font-size: 13px; color: #d5dadd; } + +input, select { + min-height: 36px; + padding: 7px 10px; + border: 1px solid #b8c0c7; + border-radius: 4px; + background: #fff; +} + +button { + min-height: 36px; + padding: 7px 13px; + border: 1px solid #aab4bc; + border-radius: 4px; + color: #26323c; + background: #fff; +} +button:hover { background: #edf1f3; } +button:disabled { cursor: default; opacity: .55; } +.primary { color: #fff; border-color: #176b57; background: #1f8a70; } +.primary:hover { background: #176b57; } +.danger { color: #a12622; border-color: #d4aaa8; } +.icon-button { width: 36px; padding: 0; font-weight: 700; } + +main { width: min(1480px, 100%); margin: 0 auto; } +section { padding: 24px 28px; } +.upload-band { background: #fff; border-bottom: 1px solid #dce1e5; } +.video-band { min-height: 400px; } +.section-heading { + min-height: 30px; + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 16px; + margin-bottom: 14px; +} + +#upload-form { + display: grid; + grid-template-columns: minmax(260px, 2fr) minmax(180px, 1fr) 150px auto; + align-items: end; + gap: 12px; +} +#upload-form label { display: grid; gap: 5px; font-size: 13px; color: #4f5b64; } +#upload-progress { width: 100%; height: 8px; margin-top: 14px; accent-color: #1f8a70; } + +.table-scroll { overflow: auto; border: 1px solid #d7dde1; background: #fff; } +table { width: 100%; border-collapse: collapse; font-size: 13px; } +th, td { padding: 10px 12px; text-align: left; border-bottom: 1px solid #e3e7ea; vertical-align: middle; } +th { position: sticky; top: 0; z-index: 1; color: #44515b; background: #edf1f3; font-weight: 600; } +tr:last-child td { border-bottom: 0; } +.actions-column { min-width: 280px; } +.actions { display: flex; gap: 6px; white-space: nowrap; } +.actions button, .actions a { min-height: 30px; padding: 5px 9px; font-size: 12px; } +.actions a { display: inline-flex; align-items: center; border: 1px solid #aab4bc; border-radius: 4px; color: #26323c; text-decoration: none; } +.status { display: inline-block; min-width: 64px; font-weight: 600; } +.status-ready { color: #167356; } +.status-failed { color: #b12d28; } +.status-processing, .status-uploaded { color: #a45d08; } +.error-message { max-width: 320px; margin-top: 3px; color: #a12622; font-size: 12px; white-space: normal; } +.empty-state { padding: 70px 20px; text-align: center; color: #6f7a82; } + +dialog { + width: min(1200px, calc(100vw - 32px)); + max-height: calc(100vh - 32px); + padding: 0; + border: 1px solid #909ba3; + border-radius: 6px; + box-shadow: 0 18px 60px rgb(0 0 0 / 25%); +} +dialog::backdrop { background: rgb(20 25 28 / 55%); } +.dialog-heading { padding: 16px 18px; display: flex; justify-content: space-between; align-items: center; border-bottom: 1px solid #dce1e5; } +.dialog-heading span { font-size: 12px; color: #68747d; } +.sentence-scroll { max-height: calc(100vh - 120px); overflow: auto; } +.sentence-scroll textarea { width: min(600px, 45vw); min-width: 260px; min-height: 58px; resize: vertical; padding: 7px; border: 1px solid #b8c0c7; border-radius: 3px; } +.sentence-scroll select { min-width: 90px; } + +@media (max-width: 820px) { + .topbar { align-items: stretch; flex-direction: column; padding: 14px 16px; } + .admin-auth { flex-wrap: wrap; } + .admin-auth input { flex: 1 1 150px; } + section { padding: 20px 16px; } + #upload-form { grid-template-columns: 1fr; } + .actions-column { min-width: 250px; } +} diff --git a/sentence_api/static/admin.html b/sentence_api/static/admin.html new file mode 100644 index 0000000..a5faf0b --- /dev/null +++ b/sentence_api/static/admin.html @@ -0,0 +1,96 @@ + + + + + + 口语训练视频管理 + + + +
+
+

口语训练视频管理

+ 检查服务 +
+
+ + + + +
+
+ +
+
+
+

上传视频

+ +
+
+ + + + +
+ +
+ +
+
+

视频库

+ 0 个视频 +
+
+ + + + + + + + + + + + + +
标题状态时长句子大小更新时间操作
+
+ +
+
+ + +
+
+

句子文本

+ +
+ +
+
+ + + + + +
#时间文本语言操作
+
+
+ + + + diff --git a/sentence_api/static/admin.js b/sentence_api/static/admin.js new file mode 100644 index 0000000..90db333 --- /dev/null +++ b/sentence_api/static/admin.js @@ -0,0 +1,282 @@ +const state = { + adminKey: localStorage.getItem("oralTrainerAdminKey") || "", + videos: [], +}; + +const elements = { + adminKey: document.querySelector("#admin-key"), + saveKey: document.querySelector("#save-key"), + refresh: document.querySelector("#refresh"), + serviceState: document.querySelector("#service-state"), + uploadForm: document.querySelector("#upload-form"), + uploadState: document.querySelector("#upload-state"), + uploadProgress: document.querySelector("#upload-progress"), + videoRows: document.querySelector("#video-rows"), + videoCount: document.querySelector("#video-count"), + emptyState: document.querySelector("#empty-state"), + dialog: document.querySelector("#sentence-dialog"), + sentenceTitle: document.querySelector("#sentence-title"), + sentenceMeta: document.querySelector("#sentence-meta"), + sentenceRows: document.querySelector("#sentence-rows"), + closeDialog: document.querySelector("#close-dialog"), +}; + +elements.adminKey.value = state.adminKey; +elements.saveKey.addEventListener("click", () => { + state.adminKey = elements.adminKey.value; + localStorage.setItem("oralTrainerAdminKey", state.adminKey); + elements.serviceState.textContent = "密钥已保存"; +}); +elements.refresh.addEventListener("click", loadVideos); +elements.closeDialog.addEventListener("click", () => elements.dialog.close()); +elements.uploadForm.addEventListener("submit", uploadVideo); + +async function api(path, options = {}) { + const headers = new Headers(options.headers || {}); + if (state.adminKey) headers.set("X-Admin-Key", state.adminKey); + const response = await fetch(path, { ...options, headers }); + if (!response.ok) { + let detail = `HTTP ${response.status}`; + try { + const payload = await response.json(); + detail = typeof payload.detail === "string" ? payload.detail : JSON.stringify(payload.detail); + } catch (_) {} + throw new Error(detail); + } + return response.status === 204 ? null : response.json(); +} + +async function checkHealth() { + try { + const health = await api("/healthz"); + elements.serviceState.textContent = health.moss_configured ? "服务正常 / MOSS 已连接" : "服务正常 / MOSS 未配置"; + } catch (error) { + elements.serviceState.textContent = `服务异常: ${error.message}`; + } +} + +async function loadVideos() { + try { + const payload = await api("/api/v1/videos"); + state.videos = payload.videos; + renderVideos(); + } catch (error) { + elements.serviceState.textContent = `加载失败: ${error.message}`; + } +} + +function renderVideos() { + elements.videoRows.replaceChildren(); + elements.videoCount.textContent = `${state.videos.length} 个视频`; + elements.emptyState.hidden = state.videos.length !== 0; + for (const video of state.videos) { + const row = document.createElement("tr"); + row.append( + cellWithError(video.title, video.error_message), + statusCell(video.status), + textCell(formatDuration(video.duration_ms)), + textCell(String(video.sentence_count)), + textCell(formatBytes(video.size_bytes)), + textCell(formatDate(video.updated_at)), + actionCell(video), + ); + elements.videoRows.append(row); + } +} + +function textCell(value) { + const cell = document.createElement("td"); + cell.textContent = value; + return cell; +} + +function cellWithError(title, error) { + const cell = textCell(title); + if (error) { + const message = document.createElement("div"); + message.className = "error-message"; + message.textContent = error; + cell.append(message); + } + return cell; +} + +function statusCell(status) { + const cell = document.createElement("td"); + const label = document.createElement("span"); + label.className = `status status-${status}`; + label.textContent = { uploaded: "已上传", processing: "处理中", ready: "可用", failed: "失败" }[status] || status; + cell.append(label); + return cell; +} + +function actionCell(video) { + const cell = document.createElement("td"); + const actions = document.createElement("div"); + actions.className = "actions"; + const play = document.createElement("a"); + play.href = video.stream_url; + play.target = "_blank"; + play.rel = "noopener"; + play.textContent = "播放"; + actions.append(play); + actions.append(button("句子", () => openSentences(video))); + actions.append(button("重新处理", () => reprocess(video.video_hash))); + const remove = button("删除", () => deleteVideo(video)); + remove.className = "danger"; + actions.append(remove); + cell.append(actions); + return cell; +} + +function button(label, handler) { + const result = document.createElement("button"); + result.type = "button"; + result.textContent = label; + result.addEventListener("click", handler); + return result; +} + +function uploadVideo(event) { + event.preventDefault(); + const file = document.querySelector("#video-file").files[0]; + if (!file) return; + const title = document.querySelector("#video-title").value; + const language = document.querySelector("#video-language").value; + const query = new URLSearchParams({ filename: file.name }); + if (title) query.set("title", title); + if (language) query.set("language", language); + elements.uploadProgress.hidden = false; + elements.uploadProgress.value = 0; + elements.uploadState.textContent = "上传中"; + const request = new XMLHttpRequest(); + request.open("PUT", `/api/v1/admin/videos/raw?${query.toString()}`); + if (state.adminKey) request.setRequestHeader("X-Admin-Key", state.adminKey); + request.setRequestHeader("Content-Type", file.type || "application/octet-stream"); + request.upload.addEventListener("progress", (progress) => { + if (progress.lengthComputable) elements.uploadProgress.value = progress.loaded / progress.total * 100; + }); + request.addEventListener("load", async () => { + if (request.status >= 200 && request.status < 300) { + elements.uploadState.textContent = "上传完成,正在处理"; + elements.uploadForm.reset(); + await loadVideos(); + } else { + elements.uploadState.textContent = `上传失败: ${readXhrError(request)}`; + } + elements.uploadProgress.hidden = true; + }); + request.addEventListener("error", () => { + elements.uploadState.textContent = "上传连接中断"; + elements.uploadProgress.hidden = true; + }); + request.send(file); +} + +function readXhrError(request) { + try { return JSON.parse(request.responseText).detail || `HTTP ${request.status}`; } + catch (_) { return `HTTP ${request.status}`; } +} + +async function openSentences(video) { + try { + const detail = await api(`/api/v1/videos/${video.video_hash}`); + elements.sentenceTitle.textContent = detail.video.title; + const sentences = detail.boundaries?.sentences || []; + elements.sentenceMeta.textContent = `${sentences.length} 句 / ${detail.video.video_hash}`; + elements.sentenceRows.replaceChildren(); + for (const sentence of sentences) { + elements.sentenceRows.append(sentenceRow(video.video_hash, sentence)); + } + elements.dialog.showModal(); + } catch (error) { + elements.serviceState.textContent = `句子加载失败: ${error.message}`; + } +} + +function sentenceRow(videoHash, sentence) { + const row = document.createElement("tr"); + const textArea = document.createElement("textarea"); + textArea.value = sentence.text || ""; + const language = document.createElement("select"); + for (const [value, label] of [["", "自动"], ["en", "英语"], ["zh", "中文"]]) { + const option = document.createElement("option"); + option.value = value; + option.textContent = label; + option.selected = value === (sentence.language || ""); + language.append(option); + } + const textContainer = document.createElement("td"); + textContainer.append(textArea); + const languageContainer = document.createElement("td"); + languageContainer.append(language); + const saveContainer = document.createElement("td"); + const save = button("保存", async () => { + save.disabled = true; + try { + await api(`/api/v1/admin/videos/${videoHash}/sentences/${sentence.index}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text: textArea.value, language: language.value || null }), + }); + save.textContent = "已保存"; + } catch (error) { + save.textContent = error.message; + } finally { + window.setTimeout(() => { save.disabled = false; save.textContent = "保存"; }, 1600); + } + }); + saveContainer.append(save); + row.append( + textCell(String(sentence.index + 1)), + textCell(`${formatDuration(sentence.start_ms)} - ${formatDuration(sentence.end_ms)}`), + textContainer, + languageContainer, + saveContainer, + ); + return row; +} + +async function reprocess(videoHash) { + try { + await api(`/api/v1/admin/videos/${videoHash}/process`, { method: "POST" }); + await loadVideos(); + } catch (error) { + elements.serviceState.textContent = `处理启动失败: ${error.message}`; + } +} + +async function deleteVideo(video) { + if (!window.confirm(`删除“${video.title}”?`)) return; + try { + await api(`/api/v1/admin/videos/${video.video_hash}`, { method: "DELETE" }); + await loadVideos(); + } catch (error) { + elements.serviceState.textContent = `删除失败: ${error.message}`; + } +} + +function formatDuration(ms) { + if (!ms) return "-"; + const total = Math.floor(ms / 1000); + const hours = Math.floor(total / 3600); + const minutes = Math.floor(total % 3600 / 60); + const seconds = total % 60; + return hours ? `${hours}:${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}` : `${minutes}:${String(seconds).padStart(2, "0")}`; +} + +function formatBytes(bytes) { + if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`; + if (bytes < 1024 ** 3) return `${(bytes / 1024 ** 2).toFixed(1)} MB`; + return `${(bytes / 1024 ** 3).toFixed(2)} GB`; +} + +function formatDate(value) { + return new Intl.DateTimeFormat("zh-CN", { dateStyle: "short", timeStyle: "short" }).format(new Date(value)); +} + +checkHealth(); +loadVideos(); +window.setInterval(() => { + if (!elements.dialog.open) loadVideos(); +}, 5000); diff --git a/sentence_api/store.py b/sentence_api/store.py index b8dc23e..1a3e3a7 100644 --- a/sentence_api/store.py +++ b/sentence_api/store.py @@ -23,7 +23,11 @@ class BoundaryStore: self.reload() def reload(self) -> None: - raw = json.loads(self.index_path.read_text(encoding="utf-8")) + raw = ( + json.loads(self.index_path.read_text(encoding="utf-8")) + if self.index_path.is_file() + else {"videos": {}} + ) entries = raw.get("videos", raw) if not isinstance(entries, dict): raise ValueError("index JSON must contain a 'videos' object") diff --git a/sentence_api/tests/test_api.py b/sentence_api/tests/test_api.py index ca45491..49c6052 100644 --- a/sentence_api/tests/test_api.py +++ b/sentence_api/tests/test_api.py @@ -1,8 +1,11 @@ import json +from dataclasses import replace from fastapi.testclient import TestClient from sentence_api.main import create_app +from sentence_api.config import Settings +from sentence_api.repository import VideoRepository from sentence_api.store import BoundaryStore @@ -26,7 +29,19 @@ def make_client(tmp_path): }), encoding="utf-8", ) - return TestClient(create_app(BoundaryStore(index_path))) + settings = replace( + Settings.from_env(), + data_dir=tmp_path / "data", + legacy_boundaries_file=index_path, + ) + settings.ensure_directories() + return TestClient( + create_app( + BoundaryStore(index_path), + settings=settings, + repository=VideoRepository(settings.database_path), + ) + ) def test_lookup_returns_boundaries(tmp_path): diff --git a/sentence_api/tests/test_assessment_api.py b/sentence_api/tests/test_assessment_api.py new file mode 100644 index 0000000..fcb19e6 --- /dev/null +++ b/sentence_api/tests/test_assessment_api.py @@ -0,0 +1,146 @@ +import io +import json +import wave +from dataclasses import replace + +import numpy as np +from fastapi.testclient import TestClient + +from sentence_api.config import Settings +from sentence_api.main import create_app +from sentence_api.models import SentenceBoundaryDocument +from sentence_api.repository import VideoRepository +from sentence_api.store import BoundaryStore +from sentence_api.transcription import Transcript + + +VIDEO_HASH = "c" * 64 + + +class FakeTranscriber: + available = True + + def transcribe(self, audio_path, language=None): + return Transcript(text="The meeting starts at nine.", segments=[]) + + +def make_wav() -> bytes: + sample_rate = 16_000 + time = np.arange(sample_rate * 2, dtype=np.float64) / sample_rate + samples = (np.sin(2 * np.pi * 220 * time) * 8000).astype(" bool: + ... + + def transcribe(self, audio_path: Path, language: Optional[str] = None) -> Transcript: + ... + + +class MossTranscriber: + def __init__( + self, + endpoint: str, + model: str, + timeout_seconds: float = 1800, + max_new_tokens: int = 65536, + ): + self.endpoint = endpoint.strip() + self.model = model + self.timeout_seconds = timeout_seconds + self.max_new_tokens = max_new_tokens + + @property + def available(self) -> bool: + return bool(self.endpoint) + + def transcribe(self, audio_path: Path, language: Optional[str] = None) -> Transcript: + if not self.available: + raise RuntimeError("MOSS transcription is not configured on this server.") + content_type = mimetypes.guess_type(audio_path.name)[0] or "application/octet-stream" + data = { + "model": self.model, + "response_format": "verbose_json", + "temperature": "0", + "max_new_tokens": str(self.max_new_tokens), + } + if language: + data["language"] = language + + try: + with audio_path.open("rb") as audio_file: + response = httpx.post( + self.endpoint, + data=data, + files={"file": (audio_path.name, audio_file, content_type)}, + timeout=httpx.Timeout(self.timeout_seconds, connect=30), + ) + response.raise_for_status() + payload = response.json() + except (httpx.HTTPError, ValueError) as exc: + raise RuntimeError(f"MOSS transcription request failed: {exc}") from exc + + raw_text = str(payload.get("text") or "").strip() + segments = _parse_json_segments(payload.get("segments")) + if not segments: + segments = _parse_compact_segments(raw_text) + plain_text = " ".join(segment.text for segment in segments).strip() or raw_text + return Transcript(text=plain_text, segments=segments) + + +def _parse_json_segments(raw_segments) -> List[TranscriptionSegment]: + if not isinstance(raw_segments, list): + return [] + segments = [] + for item in raw_segments: + if not isinstance(item, dict): + continue + try: + start = float(item["start"]) + end = float(item["end"]) + except (KeyError, TypeError, ValueError): + continue + text = str(item.get("text") or "").strip() + if text and end > start >= 0: + speaker = item.get("speaker") + segments.append( + TranscriptionSegment( + start_seconds=start, + end_seconds=end, + text=text, + speaker=str(speaker) if speaker is not None else None, + ) + ) + return segments + + +_COMPACT_SEGMENT = re.compile( + r"\[(?P\d+(?:\.\d+)?)\]" + r"\[(?PS\d+)\]" + r"(?P.*?)" + r"\[(?P\d+(?:\.\d+)?)\]", + flags=re.DOTALL, +) + + +def _parse_compact_segments(text: str) -> List[TranscriptionSegment]: + segments = [] + for match in _COMPACT_SEGMENT.finditer(text): + start = float(match.group("start")) + end = float(match.group("end")) + segment_text = match.group("text").strip() + if segment_text and end > start: + segments.append( + TranscriptionSegment( + start_seconds=start, + end_seconds=end, + text=segment_text, + speaker=match.group("speaker"), + ) + ) + return segments