add test module

This commit is contained in:
2026-08-16 15:39:52 +08:00
parent d0310620fc
commit 6e4d93cea6
46 changed files with 3880 additions and 206 deletions

1
.curlrc-homebrew-speech Normal file
View File

@@ -0,0 +1 @@
--http1.1

13
.dockerignore Normal file
View File

@@ -0,0 +1,13 @@
.git
.gitignore
.DS_Store
__pycache__
*.pyc
.pytest_cache
.venv*
android
sentence_api/data/v
sentence_api/data/work
sentence_api/data/attempts
sentence_api/data/oral_trainer.sqlite3*

9
.gitignore vendored Normal file
View File

@@ -0,0 +1,9 @@
.DS_Store
__pycache__/
*.py[cod]
.pytest_cache/
sentence_api/data/oral_trainer.sqlite3*
sentence_api/data/v/
sentence_api/data/work/
sentence_api/data/attempts/

View File

@@ -122,10 +122,11 @@ SDK 基于 AndroidX Media3 / ExoPlayer支持在线视频流、本地缓存、
详见 [android/README.md](android/README.md)。
## 句子边界 API
## 口语训练视频服务
仓库新增了 `sentence_api/` 服务。它复用桌面播放器的静音检测算法,提前为视频
生成句子边界,并根据视频 SHA-256 哈希向 Android 返回毫秒级的
`start_ms` / `end_ms` 列表。服务不会在移动端请求时临时分析视频
`sentence_api/` 已扩展为完整的视频服务:支持超过 1 GB 的流式上传、MOSS 自动
转写、句子文本校对、HTTP Range 在线播放、Android 视频目录、学生录音上传,以及
内容正确度、有效语音时长、停顿和语速评分。原有的 SHA-256 句子边界接口保持兼容
见 [sentence_api/README.md](sentence_api/README.md)
本地使用见 [sentence_api/README.md](sentence_api/README.md),生产部署见
[sentence_api/DEPLOYMENT.md](sentence_api/DEPLOYMENT.md)。

View File

@@ -2,7 +2,7 @@
Android SDK for online oral-training video playback. It provides an embeddable
Media3/ExoPlayer player, streaming cache, tablet gestures, sentence navigation,
and a placeholder interface for future imitation-quality scoring.
the server video catalog, and remote imitation-quality assessment.
## Modules
@@ -93,11 +93,36 @@ Override the service only when a staging or private deployment is required:
OralTrainerSdk.init(
context,
OralTrainerSdkConfig(
sentenceBoundaryApiBaseUrl = "https://video_service.d1kt.cn"
sentenceBoundaryApiBaseUrl = "https://video_service.d1kt.cn",
assessmentApiKey = BuildConfig.ORAL_TRAINER_CLIENT_KEY,
)
)
```
## Video Catalog
The SDK loads ready videos from `GET /api/v1/videos` and resolves both absolute
and relative stream URLs:
```kotlin
sdk.videoCatalogApi.fetch(object : VideoCatalogCallback {
override fun onSuccess(videos: List<TrainingVideoSummary>) {
val video = videos.first()
sdk.sentenceBoundaryApi.fetch(video.videoHash, object : SentenceBoundaryApiCallback {
override fun onSuccess(result: SentenceBoundaryApiResult) {
controller.loadItem(video.toTrainingMediaItem(result.sentences))
}
override fun onError(error: Throwable) {
controller.loadItem(video.toTrainingMediaItem())
}
})
}
override fun onError(error: Throwable) = Unit
})
```
## Local Video Testing
The computer path `/Users/...` is not visible to an Android device. For a
@@ -110,19 +135,40 @@ service and pass its URL as `TrainingMediaItem.uri`. The SDK streams it and
caches downloaded ranges locally. Copying large 4K files to each device is
better reserved for explicitly offline courses.
## Future Imitation Scoring
## Remote Imitation Scoring
Provide an implementation of `ImitationQualityAssessor` when the speech
assessment algorithm is ready:
Create the controller with the built-in remote assessor:
```kotlin
val remoteAssessor = sdk.createRemoteImitationQualityAssessor()
val controller = sdk.createController(
imitationAssessor = MyImitationQualityAssessor()
imitationAssessor = remoteAssessor
)
```
Then call `assessCurrentSentence(recordingUri, callback)` after the student
records a sentence.
records a sentence. The current `TrainingMediaItem.id` must be the server video
SHA-256, which is already true for items created by `TrainingVideoSummary`:
```kotlin
controller.assessCurrentSentence(
recordingUri = recordingUri,
locale = "en",
callback = object : ImitationAssessmentCallback {
override fun onResult(result: ImitationAssessmentResult) {
val passed = result.passed == true
val score = result.overallScore
val durationRatio = result.durationRatio
}
override fun onError(error: Throwable) = Unit
}
)
```
The server returns content, completeness, fluency, duration, pause, and speech
rate scores. Phoneme pronunciation and prosody are nullable until the dedicated
models are enabled.
## Build

View File

@@ -18,6 +18,17 @@ data class ImitationAssessmentResult @JvmOverloads constructor(
val completenessScore: Float? = null,
val feedback: String? = null,
val details: Map<String, String> = emptyMap(),
val passed: Boolean? = null,
val contentScore: Float? = null,
val durationScore: Float? = null,
val pauseScore: Float? = null,
val speechRateScore: Float? = null,
val durationRatio: Float? = null,
val referenceText: String? = null,
val recognizedText: String? = null,
val missingTokens: List<String> = emptyList(),
val extraTokens: List<String> = emptyList(),
val substitutions: List<Pair<String, String>> = emptyList(),
)
fun interface CancellableAssessment {

View File

@@ -4,6 +4,7 @@ import android.content.Context
import android.util.AttributeSet
import android.view.GestureDetector
import android.view.MotionEvent
import android.view.ViewConfiguration
import android.widget.FrameLayout
import androidx.media3.ui.PlayerView
import kotlin.math.abs
@@ -21,6 +22,23 @@ class OralTrainerPlayerView @JvmOverloads constructor(
private var controller: OralTrainerController? = null
private var gestureControls = GestureControlsConfig()
private val density = resources.displayMetrics.density
private val touchSlop = ViewConfiguration.get(context).scaledTouchSlop
private val longPressTimeoutMs = ViewConfiguration.getLongPressTimeout().toLong()
private var longPressStartX = 0f
private var longPressStartY = 0f
private var longPressActive = false
private var speedBeforeLongPress: Float? = null
private val longPressSpeedRunnable = Runnable {
val activeController = controller ?: return@Runnable
if (!gestureControls.enabled || !gestureControls.longPressAdjustsSpeed || width <= 0) {
return@Runnable
}
longPressActive = true
speedBeforeLongPress = activeController.snapshot().playbackSpeed
activeController.setPlaybackSpeed(speedForTouchX(longPressStartX))
activeController.dispatchGesture(GestureEvent(GestureKind.LONG_PRESS_SPEED))
}
private val gestureDetector = GestureDetector(
context,
@@ -99,6 +117,72 @@ class OralTrainerPlayerView @JvmOverloads constructor(
}
override fun dispatchTouchEvent(ev: MotionEvent): Boolean {
if (handleLongPressSpeed(ev)) {
return true
}
return gestureDetector.onTouchEvent(ev) || super.dispatchTouchEvent(ev)
}
override fun onDetachedFromWindow() {
cancelLongPressSpeed(restoreSpeed = true)
super.onDetachedFromWindow()
}
private fun handleLongPressSpeed(event: MotionEvent): Boolean {
if (!gestureControls.enabled || !gestureControls.longPressAdjustsSpeed) {
return false
}
when (event.actionMasked) {
MotionEvent.ACTION_DOWN -> {
longPressStartX = event.x
longPressStartY = event.y
longPressActive = false
speedBeforeLongPress = null
removeCallbacks(longPressSpeedRunnable)
postDelayed(longPressSpeedRunnable, longPressTimeoutMs)
return false
}
MotionEvent.ACTION_MOVE -> {
if (longPressActive) {
controller?.setPlaybackSpeed(speedForTouchX(event.x))
return true
}
val dx = abs(event.x - longPressStartX)
val dy = abs(event.y - longPressStartY)
if (dx > touchSlop || dy > touchSlop) {
removeCallbacks(longPressSpeedRunnable)
}
return false
}
MotionEvent.ACTION_UP,
MotionEvent.ACTION_CANCEL -> {
val wasActive = longPressActive
cancelLongPressSpeed(restoreSpeed = wasActive)
return wasActive
}
}
return false
}
private fun cancelLongPressSpeed(restoreSpeed: Boolean) {
removeCallbacks(longPressSpeedRunnable)
if (restoreSpeed) {
controller?.setPlaybackSpeed(speedBeforeLongPress ?: 1f)
}
longPressActive = false
speedBeforeLongPress = null
}
private fun speedForTouchX(x: Float): Float {
val center = width / 2f
val distanceRatio = if (center <= 0f) 0f else abs(x - center) / center
val farFromCenter = distanceRatio >= 0.5f
return if (x < center) {
if (farFromCenter) gestureControls.longPressSlowerSpeed else gestureControls.longPressSlowSpeed
} else {
if (farFromCenter) gestureControls.longPressFasterSpeed else gestureControls.longPressFastSpeed
}
}
}

View File

@@ -1,6 +1,7 @@
package cn.learningpad.oraltrainer.sdk
import android.content.Context
import java.util.concurrent.CopyOnWriteArrayList
class OralTrainerSdk private constructor(
context: Context,
@@ -10,6 +11,12 @@ class OralTrainerSdk private constructor(
val cache: OralTrainerCache = OralTrainerCache(appContext, config)
val sentenceBoundaryApi: SentenceBoundaryApi = SentenceBoundaryApi(appContext, config)
val videoCatalogApi: VideoCatalogApi = VideoCatalogApi(appContext, config)
private val remoteAssessors = CopyOnWriteArrayList<RemoteImitationQualityAssessor>()
fun createRemoteImitationQualityAssessor(): RemoteImitationQualityAssessor {
return RemoteImitationQualityAssessor(appContext, config).also(remoteAssessors::add)
}
@JvmOverloads
fun createController(
@@ -26,6 +33,9 @@ class OralTrainerSdk private constructor(
fun release() {
sentenceBoundaryApi.release()
videoCatalogApi.release()
remoteAssessors.forEach { it.release() }
remoteAssessors.clear()
StreamingCache.release()
}

View File

@@ -9,4 +9,5 @@ data class OralTrainerSdkConfig @JvmOverloads constructor(
val connectTimeoutMs: Int = 15_000,
val readTimeoutMs: Int = 30_000,
val sentenceBoundaryApiBaseUrl: String = "https://video_service.d1kt.cn",
val assessmentApiKey: String? = null,
)

View File

@@ -27,6 +27,7 @@ enum class GestureKind {
SINGLE_TAP,
SWIPE_LEFT,
SWIPE_RIGHT,
LONG_PRESS_SPEED,
}
interface OralTrainerListener {

View File

@@ -16,6 +16,11 @@ data class GestureControlsConfig @JvmOverloads constructor(
val rightSwipeAction: SwipeAction = SwipeAction.NEXT_SENTENCE_OR_FORWARD,
val minSwipeDistanceDp: Float = 48f,
val minSwipeVelocityDpPerSecond: Float = 160f,
val longPressAdjustsSpeed: Boolean = true,
val longPressSlowSpeed: Float = 0.75f,
val longPressSlowerSpeed: Float = 0.5f,
val longPressFastSpeed: Float = 1.5f,
val longPressFasterSpeed: Float = 2.0f,
)
enum class SwipeAction {

View File

@@ -0,0 +1,198 @@
package cn.learningpad.oraltrainer.sdk
import android.content.ContentResolver
import android.content.Context
import android.net.Uri
import android.os.Handler
import android.os.Looper
import org.json.JSONArray
import org.json.JSONObject
import java.io.BufferedInputStream
import java.io.BufferedOutputStream
import java.io.IOException
import java.net.HttpURLConnection
import java.net.URL
import java.util.UUID
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.Future
/** Uploads one sentence recording to the server-side MOSS assessment endpoint. */
class RemoteImitationQualityAssessor internal constructor(
context: Context,
private val config: OralTrainerSdkConfig,
) : ImitationQualityAssessor {
private val appContext = context.applicationContext
private val contentResolver: ContentResolver = appContext.contentResolver
private val executor: ExecutorService = Executors.newCachedThreadPool()
private val mainHandler = Handler(Looper.getMainLooper())
private val baseUri = Uri.parse(config.sentenceBoundaryApiBaseUrl.trimEnd('/'))
init {
require(baseUri.scheme == "https" || baseUri.scheme == "http") {
"sentenceBoundaryApiBaseUrl must use http or https."
}
require(!baseUri.host.isNullOrBlank()) {
"sentenceBoundaryApiBaseUrl must include a host."
}
}
override fun assess(
request: ImitationAssessmentRequest,
callback: ImitationAssessmentCallback,
): CancellableAssessment {
val future = executor.submit {
try {
val result = assessBlocking(request)
if (!Thread.currentThread().isInterrupted) {
mainHandler.post { callback.onResult(result) }
}
} catch (error: Throwable) {
if (!Thread.currentThread().isInterrupted) {
mainHandler.post { callback.onError(error) }
}
}
}
return CancellableAssessment { future.cancel(true) }
}
fun release() {
executor.shutdownNow()
}
private fun assessBlocking(request: ImitationAssessmentRequest): ImitationAssessmentResult {
val videoHash = request.metadata["video_hash"] ?: request.mediaId
require(SHA256_PATTERN.matches(videoHash)) {
"ImitationAssessmentRequest.mediaId or metadata[video_hash] must be a SHA-256 hash."
}
val endpoint = baseUri.buildUpon()
.appendPath("api")
.appendPath("v1")
.appendPath("videos")
.appendPath(videoHash.lowercase())
.appendPath("sentences")
.appendPath(request.sentence.index.toString())
.appendPath("assessments")
.build()
val connection = URL(endpoint.toString()).openConnection() as HttpURLConnection
val boundary = "----OralTrainer-${UUID.randomUUID()}"
return try {
connection.requestMethod = "POST"
connection.doOutput = true
connection.connectTimeout = config.connectTimeoutMs
connection.readTimeout = maxOf(config.readTimeoutMs, 180_000)
connection.setChunkedStreamingMode(64 * 1024)
connection.setRequestProperty("Accept", "application/json")
connection.setRequestProperty("User-Agent", config.userAgent)
connection.setRequestProperty("Content-Type", "multipart/form-data; boundary=$boundary")
config.assessmentApiKey?.takeIf { it.isNotBlank() }?.let {
connection.setRequestProperty("X-Client-Key", it)
}
BufferedOutputStream(connection.outputStream).use { output ->
writeFormField(output, boundary, "language", request.locale ?: request.sentence.language.orEmpty())
writeFilePart(output, boundary, request.recordingUri)
output.write("--$boundary--\r\n".toByteArray(Charsets.UTF_8))
}
val statusCode = connection.responseCode
val stream = if (statusCode in 200..299) connection.inputStream else connection.errorStream
val body = stream?.bufferedReader(Charsets.UTF_8)?.use { it.readText() }.orEmpty()
if (statusCode !in 200..299) {
throw RemoteAssessmentException(statusCode, body)
}
parseResponse(body)
} finally {
connection.disconnect()
}
}
private fun writeFormField(output: BufferedOutputStream, boundary: String, name: String, value: String) {
output.write("--$boundary\r\n".toByteArray(Charsets.UTF_8))
output.write("Content-Disposition: form-data; name=\"$name\"\r\n\r\n".toByteArray(Charsets.UTF_8))
output.write(value.toByteArray(Charsets.UTF_8))
output.write("\r\n".toByteArray(Charsets.UTF_8))
}
private fun writeFilePart(output: BufferedOutputStream, boundary: String, uri: Uri) {
val fileName = uri.lastPathSegment?.substringAfterLast('/')
?.replace("\r", "_")
?.replace("\n", "_")
?.replace("\"", "_")
?.ifBlank { "recording.wav" }
?: "recording.wav"
val mimeType = contentResolver.getType(uri) ?: "audio/wav"
output.write("--$boundary\r\n".toByteArray(Charsets.UTF_8))
output.write(
"Content-Disposition: form-data; name=\"audio\"; filename=\"$fileName\"\r\n".toByteArray(
Charsets.UTF_8
)
)
output.write("Content-Type: $mimeType\r\n\r\n".toByteArray(Charsets.UTF_8))
contentResolver.openInputStream(uri)?.use { input ->
BufferedInputStream(input).use { buffered ->
val buffer = ByteArray(64 * 1024)
while (true) {
if (Thread.currentThread().isInterrupted) throw IOException("Assessment request was cancelled.")
val count = buffered.read(buffer)
if (count < 0) break
output.write(buffer, 0, count)
}
}
} ?: throw IOException("Cannot open recording URI: $uri")
output.write("\r\n".toByteArray(Charsets.UTF_8))
}
private fun parseResponse(body: String): ImitationAssessmentResult {
val root = JSONObject(body)
val substitutions = buildList {
val array = root.optJSONArray("substitutions") ?: JSONArray()
for (index in 0 until array.length()) {
val item = array.optJSONObject(index) ?: continue
add(item.optString("expected") to item.optString("actual"))
}
}
val details = buildMap {
val objectValue = root.optJSONObject("details") ?: return@buildMap
for (key in objectValue.keys()) put(key, objectValue.optString(key))
}
return ImitationAssessmentResult(
overallScore = root.getDouble("overall_score").toFloat(),
pronunciationScore = root.optNullableDouble("pronunciation_score")?.toFloat(),
fluencyScore = root.optNullableDouble("fluency_score")?.toFloat(),
completenessScore = root.optNullableDouble("completeness_score")?.toFloat(),
feedback = root.optString("feedback").ifBlank { null },
details = details,
passed = if (root.has("passed")) root.optBoolean("passed") else null,
contentScore = root.optNullableDouble("content_score")?.toFloat(),
durationScore = root.optNullableDouble("duration_score")?.toFloat(),
pauseScore = root.optNullableDouble("pause_score")?.toFloat(),
speechRateScore = root.optNullableDouble("speech_rate_score")?.toFloat(),
durationRatio = root.optNullableDouble("duration_ratio")?.toFloat(),
referenceText = root.optString("reference_text").ifBlank { null },
recognizedText = root.optString("recognized_text").ifBlank { null },
missingTokens = root.optStringArray("missing_tokens"),
extraTokens = root.optStringArray("extra_tokens"),
substitutions = substitutions,
)
}
companion object {
private val SHA256_PATTERN = Regex("^[0-9a-fA-F]{64}$")
}
}
class RemoteAssessmentException(
val statusCode: Int,
responseBody: String,
) : IOException("Remote assessment returned HTTP $statusCode: $responseBody")
private fun JSONObject.optNullableDouble(name: String): Double? {
if (!has(name) || isNull(name)) return null
return optDouble(name).takeUnless { it.isNaN() }
}
private fun JSONObject.optStringArray(name: String): List<String> {
val array = optJSONArray(name) ?: return emptyList()
return buildList(array.length()) {
for (index in 0 until array.length()) add(array.optString(index))
}
}

View File

@@ -147,6 +147,12 @@ class SentenceBoundaryApi internal constructor(
startMs = sentence.getLong("start_ms"),
endMs = sentence.getLong("end_ms"),
text = if (sentence.isNull("text")) null else sentence.getString("text"),
language = if (sentence.isNull("language")) null else sentence.getString("language"),
referenceSpeechDurationMs = if (sentence.isNull("reference_speech_duration_ms")) {
null
} else {
sentence.getLong("reference_speech_duration_ms")
},
)
)
}

View File

@@ -8,11 +8,16 @@ data class SentenceBoundary @JvmOverloads constructor(
val startMs: Long,
val endMs: Long,
val text: String? = null,
val language: String? = null,
val referenceSpeechDurationMs: Long? = null,
) {
init {
require(index >= 0) { "Sentence index must be non-negative." }
require(startMs >= 0) { "Sentence startMs must be non-negative." }
require(endMs > startMs) { "Sentence endMs must be greater than startMs." }
require(referenceSpeechDurationMs == null || referenceSpeechDurationMs > 0) {
"Sentence referenceSpeechDurationMs must be positive when present."
}
}
}

View File

@@ -0,0 +1,116 @@
package cn.learningpad.oraltrainer.sdk
import android.content.Context
import android.net.Uri
import android.os.Handler
import android.os.Looper
import org.json.JSONObject
import java.io.IOException
import java.net.HttpURLConnection
import java.net.URL
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
data class TrainingVideoSummary(
val videoHash: String,
val title: String,
val streamUri: Uri,
val durationMs: Long?,
val sizeBytes: Long,
val language: String?,
val sentenceCount: Int,
val status: String,
) {
fun toTrainingMediaItem(sentences: List<SentenceBoundary> = emptyList()): TrainingMediaItem {
return TrainingMediaItem(
id = videoHash,
title = title,
uri = streamUri,
sentences = sentences,
customCacheKey = videoHash,
)
}
}
interface VideoCatalogCallback {
fun onSuccess(videos: List<TrainingVideoSummary>)
fun onError(error: Throwable)
}
class VideoCatalogApi internal constructor(
context: Context,
private val config: OralTrainerSdkConfig,
) {
private val mainHandler = Handler(Looper.getMainLooper())
private val executor: ExecutorService = Executors.newCachedThreadPool()
private val baseUrl = config.sentenceBoundaryApiBaseUrl.trimEnd('/') + "/"
fun fetch(callback: VideoCatalogCallback): CancellableRequest {
val future = executor.submit {
try {
val videos = fetchBlocking()
if (!Thread.currentThread().isInterrupted) {
mainHandler.post { callback.onSuccess(videos) }
}
} catch (error: Throwable) {
if (!Thread.currentThread().isInterrupted) {
mainHandler.post { callback.onError(error) }
}
}
}
return CancellableRequest(future)
}
fun release() {
executor.shutdownNow()
}
private fun fetchBlocking(): List<TrainingVideoSummary> {
val endpoint = URL(URL(baseUrl), "api/v1/videos")
val connection = endpoint.openConnection() as HttpURLConnection
return try {
connection.requestMethod = "GET"
connection.connectTimeout = config.connectTimeoutMs
connection.readTimeout = config.readTimeoutMs
connection.setRequestProperty("Accept", "application/json")
connection.setRequestProperty("User-Agent", config.userAgent)
val statusCode = connection.responseCode
val stream = if (statusCode in 200..299) connection.inputStream else connection.errorStream
val body = stream?.bufferedReader(Charsets.UTF_8)?.use { it.readText() }.orEmpty()
if (statusCode !in 200..299) throw VideoCatalogApiException(statusCode, body)
parseResponse(body)
} finally {
connection.disconnect()
}
}
private fun parseResponse(body: String): List<TrainingVideoSummary> {
val videos = JSONObject(body).getJSONArray("videos")
return buildList(videos.length()) {
for (index in 0 until videos.length()) {
val item = videos.getJSONObject(index)
val status = item.getString("status")
if (status != "ready") continue
val streamUrl = URL(URL(baseUrl), item.getString("stream_url"))
add(
TrainingVideoSummary(
videoHash = item.getString("video_hash"),
title = item.getString("title"),
streamUri = Uri.parse(streamUrl.toString()),
durationMs = if (item.isNull("duration_ms")) null else item.getLong("duration_ms"),
sizeBytes = item.getLong("size_bytes"),
language = if (item.isNull("language")) null else item.getString("language"),
sentenceCount = item.getInt("sentence_count"),
status = status,
)
)
}
}
}
}
class VideoCatalogApiException(
val statusCode: Int,
responseBody: String,
) : IOException("Video catalog API returned HTTP $statusCode: $responseBody")

View File

@@ -3,7 +3,7 @@
<application
android:allowBackup="true"
android:label="Oral Trainer Sample"
android:label="@string/app_name"
android:supportsRtl="true"
android:theme="@style/AppTheme">
<activity

View File

@@ -2,16 +2,24 @@ package cn.learningpad.oraltrainer.sample
import android.app.Activity
import android.content.Intent
import android.content.res.ColorStateList
import android.graphics.Color
import android.graphics.Typeface
import android.graphics.drawable.GradientDrawable
import android.net.Uri
import android.os.Bundle
import android.view.Gravity
import android.view.View
import android.view.ViewGroup
import android.widget.Button
import android.widget.FrameLayout
import android.widget.HorizontalScrollView
import android.widget.LinearLayout
import android.widget.ProgressBar
import android.widget.TextView
import androidx.media3.common.MimeTypes
import cn.learningpad.oraltrainer.sdk.GestureEvent
import cn.learningpad.oraltrainer.sdk.GestureKind
import cn.learningpad.oraltrainer.sdk.LoopMode
import cn.learningpad.oraltrainer.sdk.OralTrainerController
import cn.learningpad.oraltrainer.sdk.OralTrainerListener
@@ -21,21 +29,33 @@ import cn.learningpad.oraltrainer.sdk.PlaybackSnapshot
import cn.learningpad.oraltrainer.sdk.PlayerConfig
import cn.learningpad.oraltrainer.sdk.SentenceBoundary
import cn.learningpad.oraltrainer.sdk.SentenceBoundaryApiCallback
import cn.learningpad.oraltrainer.sdk.TrainingMediaItem
import cn.learningpad.oraltrainer.sdk.SentenceBoundaryApiResult
import cn.learningpad.oraltrainer.sdk.TrainingMediaItem
import cn.learningpad.oraltrainer.sdk.TrainingVideoSummary
import cn.learningpad.oraltrainer.sdk.VideoCatalogCallback
import java.util.Locale
class MainActivity : Activity() {
private companion object {
const val PICK_VIDEO_REQUEST = 1001
}
private lateinit var sdk: OralTrainerSdk
private lateinit var controller: OralTrainerController
private lateinit var playerView: OralTrainerPlayerView
private lateinit var lessonTitleText: TextView
private lateinit var statusText: TextView
private lateinit var sentenceText: TextView
private lateinit var sentenceMetaText: TextView
private lateinit var timeText: TextView
private lateinit var speedText: TextView
private lateinit var progressBar: ProgressBar
private lateinit var catalogList: LinearLayout
private lateinit var catalogStatusText: TextView
private var activeItemId: String = SAMPLE_ID
private var currentSentenceCount = 0
private var catalogVideos: List<TrainingVideoSummary> = emptyList()
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
configureWindow()
sdk = OralTrainerSdk.init(this)
controller = sdk.createController(
@@ -47,77 +67,10 @@ class MainActivity : Activity() {
)
controller.setLoopMode(LoopMode.ALL)
val playerView = OralTrainerPlayerView(this).apply {
bind(controller)
layoutParams = LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
0,
1f,
)
}
statusText = TextView(this).apply {
setTextColor(Color.WHITE)
textSize = 14f
setPadding(24, 16, 24, 8)
}
sentenceText = TextView(this).apply {
setTextColor(Color.WHITE)
textSize = 18f
setPadding(24, 4, 24, 16)
}
val controls = LinearLayout(this).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER
setPadding(16, 8, 16, 20)
addView(commandButton("快退") { controller.rewind() })
addView(commandButton("播放/暂停") { controller.togglePlayPause() })
addView(commandButton("快进") { controller.fastForward() })
addView(commandButton("上一句") { controller.previousSentenceOrRewind() })
addView(commandButton("下一句") { controller.nextSentenceOrForward() })
}
val root = LinearLayout(this).apply {
orientation = LinearLayout.VERTICAL
setBackgroundColor(Color.rgb(18, 18, 18))
addView(playerView)
addView(statusText)
addView(sentenceText)
addView(controls)
addView(Button(this@MainActivity).apply {
text = "选择本地视频"
setOnClickListener { openLocalVideoPicker() }
})
}
setContentView(root)
controller.addListener(object : OralTrainerListener {
override fun onPlaybackSnapshot(snapshot: PlaybackSnapshot) {
statusText.text = buildString {
append(if (snapshot.isPlaying) "播放中" else "已暂停")
append(" ")
append(format(snapshot.positionMs))
append(" / ")
append(format(snapshot.durationMs))
append(" 速度 ")
append(snapshot.playbackSpeed)
append("x")
}
}
override fun onSentenceChanged(sentence: SentenceBoundary?) {
sentenceText.text = sentence?.let {
"${it.index + 1} 句:${it.text.orEmpty()}"
} ?: "当前没有句子边界"
}
override fun onGesture(event: GestureEvent) {
statusText.text = "手势:${event.kind} ${event.action ?: ""}"
}
})
controller.loadItem(sampleOnlineLesson())
setContentView(createContentView())
bindPlayerEvents()
loadSampleLesson()
loadCatalog()
}
override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
@@ -132,21 +85,38 @@ class MainActivity : Activity() {
contentResolver.takePersistableUriPermission(uri, persistableFlags)
}
}
val item = TrainingMediaItem(
id = uri.toString(),
title = uri.lastPathSegment ?: "本地视频",
uri = uri,
)
activeItemId = item.id
currentSentenceCount = 0
controller.loadItem(item)
statusText.text = "正在获取句子边界..."
lessonTitleText.text = item.title
sentenceMetaText.text = "句子边界分析中"
sentenceText.text = "正在匹配服务端句子边界"
statusText.text = "正在获取本地视频的句子边界..."
renderCatalog()
sdk.sentenceBoundaryApi.fetchForUri(uri, contentResolver, object : SentenceBoundaryApiCallback {
override fun onSuccess(result: SentenceBoundaryApiResult) {
if (activeItemId != item.id) {
return
}
currentSentenceCount = result.sentences.size
controller.loadItem(item.copy(sentences = result.sentences))
statusText.text = "句子边界已加载:${result.sentences.size}"
}
override fun onError(error: Throwable) {
statusText.text = "句子边界获取失败,使用 10 秒跳转:${error.message.orEmpty()}"
if (activeItemId != item.id) {
return
}
statusText.text = "句子边界获取失败,使用默认快退/快进:${error.message.orEmpty()}"
sentenceMetaText.text = "暂无句子边界"
sentenceText.text = item.title
}
})
}
@@ -156,19 +126,382 @@ class MainActivity : Activity() {
super.onDestroy()
}
private fun commandButton(label: String, action: () -> Unit): Button {
@Suppress("DEPRECATION")
private fun configureWindow() {
window.statusBarColor = COLOR_BACKGROUND
window.navigationBarColor = COLOR_BACKGROUND
}
private fun createContentView(): View {
return LinearLayout(this).apply {
orientation = LinearLayout.VERTICAL
setBackgroundColor(COLOR_BACKGROUND)
addView(createHeader())
addView(createPlayerSection())
addView(createSentenceSection())
addView(createCatalogSection())
}
}
private fun createHeader(): View {
val titleBlock = LinearLayout(this).apply {
orientation = LinearLayout.VERTICAL
gravity = Gravity.CENTER_VERTICAL
layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f)
addView(TextView(this@MainActivity).apply {
text = "口语宝"
setTextColor(Color.WHITE)
textSize = 26f
typeface = Typeface.DEFAULT_BOLD
includeFontPadding = false
})
addView(TextView(this@MainActivity).apply {
text = "中英跟读训练"
setTextColor(COLOR_TEXT_MUTED)
textSize = 13f
setPadding(1.dp, 4.dp, 0, 0)
})
}
return LinearLayout(this).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
setPadding(18.dp, 16.dp, 18.dp, 12.dp)
addView(titleBlock)
addView(headerButton("导入") { openLocalVideoPicker() })
addView(headerButton("刷新") { loadCatalog() })
}
}
private fun createPlayerSection(): View {
playerView = OralTrainerPlayerView(this).apply {
bind(controller)
layoutParams = FrameLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.MATCH_PARENT,
)
}
val overlay = LinearLayout(this).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
setPadding(12.dp, 12.dp, 12.dp, 12.dp)
layoutParams = FrameLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
Gravity.TOP,
)
}
timeText = overlayPill("--:-- / --:--").apply {
layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f)
}
speedText = overlayPill("1x")
overlay.addView(timeText)
overlay.addView(speedText)
return FrameLayout(this).apply {
background = rounded(COLOR_SURFACE, 8f, COLOR_BORDER)
clipToOutline = true
layoutParams = LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
0,
1f,
).withMargins(16.dp, 0, 16.dp, 10.dp)
addView(playerView)
addView(overlay)
}
}
private fun createSentenceSection(): View {
lessonTitleText = TextView(this).apply {
setTextColor(COLOR_TEXT_DARK)
textSize = 17f
typeface = Typeface.DEFAULT_BOLD
maxLines = 1
}
sentenceMetaText = TextView(this).apply {
setTextColor(COLOR_ACCENT_DEEP)
textSize = 13f
setPadding(0, 8.dp, 0, 0)
}
sentenceText = TextView(this).apply {
setTextColor(COLOR_TEXT_DARK)
textSize = 22f
typeface = Typeface.DEFAULT_BOLD
setLineSpacing(2.dp.toFloat(), 1.05f)
setPadding(0, 10.dp, 0, 10.dp)
}
progressBar = ProgressBar(this, null, android.R.attr.progressBarStyleHorizontal).apply {
max = PROGRESS_MAX
progress = 0
progressTintList = ColorStateList.valueOf(COLOR_ACCENT)
progressBackgroundTintList = ColorStateList.valueOf(COLOR_PROGRESS_TRACK)
layoutParams = LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
6.dp,
)
}
statusText = TextView(this).apply {
setTextColor(COLOR_TEXT_SUBTLE)
textSize = 13f
setPadding(0, 10.dp, 0, 0)
}
return LinearLayout(this).apply {
orientation = LinearLayout.VERTICAL
background = rounded(Color.WHITE, 8f, COLOR_LIGHT_BORDER)
setPadding(18.dp, 16.dp, 18.dp, 16.dp)
layoutParams = LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
).withMargins(16.dp, 0, 16.dp, 12.dp)
addView(lessonTitleText)
addView(sentenceMetaText)
addView(sentenceText)
addView(progressBar)
addView(statusText)
}
}
private fun createCatalogSection(): View {
catalogStatusText = TextView(this).apply {
setTextColor(COLOR_TEXT_MUTED)
textSize = 13f
gravity = Gravity.END
text = "正在同步"
layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f)
}
catalogList = LinearLayout(this).apply {
orientation = LinearLayout.HORIZONTAL
setPadding(0, 10.dp, 0, 4.dp)
}
val header = LinearLayout(this).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
addView(TextView(this@MainActivity).apply {
text = "课程"
setTextColor(Color.WHITE)
textSize = 16f
typeface = Typeface.DEFAULT_BOLD
})
addView(catalogStatusText)
}
val scroller = HorizontalScrollView(this).apply {
isHorizontalScrollBarEnabled = false
addView(catalogList)
}
return LinearLayout(this).apply {
orientation = LinearLayout.VERTICAL
setPadding(16.dp, 0, 16.dp, 14.dp)
addView(header)
addView(scroller)
}
}
private fun bindPlayerEvents() {
controller.addListener(object : OralTrainerListener {
override fun onPlaybackSnapshot(snapshot: PlaybackSnapshot) {
timeText.text = "${formatTime(snapshot.positionMs)} / ${formatTime(snapshot.durationMs)}"
speedText.text = formatSpeed(snapshot.playbackSpeed)
statusText.text = playbackStatus(snapshot)
progressBar.progress = playbackProgress(snapshot)
}
override fun onMediaChanged(item: TrainingMediaItem?) {
lessonTitleText.text = item?.title ?: "未选择课程"
currentSentenceCount = item?.sentences?.size ?: 0
}
override fun onSentenceChanged(sentence: SentenceBoundary?) {
if (sentence == null) {
sentenceMetaText.text = "暂无句子边界"
sentenceText.text = lessonTitleText.text
return
}
val countText = if (currentSentenceCount > 0) {
" / $currentSentenceCount"
} else {
""
}
sentenceMetaText.text = "${sentence.index + 1}$countText${formatTime(sentence.startMs)}-${formatTime(sentence.endMs)}"
sentenceText.text = sentence.text?.takeIf { it.isNotBlank() } ?: "当前句子"
}
override fun onGesture(event: GestureEvent) {
statusText.text = when (event.kind) {
GestureKind.SINGLE_TAP -> "播放状态已切换"
GestureKind.SWIPE_LEFT -> "已跳到上一句"
GestureKind.SWIPE_RIGHT -> "已跳到下一句"
GestureKind.LONG_PRESS_SPEED -> "变速播放中"
}
}
override fun onPlayerError(error: Throwable) {
statusText.text = "播放失败:${error.message.orEmpty()}"
}
})
}
private fun loadSampleLesson() {
val item = sampleOnlineLesson()
activeItemId = item.id
currentSentenceCount = item.sentences.size
controller.loadItem(item)
lessonTitleText.text = item.title
renderCatalog()
}
private fun loadCatalog() {
catalogStatusText.text = "正在同步"
sdk.videoCatalogApi.fetch(object : VideoCatalogCallback {
override fun onSuccess(videos: List<TrainingVideoSummary>) {
catalogVideos = videos
catalogStatusText.text = if (videos.isEmpty()) {
"暂无云端课程"
} else {
"${videos.size} 个云端课程"
}
renderCatalog()
}
override fun onError(error: Throwable) {
catalogStatusText.text = "云端暂不可用"
renderCatalog()
}
})
}
private fun renderCatalog() {
if (!::catalogList.isInitialized) {
return
}
catalogList.removeAllViews()
catalogList.addView(
videoCard(
id = SAMPLE_ID,
title = "示例课程",
meta = "4 句 · ${formatTime(16_000L)}",
) {
loadSampleLesson()
}
)
catalogVideos.forEach { video ->
catalogList.addView(
videoCard(
id = video.videoHash,
title = video.title,
meta = "${video.sentenceCount} 句 · ${formatTime(video.durationMs ?: -1L)}",
) {
loadRemoteVideo(video)
}
)
}
}
private fun loadRemoteVideo(video: TrainingVideoSummary) {
activeItemId = video.videoHash
lessonTitleText.text = video.title
sentenceMetaText.text = "句子边界加载中"
sentenceText.text = video.title
statusText.text = "正在加载云端课程..."
renderCatalog()
val loadingItem = video.toTrainingMediaItem()
controller.loadItem(loadingItem)
currentSentenceCount = video.sentenceCount
sdk.sentenceBoundaryApi.fetch(video.videoHash, object : SentenceBoundaryApiCallback {
override fun onSuccess(result: SentenceBoundaryApiResult) {
if (activeItemId != video.videoHash) {
return
}
currentSentenceCount = result.sentences.size
controller.loadItem(video.toTrainingMediaItem(result.sentences))
statusText.text = "课程已就绪:${result.sentences.size}"
}
override fun onError(error: Throwable) {
if (activeItemId != video.videoHash) {
return
}
statusText.text = "句子边界加载失败:${error.message.orEmpty()}"
sentenceMetaText.text = "暂无句子边界"
sentenceText.text = video.title
}
})
}
private fun videoCard(
id: String,
title: String,
meta: String,
onClick: () -> Unit,
): View {
val selected = activeItemId == id
val titleColor = if (selected) COLOR_TEXT_DARK else Color.WHITE
val metaColor = if (selected) COLOR_ACCENT_DEEP else COLOR_TEXT_MUTED
return LinearLayout(this).apply {
orientation = LinearLayout.VERTICAL
gravity = Gravity.CENTER_VERTICAL
isClickable = true
background = if (selected) {
rounded(COLOR_SELECTED, 8f, COLOR_ACCENT)
} else {
rounded(COLOR_SURFACE, 8f, COLOR_BORDER)
}
setPadding(14.dp, 12.dp, 14.dp, 12.dp)
layoutParams = LinearLayout.LayoutParams(218.dp, 84.dp).withMargins(0, 0, 10.dp, 0)
setOnClickListener { onClick() }
addView(TextView(this@MainActivity).apply {
text = title
setTextColor(titleColor)
textSize = 15f
typeface = Typeface.DEFAULT_BOLD
maxLines = 2
})
addView(TextView(this@MainActivity).apply {
text = meta
setTextColor(metaColor)
textSize = 12f
setPadding(0, 6.dp, 0, 0)
})
}
}
private fun headerButton(label: String, action: () -> Unit): Button {
return Button(this).apply {
text = label
setOnClickListener { action() }
isAllCaps = false
setTextColor(Color.WHITE)
textSize = 14f
typeface = Typeface.DEFAULT_BOLD
minWidth = 0
minHeight = 0
minimumWidth = 0
minimumHeight = 0
setPadding(14.dp, 0, 14.dp, 0)
background = rounded(COLOR_BUTTON, 8f)
layoutParams = LinearLayout.LayoutParams(
0,
ViewGroup.LayoutParams.WRAP_CONTENT,
1f,
).apply {
marginStart = 4
marginEnd = 4
40.dp,
).withMargins(8.dp, 0, 0, 0)
setOnClickListener { action() }
}
}
private fun overlayPill(label: String): TextView {
return TextView(this).apply {
text = label
setTextColor(Color.WHITE)
textSize = 13f
typeface = Typeface.DEFAULT_BOLD
background = rounded(COLOR_OVERLAY, 8f)
setPadding(10.dp, 6.dp, 10.dp, 6.dp)
gravity = Gravity.CENTER
}
}
private fun openLocalVideoPicker() {
@@ -185,27 +518,103 @@ class MainActivity : Activity() {
private fun sampleOnlineLesson(): TrainingMediaItem {
return TrainingMediaItem(
id = "online_sample_01",
title = "Online Sample Lesson",
id = SAMPLE_ID,
title = "口语宝示例课",
uri = Uri.parse("https://storage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4"),
mimeType = MimeTypes.VIDEO_MP4,
customCacheKey = "online_sample_01",
customCacheKey = SAMPLE_ID,
sentences = listOf(
SentenceBoundary(0, 0L, 3_000L, "Listen once, then imitate."),
SentenceBoundary(1, 3_000L, 7_000L, "Swipe left to go back."),
SentenceBoundary(2, 7_000L, 11_000L, "Swipe right to move forward."),
SentenceBoundary(3, 11_000L, 16_000L, "Tap the video area to pause or play."),
SentenceBoundary(0, 0L, 3_000L, "Good morning, everyone."),
SentenceBoundary(1, 3_000L, 7_000L, "Today we will practice listening carefully."),
SentenceBoundary(2, 7_000L, 11_000L, "Please repeat each sentence clearly."),
SentenceBoundary(3, 11_000L, 16_000L, "Small daily practice makes progress visible."),
),
)
}
private fun format(ms: Long): String {
private fun playbackStatus(snapshot: PlaybackSnapshot): String {
val state = when {
snapshot.isPlaying -> "播放中"
snapshot.playbackState.name == "BUFFERING" -> "缓冲中"
snapshot.playbackState.name == "ENDED" -> "已结束"
else -> "已暂停"
}
val sentence = snapshot.sentenceIndex?.let { " · 第 ${it + 1}" }.orEmpty()
return "$state$sentence"
}
private fun playbackProgress(snapshot: PlaybackSnapshot): Int {
if (snapshot.durationMs <= 0L) {
return 0
}
return ((snapshot.positionMs.coerceAtMost(snapshot.durationMs) * PROGRESS_MAX) / snapshot.durationMs).toInt()
}
private fun formatTime(ms: Long): String {
if (ms < 0) {
return "--:--"
}
val totalSeconds = ms / 1000
val minutes = totalSeconds / 60
val hours = totalSeconds / 3600
val minutes = (totalSeconds % 3600) / 60
val seconds = totalSeconds % 60
return "%02d:%02d".format(minutes, seconds)
return if (hours > 0) {
"%d:%02d:%02d".format(Locale.US, hours, minutes, seconds)
} else {
"%02d:%02d".format(Locale.US, minutes, seconds)
}
}
private fun formatSpeed(speed: Float): String {
val value = String.format(Locale.US, "%.2f", speed).trimEnd('0').trimEnd('.')
return "${value}x"
}
private fun rounded(
color: Int,
radiusDp: Float,
strokeColor: Int? = null,
): GradientDrawable {
return GradientDrawable().apply {
setColor(color)
cornerRadius = radiusDp.dp.toFloat()
strokeColor?.let { setStroke(1.dp, it) }
}
}
private fun LinearLayout.LayoutParams.withMargins(
left: Int,
top: Int,
right: Int,
bottom: Int,
): LinearLayout.LayoutParams {
setMargins(left, top, right, bottom)
return this
}
private val Int.dp: Int
get() = (this * resources.displayMetrics.density).toInt()
private val Float.dp: Int
get() = (this * resources.displayMetrics.density).toInt()
private companion object {
const val PICK_VIDEO_REQUEST = 1001
const val SAMPLE_ID = "online_sample_01"
const val PROGRESS_MAX = 1000
val COLOR_BACKGROUND: Int = Color.rgb(12, 15, 18)
val COLOR_SURFACE: Int = Color.rgb(28, 34, 40)
val COLOR_BORDER: Int = Color.rgb(50, 59, 67)
val COLOR_BUTTON: Int = Color.rgb(37, 99, 235)
val COLOR_ACCENT: Int = Color.rgb(21, 184, 132)
val COLOR_ACCENT_DEEP: Int = Color.rgb(7, 118, 86)
val COLOR_SELECTED: Int = Color.rgb(229, 248, 240)
val COLOR_PROGRESS_TRACK: Int = Color.rgb(224, 231, 235)
val COLOR_LIGHT_BORDER: Int = Color.rgb(218, 226, 232)
val COLOR_OVERLAY: Int = Color.argb(178, 9, 12, 16)
val COLOR_TEXT_DARK: Int = Color.rgb(18, 24, 31)
val COLOR_TEXT_MUTED: Int = Color.rgb(151, 162, 174)
val COLOR_TEXT_SUBTLE: Int = Color.rgb(88, 98, 108)
}
}

View File

@@ -0,0 +1,3 @@
<resources>
<string name="app_name">口语宝</string>
</resources>

BIN
output/karoline_audio.m4a Normal file

Binary file not shown.

View File

@@ -0,0 +1 @@
{"error":{"message":"Model 'MOSS-Transcribe-Diarize-0.9B-MLX-5bit' failed to load: No safetensors found in /Users/liushuming/.omlx/models/aufklarer/MOSS-Transcribe-Diarize-0.9B-MLX-5bit. Reload models after fixing the files to retry.","type":"server_error","param":null,"code":null}}

View File

@@ -0,0 +1 @@
{"error":{"message":"Model type moss_transcribe_diarize not supported.","type":"server_error","param":null,"code":null}}

Binary file not shown.

14
sentence_api/.env.example Normal file
View File

@@ -0,0 +1,14 @@
ORAL_TRAINER_DATA_DIR=/app/sentence_api/data
PUBLIC_BASE_URL=https://video_service.d1kt.cn
ADMIN_API_KEY=replace-with-a-long-random-secret
CLIENT_API_KEY=replace-with-a-separate-client-secret
MAX_VIDEO_UPLOAD_BYTES=12884901888
MAX_ATTEMPT_UPLOAD_BYTES=52428800
KEEP_ATTEMPT_AUDIO=false
ASSESSMENT_PASS_SCORE=70
MOSS_TRANSCRIBE_URL=http://127.0.0.1:8001/v1/audio/transcriptions
MOSS_MODEL=OpenMOSS-Team/MOSS-Transcribe-Diarize
MOSS_TIMEOUT_SECONDS=1800
MOSS_MAX_NEW_TOKENS=65536

314
sentence_api/DEPLOYMENT.md Normal file
View File

@@ -0,0 +1,314 @@
# 口语训练视频服务部署指南
本文部署以下两个进程:
- `oral-trainer-api`视频上传、管理后台、边界查询、Range 视频播放和朗读评分。
- `MOSS-Transcribe-Diarize`GPU 转写服务,只在服务器内网监听。
推荐 Ubuntu 22.04/24.04、Python 3.12、FFmpeg、NVIDIA GPU 和已安装的 NVIDIA 驱动。MOSS 0.9B 的实际显存占用受推理框架、并发和音频长度影响,生产环境建议从 16 GB 以上显存开始验证。
## 1. 目录与端口
本文假设代码位于 `/opt/oral-trainer`
```text
/opt/oral-trainer/
sentence_api/
data/
v/ 上传的视频
work/ 临时转码文件
attempts/ 可选的学生录音
oral_trainer.sqlite3
```
端口规划:
| 服务 | 监听地址 | 用途 |
| --- | --- | --- |
| MOSS | `127.0.0.1:8001` | 内部语音转写 |
| FastAPI | `127.0.0.1:8000` | 内部应用服务 |
| Nginx | `0.0.0.0:443` | 对外 HTTPS |
不要把 MOSS 的 `8001` 端口直接暴露到公网。
## 2. 部署 MOSS
官方模型为 `OpenMOSS-Team/MOSS-Transcribe-Diarize`。CUDA 12 环境可使用官方当前说明中的 vLLM 构建:
```bash
sudo apt update
sudo apt install -y ffmpeg git curl
curl -LsSf https://astral.sh/uv/install.sh | sh
sudo mkdir -p /opt/moss-transcribe
sudo chown "$USER":"$USER" /opt/moss-transcribe
cd /opt/moss-transcribe
uv venv --python 3.12 .venv
. .venv/bin/activate
uv pip install -U vllm \
--torch-backend=auto \
--extra-index-url https://wheels.vllm.ai/68b4a1d582818e67adc903bf1b8fc5a5447da2fa/cu129
```
启动模型:
```bash
. /opt/moss-transcribe/.venv/bin/activate
vllm serve OpenMOSS-Team/MOSS-Transcribe-Diarize \
--host 127.0.0.1 \
--port 8001 \
--trust-remote-code
```
生产环境可把它注册为 systemd 服务:
```ini
# /etc/systemd/system/moss-transcribe.service
[Unit]
Description=MOSS Transcribe Service
After=network-online.target
[Service]
Type=simple
User=oraltrainer
WorkingDirectory=/opt/moss-transcribe
ExecStart=/opt/moss-transcribe/.venv/bin/vllm serve OpenMOSS-Team/MOSS-Transcribe-Diarize --host 127.0.0.1 --port 8001 --trust-remote-code
Restart=on-failure
RestartSec=10
TimeoutStartSec=1800
[Install]
WantedBy=multi-user.target
```
```bash
sudo systemctl daemon-reload
sudo systemctl enable --now moss-transcribe
sudo journalctl -u moss-transcribe -f
```
模型第一次启动会下载权重。CUDA 13 服务器也可以按照 MOSS 官方仓库说明改用 SGLang Omni。推理框架的安装地址可能随上游版本更新正式部署前应对照 [MOSS 官方仓库](https://github.com/OpenMOSS/MOSS-Transcribe-Diarize) 的 Quickstart。
验证:
```bash
curl -X POST http://127.0.0.1:8001/v1/audio/transcriptions \
-F model=OpenMOSS-Team/MOSS-Transcribe-Diarize \
-F file=@/path/to/test.wav \
-F response_format=verbose_json \
-F temperature=0
```
响应应包含 `text`;开启 `verbose_json` 后应包含带 `start``end``text``segments`
## 3. 部署 API
### Docker Compose 方式
```bash
cd /opt/oral-trainer/sentence_api
cp .env.example .env
openssl rand -hex 32
```
分别生成两个随机值,填入 `.env``ADMIN_API_KEY``CLIENT_API_KEY`,并确认:
```dotenv
PUBLIC_BASE_URL=https://video_service.d1kt.cn
MOSS_TRANSCRIBE_URL=http://127.0.0.1:8001/v1/audio/transcriptions
```
启动:
```bash
docker compose up -d --build
docker compose logs -f oral-trainer-api
```
Compose 配置使用 Linux 的 host network使容器可以访问只监听
`127.0.0.1:8001` 的 MOSS同时 API 也只监听 `127.0.0.1:8000`。如果在
Docker Desktop 上做本地测试,可移除 `network_mode: host`,恢复端口映射,并把
MOSS 地址改为 `host.docker.internal`
检查:
```bash
curl http://127.0.0.1:8000/healthz
```
`moss_configured` 应为 `true`。这里只表示地址已配置,实际连通性会在上传视频或评分时验证。
### 不使用 Docker
```bash
cd /opt/oral-trainer
python3.12 -m venv .venv-api
. .venv-api/bin/activate
python -m pip install -r sentence_api/requirements.txt
export ORAL_TRAINER_DATA_DIR=/opt/oral-trainer-data
export PUBLIC_BASE_URL=https://video_service.d1kt.cn
export ADMIN_API_KEY='替换为随机密钥'
export MOSS_TRANSCRIBE_URL=http://127.0.0.1:8001/v1/audio/transcriptions
python -m uvicorn sentence_api.main:app \
--host 127.0.0.1 --port 8000 --workers 1 \
--proxy-headers --forwarded-allow-ips='*'
```
不使用 Docker 时可创建 API 的 systemd 服务:
```ini
# /etc/systemd/system/oral-trainer-api.service
[Unit]
Description=Oral Trainer Video API
After=network-online.target moss-transcribe.service
[Service]
Type=simple
User=oraltrainer
WorkingDirectory=/opt/oral-trainer
EnvironmentFile=/opt/oral-trainer/sentence_api/.env
ExecStart=/opt/oral-trainer/.venv-api/bin/python -m uvicorn sentence_api.main:app --host 127.0.0.1 --port 8000 --workers 1 --proxy-headers --forwarded-allow-ips=*
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
```
```bash
sudo systemctl daemon-reload
sudo systemctl enable --now oral-trainer-api
sudo journalctl -u oral-trainer-api -f
```
当前上传后的处理任务运行在 API 进程中,因此只能使用一个 Uvicorn worker。需要多机或多 worker 时,应把 `VideoProcessor.process` 迁移到 Celery、RQ 或其他持久化任务队列。
## 4. 配置 Nginx 与 HTTPS
```bash
sudo cp sentence_api/nginx.conf.example /etc/nginx/sites-available/oral-trainer
sudo ln -s /etc/nginx/sites-available/oral-trainer /etc/nginx/sites-enabled/oral-trainer
sudo nginx -t
sudo systemctl reload nginx
```
配置文件中的关键项:
- `client_max_body_size 12g`:允许超过 1 GB 的视频。
- `proxy_request_buffering off`:上传时直接流向 FastAPI避免 Nginx 再完整缓存一份。
- `proxy_force_ranges on`:保留 Android 随机拖动播放所需的 HTTP Range。
- `proxy_read_timeout 7200s`:允许长视频处理和慢速上传。
签发证书:
```bash
sudo apt install -y certbot python3-certbot-nginx
sudo certbot --nginx -d video_service.d1kt.cn
```
证书完成后验证:
```bash
curl https://video_service.d1kt.cn/healthz
```
管理后台地址:
```text
https://video_service.d1kt.cn/admin
```
后台的管理密钥保存在当前浏览器的 `localStorage`,接口请求通过 `X-Admin-Key` 发送。
`CLIENT_API_KEY` 应通过 Android 构建配置注入 `OralTrainerSdkConfig.assessmentApiKey`
不要硬编码进公开代码仓库。SDK 会在评分请求中发送 `X-Client-Key`。公开视频列表和播放
接口仍可交给 CDN 缓存GPU 评分接口则受到密钥保护。
## 5. 上传与处理流程
后台上传后,服务会:
1. 管理后台通过 raw body 流式写入临时文件并同步计算 SHA-256不会先在系统临时目录保留完整副本。兼容的 multipart 接口仍以 4 MB 分块复制。
2. 保存到 `data/v/{sha256}.{extension}`
3. 使用 FFmpeg 提取 16 kHz 单声道 WAV。
4. 请求 MOSS 的 `verbose_json` 转写接口。
5. 以 MOSS 时间戳生成句子边界,并计算每句原音有效语音时长。
6. 把视频和句子写入 SQLite状态变为 `ready`
查询处理状态:
```bash
curl https://video_service.d1kt.cn/api/v1/videos
curl https://video_service.d1kt.cn/api/v1/videos/{sha256}
```
如果 MOSS 未配置,上传仍会使用原来的静音检测生成边界,但句子文本为空,无法开始测验。配置好 MOSS 后可在后台点击“重新处理”。
## 6. 视频随机播放
移动端播放地址为:
```text
GET /api/v1/videos/{sha256}/content
```
服务支持 HTTP RangeMedia3/ExoPlayer 可以随机 seek并继续使用 SDK 现有的本地缓存。第一阶段不必强制改成 HLS。
上传前建议把 MP4 处理成 H.264/AAC 并把 `moov` 元数据移动到文件头:
```bash
ffmpeg -i input.mp4 -c copy -movflags +faststart output.mp4
```
如果原始编码不被 Android 广泛支持,再转码:
```bash
ffmpeg -i input.mkv \
-c:v libx264 -preset medium -crf 22 \
-c:a aac -b:a 128k -movflags +faststart output.mp4
```
当并发量明显增长或网络波动较大时,再增加 HLS/DASH 转码和对象存储/CDN。
## 7. 朗读评分接口
```bash
curl -X POST \
https://video_service.d1kt.cn/api/v1/videos/{sha256}/sentences/0/assessments \
-H 'X-Client-Key: 替换为客户端密钥' \
-F audio=@student.wav \
-F language=en
```
当前 `asr-fluency-v1` 评分为:
```text
总分 = 内容分 * 80% + 流畅度 * 20%
流畅度 = 有效语音时长分 * 35% + 停顿分 * 40% + 语速分 * 25%
```
总分不低于 `ASSESSMENT_PASS_SCORE`(默认 70即通过。`pronunciation_score``prosody_score` 目前返回 `null`,避免把 ASR 内容匹配误报为音素发音质量。
服务只向 MOSS 发送语言提示,不发送标准句子作为 prompt 或热词,避免标准答案诱导识别结果。
## 8. 运维与备份
需要持久化备份:
```text
sentence_api/data/v/
sentence_api/data/oral_trainer.sqlite3
```
`work/` 可以清理。`KEEP_ATTEMPT_AUDIO=false` 时学生录音在评分完成后自动删除;设为 `true` 时还需备份和定期清理 `attempts/`,并在产品隐私政策中说明保存期限。
SQLite 在线备份示例:
```bash
sqlite3 sentence_api/data/oral_trainer.sqlite3 \
".backup '/backup/oral_trainer-$(date +%F).sqlite3'"
```
升级前先备份数据库和 `data/v/`。API 重启期间被中断的视频会保留 `processing` 状态,可从后台执行“重新处理”。

20
sentence_api/Dockerfile Normal file
View File

@@ -0,0 +1,20 @@
FROM python:3.12-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1
RUN apt-get update \
&& apt-get install -y --no-install-recommends ffmpeg ca-certificates \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY sentence_api/requirements.txt /app/sentence_api/requirements.txt
RUN python -m pip install --upgrade pip \
&& python -m pip install -r /app/sentence_api/requirements.txt
COPY sentence_api /app/sentence_api
COPY sentence_analysis.py /app/sentence_analysis.py
EXPOSE 8000
CMD ["python", "-m", "uvicorn", "sentence_api.main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1", "--proxy-headers", "--forwarded-allow-ips", "*"]

View File

@@ -1,92 +1,82 @@
# Sentence Boundary API
# Oral Trainer Video Service
This service looks up pre-generated sentence boundaries by the SHA-256 hash of
the exact video bytes. It does not analyze media during an API request.
FastAPI service for the Android oral-training SDK. It provides:
## Install
- chunked video upload with SHA-256 calculation;
- files stored under `data/v/` and metadata stored in SQLite;
- background sentence transcription through MOSS-Transcribe-Diarize;
- a browser-based administration page at `/admin`;
- video catalog and HTTP Range playback;
- sentence-boundary lookup compatible with the existing Android SDK;
- student recording upload and `asr-fluency-v1` assessment;
- the legacy JSON boundary index as a read-only fallback.
From the repository root:
## Local Run
```bash
python3 -m venv .venv-sentence-api
python3.12 -m venv .venv-sentence-api
. .venv-sentence-api/bin/activate
python -m pip install -r sentence_api/requirements.txt
```
## Generate The Index
Generate boundaries with the same silence detector used by the desktop player:
```bash
python -m sentence_api.generate_boundaries \
"/path/to/lesson.mp4" \
--index sentence_api/data/sentence_boundaries.json
```
The command calculates the SHA-256 hash, detects boundaries, converts seconds
to milliseconds, infers each `end_ms` from the next sentence start, and writes
the result atomically into the JSON index. The last sentence ends at the media
duration.
For a large course library, run this command in an ingestion worker and store
the same document in a database or object storage instead of committing the
JSON file to the application image.
## Run
```bash
SENTENCE_BOUNDARIES_FILE=sentence_api/data/sentence_boundaries.json \
export ADMIN_API_KEY=local-development-key
export MOSS_TRANSCRIBE_URL=http://127.0.0.1:8001/v1/audio/transcriptions
python -m uvicorn sentence_api.main:app --host 0.0.0.0 --port 8000
```
The interactive API documentation is available at `/docs`.
Open:
## Request
```http
GET /api/v1/videos/{sha256}/sentence-boundaries
```text
http://127.0.0.1:8000/admin
http://127.0.0.1:8000/docs
```
Example using the demo record in the checked-in index:
When `MOSS_TRANSCRIBE_URL` is empty, uploaded videos still receive silence-based
boundaries, but no reference transcript is produced and assessment is disabled.
## Main Endpoints
```text
GET /healthz
GET /api/v1/videos
GET /api/v1/videos/{sha256}
GET /api/v1/videos/{sha256}/content
GET /api/v1/videos/{sha256}/sentence-boundaries
POST /api/v1/videos/{sha256}/sentences/{index}/assessments
POST /api/v1/admin/videos
PUT /api/v1/admin/videos/raw
POST /api/v1/admin/videos/{sha256}/process
PUT /api/v1/admin/videos/{sha256}/sentences/{index}
DELETE /api/v1/admin/videos/{sha256}
```
Admin endpoints use `X-Admin-Key` when `ADMIN_API_KEY` is configured. Assessment
requests use `X-Client-Key` when `CLIENT_API_KEY` is configured.
## Assessment
```bash
curl http://127.0.0.1:8000/api/v1/videos/468a4d064f6ec49942b45e25ab93c500d31870f978c4f28ff8b3b408852326e0/sentence-boundaries
curl -X POST \
http://127.0.0.1:8000/api/v1/videos/{sha256}/sentences/0/assessments \
-F audio=@student.wav \
-F language=en
```
The MP4 used during development is also indexed. Its hash is
`b6631d5cf48f37fed0ecc623563dd48b7ed660689b4d25d2ebac7fecab807ddc`, and its
generated index contains 244 boundaries.
The current score intentionally measures reading content and fluency:
The response is:
```json
{
"video_hash": "...",
"duration_ms": 16000,
"algorithm_version": "silence-rms-v1",
"sentences": [
{
"index": 0,
"start_ms": 0,
"end_ms": 4230,
"text": null
}
]
}
```text
overall = content * 80% + fluency * 20%
fluency = duration * 35% + pauses * 40% + speech rate * 25%
```
Unknown hashes return `404`. A hash must be a 64-character hexadecimal
SHA-256 digest; malformed values return `422`.
`pronunciation_score` and `prosody_score` remain `null` until a phoneme/GOP
model is connected.
When testing from a physical Android phone, replace `127.0.0.1` with the
computer's LAN IP address. `127.0.0.1` on the phone refers to the phone itself.
For production, expose the API over HTTPS.
## Tests
## Android Request Flow
```bash
python -m pytest sentence_api/tests -q
```
The mobile app should calculate the hash from the selected `content://` URI in
streaming chunks, request the endpoint, map the returned `sentences` to
`SentenceBoundary`, and then call `controller.loadItem`. The hash must be
calculated from the exact bytes of the same video served to the player. For
HTTPS course videos, the course manifest can carry the hash and avoid hashing
the entire remote file on every device.
See [DEPLOYMENT.md](DEPLOYMENT.md) for Docker, MOSS, Nginx, HTTPS, large-file
upload, backup, and production operation instructions.

107
sentence_api/assessment.py Normal file
View File

@@ -0,0 +1,107 @@
import uuid
from pathlib import Path
from typing import Optional
from .audio_metrics import analyze_audio
from .models import AssessmentResult, SentenceBoundaryDocument, TextSubstitution
from .repository import VideoRepository
from .scoring import SCORING_VERSION, score_attempt
from .transcription import Transcriber
class AssessmentService:
def __init__(
self,
repository: VideoRepository,
transcriber: Transcriber,
pass_score: float,
):
self.repository = repository
self.transcriber = transcriber
self.pass_score = pass_score
def assess(
self,
*,
document: SentenceBoundaryDocument,
sentence_index: int,
audio_path: Path,
language: Optional[str] = None,
retained_audio_filename: Optional[str] = None,
) -> AssessmentResult:
if not self.transcriber.available:
raise RuntimeError("MOSS transcription is not configured on this server.")
sentence = next(
(item for item in document.sentences if item.index == sentence_index),
None,
)
if sentence is None:
raise LookupError("Sentence index was not found.")
if not sentence.text or not sentence.text.strip():
raise ValueError("The sentence has no reviewed reference text yet.")
student_metrics = analyze_audio(audio_path)
transcript = self.transcriber.transcribe(
audio_path,
language or sentence.language,
)
recognized_text = transcript.text.strip()
if not recognized_text:
raise ValueError("MOSS did not recognize any speech in the recording.")
reference_duration_ms = sentence.end_ms - sentence.start_ms
reference_speech_duration_ms = (
sentence.reference_speech_duration_ms or reference_duration_ms
)
score = score_attempt(
reference_text=sentence.text,
recognized_text=recognized_text,
reference_speech_duration_ms=reference_speech_duration_ms,
student_metrics=student_metrics,
)
attempt_id = uuid.uuid4().hex
result = AssessmentResult(
attempt_id=attempt_id,
scoring_version=SCORING_VERSION,
overall_score=score.overall_score,
passed=score.overall_score >= self.pass_score,
pass_score=self.pass_score,
content_score=score.content_score,
completeness_score=score.completeness_score,
fluency_score=score.fluency_score,
pronunciation_score=None,
prosody_score=None,
duration_score=score.duration_score,
pause_score=score.pause_score,
speech_rate_score=score.speech_rate_score,
reference_text=sentence.text,
recognized_text=recognized_text,
reference_duration_ms=reference_duration_ms,
reference_speech_duration_ms=reference_speech_duration_ms,
student_recording_duration_ms=student_metrics.recording_duration_ms,
student_speech_duration_ms=student_metrics.speech_duration_ms,
duration_ratio=score.duration_ratio,
missing_tokens=score.missing_tokens,
extra_tokens=score.extra_tokens,
substitutions=[
TextSubstitution(expected=expected, actual=actual)
for expected, actual in score.substitutions
],
feedback=score.feedback,
details={
"score_kind": "朗读内容与流畅度匹配分",
"content_weight": "0.80",
"fluency_weight": "0.20",
"duration_weight_within_fluency": "0.35",
"phoneme_scoring": "not_enabled",
},
)
if self.repository.get_video(document.video_hash) is not None:
self.repository.record_attempt(
attempt_id=attempt_id,
video_hash=document.video_hash,
sentence_index=sentence_index,
result=result.model_dump(mode="json"),
audio_filename=retained_audio_filename,
)
return result

View File

@@ -0,0 +1,123 @@
from dataclasses import dataclass
from pathlib import Path
from typing import Tuple
import numpy as np
SAMPLE_RATE = 16_000
FRAME_SAMPLES = 480
class AudioAnalysisError(RuntimeError):
pass
@dataclass(frozen=True)
class AudioMetrics:
recording_duration_ms: int
speech_duration_ms: int
internal_silence_ms: int
internal_pause_ratio: float
def decode_audio_mono(path: Path) -> Tuple[np.ndarray, int]:
try:
import av
except ImportError as exc:
raise AudioAnalysisError("PyAV is required for audio analysis.") from exc
container = av.open(str(path))
chunks = []
try:
audio_stream = next(
(stream for stream in container.streams if stream.type == "audio"),
None,
)
if audio_stream is None:
raise AudioAnalysisError("The uploaded file does not contain an audio stream.")
resampler = av.AudioResampler(format="fltp", layout="mono", rate=SAMPLE_RATE)
for packet in container.demux(audio_stream):
for frame in packet.decode():
for output in resampler.resample(frame):
chunks.append(output.to_ndarray()[0].astype(np.float32, copy=False))
for output in resampler.resample(None):
chunks.append(output.to_ndarray()[0].astype(np.float32, copy=False))
finally:
container.close()
if not chunks:
raise AudioAnalysisError("The uploaded audio is empty.")
return np.concatenate(chunks), SAMPLE_RATE
def analyze_audio(path: Path) -> AudioMetrics:
samples, sample_rate = decode_audio_mono(path)
return analyze_samples(samples, sample_rate)
def analyze_samples(samples: np.ndarray, sample_rate: int = SAMPLE_RATE) -> AudioMetrics:
if samples.ndim != 1:
samples = samples.reshape(-1)
if samples.size == 0 or sample_rate <= 0:
raise AudioAnalysisError("The uploaded audio is empty.")
recording_duration_ms = max(1, int(round(samples.size / sample_rate * 1000)))
window = max(1, int(round(sample_rate * 0.03)))
complete_frames = int(np.ceil(samples.size / window))
padded = np.pad(samples, (0, complete_frames * window - samples.size))
frames = padded.reshape(complete_frames, window).astype(np.float64, copy=False)
rms = np.sqrt(np.mean(frames * frames, axis=1))
signal_level = float(np.percentile(rms, 95))
if signal_level < 0.002:
raise AudioAnalysisError("No usable speech was detected in the recording.")
noise_floor = float(np.percentile(rms, 10))
threshold = min(signal_level * 0.45, max(0.006, noise_floor * 2.2))
speech = rms >= threshold
# Treat very short gaps inside a word as speech, then reject short clicks.
_bridge_false_runs(speech, max_frames=5)
_remove_true_runs(speech, max_frames=2)
speech_indexes = np.flatnonzero(speech)
if speech_indexes.size == 0:
raise AudioAnalysisError("No usable speech was detected in the recording.")
frame_ms = window / sample_rate * 1000
speech_duration_ms = max(1, int(round(speech.sum() * frame_ms)))
first = int(speech_indexes[0])
last = int(speech_indexes[-1])
internal_frames = max(1, last - first + 1)
internal_silence_frames = int((~speech[first : last + 1]).sum())
internal_silence_ms = int(round(internal_silence_frames * frame_ms))
internal_pause_ratio = internal_silence_frames / internal_frames
return AudioMetrics(
recording_duration_ms=recording_duration_ms,
speech_duration_ms=speech_duration_ms,
internal_silence_ms=internal_silence_ms,
internal_pause_ratio=round(internal_pause_ratio, 4),
)
def _bridge_false_runs(values: np.ndarray, max_frames: int) -> None:
start = None
for index, value in enumerate(values):
if not value and start is None:
start = index
elif value and start is not None:
if start > 0 and index - start <= max_frames:
values[start:index] = True
start = None
def _remove_true_runs(values: np.ndarray, max_frames: int) -> None:
start = None
for index, value in enumerate(values):
if value and start is None:
start = index
elif not value and start is not None:
if index - start <= max_frames:
values[start:index] = False
start = None
if start is not None and len(values) - start <= max_frames:
values[start:] = False

75
sentence_api/config.py Normal file
View File

@@ -0,0 +1,75 @@
import os
from dataclasses import dataclass
from pathlib import Path
def _bool_env(name: str, default: bool) -> bool:
value = os.getenv(name)
if value is None:
return default
return value.strip().lower() in {"1", "true", "yes", "on"}
@dataclass(frozen=True)
class Settings:
data_dir: Path
legacy_boundaries_file: Path
admin_api_key: str
client_api_key: str
public_base_url: str
max_upload_bytes: int
max_attempt_bytes: int
keep_attempt_audio: bool
moss_transcribe_url: str
moss_model: str
moss_timeout_seconds: float
moss_max_new_tokens: int
pass_score: float
@classmethod
def from_env(cls) -> "Settings":
package_dir = Path(__file__).resolve().parent
data_dir = Path(os.getenv("ORAL_TRAINER_DATA_DIR", str(package_dir / "data")))
return cls(
data_dir=data_dir,
legacy_boundaries_file=Path(
os.getenv(
"SENTENCE_BOUNDARIES_FILE",
str(package_dir / "data" / "sentence_boundaries.json"),
)
),
admin_api_key=os.getenv("ADMIN_API_KEY", ""),
client_api_key=os.getenv("CLIENT_API_KEY", ""),
public_base_url=os.getenv("PUBLIC_BASE_URL", "").rstrip("/"),
max_upload_bytes=int(os.getenv("MAX_VIDEO_UPLOAD_BYTES", str(12 * 1024**3))),
max_attempt_bytes=int(os.getenv("MAX_ATTEMPT_UPLOAD_BYTES", str(50 * 1024**2))),
keep_attempt_audio=_bool_env("KEEP_ATTEMPT_AUDIO", False),
moss_transcribe_url=os.getenv("MOSS_TRANSCRIBE_URL", "").strip(),
moss_model=os.getenv(
"MOSS_MODEL",
"OpenMOSS-Team/MOSS-Transcribe-Diarize",
),
moss_timeout_seconds=float(os.getenv("MOSS_TIMEOUT_SECONDS", "1800")),
moss_max_new_tokens=int(os.getenv("MOSS_MAX_NEW_TOKENS", "65536")),
pass_score=float(os.getenv("ASSESSMENT_PASS_SCORE", "70")),
)
@property
def videos_dir(self) -> Path:
return self.data_dir / "v"
@property
def work_dir(self) -> Path:
return self.data_dir / "work"
@property
def attempts_dir(self) -> Path:
return self.data_dir / "attempts"
@property
def database_path(self) -> Path:
return self.data_dir / "oral_trainer.sqlite3"
def ensure_directories(self) -> None:
for path in (self.data_dir, self.videos_dir, self.work_dir, self.attempts_dir):
path.mkdir(parents=True, exist_ok=True)

View File

@@ -0,0 +1,17 @@
services:
oral-trainer-api:
build:
context: ..
dockerfile: sentence_api/Dockerfile
restart: unless-stopped
network_mode: host
env_file:
- .env
command: ["python", "-m", "uvicorn", "sentence_api.main:app", "--host", "127.0.0.1", "--port", "8000", "--workers", "1", "--proxy-headers", "--forwarded-allow-ips", "*"]
volumes:
- ./data:/app/sentence_api/data
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=5)"]
interval: 30s
timeout: 10s
retries: 3

View File

@@ -1,31 +1,161 @@
import os
import hashlib
import hmac
import logging
import mimetypes
import uuid
from pathlib import Path
from typing import Any, Dict
from typing import Any, BinaryIO, Dict, Optional
from fastapi import Depends, FastAPI, HTTPException, Path as ApiPath
from .models import SentenceBoundaryDocument
from .store import BoundaryStore
DEFAULT_INDEX_PATH = Path(__file__).resolve().parent / "data" / "sentence_boundaries.json"
def create_app(store: BoundaryStore = None) -> FastAPI:
application = FastAPI(
title="Oral Trainer Sentence Boundary API",
version="1.0.0",
description="Looks up pre-generated sentence boundaries by video SHA-256.",
from fastapi import (
BackgroundTasks,
Depends,
FastAPI,
File,
Form,
Header,
HTTPException,
Path as ApiPath,
Query,
Request,
UploadFile,
)
index_path = Path(os.getenv("SENTENCE_BOUNDARIES_FILE", str(DEFAULT_INDEX_PATH)))
application.state.boundary_store = store or BoundaryStore(index_path)
from fastapi.responses import FileResponse, Response
from fastapi.staticfiles import StaticFiles
from starlette.concurrency import run_in_threadpool
def get_store() -> BoundaryStore:
return application.state.boundary_store
from .assessment import AssessmentService
from .audio_metrics import AudioAnalysisError
from .config import Settings
from .models import (
AssessmentResult,
SentenceBoundary,
SentenceBoundaryDocument,
SentenceTextUpdate,
VideoDetailResponse,
VideoListResponse,
VideoSummary,
VideoUploadResponse,
)
from .processing import VideoProcessor
from .repository import VideoRepository
from .store import BoundaryStore
from .transcription import MossTranscriber, Transcriber
logger = logging.getLogger(__name__)
SHA256_PATH = ApiPath(
min_length=64,
max_length=64,
pattern=r"^[A-Fa-f0-9]{64}$",
description="SHA-256 hex digest of the exact uploaded video bytes",
)
ALLOWED_VIDEO_EXTENSIONS = {".mp4", ".m4v", ".mov", ".mkv", ".webm"}
class UploadTooLargeError(ValueError):
pass
def create_app(
store: Optional[BoundaryStore] = None,
*,
settings: Optional[Settings] = None,
repository: Optional[VideoRepository] = None,
transcriber: Optional[Transcriber] = None,
) -> FastAPI:
service_settings = settings or Settings.from_env()
service_settings.ensure_directories()
video_repository = repository or VideoRepository(service_settings.database_path)
legacy_store = store or BoundaryStore(service_settings.legacy_boundaries_file)
moss = transcriber or MossTranscriber(
endpoint=service_settings.moss_transcribe_url,
model=service_settings.moss_model,
timeout_seconds=service_settings.moss_timeout_seconds,
max_new_tokens=service_settings.moss_max_new_tokens,
)
processor = VideoProcessor(service_settings, video_repository, moss)
assessment_service = AssessmentService(
video_repository,
moss,
service_settings.pass_score,
)
application = FastAPI(
title="Oral Trainer Video Service",
version="2.0.0",
description="Video ingestion, sentence transcription, streaming, and oral-reading assessment.",
)
application.state.settings = service_settings
application.state.boundary_store = legacy_store
application.state.video_repository = video_repository
application.state.transcriber = moss
application.state.video_processor = processor
application.state.assessment_service = assessment_service
static_dir = Path(__file__).resolve().parent / "static"
if static_dir.is_dir():
application.mount("/static", StaticFiles(directory=static_dir), name="static")
def require_admin(x_admin_key: Optional[str] = Header(default=None)) -> None:
expected = service_settings.admin_api_key
if expected and not hmac.compare_digest(x_admin_key or "", expected):
raise HTTPException(status_code=401, detail="A valid X-Admin-Key header is required.")
def require_client(x_client_key: Optional[str] = Header(default=None)) -> None:
expected = service_settings.client_api_key
if expected and not hmac.compare_digest(x_client_key or "", expected):
raise HTTPException(status_code=401, detail="A valid X-Client-Key header is required.")
def find_document(video_hash: str) -> Optional[SentenceBoundaryDocument]:
return video_repository.get_document(video_hash) or legacy_store.get(video_hash)
@application.get("/healthz")
def healthz(boundary_store: BoundaryStore = Depends(get_store)) -> Dict[str, Any]:
return {"status": "ok", "video_count": boundary_store.count()}
def healthz() -> Dict[str, Any]:
videos = video_repository.list_videos()
return {
"status": "ok",
"video_count": len(videos) + legacy_store.count(),
"managed_video_count": len(videos),
"moss_configured": moss.available,
"scoring_version": "asr-fluency-v1",
}
@application.get("/admin", include_in_schema=False)
def admin_page() -> FileResponse:
page = static_dir / "admin.html"
if not page.is_file():
raise HTTPException(status_code=404, detail="Admin UI is not installed.")
return FileResponse(page)
@application.get("/api/v1/videos", response_model=VideoListResponse)
def list_videos() -> VideoListResponse:
return VideoListResponse(
videos=[_video_summary(row, service_settings) for row in video_repository.list_videos()]
)
@application.get("/api/v1/videos/{video_hash}", response_model=VideoDetailResponse)
def get_video(video_hash: str = SHA256_PATH) -> VideoDetailResponse:
row = video_repository.get_video(video_hash)
if row is None:
raise HTTPException(status_code=404, detail="Video was not found.")
return VideoDetailResponse(
video=_video_summary(row, service_settings),
boundaries=video_repository.get_document(video_hash),
)
@application.get("/api/v1/videos/{video_hash}/content")
def stream_video(video_hash: str = SHA256_PATH) -> FileResponse:
row = video_repository.get_video(video_hash)
if row is None:
raise HTTPException(status_code=404, detail="Video was not found.")
media_path = service_settings.videos_dir / row["stored_filename"]
if not media_path.is_file():
raise HTTPException(status_code=404, detail="Stored video file is missing.")
return FileResponse(
media_path,
media_type=row["content_type"],
headers={"Accept-Ranges": "bytes", "Cache-Control": "public, max-age=3600"},
)
@application.get(
"/api/v1/videos/{video_hash}/sentence-boundaries",
@@ -36,16 +166,8 @@ def create_app(store: BoundaryStore = None) -> FastAPI:
response_model=SentenceBoundaryDocument,
include_in_schema=False,
)
def get_sentence_boundaries(
video_hash: str = ApiPath(
min_length=64,
max_length=64,
pattern=r"^[A-Fa-f0-9]{64}$",
description="SHA-256 hex digest of the exact video bytes",
),
boundary_store: BoundaryStore = Depends(get_store),
) -> SentenceBoundaryDocument:
document = boundary_store.get(video_hash)
def get_sentence_boundaries(video_hash: str = SHA256_PATH) -> SentenceBoundaryDocument:
document = find_document(video_hash)
if document is None:
raise HTTPException(
status_code=404,
@@ -56,7 +178,321 @@ def create_app(store: BoundaryStore = None) -> FastAPI:
)
return document
@application.post(
"/api/v1/admin/videos",
response_model=VideoUploadResponse,
status_code=202,
dependencies=[Depends(require_admin)],
)
async def upload_video(
background_tasks: BackgroundTasks,
file: UploadFile = File(...),
title: Optional[str] = Form(default=None),
language: Optional[str] = Form(default=None),
) -> VideoUploadResponse:
original_name = Path(file.filename or "video.mp4").name
suffix = _validate_video_extension(original_name)
temporary_path = service_settings.work_dir / f"upload-{uuid.uuid4().hex}.part"
try:
video_hash, size_bytes = await run_in_threadpool(
_persist_upload,
file.file,
temporary_path,
service_settings.max_upload_bytes,
)
except UploadTooLargeError as exc:
temporary_path.unlink(missing_ok=True)
raise HTTPException(status_code=413, detail=str(exc)) from exc
except ValueError as exc:
temporary_path.unlink(missing_ok=True)
raise HTTPException(status_code=400, detail=str(exc)) from exc
finally:
await file.close()
content_type = file.content_type or mimetypes.guess_type(original_name)[0] or "video/mp4"
response = _register_uploaded_video(
settings=service_settings,
repository=video_repository,
temporary_path=temporary_path,
video_hash=video_hash,
size_bytes=size_bytes,
original_name=original_name,
suffix=suffix,
content_type=content_type,
title=title,
language=language,
)
background_tasks.add_task(_process_safely, processor, video_hash)
return response
@application.put(
"/api/v1/admin/videos/raw",
response_model=VideoUploadResponse,
status_code=202,
dependencies=[Depends(require_admin)],
)
async def upload_video_stream(
request: Request,
background_tasks: BackgroundTasks,
filename: str = Query(min_length=1, max_length=500),
title: Optional[str] = Query(default=None, max_length=200),
language: Optional[str] = Query(default=None, max_length=32),
) -> VideoUploadResponse:
original_name = Path(filename).name
suffix = _validate_video_extension(original_name)
content_length = request.headers.get("content-length")
if content_length:
try:
declared_size = int(content_length)
except ValueError as exc:
raise HTTPException(status_code=400, detail="Invalid Content-Length header.") from exc
if declared_size > service_settings.max_upload_bytes:
raise HTTPException(status_code=413, detail="Upload exceeds the configured limit.")
temporary_path = service_settings.work_dir / f"upload-{uuid.uuid4().hex}.part"
try:
video_hash, size_bytes = await _persist_request_stream(
request,
temporary_path,
service_settings.max_upload_bytes,
)
except UploadTooLargeError as exc:
raise HTTPException(status_code=413, detail=str(exc)) from exc
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
response = _register_uploaded_video(
settings=service_settings,
repository=video_repository,
temporary_path=temporary_path,
video_hash=video_hash,
size_bytes=size_bytes,
original_name=original_name,
suffix=suffix,
content_type=request.headers.get("content-type") or "application/octet-stream",
title=title,
language=language,
)
background_tasks.add_task(_process_safely, processor, video_hash)
return response
@application.post(
"/api/v1/admin/videos/{video_hash}/process",
status_code=202,
dependencies=[Depends(require_admin)],
)
def reprocess_video(
background_tasks: BackgroundTasks,
video_hash: str = SHA256_PATH,
) -> Dict[str, str]:
if video_repository.get_video(video_hash) is None:
raise HTTPException(status_code=404, detail="Video was not found.")
background_tasks.add_task(_process_safely, processor, video_hash)
return {"video_hash": video_hash.lower(), "status": "processing_queued"}
@application.put(
"/api/v1/admin/videos/{video_hash}/sentences/{sentence_index}",
response_model=SentenceBoundary,
dependencies=[Depends(require_admin)],
)
def update_sentence(
payload: SentenceTextUpdate,
video_hash: str = SHA256_PATH,
sentence_index: int = ApiPath(ge=0),
) -> SentenceBoundary:
sentence = video_repository.update_sentence_text(
video_hash,
sentence_index,
payload.text,
payload.language,
)
if sentence is None:
raise HTTPException(status_code=404, detail="Sentence was not found.")
return sentence
@application.delete(
"/api/v1/admin/videos/{video_hash}",
status_code=204,
dependencies=[Depends(require_admin)],
)
def delete_video(video_hash: str = SHA256_PATH) -> Response:
row = video_repository.delete_video(video_hash)
if row is None:
raise HTTPException(status_code=404, detail="Video was not found.")
(service_settings.videos_dir / row["stored_filename"]).unlink(missing_ok=True)
return Response(status_code=204)
@application.post(
"/api/v1/videos/{video_hash}/sentences/{sentence_index}/assessments",
response_model=AssessmentResult,
dependencies=[Depends(require_client)],
)
async def assess_sentence(
video_hash: str = SHA256_PATH,
sentence_index: int = ApiPath(ge=0),
audio: UploadFile = File(...),
language: Optional[str] = Form(default=None),
) -> AssessmentResult:
document = find_document(video_hash)
if document is None:
raise HTTPException(status_code=404, detail="Video or sentence boundaries were not found.")
if not moss.available:
raise HTTPException(status_code=503, detail="MOSS transcription is not configured.")
suffix = Path(audio.filename or "attempt.wav").suffix.lower() or ".wav"
temporary_name = f"attempt-{uuid.uuid4().hex}{suffix}"
temporary_path = service_settings.attempts_dir / temporary_name
completed = False
try:
await run_in_threadpool(
_persist_upload,
audio.file,
temporary_path,
service_settings.max_attempt_bytes,
)
result = await run_in_threadpool(
assessment_service.assess,
document=document,
sentence_index=sentence_index,
audio_path=temporary_path,
language=language,
retained_audio_filename=temporary_name if service_settings.keep_attempt_audio else None,
)
completed = True
return result
except UploadTooLargeError as exc:
raise HTTPException(status_code=413, detail=str(exc)) from exc
except LookupError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except (ValueError, AudioAnalysisError) as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
except RuntimeError as exc:
raise HTTPException(status_code=502, detail=str(exc)) from exc
finally:
await audio.close()
if not service_settings.keep_attempt_audio or not completed:
temporary_path.unlink(missing_ok=True)
return application
def _persist_upload(source: BinaryIO, destination: Path, max_bytes: int) -> tuple[str, int]:
digest = hashlib.sha256()
size = 0
destination.parent.mkdir(parents=True, exist_ok=True)
try:
with destination.open("wb") as output:
while True:
chunk = source.read(4 * 1024 * 1024)
if not chunk:
break
size += len(chunk)
if size > max_bytes:
raise UploadTooLargeError(f"Upload exceeds the {max_bytes}-byte limit.")
digest.update(chunk)
output.write(chunk)
if size == 0:
raise ValueError("Uploaded file is empty.")
except Exception:
destination.unlink(missing_ok=True)
raise
return digest.hexdigest(), size
async def _persist_request_stream(
request: Request,
destination: Path,
max_bytes: int,
) -> tuple[str, int]:
digest = hashlib.sha256()
size = 0
destination.parent.mkdir(parents=True, exist_ok=True)
try:
with destination.open("wb") as output:
async for chunk in request.stream():
if not chunk:
continue
size += len(chunk)
if size > max_bytes:
raise UploadTooLargeError(f"Upload exceeds the {max_bytes}-byte limit.")
digest.update(chunk)
output.write(chunk)
if size == 0:
raise ValueError("Uploaded file is empty.")
except Exception:
destination.unlink(missing_ok=True)
raise
return digest.hexdigest(), size
def _validate_video_extension(filename: str) -> str:
suffix = Path(filename).suffix.lower()
if suffix not in ALLOWED_VIDEO_EXTENSIONS:
raise HTTPException(
status_code=415,
detail=f"Unsupported video extension: {suffix or '(none)'}",
)
return suffix
def _register_uploaded_video(
*,
settings: Settings,
repository: VideoRepository,
temporary_path: Path,
video_hash: str,
size_bytes: int,
original_name: str,
suffix: str,
content_type: str,
title: Optional[str],
language: Optional[str],
) -> VideoUploadResponse:
stored_filename = f"{video_hash}{suffix}"
destination = settings.videos_dir / stored_filename
if destination.exists():
temporary_path.unlink(missing_ok=True)
else:
temporary_path.replace(destination)
repository.upsert_upload(
video_hash=video_hash,
title=(title or Path(original_name).stem).strip() or video_hash,
filename=original_name,
stored_filename=stored_filename,
content_type=content_type,
size_bytes=size_bytes,
language=language.strip() if language else None,
)
return VideoUploadResponse(
video_hash=video_hash,
status="uploaded",
detail_url=f"/api/v1/videos/{video_hash}",
)
def _process_safely(processor: VideoProcessor, video_hash: str) -> None:
try:
processor.process(video_hash)
except Exception:
logger.exception("Video processing failed for %s", video_hash)
def _video_summary(row: Dict[str, Any], settings: Settings) -> VideoSummary:
relative_stream_url = f"/api/v1/videos/{row['video_hash']}/content"
stream_url = f"{settings.public_base_url}{relative_stream_url}" if settings.public_base_url else relative_stream_url
return VideoSummary(
video_hash=row["video_hash"],
title=row["title"],
filename=row["filename"],
content_type=row["content_type"],
size_bytes=row["size_bytes"],
duration_ms=row["duration_ms"],
language=row["language"],
status=row["status"],
error_message=row["error_message"],
sentence_count=row["sentence_count"],
stream_url=stream_url,
created_at=row["created_at"],
updated_at=row["updated_at"],
)
app = create_app()

View File

@@ -1,4 +1,5 @@
from typing import List, Optional
from datetime import datetime
from typing import Dict, List, Literal, Optional
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
@@ -10,6 +11,8 @@ class SentenceBoundary(BaseModel):
start_ms: int = Field(ge=0)
end_ms: int = Field(gt=0)
text: Optional[str] = None
language: Optional[str] = None
reference_speech_duration_ms: Optional[int] = Field(default=None, gt=0)
@model_validator(mode="after")
def validate_range(self):
@@ -46,3 +49,79 @@ class SentenceBoundaryDocument(BaseModel):
raise ValueError("sentence end_ms cannot exceed duration_ms")
previous_end = sentence.end_ms
return self
VideoStatus = Literal["uploaded", "processing", "ready", "failed"]
class VideoSummary(BaseModel):
model_config = ConfigDict(extra="forbid")
video_hash: str
title: str
filename: str
content_type: str
size_bytes: int = Field(ge=0)
duration_ms: Optional[int] = Field(default=None, gt=0)
language: Optional[str] = None
status: VideoStatus
error_message: Optional[str] = None
sentence_count: int = Field(ge=0)
stream_url: str
created_at: datetime
updated_at: datetime
class VideoListResponse(BaseModel):
videos: List[VideoSummary]
class VideoUploadResponse(BaseModel):
video_hash: str
status: VideoStatus
detail_url: str
class VideoDetailResponse(BaseModel):
video: VideoSummary
boundaries: Optional[SentenceBoundaryDocument] = None
class SentenceTextUpdate(BaseModel):
text: str = Field(min_length=1, max_length=4000)
language: Optional[str] = Field(default=None, max_length=32)
class TextSubstitution(BaseModel):
expected: str
actual: str
class AssessmentResult(BaseModel):
model_config = ConfigDict(extra="forbid")
attempt_id: str
scoring_version: str
overall_score: float = Field(ge=0, le=100)
passed: bool
pass_score: float = Field(ge=0, le=100)
content_score: float = Field(ge=0, le=100)
completeness_score: float = Field(ge=0, le=100)
fluency_score: float = Field(ge=0, le=100)
pronunciation_score: Optional[float] = Field(default=None, ge=0, le=100)
prosody_score: Optional[float] = Field(default=None, ge=0, le=100)
duration_score: float = Field(ge=0, le=100)
pause_score: float = Field(ge=0, le=100)
speech_rate_score: float = Field(ge=0, le=100)
reference_text: str
recognized_text: str
reference_duration_ms: int = Field(gt=0)
reference_speech_duration_ms: int = Field(gt=0)
student_recording_duration_ms: int = Field(gt=0)
student_speech_duration_ms: int = Field(gt=0)
duration_ratio: float = Field(gt=0)
missing_tokens: List[str]
extra_tokens: List[str]
substitutions: List[TextSubstitution]
feedback: str
details: Dict[str, str] = Field(default_factory=dict)

View File

@@ -0,0 +1,30 @@
server {
listen 80;
server_name video_service.d1kt.cn;
client_max_body_size 12g;
client_body_timeout 7200s;
send_timeout 7200s;
location / {
proxy_pass http://127.0.0.1:8000;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_request_buffering off;
proxy_read_timeout 7200s;
proxy_send_timeout 7200s;
}
location ~ ^/api/v1/videos/[0-9a-fA-F]{64}/content$ {
proxy_pass http://127.0.0.1:8000;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header Range $http_range;
proxy_set_header If-Range $http_if_range;
proxy_force_ranges on;
proxy_buffering off;
}
}

144
sentence_api/processing.py Normal file
View File

@@ -0,0 +1,144 @@
import shutil
import subprocess
import uuid
from pathlib import Path
from typing import List, Optional
from .audio_metrics import AudioAnalysisError, analyze_samples, decode_audio_mono
from .config import Settings
from .generate_boundaries import ALGORITHM_VERSION, make_entry
from .models import SentenceBoundary, SentenceBoundaryDocument
from .repository import VideoRepository
from .transcription import Transcript, Transcriber
MOSS_ALGORITHM_VERSION = "moss-timestamp-v1"
class VideoProcessor:
def __init__(
self,
settings: Settings,
repository: VideoRepository,
transcriber: Transcriber,
):
self.settings = settings
self.repository = repository
self.transcriber = transcriber
def process(self, video_hash: str) -> None:
video = self.repository.get_video(video_hash)
if video is None:
raise ValueError(f"Unknown video: {video_hash}")
media_path = self.settings.videos_dir / video["stored_filename"]
if not media_path.is_file():
raise FileNotFoundError(f"Stored video is missing: {media_path.name}")
self.repository.mark_processing(video_hash)
work_path: Optional[Path] = None
try:
if self.transcriber.available:
work_path = self.settings.work_dir / f"{video_hash}-{uuid.uuid4().hex}.wav"
extract_audio(media_path, work_path)
transcript = self.transcriber.transcribe(work_path, video.get("language"))
document = document_from_transcript(
video_hash=video_hash,
duration_ms=_media_duration_ms(media_path),
transcript=transcript,
language=video.get("language"),
audio_path=work_path,
)
if not document.sentences:
raise RuntimeError("MOSS returned no timestamped speech segments.")
self.repository.save_processing_result(document, transcript.text)
else:
entry, _ = make_entry(media_path, video_hash=video_hash)
document = SentenceBoundaryDocument(
video_hash=video_hash,
duration_ms=entry["duration_ms"],
algorithm_version=ALGORITHM_VERSION,
sentences=entry["sentences"],
)
self.repository.save_processing_result(document, None)
except Exception as exc:
self.repository.mark_failed(video_hash, str(exc))
raise
finally:
if work_path is not None:
work_path.unlink(missing_ok=True)
def extract_audio(media_path: Path, output_path: Path) -> None:
ffmpeg = shutil.which("ffmpeg")
if ffmpeg is None:
raise RuntimeError("ffmpeg is required for MOSS transcription but was not found.")
output_path.parent.mkdir(parents=True, exist_ok=True)
command = [
ffmpeg,
"-hide_banner",
"-loglevel",
"error",
"-y",
"-i",
str(media_path),
"-vn",
"-ac",
"1",
"-ar",
"16000",
"-c:a",
"pcm_s16le",
str(output_path),
]
completed = subprocess.run(command, capture_output=True, text=True, timeout=7200)
if completed.returncode != 0:
message = completed.stderr.strip() or "unknown ffmpeg error"
raise RuntimeError(f"Could not extract video audio: {message[-2000:]}")
def document_from_transcript(
*,
video_hash: str,
duration_ms: int,
transcript: Transcript,
language: Optional[str],
audio_path: Path,
) -> SentenceBoundaryDocument:
samples, sample_rate = decode_audio_mono(audio_path)
sentences: List[SentenceBoundary] = []
previous_end = 0
for segment in sorted(transcript.segments, key=lambda item: (item.start_seconds, item.end_seconds)):
start_ms = max(previous_end, int(round(segment.start_seconds * 1000)))
end_ms = min(duration_ms, int(round(segment.end_seconds * 1000)))
if not segment.text.strip() or end_ms <= start_ms:
continue
start_sample = max(0, int(start_ms / 1000 * sample_rate))
end_sample = min(samples.size, int(end_ms / 1000 * sample_rate))
try:
metrics = analyze_samples(samples[start_sample:end_sample], sample_rate)
speech_duration_ms = metrics.speech_duration_ms
except AudioAnalysisError:
speech_duration_ms = end_ms - start_ms
sentences.append(
SentenceBoundary(
index=len(sentences),
start_ms=start_ms,
end_ms=end_ms,
text=segment.text.strip(),
language=language,
reference_speech_duration_ms=max(1, speech_duration_ms),
)
)
previous_end = end_ms
return SentenceBoundaryDocument(
video_hash=video_hash,
duration_ms=duration_ms,
algorithm_version=MOSS_ALGORITHM_VERSION,
sentences=sentences,
)
def _media_duration_ms(path: Path) -> int:
from .generate_boundaries import media_duration_ms
return media_duration_ms(path)

305
sentence_api/repository.py Normal file
View File

@@ -0,0 +1,305 @@
import json
import sqlite3
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional
from .models import SentenceBoundary, SentenceBoundaryDocument
from .store import normalize_video_hash
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
class VideoRepository:
def __init__(self, database_path: Path):
self.database_path = Path(database_path)
self.database_path.parent.mkdir(parents=True, exist_ok=True)
self._initialize()
def _connect(self) -> sqlite3.Connection:
connection = sqlite3.connect(self.database_path, timeout=30)
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA foreign_keys = ON")
connection.execute("PRAGMA journal_mode = WAL")
return connection
def _initialize(self) -> None:
with self._connect() as connection:
connection.executescript(
"""
CREATE TABLE IF NOT EXISTS videos (
video_hash TEXT PRIMARY KEY,
title TEXT NOT NULL,
filename TEXT NOT NULL,
stored_filename TEXT NOT NULL,
content_type TEXT NOT NULL,
size_bytes INTEGER NOT NULL,
duration_ms INTEGER,
language TEXT,
status TEXT NOT NULL,
error_message TEXT,
algorithm_version TEXT,
transcription TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS sentences (
video_hash TEXT NOT NULL,
sentence_index INTEGER NOT NULL,
start_ms INTEGER NOT NULL,
end_ms INTEGER NOT NULL,
text TEXT,
language TEXT,
reference_speech_duration_ms INTEGER,
PRIMARY KEY (video_hash, sentence_index),
FOREIGN KEY (video_hash) REFERENCES videos(video_hash) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS attempts (
attempt_id TEXT PRIMARY KEY,
video_hash TEXT NOT NULL,
sentence_index INTEGER NOT NULL,
result_json TEXT NOT NULL,
audio_filename TEXT,
created_at TEXT NOT NULL,
FOREIGN KEY (video_hash) REFERENCES videos(video_hash) ON DELETE CASCADE
);
"""
)
def upsert_upload(
self,
*,
video_hash: str,
title: str,
filename: str,
stored_filename: str,
content_type: str,
size_bytes: int,
language: Optional[str],
) -> None:
normalized_hash = normalize_video_hash(video_hash)
now = utc_now()
with self._connect() as connection:
existing = connection.execute(
"SELECT created_at FROM videos WHERE video_hash = ?",
(normalized_hash,),
).fetchone()
created_at = existing["created_at"] if existing else now
connection.execute(
"""
INSERT INTO videos (
video_hash, title, filename, stored_filename, content_type,
size_bytes, duration_ms, language, status, error_message,
algorithm_version, transcription, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, NULL, ?, 'uploaded', NULL, NULL, NULL, ?, ?)
ON CONFLICT(video_hash) DO UPDATE SET
title = excluded.title,
filename = excluded.filename,
stored_filename = excluded.stored_filename,
content_type = excluded.content_type,
size_bytes = excluded.size_bytes,
language = excluded.language,
status = 'uploaded',
error_message = NULL,
updated_at = excluded.updated_at
""",
(
normalized_hash,
title,
filename,
stored_filename,
content_type,
size_bytes,
language,
created_at,
now,
),
)
def mark_processing(self, video_hash: str) -> None:
self._update_status(video_hash, "processing", None)
def mark_failed(self, video_hash: str, message: str) -> None:
self._update_status(video_hash, "failed", message[:4000])
def _update_status(self, video_hash: str, status: str, error_message: Optional[str]) -> None:
with self._connect() as connection:
connection.execute(
"UPDATE videos SET status = ?, error_message = ?, updated_at = ? WHERE video_hash = ?",
(status, error_message, utc_now(), normalize_video_hash(video_hash)),
)
def save_processing_result(
self,
document: SentenceBoundaryDocument,
transcription: Optional[str],
) -> None:
with self._connect() as connection:
connection.execute(
"DELETE FROM sentences WHERE video_hash = ?",
(document.video_hash,),
)
connection.executemany(
"""
INSERT INTO sentences (
video_hash, sentence_index, start_ms, end_ms, text,
language, reference_speech_duration_ms
) VALUES (?, ?, ?, ?, ?, ?, ?)
""",
[
(
document.video_hash,
sentence.index,
sentence.start_ms,
sentence.end_ms,
sentence.text,
sentence.language,
sentence.reference_speech_duration_ms,
)
for sentence in document.sentences
],
)
connection.execute(
"""
UPDATE videos SET duration_ms = ?, status = 'ready', error_message = NULL,
algorithm_version = ?, transcription = ?, updated_at = ?
WHERE video_hash = ?
""",
(
document.duration_ms,
document.algorithm_version,
transcription,
utc_now(),
document.video_hash,
),
)
def list_videos(self) -> List[Dict[str, Any]]:
with self._connect() as connection:
rows = connection.execute(
"""
SELECT v.*, COUNT(s.sentence_index) AS sentence_count
FROM videos v
LEFT JOIN sentences s ON s.video_hash = v.video_hash
GROUP BY v.video_hash
ORDER BY v.created_at DESC
"""
).fetchall()
return [dict(row) for row in rows]
def get_video(self, video_hash: str) -> Optional[Dict[str, Any]]:
with self._connect() as connection:
row = connection.execute(
"""
SELECT v.*, COUNT(s.sentence_index) AS sentence_count
FROM videos v
LEFT JOIN sentences s ON s.video_hash = v.video_hash
WHERE v.video_hash = ?
GROUP BY v.video_hash
""",
(normalize_video_hash(video_hash),),
).fetchone()
return dict(row) if row else None
def get_document(self, video_hash: str) -> Optional[SentenceBoundaryDocument]:
video = self.get_video(video_hash)
if video is None or video["duration_ms"] is None:
return None
with self._connect() as connection:
rows = connection.execute(
"SELECT * FROM sentences WHERE video_hash = ? ORDER BY sentence_index",
(normalize_video_hash(video_hash),),
).fetchall()
return SentenceBoundaryDocument(
video_hash=video["video_hash"],
duration_ms=video["duration_ms"],
algorithm_version=video["algorithm_version"] or "unknown",
sentences=[
SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
for row in rows
],
)
def update_sentence_text(
self,
video_hash: str,
sentence_index: int,
text: str,
language: Optional[str],
) -> Optional[SentenceBoundary]:
normalized_hash = normalize_video_hash(video_hash)
with self._connect() as connection:
cursor = connection.execute(
"""
UPDATE sentences SET text = ?, language = COALESCE(?, language)
WHERE video_hash = ? AND sentence_index = ?
""",
(text.strip(), language, normalized_hash, sentence_index),
)
if cursor.rowcount == 0:
return None
connection.execute(
"UPDATE videos SET updated_at = ? WHERE video_hash = ?",
(utc_now(), normalized_hash),
)
row = connection.execute(
"SELECT * FROM sentences WHERE video_hash = ? AND sentence_index = ?",
(normalized_hash, sentence_index),
).fetchone()
return SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
def record_attempt(
self,
*,
attempt_id: str,
video_hash: str,
sentence_index: int,
result: Dict[str, Any],
audio_filename: Optional[str],
) -> None:
with self._connect() as connection:
connection.execute(
"""
INSERT INTO attempts (
attempt_id, video_hash, sentence_index, result_json,
audio_filename, created_at
) VALUES (?, ?, ?, ?, ?, ?)
""",
(
attempt_id,
normalize_video_hash(video_hash),
sentence_index,
json.dumps(result, ensure_ascii=False),
audio_filename,
utc_now(),
),
)
def delete_video(self, video_hash: str) -> Optional[Dict[str, Any]]:
video = self.get_video(video_hash)
if video is None:
return None
with self._connect() as connection:
connection.execute(
"DELETE FROM videos WHERE video_hash = ?",
(normalize_video_hash(video_hash),),
)
return video

View File

@@ -3,5 +3,6 @@ uvicorn[standard]>=0.30,<1
pydantic>=2.7,<3
pytest>=8,<9
httpx>=0.27,<1
python-multipart>=0.0.9,<1
av>=12.0
numpy>=1.26

241
sentence_api/scoring.py Normal file
View File

@@ -0,0 +1,241 @@
import math
import unicodedata
from dataclasses import dataclass
from typing import List, Sequence, Tuple
from .audio_metrics import AudioMetrics
SCORING_VERSION = "asr-fluency-v1"
@dataclass(frozen=True)
class TextAlignment:
reference_tokens: List[str]
recognized_tokens: List[str]
matches: int
missing_tokens: List[str]
extra_tokens: List[str]
substitutions: List[Tuple[str, str]]
content_score: float
completeness_score: float
@dataclass(frozen=True)
class ScoreBreakdown:
overall_score: float
content_score: float
completeness_score: float
fluency_score: float
duration_score: float
pause_score: float
speech_rate_score: float
duration_ratio: float
missing_tokens: List[str]
extra_tokens: List[str]
substitutions: List[Tuple[str, str]]
feedback: str
def tokenize(text: str) -> List[str]:
normalized = unicodedata.normalize("NFKC", text).lower().strip()
tokens: List[str] = []
word: List[str] = []
def flush_word() -> None:
if word:
token = "".join(word).strip("'")
if token:
tokens.append(token)
word.clear()
for char in normalized:
if _is_cjk(char):
flush_word()
tokens.append(char)
elif char.isalnum() or (char == "'" and word):
word.append(char)
else:
flush_word()
flush_word()
return tokens
def align_text(reference_text: str, recognized_text: str) -> TextAlignment:
reference = tokenize(reference_text)
recognized = tokenize(recognized_text)
if not reference:
raise ValueError("Reference text does not contain any scoreable tokens.")
costs = [[0] * (len(recognized) + 1) for _ in range(len(reference) + 1)]
for row in range(1, len(reference) + 1):
costs[row][0] = row
for column in range(1, len(recognized) + 1):
costs[0][column] = column
for row in range(1, len(reference) + 1):
for column in range(1, len(recognized) + 1):
substitution_cost = 0 if reference[row - 1] == recognized[column - 1] else 1
costs[row][column] = min(
costs[row - 1][column] + 1,
costs[row][column - 1] + 1,
costs[row - 1][column - 1] + substitution_cost,
)
row = len(reference)
column = len(recognized)
matches = 0
missing: List[str] = []
extra: List[str] = []
substitutions: List[Tuple[str, str]] = []
while row > 0 or column > 0:
if row > 0 and column > 0:
same = reference[row - 1] == recognized[column - 1]
diagonal_cost = costs[row - 1][column - 1] + (0 if same else 1)
if costs[row][column] == diagonal_cost:
if same:
matches += 1
else:
substitutions.append((reference[row - 1], recognized[column - 1]))
row -= 1
column -= 1
continue
if row > 0 and costs[row][column] == costs[row - 1][column] + 1:
missing.append(reference[row - 1])
row -= 1
else:
extra.append(recognized[column - 1])
column -= 1
missing.reverse()
extra.reverse()
substitutions.reverse()
distance = len(missing) + len(extra) + len(substitutions)
accuracy = max(0.0, 1.0 - distance / len(reference)) * 100
attempted_reference_tokens = matches + len(substitutions)
completeness = attempted_reference_tokens / len(reference) * 100
content = accuracy * 0.7 + completeness * 0.3
return TextAlignment(
reference_tokens=reference,
recognized_tokens=recognized,
matches=matches,
missing_tokens=missing,
extra_tokens=extra,
substitutions=substitutions,
content_score=_round_score(content),
completeness_score=_round_score(completeness),
)
def score_attempt(
*,
reference_text: str,
recognized_text: str,
reference_speech_duration_ms: int,
student_metrics: AudioMetrics,
) -> ScoreBreakdown:
if reference_speech_duration_ms <= 0:
raise ValueError("reference_speech_duration_ms must be positive")
alignment = align_text(reference_text, recognized_text)
ratio = student_metrics.speech_duration_ms / reference_speech_duration_ms
duration = duration_similarity_score(ratio)
pause = pause_score(student_metrics.internal_pause_ratio)
reference_rate = len(alignment.reference_tokens) / (reference_speech_duration_ms / 1000)
recognized_units = max(1, len(alignment.recognized_tokens))
student_rate = recognized_units / (student_metrics.speech_duration_ms / 1000)
rate_ratio = student_rate / reference_rate if reference_rate > 0 else 1.0
rate = symmetric_rate_score(rate_ratio)
fluency = duration * 0.35 + pause * 0.40 + rate * 0.25
overall = alignment.content_score * 0.80 + fluency * 0.20
feedback = build_feedback(alignment, ratio, student_metrics.internal_pause_ratio)
return ScoreBreakdown(
overall_score=_round_score(overall),
content_score=alignment.content_score,
completeness_score=alignment.completeness_score,
fluency_score=_round_score(fluency),
duration_score=_round_score(duration),
pause_score=_round_score(pause),
speech_rate_score=_round_score(rate),
duration_ratio=round(ratio, 4),
missing_tokens=alignment.missing_tokens,
extra_tokens=alignment.extra_tokens,
substitutions=alignment.substitutions,
feedback=feedback,
)
def duration_similarity_score(ratio: float) -> float:
if ratio <= 0:
return 0.0
if 0.80 <= ratio <= 1.30:
return 100.0
if 0.65 <= ratio < 0.80:
return _interpolate(ratio, 0.65, 0.80, 60, 100)
if 1.30 < ratio <= 1.50:
return _interpolate(ratio, 1.30, 1.50, 100, 60)
if 0.50 <= ratio < 0.65:
return _interpolate(ratio, 0.50, 0.65, 20, 60)
if 1.50 < ratio <= 1.80:
return _interpolate(ratio, 1.50, 1.80, 60, 20)
if ratio < 0.50:
return max(0.0, ratio / 0.50 * 20)
return max(0.0, 20 - (ratio - 1.80) / 0.40 * 20)
def pause_score(internal_pause_ratio: float) -> float:
if internal_pause_ratio <= 0.15:
return 100.0
if internal_pause_ratio <= 0.30:
return _interpolate(internal_pause_ratio, 0.15, 0.30, 100, 70)
if internal_pause_ratio <= 0.50:
return _interpolate(internal_pause_ratio, 0.30, 0.50, 70, 20)
return max(0.0, 20 - (internal_pause_ratio - 0.50) / 0.30 * 20)
def symmetric_rate_score(ratio: float) -> float:
if ratio <= 0:
return 0.0
deviation = abs(math.log(ratio))
free_tolerance = math.log(1.20)
if deviation <= free_tolerance:
return 100.0
return max(0.0, 100 - (deviation - free_tolerance) / math.log(2.5) * 100)
def build_feedback(
alignment: TextAlignment,
duration_ratio: float,
internal_pause_ratio: float,
) -> str:
messages = []
if alignment.missing_tokens:
messages.append("存在漏读")
if alignment.substitutions:
messages.append("存在错读")
if alignment.extra_tokens:
messages.append("存在多读")
if duration_ratio < 0.65:
messages.append("朗读明显偏快或内容不完整")
elif duration_ratio > 1.50:
messages.append("朗读速度偏慢")
if internal_pause_ratio > 0.30:
messages.append("句内停顿偏多")
return "".join(messages) if messages else "内容和朗读节奏匹配良好"
def _is_cjk(char: str) -> bool:
codepoint = ord(char)
return (
0x3400 <= codepoint <= 0x4DBF
or 0x4E00 <= codepoint <= 0x9FFF
or 0xF900 <= codepoint <= 0xFAFF
)
def _interpolate(value: float, start: float, end: float, start_score: float, end_score: float) -> float:
return start_score + (value - start) / (end - start) * (end_score - start_score)
def _round_score(value: float) -> float:
return round(min(100.0, max(0.0, value)), 1)

View File

@@ -0,0 +1,117 @@
:root {
color-scheme: light;
font-family: Inter, "PingFang SC", "Microsoft YaHei", sans-serif;
color: #17202a;
background: #f4f6f7;
letter-spacing: 0;
}
* { box-sizing: border-box; }
body { margin: 0; min-width: 320px; }
button, input, select { font: inherit; letter-spacing: 0; }
button { cursor: pointer; }
.topbar {
min-height: 72px;
padding: 14px 28px;
display: flex;
align-items: center;
justify-content: space-between;
gap: 24px;
color: #fff;
background: #20262c;
border-bottom: 3px solid #1f8a70;
}
h1, h2 { margin: 0; letter-spacing: 0; }
h1 { font-size: 21px; line-height: 1.35; }
h2 { font-size: 17px; }
.service-state, .section-heading span { font-size: 13px; color: #66727d; }
.topbar .service-state { color: #c8d0d7; }
.admin-auth { display: flex; align-items: center; gap: 8px; }
.admin-auth label { font-size: 13px; color: #d5dadd; }
input, select {
min-height: 36px;
padding: 7px 10px;
border: 1px solid #b8c0c7;
border-radius: 4px;
background: #fff;
}
button {
min-height: 36px;
padding: 7px 13px;
border: 1px solid #aab4bc;
border-radius: 4px;
color: #26323c;
background: #fff;
}
button:hover { background: #edf1f3; }
button:disabled { cursor: default; opacity: .55; }
.primary { color: #fff; border-color: #176b57; background: #1f8a70; }
.primary:hover { background: #176b57; }
.danger { color: #a12622; border-color: #d4aaa8; }
.icon-button { width: 36px; padding: 0; font-weight: 700; }
main { width: min(1480px, 100%); margin: 0 auto; }
section { padding: 24px 28px; }
.upload-band { background: #fff; border-bottom: 1px solid #dce1e5; }
.video-band { min-height: 400px; }
.section-heading {
min-height: 30px;
display: flex;
align-items: baseline;
justify-content: space-between;
gap: 16px;
margin-bottom: 14px;
}
#upload-form {
display: grid;
grid-template-columns: minmax(260px, 2fr) minmax(180px, 1fr) 150px auto;
align-items: end;
gap: 12px;
}
#upload-form label { display: grid; gap: 5px; font-size: 13px; color: #4f5b64; }
#upload-progress { width: 100%; height: 8px; margin-top: 14px; accent-color: #1f8a70; }
.table-scroll { overflow: auto; border: 1px solid #d7dde1; background: #fff; }
table { width: 100%; border-collapse: collapse; font-size: 13px; }
th, td { padding: 10px 12px; text-align: left; border-bottom: 1px solid #e3e7ea; vertical-align: middle; }
th { position: sticky; top: 0; z-index: 1; color: #44515b; background: #edf1f3; font-weight: 600; }
tr:last-child td { border-bottom: 0; }
.actions-column { min-width: 280px; }
.actions { display: flex; gap: 6px; white-space: nowrap; }
.actions button, .actions a { min-height: 30px; padding: 5px 9px; font-size: 12px; }
.actions a { display: inline-flex; align-items: center; border: 1px solid #aab4bc; border-radius: 4px; color: #26323c; text-decoration: none; }
.status { display: inline-block; min-width: 64px; font-weight: 600; }
.status-ready { color: #167356; }
.status-failed { color: #b12d28; }
.status-processing, .status-uploaded { color: #a45d08; }
.error-message { max-width: 320px; margin-top: 3px; color: #a12622; font-size: 12px; white-space: normal; }
.empty-state { padding: 70px 20px; text-align: center; color: #6f7a82; }
dialog {
width: min(1200px, calc(100vw - 32px));
max-height: calc(100vh - 32px);
padding: 0;
border: 1px solid #909ba3;
border-radius: 6px;
box-shadow: 0 18px 60px rgb(0 0 0 / 25%);
}
dialog::backdrop { background: rgb(20 25 28 / 55%); }
.dialog-heading { padding: 16px 18px; display: flex; justify-content: space-between; align-items: center; border-bottom: 1px solid #dce1e5; }
.dialog-heading span { font-size: 12px; color: #68747d; }
.sentence-scroll { max-height: calc(100vh - 120px); overflow: auto; }
.sentence-scroll textarea { width: min(600px, 45vw); min-width: 260px; min-height: 58px; resize: vertical; padding: 7px; border: 1px solid #b8c0c7; border-radius: 3px; }
.sentence-scroll select { min-width: 90px; }
@media (max-width: 820px) {
.topbar { align-items: stretch; flex-direction: column; padding: 14px 16px; }
.admin-auth { flex-wrap: wrap; }
.admin-auth input { flex: 1 1 150px; }
section { padding: 20px 16px; }
#upload-form { grid-template-columns: 1fr; }
.actions-column { min-width: 250px; }
}

View File

@@ -0,0 +1,96 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>口语训练视频管理</title>
<link rel="stylesheet" href="/static/admin.css">
</head>
<body>
<header class="topbar">
<div>
<h1>口语训练视频管理</h1>
<span id="service-state" class="service-state">检查服务</span>
</div>
<div class="admin-auth">
<label for="admin-key">管理密钥</label>
<input id="admin-key" type="password" autocomplete="current-password">
<button id="save-key" type="button">保存</button>
<button id="refresh" type="button">刷新</button>
</div>
</header>
<main>
<section class="upload-band" aria-labelledby="upload-heading">
<div class="section-heading">
<h2 id="upload-heading">上传视频</h2>
<span id="upload-state"></span>
</div>
<form id="upload-form">
<label class="file-field">
<span>视频文件</span>
<input id="video-file" name="file" type="file" accept="video/*" required>
</label>
<label>
<span>标题</span>
<input id="video-title" name="title" type="text" maxlength="200">
</label>
<label>
<span>语言</span>
<select id="video-language" name="language">
<option value="">自动识别</option>
<option value="en">英语</option>
<option value="zh">中文</option>
</select>
</label>
<button class="primary" type="submit">上传并处理</button>
</form>
<progress id="upload-progress" value="0" max="100" hidden></progress>
</section>
<section class="video-band" aria-labelledby="videos-heading">
<div class="section-heading">
<h2 id="videos-heading">视频库</h2>
<span id="video-count">0 个视频</span>
</div>
<div class="table-scroll">
<table>
<thead>
<tr>
<th>标题</th>
<th>状态</th>
<th>时长</th>
<th>句子</th>
<th>大小</th>
<th>更新时间</th>
<th class="actions-column">操作</th>
</tr>
</thead>
<tbody id="video-rows"></tbody>
</table>
</div>
<div id="empty-state" class="empty-state" hidden>暂无视频</div>
</section>
</main>
<dialog id="sentence-dialog">
<div class="dialog-heading">
<div>
<h2 id="sentence-title">句子文本</h2>
<span id="sentence-meta"></span>
</div>
<button id="close-dialog" class="icon-button" type="button" aria-label="关闭">X</button>
</div>
<div class="sentence-scroll">
<table>
<thead>
<tr><th>#</th><th>时间</th><th>文本</th><th>语言</th><th>操作</th></tr>
</thead>
<tbody id="sentence-rows"></tbody>
</table>
</div>
</dialog>
<script src="/static/admin.js" defer></script>
</body>
</html>

View File

@@ -0,0 +1,282 @@
const state = {
adminKey: localStorage.getItem("oralTrainerAdminKey") || "",
videos: [],
};
const elements = {
adminKey: document.querySelector("#admin-key"),
saveKey: document.querySelector("#save-key"),
refresh: document.querySelector("#refresh"),
serviceState: document.querySelector("#service-state"),
uploadForm: document.querySelector("#upload-form"),
uploadState: document.querySelector("#upload-state"),
uploadProgress: document.querySelector("#upload-progress"),
videoRows: document.querySelector("#video-rows"),
videoCount: document.querySelector("#video-count"),
emptyState: document.querySelector("#empty-state"),
dialog: document.querySelector("#sentence-dialog"),
sentenceTitle: document.querySelector("#sentence-title"),
sentenceMeta: document.querySelector("#sentence-meta"),
sentenceRows: document.querySelector("#sentence-rows"),
closeDialog: document.querySelector("#close-dialog"),
};
elements.adminKey.value = state.adminKey;
elements.saveKey.addEventListener("click", () => {
state.adminKey = elements.adminKey.value;
localStorage.setItem("oralTrainerAdminKey", state.adminKey);
elements.serviceState.textContent = "密钥已保存";
});
elements.refresh.addEventListener("click", loadVideos);
elements.closeDialog.addEventListener("click", () => elements.dialog.close());
elements.uploadForm.addEventListener("submit", uploadVideo);
async function api(path, options = {}) {
const headers = new Headers(options.headers || {});
if (state.adminKey) headers.set("X-Admin-Key", state.adminKey);
const response = await fetch(path, { ...options, headers });
if (!response.ok) {
let detail = `HTTP ${response.status}`;
try {
const payload = await response.json();
detail = typeof payload.detail === "string" ? payload.detail : JSON.stringify(payload.detail);
} catch (_) {}
throw new Error(detail);
}
return response.status === 204 ? null : response.json();
}
async function checkHealth() {
try {
const health = await api("/healthz");
elements.serviceState.textContent = health.moss_configured ? "服务正常 / MOSS 已连接" : "服务正常 / MOSS 未配置";
} catch (error) {
elements.serviceState.textContent = `服务异常: ${error.message}`;
}
}
async function loadVideos() {
try {
const payload = await api("/api/v1/videos");
state.videos = payload.videos;
renderVideos();
} catch (error) {
elements.serviceState.textContent = `加载失败: ${error.message}`;
}
}
function renderVideos() {
elements.videoRows.replaceChildren();
elements.videoCount.textContent = `${state.videos.length} 个视频`;
elements.emptyState.hidden = state.videos.length !== 0;
for (const video of state.videos) {
const row = document.createElement("tr");
row.append(
cellWithError(video.title, video.error_message),
statusCell(video.status),
textCell(formatDuration(video.duration_ms)),
textCell(String(video.sentence_count)),
textCell(formatBytes(video.size_bytes)),
textCell(formatDate(video.updated_at)),
actionCell(video),
);
elements.videoRows.append(row);
}
}
function textCell(value) {
const cell = document.createElement("td");
cell.textContent = value;
return cell;
}
function cellWithError(title, error) {
const cell = textCell(title);
if (error) {
const message = document.createElement("div");
message.className = "error-message";
message.textContent = error;
cell.append(message);
}
return cell;
}
function statusCell(status) {
const cell = document.createElement("td");
const label = document.createElement("span");
label.className = `status status-${status}`;
label.textContent = { uploaded: "已上传", processing: "处理中", ready: "可用", failed: "失败" }[status] || status;
cell.append(label);
return cell;
}
function actionCell(video) {
const cell = document.createElement("td");
const actions = document.createElement("div");
actions.className = "actions";
const play = document.createElement("a");
play.href = video.stream_url;
play.target = "_blank";
play.rel = "noopener";
play.textContent = "播放";
actions.append(play);
actions.append(button("句子", () => openSentences(video)));
actions.append(button("重新处理", () => reprocess(video.video_hash)));
const remove = button("删除", () => deleteVideo(video));
remove.className = "danger";
actions.append(remove);
cell.append(actions);
return cell;
}
function button(label, handler) {
const result = document.createElement("button");
result.type = "button";
result.textContent = label;
result.addEventListener("click", handler);
return result;
}
function uploadVideo(event) {
event.preventDefault();
const file = document.querySelector("#video-file").files[0];
if (!file) return;
const title = document.querySelector("#video-title").value;
const language = document.querySelector("#video-language").value;
const query = new URLSearchParams({ filename: file.name });
if (title) query.set("title", title);
if (language) query.set("language", language);
elements.uploadProgress.hidden = false;
elements.uploadProgress.value = 0;
elements.uploadState.textContent = "上传中";
const request = new XMLHttpRequest();
request.open("PUT", `/api/v1/admin/videos/raw?${query.toString()}`);
if (state.adminKey) request.setRequestHeader("X-Admin-Key", state.adminKey);
request.setRequestHeader("Content-Type", file.type || "application/octet-stream");
request.upload.addEventListener("progress", (progress) => {
if (progress.lengthComputable) elements.uploadProgress.value = progress.loaded / progress.total * 100;
});
request.addEventListener("load", async () => {
if (request.status >= 200 && request.status < 300) {
elements.uploadState.textContent = "上传完成,正在处理";
elements.uploadForm.reset();
await loadVideos();
} else {
elements.uploadState.textContent = `上传失败: ${readXhrError(request)}`;
}
elements.uploadProgress.hidden = true;
});
request.addEventListener("error", () => {
elements.uploadState.textContent = "上传连接中断";
elements.uploadProgress.hidden = true;
});
request.send(file);
}
function readXhrError(request) {
try { return JSON.parse(request.responseText).detail || `HTTP ${request.status}`; }
catch (_) { return `HTTP ${request.status}`; }
}
async function openSentences(video) {
try {
const detail = await api(`/api/v1/videos/${video.video_hash}`);
elements.sentenceTitle.textContent = detail.video.title;
const sentences = detail.boundaries?.sentences || [];
elements.sentenceMeta.textContent = `${sentences.length} 句 / ${detail.video.video_hash}`;
elements.sentenceRows.replaceChildren();
for (const sentence of sentences) {
elements.sentenceRows.append(sentenceRow(video.video_hash, sentence));
}
elements.dialog.showModal();
} catch (error) {
elements.serviceState.textContent = `句子加载失败: ${error.message}`;
}
}
function sentenceRow(videoHash, sentence) {
const row = document.createElement("tr");
const textArea = document.createElement("textarea");
textArea.value = sentence.text || "";
const language = document.createElement("select");
for (const [value, label] of [["", "自动"], ["en", "英语"], ["zh", "中文"]]) {
const option = document.createElement("option");
option.value = value;
option.textContent = label;
option.selected = value === (sentence.language || "");
language.append(option);
}
const textContainer = document.createElement("td");
textContainer.append(textArea);
const languageContainer = document.createElement("td");
languageContainer.append(language);
const saveContainer = document.createElement("td");
const save = button("保存", async () => {
save.disabled = true;
try {
await api(`/api/v1/admin/videos/${videoHash}/sentences/${sentence.index}`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ text: textArea.value, language: language.value || null }),
});
save.textContent = "已保存";
} catch (error) {
save.textContent = error.message;
} finally {
window.setTimeout(() => { save.disabled = false; save.textContent = "保存"; }, 1600);
}
});
saveContainer.append(save);
row.append(
textCell(String(sentence.index + 1)),
textCell(`${formatDuration(sentence.start_ms)} - ${formatDuration(sentence.end_ms)}`),
textContainer,
languageContainer,
saveContainer,
);
return row;
}
async function reprocess(videoHash) {
try {
await api(`/api/v1/admin/videos/${videoHash}/process`, { method: "POST" });
await loadVideos();
} catch (error) {
elements.serviceState.textContent = `处理启动失败: ${error.message}`;
}
}
async function deleteVideo(video) {
if (!window.confirm(`删除“${video.title}”?`)) return;
try {
await api(`/api/v1/admin/videos/${video.video_hash}`, { method: "DELETE" });
await loadVideos();
} catch (error) {
elements.serviceState.textContent = `删除失败: ${error.message}`;
}
}
function formatDuration(ms) {
if (!ms) return "-";
const total = Math.floor(ms / 1000);
const hours = Math.floor(total / 3600);
const minutes = Math.floor(total % 3600 / 60);
const seconds = total % 60;
return hours ? `${hours}:${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}` : `${minutes}:${String(seconds).padStart(2, "0")}`;
}
function formatBytes(bytes) {
if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`;
if (bytes < 1024 ** 3) return `${(bytes / 1024 ** 2).toFixed(1)} MB`;
return `${(bytes / 1024 ** 3).toFixed(2)} GB`;
}
function formatDate(value) {
return new Intl.DateTimeFormat("zh-CN", { dateStyle: "short", timeStyle: "short" }).format(new Date(value));
}
checkHealth();
loadVideos();
window.setInterval(() => {
if (!elements.dialog.open) loadVideos();
}, 5000);

View File

@@ -23,7 +23,11 @@ class BoundaryStore:
self.reload()
def reload(self) -> None:
raw = json.loads(self.index_path.read_text(encoding="utf-8"))
raw = (
json.loads(self.index_path.read_text(encoding="utf-8"))
if self.index_path.is_file()
else {"videos": {}}
)
entries = raw.get("videos", raw)
if not isinstance(entries, dict):
raise ValueError("index JSON must contain a 'videos' object")

View File

@@ -1,8 +1,11 @@
import json
from dataclasses import replace
from fastapi.testclient import TestClient
from sentence_api.main import create_app
from sentence_api.config import Settings
from sentence_api.repository import VideoRepository
from sentence_api.store import BoundaryStore
@@ -26,7 +29,19 @@ def make_client(tmp_path):
}),
encoding="utf-8",
)
return TestClient(create_app(BoundaryStore(index_path)))
settings = replace(
Settings.from_env(),
data_dir=tmp_path / "data",
legacy_boundaries_file=index_path,
)
settings.ensure_directories()
return TestClient(
create_app(
BoundaryStore(index_path),
settings=settings,
repository=VideoRepository(settings.database_path),
)
)
def test_lookup_returns_boundaries(tmp_path):

View File

@@ -0,0 +1,146 @@
import io
import json
import wave
from dataclasses import replace
import numpy as np
from fastapi.testclient import TestClient
from sentence_api.config import Settings
from sentence_api.main import create_app
from sentence_api.models import SentenceBoundaryDocument
from sentence_api.repository import VideoRepository
from sentence_api.store import BoundaryStore
from sentence_api.transcription import Transcript
VIDEO_HASH = "c" * 64
class FakeTranscriber:
available = True
def transcribe(self, audio_path, language=None):
return Transcript(text="The meeting starts at nine.", segments=[])
def make_wav() -> bytes:
sample_rate = 16_000
time = np.arange(sample_rate * 2, dtype=np.float64) / sample_rate
samples = (np.sin(2 * np.pi * 220 * time) * 8000).astype("<i2")
output = io.BytesIO()
with wave.open(output, "wb") as wav_file:
wav_file.setnchannels(1)
wav_file.setsampwidth(2)
wav_file.setframerate(sample_rate)
wav_file.writeframes(samples.tobytes())
return output.getvalue()
def make_client(tmp_path, client_api_key=""):
legacy_path = tmp_path / "legacy.json"
legacy_path.write_text(json.dumps({"videos": {}}), encoding="utf-8")
settings = replace(
Settings.from_env(),
data_dir=tmp_path / "data",
legacy_boundaries_file=legacy_path,
admin_api_key="test-admin-key",
client_api_key=client_api_key,
)
settings.ensure_directories()
repository = VideoRepository(settings.database_path)
stored_filename = f"{VIDEO_HASH}.mp4"
(settings.videos_dir / stored_filename).write_bytes(b"video-placeholder")
repository.upsert_upload(
video_hash=VIDEO_HASH,
title="Lesson",
filename="lesson.mp4",
stored_filename=stored_filename,
content_type="video/mp4",
size_bytes=17,
language="en",
)
repository.save_processing_result(
SentenceBoundaryDocument(
video_hash=VIDEO_HASH,
duration_ms=2000,
algorithm_version="test-v1",
sentences=[
{
"index": 0,
"start_ms": 0,
"end_ms": 2000,
"text": "The meeting starts at nine.",
"language": "en",
"reference_speech_duration_ms": 2000,
}
],
),
"The meeting starts at nine.",
)
app = create_app(
BoundaryStore(legacy_path),
settings=settings,
repository=repository,
transcriber=FakeTranscriber(),
)
return TestClient(app)
def test_assessment_returns_duration_and_content_breakdown(tmp_path):
client = make_client(tmp_path)
response = client.post(
f"/api/v1/videos/{VIDEO_HASH}/sentences/0/assessments",
files={"audio": ("reading.wav", make_wav(), "audio/wav")},
data={"language": "en"},
)
assert response.status_code == 200, response.text
payload = response.json()
assert payload["overall_score"] == 100
assert payload["passed"] is True
assert 0.99 <= payload["duration_ratio"] <= 1.02
assert payload["pronunciation_score"] is None
assert payload["details"]["phoneme_scoring"] == "not_enabled"
def test_admin_routes_require_key(tmp_path):
client = make_client(tmp_path)
response = client.delete(f"/api/v1/admin/videos/{VIDEO_HASH}")
assert response.status_code == 401
def test_catalog_and_range_streaming(tmp_path):
client = make_client(tmp_path)
catalog = client.get("/api/v1/videos")
stream = client.get(
f"/api/v1/videos/{VIDEO_HASH}/content",
headers={"Range": "bytes=0-4"},
)
assert catalog.status_code == 200
assert catalog.json()["videos"][0]["stream_url"].endswith(f"/{VIDEO_HASH}/content")
assert stream.status_code == 206
assert stream.content == b"video"
def test_assessment_client_key_is_enforced_when_configured(tmp_path):
client = make_client(tmp_path, client_api_key="tablet-key")
endpoint = f"/api/v1/videos/{VIDEO_HASH}/sentences/0/assessments"
unauthorized = client.post(
endpoint,
files={"audio": ("reading.wav", make_wav(), "audio/wav")},
)
authorized = client.post(
endpoint,
headers={"X-Client-Key": "tablet-key"},
files={"audio": ("reading.wav", make_wav(), "audio/wav")},
)
assert unauthorized.status_code == 401
assert authorized.status_code == 200

View File

@@ -0,0 +1,56 @@
import numpy as np
from sentence_api.audio_metrics import AudioMetrics, analyze_samples
from sentence_api.scoring import align_text, duration_similarity_score, score_attempt, tokenize
def test_tokenize_supports_mixed_chinese_and_english():
assert tokenize("你好World! Don't stop.") == ["", "", "world", "don't", "stop"]
def test_identical_reading_with_matching_duration_scores_100():
result = score_attempt(
reference_text="The meeting starts at nine.",
recognized_text="The meeting starts at nine",
reference_speech_duration_ms=2000,
student_metrics=AudioMetrics(
recording_duration_ms=2400,
speech_duration_ms=2000,
internal_silence_ms=0,
internal_pause_ratio=0.0,
),
)
assert result.overall_score == 100
assert result.duration_score == 100
assert result.missing_tokens == []
def test_alignment_reports_missing_extra_and_substituted_tokens():
result = align_text(
"The meeting starts at nine",
"The lesson start at nine today",
)
assert result.content_score < 70
assert result.extra_tokens == ["today"]
assert ("meeting", "lesson") in result.substitutions
assert ("starts", "start") in result.substitutions
def test_duration_score_allows_students_to_read_more_slowly():
assert duration_similarity_score(0.8) == 100
assert duration_similarity_score(1.3) == 100
assert duration_similarity_score(1.5) == 60
assert duration_similarity_score(1.8) == 20
def test_vad_excludes_leading_and_trailing_silence():
sample_rate = 16_000
silence = np.zeros(sample_rate // 2, dtype=np.float32)
time = np.arange(sample_rate, dtype=np.float32) / sample_rate
speech = (0.25 * np.sin(2 * np.pi * 220 * time)).astype(np.float32)
metrics = analyze_samples(np.concatenate([silence, speech, silence]), sample_rate)
assert metrics.recording_duration_ms == 2000
assert 900 <= metrics.speech_duration_ms <= 1050

View File

@@ -0,0 +1,7 @@
from sentence_api.store import BoundaryStore
def test_missing_legacy_index_starts_empty(tmp_path):
store = BoundaryStore(tmp_path / "not-created-yet.json")
assert store.count() == 0

View File

@@ -0,0 +1,134 @@
import mimetypes
import re
from dataclasses import dataclass
from pathlib import Path
from typing import List, Optional, Protocol
import httpx
@dataclass(frozen=True)
class TranscriptionSegment:
start_seconds: float
end_seconds: float
text: str
speaker: Optional[str] = None
@dataclass(frozen=True)
class Transcript:
text: str
segments: List[TranscriptionSegment]
class Transcriber(Protocol):
@property
def available(self) -> bool:
...
def transcribe(self, audio_path: Path, language: Optional[str] = None) -> Transcript:
...
class MossTranscriber:
def __init__(
self,
endpoint: str,
model: str,
timeout_seconds: float = 1800,
max_new_tokens: int = 65536,
):
self.endpoint = endpoint.strip()
self.model = model
self.timeout_seconds = timeout_seconds
self.max_new_tokens = max_new_tokens
@property
def available(self) -> bool:
return bool(self.endpoint)
def transcribe(self, audio_path: Path, language: Optional[str] = None) -> Transcript:
if not self.available:
raise RuntimeError("MOSS transcription is not configured on this server.")
content_type = mimetypes.guess_type(audio_path.name)[0] or "application/octet-stream"
data = {
"model": self.model,
"response_format": "verbose_json",
"temperature": "0",
"max_new_tokens": str(self.max_new_tokens),
}
if language:
data["language"] = language
try:
with audio_path.open("rb") as audio_file:
response = httpx.post(
self.endpoint,
data=data,
files={"file": (audio_path.name, audio_file, content_type)},
timeout=httpx.Timeout(self.timeout_seconds, connect=30),
)
response.raise_for_status()
payload = response.json()
except (httpx.HTTPError, ValueError) as exc:
raise RuntimeError(f"MOSS transcription request failed: {exc}") from exc
raw_text = str(payload.get("text") or "").strip()
segments = _parse_json_segments(payload.get("segments"))
if not segments:
segments = _parse_compact_segments(raw_text)
plain_text = " ".join(segment.text for segment in segments).strip() or raw_text
return Transcript(text=plain_text, segments=segments)
def _parse_json_segments(raw_segments) -> List[TranscriptionSegment]:
if not isinstance(raw_segments, list):
return []
segments = []
for item in raw_segments:
if not isinstance(item, dict):
continue
try:
start = float(item["start"])
end = float(item["end"])
except (KeyError, TypeError, ValueError):
continue
text = str(item.get("text") or "").strip()
if text and end > start >= 0:
speaker = item.get("speaker")
segments.append(
TranscriptionSegment(
start_seconds=start,
end_seconds=end,
text=text,
speaker=str(speaker) if speaker is not None else None,
)
)
return segments
_COMPACT_SEGMENT = re.compile(
r"\[(?P<start>\d+(?:\.\d+)?)\]"
r"\[(?P<speaker>S\d+)\]"
r"(?P<text>.*?)"
r"\[(?P<end>\d+(?:\.\d+)?)\]",
flags=re.DOTALL,
)
def _parse_compact_segments(text: str) -> List[TranscriptionSegment]:
segments = []
for match in _COMPACT_SEGMENT.finditer(text):
start = float(match.group("start"))
end = float(match.group("end"))
segment_text = match.group("text").strip()
if segment_text and end > start:
segments.append(
TranscriptionSegment(
start_seconds=start,
end_seconds=end,
text=segment_text,
speaker=match.group("speaker"),
)
)
return segments