smshine/mediaplayer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add sentence service

Browse Source
This commit is contained in:
Shuming Liu
2026-08-14 19:06:04 +08:00
parent f8a6bf24e4
commit 69d2ba986f
19 changed files with 2379 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
android/README.md
View File

@@ -17,6 +17,10 @@ and a placeholder interface for future imitation-quality scoring.
- Swipe right: next sentence; if there is no sentence data, fast-forward by the
configured seek step.
The sample app's previous/next sentence buttons use the same fallback behavior.
Actual sentence navigation requires `SentenceBoundary` timestamps supplied by
the course service or extracted from subtitle/speech analysis data.
These defaults match the desktop player's arrow-key workflow while fitting a
tablet touch screen.
@@ -62,6 +66,50 @@ OralTrainerSdk.init(
)
```
## Sentence Boundary API
The SDK defaults to `https://video_service.d1kt.cn` and requests:
```text
GET /api/v1/videos/{sha256}/sentence-boundaries
```
For a known hash:
```kotlin
sdk.sentenceBoundaryApi.fetch(videoHash, callback)
```
For a local `content://` video, the SDK can hash the file in streaming chunks
before querying the API:
```kotlin
sdk.sentenceBoundaryApi.fetchForUri(videoUri, contentResolver, callback)
```
Override the service only when a staging or private deployment is required:
```kotlin
OralTrainerSdk.init(
context,
OralTrainerSdkConfig(
sentenceBoundaryApiBaseUrl = "https://video_service.d1kt.cn"
)
)
```
## Local Video Testing
The computer path `/Users/...` is not visible to an Android device. For a
quick test, copy the video to the device or use the `选择本地视频` button in the
sample app. The Android document picker returns a `content://` URI, which the
SDK supports without requesting broad storage permissions.
For production courses, keep the video on an HTTPS CDN or object-storage
service and pass its URL as `TrainingMediaItem.uri`. The SDK streams it and
caches downloaded ranges locally. Copying large 4K files to each device is
better reserved for explicitly offline courses.
## Future Imitation Scoring
Provide an implementation of `ImitationQualityAssessor` when the speech

24
android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerController.kt
View File

@@ -242,6 +242,18 @@ class OralTrainerController internal constructor(
return true
}
fun previousSentenceOrRewind() {
if (!config.sentenceMode || !seekToPreviousSentence()) {
rewind()
}
}
fun nextSentenceOrForward() {
if (!config.sentenceMode || !seekToNextSentence()) {
fastForward()
}
}
fun performSwipeAction(action: SwipeAction) {
when (action) {
SwipeAction.NONE -> Unit
@@ -249,16 +261,8 @@ class OralTrainerController internal constructor(
SwipeAction.FORWARD -> fastForward()
SwipeAction.PREVIOUS_SENTENCE -> seekToPreviousSentence()
SwipeAction.NEXT_SENTENCE -> seekToNextSentence()
SwipeAction.PREVIOUS_SENTENCE_OR_REWIND -> {
if (!config.sentenceMode || !seekToPreviousSentence()) {
rewind()
}
}
SwipeAction.NEXT_SENTENCE_OR_FORWARD -> {
if (!config.sentenceMode || !seekToNextSentence()) {
fastForward()
}
}
SwipeAction.PREVIOUS_SENTENCE_OR_REWIND -> previousSentenceOrRewind()
SwipeAction.NEXT_SENTENCE_OR_FORWARD -> nextSentenceOrForward()
}
}

2
android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdk.kt
View File

@@ -9,6 +9,7 @@ class OralTrainerSdk private constructor(
private val appContext = context.applicationContext
val cache: OralTrainerCache = OralTrainerCache(appContext, config)
val sentenceBoundaryApi: SentenceBoundaryApi = SentenceBoundaryApi(appContext, config)
@JvmOverloads
fun createController(
@@ -24,6 +25,7 @@ class OralTrainerSdk private constructor(
}
fun release() {
sentenceBoundaryApi.release()
StreamingCache.release()
}

1
android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdkConfig.kt
View File

@@ -8,4 +8,5 @@ data class OralTrainerSdkConfig @JvmOverloads constructor(
val userAgent: String = "OralTrainerSdk/0.1.0",
val connectTimeoutMs: Int = 15_000,
val readTimeoutMs: Int = 30_000,
val sentenceBoundaryApiBaseUrl: String = "https://video_service.d1kt.cn",
)

196
android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/SentenceBoundaryApi.kt Normal file
View File

@@ -0,0 +1,196 @@
package cn.learningpad.oraltrainer.sdk
import android.content.ContentResolver
import android.content.Context
import android.net.Uri
import android.os.Handler
import android.os.Looper
import org.json.JSONObject
import java.io.BufferedInputStream
import java.io.IOException
import java.io.InterruptedIOException
import java.net.HttpURLConnection
import java.net.URL
import java.security.MessageDigest
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.Future
data class SentenceBoundaryApiResult(
val videoHash: String,
val durationMs: Long,
val algorithmVersion: String,
val sentences: List<SentenceBoundary>,
)
interface SentenceBoundaryApiCallback {
fun onSuccess(result: SentenceBoundaryApiResult)
fun onError(error: Throwable)
}
class CancellableRequest internal constructor(
private val future: Future<*>,
) {
fun cancel() {
future.cancel(true)
}
}
class SentenceBoundaryApi internal constructor(
context: Context,
private val config: OralTrainerSdkConfig,
) {
private val appContext = context.applicationContext
private val mainHandler = Handler(Looper.getMainLooper())
private val executor: ExecutorService = Executors.newCachedThreadPool()
private val baseUri = Uri.parse(config.sentenceBoundaryApiBaseUrl.trimEnd('/'))
init {
require(baseUri.scheme == "https" || baseUri.scheme == "http") {
"sentenceBoundaryApiBaseUrl must use http or https."
}
require(!baseUri.host.isNullOrBlank()) {
"sentenceBoundaryApiBaseUrl must include a host."
}
}
fun fetch(
videoHash: String,
callback: SentenceBoundaryApiCallback,
): CancellableRequest {
validateHash(videoHash)
return submit(callback) { fetchBlocking(videoHash.lowercase()) }
}
fun fetchForUri(
uri: Uri,
contentResolver: ContentResolver = appContext.contentResolver,
callback: SentenceBoundaryApiCallback,
): CancellableRequest {
return submit(callback) {
val videoHash = sha256(uri, contentResolver)
fetchBlocking(videoHash)
}
}
fun release() {
executor.shutdownNow()
}
private fun submit(
callback: SentenceBoundaryApiCallback,
operation: () -> SentenceBoundaryApiResult,
): CancellableRequest {
val future = executor.submit {
try {
val result = operation()
if (!Thread.currentThread().isInterrupted) {
mainHandler.post { callback.onSuccess(result) }
}
} catch (error: Throwable) {
if (!Thread.currentThread().isInterrupted) {
mainHandler.post { callback.onError(error) }
}
}
}
return CancellableRequest(future)
}
private fun fetchBlocking(videoHash: String): SentenceBoundaryApiResult {
val endpoint = baseUri.buildUpon()
.appendPath("api")
.appendPath("v1")
.appendPath("videos")
.appendPath(videoHash)
.appendPath("sentence-boundaries")
.build()
val connection = URL(endpoint.toString()).openConnection() as HttpURLConnection
return try {
connection.requestMethod = "GET"
connection.connectTimeout = config.connectTimeoutMs
connection.readTimeout = config.readTimeoutMs
connection.setRequestProperty("Accept", "application/json")
connection.setRequestProperty("User-Agent", config.userAgent)
val statusCode = connection.responseCode
val stream = if (statusCode in 200..299) {
connection.inputStream
} else {
connection.errorStream
}
val body = stream?.bufferedReader(Charsets.UTF_8)?.use { it.readText() }.orEmpty()
if (statusCode !in 200..299) {
throw SentenceBoundaryApiException(statusCode, body)
}
parseResponse(body, videoHash)
} finally {
connection.disconnect()
}
}
private fun parseResponse(
body: String,
requestedHash: String,
): SentenceBoundaryApiResult {
val root = JSONObject(body)
val responseHash = root.getString("video_hash").lowercase()
if (responseHash != requestedHash.lowercase()) {
throw IOException("Sentence boundary response hash does not match the request.")
}
val jsonSentences = root.getJSONArray("sentences")
val sentences = buildList(jsonSentences.length()) {
for (index in 0 until jsonSentences.length()) {
val sentence = jsonSentences.getJSONObject(index)
add(
SentenceBoundary(
index = sentence.getInt("index"),
startMs = sentence.getLong("start_ms"),
endMs = sentence.getLong("end_ms"),
text = if (sentence.isNull("text")) null else sentence.getString("text"),
)
)
}
}
return SentenceBoundaryApiResult(
videoHash = responseHash,
durationMs = root.getLong("duration_ms"),
algorithmVersion = root.getString("algorithm_version"),
sentences = sentences,
)
}
private fun sha256(uri: Uri, contentResolver: ContentResolver): String {
val digest = MessageDigest.getInstance("SHA-256")
contentResolver.openInputStream(uri)?.use { input ->
BufferedInputStream(input).use { buffered ->
val buffer = ByteArray(1024 * 1024)
while (true) {
if (Thread.currentThread().isInterrupted) {
throw InterruptedIOException()
}
val count = buffered.read(buffer)
if (count < 0) {
break
}
digest.update(buffer, 0, count)
}
}
} ?: throw IOException("Cannot open video URI: $uri")
return digest.digest().joinToString("") { byte -> "%02x".format(byte.toInt() and 0xff) }
}
private fun validateHash(videoHash: String) {
require(SHA256_PATTERN.matches(videoHash)) {
"videoHash must be a 64-character SHA-256 hex digest."
}
}
companion object {
private val SHA256_PATTERN = Regex("^[0-9a-fA-F]{64}$")
}
}
class SentenceBoundaryApiException(
val statusCode: Int,
responseBody: String,
) : IOException("Sentence boundary API returned HTTP $statusCode: $responseBody")

4
android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/StreamingCache.kt
View File

@@ -5,6 +5,7 @@ package cn.learningpad.oraltrainer.sdk
import android.content.Context
import androidx.media3.database.StandaloneDatabaseProvider
import androidx.media3.datasource.DataSource
import androidx.media3.datasource.DefaultDataSource
import androidx.media3.datasource.cache.CacheDataSource
import androidx.media3.datasource.cache.LeastRecentlyUsedCacheEvictor
import androidx.media3.datasource.cache.SimpleCache
@@ -32,11 +33,12 @@ internal object StreamingCache {
context: Context,
config: OralTrainerSdkConfig,
): DataSource.Factory {
val upstreamFactory = DefaultHttpDataSource.Factory()
val httpDataSourceFactory = DefaultHttpDataSource.Factory()
.setUserAgent(config.userAgent)
.setConnectTimeoutMs(config.connectTimeoutMs)
.setReadTimeoutMs(config.readTimeoutMs)
.setAllowCrossProtocolRedirects(true)
val upstreamFactory = DefaultDataSource.Factory(context.applicationContext, httpDataSourceFactory)
return CacheDataSource.Factory()
.setCache(get(context, config))

61
android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt
View File

@@ -1,6 +1,7 @@
package cn.learningpad.oraltrainer.sample
import android.app.Activity
import android.content.Intent
import android.graphics.Color
import android.net.Uri
import android.os.Bundle
@@ -19,9 +20,16 @@ import cn.learningpad.oraltrainer.sdk.OralTrainerSdk
import cn.learningpad.oraltrainer.sdk.PlaybackSnapshot
import cn.learningpad.oraltrainer.sdk.PlayerConfig
import cn.learningpad.oraltrainer.sdk.SentenceBoundary
import cn.learningpad.oraltrainer.sdk.SentenceBoundaryApiCallback
import cn.learningpad.oraltrainer.sdk.TrainingMediaItem
import cn.learningpad.oraltrainer.sdk.SentenceBoundaryApiResult
class MainActivity : Activity() {
private companion object {
const val PICK_VIDEO_REQUEST = 1001
}
private lateinit var sdk: OralTrainerSdk
private lateinit var controller: OralTrainerController
private lateinit var statusText: TextView
private lateinit var sentenceText: TextView
@@ -29,7 +37,7 @@ class MainActivity : Activity() {
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
val sdk = OralTrainerSdk.init(this)
sdk = OralTrainerSdk.init(this)
controller = sdk.createController(
playerConfig = PlayerConfig(
sentenceMode = true,
@@ -66,8 +74,8 @@ class MainActivity : Activity() {
addView(commandButton("快退") { controller.rewind() })
addView(commandButton("播放/暂停") { controller.togglePlayPause() })
addView(commandButton("快进") { controller.fastForward() })
addView(commandButton("上一句") { controller.seekToPreviousSentence() })
addView(commandButton("下一句") { controller.seekToNextSentence() })
addView(commandButton("上一句") { controller.previousSentenceOrRewind() })
addView(commandButton("下一句") { controller.nextSentenceOrForward() })
}
val root = LinearLayout(this).apply {
@@ -77,6 +85,10 @@ class MainActivity : Activity() {
addView(statusText)
addView(sentenceText)
addView(controls)
addView(Button(this@MainActivity).apply {
text = "选择本地视频"
setOnClickListener { openLocalVideoPicker() }
})
}
setContentView(root)
@@ -108,6 +120,37 @@ class MainActivity : Activity() {
controller.loadItem(sampleOnlineLesson())
}
override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
super.onActivityResult(requestCode, resultCode, data)
if (requestCode != PICK_VIDEO_REQUEST || resultCode != RESULT_OK) {
return
}
val uri = data?.data ?: return
val persistableFlags = data.flags and Intent.FLAG_GRANT_READ_URI_PERMISSION
if (persistableFlags != 0) {
runCatching {
contentResolver.takePersistableUriPermission(uri, persistableFlags)
}
}
val item = TrainingMediaItem(
id = uri.toString(),
title = uri.lastPathSegment ?: "本地视频",
uri = uri,
)
controller.loadItem(item)
statusText.text = "正在获取句子边界..."
sdk.sentenceBoundaryApi.fetchForUri(uri, contentResolver, object : SentenceBoundaryApiCallback {
override fun onSuccess(result: SentenceBoundaryApiResult) {
controller.loadItem(item.copy(sentences = result.sentences))
statusText.text = "句子边界已加载：${result.sentences.size} 句"
}
override fun onError(error: Throwable) {
statusText.text = "句子边界获取失败，使用 10 秒跳转：${error.message.orEmpty()}"
}
})
}
override fun onDestroy() {
controller.release()
super.onDestroy()
@@ -128,6 +171,18 @@ class MainActivity : Activity() {
}
}
private fun openLocalVideoPicker() {
startActivityForResult(
Intent(Intent.ACTION_OPEN_DOCUMENT).apply {
addCategory(Intent.CATEGORY_OPENABLE)
type = "video/*"
addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION)
addFlags(Intent.FLAG_GRANT_PERSISTABLE_URI_PERMISSION)
},
PICK_VIDEO_REQUEST,
)
}
private fun sampleOnlineLesson(): TrainingMediaItem {
return TrainingMediaItem(
id = "online_sample_01",