Compare commits

...

40 Commits

Author SHA1 Message Date
d143eb7fed fix a bug 2026-10-29 13:25:39 +08:00
a87228cdc9 fixed bugs 2026-10-29 11:32:33 +08:00
031f3c6f21 later process 2026-10-29 08:48:49 +08:00
03e7987c7c fixed some bugs 2026-10-26 20:23:57 +08:00
ed8df744f9 add readming mode 2026-10-26 17:31:08 +08:00
9b7c7d726b add function to subtitle layer 2026-10-26 12:02:41 +08:00
fe21617c63 fixed a bug 2026-10-25 11:41:08 +08:00
5428c42ffd add debug info 2026-10-25 11:31:32 +08:00
ae84347912 fff 2026-10-25 10:30:22 +08:00
38894ba80c playing in background 2026-10-24 13:04:57 +08:00
532d9900ba repaire sentences 2026-10-21 19:39:04 +08:00
176d722c77 use fast souces 2026-10-08 11:40:52 +08:00
4746331515 pc translate sentences 2026-10-08 10:08:16 +08:00
b28f809c46 add a retranslate chinese function 2026-10-06 14:47:22 +08:00
4fd0fd36a4 regenerate something when the sentences changed 2026-10-03 21:12:12 +08:00
d5609c5c42 continue fixing 2026-10-03 20:52:56 +08:00
beb29429d2 optimized playing screen 2026-10-03 20:30:05 +08:00
9daea7b807 fixed a bug:translate maybe timeout 2026-10-03 19:49:21 +08:00
6bd64ee4a9 add chinese subtitles 2026-10-03 12:14:17 +08:00
138d446d73 remove lessonTitleText in training module 2026-10-03 09:46:38 +08:00
7b63c4eab6 version changed 2026-10-03 09:38:25 +08:00
1a5a756e82 relayout of train view 2026-10-02 11:02:36 +08:00
55fad2c0cd add a refresh action 2026-09-26 10:05:48 +08:00
7fae2bb8c9 title modification 2026-09-26 08:03:20 +08:00
37ee850ee1 add Markdown surpport 2026-09-24 08:42:11 +08:00
2a7f24b1bd ios full screen 2026-09-14 17:48:42 +08:00
3aaa9f4229 add save function of explanation 2026-09-14 16:27:55 +08:00
9c78bb9b04 undo the media format convertion action 2026-09-13 17:30:28 +08:00
53e947c506 try again 2026-09-13 13:12:47 +08:00
5328898971 fixed bugs 2026-09-13 12:58:45 +08:00
f810cfedd4 fixed some bugs on ios 2026-09-13 11:35:46 +08:00
1499db0ed7 fixed some bugs 2026-09-13 09:42:39 +08:00
5856832eca fixed a bug 2026-09-12 21:27:51 +08:00
55d6860cc0 delete .tmp from git 2026-09-12 20:54:02 +08:00
ce8b765103 add Moss result to db 2026-09-12 20:50:54 +08:00
96d0ff5318 fixed a bug 2026-09-12 20:27:42 +08:00
93232a64fa return android 2026-09-12 19:58:52 +08:00
e2cc793bdb fff 2026-09-12 15:55:12 +08:00
8414f36cca ios fixed 2026-09-12 14:01:20 +08:00
ab4ff6c25a webm file supported 2026-09-12 11:28:32 +08:00
80 changed files with 9121 additions and 822 deletions

2
.gitignore vendored
View File

@@ -9,3 +9,5 @@ sentence_api/data/v/
sentence_api/data/work/
sentence_api/data/attempts/
android/sample-app/release/
.tmp/

View File

@@ -14,6 +14,9 @@ pip 安装依赖后即可直接播放视频。
视频的句边界;有则直接取回,没有时才本地提取音频并上传。生成完成后才能
勾选“按句跳转”,之后 `, / .` 或 `← / →` 可跳句。PC 会按文件 SHA-256
缓存结果;生成过程有进度条。
- 画面字幕:句边界就绪后,当前句会叠在视频画面底部。可勾选“中文字幕”
(快捷键 `C`)显示或关闭中文翻译;翻译由服务端生成并缓存。
- 导出字幕:句边界就绪后可点“导出字幕”,默认保存为 UTF-8(带 BOM)的 `.ass`(中英同一条字幕避免重叠),也可另存 `.srt`。
- AI 讲解:播放中按 `E` 或点击“AI讲解 (E)”,由服务端 AI 讲解当前句子。
- 服务配置:点击“服务…”可填写服务器地址,或使用账号密码登录后保存令牌。
- 调试日志:点击“调试日志”可查看/复制请求状态、耗时和错误摘要;日志保存在
@@ -83,6 +86,7 @@ pip install -r requirements.txt
| `N` / `P` | 下一首 / 上一首 |
| `[` / `]` | 减速 / 加速(0.5x - 2.0x) |
| `L` | 切换循环模式(关 / 单曲 / 列表) |
| `C` | 开 / 关中文字幕 |
| 双击画面 | 进入 / 退出全屏 |
| `F` | 视频全屏切换 |
| `Esc` | 退出全屏 |
@@ -105,7 +109,8 @@ pip install -r requirements.txt
静音阈值会根据每个文件的背景噪声自动调整,带环境噪音的录音也能正常分句。
分析在后台进行,首次播放某文件时状态栏会提示“正在分析句边界…”,完成后
自动缓存;无法识别(如纯音乐、无音频轨)时会提示并回退为普通步进跳转。
“按句跳转”的开关状态同样会自动保存。
“按句跳转”和“中文字幕”的开关状态同样会自动保存。句边界就绪后,原文会叠在
视频画面底部;打开中文字幕时会向服务器请求翻译并一并显示。
## 常见问题

View File

@@ -1,3 +1,5 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.WAKE_LOCK" />
<uses-permission android:name="android.permission.ACCESS_WIFI_STATE" />
</manifest>

View File

@@ -10,8 +10,10 @@ import androidx.media3.common.C
import androidx.media3.common.MediaItem
import androidx.media3.common.PlaybackException
import androidx.media3.common.Player
import androidx.media3.common.TrackSelectionParameters.AudioOffloadPreferences
import androidx.media3.exoplayer.ExoPlayer
import androidx.media3.exoplayer.PlayerMessage
import androidx.media3.exoplayer.SeekParameters
import androidx.media3.exoplayer.source.DefaultMediaSourceFactory
import java.util.concurrent.CopyOnWriteArraySet
import kotlin.math.max
@@ -31,8 +33,15 @@ class OralTrainerController internal constructor(
private var lastSentenceIndex: Int? = null
private var stopAtSentenceEnd = false
private var stopAtSentenceEndIndex: Int? = null
private var boundaryMessage: PlayerMessage? = null
private var boundaryGuardToken = 0
private var baseItemIndex = 0
private var clippedSentence: SentenceBoundary? = null
private var updatingPlaybackWindow = false
private var unclippedDurationMs = -1L
private var sentenceEndMessage: PlayerMessage? = null
private var playFromStartAfterReady = false
private var clipStartNeedsWarmup = false
private var clipStartWarmupVolume: Float? = null
private val finishClipStartWarmupRunnable = Runnable { finishClipStartWarmup() }
var config: PlayerConfig = initialConfig
private set
@@ -47,10 +56,10 @@ class OralTrainerController internal constructor(
if (released) {
return
}
enforceSentenceBoundary()
enforceStopAtSentenceEnd()
notifySentenceIfChanged()
notifySnapshot()
mainHandler.postDelayed(this, 250L)
mainHandler.postDelayed(this, IDLE_POLL_MS)
}
}
@@ -60,30 +69,63 @@ class OralTrainerController internal constructor(
)
player = ExoPlayer.Builder(appContext)
.setMediaSourceFactory(mediaSourceFactory)
.setWakeMode(C.WAKE_MODE_NETWORK)
.build()
.also { exoPlayer ->
exoPlayer.repeatMode = Player.REPEAT_MODE_ALL
exoPlayer.playWhenReady = initialConfig.autoPlay
exoPlayer.setSeekParameters(SeekParameters.EXACT)
exoPlayer.setForegroundMode(true)
exoPlayer.trackSelectionParameters = exoPlayer.trackSelectionParameters
.buildUpon()
.setAudioOffloadPreferences(
AudioOffloadPreferences.Builder()
.setAudioOffloadMode(AudioOffloadPreferences.AUDIO_OFFLOAD_MODE_DISABLED)
.build()
)
.build()
exoPlayer.addListener(object : Player.Listener {
override fun onPlaybackStateChanged(playbackState: Int) {
notifySnapshot()
}
override fun onIsPlayingChanged(isPlaying: Boolean) {
if (isPlaying) {
refreshSentenceBoundaryGuard()
} else {
cancelSentenceBoundaryGuard()
if (playbackState == Player.STATE_ENDED) {
if (clipStartWarmupVolume != null) {
finishClipStartWarmup()
} else {
onClippedSentenceEnded()
}
}
if (playbackState == Player.STATE_READY) {
if (playFromStartAfterReady) {
playFromStartAfterReady = false
playClippedSentenceFromStart()
}
if (stopAtSentenceEnd) {
scheduleSentenceEndGuard()
}
if (unclippedDurationMs <= 0L) {
val duration = player.duration
if (duration != C.TIME_UNSET && clippedSentence == null) {
unclippedDurationMs = duration
}
}
}
notifySnapshot()
}
override fun onIsPlayingChanged(isPlaying: Boolean) {
notifySnapshot()
}
override fun onMediaItemTransition(mediaItem: MediaItem?, reason: Int) {
lastSentenceIndex = null
cancelSentenceBoundaryGuard()
if (updatingPlaybackWindow) {
notifySnapshot()
return
}
if (clippedSentence == null) {
baseItemIndex = player.currentMediaItemIndex
lastSentenceIndex = null
}
listeners.forEach { it.onMediaChanged(currentTrainingItem()) }
notifySentenceIfChanged(force = true)
refreshSentenceBoundaryGuard()
notifySnapshot()
}
@@ -93,7 +135,6 @@ class OralTrainerController internal constructor(
reason: Int,
) {
notifySentenceIfChanged(force = true)
refreshSentenceBoundaryGuard()
notifySnapshot()
}
@@ -140,9 +181,18 @@ class OralTrainerController internal constructor(
lastSentenceIndex = null
stopAtSentenceEnd = false
stopAtSentenceEndIndex = null
player.setMediaItems(items.map { it.toMedia3Item() }, safeIndex, max(0L, startPositionMs))
cancelClipStartWarmup(restoreVolume = true)
cancelSentenceEndGuard()
baseItemIndex = safeIndex
clippedSentence = null
unclippedDurationMs = -1L
val startMs = max(0L, startPositionMs)
player.setMediaItems(items.map { it.toMedia3Item() }, safeIndex, startMs)
player.prepare()
player.playWhenReady = config.autoPlay
applyLoopMode()
lockSentenceAt(startMs)
syncPlaybackWindow(startMs)
listeners.forEach { it.onMediaChanged(currentTrainingItem()) }
notifySentenceIfChanged(force = true)
notifySnapshot()
@@ -166,25 +216,32 @@ class OralTrainerController internal constructor(
return true
}
mediaItems[index] = updatedItem
if (index == player.currentMediaItemIndex) {
if (index == baseItemIndex) {
listeners.forEach { it.onMediaChanged(updatedItem) }
lockSentenceAt(playbackPositionMs())
syncPlaybackWindow()
notifySentenceIfChanged(force = true)
refreshSentenceBoundaryGuard()
notifySnapshot()
}
return true
}
fun play() {
syncPlaybackWindow()
if (clippedSentence != null && !player.isPlaying && shouldWarmClipStart()) {
playClippedSentenceFromStart()
return
}
player.play()
refreshSentenceBoundaryGuard()
}
fun pause() {
stopAtSentenceEnd = false
stopAtSentenceEndIndex = null
cancelSentenceBoundaryGuard()
cancelClipStartWarmup(restoreVolume = true)
cancelSentenceEndGuard()
player.pause()
applySentenceRepeatMode()
}
fun togglePlayPause() {
@@ -198,19 +255,26 @@ class OralTrainerController internal constructor(
fun stop() {
stopAtSentenceEnd = false
stopAtSentenceEndIndex = null
cancelSentenceBoundaryGuard()
cancelClipStartWarmup(restoreVolume = true)
cancelSentenceEndGuard()
player.stop()
}
fun seekTo(positionMs: Long) {
stopAtSentenceEnd = false
stopAtSentenceEndIndex = null
player.seekTo(max(0L, positionMs))
refreshSentenceBoundaryGuard()
cancelClipStartWarmup(restoreVolume = true)
cancelSentenceEndGuard()
val safePositionMs = max(0L, positionMs)
lockSentenceAt(safePositionMs)
syncPlaybackWindow(safePositionMs)
seekInCurrentWindow(safePositionMs)
notifySentenceIfChanged(force = true)
notifySnapshot()
}
fun seekRelative(deltaMs: Long) {
seekTo(player.currentPosition + deltaMs)
seekTo(playbackPositionMs() + deltaMs)
}
fun rewind(stepMs: Long = config.defaultSeekStepMs) {
@@ -222,11 +286,11 @@ class OralTrainerController internal constructor(
}
fun previousItem() {
player.seekToPreviousMediaItem()
switchBaseItem(baseItemIndex - 1)
}
fun nextItem() {
player.seekToNextMediaItem()
switchBaseItem(baseItemIndex + 1)
}
fun setPlaybackSpeed(speed: Float) {
@@ -240,20 +304,21 @@ class OralTrainerController internal constructor(
fun setLoopMode(loopMode: LoopMode) {
this.loopMode = loopMode
player.repeatMode = when (loopMode) {
LoopMode.OFF -> Player.REPEAT_MODE_OFF
LoopMode.ONE -> Player.REPEAT_MODE_ONE
LoopMode.ALL -> Player.REPEAT_MODE_ALL
}
applySentenceRepeatMode()
}
fun setSentenceMode(enabled: Boolean) {
config = config.copy(sentenceMode = enabled)
syncPlaybackWindow()
}
fun setContinuousPlayback(enabled: Boolean) {
config = config.copy(continuousPlayback = enabled)
refreshSentenceBoundaryGuard()
if (!enabled) {
lockSentenceAt(playbackPositionMs())
}
syncPlaybackWindow()
notifySentenceIfChanged(force = true)
}
fun playCurrentSentenceAndStop() {
@@ -263,24 +328,47 @@ class OralTrainerController internal constructor(
/** Seek to a known sentence and stop automatically when that sentence ends. */
fun playSentenceAndStop(startMs: Long, sentenceIndex: Int) {
val item = currentTrainingItem()
val sentence = item?.sentences?.getOrNull(sentenceIndex)
cancelClipStartWarmup(restoreVolume = true)
stopAtSentenceEnd = true
stopAtSentenceEndIndex = sentenceIndex
player.seekTo(max(0L, startMs))
player.play()
refreshSentenceBoundaryGuard()
lastSentenceIndex = sentenceIndex
player.playWhenReady = false
if (item != null && sentence != null) {
if (!isCurrentClip(sentence)) {
installSentenceClip(item, sentence, 0L)
player.playWhenReady = false
}
playClippedSentenceFromStart()
} else {
cancelSentenceEndGuard()
player.seekTo(max(0L, startMs))
player.playWhenReady = true
player.play()
}
notifySentenceIfChanged(force = true)
notifySnapshot()
}
fun currentTrainingItem(): TrainingMediaItem? {
val index = player.currentMediaItemIndex
return mediaItems.getOrNull(index)
return mediaItems.getOrNull(baseItemIndex)
}
fun currentSentence(): SentenceBoundary? {
val item = currentTrainingItem() ?: return null
val position = player.currentPosition
return item.sentences.lastOrNull { sentence ->
position >= sentence.startMs && position < sentence.endMs
if (item.sentences.isEmpty()) {
return null
}
clippedSentence?.let { window ->
return item.sentences.getOrNull(window.index)
}
if (shouldBoundCurrentSentence()) {
lastSentenceIndex?.let { index ->
item.sentences.getOrNull(index)?.let { return it }
}
}
return sentenceAt(item, playbackPositionMs())
}
fun seekToPreviousSentence(): Boolean {
@@ -288,11 +376,16 @@ class OralTrainerController internal constructor(
if (item.sentences.isEmpty()) {
return false
}
val position = player.currentPosition
val target = item.sentences
.asReversed()
.firstOrNull { it.startMs < position - PREVIOUS_SENTENCE_TOLERANCE_MS }
?: item.sentences.first()
val target = if (shouldBoundCurrentSentence()) {
val index = anchoredSentenceIndex(item) ?: 0
item.sentences.getOrNull((index - 1).coerceAtLeast(0)) ?: item.sentences.first()
} else {
val position = playbackPositionMs()
item.sentences
.asReversed()
.firstOrNull { it.startMs < position - PREVIOUS_SENTENCE_TOLERANCE_MS }
?: item.sentences.first()
}
seekTo(target.startMs)
return true
}
@@ -302,10 +395,15 @@ class OralTrainerController internal constructor(
if (item.sentences.isEmpty()) {
return false
}
val position = player.currentPosition
val target = item.sentences.firstOrNull {
it.startMs > position + NEXT_SENTENCE_TOLERANCE_MS
} ?: return false
val target = if (shouldBoundCurrentSentence()) {
val index = anchoredSentenceIndex(item) ?: return false
item.sentences.getOrNull(index + 1) ?: return false
} else {
val position = playbackPositionMs()
item.sentences.firstOrNull {
it.startMs > position + NEXT_SENTENCE_TOLERANCE_MS
} ?: return false
}
seekTo(target.startMs)
return true
}
@@ -362,12 +460,16 @@ class OralTrainerController internal constructor(
fun snapshot(): PlaybackSnapshot {
val item = currentTrainingItem()
val duration = player.duration.takeUnless { it == C.TIME_UNSET } ?: -1L
val duration = when {
unclippedDurationMs > 0L -> unclippedDurationMs
clippedSentence != null -> clippedSentence?.endMs ?: -1L
else -> player.duration.takeUnless { it == C.TIME_UNSET } ?: -1L
}
return PlaybackSnapshot(
mediaId = item?.id,
positionMs = max(0L, player.currentPosition),
positionMs = playbackPositionMs(),
durationMs = duration,
bufferedPositionMs = max(0L, player.bufferedPosition),
bufferedPositionMs = playbackBufferedPositionMs(),
isPlaying = player.isPlaying,
playbackState = player.playbackState.toOralTrainerState(),
playbackSpeed = player.playbackParameters.speed,
@@ -380,7 +482,8 @@ class OralTrainerController internal constructor(
return
}
released = true
cancelSentenceBoundaryGuard()
cancelClipStartWarmup(restoreVolume = false)
cancelSentenceEndGuard()
mainHandler.removeCallbacks(ticker)
listeners.clear()
player.release()
@@ -400,127 +503,295 @@ class OralTrainerController internal constructor(
if (!force && index == lastSentenceIndex) {
return
}
val changed = index != lastSentenceIndex
lastSentenceIndex = index
listeners.forEach { it.onSentenceChanged(sentence) }
if (changed && !config.continuousPlayback && !stopAtSentenceEnd) {
refreshSentenceBoundaryGuard()
}
}
private fun enforceSentenceBoundary() {
if (!config.sentenceMode || !player.isPlaying) {
private fun syncPlaybackWindow(positionMs: Long = playbackPositionMs()) {
if (released || updatingPlaybackWindow) {
return
}
val item = currentTrainingItem() ?: return
if (item.sentences.isEmpty()) {
val item = currentTrainingItem()
val target = sentenceForWindow(item)
if (target == null || item == null) {
restoreUnclipped(positionMs)
return
}
val position = player.currentPosition
if (stopAtSentenceEnd) {
val index = stopAtSentenceEndIndex ?: return
val sentence = item.sentences.getOrNull(index) ?: return
if (position >= sentence.endMs) {
applySentenceEndAction(sentence)
}
val current = clippedSentence
if (current != null &&
current.index == target.index &&
current.startMs == target.startMs &&
current.endMs == target.endMs
) {
applySentenceRepeatMode()
return
}
if (config.continuousPlayback) {
return
}
val anchorIndex = lastSentenceIndex ?: currentSentence()?.index ?: return
val anchor = item.sentences.getOrNull(anchorIndex) ?: return
if (position >= anchor.endMs) {
applySentenceEndAction(anchor)
}
val relativeMs = (positionMs - target.startMs).coerceAtLeast(0L)
installSentenceClip(item, target, relativeMs)
}
private fun refreshSentenceBoundaryGuard() {
cancelSentenceBoundaryGuard()
if (released || !config.sentenceMode) {
return
private fun sentenceForWindow(item: TrainingMediaItem?): SentenceBoundary? {
if (item == null || !config.sentenceMode || item.sentences.isEmpty()) {
return null
}
if (!player.isPlaying && !player.playWhenReady) {
return
}
val item = currentTrainingItem() ?: return
if (item.sentences.isEmpty()) {
return
}
val sentence = when {
stopAtSentenceEnd -> item.sentences.getOrNull(stopAtSentenceEndIndex ?: return)
return when {
stopAtSentenceEnd -> item.sentences.getOrNull(stopAtSentenceEndIndex ?: return null)
config.continuousPlayback -> null
else -> {
val index = lastSentenceIndex ?: currentSentence()?.index
item.sentences.getOrNull(index ?: return)
val index = lastSentenceIndex ?: sentenceAt(item, playbackPositionMs())?.index
item.sentences.getOrNull(index ?: return null)
}
} ?: return
}
}
val mediaItemIndex = player.currentMediaItemIndex
if (player.currentPosition >= sentence.endMs) {
applySentenceEndAction(sentence)
private fun installSentenceClip(
item: TrainingMediaItem,
sentence: SentenceBoundary,
relativePositionMs: Long,
) {
if (released) {
return
}
val token = ++boundaryGuardToken
boundaryMessage = player.createMessage { _, _ ->
if (token != boundaryGuardToken || released) {
return@createMessage
rememberUnclippedDuration()
cancelSentenceEndGuard()
val lengthMs = (sentence.endMs - sentence.startMs).coerceAtLeast(0L)
val startPositionMs = relativePositionMs.coerceIn(0L, lengthMs)
val playWhenReady = player.playWhenReady
val warmFromStart = startPositionMs <= CLIP_START_WARMUP_POSITION_MS
clipStartNeedsWarmup = warmFromStart
updatingPlaybackWindow = true
clippedSentence = sentence
lastSentenceIndex = sentence.index
player.setMediaItem(item.toMedia3Item(sentence.startMs, sentence.endMs), startPositionMs)
player.prepare()
player.playWhenReady = false
applySentenceRepeatMode()
updatingPlaybackWindow = false
if (playWhenReady) {
if (warmFromStart) {
playClippedSentenceFromStart()
} else {
player.playWhenReady = true
player.play()
}
val active = currentTrainingItem() ?: return@createMessage
val latest = when {
stopAtSentenceEnd -> active.sentences.getOrNull(stopAtSentenceEndIndex ?: return@createMessage)
else -> active.sentences.getOrNull(sentence.index)
} ?: return@createMessage
if (player.currentMediaItemIndex != mediaItemIndex) {
return@createMessage
}
applySentenceEndAction(latest)
}
.setLooper(mainHandler.looper)
.setPosition(mediaItemIndex, sentence.endMs)
}
private fun restoreUnclipped(positionMs: Long) {
if (released) {
return
}
if (clippedSentence == null && player.mediaItemCount == mediaItems.size) {
applyLoopMode()
return
}
val playWhenReady = player.playWhenReady
updatingPlaybackWindow = true
clippedSentence = null
clipStartNeedsWarmup = false
player.setMediaItems(
mediaItems.map { it.toMedia3Item() },
baseItemIndex.coerceIn(0, mediaItems.lastIndex),
max(0L, positionMs),
)
player.prepare()
player.playWhenReady = playWhenReady
applyLoopMode()
updatingPlaybackWindow = false
}
private fun seekInCurrentWindow(positionMs: Long) {
val sentence = clippedSentence
if (sentence == null) {
player.seekTo(max(0L, positionMs))
return
}
val lengthMs = (sentence.endMs - sentence.startMs).coerceAtLeast(0L)
player.seekTo((positionMs - sentence.startMs).coerceIn(0L, lengthMs))
}
private fun onClippedSentenceEnded() {
val sentence = clippedSentence ?: return
if (stopAtSentenceEnd && stopAtSentenceEndIndex == sentence.index) {
handleStopAtSentenceEnd(sentence)
return
}
clipStartNeedsWarmup = false
player.seekTo(0L)
if (player.playbackState == Player.STATE_ENDED) {
player.prepare()
}
if (!config.continuousPlayback) {
player.play()
}
lastSentenceIndex = sentence.index
notifySentenceIfChanged(force = true)
notifySnapshot()
}
private fun enforceStopAtSentenceEnd() {
if (!stopAtSentenceEnd) {
return
}
val index = stopAtSentenceEndIndex ?: return
val sentence = currentTrainingItem()?.sentences?.getOrNull(index) ?: return
if (playbackPositionMs() >= sentence.endMs) {
handleStopAtSentenceEnd(sentence)
}
}
private fun handleStopAtSentenceEnd(sentence: SentenceBoundary) {
if (!stopAtSentenceEnd || stopAtSentenceEndIndex != sentence.index) {
return
}
cancelSentenceEndGuard()
stopAtSentenceEnd = false
stopAtSentenceEndIndex = null
player.pause()
if (clippedSentence?.index == sentence.index) {
player.seekTo(0L)
if (player.playbackState == Player.STATE_ENDED) {
player.prepare()
}
} else {
seekInCurrentWindow(sentence.startMs)
}
applySentenceRepeatMode()
lastSentenceIndex = sentence.index
notifySentenceIfChanged(force = true)
notifySnapshot()
}
private fun scheduleSentenceEndGuard() {
val index = stopAtSentenceEndIndex ?: return
val sentence = currentTrainingItem()?.sentences?.getOrNull(index) ?: return
cancelSentenceEndGuard()
val endInWindowMs = if (clippedSentence?.index == sentence.index) {
(sentence.endMs - sentence.startMs - 1L).coerceAtLeast(0L)
} else {
sentence.endMs
}
sentenceEndMessage = player.createMessage { _, _ ->
if (stopAtSentenceEnd && stopAtSentenceEndIndex == sentence.index) {
handleStopAtSentenceEnd(sentence)
}
}.setLooper(mainHandler.looper)
.setPosition(endInWindowMs)
.setDeleteAfterDelivery(true)
.send()
}
private fun cancelSentenceBoundaryGuard() {
boundaryGuardToken++
boundaryMessage?.let { message ->
runCatching { message.cancel() }
}
boundaryMessage = null
private fun cancelSentenceEndGuard() {
sentenceEndMessage?.cancel()
sentenceEndMessage = null
}
private fun applySentenceEndAction(sentence: SentenceBoundary) {
if (stopAtSentenceEnd && stopAtSentenceEndIndex == sentence.index) {
stopAtSentenceEnd = false
stopAtSentenceEndIndex = null
cancelSentenceBoundaryGuard()
player.pause()
player.seekTo(sentence.startMs)
lastSentenceIndex = sentence.index
notifySentenceIfChanged(force = true)
notifySnapshot()
private fun switchBaseItem(targetIndex: Int) {
if (targetIndex !in mediaItems.indices) {
return
}
if (config.continuousPlayback) {
return
baseItemIndex = targetIndex
lastSentenceIndex = null
stopAtSentenceEnd = false
stopAtSentenceEndIndex = null
cancelClipStartWarmup(restoreVolume = true)
cancelSentenceEndGuard()
clippedSentence = null
unclippedDurationMs = -1L
val item = mediaItems[targetIndex]
val firstSentence = item.sentences.firstOrNull()
if (shouldBoundCurrentSentence() && firstSentence != null) {
lastSentenceIndex = firstSentence.index
installSentenceClip(item, firstSentence, 0L)
} else {
player.setMediaItem(item.toMedia3Item(), 0L)
player.prepare()
applyLoopMode()
}
// Loop the current sentence without leaking into the next one.
player.seekTo(sentence.startMs)
lastSentenceIndex = sentence.index
listeners.forEach { it.onMediaChanged(item) }
notifySentenceIfChanged(force = true)
refreshSentenceBoundaryGuard()
notifySnapshot()
}
private fun TrainingMediaItem.toMedia3Item(): MediaItem {
private fun playbackPositionMs(): Long {
val sentence = clippedSentence ?: return max(0L, player.currentPosition)
val lengthMs = (sentence.endMs - sentence.startMs).coerceAtLeast(0L)
return sentence.startMs + player.currentPosition.coerceIn(0L, lengthMs)
}
private fun playbackBufferedPositionMs(): Long {
val sentence = clippedSentence ?: return max(0L, player.bufferedPosition)
val lengthMs = (sentence.endMs - sentence.startMs).coerceAtLeast(0L)
return sentence.startMs + player.bufferedPosition.coerceIn(0L, lengthMs)
}
private fun shouldBoundCurrentSentence(): Boolean {
return config.sentenceMode && (!config.continuousPlayback || stopAtSentenceEnd)
}
private fun lockSentenceAt(positionMs: Long) {
if (!shouldBoundCurrentSentence()) {
return
}
val item = currentTrainingItem() ?: return
lastSentenceIndex = sentenceAt(item, positionMs)?.index
}
private fun anchoredSentenceIndex(item: TrainingMediaItem): Int? {
return lastSentenceIndex?.takeIf { it in item.sentences.indices }
?: clippedSentence?.index
?: sentenceAt(item, playbackPositionMs())?.index
}
private fun sentenceAt(item: TrainingMediaItem, positionMs: Long): SentenceBoundary? {
return item.sentences.lastOrNull { positionMs >= it.startMs && positionMs < it.endMs }
?: item.sentences.lastOrNull { it.startMs <= positionMs }
}
private fun rememberUnclippedDuration() {
if (unclippedDurationMs > 0L || clippedSentence != null) {
return
}
val duration = player.duration
if (duration != C.TIME_UNSET) {
unclippedDurationMs = duration
}
}
private fun applyLoopMode() {
player.repeatMode = when (loopMode) {
LoopMode.OFF -> Player.REPEAT_MODE_OFF
LoopMode.ONE -> Player.REPEAT_MODE_ONE
LoopMode.ALL -> Player.REPEAT_MODE_ALL
}
}
private fun applySentenceRepeatMode() {
if (clippedSentence == null) {
applyLoopMode()
return
}
player.repeatMode = Player.REPEAT_MODE_OFF
}
private fun TrainingMediaItem.toMedia3Item(
clippingStartPositionMs: Long = 0L,
clippingEndPositionMs: Long? = null,
): MediaItem {
val builder = MediaItem.Builder()
.setMediaId(id)
.setUri(uri)
mimeType?.let(builder::setMimeType)
customCacheKey?.let(builder::setCustomCacheKey)
if (clippingEndPositionMs != null) {
builder.setClippingConfiguration(
MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(max(0L, clippingStartPositionMs))
.setEndPositionMs(max(0L, clippingEndPositionMs))
.setStartsAtKeyFrame(false)
.build()
)
}
subtitleUri?.let { uri ->
builder.setSubtitleConfigurations(
listOf(
@@ -549,8 +820,93 @@ class OralTrainerController internal constructor(
return CancellableAssessment {}
}
private fun isCurrentClip(sentence: SentenceBoundary): Boolean {
val current = clippedSentence ?: return false
return current.index == sentence.index &&
current.startMs == sentence.startMs &&
current.endMs == sentence.endMs
}
private fun shouldWarmClipStart(): Boolean {
if (clippedSentence == null) {
return false
}
return clipStartNeedsWarmup || player.currentPosition <= CLIP_START_WARMUP_POSITION_MS
}
private fun playClippedSentenceFromStart() {
if (released) {
return
}
if (player.playbackState == Player.STATE_IDLE ||
player.playbackState == Player.STATE_ENDED ||
player.playbackState == Player.STATE_BUFFERING
) {
playFromStartAfterReady = true
player.playWhenReady = false
if (player.playbackState == Player.STATE_IDLE || player.playbackState == Player.STATE_ENDED) {
player.prepare()
}
return
}
startClipStartWarmup()
}
private fun startClipStartWarmup() {
if (released) {
return
}
mainHandler.removeCallbacks(finishClipStartWarmupRunnable)
if (clipStartWarmupVolume == null) {
clipStartWarmupVolume = player.volume
}
player.volume = 0f
player.seekTo(0L)
if (player.playbackState == Player.STATE_ENDED) {
playFromStartAfterReady = true
player.prepare()
return
}
player.playWhenReady = true
player.play()
mainHandler.postDelayed(finishClipStartWarmupRunnable, CLIP_START_WARMUP_MS)
}
private fun finishClipStartWarmup() {
if (released) {
return
}
val restoreVolume = clipStartWarmupVolume ?: return
clipStartWarmupVolume = null
clipStartNeedsWarmup = false
mainHandler.removeCallbacks(finishClipStartWarmupRunnable)
player.seekTo(0L)
if (player.playbackState == Player.STATE_ENDED) {
player.prepare()
}
player.volume = restoreVolume
player.playWhenReady = true
player.play()
if (stopAtSentenceEnd) {
scheduleSentenceEndGuard()
}
}
private fun cancelClipStartWarmup(restoreVolume: Boolean) {
playFromStartAfterReady = false
mainHandler.removeCallbacks(finishClipStartWarmupRunnable)
val volume = clipStartWarmupVolume ?: return
clipStartWarmupVolume = null
if (restoreVolume) {
player.volume = volume
}
}
private companion object {
const val PREVIOUS_SENTENCE_TOLERANCE_MS = 250L
const val NEXT_SENTENCE_TOLERANCE_MS = 150L
const val IDLE_POLL_MS = 250L
const val CLIP_START_WARMUP_MS = 180L
const val CLIP_START_WARMUP_POSITION_MS = 120L
}
}

View File

@@ -4,6 +4,7 @@ import android.content.Context
import android.util.AttributeSet
import android.view.GestureDetector
import android.view.MotionEvent
import android.view.View
import android.view.ViewConfiguration
import android.widget.FrameLayout
import androidx.media3.ui.PlayerView
@@ -16,6 +17,7 @@ class OralTrainerPlayerView @JvmOverloads constructor(
) : FrameLayout(context, attrs, defStyleAttr) {
val playerView: PlayerView = PlayerView(context).apply {
useController = false
setKeepContentOnPlayerReset(true)
layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT)
}
@@ -122,6 +124,26 @@ class OralTrainerPlayerView @JvmOverloads constructor(
verticalSwipeListener = listener
}
/** Forwards touches from an overlay (for example a subtitle layer) as player gestures. */
fun forwardGestureFrom(source: View, event: MotionEvent): Boolean {
if (controller == null || !gestureControls.enabled) {
return false
}
val sourceLocation = IntArray(2).also(source::getLocationOnScreen)
val playerLocation = IntArray(2).also(::getLocationOnScreen)
val playerEvent = MotionEvent.obtain(event).apply {
offsetLocation(
(playerLocation[0] - sourceLocation[0]).toFloat(),
(playerLocation[1] - sourceLocation[1]).toFloat(),
)
}
return try {
handleLongPressSpeed(playerEvent) || gestureDetector.onTouchEvent(playerEvent)
} finally {
playerEvent.recycle()
}
}
override fun onTouchEvent(event: MotionEvent): Boolean {
return gestureDetector.onTouchEvent(event) || super.onTouchEvent(event)
}

View File

@@ -159,6 +159,11 @@ class SentenceBoundaryApi internal constructor(
} else {
sentence.getLong("reference_speech_duration_ms")
},
zhText = if (sentence.isNull("zh_text")) {
null
} else {
sentence.getString("zh_text").takeIf { it.isNotBlank() }
},
)
)
}

View File

@@ -10,6 +10,7 @@ data class SentenceBoundary @JvmOverloads constructor(
val text: String? = null,
val language: String? = null,
val referenceSpeechDurationMs: Long? = null,
val zhText: String? = null,
) {
init {
require(index >= 0) { "Sentence index must be non-negative." }

View File

@@ -19,7 +19,7 @@ android {
minSdk = 26
targetSdk = 35
versionCode = 1
versionName = "0.1.4"
versionName = "0.1.6"
}
compileOptions {
@@ -37,6 +37,7 @@ kotlin {
dependencies {
implementation(project(":oral-trainer-sdk"))
implementation("androidx.core:core-ktx:1.13.1")
implementation("androidx.recyclerview:recyclerview:1.3.2")
implementation("androidx.media3:media3-common:1.11.0")
implementation("androidx.media3:media3-exoplayer:1.11.0")
}

View File

@@ -1,6 +1,9 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PLAYBACK" />
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
<application
android:allowBackup="true"
@@ -19,6 +22,11 @@
</intent-filter>
</activity>
<service
android:name=".TrainingPlaybackService"
android:exported="false"
android:foregroundServiceType="mediaPlayback" />
<provider
android:name="androidx.core.content.FileProvider"
android:authorities="${applicationId}.fileprovider"

View File

@@ -3,6 +3,7 @@ package cn.learningpad.oraltrainer.sample
enum class Module {
COURSE,
TRAIN,
READING,
TEST,
MINE,
}
@@ -13,7 +14,6 @@ internal const val ASSESSMENT_API_KEY =
internal const val SSO_LOGIN_URL = "https://d1kt.cn/login"
internal const val PICK_VIDEO_REQUEST = 1001
internal const val RECORD_AUDIO_REQUEST = 1002
internal const val PROGRESS_MAX = 1000

View File

@@ -2,6 +2,7 @@ package cn.learningpad.oraltrainer.sample
import cn.learningpad.oraltrainer.sdk.OralTrainerController
import cn.learningpad.oraltrainer.sdk.OralTrainerSdk
import cn.learningpad.oraltrainer.sdk.SentenceBoundary
import cn.learningpad.oraltrainer.sdk.SentenceBoundaryApiCallback
import cn.learningpad.oraltrainer.sdk.SentenceBoundaryApiResult
import cn.learningpad.oraltrainer.sdk.TrainingVideoSummary
@@ -26,23 +27,45 @@ internal class CourseCatalogController(
fun onSentencePlaceholder(meta: String, text: String)
fun refreshPlayerUi()
fun openTrainingModule()
fun syncSentenceExplanations(videoHash: String, sentences: List<SentenceBoundary>)
}
fun loadCatalog() {
loadCatalog(refreshBoundaries = false)
}
fun refreshCourses() {
if (screenState.catalogIsRefreshing) return
loadCatalog(refreshBoundaries = true)
}
private fun loadCatalog(refreshBoundaries: Boolean) {
val generation = screenState.beginCatalogRequest()
screenState.catalogIsRefreshing = refreshBoundaries
screenState.catalogStatus = "正在同步"
host.onCatalogStatus(screenState.catalogStatus)
host.onCatalogLoadingPlaceholder()
sdk.userApi.courses(tokenProvider()) { result ->
if (!screenState.isCurrentCatalogRequest(generation)) return@courses
result.fold(
onSuccess = { courses ->
screenState.catalogVideos = courses.map { it.toVideoSummary() }
screenState.courseEnrollment.clear()
courses.forEach { screenState.courseEnrollment[it.videoHash] = it.enrolled }
screenState.catalogStatus = "${screenState.catalogVideos.size} 个课程"
host.onCatalogStatus(screenState.catalogStatus)
host.onCatalogChanged()
if (refreshBoundaries) {
screenState.catalogStatus = "${courses.size} 个课程,正在刷新句界"
host.onCatalogStatus(screenState.catalogStatus)
host.onCatalogChanged()
refreshAllCourseBoundaries(generation, index = 0, succeeded = 0, failed = 0)
} else {
screenState.catalogIsRefreshing = false
screenState.catalogStatus = "${screenState.catalogVideos.size} 个课程"
host.onCatalogStatus(screenState.catalogStatus)
host.onCatalogChanged()
}
},
onFailure = {
screenState.catalogIsRefreshing = false
screenState.catalogStatus = "课程加载失败:${it.message}"
host.onCatalogStatus(screenState.catalogStatus)
screenState.courseEnrollment.clear()
@@ -52,6 +75,68 @@ internal class CourseCatalogController(
}
}
private fun refreshAllCourseBoundaries(
generation: Int,
index: Int,
succeeded: Int,
failed: Int,
) {
if (!screenState.isCurrentCatalogRequest(generation)) return
val videos = screenState.catalogVideos
if (index >= videos.size) {
screenState.catalogIsRefreshing = false
screenState.catalogStatus = when {
failed == 0 -> "${videos.size} 个课程 · 句界已刷新"
succeeded == 0 -> "${videos.size} 个课程 · 句界刷新失败"
else -> "${videos.size} 个课程 · 句界部分刷新(失败 $failed)"
}
host.onCatalogStatus(screenState.catalogStatus)
host.onCatalogChanged()
return
}
val video = videos[index]
screenState.catalogStatus =
"正在刷新句界 ${index + 1}/${videos.size}"
host.onCatalogStatus(screenState.catalogStatus)
sdk.sentenceBoundaryApi.fetch(video.videoHash, object : SentenceBoundaryApiCallback {
override fun onSuccess(result: SentenceBoundaryApiResult) {
if (!screenState.isCurrentCatalogRequest(generation)) return
screenState.courseBoundaries[result.videoHash] = result
host.syncSentenceExplanations(result.videoHash, result.sentences)
updateCatalogVideo(video.videoHash, result)
if (screenState.activeItemId == result.videoHash) {
controller.updateSentences(result.videoHash, result.sentences)
screenState.currentSentenceCount = result.sentences.size
host.refreshPlayerUi()
}
host.onCatalogChanged()
refreshAllCourseBoundaries(generation, index + 1, succeeded + 1, failed)
}
override fun onError(error: Throwable) {
if (!screenState.isCurrentCatalogRequest(generation)) return
refreshAllCourseBoundaries(generation, index + 1, succeeded, failed + 1)
}
})
}
private fun updateCatalogVideo(videoHash: String, result: SentenceBoundaryApiResult) {
screenState.catalogVideos = screenState.catalogVideos.map { video ->
if (video.videoHash != videoHash) {
video
} else {
video.copy(
durationMs = result.durationMs,
sentenceCount = result.sentences.size,
)
}
}
if (screenState.currentCourse?.videoHash == videoHash) {
screenState.currentCourse = screenState.catalogVideos.firstOrNull { it.videoHash == videoHash }
}
}
fun loadUserData(target: Module, onMine: () -> Unit) {
when (target) {
Module.COURSE -> loadCatalog()
@@ -85,23 +170,24 @@ internal class CourseCatalogController(
}
}
fun useCourseInPlace(video: TrainingVideoSummary) {
// Keep the current module (e.g. Train) while syncing course selection.
if (screenState.courseEnrollment[video.videoHash] != true) {
enrollCourse(video)
}
loadRemoteVideo(video)
}
fun loadRemoteVideo(video: TrainingVideoSummary) {
screenState.currentCourse = video
screenState.activeItemId = video.videoHash
host.onPlayerLoading(video)
host.onCatalogChanged()
controller.loadItem(video.toTrainingMediaItem())
screenState.currentSentenceCount = video.sentenceCount
refreshSentenceBoundaries(video, background = false)
val cachedBoundary = screenState.courseBoundaries[video.videoHash]
if (cachedBoundary != null) {
host.syncSentenceExplanations(cachedBoundary.videoHash, cachedBoundary.sentences)
}
controller.loadItem(video.toTrainingMediaItem(cachedBoundary?.sentences.orEmpty()))
screenState.currentSentenceCount = cachedBoundary?.sentences?.size ?: video.sentenceCount
if (cachedBoundary == null) {
refreshSentenceBoundaries(video, background = false)
} else {
host.refreshPlayerUi()
host.onStatus("课程已就绪:${cachedBoundary.sentences.size} 句")
}
}
fun refreshSentenceBoundaries(
@@ -117,8 +203,12 @@ internal class CourseCatalogController(
if (!screenState.isCurrentBoundaryRequest(requestGeneration, video.videoHash)) {
return
}
screenState.courseBoundaries[result.videoHash] = result
host.syncSentenceExplanations(result.videoHash, result.sentences)
updateCatalogVideo(video.videoHash, result)
screenState.currentSentenceCount = result.sentences.size
controller.updateSentences(video.videoHash, result.sentences)
host.onCatalogChanged()
host.refreshPlayerUi()
host.onStatus("课程已就绪:${result.sentences.size} 句")
}

View File

@@ -7,7 +7,6 @@ import android.content.Intent
import android.content.IntentFilter
import android.content.pm.PackageManager
import android.content.res.Configuration
import android.net.Uri
import android.os.Bundle
import cn.learningpad.oraltrainer.sdk.AuthSession
import cn.learningpad.oraltrainer.sdk.GestureEvent
@@ -17,8 +16,6 @@ import cn.learningpad.oraltrainer.sdk.OralTrainerListener
import cn.learningpad.oraltrainer.sdk.OralTrainerSdk
import cn.learningpad.oraltrainer.sdk.PlaybackSnapshot
import cn.learningpad.oraltrainer.sdk.SentenceBoundary
import cn.learningpad.oraltrainer.sdk.SentenceBoundaryApiCallback
import cn.learningpad.oraltrainer.sdk.SentenceBoundaryApiResult
import cn.learningpad.oraltrainer.sdk.TrainingMediaItem
import cn.learningpad.oraltrainer.sdk.TrainingVideoSummary
@@ -43,13 +40,21 @@ class MainActivity : Activity() {
private lateinit var catalogController: CourseCatalogController
private lateinit var testSession: TestSessionController
private lateinit var explanationOverlay: SentenceExplanationOverlay
private lateinit var translationApi: SubtitleTranslationApi
private lateinit var sentenceFavoriteStore: SentenceFavoriteStore
private var translationRequest: CancellableTranslationRequest? = null
private var progressBarDragging = false
private var trainingPlaybackServiceActive = false
private val screenActionReceiver = object : BroadcastReceiver() {
override fun onReceive(context: Context?, intent: Intent?) {
if (intent?.action == Intent.ACTION_SCREEN_OFF) {
testSession.onScreenOff()
// Training playback intentionally keeps running with the screen off.
// The test module still pauses because recording/assessment state is UI-dependent.
if (screenState.activeModule == Module.TEST) {
testSession.onScreenOff()
}
}
}
}
@@ -73,6 +78,10 @@ class MainActivity : Activity() {
authSessionStore.session?.token.orEmpty()
},
)
translationApi = SubtitleTranslationApi(this, SERVER_BASE_URL) {
authSessionStore.session?.token.orEmpty()
}
sentenceFavoriteStore = SentenceFavoriteStore(this)
screenFactory = SampleScreenFactory(
activity = this,
@@ -147,8 +156,9 @@ class MainActivity : Activity() {
if (bindings.hasPlayerChrome) {
bindings.statusText.text = "已停止配音播放"
}
} else {
} else if (screenState.activeModule != Module.TRAIN) {
controller.pause()
stopTrainingPlaybackService()
}
super.onPause()
}
@@ -173,58 +183,14 @@ class MainActivity : Activity() {
super.onBackPressed()
}
override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
super.onActivityResult(requestCode, resultCode, data)
if (requestCode != PICK_VIDEO_REQUEST || resultCode != RESULT_OK) return
val uri = data?.data ?: return
val persistableFlags = data.flags and Intent.FLAG_GRANT_READ_URI_PERMISSION
if (persistableFlags != 0) {
runCatching { contentResolver.takePersistableUriPermission(uri, persistableFlags) }
}
val item = TrainingMediaItem(
id = uri.toString(),
title = uri.lastPathSegment ?: "本地视频",
uri = uri,
)
screenState.activeItemId = item.id
screenState.currentSentenceCount = 0
controller.loadItem(item)
if (bindings.hasPlayerChrome) {
bindings.lessonTitleText.text = item.title
bindings.sentenceMetaText.text = "句子边界分析中"
bindings.sentenceText.text = "正在匹配服务端句子边界"
bindings.statusText.text = "正在获取本地视频的句子边界..."
}
screenFactory.renderCatalog()
sdk.sentenceBoundaryApi.fetchForUri(uri, contentResolver, object : SentenceBoundaryApiCallback {
override fun onSuccess(result: SentenceBoundaryApiResult) {
if (screenState.activeItemId != item.id) return
screenState.currentSentenceCount = result.sentences.size
controller.loadItem(item.copy(sentences = result.sentences))
if (bindings.hasPlayerChrome) {
bindings.statusText.text = "句子边界已加载:${result.sentences.size} 句"
}
}
override fun onError(error: Throwable) {
if (screenState.activeItemId != item.id) return
if (bindings.hasPlayerChrome) {
bindings.statusText.text =
"句子边界获取失败,使用默认快退/快进:${error.message.orEmpty()}"
bindings.sentenceMetaText.text = "暂无句子边界"
bindings.sentenceText.text = ""
}
}
})
}
override fun onDestroy() {
authCoordinator.reset()
testSession.release()
stopTrainingPlaybackService()
controller.release()
explanationOverlay.release()
translationRequest?.cancel()
translationApi.release()
super.onDestroy()
}
@@ -264,28 +230,20 @@ class MainActivity : Activity() {
sdk.config.userToken = null
screenState.clearCourse()
controller.pause()
stopTrainingPlaybackService()
screenState.activeModule = Module.COURSE
setRootView()
}
private fun openLocalVideoPicker() {
@Suppress("DEPRECATION")
startActivityForResult(
Intent(Intent.ACTION_OPEN_DOCUMENT).apply {
addCategory(Intent.CATEGORY_OPENABLE)
type = "video/*"
addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION)
addFlags(Intent.FLAG_GRANT_PERSISTABLE_URI_PERMISSION)
},
PICK_VIDEO_REQUEST,
)
}
private fun switchModule(module: Module) {
if (module != Module.TEST) {
testSession.stopDubbingIfNeeded()
}
screenState.activeModule = module
if (module != Module.TRAIN) {
controller.pause()
stopTrainingPlaybackService()
}
if (module == Module.COURSE || module == Module.MINE) {
catalogController.loadUserData(module) {
screenFactory.refreshMyContent(sdk.userApi, authSessionStore.session?.token.orEmpty())
@@ -305,10 +263,23 @@ class MainActivity : Activity() {
if (screenState.activeModule == Module.MINE) {
screenFactory.refreshMyContent(sdk.userApi, authSessionStore.session?.token.orEmpty())
}
if (screenState.activeModule == Module.READING) {
screenFactory.scrollToReadingCurrentSentence()
}
}
private fun refreshCurrentUi() {
if (screenState.activeModule != Module.TRAIN && screenState.activeModule != Module.TEST) {
if (screenState.activeModule != Module.TRAIN &&
screenState.activeModule != Module.TEST &&
screenState.activeModule != Module.READING
) {
screenFactory.setCatalogStatus(screenState.catalogStatus)
return
}
if (screenState.activeModule == Module.READING) {
screenState.currentSentenceCount = controller.currentTrainingItem()?.sentences?.size ?: 0
screenFactory.refreshReadingSentences()
ensureChineseTranslations()
screenFactory.setCatalogStatus(screenState.catalogStatus)
return
}
@@ -318,6 +289,7 @@ class MainActivity : Activity() {
screenState.currentSentenceCount = item?.sentences?.size ?: 0
applyPlaybackSnapshot(controller.snapshot())
applySentence(controller.currentSentence())
ensureChineseTranslations()
screenFactory.setCatalogStatus(screenState.catalogStatus)
}
@@ -327,9 +299,13 @@ class MainActivity : Activity() {
}
override fun onMediaChanged(item: TrainingMediaItem?) {
screenState.currentSentenceCount = item?.sentences?.size ?: 0
if (bindings.hasPlayerChrome) {
bindings.lessonTitleText.text = item?.title ?: "未选择课程"
screenState.currentSentenceCount = item?.sentences?.size ?: 0
}
ensureChineseTranslations()
if (screenState.activeModule == Module.READING) {
screenFactory.refreshReadingSentences()
}
testSession.ensureHistoryForCurrentCourse()
if (screenState.activeModule == Module.TEST) {
@@ -359,6 +335,11 @@ class MainActivity : Activity() {
private fun applyPlaybackSnapshot(snapshot: PlaybackSnapshot) {
if (!bindings.hasPlayerChrome) return
if (screenState.activeModule == Module.TRAIN && snapshot.isPlaying) {
startTrainingPlaybackService()
} else if (!snapshot.isPlaying) {
stopTrainingPlaybackService()
}
if (!progressBarDragging) {
bindings.timeText.text =
"${PlaybackFormat.time(snapshot.positionMs)} / ${PlaybackFormat.time(snapshot.durationMs)}"
@@ -368,12 +349,25 @@ class MainActivity : Activity() {
bindings.statusText.text = PlaybackFormat.status(snapshot)
}
private fun startTrainingPlaybackService() {
if (trainingPlaybackServiceActive) return
TrainingPlaybackService.start(this)
trainingPlaybackServiceActive = true
}
private fun stopTrainingPlaybackService() {
if (!trainingPlaybackServiceActive) return
TrainingPlaybackService.stop(this)
trainingPlaybackServiceActive = false
}
private fun applySentence(sentence: SentenceBoundary?) {
if (!bindings.hasPlayerChrome) return
if (sentence == null) {
bindings.sentenceMetaText.text = "暂无句子边界"
bindings.sentenceText.text = ""
bindings.explainSentenceButton?.isEnabled = false
applyChineseSubtitle(null)
return
}
val countText = if (screenState.currentSentenceCount > 0) {
@@ -385,18 +379,113 @@ class MainActivity : Activity() {
"第 ${sentence.index + 1}$countText 句 ${PlaybackFormat.time(sentence.startMs)}-${PlaybackFormat.time(sentence.endMs)}"
bindings.sentenceText.text = sentence.text?.takeIf { it.isNotBlank() } ?: "当前句子"
bindings.explainSentenceButton?.isEnabled = !sentence.text.isNullOrBlank()
applyChineseSubtitle(sentence)
if (screenState.activeModule == Module.TEST) {
testSession.refreshTestUi()
}
}
private fun applyChineseSubtitle(sentence: SentenceBoundary?) {
if (!bindings.hasPlayerChrome) return
val showChinese = screenState.activeModule == Module.TRAIN &&
screenState.chineseSubtitlesEnabled &&
sentence != null &&
!sentence.text.isNullOrBlank()
if (!showChinese || sentence == null) {
bindings.sentenceZhText.visibility = android.view.View.GONE
return
}
bindings.sentenceZhText.visibility = android.view.View.VISIBLE
bindings.sentenceZhText.text = when {
!sentence.zhText.isNullOrBlank() -> sentence.zhText
screenState.chineseTranslationLoading -> "正在翻译中文字幕…"
screenState.chineseTranslationError.isNotBlank() -> screenState.chineseTranslationError
else -> "暂无中文字幕"
}
}
private fun applyTranslatedSentences(itemId: String, translations: Map<Int, String>) {
if (translations.isEmpty()) return
val current = controller.currentTrainingItem() ?: return
if (current.id != itemId) return
val updated = current.sentences.map { sentence ->
sentence.copy(zhText = translations[sentence.index] ?: sentence.zhText)
}
controller.updateSentences(current.id, updated)
screenState.courseBoundaries[current.id]?.let { cached ->
screenState.courseBoundaries[current.id] = cached.copy(sentences = updated)
}
applyChineseSubtitle(controller.currentSentence())
screenFactory.refreshReadingTranslations()
}
private fun favoriteUserKey(): String =
SentenceFavoriteStore.userKey(authSessionStore.session?.token.orEmpty())
private fun ensureChineseTranslations() {
val translationEnabled = when (screenState.activeModule) {
Module.TRAIN -> screenState.chineseSubtitlesEnabled
Module.READING -> screenState.readingTranslationVisible
else -> false
}
if (!translationEnabled) {
return
}
val item = controller.currentTrainingItem() ?: return
val sentences = item.sentences
val needsTranslation = sentences.any { !it.text.isNullOrBlank() && it.zhText.isNullOrBlank() }
if (!needsTranslation) {
screenState.chineseTranslationLoading = false
screenState.chineseTranslationError = ""
applyChineseSubtitle(controller.currentSentence())
screenFactory.refreshReadingTranslations()
return
}
val generation = screenState.beginTranslationRequest()
screenState.chineseTranslationLoading = true
screenState.chineseTranslationError = ""
applyChineseSubtitle(controller.currentSentence())
screenFactory.refreshReadingTranslations()
translationRequest?.cancel()
val expectedIndexes = sentences.mapNotNull { sentence ->
sentence.index.takeIf { !sentence.text.isNullOrBlank() }
}
translationRequest = translationApi.translate(
item.id,
expectedIndexes,
object : SubtitleTranslationCallback {
override fun onProgress(result: SubtitleTranslationResult) {
if (!screenState.isCurrentTranslationRequest(generation, item.id)) return
applyTranslatedSentences(item.id, result.translations)
}
override fun onSuccess(result: SubtitleTranslationResult) {
if (!screenState.isCurrentTranslationRequest(generation, item.id)) return
applyTranslatedSentences(item.id, result.translations)
screenState.chineseTranslationLoading = false
screenState.chineseTranslationError = ""
applyChineseSubtitle(controller.currentSentence())
if (bindings.hasPlayerChrome) {
bindings.statusText.text = "已加载中文字幕"
}
}
override fun onError(error: Throwable) {
if (!screenState.isCurrentTranslationRequest(generation, item.id)) return
screenState.chineseTranslationLoading = false
screenState.chineseTranslationError = "中文字幕翻译失败"
applyChineseSubtitle(controller.currentSentence())
if (bindings.hasPlayerChrome) {
bindings.statusText.text = "中文字幕翻译失败:${error.message.orEmpty()}"
}
}
},
)
}
private fun screenActions() = object : SampleScreenActions {
override fun onModuleSelected(module: Module) = switchModule(module)
override fun onImportLocalVideo() = openLocalVideoPicker()
override fun onRefreshCatalog() = catalogController.loadCatalog()
override fun onSeekProgressPreview(progress: Int) {
if (!bindings.hasPlayerChrome) return
val duration = controller.snapshot().durationMs
@@ -427,13 +516,66 @@ class MainActivity : Activity() {
}
}
override fun onChineseSubtitlesChanged(enabled: Boolean) {
screenState.chineseSubtitlesEnabled = enabled
if (bindings.hasPlayerChrome) {
bindings.statusText.text = if (enabled) {
"已开启中文字幕"
} else {
"已关闭中文字幕"
}
}
if (enabled) {
ensureChineseTranslations()
} else {
translationRequest?.cancel()
screenState.chineseTranslationLoading = false
applyChineseSubtitle(controller.currentSentence())
}
}
override fun onReadingTranslationVisible(visible: Boolean) {
screenState.readingTranslationVisible = visible
if (!visible) {
translationRequest?.cancel()
screenState.chineseTranslationLoading = false
} else {
ensureChineseTranslations()
}
screenFactory.refreshReadingTranslations()
}
override fun isSentenceFavorite(sentence: SentenceBoundary): Boolean {
val item = controller.currentTrainingItem() ?: return false
return sentence.index in sentenceFavoriteStore.indexes(favoriteUserKey(), item.id)
}
override fun toggleSentenceFavorite(sentence: SentenceBoundary): Boolean {
val item = controller.currentTrainingItem() ?: return false
return sentenceFavoriteStore.toggle(favoriteUserKey(), item.id, sentence.index)
}
override fun onSentenceDoubleTapped(sentence: SentenceBoundary) {
screenState.continuousPlaybackEnabled = false
switchModule(Module.TRAIN)
controller.setContinuousPlayback(false)
controller.seekTo(sentence.startMs)
controller.play()
if (bindings.hasPlayerChrome) {
bindings.statusText.text = "已定位到第 ${sentence.index + 1} 句,单句循环播放中"
}
}
override fun onExplainCurrentSentence() {
val sentence = controller.currentSentence() ?: return
if (sentence.text.isNullOrBlank()) return
controller.pause()
val item = controller.currentTrainingItem()
explanationOverlay.show(
sentence,
controller.currentTrainingItem()?.title.orEmpty(),
item?.title.orEmpty(),
item?.id.orEmpty(),
item?.sentences.orEmpty(),
)
}
@@ -449,8 +591,8 @@ class MainActivity : Activity() {
override fun onReplaySentence() {
testSession.stopDubbingIfNeeded()
testSession.refreshTestUi()
controller.playCurrentSentenceAndStop()
testSession.refreshTestUi()
}
override fun onNextSentence() {
@@ -471,14 +613,18 @@ class MainActivity : Activity() {
catalogController.selectCourse(video)
}
override fun onTrainCatalogVideoClick(video: TrainingVideoSummary) {
catalogController.useCourseInPlace(video)
}
override fun onRefreshCourses() = catalogController.refreshCourses()
override fun onLogout() = logout()
override fun nickname(): String =
authSessionStore.session?.user?.nickname.orEmpty()
override fun favoriteSentences(): List<SentenceBoundary> {
val item = controller.currentTrainingItem() ?: return emptyList()
val indexes = sentenceFavoriteStore.indexes(favoriteUserKey(), item.id)
return item.sentences.filter { it.index in indexes }
}
}
private fun catalogHost() = object : CourseCatalogController.Host {
@@ -513,6 +659,13 @@ class MainActivity : Activity() {
screenFactory.renderCatalog()
refreshCurrentUi()
}
override fun syncSentenceExplanations(
videoHash: String,
sentences: List<SentenceBoundary>,
) {
SentenceExplanationStore(this@MainActivity).sync(videoHash, sentences)
}
}
private fun testHost() = object : TestSessionController.Host {

View File

@@ -0,0 +1,130 @@
package cn.learningpad.oraltrainer.sample
import android.widget.TextView
import androidx.core.text.HtmlCompat
/** Renders common LLM Markdown into a TextView without extra libraries. */
internal object MarkdownText {
private val heading = Regex("^(#{1,6})\\s*(.*)$")
private val unordered = Regex("^[-*+]\\s+(.*)$")
private val ordered = Regex("^\\d+[.)]\\s+(.*)$")
private val boldItalic = Regex("\\*\\*\\*(.+?)\\*\\*\\*")
private val bold = Regex("(\\*\\*|__)(.+?)\\1")
private val italic = Regex("(?<!\\*)\\*(?!\\*)(.+?)(?<!\\*)\\*(?!\\*)")
private val italicUnderscore = Regex("(?<!\\w)_(?!_)(.+?)(?<!_)_(?!\\w)")
private val inlineCode = Regex("`([^`]+)`")
private val image = Regex("!\\[([^\\]]*)]\\(([^)]+)\\)")
private val link = Regex("\\[([^\\]]+)]\\(([^)]+)\\)")
fun set(view: TextView, markdown: String) {
view.text = render(markdown)
}
fun render(markdown: String): CharSequence {
return HtmlCompat.fromHtml(toHtml(markdown), HtmlCompat.FROM_HTML_MODE_COMPACT)
}
internal fun toHtml(markdown: String): String {
val normalized = markdown.replace("\r\n", "\n").replace('\r', '\n')
val html = StringBuilder()
var inFence = false
val code = StringBuilder()
var listTag: String? = null
fun closeList() {
listTag?.let { html.append("</$it>") }
listTag = null
}
fun openList(tag: String) {
if (listTag != tag) {
closeList()
listTag = tag
html.append('<').append(tag).append('>')
}
}
for (line in normalized.split('\n')) {
val trimmed = line.trim()
if (trimmed.startsWith("```")) {
if (inFence) {
html.append("<pre><code>").append(code).append("</code></pre>")
code.setLength(0)
inFence = false
} else {
closeList()
inFence = true
}
continue
}
if (inFence) {
if (code.isNotEmpty()) code.append('\n')
code.append(escape(line))
continue
}
if (trimmed.isEmpty()) {
closeList()
continue
}
val headingMatch = heading.matchEntire(trimmed)
if (headingMatch != null) {
closeList()
val level = headingMatch.groupValues[1].length
html.append("<h$level>").append(inline(headingMatch.groupValues[2])).append("</h$level>")
continue
}
val unorderedMatch = unordered.matchEntire(trimmed)
if (unorderedMatch != null) {
openList("ul")
html.append("<li>").append(inline(unorderedMatch.groupValues[1])).append("</li>")
continue
}
val orderedMatch = ordered.matchEntire(trimmed)
if (orderedMatch != null) {
openList("ol")
html.append("<li>").append(inline(orderedMatch.groupValues[1])).append("</li>")
continue
}
closeList()
if (trimmed.startsWith(">")) {
html.append("<blockquote>")
.append(inline(trimmed.removePrefix(">").trimStart()))
.append("</blockquote>")
} else {
html.append("<p>").append(inline(trimmed)).append("</p>")
}
}
if (inFence) {
html.append("<pre><code>").append(code).append("</code></pre>")
}
closeList()
return html.toString()
}
private fun inline(text: String): String {
val codes = mutableListOf<String>()
var current = escape(text)
current = inlineCode.replace(current) { match ->
codes += "<code>${match.groupValues[1]}</code>"
"%%MDCODE${codes.lastIndex}%%"
}
current = image.replace(current) { match -> escape(match.groupValues[1]) }
current = link.replace(current) { match ->
"""<a href="${match.groupValues[2]}">${match.groupValues[1]}</a>"""
}
current = boldItalic.replace(current) { match -> "<b><i>${match.groupValues[1]}</i></b>" }
current = bold.replace(current) { match -> "<b>${match.groupValues[2]}</b>" }
current = italic.replace(current) { match -> "<i>${match.groupValues[1]}</i>" }
current = italicUnderscore.replace(current) { match -> "<i>${match.groupValues[1]}</i>" }
codes.forEachIndexed { index, value ->
current = current.replace("%%MDCODE$index%%", value)
}
return current
}
private fun escape(text: String): String =
text.replace("&", "&amp;")
.replace("<", "&lt;")
.replace(">", "&gt;")
}

View File

@@ -13,12 +13,14 @@ internal class SampleBindings {
lateinit var lessonTitleText: TextView
lateinit var statusText: TextView
lateinit var sentenceText: TextView
lateinit var sentenceZhText: TextView
lateinit var sentenceMetaText: TextView
lateinit var timeText: TextView
lateinit var speedText: TextView
lateinit var progressBar: SeekBar
lateinit var catalogList: LinearLayout
lateinit var catalogStatusText: TextView
var catalogRefreshButton: Button? = null
lateinit var testStatusText: TextView
lateinit var scoreSummaryText: TextView
lateinit var scoreDetailText: TextView
@@ -29,11 +31,13 @@ internal class SampleBindings {
var dubbingStatusText: TextView? = null
var explainSentenceButton: ImageButton? = null
var myContentList: LinearLayout? = null
var favoriteSentenceList: LinearLayout? = null
val hasPlayerChrome: Boolean
get() = ::lessonTitleText.isInitialized &&
::statusText.isInitialized &&
::sentenceText.isInitialized &&
::sentenceZhText.isInitialized &&
::sentenceMetaText.isInitialized &&
::timeText.isInitialized &&
::speedText.isInitialized &&

View File

@@ -1,5 +1,6 @@
package cn.learningpad.oraltrainer.sample
import cn.learningpad.oraltrainer.sdk.SentenceBoundary
import cn.learningpad.oraltrainer.sdk.TrainingVideoSummary
internal interface ModuleTabActions {
@@ -8,6 +9,7 @@ internal interface ModuleTabActions {
internal interface CourseScreenActions {
fun onSelectCourse(video: TrainingVideoSummary)
fun onRefreshCourses()
}
internal interface SentenceExplainActions {
@@ -16,13 +18,18 @@ internal interface SentenceExplainActions {
internal interface TrainScreenActions :
SentenceExplainActions {
fun onImportLocalVideo()
fun onRefreshCatalog()
fun onSeekProgressPreview(progress: Int)
fun onSeekStart()
fun onSeekStop(progress: Int)
fun onContinuousPlaybackChanged(enabled: Boolean)
fun onTrainCatalogVideoClick(video: TrainingVideoSummary)
fun onChineseSubtitlesChanged(enabled: Boolean)
}
internal interface ReadingScreenActions {
fun onReadingTranslationVisible(visible: Boolean)
fun isSentenceFavorite(sentence: SentenceBoundary): Boolean
fun toggleSentenceFavorite(sentence: SentenceBoundary): Boolean
fun onSentenceDoubleTapped(sentence: SentenceBoundary)
}
internal interface TestScreenActions :
@@ -43,11 +50,14 @@ internal interface TestScreenActions :
internal interface MineScreenActions {
fun onLogout()
fun nickname(): String
fun favoriteSentences(): List<SentenceBoundary>
fun toggleSentenceFavorite(sentence: SentenceBoundary): Boolean
}
internal interface SampleScreenActions :
ModuleTabActions,
CourseScreenActions,
TrainScreenActions,
ReadingScreenActions,
TestScreenActions,
MineScreenActions

View File

@@ -7,6 +7,7 @@ import android.widget.LinearLayout
import cn.learningpad.oraltrainer.sample.ui.CourseScreen
import cn.learningpad.oraltrainer.sample.ui.MineScreen
import cn.learningpad.oraltrainer.sample.ui.PlayerChrome
import cn.learningpad.oraltrainer.sample.ui.ReadingScreen
import cn.learningpad.oraltrainer.sample.ui.SampleWidgets
import cn.learningpad.oraltrainer.sample.ui.TestScreen
import cn.learningpad.oraltrainer.sample.ui.TrainScreen
@@ -33,13 +34,16 @@ internal class SampleScreenFactory(
private val courseScreen = CourseScreen(activity, bindings, screenState, widgets, actions)
private val trainScreen = TrainScreen(
activity = activity,
bindings = bindings,
screenState = screenState,
widgets = widgets,
playerChrome = playerChrome,
actions = actions,
)
private val testScreen = TestScreen(activity, bindings, screenState, playerChrome, actions)
private val readingScreen = ReadingScreen(
activity = activity,
screenState = screenState,
controller = controller,
widgets = widgets,
actions = actions,
)
private val mineScreen = MineScreen(activity, bindings, screenState, widgets, actions)
fun createContentView(): View {
@@ -53,36 +57,54 @@ internal class SampleScreenFactory(
return LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
layoutParams = LinearLayout.LayoutParams(
LinearLayout.LayoutParams.MATCH_PARENT,
LinearLayout.LayoutParams.MATCH_PARENT,
)
setBackgroundColor(SampleTheme.background)
addView(widgets.moduleTabBar(actions))
when (screenState.activeModule) {
Module.COURSE -> addView(courseScreen.createSection())
Module.MINE -> addView(mineScreen.createSection())
Module.TRAIN -> trainScreen.attachPortraitSections(this)
Module.READING -> addView(readingScreen.createSection())
Module.TEST -> testScreen.attachSections(this)
}
}
}
fun refreshReadingSentences() {
if (screenState.activeModule == Module.READING) {
readingScreen.renderSentences()
}
}
fun refreshReadingTranslations() {
if (screenState.activeModule == Module.READING) {
readingScreen.refreshTranslations()
}
}
fun scrollToReadingCurrentSentence() {
if (screenState.activeModule == Module.READING) {
readingScreen.scrollToCurrentSentence()
}
}
fun renderCatalog() {
when (screenState.activeModule) {
Module.COURSE -> courseScreen.renderCatalog()
Module.TRAIN -> trainScreen.renderCatalogStrip()
else -> Unit
if (screenState.activeModule == Module.COURSE) {
courseScreen.renderCatalog()
}
}
fun showCatalogLoading() {
when (screenState.activeModule) {
Module.COURSE -> courseScreen.showLoading()
Module.TRAIN -> trainScreen.showCatalogLoading()
else -> Unit
if (screenState.activeModule == Module.COURSE) {
courseScreen.showLoading()
}
}
fun setCatalogStatus(text: String) {
courseScreen.setStatus(text)
trainScreen.setCatalogStatus(text)
}
fun refreshMyContent(userApi: UserApi, token: String) {

View File

@@ -19,6 +19,7 @@ internal object SampleTheme {
val textDark = Color.rgb(18, 24, 31)
val textMuted = Color.rgb(151, 162, 174)
val textSubtle = Color.rgb(88, 98, 108)
val readingTranslation = Color.rgb(147, 197, 253)
val danger = Color.rgb(220, 38, 38)
val textOverlay = Color.rgb(163, 172, 182)
}

View File

@@ -91,7 +91,7 @@ internal class SentenceExplanationApi(
connection.useCaches = false
connection.setRequestProperty("Accept", "application/json")
connection.setRequestProperty("Content-Type", "application/json")
connection.setRequestProperty("User-Agent", "OralTrainerSample/0.1.4 (Android)")
connection.setRequestProperty("User-Agent", "OralTrainerSample/0.1.5 (Android)")
tokenProvider().takeIf { it.isNotBlank() }?.let {
connection.setRequestProperty("Authorization", "Bearer $it")
}

View File

@@ -18,11 +18,20 @@ internal class SentenceExplanationOverlay(
private val activity: Activity,
private val api: SentenceExplanationApi,
) {
private val store = SentenceExplanationStore(activity)
private var currentOverlay: View? = null
private var currentRequest: CancellableExplanationRequest? = null
private var requestGeneration = 0
fun show(sentence: SentenceBoundary, courseTitle: String) {
fun show(
sentence: SentenceBoundary,
courseTitle: String,
mediaId: String,
sentences: List<SentenceBoundary> = emptyList(),
) {
if (sentences.isNotEmpty()) {
store.sync(mediaId, sentences)
}
close()
val generation = ++requestGeneration
val root = FrameLayout(activity).apply {
@@ -51,13 +60,38 @@ internal class SentenceExplanationOverlay(
setTextColor(SampleTheme.textDark)
textSize = 19f
typeface = Typeface.DEFAULT_BOLD
layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f)
}
val saveHint = TextView(activity).apply {
text = HINT_UNSAVED
setTextColor(SampleTheme.textSubtle)
textSize = 12f
setPadding(6.dp, 0, 0, 0)
}
val saveButton = TextView(activity).apply {
text = "保存"
setTextColor(Color.WHITE)
textSize = 13f
typeface = Typeface.DEFAULT_BOLD
gravity = Gravity.CENTER
background = rounded(SampleTheme.button, 8f)
setPadding(12.dp, 6.dp, 12.dp, 6.dp)
isEnabled = false
alpha = 0.45f
}
val titleRow = LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
addView(title)
addView(saveButton)
addView(saveHint)
}
val subtitle = TextView(activity).apply {
text = sentence.text?.takeIf { it.isNotBlank() } ?: courseTitle
setTextColor(SampleTheme.accentDeep)
textSize = 14f
setPadding(0, 8.dp, 0, 12.dp)
maxLines = 3
maxLines = 5
}
val message = TextView(activity).apply {
text = "AI 正在讲解这句话…"
@@ -86,7 +120,7 @@ internal class SentenceExplanationOverlay(
).apply { topMargin = 16.dp }
setOnClickListener { close() }
}
card.addView(title)
card.addView(titleRow)
card.addView(subtitle)
card.addView(scroll)
card.addView(close)
@@ -94,6 +128,21 @@ internal class SentenceExplanationOverlay(
addToWindow(root)
currentOverlay = root
val cached = store.load(mediaId, sentence.index)
if (!cached.isNullOrBlank()) {
showMarkdown(message, cached)
markSaved(saveButton, saveHint)
return
}
var loadedExplanation: String? = null
saveButton.setOnClickListener {
val explanation = loadedExplanation ?: return@setOnClickListener
if (store.save(mediaId, sentence.index, explanation)) {
markSaved(saveButton, saveHint)
}
}
currentRequest = api.explain(
sentence.text.orEmpty(),
courseTitle,
@@ -101,14 +150,19 @@ internal class SentenceExplanationOverlay(
sentence.language,
object : SentenceExplanationCallback {
override fun onSuccess(result: SentenceExplanationResult) {
if (generation == requestGeneration) message.text = result.explanation
if (generation != requestGeneration) return
loadedExplanation = result.explanation
showMarkdown(message, result.explanation)
saveButton.isEnabled = true
saveButton.alpha = 1f
}
override fun onError(error: Throwable) {
if (generation == requestGeneration) {
message.text = "讲解失败:${error.message ?: "请稍后重试"}"
message.setTextColor(SampleTheme.danger)
}
if (generation != requestGeneration) return
message.text = "讲解失败:${error.message ?: "请稍后重试"}"
message.setTextColor(SampleTheme.danger)
saveButton.isEnabled = false
saveButton.alpha = 0.45f
}
},
)
@@ -128,6 +182,18 @@ internal class SentenceExplanationOverlay(
api.release()
}
private fun showMarkdown(message: TextView, explanation: String) {
message.setTextColor(SampleTheme.textDark)
MarkdownText.set(message, explanation)
}
private fun markSaved(saveButton: TextView, saveHint: TextView) {
saveButton.text = "已保存"
saveButton.isEnabled = false
saveButton.alpha = 0.7f
saveHint.text = HINT_SAVED
}
private fun addToWindow(view: View) {
(activity.window?.decorView as? ViewGroup)?.addView(view)
}
@@ -147,4 +213,9 @@ internal class SentenceExplanationOverlay(
cornerRadius = radiusDp.dp.toFloat()
strokeColor?.let { setStroke(1.dp, it) }
}
private companion object {
const val HINT_UNSAVED = "(下次直接看本地)"
const val HINT_SAVED = "(已保存本地)"
}
}

View File

@@ -0,0 +1,62 @@
package cn.learningpad.oraltrainer.sample
import android.content.Context
import cn.learningpad.oraltrainer.sdk.SentenceBoundary
import java.io.File
import java.util.Locale
/** Persists sentence explanations under app filesDir for reuse without another API call. */
internal class SentenceExplanationStore(context: Context) {
private val rootDir = File(context.applicationContext.filesDir, "sentence-explanations")
fun load(mediaId: String, sentenceIndex: Int): String? {
val file = fileFor(mediaId, sentenceIndex) ?: return null
if (!file.isFile || file.length() <= 0L) return null
return file.readText()
}
fun save(mediaId: String, sentenceIndex: Int, explanation: String): Boolean {
val text = explanation.trim()
if (text.isEmpty()) return false
val file = fileFor(mediaId, sentenceIndex) ?: return false
file.parentFile?.mkdirs()
file.writeText(text)
return file.isFile && file.length() > 0L
}
fun sync(mediaId: String, sentences: List<SentenceBoundary>) {
if (sentences.isEmpty()) return
val directoryName = sanitize(mediaId) ?: return
val dir = File(rootDir, directoryName)
val revision = revisionFor(sentences)
val marker = File(dir, REVISION_FILE)
val previous = marker.takeIf { it.isFile }?.readText()?.trim().orEmpty()
if (previous == revision) return
if (dir.exists()) {
dir.deleteRecursively()
}
dir.mkdirs()
marker.writeText(revision)
}
private fun fileFor(mediaId: String, sentenceIndex: Int): File? {
val directoryName = sanitize(mediaId) ?: return null
return File(File(rootDir, directoryName), "$sentenceIndex.txt")
}
companion object {
private const val REVISION_FILE = "revision.txt"
fun revisionFor(sentences: List<SentenceBoundary>): String =
sentences.joinToString("\n") { sentence ->
"${sentence.index}\t${sentence.text.orEmpty().trim()}"
}
fun sanitize(mediaId: String): String? {
val filtered = mediaId.trim().lowercase(Locale.US)
.replace(Regex("[^a-z0-9._-]"), "_")
.trim('_')
return filtered.takeIf { it.isNotBlank() && it != "." && it != ".." }
}
}
}

View File

@@ -0,0 +1,55 @@
package cn.learningpad.oraltrainer.sample
import android.content.Context
import java.security.MessageDigest
/** Persists reading-mode favorites locally, scoped by signed-in user and video. */
internal class SentenceFavoriteStore(context: Context) {
private val preferences =
context.getSharedPreferences(PREFERENCES_NAME, Context.MODE_PRIVATE)
fun toggle(userKey: String, mediaId: String, sentenceIndex: Int): Boolean {
val key = storageKey(userKey, mediaId) ?: return false
val favorites = readIndexes(key)
val isFavorite = sentenceIndex in favorites
val updated = if (isFavorite) {
favorites - sentenceIndex
} else {
favorites + sentenceIndex
}
preferences.edit()
.putString(key, updated.sorted().joinToString(SEPARATOR))
.apply()
return !isFavorite
}
fun indexes(userKey: String, mediaId: String): Set<Int> {
return storageKey(userKey, mediaId)?.let(::readIndexes).orEmpty()
}
private fun readIndexes(key: String): Set<Int> {
return preferences.getString(key, null).orEmpty()
.split(SEPARATOR)
.mapNotNull { it.trim().toIntOrNull() }
.toSet()
}
private fun storageKey(userKey: String, mediaId: String): String? {
val normalizedUser = userKey.trim()
val normalizedMedia = mediaId.trim()
if (normalizedUser.isEmpty() || normalizedMedia.isEmpty()) return null
return "$KEY_PREFIX$normalizedUser:$normalizedMedia"
}
companion object {
private const val PREFERENCES_NAME = "sentence_reading_favorites"
private const val KEY_PREFIX = "indexes"
private const val SEPARATOR = ","
fun userKey(token: String): String {
val digest = MessageDigest.getInstance("SHA-256")
.digest(token.toByteArray(Charsets.UTF_8))
return digest.joinToString("") { "%02x".format(it) }.take(24)
}
}
}

View File

@@ -0,0 +1,197 @@
package cn.learningpad.oraltrainer.sample
import android.content.Context
import android.os.Handler
import android.os.Looper
import android.os.SystemClock
import org.json.JSONObject
import java.io.IOException
import java.net.HttpURLConnection
import java.net.SocketTimeoutException
import java.net.URL
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.Future
data class SubtitleTranslationResult(
val videoHash: String,
val translations: Map<Int, String>,
val complete: Boolean = false,
val status: String = "",
val error: String = "",
)
interface SubtitleTranslationCallback {
fun onProgress(result: SubtitleTranslationResult) {}
fun onSuccess(result: SubtitleTranslationResult)
fun onError(error: Throwable)
}
class CancellableTranslationRequest internal constructor(
private val future: Future<*>,
) {
fun cancel() {
future.cancel(true)
}
}
/** Calls the app server; AI credentials never live in the mobile app. */
internal class SubtitleTranslationApi(
context: Context,
private val baseUrl: String,
private val tokenProvider: () -> String,
) {
private val appContext = context.applicationContext
private val mainHandler = Handler(Looper.getMainLooper())
private val executor: ExecutorService = Executors.newCachedThreadPool()
fun translate(
videoHash: String,
expectedIndexes: Collection<Int> = emptyList(),
callback: SubtitleTranslationCallback,
): CancellableTranslationRequest {
val future = executor.submit {
try {
val result = loadBlocking(videoHash, expectedIndexes) { partial ->
if (!Thread.currentThread().isInterrupted) {
mainHandler.post { callback.onProgress(partial) }
}
}
if (!Thread.currentThread().isInterrupted) {
mainHandler.post { callback.onSuccess(result) }
}
} catch (error: Throwable) {
if (!Thread.currentThread().isInterrupted) {
mainHandler.post { callback.onError(error) }
}
}
}
return CancellableTranslationRequest(future)
}
fun release() {
executor.shutdownNow()
}
private fun loadBlocking(
videoHash: String,
expectedIndexes: Collection<Int>,
onProgress: (SubtitleTranslationResult) -> Unit,
): SubtitleTranslationResult {
val expected = expectedIndexes.toSet()
val deadline = SystemClock.elapsedRealtime() + TRANSLATION_DEADLINE_MS
var kickJob = true
var lastError: Throwable? = null
var lastCount = -1
var stalledRounds = 0
while (SystemClock.elapsedRealtime() < deadline) {
if (Thread.currentThread().isInterrupted) {
throw InterruptedException("Subtitle translation cancelled")
}
val stored = runCatching {
requestBlocking(videoHash, method = "GET", readTimeoutMs = 15_000)
}.onFailure { lastError = it }.getOrNull()
if (stored == null) {
kickJob = true
} else {
onProgress(stored)
if (isComplete(stored, expected)) {
return stored
}
if (stored.translations.size > lastCount) {
lastCount = stored.translations.size
stalledRounds = 0
} else {
stalledRounds += 1
}
if (stored.status != "running" && stalledRounds >= 2) {
kickJob = true
}
}
if (kickJob) {
val posted = runCatching {
requestBlocking(videoHash, method = "POST", readTimeoutMs = 30_000)
}.onFailure { error ->
lastError = error
if (error !is SocketTimeoutException) {
stalledRounds += 1
}
}.getOrNull()
kickJob = false
if (posted != null) {
onProgress(posted)
if (isComplete(posted, expected)) {
return posted
}
if (posted.status == "error") {
lastError = IOException(posted.error.ifBlank { "Subtitle translation failed" })
}
}
}
Thread.sleep(POLL_INTERVAL_MS)
}
throw lastError ?: IOException("Subtitle translation timed out")
}
private fun isComplete(
result: SubtitleTranslationResult,
expected: Set<Int>,
): Boolean {
if (expected.isNotEmpty()) {
return expected.all { result.translations.containsKey(it) }
}
return result.complete
}
private fun requestBlocking(
videoHash: String,
method: String,
readTimeoutMs: Int,
): SubtitleTranslationResult {
val endpoint = baseUrl.trimEnd('/') + "/api/v1/videos/$videoHash/subtitle-translation"
val connection = URL(endpoint).openConnection() as HttpURLConnection
try {
connection.requestMethod = method
connection.connectTimeout = 15_000
connection.readTimeout = readTimeoutMs
connection.doOutput = false
connection.useCaches = false
connection.setRequestProperty("Accept", "application/json")
connection.setRequestProperty("User-Agent", "OralTrainerSample/0.1.5 (Android)")
tokenProvider().takeIf { it.isNotBlank() }?.let {
connection.setRequestProperty("Authorization", "Bearer $it")
}
val statusCode = connection.responseCode
val stream = if (statusCode in 200..299) connection.inputStream else connection.errorStream
val response = stream?.bufferedReader(Charsets.UTF_8)?.use { it.readText() }.orEmpty()
if (statusCode !in 200..299) {
throw IOException("Subtitle translation API returned HTTP $statusCode: $response")
}
val root = JSONObject(response)
val items = root.getJSONArray("translations")
val translations = buildMap {
for (index in 0 until items.length()) {
val item = items.getJSONObject(index)
val zh = item.optString("zh_text")
if (zh.isNotBlank()) {
put(item.getInt("index"), zh)
}
}
}
return SubtitleTranslationResult(
videoHash = root.getString("video_hash"),
translations = translations,
complete = root.optBoolean("complete", false),
status = root.optString("status"),
error = root.optString("error"),
)
} finally {
connection.disconnect()
}
}
private companion object {
const val POLL_INTERVAL_MS = 1_500L
const val TRANSLATION_DEADLINE_MS = 15 * 60 * 1000L
}
}

View File

@@ -0,0 +1,79 @@
package cn.learningpad.oraltrainer.sample
import android.app.Notification
import android.app.NotificationChannel
import android.app.NotificationManager
import android.app.PendingIntent
import android.app.Service
import android.content.Context
import android.content.Intent
import android.content.pm.ServiceInfo
import android.os.Build
import android.os.IBinder
/** Keeps the training player alive after the screen turns off or the app backgrounds. */
class TrainingPlaybackService : Service() {
override fun onBind(intent: Intent?): IBinder? = null
override fun onCreate() {
super.onCreate()
val manager = getSystemService(NOTIFICATION_SERVICE) as NotificationManager
manager.createNotificationChannel(
NotificationChannel(
CHANNEL_ID,
getString(R.string.training_playback_channel_name),
NotificationManager.IMPORTANCE_LOW,
),
)
}
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
val notification = buildNotification()
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
startForeground(
NOTIFICATION_ID,
notification,
ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PLAYBACK,
)
} else {
startForeground(NOTIFICATION_ID, notification)
}
return START_NOT_STICKY
}
private fun buildNotification(): Notification {
val contentIntent = PendingIntent.getActivity(
this,
0,
Intent(this, MainActivity::class.java).addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP),
PendingIntent.FLAG_IMMUTABLE,
)
return Notification.Builder(this, CHANNEL_ID)
.setContentTitle(getString(R.string.training_playback_notification_title))
.setContentText(getString(R.string.training_playback_notification_text))
.setSmallIcon(R.drawable.ic_notification_play)
.setContentIntent(contentIntent)
.setOngoing(true)
.setCategory(Notification.CATEGORY_TRANSPORT)
.build()
}
companion object {
private const val CHANNEL_ID = "training_playback"
private const val NOTIFICATION_ID = 1001
fun start(context: Context) {
val intent = Intent(context, TrainingPlaybackService::class.java)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
context.startForegroundService(intent)
} else {
context.startService(intent)
}
}
fun stop(context: Context) {
context.stopService(Intent(context, TrainingPlaybackService::class.java))
}
}
}

View File

@@ -1,5 +1,6 @@
package cn.learningpad.oraltrainer.sample
import cn.learningpad.oraltrainer.sdk.SentenceBoundaryApiResult
import cn.learningpad.oraltrainer.sdk.TrainingVideoSummary
internal class TrainingScreenState {
@@ -7,23 +8,46 @@ internal class TrainingScreenState {
var currentSentenceCount = 0
var catalogVideos: List<TrainingVideoSummary> = emptyList()
val courseEnrollment = mutableMapOf<String, Boolean>()
val courseBoundaries = mutableMapOf<String, SentenceBoundaryApiResult>()
var catalogStatus = "正在同步"
var catalogIsRefreshing = false
var activeModule = Module.COURSE
var currentCourse: TrainingVideoSummary? = null
var continuousPlaybackEnabled = false
var chineseSubtitlesEnabled = false
var readingTranslationVisible = true
var chineseTranslationLoading = false
var chineseTranslationError = ""
var testSubtitleVisible = false
var landscapeSubtitleVisible = true
private var boundaryRequestGeneration = 0
private var catalogRequestGeneration = 0
private var translationRequestGeneration = 0
fun beginBoundaryRequest(): Int = ++boundaryRequestGeneration
fun isCurrentBoundaryRequest(generation: Int, itemId: String): Boolean =
boundaryRequestGeneration == generation && activeItemId == itemId
fun beginCatalogRequest(): Int = ++catalogRequestGeneration
fun isCurrentCatalogRequest(generation: Int): Boolean =
catalogRequestGeneration == generation
fun beginTranslationRequest(): Int = ++translationRequestGeneration
fun isCurrentTranslationRequest(generation: Int, itemId: String): Boolean =
translationRequestGeneration == generation && activeItemId == itemId
fun clearCourse() {
currentCourse = null
activeItemId = ""
currentSentenceCount = 0
catalogVideos = emptyList()
courseEnrollment.clear()
courseBoundaries.clear()
catalogStatus = "正在同步"
catalogIsRefreshing = false
}
}

View File

@@ -3,6 +3,7 @@ package cn.learningpad.oraltrainer.sample.ui
import android.app.Activity
import android.graphics.Color
import android.graphics.Typeface
import android.view.Gravity
import android.view.View
import android.view.ViewGroup
import android.widget.LinearLayout
@@ -34,18 +35,31 @@ internal class CourseScreen(
orientation = LinearLayout.VERTICAL
setPadding(16.dp, 8.dp, 16.dp, 16.dp)
}
val header = LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
setPadding(16.dp, 0.dp, 16.dp, 0.dp)
addView(TextView(activity).apply {
text = "课程首页"
textSize = 26f
typeface = Typeface.DEFAULT_BOLD
setTextColor(Color.WHITE)
layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f)
})
bindings.catalogRefreshButton = widgets.headerButton("刷新课程") {
actions.onRefreshCourses()
}.also { button ->
button.isEnabled = !screenState.catalogIsRefreshing
button.alpha = if (screenState.catalogIsRefreshing) 0.55f else 1f
addView(button)
}
}
return ScrollView(activity).apply {
layoutParams = LinearLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, 0, 1f)
addView(LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
setPadding(0.dp, 12.dp, 0.dp, 16.dp)
addView(TextView(activity).apply {
text = "课程首页"
textSize = 26f
typeface = Typeface.DEFAULT_BOLD
setTextColor(Color.WHITE)
setPadding(16.dp, 0.dp, 16.dp, 0.dp)
})
addView(header)
addView(TextView(activity).apply {
text = "选择一门课程开始训练"
textSize = 13f
@@ -83,5 +97,10 @@ internal class CourseScreen(
if (bindings.hasCatalogChrome) {
bindings.catalogStatusText.text = text
}
bindings.catalogRefreshButton?.let { button ->
button.isEnabled = !screenState.catalogIsRefreshing
button.alpha = if (screenState.catalogIsRefreshing) 0.55f else 1f
button.text = if (screenState.catalogIsRefreshing) "刷新中" else "刷新课程"
}
}
}

View File

@@ -16,6 +16,7 @@ import cn.learningpad.oraltrainer.sample.SampleBindings
import cn.learningpad.oraltrainer.sample.SampleTheme
import cn.learningpad.oraltrainer.sample.TrainingScreenState
import cn.learningpad.oraltrainer.sample.UiStyled
import cn.learningpad.oraltrainer.sdk.SentenceBoundary
import cn.learningpad.oraltrainer.sdk.UserApi
internal class MineScreen(
@@ -33,6 +34,9 @@ internal class MineScreen(
bindings.myContentList = LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
}
bindings.favoriteSentenceList = LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
}
return ScrollView(activity).apply {
layoutParams = LinearLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, 0, 1f)
addView(LinearLayout(activity).apply {
@@ -52,6 +56,13 @@ internal class MineScreen(
44.dp,
).withMargins(0, 12.dp, 0, 20.dp)
})
addView(TextView(activity).apply {
text = "本视频收藏句子"
textSize = 18f
typeface = Typeface.DEFAULT_BOLD
setTextColor(Color.WHITE)
})
addView(bindings.favoriteSentenceList)
addView(TextView(activity).apply {
text = "我的测试结果"
textSize = 18f
@@ -64,6 +75,7 @@ internal class MineScreen(
}
fun refreshContent(userApi: UserApi, token: String) {
renderFavoriteSentences()
val list = bindings.myContentList ?: return
list.removeAllViews()
val course = screenState.currentCourse
@@ -125,6 +137,74 @@ internal class MineScreen(
}
}
private fun renderFavoriteSentences() {
val list = bindings.favoriteSentenceList ?: return
list.removeAllViews()
if (screenState.currentCourse == null) {
list.addView(widgets.infoCard("请先在「课程」中选择一门课程"))
return
}
val favorites = actions.favoriteSentences()
if (favorites.isEmpty()) {
list.addView(widgets.infoCard("当前课程还没有收藏句子"))
return
}
favorites.forEach { sentence ->
list.addView(favoriteCard(sentence) {
if (!actions.toggleSentenceFavorite(sentence)) {
renderFavoriteSentences()
}
})
}
}
private fun favoriteCard(
sentence: SentenceBoundary,
onRemove: () -> Unit,
): View {
val foreign = sentence.text?.takeIf { it.isNotBlank() } ?: "未提供文本"
val translation = sentence.zhText?.takeIf { it.isNotBlank() }
val cardText = buildString {
append("第 ${sentence.index + 1} 句")
append("\n$foreign")
translation?.let { append("\n$it") }
}
return LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
background = rounded(SampleTheme.selected, 8f)
setPadding(12.dp, 10.dp, 10.dp, 10.dp)
layoutParams = LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
).withMargins(0, 0, 0, 8.dp)
addView(TextView(activity).apply {
text = cardText
setTextColor(SampleTheme.textDark)
textSize = 13f
layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f)
})
addView(Button(activity).apply {
text = "移除"
isAllCaps = false
setTextColor(Color.WHITE)
textSize = 12f
typeface = Typeface.DEFAULT_BOLD
minWidth = 0
minHeight = 0
minimumWidth = 0
minimumHeight = 0
setPadding(10.dp, 0, 10.dp, 0)
background = rounded(SampleTheme.danger, 8f)
layoutParams = LinearLayout.LayoutParams(
ViewGroup.LayoutParams.WRAP_CONTENT,
32.dp,
).withMargins(8.dp, 0, 0, 0)
setOnClickListener { onRemove() }
})
}
}
private fun deleteResult(userApi: UserApi, token: String, attemptId: String) {
userApi.deleteResult(token, attemptId) { result ->
result.fold(

View File

@@ -4,7 +4,11 @@ import android.app.Activity
import android.content.pm.ActivityInfo
import android.content.res.ColorStateList
import android.graphics.Color
import android.graphics.drawable.GradientDrawable
import android.graphics.drawable.InsetDrawable
import android.graphics.Typeface
import android.text.TextUtils
import android.view.MotionEvent
import android.view.Gravity
import android.view.View
import android.view.ViewGroup
@@ -14,9 +18,12 @@ import android.widget.FrameLayout
import android.widget.ImageButton
import android.widget.ImageView
import android.widget.LinearLayout
import android.widget.ScrollView
import android.widget.SeekBar
import android.widget.Switch
import android.widget.TextView
import androidx.core.view.ViewCompat
import androidx.core.view.WindowInsetsCompat
import cn.learningpad.oraltrainer.sample.Module
import cn.learningpad.oraltrainer.sample.PROGRESS_MAX
import cn.learningpad.oraltrainer.sample.R
@@ -40,26 +47,50 @@ internal class PlayerChrome(
override val uiContext get() = activity
fun createPlayerSection(): View =
FrameLayout(activity).apply {
LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
background = rounded(SampleTheme.surface, 8f, SampleTheme.border)
clipToOutline = true
layoutParams = LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
0,
1f,
ViewGroup.LayoutParams.WRAP_CONTENT,
).withMargins(16.dp, 0, 16.dp, 10.dp)
addView(createPlayerView())
addView(createPlayerOverlay())
addView(
createRotationButton().apply {
layoutParams = FrameLayout.LayoutParams(
44.dp,
44.dp,
Gravity.BOTTOM or Gravity.END,
).apply {
marginEnd = 12.dp
bottomMargin = 12.dp
createPlayerOverlay().apply {
layoutParams = LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
)
},
)
addView(
object : FrameLayout(activity) {
override fun onMeasure(widthMeasureSpec: Int, heightMeasureSpec: Int) {
val width = MeasureSpec.getSize(widthMeasureSpec)
val height = (width * 9) / 16
super.onMeasure(
MeasureSpec.makeMeasureSpec(width, MeasureSpec.EXACTLY),
MeasureSpec.makeMeasureSpec(height, MeasureSpec.EXACTLY),
)
}
}.apply {
layoutParams = LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
)
addView(createPlayerView())
addView(
createRotationButton().apply {
layoutParams = FrameLayout.LayoutParams(
44.dp,
44.dp,
Gravity.BOTTOM or Gravity.END,
).apply {
marginEnd = 12.dp
bottomMargin = 12.dp
}
},
)
},
)
}
@@ -70,34 +101,114 @@ internal class PlayerChrome(
setPadding(18.dp, 16.dp, 18.dp, 16.dp)
layoutParams = LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
0,
1f,
).withMargins(16.dp, 0, 16.dp, 12.dp)
}
fun createLandscapeContentView(): View =
FrameLayout(activity).apply {
LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
setBackgroundColor(Color.BLACK)
addView(createPlayerView())
addView(createPlayerOverlay())
val subtitleContainer = LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
layoutParams = FrameLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
Gravity.BOTTOM,
ViewCompat.setOnApplyWindowInsetsListener(this) { view, insets ->
val bars = insets.getInsets(
WindowInsetsCompat.Type.systemBars() or WindowInsetsCompat.Type.displayCutout(),
)
addView(
createLandscapeSentencePanel().apply {
layoutParams = LinearLayout.LayoutParams(
val gestures = insets.getInsets(WindowInsetsCompat.Type.mandatorySystemGestures())
view.setPadding(
bars.left,
bars.top,
bars.right,
maxOf(bars.bottom, gestures.bottom, 24.dp),
)
WindowInsetsCompat.CONSUMED
}
addView(
createPlayerOverlay(compact = true).apply {
layoutParams = LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
)
},
)
addView(
FrameLayout(activity).apply {
layoutParams = LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
0,
1f,
)
addView(
object : FrameLayout(activity) {
override fun onMeasure(widthMeasureSpec: Int, heightMeasureSpec: Int) {
val maxWidth = MeasureSpec.getSize(widthMeasureSpec)
val maxHeight = MeasureSpec.getSize(heightMeasureSpec)
val heightFromWidth = if (maxWidth > 0) (maxWidth * 9) / 16 else 0
val width: Int
val height: Int
if (heightFromWidth <= maxHeight) {
width = maxWidth
height = heightFromWidth
} else {
height = maxHeight
width = (maxHeight * 16) / 9
}
super.onMeasure(
MeasureSpec.makeMeasureSpec(width, MeasureSpec.EXACTLY),
MeasureSpec.makeMeasureSpec(height, MeasureSpec.EXACTLY),
)
}
}.apply {
layoutParams = FrameLayout.LayoutParams(
ViewGroup.LayoutParams.WRAP_CONTENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
Gravity.TOP or Gravity.CENTER_HORIZONTAL,
)
addView(createPlayerView())
},
)
val subtitleContainer = object : LinearLayout(activity) {
private var forwardTouchToPlayer = false
override fun dispatchTouchEvent(event: MotionEvent): Boolean {
if (event.actionMasked == MotionEvent.ACTION_DOWN) {
forwardTouchToPlayer = !isTouchOverInteractiveControl(this, event)
}
val forwarded = if (forwardTouchToPlayer) {
bindings.playerView.forwardGestureFrom(this, event)
} else {
false
}
val handled = forwarded || super.dispatchTouchEvent(event)
if (event.actionMasked == MotionEvent.ACTION_UP ||
event.actionMasked == MotionEvent.ACTION_CANCEL
) {
forwardTouchToPlayer = false
}
return handled
}
}.apply {
orientation = LinearLayout.VERTICAL
layoutParams = FrameLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
Gravity.BOTTOM,
)
},
)
}
addView(subtitleContainer)
applyLandscapeSubtitleState(subtitleContainer, animate = false)
installLandscapeSwipeGesture(subtitleContainer)
addView(
createLandscapeSentencePanel().apply {
layoutParams = LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
)
},
)
}
addView(subtitleContainer)
applyLandscapeSubtitleState(subtitleContainer, animate = false)
installLandscapeSwipeGesture(subtitleContainer)
},
)
}
fun createSentenceContent(translucent: Boolean): LinearLayout {
@@ -106,6 +217,7 @@ internal class PlayerChrome(
textSize = 17f
typeface = Typeface.DEFAULT_BOLD
maxLines = 1
ellipsize = TextUtils.TruncateAt.END
}
bindings.sentenceMetaText = TextView(activity).apply {
setTextColor(if (translucent) SampleTheme.accentLight else SampleTheme.accentDeep)
@@ -117,7 +229,14 @@ internal class PlayerChrome(
textSize = 22f
typeface = Typeface.DEFAULT_BOLD
setLineSpacing(2.dp.toFloat(), 1.05f)
setPadding(0, 10.dp, 0, 10.dp)
setPadding(0, 10.dp, 0, 4.dp)
}
bindings.sentenceZhText = TextView(activity).apply {
setTextColor(if (translucent) SampleTheme.textOverlay else SampleTheme.textSubtle)
textSize = 16f
setLineSpacing(2.dp.toFloat(), 1.05f)
setPadding(0, 0, 0, 8.dp)
visibility = View.GONE
}
bindings.progressBar = SeekBar(activity).apply {
max = PROGRESS_MAX
@@ -157,28 +276,69 @@ internal class PlayerChrome(
background = rounded(SampleTheme.overlayFaint, 0f, null)
setPadding(18.dp, 14.dp, 18.dp, 16.dp)
}
addView(bindings.lessonTitleText)
if (screenState.activeModule == Module.TRAIN) {
addView(createContinuousPlaybackRow(translucent))
val topContent = LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
if (screenState.activeModule != Module.TRAIN) {
addView(bindings.lessonTitleText)
}
if (screenState.activeModule == Module.TRAIN) {
addView(createContinuousPlaybackRow(translucent))
addView(createChineseSubtitleRow(translucent))
}
addView(bindings.sentenceMetaText)
addView(LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.TOP
addView(
LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
addView(
bindings.sentenceText,
LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
),
)
addView(
bindings.sentenceZhText,
LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
),
)
},
LinearLayout.LayoutParams(
0,
ViewGroup.LayoutParams.WRAP_CONTENT,
1f,
),
)
bindings.explainSentenceButton = createExplainButton(translucent)
addView(
bindings.explainSentenceButton,
LinearLayout.LayoutParams(32.dp, 32.dp).apply { marginStart = 10.dp },
)
})
}
addView(bindings.sentenceMetaText)
addView(LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
addView(
bindings.sentenceText,
LinearLayout.LayoutParams(
0,
ViewGroup.LayoutParams.WRAP_CONTENT,
1f,
),
)
bindings.explainSentenceButton = createExplainButton(translucent)
addView(
bindings.explainSentenceButton,
LinearLayout.LayoutParams(32.dp, 32.dp).apply { marginStart = 10.dp },
)
})
addView(
ScrollView(activity).apply {
isFillViewport = true
isVerticalScrollBarEnabled = false
overScrollMode = View.OVER_SCROLL_NEVER
addView(
topContent,
LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
),
)
},
LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
if (translucent) ViewGroup.LayoutParams.WRAP_CONTENT else 0,
if (translucent) 0f else 1f,
),
)
addView(bindings.progressBar)
addView(bindings.statusText)
}
@@ -193,16 +353,15 @@ internal class PlayerChrome(
)
}.also { bindings.playerView = it }
private fun createPlayerOverlay(): View {
private fun createPlayerOverlay(compact: Boolean = false): View {
val overlay = LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
setPadding(12.dp, 12.dp, 12.dp, 12.dp)
layoutParams = FrameLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
Gravity.TOP,
)
if (compact) {
setPadding(12.dp, 4.dp, 12.dp, 0)
} else {
setPadding(12.dp, 10.dp, 12.dp, 8.dp)
}
}
bindings.timeText = widgets.overlayPill("--:-- / --:--").apply {
layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f)
@@ -217,11 +376,27 @@ internal class PlayerChrome(
ImageButton(activity).apply {
setImageResource(R.drawable.ic_screen_rotation)
imageTintList = ColorStateList.valueOf(Color.WHITE)
background = rounded(Color.argb(178, 9, 12, 16), 22.dp.toFloat())
val backgroundSize = 22.dp
val buttonSize = 44.dp
val inset = (buttonSize - backgroundSize) / 2
background = InsetDrawable(
GradientDrawable().apply {
shape = GradientDrawable.OVAL
setColor(Color.argb(102, 9, 12, 16))
setSize(backgroundSize, backgroundSize)
},
inset,
)
scaleType = ImageView.ScaleType.CENTER_INSIDE
setPadding(10.dp, 10.dp, 10.dp, 10.dp)
contentDescription = "横屏"
layoutParams = LinearLayout.LayoutParams(44.dp, 44.dp)
minimumWidth = 0
minimumHeight = 0
clipToOutline = false
if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.M) {
foreground = null
}
layoutParams = LinearLayout.LayoutParams(buttonSize, buttonSize)
setOnClickListener {
activity.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_SENSOR_LANDSCAPE
}
@@ -276,6 +451,29 @@ internal class PlayerChrome(
})
}
private fun createChineseSubtitleRow(translucent: Boolean): View =
LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
setPadding(0, 6.dp, 0, 0)
addView(TextView(activity).apply {
text = "中文字幕"
setTextColor(if (translucent) SampleTheme.textOverlay else SampleTheme.textSubtle)
textSize = 13f
typeface = Typeface.DEFAULT_BOLD
layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f)
})
addView(Switch(activity).apply {
isChecked = screenState.chineseSubtitlesEnabled
buttonTintList = ColorStateList.valueOf(
if (translucent) SampleTheme.accentLight else SampleTheme.accent,
)
setOnCheckedChangeListener { _, checked ->
seekActions.onChineseSubtitlesChanged(checked)
}
})
}
private fun installLandscapeSwipeGesture(subtitleContainer: View) {
bindings.playerView.setVerticalSwipeListener { isSwipeUp ->
if (isSwipeUp == screenState.landscapeSubtitleVisible) return@setVerticalSwipeListener
@@ -327,4 +525,33 @@ internal class PlayerChrome(
}
view.startAnimation(animation)
}
/** Lets controls own their touches while the subtitle background drives player gestures. */
private fun isTouchOverInteractiveControl(root: View, event: MotionEvent): Boolean =
hasInteractiveTouchTarget(root, event.rawX, event.rawY)
private fun hasInteractiveTouchTarget(view: View, screenX: Float, screenY: Float): Boolean {
if (view.visibility != View.VISIBLE || !view.isEnabled) {
return false
}
if (view.isClickable || view.isLongClickable || view.isContextClickable) {
val location = IntArray(2).also(view::getLocationOnScreen)
if (screenX >= location[0] &&
screenX < location[0] + view.width &&
screenY >= location[1] &&
screenY < location[1] + view.height
) {
return true
}
}
if (view !is ViewGroup) {
return false
}
for (index in view.childCount - 1 downTo 0) {
if (hasInteractiveTouchTarget(view.getChildAt(index), screenX, screenY)) {
return true
}
}
return false
}
}

View File

@@ -0,0 +1,306 @@
package cn.learningpad.oraltrainer.sample.ui
import android.app.Activity
import android.graphics.Color
import android.graphics.Typeface
import android.os.SystemClock
import android.view.Gravity
import android.view.View
import android.view.ViewGroup
import android.widget.Button
import android.widget.LinearLayout
import android.widget.Switch
import android.widget.TextView
import androidx.recyclerview.widget.LinearLayoutManager
import androidx.recyclerview.widget.RecyclerView
import cn.learningpad.oraltrainer.sample.ReadingScreenActions
import cn.learningpad.oraltrainer.sample.SampleTheme
import cn.learningpad.oraltrainer.sample.TrainingScreenState
import cn.learningpad.oraltrainer.sample.UiStyled
import cn.learningpad.oraltrainer.sdk.OralTrainerController
import cn.learningpad.oraltrainer.sdk.SentenceBoundary
/** Lazy-loading full-text reading view for the selected course. */
internal class ReadingScreen(
private val activity: Activity,
private val screenState: TrainingScreenState,
private val controller: OralTrainerController,
private val widgets: SampleWidgets,
private val actions: ReadingScreenActions,
) : UiStyled {
override val uiContext get() = activity
private var readingAdapter: ReadingSentenceAdapter? = null
private var titleText: TextView? = null
private var sentenceListView: RecyclerView? = null
private var lastClickedSentenceIndex = Int.MIN_VALUE
private var lastClickedAtMs = 0L
fun createSection(): View {
val root = LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
setPadding(16.dp, 12.dp, 16.dp, 16.dp)
}
val adapter = ReadingSentenceAdapter()
readingAdapter = adapter
val list = RecyclerView(activity).apply {
layoutManager = LinearLayoutManager(activity)
this.adapter = adapter
itemAnimator = null
setItemViewCacheSize(12)
clipToPadding = false
setPadding(0, 0, 0, 16.dp)
layoutParams = LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
0,
1f,
)
}
sentenceListView = list
val title = TextView(activity).apply {
setTextColor(Color.WHITE)
textSize = 15f
typeface = Typeface.DEFAULT_BOLD
setPadding(0, 0, 0, 6.dp)
}
titleText = title
root.addView(title)
root.addView(createTranslationToggle())
root.addView(list)
renderSentences()
return root
}
fun renderSentences() {
val item = controller.currentTrainingItem()
val baseTitle = screenState.currentCourse?.title.orEmpty()
.ifBlank { item?.title ?: "阅读模式" }
val sentences = item?.sentences.orEmpty()
titleText?.text = if (sentences.isEmpty()) {
baseTitle
} else {
"$baseTitle · ${sentences.size} 句"
}
readingAdapter?.submitSentences(sentences)
}
fun refreshTranslations() {
readingAdapter?.notifyDataSetChanged()
}
fun refreshFavorites() {
readingAdapter?.notifyDataSetChanged()
}
fun scrollToCurrentSentence() {
val list = sentenceListView ?: return
val currentSentence = controller.currentSentence() ?: return
val item = controller.currentTrainingItem() ?: return
val position = item.sentences.indexOfFirst { it.index == currentSentence.index }
if (position < 0) return
list.post {
if (sentenceListView !== list) return@post
val layoutManager = list.layoutManager as? LinearLayoutManager ?: return@post
val offset = (list.height * 0.25f).toInt().coerceAtLeast(0)
layoutManager.scrollToPositionWithOffset(position, offset)
}
}
private fun createTranslationToggle(): View =
LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
setPadding(0, 4.dp, 0, 10.dp)
addView(TextView(activity).apply {
text = "中文翻译"
setTextColor(Color.WHITE)
textSize = 15f
typeface = Typeface.DEFAULT_BOLD
layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f)
})
addView(Switch(activity).apply {
isChecked = screenState.readingTranslationVisible
buttonTintList = android.content.res.ColorStateList.valueOf(SampleTheme.accent)
setOnCheckedChangeListener { _, checked ->
actions.onReadingTranslationVisible(checked)
}
})
}
private fun translationText(sentence: SentenceBoundary): String = when {
!sentence.zhText.isNullOrBlank() -> sentence.zhText.orEmpty()
screenState.chineseTranslationLoading -> "正在翻译中文…"
screenState.chineseTranslationError.isNotBlank() -> screenState.chineseTranslationError
else -> "暂无中文翻译"
}
private inner class ReadingSentenceAdapter :
RecyclerView.Adapter<RecyclerView.ViewHolder>() {
private var sentences: List<SentenceBoundary> = emptyList()
init {
setHasStableIds(true)
}
override fun getItemId(position: Int): Long {
return if (sentences.isEmpty()) EMPTY_ID else sentences[position].index.toLong()
}
override fun getItemViewType(position: Int): Int {
return if (sentences.isEmpty()) TYPE_EMPTY else TYPE_SENTENCE
}
override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): RecyclerView.ViewHolder {
return if (viewType == TYPE_EMPTY) {
val empty = TextView(activity).apply {
text = "当前课程还没有可阅读的句子文本"
setTextColor(SampleTheme.textDark)
textSize = 13f
background = rounded(SampleTheme.selected, 8f)
setPadding(12.dp, 10.dp, 12.dp, 10.dp)
layoutParams = RecyclerView.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
)
}
EmptyViewHolder(empty)
} else {
SentenceViewHolder(createSentenceView())
}
}
override fun onBindViewHolder(holder: RecyclerView.ViewHolder, position: Int) {
if (holder !is SentenceViewHolder || sentences.isEmpty()) return
val sentence = sentences[position]
attachDoubleTap(holder.itemView, sentence)
holder.text.text =
"${sentence.index + 1}. ${sentence.text?.takeIf { it.isNotBlank() } ?: "未提供文本"}"
holder.translation.visibility = if (screenState.readingTranslationVisible) {
View.VISIBLE
} else {
View.GONE
}
holder.translation.text = translationText(sentence)
bindFavoriteButton(holder.favoriteButton, sentence)
}
override fun getItemCount(): Int {
return if (sentences.isEmpty()) 1 else sentences.size
}
fun submitSentences(updated: List<SentenceBoundary>) {
sentences = updated
notifyDataSetChanged()
}
private fun createSentenceView(): View {
val text = TextView(activity).apply {
setTextColor(Color.WHITE)
textSize = 16f
}
val translation = TextView(activity).apply {
setTextColor(SampleTheme.readingTranslation)
textSize = 14f
setPadding(0, 6.dp, 0, 0)
}
val favoriteButton = Button(activity).apply {
isAllCaps = false
setTextColor(Color.WHITE)
textSize = 12f
typeface = Typeface.DEFAULT_BOLD
minWidth = 0
minHeight = 0
minimumWidth = 0
minimumHeight = 0
setPadding(12.dp, 0, 12.dp, 0)
layoutParams = LinearLayout.LayoutParams(
ViewGroup.LayoutParams.WRAP_CONTENT,
32.dp,
).withMargins(10.dp, 0, 0, 0)
}
val textColumn = LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f)
addView(text)
addView(translation)
}
return LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
background = rounded(SampleTheme.surface, 10f, SampleTheme.border)
setPadding(14.dp, 12.dp, 10.dp, 12.dp)
layoutParams = RecyclerView.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
).apply {
bottomMargin = 10.dp
}
addView(textColumn)
addView(favoriteButton)
}
}
private fun bindFavoriteButton(button: Button, sentence: SentenceBoundary) {
val isFavorite = actions.isSentenceFavorite(sentence)
button.setText(if (isFavorite) "已收藏" else "收藏")
button.background = rounded(
if (isFavorite) SampleTheme.accent else SampleTheme.button,
8f,
)
button.setOnClickListener {
val favorite = actions.toggleSentenceFavorite(sentence)
button.setText(if (favorite) "已收藏" else "收藏")
button.background = rounded(
if (favorite) SampleTheme.accent else SampleTheme.button,
8f,
)
}
}
private fun attachDoubleTap(view: View, sentence: SentenceBoundary) {
view.isClickable = true
view.setOnClickListener {
val now = SystemClock.elapsedRealtime()
val isDoubleTap = lastClickedSentenceIndex == sentence.index &&
now - lastClickedAtMs <= DOUBLE_TAP_INTERVAL_MS
if (isDoubleTap) {
lastClickedSentenceIndex = Int.MIN_VALUE
lastClickedAtMs = 0L
actions.onSentenceDoubleTapped(sentence)
} else {
lastClickedSentenceIndex = sentence.index
lastClickedAtMs = now
}
}
}
private inner class EmptyViewHolder(root: View) : RecyclerView.ViewHolder(root)
private inner class SentenceViewHolder(
root: View,
) : RecyclerView.ViewHolder(root) {
val text: TextView
val translation: TextView
val favoriteButton: Button
init {
val textColumn = root as ViewGroup
val content = textColumn.getChildAt(0) as ViewGroup
text = content.getChildAt(0) as TextView
translation = content.getChildAt(1) as TextView
favoriteButton = textColumn.getChildAt(1) as Button
}
}
}
private companion object {
private const val TYPE_EMPTY = 0
private const val TYPE_SENTENCE = 1
private const val EMPTY_ID = Long.MIN_VALUE
private const val DOUBLE_TAP_INTERVAL_MS = 300L
}
}

View File

@@ -30,6 +30,7 @@ internal class SampleWidgets(
setPadding(16.dp, 40.dp, 16.dp, 0)
addView(moduleTab("课程", Module.COURSE, actions))
addView(moduleTab("训练", Module.TRAIN, actions))
addView(moduleTab("阅读", Module.READING, actions))
addView(moduleTab("测试", Module.TEST, actions))
addView(moduleTab("我的", Module.MINE, actions))
}
@@ -153,40 +154,4 @@ internal class SampleWidgets(
}
}
fun videoCard(
id: String,
title: String,
meta: String,
onClick: () -> Unit,
): View {
val selected = screenState.activeItemId == id
val titleColor = if (selected) SampleTheme.textDark else Color.WHITE
val metaColor = if (selected) SampleTheme.accentDeep else SampleTheme.textMuted
return LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
gravity = Gravity.CENTER_VERTICAL
isClickable = true
background = if (selected) {
rounded(SampleTheme.selected, 8f, SampleTheme.accent)
} else {
rounded(SampleTheme.surface, 8f, SampleTheme.border)
}
setPadding(14.dp, 12.dp, 14.dp, 12.dp)
layoutParams = LinearLayout.LayoutParams(218.dp, 84.dp).withMargins(0, 0, 10.dp, 0)
setOnClickListener { onClick() }
addView(TextView(activity).apply {
text = title
setTextColor(titleColor)
textSize = 15f
typeface = Typeface.DEFAULT_BOLD
maxLines = 2
})
addView(TextView(activity).apply {
text = meta
setTextColor(metaColor)
textSize = 12f
setPadding(0, 6.dp, 0, 0)
})
}
}
}

View File

@@ -5,23 +5,14 @@ import android.graphics.Color
import android.graphics.Typeface
import android.view.Gravity
import android.view.View
import android.view.ViewGroup
import android.widget.HorizontalScrollView
import android.widget.LinearLayout
import android.widget.TextView
import cn.learningpad.oraltrainer.sample.SampleBindings
import cn.learningpad.oraltrainer.sample.SampleTheme
import cn.learningpad.oraltrainer.sample.TrainScreenActions
import cn.learningpad.oraltrainer.sample.TrainingScreenState
import cn.learningpad.oraltrainer.sample.UiStyled
internal class TrainScreen(
private val activity: Activity,
private val bindings: SampleBindings,
private val screenState: TrainingScreenState,
private val widgets: SampleWidgets,
private val playerChrome: PlayerChrome,
private val actions: TrainScreenActions,
) : UiStyled {
override val uiContext get() = activity
@@ -29,47 +20,13 @@ internal class TrainScreen(
root.addView(createHeader())
root.addView(playerChrome.createPlayerSection())
root.addView(playerChrome.createSentenceSection())
root.addView(createCatalogSection())
}
fun renderCatalogStrip() {
if (!bindings.hasCatalogChrome) return
bindings.catalogList.removeAllViews()
bindings.catalogStatusText.text = screenState.catalogStatus
screenState.catalogVideos.forEach { video ->
val enrolled = screenState.courseEnrollment[video.videoHash] == true
bindings.catalogList.addView(
widgets.videoCard(
id = video.videoHash,
title = video.title,
meta = if (enrolled) {
"已选修 · ${video.sentenceCount} 句"
} else {
"未选修 · 点击选修并进入"
},
onClick = { actions.onTrainCatalogVideoClick(video) },
),
)
}
}
fun showCatalogLoading() {
if (!bindings.hasCatalogChrome) return
bindings.catalogList.removeAllViews()
bindings.catalogList.addView(widgets.infoCard("课程加载中..."))
}
fun setCatalogStatus(text: String) {
if (bindings.hasCatalogChrome) {
bindings.catalogStatusText.text = text
}
}
private fun createHeader(): View {
val titleBlock = LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
return LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f)
setPadding(18.dp, 16.dp, 18.dp, 12.dp)
addView(TextView(activity).apply {
text = "八哥口语"
setTextColor(Color.WHITE)
@@ -80,52 +37,10 @@ internal class TrainScreen(
addView(TextView(activity).apply {
text = "外语跟读训练神器"
setTextColor(SampleTheme.textMuted)
textSize = 20f
setPadding(1.dp, 4.dp, 0, 0)
textSize = 18f
includeFontPadding = false
setPadding(12.dp, 6.dp, 0, 0)
})
}
return LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
setPadding(18.dp, 16.dp, 18.dp, 12.dp)
addView(titleBlock)
addView(widgets.headerButton("导入") { actions.onImportLocalVideo() })
addView(widgets.headerButton("刷新") { actions.onRefreshCatalog() })
}
}
private fun createCatalogSection(): View {
bindings.catalogStatusText = TextView(activity).apply {
setTextColor(SampleTheme.textMuted)
textSize = 13f
gravity = Gravity.END
text = "正在同步"
layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f)
}
bindings.catalogList = LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
setPadding(0, 10.dp, 0, 4.dp)
}
val header = LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
addView(TextView(activity).apply {
text = "课程"
setTextColor(Color.WHITE)
textSize = 16f
typeface = Typeface.DEFAULT_BOLD
})
addView(bindings.catalogStatusText)
}
val scroller = HorizontalScrollView(activity).apply {
isHorizontalScrollBarEnabled = false
addView(bindings.catalogList)
}
return LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
setPadding(16.dp, 0, 16.dp, 14.dp)
addView(header)
addView(scroller)
}
}
}

View File

@@ -0,0 +1,9 @@
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="24dp"
android:height="24dp"
android:viewportWidth="24"
android:viewportHeight="24">
<path
android:fillColor="#FFFFFFFF"
android:pathData="M8,5v14l11,-7z" />
</vector>

View File

@@ -1,4 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<resources>
<string name="app_name">八哥口语</string>
<string name="training_playback_channel_name">训练播放</string>
<string name="training_playback_notification_title">口语训练</string>
<string name="training_playback_notification_text">正在后台播放</string>
</resources>

View File

@@ -1,3 +1,6 @@
<resources>
<string name="app_name">MynaTalk</string>
<string name="training_playback_channel_name">Training playback</string>
<string name="training_playback_notification_title">Oral training</string>
<string name="training_playback_notification_text">Playing in the background</string>
</resources>

View File

@@ -10,6 +10,7 @@
AA0000000000000000000090 /* AuthAPI.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000091 /* AuthAPI.swift */; };
AA00000000000000000000A1 /* UserHistory.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA00000000000000000000A2 /* UserHistory.swift */; };
AA00000000000000000000A3 /* TrainingHistoryStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA00000000000000000000A4 /* TrainingHistoryStore.swift */; };
AA00000000000000000000B0 /* SentenceExplanationStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA00000000000000000000B1 /* SentenceExplanationStore.swift */; };
AA0000000000000000000092 /* AuthView.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000093 /* AuthView.swift */; };
AA0000000000000000000040 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000020 /* Assets.xcassets */; };
AA0000000000000000000042 /* OralTrainerApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA0000000000000000000022 /* OralTrainerApp.swift */; };
@@ -41,6 +42,7 @@
AA0000000000000000000091 /* AuthAPI.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AuthAPI.swift; sourceTree = "<group>"; };
AA00000000000000000000A2 /* UserHistory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UserHistory.swift; sourceTree = "<group>"; };
AA00000000000000000000A4 /* TrainingHistoryStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TrainingHistoryStore.swift; sourceTree = "<group>"; };
AA00000000000000000000B1 /* SentenceExplanationStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentenceExplanationStore.swift; sourceTree = "<group>"; };
AA0000000000000000000093 /* AuthView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AuthView.swift; sourceTree = "<group>"; };
AA000000000000000000000A /* OralTrainer.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = OralTrainer.app; sourceTree = BUILT_PRODUCTS_DIR; };
AA000000000000000000001F /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
@@ -131,6 +133,7 @@
AA000000000000000000002A /* SentenceBoundaryAPI.swift */,
AA000000000000000000002B /* ImitationAssessor.swift */,
AA00000000000000000000A4 /* TrainingHistoryStore.swift */,
AA00000000000000000000B1 /* SentenceExplanationStore.swift */,
AA00000000000000000000A2 /* UserHistory.swift */,
AA000000000000000000007E /* DubbingCompositor.swift */,
AA0000000000000000000081 /* AudioRecorderSession.swift */,
@@ -267,6 +270,7 @@
AA0000000000000000000090 /* AuthAPI.swift in Sources */,
AA00000000000000000000A1 /* UserHistory.swift in Sources */,
AA00000000000000000000A3 /* TrainingHistoryStore.swift in Sources */,
AA00000000000000000000B0 /* SentenceExplanationStore.swift in Sources */,
AA0000000000000000000092 /* AuthView.swift in Sources */,
AA0000000000000000000053 /* TestView.swift in Sources */,
AA0000000000000000000054 /* FullscreenPlayerView.swift in Sources */,

View File

@@ -0,0 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<array/>
</plist>

View File

@@ -9,4 +9,38 @@ final class AppDelegate: NSObject, UIApplicationDelegate {
) -> UIInterfaceOrientationMask {
AppDelegate.orientationLock
}
static func lockOrientation(_ mask: UIInterfaceOrientationMask) {
orientationLock = mask
guard let scene = UIApplication.shared.connectedScenes
.compactMap({ $0 as? UIWindowScene })
.first(where: { $0.activationState == .foregroundActive })
?? UIApplication.shared.connectedScenes.compactMap({ $0 as? UIWindowScene }).first
else {
return
}
func update(_ controller: UIViewController?) {
guard let controller else { return }
controller.setNeedsUpdateOfSupportedInterfaceOrientations()
update(controller.presentedViewController)
controller.children.forEach { update($0) }
}
let requested: UIInterfaceOrientationMask
if mask.contains(.landscapeRight) {
requested = .landscapeRight
} else if mask.contains(.landscapeLeft) {
requested = .landscapeLeft
} else {
requested = mask
}
scene.windows.forEach { update($0.rootViewController) }
scene.requestGeometryUpdate(.iOS(interfaceOrientations: requested)) { _ in }
DispatchQueue.main.async {
scene.windows.forEach { update($0.rootViewController) }
scene.requestGeometryUpdate(.iOS(interfaceOrientations: requested)) { _ in }
}
}
}

View File

@@ -15,6 +15,9 @@ enum AppConfig {
static let connectTimeout: TimeInterval = 15
static let readTimeout: TimeInterval = 30
static let assessmentTimeout: TimeInterval = 180
static let translationTimeout: TimeInterval = 30
static let translationDeadline: TimeInterval = 900
static let translationPollIntervalNs: UInt64 = 1_500_000_000
static let sha256Pattern = #"^[0-9a-fA-F]{64}$"#

View File

@@ -63,3 +63,22 @@ enum BoundaryState: Equatable {
case loaded(count: Int)
case failed(message: String)
}
/// 云端视频按需流式播放;若本地已有完整缓存文件则优先使用。
enum VideoCacheState: Equatable {
case notRequired
case downloading
case ready
case failed(message: String)
var statusMessage: String? {
switch self {
case .notRequired, .ready:
return nil
case .downloading:
return "正在缓冲视频…"
case .failed(let message):
return "视频加载失败:\(message)"
}
}
}

View File

@@ -7,6 +7,7 @@ struct SentenceBoundary: Identifiable, Equatable, Decodable {
let text: String?
let language: String?
let referenceSpeechDurationMs: Int64?
let zhText: String?
var id: Int { index }
@@ -15,6 +16,48 @@ struct SentenceBoundary: Identifiable, Equatable, Decodable {
case startMs = "start_ms"
case endMs = "end_ms"
case referenceSpeechDurationMs = "reference_speech_duration_ms"
case zhText = "zh_text"
}
init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
index = try container.decode(Int.self, forKey: .index)
startMs = try container.decode(Int64.self, forKey: .startMs)
endMs = try container.decode(Int64.self, forKey: .endMs)
text = try container.decodeIfPresent(String.self, forKey: .text)
language = try container.decodeIfPresent(String.self, forKey: .language)
referenceSpeechDurationMs = try container.decodeIfPresent(Int64.self, forKey: .referenceSpeechDurationMs)
zhText = try container.decodeIfPresent(String.self, forKey: .zhText)
}
init(
index: Int,
startMs: Int64,
endMs: Int64,
text: String?,
language: String?,
referenceSpeechDurationMs: Int64?,
zhText: String? = nil
) {
self.index = index
self.startMs = startMs
self.endMs = endMs
self.text = text
self.language = language
self.referenceSpeechDurationMs = referenceSpeechDurationMs
self.zhText = zhText
}
func withZhText(_ zhText: String?) -> SentenceBoundary {
SentenceBoundary(
index: index,
startMs: startMs,
endMs: endMs,
text: text,
language: language,
referenceSpeechDurationMs: referenceSpeechDurationMs,
zhText: zhText
)
}
}
@@ -66,6 +109,19 @@ struct TrainingVideoSummary: Identifiable, Decodable {
}
return URL(string: absolute)
}
func copy(durationMs: Int64? = self.durationMs, sentenceCount: Int = self.sentenceCount) -> Self {
TrainingVideoSummary(
videoHash: videoHash,
title: title,
streamUrl: streamUrl,
durationMs: durationMs,
sizeBytes: sizeBytes,
language: language,
sentenceCount: sentenceCount,
status: status
)
}
}
struct TrainingMediaItem: Identifiable, Equatable {

View File

@@ -10,6 +10,7 @@ final class PlayerController: ObservableObject {
@Published private(set) var currentSentence: SentenceBoundary?
@Published var boundaryState: BoundaryState = .idle
@Published var transientMessage: String?
@Published private(set) var cacheState: VideoCacheState = .notRequired
let player = AVPlayer()
@@ -30,8 +31,9 @@ final class PlayerController: ObservableObject {
private var swipeBackAnchorTime: TimeInterval = 0
private var speedBeforeLongPress: Float?
private var transientTask: Task<Void, Never>?
private var cacheSwapTask: Task<Void, Never>?
private var cacheFallbackTask: Task<Void, Never>?
private var activePlaybackURL: URL?
private var didAttemptFullFileFallback = false
private var sentenceEndTimeObserver: Any?
private var sentenceEndGuardToken = 0
@@ -39,6 +41,8 @@ final class PlayerController: ObservableObject {
/// AVPlayer 在个别句子边界会把 seek 结果上报得比目标早 1ms 左右。
/// 如果恰好落在句边界前,tick 会把句子重新判定回上一句;稍微进入目标句可消除该问题。
private static let sentenceSeekInsetMs: Int64 = 20
private static let previousSentenceToleranceMs: Int64 = 250
private static let nextSentenceToleranceMs: Int64 = 150
private var timeObserver: Any?
private var endObserver: NSObjectProtocol?
@@ -88,33 +92,29 @@ final class PlayerController: ObservableObject {
@MainActor
func load(_ item: TrainingMediaItem) {
cancelSentenceBoundaryGuard()
cacheFallbackTask?.cancel()
cacheFallbackTask = nil
didAttemptFullFileFallback = false
lastSentenceIndex = nil
swipeBackAnchorIndex = nil
pendingSeekPositionMs = nil
stopAtSentenceEnd = false
stopAtSentenceEndIndex = nil
playbackIntent = false
mediaItem = item
currentItem = item
boundaryState = .idle
let playbackURL: URL
if item.isLocalFile {
playbackURL = item.url
} else if let cachedURL = StreamingVideoCache.shared.cachedFileIfAvailable(for: item.url) {
playbackURL = cachedURL
} else {
playbackURL = item.url
}
activePlaybackURL = playbackURL
let playerItem = AVPlayerItem(url: playbackURL)
attachItemObservers(to: playerItem)
player.replaceCurrentItem(with: playerItem)
if config.autoPlay {
player.play()
if item.isLocalFile {
cacheState = .notRequired
preparePlaybackItem(with: item.url)
} else if let cachedURL = StreamingVideoCache.shared.cachedFileIfAvailable(for: item.url) {
cacheState = .ready
preparePlaybackItem(with: cachedURL)
} else {
cacheState = .notRequired
preparePlaybackItem(with: item.url)
}
playbackIntent = config.autoPlay
scheduleCachePrefetchIfNeeded(for: item, activeURL: playbackURL)
tick()
}
/// 句边界请求完成后只更新训练数据,不重新创建 AVPlayerItem。
@@ -201,8 +201,13 @@ final class PlayerController: ObservableObject {
guard let item = mediaItem, !item.sentences.isEmpty else {
return false
}
guard let currentIndex = sentenceAnchorIndex() else { return false }
let target = item.sentences[max(0, currentIndex - 1)]
// 用播放位置查找目标,而不是用可能已经过期的 sentence index 推导。
// 播放点落在两句之间的空隙时也能正确定位前一句。
let position = referencePositionMs()
let target = item.sentences
.reversed()
.first { $0.startMs < position - Self.previousSentenceToleranceMs }
?? item.sentences[0]
seekToSentenceBeginning(target)
return true
}
@@ -212,12 +217,14 @@ final class PlayerController: ObservableObject {
guard let item = mediaItem, !item.sentences.isEmpty else {
return false
}
guard let currentIndex = sentenceAnchorIndex() else { return false }
let targetIndex = currentIndex + 1
guard item.sentences.indices.contains(targetIndex) else {
// Android 端同样按当前位置向后查找下一个 startMs。即使边界表存在句间
// 空隙,或 AVPlayer 的时间回调仍停留在上一句,也能跳到实际的下一句。
let position = referencePositionMs()
guard let target = item.sentences.first(where: {
$0.startMs > position + Self.nextSentenceToleranceMs
}) else {
return false
}
let target = item.sentences[targetIndex]
seekToSentenceBeginning(target)
return true
}
@@ -309,21 +316,27 @@ final class PlayerController: ObservableObject {
}
/// 只播放当前句子,播完自动暂停并回到句首(评测“播放原句”)。
/// 必须先打上句尾标记再 seek,这样 completion 里的 forwardPlaybackEndTime
/// 会卡在当前句 endMs,而不是等 250ms 观察器才停。
func playCurrentSentenceAndStop() {
guard let sentence = currentSentence else {
return
}
performSeek(to: sentenceSeekPosition(sentence))
stopAtSentenceEnd = true
stopAtSentenceEndIndex = sentence.index
play()
lastSentenceIndex = sentence.index
playbackIntent = true
performSeek(to: sentenceSeekPosition(sentence), resumePlayback: true)
}
// MARK: - 设置
func setContinuousPlayback(_ enabled: Bool) {
config.continuousPlayback = enabled
refreshSentenceBoundaryGuard()
if !enabled {
lastSentenceIndex = currentSentence?.index ?? lastSentenceIndex
}
refreshSentenceBoundaryGuard(force: true)
}
func setSentenceMode(_ enabled: Bool) {
@@ -465,41 +478,30 @@ final class PlayerController: ObservableObject {
pendingSeekPositionMs ?? currentPositionMs()
}
@MainActor
private func scheduleCachePrefetchIfNeeded(for item: TrainingMediaItem, activeURL: URL) {
cacheSwapTask?.cancel()
cacheSwapTask = nil
guard !item.isLocalFile,
activeURL != item.url else {
return
}
let remoteURL = item.url
cacheSwapTask = Task { [weak self] in
guard let cachedURL = await StreamingVideoCache.shared.prefetch(remoteURL) else {
return
}
self?.swapToCachedFileIfNeeded(remoteURL: remoteURL, cachedURL: cachedURL)
}
}
@MainActor
private func swapToCachedFileIfNeeded(remoteURL: URL, cachedURL: URL) {
guard !Task.isCancelled,
mediaItem?.url == remoteURL,
activePlaybackURL != cachedURL else {
return
}
let targetPositionMs = pendingSeekPositionMs ?? currentPositionMs()
let newItem = AVPlayerItem(url: cachedURL)
activePlaybackURL = cachedURL
// 递增 seek generation,让旧远程 Item 上未完成的 seek 回调失效。
seekGeneration += 1
private func preparePlaybackItem(with playbackURL: URL) {
let newItem = makePlayerItem(url: playbackURL)
activePlaybackURL = playbackURL
attachItemObservers(to: newItem)
player.replaceCurrentItem(with: newItem)
performSeek(to: targetPositionMs)
let shouldPlay = config.autoPlay || playbackIntent
if shouldPlay {
player.play()
}
playbackIntent = shouldPlay
tick()
}
private func makePlayerItem(url: URL) -> AVPlayerItem {
guard !url.isFileURL else {
return AVPlayerItem(url: url)
}
var options: [String: Any] = [:]
let ext = url.pathExtension.lowercased()
if ext.isEmpty || ["mp4", "m4v", "mov"].contains(ext) {
options["AVURLAssetOverrideMIMETypeKey"] = "video/mp4"
}
let asset = AVURLAsset(url: url, options: options.isEmpty ? nil : options)
return AVPlayerItem(asset: asset, automaticallyLoadedAssetKeys: ["playable", "tracks", "duration"])
}
/// AVPlayer 的 seek 是异步的;所有 seek 必须统一走这里,避免播放循环、
@@ -511,6 +513,11 @@ final class PlayerController: ObservableObject {
) {
let targetPositionMs = max(0, positionMs)
let shouldResumePlayback = resumePlayback ?? playbackIntent
// 必须在发起 seek 前清掉上一句的硬边界。否则从第 1 句右划到第 2 句时,
// AVPlayerItem 上的 forwardPlaybackEndTime 仍是第 1 句 endMs;seek 可能
// 被这个旧边界截断,UI 先显示第 2 句,然后立刻又回到第 1 句。
cancelSentenceBoundaryGuard()
clearForwardPlaybackEnd()
seekGeneration += 1
let generation = seekGeneration
pendingSeekPositionMs = targetPositionMs
@@ -529,9 +536,11 @@ final class PlayerController: ObservableObject {
self.player.play()
}
completion?()
// seek 已真正落在目标句后,再按目标句注册新的句尾硬边界。
self.refreshSentenceBoundaryGuard(force: shouldResumePlayback)
self.tick()
}
}
refreshSentenceBoundaryGuard()
tick()
}
@@ -760,6 +769,13 @@ final class PlayerController: ObservableObject {
tick()
return
}
// 评测“播放原句”即使处于连续播放,也要用当前句硬边界,不能滑进下一句。
if stopAtSentenceEnd,
let index = stopAtSentenceEndIndex,
let sentence = mediaItem?.sentences.first(where: { $0.index == index }) {
applySentenceEndAction(sentence)
return
}
// 若最后一个句子的 endMs 就是视频末尾,forwardPlaybackEndTime 和自然结束
// 是同一时间点。此时必须执行“当前句”的句尾动作,而不能整条视频回零。
if config.sentenceMode,
@@ -808,12 +824,57 @@ final class PlayerController: ObservableObject {
if item.status == .failed {
let message = item.error?.localizedDescription ?? "未知错误"
self.showTransientMessage("播放失败:\(message)")
} else if item.status == .readyToPlay {
self.considerFullFileFallback(for: item)
}
self.tick()
}
}
}
private func considerFullFileFallback(for item: AVPlayerItem) {
guard !didAttemptFullFileFallback,
let media = mediaItem,
!media.isLocalFile,
!(activePlaybackURL?.isFileURL ?? false) else {
return
}
let hasVisibleVideo = item.presentationSize.height >= 2 && item.presentationSize.width >= 2
if hasVisibleVideo {
return
}
didAttemptFullFileFallback = true
cacheFallbackTask?.cancel()
cacheFallbackTask = Task { @MainActor [weak self] in
try? await Task.sleep(nanoseconds: 800_000_000)
guard let self, !Task.isCancelled else {
return
}
guard self.mediaItem?.id == media.id,
self.player.currentItem === item,
item.presentationSize.height < 2 else {
return
}
do {
let cachedURL = try await StreamingVideoCache.shared.prefetch(media.url)
guard self.mediaItem?.id == media.id else {
return
}
let shouldPlay = self.playbackIntent
let positionMs = self.pendingSeekPositionMs ?? self.currentPositionMs()
self.preparePlaybackItem(with: cachedURL)
self.cacheState = .ready
if positionMs > 0 {
self.performSeek(to: positionMs, resumePlayback: shouldPlay)
}
} catch is CancellationError {
return
} catch {
self.showTransientMessage("视频画面加载失败:\(error.localizedDescription)")
}
}
}
private func configureAudioSession() {
let session = AVAudioSession.sharedInstance()
try? session.setCategory(.playback, mode: .moviePlayback)

View File

@@ -13,15 +13,20 @@ struct PlayerSurfaceView: UIViewRepresentable {
func makeUIView(context: Context) -> VideoSurfaceView {
let view = VideoSurfaceView()
view.playerLayer.player = controller.player
view.controller = controller
view.onVerticalSwipe = onVerticalSwipe
view.attach(player: controller.player)
view.setContentHuggingPriority(.defaultLow, for: .vertical)
view.setContentHuggingPriority(.defaultLow, for: .horizontal)
view.setContentCompressionResistancePriority(.defaultLow, for: .vertical)
view.setContentCompressionResistancePriority(.defaultLow, for: .horizontal)
return view
}
func updateUIView(_ uiView: VideoSurfaceView, context: Context) {
uiView.playerLayer.player = controller.player
uiView.controller = controller
uiView.onVerticalSwipe = onVerticalSwipe
uiView.attach(player: controller.player)
}
}
@@ -29,13 +34,9 @@ final class VideoSurfaceView: UIView {
weak var controller: PlayerController?
var onVerticalSwipe: ((Bool) -> Void)?
override class var layerClass: AnyClass {
AVPlayerLayer.self
}
private let videoLayer = AVPlayerLayer()
var playerLayer: AVPlayerLayer {
layer as! AVPlayerLayer
}
var playerLayer: AVPlayerLayer { videoLayer }
private let minSwipeDistance: CGFloat = 48
private let minSwipeVelocity: CGFloat = 160
@@ -55,7 +56,11 @@ final class VideoSurfaceView: UIView {
override init(frame: CGRect) {
super.init(frame: frame)
backgroundColor = .black
playerLayer.videoGravity = .resizeAspect
isOpaque = true
clipsToBounds = true
videoLayer.videoGravity = .resizeAspect
videoLayer.backgroundColor = UIColor.black.cgColor
layer.addSublayer(videoLayer)
tapRecognizer = UITapGestureRecognizer(target: self, action: #selector(handleTap))
panRecognizer = UIPanGestureRecognizer(target: self, action: #selector(handlePan))
@@ -82,6 +87,25 @@ final class VideoSurfaceView: UIView {
fatalError("init(coder:) has not been implemented")
}
override func layoutSubviews() {
super.layoutSubviews()
CATransaction.begin()
CATransaction.setDisableActions(true)
videoLayer.frame = bounds
CATransaction.commit()
}
func attach(player: AVPlayer) {
if videoLayer.player !== player {
videoLayer.player = player
}
if videoLayer.superlayer !== layer {
layer.addSublayer(videoLayer)
}
videoLayer.frame = bounds
videoLayer.isHidden = false
}
@objc private func handleTap() {
controller?.handleGesture(.singleTap)
}
@@ -96,7 +120,6 @@ final class VideoSurfaceView: UIView {
case .began:
resetSwipeTracking()
case .changed:
// 在滑动达到阈值时立即派发,并且同一个手势只允许派发一次。
evaluateSwipe(dx: dx, dy: dy, velocity: velocity)
case .ended, .cancelled, .failed:
if !didDispatchSwipe {
@@ -112,7 +135,6 @@ final class VideoSurfaceView: UIView {
guard !didDispatchSwipe else { return }
if swipeAxis == nil {
// 只有方向足够明确时锁定轴向,避免轻微斜滑被误判为上下滑。
if abs(dx) > abs(dy) * 1.2 {
swipeAxis = .horizontal
} else if abs(dy) > abs(dx) * 1.2 {

View File

@@ -33,6 +33,8 @@
<string>需要麦克风权限才能进行录音评测</string>
<key>UILaunchScreen</key>
<dict/>
<key>UIRequiresFullScreen</key>
<true/>
<key>UISupportedInterfaceOrientations</key>
<array>
<string>UIInterfaceOrientationPortrait</string>

View File

@@ -88,16 +88,20 @@ enum ImitationAssessor {
appendFile(&body, boundary: boundary, name: "audio", filename: filename, mimeType: mimeType, data: recordingData)
body.append(Data("--\(boundary)--\r\n".utf8))
var headers = [
"Accept": "application/json",
"Content-Type": "multipart/form-data; boundary=\(boundary)",
"X-Client-Key": AppConfig.assessmentAPIKey,
]
if !AppConfig.authToken.isEmpty {
headers["Authorization"] = "Bearer \(AppConfig.authToken)"
}
let result = try await HTTPClient.execute(
baseURL: AppConfig.serverBaseURL,
path: path,
method: "POST",
timeout: AppConfig.assessmentTimeout,
headers: [
"Accept": "application/json",
"Content-Type": "multipart/form-data; boundary=\(boundary)",
"X-Client-Key": AppConfig.assessmentAPIKey,
],
headers: headers,
body: body
)
guard (200...299).contains(result.statusCode) else {

View File

@@ -46,6 +46,40 @@ struct SentenceExplanationResponse: Decodable {
}
}
struct SubtitleTranslationItem: Decodable {
let index: Int
let zhText: String
private enum CodingKeys: String, CodingKey {
case index
case zhText = "zh_text"
}
}
struct SubtitleTranslationResponse: Decodable {
let videoHash: String
let translations: [SubtitleTranslationItem]
let complete: Bool?
let status: String?
let error: String?
private enum CodingKeys: String, CodingKey {
case translations
case videoHash = "video_hash"
case complete
case status
case error
}
var mapped: [Int: String] {
var values: [Int: String] = [:]
for item in translations where !item.zhText.isEmpty {
values[item.index] = item.zhText
}
return values
}
}
extension SentenceBoundaryAPI {
static func explain(
_ sentence: SentenceBoundary,
@@ -79,4 +113,110 @@ extension SentenceBoundaryAPI {
.decode(SentenceExplanationResponse.self, from: result.body)
.explanation
}
static func fetchStoredSubtitles(videoHash: String) async throws -> SubtitleTranslationResponse {
try await requestSubtitles(videoHash: videoHash, method: "GET", timeout: AppConfig.readTimeout)
}
static func translateSubtitles(videoHash: String) async throws -> SubtitleTranslationResponse {
try await requestSubtitles(videoHash: videoHash, method: "POST", timeout: AppConfig.translationTimeout)
}
static func loadSubtitles(
videoHash: String,
expectedIndexes: [Int],
onProgress: (([Int: String]) -> Void)? = nil
) async throws -> [Int: String] {
let expected = Set(expectedIndexes)
let deadline = Date().addingTimeInterval(AppConfig.translationDeadline)
var kickJob = true
var lastError: Error?
var lastCount = -1
var stalledRounds = 0
while Date() < deadline {
try Task.checkCancellation()
if let stored = try? await fetchStoredSubtitles(videoHash: videoHash) {
let mapped = stored.mapped
onProgress?(mapped)
if isComplete(mapped, expected: expected, payload: stored) {
return mapped
}
if mapped.count > lastCount {
lastCount = mapped.count
stalledRounds = 0
} else {
stalledRounds += 1
}
if stored.status != "running" && stalledRounds >= 2 {
kickJob = true
}
} else {
kickJob = true
}
if kickJob {
do {
let posted = try await translateSubtitles(videoHash: videoHash)
let mapped = posted.mapped
onProgress?(mapped)
if isComplete(mapped, expected: expected, payload: posted) {
return mapped
}
if posted.status == "error" {
lastError = HTTPClientError.badStatus(503, posted.error ?? "Subtitle translation failed")
}
} catch {
lastError = error
if !isTimeout(error) {
stalledRounds += 1
}
}
kickJob = false
}
try await Task.sleep(nanoseconds: AppConfig.translationPollIntervalNs)
}
throw lastError ?? HTTPClientError.badStatus(408, "Subtitle translation timed out")
}
private static func isComplete(
_ mapped: [Int: String],
expected: Set<Int>,
payload: SubtitleTranslationResponse
) -> Bool {
if !expected.isEmpty {
return expected.isSubset(of: Set(mapped.keys))
}
return payload.complete == true
}
private static func isTimeout(_ error: Error) -> Bool {
let nsError = error as NSError
return nsError.domain == NSURLErrorDomain && nsError.code == NSURLErrorTimedOut
}
private static func requestSubtitles(
videoHash: String,
method: String,
timeout: TimeInterval
) async throws -> SubtitleTranslationResponse {
let hash = videoHash.lowercased()
guard hash.range(of: AppConfig.sha256Pattern, options: .regularExpression) != nil else {
throw SentenceBoundaryAPIError.invalidHash
}
var headers: [String: String] = ["Accept": "application/json"]
if !AppConfig.authToken.isEmpty {
headers["Authorization"] = "Bearer \(AppConfig.authToken)"
}
let result = try await HTTPClient.execute(
baseURL: AppConfig.serverBaseURL,
path: "api/v1/videos/\(hash)/subtitle-translation",
method: method,
timeout: timeout,
headers: headers
)
guard (200...299).contains(result.statusCode) else {
let detail = String(data: result.body, encoding: .utf8) ?? ""
throw HTTPClientError.badStatus(result.statusCode, detail)
}
return try JSONDecoder().decode(SubtitleTranslationResponse.self, from: result.body)
}
}

View File

@@ -0,0 +1,67 @@
import Foundation
final class SentenceExplanationStore {
static let shared = SentenceExplanationStore()
private let root: URL
init() {
let documents = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0]
root = documents.appendingPathComponent("SentenceExplanations", isDirectory: true)
try? FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)
}
func load(mediaId: String, sentenceIndex: Int) -> String? {
let url = fileURL(mediaId: mediaId, sentenceIndex: sentenceIndex)
guard FileManager.default.fileExists(atPath: url.path) else { return nil }
let text = try? String(contentsOf: url, encoding: .utf8)
let trimmed = text?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
return trimmed.isEmpty ? nil : trimmed
}
@discardableResult
func save(mediaId: String, sentenceIndex: Int, explanation: String) -> Bool {
let trimmed = explanation.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return false }
let url = fileURL(mediaId: mediaId, sentenceIndex: sentenceIndex)
try? FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true)
do {
try trimmed.write(to: url, atomically: true, encoding: .utf8)
return true
} catch {
return false
}
}
func sync(mediaId: String, sentences: [SentenceBoundary]) {
guard !sentences.isEmpty else { return }
let directory = root.appendingPathComponent(Self.sanitize(mediaId), isDirectory: true)
let revision = Self.revision(for: sentences)
let marker = directory.appendingPathComponent("revision.txt")
let previous = (try? String(contentsOf: marker, encoding: .utf8))?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
guard previous != revision else { return }
try? FileManager.default.removeItem(at: directory)
try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
try? revision.write(to: marker, atomically: true, encoding: .utf8)
}
private func fileURL(mediaId: String, sentenceIndex: Int) -> URL {
let directory = root.appendingPathComponent(Self.sanitize(mediaId), isDirectory: true)
return directory.appendingPathComponent("\(sentenceIndex).txt")
}
static func revision(for sentences: [SentenceBoundary]) -> String {
sentences.map { sentence in
let text = (sentence.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
return "\(sentence.index)\t\(text)"
}.joined(separator: "\n")
}
static func sanitize(_ mediaId: String) -> String {
let allowed = CharacterSet.alphanumerics.union(CharacterSet(charactersIn: "._-"))
let filtered = mediaId.lowercased().unicodeScalars.map { allowed.contains($0) ? Character($0) : "_" }
let value = String(filtered).trimmingCharacters(in: CharacterSet(charactersIn: "_"))
return value.isEmpty || value == "." || value == ".." ? "unknown" : value
}
}

View File

@@ -1,12 +1,10 @@
import CryptoKit
import Foundation
/// iOS 端的持久化整文件视频缓存。
/// iOS 端的可选整文件视频缓存。
///
/// Android 端使用 ExoPlayer 的 CacheDataSource 边播边缓存;AVFoundation 没有公开
/// 等价的持久字节缓存 API。这里在后台把远程 MP4 下载成完整本地文件,下次装载时
/// 直接交给 AVPlayer,并在本次播放完成下载后热切换到本地文件,避免回跳句子时
/// 再次依赖网络。
/// 播放默认走远程 URL 按需加载(AVPlayer HTTP Range)。若本地已有完整缓存文件,
/// 播放器会优先使用该文件,避免重复拉流。
@MainActor
final class StreamingVideoCache {
static let shared = StreamingVideoCache()
@@ -18,7 +16,7 @@ final class StreamingVideoCache {
private let directory: URL
private let fileManager = FileManager.default
private var downloadsByURL: [URL: Task<URL?, Never>] = [:]
private var downloadsByURL: [URL: Task<URL, Error>] = [:]
private init() {
let caches = fileManager.urls(for: .cachesDirectory, in: .userDomainMask)[0]
@@ -44,31 +42,40 @@ final class StreamingVideoCache {
return file
}
/// 重复调用同一 URL 时复用同一个后台下载;完成后返回本地文件。
func prefetch(_ remoteURL: URL) async -> URL? {
/// 重复调用同一 URL 时复用同一个后台下载;只有下载完成后才返回本地文件。
func prefetch(_ remoteURL: URL) async throws -> URL {
if let cached = cachedFileIfAvailable(for: remoteURL) {
return cached
}
guard isCacheable(remoteURL), let destination = destinationFile(for: remoteURL) else {
return nil
guard let destination = destinationFile(for: remoteURL) else {
throw StreamingVideoCacheError.unsupportedURL
}
guard isCacheable(remoteURL) else {
throw StreamingVideoCacheError.unsupportedURL
}
if let existing = downloadsByURL[remoteURL] {
return await existing.value
return try await existing.value
}
let task = Task {
await Self.download(from: remoteURL, to: destination)
try await Self.download(from: remoteURL, to: destination)
}
downloadsByURL[remoteURL] = task
let result = await task.value
let result: URL
do {
result = try await task.value
} catch {
if !task.isCancelled {
downloadsByURL[remoteURL] = nil
}
throw error
}
if downloadsByURL[remoteURL]?.isCancelled != true {
downloadsByURL[remoteURL] = nil
}
if result != nil {
evictIfNeeded(keeping: remoteURL)
}
evictIfNeeded(keeping: remoteURL)
return result
}
@@ -137,19 +144,17 @@ final class StreamingVideoCache {
}
}
private static func download(from remoteURL: URL, to destination: URL) async -> URL? {
private static func download(from remoteURL: URL, to destination: URL) async throws -> URL {
var request = URLRequest(url: remoteURL)
request.timeoutInterval = AppConfig.connectTimeout
request.setValue(AppConfig.userAgent, forHTTPHeaderField: "User-Agent")
request.cachePolicy = .reloadIgnoringLocalCacheData
let (tempFile, response) = try await URLSession.shared.download(for: request)
do {
let (tempFile, response) = try await URLSession.shared.download(for: request)
guard let http = response as? HTTPURLResponse, (200...299).contains(http.statusCode) else {
let status = (response as? HTTPURLResponse)?.statusCode ?? -1
FileManager.default.removeItemIfPossible(at: tempFile)
logFailure(remoteURL, message: "HTTP \(status)")
return nil
throw StreamingVideoCacheError.downloadFailed(message: "HTTP \(status)")
}
let fileManager = FileManager.default
@@ -160,11 +165,16 @@ final class StreamingVideoCache {
)
try fileManager.moveItem(at: tempFile, to: destination)
return destination
} catch is CancellationError {
return nil
} catch {
FileManager.default.removeItemIfPossible(at: tempFile)
if let cacheError = error as? StreamingVideoCacheError {
logFailure(remoteURL, message: cacheError.localizedDescription)
throw cacheError
}
logFailure(remoteURL, message: error.localizedDescription)
return nil
throw StreamingVideoCacheError.downloadFailed(
message: error.localizedDescription
)
}
}
@@ -175,6 +185,20 @@ final class StreamingVideoCache {
}
}
enum StreamingVideoCacheError: LocalizedError {
case unsupportedURL
case downloadFailed(message: String)
var errorDescription: String? {
switch self {
case .unsupportedURL:
return "仅支持 MP4/M4V/MOV 整文件缓存"
case .downloadFailed(let message):
return message
}
}
}
private extension FileManager {
@discardableResult
func removeItemIfPossible(at url: URL) -> Bool {

View File

@@ -10,6 +10,7 @@ final class CatalogViewModel: ObservableObject {
private var baseURL = AppConfig.serverBaseURL
private var hasLoaded = false
private var boundaryDocuments: [String: SentenceBoundaryDocument] = [:]
func loadIfNeeded() async {
guard !hasLoaded else {
@@ -31,6 +32,42 @@ final class CatalogViewModel: ObservableObject {
}
}
func preloadBoundaries(for videos: [TrainingVideoSummary]) async -> (
videos: [TrainingVideoSummary],
succeeded: Int,
failed: Int
) {
var updatedVideos = videos
var succeeded = 0
var failed = 0
for (index, video) in videos.enumerated() {
do {
let document = try await SentenceBoundaryAPI.fetch(videoHash: video.videoHash)
boundaryDocuments[document.videoHash] = document
SentenceExplanationStore.shared.sync(
mediaId: document.videoHash,
sentences: document.sentences
)
updatedVideos[index] = video.copy(
durationMs: document.durationMs,
sentenceCount: document.sentences.count
)
if activeVideoHash == video.videoHash {
controller?.updateBoundaries(document.sentences)
controller?.boundaryState = .loaded(count: document.sentences.count)
controller?.showTransientMessage("课程已就绪:\(document.sentences.count) 句")
}
succeeded += 1
} catch {
failed += 1
}
}
videos = updatedVideos
return (updatedVideos, succeeded, failed)
}
/// 与安卓端 loadRemoteVideo() 一致:先以无句边界装载,再拉取句边界后重新装载。
func select(_ video: TrainingVideoSummary) {
guard let controller = controller else {
@@ -41,6 +78,23 @@ final class CatalogViewModel: ObservableObject {
controller.boundaryState = .failed(message: "无法解析视频流地址")
return
}
if let document = boundaryDocuments[video.videoHash] {
SentenceExplanationStore.shared.sync(
mediaId: document.videoHash,
sentences: document.sentences
)
let item = TrainingMediaItem(
id: video.videoHash,
title: video.title,
url: url,
sentences: document.sentences
)
controller.load(item)
controller.boundaryState = .loaded(count: document.sentences.count)
controller.showTransientMessage("课程已就绪:\(document.sentences.count) 句")
return
}
let loadingItem = TrainingMediaItem(id: video.videoHash, title: video.title, url: url)
controller.load(loadingItem)
controller.boundaryState = .loading(isLocal: false)
@@ -51,6 +105,11 @@ final class CatalogViewModel: ObservableObject {
guard activeVideoHash == video.videoHash else {
return
}
boundaryDocuments[document.videoHash] = document
SentenceExplanationStore.shared.sync(
mediaId: document.videoHash,
sentences: document.sentences
)
controller.updateBoundaries(document.sentences)
controller.boundaryState = .loaded(count: document.sentences.count)
controller.showTransientMessage("课程已就绪:\(document.sentences.count) 句")
@@ -95,6 +154,10 @@ final class CatalogViewModel: ObservableObject {
guard activeVideoHash == hash else {
return
}
SentenceExplanationStore.shared.sync(
mediaId: document.videoHash,
sentences: document.sentences
)
controller.updateBoundaries(document.sentences)
controller.boundaryState = .loaded(count: document.sentences.count)
controller.showTransientMessage("句子边界已加载:\(document.sentences.count) 句")

View File

@@ -3,12 +3,39 @@ import SwiftUI
/// 与安卓端横屏布局一致:全屏播放 + 半透明句面板 + 退出全屏按钮。
struct FullscreenPlayerView: View {
@ObservedObject var controller: PlayerController
var showsLessonTitle: Bool = true
var showChineseSubtitles: Bool = false
var chineseSubtitlePlaceholder: String = "正在翻译中文字幕…"
let onExit: () -> Void
@State private var subtitleVisible = true
@State private var explanationSentence: SentenceBoundary?
var body: some View {
GeometryReader { proxy in
let fakeLandscape = proxy.size.height > proxy.size.width
playerChrome
.frame(
width: fakeLandscape ? proxy.size.height : proxy.size.width,
height: fakeLandscape ? proxy.size.width : proxy.size.height
)
.rotationEffect(.degrees(fakeLandscape ? -90 : 0))
.frame(width: proxy.size.width, height: proxy.size.height)
}
.background(Color.black)
.ignoresSafeArea()
.statusBarHidden(true)
.sheet(item: $explanationSentence) { sentence in
SentenceExplanationSheet(
mediaId: controller.currentItem?.id ?? "",
courseTitle: controller.currentItem?.title ?? "口语训练课程",
sentence: sentence,
sentences: controller.currentItem?.sentences ?? []
)
}
}
private var playerChrome: some View {
ZStack {
Color.black.ignoresSafeArea()
PlayerSurfaceView(controller: controller) { isSwipeUp in
@@ -32,18 +59,28 @@ struct FullscreenPlayerView: View {
Spacer()
if subtitleVisible {
VStack(alignment: .leading, spacing: 6) {
Text(controller.currentItem?.title ?? "未选择课程")
.font(.system(size: 17, weight: .bold))
.foregroundColor(.white)
.lineLimit(1)
if showsLessonTitle {
Text(controller.currentItem?.title ?? "未选择课程")
.font(.system(size: 17, weight: .bold))
.foregroundColor(.white)
.lineLimit(1)
}
Text(sentenceMetaText)
.font(.system(size: 13))
.foregroundColor(Theme.textOverlay)
HStack(alignment: .center, spacing: 8) {
Text(sentenceTextText)
.font(.system(size: 15))
.foregroundColor(.white)
.lineLimit(2)
VStack(alignment: .leading, spacing: 4) {
Text(sentenceTextText)
.font(.system(size: 15))
.foregroundColor(.white)
.lineLimit(2)
if let chineseSubtitleText = chineseSubtitleText {
Text(chineseSubtitleText)
.font(.system(size: 14))
.foregroundColor(Theme.textOverlay)
.lineLimit(2)
}
}
Spacer()
Button {
guard let sentence = controller.currentSentence,
@@ -95,12 +132,6 @@ struct FullscreenPlayerView: View {
}
.padding(16)
}
.sheet(item: $explanationSentence) { sentence in
SentenceExplanationSheet(
courseTitle: controller.currentItem?.title ?? "口语训练课程",
sentence: sentence
)
}
}
@State private var progressDragging = false
@@ -183,7 +214,24 @@ struct FullscreenPlayerView: View {
}
}
private var chineseSubtitleText: String? {
guard showChineseSubtitles else {
return nil
}
if let zh = controller.currentSentence?.zhText?.trimmingCharacters(in: .whitespacesAndNewlines),
!zh.isEmpty {
return zh
}
if (controller.currentSentence?.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
return nil
}
return chineseSubtitlePlaceholder
}
private var statusText: String {
if let message = controller.cacheState.statusMessage {
return message
}
if let message = controller.transientMessage {
return message
}

View File

@@ -6,6 +6,9 @@ struct PlayerSectionView: View {
var showsContinuousToggle: Bool
@Binding var continuousPlayback: Bool
var showsSubtitle: Binding<Bool>? = .constant(true)
var showsLessonTitle: Bool = true
var showChineseSubtitles: Binding<Bool>? = nil
var chineseSubtitlePlaceholder: String = "正在翻译中文字幕…"
var onFullscreen: () -> Void
@State private var progressDragging = false
@@ -22,8 +25,10 @@ struct PlayerSectionView: View {
.padding(.horizontal, 16)
.sheet(item: $explanationSentence) { sentence in
SentenceExplanationSheet(
mediaId: controller.currentItem?.id ?? "",
courseTitle: controller.currentItem?.title ?? "口语训练课程",
sentence: sentence
sentence: sentence,
sentences: controller.currentItem?.sentences ?? []
)
}
}
@@ -73,10 +78,12 @@ struct PlayerSectionView: View {
private var sentencePanel: some View {
VStack(alignment: .leading, spacing: 6) {
Text(controller.currentItem?.title ?? "未选择课程")
.font(.system(size: 17, weight: .bold))
.foregroundColor(.white)
.lineLimit(1)
if showsLessonTitle {
Text(controller.currentItem?.title ?? "未选择课程")
.font(.system(size: 17, weight: .bold))
.foregroundColor(.white)
.lineLimit(1)
}
if showsContinuousToggle {
HStack(spacing: 0) {
Text("连续播放")
@@ -89,14 +96,34 @@ struct PlayerSectionView: View {
}
.padding(.vertical, 2)
}
if let showChineseSubtitles = showChineseSubtitles {
HStack(spacing: 0) {
Text("中文字幕")
.font(.system(size: 13, weight: .bold))
.foregroundColor(Theme.textSubtle)
Spacer()
Toggle("", isOn: showChineseSubtitles)
.labelsHidden()
.tint(Theme.accent)
}
.padding(.vertical, 2)
}
Text(sentenceMetaText)
.font(.system(size: 13))
.foregroundColor(Theme.textSubtle)
HStack(alignment: .center, spacing: 8) {
Text(sentenceTextText)
.font(.system(size: 15))
.foregroundColor(.white)
.lineLimit(2)
VStack(alignment: .leading, spacing: 4) {
Text(sentenceTextText)
.font(.system(size: 15))
.foregroundColor(.white)
.lineLimit(4)
if let chineseSubtitleText = chineseSubtitleText {
Text(chineseSubtitleText)
.font(.system(size: 14))
.foregroundColor(Theme.textMuted)
.lineLimit(4)
}
}
Spacer()
Button {
guard let sentence = controller.currentSentence, !(sentence.text ?? "").isEmpty else {
@@ -201,7 +228,24 @@ struct PlayerSectionView: View {
}
}
private var chineseSubtitleText: String? {
guard showChineseSubtitles?.wrappedValue == true else {
return nil
}
if let zh = controller.currentSentence?.zhText?.trimmingCharacters(in: .whitespacesAndNewlines),
!zh.isEmpty {
return zh
}
if (controller.currentSentence?.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
return nil
}
return chineseSubtitlePlaceholder
}
private var statusText: String {
if let message = controller.cacheState.statusMessage {
return message
}
if let message = controller.transientMessage {
return message
}
@@ -227,11 +271,15 @@ struct PlayerSectionView: View {
}
struct SentenceExplanationSheet: View {
let mediaId: String
let courseTitle: String
let sentence: SentenceBoundary
var sentences: [SentenceBoundary] = []
@Environment(\.dismiss) private var dismiss
@State private var explanation = ""
@State private var errorMessage: String?
@State private var saved = false
var body: some View {
NavigationView {
@@ -249,10 +297,7 @@ struct SentenceExplanationSheet: View {
.font(.system(size: 15))
.foregroundColor(Theme.textSubtle)
} else {
Text(explanation)
.font(.system(size: 16))
.foregroundColor(Theme.textDark)
.lineSpacing(5)
MarkdownExplanationText(text: explanation)
}
}
.frame(maxWidth: .infinity, alignment: .leading)
@@ -261,8 +306,38 @@ struct SentenceExplanationSheet: View {
.background(Color.white.ignoresSafeArea())
.navigationTitle("句子讲解")
.navigationBarTitleDisplayMode(.inline)
.toolbar {
ToolbarItem(placement: .navigationBarTrailing) {
HStack(spacing: 6) {
Button(saved ? "已保存" : "保存") {
saveExplanation()
}
.disabled(saved || explanation.isEmpty || errorMessage != nil)
Text(saved ? "(已保存本地)" : "(下次直接看本地)")
.font(.system(size: 12))
.foregroundColor(Theme.textMuted)
Button {
dismiss()
} label: {
Image(systemName: "xmark.circle.fill")
.font(.system(size: 22, weight: .semibold))
.foregroundColor(Theme.textMuted)
}
.accessibilityLabel("关闭句子讲解")
}
}
}
}
.task {
SentenceExplanationStore.shared.sync(mediaId: mediaId, sentences: sentences)
if let cached = SentenceExplanationStore.shared.load(
mediaId: mediaId,
sentenceIndex: sentence.index
) {
explanation = cached
saved = true
return
}
guard explanation.isEmpty, errorMessage == nil else { return }
do {
explanation = try await SentenceBoundaryAPI.explain(
@@ -274,4 +349,64 @@ struct SentenceExplanationSheet: View {
}
}
}
private func saveExplanation() {
guard !saved, !explanation.isEmpty, errorMessage == nil else { return }
saved = SentenceExplanationStore.shared.save(
mediaId: mediaId,
sentenceIndex: sentence.index,
explanation: explanation
)
}
}
private struct MarkdownExplanationText: View {
let text: String
var body: some View {
Text(Self.attributed(from: text))
.font(.system(size: 16))
.foregroundColor(Theme.textDark)
.tint(Theme.buttonBlue)
.lineSpacing(5)
.frame(maxWidth: .infinity, alignment: .leading)
}
private static func attributed(from source: String) -> AttributedString {
let options = AttributedString.MarkdownParsingOptions(
interpretedSyntax: .full,
failurePolicy: .returnPartiallyParsedIfPossible
)
if let parsed = try? AttributedString(
markdown: preservingLineBreaks(source),
options: options
) {
return parsed
}
return AttributedString(source)
}
private static func preservingLineBreaks(_ source: String) -> String {
let normalized = source
.replacingOccurrences(of: "\r\n", with: "\n")
.replacingOccurrences(of: "\r", with: "\n")
var result = ""
var inFence = false
let lines = normalized.split(separator: "\n", omittingEmptySubsequences: false)
for (index, rawLine) in lines.enumerated() {
let line = String(rawLine)
let trimmed = line.trimmingCharacters(in: .whitespaces)
if trimmed.hasPrefix("```") {
inFence.toggle()
}
result += line
guard index < lines.count - 1 else { continue }
if inFence || trimmed.isEmpty {
result += "\n"
} else {
result += "\n\n"
}
}
return result
}
}

View File

@@ -9,11 +9,16 @@ struct RootView: View {
@StateObject private var historyManager = TrainingHistoryManager()
@State private var activeModule: Module = .course
@State private var continuousPlayback = false
@State private var showChineseSubtitles = false
@State private var chineseTranslationLoading = false
@State private var chineseTranslationError = ""
@State private var chineseTranslationTask: Task<Void, Never>?
@State private var showImporter = false
@State private var showFullscreenPlayer = false
@State private var courses: [Course] = []
@State private var currentCourse: Course?
@State private var courseStatusText = "正在同步"
@State private var isRefreshingCourses = false
@State private var showCourseGuardToast = false
@State private var managedResults: [UserResultSummary] = []
@State private var managedDubShares: [UserDubShareSummary] = []
@@ -52,6 +57,8 @@ struct RootView: View {
controller: controller,
catalogVM: catalogVM,
continuousPlayback: $continuousPlayback,
showChineseSubtitles: $showChineseSubtitles,
chineseSubtitlePlaceholder: chineseSubtitlePlaceholder,
onFullscreen: enterFullscreen
)
case .test:
@@ -87,9 +94,27 @@ struct RootView: View {
Task { await loadCourses() }
}
}
.onChange(of: showChineseSubtitles) { enabled in
if enabled {
ensureChineseTranslations()
} else {
chineseTranslationTask?.cancel()
chineseTranslationLoading = false
}
}
.onChange(of: controller.currentItem?.id) { _ in
ensureChineseTranslations()
}
.onChange(of: controller.currentItem?.sentences.count) { _ in
ensureChineseTranslations()
}
.onChange(of: activeModule) { module in
if module == .train {
controller.setContinuousPlayback(continuousPlayback)
ensureChineseTranslations()
} else if module == .test {
// 与安卓测试页一致:按当前句窗口播放,避免滑进下一句。
controller.setContinuousPlayback(false)
} else {
controller.setContinuousPlayback(true)
}
@@ -109,10 +134,17 @@ struct RootView: View {
syncCurrentCourse(with: videoHash)
}
.fullScreenCover(isPresented: $showFullscreenPlayer) {
FullscreenPlayerView(controller: controller) {
FullscreenPlayerView(
controller: controller,
showsLessonTitle: activeModule != .train,
showChineseSubtitles: showChineseSubtitles,
chineseSubtitlePlaceholder: chineseSubtitlePlaceholder
) {
exitFullscreen()
}
.onAppear(perform: enterFullscreen)
.onAppear {
AppDelegate.lockOrientation(.landscape)
}
}
}
@@ -203,11 +235,29 @@ struct RootView: View {
.font(.system(size: 26, weight: .bold))
.foregroundColor(.white)
.padding(.horizontal, 16)
Text("选择一门课程开始训练")
.font(.system(size: 13))
.foregroundColor(Theme.textMuted)
.padding(.horizontal, 16)
.padding(.top, 4)
HStack(spacing: 10) {
Text("选择一门课程开始训练")
.font(.system(size: 13))
.foregroundColor(Theme.textMuted)
Spacer()
Button {
Task { await loadCourses() }
} label: {
Text(isRefreshingCourses ? "刷新中" : "刷新课程")
.font(.system(size: 13, weight: .bold))
.foregroundColor(.white)
.padding(.horizontal, 12)
.frame(height: 32)
.background(Theme.buttonBlue)
.clipShape(RoundedRectangle(cornerRadius: 8))
}
.buttonStyle(.plain)
.disabled(isRefreshingCourses)
.opacity(isRefreshingCourses ? 0.6 : 1)
.padding(.trailing, 16)
}
.padding(.horizontal, 16)
.padding(.top, 4)
Text(courseStatusText)
.font(.system(size: 13))
.foregroundColor(Theme.textMuted)
@@ -504,10 +554,50 @@ struct RootView: View {
// MARK: - Load Courses
private func loadCourses() async {
guard !isRefreshingCourses else { return }
isRefreshingCourses = true
defer { isRefreshingCourses = false }
courseStatusText = "正在同步"
do {
courses = try await AuthAPI.courses(token: auth.token)
courseStatusText = "\(courses.count) 个课程"
var fetched = try await AuthAPI.courses(token: auth.token)
var boundarySucceeded = 0
var boundaryFailed = 0
courses = fetched
if !fetched.isEmpty {
courseStatusText = "\(fetched.count) 个课程,正在刷新句界"
let refreshed = await catalogVM.preloadBoundaries(
for: fetched.map { $0.toVideoSummary() }
)
boundarySucceeded = refreshed.succeeded
boundaryFailed = refreshed.failed
let refreshedByHash = Dictionary(uniqueKeysWithValues: refreshed.videos.map { ($0.videoHash, $0) })
fetched = fetched.map { course in
guard let video = refreshedByHash[course.videoHash] else { return course }
return Course(
videoHash: course.videoHash,
title: course.title,
durationMs: video.durationMs,
language: course.language,
sentenceCount: video.sentenceCount,
streamUrl: course.streamUrl,
enrolled: course.enrolled
)
}
if currentCourse != nil,
let video = refreshed.videos.first(where: { $0.videoHash == currentCourse?.videoHash }) {
currentCourse = fetched.first(where: { $0.videoHash == video.videoHash })
}
}
courses = fetched
if fetched.isEmpty {
courseStatusText = "0 个课程"
} else if boundaryFailed == 0 {
courseStatusText = "\(fetched.count) 个课程 · 句界已刷新"
} else if boundarySucceeded == 0 {
courseStatusText = "\(fetched.count) 个课程 · 句界刷新失败"
} else {
courseStatusText = "\(fetched.count) 个课程 · 句界部分刷新(失败 \(boundaryFailed))"
}
} catch {
courseStatusText = "课程加载失败:\(error.localizedDescription)"
}
@@ -516,25 +606,84 @@ struct RootView: View {
// MARK: - Fullscreen
private func enterFullscreen() {
AppDelegate.orientationLock = UIDevice.current.userInterfaceIdiom == .pad ? .all : .landscape
updateSupportedInterfaceOrientations()
AppDelegate.lockOrientation(.landscape)
showFullscreenPlayer = true
}
private func exitFullscreen() {
AppDelegate.orientationLock = UIDevice.current.userInterfaceIdiom == .pad ? .all : .portrait
updateSupportedInterfaceOrientations()
let restored: UIInterfaceOrientationMask =
UIDevice.current.userInterfaceIdiom == .pad ? .all : .portrait
AppDelegate.lockOrientation(restored)
showFullscreenPlayer = false
}
private func updateSupportedInterfaceOrientations() {
let scenes = UIApplication.shared.connectedScenes.compactMap { $0 as? UIWindowScene }
let controllers = scenes.flatMap { $0.windows }.compactMap(\.rootViewController)
for controller in controllers {
if #available(iOS 16.0, *) {
controller.setNeedsUpdateOfSupportedInterfaceOrientations()
} else {
UIViewController.attemptRotationToDeviceOrientation()
private var chineseSubtitlePlaceholder: String {
if chineseTranslationLoading {
return "正在翻译中文字幕…"
}
if !chineseTranslationError.isEmpty {
return chineseTranslationError
}
return "暂无中文字幕"
}
@MainActor
private func applyChineseTranslations(_ map: [Int: String], mediaId: String) {
guard controller.currentItem?.id == mediaId else { return }
let updated = (controller.currentItem?.sentences ?? []).map { sentence in
sentence.withZhText(map[sentence.index] ?? sentence.zhText)
}
controller.updateBoundaries(updated)
}
private func ensureChineseTranslations() {
guard showChineseSubtitles, activeModule == .train else {
return
}
guard let item = controller.currentItem else {
return
}
let needsTranslation = item.sentences.contains {
!($0.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty &&
($0.zhText ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
}
guard needsTranslation else {
chineseTranslationLoading = false
chineseTranslationError = ""
return
}
chineseTranslationTask?.cancel()
chineseTranslationLoading = true
chineseTranslationError = ""
let mediaId = item.id
chineseTranslationTask = Task {
do {
let expectedIndexes = item.sentences.compactMap { sentence -> Int? in
let text = (sentence.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
return text.isEmpty ? nil : sentence.index
}
let map = try await SentenceBoundaryAPI.loadSubtitles(
videoHash: mediaId,
expectedIndexes: expectedIndexes
) { partial in
Task { @MainActor in
applyChineseTranslations(partial, mediaId: mediaId)
}
}
guard !Task.isCancelled else { return }
await MainActor.run {
applyChineseTranslations(map, mediaId: mediaId)
chineseTranslationLoading = false
chineseTranslationError = ""
controller.showTransientMessage("已加载中文字幕")
}
} catch {
guard !Task.isCancelled else { return }
await MainActor.run {
chineseTranslationLoading = false
chineseTranslationError = "中文字幕翻译失败"
controller.showTransientMessage("中文字幕翻译失败:\(error.localizedDescription)")
}
}
}
}

View File

@@ -35,7 +35,7 @@ struct TestView: View {
PlayerSectionView(
controller: controller,
showsContinuousToggle: false,
continuousPlayback: .constant(true),
continuousPlayback: .constant(false),
showsSubtitle: $testSubtitleVisible,
onFullscreen: onFullscreen
)
@@ -321,7 +321,7 @@ struct TestView: View {
dubSegments[sentence.index] = audioURL
dubRawSegments[sentence.index] = rawPersistedURL
mergedDubURL = nil
submitAssessment(recordingURL: url)
submitAssessment(recordingURL: audioURL)
}
private func restorePlaybackAudioSession() {

View File

@@ -4,6 +4,8 @@ struct TrainingView: View {
@ObservedObject var controller: PlayerController
@ObservedObject var catalogVM: CatalogViewModel
@Binding var continuousPlayback: Bool
@Binding var showChineseSubtitles: Bool
var chineseSubtitlePlaceholder: String = "正在翻译中文字幕…"
let onFullscreen: () -> Void
var body: some View {
@@ -13,6 +15,9 @@ struct TrainingView: View {
controller: controller,
showsContinuousToggle: true,
continuousPlayback: $continuousPlayback,
showsLessonTitle: false,
showChineseSubtitles: $showChineseSubtitles,
chineseSubtitlePlaceholder: chineseSubtitlePlaceholder,
onFullscreen: onFullscreen
)
CatalogView(vm: catalogVM)
@@ -27,5 +32,8 @@ struct TrainingView: View {
: "已关闭连续播放,将循环播放当前句子"
)
}
.onChange(of: showChineseSubtitles) { enabled in
controller.showTransientMessage(enabled ? "已开启中文字幕" : "已关闭中文字幕")
}
}
}

View File

@@ -3,7 +3,8 @@
iOS 端与 [android/](../android/README.md) 示例 App 功能对齐的移动端:
- 在线视频目录(`GET /api/v1/videos`,仅展示 `ready` 课程,自动解析相对流地址)
- AVPlayer 在线播放 + 本地缓存导入;单击播放/暂停
- 云端视频先完整下载为本地整文件缓存,下载完成后才允许 AVPlayer 播放;
支持本地缓存导入;单击播放/暂停
- 左划 / 右划:上一句 / 下一句(无句边界时回退为 10 秒快退 / 快进)
- 长按播放区:按触摸点水平位置变速(左慢右快,0.5x - 2.0x),松手恢复原速
- 按句跳转:句边界来自 `GET /api/v1/videos/{sha256}/sentence-boundaries`

File diff suppressed because it is too large Load Diff

View File

@@ -9,6 +9,7 @@ MAX_VIDEO_UPLOAD_BYTES=12884901888
MAX_ATTEMPT_UPLOAD_BYTES=52428800
KEEP_ATTEMPT_AUDIO=false
ASSESSMENT_PASS_SCORE=70
LOG_LEVEL=INFO
MOSS_TRANSCRIBE_URL=http://127.0.0.1:8001/v1/audio/transcriptions
MOSS_MODEL=OpenMOSS-Team/MOSS-Transcribe-Diarize
@@ -24,6 +25,8 @@ AI_API_URL=https://api.example.com/v1/chat/completions
AI_API_KEY=replace-with-your-ai-api-key
AI_MODEL=gpt-4o-mini
AI_TIMEOUT_SECONDS=45
# 后台批量翻译可使用更长时间;实际翻译超时 = max(AI_TIMEOUT_SECONDS, AI_TRANSLATION_TIMEOUT_SECONDS)
AI_TRANSLATION_TIMEOUT_SECONDS=120
# 火山引擎对象存储(TOS / 火山 OSS)。桶请设置为公共读,或把下方地址配置为可公开访问的 CDN 域名。
VOLCANO_OSS_ACCESS_KEY=replace-with-volcano-access-key

View File

@@ -51,6 +51,8 @@ cp -a whisper/. /opt/whisper/ # 用 whisper/. 而不是 whisper/*,否则不
cd /opt/whisper
cp .env.example .env # 按需修改 WHISPER_MODEL
# 默认从 DaoCloud 拉 ghcr 镜像;能直连 GitHub 时在 .env 里设
# WHISPER_IMAGE=ghcr.io/speaches-ai/speaches:latest-cuda
docker compose up -d
docker compose logs -f whisper # 首次启动下载模型(约 3 GB)
```
@@ -156,8 +158,9 @@ curl -X POST http://127.0.0.1:8001/v1/audio/transcriptions \
`sentence_api` 不以 Whisper 返回的 segment 直接作为句子:文本会跨 segment 累积,
只有碰到句号(`.`)、问号(`?`)或逗号前的单词数达到 50 时才算一句结束;
逗号前内容不足 50 个单词时不在此处断句,会继续与后续内容合并。Whisper 在没有
标点处断开的 segment 会自动合并成同一句。管理后台编辑句子文本时,换行并保存会把
该句拆成多句;也支持将相邻两句合并成一句;拆分点优先使用对该句音频再次转写得到的词级时间戳,转写不可用时按
标点处断开的 segment 会自动合并成同一句。管理后台的文本保存、换行拆分占位、相邻句
合并和结束时间微调会立即入库,但不会触发翻译或转写;关闭句子对话框时才统一执行
一轮拆分时间戳处理和翻译。拆分点优先使用对原句音频再次转写得到的词级时间戳,转写不可用时按
文本长度估算。管理后台的结束时间旁支持微调按钮;调整后下一句的起始时间联动更新,
并保证相邻句子不重叠。若转写服务支持词级时间戳
(`timestamp_granularities[]=word`,Whisper/Speaches 支持),句子的结束时间用
@@ -195,6 +198,33 @@ AI_MODEL=gpt-4o-mini
`http://127.0.0.1:9000/v1/audio/transcriptions`,并把 `MOSS_MODEL` 设为
`Systran/faster-whisper-large-v3`。
Dockerfile 默认使用国内镜像源,避免构建时从 Docker Hub / Debian / PyPI 直连过慢:
- 基础镜像:`docker.m.daocloud.io/library/python:3.12-slim`
- apt:`mirrors.aliyun.com`
- pip:清华 `https://pypi.tuna.tsinghua.edu.cn/simple`
如需覆盖,可在 `sentence_api/.env` 或构建命令前设置:
```bash
export PYTHON_IMAGE=python:3.12-slim # 能直连 Docker Hub 时
export PIP_INDEX_URL=https://mirrors.aliyun.com/pypi/simple
export PIP_TRUSTED_HOST=mirrors.aliyun.com
export APT_MIRROR=mirrors.tuna.tsinghua.edu.cn
```
拉 Docker Hub 官方镜像仍然慢时,把仓库里的 `sentence_api/docker-daemon.json.example`
拷到 Docker 配置并重启:
```bash
sudo mkdir -p /etc/docker
sudo cp sentence_api/docker-daemon.json.example /etc/docker/daemon.json
sudo systemctl restart docker
```
`registry-mirrors` 只加速 `docker.io`。Whisper 的 `ghcr.io` 镜像请用 compose 里默认的
DaoCloud 前缀(见第 2A 节)。
启动:
```bash

View File

@@ -1,10 +1,27 @@
FROM python:3.12-slim
ARG PYTHON_IMAGE=docker.m.daocloud.io/library/python:3.12-slim
FROM ${PYTHON_IMAGE}
ARG APT_MIRROR=mirrors.aliyun.com
ARG PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
ARG PIP_TRUSTED_HOST=pypi.tuna.tsinghua.edu.cn
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1 \
PIP_INDEX_URL=${PIP_INDEX_URL} \
PIP_TRUSTED_HOST=${PIP_TRUSTED_HOST}
RUN apt-get update \
RUN set -eux; \
if [ -f /etc/apt/sources.list.d/debian.sources ]; then \
sed -i "s|deb.debian.org|${APT_MIRROR}|g; s|security.debian.org|${APT_MIRROR}|g" \
/etc/apt/sources.list.d/debian.sources; \
fi; \
if [ -f /etc/apt/sources.list ]; then \
sed -i "s|deb.debian.org|${APT_MIRROR}|g; s|security.debian.org|${APT_MIRROR}|g" \
/etc/apt/sources.list; \
fi; \
apt-get update \
&& apt-get install -y --no-install-recommends ffmpeg ca-certificates \
&& rm -rf /var/lib/apt/lists/*

View File

@@ -59,7 +59,12 @@ POST /api/v1/videos/{sha256}/sentences/{index}/assessments
POST /api/v1/admin/videos
PUT /api/v1/admin/videos/raw
POST /api/v1/admin/videos/{sha256}/process
POST /api/v1/admin/videos/{sha256}/repair-gaps
GET /api/v1/admin/videos/{sha256}/subtitle-translation
POST /api/v1/admin/videos/{sha256}/subtitle-translation
POST /api/v1/admin/videos/{sha256}/subtitle-translation/repair
PUT /api/v1/admin/videos/{sha256}/sentences/{index}
PUT /api/v1/admin/videos/{sha256}/sentences/batch
PUT /api/v1/admin/videos/{sha256}/sentences/{index}/split
PUT /api/v1/admin/videos/{sha256}/sentences/{index}/boundary
PUT /api/v1/admin/users/password
@@ -100,6 +105,21 @@ The endpoint requires a signed-in user and accepts only the sentence and course
context. `AI_API_URL` and `AI_API_KEY` remain on the server and are never sent
to the mobile clients.
Chinese captions are stored on the course in the server database and shared by
every user. `GET /api/v1/videos/{sha256}/subtitle-translation` returns the saved
captions without calling the AI, including `complete` and `status` so clients can
poll long videos. `POST` starts missing-caption generation in the background and
returns quickly; it translates in small windows with neighboring context, runs
several windows in parallel, retries and splits a window if the model response is
invalid, and writes each finished window immediately so later viewers can reuse
it. English sentence edits clear the saved Chinese captions for that course.
Desktop playback of a local file can `POST` the same endpoint with the locally
generated English cues (`duration_ms`, `algorithm_version`, `sentences`). If that
SHA-256 is not already a course, the server stores the English subtitles and
translates them; later `GET`/`POST` calls poll and reuse the saved Chinese
captions. Official course sentences are never overwritten by a local upload.
## Assessment
```bash

View File

@@ -180,8 +180,8 @@ sudo systemctl restart whisper-transcribe
| 现象 | 原因 | 处理 |
| --- | --- | --- |
| 更新后接口行为没变 | 忘了 `--build`,容器还在跑旧镜像 | 执行 `docker compose up -d --build` |
| 构建时 pip 安装超时 | 服务器访问 PyPI 受限 | 给 Dockerfile 的 pip install 加 `-i https://pypi.tuna.tsinghua.edu.cn/simple` 后重新构建 |
| 构建时拉基础镜像超时 | Docker Hub 网络问题 | 检查 `/etc/docker/daemon.json` 的 `registry-mirrors`(已配置 DaoCloud) |
| 构建时 pip 安装超时 | 服务器访问 PyPI 受限 | Dockerfile 默认已用清华 PyPI;仍慢可改 `PIP_INDEX_URL` 后 `--build` |
| 构建时拉基础镜像超时 | Docker Hub 网络问题 | Dockerfile 默认从 DaoCloud 拉 `python:3.12-slim`;也可把 `docker-daemon.json.example` 拷到 `/etc/docker/daemon.json` 后 `systemctl restart docker` |
| `healthz` 里 `moss_configured: false` | `.env` 里 `MOSS_TRANSCRIBE_URL` 为空或改了未重建 | 改 `.env` 后执行 `docker compose up -d` |
| Whisper 9000 端口连不上 | 服务未运行或容器被删 | `systemctl status whisper-transcribe`;容器被删时用 `systemctl start` 重建 |
| API 重启后视频卡在 `processing` | 重启中断了处理流程 | 后台点“重新处理”即可 |

View File

@@ -1,7 +1,11 @@
"""AI-backed sentence explanations for the mobile clients."""
"""AI-backed sentence explanations and subtitle translation."""
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
import json
import logging
import re
import threading
from urllib.parse import urlsplit
import requests
@@ -9,6 +13,17 @@ import requests
logger = logging.getLogger(__name__)
_JSON_FENCE_RE = re.compile(r"^```(?:json)?\s*|\s*```$", re.IGNORECASE)
_CJK_RE = re.compile("[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]")
_NUMBERED_LINE_RE = re.compile(r"^\s*[\[(]?(\d+)[\])]?[\.、::\)]\s*(.+?)\s*$")
_TRANSLATE_BATCH_SENTENCES = 12
_TRANSLATE_BATCH_CHARS = 2_200
_TRANSLATE_CONTEXT_BEFORE = 8
_TRANSLATE_CONTEXT_AFTER = 4
_TRANSLATE_WORKERS = 4
_TRANSLATE_TIMEOUT_SECONDS = 120.0
_TRANSLATE_SPLIT_THRESHOLD = 4
class AITutorError(RuntimeError):
"""Raised when the configured AI service cannot answer a request."""
@@ -20,6 +35,7 @@ class AITutor:
api_key: str
model: str
timeout_seconds: float
translation_timeout_seconds: float = _TRANSLATE_TIMEOUT_SECONDS
@property
def available(self) -> bool:
@@ -54,15 +70,302 @@ class AITutor:
language_note = f"句子语言:{language.strip()}。" if language.strip() else ""
system_prompt = (
"你是一位耐心的外语口语老师。请用简体中文讲解用户提供的影视/课程句子。"
"先给出自然的中文翻译,再解释关键词、常用搭配或语法要点,并简要说明这句话适合怎样跟读。"
"回答使用简明清晰的分段短文,总长度不超过 300 字,不要编造视频里没有的信息。"
"先给出自然的中文翻译,再解释关键词、常用搭配、语法要点,并简要说明这句话适合怎样跟读。"
"回答使用简明清晰的分段短文,总长度不超过 600 字,不要编造视频里没有的信息。"
)
user_prompt = (
f"课程:{context}\n位置:{position}\n{language_note}句子:{sentence_text}"
)
return self._complete(system_prompt, user_prompt, temperature=0.3)
def translate_sentences(
self,
sentences: list[tuple[int, str]],
*,
course_title: str = "",
existing: dict[int, str] | None = None,
persist=None,
terms: list[tuple[str, str]] | None = None,
) -> dict[int, str]:
if not self.available:
raise AITutorError("AI translation is not configured.")
if not sentences:
raise AITutorError("No sentences were provided for translation.")
ordered = [(index, text.strip()) for index, text in sentences if text.strip()]
if not ordered:
raise AITutorError("No sentence text was provided for translation.")
glossary = _normalize_translation_terms(terms)
source_by_index = {index: text for index, text in ordered}
translations = {
index: text.strip()
for index, text in (existing or {}).items()
if isinstance(text, str)
and text.strip()
and not translation_needs_repair(source_by_index.get(index, ""), text)
}
pending = [(index, text) for index, text in ordered if index not in translations]
if not pending:
return translations
persist_lock = threading.Lock()
def save_partial(partial: dict[int, str]) -> None:
if not partial:
return
translations.update(partial)
if persist is None:
return
with persist_lock:
persist(partial)
batches = _translation_batches(pending, ordered)
workers = min(_TRANSLATE_WORKERS, len(batches)) or 1
failures: list[tuple[list[tuple[int, str]], Exception]] = []
def run_batch(batch: list[tuple[int, str]]) -> dict[int, str]:
return self._translate_batch_resilient(
batch,
all_sentences=ordered,
existing=translations,
course_title=course_title,
terms=glossary,
)
if workers == 1:
for batch in batches:
try:
save_partial(run_batch(batch))
except Exception as exc: # noqa: BLE001 - collect and retry below
failures.append((batch, exc))
else:
with ThreadPoolExecutor(max_workers=workers) as pool:
futures = {pool.submit(run_batch, batch): batch for batch in batches}
for future in as_completed(futures):
batch = futures[future]
try:
save_partial(future.result())
except Exception as exc: # noqa: BLE001
failures.append((batch, exc))
for batch, error in failures:
logger.warning(
"Translation batch failed, retrying with smaller windows: size=%s error=%s",
len(batch),
error,
)
save_partial(
self._translate_batch_resilient(
batch,
all_sentences=ordered,
existing=translations,
course_title=course_title,
terms=glossary,
force_split=len(batch) > 1,
)
)
missing = [index for index, _ in pending if not translations.get(index, "").strip()]
if missing:
raise AITutorError("AI translation did not cover every sentence.")
if glossary:
source_by_index = {index: text for index, text in ordered}
translations = {
index: _apply_translation_terms(source_by_index.get(index, ""), zh_text, glossary)
for index, zh_text in translations.items()
}
return translations
def _translate_batch_resilient(
self,
batch: list[tuple[int, str]],
*,
all_sentences: list[tuple[int, str]],
existing: dict[int, str],
course_title: str,
terms: list[tuple[str, str]] | None = None,
force_split: bool = False,
) -> dict[int, str]:
if not batch:
return {}
if force_split and len(batch) > 1:
return self._translate_split_batch(
batch,
all_sentences=all_sentences,
existing=existing,
course_title=course_title,
terms=terms,
)
last_error: Exception | None = None
attempts = 2 if len(batch) <= _TRANSLATE_SPLIT_THRESHOLD else 1
for _ in range(attempts):
try:
return self._translate_batch(
batch,
all_sentences=all_sentences,
existing=existing,
course_title=course_title,
terms=terms,
)
except AITutorError as exc:
last_error = exc
logger.warning(
"Translation batch attempt failed: size=%s indexes=%s error=%s",
len(batch),
[index for index, _ in batch],
exc,
)
if len(batch) > 1:
return self._translate_split_batch(
batch,
all_sentences=all_sentences,
existing=existing,
course_title=course_title,
terms=terms,
)
if last_error is not None:
raise last_error
raise AITutorError("AI translation did not cover every sentence.")
def _translate_split_batch(
self,
batch: list[tuple[int, str]],
*,
all_sentences: list[tuple[int, str]],
existing: dict[int, str],
course_title: str,
terms: list[tuple[str, str]] | None = None,
) -> dict[int, str]:
mid = max(1, len(batch) // 2)
merged: dict[int, str] = {}
for part in (batch[:mid], batch[mid:]):
merged.update(
self._translate_batch_resilient(
part,
all_sentences=all_sentences,
existing=existing,
course_title=course_title,
terms=terms,
)
)
return merged
def _translate_batch(
self,
batch: list[tuple[int, str]],
*,
all_sentences: list[tuple[int, str]],
existing: dict[int, str],
course_title: str,
terms: list[tuple[str, str]] | None = None,
) -> dict[int, str]:
context = course_title.strip() or "口语训练课程"
existing = dict(existing)
before, after = _neighbor_context(batch, all_sentences)
expected_indexes = [index for index, _ in batch]
glossary = _normalize_translation_terms(terms)
system_prompt = (
"你是专业的影视字幕翻译。请结合前后文语境,把指定编号的外语句子翻译成自然、口语化的简体中文。"
"必须一句对一句,不能合并、拆分、跳过或改写编号。"
"只输出 JSON 对象,键是句子编号,值是对应中文,不要输出解释或 markdown。"
)
if glossary:
system_prompt += "必须严格使用用户指定的专有名词译法,禁止自行音译成其他写法。"
user_parts = [
f"课程:{context}",
f"本次需要翻译 {len(batch)} 句,编号必须与输入完全一致。",
]
if glossary:
user_parts.append("专有名词译法(必须严格遵守,原文出现时译文必须使用指定中文):")
user_parts.extend(f"{source} => {zh}" for source, zh in glossary)
previous = [
(index, source, existing[index])
for index, source in before
if existing.get(index, "").strip()
]
if previous:
user_parts.append("已有译文(供用词一致,不要重复输出):")
user_parts.extend(
f"[{index}] {source} => {zh}" for index, source, zh in previous[-6:]
)
elif before:
user_parts.append("前文(只供理解语境,不要翻译):")
user_parts.extend(f"[{index}] {text}" for index, text in before)
if after:
user_parts.append("后文(只供理解语境,不要翻译):")
user_parts.extend(f"[{index}] {text}" for index, text in after)
user_parts.append("待翻译句子:")
user_parts.extend(f"[{index}] {text}" for index, text in batch)
user_parts.append(
"输出示例:{" + ",".join(f'"{index}":"中文"' for index in expected_indexes[:2]) + "}"
)
user_prompt = "\n".join(user_parts)
content = self._complete(
system_prompt,
user_prompt,
temperature=0.2,
timeout_seconds=max(self.timeout_seconds, self.translation_timeout_seconds),
)
try:
parsed = _parse_translation_payload(content, expected_indexes)
except AITutorError:
logger.warning(
"Translation response parse failed: endpoint=%s model=%s "
"expected_indexes=%s prompt=%r raw_content=%r",
self.request_url,
self.model,
expected_indexes,
user_prompt[:2000],
content[:2000],
)
raise
source_by_index = {index: text for index, text in batch}
missing = [
index
for index in expected_indexes
if translation_needs_repair(source_by_index.get(index, ""), parsed.get(index, ""))
]
if missing:
invalid = [
{
"index": index,
"source": source_by_index.get(index, ""),
"translation": parsed.get(index, ""),
}
for index in missing
]
logger.warning(
"Translation response failed validation: endpoint=%s model=%s "
"expected_indexes=%s parsed=%r invalid=%r prompt=%r raw_content=%r",
self.request_url,
self.model,
expected_indexes,
parsed,
invalid,
user_prompt[:2000],
content[:2000],
)
raise AITutorError("AI translation returned an unexpected response.")
return {
index: _apply_translation_terms(source_by_index.get(index, ""), parsed[index], glossary)
for index in expected_indexes
}
def _complete(
self,
system_prompt: str,
user_prompt: str,
*,
temperature: float,
timeout_seconds: float | None = None,
) -> str:
headers = {"Content-Type": "application/json"}
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
timeout = self.timeout_seconds if timeout_seconds is None else timeout_seconds
try:
response = requests.post(
self.request_url,
@@ -73,14 +376,24 @@ class AITutor:
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt},
],
"temperature": 0.3,
"temperature": temperature,
"stream": False,
},
timeout=self.timeout_seconds,
timeout=timeout,
)
except requests.Timeout as exc:
logger.warning(
"AI request timed out: endpoint=%s model=%s timeout_seconds=%s",
self.request_url,
self.model,
timeout,
)
raise AITutorError(
f"AI service request timed out after {timeout:g} seconds."
) from exc
except requests.RequestException as exc:
logger.exception(
"AI explanation request failed: endpoint=%s model=%s",
"AI request failed: endpoint=%s model=%s",
self.request_url,
self.model,
)
@@ -88,7 +401,7 @@ class AITutor:
if response.status_code < 200 or response.status_code >= 300:
logger.warning(
"AI explanation returned HTTP %s: endpoint=%s model=%s body=%s",
"AI service returned HTTP %s: endpoint=%s model=%s body=%s",
response.status_code,
self.request_url,
self.model,
@@ -97,19 +410,246 @@ class AITutor:
raise AITutorError(f"AI service returned HTTP {response.status_code}.")
try:
body = response.json()
explanation = body["choices"][0]["message"]["content"]
content = body["choices"][0]["message"]["content"]
except (KeyError, IndexError, TypeError, ValueError) as exc:
logger.warning(
"AI explanation returned an unexpected body: endpoint=%s body=%s",
"AI service returned an unexpected body: endpoint=%s body=%s",
self.request_url,
response.text[:1000],
)
raise AITutorError("AI service returned an unexpected response.") from exc
if not isinstance(explanation, str) or not explanation.strip():
if not isinstance(content, str) or not content.strip():
logger.warning(
"AI explanation returned empty content: endpoint=%s body=%s",
"AI service returned empty content: endpoint=%s body=%s",
self.request_url,
response.text[:1000],
)
raise AITutorError("AI service returned an empty explanation.")
return explanation.strip()
return content.strip()
def translation_needs_repair(source: str | None, zh_text: str | None) -> bool:
"""Return True when a sentence still needs a Chinese translation."""
source_text = (source or "").strip()
if not source_text:
return False
zh = (zh_text or "").strip()
if not zh:
return True
if _CJK_RE.search(zh):
return False
return _CJK_RE.search(source_text) is None
def _translation_batches(
pending: list[tuple[int, str]],
all_sentences: list[tuple[int, str]],
) -> list[list[tuple[int, str]]]:
position = {index: offset for offset, (index, _) in enumerate(all_sentences)}
ordered_pending = sorted(pending, key=lambda item: position[item[0]])
runs: list[list[tuple[int, str]]] = []
current: list[tuple[int, str]] = []
last_pos: int | None = None
for item in ordered_pending:
pos = position[item[0]]
if current and last_pos is not None and pos != last_pos + 1:
runs.append(current)
current = []
current.append(item)
last_pos = pos
if current:
runs.append(current)
batches: list[list[tuple[int, str]]] = []
for run in runs:
batch: list[tuple[int, str]] = []
batch_chars = 0
for item in run:
extra = len(item[1]) + 8
too_many = len(batch) >= _TRANSLATE_BATCH_SENTENCES
too_long = batch and batch_chars + extra > _TRANSLATE_BATCH_CHARS
if too_many or too_long:
batches.append(batch)
batch = []
batch_chars = 0
batch.append(item)
batch_chars += extra
if batch:
batches.append(batch)
return batches
def _neighbor_context(
batch: list[tuple[int, str]],
all_sentences: list[tuple[int, str]],
) -> tuple[list[tuple[int, str]], list[tuple[int, str]]]:
position = {index: offset for offset, (index, _) in enumerate(all_sentences)}
first = position[batch[0][0]]
last = position[batch[-1][0]]
before = all_sentences[max(0, first - _TRANSLATE_CONTEXT_BEFORE):first]
after = all_sentences[last + 1:last + 1 + _TRANSLATE_CONTEXT_AFTER]
return before, after
def _parse_translation_payload(content: str, expected_indexes: list[int] | int) -> dict[int, str]:
if isinstance(expected_indexes, int):
expected = list(range(expected_indexes))
else:
expected = list(expected_indexes)
text = _JSON_FENCE_RE.sub("", content.strip()).strip()
parse_errors: list[str] = []
for candidate in (text, *_json_snippets(text)):
try:
parsed = _translations_from_json(json.loads(candidate), expected)
except (TypeError, ValueError) as exc:
parsed = None
parse_errors.append(str(exc))
if parsed:
return parsed
numbered: dict[int, str] = {}
for line in text.splitlines():
match = _NUMBERED_LINE_RE.match(line)
if match is None:
continue
numbered[int(match.group(1))] = match.group(2).strip()
by_original = {
index: numbered[index]
for index in expected
if numbered.get(index, "").strip()
}
if len(by_original) == len(expected):
return by_original
by_order = {
expected[offset]: numbered[offset + 1]
for offset in range(len(expected))
if numbered.get(offset + 1, "").strip()
}
if len(by_order) == len(expected):
return by_order
logger.warning(
"Translation payload parser exhausted all formats: expected_indexes=%s "
"json_parse_errors=%r numbered_line_keys=%s content=%r",
expected,
parse_errors,
sorted(numbered),
text[:2000],
)
raise AITutorError("AI translation returned an unexpected response.")
def _json_snippets(text: str) -> list[str]:
snippets: list[str] = []
for opener, closer in (("{", "}"), ("[", "]")):
start = text.find(opener)
while start != -1:
depth = 0
in_string = False
escape = False
for offset, char in enumerate(text[start:], start=start):
if in_string:
if escape:
escape = False
elif char == "\\":
escape = True
elif char == '"':
in_string = False
continue
if char == '"':
in_string = True
elif char == opener:
depth += 1
elif char == closer:
depth -= 1
if depth == 0:
snippets.append(text[start:offset + 1])
break
start = text.find(opener, start + 1)
return snippets
def _translations_from_json(data: object, expected: list[int]) -> dict[int, str] | None:
expected_set = set(expected)
if isinstance(data, dict):
nested = None
for key in ("translations", "sentences", "zh", "items"):
if key in data:
nested = data[key]
break
if nested is not None:
parsed = _translations_from_json(nested, expected)
if parsed:
return parsed
values: dict[int, str] = {}
for raw_key, raw_value in data.items():
try:
index = int(raw_key)
except (TypeError, ValueError):
continue
if index not in expected_set or not isinstance(raw_value, str) or not raw_value.strip():
continue
values[index] = raw_value.strip()
if len(values) == len(expected):
return values
return None
if isinstance(data, list) and len(data) == len(expected):
if all(isinstance(item, str) and item.strip() for item in data):
return {
index: item.strip()
for index, item in zip(expected, data)
}
if all(isinstance(item, dict) for item in data):
values = {}
for item in data:
raw_index = item.get("index")
zh = item.get("zh_text") or item.get("zh") or item.get("text")
try:
index = int(raw_index)
except (TypeError, ValueError):
continue
if index not in expected_set or not isinstance(zh, str) or not zh.strip():
continue
values[index] = zh.strip()
if len(values) == len(expected):
return values
return None
def _normalize_translation_terms(terms: list[tuple[str, str]] | None) -> list[tuple[str, str]]:
cleaned: list[tuple[str, str]] = []
seen: dict[str, int] = {}
for item in terms or []:
if not isinstance(item, (tuple, list)) or len(item) < 2:
continue
source = str(item[0] or "").strip()[:120]
zh = str(item[1] or "").strip()[:120]
if not source or not zh:
continue
key = source.casefold()
if key in seen:
cleaned[seen[key]] = (source, zh)
continue
seen[key] = len(cleaned)
cleaned.append((source, zh))
if len(cleaned) >= 50:
break
return cleaned
def _term_pattern(source: str) -> re.Pattern[str]:
escaped = re.escape(source)
if re.fullmatch(r"[A-Za-z][A-Za-z0-9'’.\- ]*", source):
return re.compile(rf"(?<![A-Za-z0-9]){escaped}(?![A-Za-z0-9])", re.IGNORECASE)
return re.compile(escaped)
def _apply_translation_terms(
source_text: str,
zh_text: str,
terms: list[tuple[str, str]] | None,
) -> str:
del source_text
result = zh_text
for source, zh in _normalize_translation_terms(terms):
result = _term_pattern(source).sub(zh, result)
return result

View File

@@ -26,10 +26,12 @@ class Settings:
moss_timeout_seconds: float
moss_max_new_tokens: int
moss_end_padding_ms: int
log_level: str
ai_api_url: str
ai_api_key: str
ai_model: str
ai_timeout_seconds: float
ai_translation_timeout_seconds: float
pass_score: float
volcano_oss_access_key: str
volcano_oss_secret_key: str
@@ -65,10 +67,14 @@ class Settings:
moss_timeout_seconds=float(os.getenv("MOSS_TIMEOUT_SECONDS", "1800")),
moss_max_new_tokens=int(os.getenv("MOSS_MAX_NEW_TOKENS", "65536")),
moss_end_padding_ms=int(os.getenv("MOSS_END_PADDING_MS", "300")),
log_level=os.getenv("LOG_LEVEL", "INFO").strip().upper() or "INFO",
ai_api_url=os.getenv("AI_API_URL", "").strip().rstrip("/"),
ai_api_key=os.getenv("AI_API_KEY", "").strip(),
ai_model=os.getenv("AI_MODEL", "gpt-4o-mini"),
ai_timeout_seconds=float(os.getenv("AI_TIMEOUT_SECONDS", "45")),
ai_translation_timeout_seconds=float(
os.getenv("AI_TRANSLATION_TIMEOUT_SECONDS", "120")
),
pass_score=float(os.getenv("ASSESSMENT_PASS_SCORE", "70")),
volcano_oss_access_key=os.getenv("VOLCANO_OSS_ACCESS_KEY", "").strip(),
volcano_oss_secret_key=os.getenv("VOLCANO_OSS_SECRET_KEY", "").strip(),

View File

@@ -3,6 +3,11 @@ services:
build:
context: ..
dockerfile: sentence_api/Dockerfile
args:
PYTHON_IMAGE: ${PYTHON_IMAGE:-docker.m.daocloud.io/library/python:3.12-slim}
APT_MIRROR: ${APT_MIRROR:-mirrors.aliyun.com}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.tuna.tsinghua.edu.cn/simple}
PIP_TRUSTED_HOST: ${PIP_TRUSTED_HOST:-pypi.tuna.tsinghua.edu.cn}
restart: unless-stopped
network_mode: host
env_file:

View File

@@ -0,0 +1,5 @@
{
"registry-mirrors": [
"https://docker.m.daocloud.io"
]
}

View File

@@ -24,11 +24,13 @@ from fastapi import (
Request,
UploadFile,
)
from pydantic import ValidationError
from fastapi.responses import FileResponse, RedirectResponse, Response
from fastapi.staticfiles import StaticFiles
from starlette.concurrency import run_in_threadpool
from starlette.requests import ClientDisconnect
from .ai_tutor import AITutor, AITutorError
from .translation_jobs import SubtitleTranslationService
from .assessment import AssessmentService
from .auth import AuthenticatedUser
@@ -39,11 +41,20 @@ from .models import (
AuthRequest,
AuthResponse,
CourseSummary,
CourseTitleUpdate,
SentenceBoundaryAdjust,
SentenceGapRepairRequest,
SentenceGapRepairResponse,
SentenceBoundary,
SentenceBoundaryDocument,
SentenceExplanationRequest,
SentenceExplanationResponse,
AdminSubtitleTranslationResponse,
AdminSentenceBatchItem,
AdminSentenceBatchUpdate,
LocalSubtitleTranslationRequest,
SubtitleRetranslateRequest,
SubtitleTranslationResponse,
PasswordResetRequest,
SentenceTextUpdate,
UserDubShareSummary,
@@ -90,6 +101,10 @@ def create_app(
oss_uploader: Optional[VolcanoOSSUploader] = None,
) -> FastAPI:
service_settings = settings or Settings.from_env()
logging.basicConfig(
level=getattr(logging, service_settings.log_level, logging.INFO),
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
service_settings.ensure_directories()
video_repository = repository or VideoRepository(service_settings.database_path)
legacy_store = store or BoundaryStore(service_settings.legacy_boundaries_file)
@@ -104,6 +119,7 @@ def create_app(
api_key=service_settings.ai_api_key,
model=service_settings.ai_model,
timeout_seconds=service_settings.ai_timeout_seconds,
translation_timeout_seconds=service_settings.ai_translation_timeout_seconds,
)
oss = oss_uploader or VolcanoOSSUploader(service_settings)
processor = VideoProcessor(service_settings, video_repository, moss, oss)
@@ -126,6 +142,10 @@ def create_app(
application.state.oss_uploader = oss
application.state.video_processor = processor
application.state.assessment_service = assessment_service
application.state.subtitle_translations = SubtitleTranslationService(
video_repository,
lambda: application.state.ai_tutor,
)
static_dir = Path(__file__).resolve().parent / "static"
if static_dir.is_dir():
@@ -332,6 +352,59 @@ def create_app(
raise HTTPException(status_code=503, detail=str(exc)) from exc
return SentenceExplanationResponse(explanation=explanation)
def stored_subtitle_translations(
document: SentenceBoundaryDocument,
) -> SubtitleTranslationResponse:
return application.state.subtitle_translations.snapshot(document)
@application.get(
"/api/v1/videos/{video_hash}/subtitle-translation",
response_model=SubtitleTranslationResponse,
)
def get_subtitle_translation(
video_hash: str = SHA256_PATH,
user: AuthenticatedUser = Depends(current_user),
) -> SubtitleTranslationResponse:
del user
snapshot = application.state.subtitle_translations.current(video_hash)
if snapshot is None:
raise HTTPException(status_code=404, detail="Course is not available.")
return snapshot
@application.post(
"/api/v1/videos/{video_hash}/subtitle-translation",
response_model=SubtitleTranslationResponse,
)
async def translate_subtitles(
request: Request,
video_hash: str = SHA256_PATH,
user: AuthenticatedUser = Depends(current_user),
) -> SubtitleTranslationResponse:
del user
raw = await request.body()
local_document = None
if raw.strip():
try:
payload = LocalSubtitleTranslationRequest.model_validate_json(raw)
local_document = SentenceBoundaryDocument(
video_hash=video_hash,
duration_ms=payload.duration_ms,
algorithm_version=payload.algorithm_version,
sentences=payload.sentences,
)
except ValidationError as exc:
raise HTTPException(status_code=422, detail=exc.errors()) from exc
try:
if local_document is not None:
return application.state.subtitle_translations.ensure_from_local(local_document)
return application.state.subtitle_translations.ensure(video_hash)
except KeyError:
raise HTTPException(status_code=404, detail="Course is not available.") from None
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
except AITutorError as exc:
raise HTTPException(status_code=503, detail=str(exc)) from exc
@application.post(
"/api/v1/audio/sentence-boundaries",
response_model=SentenceBoundaryDocument,
@@ -481,9 +554,13 @@ def create_app(
media_path = service_settings.videos_dir / row["stored_filename"]
if not media_path.is_file():
raise HTTPException(status_code=404, detail="Stored video file is missing.")
media_type = row.get("content_type") or "video/mp4"
if not str(media_type).lower().startswith("video/"):
guessed = mimetypes.guess_type(str(row.get("stored_filename") or ""))[0]
media_type = guessed if guessed and guessed.lower().startswith("video/") else "video/mp4"
return FileResponse(
media_path,
media_type=row["content_type"],
media_type=media_type,
headers={"Accept-Ranges": "bytes", "Cache-Control": "public, max-age=3600"},
)
@@ -620,6 +697,127 @@ def create_app(
background_tasks.add_task(_process_safely, processor, video_hash)
return {"video_hash": video_hash.lower(), "status": "processing_queued"}
@application.post(
"/api/v1/admin/videos/{video_hash}/repair-gaps",
response_model=SentenceGapRepairResponse,
dependencies=[Depends(require_admin)],
)
async def repair_sentence_gaps(
payload: Optional[SentenceGapRepairRequest] = None,
video_hash: str = SHA256_PATH,
) -> SentenceGapRepairResponse:
request = payload or SentenceGapRepairRequest()
try:
result = await run_in_threadpool(
processor.repair_gaps,
video_hash,
request.min_gap_ms,
)
except KeyError:
raise HTTPException(status_code=404, detail="Video was not found.") from None
except FileNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except RuntimeError as exc:
raise HTTPException(status_code=503, detail=str(exc)) from exc
if result["inserted_count"]:
application.state.subtitle_translations.start_missing(video_hash)
return SentenceGapRepairResponse(video_hash=video_hash.lower(), **result)
@application.get(
"/api/v1/admin/videos/{video_hash}/subtitle-translation",
response_model=AdminSubtitleTranslationResponse,
dependencies=[Depends(require_admin)],
)
def get_admin_subtitle_translation(
video_hash: str = SHA256_PATH,
) -> AdminSubtitleTranslationResponse:
snapshot = application.state.subtitle_translations.admin_snapshot(video_hash)
if snapshot is None:
raise HTTPException(status_code=404, detail="Course is not available.")
return snapshot
@application.post(
"/api/v1/admin/videos/{video_hash}/subtitle-translation",
response_model=AdminSubtitleTranslationResponse,
dependencies=[Depends(require_admin)],
)
def retranslate_subtitles(
payload: Optional[SubtitleRetranslateRequest] = None,
video_hash: str = SHA256_PATH,
) -> AdminSubtitleTranslationResponse:
request = payload or SubtitleRetranslateRequest()
try:
return application.state.subtitle_translations.retranslate(
video_hash,
terms=[(item.source, item.zh) for item in request.terms],
)
except KeyError:
raise HTTPException(status_code=404, detail="Course is not available.") from None
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
except AITutorError as exc:
raise HTTPException(status_code=503, detail=str(exc)) from exc
@application.post(
"/api/v1/admin/videos/{video_hash}/subtitle-translation/repair",
response_model=AdminSubtitleTranslationResponse,
dependencies=[Depends(require_admin)],
)
def repair_subtitle_translation(
video_hash: str = SHA256_PATH,
) -> AdminSubtitleTranslationResponse:
try:
return application.state.subtitle_translations.repair_missing(video_hash)
except KeyError:
raise HTTPException(status_code=404, detail="Course is not available.") from None
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
except AITutorError as exc:
raise HTTPException(status_code=503, detail=str(exc)) from exc
@application.put(
"/api/v1/admin/videos/{video_hash}/title",
response_model=VideoSummary,
dependencies=[Depends(require_admin)],
)
def update_course_title(
payload: CourseTitleUpdate,
video_hash: str = SHA256_PATH,
) -> VideoSummary:
video = video_repository.update_title(video_hash, payload.title)
if video is None:
raise HTTPException(status_code=404, detail="Video was not found.")
return _video_summary(video, service_settings)
@application.put(
"/api/v1/admin/videos/{video_hash}/sentences/batch",
response_model=List[SentenceBoundary],
dependencies=[Depends(require_admin)],
)
def batch_admin_sentences(
payload: AdminSentenceBatchUpdate,
video_hash: str = SHA256_PATH,
) -> List[SentenceBoundary]:
try:
result = _replace_admin_sentences(
repository=video_repository,
transcriber=moss,
settings=service_settings,
video_hash=video_hash,
payload=payload,
)
except KeyError:
raise HTTPException(status_code=404, detail="Course is not available.") from None
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
if payload.process_timings:
application.state.subtitle_translations.refresh(video_hash)
else:
application.state.subtitle_translations.invalidate(video_hash)
return result
@application.put(
"/api/v1/admin/videos/{video_hash}/sentences/{sentence_index}",
response_model=SentenceBoundary,
@@ -638,6 +836,7 @@ def create_app(
)
if sentence is None:
raise HTTPException(status_code=404, detail="Sentence was not found.")
application.state.subtitle_translations.invalidate(video_hash)
return sentence
@application.put(
@@ -673,6 +872,7 @@ def create_app(
raise HTTPException(status_code=400, detail=str(exc)) from exc
if result is None:
raise HTTPException(status_code=404, detail="Sentence was not found.")
application.state.subtitle_translations.refresh(video_hash)
return result
@@ -694,6 +894,7 @@ def create_app(
raise HTTPException(status_code=400, detail=str(exc)) from exc
if sentence is None:
raise HTTPException(status_code=404, detail="Sentence was not found.")
application.state.subtitle_translations.invalidate(video_hash)
return sentence
@application.put(
@@ -1030,6 +1231,161 @@ def _process_safely(processor: VideoProcessor, video_hash: str) -> None:
logger.exception("Video processing failed for %s", video_hash)
def _replace_admin_sentences(
*,
repository: VideoRepository,
transcriber: Transcriber,
settings: Settings,
video_hash: str,
payload: AdminSentenceBatchUpdate,
) -> List[SentenceBoundary]:
"""Apply queued admin edits once, instead of after every split/merge."""
video = repository.get_video(video_hash)
old_document = repository.get_document(video_hash)
if video is None or old_document is None:
raise KeyError(video_hash)
old_sentences = {
(sentence.start_ms, sentence.end_ms): sentence
for sentence in old_document.sentences
}
rows: List[SentenceBoundary] = []
index = 0
position = 0
while position < len(payload.sentences):
item = payload.sentences[position]
if item.source_start_ms is None or item.source_end_ms is None:
if "\n" in item.text or "\r" in item.text:
raise ValueError("Unsplit batch sentences cannot contain line breaks.")
rows.append(
SentenceBoundary(
index=index,
start_ms=item.start_ms,
end_ms=item.end_ms,
text=item.text.strip(),
language=item.language,
zh_text=item.zh_text,
reference_speech_duration_ms=item.reference_speech_duration_ms,
)
)
index += 1
position += 1
continue
source_key = (item.source_start_ms, item.source_end_ms)
group_end = position + 1
while group_end < len(payload.sentences):
following = payload.sentences[group_end]
if following.source_start_ms != source_key[0] or following.source_end_ms != source_key[1]:
break
group_end += 1
group = payload.sentences[position:group_end]
source = old_sentences.get(source_key)
if source is None:
if payload.process_timings:
# A split may already have been persisted with provisional
# boundaries. Reconstruct its audio range from the saved rows so
# timing refinement can be retried after a dialog is reopened.
missing = [
item
for item in group
if (item.start_ms, item.end_ms) not in old_sentences
]
if missing:
raise ValueError("The original sentence for a queued split no longer exists.")
source = SentenceBoundary(
index=0,
start_ms=source_key[0],
end_ms=source_key[1],
text="\n".join(item.text.strip() for item in group),
language=next(
(item.language for item in group if item.language is not None),
None,
),
reference_speech_duration_ms=sum(
item.reference_speech_duration_ms for item in group
),
)
else:
source = None
if len(group) == 1:
if "\n" in group[0].text or "\r" in group[0].text:
raise ValueError("Unsplit batch sentences cannot contain line breaks.")
rows.append(
SentenceBoundary(
index=index,
start_ms=group[0].start_ms,
end_ms=group[0].end_ms,
text=group[0].text.strip(),
language=group[0].language,
zh_text=group[0].zh_text,
reference_speech_duration_ms=group[0].reference_speech_duration_ms,
)
)
elif not payload.process_timings:
for item in group:
rows.append(
SentenceBoundary(
index=index,
start_ms=item.start_ms,
end_ms=item.end_ms,
text=item.text.strip(),
language=item.language,
zh_text=item.zh_text,
reference_speech_duration_ms=item.reference_speech_duration_ms,
)
)
index += 1
else:
split_text = "\n".join(item.text.strip() for item in group if item.text.strip())
replacements = split_sentence_text(
transcriber=transcriber,
media_path=settings.videos_dir / video["stored_filename"],
sentence=source,
text=split_text,
language=next(
(item.language for item in group if item.language is not None),
source.language,
),
)
if len(replacements) != len(group):
raise ValueError("Split result count does not match the queued sentences.")
for replacement, item in zip(replacements, group):
rows.append(
SentenceBoundary(
index=index,
start_ms=replacement.start_ms,
end_ms=replacement.end_ms,
text=item.text.strip(),
language=(
item.language
if item.language is not None
else source.language
),
zh_text=None,
reference_speech_duration_ms=replacement.reference_speech_duration_ms,
)
)
index += 1
index = len(rows)
position = group_end
duration_ms = video["duration_ms"]
if duration_ms is None:
raise ValueError("Course duration is not available.")
previous_end = 0
for row in rows:
if row.start_ms < previous_end:
raise ValueError("Batch sentences must be sorted and non-overlapping.")
if row.end_ms > duration_ms:
raise ValueError("Batch sentences cannot exceed the course duration.")
previous_end = row.end_ms
document = old_document.model_copy(update={"sentences": rows})
return repository.replace_sentences(document).sentences
def _video_summary(row: Dict[str, Any], settings: Settings) -> VideoSummary:
if row.get("remote_url"):
stream_url = row["remote_url"]

View File

@@ -12,6 +12,7 @@ class SentenceBoundary(BaseModel):
end_ms: int = Field(gt=0)
text: Optional[str] = None
language: Optional[str] = None
zh_text: Optional[str] = None
reference_speech_duration_ms: Optional[int] = Field(default=None, gt=0)
@model_validator(mode="after")
@@ -51,7 +52,7 @@ class SentenceBoundaryDocument(BaseModel):
return self
VideoStatus = Literal["uploaded", "processing", "ready", "failed"]
VideoStatus = Literal["uploaded", "processing", "ready", "failed", "local_subtitles"]
class VideoSummary(BaseModel):
@@ -92,6 +93,50 @@ class SentenceTextUpdate(BaseModel):
language: Optional[str] = Field(default=None, max_length=32)
class AdminSentenceBatchItem(BaseModel):
model_config = ConfigDict(extra="forbid")
start_ms: int = Field(ge=0)
end_ms: int = Field(gt=0)
text: str = Field(min_length=1, max_length=4000)
language: Optional[str] = Field(default=None, max_length=32)
zh_text: Optional[str] = Field(default=None, max_length=4000)
reference_speech_duration_ms: int = Field(gt=0)
source_start_ms: Optional[int] = Field(default=None, ge=0)
source_end_ms: Optional[int] = Field(default=None, gt=0)
@model_validator(mode="after")
def validate_range(self):
if self.end_ms <= self.start_ms:
raise ValueError("end_ms must be greater than start_ms")
if (self.source_start_ms is None) != (self.source_end_ms is None):
raise ValueError("source_start_ms and source_end_ms must be set together")
elif self.source_start_ms is not None and self.source_end_ms <= self.source_start_ms:
raise ValueError("source end_ms must be greater than source start_ms")
return self
class AdminSentenceBatchUpdate(BaseModel):
model_config = ConfigDict(extra="forbid")
sentences: List[AdminSentenceBatchItem] = Field(min_length=1, max_length=10000)
process_timings: bool = Field(default=True)
class CourseTitleUpdate(BaseModel):
model_config = ConfigDict(extra="forbid")
title: str = Field(max_length=200)
@field_validator("title")
@classmethod
def validate_title(cls, value: str) -> str:
cleaned = value.strip()
if not cleaned:
raise ValueError("title cannot be empty")
return cleaned
class PasswordResetRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
@@ -112,12 +157,83 @@ class SentenceExplanationResponse(BaseModel):
explanation: str = Field(min_length=1, max_length=6000)
class SubtitleTranslationItem(BaseModel):
model_config = ConfigDict(extra="forbid")
index: int = Field(ge=0)
zh_text: str = Field(min_length=1, max_length=4000)
class SubtitleTranslationResponse(BaseModel):
model_config = ConfigDict(extra="forbid")
video_hash: str
translations: List[SubtitleTranslationItem]
complete: bool = True
status: str = "ready"
error: Optional[str] = None
class LocalSubtitleTranslationRequest(BaseModel):
"""Upload locally generated English subtitles so the server can translate them."""
model_config = ConfigDict(extra="forbid")
duration_ms: int = Field(gt=0)
algorithm_version: str = Field(default="local-pc", min_length=1, max_length=64)
sentences: List[SentenceBoundary] = Field(min_length=1)
class TranslationTerm(BaseModel):
model_config = ConfigDict(extra="forbid")
source: str = Field(min_length=1, max_length=120)
zh: str = Field(min_length=1, max_length=120)
@field_validator("source", "zh")
@classmethod
def strip_term(cls, value: str) -> str:
cleaned = value.strip()
if not cleaned:
raise ValueError("translation term cannot be empty")
return cleaned
class SubtitleRetranslateRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
terms: List[TranslationTerm] = Field(default_factory=list, max_length=50)
class AdminSubtitleTranslationResponse(SubtitleTranslationResponse):
terms: List[TranslationTerm] = Field(default_factory=list)
missing_count: int = Field(default=0, ge=0)
attempted_count: int = Field(default=0, ge=0)
class SentenceBoundaryAdjust(BaseModel):
model_config = ConfigDict(extra="forbid")
delta_ms: int = Field(ge=-10_000, le=10_000)
class SentenceGapRepairRequest(BaseModel):
model_config = ConfigDict(extra="forbid")
min_gap_ms: int = Field(default=8000, ge=2000, le=180_000)
class SentenceGapRepairResponse(BaseModel):
model_config = ConfigDict(extra="forbid")
video_hash: str
min_gap_ms: int = Field(ge=1)
gap_count: int = Field(ge=0)
transcribed_gaps: int = Field(ge=0)
inserted_count: int = Field(ge=0)
sentence_count: int = Field(ge=0)
class TextSubstitution(BaseModel):
expected: str
actual: str

View File

@@ -4,14 +4,16 @@ import subprocess
import tempfile
import uuid
from pathlib import Path
from typing import List, Optional
from typing import Any, Dict, List, Optional, Sequence, Tuple
from .audio_metrics import (
AudioAnalysisError,
analyze_samples,
decode_audio_mono,
FRAME_SAMPLES,
refine_sentence_end_ms,
refine_sentence_start_ms,
speech_frame_mask,
)
from .config import Settings
from .generate_boundaries import ALGORITHM_VERSION, make_entry, media_duration_ms
@@ -27,8 +29,72 @@ from .transcription import (
logger = logging.getLogger(__name__)
MOSS_ALGORITHM_VERSION = "moss-punctuation-v3"
MOSS_ALGORITHM_VERSION = "moss-punctuation-v4"
SILENCE_ALGORITHM_VERSION = "silence-rms-v1"
GAP_REPAIR_MIN_MS = 8_000
GAP_REPAIR_INSET_MS = 150
GAP_REPAIR_MIN_SPEECH_MS = 400
GAP_REPAIR_MIN_SENTENCE_MS = 100
GAP_REPAIR_TARGET_CHUNK_MS = 15_000
GAP_REPAIR_MAX_CHUNK_MS = 20_000
GAP_REPAIR_MIN_CHUNK_MS = 5_000
GAP_REPAIR_MIN_SPLIT_SILENCE_MS = 120
GAP_REPAIR_FALLBACK_LANGUAGE = "en"
_CUES_ID = b"\x1e\x53\xb6\x80"
_CLUSTER_ID = b"\x1f\x43\xb6\x75"
_HEADER_SCAN_SIZE = 64 * 1024
_WEBM_SUFFIXES = {".webm", ".mkv"}
def _webm_cues_at_front(filepath: Path) -> bool:
"""Return True if WebM/MKV CUES appear before the first Cluster."""
try:
with open(filepath, "rb") as f:
header = f.read(_HEADER_SCAN_SIZE)
except OSError:
return True # can't read → assume OK, don't touch
cues_pos = header.find(_CUES_ID)
cluster_pos = header.find(_CLUSTER_ID)
if cues_pos != -1 and (cluster_pos == -1 or cues_pos < cluster_pos):
return True
return False
def _ensure_webm_cues_front(filepath: Path) -> None:
"""Re-mux a WebM/MKV file (stream copy) if CUES are not at the front.
YouTube-dl / yt-dlp writes CUES at the end by default, which slows HTTP
Range seeking. This runs once during upload processing and takes seconds.
"""
if filepath.suffix.lower() not in _WEBM_SUFFIXES:
return
if _webm_cues_at_front(filepath):
return
ffmpeg = shutil.which("ffmpeg")
if not ffmpeg:
logger.warning("ffmpeg not found; skipping WebM CUES optimization.")
return
tmp = filepath.with_suffix(".cues_tmp" + filepath.suffix)
try:
result = subprocess.run(
[ffmpeg, "-y", "-loglevel", "error",
"-i", str(filepath), "-c", "copy",
"-cues_to_front", "1", str(tmp)],
capture_output=True, timeout=300,
)
if result.returncode == 0 and tmp.is_file() and tmp.stat().st_size > 0:
tmp.replace(filepath)
logger.info("WebM CUES moved to front: %s", filepath.name)
else:
logger.warning(
"WebM CUES re-mux failed for %s: rc=%d stderr=%s",
filepath.name, result.returncode, result.stderr.decode()[-200:],
)
except (subprocess.TimeoutExpired, OSError) as exc:
logger.warning("WebM CUES re-mux error for %s: %s", filepath.name, exc)
finally:
tmp.unlink(missing_ok=True)
class VideoProcessor:
@@ -55,6 +121,7 @@ class VideoProcessor:
self.repository.mark_processing(video_hash)
work_path: Optional[Path] = None
try:
_ensure_webm_cues_front(media_path)
if self.transcriber.available:
work_path = self.settings.work_dir / f"{video_hash}-{uuid.uuid4().hex}.wav"
extract_audio(media_path, work_path)
@@ -69,7 +136,11 @@ class VideoProcessor:
)
if not document.sentences:
raise RuntimeError("MOSS returned no timestamped speech segments.")
self._finish_processing(document, transcript.text)
self._finish_processing(
document,
transcript.text,
moss_segments=_serialize_moss_segments(transcript),
)
else:
entry, _ = make_entry(media_path, video_hash=video_hash)
document = SentenceBoundaryDocument(
@@ -90,13 +161,18 @@ class VideoProcessor:
self,
document: SentenceBoundaryDocument,
transcription: Optional[str],
moss_segments: Optional[List[Dict[str, Any]]] = None,
) -> None:
if self.oss_uploader is None or not self.oss_uploader.enabled:
self.repository.save_processing_result(document, transcription, ready=True)
self.repository.save_processing_result(
document, transcription, ready=True, moss_segments=moss_segments
)
return
# Keep the course invisible while the mandatory OSS upload is running.
self.repository.save_processing_result(document, transcription, ready=False)
self.repository.save_processing_result(
document, transcription, ready=False, moss_segments=moss_segments
)
video = self.repository.get_video(document.video_hash)
if video is None:
raise ValueError(f"Unknown video: {document.video_hash}")
@@ -108,6 +184,338 @@ class VideoProcessor:
)
self.repository.mark_oss_uploaded(document.video_hash, remote_url)
def repair_gaps(
self,
video_hash: str,
min_gap_ms: int = GAP_REPAIR_MIN_MS,
) -> Dict[str, int]:
"""Re-transcribe long unsentenced gaps without replacing existing cues."""
if min_gap_ms < 1:
raise ValueError("min_gap_ms must be positive")
if not self.transcriber.available:
raise RuntimeError("MOSS transcription is not configured on this server.")
video = self.repository.get_video(video_hash)
if video is None:
raise KeyError(video_hash)
document = self.repository.get_document(video_hash)
if document is None or not document.sentences:
raise ValueError("Sentence boundaries are not available for this video.")
media_path = self.settings.videos_dir / video["stored_filename"]
if not media_path.is_file():
raise FileNotFoundError(f"Stored video is missing: {media_path.name}")
gaps = find_sentence_gaps(document, min_gap_ms)
inserted: List[SentenceBoundary] = []
transcribed_gaps = 0
logger.info(
"Gap repair started: video=%s min_gap_ms=%d language=%s gaps=%d "
"sentences=%d duration_ms=%d transcriber=%s",
video_hash,
min_gap_ms,
video.get("language") or "auto",
len(gaps),
len(document.sentences),
document.duration_ms,
type(self.transcriber).__name__,
)
if gaps:
language = video.get("language")
with tempfile.TemporaryDirectory(prefix="gap-repair-") as temp_dir:
for gap_index, (raw_start, raw_end) in enumerate(gaps):
logger.info(
"Gap %d/%d: raw=%d-%dms duration=%dms",
gap_index + 1,
len(gaps),
raw_start,
raw_end,
raw_end - raw_start,
)
clipped = _inset_gap(raw_start, raw_end, document.duration_ms)
if clipped is None:
logger.warning(
"Gap %d/%d skipped after inset: raw=%d-%dms",
gap_index + 1,
len(gaps),
raw_start,
raw_end,
)
continue
start_ms, end_ms = clipped
chunk_path = Path(temp_dir) / f"gap-{gap_index}-full.wav"
logger.info(
"Gap %d/%d extracted range=%d-%dms duration=%dms file=%s",
gap_index + 1,
len(gaps),
start_ms,
end_ms,
end_ms - start_ms,
chunk_path,
)
try:
extract_audio_segment(
media_path,
chunk_path,
start_ms=start_ms,
duration_ms=end_ms - start_ms,
)
samples, sample_rate = decode_audio_mono(chunk_path)
metrics = analyze_samples(samples, sample_rate)
except (AudioAnalysisError, RuntimeError) as exc:
logger.warning(
"Skipping sentence gap %s-%sms: %s",
start_ms,
end_ms,
exc,
exc_info=True,
)
continue
logger.info(
"Gap %d/%d audio metrics: duration_ms=%d speech_duration_ms=%d "
"internal_silence_ms=%d pause_ratio=%.3f threshold_ms=%d",
gap_index + 1,
len(gaps),
metrics.recording_duration_ms,
metrics.speech_duration_ms,
metrics.internal_silence_ms,
metrics.internal_pause_ratio,
GAP_REPAIR_MIN_SPEECH_MS,
)
if metrics.speech_duration_ms < GAP_REPAIR_MIN_SPEECH_MS:
logger.warning(
"Gap %d/%d skipped as silence: speech_duration_ms=%d < %d",
gap_index + 1,
len(gaps),
metrics.speech_duration_ms,
GAP_REPAIR_MIN_SPEECH_MS,
)
continue
relative_chunks = _plan_gap_chunks(samples, sample_rate, end_ms - start_ms)
logger.info(
"Gap %d/%d split into %d transcription chunks: %s",
gap_index + 1,
len(gaps),
len(relative_chunks),
relative_chunks,
)
gap_inserted_count = 0
for chunk_index, (chunk_offset_start, chunk_offset_end) in enumerate(
relative_chunks
):
chunk_start = start_ms + chunk_offset_start
chunk_end = start_ms + chunk_offset_end
chunk_path = (
Path(temp_dir) / f"gap-{gap_index}-chunk-{chunk_index}.wav"
)
try:
extract_audio_segment(
media_path,
chunk_path,
start_ms=chunk_start,
duration_ms=chunk_end - chunk_start,
)
chunk_samples, chunk_sample_rate = decode_audio_mono(chunk_path)
chunk_metrics = analyze_samples(
chunk_samples, chunk_sample_rate
)
except (AudioAnalysisError, RuntimeError) as exc:
logger.warning(
"Could not prepare gap chunk %d/%d %s-%sms: %s",
chunk_index + 1,
len(relative_chunks),
chunk_start,
chunk_end,
exc,
exc_info=True,
)
continue
if chunk_metrics.speech_duration_ms < GAP_REPAIR_MIN_SPEECH_MS:
logger.info(
"Skipping quiet gap chunk %d/%d: range=%d-%dms "
"speech_duration_ms=%d",
chunk_index + 1,
len(relative_chunks),
chunk_start,
chunk_end,
chunk_metrics.speech_duration_ms,
)
continue
languages_to_try = [language]
if not language or language.strip().lower() == "auto":
fallback = GAP_REPAIR_FALLBACK_LANGUAGE
if fallback not in languages_to_try:
languages_to_try.append(fallback)
transcript = None
for attempt_index, attempt_language in enumerate(languages_to_try):
try:
logger.info(
"Transcribing gap chunk %d/%d: gap=%d/%d "
"range=%d-%dms language=%s",
chunk_index + 1,
len(relative_chunks),
gap_index + 1,
len(gaps),
chunk_start,
chunk_end,
attempt_language or "auto",
)
transcript = self.transcriber.transcribe(
chunk_path, attempt_language
)
except Exception as exc:
logger.warning(
"Gap chunk transcription failed for %s %s-%sms: %s",
video_hash,
chunk_start,
chunk_end,
exc,
exc_info=True,
)
continue
logger.info(
"Gap chunk transcript returned: gap=%d/%d chunk=%d/%d "
"segments=%d text=%r",
gap_index + 1,
len(gaps),
chunk_index + 1,
len(relative_chunks),
len(transcript.segments),
transcript.text,
)
if transcript.segments:
break
if attempt_index + 1 < len(languages_to_try):
logger.warning(
"Gap chunk returned no segments; retrying with %s: "
"range=%d-%dms",
languages_to_try[attempt_index + 1],
chunk_start,
chunk_end,
)
if transcript is None or not transcript.segments:
logger.warning(
"Gap chunk produced no transcript segments: gap=%d/%d "
"chunk=%d/%d range=%d-%dms attempts=%s",
gap_index + 1,
len(gaps),
chunk_index + 1,
len(relative_chunks),
chunk_start,
chunk_end,
[item or "auto" for item in languages_to_try],
)
continue
gap_document = document_from_transcript(
video_hash=video_hash,
duration_ms=chunk_end - chunk_start,
transcript=transcript,
language=language,
audio_path=chunk_path,
end_padding_ms=self.settings.moss_end_padding_ms,
)
shifted = _shift_gap_sentences(
gap_document.sentences,
offset_ms=chunk_start,
limit_end_ms=chunk_end,
)
if not shifted:
logger.warning(
"Gap chunk produced no insertable sentences: gap=%d/%d "
"chunk=%d/%d transcript_segments=%d "
"boundary_sentences=%d range=%d-%dms",
gap_index + 1,
len(gaps),
chunk_index + 1,
len(relative_chunks),
len(transcript.segments),
len(gap_document.sentences),
chunk_start,
chunk_end,
)
for boundary in gap_document.sentences:
logger.warning(
"Candidate boundary before shift: start=%d end=%d text=%r",
boundary.start_ms,
boundary.end_ms,
boundary.text,
)
continue
inserted.extend(shifted)
gap_inserted_count += len(shifted)
logger.info(
"Gap chunk produced %d insertable sentences: gap=%d/%d "
"chunk=%d/%d range=%d-%dms",
len(shifted),
gap_index + 1,
len(gaps),
chunk_index + 1,
len(relative_chunks),
chunk_start,
chunk_end,
)
if gap_inserted_count:
transcribed_gaps += 1
logger.info(
"Gap %d/%d produced %d insertable sentences total: range=%d-%dms",
gap_index + 1,
len(gaps),
gap_inserted_count,
start_ms,
end_ms,
)
else:
logger.warning(
"Gap %d/%d produced no insertable sentences after chunking: "
"range=%d-%dms chunks=%d",
gap_index + 1,
len(gaps),
start_ms,
end_ms,
len(relative_chunks),
)
if not inserted:
logger.warning(
"Gap repair finished without inserts: video=%s gaps=%d "
"transcribed_gaps=%d inserted=0",
video_hash,
len(gaps),
transcribed_gaps,
)
return {
"min_gap_ms": min_gap_ms,
"gap_count": len(gaps),
"transcribed_gaps": transcribed_gaps,
"inserted_count": 0,
"sentence_count": len(document.sentences),
}
merged = _merge_sentence_lists(document.sentences, inserted)
updated = document.model_copy(update={"sentences": merged})
saved = self.repository.replace_sentences(updated)
logger.info(
"Gap repair finished: video=%s gaps=%d transcribed_gaps=%d inserted=%d "
"sentences=%d->%d",
video_hash,
len(gaps),
transcribed_gaps,
len(saved.sentences) - len(document.sentences),
len(document.sentences),
len(saved.sentences),
)
return {
"min_gap_ms": min_gap_ms,
"gap_count": len(gaps),
"transcribed_gaps": transcribed_gaps,
"inserted_count": len(saved.sentences) - len(document.sentences),
"sentence_count": len(saved.sentences),
}
def extract_audio(media_path: Path, output_path: Path) -> None:
ffmpeg = shutil.which("ffmpeg")
@@ -176,6 +584,169 @@ def extract_audio_segment(
raise RuntimeError(f"Could not extract sentence audio: {message[-2000:]}")
def find_sentence_gaps(
document: SentenceBoundaryDocument,
min_gap_ms: int = GAP_REPAIR_MIN_MS,
) -> List[Tuple[int, int]]:
"""Return [start_ms, end_ms) ranges that are missing sentence coverage."""
if min_gap_ms < 1:
raise ValueError("min_gap_ms must be positive")
gaps: List[Tuple[int, int]] = []
cursor = 0
for sentence in document.sentences:
if sentence.start_ms - cursor >= min_gap_ms:
gaps.append((cursor, sentence.start_ms))
cursor = max(cursor, sentence.end_ms)
if document.duration_ms - cursor >= min_gap_ms:
gaps.append((cursor, document.duration_ms))
return gaps
def _inset_gap(
start_ms: int,
end_ms: int,
duration_ms: int,
inset_ms: int = GAP_REPAIR_INSET_MS,
) -> Optional[Tuple[int, int]]:
start = start_ms + (inset_ms if start_ms > 0 else 0)
end = end_ms - (inset_ms if end_ms < duration_ms else 0)
if end - start < GAP_REPAIR_MIN_SPEECH_MS:
return None
return start, end
def _plan_gap_chunks(
samples,
sample_rate: int,
duration_ms: int,
) -> List[Tuple[int, int]]:
"""Split a long gap near silence so no ASR request exceeds Whisper's window."""
if duration_ms <= GAP_REPAIR_MAX_CHUNK_MS:
return [(0, duration_ms)]
chunks: List[Tuple[int, int]] = []
cursor = 0
while cursor < duration_ms:
remaining = duration_ms - cursor
if remaining <= GAP_REPAIR_MAX_CHUNK_MS:
chunks.append((cursor, duration_ms))
break
preferred = min(cursor + GAP_REPAIR_TARGET_CHUNK_MS, duration_ms)
lower = min(cursor + GAP_REPAIR_MIN_CHUNK_MS, preferred)
upper = min(cursor + GAP_REPAIR_MAX_CHUNK_MS, duration_ms)
cut = _find_silence_cut(
samples,
sample_rate,
lower,
upper,
preferred,
min_silence_ms=GAP_REPAIR_MIN_SPLIT_SILENCE_MS,
)
cut = max(cursor + 1, min(duration_ms, cut))
chunks.append((cursor, cut))
cursor = cut
return chunks
def _find_silence_cut(
samples,
sample_rate: int,
lower_ms: int,
upper_ms: int,
preferred_ms: int,
*,
min_silence_ms: int,
) -> int:
"""Choose the silence midpoint nearest ``preferred_ms``."""
frame_ms = 1000.0 * FRAME_SAMPLES / sample_rate
min_silence_frames = max(1, int(round(min_silence_ms / frame_ms)))
start_sample = max(0, int(lower_ms / 1000 * sample_rate))
end_sample = min(samples.size, int(upper_ms / 1000 * sample_rate) + 1)
if end_sample <= start_sample:
return preferred_ms
try:
speech = speech_frame_mask(samples[start_sample:end_sample], sample_rate)
except AudioAnalysisError:
return preferred_ms
run_start: Optional[int] = None
candidates: List[Tuple[int, int]] = []
for index, is_speech in enumerate(speech):
if not is_speech:
if run_start is None:
run_start = index
continue
if run_start is not None:
candidates.append((run_start, index))
run_start = None
if run_start is not None:
candidates.append((run_start, len(speech)))
valid = [
(run_start, run_end)
for run_start, run_end in candidates
if run_end - run_start >= min_silence_frames
]
if not valid:
return preferred_ms
def distance(run: Tuple[int, int]) -> float:
midpoint_frame = (run[0] + run[1]) / 2.0
midpoint_ms = lower_ms + midpoint_frame * frame_ms
return abs(midpoint_ms - preferred_ms)
run_start, run_end = min(valid, key=distance)
midpoint_ms = lower_ms + ((run_start + run_end) / 2.0) * frame_ms
return int(round(midpoint_ms))
def _shift_gap_sentences(
sentences: Sequence[SentenceBoundary],
*,
offset_ms: int,
limit_end_ms: int,
) -> List[SentenceBoundary]:
shifted: List[SentenceBoundary] = []
previous_end = offset_ms
for sentence in sentences:
start_ms = max(previous_end, sentence.start_ms + offset_ms)
end_ms = min(limit_end_ms, sentence.end_ms + offset_ms)
text = (sentence.text or "").strip()
if not text or end_ms - start_ms < GAP_REPAIR_MIN_SENTENCE_MS:
continue
shifted.append(
sentence.model_copy(
update={
"index": len(shifted),
"start_ms": start_ms,
"end_ms": end_ms,
"text": text,
"zh_text": None,
}
)
)
previous_end = end_ms
return shifted
def _merge_sentence_lists(
existing: Sequence[SentenceBoundary],
inserted: Sequence[SentenceBoundary],
) -> List[SentenceBoundary]:
combined = [*existing, *inserted]
combined.sort(key=lambda sentence: (sentence.start_ms, sentence.end_ms))
merged: List[SentenceBoundary] = []
previous_end = 0
for sentence in combined:
if sentence.start_ms < previous_end:
continue
merged.append(sentence.model_copy(update={"index": len(merged)}))
previous_end = sentence.end_ms
return merged
def split_sentence_text(
*,
transcriber: Transcriber,
@@ -484,3 +1055,32 @@ def _media_duration_ms(path: Path) -> int:
from .generate_boundaries import media_duration_ms
return media_duration_ms(path)
def _serialize_moss_segments(transcript: Transcript) -> List[Dict[str, Any]]:
rows: List[Dict[str, Any]] = []
for segment in transcript.segments:
text = segment.text.strip()
if not text:
continue
words = None
if segment.words:
words = [
{
"start_seconds": word.start_seconds,
"end_seconds": word.end_seconds,
"text": word.text,
}
for word in segment.words
]
rows.append(
{
"start_seconds": segment.start_seconds,
"end_seconds": segment.end_seconds,
"text": text,
"speaker": segment.speaker,
"words": words,
}
)
return rows

View File

@@ -2,7 +2,7 @@ import json
import sqlite3
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, Sequence, Tuple
import uuid
from .auth import hash_password, hash_token, new_token, verify_password
@@ -56,6 +56,7 @@ class VideoRepository:
end_ms INTEGER NOT NULL,
text TEXT,
language TEXT,
zh_text TEXT,
reference_speech_duration_ms INTEGER,
PRIMARY KEY (video_hash, sentence_index),
FOREIGN KEY (video_hash) REFERENCES videos(video_hash) ON DELETE CASCADE
@@ -120,6 +121,20 @@ class VideoRepository:
FOREIGN KEY (share_id) REFERENCES dub_shares(share_id) ON DELETE CASCADE,
UNIQUE (share_id, sentence_index)
);
CREATE TABLE IF NOT EXISTS moss_segments (
video_hash TEXT NOT NULL,
segment_index INTEGER NOT NULL,
start_seconds REAL NOT NULL,
end_seconds REAL NOT NULL,
start_ms INTEGER NOT NULL,
end_ms INTEGER NOT NULL,
text TEXT NOT NULL,
speaker TEXT,
words_json TEXT,
PRIMARY KEY (video_hash, segment_index),
FOREIGN KEY (video_hash) REFERENCES videos(video_hash) ON DELETE CASCADE
);
"""
)
@@ -179,6 +194,22 @@ class VideoRepository:
if "remote_url" not in video_columns:
connection.execute("ALTER TABLE videos ADD COLUMN remote_url TEXT")
with self._connect() as connection:
sentence_columns = {
row["name"]
for row in connection.execute("PRAGMA table_info(sentences)")
}
if "zh_text" not in sentence_columns:
connection.execute("ALTER TABLE sentences ADD COLUMN zh_text TEXT")
with self._connect() as connection:
video_columns = {
row["name"]
for row in connection.execute("PRAGMA table_info(videos)")
}
if "translation_glossary" not in video_columns:
connection.execute("ALTER TABLE videos ADD COLUMN translation_glossary TEXT")
def upsert_upload(
self,
*,
@@ -230,7 +261,24 @@ class VideoRepository:
)
def mark_processing(self, video_hash: str) -> None:
self._update_status(video_hash, "processing", None)
normalized_hash = normalize_video_hash(video_hash)
with self._connect() as connection:
connection.execute(
"DELETE FROM sentences WHERE video_hash = ?",
(normalized_hash,),
)
connection.execute(
"DELETE FROM moss_segments WHERE video_hash = ?",
(normalized_hash,),
)
connection.execute(
"""
UPDATE videos
SET status = ?, error_message = NULL, transcription = NULL, updated_at = ?
WHERE video_hash = ?
""",
("processing", utc_now(), normalized_hash),
)
def mark_failed(self, video_hash: str, message: str) -> None:
self._update_status(video_hash, "failed", message[:4000])
@@ -247,12 +295,17 @@ class VideoRepository:
document: SentenceBoundaryDocument,
transcription: Optional[str],
ready: bool = True,
moss_segments: Optional[List[Dict[str, Any]]] = None,
) -> None:
with self._connect() as connection:
connection.execute(
"DELETE FROM sentences WHERE video_hash = ?",
(document.video_hash,),
)
connection.execute(
"DELETE FROM moss_segments WHERE video_hash = ?",
(document.video_hash,),
)
connection.executemany(
"""
INSERT INTO sentences (
@@ -288,6 +341,111 @@ class VideoRepository:
document.video_hash,
),
)
if moss_segments:
connection.executemany(
"""
INSERT INTO moss_segments (
video_hash, segment_index, start_seconds, end_seconds,
start_ms, end_ms, text, speaker, words_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
[
_moss_segment_row(document.video_hash, index, segment)
for index, segment in enumerate(moss_segments)
],
)
def import_local_subtitle_document(
self,
document: SentenceBoundaryDocument,
) -> SentenceBoundaryDocument:
"""Save locally generated English subtitles for translation.
Official course videos are left unchanged. A previous local-only
import is replaced when the English cues changed.
"""
normalized_hash = normalize_video_hash(document.video_hash)
document = document.model_copy(update={"video_hash": normalized_hash})
existing_video = self.get_video(normalized_hash)
existing_document = self.get_document(normalized_hash)
if existing_video is not None and existing_video.get("status") != "local_subtitles":
if existing_document is None:
raise KeyError(normalized_hash)
return existing_document
if existing_document is not None and _english_fingerprint(existing_document) == _english_fingerprint(document):
return existing_document
now = utc_now()
language = next((sentence.language for sentence in document.sentences if sentence.language), None)
with self._connect() as connection:
if existing_video is None:
connection.execute(
"""
INSERT INTO videos (
video_hash, title, filename, stored_filename, content_type,
size_bytes, duration_ms, language, status, error_message,
algorithm_version, transcription, remote_url, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'local_subtitles', NULL, ?, NULL, NULL, ?, ?)
""",
(
normalized_hash,
"Local subtitles",
"local-subtitles.json",
"",
"application/json",
0,
document.duration_ms,
language,
document.algorithm_version,
now,
now,
),
)
else:
connection.execute(
"""
UPDATE videos
SET duration_ms = ?, language = COALESCE(?, language),
algorithm_version = ?, status = 'local_subtitles',
error_message = NULL, updated_at = ?
WHERE video_hash = ?
""",
(
document.duration_ms,
language,
document.algorithm_version,
now,
normalized_hash,
),
)
connection.execute(
"DELETE FROM sentences WHERE video_hash = ?",
(normalized_hash,),
)
connection.executemany(
"""
INSERT INTO sentences (
video_hash, sentence_index, start_ms, end_ms, text,
language, zh_text, reference_speech_duration_ms
) VALUES (?, ?, ?, ?, ?, ?, NULL, ?)
""",
[
(
normalized_hash,
sentence.index,
sentence.start_ms,
sentence.end_ms,
sentence.text,
sentence.language,
sentence.reference_speech_duration_ms,
)
for sentence in document.sentences
],
)
imported = self.get_document(normalized_hash)
if imported is None:
raise KeyError(normalized_hash)
return imported
def mark_oss_uploaded(self, video_hash: str, remote_url: str) -> None:
with self._connect() as connection:
@@ -300,6 +458,22 @@ class VideoRepository:
(remote_url, utc_now(), normalize_video_hash(video_hash)),
)
def update_title(self, video_hash: str, title: str) -> Optional[Dict[str, Any]]:
normalized_hash = normalize_video_hash(video_hash)
cleaned_title = title.strip()[:200]
if not cleaned_title:
raise ValueError("Title cannot be empty.")
with self._connect() as connection:
cursor = connection.execute(
"UPDATE videos SET title = ?, updated_at = ? WHERE video_hash = ?",
(cleaned_title, utc_now(), normalized_hash),
)
if cursor.rowcount == 0:
return None
return self.get_video(normalized_hash)
def list_videos(self) -> List[Dict[str, Any]]:
with self._connect() as connection:
rows = connection.execute(
@@ -307,6 +481,7 @@ class VideoRepository:
SELECT v.*, COUNT(s.sentence_index) AS sentence_count
FROM videos v
LEFT JOIN sentences s ON s.video_hash = v.video_hash
WHERE v.status != 'local_subtitles'
GROUP BY v.video_hash
ORDER BY v.created_at DESC
"""
@@ -327,6 +502,32 @@ class VideoRepository:
).fetchone()
return dict(row) if row else None
def get_moss_segments(self, video_hash: str) -> List[Dict[str, Any]]:
with self._connect() as connection:
rows = connection.execute(
"""
SELECT * FROM moss_segments
WHERE video_hash = ?
ORDER BY segment_index
""",
(normalize_video_hash(video_hash),),
).fetchall()
segments = []
for row in rows:
item = dict(row)
raw_words = item.pop("words_json", None)
words = None
if raw_words:
try:
parsed = json.loads(raw_words)
except json.JSONDecodeError:
parsed = None
if isinstance(parsed, list):
words = parsed
item["words"] = words
segments.append(item)
return segments
def get_document(self, video_hash: str) -> Optional[SentenceBoundaryDocument]:
video = self.get_video(video_hash)
if video is None or video["duration_ms"] is None:
@@ -340,19 +541,56 @@ class VideoRepository:
video_hash=video["video_hash"],
duration_ms=video["duration_ms"],
algorithm_version=video["algorithm_version"] or "unknown",
sentences=[
SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
for row in rows
],
sentences=[_sentence_from_row(row) for row in rows],
)
def replace_sentences(
self,
document: SentenceBoundaryDocument,
) -> SentenceBoundaryDocument:
"""Replace stored sentences while preserving video status and translations."""
normalized_hash = normalize_video_hash(document.video_hash)
with self._connect() as connection:
existing = connection.execute(
"SELECT video_hash FROM videos WHERE video_hash = ?",
(normalized_hash,),
).fetchone()
if existing is None:
raise KeyError(normalized_hash)
connection.execute(
"DELETE FROM sentences WHERE video_hash = ?",
(normalized_hash,),
)
connection.executemany(
"""
INSERT INTO sentences (
video_hash, sentence_index, start_ms, end_ms, text,
language, zh_text, reference_speech_duration_ms
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
[
(
normalized_hash,
sentence.index,
sentence.start_ms,
sentence.end_ms,
sentence.text,
sentence.language,
sentence.zh_text,
sentence.reference_speech_duration_ms,
)
for sentence in document.sentences
],
)
connection.execute(
"UPDATE videos SET updated_at = ? WHERE video_hash = ?",
(utc_now(), normalized_hash),
)
saved = self.get_document(normalized_hash)
if saved is None:
raise KeyError(normalized_hash)
return saved
def update_sentence_text(
self,
video_hash: str,
@@ -364,7 +602,8 @@ class VideoRepository:
with self._connect() as connection:
cursor = connection.execute(
"""
UPDATE sentences SET text = ?, language = COALESCE(?, language)
UPDATE sentences SET text = ?, language = COALESCE(?, language),
zh_text = NULL
WHERE video_hash = ? AND sentence_index = ?
""",
(text.strip(), language, normalized_hash, sentence_index),
@@ -379,14 +618,7 @@ class VideoRepository:
"SELECT * FROM sentences WHERE video_hash = ? AND sentence_index = ?",
(normalized_hash, sentence_index),
).fetchone()
return SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
return _sentence_from_row(row)
def get_sentence(
self,
@@ -404,14 +636,7 @@ class VideoRepository:
).fetchone()
if row is None:
return None
return SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
return _sentence_from_row(row)
def replace_sentence(
self,
@@ -523,6 +748,10 @@ class VideoRepository:
replacement.reference_speech_duration_ms,
),
)
connection.execute(
"UPDATE sentences SET zh_text = NULL WHERE video_hash = ?",
(normalized_hash,),
)
connection.execute(
"UPDATE videos SET updated_at = ? WHERE video_hash = ?",
(utc_now(), normalized_hash),
@@ -536,17 +765,7 @@ class VideoRepository:
(normalized_hash, sentence_index, sentence_index + len(rows)),
).fetchall()
return [
SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
for row in result_rows
]
return [_sentence_from_row(row) for row in result_rows]
def merge_with_next_sentence(
@@ -595,7 +814,7 @@ class VideoRepository:
"""
UPDATE sentences
SET end_ms = ?, text = ?, language = ?,
reference_speech_duration_ms = ?
reference_speech_duration_ms = ?, zh_text = NULL
WHERE video_hash = ? AND sentence_index = ?
""",
(
@@ -648,14 +867,7 @@ class VideoRepository:
(normalized_hash, sentence_index),
).fetchone()
return SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
return _sentence_from_row(row)
def adjust_sentence_boundary(
self,
@@ -760,17 +972,83 @@ class VideoRepository:
(normalized_hash, *updated_indexes),
).fetchall()
return [
SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
return [_sentence_from_row(row) for row in result_rows]
def get_translation_glossary(self, video_hash: str) -> List[Tuple[str, str]]:
video = self.get_video(video_hash)
if video is None:
return []
return _parse_translation_glossary(video.get("translation_glossary"))
def save_translation_glossary(
self,
video_hash: str,
terms: Sequence[Any],
) -> bool:
normalized_hash = normalize_video_hash(video_hash)
payload = json.dumps(
[
{"source": source, "zh": zh}
for source, zh in _normalize_translation_glossary(terms)
],
ensure_ascii=False,
)
with self._connect() as connection:
cursor = connection.execute(
"""
UPDATE videos
SET translation_glossary = ?, updated_at = ?
WHERE video_hash = ?
""",
(payload, utc_now(), normalized_hash),
)
for row in result_rows
]
return cursor.rowcount > 0
def clear_sentence_translations(
self,
video_hash: str,
indexes: Optional[Sequence[int]] = None,
) -> None:
normalized_hash = normalize_video_hash(video_hash)
with self._connect() as connection:
if indexes is None:
connection.execute(
"UPDATE sentences SET zh_text = NULL WHERE video_hash = ?",
(normalized_hash,),
)
return
unique_indexes = list(dict.fromkeys(indexes))
if not unique_indexes:
return
connection.executemany(
"""
UPDATE sentences SET zh_text = NULL
WHERE video_hash = ? AND sentence_index = ?
""",
[(normalized_hash, index) for index in unique_indexes],
)
def save_sentence_translations(
self,
video_hash: str,
translations: Dict[int, str],
) -> Optional[SentenceBoundaryDocument]:
normalized_hash = normalize_video_hash(video_hash)
with self._connect() as connection:
for index, zh_text in translations.items():
connection.execute(
"""
UPDATE sentences
SET zh_text = ?
WHERE video_hash = ? AND sentence_index = ?
""",
(zh_text.strip(), normalized_hash, index),
)
connection.execute(
"UPDATE videos SET updated_at = ? WHERE video_hash = ?",
(utc_now(), normalized_hash),
)
return self.get_document(normalized_hash)
def create_dub_share(
self,
@@ -1187,6 +1465,28 @@ class VideoRepository:
return video
def _english_fingerprint(document: SentenceBoundaryDocument) -> tuple:
return tuple(
(sentence.index, sentence.start_ms, sentence.end_ms, (sentence.text or "").strip())
for sentence in document.sentences
)
def _sentence_from_row(row: sqlite3.Row) -> SentenceBoundary:
keys = set(row.keys())
zh_text = row["zh_text"] if "zh_text" in keys else None
return SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
zh_text=zh_text,
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
def _join_adjacent_sentence_text(
left: str,
right: str,
@@ -1221,3 +1521,62 @@ def _is_cjk_char(char: str) -> bool:
or 0xF900 <= code <= 0xFAFF
or 0x20000 <= code <= 0x2CEAF
)
def _moss_segment_row(video_hash: str, index: int, segment: Dict[str, Any]) -> tuple:
start_seconds = float(segment["start_seconds"])
end_seconds = float(segment["end_seconds"])
words = segment.get("words")
words_json = json.dumps(words, ensure_ascii=False) if words else None
speaker = segment.get("speaker")
return (
video_hash,
index,
start_seconds,
end_seconds,
int(round(start_seconds * 1000)),
int(round(end_seconds * 1000)),
str(segment.get("text") or "").strip(),
str(speaker) if speaker is not None else None,
words_json,
)
def _normalize_translation_glossary(terms: Sequence[Any]) -> List[Tuple[str, str]]:
cleaned: List[Tuple[str, str]] = []
seen: Dict[str, int] = {}
for item in terms or []:
source = ""
zh = ""
if isinstance(item, dict):
source = str(item.get("source") or "")
zh = str(item.get("zh") or "")
elif isinstance(item, (tuple, list)) and len(item) >= 2:
source = str(item[0] or "")
zh = str(item[1] or "")
source = source.strip()[:120]
zh = zh.strip()[:120]
if not source or not zh:
continue
key = source.casefold()
if key in seen:
cleaned[seen[key]] = (source, zh)
continue
seen[key] = len(cleaned)
cleaned.append((source, zh))
if len(cleaned) >= 50:
break
return cleaned
def _parse_translation_glossary(raw: Any) -> List[Tuple[str, str]]:
if not raw:
return []
if isinstance(raw, str):
try:
raw = json.loads(raw)
except json.JSONDecodeError:
return []
if not isinstance(raw, list):
return []
return _normalize_translation_glossary(raw)

View File

@@ -81,7 +81,7 @@ table { width: 100%; border-collapse: collapse; font-size: 13px; }
th, td { padding: 10px 12px; text-align: left; border-bottom: 1px solid #e3e7ea; vertical-align: middle; }
th { position: sticky; top: 0; z-index: 1; color: #44515b; background: #edf1f3; font-weight: 600; }
tr:last-child td { border-bottom: 0; }
.actions-column { min-width: 280px; }
.actions-column { min-width: 340px; }
.actions { display: flex; gap: 6px; white-space: nowrap; }
.actions button, .actions a { min-height: 30px; padding: 5px 9px; font-size: 12px; }
.actions a { display: inline-flex; align-items: center; border: 1px solid #aab4bc; border-radius: 4px; color: #26323c; text-decoration: none; }
@@ -103,6 +103,9 @@ dialog {
dialog::backdrop { background: rgb(20 25 28 / 55%); }
.dialog-heading { padding: 16px 18px; display: flex; justify-content: space-between; align-items: center; border-bottom: 1px solid #dce1e5; }
.dialog-heading span { font-size: 12px; color: #68747d; }
.dialog-heading-actions { display: flex; align-items: center; gap: 8px; }
.gap-row td { background: #fff8ee; color: #9a5b12; font-style: italic; }
.missing-zh-row .zh-text { color: #a45d08; }
.sentence-scroll { max-height: calc(100vh - 120px); overflow: auto; }
.sentence-scroll textarea { width: min(600px, 45vw); min-width: 260px; min-height: 58px; resize: vertical; padding: 7px; border: 1px solid #b8c0c7; border-radius: 3px; }
.sentence-scroll select { min-width: 90px; }
@@ -119,3 +122,11 @@ dialog::backdrop { background: rgb(20 25 28 / 55%); }
#upload-form { grid-template-columns: 1fr; }
.actions-column { min-width: 250px; }
}
.zh-text { margin-top: 6px; color: #66727d; font-size: 12px; white-space: pre-wrap; line-height: 1.45; }
.retranslate-form { padding: 16px 18px 20px; display: grid; gap: 12px; }
.retranslate-form p { margin: 0; font-size: 13px; color: #4f5b64; line-height: 1.55; }
.retranslate-form label { display: grid; gap: 6px; font-size: 13px; color: #4f5b64; }
.retranslate-form textarea { width: 100%; min-height: 160px; resize: vertical; }
.retranslate-form code { font-size: 12px; }
.retranslate-actions { display: flex; justify-content: flex-end; }

View File

@@ -96,13 +96,38 @@
</section>
</main>
<dialog id="retranslate-dialog">
<div class="dialog-heading">
<div>
<h2 id="retranslate-title">重新翻译中文字幕</h2>
<span id="retranslate-meta"></span>
</div>
<button id="close-retranslate-dialog" class="icon-button" type="button" aria-label="关闭">X</button>
</div>
<form id="retranslate-form" class="retranslate-form">
<p>会清空当前中文字幕并重新翻译。可指定专有名词译法,每行一条,格式为 <code>原文=中文</code>。例如 <code>Harimoto=哈瑞么托</code>,也可以把错误译名写在左边:<code>哈达马托=哈瑞么托</code>。</p>
<label>
<span>专有名词译法</span>
<textarea id="retranslate-terms" placeholder="Harimoto=哈瑞么托"></textarea>
</label>
<div class="retranslate-actions">
<button id="repair-translations" type="button">只补译缺失</button>
<button class="primary" type="submit">开始重新翻译</button>
</div>
</form>
</dialog>
<dialog id="sentence-dialog">
<div class="dialog-heading">
<div>
<h2 id="sentence-title">句子文本</h2>
<span id="sentence-meta"></span>
</div>
<button id="close-dialog" class="icon-button" type="button" aria-label="关闭">X</button>
<div class="dialog-heading-actions">
<button id="repair-gaps" type="button">修复空白</button>
<button id="repair-missing-zh" type="button">补译缺失</button>
<button id="close-dialog" class="icon-button" type="button" aria-label="关闭" title="关闭时批量处理暂存修改">X</button>
</div>
</div>
<div class="sentence-scroll">
<table>
@@ -114,6 +139,6 @@
</div>
</dialog>
<script src="/static/admin.js?v=admin-merge-sentence-v1" defer></script>
<script src="/static/admin.js?v=admin-sentence-batch-v5" defer></script>
</body>
</html>

View File

@@ -1,7 +1,17 @@
const GAP_REPAIR_MIN_MS = 8000;
const CJK_RE = /[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/;
const state = {
adminKey: localStorage.getItem("oralTrainerAdminKey") || "",
videos: [],
users: [],
retranslateVideo: null,
sentenceVideo: null,
sentenceDraft: null,
sentenceBaseMeta: "",
sentencePending: false,
sentenceSaving: false,
sentenceNeedsProcessing: false,
};
const elements = {
@@ -22,6 +32,15 @@ const elements = {
sentenceMeta: document.querySelector("#sentence-meta"),
sentenceRows: document.querySelector("#sentence-rows"),
closeDialog: document.querySelector("#close-dialog"),
repairGaps: document.querySelector("#repair-gaps"),
repairMissingZh: document.querySelector("#repair-missing-zh"),
retranslateDialog: document.querySelector("#retranslate-dialog"),
retranslateTitle: document.querySelector("#retranslate-title"),
retranslateMeta: document.querySelector("#retranslate-meta"),
retranslateForm: document.querySelector("#retranslate-form"),
retranslateTerms: document.querySelector("#retranslate-terms"),
repairTranslations: document.querySelector("#repair-translations"),
closeRetranslateDialog: document.querySelector("#close-retranslate-dialog"),
userRows: document.querySelector("#user-rows"),
userCount: document.querySelector("#user-count"),
userEmptyState: document.querySelector("#user-empty-state"),
@@ -38,6 +57,26 @@ elements.refresh.addEventListener("click", () => {
loadUsers();
});
elements.closeDialog.addEventListener("click", () => elements.dialog.close());
elements.dialog.addEventListener("close", handleSentenceDialogClose);
elements.repairGaps.addEventListener("click", () => {
if (state.sentencePending) {
elements.serviceState.textContent = "句子修改已暂存,请先关闭对话框完成处理";
return;
}
if (state.sentenceVideo) repairGaps(state.sentenceVideo, { reloadSentences: true });
});
elements.repairMissingZh.addEventListener("click", () => {
if (state.sentencePending) {
elements.serviceState.textContent = "句子修改已暂存,请先关闭对话框完成处理";
return;
}
if (state.sentenceVideo) repairMissingTranslations(state.sentenceVideo, { reloadSentences: true });
});
elements.closeRetranslateDialog.addEventListener("click", () => elements.retranslateDialog.close());
elements.repairTranslations.addEventListener("click", () => {
if (state.retranslateVideo) repairMissingTranslations(state.retranslateVideo, { closeRetranslate: true });
});
elements.retranslateForm.addEventListener("submit", submitRetranslate);
elements.uploadForm.addEventListener("submit", uploadVideo);
elements.videoFile.addEventListener("change", () => {
if (!elements.videoFile.files[0]) return;
@@ -181,7 +220,10 @@ function actionCell(video) {
play.rel = "noopener";
play.textContent = "播放";
actions.append(play);
actions.append(button("改标题", () => renameVideo(video)));
actions.append(button("句子", () => openSentences(video)));
actions.append(button("修复空白", () => repairGaps(video)));
actions.append(button("重新翻译", () => openRetranslate(video)));
actions.append(button("重新处理", () => reprocess(video.video_hash)));
const remove = button("删除", () => deleteVideo(video));
remove.className = "danger";
@@ -198,6 +240,28 @@ function button(label, handler) {
return result;
}
async function renameVideo(video) {
const title = window.prompt("请输入新的课程标题", video.title || "");
if (title === null) return;
const cleanedTitle = title.trim();
if (!cleanedTitle) {
elements.serviceState.textContent = "课程标题不能为空";
return;
}
if (cleanedTitle === video.title) return;
try {
await api(`/api/v1/admin/videos/${video.video_hash}/title`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ title: cleanedTitle }),
});
elements.serviceState.textContent = "课程标题已更新";
await loadVideos();
} catch (error) {
elements.serviceState.textContent = `标题修改失败: ${error.message}`;
}
}
function uploadVideo(event) {
event.preventDefault();
const file = document.querySelector("#video-file").files[0];
@@ -247,25 +311,306 @@ function readXhrError(request) {
}
async function openSentences(video) {
if (state.sentenceSaving) {
elements.serviceState.textContent = "句子批量处理中,请稍后再打开";
return;
}
if (
state.sentenceVideo?.video_hash === video.video_hash &&
Array.isArray(state.sentenceDraft)
) {
renderSentenceDraft(video);
if (!elements.dialog.open) elements.dialog.showModal();
return;
}
try {
const detail = await api(`/api/v1/videos/${video.video_hash}`);
state.sentenceVideo = video;
elements.sentenceTitle.textContent = detail.video.title;
const sentences = detail.boundaries?.sentences || [];
elements.sentenceMeta.textContent = `${sentences.length} 句 / ${detail.video.video_hash}`;
elements.sentenceRows.replaceChildren();
for (const sentence of sentences) {
elements.sentenceRows.append(sentenceRow(video, sentence, sentences.length));
}
state.sentenceDraft = (detail.boundaries?.sentences || []).map((sentence) => ({
...sentence,
originStartMs: sentence.start_ms,
originEndMs: sentence.end_ms,
sourceStartMs: undefined,
sourceEndMs: undefined,
}));
state.sentencePending = false;
state.sentenceSaving = false;
state.sentenceNeedsProcessing = false;
const durationMs = detail.video.duration_ms || 0;
const sentences = state.sentenceDraft;
const gaps = collectSentenceGaps(sentences, durationMs);
const missingZh = sentences.filter(translationNeedsRepair).length;
const gapLabel = gaps.length ? ` / ${gaps.length} 段空白待修复` : "";
const zhLabel = missingZh ? ` / ${missingZh} 句未译` : "";
state.sentenceBaseMeta = `${sentences.length} 句${gapLabel}${zhLabel} / ${detail.video.video_hash}`;
renderSentenceDraft(video);
if (!elements.dialog.open) elements.dialog.showModal();
} catch (error) {
elements.serviceState.textContent = `句子加载失败: ${error.message}`;
}
}
function renderSentenceDraft(video) {
const sentences = state.sentenceDraft || [];
const durationMs = state.sentenceVideo?.duration_ms || 0;
const gaps = collectSentenceGaps(sentences, durationMs);
const missingZh = sentences.filter(translationNeedsRepair).length;
elements.sentenceMeta.textContent = [
state.sentenceBaseMeta,
state.sentencePending ? "有暂存修改" : "",
state.sentenceNeedsProcessing ? "已入库待处理" : "",
state.sentenceSaving ? "正在处理" : "",
].filter(Boolean).join(" / ");
elements.repairGaps.disabled = gaps.length === 0 || state.sentencePending;
elements.repairMissingZh.disabled = missingZh === 0 || state.sentencePending;
elements.closeDialog.disabled = state.sentenceSaving;
elements.closeDialog.textContent = state.sentenceSaving ? "..." : "X";
elements.sentenceRows.replaceChildren();
let gapIndex = 0;
for (const sentence of sentences) {
if (gapIndex < gaps.length && gaps[gapIndex].end_ms === sentence.start_ms) {
elements.sentenceRows.append(gapRow(gaps[gapIndex]));
gapIndex += 1;
}
elements.sentenceRows.append(sentenceRow(video, sentence, sentences.length));
}
if (gapIndex < gaps.length) {
elements.sentenceRows.append(gapRow(gaps[gapIndex]));
}
}
async function handleSentenceDialogClose() {
if (!state.sentenceDraft || state.sentenceSaving) return;
if (!state.sentencePending && !state.sentenceNeedsProcessing) {
resetSentenceDraft();
return;
}
if (state.sentencePending) {
const persisted = await saveSentenceDraft({ processTimings: false });
if (!persisted) {
return;
}
}
const processed = await saveSentenceDraft({ processTimings: true });
if (!processed) {
return;
}
resetSentenceDraft();
loadVideos();
}
async function saveSentenceDraft({ processTimings = false } = {}) {
const video = state.sentenceVideo;
if (!video) return false;
if (!processTimings && !state.sentencePending) return true;
if (processTimings && !state.sentencePending && !state.sentenceNeedsProcessing) return true;
materializeSentenceDraftSplits(video);
state.sentenceSaving = true;
renderSentenceDraft(video);
try {
const body = {
process_timings: processTimings,
sentences: state.sentenceDraft.map(({
index,
originStartMs,
originEndMs,
sourceStartMs,
sourceEndMs,
...sentence
}) => ({
...sentence,
...(sourceStartMs !== undefined && sourceEndMs !== undefined ? {
source_start_ms: sourceStartMs,
source_end_ms: sourceEndMs,
} : {}),
}),
),
};
const saved = await api(
`/api/v1/admin/videos/${video.video_hash}/sentences/batch`,
{
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
},
);
const previousByRange = new Map(
state.sentenceDraft.map((sentence) => [`${sentence.start_ms}:${sentence.end_ms}`, sentence]),
);
state.sentenceDraft = saved.map((sentence) => {
const previous = previousByRange.get(`${sentence.start_ms}:${sentence.end_ms}`);
return {
...sentence,
originStartMs: previous?.originStartMs ?? sentence.start_ms,
originEndMs: previous?.originEndMs ?? sentence.end_ms,
sourceStartMs: previous?.sourceStartMs,
sourceEndMs: previous?.sourceEndMs,
};
});
state.sentencePending = false;
state.sentenceNeedsProcessing = !processTimings;
return true;
} catch (error) {
elements.serviceState.textContent = processTimings
? `句子批量处理失败: ${error.message}`
: `句子入库失败: ${error.message}`;
return false;
} finally {
state.sentenceSaving = false;
if (state.sentenceDraft) renderSentenceDraft(video);
}
}
function materializeSentenceDraftSplits(video) {
for (const sentence of [...(state.sentenceDraft || [])]) {
if (/\r?\n/.test(sentence.text || "")) {
splitDraftSentence(video, sentence, sentence.text, sentence.language);
}
}
}
function resetSentenceDraft() {
state.sentenceDraft = null;
state.sentencePending = false;
state.sentenceSaving = false;
state.sentenceNeedsProcessing = false;
state.sentenceVideo = null;
state.sentenceBaseMeta = "";
}
function markSentenceDraftChanged(video) {
state.sentencePending = true;
renderSentenceDraft(video);
}
function markSentenceDraftPending() {
state.sentencePending = true;
elements.sentenceMeta.textContent = [
state.sentenceBaseMeta,
"有暂存修改",
].filter(Boolean).join(" / ");
elements.repairGaps.disabled = true;
elements.repairMissingZh.disabled = true;
}
function joinDraftSentenceText(left, right, language) {
const leftText = (left || "").trim();
const rightText = (right || "").trim();
if (!leftText) return rightText;
if (!rightText) return leftText;
const lang = (language || "").toLowerCase();
if (lang.startsWith("zh") || lang.startsWith("ja") || lang.startsWith("ko")) {
return leftText + rightText;
}
const leftEnd = leftText.slice(-1);
const rightStart = rightText.slice(0, 1);
const cjk = /[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/;
if (cjk.test(leftEnd) || cjk.test(rightStart)) return leftText + rightText;
if (rightStart && ",.;:!?)]}、,。!?;:”’》〉】".includes(rightStart)) {
return leftText + rightText;
}
return `${leftText} ${rightText}`;
}
function splitDraftSentence(video, sentence, text, language) {
const lines = text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
const duration = Math.max(1, sentence.end_ms - sentence.start_ms);
const weights = lines.map((line) => Math.max(1, line.replace(/\s+/g, "").length));
const totalWeight = Math.max(1, weights.reduce((total, weight) => total + weight, 0));
const sourceStart = sentence.sourceStartMs ?? sentence.originStartMs ?? sentence.start_ms;
const sourceEnd = sentence.sourceEndMs ?? sentence.originEndMs ?? sentence.end_ms;
const replacements = [];
let cursor = sentence.start_ms;
let remaining = lines.length;
let weightSum = 0;
for (const [lineIndex, line] of lines.entries()) {
const isLast = lineIndex === lines.length - 1;
weightSum += weights[lineIndex];
let end = sentence.end_ms;
if (!isLast) {
end = sentence.start_ms + Math.round(duration * weightSum / totalWeight);
end = Math.min(Math.max(end, cursor + 1), sentence.end_ms - (remaining - 1));
}
const sourceDuration = Math.max(
1,
(sentence.reference_speech_duration_ms || duration) * weights[lineIndex] / totalWeight,
);
replacements.push({
index: sentence.index + lineIndex,
start_ms: cursor,
end_ms: end,
text: line,
language: language || sentence.language,
zh_text: null,
reference_speech_duration_ms: Math.max(1, Math.round(sourceDuration)),
sourceStartMs: sourceStart,
sourceEndMs: sourceEnd,
originStartMs: sourceStart,
originEndMs: sourceEnd,
});
cursor = end;
remaining -= 1;
}
state.sentenceDraft.splice(sentence.index, 1, ...replacements);
renumberSentenceDraft();
markSentenceDraftChanged(video);
}
function renumberSentenceDraft() {
(state.sentenceDraft || []).forEach((sentence, index) => {
sentence.index = index;
});
}
function collectSentenceGaps(sentences, durationMs, minGapMs = GAP_REPAIR_MIN_MS) {
const gaps = [];
let cursor = 0;
for (const sentence of sentences) {
if (sentence.start_ms - cursor >= minGapMs) {
gaps.push({ start_ms: cursor, end_ms: sentence.start_ms });
}
cursor = Math.max(cursor, sentence.end_ms);
}
const endMs = durationMs || cursor;
if (endMs - cursor >= minGapMs) {
gaps.push({ start_ms: cursor, end_ms: endMs });
}
return gaps;
}
function gapRow(gap) {
const row = document.createElement("tr");
row.className = "gap-row";
row.append(
textCell(""),
textCell(`${formatDuration(gap.start_ms)} — ${formatDuration(gap.end_ms)}`),
textCell(`未识别空白 ${formatDuration(gap.end_ms - gap.start_ms)}`),
textCell(""),
textCell(""),
);
return row;
}
function translationNeedsRepair(sentence) {
const source = (sentence.text || "").trim();
const zh = (sentence.zh_text || "").trim();
if (!source) return false;
if (!zh) return true;
if (CJK_RE.test(zh)) return false;
return !CJK_RE.test(source);
}
function sentenceRow(video, sentence, totalCount) {
const row = document.createElement("tr");
if (translationNeedsRepair(sentence)) row.className = "missing-zh-row";
const textArea = document.createElement("textarea");
textArea.value = sentence.text || "";
textArea.addEventListener("input", () => {
sentence.text = textArea.value;
sentence.zh_text = null;
markSentenceDraftPending();
});
const language = document.createElement("select");
for (const [value, label] of [["", "自动"], ["en", "英语"], ["zh", "中文"]]) {
const option = document.createElement("option");
@@ -274,55 +619,77 @@ function sentenceRow(video, sentence, totalCount) {
option.selected = value === (sentence.language || "");
language.append(option);
}
language.addEventListener("change", () => {
sentence.language = language.value || null;
markSentenceDraftPending();
});
const textContainer = document.createElement("td");
textContainer.append(textArea);
const zhNote = document.createElement("div");
zhNote.className = "zh-text";
zhNote.textContent = translationNeedsRepair(sentence)
? (sentence.zh_text ? `仍是英文:${sentence.zh_text}` : "暂无中文翻译")
: sentence.zh_text;
textContainer.append(textArea, zhNote);
const languageContainer = document.createElement("td");
languageContainer.append(language);
const saveContainer = document.createElement("td");
const save = button("保存", async () => {
save.disabled = true;
save.textContent = "保存中";
try {
const body = JSON.stringify({ text: textArea.value, language: language.value || null });
if (/\r?\n/.test(textArea.value)) {
await api(`/api/v1/admin/videos/${video.video_hash}/sentences/${sentence.index}/split`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body,
});
save.textContent = "已拆分";
await openSentences(video);
if (/\r?\n/.test(sentence.text || "")) {
splitDraftSentence(video, sentence, sentence.text, language.value || null);
} else {
await api(`/api/v1/admin/videos/${video.video_hash}/sentences/${sentence.index}`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body,
});
save.textContent = "已保存";
sentence.text = textArea.value;
sentence.language = language.value || null;
sentence.zh_text = null;
markSentenceDraftChanged(video);
}
} catch (error) {
elements.serviceState.textContent = `句子保存失败: ${error.message}`;
save.textContent = error.message;
await saveSentenceDraft({ processTimings: false });
} finally {
window.setTimeout(() => { save.disabled = false; save.textContent = "保存"; }, 1600);
save.disabled = false;
save.textContent = "保存";
}
});
saveContainer.append(save);
if (sentence.index < totalCount - 1) {
const merge = button("合并下句", async () => {
const preview = `${(textArea.value || sentence.text || "").trim()} + 下一句`;
const preview = `${(sentence.text || "").trim()} + 下一句`;
const confirmed = window.confirm(`确定将第 ${sentence.index + 1} 句与下一句合并成一句吗?\n${preview}`);
if (!confirmed) return;
const sourceStart = sentence.start_ms;
const sourceEnd = sentence.end_ms;
merge.disabled = true;
merge.textContent = "入库中";
try {
await api(`/api/v1/admin/videos/${video.video_hash}/sentences/${sentence.index}/merge`, {
method: "PUT",
if (state.sentencePending) {
const persisted = await saveSentenceDraft({ processTimings: false });
if (!persisted) return;
}
const targetIndex = state.sentenceDraft.findIndex(
(item) => item.start_ms === sourceStart && item.end_ms === sourceEnd,
);
if (targetIndex < 0) throw new Error("找不到要合并的句子");
const merged = await api(
`/api/v1/admin/videos/${video.video_hash}/sentences/${targetIndex}/merge`,
{ method: "PUT" },
);
state.sentenceDraft.splice(targetIndex, 2, {
...merged,
originStartMs: merged.start_ms,
originEndMs: merged.end_ms,
sourceStartMs: undefined,
sourceEndMs: undefined,
});
merge.textContent = "已合并";
await openSentences(video);
renumberSentenceDraft();
state.sentencePending = false;
state.sentenceNeedsProcessing = true;
} catch (error) {
elements.serviceState.textContent = `句子合并失败: ${error.message}`;
merge.textContent = error.message;
window.setTimeout(() => { merge.disabled = false; merge.textContent = "合并下句"; }, 1600);
} finally {
merge.disabled = false;
merge.textContent = "合并下句";
renderSentenceDraft(video);
}
});
saveContainer.append(merge);
@@ -350,41 +717,141 @@ function boundaryControl(video, sentence) {
const status = document.createElement("span");
status.className = "boundary-status";
async function adjust(deltaMs, changedButton) {
const buttons = [minus, plus];
buttons.forEach((item) => { item.disabled = true; });
status.textContent = "";
try {
await api(
`/api/v1/admin/videos/${video.video_hash}/sentences/${sentence.index}/boundary`,
{
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ delta_ms: deltaMs }),
},
);
await openSentences(video);
} catch (error) {
elements.serviceState.textContent = `分割点调整失败: ${error.message}`;
status.textContent = "失败";
buttons.forEach((item) => { item.disabled = false; });
window.setTimeout(() => { status.textContent = ""; }, 1800);
}
if (changedButton) {
window.setTimeout(() => {
changedButton.textContent = deltaMs < 0 ? "−0.1" : "+0.1";
}, 900);
async function adjust(deltaMs) {
const next = state.sentenceDraft[sentence.index + 1];
if (!next) return;
const nextBoundary = sentence.end_ms + deltaMs;
if (
nextBoundary <= sentence.start_ms ||
nextBoundary >= next.end_ms
) {
elements.serviceState.textContent = "分割点必须保留在两句时间范围内";
return;
}
sentence.end_ms = nextBoundary;
next.start_ms = nextBoundary;
sentence.sourceStartMs = undefined;
sentence.sourceEndMs = undefined;
next.sourceStartMs = undefined;
next.sourceEndMs = undefined;
markSentenceDraftChanged(video);
await saveSentenceDraft({ processTimings: false });
}
const minus = button("−0.1", (event) => adjust(event.shiftKey ? -10 : -100, minus));
const plus = button("+0.1", (event) => adjust(event.shiftKey ? 10 : 100, plus));
const minus = button("−0.1", (event) => adjust(event.shiftKey ? -10 : -100));
const plus = button("+0.1", (event) => adjust(event.shiftKey ? 10 : 100));
minus.title = "结束点提前 0.1 秒;按住 Shift 微调 0.01 秒";
plus.title = "结束点延后 0.1 秒;按住 Shift 微调 0.01 秒";
container.append(minus, label, plus, status);
return container;
}
async function openRetranslate(video) {
state.retranslateVideo = video;
elements.retranslateTitle.textContent = `重新翻译:${video.title}`;
elements.retranslateMeta.textContent = video.video_hash;
elements.retranslateTerms.value = "";
try {
const payload = await api(`/api/v1/admin/videos/${video.video_hash}/subtitle-translation`);
elements.retranslateTerms.value = formatTranslationTerms(payload.terms || []);
const missing = payload.missing_count || 0;
const status = missing
? `${missing} 句未译`
: (payload.complete ? "中文字幕已就绪" : (payload.status === "running" ? "正在翻译" : "尚未翻译完成"));
elements.retranslateMeta.textContent = `${status} / ${video.video_hash}`;
elements.repairTranslations.disabled = missing === 0;
} catch (error) {
elements.serviceState.textContent = `加载翻译信息失败: ${error.message}`;
}
if (!elements.retranslateDialog.open) elements.retranslateDialog.showModal();
}
function formatTranslationTerms(terms) {
return (terms || []).map((item) => `${item.source}=${item.zh}`).join("\n");
}
function parseTranslationTerms(text) {
const terms = [];
for (const rawLine of (text || "").split(/\r?\n/)) {
const line = rawLine.trim();
if (!line || line.startsWith("#")) continue;
const match = line.match(/^(.+?)\s*(?:=|=>|:|:)\s*(.+)$/);
if (!match) {
throw new Error(`无法解析专有名词:${line}`);
}
const source = match[1].trim();
const zh = match[2].trim();
if (!source || !zh) {
throw new Error(`无法解析专有名词:${line}`);
}
terms.push({ source, zh });
}
return terms;
}
async function submitRetranslate(event) {
event.preventDefault();
const video = state.retranslateVideo;
if (!video) return;
let terms;
try {
terms = parseTranslationTerms(elements.retranslateTerms.value);
} catch (error) {
elements.serviceState.textContent = error.message;
return;
}
const submit = elements.retranslateForm.querySelector("button[type='submit']");
submit.disabled = true;
try {
const payload = await api(`/api/v1/admin/videos/${video.video_hash}/subtitle-translation`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ terms }),
});
const status = payload.complete ? "中文字幕已重新翻译" : "已开始重新翻译";
elements.serviceState.textContent = terms.length ? `${status},已应用 ${terms.length} 条专有名词` : status;
elements.retranslateDialog.close();
await loadVideos();
} catch (error) {
elements.serviceState.textContent = `重新翻译失败: ${error.message}`;
} finally {
submit.disabled = false;
}
}
async function repairMissingTranslations(video, options = {}) {
const confirmed = window.confirm(
`将只补译“${video.title}”里缺失或仍是英文的中文,已有中文会保留。继续?`,
);
if (!confirmed) return;
const buttons = [elements.repairMissingZh, elements.repairTranslations].filter(Boolean);
buttons.forEach((button) => { button.disabled = true; });
elements.serviceState.textContent = "正在补译缺失的中文…";
try {
const payload = await api(`/api/v1/admin/videos/${video.video_hash}/subtitle-translation/repair`, {
method: "POST",
});
const attempted = payload.attempted_count || 0;
if (!attempted) {
elements.serviceState.textContent = "没有需要补译的句子";
} else if (payload.complete) {
elements.serviceState.textContent = `补译完成:处理了 ${attempted} 句`;
} else {
const remaining = payload.missing_count ? `,还剩 ${payload.missing_count} 句` : "";
elements.serviceState.textContent = `已开始补译 ${attempted} 句${remaining},请稍后刷新`;
}
if (options.closeRetranslate) elements.retranslateDialog.close();
await loadVideos();
if (options.reloadSentences) await openSentences(video);
} catch (error) {
elements.serviceState.textContent = `补译失败: ${error.message}`;
} finally {
if (!options.reloadSentences) {
buttons.forEach((button) => { button.disabled = false; });
}
}
}
async function reprocess(videoHash) {
try {
await api(`/api/v1/admin/videos/${videoHash}/process`, { method: "POST" });
@@ -394,6 +861,36 @@ async function reprocess(videoHash) {
}
}
async function repairGaps(video, options = {}) {
const confirmed = window.confirm(
`将对“${video.title}”中间隔超过 8 秒的空白重新识别。现有句子会保留,可能需要几分钟。继续?`,
);
if (!confirmed) return;
const button = elements.repairGaps;
if (button) button.disabled = true;
elements.serviceState.textContent = "正在重新识别空白段…";
try {
const payload = await api(`/api/v1/admin/videos/${video.video_hash}/repair-gaps`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ min_gap_ms: GAP_REPAIR_MIN_MS }),
});
if (payload.inserted_count) {
elements.serviceState.textContent = `空白修复完成:补了 ${payload.inserted_count} 句(${payload.transcribed_gaps}/${payload.gap_count} 段)`;
} else if (payload.gap_count) {
elements.serviceState.textContent = `发现 ${payload.gap_count} 段空白,但没有识别到新句子`;
} else {
elements.serviceState.textContent = "没有发现需要修复的空白段";
}
await loadVideos();
if (options.reloadSentences) await openSentences(video);
} catch (error) {
elements.serviceState.textContent = `空白修复失败: ${error.message}`;
} finally {
if (button && !options.reloadSentences) button.disabled = false;
}
}
async function deleteVideo(video) {
if (!window.confirm(`删除“${video.title}”?`)) return;
try {

View File

@@ -1,6 +1,11 @@
import pytest
from sentence_api.ai_tutor import AITutor, AITutorError
import json
import logging
import re
import threading
from sentence_api.ai_tutor import AITutor, AITutorError, _apply_translation_terms, _parse_translation_payload
class FakeResponse:
@@ -71,3 +76,238 @@ def test_ai_tutor_rejects_unexpected_response(monkeypatch):
with pytest.raises(AITutorError):
tutor.explain("Hello.")
def test_ai_tutor_translates_full_document(monkeypatch):
requests_payloads = []
def fake_post(endpoint, headers, json, timeout):
requests_payloads.append((endpoint, headers, json, timeout))
return FakeResponse(body={"choices": [{"message": {"content": '{"0":"你好。","1":"见到你很高兴。"}'}}]})
monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post)
tutor = AITutor("https://ai.test/v1", "secret", "test-model", 1.5)
translated = tutor.translate_sentences(
[(0, "Hello."), (1, "Nice to meet you.")],
course_title="Greetings",
)
assert translated == {0: "你好。", 1: "见到你很高兴。"}
_endpoint, _headers, payload, timeout = requests_payloads[0]
user_prompt = payload["messages"][1]["content"]
assert "[0] Hello." in user_prompt
assert "[1] Nice to meet you." in user_prompt
assert timeout == 120.0
def test_ai_tutor_translation_timeout_can_be_configured(monkeypatch):
request_payloads = []
def fake_post(endpoint, headers, json, timeout):
request_payloads.append((json, timeout))
return _translation_response_from_prompt(json)
monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post)
tutor = AITutor(
"https://ai.test/v1",
"secret",
"test-model",
1,
translation_timeout_seconds=180,
)
translated = tutor.translate_sentences([(0, "Hello.")])
assert translated == {0: "中文0"}
assert request_payloads[0][1] == 180
def _translation_response_from_prompt(payload):
user_prompt = payload["messages"][1]["content"]
section = user_prompt.split("待翻译句子:", 1)[1]
indexes = [int(match) for match in re.findall(r"^\[(\d+)\]", section, re.M)]
body = {str(index): f"中文{index}" for index in indexes}
return FakeResponse(
body={"choices": [{"message": {"content": json.dumps(body, ensure_ascii=False)}}]}
)
def test_ai_tutor_translates_long_input_in_batches(monkeypatch):
requests_payloads = []
lock = threading.Lock()
def fake_post(endpoint, headers, json, timeout):
with lock:
requests_payloads.append(json)
return _translation_response_from_prompt(json)
monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post)
tutor = AITutor("https://ai.test/v1", "secret", "test-model", 1)
sentences = [(index, f"This is sentence {index} with enough text.") for index in range(30)]
translated = tutor.translate_sentences(sentences, course_title="Long talk")
assert translated == {index: f"中文{index}" for index in range(30)}
assert len(requests_payloads) >= 3
first_prompt = requests_payloads[0]["messages"][1]["content"]
assert "前文" in first_prompt or "后文" in first_prompt or "待翻译句子:" in first_prompt
def test_ai_tutor_skips_cached_sentences_and_persists_windows(monkeypatch):
persisted = []
def fake_post(endpoint, headers, json, timeout):
return _translation_response_from_prompt(json)
monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post)
tutor = AITutor("https://ai.test/v1", "secret", "test-model", 1)
translated = tutor.translate_sentences(
[(0, "Hello."), (1, "Thanks."), (2, "Bye.")],
existing={0: "你好。"},
persist=persisted.append,
course_title="Greetings",
)
assert translated[0] == "你好。"
assert translated[1] == "中文1"
assert translated[2] == "中文2"
assert persisted
assert 0 not in persisted[0]
def test_ai_tutor_splits_invalid_windows_and_retries(monkeypatch):
calls = {"count": 0}
def fake_post(endpoint, headers, json, timeout):
calls["count"] += 1
if calls["count"] == 1:
return FakeResponse(body={"choices": [{"message": {"content": "not-json"}}]})
return _translation_response_from_prompt(json)
monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post)
tutor = AITutor("https://ai.test/v1", "secret", "test-model", 1)
sentences = [(index, f"This is sentence {index} with enough text.") for index in range(12)]
translated = tutor.translate_sentences(sentences)
assert translated == {index: f"中文{index}" for index in range(12)}
assert calls["count"] >= 3
def test_ai_tutor_includes_name_terms_and_rewrites_output(monkeypatch):
requests_payloads = []
def fake_post(endpoint, headers, json, timeout):
requests_payloads.append(json)
return FakeResponse(
body={"choices": [{"message": {"content": '{"0":"哈达马托击败了 Harimoto。"}'}}]}
)
monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post)
tutor = AITutor("https://ai.test/v1", "secret", "test-model", 1)
translated = tutor.translate_sentences(
[(0, "Harimoto won.")],
course_title="Table tennis",
terms=[("Harimoto", "哈瑞么托"), ("哈达马托", "哈瑞么托")],
)
assert translated == {0: "哈瑞么托击败了 哈瑞么托。"}
user_prompt = requests_payloads[0]["messages"][1]["content"]
assert "Harimoto => 哈瑞么托" in user_prompt
assert "哈达马托 => 哈瑞么托" in user_prompt
def test_apply_translation_terms_replaces_english_and_wrong_chinese():
assert _apply_translation_terms(
"Harimoto won.",
"哈达马托击败了 Harimoto。",
[("Harimoto", "哈瑞么托"), ("哈达马托", "哈瑞么托")],
) == "哈瑞么托击败了 哈瑞么托。"
def test_parse_translation_payload_accepts_fenced_json_and_numbered_lines():
assert _parse_translation_payload('```json\n["一", "二"]\n```', 2) == {0: "一", 1: "二"}
assert _parse_translation_payload("1. 一\n2. 二", 2) == {0: "一", 1: "二"}
assert _parse_translation_payload('prefix {"12":"甲","13":"乙"} suffix', [12, 13]) == {
12: "甲",
13: "乙",
}
def test_ai_tutor_retranslates_english_existing_cache(monkeypatch):
def fake_post(endpoint, headers, json, timeout):
return _translation_response_from_prompt(json)
monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post)
tutor = AITutor("https://ai.test/v1", "secret", "test-model", 1)
translated = tutor.translate_sentences(
[(0, "Hello."), (1, "Thanks.")],
existing={0: "Hello.", 1: "谢谢。"},
)
assert translated[0] == "中文0"
assert translated[1] == "谢谢。"
def test_ai_tutor_retries_when_translation_stays_english(monkeypatch):
calls = {"count": 0}
def fake_post(endpoint, headers, json, timeout):
calls["count"] += 1
if calls["count"] == 1:
return FakeResponse(
body={"choices": [{"message": {"content": '{"0":"Hello."}'}}]}
)
return FakeResponse(
body={"choices": [{"message": {"content": '{"0":"你好。"}'}}]}
)
monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post)
tutor = AITutor("https://ai.test/v1", "secret", "test-model", 1)
translated = tutor.translate_sentences([(0, "Hello.")])
assert translated == {0: "你好。"}
assert calls["count"] >= 2
def test_ai_tutor_logs_raw_response_when_translation_json_is_invalid(monkeypatch, caplog):
def fake_post(endpoint, headers, json, timeout):
return FakeResponse(
body={"choices": [{"message": {"content": "definitely not translation json"}}]}
)
monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post)
tutor = AITutor("https://ai.test/v1", "secret", "test-model", 1)
with caplog.at_level(logging.WARNING, logger="sentence_api.ai_tutor"):
with pytest.raises(AITutorError):
tutor.translate_sentences([(0, "Hello.")])
assert any(
"Translation response parse failed" in record.getMessage()
and "definitely not translation json" in record.getMessage()
for record in caplog.records
)
def test_ai_tutor_logs_invalid_translation_before_repair(monkeypatch, caplog):
def fake_post(endpoint, headers, json, timeout):
return FakeResponse(
body={"choices": [{"message": {"content": '{"0":"Still English."}'}}]}
)
monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post)
tutor = AITutor("https://ai.test/v1", "secret", "test-model", 1)
with caplog.at_level(logging.WARNING, logger="sentence_api.ai_tutor"):
with pytest.raises(AITutorError):
tutor.translate_sentences([(0, "Hello.")])
assert any(
"Translation response failed validation" in record.getMessage()
and "Still English." in record.getMessage()
for record in caplog.records
)

View File

@@ -1,4 +1,6 @@
import json
import threading
import time
from dataclasses import replace
from fastapi.testclient import TestClient
@@ -14,6 +16,18 @@ from sentence_api.store import BoundaryStore
VIDEO_HASH = "a" * 64
def wait_for_subtitle_translation(client, video_hash, timeout=5.0):
deadline = time.time() + timeout
last = None
while time.time() < deadline:
last = client.get(f"/api/v1/videos/{video_hash}/subtitle-translation")
if last.status_code == 200 and last.json().get("complete"):
return last
time.sleep(0.05)
raise AssertionError(f"subtitle translation did not complete: {getattr(last, 'text', last)}")
def make_client(tmp_path, admin_api_key=""):
index_path = tmp_path / "boundaries.json"
@@ -172,7 +186,7 @@ def test_sentence_explanation_uses_server_side_ai_client(tmp_path, monkeypatch):
return "这句话表示打招呼。"
class StubAITutor:
pass
available = True
stub = StubAITutor()
stub.explain = explain
@@ -202,6 +216,319 @@ def test_sentence_explanation_uses_server_side_ai_client(tmp_path, monkeypatch):
]
def _ready_video(repository):
repository.upsert_upload(
video_hash=VIDEO_HASH,
title="Greetings",
filename="lesson.mp4",
stored_filename=f"{VIDEO_HASH}.mp4",
content_type="video/mp4",
size_bytes=5,
language="en",
)
repository.save_processing_result(
SentenceBoundaryDocument(
video_hash=VIDEO_HASH,
duration_ms=9000,
algorithm_version="test",
sentences=[
SentenceBoundary(index=0, start_ms=0, end_ms=4000, text="Hello.", language="en", reference_speech_duration_ms=3000),
SentenceBoundary(index=1, start_ms=4000, end_ms=9000, text="Nice to meet you.", language="en", reference_speech_duration_ms=4000),
],
),
None,
)
def test_subtitle_translation_requires_login(tmp_path):
client = make_client(tmp_path)
client.headers.clear()
response = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert response.status_code == 401
LOCAL_SUBTITLE_HASH = "b" * 64
def _stub_translator(monkeypatch, client, requested=None):
requested = requested if requested is not None else []
def translate_sentences(sentences, **kwargs):
requested.append((sentences, kwargs))
return {index: f"中文{index}" for index, _text in sentences}
class StubAITutor:
available = True
stub = StubAITutor()
stub.translate_sentences = translate_sentences
monkeypatch.setattr(client.app.state, "ai_tutor", stub)
return requested
def test_local_english_subtitles_can_be_uploaded_for_translation(tmp_path, monkeypatch):
client = make_client(tmp_path)
requested = _stub_translator(monkeypatch, client)
missing = client.post(f"/api/v1/videos/{LOCAL_SUBTITLE_HASH}/subtitle-translation")
assert missing.status_code == 404
response = client.post(
f"/api/v1/videos/{LOCAL_SUBTITLE_HASH}/subtitle-translation",
json={
"duration_ms": 9000,
"algorithm_version": "local-pc",
"sentences": [
{"index": 0, "start_ms": 0, "end_ms": 4000, "text": "Hello.", "language": "en"},
{"index": 1, "start_ms": 4000, "end_ms": 9000, "text": "Nice to meet you.", "language": "en"},
],
},
)
assert response.status_code == 200, response.text
if not response.json().get("complete"):
response = wait_for_subtitle_translation(client, LOCAL_SUBTITLE_HASH)
payload = response.json()
assert payload["complete"] is True
assert payload["translations"] == [
{"index": 0, "zh_text": "中文0"},
{"index": 1, "zh_text": "中文1"},
]
assert requested[0][0] == [(0, "Hello."), (1, "Nice to meet you.")]
stored = client.get(f"/api/v1/videos/{LOCAL_SUBTITLE_HASH}/subtitle-translation")
assert stored.status_code == 200
assert stored.json()["translations"] == payload["translations"]
courses = client.get("/api/v1/courses")
assert courses.status_code == 200
assert all(item["video_hash"] != LOCAL_SUBTITLE_HASH for item in courses.json())
videos = client.get("/api/v1/videos")
assert videos.status_code == 200
assert all(item["video_hash"] != LOCAL_SUBTITLE_HASH for item in videos.json()["videos"])
def test_local_subtitle_upload_does_not_overwrite_course_sentences(tmp_path, monkeypatch):
client = make_client(tmp_path)
_ready_video(client.app.state.video_repository)
requested = _stub_translator(monkeypatch, client)
response = client.post(
f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation",
json={
"duration_ms": 9000,
"algorithm_version": "local-pc",
"sentences": [
{"index": 0, "start_ms": 0, "end_ms": 4000, "text": "Changed.", "language": "en"},
{"index": 1, "start_ms": 4000, "end_ms": 9000, "text": "Also changed.", "language": "en"},
],
},
)
assert response.status_code == 200, response.text
if not response.json().get("complete"):
response = wait_for_subtitle_translation(client, VIDEO_HASH)
assert requested[0][0] == [(0, "Hello."), (1, "Nice to meet you.")]
document = client.app.state.video_repository.get_document(VIDEO_HASH)
assert [sentence.text for sentence in document.sentences] == ["Hello.", "Nice to meet you."]
def test_subtitle_translation_uses_full_document_and_is_cached(tmp_path, monkeypatch):
client = make_client(tmp_path)
_ready_video(client.app.state.video_repository)
requested = []
def translate_sentences(sentences, **kwargs):
requested.append((sentences, kwargs))
return {index: f"中文{index}" for index, _text in sentences}
class StubAITutor:
available = True
stub = StubAITutor()
stub.translate_sentences = translate_sentences
monkeypatch.setattr(client.app.state, "ai_tutor", stub)
response = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert response.status_code == 200, response.text
if not response.json().get("complete"):
response = wait_for_subtitle_translation(client, VIDEO_HASH)
payload = response.json()
assert payload["video_hash"] == VIDEO_HASH
assert payload["complete"] is True
assert payload["status"] == "ready"
assert payload["translations"] == [
{"index": 0, "zh_text": "中文0"},
{"index": 1, "zh_text": "中文1"},
]
document = client.app.state.video_repository.get_document(VIDEO_HASH)
assert [sentence.zh_text for sentence in document.sentences] == ["中文0", "中文1"]
second = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert second.status_code == 200
assert second.json()["translations"] == payload["translations"]
assert second.json()["complete"] is True
assert len(requested) == 1
assert requested[0][0] == [(0, "Hello."), (1, "Nice to meet you.")]
def test_subtitle_translation_is_shared_with_other_users(tmp_path, monkeypatch):
client = make_client(tmp_path)
_ready_video(client.app.state.video_repository)
requested = []
def translate_sentences(sentences, **kwargs):
requested.append((sentences, kwargs))
return {index: f"中文{index}" for index, _text in sentences}
class StubAITutor:
available = True
stub = StubAITutor()
stub.translate_sentences = translate_sentences
monkeypatch.setattr(client.app.state, "ai_tutor", stub)
first = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert first.status_code == 200, first.text
if not first.json().get("complete"):
first = wait_for_subtitle_translation(client, VIDEO_HASH)
stored = client.get(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert stored.status_code == 200
assert stored.json()["translations"] == first.json()["translations"]
assert stored.json()["complete"] is True
other = TestClient(client.app)
signup = other.post(
"/api/v1/auth/register",
json={"username": "viewer2", "password": "password-123"},
)
assert signup.status_code == 201, signup.text
other.headers.update({"Authorization": f"Bearer {signup.json()['token']}"})
shared = other.get(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert shared.status_code == 200
assert shared.json()["translations"] == [
{"index": 0, "zh_text": "中文0"},
{"index": 1, "zh_text": "中文1"},
]
boundaries = other.get(f"/api/v1/videos/{VIDEO_HASH}/sentence-boundaries")
assert boundaries.status_code == 200
assert [item["zh_text"] for item in boundaries.json()["sentences"]] == ["中文0", "中文1"]
reused = other.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert reused.status_code == 200
assert reused.json()["translations"] == first.json()["translations"]
assert reused.json()["complete"] is True
assert len(requested) == 1
def test_subtitle_translation_does_not_block_http_on_long_jobs(tmp_path, monkeypatch):
client = make_client(tmp_path)
_ready_video(client.app.state.video_repository)
started = threading.Event()
release = threading.Event()
requested = []
def translate_sentences(sentences, **kwargs):
requested.append(sentences)
persist = kwargs.get("persist")
if persist is not None:
persist({0: "中文0"})
started.set()
assert release.wait(timeout=5)
if persist is not None:
persist({1: "中文1"})
return {0: "中文0", 1: "中文1"}
class StubAITutor:
available = True
stub = StubAITutor()
stub.translate_sentences = translate_sentences
monkeypatch.setattr(client.app.state, "ai_tutor", stub)
client.app.state.subtitle_translations.ai_tutor = stub
client.app.state.subtitle_translations.post_wait_seconds = 0.2
response = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert started.wait(timeout=2)
assert response.status_code == 200, response.text
payload = response.json()
assert payload["complete"] is False
assert payload["status"] == "running"
if {"index": 0, "zh_text": "中文0"} not in payload["translations"]:
payload = client.get(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation").json()
assert {"index": 0, "zh_text": "中文0"} in payload["translations"]
stored = client.get(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert stored.status_code == 200
assert stored.json()["status"] == "running"
assert stored.json()["complete"] is False
release.set()
finished = wait_for_subtitle_translation(client, VIDEO_HASH)
assert finished.json()["complete"] is True
assert finished.json()["translations"] == [
{"index": 0, "zh_text": "中文0"},
{"index": 1, "zh_text": "中文1"},
]
assert len(requested) == 1
def test_admin_can_retranslate_subtitles_with_name_terms(tmp_path, monkeypatch):
client = make_client(tmp_path)
_ready_video(client.app.state.video_repository)
requested = []
def translate_sentences(sentences, **kwargs):
requested.append((sentences, kwargs.get("terms")))
terms = dict(kwargs.get("terms") or [])
return {
index: terms.get("Harimoto", "哈达马托") + f":{text}"
for index, text in sentences
}
class StubAITutor:
available = True
stub = StubAITutor()
stub.translate_sentences = translate_sentences
monkeypatch.setattr(client.app.state, "ai_tutor", stub)
client.app.state.subtitle_translations.ai_tutor = stub
client.app.state.subtitle_translations.post_wait_seconds = 0.2
first = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
if not first.json().get("complete"):
first = wait_for_subtitle_translation(client, VIDEO_HASH)
assert first.json()["translations"] == [
{"index": 0, "zh_text": "哈达马托:Hello."},
{"index": 1, "zh_text": "哈达马托:Nice to meet you."},
]
response = client.post(
f"/api/v1/admin/videos/{VIDEO_HASH}/subtitle-translation",
json={"terms": [{"source": "Harimoto", "zh": "哈瑞么托"}]},
)
assert response.status_code == 200, response.text
if not response.json().get("complete"):
response = wait_for_subtitle_translation(client, VIDEO_HASH)
stored = client.get(f"/api/v1/admin/videos/{VIDEO_HASH}/subtitle-translation")
assert stored.status_code == 200
payload = stored.json()
else:
payload = response.json()
assert payload["translations"] == [
{"index": 0, "zh_text": "哈瑞么托:Hello."},
{"index": 1, "zh_text": "哈瑞么托:Nice to meet you."},
]
assert payload["terms"] == [{"source": "Harimoto", "zh": "哈瑞么托"}]
assert requested[-1][1] == [("Harimoto", "哈瑞么托")]
assert client.app.state.video_repository.get_translation_glossary(VIDEO_HASH) == [
("Harimoto", "哈瑞么托")
]
def test_oss_video_url_is_used_for_catalog_and_content(tmp_path):
index_path = tmp_path / "boundaries.json"
index_path.write_text("{}", encoding="utf-8")
@@ -366,6 +693,10 @@ def test_admin_can_split_sentence_and_reindexes_following_sentences(tmp_path):
),
None,
)
repository.save_sentence_translations(
VIDEO_HASH,
{0: "之前。", 1: "第一段第二段", 2: "下一句。"},
)
class UnavailableTranscriber:
available = False
@@ -397,6 +728,7 @@ def test_admin_can_split_sentence_and_reindexes_following_sentences(tmp_path):
"Next sentence.",
]
assert [item.index for item in document.sentences] == [0, 1, 2, 3]
assert all(not (item.zh_text or "").strip() for item in document.sentences)
def test_admin_can_merge_adjacent_sentences_and_reindexes_following_sentences(tmp_path):
@@ -454,6 +786,11 @@ def test_admin_can_merge_adjacent_sentences_and_reindexes_following_sentences(tm
),
None,
)
repository.save_sentence_translations(
VIDEO_HASH,
{0: "之前。", 1: "第一段", 2: "第二段", 3: "下一句。"},
)
(settings.videos_dir / f"{VIDEO_HASH}.mp4").write_bytes(b"video")
class UnavailableTranscriber:
available = False
@@ -487,6 +824,7 @@ def test_admin_can_merge_adjacent_sentences_and_reindexes_following_sentences(tm
assert [item.index for item in document.sentences] == [0, 1, 2]
assert [item.start_ms for item in document.sentences] == [0, 2000, 9000]
assert [item.end_ms for item in document.sentences] == [2000, 9000, 12000]
assert [item.zh_text for item in document.sentences] == ["之前。", None, "下一句。"]
last_response = client.put(
f"/api/v1/admin/videos/{VIDEO_HASH}/sentences/2/merge",
@@ -495,6 +833,257 @@ def test_admin_can_merge_adjacent_sentences_and_reindexes_following_sentences(tm
assert last_response.json()["detail"] == "No next sentence to merge with."
def test_admin_batch_applies_merge_and_split_once(tmp_path, monkeypatch):
settings = replace(Settings.from_env(), data_dir=tmp_path / "data")
settings.ensure_directories()
repository = VideoRepository(settings.database_path)
repository.upsert_upload(
video_hash=VIDEO_HASH,
title="Batch edit lesson",
filename="lesson.mp4",
stored_filename=f"{VIDEO_HASH}.mp4",
content_type="video/mp4",
size_bytes=5,
language="en",
)
repository.save_processing_result(
SentenceBoundaryDocument(
video_hash=VIDEO_HASH,
duration_ms=12000,
algorithm_version="test",
sentences=[
SentenceBoundary(index=0, start_ms=0, end_ms=2000, text="Before.", language="en", reference_speech_duration_ms=1800),
SentenceBoundary(index=1, start_ms=2000, end_ms=5000, text="First part", language="en", reference_speech_duration_ms=2800),
SentenceBoundary(index=2, start_ms=5000, end_ms=9000, text="second part", language="en", reference_speech_duration_ms=3600),
SentenceBoundary(index=3, start_ms=9000, end_ms=12000, text="Next sentence.", language="en", reference_speech_duration_ms=2800),
],
),
None,
)
repository.save_sentence_translations(
VIDEO_HASH,
{0: "之前。", 1: "第一段", 2: "第二段", 3: "下一句。"},
)
(settings.videos_dir / f"{VIDEO_HASH}.mp4").write_bytes(b"video")
calls = []
from sentence_api.transcription import Transcript, TranscriptionSegment, WordTimestamp
class OnceTranscriber:
available = True
def transcribe(self, audio_path, language=None):
calls.append(audio_path)
return Transcript(
text="Next sentence.",
segments=[
TranscriptionSegment(
start_seconds=0.0,
end_seconds=1.0,
text="Next sentence.",
words=[
WordTimestamp(0.0, 0.2, "Next"),
WordTimestamp(0.2, 1.0, "sentence."),
],
)
],
)
monkeypatch.setattr(
"sentence_api.processing.extract_audio_segment",
lambda media_path, output_path, *, start_ms, duration_ms: output_path.write_bytes(b"audio"),
)
client = TestClient(
create_app(
BoundaryStore(tmp_path / "boundaries.json"),
settings=settings,
repository=repository,
transcriber=OnceTranscriber(),
)
)
response = client.put(
f"/api/v1/admin/videos/{VIDEO_HASH}/sentences/batch",
json={
"sentences": [
{"start_ms": 0, "end_ms": 2000, "text": "Before.", "language": "en", "reference_speech_duration_ms": 1800},
{"start_ms": 2000, "end_ms": 9000, "text": "First part second part", "language": "en", "reference_speech_duration_ms": 6400},
{"start_ms": 9000, "end_ms": 9200, "text": "Next", "language": "en", "reference_speech_duration_ms": 1000, "source_start_ms": 9000, "source_end_ms": 12000},
{"start_ms": 9200, "end_ms": 12000, "text": "sentence.", "language": "en", "reference_speech_duration_ms": 1000, "source_start_ms": 9000, "source_end_ms": 12000},
]
},
)
assert response.status_code == 200, response.text
payload = response.json()
assert [item["text"] for item in payload] == [
"Before.",
"First part second part",
"Next",
"sentence.",
]
assert [item["start_ms"] for item in payload] == [0, 2000, 9000, 9600]
assert [item["end_ms"] for item in payload] == [2000, 9000, 9600, 12000]
assert [item["reference_speech_duration_ms"] for item in payload] == [1800, 6400, 862, 1938]
assert len(calls) == 1
document = repository.get_document(VIDEO_HASH)
assert [item.text for item in document.sentences] == [
"Before.",
"First part second part",
"Next",
"sentence.",
]
assert all(not (item.zh_text or "").strip() for item in document.sentences)
def test_admin_batch_can_persist_provisional_split_without_processing(tmp_path, monkeypatch):
settings = replace(Settings.from_env(), data_dir=tmp_path / "data")
settings.ensure_directories()
repository = VideoRepository(settings.database_path)
repository.upsert_upload(
video_hash=VIDEO_HASH,
title="Provisional split lesson",
filename="lesson.mp4",
stored_filename=f"{VIDEO_HASH}.mp4",
content_type="video/mp4",
size_bytes=5,
language="en",
)
repository.save_processing_result(
SentenceBoundaryDocument(
video_hash=VIDEO_HASH,
duration_ms=12000,
algorithm_version="test",
sentences=[
SentenceBoundary(index=0, start_ms=0, end_ms=5000, text="First part second part", language="en", reference_speech_duration_ms=4600),
SentenceBoundary(index=1, start_ms=5000, end_ms=12000, text="Unchanged.", language="en", reference_speech_duration_ms=6400),
],
),
None,
)
repository.save_sentence_translations(VIDEO_HASH, {1: "未变化。"})
(settings.videos_dir / f"{VIDEO_HASH}.mp4").write_bytes(b"video")
from sentence_api.transcription import Transcript, TranscriptionSegment, WordTimestamp
calls = []
class DeferredTranscriber:
available = True
allowed = False
def transcribe(self, audio_path, language=None):
assert self.allowed, "provisional save must not transcribe"
calls.append(audio_path)
return Transcript(
text="First part second part",
segments=[
TranscriptionSegment(
start_seconds=0.0,
end_seconds=1.0,
text="First part second part",
words=[
WordTimestamp(0.0, 0.2, "First"),
WordTimestamp(0.2, 1.0, "part second part"),
],
)
],
)
monkeypatch.setattr(
"sentence_api.processing.extract_audio_segment",
lambda media_path, output_path, *, start_ms, duration_ms: output_path.write_bytes(b"audio"),
)
client = TestClient(
create_app(
BoundaryStore(tmp_path / "boundaries.json"),
settings=settings,
repository=repository,
transcriber=DeferredTranscriber(),
)
)
response = client.put(
f"/api/v1/admin/videos/{VIDEO_HASH}/sentences/batch",
json={
"process_timings": False,
"sentences": [
{"start_ms": 0, "end_ms": 2500, "text": "First part", "language": "en", "zh_text": None, "reference_speech_duration_ms": 2300, "source_start_ms": 0, "source_end_ms": 5000},
{"start_ms": 2500, "end_ms": 5000, "text": "second part", "language": "en", "zh_text": None, "reference_speech_duration_ms": 2300, "source_start_ms": 0, "source_end_ms": 5000},
{"start_ms": 5000, "end_ms": 12000, "text": "Unchanged.", "language": "en", "zh_text": "未变化。", "reference_speech_duration_ms": 6400},
],
},
)
assert response.status_code == 200, response.text
payload = response.json()
assert [item["text"] for item in payload] == ["First part", "second part", "Unchanged."]
assert [item["start_ms"] for item in payload] == [0, 2500, 5000]
assert [item["zh_text"] for item in payload] == [None, None, "未变化。"]
assert calls == []
document = repository.get_document(VIDEO_HASH)
assert [item.text for item in document.sentences] == ["First part", "second part", "Unchanged."]
assert [item.zh_text for item in document.sentences] == [None, None, "未变化。"]
client.app.state.transcriber.allowed = True
processed = client.put(
f"/api/v1/admin/videos/{VIDEO_HASH}/sentences/batch",
json={
"process_timings": True,
"sentences": [
{"start_ms": 0, "end_ms": 2500, "text": "First part", "language": "en", "zh_text": None, "reference_speech_duration_ms": 2300, "source_start_ms": 0, "source_end_ms": 5000},
{"start_ms": 2500, "end_ms": 5000, "text": "second part", "language": "en", "zh_text": None, "reference_speech_duration_ms": 2300, "source_start_ms": 0, "source_end_ms": 5000},
{"start_ms": 5000, "end_ms": 12000, "text": "Unchanged.", "language": "en", "zh_text": "未变化。", "reference_speech_duration_ms": 6400},
],
},
)
assert processed.status_code == 200, processed.text
processed_payload = processed.json()
assert [item["start_ms"] for item in processed_payload] == [0, 1000, 5000]
assert [item["end_ms"] for item in processed_payload] == [1000, 5000, 12000]
assert len(calls) == 1
def test_admin_sentence_merge_does_not_start_subtitle_translation(tmp_path, monkeypatch):
client = make_client(tmp_path)
_ready_video(client.app.state.video_repository)
requested = []
def translate_sentences(sentences, **kwargs):
requested.append([(index, text) for index, text in sentences])
return {index: f"中文:{text}" for index, text in sentences}
class StubAITutor:
available = True
stub = StubAITutor()
stub.translate_sentences = translate_sentences
monkeypatch.setattr(client.app.state, "ai_tutor", stub)
client.app.state.subtitle_translations.ai_tutor = stub
client.app.state.subtitle_translations.post_wait_seconds = 0.2
first = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
if not first.json().get("complete"):
first = wait_for_subtitle_translation(client, VIDEO_HASH)
assert first.json()["translations"] == [
{"index": 0, "zh_text": "中文:Hello."},
{"index": 1, "zh_text": "中文:Nice to meet you."},
]
merge = client.put(f"/api/v1/admin/videos/{VIDEO_HASH}/sentences/0/merge")
assert merge.status_code == 200, merge.text
assert merge.json()["text"] == "Hello. Nice to meet you."
snapshot = client.get(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert snapshot.status_code == 200
assert snapshot.json()["status"] == "idle"
assert snapshot.json()["complete"] is False
assert snapshot.json()["translations"] == []
assert len(requested) == 1
document = client.app.state.video_repository.get_document(VIDEO_HASH)
assert [sentence.text for sentence in document.sentences] == ["Hello. Nice to meet you."]
assert [sentence.zh_text for sentence in document.sentences] == [None]
def test_admin_boundary_adjustment_updates_linked_next_start(tmp_path):
settings = replace(Settings.from_env(), data_dir=tmp_path / "data")
settings.ensure_directories()
@@ -562,3 +1151,264 @@ def test_admin_boundary_adjustment_updates_linked_next_start(tmp_path):
document = repository.get_document(VIDEO_HASH)
assert [item.end_ms for item in document.sentences] == [4250, 9000]
assert [item.start_ms for item in document.sentences] == [0, 4250]
def test_admin_can_update_course_title(tmp_path):
settings = replace(Settings.from_env(), data_dir=tmp_path / "data")
settings.ensure_directories()
repository = VideoRepository(settings.database_path)
repository.upsert_upload(
video_hash=VIDEO_HASH,
title="Old course title",
filename="lesson.mp4",
stored_filename=f"{VIDEO_HASH}.mp4",
content_type="video/mp4",
size_bytes=5,
language="en",
)
client = TestClient(
create_app(
BoundaryStore(tmp_path / "boundaries.json"),
settings=settings,
repository=repository,
)
)
response = client.put(
f"/api/v1/admin/videos/{VIDEO_HASH}/title",
json={"title": " New course title "},
)
assert response.status_code == 200
assert response.json()["title"] == "New course title"
assert repository.get_video(VIDEO_HASH)["title"] == "New course title"
missing_response = client.put(
f"/api/v1/admin/videos/{'b' * 64}/title",
json={"title": "Missing course"},
)
assert missing_response.status_code == 404
def test_admin_can_repair_sentence_gaps(tmp_path, monkeypatch):
settings = replace(Settings.from_env(), data_dir=tmp_path / "data")
settings.ensure_directories()
repository = VideoRepository(settings.database_path)
repository.upsert_upload(
video_hash=VIDEO_HASH,
title="Repair gaps lesson",
filename="lesson.mp4",
stored_filename=f"{VIDEO_HASH}.mp4",
content_type="video/mp4",
size_bytes=5,
language="en",
)
(settings.videos_dir / f"{VIDEO_HASH}.mp4").write_bytes(b"fake-video")
repository.save_processing_result(
SentenceBoundaryDocument(
video_hash=VIDEO_HASH,
duration_ms=18000,
algorithm_version="test",
sentences=[
SentenceBoundary(
index=0,
start_ms=0,
end_ms=2000,
text="Hello.",
language="en",
reference_speech_duration_ms=1800,
),
SentenceBoundary(
index=1,
start_ms=15000,
end_ms=18000,
text="Goodbye.",
language="en",
reference_speech_duration_ms=2500,
),
],
),
"Hello. Goodbye.",
)
repository.save_sentence_translations(VIDEO_HASH, {0: "你好。", 1: "再见。"})
from sentence_api.transcription import Transcript, TranscriptionSegment, _wav_bytes
import numpy as np
class GapTranscriber:
available = True
def transcribe(self, audio_path, language=None):
return Transcript(
text="The missing sentence.",
segments=[
TranscriptionSegment(
start_seconds=1.0,
end_seconds=5.0,
text="The missing sentence.",
)
],
)
def fake_extract(media_path, output_path, *, start_ms, duration_ms):
sample_rate = 16000
t = np.arange(int(max(1.0, duration_ms / 1000) * sample_rate)) / sample_rate
samples = (0.25 * np.sin(2 * np.pi * 220 * t)).astype(np.float32)
output_path.write_bytes(_wav_bytes(samples, sample_rate))
monkeypatch.setattr("sentence_api.processing.extract_audio_segment", fake_extract)
client = TestClient(
create_app(
BoundaryStore(tmp_path / "boundaries.json"),
settings=settings,
repository=repository,
transcriber=GapTranscriber(),
)
)
response = client.post(f"/api/v1/admin/videos/{VIDEO_HASH}/repair-gaps")
assert response.status_code == 200, response.text
payload = response.json()
assert payload["gap_count"] == 1
assert payload["inserted_count"] == 1
assert payload["sentence_count"] == 3
document = repository.get_document(VIDEO_HASH)
assert [item.text for item in document.sentences] == [
"Hello.",
"The missing sentence.",
"Goodbye.",
]
assert [item.zh_text for item in document.sentences] == ["你好。", None, "再见。"]
def test_admin_gap_repair_requires_transcriber(tmp_path):
settings = replace(Settings.from_env(), data_dir=tmp_path / "data")
settings.ensure_directories()
repository = VideoRepository(settings.database_path)
repository.upsert_upload(
video_hash=VIDEO_HASH,
title="Repair gaps lesson",
filename="lesson.mp4",
stored_filename=f"{VIDEO_HASH}.mp4",
content_type="video/mp4",
size_bytes=5,
language="en",
)
(settings.videos_dir / f"{VIDEO_HASH}.mp4").write_bytes(b"fake-video")
repository.save_processing_result(
SentenceBoundaryDocument(
video_hash=VIDEO_HASH,
duration_ms=18000,
algorithm_version="test",
sentences=[
SentenceBoundary(index=0, start_ms=0, end_ms=2000, text="Hello."),
SentenceBoundary(index=1, start_ms=15000, end_ms=18000, text="Goodbye."),
],
),
None,
)
class UnavailableTranscriber:
available = False
client = TestClient(
create_app(
BoundaryStore(tmp_path / "boundaries.json"),
settings=settings,
repository=repository,
transcriber=UnavailableTranscriber(),
)
)
response = client.post(f"/api/v1/admin/videos/{VIDEO_HASH}/repair-gaps")
assert response.status_code == 503
def test_admin_can_repair_missing_and_english_translations(tmp_path, monkeypatch):
client = make_client(tmp_path)
repository = client.app.state.video_repository
_ready_video(repository)
repository.save_sentence_translations(
VIDEO_HASH,
{0: "你好。", 1: "Nice to meet you."},
)
requested = []
def translate_sentences(sentences, **kwargs):
requested.append([index for index, _text in sentences])
existing = dict(kwargs.get("existing") or {})
result = {}
for index, _text in sentences:
current = existing.get(index, "")
if current and any("\u4e00" <= char <= "\u9fff" for char in current):
result[index] = current
else:
result[index] = f"补译{index}"
return result
class StubAITutor:
available = True
stub = StubAITutor()
stub.translate_sentences = translate_sentences
monkeypatch.setattr(client.app.state, "ai_tutor", stub)
client.app.state.subtitle_translations.ai_tutor = stub
client.app.state.subtitle_translations.post_wait_seconds = 0.2
snapshot = client.get(f"/api/v1/admin/videos/{VIDEO_HASH}/subtitle-translation")
assert snapshot.status_code == 200
assert snapshot.json()["missing_count"] == 1
response = client.post(
f"/api/v1/admin/videos/{VIDEO_HASH}/subtitle-translation/repair"
)
assert response.status_code == 200, response.text
payload = response.json()
if not payload.get("complete"):
finished = wait_for_subtitle_translation(client, VIDEO_HASH)
payload = client.get(f"/api/v1/admin/videos/{VIDEO_HASH}/subtitle-translation").json()
payload["attempted_count"] = response.json()["attempted_count"]
payload["complete"] = finished.json()["complete"]
assert payload["attempted_count"] == 1
assert payload["complete"] is True
assert payload["missing_count"] == 0
document = repository.get_document(VIDEO_HASH)
assert [sentence.zh_text for sentence in document.sentences] == ["你好。", "补译1"]
assert requested
def test_admin_translation_repair_keeps_complete_chinese(tmp_path, monkeypatch):
client = make_client(tmp_path)
repository = client.app.state.video_repository
_ready_video(repository)
repository.save_sentence_translations(
VIDEO_HASH,
{0: "你好。", 1: "见到你很高兴。"},
)
called = {"count": 0}
def translate_sentences(sentences, **kwargs):
called["count"] += 1
return {index: f"新{index}" for index, _text in sentences}
class StubAITutor:
available = True
stub = StubAITutor()
stub.translate_sentences = translate_sentences
monkeypatch.setattr(client.app.state, "ai_tutor", stub)
client.app.state.subtitle_translations.ai_tutor = stub
response = client.post(
f"/api/v1/admin/videos/{VIDEO_HASH}/subtitle-translation/repair"
)
assert response.status_code == 200, response.text
assert response.json()["attempted_count"] == 0
assert response.json()["missing_count"] == 0
assert called["count"] == 0
document = repository.get_document(VIDEO_HASH)
assert [sentence.zh_text for sentence in document.sentences] == ["你好。", "见到你很高兴。"]

View File

@@ -2,10 +2,14 @@ import numpy as np
from sentence_api.processing import (
MOSS_ALGORITHM_VERSION,
VideoProcessor,
document_from_transcript,
find_sentence_gaps,
_plan_gap_chunks,
split_sentence_text,
)
from sentence_api.models import SentenceBoundary
from sentence_api.models import SentenceBoundary, SentenceBoundaryDocument
from sentence_api.repository import VideoRepository
from sentence_api.transcription import (
Transcript,
TranscriptionSegment,
@@ -158,3 +162,415 @@ def test_split_sentence_text_uses_whisper_word_timestamps(tmp_path):
("Alpha beta", 1000, 3000),
("Gamma delta", 3000, 5000),
]
def test_processor_persists_original_moss_segments(tmp_path, monkeypatch):
from sentence_api.config import Settings
from dataclasses import replace
settings = replace(Settings.from_env(), data_dir=tmp_path / "data")
settings.ensure_directories()
media_path = settings.videos_dir / f"{VIDEO_HASH}.mp4"
media_path.write_bytes(b"video")
repository = VideoRepository(settings.database_path)
repository.upsert_upload(
video_hash=VIDEO_HASH,
title="MOSS lesson",
filename="lesson.mp4",
stored_filename=f"{VIDEO_HASH}.mp4",
content_type="video/mp4",
size_bytes=5,
language="en",
)
transcript = Transcript(
text="Hello world. Next.",
segments=[
TranscriptionSegment(
start_seconds=0.0,
end_seconds=1.2,
text="Hello world.",
speaker="S01",
words=[WordTimestamp(0.0, 0.5, "Hello"), WordTimestamp(0.5, 1.2, "world.")],
),
TranscriptionSegment(
start_seconds=1.2,
end_seconds=2.4,
text="Next.",
speaker="S01",
),
],
)
class AvailableTranscriber:
available = True
def transcribe(self, audio_path, language=None):
return transcript
monkeypatch.setattr("sentence_api.processing._ensure_webm_cues_front", lambda path: None)
monkeypatch.setattr(
"sentence_api.processing.extract_audio",
lambda src, dst: dst.write_bytes(b"RIFF"),
)
monkeypatch.setattr("sentence_api.processing._media_duration_ms", lambda path: 2400)
monkeypatch.setattr(
"sentence_api.processing.document_from_transcript",
lambda **kwargs: SentenceBoundaryDocument(
video_hash=kwargs["video_hash"],
duration_ms=2400,
algorithm_version=MOSS_ALGORITHM_VERSION,
sentences=[
SentenceBoundary(
index=0,
start_ms=0,
end_ms=1200,
text="Hello world.",
language="en",
reference_speech_duration_ms=1100,
),
SentenceBoundary(
index=1,
start_ms=1200,
end_ms=2400,
text="Next.",
language="en",
reference_speech_duration_ms=1100,
),
],
),
)
VideoProcessor(settings, repository, AvailableTranscriber()).process(VIDEO_HASH)
segments = repository.get_moss_segments(VIDEO_HASH)
assert [(item["start_ms"], item["end_ms"], item["text"], item["speaker"]) for item in segments] == [
(0, 1200, "Hello world.", "S01"),
(1200, 2400, "Next.", "S01"),
]
assert segments[0]["words"] == [
{"start_seconds": 0.0, "end_seconds": 0.5, "text": "Hello"},
{"start_seconds": 0.5, "end_seconds": 1.2, "text": "world."},
]
assert segments[1]["words"] is None
def test_reprocess_without_moss_clears_original_segments(tmp_path, monkeypatch):
from sentence_api.config import Settings
from dataclasses import replace
settings = replace(Settings.from_env(), data_dir=tmp_path / "data")
settings.ensure_directories()
media_path = settings.videos_dir / f"{VIDEO_HASH}.mp4"
media_path.write_bytes(b"video")
repository = VideoRepository(settings.database_path)
repository.upsert_upload(
video_hash=VIDEO_HASH,
title="MOSS lesson",
filename="lesson.mp4",
stored_filename=f"{VIDEO_HASH}.mp4",
content_type="video/mp4",
size_bytes=5,
language="en",
)
repository.save_processing_result(
SentenceBoundaryDocument(
video_hash=VIDEO_HASH,
duration_ms=2400,
algorithm_version=MOSS_ALGORITHM_VERSION,
sentences=[
SentenceBoundary(
index=0,
start_ms=0,
end_ms=2400,
text="Hello.",
language="en",
reference_speech_duration_ms=2000,
)
],
),
"Hello.",
moss_segments=[
{
"start_seconds": 0.0,
"end_seconds": 2.4,
"text": "Hello.",
"speaker": "S01",
}
],
)
assert repository.get_moss_segments(VIDEO_HASH)
class UnavailableTranscriber:
available = False
monkeypatch.setattr("sentence_api.processing._ensure_webm_cues_front", lambda path: None)
monkeypatch.setattr(
"sentence_api.processing.make_entry",
lambda *args, **kwargs: (
{
"duration_ms": 2400,
"sentences": [
SentenceBoundary(
index=0,
start_ms=0,
end_ms=2400,
text=None,
language="en",
)
],
},
{},
),
)
VideoProcessor(settings, repository, UnavailableTranscriber()).process(VIDEO_HASH)
assert repository.get_moss_segments(VIDEO_HASH) == []
def test_mark_processing_clears_previous_sentence_data(tmp_path):
repository = VideoRepository(tmp_path / "oral.sqlite3")
repository.upsert_upload(
video_hash=VIDEO_HASH,
title="MOSS lesson",
filename="lesson.mp4",
stored_filename=f"{VIDEO_HASH}.mp4",
content_type="video/mp4",
size_bytes=5,
language="en",
)
repository.save_processing_result(
SentenceBoundaryDocument(
video_hash=VIDEO_HASH,
duration_ms=2400,
algorithm_version=MOSS_ALGORITHM_VERSION,
sentences=[
SentenceBoundary(
index=0,
start_ms=0,
end_ms=1200,
text="Old first.",
language="en",
reference_speech_duration_ms=1000,
),
SentenceBoundary(
index=1,
start_ms=1200,
end_ms=2400,
text="Old second.",
language="en",
reference_speech_duration_ms=1000,
),
],
),
"Old first. Old second.",
moss_segments=[
{
"start_seconds": 0.0,
"end_seconds": 1.2,
"text": "Old first.",
"speaker": "S01",
}
],
)
repository.mark_processing(VIDEO_HASH)
video = repository.get_video(VIDEO_HASH)
document = repository.get_document(VIDEO_HASH)
assert video["status"] == "processing"
assert video["transcription"] is None
assert video["sentence_count"] == 0
assert document is not None
assert document.sentences == []
assert repository.get_moss_segments(VIDEO_HASH) == []
def test_find_sentence_gaps_detects_leading_middle_and_trailing():
document = SentenceBoundaryDocument(
video_hash=VIDEO_HASH,
duration_ms=60000,
algorithm_version="test",
sentences=[
SentenceBoundary(index=0, start_ms=10000, end_ms=12000, text="A"),
SentenceBoundary(index=1, start_ms=25000, end_ms=27000, text="B"),
],
)
assert find_sentence_gaps(document, min_gap_ms=8000) == [
(0, 10000),
(12000, 25000),
(27000, 60000),
]
def test_find_sentence_gaps_ignores_short_pauses():
document = SentenceBoundaryDocument(
video_hash=VIDEO_HASH,
duration_ms=9000,
algorithm_version="test",
sentences=[
SentenceBoundary(index=0, start_ms=0, end_ms=4000, text="A"),
SentenceBoundary(index=1, start_ms=4500, end_ms=9000, text="B"),
],
)
assert find_sentence_gaps(document) == []
def test_plan_gap_chunks_splits_long_gap_at_silence():
samples = np.concatenate(
[_speech(14.0), _silence(0.6), _speech(16.0)]
).astype(np.float32)
chunks = _plan_gap_chunks(samples, SAMPLE_RATE, int(samples.size / SAMPLE_RATE * 1000))
assert len(chunks) == 2
first_start, first_end = chunks[0]
second_start, second_end = chunks[1]
assert first_start == 0
assert 13_000 <= first_end <= 17_000
assert second_start == first_end
assert second_end == int(samples.size / SAMPLE_RATE * 1000)
assert all(end - start <= 20_000 for start, end in chunks)
def test_repair_gaps_inserts_transcribed_sentences_and_keeps_existing(tmp_path, monkeypatch):
from dataclasses import replace
from sentence_api.config import Settings
settings = replace(Settings.from_env(), data_dir=tmp_path / "data")
settings.ensure_directories()
repository = VideoRepository(settings.database_path)
repository.upsert_upload(
video_hash=VIDEO_HASH,
title="Gap repair lesson",
filename="lesson.mp4",
stored_filename=f"{VIDEO_HASH}.mp4",
content_type="video/mp4",
size_bytes=5,
language="en",
)
(settings.videos_dir / f"{VIDEO_HASH}.mp4").write_bytes(b"fake-video")
repository.save_processing_result(
SentenceBoundaryDocument(
video_hash=VIDEO_HASH,
duration_ms=18000,
algorithm_version="test",
sentences=[
SentenceBoundary(
index=0,
start_ms=0,
end_ms=2000,
text="Hello.",
language="en",
reference_speech_duration_ms=1800,
),
SentenceBoundary(
index=1,
start_ms=15000,
end_ms=18000,
text="Goodbye.",
language="en",
reference_speech_duration_ms=2500,
),
],
),
"Hello. Goodbye.",
)
repository.save_sentence_translations(VIDEO_HASH, {0: "你好。", 1: "再见。"})
class GapTranscriber:
available = True
calls = 0
def transcribe(self, audio_path, language=None):
self.calls += 1
return Transcript(
text="The missing sentence.",
segments=[
TranscriptionSegment(
start_seconds=1.0,
end_seconds=5.0,
text="The missing sentence.",
)
],
)
def fake_extract(media_path, output_path, *, start_ms, duration_ms):
output_path.write_bytes(_wav_bytes(_speech(max(1.0, duration_ms / 1000)), SAMPLE_RATE))
monkeypatch.setattr("sentence_api.processing.extract_audio_segment", fake_extract)
transcriber = GapTranscriber()
result = VideoProcessor(settings, repository, transcriber).repair_gaps(VIDEO_HASH)
assert result["gap_count"] == 1
assert result["transcribed_gaps"] == 1
assert result["inserted_count"] == 1
assert transcriber.calls == 1
document = repository.get_document(VIDEO_HASH)
assert [sentence.text for sentence in document.sentences] == [
"Hello.",
"The missing sentence.",
"Goodbye.",
]
assert document.sentences[0].zh_text == "你好。"
assert document.sentences[1].zh_text is None
assert document.sentences[2].zh_text == "再见。"
assert document.sentences[0].end_ms <= document.sentences[1].start_ms
assert document.sentences[1].end_ms <= document.sentences[2].start_ms
assert document.sentences[1].start_ms >= 2000
assert document.sentences[1].end_ms <= 15000
def test_repair_gaps_skips_silent_gaps(tmp_path, monkeypatch):
from dataclasses import replace
from sentence_api.config import Settings
settings = replace(Settings.from_env(), data_dir=tmp_path / "data")
settings.ensure_directories()
repository = VideoRepository(settings.database_path)
repository.upsert_upload(
video_hash=VIDEO_HASH,
title="Silent gap lesson",
filename="lesson.mp4",
stored_filename=f"{VIDEO_HASH}.mp4",
content_type="video/mp4",
size_bytes=5,
language="en",
)
(settings.videos_dir / f"{VIDEO_HASH}.mp4").write_bytes(b"fake-video")
repository.save_processing_result(
SentenceBoundaryDocument(
video_hash=VIDEO_HASH,
duration_ms=18000,
algorithm_version="test",
sentences=[
SentenceBoundary(index=0, start_ms=0, end_ms=2000, text="Hello."),
SentenceBoundary(index=1, start_ms=15000, end_ms=18000, text="Goodbye."),
],
),
None,
)
class TrackingTranscriber:
available = True
calls = 0
def transcribe(self, audio_path, language=None):
self.calls += 1
return Transcript(text="hallucinated", segments=[])
def fake_extract(media_path, output_path, *, start_ms, duration_ms):
output_path.write_bytes(_wav_bytes(_silence(max(1.0, duration_ms / 1000)), SAMPLE_RATE))
monkeypatch.setattr("sentence_api.processing.extract_audio_segment", fake_extract)
transcriber = TrackingTranscriber()
result = VideoProcessor(settings, repository, transcriber).repair_gaps(VIDEO_HASH)
assert result["gap_count"] == 1
assert result["inserted_count"] == 0
assert transcriber.calls == 0
document = repository.get_document(VIDEO_HASH)
assert [sentence.text for sentence in document.sentences] == ["Hello.", "Goodbye."]

View File

@@ -423,6 +423,36 @@ def test_split_sentences_ignores_am_pm_abbreviations():
]
def test_split_sentences_ignores_clock_times_with_pm_period():
compact = TranscriptionSegment(
start_seconds=0.0,
end_seconds=6.0,
text="The briefing is at 3:00 PM. Eastern officials will attend later.",
)
assert [s.text for s in split_sentences_at_punctuation([compact])] == [
"The briefing is at 3:00 PM. Eastern officials will attend later."
]
spoken = TranscriptionSegment(
start_seconds=0.0,
end_seconds=6.0,
text="We start at 5 pm. Eastern time and then break.",
)
assert [s.text for s in split_sentences_at_punctuation([spoken])] == [
"We start at 5 pm. Eastern time and then break."
]
morning = TranscriptionSegment(
start_seconds=0.0,
end_seconds=6.0,
text="It begins at 9:30 AM. Please arrive early.",
)
assert [s.text for s in split_sentences_at_punctuation([morning])] == [
"It begins at 9:30 AM.",
"Please arrive early.",
]
def test_split_sentences_merges_segment_ending_with_abbreviation():
segments = [
TranscriptionSegment(0.0, 5.0, "The meeting ends at 5 p.m."),
@@ -460,6 +490,77 @@ def test_split_sentences_ignores_common_abbreviations():
]
def test_split_sentences_merges_same_speaker_line_wraps():
segments = [
TranscriptionSegment(
11.18,
15.45,
"[S01]Let's begin with some scheduling announcements for the remainder of President Trump's very",
),
TranscriptionSegment(15.45, 16.29, "[S01]busy week."),
TranscriptionSegment(
16.29,
21.8,
"[S01]Today at 3:00 PM, President Trump will host a Black History Month reception in the East",
),
TranscriptionSegment(21.8, 23.29, "[S01]Room of the White House."),
]
sentences = split_sentences_at_punctuation(segments)
assert [s.text for s in sentences] == [
(
"Let's begin with some scheduling announcements for the remainder of "
"President Trump's very busy week."
),
(
"Today at 3:00 PM, President Trump will host a Black History Month "
"reception in the East Room of the White House."
),
]
assert sentences[0].start_seconds == 11.18
assert sentences[0].end_seconds == 16.29
assert sentences[1].start_seconds == 16.29
assert sentences[1].end_seconds == 23.29
def test_split_sentences_splits_when_speaker_changes():
segments = [
TranscriptionSegment(4.63, 5.93, "[S01]Hello, everybody."),
TranscriptionSegment(6.0, 6.53, "[S02]Hello."),
TranscriptionSegment(6.56, 7.83, "[S01]Good afternoon."),
]
sentences = split_sentences_at_punctuation(segments)
assert [s.text for s in sentences] == [
"Hello, everybody.",
"Hello.",
"Good afternoon.",
]
def test_split_sentences_keeps_same_speaker_sentences_apart():
segments = [
TranscriptionSegment(6.56, 7.83, "[S01]Good afternoon."),
TranscriptionSegment(7.93, 9.86, "[S01]Thank you for your patience, as always."),
]
sentences = split_sentences_at_punctuation(segments)
assert [s.text for s in sentences] == [
"Good afternoon.",
"Thank you for your patience, as always.",
]
def test_split_sentences_ignores_initials_and_version_numbers():
segment = TranscriptionSegment(
start_seconds=0.0,
end_seconds=8.0,
text="Donald J. Trump discussed version 1.9 today. Then the meeting ended.",
)
sentences = split_sentences_at_punctuation([segment])
assert [s.text for s in sentences] == [
"Donald J. Trump discussed version 1.9 today.",
"Then the meeting ended.",
]
def test_post_audio_requests_word_timestamps_and_falls_back(monkeypatch):
calls = []

View File

@@ -1,4 +1,5 @@
import io
import logging
import mimetypes
import re
import wave
@@ -12,6 +13,9 @@ import numpy as np
from .audio_metrics import AudioAnalysisError, decode_audio_mono, speech_frame_mask
logger = logging.getLogger(__name__)
# Whisper was trained on 30-second clips. Sending longer audio in a single request
# makes the model enter a repetition loop (every later segment becomes the same
# sentence while timestamps keep advancing), so long audio is split into short
@@ -93,12 +97,44 @@ class MossTranscriber:
self, audio_path: Path, language: Optional[str]
) -> Transcript:
content_type = mimetypes.guess_type(audio_path.name)[0] or "application/octet-stream"
logger.info(
"Transcription request: endpoint=%s model=%s audio=%s language=%s",
self.endpoint,
self.model,
audio_path.name,
language or "auto",
)
with audio_path.open("rb") as audio_file:
payload = self._post_audio(audio_file, audio_path.name, content_type, language)
raw_text = str(payload.get("text") or "").strip()
raw_segments = payload.get("segments")
segments = _parse_json_segments(payload.get("segments"))
logger.info(
"Transcription response: audio=%s raw_text_chars=%d raw_segments=%s "
"parsed_segments=%d text=%r",
audio_path.name,
len(raw_text),
len(raw_segments) if isinstance(raw_segments, list) else "missing",
len(segments),
_log_text_preview(raw_text),
)
if not segments:
segments = _parse_compact_segments(raw_text)
if segments:
logger.info(
"Recovered %d compact-format segments from response text: audio=%s",
len(segments),
audio_path.name,
)
if not segments:
logger.warning(
"Transcription returned no usable timestamped segments: audio=%s "
"payload_keys=%s raw_segments=%s text=%r",
audio_path.name,
sorted(payload.keys()),
_log_segments_preview(raw_segments),
_log_text_preview(raw_text, 500),
)
plain_text = " ".join(segment.text for segment in segments).strip()
plain_text = _strip_speaker_tags(plain_text) or _strip_speaker_tags(raw_text)
return Transcript(text=plain_text, segments=segments)
@@ -117,11 +153,38 @@ class MossTranscriber:
"audio/wav",
language,
)
raw_chunk_segments = payload.get("segments")
chunk_segments = _parse_json_segments(
payload.get("segments"), compression_limit=COMPRESSION_RATIO_LIMIT
)
logger.info(
"Chunk transcription response: range=%.3f-%.3fs raw_segments=%s "
"parsed_segments=%d text=%r",
start_seconds,
end_seconds,
len(raw_chunk_segments) if isinstance(raw_chunk_segments, list) else "missing",
len(chunk_segments),
_log_text_preview(str(payload.get("text") or "")),
)
if not chunk_segments:
chunk_segments = _parse_compact_segments(str(payload.get("text") or "").strip())
if chunk_segments:
logger.info(
"Recovered %d compact-format segments from chunk text: range=%.3f-%.3fs",
len(chunk_segments),
start_seconds,
end_seconds,
)
if not chunk_segments:
logger.warning(
"Chunk returned no usable timestamped segments: range=%.3f-%.3fs "
"payload_keys=%s raw_segments=%s text=%r",
start_seconds,
end_seconds,
sorted(payload.keys()),
_log_segments_preview(raw_chunk_segments),
_log_text_preview(str(payload.get("text") or ""), 500),
)
for segment in chunk_segments:
words = None
if segment.words:
@@ -172,8 +235,20 @@ class MossTranscriber:
word_payload, audio_file, filename, content_type
)
if response.status_code in (400, 422):
logger.warning(
"Transcription endpoint rejected word timestamps; retrying without them: "
"status=%d body=%s",
response.status_code,
_log_text_preview(getattr(response, "text", ""), 1000),
)
audio_file.seek(0)
response = self._post_once(data, audio_file, filename, content_type)
if response.status_code >= 400:
logger.warning(
"Transcription endpoint returned HTTP %d: body=%s",
response.status_code,
_log_text_preview(getattr(response, "text", ""), 2000),
)
response.raise_for_status()
return response.json()
except (httpx.HTTPError, ValueError) as exc:
@@ -246,9 +321,18 @@ _PUNCTUATION_BOUNDARY = re.compile(r"[.?]+|,+")
MIN_WORDS_FOR_COMMA_SPLIT = 50
_ABBREVIATION_RE = re.compile(
r"(?i)\b(?:a\.m\.|p\.m\.|mr\.|mrs\.|ms\.|dr\.|prof\.|st\.|vs\.|etc\.|"
r"e\.g\.|i\.e\.|no\.|approx\.|fig\.|inc\.|ltd\.|jr\.|sr\.|u\.s\.|u\.k\.)"
r"e\.g\.|i\.e\.|no\.|approx\.|fig\.|inc\.|ltd\.|jr\.|sr\.|u\.s\.|u\.k\.|"
r"ph\.d\.|m\.d\.|b\.a\.|m\.a\.|d\.c\.|p\.s\.)"
r"(?![A-Za-z0-9])"
)
_TIME_MERIDIEM_RE = re.compile(
r"(?i)\b(?:[01]?\d|2[0-3])(?::[0-5]\d)?\s*(?:a\.m\.|p\.m\.|am\.|pm\.)"
r"(?![A-Za-z0-9])"
)
_TIME_CONTINUATION_RE = re.compile(
r"(?i)^(eastern|western|central|pacific|mountain|time|"
r"et|est|edt|pt|pst|pdt|ct|cst|cdt|mt|mst|mdt|gmt|utc|local)\b"
)
_SPEAKER_TAG_RE = re.compile(r"\[(S\d+)\]")
@@ -319,8 +403,10 @@ def split_sentences_at_punctuation(
are also eligible boundaries, but only when the text before the comma has
enough words; otherwise the comma stays with the following text.
Inline ``[S01]`` speaker tags are stripped and each speaker turn always
starts a new sentence, even when no sentence-ending punctuation exists.
Inline ``[S01]`` speaker tags are stripped. A new sentence starts when the
speaker actually changes; same-speaker line wraps stay merged until
sentence punctuation. Decimal points, initials, and common abbreviations
are not treated as sentence ends.
"""
sentences: List[TranscriptionSegment] = []
pieces: List[str] = []
@@ -354,10 +440,14 @@ def split_sentences_at_punctuation(
text = segment.text.strip()
if not text:
continue
# Each speaker-tagged sub-segment always starts a new sentence,
# regardless of whether the speaker actually changed — MOSS emits
# a tag per phrase/line even for the same speaker.
if pieces and segment.speaker is not None:
# MOSS often tags every phrase/line with [S01] even when the speaker
# did not change. Only a real speaker change starts a new sentence.
if (
pieces
and segment.speaker is not None
and current_speaker is not None
and segment.speaker != current_speaker
):
flush(segment.start_seconds)
if not pieces:
start_seconds = segment.start_seconds
@@ -418,31 +508,70 @@ def _is_sentence_end_run(text: str, run_start: int, run_end: int) -> bool:
punctuation = text[run_start:run_end]
if "?" in punctuation:
return True
if (
run_start > 0
and run_end < len(text)
and text[run_start - 1].isdigit()
and text[run_end].isdigit()
):
return False # decimal point, e.g. "0.4"
if _is_decimal_point(text, run_start, run_end):
return False
if len(punctuation) > 1:
return True # ellipsis or "!?" etc.
if _is_abbreviation_run(text, run_start, run_end):
return False # e.g. "a.m.", "p.m.", "Mr."
if _is_time_meridiem_run(text, run_start, run_end):
return False # e.g. "3:00 PM.", "5 pm."
if _is_single_letter_initial(text, run_start, run_end):
return False # e.g. "Donald J. Trump"
rest = text[run_end:].lstrip()
if rest and rest[0].islower():
return False # period followed by a lowercase word is an abbreviation
return True
def _is_decimal_point(text: str, run_start: int, run_end: int) -> bool:
if run_end - run_start != 1 or text[run_start] != ".":
return False
if run_start == 0 or run_end >= len(text):
return False
return text[run_start - 1].isdigit() and text[run_end].isdigit()
def _is_single_letter_initial(text: str, run_start: int, run_end: int) -> bool:
if run_end - run_start != 1 or text[run_start] != ".":
return False
if run_start == 0 or not text[run_start - 1].isalpha():
return False
return run_start == 1 or not text[run_start - 2].isalpha()
def _meridiem_continues(text: str, after: int) -> bool:
rest = text[after:].lstrip()
if not rest:
return True
if rest[0].islower():
return True
return _TIME_CONTINUATION_RE.match(rest) is not None
def _is_time_meridiem_run(text: str, run_start: int, run_end: int) -> bool:
for match in _TIME_MERIDIEM_RE.finditer(text):
if not (match.start() < run_end <= match.end()):
continue
if run_end < match.end():
return True
return _meridiem_continues(text, match.end())
return False
def _is_abbreviation_run(text: str, run_start: int, run_end: int) -> bool:
for match in _ABBREVIATION_RE.finditer(text):
if not (match.start() < run_end <= match.end()):
continue
if match.group(0).lower() == "no.":
token = match.group(0).lower()
if token == "no.":
rest = text[match.end():].lstrip()
if not rest or not rest[0].isdigit():
return False # "No." as a reply ends the sentence
if token in {"a.m.", "p.m."}:
if run_end < match.end():
return True
return _meridiem_continues(text, match.end())
return True
return False
@@ -502,6 +631,30 @@ def _wav_duration_seconds(path: Path) -> Optional[float]:
return None
def _log_text_preview(text: str, limit: int = 240) -> str:
text = " ".join(text.split())
return text if len(text) <= limit else f"{text[:limit]}... [{len(text)} chars]"
def _log_segments_preview(raw_segments, limit: int = 5):
if not isinstance(raw_segments, list):
return raw_segments
preview = []
for item in raw_segments[:limit]:
if not isinstance(item, dict):
preview.append(item)
continue
preview.append(
{
"start": item.get("start"),
"end": item.get("end"),
"compression_ratio": item.get("compression_ratio"),
"text": _log_text_preview(str(item.get("text") or "")),
}
)
return {"count": len(raw_segments), "items": preview}
def _wav_bytes(samples: np.ndarray, sample_rate: int) -> bytes:
pcm = np.clip(np.round(samples * 32767), -32768, 32767).astype("<i2")
buffer = io.BytesIO()

View File

@@ -0,0 +1,306 @@
"""Background subtitle translation so long videos do not block HTTP."""
from __future__ import annotations
import logging
import threading
from dataclasses import dataclass
from .ai_tutor import AITutor, AITutorError, translation_needs_repair
from .models import (
AdminSubtitleTranslationResponse,
SentenceBoundaryDocument,
SubtitleTranslationItem,
SubtitleTranslationResponse,
TranslationTerm,
)
from .repository import VideoRepository
logger = logging.getLogger(__name__)
@dataclass
class _Job:
video_hash: str
done: threading.Event
lock: threading.Lock
running: bool = True
cancelled: bool = False
error: str | None = None
thread: threading.Thread | None = None
class SubtitleTranslationService:
def __init__(
self,
repository: VideoRepository,
ai_tutor: AITutor,
*,
post_wait_seconds: float = 8.0,
) -> None:
self.repository = repository
self._ai_tutor = ai_tutor
self.post_wait_seconds = post_wait_seconds
self._lock = threading.Lock()
self._jobs: dict[str, _Job] = {}
@property
def ai_tutor(self) -> AITutor:
value = self._ai_tutor
return value() if callable(value) else value
@ai_tutor.setter
def ai_tutor(self, value) -> None:
self._ai_tutor = value
def snapshot(self, document: SentenceBoundaryDocument) -> SubtitleTranslationResponse:
translations = [
SubtitleTranslationItem(index=sentence.index, zh_text=sentence.zh_text)
for sentence in document.sentences
if sentence.zh_text and sentence.zh_text.strip()
]
needed = [
sentence
for sentence in document.sentences
if (sentence.text or "").strip()
]
complete = bool(needed) and all((sentence.zh_text or "").strip() for sentence in needed)
job = self._jobs.get(document.video_hash)
running = False
error: str | None = None
if job is not None:
with job.lock:
running = job.running
error = job.error
if complete:
status = "ready"
error = None
elif running:
status = "running"
elif error:
status = "error"
else:
status = "idle"
return SubtitleTranslationResponse(
video_hash=document.video_hash,
translations=translations,
complete=complete,
status=status,
error=error,
)
def current(self, video_hash: str) -> SubtitleTranslationResponse | None:
document = self.repository.get_document(video_hash)
if document is None:
return None
return self.snapshot(document)
def invalidate(self, video_hash: str) -> None:
"""Stop an in-flight job so stale translations cannot be written back."""
with self._lock:
job = self._jobs.pop(video_hash, None)
if job is None:
return
with job.lock:
job.cancelled = True
def refresh(self, video_hash: str) -> None:
"""Drop saved Chinese subtitles and translate the current sentences again."""
self.invalidate(video_hash)
self.repository.clear_sentence_translations(video_hash)
if not getattr(self.ai_tutor, "available", False):
return
try:
self._start_if_needed(video_hash)
except (KeyError, ValueError, AITutorError):
logger.exception("Could not restart subtitle translation: video=%s", video_hash)
def start_missing(self, video_hash: str) -> None:
"""Translate sentences that still lack Chinese text, keeping existing ones."""
if not getattr(self.ai_tutor, "available", False):
return
document = self.repository.get_document(video_hash)
if document is None:
return
if self.snapshot(document).complete:
return
try:
self._start_if_needed(video_hash)
except (KeyError, ValueError, AITutorError):
logger.exception(
"Could not start subtitle translation after gap repair: video=%s",
video_hash,
)
def repair_missing(self, video_hash: str) -> AdminSubtitleTranslationResponse:
"""Re-translate missing or still-English sentences, keeping good Chinese."""
if self.repository.get_video(video_hash) is None:
raise KeyError(video_hash)
document = self.repository.get_document(video_hash)
if document is None:
raise KeyError(video_hash)
missing = missing_translation_indexes(document)
if missing:
self.invalidate(video_hash)
self.repository.clear_sentence_translations(video_hash, missing)
snapshot = self.ensure(video_hash)
else:
snapshot = self.snapshot(document)
result = self._with_terms(snapshot)
return result.model_copy(update={"attempted_count": len(missing)})
def admin_snapshot(self, video_hash: str) -> AdminSubtitleTranslationResponse | None:
current = self.current(video_hash)
if current is None:
return None
return self._with_terms(current)
def retranslate(
self,
video_hash: str,
terms: list[tuple[str, str]] | None = None,
) -> AdminSubtitleTranslationResponse:
if self.repository.get_video(video_hash) is None:
raise KeyError(video_hash)
if terms is not None:
self.repository.save_translation_glossary(video_hash, terms)
self.invalidate(video_hash)
self.repository.clear_sentence_translations(video_hash)
return self._with_terms(self.ensure(video_hash))
def _with_terms(self, snapshot: SubtitleTranslationResponse) -> AdminSubtitleTranslationResponse:
terms = [
TranslationTerm(source=source, zh=zh)
for source, zh in self.repository.get_translation_glossary(snapshot.video_hash)
]
document = self.repository.get_document(snapshot.video_hash)
missing_count = 0 if document is None else len(missing_translation_indexes(document))
payload = snapshot.model_dump()
payload["terms"] = terms
payload["missing_count"] = missing_count
payload["attempted_count"] = 0
return AdminSubtitleTranslationResponse(**payload)
def ensure_from_local(self, document: SentenceBoundaryDocument) -> SubtitleTranslationResponse:
"""Accept locally generated English subtitles, then translate like a course."""
self.repository.import_local_subtitle_document(document)
return self.ensure(document.video_hash)
def ensure(self, video_hash: str) -> SubtitleTranslationResponse:
if not getattr(self.ai_tutor, "available", False):
raise AITutorError("AI translation is not configured.")
document = self.repository.get_document(video_hash)
if document is None:
raise KeyError(video_hash)
current = self.snapshot(document)
if current.complete:
return current
payload = [
(sentence.index, sentence.text or "")
for sentence in document.sentences
if (sentence.text or "").strip()
]
if not payload:
raise ValueError("No sentence text is available to translate.")
job = self._start_if_needed(video_hash)
wait_seconds = max(0.0, float(self.post_wait_seconds))
if wait_seconds > 0:
job.done.wait(timeout=wait_seconds)
latest = self.current(video_hash)
if latest is None:
raise KeyError(video_hash)
return latest
def _start_if_needed(self, video_hash: str) -> _Job:
with self._lock:
existing = self._jobs.get(video_hash)
if existing is not None:
with existing.lock:
still_running = (
not existing.cancelled
and (
existing.running
or (
existing.thread is not None
and existing.thread.is_alive()
)
)
)
if still_running:
return existing
job = _Job(
video_hash=video_hash,
done=threading.Event(),
lock=threading.Lock(),
)
thread = threading.Thread(
target=self._run_job,
args=(job,),
name=f"subtitle-translate-{video_hash[:8]}",
daemon=True,
)
job.thread = thread
self._jobs[video_hash] = job
thread.start()
return job
def _persist(self, job: _Job, translations: dict[int, str]) -> None:
with job.lock:
if job.cancelled:
return
self.repository.save_sentence_translations(job.video_hash, translations)
def _run_job(self, job: _Job) -> None:
try:
with job.lock:
if job.cancelled:
return
document = self.repository.get_document(job.video_hash)
if document is None:
raise AITutorError("Course is not available.")
payload = [
(sentence.index, sentence.text or "")
for sentence in document.sentences
if (sentence.text or "").strip()
]
if not payload:
return
existing = {
sentence.index: sentence.zh_text
for sentence in document.sentences
if sentence.zh_text and sentence.zh_text.strip()
}
video = self.repository.get_video(job.video_hash)
translated = self.ai_tutor.translate_sentences(
payload,
course_title=(video["title"] if video is not None else "") or "",
existing=existing,
persist=lambda part: self._persist(job, part),
terms=self.repository.get_translation_glossary(job.video_hash),
)
if translated:
self._persist(job, translated)
with job.lock:
if not job.cancelled:
job.error = None
except Exception as exc: # noqa: BLE001 - surface job failure to clients
with job.lock:
cancelled = job.cancelled
if cancelled:
return
logger.exception("Subtitle translation job failed: video=%s", job.video_hash)
with job.lock:
job.error = str(exc)
finally:
with job.lock:
job.running = False
job.done.set()
def missing_translation_indexes(document: SentenceBoundaryDocument) -> list[int]:
return [
sentence.index
for sentence in document.sentences
if translation_needs_repair(sentence.text, sentence.zh_text)
]

View File

@@ -15,3 +15,7 @@ WHISPER_TTL=-1
# 模型下载地址:国内默认走 hf-mirror.com 镜像;能直连 Hugging Face 时改为空值(HF_ENDPOINT=)
HF_ENDPOINT=https://hf-mirror.com
# 国内拉取 ghcr 较慢时走 DaoCloud;能直连 GitHub 时改为:
# WHISPER_IMAGE=ghcr.io/speaches-ai/speaches:latest-cuda
WHISPER_IMAGE=docker.m.daocloud.io/ghcr.io/speaches-ai/speaches:latest-cuda

View File

@@ -9,7 +9,7 @@ OpenAI 兼容语音转写服务,使用 faster-whisper(CTranslate2)推理
```text
whisper/
docker-compose.yml GPU 服务编排(ghcr.io/speaches-ai/speaches:latest-cuda)
docker-compose.yml GPU 服务编排(默认经 DaoCloud 拉 ghcr 镜像)
.env.example 配置模板(模型、端口、计算类型)
systemd/whisper-transcribe.service systemd 模板
verify.sh 部署验证脚本
@@ -27,7 +27,7 @@ sudo chown "$USER":"$USER" /opt/whisper
cp -a whisper/. /opt/whisper/ # 注意用 whisper/. 而不是 whisper/*,否则不会复制 .env.example 等隐藏文件
cd /opt/whisper
cp .env.example .env # 按需修改 WHISPER_MODEL
cp .env.example .env # 按需修改 WHISPER_MODEL / WHISPER_IMAGE
docker compose up -d
docker compose logs -f whisper # 首次启动下载模型(约 3 GB),看到启动完成即可
```
@@ -79,6 +79,8 @@ L4(24 GB 显存)可同时常驻两个模型;Speaches 支持在请求的 `m
## 常见问题
- 拉取 Speaches 镜像慢:默认镜像为 `docker.m.daocloud.io/ghcr.io/speaches-ai/speaches:latest-cuda`;
能直连 GitHub 时在 `.env` 设 `WHISPER_IMAGE=ghcr.io/speaches-ai/speaches:latest-cuda`。
- 首次启动下载模型慢或失败:默认已使用国内镜像 `https://hf-mirror.com`(可在 `.env`
的 `HF_ENDPOINT` 覆盖);仍失败可改用代理,在 `docker-compose.yml` 的
`environment` 里加 `HTTP_PROXY`/`HTTPS_PROXY` 后 `docker compose up -d`。

View File

@@ -1,6 +1,6 @@
services:
whisper:
image: ghcr.io/speaches-ai/speaches:latest-cuda
image: ${WHISPER_IMAGE:-docker.m.daocloud.io/ghcr.io/speaches-ai/speaches:latest-cuda}
container_name: whisper-transcribe
restart: unless-stopped
ports: