add chinese subtitles

This commit is contained in:
2026-10-03 12:14:17 +08:00
parent 138d446d73
commit 6bd64ee4a9
22 changed files with 922 additions and 77 deletions

View File

@@ -159,6 +159,11 @@ class SentenceBoundaryApi internal constructor(
} else { } else {
sentence.getLong("reference_speech_duration_ms") sentence.getLong("reference_speech_duration_ms")
}, },
zhText = if (sentence.isNull("zh_text")) {
null
} else {
sentence.getString("zh_text").takeIf { it.isNotBlank() }
},
) )
) )
} }

View File

@@ -10,6 +10,7 @@ data class SentenceBoundary @JvmOverloads constructor(
val text: String? = null, val text: String? = null,
val language: String? = null, val language: String? = null,
val referenceSpeechDurationMs: Long? = null, val referenceSpeechDurationMs: Long? = null,
val zhText: String? = null,
) { ) {
init { init {
require(index >= 0) { "Sentence index must be non-negative." } require(index >= 0) { "Sentence index must be non-negative." }

View File

@@ -40,6 +40,8 @@ class MainActivity : Activity() {
private lateinit var catalogController: CourseCatalogController private lateinit var catalogController: CourseCatalogController
private lateinit var testSession: TestSessionController private lateinit var testSession: TestSessionController
private lateinit var explanationOverlay: SentenceExplanationOverlay private lateinit var explanationOverlay: SentenceExplanationOverlay
private lateinit var translationApi: SubtitleTranslationApi
private var translationRequest: CancellableTranslationRequest? = null
private var progressBarDragging = false private var progressBarDragging = false
@@ -70,6 +72,9 @@ class MainActivity : Activity() {
authSessionStore.session?.token.orEmpty() authSessionStore.session?.token.orEmpty()
}, },
) )
translationApi = SubtitleTranslationApi(this, SERVER_BASE_URL) {
authSessionStore.session?.token.orEmpty()
}
screenFactory = SampleScreenFactory( screenFactory = SampleScreenFactory(
activity = this, activity = this,
@@ -175,6 +180,8 @@ class MainActivity : Activity() {
testSession.release() testSession.release()
controller.release() controller.release()
explanationOverlay.release() explanationOverlay.release()
translationRequest?.cancel()
translationApi.release()
super.onDestroy() super.onDestroy()
} }
@@ -255,6 +262,7 @@ class MainActivity : Activity() {
screenState.currentSentenceCount = item?.sentences?.size ?: 0 screenState.currentSentenceCount = item?.sentences?.size ?: 0
applyPlaybackSnapshot(controller.snapshot()) applyPlaybackSnapshot(controller.snapshot())
applySentence(controller.currentSentence()) applySentence(controller.currentSentence())
ensureChineseTranslations()
screenFactory.setCatalogStatus(screenState.catalogStatus) screenFactory.setCatalogStatus(screenState.catalogStatus)
} }
@@ -268,6 +276,7 @@ class MainActivity : Activity() {
bindings.lessonTitleText.text = item?.title ?: "未选择课程" bindings.lessonTitleText.text = item?.title ?: "未选择课程"
screenState.currentSentenceCount = item?.sentences?.size ?: 0 screenState.currentSentenceCount = item?.sentences?.size ?: 0
} }
ensureChineseTranslations()
testSession.ensureHistoryForCurrentCourse() testSession.ensureHistoryForCurrentCourse()
if (screenState.activeModule == Module.TEST) { if (screenState.activeModule == Module.TEST) {
testSession.refreshTestUi() testSession.refreshTestUi()
@@ -311,6 +320,7 @@ class MainActivity : Activity() {
bindings.sentenceMetaText.text = "暂无句子边界" bindings.sentenceMetaText.text = "暂无句子边界"
bindings.sentenceText.text = "" bindings.sentenceText.text = ""
bindings.explainSentenceButton?.isEnabled = false bindings.explainSentenceButton?.isEnabled = false
applyChineseSubtitle(null)
return return
} }
val countText = if (screenState.currentSentenceCount > 0) { val countText = if (screenState.currentSentenceCount > 0) {
@@ -322,11 +332,83 @@ class MainActivity : Activity() {
"第 ${sentence.index + 1}$countText 句 ${PlaybackFormat.time(sentence.startMs)}-${PlaybackFormat.time(sentence.endMs)}" "第 ${sentence.index + 1}$countText 句 ${PlaybackFormat.time(sentence.startMs)}-${PlaybackFormat.time(sentence.endMs)}"
bindings.sentenceText.text = sentence.text?.takeIf { it.isNotBlank() } ?: "当前句子" bindings.sentenceText.text = sentence.text?.takeIf { it.isNotBlank() } ?: "当前句子"
bindings.explainSentenceButton?.isEnabled = !sentence.text.isNullOrBlank() bindings.explainSentenceButton?.isEnabled = !sentence.text.isNullOrBlank()
applyChineseSubtitle(sentence)
if (screenState.activeModule == Module.TEST) { if (screenState.activeModule == Module.TEST) {
testSession.refreshTestUi() testSession.refreshTestUi()
} }
} }
private fun applyChineseSubtitle(sentence: SentenceBoundary?) {
if (!bindings.hasPlayerChrome) return
val showChinese = screenState.activeModule == Module.TRAIN &&
screenState.chineseSubtitlesEnabled &&
sentence != null &&
!sentence.text.isNullOrBlank()
if (!showChinese || sentence == null) {
bindings.sentenceZhText.visibility = android.view.View.GONE
return
}
bindings.sentenceZhText.visibility = android.view.View.VISIBLE
bindings.sentenceZhText.text = when {
!sentence.zhText.isNullOrBlank() -> sentence.zhText
screenState.chineseTranslationLoading -> "正在翻译中文字幕…"
screenState.chineseTranslationError.isNotBlank() -> screenState.chineseTranslationError
else -> "暂无中文字幕"
}
}
private fun ensureChineseTranslations() {
if (screenState.activeModule != Module.TRAIN || !screenState.chineseSubtitlesEnabled) {
return
}
val item = controller.currentTrainingItem() ?: return
val sentences = item.sentences
val needsTranslation = sentences.any { !it.text.isNullOrBlank() && it.zhText.isNullOrBlank() }
if (!needsTranslation) {
screenState.chineseTranslationLoading = false
screenState.chineseTranslationError = ""
applyChineseSubtitle(controller.currentSentence())
return
}
val generation = screenState.beginTranslationRequest()
screenState.chineseTranslationLoading = true
screenState.chineseTranslationError = ""
applyChineseSubtitle(controller.currentSentence())
translationRequest?.cancel()
translationRequest = translationApi.translate(
item.id,
object : SubtitleTranslationCallback {
override fun onSuccess(result: SubtitleTranslationResult) {
if (!screenState.isCurrentTranslationRequest(generation, item.id)) return
screenState.chineseTranslationLoading = false
val current = controller.currentTrainingItem() ?: return
if (current.id != item.id) return
val updated = current.sentences.map { sentence ->
sentence.copy(zhText = result.translations[sentence.index] ?: sentence.zhText)
}
controller.updateSentences(current.id, updated)
screenState.courseBoundaries[current.id]?.let { cached ->
screenState.courseBoundaries[current.id] = cached.copy(sentences = updated)
}
applyChineseSubtitle(controller.currentSentence())
if (bindings.hasPlayerChrome) {
bindings.statusText.text = "已加载中文字幕"
}
}
override fun onError(error: Throwable) {
if (!screenState.isCurrentTranslationRequest(generation, item.id)) return
screenState.chineseTranslationLoading = false
screenState.chineseTranslationError = "中文字幕翻译失败"
applyChineseSubtitle(controller.currentSentence())
if (bindings.hasPlayerChrome) {
bindings.statusText.text = "中文字幕翻译失败:${error.message.orEmpty()}"
}
}
},
)
}
private fun screenActions() = object : SampleScreenActions { private fun screenActions() = object : SampleScreenActions {
override fun onModuleSelected(module: Module) = switchModule(module) override fun onModuleSelected(module: Module) = switchModule(module)
@@ -360,6 +442,24 @@ class MainActivity : Activity() {
} }
} }
override fun onChineseSubtitlesChanged(enabled: Boolean) {
screenState.chineseSubtitlesEnabled = enabled
if (bindings.hasPlayerChrome) {
bindings.statusText.text = if (enabled) {
"已开启中文字幕"
} else {
"已关闭中文字幕"
}
}
if (enabled) {
ensureChineseTranslations()
} else {
translationRequest?.cancel()
screenState.chineseTranslationLoading = false
applyChineseSubtitle(controller.currentSentence())
}
}
override fun onExplainCurrentSentence() { override fun onExplainCurrentSentence() {
val sentence = controller.currentSentence() ?: return val sentence = controller.currentSentence() ?: return
if (sentence.text.isNullOrBlank()) return if (sentence.text.isNullOrBlank()) return

View File

@@ -13,6 +13,7 @@ internal class SampleBindings {
lateinit var lessonTitleText: TextView lateinit var lessonTitleText: TextView
lateinit var statusText: TextView lateinit var statusText: TextView
lateinit var sentenceText: TextView lateinit var sentenceText: TextView
lateinit var sentenceZhText: TextView
lateinit var sentenceMetaText: TextView lateinit var sentenceMetaText: TextView
lateinit var timeText: TextView lateinit var timeText: TextView
lateinit var speedText: TextView lateinit var speedText: TextView
@@ -35,6 +36,7 @@ internal class SampleBindings {
get() = ::lessonTitleText.isInitialized && get() = ::lessonTitleText.isInitialized &&
::statusText.isInitialized && ::statusText.isInitialized &&
::sentenceText.isInitialized && ::sentenceText.isInitialized &&
::sentenceZhText.isInitialized &&
::sentenceMetaText.isInitialized && ::sentenceMetaText.isInitialized &&
::timeText.isInitialized && ::timeText.isInitialized &&
::speedText.isInitialized && ::speedText.isInitialized &&

View File

@@ -21,6 +21,7 @@ internal interface TrainScreenActions :
fun onSeekStart() fun onSeekStart()
fun onSeekStop(progress: Int) fun onSeekStop(progress: Int)
fun onContinuousPlaybackChanged(enabled: Boolean) fun onContinuousPlaybackChanged(enabled: Boolean)
fun onChineseSubtitlesChanged(enabled: Boolean)
} }
internal interface TestScreenActions : internal interface TestScreenActions :

View File

@@ -0,0 +1,104 @@
package cn.learningpad.oraltrainer.sample
import android.content.Context
import android.os.Handler
import android.os.Looper
import org.json.JSONObject
import java.io.IOException
import java.net.HttpURLConnection
import java.net.URL
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.Future
data class SubtitleTranslationResult(
val videoHash: String,
val translations: Map<Int, String>,
)
interface SubtitleTranslationCallback {
fun onSuccess(result: SubtitleTranslationResult)
fun onError(error: Throwable)
}
class CancellableTranslationRequest internal constructor(
private val future: Future<*>,
) {
fun cancel() {
future.cancel(true)
}
}
/** Calls the app server; AI credentials never live in the mobile app. */
internal class SubtitleTranslationApi(
context: Context,
private val baseUrl: String,
private val tokenProvider: () -> String,
) {
private val appContext = context.applicationContext
private val mainHandler = Handler(Looper.getMainLooper())
private val executor: ExecutorService = Executors.newCachedThreadPool()
fun translate(
videoHash: String,
callback: SubtitleTranslationCallback,
): CancellableTranslationRequest {
val future = executor.submit {
try {
val result = requestBlocking(videoHash)
if (!Thread.currentThread().isInterrupted) {
mainHandler.post { callback.onSuccess(result) }
}
} catch (error: Throwable) {
if (!Thread.currentThread().isInterrupted) {
mainHandler.post { callback.onError(error) }
}
}
}
return CancellableTranslationRequest(future)
}
fun release() {
executor.shutdownNow()
}
private fun requestBlocking(videoHash: String): SubtitleTranslationResult {
val endpoint = baseUrl.trimEnd('/') + "/api/v1/videos/$videoHash/subtitle-translation"
val connection = URL(endpoint).openConnection() as HttpURLConnection
try {
connection.requestMethod = "POST"
connection.connectTimeout = 15_000
connection.readTimeout = 120_000
connection.doOutput = false
connection.useCaches = false
connection.setRequestProperty("Accept", "application/json")
connection.setRequestProperty("User-Agent", "OralTrainerSample/0.1.5 (Android)")
tokenProvider().takeIf { it.isNotBlank() }?.let {
connection.setRequestProperty("Authorization", "Bearer $it")
}
val statusCode = connection.responseCode
val stream = if (statusCode in 200..299) connection.inputStream else connection.errorStream
val response = stream?.bufferedReader(Charsets.UTF_8)?.use { it.readText() }.orEmpty()
if (statusCode !in 200..299) {
throw IOException("Subtitle translation API returned HTTP $statusCode: $response")
}
val root = JSONObject(response)
val items = root.getJSONArray("translations")
val translations = buildMap {
for (index in 0 until items.length()) {
val item = items.getJSONObject(index)
val zh = item.optString("zh_text")
if (zh.isNotBlank()) {
put(item.getInt("index"), zh)
}
}
}
return SubtitleTranslationResult(
videoHash = root.getString("video_hash"),
translations = translations,
)
} finally {
connection.disconnect()
}
}
}

View File

@@ -14,11 +14,15 @@ internal class TrainingScreenState {
var activeModule = Module.COURSE var activeModule = Module.COURSE
var currentCourse: TrainingVideoSummary? = null var currentCourse: TrainingVideoSummary? = null
var continuousPlaybackEnabled = false var continuousPlaybackEnabled = false
var chineseSubtitlesEnabled = false
var chineseTranslationLoading = false
var chineseTranslationError = ""
var testSubtitleVisible = false var testSubtitleVisible = false
var landscapeSubtitleVisible = true var landscapeSubtitleVisible = true
private var boundaryRequestGeneration = 0 private var boundaryRequestGeneration = 0
private var catalogRequestGeneration = 0 private var catalogRequestGeneration = 0
private var translationRequestGeneration = 0
fun beginBoundaryRequest(): Int = ++boundaryRequestGeneration fun beginBoundaryRequest(): Int = ++boundaryRequestGeneration
@@ -30,6 +34,11 @@ internal class TrainingScreenState {
fun isCurrentCatalogRequest(generation: Int): Boolean = fun isCurrentCatalogRequest(generation: Int): Boolean =
catalogRequestGeneration == generation catalogRequestGeneration == generation
fun beginTranslationRequest(): Int = ++translationRequestGeneration
fun isCurrentTranslationRequest(generation: Int, itemId: String): Boolean =
translationRequestGeneration == generation && activeItemId == itemId
fun clearCourse() { fun clearCourse() {
currentCourse = null currentCourse = null
activeItemId = "" activeItemId = ""

View File

@@ -155,7 +155,14 @@ internal class PlayerChrome(
textSize = 22f textSize = 22f
typeface = Typeface.DEFAULT_BOLD typeface = Typeface.DEFAULT_BOLD
setLineSpacing(2.dp.toFloat(), 1.05f) setLineSpacing(2.dp.toFloat(), 1.05f)
setPadding(0, 10.dp, 0, 10.dp) setPadding(0, 10.dp, 0, 4.dp)
}
bindings.sentenceZhText = TextView(activity).apply {
setTextColor(if (translucent) SampleTheme.textOverlay else SampleTheme.textSubtle)
textSize = 16f
setLineSpacing(2.dp.toFloat(), 1.05f)
setPadding(0, 0, 0, 8.dp)
visibility = View.GONE
} }
bindings.progressBar = SeekBar(activity).apply { bindings.progressBar = SeekBar(activity).apply {
max = PROGRESS_MAX max = PROGRESS_MAX
@@ -202,13 +209,30 @@ internal class PlayerChrome(
} }
if (screenState.activeModule == Module.TRAIN) { if (screenState.activeModule == Module.TRAIN) {
addView(createContinuousPlaybackRow(translucent)) addView(createContinuousPlaybackRow(translucent))
addView(createChineseSubtitleRow(translucent))
} }
addView(bindings.sentenceMetaText) addView(bindings.sentenceMetaText)
addView(LinearLayout(activity).apply { addView(LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL orientation = LinearLayout.HORIZONTAL
gravity = Gravity.TOP gravity = Gravity.TOP
addView(
LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
addView( addView(
bindings.sentenceText, bindings.sentenceText,
LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
),
)
addView(
bindings.sentenceZhText,
LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
),
)
},
LinearLayout.LayoutParams( LinearLayout.LayoutParams(
0, 0,
ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT,
@@ -349,6 +373,29 @@ internal class PlayerChrome(
}) })
} }
private fun createChineseSubtitleRow(translucent: Boolean): View =
LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
setPadding(0, 6.dp, 0, 0)
addView(TextView(activity).apply {
text = "中文字幕"
setTextColor(if (translucent) SampleTheme.textOverlay else SampleTheme.textSubtle)
textSize = 13f
typeface = Typeface.DEFAULT_BOLD
layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f)
})
addView(Switch(activity).apply {
isChecked = screenState.chineseSubtitlesEnabled
buttonTintList = ColorStateList.valueOf(
if (translucent) SampleTheme.accentLight else SampleTheme.accent,
)
setOnCheckedChangeListener { _, checked ->
seekActions.onChineseSubtitlesChanged(checked)
}
})
}
private fun installLandscapeSwipeGesture(subtitleContainer: View) { private fun installLandscapeSwipeGesture(subtitleContainer: View) {
bindings.playerView.setVerticalSwipeListener { isSwipeUp -> bindings.playerView.setVerticalSwipeListener { isSwipeUp ->
if (isSwipeUp == screenState.landscapeSubtitleVisible) return@setVerticalSwipeListener if (isSwipeUp == screenState.landscapeSubtitleVisible) return@setVerticalSwipeListener

View File

@@ -15,6 +15,7 @@ enum AppConfig {
static let connectTimeout: TimeInterval = 15 static let connectTimeout: TimeInterval = 15
static let readTimeout: TimeInterval = 30 static let readTimeout: TimeInterval = 30
static let assessmentTimeout: TimeInterval = 180 static let assessmentTimeout: TimeInterval = 180
static let translationTimeout: TimeInterval = 120
static let sha256Pattern = #"^[0-9a-fA-F]{64}$"# static let sha256Pattern = #"^[0-9a-fA-F]{64}$"#

View File

@@ -7,6 +7,7 @@ struct SentenceBoundary: Identifiable, Equatable, Decodable {
let text: String? let text: String?
let language: String? let language: String?
let referenceSpeechDurationMs: Int64? let referenceSpeechDurationMs: Int64?
let zhText: String?
var id: Int { index } var id: Int { index }
@@ -15,6 +16,48 @@ struct SentenceBoundary: Identifiable, Equatable, Decodable {
case startMs = "start_ms" case startMs = "start_ms"
case endMs = "end_ms" case endMs = "end_ms"
case referenceSpeechDurationMs = "reference_speech_duration_ms" case referenceSpeechDurationMs = "reference_speech_duration_ms"
case zhText = "zh_text"
}
init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
index = try container.decode(Int.self, forKey: .index)
startMs = try container.decode(Int64.self, forKey: .startMs)
endMs = try container.decode(Int64.self, forKey: .endMs)
text = try container.decodeIfPresent(String.self, forKey: .text)
language = try container.decodeIfPresent(String.self, forKey: .language)
referenceSpeechDurationMs = try container.decodeIfPresent(Int64.self, forKey: .referenceSpeechDurationMs)
zhText = try container.decodeIfPresent(String.self, forKey: .zhText)
}
init(
index: Int,
startMs: Int64,
endMs: Int64,
text: String?,
language: String?,
referenceSpeechDurationMs: Int64?,
zhText: String? = nil
) {
self.index = index
self.startMs = startMs
self.endMs = endMs
self.text = text
self.language = language
self.referenceSpeechDurationMs = referenceSpeechDurationMs
self.zhText = zhText
}
func withZhText(_ zhText: String?) -> SentenceBoundary {
SentenceBoundary(
index: index,
startMs: startMs,
endMs: endMs,
text: text,
language: language,
referenceSpeechDurationMs: referenceSpeechDurationMs,
zhText: zhText
)
} }
} }

View File

@@ -46,6 +46,26 @@ struct SentenceExplanationResponse: Decodable {
} }
} }
struct SubtitleTranslationItem: Decodable {
let index: Int
let zhText: String
private enum CodingKeys: String, CodingKey {
case index
case zhText = "zh_text"
}
}
struct SubtitleTranslationResponse: Decodable {
let videoHash: String
let translations: [SubtitleTranslationItem]
private enum CodingKeys: String, CodingKey {
case translations
case videoHash = "video_hash"
}
}
extension SentenceBoundaryAPI { extension SentenceBoundaryAPI {
static func explain( static func explain(
_ sentence: SentenceBoundary, _ sentence: SentenceBoundary,
@@ -79,4 +99,32 @@ extension SentenceBoundaryAPI {
.decode(SentenceExplanationResponse.self, from: result.body) .decode(SentenceExplanationResponse.self, from: result.body)
.explanation .explanation
} }
static func translateSubtitles(videoHash: String) async throws -> [Int: String] {
let hash = videoHash.lowercased()
guard hash.range(of: AppConfig.sha256Pattern, options: .regularExpression) != nil else {
throw SentenceBoundaryAPIError.invalidHash
}
var headers: [String: String] = ["Accept": "application/json"]
if !AppConfig.authToken.isEmpty {
headers["Authorization"] = "Bearer \(AppConfig.authToken)"
}
let result = try await HTTPClient.execute(
baseURL: AppConfig.serverBaseURL,
path: "api/v1/videos/\(hash)/subtitle-translation",
method: "POST",
timeout: AppConfig.translationTimeout,
headers: headers
)
guard (200...299).contains(result.statusCode) else {
let detail = String(data: result.body, encoding: .utf8) ?? ""
throw HTTPClientError.badStatus(result.statusCode, detail)
}
let payload = try JSONDecoder().decode(SubtitleTranslationResponse.self, from: result.body)
var mapped: [Int: String] = [:]
for item in payload.translations where !item.zhText.isEmpty {
mapped[item.index] = item.zhText
}
return mapped
}
} }

View File

@@ -4,6 +4,8 @@ import SwiftUI
struct FullscreenPlayerView: View { struct FullscreenPlayerView: View {
@ObservedObject var controller: PlayerController @ObservedObject var controller: PlayerController
var showsLessonTitle: Bool = true var showsLessonTitle: Bool = true
var showChineseSubtitles: Bool = false
var chineseSubtitlePlaceholder: String = "正在翻译中文字幕…"
let onExit: () -> Void let onExit: () -> Void
@State private var subtitleVisible = true @State private var subtitleVisible = true
@@ -66,10 +68,18 @@ struct FullscreenPlayerView: View {
.font(.system(size: 13)) .font(.system(size: 13))
.foregroundColor(Theme.textOverlay) .foregroundColor(Theme.textOverlay)
HStack(alignment: .center, spacing: 8) { HStack(alignment: .center, spacing: 8) {
VStack(alignment: .leading, spacing: 4) {
Text(sentenceTextText) Text(sentenceTextText)
.font(.system(size: 15)) .font(.system(size: 15))
.foregroundColor(.white) .foregroundColor(.white)
.lineLimit(2) .lineLimit(2)
if let chineseSubtitleText = chineseSubtitleText {
Text(chineseSubtitleText)
.font(.system(size: 14))
.foregroundColor(Theme.textOverlay)
.lineLimit(2)
}
}
Spacer() Spacer()
Button { Button {
guard let sentence = controller.currentSentence, guard let sentence = controller.currentSentence,
@@ -203,6 +213,20 @@ struct FullscreenPlayerView: View {
} }
} }
private var chineseSubtitleText: String? {
guard showChineseSubtitles else {
return nil
}
if let zh = controller.currentSentence?.zhText?.trimmingCharacters(in: .whitespacesAndNewlines),
!zh.isEmpty {
return zh
}
if (controller.currentSentence?.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
return nil
}
return chineseSubtitlePlaceholder
}
private var statusText: String { private var statusText: String {
if let message = controller.cacheState.statusMessage { if let message = controller.cacheState.statusMessage {
return message return message

View File

@@ -7,6 +7,8 @@ struct PlayerSectionView: View {
@Binding var continuousPlayback: Bool @Binding var continuousPlayback: Bool
var showsSubtitle: Binding<Bool>? = .constant(true) var showsSubtitle: Binding<Bool>? = .constant(true)
var showsLessonTitle: Bool = true var showsLessonTitle: Bool = true
var showChineseSubtitles: Binding<Bool>? = nil
var chineseSubtitlePlaceholder: String = "正在翻译中文字幕…"
var onFullscreen: () -> Void var onFullscreen: () -> Void
@State private var progressDragging = false @State private var progressDragging = false
@@ -93,14 +95,34 @@ struct PlayerSectionView: View {
} }
.padding(.vertical, 2) .padding(.vertical, 2)
} }
if let showChineseSubtitles = showChineseSubtitles {
HStack(spacing: 0) {
Text("中文字幕")
.font(.system(size: 13, weight: .bold))
.foregroundColor(Theme.textSubtle)
Spacer()
Toggle("", isOn: showChineseSubtitles)
.labelsHidden()
.tint(Theme.accent)
}
.padding(.vertical, 2)
}
Text(sentenceMetaText) Text(sentenceMetaText)
.font(.system(size: 13)) .font(.system(size: 13))
.foregroundColor(Theme.textSubtle) .foregroundColor(Theme.textSubtle)
HStack(alignment: .center, spacing: 8) { HStack(alignment: .center, spacing: 8) {
VStack(alignment: .leading, spacing: 4) {
Text(sentenceTextText) Text(sentenceTextText)
.font(.system(size: 15)) .font(.system(size: 15))
.foregroundColor(.white) .foregroundColor(.white)
.lineLimit(4) .lineLimit(4)
if let chineseSubtitleText = chineseSubtitleText {
Text(chineseSubtitleText)
.font(.system(size: 14))
.foregroundColor(Theme.textMuted)
.lineLimit(4)
}
}
Spacer() Spacer()
Button { Button {
guard let sentence = controller.currentSentence, !(sentence.text ?? "").isEmpty else { guard let sentence = controller.currentSentence, !(sentence.text ?? "").isEmpty else {
@@ -205,6 +227,20 @@ struct PlayerSectionView: View {
} }
} }
private var chineseSubtitleText: String? {
guard showChineseSubtitles?.wrappedValue == true else {
return nil
}
if let zh = controller.currentSentence?.zhText?.trimmingCharacters(in: .whitespacesAndNewlines),
!zh.isEmpty {
return zh
}
if (controller.currentSentence?.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
return nil
}
return chineseSubtitlePlaceholder
}
private var statusText: String { private var statusText: String {
if let message = controller.cacheState.statusMessage { if let message = controller.cacheState.statusMessage {
return message return message

View File

@@ -9,6 +9,10 @@ struct RootView: View {
@StateObject private var historyManager = TrainingHistoryManager() @StateObject private var historyManager = TrainingHistoryManager()
@State private var activeModule: Module = .course @State private var activeModule: Module = .course
@State private var continuousPlayback = false @State private var continuousPlayback = false
@State private var showChineseSubtitles = false
@State private var chineseTranslationLoading = false
@State private var chineseTranslationError = ""
@State private var chineseTranslationTask: Task<Void, Never>?
@State private var showImporter = false @State private var showImporter = false
@State private var showFullscreenPlayer = false @State private var showFullscreenPlayer = false
@State private var courses: [Course] = [] @State private var courses: [Course] = []
@@ -53,6 +57,8 @@ struct RootView: View {
controller: controller, controller: controller,
catalogVM: catalogVM, catalogVM: catalogVM,
continuousPlayback: $continuousPlayback, continuousPlayback: $continuousPlayback,
showChineseSubtitles: $showChineseSubtitles,
chineseSubtitlePlaceholder: chineseSubtitlePlaceholder,
onFullscreen: enterFullscreen onFullscreen: enterFullscreen
) )
case .test: case .test:
@@ -88,9 +94,24 @@ struct RootView: View {
Task { await loadCourses() } Task { await loadCourses() }
} }
} }
.onChange(of: showChineseSubtitles) { enabled in
if enabled {
ensureChineseTranslations()
} else {
chineseTranslationTask?.cancel()
chineseTranslationLoading = false
}
}
.onChange(of: controller.currentItem?.id) { _ in
ensureChineseTranslations()
}
.onChange(of: controller.currentItem?.sentences.count) { _ in
ensureChineseTranslations()
}
.onChange(of: activeModule) { module in .onChange(of: activeModule) { module in
if module == .train { if module == .train {
controller.setContinuousPlayback(continuousPlayback) controller.setContinuousPlayback(continuousPlayback)
ensureChineseTranslations()
} else if module == .test { } else if module == .test {
// 与安卓测试页一致:按当前句窗口播放,避免滑进下一句。 // 与安卓测试页一致:按当前句窗口播放,避免滑进下一句。
controller.setContinuousPlayback(false) controller.setContinuousPlayback(false)
@@ -115,7 +136,9 @@ struct RootView: View {
.fullScreenCover(isPresented: $showFullscreenPlayer) { .fullScreenCover(isPresented: $showFullscreenPlayer) {
FullscreenPlayerView( FullscreenPlayerView(
controller: controller, controller: controller,
showsLessonTitle: activeModule != .train showsLessonTitle: activeModule != .train,
showChineseSubtitles: showChineseSubtitles,
chineseSubtitlePlaceholder: chineseSubtitlePlaceholder
) { ) {
exitFullscreen() exitFullscreen()
} }
@@ -593,4 +616,59 @@ struct RootView: View {
AppDelegate.lockOrientation(restored) AppDelegate.lockOrientation(restored)
showFullscreenPlayer = false showFullscreenPlayer = false
} }
private var chineseSubtitlePlaceholder: String {
if chineseTranslationLoading {
return "正在翻译中文字幕…"
}
if !chineseTranslationError.isEmpty {
return chineseTranslationError
}
return "暂无中文字幕"
}
private func ensureChineseTranslations() {
guard showChineseSubtitles, activeModule == .train else {
return
}
guard let item = controller.currentItem else {
return
}
let needsTranslation = item.sentences.contains {
!($0.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty &&
($0.zhText ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
}
guard needsTranslation else {
chineseTranslationLoading = false
chineseTranslationError = ""
return
}
chineseTranslationTask?.cancel()
chineseTranslationLoading = true
chineseTranslationError = ""
let mediaId = item.id
chineseTranslationTask = Task {
do {
let map = try await SentenceBoundaryAPI.translateSubtitles(videoHash: mediaId)
guard !Task.isCancelled else { return }
await MainActor.run {
guard controller.currentItem?.id == mediaId else { return }
let updated = (controller.currentItem?.sentences ?? []).map { sentence in
sentence.withZhText(map[sentence.index] ?? sentence.zhText)
}
controller.updateBoundaries(updated)
chineseTranslationLoading = false
chineseTranslationError = ""
controller.showTransientMessage("已加载中文字幕")
}
} catch {
guard !Task.isCancelled else { return }
await MainActor.run {
chineseTranslationLoading = false
chineseTranslationError = "中文字幕翻译失败"
controller.showTransientMessage("中文字幕翻译失败:\(error.localizedDescription)")
}
}
}
}
} }

View File

@@ -4,6 +4,8 @@ struct TrainingView: View {
@ObservedObject var controller: PlayerController @ObservedObject var controller: PlayerController
@ObservedObject var catalogVM: CatalogViewModel @ObservedObject var catalogVM: CatalogViewModel
@Binding var continuousPlayback: Bool @Binding var continuousPlayback: Bool
@Binding var showChineseSubtitles: Bool
var chineseSubtitlePlaceholder: String = "正在翻译中文字幕…"
let onFullscreen: () -> Void let onFullscreen: () -> Void
var body: some View { var body: some View {
@@ -14,6 +16,8 @@ struct TrainingView: View {
showsContinuousToggle: true, showsContinuousToggle: true,
continuousPlayback: $continuousPlayback, continuousPlayback: $continuousPlayback,
showsLessonTitle: false, showsLessonTitle: false,
showChineseSubtitles: $showChineseSubtitles,
chineseSubtitlePlaceholder: chineseSubtitlePlaceholder,
onFullscreen: onFullscreen onFullscreen: onFullscreen
) )
CatalogView(vm: catalogVM) CatalogView(vm: catalogVM)
@@ -28,5 +32,8 @@ struct TrainingView: View {
: "已关闭连续播放,将循环播放当前句子" : "已关闭连续播放,将循环播放当前句子"
) )
} }
.onChange(of: showChineseSubtitles) { enabled in
controller.showTransientMessage(enabled ? "已开启中文字幕" : "已关闭中文字幕")
}
} }
} }

View File

@@ -100,6 +100,11 @@ The endpoint requires a signed-in user and accepts only the sentence and course
context. `AI_API_URL` and `AI_API_KEY` remain on the server and are never sent context. `AI_API_URL` and `AI_API_KEY` remain on the server and are never sent
to the mobile clients. to the mobile clients.
`POST /api/v1/videos/{sha256}/subtitle-translation` translates the full English
transcript in one AI request, then stores one Chinese caption per sentence.
Later requests reuse the cached Chinese captions until the English sentences
change.
## Assessment ## Assessment
```bash ```bash

View File

@@ -1,7 +1,9 @@
"""AI-backed sentence explanations for the mobile clients.""" """AI-backed sentence explanations and subtitle translation."""
from dataclasses import dataclass from dataclasses import dataclass
import json
import logging import logging
import re
from urllib.parse import urlsplit from urllib.parse import urlsplit
import requests import requests
@@ -9,6 +11,10 @@ import requests
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_JSON_FENCE_RE = re.compile(r"^```(?:json)?\s*|\s*```$", re.IGNORECASE)
_NUMBERED_LINE_RE = re.compile(r"^\s*(\d+)[\.、::]\s*(.+?)\s*$")
_TRANSLATE_BATCH_CHARS = 12_000
class AITutorError(RuntimeError): class AITutorError(RuntimeError):
"""Raised when the configured AI service cannot answer a request.""" """Raised when the configured AI service cannot answer a request."""
@@ -60,9 +66,88 @@ class AITutor:
user_prompt = ( user_prompt = (
f"课程:{context}\n位置:{position}\n{language_note}句子:{sentence_text}" f"课程:{context}\n位置:{position}\n{language_note}句子:{sentence_text}"
) )
return self._complete(system_prompt, user_prompt, temperature=0.3)
def translate_sentences(
self,
sentences: list[tuple[int, str]],
*,
course_title: str = "",
) -> dict[int, str]:
if not self.available:
raise AITutorError("AI translation is not configured.")
if not sentences:
raise AITutorError("No sentences were provided for translation.")
translations: dict[int, str] = {}
numbered = [(index, text.strip()) for index, text in sentences if text.strip()]
if not numbered:
raise AITutorError("No sentence text was provided for translation.")
for batch, context_rows in _translation_batches(numbered):
translations.update(
self._translate_batch(
batch,
context_rows=context_rows,
course_title=course_title,
total_count=len(numbered),
)
)
missing = [index for index, _ in numbered if not translations.get(index, "").strip()]
if missing:
raise AITutorError("AI translation did not cover every sentence.")
return translations
def _translate_batch(
self,
batch: list[tuple[int, str]],
*,
context_rows: list[tuple[int, str]],
course_title: str,
total_count: int,
) -> dict[int, str]:
context = course_title.strip() or "口语训练课程"
display_offset = {index: position for position, (index, _) in enumerate(batch, start=1)}
system_prompt = (
"你是专业的影视字幕翻译。请结合全文语境,把指定的外语句子翻译成自然、口语化的简体中文。"
"必须一句对一句,不能合并、拆分、跳过或改写编号。"
"只输出 JSON 数组,数组长度必须等于待翻译句子数,第 i 项对应第 i 句中文。"
"不要输出解释、markdown 或编号。"
)
lines = [f"{display_offset[index]}. {text}" for index, text in batch]
user_parts = [
f"课程:{context}",
f"全片共 {total_count} 句,本次需要翻译 {len(batch)} 句。",
]
if context_rows:
user_parts.append("前文(只供理解语境,不要翻译):")
user_parts.extend(f"{index + 1}. {text}" for index, text in context_rows)
user_parts.append("待翻译句子:")
user_parts.extend(lines)
content = self._complete(
system_prompt,
"\n".join(user_parts),
temperature=0.2,
timeout_seconds=max(self.timeout_seconds, 90.0),
)
zh_rows = _parse_translation_payload(content, len(batch))
return {
index: zh_rows[offset - 1]
for index, offset in display_offset.items()
}
def _complete(
self,
system_prompt: str,
user_prompt: str,
*,
temperature: float,
timeout_seconds: float | None = None,
) -> str:
headers = {"Content-Type": "application/json"} headers = {"Content-Type": "application/json"}
if self.api_key: if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}" headers["Authorization"] = f"Bearer {self.api_key}"
timeout = self.timeout_seconds if timeout_seconds is None else timeout_seconds
try: try:
response = requests.post( response = requests.post(
self.request_url, self.request_url,
@@ -73,14 +158,14 @@ class AITutor:
{"role": "system", "content": system_prompt}, {"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}, {"role": "user", "content": user_prompt},
], ],
"temperature": 0.3, "temperature": temperature,
"stream": False, "stream": False,
}, },
timeout=self.timeout_seconds, timeout=timeout,
) )
except requests.RequestException as exc: except requests.RequestException as exc:
logger.exception( logger.exception(
"AI explanation request failed: endpoint=%s model=%s", "AI request failed: endpoint=%s model=%s",
self.request_url, self.request_url,
self.model, self.model,
) )
@@ -88,7 +173,7 @@ class AITutor:
if response.status_code < 200 or response.status_code >= 300: if response.status_code < 200 or response.status_code >= 300:
logger.warning( logger.warning(
"AI explanation returned HTTP %s: endpoint=%s model=%s body=%s", "AI service returned HTTP %s: endpoint=%s model=%s body=%s",
response.status_code, response.status_code,
self.request_url, self.request_url,
self.model, self.model,
@@ -97,19 +182,86 @@ class AITutor:
raise AITutorError(f"AI service returned HTTP {response.status_code}.") raise AITutorError(f"AI service returned HTTP {response.status_code}.")
try: try:
body = response.json() body = response.json()
explanation = body["choices"][0]["message"]["content"] content = body["choices"][0]["message"]["content"]
except (KeyError, IndexError, TypeError, ValueError) as exc: except (KeyError, IndexError, TypeError, ValueError) as exc:
logger.warning( logger.warning(
"AI explanation returned an unexpected body: endpoint=%s body=%s", "AI service returned an unexpected body: endpoint=%s body=%s",
self.request_url, self.request_url,
response.text[:1000], response.text[:1000],
) )
raise AITutorError("AI service returned an unexpected response.") from exc raise AITutorError("AI service returned an unexpected response.") from exc
if not isinstance(explanation, str) or not explanation.strip(): if not isinstance(content, str) or not content.strip():
logger.warning( logger.warning(
"AI explanation returned empty content: endpoint=%s body=%s", "AI service returned empty content: endpoint=%s body=%s",
self.request_url, self.request_url,
response.text[:1000], response.text[:1000],
) )
raise AITutorError("AI service returned an empty explanation.") raise AITutorError("AI service returned an empty explanation.")
return explanation.strip() return content.strip()
def _translation_batches(
sentences: list[tuple[int, str]],
) -> list[tuple[list[tuple[int, str]], list[tuple[int, str]]]]:
batches: list[list[tuple[int, str]]] = []
current: list[tuple[int, str]] = []
current_chars = 0
for item in sentences:
extra = len(item[1]) + 8
if current and current_chars + extra > _TRANSLATE_BATCH_CHARS:
batches.append(current)
current = []
current_chars = 0
current.append(item)
current_chars += extra
if current:
batches.append(current)
result: list[tuple[list[tuple[int, str]], list[tuple[int, str]]]] = []
consumed = 0
for batch in batches:
context_rows = sentences[max(0, consumed - 8):consumed]
result.append((batch, context_rows))
consumed += len(batch)
return result
def _parse_translation_payload(content: str, expected_count: int) -> list[str]:
text = _JSON_FENCE_RE.sub("", content.strip()).strip()
try:
data = json.loads(text)
parsed = _translations_from_json(data, expected_count)
if parsed is not None:
return parsed
except (TypeError, ValueError):
pass
numbered: dict[int, str] = {}
for line in text.splitlines():
match = _NUMBERED_LINE_RE.match(line)
if match is None:
continue
numbered[int(match.group(1))] = match.group(2).strip()
if len(numbered) == expected_count and all(
index in numbered and numbered[index] for index in range(1, expected_count + 1)
):
return [numbered[index] for index in range(1, expected_count + 1)]
raise AITutorError("AI translation returned an unexpected response.")
def _translations_from_json(data: object, expected_count: int) -> list[str] | None:
if isinstance(data, dict):
for key in ("translations", "sentences", "zh", "items"):
if key in data:
data = data[key]
break
if isinstance(data, list) and len(data) == expected_count:
if all(isinstance(item, str) and item.strip() for item in data):
return [item.strip() for item in data]
if all(isinstance(item, dict) for item in data):
values: list[str] = []
for item in data:
zh = item.get("zh_text") or item.get("zh") or item.get("text")
if not isinstance(zh, str) or not zh.strip():
return None
values.append(zh.strip())
return values
return None

View File

@@ -45,6 +45,8 @@ from .models import (
SentenceBoundaryDocument, SentenceBoundaryDocument,
SentenceExplanationRequest, SentenceExplanationRequest,
SentenceExplanationResponse, SentenceExplanationResponse,
SubtitleTranslationItem,
SubtitleTranslationResponse,
PasswordResetRequest, PasswordResetRequest,
SentenceTextUpdate, SentenceTextUpdate,
UserDubShareSummary, UserDubShareSummary,
@@ -333,6 +335,60 @@ def create_app(
raise HTTPException(status_code=503, detail=str(exc)) from exc raise HTTPException(status_code=503, detail=str(exc)) from exc
return SentenceExplanationResponse(explanation=explanation) return SentenceExplanationResponse(explanation=explanation)
@application.post(
"/api/v1/videos/{video_hash}/subtitle-translation",
response_model=SubtitleTranslationResponse,
)
def translate_subtitles(
video_hash: str = SHA256_PATH,
user: AuthenticatedUser = Depends(current_user),
) -> SubtitleTranslationResponse:
del user
document = video_repository.get_document(video_hash)
if document is None:
raise HTTPException(status_code=404, detail="Course is not available.")
payload = [
(sentence.index, sentence.text or "")
for sentence in document.sentences
if (sentence.text or "").strip()
]
if not payload:
raise HTTPException(status_code=422, detail="No sentence text is available to translate.")
cached = [
SubtitleTranslationItem(index=sentence.index, zh_text=sentence.zh_text)
for sentence in document.sentences
if sentence.zh_text and sentence.zh_text.strip()
]
complete = len(cached) == len(payload) and all(
(sentence.zh_text or "").strip()
for sentence in document.sentences
if (sentence.text or "").strip()
)
if complete:
return SubtitleTranslationResponse(
video_hash=document.video_hash,
translations=cached,
)
video = video_repository.get_video(video_hash)
try:
translated = application.state.ai_tutor.translate_sentences(
payload,
course_title=(video["title"] if video is not None else "") or "",
)
except AITutorError as exc:
raise HTTPException(status_code=503, detail=str(exc)) from exc
saved = video_repository.save_sentence_translations(video_hash, translated)
if saved is None:
raise HTTPException(status_code=404, detail="Course is not available.")
return SubtitleTranslationResponse(
video_hash=saved.video_hash,
translations=[
SubtitleTranslationItem(index=sentence.index, zh_text=sentence.zh_text)
for sentence in saved.sentences
if sentence.zh_text and sentence.zh_text.strip()
],
)
@application.post( @application.post(
"/api/v1/audio/sentence-boundaries", "/api/v1/audio/sentence-boundaries",
response_model=SentenceBoundaryDocument, response_model=SentenceBoundaryDocument,

View File

@@ -12,6 +12,7 @@ class SentenceBoundary(BaseModel):
end_ms: int = Field(gt=0) end_ms: int = Field(gt=0)
text: Optional[str] = None text: Optional[str] = None
language: Optional[str] = None language: Optional[str] = None
zh_text: Optional[str] = None
reference_speech_duration_ms: Optional[int] = Field(default=None, gt=0) reference_speech_duration_ms: Optional[int] = Field(default=None, gt=0)
@model_validator(mode="after") @model_validator(mode="after")
@@ -126,6 +127,20 @@ class SentenceExplanationResponse(BaseModel):
explanation: str = Field(min_length=1, max_length=6000) explanation: str = Field(min_length=1, max_length=6000)
class SubtitleTranslationItem(BaseModel):
model_config = ConfigDict(extra="forbid")
index: int = Field(ge=0)
zh_text: str = Field(min_length=1, max_length=4000)
class SubtitleTranslationResponse(BaseModel):
model_config = ConfigDict(extra="forbid")
video_hash: str
translations: List[SubtitleTranslationItem]
class SentenceBoundaryAdjust(BaseModel): class SentenceBoundaryAdjust(BaseModel):
model_config = ConfigDict(extra="forbid") model_config = ConfigDict(extra="forbid")

View File

@@ -56,6 +56,7 @@ class VideoRepository:
end_ms INTEGER NOT NULL, end_ms INTEGER NOT NULL,
text TEXT, text TEXT,
language TEXT, language TEXT,
zh_text TEXT,
reference_speech_duration_ms INTEGER, reference_speech_duration_ms INTEGER,
PRIMARY KEY (video_hash, sentence_index), PRIMARY KEY (video_hash, sentence_index),
FOREIGN KEY (video_hash) REFERENCES videos(video_hash) ON DELETE CASCADE FOREIGN KEY (video_hash) REFERENCES videos(video_hash) ON DELETE CASCADE
@@ -193,6 +194,14 @@ class VideoRepository:
if "remote_url" not in video_columns: if "remote_url" not in video_columns:
connection.execute("ALTER TABLE videos ADD COLUMN remote_url TEXT") connection.execute("ALTER TABLE videos ADD COLUMN remote_url TEXT")
with self._connect() as connection:
sentence_columns = {
row["name"]
for row in connection.execute("PRAGMA table_info(sentences)")
}
if "zh_text" not in sentence_columns:
connection.execute("ALTER TABLE sentences ADD COLUMN zh_text TEXT")
def upsert_upload( def upsert_upload(
self, self,
*, *,
@@ -431,17 +440,7 @@ class VideoRepository:
video_hash=video["video_hash"], video_hash=video["video_hash"],
duration_ms=video["duration_ms"], duration_ms=video["duration_ms"],
algorithm_version=video["algorithm_version"] or "unknown", algorithm_version=video["algorithm_version"] or "unknown",
sentences=[ sentences=[_sentence_from_row(row) for row in rows],
SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
for row in rows
],
) )
def update_sentence_text( def update_sentence_text(
@@ -462,6 +461,10 @@ class VideoRepository:
) )
if cursor.rowcount == 0: if cursor.rowcount == 0:
return None return None
connection.execute(
"UPDATE sentences SET zh_text = NULL WHERE video_hash = ?",
(normalized_hash,),
)
connection.execute( connection.execute(
"UPDATE videos SET updated_at = ? WHERE video_hash = ?", "UPDATE videos SET updated_at = ? WHERE video_hash = ?",
(utc_now(), normalized_hash), (utc_now(), normalized_hash),
@@ -470,14 +473,7 @@ class VideoRepository:
"SELECT * FROM sentences WHERE video_hash = ? AND sentence_index = ?", "SELECT * FROM sentences WHERE video_hash = ? AND sentence_index = ?",
(normalized_hash, sentence_index), (normalized_hash, sentence_index),
).fetchone() ).fetchone()
return SentenceBoundary( return _sentence_from_row(row)
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
def get_sentence( def get_sentence(
self, self,
@@ -495,14 +491,7 @@ class VideoRepository:
).fetchone() ).fetchone()
if row is None: if row is None:
return None return None
return SentenceBoundary( return _sentence_from_row(row)
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
def replace_sentence( def replace_sentence(
self, self,
@@ -614,6 +603,10 @@ class VideoRepository:
replacement.reference_speech_duration_ms, replacement.reference_speech_duration_ms,
), ),
) )
connection.execute(
"UPDATE sentences SET zh_text = NULL WHERE video_hash = ?",
(normalized_hash,),
)
connection.execute( connection.execute(
"UPDATE videos SET updated_at = ? WHERE video_hash = ?", "UPDATE videos SET updated_at = ? WHERE video_hash = ?",
(utc_now(), normalized_hash), (utc_now(), normalized_hash),
@@ -627,17 +620,7 @@ class VideoRepository:
(normalized_hash, sentence_index, sentence_index + len(rows)), (normalized_hash, sentence_index, sentence_index + len(rows)),
).fetchall() ).fetchall()
return [ return [_sentence_from_row(row) for row in result_rows]
SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
for row in result_rows
]
def merge_with_next_sentence( def merge_with_next_sentence(
@@ -727,6 +710,10 @@ class VideoRepository:
(old_index - 1, normalized_hash, old_index), (old_index - 1, normalized_hash, old_index),
) )
connection.execute(
"UPDATE sentences SET zh_text = NULL WHERE video_hash = ?",
(normalized_hash,),
)
connection.execute( connection.execute(
"UPDATE videos SET updated_at = ? WHERE video_hash = ?", "UPDATE videos SET updated_at = ? WHERE video_hash = ?",
(utc_now(), normalized_hash), (utc_now(), normalized_hash),
@@ -739,14 +726,7 @@ class VideoRepository:
(normalized_hash, sentence_index), (normalized_hash, sentence_index),
).fetchone() ).fetchone()
return SentenceBoundary( return _sentence_from_row(row)
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
def adjust_sentence_boundary( def adjust_sentence_boundary(
self, self,
@@ -851,17 +831,36 @@ class VideoRepository:
(normalized_hash, *updated_indexes), (normalized_hash, *updated_indexes),
).fetchall() ).fetchall()
return [ return [_sentence_from_row(row) for row in result_rows]
SentenceBoundary(
index=row["sentence_index"], def clear_sentence_translations(self, video_hash: str) -> None:
start_ms=row["start_ms"], with self._connect() as connection:
end_ms=row["end_ms"], connection.execute(
text=row["text"], "UPDATE sentences SET zh_text = NULL WHERE video_hash = ?",
language=row["language"], (normalize_video_hash(video_hash),),
reference_speech_duration_ms=row["reference_speech_duration_ms"],
) )
for row in result_rows
] def save_sentence_translations(
self,
video_hash: str,
translations: Dict[int, str],
) -> Optional[SentenceBoundaryDocument]:
normalized_hash = normalize_video_hash(video_hash)
with self._connect() as connection:
for index, zh_text in translations.items():
connection.execute(
"""
UPDATE sentences
SET zh_text = ?
WHERE video_hash = ? AND sentence_index = ?
""",
(zh_text.strip(), normalized_hash, index),
)
connection.execute(
"UPDATE videos SET updated_at = ? WHERE video_hash = ?",
(utc_now(), normalized_hash),
)
return self.get_document(normalized_hash)
def create_dub_share( def create_dub_share(
self, self,
@@ -1278,6 +1277,21 @@ class VideoRepository:
return video return video
def _sentence_from_row(row: sqlite3.Row) -> SentenceBoundary:
keys = set(row.keys())
zh_text = row["zh_text"] if "zh_text" in keys else None
return SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
zh_text=zh_text,
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
def _join_adjacent_sentence_text( def _join_adjacent_sentence_text(
left: str, left: str,
right: str, right: str,

View File

@@ -71,3 +71,34 @@ def test_ai_tutor_rejects_unexpected_response(monkeypatch):
with pytest.raises(AITutorError): with pytest.raises(AITutorError):
tutor.explain("Hello.") tutor.explain("Hello.")
from sentence_api.ai_tutor import _parse_translation_payload
def test_ai_tutor_translates_full_document(monkeypatch):
requests_payloads = []
def fake_post(endpoint, headers, json, timeout):
requests_payloads.append((endpoint, headers, json, timeout))
return FakeResponse(body={"choices": [{"message": {"content": '["你好。", "见到你很高兴。"]'}}]})
monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post)
tutor = AITutor("https://ai.test/v1", "secret", "test-model", 1.5)
translated = tutor.translate_sentences(
[(0, "Hello."), (1, "Nice to meet you.")],
course_title="Greetings",
)
assert translated == {0: "你好。", 1: "见到你很高兴。"}
_endpoint, _headers, payload, timeout = requests_payloads[0]
user_prompt = payload["messages"][1]["content"]
assert "Hello." in user_prompt
assert "Nice to meet you." in user_prompt
assert timeout == 90.0
def test_parse_translation_payload_accepts_fenced_json_and_numbered_lines():
assert _parse_translation_payload('```json\n["一", "二"]\n```', 2) == ["一", "二"]
assert _parse_translation_payload("1. 一\n2. 二", 2) == ["一", "二"]

View File

@@ -202,6 +202,72 @@ def test_sentence_explanation_uses_server_side_ai_client(tmp_path, monkeypatch):
] ]
def _ready_video(repository):
repository.upsert_upload(
video_hash=VIDEO_HASH,
title="Greetings",
filename="lesson.mp4",
stored_filename=f"{VIDEO_HASH}.mp4",
content_type="video/mp4",
size_bytes=5,
language="en",
)
repository.save_processing_result(
SentenceBoundaryDocument(
video_hash=VIDEO_HASH,
duration_ms=9000,
algorithm_version="test",
sentences=[
SentenceBoundary(index=0, start_ms=0, end_ms=4000, text="Hello.", language="en", reference_speech_duration_ms=3000),
SentenceBoundary(index=1, start_ms=4000, end_ms=9000, text="Nice to meet you.", language="en", reference_speech_duration_ms=4000),
],
),
None,
)
def test_subtitle_translation_requires_login(tmp_path):
client = make_client(tmp_path)
client.headers.clear()
response = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert response.status_code == 401
def test_subtitle_translation_uses_full_document_and_is_cached(tmp_path, monkeypatch):
client = make_client(tmp_path)
_ready_video(client.app.state.video_repository)
requested = []
def translate_sentences(sentences, **kwargs):
requested.append((sentences, kwargs))
return {index: f"中文{index}" for index, _text in sentences}
class StubAITutor:
pass
stub = StubAITutor()
stub.translate_sentences = translate_sentences
monkeypatch.setattr(client.app.state, "ai_tutor", stub)
response = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert response.status_code == 200, response.text
payload = response.json()
assert payload["video_hash"] == VIDEO_HASH
assert payload["translations"] == [
{"index": 0, "zh_text": "中文0"},
{"index": 1, "zh_text": "中文1"},
]
document = client.app.state.video_repository.get_document(VIDEO_HASH)
assert [sentence.zh_text for sentence in document.sentences] == ["中文0", "中文1"]
second = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert second.status_code == 200
assert second.json() == payload
assert len(requested) == 1
assert requested[0][0] == [(0, "Hello."), (1, "Nice to meet you.")]
def test_oss_video_url_is_used_for_catalog_and_content(tmp_path): def test_oss_video_url_is_used_for_catalog_and_content(tmp_path):
index_path = tmp_path / "boundaries.json" index_path = tmp_path / "boundaries.json"
index_path.write_text("{}", encoding="utf-8") index_path.write_text("{}", encoding="utf-8")