add chinese subtitles

This commit is contained in:
2026-10-03 12:14:17 +08:00
parent 138d446d73
commit 6bd64ee4a9
22 changed files with 922 additions and 77 deletions

View File

@@ -159,6 +159,11 @@ class SentenceBoundaryApi internal constructor(
} else {
sentence.getLong("reference_speech_duration_ms")
},
zhText = if (sentence.isNull("zh_text")) {
null
} else {
sentence.getString("zh_text").takeIf { it.isNotBlank() }
},
)
)
}

View File

@@ -10,6 +10,7 @@ data class SentenceBoundary @JvmOverloads constructor(
val text: String? = null,
val language: String? = null,
val referenceSpeechDurationMs: Long? = null,
val zhText: String? = null,
) {
init {
require(index >= 0) { "Sentence index must be non-negative." }

View File

@@ -40,6 +40,8 @@ class MainActivity : Activity() {
private lateinit var catalogController: CourseCatalogController
private lateinit var testSession: TestSessionController
private lateinit var explanationOverlay: SentenceExplanationOverlay
private lateinit var translationApi: SubtitleTranslationApi
private var translationRequest: CancellableTranslationRequest? = null
private var progressBarDragging = false
@@ -70,6 +72,9 @@ class MainActivity : Activity() {
authSessionStore.session?.token.orEmpty()
},
)
translationApi = SubtitleTranslationApi(this, SERVER_BASE_URL) {
authSessionStore.session?.token.orEmpty()
}
screenFactory = SampleScreenFactory(
activity = this,
@@ -175,6 +180,8 @@ class MainActivity : Activity() {
testSession.release()
controller.release()
explanationOverlay.release()
translationRequest?.cancel()
translationApi.release()
super.onDestroy()
}
@@ -255,6 +262,7 @@ class MainActivity : Activity() {
screenState.currentSentenceCount = item?.sentences?.size ?: 0
applyPlaybackSnapshot(controller.snapshot())
applySentence(controller.currentSentence())
ensureChineseTranslations()
screenFactory.setCatalogStatus(screenState.catalogStatus)
}
@@ -268,6 +276,7 @@ class MainActivity : Activity() {
bindings.lessonTitleText.text = item?.title ?: "未选择课程"
screenState.currentSentenceCount = item?.sentences?.size ?: 0
}
ensureChineseTranslations()
testSession.ensureHistoryForCurrentCourse()
if (screenState.activeModule == Module.TEST) {
testSession.refreshTestUi()
@@ -311,6 +320,7 @@ class MainActivity : Activity() {
bindings.sentenceMetaText.text = "暂无句子边界"
bindings.sentenceText.text = ""
bindings.explainSentenceButton?.isEnabled = false
applyChineseSubtitle(null)
return
}
val countText = if (screenState.currentSentenceCount > 0) {
@@ -322,11 +332,83 @@ class MainActivity : Activity() {
"第 ${sentence.index + 1}$countText 句 ${PlaybackFormat.time(sentence.startMs)}-${PlaybackFormat.time(sentence.endMs)}"
bindings.sentenceText.text = sentence.text?.takeIf { it.isNotBlank() } ?: "当前句子"
bindings.explainSentenceButton?.isEnabled = !sentence.text.isNullOrBlank()
applyChineseSubtitle(sentence)
if (screenState.activeModule == Module.TEST) {
testSession.refreshTestUi()
}
}
private fun applyChineseSubtitle(sentence: SentenceBoundary?) {
if (!bindings.hasPlayerChrome) return
val showChinese = screenState.activeModule == Module.TRAIN &&
screenState.chineseSubtitlesEnabled &&
sentence != null &&
!sentence.text.isNullOrBlank()
if (!showChinese || sentence == null) {
bindings.sentenceZhText.visibility = android.view.View.GONE
return
}
bindings.sentenceZhText.visibility = android.view.View.VISIBLE
bindings.sentenceZhText.text = when {
!sentence.zhText.isNullOrBlank() -> sentence.zhText
screenState.chineseTranslationLoading -> "正在翻译中文字幕…"
screenState.chineseTranslationError.isNotBlank() -> screenState.chineseTranslationError
else -> "暂无中文字幕"
}
}
private fun ensureChineseTranslations() {
if (screenState.activeModule != Module.TRAIN || !screenState.chineseSubtitlesEnabled) {
return
}
val item = controller.currentTrainingItem() ?: return
val sentences = item.sentences
val needsTranslation = sentences.any { !it.text.isNullOrBlank() && it.zhText.isNullOrBlank() }
if (!needsTranslation) {
screenState.chineseTranslationLoading = false
screenState.chineseTranslationError = ""
applyChineseSubtitle(controller.currentSentence())
return
}
val generation = screenState.beginTranslationRequest()
screenState.chineseTranslationLoading = true
screenState.chineseTranslationError = ""
applyChineseSubtitle(controller.currentSentence())
translationRequest?.cancel()
translationRequest = translationApi.translate(
item.id,
object : SubtitleTranslationCallback {
override fun onSuccess(result: SubtitleTranslationResult) {
if (!screenState.isCurrentTranslationRequest(generation, item.id)) return
screenState.chineseTranslationLoading = false
val current = controller.currentTrainingItem() ?: return
if (current.id != item.id) return
val updated = current.sentences.map { sentence ->
sentence.copy(zhText = result.translations[sentence.index] ?: sentence.zhText)
}
controller.updateSentences(current.id, updated)
screenState.courseBoundaries[current.id]?.let { cached ->
screenState.courseBoundaries[current.id] = cached.copy(sentences = updated)
}
applyChineseSubtitle(controller.currentSentence())
if (bindings.hasPlayerChrome) {
bindings.statusText.text = "已加载中文字幕"
}
}
override fun onError(error: Throwable) {
if (!screenState.isCurrentTranslationRequest(generation, item.id)) return
screenState.chineseTranslationLoading = false
screenState.chineseTranslationError = "中文字幕翻译失败"
applyChineseSubtitle(controller.currentSentence())
if (bindings.hasPlayerChrome) {
bindings.statusText.text = "中文字幕翻译失败:${error.message.orEmpty()}"
}
}
},
)
}
private fun screenActions() = object : SampleScreenActions {
override fun onModuleSelected(module: Module) = switchModule(module)
@@ -360,6 +442,24 @@ class MainActivity : Activity() {
}
}
override fun onChineseSubtitlesChanged(enabled: Boolean) {
screenState.chineseSubtitlesEnabled = enabled
if (bindings.hasPlayerChrome) {
bindings.statusText.text = if (enabled) {
"已开启中文字幕"
} else {
"已关闭中文字幕"
}
}
if (enabled) {
ensureChineseTranslations()
} else {
translationRequest?.cancel()
screenState.chineseTranslationLoading = false
applyChineseSubtitle(controller.currentSentence())
}
}
override fun onExplainCurrentSentence() {
val sentence = controller.currentSentence() ?: return
if (sentence.text.isNullOrBlank()) return

View File

@@ -13,6 +13,7 @@ internal class SampleBindings {
lateinit var lessonTitleText: TextView
lateinit var statusText: TextView
lateinit var sentenceText: TextView
lateinit var sentenceZhText: TextView
lateinit var sentenceMetaText: TextView
lateinit var timeText: TextView
lateinit var speedText: TextView
@@ -35,6 +36,7 @@ internal class SampleBindings {
get() = ::lessonTitleText.isInitialized &&
::statusText.isInitialized &&
::sentenceText.isInitialized &&
::sentenceZhText.isInitialized &&
::sentenceMetaText.isInitialized &&
::timeText.isInitialized &&
::speedText.isInitialized &&

View File

@@ -21,6 +21,7 @@ internal interface TrainScreenActions :
fun onSeekStart()
fun onSeekStop(progress: Int)
fun onContinuousPlaybackChanged(enabled: Boolean)
fun onChineseSubtitlesChanged(enabled: Boolean)
}
internal interface TestScreenActions :

View File

@@ -0,0 +1,104 @@
package cn.learningpad.oraltrainer.sample
import android.content.Context
import android.os.Handler
import android.os.Looper
import org.json.JSONObject
import java.io.IOException
import java.net.HttpURLConnection
import java.net.URL
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.Future
data class SubtitleTranslationResult(
val videoHash: String,
val translations: Map<Int, String>,
)
interface SubtitleTranslationCallback {
fun onSuccess(result: SubtitleTranslationResult)
fun onError(error: Throwable)
}
class CancellableTranslationRequest internal constructor(
private val future: Future<*>,
) {
fun cancel() {
future.cancel(true)
}
}
/** Calls the app server; AI credentials never live in the mobile app. */
internal class SubtitleTranslationApi(
context: Context,
private val baseUrl: String,
private val tokenProvider: () -> String,
) {
private val appContext = context.applicationContext
private val mainHandler = Handler(Looper.getMainLooper())
private val executor: ExecutorService = Executors.newCachedThreadPool()
fun translate(
videoHash: String,
callback: SubtitleTranslationCallback,
): CancellableTranslationRequest {
val future = executor.submit {
try {
val result = requestBlocking(videoHash)
if (!Thread.currentThread().isInterrupted) {
mainHandler.post { callback.onSuccess(result) }
}
} catch (error: Throwable) {
if (!Thread.currentThread().isInterrupted) {
mainHandler.post { callback.onError(error) }
}
}
}
return CancellableTranslationRequest(future)
}
fun release() {
executor.shutdownNow()
}
private fun requestBlocking(videoHash: String): SubtitleTranslationResult {
val endpoint = baseUrl.trimEnd('/') + "/api/v1/videos/$videoHash/subtitle-translation"
val connection = URL(endpoint).openConnection() as HttpURLConnection
try {
connection.requestMethod = "POST"
connection.connectTimeout = 15_000
connection.readTimeout = 120_000
connection.doOutput = false
connection.useCaches = false
connection.setRequestProperty("Accept", "application/json")
connection.setRequestProperty("User-Agent", "OralTrainerSample/0.1.5 (Android)")
tokenProvider().takeIf { it.isNotBlank() }?.let {
connection.setRequestProperty("Authorization", "Bearer $it")
}
val statusCode = connection.responseCode
val stream = if (statusCode in 200..299) connection.inputStream else connection.errorStream
val response = stream?.bufferedReader(Charsets.UTF_8)?.use { it.readText() }.orEmpty()
if (statusCode !in 200..299) {
throw IOException("Subtitle translation API returned HTTP $statusCode: $response")
}
val root = JSONObject(response)
val items = root.getJSONArray("translations")
val translations = buildMap {
for (index in 0 until items.length()) {
val item = items.getJSONObject(index)
val zh = item.optString("zh_text")
if (zh.isNotBlank()) {
put(item.getInt("index"), zh)
}
}
}
return SubtitleTranslationResult(
videoHash = root.getString("video_hash"),
translations = translations,
)
} finally {
connection.disconnect()
}
}
}

View File

@@ -14,11 +14,15 @@ internal class TrainingScreenState {
var activeModule = Module.COURSE
var currentCourse: TrainingVideoSummary? = null
var continuousPlaybackEnabled = false
var chineseSubtitlesEnabled = false
var chineseTranslationLoading = false
var chineseTranslationError = ""
var testSubtitleVisible = false
var landscapeSubtitleVisible = true
private var boundaryRequestGeneration = 0
private var catalogRequestGeneration = 0
private var translationRequestGeneration = 0
fun beginBoundaryRequest(): Int = ++boundaryRequestGeneration
@@ -30,6 +34,11 @@ internal class TrainingScreenState {
fun isCurrentCatalogRequest(generation: Int): Boolean =
catalogRequestGeneration == generation
fun beginTranslationRequest(): Int = ++translationRequestGeneration
fun isCurrentTranslationRequest(generation: Int, itemId: String): Boolean =
translationRequestGeneration == generation && activeItemId == itemId
fun clearCourse() {
currentCourse = null
activeItemId = ""

View File

@@ -155,7 +155,14 @@ internal class PlayerChrome(
textSize = 22f
typeface = Typeface.DEFAULT_BOLD
setLineSpacing(2.dp.toFloat(), 1.05f)
setPadding(0, 10.dp, 0, 10.dp)
setPadding(0, 10.dp, 0, 4.dp)
}
bindings.sentenceZhText = TextView(activity).apply {
setTextColor(if (translucent) SampleTheme.textOverlay else SampleTheme.textSubtle)
textSize = 16f
setLineSpacing(2.dp.toFloat(), 1.05f)
setPadding(0, 0, 0, 8.dp)
visibility = View.GONE
}
bindings.progressBar = SeekBar(activity).apply {
max = PROGRESS_MAX
@@ -202,13 +209,30 @@ internal class PlayerChrome(
}
if (screenState.activeModule == Module.TRAIN) {
addView(createContinuousPlaybackRow(translucent))
addView(createChineseSubtitleRow(translucent))
}
addView(bindings.sentenceMetaText)
addView(LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.TOP
addView(
bindings.sentenceText,
LinearLayout(activity).apply {
orientation = LinearLayout.VERTICAL
addView(
bindings.sentenceText,
LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
),
)
addView(
bindings.sentenceZhText,
LinearLayout.LayoutParams(
ViewGroup.LayoutParams.MATCH_PARENT,
ViewGroup.LayoutParams.WRAP_CONTENT,
),
)
},
LinearLayout.LayoutParams(
0,
ViewGroup.LayoutParams.WRAP_CONTENT,
@@ -349,6 +373,29 @@ internal class PlayerChrome(
})
}
private fun createChineseSubtitleRow(translucent: Boolean): View =
LinearLayout(activity).apply {
orientation = LinearLayout.HORIZONTAL
gravity = Gravity.CENTER_VERTICAL
setPadding(0, 6.dp, 0, 0)
addView(TextView(activity).apply {
text = "中文字幕"
setTextColor(if (translucent) SampleTheme.textOverlay else SampleTheme.textSubtle)
textSize = 13f
typeface = Typeface.DEFAULT_BOLD
layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f)
})
addView(Switch(activity).apply {
isChecked = screenState.chineseSubtitlesEnabled
buttonTintList = ColorStateList.valueOf(
if (translucent) SampleTheme.accentLight else SampleTheme.accent,
)
setOnCheckedChangeListener { _, checked ->
seekActions.onChineseSubtitlesChanged(checked)
}
})
}
private fun installLandscapeSwipeGesture(subtitleContainer: View) {
bindings.playerView.setVerticalSwipeListener { isSwipeUp ->
if (isSwipeUp == screenState.landscapeSubtitleVisible) return@setVerticalSwipeListener

View File

@@ -15,6 +15,7 @@ enum AppConfig {
static let connectTimeout: TimeInterval = 15
static let readTimeout: TimeInterval = 30
static let assessmentTimeout: TimeInterval = 180
static let translationTimeout: TimeInterval = 120
static let sha256Pattern = #"^[0-9a-fA-F]{64}$"#

View File

@@ -7,6 +7,7 @@ struct SentenceBoundary: Identifiable, Equatable, Decodable {
let text: String?
let language: String?
let referenceSpeechDurationMs: Int64?
let zhText: String?
var id: Int { index }
@@ -15,6 +16,48 @@ struct SentenceBoundary: Identifiable, Equatable, Decodable {
case startMs = "start_ms"
case endMs = "end_ms"
case referenceSpeechDurationMs = "reference_speech_duration_ms"
case zhText = "zh_text"
}
init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
index = try container.decode(Int.self, forKey: .index)
startMs = try container.decode(Int64.self, forKey: .startMs)
endMs = try container.decode(Int64.self, forKey: .endMs)
text = try container.decodeIfPresent(String.self, forKey: .text)
language = try container.decodeIfPresent(String.self, forKey: .language)
referenceSpeechDurationMs = try container.decodeIfPresent(Int64.self, forKey: .referenceSpeechDurationMs)
zhText = try container.decodeIfPresent(String.self, forKey: .zhText)
}
init(
index: Int,
startMs: Int64,
endMs: Int64,
text: String?,
language: String?,
referenceSpeechDurationMs: Int64?,
zhText: String? = nil
) {
self.index = index
self.startMs = startMs
self.endMs = endMs
self.text = text
self.language = language
self.referenceSpeechDurationMs = referenceSpeechDurationMs
self.zhText = zhText
}
func withZhText(_ zhText: String?) -> SentenceBoundary {
SentenceBoundary(
index: index,
startMs: startMs,
endMs: endMs,
text: text,
language: language,
referenceSpeechDurationMs: referenceSpeechDurationMs,
zhText: zhText
)
}
}

View File

@@ -46,6 +46,26 @@ struct SentenceExplanationResponse: Decodable {
}
}
struct SubtitleTranslationItem: Decodable {
let index: Int
let zhText: String
private enum CodingKeys: String, CodingKey {
case index
case zhText = "zh_text"
}
}
struct SubtitleTranslationResponse: Decodable {
let videoHash: String
let translations: [SubtitleTranslationItem]
private enum CodingKeys: String, CodingKey {
case translations
case videoHash = "video_hash"
}
}
extension SentenceBoundaryAPI {
static func explain(
_ sentence: SentenceBoundary,
@@ -79,4 +99,32 @@ extension SentenceBoundaryAPI {
.decode(SentenceExplanationResponse.self, from: result.body)
.explanation
}
static func translateSubtitles(videoHash: String) async throws -> [Int: String] {
let hash = videoHash.lowercased()
guard hash.range(of: AppConfig.sha256Pattern, options: .regularExpression) != nil else {
throw SentenceBoundaryAPIError.invalidHash
}
var headers: [String: String] = ["Accept": "application/json"]
if !AppConfig.authToken.isEmpty {
headers["Authorization"] = "Bearer \(AppConfig.authToken)"
}
let result = try await HTTPClient.execute(
baseURL: AppConfig.serverBaseURL,
path: "api/v1/videos/\(hash)/subtitle-translation",
method: "POST",
timeout: AppConfig.translationTimeout,
headers: headers
)
guard (200...299).contains(result.statusCode) else {
let detail = String(data: result.body, encoding: .utf8) ?? ""
throw HTTPClientError.badStatus(result.statusCode, detail)
}
let payload = try JSONDecoder().decode(SubtitleTranslationResponse.self, from: result.body)
var mapped: [Int: String] = [:]
for item in payload.translations where !item.zhText.isEmpty {
mapped[item.index] = item.zhText
}
return mapped
}
}

View File

@@ -4,6 +4,8 @@ import SwiftUI
struct FullscreenPlayerView: View {
@ObservedObject var controller: PlayerController
var showsLessonTitle: Bool = true
var showChineseSubtitles: Bool = false
var chineseSubtitlePlaceholder: String = "正在翻译中文字幕…"
let onExit: () -> Void
@State private var subtitleVisible = true
@@ -66,10 +68,18 @@ struct FullscreenPlayerView: View {
.font(.system(size: 13))
.foregroundColor(Theme.textOverlay)
HStack(alignment: .center, spacing: 8) {
Text(sentenceTextText)
.font(.system(size: 15))
.foregroundColor(.white)
.lineLimit(2)
VStack(alignment: .leading, spacing: 4) {
Text(sentenceTextText)
.font(.system(size: 15))
.foregroundColor(.white)
.lineLimit(2)
if let chineseSubtitleText = chineseSubtitleText {
Text(chineseSubtitleText)
.font(.system(size: 14))
.foregroundColor(Theme.textOverlay)
.lineLimit(2)
}
}
Spacer()
Button {
guard let sentence = controller.currentSentence,
@@ -203,6 +213,20 @@ struct FullscreenPlayerView: View {
}
}
private var chineseSubtitleText: String? {
guard showChineseSubtitles else {
return nil
}
if let zh = controller.currentSentence?.zhText?.trimmingCharacters(in: .whitespacesAndNewlines),
!zh.isEmpty {
return zh
}
if (controller.currentSentence?.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
return nil
}
return chineseSubtitlePlaceholder
}
private var statusText: String {
if let message = controller.cacheState.statusMessage {
return message

View File

@@ -7,6 +7,8 @@ struct PlayerSectionView: View {
@Binding var continuousPlayback: Bool
var showsSubtitle: Binding<Bool>? = .constant(true)
var showsLessonTitle: Bool = true
var showChineseSubtitles: Binding<Bool>? = nil
var chineseSubtitlePlaceholder: String = "正在翻译中文字幕…"
var onFullscreen: () -> Void
@State private var progressDragging = false
@@ -93,14 +95,34 @@ struct PlayerSectionView: View {
}
.padding(.vertical, 2)
}
if let showChineseSubtitles = showChineseSubtitles {
HStack(spacing: 0) {
Text("中文字幕")
.font(.system(size: 13, weight: .bold))
.foregroundColor(Theme.textSubtle)
Spacer()
Toggle("", isOn: showChineseSubtitles)
.labelsHidden()
.tint(Theme.accent)
}
.padding(.vertical, 2)
}
Text(sentenceMetaText)
.font(.system(size: 13))
.foregroundColor(Theme.textSubtle)
HStack(alignment: .center, spacing: 8) {
Text(sentenceTextText)
.font(.system(size: 15))
.foregroundColor(.white)
.lineLimit(4)
VStack(alignment: .leading, spacing: 4) {
Text(sentenceTextText)
.font(.system(size: 15))
.foregroundColor(.white)
.lineLimit(4)
if let chineseSubtitleText = chineseSubtitleText {
Text(chineseSubtitleText)
.font(.system(size: 14))
.foregroundColor(Theme.textMuted)
.lineLimit(4)
}
}
Spacer()
Button {
guard let sentence = controller.currentSentence, !(sentence.text ?? "").isEmpty else {
@@ -205,6 +227,20 @@ struct PlayerSectionView: View {
}
}
private var chineseSubtitleText: String? {
guard showChineseSubtitles?.wrappedValue == true else {
return nil
}
if let zh = controller.currentSentence?.zhText?.trimmingCharacters(in: .whitespacesAndNewlines),
!zh.isEmpty {
return zh
}
if (controller.currentSentence?.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
return nil
}
return chineseSubtitlePlaceholder
}
private var statusText: String {
if let message = controller.cacheState.statusMessage {
return message

View File

@@ -9,6 +9,10 @@ struct RootView: View {
@StateObject private var historyManager = TrainingHistoryManager()
@State private var activeModule: Module = .course
@State private var continuousPlayback = false
@State private var showChineseSubtitles = false
@State private var chineseTranslationLoading = false
@State private var chineseTranslationError = ""
@State private var chineseTranslationTask: Task<Void, Never>?
@State private var showImporter = false
@State private var showFullscreenPlayer = false
@State private var courses: [Course] = []
@@ -53,6 +57,8 @@ struct RootView: View {
controller: controller,
catalogVM: catalogVM,
continuousPlayback: $continuousPlayback,
showChineseSubtitles: $showChineseSubtitles,
chineseSubtitlePlaceholder: chineseSubtitlePlaceholder,
onFullscreen: enterFullscreen
)
case .test:
@@ -88,9 +94,24 @@ struct RootView: View {
Task { await loadCourses() }
}
}
.onChange(of: showChineseSubtitles) { enabled in
if enabled {
ensureChineseTranslations()
} else {
chineseTranslationTask?.cancel()
chineseTranslationLoading = false
}
}
.onChange(of: controller.currentItem?.id) { _ in
ensureChineseTranslations()
}
.onChange(of: controller.currentItem?.sentences.count) { _ in
ensureChineseTranslations()
}
.onChange(of: activeModule) { module in
if module == .train {
controller.setContinuousPlayback(continuousPlayback)
ensureChineseTranslations()
} else if module == .test {
// 与安卓测试页一致:按当前句窗口播放,避免滑进下一句。
controller.setContinuousPlayback(false)
@@ -115,7 +136,9 @@ struct RootView: View {
.fullScreenCover(isPresented: $showFullscreenPlayer) {
FullscreenPlayerView(
controller: controller,
showsLessonTitle: activeModule != .train
showsLessonTitle: activeModule != .train,
showChineseSubtitles: showChineseSubtitles,
chineseSubtitlePlaceholder: chineseSubtitlePlaceholder
) {
exitFullscreen()
}
@@ -593,4 +616,59 @@ struct RootView: View {
AppDelegate.lockOrientation(restored)
showFullscreenPlayer = false
}
private var chineseSubtitlePlaceholder: String {
if chineseTranslationLoading {
return "正在翻译中文字幕…"
}
if !chineseTranslationError.isEmpty {
return chineseTranslationError
}
return "暂无中文字幕"
}
private func ensureChineseTranslations() {
guard showChineseSubtitles, activeModule == .train else {
return
}
guard let item = controller.currentItem else {
return
}
let needsTranslation = item.sentences.contains {
!($0.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty &&
($0.zhText ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
}
guard needsTranslation else {
chineseTranslationLoading = false
chineseTranslationError = ""
return
}
chineseTranslationTask?.cancel()
chineseTranslationLoading = true
chineseTranslationError = ""
let mediaId = item.id
chineseTranslationTask = Task {
do {
let map = try await SentenceBoundaryAPI.translateSubtitles(videoHash: mediaId)
guard !Task.isCancelled else { return }
await MainActor.run {
guard controller.currentItem?.id == mediaId else { return }
let updated = (controller.currentItem?.sentences ?? []).map { sentence in
sentence.withZhText(map[sentence.index] ?? sentence.zhText)
}
controller.updateBoundaries(updated)
chineseTranslationLoading = false
chineseTranslationError = ""
controller.showTransientMessage("已加载中文字幕")
}
} catch {
guard !Task.isCancelled else { return }
await MainActor.run {
chineseTranslationLoading = false
chineseTranslationError = "中文字幕翻译失败"
controller.showTransientMessage("中文字幕翻译失败:\(error.localizedDescription)")
}
}
}
}
}

View File

@@ -4,6 +4,8 @@ struct TrainingView: View {
@ObservedObject var controller: PlayerController
@ObservedObject var catalogVM: CatalogViewModel
@Binding var continuousPlayback: Bool
@Binding var showChineseSubtitles: Bool
var chineseSubtitlePlaceholder: String = "正在翻译中文字幕…"
let onFullscreen: () -> Void
var body: some View {
@@ -14,6 +16,8 @@ struct TrainingView: View {
showsContinuousToggle: true,
continuousPlayback: $continuousPlayback,
showsLessonTitle: false,
showChineseSubtitles: $showChineseSubtitles,
chineseSubtitlePlaceholder: chineseSubtitlePlaceholder,
onFullscreen: onFullscreen
)
CatalogView(vm: catalogVM)
@@ -28,5 +32,8 @@ struct TrainingView: View {
: "已关闭连续播放,将循环播放当前句子"
)
}
.onChange(of: showChineseSubtitles) { enabled in
controller.showTransientMessage(enabled ? "已开启中文字幕" : "已关闭中文字幕")
}
}
}

View File

@@ -100,6 +100,11 @@ The endpoint requires a signed-in user and accepts only the sentence and course
context. `AI_API_URL` and `AI_API_KEY` remain on the server and are never sent
to the mobile clients.
`POST /api/v1/videos/{sha256}/subtitle-translation` translates the full English
transcript in one AI request, then stores one Chinese caption per sentence.
Later requests reuse the cached Chinese captions until the English sentences
change.
## Assessment
```bash

View File

@@ -1,7 +1,9 @@
"""AI-backed sentence explanations for the mobile clients."""
"""AI-backed sentence explanations and subtitle translation."""
from dataclasses import dataclass
import json
import logging
import re
from urllib.parse import urlsplit
import requests
@@ -9,6 +11,10 @@ import requests
logger = logging.getLogger(__name__)
_JSON_FENCE_RE = re.compile(r"^```(?:json)?\s*|\s*```$", re.IGNORECASE)
_NUMBERED_LINE_RE = re.compile(r"^\s*(\d+)[\.、::]\s*(.+?)\s*$")
_TRANSLATE_BATCH_CHARS = 12_000
class AITutorError(RuntimeError):
"""Raised when the configured AI service cannot answer a request."""
@@ -60,9 +66,88 @@ class AITutor:
user_prompt = (
f"课程:{context}\n位置:{position}\n{language_note}句子:{sentence_text}"
)
return self._complete(system_prompt, user_prompt, temperature=0.3)
def translate_sentences(
self,
sentences: list[tuple[int, str]],
*,
course_title: str = "",
) -> dict[int, str]:
if not self.available:
raise AITutorError("AI translation is not configured.")
if not sentences:
raise AITutorError("No sentences were provided for translation.")
translations: dict[int, str] = {}
numbered = [(index, text.strip()) for index, text in sentences if text.strip()]
if not numbered:
raise AITutorError("No sentence text was provided for translation.")
for batch, context_rows in _translation_batches(numbered):
translations.update(
self._translate_batch(
batch,
context_rows=context_rows,
course_title=course_title,
total_count=len(numbered),
)
)
missing = [index for index, _ in numbered if not translations.get(index, "").strip()]
if missing:
raise AITutorError("AI translation did not cover every sentence.")
return translations
def _translate_batch(
self,
batch: list[tuple[int, str]],
*,
context_rows: list[tuple[int, str]],
course_title: str,
total_count: int,
) -> dict[int, str]:
context = course_title.strip() or "口语训练课程"
display_offset = {index: position for position, (index, _) in enumerate(batch, start=1)}
system_prompt = (
"你是专业的影视字幕翻译。请结合全文语境,把指定的外语句子翻译成自然、口语化的简体中文。"
"必须一句对一句,不能合并、拆分、跳过或改写编号。"
"只输出 JSON 数组,数组长度必须等于待翻译句子数,第 i 项对应第 i 句中文。"
"不要输出解释、markdown 或编号。"
)
lines = [f"{display_offset[index]}. {text}" for index, text in batch]
user_parts = [
f"课程:{context}",
f"全片共 {total_count} 句,本次需要翻译 {len(batch)} 句。",
]
if context_rows:
user_parts.append("前文(只供理解语境,不要翻译):")
user_parts.extend(f"{index + 1}. {text}" for index, text in context_rows)
user_parts.append("待翻译句子:")
user_parts.extend(lines)
content = self._complete(
system_prompt,
"\n".join(user_parts),
temperature=0.2,
timeout_seconds=max(self.timeout_seconds, 90.0),
)
zh_rows = _parse_translation_payload(content, len(batch))
return {
index: zh_rows[offset - 1]
for index, offset in display_offset.items()
}
def _complete(
self,
system_prompt: str,
user_prompt: str,
*,
temperature: float,
timeout_seconds: float | None = None,
) -> str:
headers = {"Content-Type": "application/json"}
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
timeout = self.timeout_seconds if timeout_seconds is None else timeout_seconds
try:
response = requests.post(
self.request_url,
@@ -73,14 +158,14 @@ class AITutor:
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt},
],
"temperature": 0.3,
"temperature": temperature,
"stream": False,
},
timeout=self.timeout_seconds,
timeout=timeout,
)
except requests.RequestException as exc:
logger.exception(
"AI explanation request failed: endpoint=%s model=%s",
"AI request failed: endpoint=%s model=%s",
self.request_url,
self.model,
)
@@ -88,7 +173,7 @@ class AITutor:
if response.status_code < 200 or response.status_code >= 300:
logger.warning(
"AI explanation returned HTTP %s: endpoint=%s model=%s body=%s",
"AI service returned HTTP %s: endpoint=%s model=%s body=%s",
response.status_code,
self.request_url,
self.model,
@@ -97,19 +182,86 @@ class AITutor:
raise AITutorError(f"AI service returned HTTP {response.status_code}.")
try:
body = response.json()
explanation = body["choices"][0]["message"]["content"]
content = body["choices"][0]["message"]["content"]
except (KeyError, IndexError, TypeError, ValueError) as exc:
logger.warning(
"AI explanation returned an unexpected body: endpoint=%s body=%s",
"AI service returned an unexpected body: endpoint=%s body=%s",
self.request_url,
response.text[:1000],
)
raise AITutorError("AI service returned an unexpected response.") from exc
if not isinstance(explanation, str) or not explanation.strip():
if not isinstance(content, str) or not content.strip():
logger.warning(
"AI explanation returned empty content: endpoint=%s body=%s",
"AI service returned empty content: endpoint=%s body=%s",
self.request_url,
response.text[:1000],
)
raise AITutorError("AI service returned an empty explanation.")
return explanation.strip()
return content.strip()
def _translation_batches(
sentences: list[tuple[int, str]],
) -> list[tuple[list[tuple[int, str]], list[tuple[int, str]]]]:
batches: list[list[tuple[int, str]]] = []
current: list[tuple[int, str]] = []
current_chars = 0
for item in sentences:
extra = len(item[1]) + 8
if current and current_chars + extra > _TRANSLATE_BATCH_CHARS:
batches.append(current)
current = []
current_chars = 0
current.append(item)
current_chars += extra
if current:
batches.append(current)
result: list[tuple[list[tuple[int, str]], list[tuple[int, str]]]] = []
consumed = 0
for batch in batches:
context_rows = sentences[max(0, consumed - 8):consumed]
result.append((batch, context_rows))
consumed += len(batch)
return result
def _parse_translation_payload(content: str, expected_count: int) -> list[str]:
text = _JSON_FENCE_RE.sub("", content.strip()).strip()
try:
data = json.loads(text)
parsed = _translations_from_json(data, expected_count)
if parsed is not None:
return parsed
except (TypeError, ValueError):
pass
numbered: dict[int, str] = {}
for line in text.splitlines():
match = _NUMBERED_LINE_RE.match(line)
if match is None:
continue
numbered[int(match.group(1))] = match.group(2).strip()
if len(numbered) == expected_count and all(
index in numbered and numbered[index] for index in range(1, expected_count + 1)
):
return [numbered[index] for index in range(1, expected_count + 1)]
raise AITutorError("AI translation returned an unexpected response.")
def _translations_from_json(data: object, expected_count: int) -> list[str] | None:
if isinstance(data, dict):
for key in ("translations", "sentences", "zh", "items"):
if key in data:
data = data[key]
break
if isinstance(data, list) and len(data) == expected_count:
if all(isinstance(item, str) and item.strip() for item in data):
return [item.strip() for item in data]
if all(isinstance(item, dict) for item in data):
values: list[str] = []
for item in data:
zh = item.get("zh_text") or item.get("zh") or item.get("text")
if not isinstance(zh, str) or not zh.strip():
return None
values.append(zh.strip())
return values
return None

View File

@@ -45,6 +45,8 @@ from .models import (
SentenceBoundaryDocument,
SentenceExplanationRequest,
SentenceExplanationResponse,
SubtitleTranslationItem,
SubtitleTranslationResponse,
PasswordResetRequest,
SentenceTextUpdate,
UserDubShareSummary,
@@ -333,6 +335,60 @@ def create_app(
raise HTTPException(status_code=503, detail=str(exc)) from exc
return SentenceExplanationResponse(explanation=explanation)
@application.post(
"/api/v1/videos/{video_hash}/subtitle-translation",
response_model=SubtitleTranslationResponse,
)
def translate_subtitles(
video_hash: str = SHA256_PATH,
user: AuthenticatedUser = Depends(current_user),
) -> SubtitleTranslationResponse:
del user
document = video_repository.get_document(video_hash)
if document is None:
raise HTTPException(status_code=404, detail="Course is not available.")
payload = [
(sentence.index, sentence.text or "")
for sentence in document.sentences
if (sentence.text or "").strip()
]
if not payload:
raise HTTPException(status_code=422, detail="No sentence text is available to translate.")
cached = [
SubtitleTranslationItem(index=sentence.index, zh_text=sentence.zh_text)
for sentence in document.sentences
if sentence.zh_text and sentence.zh_text.strip()
]
complete = len(cached) == len(payload) and all(
(sentence.zh_text or "").strip()
for sentence in document.sentences
if (sentence.text or "").strip()
)
if complete:
return SubtitleTranslationResponse(
video_hash=document.video_hash,
translations=cached,
)
video = video_repository.get_video(video_hash)
try:
translated = application.state.ai_tutor.translate_sentences(
payload,
course_title=(video["title"] if video is not None else "") or "",
)
except AITutorError as exc:
raise HTTPException(status_code=503, detail=str(exc)) from exc
saved = video_repository.save_sentence_translations(video_hash, translated)
if saved is None:
raise HTTPException(status_code=404, detail="Course is not available.")
return SubtitleTranslationResponse(
video_hash=saved.video_hash,
translations=[
SubtitleTranslationItem(index=sentence.index, zh_text=sentence.zh_text)
for sentence in saved.sentences
if sentence.zh_text and sentence.zh_text.strip()
],
)
@application.post(
"/api/v1/audio/sentence-boundaries",
response_model=SentenceBoundaryDocument,

View File

@@ -12,6 +12,7 @@ class SentenceBoundary(BaseModel):
end_ms: int = Field(gt=0)
text: Optional[str] = None
language: Optional[str] = None
zh_text: Optional[str] = None
reference_speech_duration_ms: Optional[int] = Field(default=None, gt=0)
@model_validator(mode="after")
@@ -126,6 +127,20 @@ class SentenceExplanationResponse(BaseModel):
explanation: str = Field(min_length=1, max_length=6000)
class SubtitleTranslationItem(BaseModel):
model_config = ConfigDict(extra="forbid")
index: int = Field(ge=0)
zh_text: str = Field(min_length=1, max_length=4000)
class SubtitleTranslationResponse(BaseModel):
model_config = ConfigDict(extra="forbid")
video_hash: str
translations: List[SubtitleTranslationItem]
class SentenceBoundaryAdjust(BaseModel):
model_config = ConfigDict(extra="forbid")

View File

@@ -56,6 +56,7 @@ class VideoRepository:
end_ms INTEGER NOT NULL,
text TEXT,
language TEXT,
zh_text TEXT,
reference_speech_duration_ms INTEGER,
PRIMARY KEY (video_hash, sentence_index),
FOREIGN KEY (video_hash) REFERENCES videos(video_hash) ON DELETE CASCADE
@@ -193,6 +194,14 @@ class VideoRepository:
if "remote_url" not in video_columns:
connection.execute("ALTER TABLE videos ADD COLUMN remote_url TEXT")
with self._connect() as connection:
sentence_columns = {
row["name"]
for row in connection.execute("PRAGMA table_info(sentences)")
}
if "zh_text" not in sentence_columns:
connection.execute("ALTER TABLE sentences ADD COLUMN zh_text TEXT")
def upsert_upload(
self,
*,
@@ -431,17 +440,7 @@ class VideoRepository:
video_hash=video["video_hash"],
duration_ms=video["duration_ms"],
algorithm_version=video["algorithm_version"] or "unknown",
sentences=[
SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
for row in rows
],
sentences=[_sentence_from_row(row) for row in rows],
)
def update_sentence_text(
@@ -462,6 +461,10 @@ class VideoRepository:
)
if cursor.rowcount == 0:
return None
connection.execute(
"UPDATE sentences SET zh_text = NULL WHERE video_hash = ?",
(normalized_hash,),
)
connection.execute(
"UPDATE videos SET updated_at = ? WHERE video_hash = ?",
(utc_now(), normalized_hash),
@@ -470,14 +473,7 @@ class VideoRepository:
"SELECT * FROM sentences WHERE video_hash = ? AND sentence_index = ?",
(normalized_hash, sentence_index),
).fetchone()
return SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
return _sentence_from_row(row)
def get_sentence(
self,
@@ -495,14 +491,7 @@ class VideoRepository:
).fetchone()
if row is None:
return None
return SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
return _sentence_from_row(row)
def replace_sentence(
self,
@@ -614,6 +603,10 @@ class VideoRepository:
replacement.reference_speech_duration_ms,
),
)
connection.execute(
"UPDATE sentences SET zh_text = NULL WHERE video_hash = ?",
(normalized_hash,),
)
connection.execute(
"UPDATE videos SET updated_at = ? WHERE video_hash = ?",
(utc_now(), normalized_hash),
@@ -627,17 +620,7 @@ class VideoRepository:
(normalized_hash, sentence_index, sentence_index + len(rows)),
).fetchall()
return [
SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
for row in result_rows
]
return [_sentence_from_row(row) for row in result_rows]
def merge_with_next_sentence(
@@ -727,6 +710,10 @@ class VideoRepository:
(old_index - 1, normalized_hash, old_index),
)
connection.execute(
"UPDATE sentences SET zh_text = NULL WHERE video_hash = ?",
(normalized_hash,),
)
connection.execute(
"UPDATE videos SET updated_at = ? WHERE video_hash = ?",
(utc_now(), normalized_hash),
@@ -739,14 +726,7 @@ class VideoRepository:
(normalized_hash, sentence_index),
).fetchone()
return SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
return _sentence_from_row(row)
def adjust_sentence_boundary(
self,
@@ -851,17 +831,36 @@ class VideoRepository:
(normalized_hash, *updated_indexes),
).fetchall()
return [
SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
reference_speech_duration_ms=row["reference_speech_duration_ms"],
return [_sentence_from_row(row) for row in result_rows]
def clear_sentence_translations(self, video_hash: str) -> None:
with self._connect() as connection:
connection.execute(
"UPDATE sentences SET zh_text = NULL WHERE video_hash = ?",
(normalize_video_hash(video_hash),),
)
for row in result_rows
]
def save_sentence_translations(
self,
video_hash: str,
translations: Dict[int, str],
) -> Optional[SentenceBoundaryDocument]:
normalized_hash = normalize_video_hash(video_hash)
with self._connect() as connection:
for index, zh_text in translations.items():
connection.execute(
"""
UPDATE sentences
SET zh_text = ?
WHERE video_hash = ? AND sentence_index = ?
""",
(zh_text.strip(), normalized_hash, index),
)
connection.execute(
"UPDATE videos SET updated_at = ? WHERE video_hash = ?",
(utc_now(), normalized_hash),
)
return self.get_document(normalized_hash)
def create_dub_share(
self,
@@ -1278,6 +1277,21 @@ class VideoRepository:
return video
def _sentence_from_row(row: sqlite3.Row) -> SentenceBoundary:
keys = set(row.keys())
zh_text = row["zh_text"] if "zh_text" in keys else None
return SentenceBoundary(
index=row["sentence_index"],
start_ms=row["start_ms"],
end_ms=row["end_ms"],
text=row["text"],
language=row["language"],
zh_text=zh_text,
reference_speech_duration_ms=row["reference_speech_duration_ms"],
)
def _join_adjacent_sentence_text(
left: str,
right: str,

View File

@@ -71,3 +71,34 @@ def test_ai_tutor_rejects_unexpected_response(monkeypatch):
with pytest.raises(AITutorError):
tutor.explain("Hello.")
from sentence_api.ai_tutor import _parse_translation_payload
def test_ai_tutor_translates_full_document(monkeypatch):
requests_payloads = []
def fake_post(endpoint, headers, json, timeout):
requests_payloads.append((endpoint, headers, json, timeout))
return FakeResponse(body={"choices": [{"message": {"content": '["你好。", "见到你很高兴。"]'}}]})
monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post)
tutor = AITutor("https://ai.test/v1", "secret", "test-model", 1.5)
translated = tutor.translate_sentences(
[(0, "Hello."), (1, "Nice to meet you.")],
course_title="Greetings",
)
assert translated == {0: "你好。", 1: "见到你很高兴。"}
_endpoint, _headers, payload, timeout = requests_payloads[0]
user_prompt = payload["messages"][1]["content"]
assert "Hello." in user_prompt
assert "Nice to meet you." in user_prompt
assert timeout == 90.0
def test_parse_translation_payload_accepts_fenced_json_and_numbered_lines():
assert _parse_translation_payload('```json\n["一", "二"]\n```', 2) == ["一", "二"]
assert _parse_translation_payload("1. 一\n2. 二", 2) == ["一", "二"]

View File

@@ -202,6 +202,72 @@ def test_sentence_explanation_uses_server_side_ai_client(tmp_path, monkeypatch):
]
def _ready_video(repository):
repository.upsert_upload(
video_hash=VIDEO_HASH,
title="Greetings",
filename="lesson.mp4",
stored_filename=f"{VIDEO_HASH}.mp4",
content_type="video/mp4",
size_bytes=5,
language="en",
)
repository.save_processing_result(
SentenceBoundaryDocument(
video_hash=VIDEO_HASH,
duration_ms=9000,
algorithm_version="test",
sentences=[
SentenceBoundary(index=0, start_ms=0, end_ms=4000, text="Hello.", language="en", reference_speech_duration_ms=3000),
SentenceBoundary(index=1, start_ms=4000, end_ms=9000, text="Nice to meet you.", language="en", reference_speech_duration_ms=4000),
],
),
None,
)
def test_subtitle_translation_requires_login(tmp_path):
client = make_client(tmp_path)
client.headers.clear()
response = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert response.status_code == 401
def test_subtitle_translation_uses_full_document_and_is_cached(tmp_path, monkeypatch):
client = make_client(tmp_path)
_ready_video(client.app.state.video_repository)
requested = []
def translate_sentences(sentences, **kwargs):
requested.append((sentences, kwargs))
return {index: f"中文{index}" for index, _text in sentences}
class StubAITutor:
pass
stub = StubAITutor()
stub.translate_sentences = translate_sentences
monkeypatch.setattr(client.app.state, "ai_tutor", stub)
response = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert response.status_code == 200, response.text
payload = response.json()
assert payload["video_hash"] == VIDEO_HASH
assert payload["translations"] == [
{"index": 0, "zh_text": "中文0"},
{"index": 1, "zh_text": "中文1"},
]
document = client.app.state.video_repository.get_document(VIDEO_HASH)
assert [sentence.zh_text for sentence in document.sentences] == ["中文0", "中文1"]
second = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation")
assert second.status_code == 200
assert second.json() == payload
assert len(requested) == 1
assert requested[0][0] == [(0, "Hello."), (1, "Nice to meet you.")]
def test_oss_video_url_is_used_for_catalog_and_content(tmp_path):
index_path = tmp_path / "boundaries.json"
index_path.write_text("{}", encoding="utf-8")