From 6bd64ee4a98ed24f061c06acc0d801dccf72930a Mon Sep 17 00:00:00 2001 From: Shuming Liu Date: Sat, 3 Oct 2026 12:14:17 +0800 Subject: [PATCH] add chinese subtitles --- .../oraltrainer/sdk/SentenceBoundaryApi.kt | 5 + .../oraltrainer/sdk/TrainingModels.kt | 1 + .../oraltrainer/sample/MainActivity.kt | 100 ++++++++++ .../oraltrainer/sample/SampleBindings.kt | 2 + .../oraltrainer/sample/SampleScreenActions.kt | 1 + .../sample/SubtitleTranslationApi.kt | 104 +++++++++++ .../oraltrainer/sample/TrainingScreenState.kt | 9 + .../oraltrainer/sample/ui/PlayerChrome.kt | 51 +++++- ios/OralTrainer/Config.swift | 1 + ios/OralTrainer/Models/TrainingModels.swift | 43 +++++ .../Services/SentenceBoundaryAPI.swift | 48 +++++ .../Views/FullscreenPlayerView.swift | 32 +++- ios/OralTrainer/Views/PlayerSectionView.swift | 44 ++++- ios/OralTrainer/Views/RootView.swift | 80 +++++++- ios/OralTrainer/Views/TrainingView.swift | 7 + sentence_api/README.md | 5 + sentence_api/ai_tutor.py | 172 +++++++++++++++++- sentence_api/main.py | 56 ++++++ sentence_api/models.py | 15 ++ sentence_api/repository.py | 126 +++++++------ sentence_api/tests/test_ai_tutor.py | 31 ++++ sentence_api/tests/test_api.py | 66 +++++++ 22 files changed, 922 insertions(+), 77 deletions(-) create mode 100644 android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SubtitleTranslationApi.kt diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/SentenceBoundaryApi.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/SentenceBoundaryApi.kt index f7252d0..165c0f5 100644 --- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/SentenceBoundaryApi.kt +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/SentenceBoundaryApi.kt @@ -159,6 +159,11 @@ class SentenceBoundaryApi internal constructor( } else { sentence.getLong("reference_speech_duration_ms") }, + zhText = if (sentence.isNull("zh_text")) { + null + } else { + sentence.getString("zh_text").takeIf { it.isNotBlank() } + }, ) ) } diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/TrainingModels.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/TrainingModels.kt index 736eb4e..af219d6 100644 --- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/TrainingModels.kt +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/TrainingModels.kt @@ -10,6 +10,7 @@ data class SentenceBoundary @JvmOverloads constructor( val text: String? = null, val language: String? = null, val referenceSpeechDurationMs: Long? = null, + val zhText: String? = null, ) { init { require(index >= 0) { "Sentence index must be non-negative." } diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt index 6b58036..95d7384 100644 --- a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt @@ -40,6 +40,8 @@ class MainActivity : Activity() { private lateinit var catalogController: CourseCatalogController private lateinit var testSession: TestSessionController private lateinit var explanationOverlay: SentenceExplanationOverlay + private lateinit var translationApi: SubtitleTranslationApi + private var translationRequest: CancellableTranslationRequest? = null private var progressBarDragging = false @@ -70,6 +72,9 @@ class MainActivity : Activity() { authSessionStore.session?.token.orEmpty() }, ) + translationApi = SubtitleTranslationApi(this, SERVER_BASE_URL) { + authSessionStore.session?.token.orEmpty() + } screenFactory = SampleScreenFactory( activity = this, @@ -175,6 +180,8 @@ class MainActivity : Activity() { testSession.release() controller.release() explanationOverlay.release() + translationRequest?.cancel() + translationApi.release() super.onDestroy() } @@ -255,6 +262,7 @@ class MainActivity : Activity() { screenState.currentSentenceCount = item?.sentences?.size ?: 0 applyPlaybackSnapshot(controller.snapshot()) applySentence(controller.currentSentence()) + ensureChineseTranslations() screenFactory.setCatalogStatus(screenState.catalogStatus) } @@ -268,6 +276,7 @@ class MainActivity : Activity() { bindings.lessonTitleText.text = item?.title ?: "未选择课程" screenState.currentSentenceCount = item?.sentences?.size ?: 0 } + ensureChineseTranslations() testSession.ensureHistoryForCurrentCourse() if (screenState.activeModule == Module.TEST) { testSession.refreshTestUi() @@ -311,6 +320,7 @@ class MainActivity : Activity() { bindings.sentenceMetaText.text = "暂无句子边界" bindings.sentenceText.text = "" bindings.explainSentenceButton?.isEnabled = false + applyChineseSubtitle(null) return } val countText = if (screenState.currentSentenceCount > 0) { @@ -322,11 +332,83 @@ class MainActivity : Activity() { "第 ${sentence.index + 1}$countText 句 ${PlaybackFormat.time(sentence.startMs)}-${PlaybackFormat.time(sentence.endMs)}" bindings.sentenceText.text = sentence.text?.takeIf { it.isNotBlank() } ?: "当前句子" bindings.explainSentenceButton?.isEnabled = !sentence.text.isNullOrBlank() + applyChineseSubtitle(sentence) if (screenState.activeModule == Module.TEST) { testSession.refreshTestUi() } } + private fun applyChineseSubtitle(sentence: SentenceBoundary?) { + if (!bindings.hasPlayerChrome) return + val showChinese = screenState.activeModule == Module.TRAIN && + screenState.chineseSubtitlesEnabled && + sentence != null && + !sentence.text.isNullOrBlank() + if (!showChinese || sentence == null) { + bindings.sentenceZhText.visibility = android.view.View.GONE + return + } + bindings.sentenceZhText.visibility = android.view.View.VISIBLE + bindings.sentenceZhText.text = when { + !sentence.zhText.isNullOrBlank() -> sentence.zhText + screenState.chineseTranslationLoading -> "正在翻译中文字幕…" + screenState.chineseTranslationError.isNotBlank() -> screenState.chineseTranslationError + else -> "暂无中文字幕" + } + } + + private fun ensureChineseTranslations() { + if (screenState.activeModule != Module.TRAIN || !screenState.chineseSubtitlesEnabled) { + return + } + val item = controller.currentTrainingItem() ?: return + val sentences = item.sentences + val needsTranslation = sentences.any { !it.text.isNullOrBlank() && it.zhText.isNullOrBlank() } + if (!needsTranslation) { + screenState.chineseTranslationLoading = false + screenState.chineseTranslationError = "" + applyChineseSubtitle(controller.currentSentence()) + return + } + val generation = screenState.beginTranslationRequest() + screenState.chineseTranslationLoading = true + screenState.chineseTranslationError = "" + applyChineseSubtitle(controller.currentSentence()) + translationRequest?.cancel() + translationRequest = translationApi.translate( + item.id, + object : SubtitleTranslationCallback { + override fun onSuccess(result: SubtitleTranslationResult) { + if (!screenState.isCurrentTranslationRequest(generation, item.id)) return + screenState.chineseTranslationLoading = false + val current = controller.currentTrainingItem() ?: return + if (current.id != item.id) return + val updated = current.sentences.map { sentence -> + sentence.copy(zhText = result.translations[sentence.index] ?: sentence.zhText) + } + controller.updateSentences(current.id, updated) + screenState.courseBoundaries[current.id]?.let { cached -> + screenState.courseBoundaries[current.id] = cached.copy(sentences = updated) + } + applyChineseSubtitle(controller.currentSentence()) + if (bindings.hasPlayerChrome) { + bindings.statusText.text = "已加载中文字幕" + } + } + + override fun onError(error: Throwable) { + if (!screenState.isCurrentTranslationRequest(generation, item.id)) return + screenState.chineseTranslationLoading = false + screenState.chineseTranslationError = "中文字幕翻译失败" + applyChineseSubtitle(controller.currentSentence()) + if (bindings.hasPlayerChrome) { + bindings.statusText.text = "中文字幕翻译失败：${error.message.orEmpty()}" + } + } + }, + ) + } + private fun screenActions() = object : SampleScreenActions { override fun onModuleSelected(module: Module) = switchModule(module) @@ -360,6 +442,24 @@ class MainActivity : Activity() { } } + override fun onChineseSubtitlesChanged(enabled: Boolean) { + screenState.chineseSubtitlesEnabled = enabled + if (bindings.hasPlayerChrome) { + bindings.statusText.text = if (enabled) { + "已开启中文字幕" + } else { + "已关闭中文字幕" + } + } + if (enabled) { + ensureChineseTranslations() + } else { + translationRequest?.cancel() + screenState.chineseTranslationLoading = false + applyChineseSubtitle(controller.currentSentence()) + } + } + override fun onExplainCurrentSentence() { val sentence = controller.currentSentence() ?: return if (sentence.text.isNullOrBlank()) return diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleBindings.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleBindings.kt index c9dbc1d..913f099 100644 --- a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleBindings.kt +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleBindings.kt @@ -13,6 +13,7 @@ internal class SampleBindings { lateinit var lessonTitleText: TextView lateinit var statusText: TextView lateinit var sentenceText: TextView + lateinit var sentenceZhText: TextView lateinit var sentenceMetaText: TextView lateinit var timeText: TextView lateinit var speedText: TextView @@ -35,6 +36,7 @@ internal class SampleBindings { get() = ::lessonTitleText.isInitialized && ::statusText.isInitialized && ::sentenceText.isInitialized && + ::sentenceZhText.isInitialized && ::sentenceMetaText.isInitialized && ::timeText.isInitialized && ::speedText.isInitialized && diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleScreenActions.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleScreenActions.kt index dce1602..43d681e 100644 --- a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleScreenActions.kt +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleScreenActions.kt @@ -21,6 +21,7 @@ internal interface TrainScreenActions : fun onSeekStart() fun onSeekStop(progress: Int) fun onContinuousPlaybackChanged(enabled: Boolean) + fun onChineseSubtitlesChanged(enabled: Boolean) } internal interface TestScreenActions : diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SubtitleTranslationApi.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SubtitleTranslationApi.kt new file mode 100644 index 0000000..2ff8cea --- /dev/null +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SubtitleTranslationApi.kt @@ -0,0 +1,104 @@ +package cn.learningpad.oraltrainer.sample + +import android.content.Context +import android.os.Handler +import android.os.Looper +import org.json.JSONObject +import java.io.IOException +import java.net.HttpURLConnection +import java.net.URL +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.Future + +data class SubtitleTranslationResult( + val videoHash: String, + val translations: Map, +) + +interface SubtitleTranslationCallback { + fun onSuccess(result: SubtitleTranslationResult) + fun onError(error: Throwable) +} + +class CancellableTranslationRequest internal constructor( + private val future: Future<*>, +) { + fun cancel() { + future.cancel(true) + } +} + +/** Calls the app server; AI credentials never live in the mobile app. */ +internal class SubtitleTranslationApi( + context: Context, + private val baseUrl: String, + private val tokenProvider: () -> String, +) { + private val appContext = context.applicationContext + private val mainHandler = Handler(Looper.getMainLooper()) + private val executor: ExecutorService = Executors.newCachedThreadPool() + + fun translate( + videoHash: String, + callback: SubtitleTranslationCallback, + ): CancellableTranslationRequest { + val future = executor.submit { + try { + val result = requestBlocking(videoHash) + if (!Thread.currentThread().isInterrupted) { + mainHandler.post { callback.onSuccess(result) } + } + } catch (error: Throwable) { + if (!Thread.currentThread().isInterrupted) { + mainHandler.post { callback.onError(error) } + } + } + } + return CancellableTranslationRequest(future) + } + + fun release() { + executor.shutdownNow() + } + + private fun requestBlocking(videoHash: String): SubtitleTranslationResult { + val endpoint = baseUrl.trimEnd('/') + "/api/v1/videos/$videoHash/subtitle-translation" + val connection = URL(endpoint).openConnection() as HttpURLConnection + try { + connection.requestMethod = "POST" + connection.connectTimeout = 15_000 + connection.readTimeout = 120_000 + connection.doOutput = false + connection.useCaches = false + connection.setRequestProperty("Accept", "application/json") + connection.setRequestProperty("User-Agent", "OralTrainerSample/0.1.5 (Android)") + tokenProvider().takeIf { it.isNotBlank() }?.let { + connection.setRequestProperty("Authorization", "Bearer $it") + } + val statusCode = connection.responseCode + val stream = if (statusCode in 200..299) connection.inputStream else connection.errorStream + val response = stream?.bufferedReader(Charsets.UTF_8)?.use { it.readText() }.orEmpty() + if (statusCode !in 200..299) { + throw IOException("Subtitle translation API returned HTTP $statusCode: $response") + } + val root = JSONObject(response) + val items = root.getJSONArray("translations") + val translations = buildMap { + for (index in 0 until items.length()) { + val item = items.getJSONObject(index) + val zh = item.optString("zh_text") + if (zh.isNotBlank()) { + put(item.getInt("index"), zh) + } + } + } + return SubtitleTranslationResult( + videoHash = root.getString("video_hash"), + translations = translations, + ) + } finally { + connection.disconnect() + } + } +} diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/TrainingScreenState.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/TrainingScreenState.kt index 05a1bcc..7112a30 100644 --- a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/TrainingScreenState.kt +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/TrainingScreenState.kt @@ -14,11 +14,15 @@ internal class TrainingScreenState { var activeModule = Module.COURSE var currentCourse: TrainingVideoSummary? = null var continuousPlaybackEnabled = false + var chineseSubtitlesEnabled = false + var chineseTranslationLoading = false + var chineseTranslationError = "" var testSubtitleVisible = false var landscapeSubtitleVisible = true private var boundaryRequestGeneration = 0 private var catalogRequestGeneration = 0 + private var translationRequestGeneration = 0 fun beginBoundaryRequest(): Int = ++boundaryRequestGeneration @@ -30,6 +34,11 @@ internal class TrainingScreenState { fun isCurrentCatalogRequest(generation: Int): Boolean = catalogRequestGeneration == generation + fun beginTranslationRequest(): Int = ++translationRequestGeneration + + fun isCurrentTranslationRequest(generation: Int, itemId: String): Boolean = + translationRequestGeneration == generation && activeItemId == itemId + fun clearCourse() { currentCourse = null activeItemId = "" diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/ui/PlayerChrome.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/ui/PlayerChrome.kt index a3193ff..7a798e0 100644 --- a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/ui/PlayerChrome.kt +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/ui/PlayerChrome.kt @@ -155,7 +155,14 @@ internal class PlayerChrome( textSize = 22f typeface = Typeface.DEFAULT_BOLD setLineSpacing(2.dp.toFloat(), 1.05f) - setPadding(0, 10.dp, 0, 10.dp) + setPadding(0, 10.dp, 0, 4.dp) + } + bindings.sentenceZhText = TextView(activity).apply { + setTextColor(if (translucent) SampleTheme.textOverlay else SampleTheme.textSubtle) + textSize = 16f + setLineSpacing(2.dp.toFloat(), 1.05f) + setPadding(0, 0, 0, 8.dp) + visibility = View.GONE } bindings.progressBar = SeekBar(activity).apply { max = PROGRESS_MAX @@ -202,13 +209,30 @@ internal class PlayerChrome( } if (screenState.activeModule == Module.TRAIN) { addView(createContinuousPlaybackRow(translucent)) + addView(createChineseSubtitleRow(translucent)) } addView(bindings.sentenceMetaText) addView(LinearLayout(activity).apply { orientation = LinearLayout.HORIZONTAL gravity = Gravity.TOP addView( - bindings.sentenceText, + LinearLayout(activity).apply { + orientation = LinearLayout.VERTICAL + addView( + bindings.sentenceText, + LinearLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + ViewGroup.LayoutParams.WRAP_CONTENT, + ), + ) + addView( + bindings.sentenceZhText, + LinearLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + ViewGroup.LayoutParams.WRAP_CONTENT, + ), + ) + }, LinearLayout.LayoutParams( 0, ViewGroup.LayoutParams.WRAP_CONTENT, @@ -349,6 +373,29 @@ internal class PlayerChrome( }) } + private fun createChineseSubtitleRow(translucent: Boolean): View = + LinearLayout(activity).apply { + orientation = LinearLayout.HORIZONTAL + gravity = Gravity.CENTER_VERTICAL + setPadding(0, 6.dp, 0, 0) + addView(TextView(activity).apply { + text = "中文字幕" + setTextColor(if (translucent) SampleTheme.textOverlay else SampleTheme.textSubtle) + textSize = 13f + typeface = Typeface.DEFAULT_BOLD + layoutParams = LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f) + }) + addView(Switch(activity).apply { + isChecked = screenState.chineseSubtitlesEnabled + buttonTintList = ColorStateList.valueOf( + if (translucent) SampleTheme.accentLight else SampleTheme.accent, + ) + setOnCheckedChangeListener { _, checked -> + seekActions.onChineseSubtitlesChanged(checked) + } + }) + } + private fun installLandscapeSwipeGesture(subtitleContainer: View) { bindings.playerView.setVerticalSwipeListener { isSwipeUp -> if (isSwipeUp == screenState.landscapeSubtitleVisible) return@setVerticalSwipeListener diff --git a/ios/OralTrainer/Config.swift b/ios/OralTrainer/Config.swift index 30ba6c6..c29b678 100644 --- a/ios/OralTrainer/Config.swift +++ b/ios/OralTrainer/Config.swift @@ -15,6 +15,7 @@ enum AppConfig { static let connectTimeout: TimeInterval = 15 static let readTimeout: TimeInterval = 30 static let assessmentTimeout: TimeInterval = 180 + static let translationTimeout: TimeInterval = 120 static let sha256Pattern = #"^[0-9a-fA-F]{64}$"# diff --git a/ios/OralTrainer/Models/TrainingModels.swift b/ios/OralTrainer/Models/TrainingModels.swift index 99da78d..fd4c387 100644 --- a/ios/OralTrainer/Models/TrainingModels.swift +++ b/ios/OralTrainer/Models/TrainingModels.swift @@ -7,6 +7,7 @@ struct SentenceBoundary: Identifiable, Equatable, Decodable { let text: String? let language: String? let referenceSpeechDurationMs: Int64? + let zhText: String? var id: Int { index } @@ -15,6 +16,48 @@ struct SentenceBoundary: Identifiable, Equatable, Decodable { case startMs = "start_ms" case endMs = "end_ms" case referenceSpeechDurationMs = "reference_speech_duration_ms" + case zhText = "zh_text" + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + index = try container.decode(Int.self, forKey: .index) + startMs = try container.decode(Int64.self, forKey: .startMs) + endMs = try container.decode(Int64.self, forKey: .endMs) + text = try container.decodeIfPresent(String.self, forKey: .text) + language = try container.decodeIfPresent(String.self, forKey: .language) + referenceSpeechDurationMs = try container.decodeIfPresent(Int64.self, forKey: .referenceSpeechDurationMs) + zhText = try container.decodeIfPresent(String.self, forKey: .zhText) + } + + init( + index: Int, + startMs: Int64, + endMs: Int64, + text: String?, + language: String?, + referenceSpeechDurationMs: Int64?, + zhText: String? = nil + ) { + self.index = index + self.startMs = startMs + self.endMs = endMs + self.text = text + self.language = language + self.referenceSpeechDurationMs = referenceSpeechDurationMs + self.zhText = zhText + } + + func withZhText(_ zhText: String?) -> SentenceBoundary { + SentenceBoundary( + index: index, + startMs: startMs, + endMs: endMs, + text: text, + language: language, + referenceSpeechDurationMs: referenceSpeechDurationMs, + zhText: zhText + ) } } diff --git a/ios/OralTrainer/Services/SentenceBoundaryAPI.swift b/ios/OralTrainer/Services/SentenceBoundaryAPI.swift index e7690fb..d82db6d 100644 --- a/ios/OralTrainer/Services/SentenceBoundaryAPI.swift +++ b/ios/OralTrainer/Services/SentenceBoundaryAPI.swift @@ -46,6 +46,26 @@ struct SentenceExplanationResponse: Decodable { } } +struct SubtitleTranslationItem: Decodable { + let index: Int + let zhText: String + + private enum CodingKeys: String, CodingKey { + case index + case zhText = "zh_text" + } +} + +struct SubtitleTranslationResponse: Decodable { + let videoHash: String + let translations: [SubtitleTranslationItem] + + private enum CodingKeys: String, CodingKey { + case translations + case videoHash = "video_hash" + } +} + extension SentenceBoundaryAPI { static func explain( _ sentence: SentenceBoundary, @@ -79,4 +99,32 @@ extension SentenceBoundaryAPI { .decode(SentenceExplanationResponse.self, from: result.body) .explanation } + + static func translateSubtitles(videoHash: String) async throws -> [Int: String] { + let hash = videoHash.lowercased() + guard hash.range(of: AppConfig.sha256Pattern, options: .regularExpression) != nil else { + throw SentenceBoundaryAPIError.invalidHash + } + var headers: [String: String] = ["Accept": "application/json"] + if !AppConfig.authToken.isEmpty { + headers["Authorization"] = "Bearer \(AppConfig.authToken)" + } + let result = try await HTTPClient.execute( + baseURL: AppConfig.serverBaseURL, + path: "api/v1/videos/\(hash)/subtitle-translation", + method: "POST", + timeout: AppConfig.translationTimeout, + headers: headers + ) + guard (200...299).contains(result.statusCode) else { + let detail = String(data: result.body, encoding: .utf8) ?? "" + throw HTTPClientError.badStatus(result.statusCode, detail) + } + let payload = try JSONDecoder().decode(SubtitleTranslationResponse.self, from: result.body) + var mapped: [Int: String] = [:] + for item in payload.translations where !item.zhText.isEmpty { + mapped[item.index] = item.zhText + } + return mapped + } } diff --git a/ios/OralTrainer/Views/FullscreenPlayerView.swift b/ios/OralTrainer/Views/FullscreenPlayerView.swift index e4ab725..a2ccbd7 100644 --- a/ios/OralTrainer/Views/FullscreenPlayerView.swift +++ b/ios/OralTrainer/Views/FullscreenPlayerView.swift @@ -4,6 +4,8 @@ import SwiftUI struct FullscreenPlayerView: View { @ObservedObject var controller: PlayerController var showsLessonTitle: Bool = true + var showChineseSubtitles: Bool = false + var chineseSubtitlePlaceholder: String = "正在翻译中文字幕…" let onExit: () -> Void @State private var subtitleVisible = true @@ -66,10 +68,18 @@ struct FullscreenPlayerView: View { .font(.system(size: 13)) .foregroundColor(Theme.textOverlay) HStack(alignment: .center, spacing: 8) { - Text(sentenceTextText) - .font(.system(size: 15)) - .foregroundColor(.white) - .lineLimit(2) + VStack(alignment: .leading, spacing: 4) { + Text(sentenceTextText) + .font(.system(size: 15)) + .foregroundColor(.white) + .lineLimit(2) + if let chineseSubtitleText = chineseSubtitleText { + Text(chineseSubtitleText) + .font(.system(size: 14)) + .foregroundColor(Theme.textOverlay) + .lineLimit(2) + } + } Spacer() Button { guard let sentence = controller.currentSentence, @@ -203,6 +213,20 @@ struct FullscreenPlayerView: View { } } + private var chineseSubtitleText: String? { + guard showChineseSubtitles else { + return nil + } + if let zh = controller.currentSentence?.zhText?.trimmingCharacters(in: .whitespacesAndNewlines), + !zh.isEmpty { + return zh + } + if (controller.currentSentence?.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + return nil + } + return chineseSubtitlePlaceholder + } + private var statusText: String { if let message = controller.cacheState.statusMessage { return message diff --git a/ios/OralTrainer/Views/PlayerSectionView.swift b/ios/OralTrainer/Views/PlayerSectionView.swift index 6a8746b..a7ec0ce 100644 --- a/ios/OralTrainer/Views/PlayerSectionView.swift +++ b/ios/OralTrainer/Views/PlayerSectionView.swift @@ -7,6 +7,8 @@ struct PlayerSectionView: View { @Binding var continuousPlayback: Bool var showsSubtitle: Binding? = .constant(true) var showsLessonTitle: Bool = true + var showChineseSubtitles: Binding? = nil + var chineseSubtitlePlaceholder: String = "正在翻译中文字幕…" var onFullscreen: () -> Void @State private var progressDragging = false @@ -93,14 +95,34 @@ struct PlayerSectionView: View { } .padding(.vertical, 2) } + if let showChineseSubtitles = showChineseSubtitles { + HStack(spacing: 0) { + Text("中文字幕") + .font(.system(size: 13, weight: .bold)) + .foregroundColor(Theme.textSubtle) + Spacer() + Toggle("", isOn: showChineseSubtitles) + .labelsHidden() + .tint(Theme.accent) + } + .padding(.vertical, 2) + } Text(sentenceMetaText) .font(.system(size: 13)) .foregroundColor(Theme.textSubtle) HStack(alignment: .center, spacing: 8) { - Text(sentenceTextText) - .font(.system(size: 15)) - .foregroundColor(.white) - .lineLimit(4) + VStack(alignment: .leading, spacing: 4) { + Text(sentenceTextText) + .font(.system(size: 15)) + .foregroundColor(.white) + .lineLimit(4) + if let chineseSubtitleText = chineseSubtitleText { + Text(chineseSubtitleText) + .font(.system(size: 14)) + .foregroundColor(Theme.textMuted) + .lineLimit(4) + } + } Spacer() Button { guard let sentence = controller.currentSentence, !(sentence.text ?? "").isEmpty else { @@ -205,6 +227,20 @@ struct PlayerSectionView: View { } } + private var chineseSubtitleText: String? { + guard showChineseSubtitles?.wrappedValue == true else { + return nil + } + if let zh = controller.currentSentence?.zhText?.trimmingCharacters(in: .whitespacesAndNewlines), + !zh.isEmpty { + return zh + } + if (controller.currentSentence?.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + return nil + } + return chineseSubtitlePlaceholder + } + private var statusText: String { if let message = controller.cacheState.statusMessage { return message diff --git a/ios/OralTrainer/Views/RootView.swift b/ios/OralTrainer/Views/RootView.swift index d79259b..099a5e2 100644 --- a/ios/OralTrainer/Views/RootView.swift +++ b/ios/OralTrainer/Views/RootView.swift @@ -9,6 +9,10 @@ struct RootView: View { @StateObject private var historyManager = TrainingHistoryManager() @State private var activeModule: Module = .course @State private var continuousPlayback = false + @State private var showChineseSubtitles = false + @State private var chineseTranslationLoading = false + @State private var chineseTranslationError = "" + @State private var chineseTranslationTask: Task? @State private var showImporter = false @State private var showFullscreenPlayer = false @State private var courses: [Course] = [] @@ -53,6 +57,8 @@ struct RootView: View { controller: controller, catalogVM: catalogVM, continuousPlayback: $continuousPlayback, + showChineseSubtitles: $showChineseSubtitles, + chineseSubtitlePlaceholder: chineseSubtitlePlaceholder, onFullscreen: enterFullscreen ) case .test: @@ -88,9 +94,24 @@ struct RootView: View { Task { await loadCourses() } } } + .onChange(of: showChineseSubtitles) { enabled in + if enabled { + ensureChineseTranslations() + } else { + chineseTranslationTask?.cancel() + chineseTranslationLoading = false + } + } + .onChange(of: controller.currentItem?.id) { _ in + ensureChineseTranslations() + } + .onChange(of: controller.currentItem?.sentences.count) { _ in + ensureChineseTranslations() + } .onChange(of: activeModule) { module in if module == .train { controller.setContinuousPlayback(continuousPlayback) + ensureChineseTranslations() } else if module == .test { // 与安卓测试页一致：按当前句窗口播放，避免滑进下一句。 controller.setContinuousPlayback(false) @@ -115,7 +136,9 @@ struct RootView: View { .fullScreenCover(isPresented: $showFullscreenPlayer) { FullscreenPlayerView( controller: controller, - showsLessonTitle: activeModule != .train + showsLessonTitle: activeModule != .train, + showChineseSubtitles: showChineseSubtitles, + chineseSubtitlePlaceholder: chineseSubtitlePlaceholder ) { exitFullscreen() } @@ -593,4 +616,59 @@ struct RootView: View { AppDelegate.lockOrientation(restored) showFullscreenPlayer = false } + + private var chineseSubtitlePlaceholder: String { + if chineseTranslationLoading { + return "正在翻译中文字幕…" + } + if !chineseTranslationError.isEmpty { + return chineseTranslationError + } + return "暂无中文字幕" + } + + private func ensureChineseTranslations() { + guard showChineseSubtitles, activeModule == .train else { + return + } + guard let item = controller.currentItem else { + return + } + let needsTranslation = item.sentences.contains { + !($0.text ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty && + ($0.zhText ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + } + guard needsTranslation else { + chineseTranslationLoading = false + chineseTranslationError = "" + return + } + chineseTranslationTask?.cancel() + chineseTranslationLoading = true + chineseTranslationError = "" + let mediaId = item.id + chineseTranslationTask = Task { + do { + let map = try await SentenceBoundaryAPI.translateSubtitles(videoHash: mediaId) + guard !Task.isCancelled else { return } + await MainActor.run { + guard controller.currentItem?.id == mediaId else { return } + let updated = (controller.currentItem?.sentences ?? []).map { sentence in + sentence.withZhText(map[sentence.index] ?? sentence.zhText) + } + controller.updateBoundaries(updated) + chineseTranslationLoading = false + chineseTranslationError = "" + controller.showTransientMessage("已加载中文字幕") + } + } catch { + guard !Task.isCancelled else { return } + await MainActor.run { + chineseTranslationLoading = false + chineseTranslationError = "中文字幕翻译失败" + controller.showTransientMessage("中文字幕翻译失败：\(error.localizedDescription)") + } + } + } + } } diff --git a/ios/OralTrainer/Views/TrainingView.swift b/ios/OralTrainer/Views/TrainingView.swift index dc55b2d..7a3cacc 100644 --- a/ios/OralTrainer/Views/TrainingView.swift +++ b/ios/OralTrainer/Views/TrainingView.swift @@ -4,6 +4,8 @@ struct TrainingView: View { @ObservedObject var controller: PlayerController @ObservedObject var catalogVM: CatalogViewModel @Binding var continuousPlayback: Bool + @Binding var showChineseSubtitles: Bool + var chineseSubtitlePlaceholder: String = "正在翻译中文字幕…" let onFullscreen: () -> Void var body: some View { @@ -14,6 +16,8 @@ struct TrainingView: View { showsContinuousToggle: true, continuousPlayback: $continuousPlayback, showsLessonTitle: false, + showChineseSubtitles: $showChineseSubtitles, + chineseSubtitlePlaceholder: chineseSubtitlePlaceholder, onFullscreen: onFullscreen ) CatalogView(vm: catalogVM) @@ -28,5 +32,8 @@ struct TrainingView: View { : "已关闭连续播放，将循环播放当前句子" ) } + .onChange(of: showChineseSubtitles) { enabled in + controller.showTransientMessage(enabled ? "已开启中文字幕" : "已关闭中文字幕") + } } } diff --git a/sentence_api/README.md b/sentence_api/README.md index 252ed6b..e5ab89e 100644 --- a/sentence_api/README.md +++ b/sentence_api/README.md @@ -100,6 +100,11 @@ The endpoint requires a signed-in user and accepts only the sentence and course context. `AI_API_URL` and `AI_API_KEY` remain on the server and are never sent to the mobile clients. +`POST /api/v1/videos/{sha256}/subtitle-translation` translates the full English +transcript in one AI request, then stores one Chinese caption per sentence. +Later requests reuse the cached Chinese captions until the English sentences +change. + ## Assessment ```bash diff --git a/sentence_api/ai_tutor.py b/sentence_api/ai_tutor.py index 2c58ae2..f28d040 100644 --- a/sentence_api/ai_tutor.py +++ b/sentence_api/ai_tutor.py @@ -1,7 +1,9 @@ -"""AI-backed sentence explanations for the mobile clients.""" +"""AI-backed sentence explanations and subtitle translation.""" from dataclasses import dataclass +import json import logging +import re from urllib.parse import urlsplit import requests @@ -9,6 +11,10 @@ import requests logger = logging.getLogger(__name__) +_JSON_FENCE_RE = re.compile(r"^```(?:json)?\s*|\s*```$", re.IGNORECASE) +_NUMBERED_LINE_RE = re.compile(r"^\s*(\d+)[\.、:：]\s*(.+?)\s*$") +_TRANSLATE_BATCH_CHARS = 12_000 + class AITutorError(RuntimeError): """Raised when the configured AI service cannot answer a request.""" @@ -60,9 +66,88 @@ class AITutor: user_prompt = ( f"课程：{context}\n位置：{position}\n{language_note}句子：{sentence_text}" ) + return self._complete(system_prompt, user_prompt, temperature=0.3) + + def translate_sentences( + self, + sentences: list[tuple[int, str]], + *, + course_title: str = "", + ) -> dict[int, str]: + if not self.available: + raise AITutorError("AI translation is not configured.") + if not sentences: + raise AITutorError("No sentences were provided for translation.") + + translations: dict[int, str] = {} + numbered = [(index, text.strip()) for index, text in sentences if text.strip()] + if not numbered: + raise AITutorError("No sentence text was provided for translation.") + + for batch, context_rows in _translation_batches(numbered): + translations.update( + self._translate_batch( + batch, + context_rows=context_rows, + course_title=course_title, + total_count=len(numbered), + ) + ) + missing = [index for index, _ in numbered if not translations.get(index, "").strip()] + if missing: + raise AITutorError("AI translation did not cover every sentence.") + return translations + + def _translate_batch( + self, + batch: list[tuple[int, str]], + *, + context_rows: list[tuple[int, str]], + course_title: str, + total_count: int, + ) -> dict[int, str]: + context = course_title.strip() or "口语训练课程" + display_offset = {index: position for position, (index, _) in enumerate(batch, start=1)} + system_prompt = ( + "你是专业的影视字幕翻译。请结合全文语境，把指定的外语句子翻译成自然、口语化的简体中文。" + "必须一句对一句，不能合并、拆分、跳过或改写编号。" + "只输出 JSON 数组，数组长度必须等于待翻译句子数，第 i 项对应第 i 句中文。" + "不要输出解释、markdown 或编号。" + ) + lines = [f"{display_offset[index]}. {text}" for index, text in batch] + user_parts = [ + f"课程：{context}", + f"全片共 {total_count} 句，本次需要翻译 {len(batch)} 句。", + ] + if context_rows: + user_parts.append("前文（只供理解语境，不要翻译）：") + user_parts.extend(f"{index + 1}. {text}" for index, text in context_rows) + user_parts.append("待翻译句子：") + user_parts.extend(lines) + content = self._complete( + system_prompt, + "\n".join(user_parts), + temperature=0.2, + timeout_seconds=max(self.timeout_seconds, 90.0), + ) + zh_rows = _parse_translation_payload(content, len(batch)) + return { + index: zh_rows[offset - 1] + for index, offset in display_offset.items() + } + + def _complete( + self, + system_prompt: str, + user_prompt: str, + *, + temperature: float, + timeout_seconds: float | None = None, + ) -> str: headers = {"Content-Type": "application/json"} if self.api_key: headers["Authorization"] = f"Bearer {self.api_key}" + timeout = self.timeout_seconds if timeout_seconds is None else timeout_seconds try: response = requests.post( self.request_url, @@ -73,14 +158,14 @@ class AITutor: {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}, ], - "temperature": 0.3, + "temperature": temperature, "stream": False, }, - timeout=self.timeout_seconds, + timeout=timeout, ) except requests.RequestException as exc: logger.exception( - "AI explanation request failed: endpoint=%s model=%s", + "AI request failed: endpoint=%s model=%s", self.request_url, self.model, ) @@ -88,7 +173,7 @@ class AITutor: if response.status_code < 200 or response.status_code >= 300: logger.warning( - "AI explanation returned HTTP %s: endpoint=%s model=%s body=%s", + "AI service returned HTTP %s: endpoint=%s model=%s body=%s", response.status_code, self.request_url, self.model, @@ -97,19 +182,86 @@ class AITutor: raise AITutorError(f"AI service returned HTTP {response.status_code}.") try: body = response.json() - explanation = body["choices"][0]["message"]["content"] + content = body["choices"][0]["message"]["content"] except (KeyError, IndexError, TypeError, ValueError) as exc: logger.warning( - "AI explanation returned an unexpected body: endpoint=%s body=%s", + "AI service returned an unexpected body: endpoint=%s body=%s", self.request_url, response.text[:1000], ) raise AITutorError("AI service returned an unexpected response.") from exc - if not isinstance(explanation, str) or not explanation.strip(): + if not isinstance(content, str) or not content.strip(): logger.warning( - "AI explanation returned empty content: endpoint=%s body=%s", + "AI service returned empty content: endpoint=%s body=%s", self.request_url, response.text[:1000], ) raise AITutorError("AI service returned an empty explanation.") - return explanation.strip() + return content.strip() + + +def _translation_batches( + sentences: list[tuple[int, str]], +) -> list[tuple[list[tuple[int, str]], list[tuple[int, str]]]]: + batches: list[list[tuple[int, str]]] = [] + current: list[tuple[int, str]] = [] + current_chars = 0 + for item in sentences: + extra = len(item[1]) + 8 + if current and current_chars + extra > _TRANSLATE_BATCH_CHARS: + batches.append(current) + current = [] + current_chars = 0 + current.append(item) + current_chars += extra + if current: + batches.append(current) + result: list[tuple[list[tuple[int, str]], list[tuple[int, str]]]] = [] + consumed = 0 + for batch in batches: + context_rows = sentences[max(0, consumed - 8):consumed] + result.append((batch, context_rows)) + consumed += len(batch) + return result + + +def _parse_translation_payload(content: str, expected_count: int) -> list[str]: + text = _JSON_FENCE_RE.sub("", content.strip()).strip() + try: + data = json.loads(text) + parsed = _translations_from_json(data, expected_count) + if parsed is not None: + return parsed + except (TypeError, ValueError): + pass + numbered: dict[int, str] = {} + for line in text.splitlines(): + match = _NUMBERED_LINE_RE.match(line) + if match is None: + continue + numbered[int(match.group(1))] = match.group(2).strip() + if len(numbered) == expected_count and all( + index in numbered and numbered[index] for index in range(1, expected_count + 1) + ): + return [numbered[index] for index in range(1, expected_count + 1)] + raise AITutorError("AI translation returned an unexpected response.") + + +def _translations_from_json(data: object, expected_count: int) -> list[str] | None: + if isinstance(data, dict): + for key in ("translations", "sentences", "zh", "items"): + if key in data: + data = data[key] + break + if isinstance(data, list) and len(data) == expected_count: + if all(isinstance(item, str) and item.strip() for item in data): + return [item.strip() for item in data] + if all(isinstance(item, dict) for item in data): + values: list[str] = [] + for item in data: + zh = item.get("zh_text") or item.get("zh") or item.get("text") + if not isinstance(zh, str) or not zh.strip(): + return None + values.append(zh.strip()) + return values + return None diff --git a/sentence_api/main.py b/sentence_api/main.py index aa4dd95..0561f0b 100644 --- a/sentence_api/main.py +++ b/sentence_api/main.py @@ -45,6 +45,8 @@ from .models import ( SentenceBoundaryDocument, SentenceExplanationRequest, SentenceExplanationResponse, + SubtitleTranslationItem, + SubtitleTranslationResponse, PasswordResetRequest, SentenceTextUpdate, UserDubShareSummary, @@ -333,6 +335,60 @@ def create_app( raise HTTPException(status_code=503, detail=str(exc)) from exc return SentenceExplanationResponse(explanation=explanation) + @application.post( + "/api/v1/videos/{video_hash}/subtitle-translation", + response_model=SubtitleTranslationResponse, + ) + def translate_subtitles( + video_hash: str = SHA256_PATH, + user: AuthenticatedUser = Depends(current_user), + ) -> SubtitleTranslationResponse: + del user + document = video_repository.get_document(video_hash) + if document is None: + raise HTTPException(status_code=404, detail="Course is not available.") + payload = [ + (sentence.index, sentence.text or "") + for sentence in document.sentences + if (sentence.text or "").strip() + ] + if not payload: + raise HTTPException(status_code=422, detail="No sentence text is available to translate.") + cached = [ + SubtitleTranslationItem(index=sentence.index, zh_text=sentence.zh_text) + for sentence in document.sentences + if sentence.zh_text and sentence.zh_text.strip() + ] + complete = len(cached) == len(payload) and all( + (sentence.zh_text or "").strip() + for sentence in document.sentences + if (sentence.text or "").strip() + ) + if complete: + return SubtitleTranslationResponse( + video_hash=document.video_hash, + translations=cached, + ) + video = video_repository.get_video(video_hash) + try: + translated = application.state.ai_tutor.translate_sentences( + payload, + course_title=(video["title"] if video is not None else "") or "", + ) + except AITutorError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + saved = video_repository.save_sentence_translations(video_hash, translated) + if saved is None: + raise HTTPException(status_code=404, detail="Course is not available.") + return SubtitleTranslationResponse( + video_hash=saved.video_hash, + translations=[ + SubtitleTranslationItem(index=sentence.index, zh_text=sentence.zh_text) + for sentence in saved.sentences + if sentence.zh_text and sentence.zh_text.strip() + ], + ) + @application.post( "/api/v1/audio/sentence-boundaries", response_model=SentenceBoundaryDocument, diff --git a/sentence_api/models.py b/sentence_api/models.py index 311bd95..d59f57a 100644 --- a/sentence_api/models.py +++ b/sentence_api/models.py @@ -12,6 +12,7 @@ class SentenceBoundary(BaseModel): end_ms: int = Field(gt=0) text: Optional[str] = None language: Optional[str] = None + zh_text: Optional[str] = None reference_speech_duration_ms: Optional[int] = Field(default=None, gt=0) @model_validator(mode="after") @@ -126,6 +127,20 @@ class SentenceExplanationResponse(BaseModel): explanation: str = Field(min_length=1, max_length=6000) +class SubtitleTranslationItem(BaseModel): + model_config = ConfigDict(extra="forbid") + + index: int = Field(ge=0) + zh_text: str = Field(min_length=1, max_length=4000) + + +class SubtitleTranslationResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + video_hash: str + translations: List[SubtitleTranslationItem] + + class SentenceBoundaryAdjust(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/sentence_api/repository.py b/sentence_api/repository.py index 9cc92d1..0a4eb71 100644 --- a/sentence_api/repository.py +++ b/sentence_api/repository.py @@ -56,6 +56,7 @@ class VideoRepository: end_ms INTEGER NOT NULL, text TEXT, language TEXT, + zh_text TEXT, reference_speech_duration_ms INTEGER, PRIMARY KEY (video_hash, sentence_index), FOREIGN KEY (video_hash) REFERENCES videos(video_hash) ON DELETE CASCADE @@ -193,6 +194,14 @@ class VideoRepository: if "remote_url" not in video_columns: connection.execute("ALTER TABLE videos ADD COLUMN remote_url TEXT") + with self._connect() as connection: + sentence_columns = { + row["name"] + for row in connection.execute("PRAGMA table_info(sentences)") + } + if "zh_text" not in sentence_columns: + connection.execute("ALTER TABLE sentences ADD COLUMN zh_text TEXT") + def upsert_upload( self, *, @@ -431,17 +440,7 @@ class VideoRepository: video_hash=video["video_hash"], duration_ms=video["duration_ms"], algorithm_version=video["algorithm_version"] or "unknown", - sentences=[ - SentenceBoundary( - index=row["sentence_index"], - start_ms=row["start_ms"], - end_ms=row["end_ms"], - text=row["text"], - language=row["language"], - reference_speech_duration_ms=row["reference_speech_duration_ms"], - ) - for row in rows - ], + sentences=[_sentence_from_row(row) for row in rows], ) def update_sentence_text( @@ -462,6 +461,10 @@ class VideoRepository: ) if cursor.rowcount == 0: return None + connection.execute( + "UPDATE sentences SET zh_text = NULL WHERE video_hash = ?", + (normalized_hash,), + ) connection.execute( "UPDATE videos SET updated_at = ? WHERE video_hash = ?", (utc_now(), normalized_hash), @@ -470,14 +473,7 @@ class VideoRepository: "SELECT * FROM sentences WHERE video_hash = ? AND sentence_index = ?", (normalized_hash, sentence_index), ).fetchone() - return SentenceBoundary( - index=row["sentence_index"], - start_ms=row["start_ms"], - end_ms=row["end_ms"], - text=row["text"], - language=row["language"], - reference_speech_duration_ms=row["reference_speech_duration_ms"], - ) + return _sentence_from_row(row) def get_sentence( self, @@ -495,14 +491,7 @@ class VideoRepository: ).fetchone() if row is None: return None - return SentenceBoundary( - index=row["sentence_index"], - start_ms=row["start_ms"], - end_ms=row["end_ms"], - text=row["text"], - language=row["language"], - reference_speech_duration_ms=row["reference_speech_duration_ms"], - ) + return _sentence_from_row(row) def replace_sentence( self, @@ -614,6 +603,10 @@ class VideoRepository: replacement.reference_speech_duration_ms, ), ) + connection.execute( + "UPDATE sentences SET zh_text = NULL WHERE video_hash = ?", + (normalized_hash,), + ) connection.execute( "UPDATE videos SET updated_at = ? WHERE video_hash = ?", (utc_now(), normalized_hash), @@ -627,17 +620,7 @@ class VideoRepository: (normalized_hash, sentence_index, sentence_index + len(rows)), ).fetchall() - return [ - SentenceBoundary( - index=row["sentence_index"], - start_ms=row["start_ms"], - end_ms=row["end_ms"], - text=row["text"], - language=row["language"], - reference_speech_duration_ms=row["reference_speech_duration_ms"], - ) - for row in result_rows - ] + return [_sentence_from_row(row) for row in result_rows] def merge_with_next_sentence( @@ -727,6 +710,10 @@ class VideoRepository: (old_index - 1, normalized_hash, old_index), ) + connection.execute( + "UPDATE sentences SET zh_text = NULL WHERE video_hash = ?", + (normalized_hash,), + ) connection.execute( "UPDATE videos SET updated_at = ? WHERE video_hash = ?", (utc_now(), normalized_hash), @@ -739,14 +726,7 @@ class VideoRepository: (normalized_hash, sentence_index), ).fetchone() - return SentenceBoundary( - index=row["sentence_index"], - start_ms=row["start_ms"], - end_ms=row["end_ms"], - text=row["text"], - language=row["language"], - reference_speech_duration_ms=row["reference_speech_duration_ms"], - ) + return _sentence_from_row(row) def adjust_sentence_boundary( self, @@ -851,17 +831,36 @@ class VideoRepository: (normalized_hash, *updated_indexes), ).fetchall() - return [ - SentenceBoundary( - index=row["sentence_index"], - start_ms=row["start_ms"], - end_ms=row["end_ms"], - text=row["text"], - language=row["language"], - reference_speech_duration_ms=row["reference_speech_duration_ms"], + return [_sentence_from_row(row) for row in result_rows] + + def clear_sentence_translations(self, video_hash: str) -> None: + with self._connect() as connection: + connection.execute( + "UPDATE sentences SET zh_text = NULL WHERE video_hash = ?", + (normalize_video_hash(video_hash),), ) - for row in result_rows - ] + + def save_sentence_translations( + self, + video_hash: str, + translations: Dict[int, str], + ) -> Optional[SentenceBoundaryDocument]: + normalized_hash = normalize_video_hash(video_hash) + with self._connect() as connection: + for index, zh_text in translations.items(): + connection.execute( + """ + UPDATE sentences + SET zh_text = ? + WHERE video_hash = ? AND sentence_index = ? + """, + (zh_text.strip(), normalized_hash, index), + ) + connection.execute( + "UPDATE videos SET updated_at = ? WHERE video_hash = ?", + (utc_now(), normalized_hash), + ) + return self.get_document(normalized_hash) def create_dub_share( self, @@ -1278,6 +1277,21 @@ class VideoRepository: return video + +def _sentence_from_row(row: sqlite3.Row) -> SentenceBoundary: + keys = set(row.keys()) + zh_text = row["zh_text"] if "zh_text" in keys else None + return SentenceBoundary( + index=row["sentence_index"], + start_ms=row["start_ms"], + end_ms=row["end_ms"], + text=row["text"], + language=row["language"], + zh_text=zh_text, + reference_speech_duration_ms=row["reference_speech_duration_ms"], + ) + + def _join_adjacent_sentence_text( left: str, right: str, diff --git a/sentence_api/tests/test_ai_tutor.py b/sentence_api/tests/test_ai_tutor.py index de778b0..2501c06 100644 --- a/sentence_api/tests/test_ai_tutor.py +++ b/sentence_api/tests/test_ai_tutor.py @@ -71,3 +71,34 @@ def test_ai_tutor_rejects_unexpected_response(monkeypatch): with pytest.raises(AITutorError): tutor.explain("Hello.") + + +from sentence_api.ai_tutor import _parse_translation_payload + + +def test_ai_tutor_translates_full_document(monkeypatch): + requests_payloads = [] + + def fake_post(endpoint, headers, json, timeout): + requests_payloads.append((endpoint, headers, json, timeout)) + return FakeResponse(body={"choices": [{"message": {"content": '["你好。", "见到你很高兴。"]'}}]}) + + monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post) + tutor = AITutor("https://ai.test/v1", "secret", "test-model", 1.5) + + translated = tutor.translate_sentences( + [(0, "Hello."), (1, "Nice to meet you.")], + course_title="Greetings", + ) + + assert translated == {0: "你好。", 1: "见到你很高兴。"} + _endpoint, _headers, payload, timeout = requests_payloads[0] + user_prompt = payload["messages"][1]["content"] + assert "Hello." in user_prompt + assert "Nice to meet you." in user_prompt + assert timeout == 90.0 + + +def test_parse_translation_payload_accepts_fenced_json_and_numbered_lines(): + assert _parse_translation_payload('```json\n["一", "二"]\n```', 2) == ["一", "二"] + assert _parse_translation_payload("1. 一\n2. 二", 2) == ["一", "二"] diff --git a/sentence_api/tests/test_api.py b/sentence_api/tests/test_api.py index ecc9740..bcd31a6 100644 --- a/sentence_api/tests/test_api.py +++ b/sentence_api/tests/test_api.py @@ -202,6 +202,72 @@ def test_sentence_explanation_uses_server_side_ai_client(tmp_path, monkeypatch): ] + +def _ready_video(repository): + repository.upsert_upload( + video_hash=VIDEO_HASH, + title="Greetings", + filename="lesson.mp4", + stored_filename=f"{VIDEO_HASH}.mp4", + content_type="video/mp4", + size_bytes=5, + language="en", + ) + repository.save_processing_result( + SentenceBoundaryDocument( + video_hash=VIDEO_HASH, + duration_ms=9000, + algorithm_version="test", + sentences=[ + SentenceBoundary(index=0, start_ms=0, end_ms=4000, text="Hello.", language="en", reference_speech_duration_ms=3000), + SentenceBoundary(index=1, start_ms=4000, end_ms=9000, text="Nice to meet you.", language="en", reference_speech_duration_ms=4000), + ], + ), + None, + ) + + +def test_subtitle_translation_requires_login(tmp_path): + client = make_client(tmp_path) + client.headers.clear() + response = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation") + assert response.status_code == 401 + + +def test_subtitle_translation_uses_full_document_and_is_cached(tmp_path, monkeypatch): + client = make_client(tmp_path) + _ready_video(client.app.state.video_repository) + requested = [] + + def translate_sentences(sentences, **kwargs): + requested.append((sentences, kwargs)) + return {index: f"中文{index}" for index, _text in sentences} + + class StubAITutor: + pass + + stub = StubAITutor() + stub.translate_sentences = translate_sentences + monkeypatch.setattr(client.app.state, "ai_tutor", stub) + + response = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation") + assert response.status_code == 200, response.text + payload = response.json() + assert payload["video_hash"] == VIDEO_HASH + assert payload["translations"] == [ + {"index": 0, "zh_text": "中文0"}, + {"index": 1, "zh_text": "中文1"}, + ] + document = client.app.state.video_repository.get_document(VIDEO_HASH) + assert [sentence.zh_text for sentence in document.sentences] == ["中文0", "中文1"] + + second = client.post(f"/api/v1/videos/{VIDEO_HASH}/subtitle-translation") + assert second.status_code == 200 + assert second.json() == payload + assert len(requested) == 1 + assert requested[0][0] == [(0, "Hello."), (1, "Nice to meet you.")] + + def test_oss_video_url_is_used_for_catalog_and_content(tmp_path): index_path = tmp_path / "boundaries.json" index_path.write_text("{}", encoding="utf-8")