From 701148e00eaf904db142d674f2fbaf1a0c390e07 Mon Sep 17 00:00:00 2001 From: Shuming Liu Date: Sun, 6 Sep 2026 07:38:56 +0800 Subject: [PATCH] add explain module --- android/sample-app/build.gradle.kts | 2 +- .../oraltrainer/sample/MainActivity.kt | 20 +++ .../oraltrainer/sample/SampleBindings.kt | 2 + .../oraltrainer/sample/SampleScreenActions.kt | 10 +- .../sample/SentenceExplanationApi.kt | 113 +++++++++++++ .../sample/SentenceExplanationOverlay.kt | 150 ++++++++++++++++++ .../oraltrainer/sample/ui/PlayerChrome.kt | 35 +++- .../src/main/res/drawable/ic_ai_explain.xml | 14 ++ .../Services/SentenceBoundaryAPI.swift | 43 +++++ .../Views/FullscreenPlayerView.swift | 33 +++- ios/OralTrainer/Views/PlayerSectionView.swift | 86 +++++++++- sentence_api/.env.example | 7 + sentence_api/DEPLOYMENT.md | 3 + sentence_api/README.md | 17 ++ sentence_api/ai_tutor.py | 75 +++++++++ sentence_api/config.py | 8 + sentence_api/main.py | 31 ++++ sentence_api/models.py | 13 ++ sentence_api/tests/test_ai_tutor.py | 57 +++++++ sentence_api/tests/test_api.py | 64 ++++++++ 20 files changed, 775 insertions(+), 8 deletions(-) create mode 100644 android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SentenceExplanationApi.kt create mode 100644 android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SentenceExplanationOverlay.kt create mode 100644 android/sample-app/src/main/res/drawable/ic_ai_explain.xml create mode 100644 sentence_api/ai_tutor.py create mode 100644 sentence_api/tests/test_ai_tutor.py diff --git a/android/sample-app/build.gradle.kts b/android/sample-app/build.gradle.kts index 13b9b49..bf96a29 100644 --- a/android/sample-app/build.gradle.kts +++ b/android/sample-app/build.gradle.kts @@ -19,7 +19,7 @@ android { minSdk = 26 targetSdk = 35 versionCode = 1 - versionName = "0.1.3" + versionName = "0.1.4" } compileOptions { diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt index 6589f2e..c462834 100644 --- a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt @@ -42,6 +42,7 @@ class MainActivity : Activity() { private lateinit var screenFactory: SampleScreenFactory private lateinit var catalogController: CourseCatalogController private lateinit var testSession: TestSessionController + private lateinit var explanationOverlay: SentenceExplanationOverlay private var progressBarDragging = false @@ -66,6 +67,12 @@ class MainActivity : Activity() { sdk = sdkComponents.sdk controller = sdkComponents.controller authCoordinator = AuthCoordinator(this, sdk.userApi, ::saveAuthSession) + explanationOverlay = SentenceExplanationOverlay( + this, + SentenceExplanationApi(this, SERVER_BASE_URL) { + authSessionStore.session?.token.orEmpty() + }, + ) screenFactory = SampleScreenFactory( activity = this, @@ -217,6 +224,7 @@ class MainActivity : Activity() { authCoordinator.reset() testSession.release() controller.release() + explanationOverlay.release() super.onDestroy() } @@ -365,6 +373,7 @@ class MainActivity : Activity() { if (sentence == null) { bindings.sentenceMetaText.text = "暂无句子边界" bindings.sentenceText.text = bindings.lessonTitleText.text + bindings.explainSentenceButton?.isEnabled = false return } val countText = if (screenState.currentSentenceCount > 0) { @@ -375,6 +384,7 @@ class MainActivity : Activity() { bindings.sentenceMetaText.text = "第 ${sentence.index + 1}$countText 句 ${PlaybackFormat.time(sentence.startMs)}-${PlaybackFormat.time(sentence.endMs)}" bindings.sentenceText.text = sentence.text?.takeIf { it.isNotBlank() } ?: "当前句子" + bindings.explainSentenceButton?.isEnabled = !sentence.text.isNullOrBlank() if (screenState.activeModule == Module.TEST) { testSession.refreshTestUi() } @@ -417,6 +427,16 @@ class MainActivity : Activity() { } } + override fun onExplainCurrentSentence() { + val sentence = controller.currentSentence() ?: return + if (sentence.text.isNullOrBlank()) return + controller.pause() + explanationOverlay.show( + sentence, + controller.currentTrainingItem()?.title.orEmpty(), + ) + } + override fun onTestSubtitleChanged(visible: Boolean) { testSession.onTestSubtitleChanged(visible) } diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleBindings.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleBindings.kt index a805739..46fa86a 100644 --- a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleBindings.kt +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleBindings.kt @@ -1,6 +1,7 @@ package cn.learningpad.oraltrainer.sample import android.widget.Button +import android.widget.ImageButton import android.widget.LinearLayout import android.widget.SeekBar import android.widget.TextView @@ -26,6 +27,7 @@ internal class SampleBindings { lateinit var mergeDubButton: Button lateinit var shareDubButton: Button var dubbingStatusText: TextView? = null + var explainSentenceButton: ImageButton? = null var myContentList: LinearLayout? = null val hasPlayerChrome: Boolean diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleScreenActions.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleScreenActions.kt index b3e469e..276f874 100644 --- a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleScreenActions.kt +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SampleScreenActions.kt @@ -10,7 +10,12 @@ internal interface CourseScreenActions { fun onSelectCourse(video: TrainingVideoSummary) } -internal interface TrainScreenActions { +internal interface SentenceExplainActions { + fun onExplainCurrentSentence() +} + +internal interface TrainScreenActions : + SentenceExplainActions { fun onImportLocalVideo() fun onRefreshCatalog() fun onSeekProgressPreview(progress: Int) @@ -20,7 +25,8 @@ internal interface TrainScreenActions { fun onTrainCatalogVideoClick(video: TrainingVideoSummary) } -internal interface TestScreenActions { +internal interface TestScreenActions : + SentenceExplainActions { fun onSeekProgressPreview(progress: Int) fun onSeekStart() fun onSeekStop(progress: Int) diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SentenceExplanationApi.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SentenceExplanationApi.kt new file mode 100644 index 0000000..67d63ad --- /dev/null +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SentenceExplanationApi.kt @@ -0,0 +1,113 @@ +package cn.learningpad.oraltrainer.sample + +import android.content.Context +import android.os.Handler +import android.os.Looper +import org.json.JSONObject +import java.io.IOException +import java.net.HttpURLConnection +import java.net.URL +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.Future + +data class SentenceExplanationResult( + val explanation: String, +) + +interface SentenceExplanationCallback { + fun onSuccess(result: SentenceExplanationResult) + fun onError(error: Throwable) +} + +class CancellableExplanationRequest internal constructor( + private val future: Future<*>, +) { + fun cancel() { + future.cancel(true) + } +} + +/** Calls the app server; AI credentials never live in the mobile app. */ +internal class SentenceExplanationApi( + context: Context, + private val baseUrl: String, + private val tokenProvider: () -> String, +) { + private val appContext = context.applicationContext + private val mainHandler = Handler(Looper.getMainLooper()) + private val executor: ExecutorService = Executors.newCachedThreadPool() + + fun explain( + sentenceText: String, + courseTitle: String, + sentenceIndex: Int?, + language: String?, + callback: SentenceExplanationCallback, + ): CancellableExplanationRequest { + val future = executor.submit { + try { + val result = requestBlocking( + sentenceText, + courseTitle, + sentenceIndex, + language, + ) + if (!Thread.currentThread().isInterrupted) { + mainHandler.post { callback.onSuccess(result) } + } + } catch (error: Throwable) { + if (!Thread.currentThread().isInterrupted) { + mainHandler.post { callback.onError(error) } + } + } + } + return CancellableExplanationRequest(future) + } + + fun release() { + executor.shutdownNow() + } + + private fun requestBlocking( + sentenceText: String, + courseTitle: String, + sentenceIndex: Int?, + language: String?, + ): SentenceExplanationResult { + val endpoint = baseUrl.trimEnd('/') + "/api/v1/sentences/explain" + val body = JSONObject().apply { + put("sentence_text", sentenceText) + put("course_title", courseTitle) + sentenceIndex?.let { put("sentence_index", it) } + language?.takeIf { it.isNotBlank() }?.let { put("language", it) } + } + val connection = URL(endpoint).openConnection() as HttpURLConnection + try { + connection.requestMethod = "POST" + connection.connectTimeout = 15_000 + connection.readTimeout = 60_000 + connection.doOutput = true + connection.useCaches = false + connection.setRequestProperty("Accept", "application/json") + connection.setRequestProperty("Content-Type", "application/json") + connection.setRequestProperty("User-Agent", "OralTrainerSample/0.1.4 (Android)") + tokenProvider().takeIf { it.isNotBlank() }?.let { + connection.setRequestProperty("Authorization", "Bearer $it") + } + connection.outputStream.use { output -> + output.write(body.toString().toByteArray(Charsets.UTF_8)) + } + val statusCode = connection.responseCode + val stream = if (statusCode in 200..299) connection.inputStream else connection.errorStream + val response = stream?.bufferedReader(Charsets.UTF_8)?.use { it.readText() }.orEmpty() + if (statusCode !in 200..299) { + throw IOException("AI explain API returned HTTP $statusCode: $response") + } + val explanation = JSONObject(response).getString("explanation") + return SentenceExplanationResult(explanation) + } finally { + connection.disconnect() + } + } +} diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SentenceExplanationOverlay.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SentenceExplanationOverlay.kt new file mode 100644 index 0000000..8b332eb --- /dev/null +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/SentenceExplanationOverlay.kt @@ -0,0 +1,150 @@ +package cn.learningpad.oraltrainer.sample + +import android.app.Activity +import android.graphics.Color +import android.graphics.Typeface +import android.graphics.drawable.GradientDrawable +import android.view.Gravity +import android.view.View +import android.view.ViewGroup +import android.widget.FrameLayout +import android.widget.LinearLayout +import android.widget.ScrollView +import android.widget.TextView +import cn.learningpad.oraltrainer.sdk.SentenceBoundary + +/** Modal explanation layer shown after the user taps the subtitle AI button. */ +internal class SentenceExplanationOverlay( + private val activity: Activity, + private val api: SentenceExplanationApi, +) { + private var currentOverlay: View? = null + private var currentRequest: CancellableExplanationRequest? = null + private var requestGeneration = 0 + + fun show(sentence: SentenceBoundary, courseTitle: String) { + close() + val generation = ++requestGeneration + val root = FrameLayout(activity).apply { + layoutParams = ViewGroup.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + ViewGroup.LayoutParams.MATCH_PARENT, + ) + setBackgroundColor(Color.argb(160, 0, 0, 0)) + isClickable = true + isFocusable = true + } + val card = LinearLayout(activity).apply { + orientation = LinearLayout.VERTICAL + background = rounded(Color.WHITE, 16f, SampleTheme.lightBorder) + setPadding(20.dp, 20.dp, 20.dp, 18.dp) + layoutParams = FrameLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + ViewGroup.LayoutParams.MATCH_PARENT, + Gravity.CENTER, + ).apply { + setMargins(24.dp, 48.dp, 24.dp, 48.dp) + } + } + val title = TextView(activity).apply { + text = "句子讲解" + setTextColor(SampleTheme.textDark) + textSize = 19f + typeface = Typeface.DEFAULT_BOLD + } + val subtitle = TextView(activity).apply { + text = sentence.text?.takeIf { it.isNotBlank() } ?: courseTitle + setTextColor(SampleTheme.accentDeep) + textSize = 14f + setPadding(0, 8.dp, 0, 12.dp) + maxLines = 3 + } + val message = TextView(activity).apply { + text = "AI 正在讲解这句话…" + setTextColor(SampleTheme.textSubtle) + textSize = 15f + setLineSpacing(3.dp.toFloat(), 1.1f) + } + val scroll = ScrollView(activity).apply { + addView(message) + layoutParams = LinearLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + 0, + 1f, + ) + } + val close = TextView(activity).apply { + text = "关闭" + setTextColor(Color.WHITE) + textSize = 15f + typeface = Typeface.DEFAULT_BOLD + gravity = Gravity.CENTER + background = rounded(SampleTheme.button, 10f) + layoutParams = LinearLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + 44.dp, + ).apply { topMargin = 16.dp } + setOnClickListener { close() } + } + card.addView(title) + card.addView(subtitle) + card.addView(scroll) + card.addView(close) + root.addView(card) + + addToWindow(root) + currentOverlay = root + currentRequest = api.explain( + sentence.text.orEmpty(), + courseTitle, + sentence.index, + sentence.language, + object : SentenceExplanationCallback { + override fun onSuccess(result: SentenceExplanationResult) { + if (generation == requestGeneration) message.text = result.explanation + } + + override fun onError(error: Throwable) { + if (generation == requestGeneration) { + message.text = "讲解失败：${error.message ?: "请稍后重试"}" + message.setTextColor(SampleTheme.danger) + } + } + }, + ) + } + + fun close() { + currentRequest?.cancel() + currentRequest = null + currentOverlay?.let { overlay -> + (overlay.parent as? ViewGroup)?.removeView(overlay) + } + currentOverlay = null + } + + fun release() { + close() + api.release() + } + + private fun addToWindow(view: View) { + (activity.window?.decorView as? ViewGroup)?.addView(view) + } + + private val Int.dp: Int + get() = (this * activity.resources.displayMetrics.density).toInt() + + private val Float.dp: Int + get() = (this * activity.resources.displayMetrics.density).toInt() + + private fun rounded( + color: Int, + radiusDp: Float, + strokeColor: Int? = null, + ): GradientDrawable = GradientDrawable().apply { + setColor(color) + cornerRadius = radiusDp.dp.toFloat() + strokeColor?.let { setStroke(1.dp, it) } + } +} diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/ui/PlayerChrome.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/ui/PlayerChrome.kt index c458ead..c5fa71e 100644 --- a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/ui/PlayerChrome.kt +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/ui/PlayerChrome.kt @@ -162,7 +162,23 @@ internal class PlayerChrome( addView(createContinuousPlaybackRow(translucent)) } addView(bindings.sentenceMetaText) - addView(bindings.sentenceText) + addView(LinearLayout(activity).apply { + orientation = LinearLayout.HORIZONTAL + gravity = Gravity.CENTER_VERTICAL + addView( + bindings.sentenceText, + LinearLayout.LayoutParams( + 0, + ViewGroup.LayoutParams.WRAP_CONTENT, + 1f, + ), + ) + bindings.explainSentenceButton = createExplainButton(translucent) + addView( + bindings.explainSentenceButton, + LinearLayout.LayoutParams(32.dp, 32.dp).apply { marginStart = 10.dp }, + ) + }) addView(bindings.progressBar) addView(bindings.statusText) } @@ -211,6 +227,23 @@ internal class PlayerChrome( } } + private fun createExplainButton(translucent: Boolean): ImageButton = + ImageButton(activity).apply { + setImageResource(R.drawable.ic_ai_explain) + imageTintList = ColorStateList.valueOf( + if (translucent) SampleTheme.accentLight else SampleTheme.accentDeep, + ) + background = rounded( + if (translucent) SampleTheme.overlay else SampleTheme.selected, + 16.dp.toFloat(), + ) + scaleType = ImageView.ScaleType.CENTER_INSIDE + setPadding(6.dp, 6.dp, 6.dp, 6.dp) + contentDescription = "AI 讲解这句话" + isEnabled = false + setOnClickListener { seekActions.onExplainCurrentSentence() } + } + private fun createLandscapeSentencePanel(): View = createSentenceContent(translucent = true).apply { layoutParams = FrameLayout.LayoutParams( diff --git a/android/sample-app/src/main/res/drawable/ic_ai_explain.xml b/android/sample-app/src/main/res/drawable/ic_ai_explain.xml new file mode 100644 index 0000000..429018b --- /dev/null +++ b/android/sample-app/src/main/res/drawable/ic_ai_explain.xml @@ -0,0 +1,14 @@ + + + + diff --git a/ios/OralTrainer/Services/SentenceBoundaryAPI.swift b/ios/OralTrainer/Services/SentenceBoundaryAPI.swift index 656f923..e7690fb 100644 --- a/ios/OralTrainer/Services/SentenceBoundaryAPI.swift +++ b/ios/OralTrainer/Services/SentenceBoundaryAPI.swift @@ -37,3 +37,46 @@ enum SentenceBoundaryAPI { return result.value } } + +struct SentenceExplanationResponse: Decodable { + let explanation: String + + private enum CodingKeys: String, CodingKey { + case explanation + } +} + +extension SentenceBoundaryAPI { + static func explain( + _ sentence: SentenceBoundary, + courseTitle: String + ) async throws -> String { + var body: [String: Any] = [ + "sentence_text": sentence.text ?? "", + "course_title": courseTitle, + "sentence_index": sentence.index, + ] + if let language = sentence.language { + body["language"] = language + } + var headers: [String: String] = ["Content-Type": "application/json"] + if !AppConfig.authToken.isEmpty { + headers["Authorization"] = "Bearer \(AppConfig.authToken)" + } + let result = try await HTTPClient.execute( + baseURL: AppConfig.serverBaseURL, + path: "api/v1/sentences/explain", + method: "POST", + timeout: AppConfig.readTimeout, + headers: headers, + body: try JSONSerialization.data(withJSONObject: body) + ) + guard (200...299).contains(result.statusCode) else { + let detail = String(data: result.body, encoding: .utf8) ?? "" + throw HTTPClientError.badStatus(result.statusCode, detail) + } + return try JSONDecoder() + .decode(SentenceExplanationResponse.self, from: result.body) + .explanation + } +} diff --git a/ios/OralTrainer/Views/FullscreenPlayerView.swift b/ios/OralTrainer/Views/FullscreenPlayerView.swift index 7c4cdfd..8f52569 100644 --- a/ios/OralTrainer/Views/FullscreenPlayerView.swift +++ b/ios/OralTrainer/Views/FullscreenPlayerView.swift @@ -6,6 +6,7 @@ struct FullscreenPlayerView: View { let onExit: () -> Void @State private var subtitleVisible = true + @State private var explanationSentence: SentenceBoundary? var body: some View { ZStack { @@ -38,10 +39,34 @@ struct FullscreenPlayerView: View { Text(sentenceMetaText) .font(.system(size: 13)) .foregroundColor(Theme.textOverlay) - Text(sentenceTextText) + HStack(alignment: .center, spacing: 8) { + Text(sentenceTextText) .font(.system(size: 15)) .foregroundColor(.white) .lineLimit(2) + Spacer() + Button { + guard let sentence = controller.currentSentence, + !(sentence.text ?? "").isEmpty else { + return + } + controller.pause() + explanationSentence = sentence + } label: { + Image(systemName: "lightbulb.circle.fill") + .font(.system(size: 21, weight: .semibold)) + .foregroundColor( + controller.currentSentence?.text?.isEmpty == false + ? Theme.accentLight + : Theme.textSubtle + ) + .frame(width: 30, height: 30) + .background(Theme.overlay) + .clipShape(Circle()) + .accessibilityLabel("AI 讲解这句话") + } + .buttonStyle(.plain) + } progressRow Text(statusText) .font(.system(size: 13)) @@ -70,6 +95,12 @@ struct FullscreenPlayerView: View { } .padding(16) } + .sheet(item: $explanationSentence) { sentence in + SentenceExplanationSheet( + courseTitle: controller.currentItem?.title ?? "口语训练课程", + sentence: sentence + ) + } } @State private var progressDragging = false diff --git a/ios/OralTrainer/Views/PlayerSectionView.swift b/ios/OralTrainer/Views/PlayerSectionView.swift index b28e377..68a899b 100644 --- a/ios/OralTrainer/Views/PlayerSectionView.swift +++ b/ios/OralTrainer/Views/PlayerSectionView.swift @@ -10,6 +10,7 @@ struct PlayerSectionView: View { @State private var progressDragging = false @State private var dragPositionMs: Int64 = 0 + @State private var explanationSentence: SentenceBoundary? var body: some View { VStack(spacing: 0) { @@ -19,6 +20,12 @@ struct PlayerSectionView: View { } } .padding(.horizontal, 16) + .sheet(item: $explanationSentence) { sentence in + SentenceExplanationSheet( + courseTitle: controller.currentItem?.title ?? "口语训练课程", + sentence: sentence + ) + } } private var playerSurface: some View { @@ -85,10 +92,33 @@ struct PlayerSectionView: View { Text(sentenceMetaText) .font(.system(size: 13)) .foregroundColor(Theme.textSubtle) - Text(sentenceTextText) + HStack(alignment: .center, spacing: 8) { + Text(sentenceTextText) .font(.system(size: 15)) - .foregroundColor(.white) - .lineLimit(2) + .foregroundColor(.white) + .lineLimit(2) + Spacer() + Button { + guard let sentence = controller.currentSentence, !(sentence.text ?? "").isEmpty else { + return + } + controller.pause() + explanationSentence = sentence + } label: { + Image(systemName: "lightbulb.circle.fill") + .font(.system(size: 23, weight: .semibold)) + .foregroundColor( + controller.currentSentence?.text?.isEmpty == false + ? Theme.accentLight + : Theme.textSubtle + ) + .frame(width: 32, height: 32) + .background(Theme.overlay) + .clipShape(Circle()) + .accessibilityLabel("AI 讲解这句话") + } + .buttonStyle(.plain) + } progressRow Text(statusText) .font(.system(size: 13)) @@ -195,3 +225,53 @@ struct PlayerSectionView: View { controller.setPlaybackSpeed(next) } } + +struct SentenceExplanationSheet: View { + let courseTitle: String + let sentence: SentenceBoundary + + @State private var explanation = "" + @State private var errorMessage: String? + + var body: some View { + NavigationView { + ScrollView { + VStack(alignment: .leading, spacing: 14) { + Text(sentence.text ?? "") + .font(.system(size: 18, weight: .bold)) + .foregroundColor(Theme.textDark) + if let errorMessage { + Text(errorMessage) + .font(.system(size: 15)) + .foregroundColor(Theme.danger) + } else if explanation.isEmpty { + Text("AI 正在讲解这句话…") + .font(.system(size: 15)) + .foregroundColor(Theme.textSubtle) + } else { + Text(explanation) + .font(.system(size: 16)) + .foregroundColor(Theme.textDark) + .lineSpacing(5) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(20) + } + .background(Color.white.ignoresSafeArea()) + .navigationTitle("句子讲解") + .navigationBarTitleDisplayMode(.inline) + } + .task { + guard explanation.isEmpty, errorMessage == nil else { return } + do { + explanation = try await SentenceBoundaryAPI.explain( + sentence, + courseTitle: courseTitle + ) + } catch { + errorMessage = "讲解失败：\(error.localizedDescription)" + } + } + } +} diff --git a/sentence_api/.env.example b/sentence_api/.env.example index b753695..8c7bb89 100644 --- a/sentence_api/.env.example +++ b/sentence_api/.env.example @@ -17,6 +17,13 @@ MOSS_MAX_NEW_TOKENS=65536 # 每句结束时间向后顺延的毫秒数，避免句子末尾发音被切掉；按需在 200-500 之间调整 MOSS_END_PADDING_MS=300 +# AI sentence explanation (OpenAI-compatible chat completions endpoint). +# The URL and key stay on the server; mobile clients only send the sentence text. +AI_API_URL=https://api.example.com/v1/chat/completions +AI_API_KEY=replace-with-your-ai-api-key +AI_MODEL=gpt-4o-mini +AI_TIMEOUT_SECONDS=45 + # 火山引擎对象存储（TOS / 火山 OSS）。桶请设置为公共读，或把下方地址配置为可公开访问的 CDN 域名。 VOLCANO_OSS_ACCESS_KEY=replace-with-volcano-access-key VOLCANO_OSS_SECRET_KEY=replace-with-volcano-secret-key diff --git a/sentence_api/DEPLOYMENT.md b/sentence_api/DEPLOYMENT.md index 6474d0a..9d57d74 100644 --- a/sentence_api/DEPLOYMENT.md +++ b/sentence_api/DEPLOYMENT.md @@ -185,6 +185,9 @@ openssl rand -hex 32 ```dotenv PUBLIC_BASE_URL=https://videoservice.d1kt.cn MOSS_TRANSCRIBE_URL=http://127.0.0.1:8001/v1/audio/transcriptions +AI_API_URL=https://api.example.com/v1/chat/completions +AI_API_KEY=replace-with-your-ai-api-key +AI_MODEL=gpt-4o-mini ``` 若使用 Whisper 方案（第 2A 节），把 `MOSS_TRANSCRIBE_URL` 改为 diff --git a/sentence_api/README.md b/sentence_api/README.md index e33b7fa..2dbe940 100644 --- a/sentence_api/README.md +++ b/sentence_api/README.md @@ -12,6 +12,7 @@ FastAPI service for the Android oral-training SDK. It provides: - video catalog and HTTP Range playback; - sentence-boundary lookup compatible with the existing Android SDK; - student recording upload and `asr-fluency-v1` assessment; +- AI sentence explanation for signed-in mobile users; - the legacy JSON boundary index as a read-only fallback. ## Local Run @@ -50,6 +51,7 @@ GET /api/v1/videos GET /api/v1/videos/{sha256} GET /api/v1/videos/{sha256}/content GET /api/v1/videos/{sha256}/sentence-boundaries +POST /api/v1/sentences/explain POST /api/v1/videos/{sha256}/sentences/{index}/assessments POST /api/v1/admin/videos @@ -64,6 +66,21 @@ DELETE /api/v1/admin/videos/{sha256} Admin endpoints use `X-Admin-Key` when `ADMIN_API_KEY` is configured. Assessment requests use `X-Client-Key` when `CLIENT_API_KEY` is configured. +## AI Sentence Explanation + +Configure an OpenAI-compatible chat-completions endpoint in `sentence_api/.env`: + +```dotenv +AI_API_URL=https://api.example.com/v1/chat/completions +AI_API_KEY=replace-with-your-ai-api-key +AI_MODEL=gpt-4o-mini +AI_TIMEOUT_SECONDS=45 +``` + +The endpoint requires a signed-in user and accepts only the sentence and course +context. `AI_API_URL` and `AI_API_KEY` remain on the server and are never sent +to the mobile clients. + ## Assessment ```bash diff --git a/sentence_api/ai_tutor.py b/sentence_api/ai_tutor.py new file mode 100644 index 0000000..81fe3c2 --- /dev/null +++ b/sentence_api/ai_tutor.py @@ -0,0 +1,75 @@ +"""AI-backed sentence explanations for the mobile clients.""" + +from dataclasses import dataclass + +import requests + + +class AITutorError(RuntimeError): + """Raised when the configured AI service cannot answer a request.""" + + +@dataclass(frozen=True) +class AITutor: + endpoint: str + api_key: str + model: str + timeout_seconds: float + + @property + def available(self) -> bool: + return bool(self.endpoint and self.model) + + def explain( + self, + sentence_text: str, + *, + course_title: str = "", + sentence_index: int | None = None, + language: str = "", + ) -> str: + if not self.available: + raise AITutorError("AI explanation is not configured.") + + context = course_title.strip() or "口语训练课程" + position = f"第 {sentence_index + 1} 句" if sentence_index is not None else "当前句子" + language_note = f"句子语言：{language.strip()}。" if language.strip() else "" + system_prompt = ( + "你是一位耐心的外语口语老师。请用简体中文讲解用户提供的影视/课程句子。" + "先给出自然的中文翻译，再解释关键词、常用搭配或语法要点，并简要说明这句话适合怎样跟读。" + "回答使用简明清晰的分段短文，总长度不超过 300 字，不要编造视频里没有的信息。" + ) + user_prompt = ( + f"课程：{context}\n位置：{position}\n{language_note}句子：{sentence_text}" + ) + headers = {"Content-Type": "application/json"} + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + try: + response = requests.post( + self.endpoint, + headers=headers, + json={ + "model": self.model, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + "temperature": 0.3, + "stream": False, + }, + timeout=self.timeout_seconds, + ) + except requests.RequestException as exc: + raise AITutorError("AI service request failed.") from exc + + if response.status_code < 200 or response.status_code >= 300: + raise AITutorError(f"AI service returned HTTP {response.status_code}.") + try: + body = response.json() + explanation = body["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError, ValueError) as exc: + raise AITutorError("AI service returned an unexpected response.") from exc + if not isinstance(explanation, str) or not explanation.strip(): + raise AITutorError("AI service returned an empty explanation.") + return explanation.strip() diff --git a/sentence_api/config.py b/sentence_api/config.py index 1694d8e..0a7b5fe 100644 --- a/sentence_api/config.py +++ b/sentence_api/config.py @@ -26,6 +26,10 @@ class Settings: moss_timeout_seconds: float moss_max_new_tokens: int moss_end_padding_ms: int + ai_api_url: str + ai_api_key: str + ai_model: str + ai_timeout_seconds: float pass_score: float volcano_oss_access_key: str volcano_oss_secret_key: str @@ -61,6 +65,10 @@ class Settings: moss_timeout_seconds=float(os.getenv("MOSS_TIMEOUT_SECONDS", "1800")), moss_max_new_tokens=int(os.getenv("MOSS_MAX_NEW_TOKENS", "65536")), moss_end_padding_ms=int(os.getenv("MOSS_END_PADDING_MS", "300")), + ai_api_url=os.getenv("AI_API_URL", "").strip().rstrip("/"), + ai_api_key=os.getenv("AI_API_KEY", "").strip(), + ai_model=os.getenv("AI_MODEL", "gpt-4o-mini"), + ai_timeout_seconds=float(os.getenv("AI_TIMEOUT_SECONDS", "45")), pass_score=float(os.getenv("ASSESSMENT_PASS_SCORE", "70")), volcano_oss_access_key=os.getenv("VOLCANO_OSS_ACCESS_KEY", "").strip(), volcano_oss_secret_key=os.getenv("VOLCANO_OSS_SECRET_KEY", "").strip(), diff --git a/sentence_api/main.py b/sentence_api/main.py index 0f6756f..c42a17c 100644 --- a/sentence_api/main.py +++ b/sentence_api/main.py @@ -27,6 +27,7 @@ from fastapi.responses import FileResponse, RedirectResponse, Response from fastapi.staticfiles import StaticFiles from starlette.concurrency import run_in_threadpool from starlette.requests import ClientDisconnect +from .ai_tutor import AITutor, AITutorError from .assessment import AssessmentService from .auth import AuthenticatedUser @@ -40,6 +41,8 @@ from .models import ( SentenceBoundaryAdjust, SentenceBoundary, SentenceBoundaryDocument, + SentenceExplanationRequest, + SentenceExplanationResponse, SentenceTextUpdate, UserDubShareSummary, SsoExchangeRequest, @@ -93,6 +96,12 @@ def create_app( timeout_seconds=service_settings.moss_timeout_seconds, max_new_tokens=service_settings.moss_max_new_tokens, ) + ai_tutor = AITutor( + endpoint=service_settings.ai_api_url, + api_key=service_settings.ai_api_key, + model=service_settings.ai_model, + timeout_seconds=service_settings.ai_timeout_seconds, + ) oss = oss_uploader or VolcanoOSSUploader(service_settings) processor = VideoProcessor(service_settings, video_repository, moss, oss) assessment_service = AssessmentService( @@ -110,6 +119,7 @@ def create_app( application.state.boundary_store = legacy_store application.state.video_repository = video_repository application.state.transcriber = moss + application.state.ai_tutor = ai_tutor application.state.oss_uploader = oss application.state.video_processor = processor application.state.assessment_service = assessment_service @@ -161,6 +171,7 @@ def create_app( "video_count": len(videos) + legacy_store.count(), "managed_video_count": len(videos), "moss_configured": moss.available, + "ai_explanation_configured": application.state.ai_tutor.available, "volcano_oss_configured": oss.enabled, "scoring_version": "asr-fluency-v1", } @@ -298,6 +309,26 @@ def create_app( raise HTTPException(status_code=404, detail="Course is not available.") return Response(status_code=204) + @application.post( + "/api/v1/sentences/explain", + response_model=SentenceExplanationResponse, + ) + def explain_sentence( + payload: SentenceExplanationRequest, + user: AuthenticatedUser = Depends(current_user), + ) -> SentenceExplanationResponse: + del user + try: + explanation = application.state.ai_tutor.explain( + payload.sentence_text, + course_title=payload.course_title, + sentence_index=payload.sentence_index, + language=payload.language or "", + ) + except AITutorError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + return SentenceExplanationResponse(explanation=explanation) + @application.delete("/api/v1/courses/{video_hash}/enroll", status_code=204) def unenroll_course( video_hash: str = SHA256_PATH, diff --git a/sentence_api/models.py b/sentence_api/models.py index 8e461e7..f388ed5 100644 --- a/sentence_api/models.py +++ b/sentence_api/models.py @@ -92,6 +92,19 @@ class SentenceTextUpdate(BaseModel): language: Optional[str] = Field(default=None, max_length=32) +class SentenceExplanationRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + sentence_text: str = Field(min_length=1, max_length=4000) + course_title: str = Field(default="", max_length=200) + sentence_index: Optional[int] = Field(default=None, ge=0) + language: Optional[str] = Field(default=None, max_length=32) + + +class SentenceExplanationResponse(BaseModel): + explanation: str = Field(min_length=1, max_length=6000) + + class SentenceBoundaryAdjust(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/sentence_api/tests/test_ai_tutor.py b/sentence_api/tests/test_ai_tutor.py new file mode 100644 index 0000000..ad0a564 --- /dev/null +++ b/sentence_api/tests/test_ai_tutor.py @@ -0,0 +1,57 @@ +import pytest + +from sentence_api.ai_tutor import AITutor, AITutorError + + +class FakeResponse: + def __init__(self, status_code=200, body=None): + self.status_code = status_code + self._body = body or {} + + def json(self): + if self._body is None: + raise ValueError("invalid JSON") + return self._body + + +def test_ai_tutor_parses_chat_completion(monkeypatch): + requests_payloads = [] + + def fake_post(endpoint, headers, json, timeout): + requests_payloads.append((endpoint, headers, json, timeout)) + return FakeResponse( + body={ + "choices": [ + {"message": {"content": " 这句话的意思是…… "}} + ] + } + ) + + monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post) + tutor = AITutor("https://ai.test/chat", "secret", "test-model", 1.5) + + explanation = tutor.explain( + "The early bird catches the worm.", + course_title="English idioms", + sentence_index=1, + language="en", + ) + + endpoint, headers, payload, timeout = requests_payloads[0] + assert explanation == "这句话的意思是……" + assert endpoint == "https://ai.test/chat" + assert headers["Authorization"] == "Bearer secret" + assert payload["model"] == "test-model" + assert "The early bird catches the worm." in payload["messages"][1]["content"] + assert timeout == 1.5 + + +def test_ai_tutor_rejects_unexpected_response(monkeypatch): + monkeypatch.setattr( + "sentence_api.ai_tutor.requests.post", + lambda *args, **kwargs: FakeResponse(body={"choices": []}), + ) + tutor = AITutor("https://ai.test/chat", "secret", "test-model", 1) + + with pytest.raises(AITutorError): + tutor.explain("Hello.") diff --git a/sentence_api/tests/test_api.py b/sentence_api/tests/test_api.py index 760da13..5fe445a 100644 --- a/sentence_api/tests/test_api.py +++ b/sentence_api/tests/test_api.py @@ -80,6 +80,70 @@ def test_invalid_hash_is_rejected(tmp_path): assert response.status_code == 422 +def test_sentence_explanation_requires_login(tmp_path): + index_path = tmp_path / "boundaries.json" + index_path.write_text("{}", encoding="utf-8") + settings = replace( + Settings.from_env(), + data_dir=tmp_path / "data", + legacy_boundaries_file=index_path, + ) + settings.ensure_directories() + client = TestClient( + create_app( + BoundaryStore(index_path), + settings=settings, + repository=VideoRepository(settings.database_path), + ) + ) + + response = client.post( + "/api/v1/sentences/explain", + json={"sentence_text": "Hello."}, + ) + + assert response.status_code == 401 + + +def test_sentence_explanation_uses_server_side_ai_client(tmp_path, monkeypatch): + client = make_client(tmp_path) + requested = [] + + def explain(text, **kwargs): + requested.append((text, kwargs)) + return "这句话表示打招呼。" + + class StubAITutor: + pass + + stub = StubAITutor() + stub.explain = explain + monkeypatch.setattr(client.app.state, "ai_tutor", stub) + + response = client.post( + "/api/v1/sentences/explain", + json={ + "sentence_text": "Hello.", + "course_title": "Greetings", + "sentence_index": 0, + "language": "en", + }, + ) + + assert response.status_code == 200 + assert response.json() == {"explanation": "这句话表示打招呼。"} + assert requested == [ + ( + "Hello.", + { + "course_title": "Greetings", + "sentence_index": 0, + "language": "en", + }, + ) + ] + + def test_oss_video_url_is_used_for_catalog_and_content(tmp_path): index_path = tmp_path / "boundaries.json" index_path.write_text("{}", encoding="utf-8")