From ae843479120ad185410428f24b793057768cf6b3 Mon Sep 17 00:00:00 2001 From: Shuming Liu Date: Sun, 25 Oct 2026 10:30:22 +0800 Subject: [PATCH] fff --- sentence_api/README.md | 1 + sentence_api/ai_tutor.py | 27 ++++++++- sentence_api/main.py | 17 ++++++ sentence_api/models.py | 2 + sentence_api/repository.py | 25 ++++++-- sentence_api/static/admin.css | 1 + sentence_api/static/admin.html | 4 +- sentence_api/static/admin.js | 67 +++++++++++++++++++++- sentence_api/tests/test_ai_tutor.py | 36 ++++++++++++ sentence_api/tests/test_api.py | 88 +++++++++++++++++++++++++++++ sentence_api/translation_jobs.py | 31 +++++++++- 11 files changed, 287 insertions(+), 12 deletions(-) diff --git a/sentence_api/README.md b/sentence_api/README.md index c4aa92b..e4f4a4a 100644 --- a/sentence_api/README.md +++ b/sentence_api/README.md @@ -62,6 +62,7 @@ POST /api/v1/admin/videos/{sha256}/process POST /api/v1/admin/videos/{sha256}/repair-gaps GET /api/v1/admin/videos/{sha256}/subtitle-translation POST /api/v1/admin/videos/{sha256}/subtitle-translation +POST /api/v1/admin/videos/{sha256}/subtitle-translation/repair PUT /api/v1/admin/videos/{sha256}/sentences/{index} PUT /api/v1/admin/videos/{sha256}/sentences/{index}/split PUT /api/v1/admin/videos/{sha256}/sentences/{index}/boundary diff --git a/sentence_api/ai_tutor.py b/sentence_api/ai_tutor.py index a2bc569..44125cf 100644 --- a/sentence_api/ai_tutor.py +++ b/sentence_api/ai_tutor.py @@ -14,6 +14,7 @@ import requests logger = logging.getLogger(__name__) _JSON_FENCE_RE = re.compile(r"^```(?:json)?\s*|\s*```$", re.IGNORECASE) +_CJK_RE = re.compile("[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]") _NUMBERED_LINE_RE = re.compile(r"^\s*[\[(]?(\d+)[\])]?[\.、:：\)]\s*(.+?)\s*$") _TRANSLATE_BATCH_SENTENCES = 12 _TRANSLATE_BATCH_CHARS = 2_200 @@ -95,10 +96,13 @@ class AITutor: raise AITutorError("No sentence text was provided for translation.") glossary = _normalize_translation_terms(terms) + source_by_index = {index: text for index, text in ordered} translations = { index: text.strip() for index, text in (existing or {}).items() - if isinstance(text, str) and text.strip() + if isinstance(text, str) + and text.strip() + and not translation_needs_repair(source_by_index.get(index, ""), text) } pending = [(index, text) for index, text in ordered if index not in translations] if not pending: @@ -304,10 +308,14 @@ class AITutor: timeout_seconds=max(self.timeout_seconds, _TRANSLATE_TIMEOUT_SECONDS), ) parsed = _parse_translation_payload(content, expected_indexes) - missing = [index for index in expected_indexes if not parsed.get(index, "").strip()] + source_by_index = {index: text for index, text in batch} + missing = [ + index + for index in expected_indexes + if translation_needs_repair(source_by_index.get(index, ""), parsed.get(index, "")) + ] if missing: raise AITutorError("AI translation returned an unexpected response.") - source_by_index = {index: text for index, text in batch} return { index: _apply_translation_terms(source_by_index.get(index, ""), parsed[index], glossary) for index in expected_indexes @@ -377,6 +385,19 @@ class AITutor: return content.strip() +def translation_needs_repair(source: str | None, zh_text: str | None) -> bool: + """Return True when a sentence still needs a Chinese translation.""" + source_text = (source or "").strip() + if not source_text: + return False + zh = (zh_text or "").strip() + if not zh: + return True + if _CJK_RE.search(zh): + return False + return _CJK_RE.search(source_text) is None + + def _translation_batches( pending: list[tuple[int, str]], all_sentences: list[tuple[int, str]], diff --git a/sentence_api/main.py b/sentence_api/main.py index c5c3619..b6bec6f 100644 --- a/sentence_api/main.py +++ b/sentence_api/main.py @@ -753,6 +753,23 @@ def create_app( except AITutorError as exc: raise HTTPException(status_code=503, detail=str(exc)) from exc + @application.post( + "/api/v1/admin/videos/{video_hash}/subtitle-translation/repair", + response_model=AdminSubtitleTranslationResponse, + dependencies=[Depends(require_admin)], + ) + def repair_subtitle_translation( + video_hash: str = SHA256_PATH, + ) -> AdminSubtitleTranslationResponse: + try: + return application.state.subtitle_translations.repair_missing(video_hash) + except KeyError: + raise HTTPException(status_code=404, detail="Course is not available.") from None + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + except AITutorError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + @application.put( "/api/v1/admin/videos/{video_hash}/title", response_model=VideoSummary, diff --git a/sentence_api/models.py b/sentence_api/models.py index afba7ba..a1b7d1c 100644 --- a/sentence_api/models.py +++ b/sentence_api/models.py @@ -177,6 +177,8 @@ class SubtitleRetranslateRequest(BaseModel): class AdminSubtitleTranslationResponse(SubtitleTranslationResponse): terms: List[TranslationTerm] = Field(default_factory=list) + missing_count: int = Field(default=0, ge=0) + attempted_count: int = Field(default=0, ge=0) class SentenceBoundaryAdjust(BaseModel): diff --git a/sentence_api/repository.py b/sentence_api/repository.py index c79f999..09cc017 100644 --- a/sentence_api/repository.py +++ b/sentence_api/repository.py @@ -1011,11 +1011,28 @@ class VideoRepository: ) return cursor.rowcount > 0 - def clear_sentence_translations(self, video_hash: str) -> None: + def clear_sentence_translations( + self, + video_hash: str, + indexes: Optional[Sequence[int]] = None, + ) -> None: + normalized_hash = normalize_video_hash(video_hash) with self._connect() as connection: - connection.execute( - "UPDATE sentences SET zh_text = NULL WHERE video_hash = ?", - (normalize_video_hash(video_hash),), + if indexes is None: + connection.execute( + "UPDATE sentences SET zh_text = NULL WHERE video_hash = ?", + (normalized_hash,), + ) + return + unique_indexes = list(dict.fromkeys(indexes)) + if not unique_indexes: + return + connection.executemany( + """ + UPDATE sentences SET zh_text = NULL + WHERE video_hash = ? AND sentence_index = ? + """, + [(normalized_hash, index) for index in unique_indexes], ) def save_sentence_translations( diff --git a/sentence_api/static/admin.css b/sentence_api/static/admin.css index 532e8eb..c857673 100644 --- a/sentence_api/static/admin.css +++ b/sentence_api/static/admin.css @@ -105,6 +105,7 @@ dialog::backdrop { background: rgb(20 25 28 / 55%); } .dialog-heading span { font-size: 12px; color: #68747d; } .dialog-heading-actions { display: flex; align-items: center; gap: 8px; } .gap-row td { background: #fff8ee; color: #9a5b12; font-style: italic; } +.missing-zh-row .zh-text { color: #a45d08; } .sentence-scroll { max-height: calc(100vh - 120px); overflow: auto; } .sentence-scroll textarea { width: min(600px, 45vw); min-width: 260px; min-height: 58px; resize: vertical; padding: 7px; border: 1px solid #b8c0c7; border-radius: 3px; } .sentence-scroll select { min-width: 90px; } diff --git a/sentence_api/static/admin.html b/sentence_api/static/admin.html index c7d637e..20690e3 100644 --- a/sentence_api/static/admin.html +++ b/sentence_api/static/admin.html @@ -111,6 +111,7 @@
+
@@ -124,6 +125,7 @@
+
@@ -137,6 +139,6 @@ - + diff --git a/sentence_api/static/admin.js b/sentence_api/static/admin.js index 234f2ba..b3c8ea4 100644 --- a/sentence_api/static/admin.js +++ b/sentence_api/static/admin.js @@ -1,4 +1,5 @@ const GAP_REPAIR_MIN_MS = 8000; +const CJK_RE = /[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/; const state = { adminKey: localStorage.getItem("oralTrainerAdminKey") || "", @@ -27,11 +28,13 @@ const elements = { sentenceRows: document.querySelector("#sentence-rows"), closeDialog: document.querySelector("#close-dialog"), repairGaps: document.querySelector("#repair-gaps"), + repairMissingZh: document.querySelector("#repair-missing-zh"), retranslateDialog: document.querySelector("#retranslate-dialog"), retranslateTitle: document.querySelector("#retranslate-title"), retranslateMeta: document.querySelector("#retranslate-meta"), retranslateForm: document.querySelector("#retranslate-form"), retranslateTerms: document.querySelector("#retranslate-terms"), + repairTranslations: document.querySelector("#repair-translations"), closeRetranslateDialog: document.querySelector("#close-retranslate-dialog"), userRows: document.querySelector("#user-rows"), userCount: document.querySelector("#user-count"), @@ -52,7 +55,13 @@ elements.closeDialog.addEventListener("click", () => elements.dialog.close()); elements.repairGaps.addEventListener("click", () => { if (state.sentenceVideo) repairGaps(state.sentenceVideo, { reloadSentences: true }); }); +elements.repairMissingZh.addEventListener("click", () => { + if (state.sentenceVideo) repairMissingTranslations(state.sentenceVideo, { reloadSentences: true }); +}); elements.closeRetranslateDialog.addEventListener("click", () => elements.retranslateDialog.close()); +elements.repairTranslations.addEventListener("click", () => { + if (state.retranslateVideo) repairMissingTranslations(state.retranslateVideo, { closeRetranslate: true }); +}); elements.retranslateForm.addEventListener("submit", submitRetranslate); elements.uploadForm.addEventListener("submit", uploadVideo); elements.videoFile.addEventListener("change", () => { @@ -295,9 +304,12 @@ async function openSentences(video) { const sentences = detail.boundaries?.sentences || []; const durationMs = detail.video.duration_ms || 0; const gaps = collectSentenceGaps(sentences, durationMs); + const missingZh = sentences.filter(translationNeedsRepair).length; const gapLabel = gaps.length ? ` / ${gaps.length} 段空白待修复` : ""; - elements.sentenceMeta.textContent = `${sentences.length} 句${gapLabel} / ${detail.video.video_hash}`; + const zhLabel = missingZh ? ` / ${missingZh} 句未译` : ""; + elements.sentenceMeta.textContent = `${sentences.length} 句${gapLabel}${zhLabel} / ${detail.video.video_hash}`; elements.repairGaps.disabled = gaps.length === 0; + elements.repairMissingZh.disabled = missingZh === 0; elements.sentenceRows.replaceChildren(); let gapIndex = 0; for (const sentence of sentences) { @@ -345,8 +357,18 @@ function gapRow(gap) { return row; } +function translationNeedsRepair(sentence) { + const source = (sentence.text || "").trim(); + const zh = (sentence.zh_text || "").trim(); + if (!source) return false; + if (!zh) return true; + if (CJK_RE.test(zh)) return false; + return !CJK_RE.test(source); +} + function sentenceRow(video, sentence, totalCount) { const row = document.createElement("tr"); + if (translationNeedsRepair(sentence)) row.className = "missing-zh-row"; const textArea = document.createElement("textarea"); textArea.value = sentence.text || ""; const language = document.createElement("select"); @@ -360,7 +382,9 @@ function sentenceRow(video, sentence, totalCount) { const textContainer = document.createElement("td"); const zhNote = document.createElement("div"); zhNote.className = "zh-text"; - zhNote.textContent = sentence.zh_text || "暂无中文翻译"; + zhNote.textContent = translationNeedsRepair(sentence) + ? (sentence.zh_text ? `仍是英文：${sentence.zh_text}` : "暂无中文翻译") + : sentence.zh_text; textContainer.append(textArea, zhNote); const languageContainer = document.createElement("td"); languageContainer.append(language); @@ -479,8 +503,12 @@ async function openRetranslate(video) { try { const payload = await api(`/api/v1/admin/videos/${video.video_hash}/subtitle-translation`); elements.retranslateTerms.value = formatTranslationTerms(payload.terms || []); - const status = payload.complete ? "中文字幕已就绪" : (payload.status === "running" ? "正在翻译" : "尚未翻译完成"); + const missing = payload.missing_count || 0; + const status = missing + ? `${missing} 句未译` + : (payload.complete ? "中文字幕已就绪" : (payload.status === "running" ? "正在翻译" : "尚未翻译完成")); elements.retranslateMeta.textContent = `${status} / ${video.video_hash}`; + elements.repairTranslations.disabled = missing === 0; } catch (error) { elements.serviceState.textContent = `加载翻译信息失败: ${error.message}`; } @@ -540,6 +568,39 @@ async function submitRetranslate(event) { } } +async function repairMissingTranslations(video, options = {}) { + const confirmed = window.confirm( + `将只补译“${video.title}”里缺失或仍是英文的中文，已有中文会保留。继续？`, + ); + if (!confirmed) return; + const buttons = [elements.repairMissingZh, elements.repairTranslations].filter(Boolean); + buttons.forEach((button) => { button.disabled = true; }); + elements.serviceState.textContent = "正在补译缺失的中文…"; + try { + const payload = await api(`/api/v1/admin/videos/${video.video_hash}/subtitle-translation/repair`, { + method: "POST", + }); + const attempted = payload.attempted_count || 0; + if (!attempted) { + elements.serviceState.textContent = "没有需要补译的句子"; + } else if (payload.complete) { + elements.serviceState.textContent = `补译完成：处理了 ${attempted} 句`; + } else { + const remaining = payload.missing_count ? `，还剩 ${payload.missing_count} 句` : ""; + elements.serviceState.textContent = `已开始补译 ${attempted} 句${remaining}，请稍后刷新`; + } + if (options.closeRetranslate) elements.retranslateDialog.close(); + await loadVideos(); + if (options.reloadSentences) await openSentences(video); + } catch (error) { + elements.serviceState.textContent = `补译失败: ${error.message}`; + } finally { + if (!options.reloadSentences) { + buttons.forEach((button) => { button.disabled = false; }); + } + } +} + async function reprocess(videoHash) { try { await api(`/api/v1/admin/videos/${videoHash}/process`, { method: "POST" }); diff --git a/sentence_api/tests/test_ai_tutor.py b/sentence_api/tests/test_ai_tutor.py index 21fab38..dc43974 100644 --- a/sentence_api/tests/test_ai_tutor.py +++ b/sentence_api/tests/test_ai_tutor.py @@ -212,3 +212,39 @@ def test_parse_translation_payload_accepts_fenced_json_and_numbered_lines(): 12: "甲", 13: "乙", } + + +def test_ai_tutor_retranslates_english_existing_cache(monkeypatch): + def fake_post(endpoint, headers, json, timeout): + return _translation_response_from_prompt(json) + + monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post) + tutor = AITutor("https://ai.test/v1", "secret", "test-model", 1) + translated = tutor.translate_sentences( + [(0, "Hello."), (1, "Thanks.")], + existing={0: "Hello.", 1: "谢谢。"}, + ) + + assert translated[0] == "中文0" + assert translated[1] == "谢谢。" + + +def test_ai_tutor_retries_when_translation_stays_english(monkeypatch): + calls = {"count": 0} + + def fake_post(endpoint, headers, json, timeout): + calls["count"] += 1 + if calls["count"] == 1: + return FakeResponse( + body={"choices": [{"message": {"content": '{"0":"Hello."}'}}]} + ) + return FakeResponse( + body={"choices": [{"message": {"content": '{"0":"你好。"}'}}]} + ) + + monkeypatch.setattr("sentence_api.ai_tutor.requests.post", fake_post) + tutor = AITutor("https://ai.test/v1", "secret", "test-model", 1) + translated = tutor.translate_sentences([(0, "Hello.")]) + + assert translated == {0: "你好。"} + assert calls["count"] >= 2 diff --git a/sentence_api/tests/test_api.py b/sentence_api/tests/test_api.py index 44a7864..b039776 100644 --- a/sentence_api/tests/test_api.py +++ b/sentence_api/tests/test_api.py @@ -1165,3 +1165,91 @@ def test_admin_gap_repair_requires_transcriber(tmp_path): response = client.post(f"/api/v1/admin/videos/{VIDEO_HASH}/repair-gaps") assert response.status_code == 503 + + +def test_admin_can_repair_missing_and_english_translations(tmp_path, monkeypatch): + client = make_client(tmp_path) + repository = client.app.state.video_repository + _ready_video(repository) + repository.save_sentence_translations( + VIDEO_HASH, + {0: "你好。", 1: "Nice to meet you."}, + ) + + requested = [] + + def translate_sentences(sentences, **kwargs): + requested.append([index for index, _text in sentences]) + existing = dict(kwargs.get("existing") or {}) + result = {} + for index, _text in sentences: + current = existing.get(index, "") + if current and any("\u4e00" <= char <= "\u9fff" for char in current): + result[index] = current + else: + result[index] = f"补译{index}" + return result + + class StubAITutor: + available = True + + stub = StubAITutor() + stub.translate_sentences = translate_sentences + monkeypatch.setattr(client.app.state, "ai_tutor", stub) + client.app.state.subtitle_translations.ai_tutor = stub + client.app.state.subtitle_translations.post_wait_seconds = 0.2 + + snapshot = client.get(f"/api/v1/admin/videos/{VIDEO_HASH}/subtitle-translation") + assert snapshot.status_code == 200 + assert snapshot.json()["missing_count"] == 1 + + response = client.post( + f"/api/v1/admin/videos/{VIDEO_HASH}/subtitle-translation/repair" + ) + assert response.status_code == 200, response.text + payload = response.json() + if not payload.get("complete"): + finished = wait_for_subtitle_translation(client, VIDEO_HASH) + payload = client.get(f"/api/v1/admin/videos/{VIDEO_HASH}/subtitle-translation").json() + payload["attempted_count"] = response.json()["attempted_count"] + payload["complete"] = finished.json()["complete"] + + assert payload["attempted_count"] == 1 + assert payload["complete"] is True + assert payload["missing_count"] == 0 + document = repository.get_document(VIDEO_HASH) + assert [sentence.zh_text for sentence in document.sentences] == ["你好。", "补译1"] + assert requested + + +def test_admin_translation_repair_keeps_complete_chinese(tmp_path, monkeypatch): + client = make_client(tmp_path) + repository = client.app.state.video_repository + _ready_video(repository) + repository.save_sentence_translations( + VIDEO_HASH, + {0: "你好。", 1: "见到你很高兴。"}, + ) + called = {"count": 0} + + def translate_sentences(sentences, **kwargs): + called["count"] += 1 + return {index: f"新{index}" for index, _text in sentences} + + class StubAITutor: + available = True + + stub = StubAITutor() + stub.translate_sentences = translate_sentences + monkeypatch.setattr(client.app.state, "ai_tutor", stub) + client.app.state.subtitle_translations.ai_tutor = stub + + response = client.post( + f"/api/v1/admin/videos/{VIDEO_HASH}/subtitle-translation/repair" + ) + assert response.status_code == 200, response.text + assert response.json()["attempted_count"] == 0 + assert response.json()["missing_count"] == 0 + assert called["count"] == 0 + document = repository.get_document(VIDEO_HASH) + assert [sentence.zh_text for sentence in document.sentences] == ["你好。", "见到你很高兴。"] diff --git a/sentence_api/translation_jobs.py b/sentence_api/translation_jobs.py index 3c47691..4bea816 100644 --- a/sentence_api/translation_jobs.py +++ b/sentence_api/translation_jobs.py @@ -6,7 +6,7 @@ import logging import threading from dataclasses import dataclass -from .ai_tutor import AITutor, AITutorError +from .ai_tutor import AITutor, AITutorError, translation_needs_repair from .models import ( AdminSubtitleTranslationResponse, SentenceBoundaryDocument, @@ -133,6 +133,23 @@ class SubtitleTranslationService: video_hash, ) + def repair_missing(self, video_hash: str) -> AdminSubtitleTranslationResponse: + """Re-translate missing or still-English sentences, keeping good Chinese.""" + if self.repository.get_video(video_hash) is None: + raise KeyError(video_hash) + document = self.repository.get_document(video_hash) + if document is None: + raise KeyError(video_hash) + missing = missing_translation_indexes(document) + if missing: + self.invalidate(video_hash) + self.repository.clear_sentence_translations(video_hash, missing) + snapshot = self.ensure(video_hash) + else: + snapshot = self.snapshot(document) + result = self._with_terms(snapshot) + return result.model_copy(update={"attempted_count": len(missing)}) + def admin_snapshot(self, video_hash: str) -> AdminSubtitleTranslationResponse | None: current = self.current(video_hash) if current is None: @@ -157,8 +174,12 @@ class SubtitleTranslationService: TranslationTerm(source=source, zh=zh) for source, zh in self.repository.get_translation_glossary(snapshot.video_hash) ] + document = self.repository.get_document(snapshot.video_hash) + missing_count = 0 if document is None else len(missing_translation_indexes(document)) payload = snapshot.model_dump() payload["terms"] = terms + payload["missing_count"] = missing_count + payload["attempted_count"] = 0 return AdminSubtitleTranslationResponse(**payload) def ensure_from_local(self, document: SentenceBoundaryDocument) -> SubtitleTranslationResponse: @@ -275,3 +296,11 @@ class SubtitleTranslationService: with job.lock: job.running = False job.done.set() + + +def missing_translation_indexes(document: SentenceBoundaryDocument) -> list[int]: + return [ + sentence.index + for sentence in document.sentences + if translation_needs_repair(sentence.text, sentence.zh_text) + ]