"""AI-backed sentence explanations and subtitle translation.""" from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass import json import logging import re import threading from urllib.parse import urlsplit import requests logger = logging.getLogger(__name__) _JSON_FENCE_RE = re.compile(r"^```(?:json)?\s*|\s*```$", re.IGNORECASE) _CJK_RE = re.compile("[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]") _NUMBERED_LINE_RE = re.compile(r"^\s*[\[(]?(\d+)[\])]?[\.、::\)]\s*(.+?)\s*$") _TRANSLATE_BATCH_SENTENCES = 12 _TRANSLATE_BATCH_CHARS = 2_200 _TRANSLATE_CONTEXT_BEFORE = 8 _TRANSLATE_CONTEXT_AFTER = 4 _TRANSLATE_WORKERS = 4 _TRANSLATE_TIMEOUT_SECONDS = 120.0 _TRANSLATE_SPLIT_THRESHOLD = 4 class AITutorError(RuntimeError): """Raised when the configured AI service cannot answer a request.""" @dataclass(frozen=True) class AITutor: endpoint: str api_key: str model: str timeout_seconds: float translation_timeout_seconds: float = _TRANSLATE_TIMEOUT_SECONDS @property def available(self) -> bool: return bool(self.endpoint and self.model) @property def request_url(self) -> str: """Resolve either a full chat URL or an OpenAI-compatible base URL.""" parsed = urlsplit(self.endpoint) path = parsed.path.rstrip("/") if parsed.query or path.endswith("/chat/completions"): return self.endpoint if path.endswith("/v1") or path.endswith("/v1/openai"): return self.endpoint + "/chat/completions" if not path: return self.endpoint + "/v1/chat/completions" return self.endpoint def explain( self, sentence_text: str, *, course_title: str = "", sentence_index: int | None = None, language: str = "", ) -> str: if not self.available: raise AITutorError("AI explanation is not configured.") context = course_title.strip() or "口语训练课程" position = f"第 {sentence_index + 1} 句" if sentence_index is not None else "当前句子" language_note = f"句子语言:{language.strip()}。" if language.strip() else "" system_prompt = ( "你是一位耐心的外语口语老师。请用简体中文讲解用户提供的影视/课程句子。" "先给出自然的中文翻译,再解释关键词、常用搭配、语法要点,并简要说明这句话适合怎样跟读。" "回答使用简明清晰的分段短文,总长度不超过 600 字,不要编造视频里没有的信息。" ) user_prompt = ( f"课程:{context}\n位置:{position}\n{language_note}句子:{sentence_text}" ) return self._complete(system_prompt, user_prompt, temperature=0.3) def translate_sentences( self, sentences: list[tuple[int, str]], *, course_title: str = "", existing: dict[int, str] | None = None, persist=None, terms: list[tuple[str, str]] | None = None, ) -> dict[int, str]: if not self.available: raise AITutorError("AI translation is not configured.") if not sentences: raise AITutorError("No sentences were provided for translation.") ordered = [(index, text.strip()) for index, text in sentences if text.strip()] if not ordered: raise AITutorError("No sentence text was provided for translation.") glossary = _normalize_translation_terms(terms) source_by_index = {index: text for index, text in ordered} translations = { index: text.strip() for index, text in (existing or {}).items() if isinstance(text, str) and text.strip() and not translation_needs_repair(source_by_index.get(index, ""), text) } pending = [(index, text) for index, text in ordered if index not in translations] if not pending: return translations persist_lock = threading.Lock() def save_partial(partial: dict[int, str]) -> None: if not partial: return translations.update(partial) if persist is None: return with persist_lock: persist(partial) batches = _translation_batches(pending, ordered) workers = min(_TRANSLATE_WORKERS, len(batches)) or 1 failures: list[tuple[list[tuple[int, str]], Exception]] = [] def run_batch(batch: list[tuple[int, str]]) -> dict[int, str]: return self._translate_batch_resilient( batch, all_sentences=ordered, existing=translations, course_title=course_title, terms=glossary, ) if workers == 1: for batch in batches: try: save_partial(run_batch(batch)) except Exception as exc: # noqa: BLE001 - collect and retry below failures.append((batch, exc)) else: with ThreadPoolExecutor(max_workers=workers) as pool: futures = {pool.submit(run_batch, batch): batch for batch in batches} for future in as_completed(futures): batch = futures[future] try: save_partial(future.result()) except Exception as exc: # noqa: BLE001 failures.append((batch, exc)) for batch, error in failures: logger.warning( "Translation batch failed, retrying with smaller windows: size=%s error=%s", len(batch), error, ) save_partial( self._translate_batch_resilient( batch, all_sentences=ordered, existing=translations, course_title=course_title, terms=glossary, force_split=len(batch) > 1, ) ) missing = [index for index, _ in pending if not translations.get(index, "").strip()] if missing: raise AITutorError("AI translation did not cover every sentence.") if glossary: source_by_index = {index: text for index, text in ordered} translations = { index: _apply_translation_terms(source_by_index.get(index, ""), zh_text, glossary) for index, zh_text in translations.items() } return translations def _translate_batch_resilient( self, batch: list[tuple[int, str]], *, all_sentences: list[tuple[int, str]], existing: dict[int, str], course_title: str, terms: list[tuple[str, str]] | None = None, force_split: bool = False, ) -> dict[int, str]: if not batch: return {} if force_split and len(batch) > 1: return self._translate_split_batch( batch, all_sentences=all_sentences, existing=existing, course_title=course_title, terms=terms, ) last_error: Exception | None = None attempts = 2 if len(batch) <= _TRANSLATE_SPLIT_THRESHOLD else 1 for _ in range(attempts): try: return self._translate_batch( batch, all_sentences=all_sentences, existing=existing, course_title=course_title, terms=terms, ) except AITutorError as exc: last_error = exc logger.warning( "Translation batch attempt failed: size=%s indexes=%s error=%s", len(batch), [index for index, _ in batch], exc, ) if len(batch) > 1: return self._translate_split_batch( batch, all_sentences=all_sentences, existing=existing, course_title=course_title, terms=terms, ) if last_error is not None: raise last_error raise AITutorError("AI translation did not cover every sentence.") def _translate_split_batch( self, batch: list[tuple[int, str]], *, all_sentences: list[tuple[int, str]], existing: dict[int, str], course_title: str, terms: list[tuple[str, str]] | None = None, ) -> dict[int, str]: mid = max(1, len(batch) // 2) merged: dict[int, str] = {} for part in (batch[:mid], batch[mid:]): merged.update( self._translate_batch_resilient( part, all_sentences=all_sentences, existing=existing, course_title=course_title, terms=terms, ) ) return merged def _translate_batch( self, batch: list[tuple[int, str]], *, all_sentences: list[tuple[int, str]], existing: dict[int, str], course_title: str, terms: list[tuple[str, str]] | None = None, ) -> dict[int, str]: context = course_title.strip() or "口语训练课程" existing = dict(existing) before, after = _neighbor_context(batch, all_sentences) expected_indexes = [index for index, _ in batch] glossary = _normalize_translation_terms(terms) system_prompt = ( "你是专业的影视字幕翻译。请结合前后文语境,把指定编号的外语句子翻译成自然、口语化的简体中文。" "必须一句对一句,不能合并、拆分、跳过或改写编号。" "只输出 JSON 对象,键是句子编号,值是对应中文,不要输出解释或 markdown。" ) if glossary: system_prompt += "必须严格使用用户指定的专有名词译法,禁止自行音译成其他写法。" user_parts = [ f"课程:{context}", f"本次需要翻译 {len(batch)} 句,编号必须与输入完全一致。", ] if glossary: user_parts.append("专有名词译法(必须严格遵守,原文出现时译文必须使用指定中文):") user_parts.extend(f"{source} => {zh}" for source, zh in glossary) previous = [ (index, source, existing[index]) for index, source in before if existing.get(index, "").strip() ] if previous: user_parts.append("已有译文(供用词一致,不要重复输出):") user_parts.extend( f"[{index}] {source} => {zh}" for index, source, zh in previous[-6:] ) elif before: user_parts.append("前文(只供理解语境,不要翻译):") user_parts.extend(f"[{index}] {text}" for index, text in before) if after: user_parts.append("后文(只供理解语境,不要翻译):") user_parts.extend(f"[{index}] {text}" for index, text in after) user_parts.append("待翻译句子:") user_parts.extend(f"[{index}] {text}" for index, text in batch) user_parts.append( "输出示例:{" + ",".join(f'"{index}":"中文"' for index in expected_indexes[:2]) + "}" ) user_prompt = "\n".join(user_parts) content = self._complete( system_prompt, user_prompt, temperature=0.2, timeout_seconds=max(self.timeout_seconds, self.translation_timeout_seconds), ) try: parsed = _parse_translation_payload(content, expected_indexes) except AITutorError: logger.warning( "Translation response parse failed: endpoint=%s model=%s " "expected_indexes=%s prompt=%r raw_content=%r", self.request_url, self.model, expected_indexes, user_prompt[:2000], content[:2000], ) raise source_by_index = {index: text for index, text in batch} missing = [ index for index in expected_indexes if translation_needs_repair(source_by_index.get(index, ""), parsed.get(index, "")) ] if missing: invalid = [ { "index": index, "source": source_by_index.get(index, ""), "translation": parsed.get(index, ""), } for index in missing ] logger.warning( "Translation response failed validation: endpoint=%s model=%s " "expected_indexes=%s parsed=%r invalid=%r prompt=%r raw_content=%r", self.request_url, self.model, expected_indexes, parsed, invalid, user_prompt[:2000], content[:2000], ) raise AITutorError("AI translation returned an unexpected response.") return { index: _apply_translation_terms(source_by_index.get(index, ""), parsed[index], glossary) for index in expected_indexes } def _complete( self, system_prompt: str, user_prompt: str, *, temperature: float, timeout_seconds: float | None = None, ) -> str: headers = {"Content-Type": "application/json"} if self.api_key: headers["Authorization"] = f"Bearer {self.api_key}" timeout = self.timeout_seconds if timeout_seconds is None else timeout_seconds try: response = requests.post( self.request_url, headers=headers, json={ "model": self.model, "messages": [ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}, ], "temperature": temperature, "stream": False, }, timeout=timeout, ) except requests.Timeout as exc: logger.warning( "AI request timed out: endpoint=%s model=%s timeout_seconds=%s", self.request_url, self.model, timeout, ) raise AITutorError( f"AI service request timed out after {timeout:g} seconds." ) from exc except requests.RequestException as exc: logger.exception( "AI request failed: endpoint=%s model=%s", self.request_url, self.model, ) raise AITutorError("AI service request failed.") from exc if response.status_code < 200 or response.status_code >= 300: logger.warning( "AI service returned HTTP %s: endpoint=%s model=%s body=%s", response.status_code, self.request_url, self.model, response.text[:1000], ) raise AITutorError(f"AI service returned HTTP {response.status_code}.") try: body = response.json() content = body["choices"][0]["message"]["content"] except (KeyError, IndexError, TypeError, ValueError) as exc: logger.warning( "AI service returned an unexpected body: endpoint=%s body=%s", self.request_url, response.text[:1000], ) raise AITutorError("AI service returned an unexpected response.") from exc if not isinstance(content, str) or not content.strip(): logger.warning( "AI service returned empty content: endpoint=%s body=%s", self.request_url, response.text[:1000], ) raise AITutorError("AI service returned an empty explanation.") return content.strip() def translation_needs_repair(source: str | None, zh_text: str | None) -> bool: """Return True when a sentence still needs a Chinese translation.""" source_text = (source or "").strip() if not source_text: return False zh = (zh_text or "").strip() if not zh: return True if _CJK_RE.search(zh): return False return _CJK_RE.search(source_text) is None def _translation_batches( pending: list[tuple[int, str]], all_sentences: list[tuple[int, str]], ) -> list[list[tuple[int, str]]]: position = {index: offset for offset, (index, _) in enumerate(all_sentences)} ordered_pending = sorted(pending, key=lambda item: position[item[0]]) runs: list[list[tuple[int, str]]] = [] current: list[tuple[int, str]] = [] last_pos: int | None = None for item in ordered_pending: pos = position[item[0]] if current and last_pos is not None and pos != last_pos + 1: runs.append(current) current = [] current.append(item) last_pos = pos if current: runs.append(current) batches: list[list[tuple[int, str]]] = [] for run in runs: batch: list[tuple[int, str]] = [] batch_chars = 0 for item in run: extra = len(item[1]) + 8 too_many = len(batch) >= _TRANSLATE_BATCH_SENTENCES too_long = batch and batch_chars + extra > _TRANSLATE_BATCH_CHARS if too_many or too_long: batches.append(batch) batch = [] batch_chars = 0 batch.append(item) batch_chars += extra if batch: batches.append(batch) return batches def _neighbor_context( batch: list[tuple[int, str]], all_sentences: list[tuple[int, str]], ) -> tuple[list[tuple[int, str]], list[tuple[int, str]]]: position = {index: offset for offset, (index, _) in enumerate(all_sentences)} first = position[batch[0][0]] last = position[batch[-1][0]] before = all_sentences[max(0, first - _TRANSLATE_CONTEXT_BEFORE):first] after = all_sentences[last + 1:last + 1 + _TRANSLATE_CONTEXT_AFTER] return before, after def _parse_translation_payload(content: str, expected_indexes: list[int] | int) -> dict[int, str]: if isinstance(expected_indexes, int): expected = list(range(expected_indexes)) else: expected = list(expected_indexes) text = _JSON_FENCE_RE.sub("", content.strip()).strip() parse_errors: list[str] = [] for candidate in (text, *_json_snippets(text)): try: parsed = _translations_from_json(json.loads(candidate), expected) except (TypeError, ValueError) as exc: parsed = None parse_errors.append(str(exc)) if parsed: return parsed numbered: dict[int, str] = {} for line in text.splitlines(): match = _NUMBERED_LINE_RE.match(line) if match is None: continue numbered[int(match.group(1))] = match.group(2).strip() by_original = { index: numbered[index] for index in expected if numbered.get(index, "").strip() } if len(by_original) == len(expected): return by_original by_order = { expected[offset]: numbered[offset + 1] for offset in range(len(expected)) if numbered.get(offset + 1, "").strip() } if len(by_order) == len(expected): return by_order logger.warning( "Translation payload parser exhausted all formats: expected_indexes=%s " "json_parse_errors=%r numbered_line_keys=%s content=%r", expected, parse_errors, sorted(numbered), text[:2000], ) raise AITutorError("AI translation returned an unexpected response.") def _json_snippets(text: str) -> list[str]: snippets: list[str] = [] for opener, closer in (("{", "}"), ("[", "]")): start = text.find(opener) while start != -1: depth = 0 in_string = False escape = False for offset, char in enumerate(text[start:], start=start): if in_string: if escape: escape = False elif char == "\\": escape = True elif char == '"': in_string = False continue if char == '"': in_string = True elif char == opener: depth += 1 elif char == closer: depth -= 1 if depth == 0: snippets.append(text[start:offset + 1]) break start = text.find(opener, start + 1) return snippets def _translations_from_json(data: object, expected: list[int]) -> dict[int, str] | None: expected_set = set(expected) if isinstance(data, dict): nested = None for key in ("translations", "sentences", "zh", "items"): if key in data: nested = data[key] break if nested is not None: parsed = _translations_from_json(nested, expected) if parsed: return parsed values: dict[int, str] = {} for raw_key, raw_value in data.items(): try: index = int(raw_key) except (TypeError, ValueError): continue if index not in expected_set or not isinstance(raw_value, str) or not raw_value.strip(): continue values[index] = raw_value.strip() if len(values) == len(expected): return values return None if isinstance(data, list) and len(data) == len(expected): if all(isinstance(item, str) and item.strip() for item in data): return { index: item.strip() for index, item in zip(expected, data) } if all(isinstance(item, dict) for item in data): values = {} for item in data: raw_index = item.get("index") zh = item.get("zh_text") or item.get("zh") or item.get("text") try: index = int(raw_index) except (TypeError, ValueError): continue if index not in expected_set or not isinstance(zh, str) or not zh.strip(): continue values[index] = zh.strip() if len(values) == len(expected): return values return None def _normalize_translation_terms(terms: list[tuple[str, str]] | None) -> list[tuple[str, str]]: cleaned: list[tuple[str, str]] = [] seen: dict[str, int] = {} for item in terms or []: if not isinstance(item, (tuple, list)) or len(item) < 2: continue source = str(item[0] or "").strip()[:120] zh = str(item[1] or "").strip()[:120] if not source or not zh: continue key = source.casefold() if key in seen: cleaned[seen[key]] = (source, zh) continue seen[key] = len(cleaned) cleaned.append((source, zh)) if len(cleaned) >= 50: break return cleaned def _term_pattern(source: str) -> re.Pattern[str]: escaped = re.escape(source) if re.fullmatch(r"[A-Za-z][A-Za-z0-9'’.\- ]*", source): return re.compile(rf"(? str: del source_text result = zh_text for source, zh in _normalize_translation_terms(terms): result = _term_pattern(source).sub(zh, result) return result