Files
mediaplayer/sentence_api/ai_tutor.py
2026-10-26 17:31:08 +08:00

656 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""AI-backed sentence explanations and subtitle translation."""
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
import json
import logging
import re
import threading
from urllib.parse import urlsplit
import requests
logger = logging.getLogger(__name__)
_JSON_FENCE_RE = re.compile(r"^```(?:json)?\s*|\s*```$", re.IGNORECASE)
_CJK_RE = re.compile("[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]")
_NUMBERED_LINE_RE = re.compile(r"^\s*[\[(]?(\d+)[\])]?[\.、::\)]\s*(.+?)\s*$")
_TRANSLATE_BATCH_SENTENCES = 12
_TRANSLATE_BATCH_CHARS = 2_200
_TRANSLATE_CONTEXT_BEFORE = 8
_TRANSLATE_CONTEXT_AFTER = 4
_TRANSLATE_WORKERS = 4
_TRANSLATE_TIMEOUT_SECONDS = 120.0
_TRANSLATE_SPLIT_THRESHOLD = 4
class AITutorError(RuntimeError):
"""Raised when the configured AI service cannot answer a request."""
@dataclass(frozen=True)
class AITutor:
endpoint: str
api_key: str
model: str
timeout_seconds: float
translation_timeout_seconds: float = _TRANSLATE_TIMEOUT_SECONDS
@property
def available(self) -> bool:
return bool(self.endpoint and self.model)
@property
def request_url(self) -> str:
"""Resolve either a full chat URL or an OpenAI-compatible base URL."""
parsed = urlsplit(self.endpoint)
path = parsed.path.rstrip("/")
if parsed.query or path.endswith("/chat/completions"):
return self.endpoint
if path.endswith("/v1") or path.endswith("/v1/openai"):
return self.endpoint + "/chat/completions"
if not path:
return self.endpoint + "/v1/chat/completions"
return self.endpoint
def explain(
self,
sentence_text: str,
*,
course_title: str = "",
sentence_index: int | None = None,
language: str = "",
) -> str:
if not self.available:
raise AITutorError("AI explanation is not configured.")
context = course_title.strip() or "口语训练课程"
position = f"第 {sentence_index + 1} 句" if sentence_index is not None else "当前句子"
language_note = f"句子语言:{language.strip()}。" if language.strip() else ""
system_prompt = (
"你是一位耐心的外语口语老师。请用简体中文讲解用户提供的影视/课程句子。"
"先给出自然的中文翻译,再解释关键词、常用搭配、语法要点,并简要说明这句话适合怎样跟读。"
"回答使用简明清晰的分段短文,总长度不超过 600 字,不要编造视频里没有的信息。"
)
user_prompt = (
f"课程:{context}\n位置:{position}\n{language_note}句子:{sentence_text}"
)
return self._complete(system_prompt, user_prompt, temperature=0.3)
def translate_sentences(
self,
sentences: list[tuple[int, str]],
*,
course_title: str = "",
existing: dict[int, str] | None = None,
persist=None,
terms: list[tuple[str, str]] | None = None,
) -> dict[int, str]:
if not self.available:
raise AITutorError("AI translation is not configured.")
if not sentences:
raise AITutorError("No sentences were provided for translation.")
ordered = [(index, text.strip()) for index, text in sentences if text.strip()]
if not ordered:
raise AITutorError("No sentence text was provided for translation.")
glossary = _normalize_translation_terms(terms)
source_by_index = {index: text for index, text in ordered}
translations = {
index: text.strip()
for index, text in (existing or {}).items()
if isinstance(text, str)
and text.strip()
and not translation_needs_repair(source_by_index.get(index, ""), text)
}
pending = [(index, text) for index, text in ordered if index not in translations]
if not pending:
return translations
persist_lock = threading.Lock()
def save_partial(partial: dict[int, str]) -> None:
if not partial:
return
translations.update(partial)
if persist is None:
return
with persist_lock:
persist(partial)
batches = _translation_batches(pending, ordered)
workers = min(_TRANSLATE_WORKERS, len(batches)) or 1
failures: list[tuple[list[tuple[int, str]], Exception]] = []
def run_batch(batch: list[tuple[int, str]]) -> dict[int, str]:
return self._translate_batch_resilient(
batch,
all_sentences=ordered,
existing=translations,
course_title=course_title,
terms=glossary,
)
if workers == 1:
for batch in batches:
try:
save_partial(run_batch(batch))
except Exception as exc: # noqa: BLE001 - collect and retry below
failures.append((batch, exc))
else:
with ThreadPoolExecutor(max_workers=workers) as pool:
futures = {pool.submit(run_batch, batch): batch for batch in batches}
for future in as_completed(futures):
batch = futures[future]
try:
save_partial(future.result())
except Exception as exc: # noqa: BLE001
failures.append((batch, exc))
for batch, error in failures:
logger.warning(
"Translation batch failed, retrying with smaller windows: size=%s error=%s",
len(batch),
error,
)
save_partial(
self._translate_batch_resilient(
batch,
all_sentences=ordered,
existing=translations,
course_title=course_title,
terms=glossary,
force_split=len(batch) > 1,
)
)
missing = [index for index, _ in pending if not translations.get(index, "").strip()]
if missing:
raise AITutorError("AI translation did not cover every sentence.")
if glossary:
source_by_index = {index: text for index, text in ordered}
translations = {
index: _apply_translation_terms(source_by_index.get(index, ""), zh_text, glossary)
for index, zh_text in translations.items()
}
return translations
def _translate_batch_resilient(
self,
batch: list[tuple[int, str]],
*,
all_sentences: list[tuple[int, str]],
existing: dict[int, str],
course_title: str,
terms: list[tuple[str, str]] | None = None,
force_split: bool = False,
) -> dict[int, str]:
if not batch:
return {}
if force_split and len(batch) > 1:
return self._translate_split_batch(
batch,
all_sentences=all_sentences,
existing=existing,
course_title=course_title,
terms=terms,
)
last_error: Exception | None = None
attempts = 2 if len(batch) <= _TRANSLATE_SPLIT_THRESHOLD else 1
for _ in range(attempts):
try:
return self._translate_batch(
batch,
all_sentences=all_sentences,
existing=existing,
course_title=course_title,
terms=terms,
)
except AITutorError as exc:
last_error = exc
logger.warning(
"Translation batch attempt failed: size=%s indexes=%s error=%s",
len(batch),
[index for index, _ in batch],
exc,
)
if len(batch) > 1:
return self._translate_split_batch(
batch,
all_sentences=all_sentences,
existing=existing,
course_title=course_title,
terms=terms,
)
if last_error is not None:
raise last_error
raise AITutorError("AI translation did not cover every sentence.")
def _translate_split_batch(
self,
batch: list[tuple[int, str]],
*,
all_sentences: list[tuple[int, str]],
existing: dict[int, str],
course_title: str,
terms: list[tuple[str, str]] | None = None,
) -> dict[int, str]:
mid = max(1, len(batch) // 2)
merged: dict[int, str] = {}
for part in (batch[:mid], batch[mid:]):
merged.update(
self._translate_batch_resilient(
part,
all_sentences=all_sentences,
existing=existing,
course_title=course_title,
terms=terms,
)
)
return merged
def _translate_batch(
self,
batch: list[tuple[int, str]],
*,
all_sentences: list[tuple[int, str]],
existing: dict[int, str],
course_title: str,
terms: list[tuple[str, str]] | None = None,
) -> dict[int, str]:
context = course_title.strip() or "口语训练课程"
existing = dict(existing)
before, after = _neighbor_context(batch, all_sentences)
expected_indexes = [index for index, _ in batch]
glossary = _normalize_translation_terms(terms)
system_prompt = (
"你是专业的影视字幕翻译。请结合前后文语境,把指定编号的外语句子翻译成自然、口语化的简体中文。"
"必须一句对一句,不能合并、拆分、跳过或改写编号。"
"只输出 JSON 对象,键是句子编号,值是对应中文,不要输出解释或 markdown。"
)
if glossary:
system_prompt += "必须严格使用用户指定的专有名词译法,禁止自行音译成其他写法。"
user_parts = [
f"课程:{context}",
f"本次需要翻译 {len(batch)} 句,编号必须与输入完全一致。",
]
if glossary:
user_parts.append("专有名词译法(必须严格遵守,原文出现时译文必须使用指定中文):")
user_parts.extend(f"{source} => {zh}" for source, zh in glossary)
previous = [
(index, source, existing[index])
for index, source in before
if existing.get(index, "").strip()
]
if previous:
user_parts.append("已有译文(供用词一致,不要重复输出):")
user_parts.extend(
f"[{index}] {source} => {zh}" for index, source, zh in previous[-6:]
)
elif before:
user_parts.append("前文(只供理解语境,不要翻译):")
user_parts.extend(f"[{index}] {text}" for index, text in before)
if after:
user_parts.append("后文(只供理解语境,不要翻译):")
user_parts.extend(f"[{index}] {text}" for index, text in after)
user_parts.append("待翻译句子:")
user_parts.extend(f"[{index}] {text}" for index, text in batch)
user_parts.append(
"输出示例:{" + ",".join(f'"{index}":"中文"' for index in expected_indexes[:2]) + "}"
)
user_prompt = "\n".join(user_parts)
content = self._complete(
system_prompt,
user_prompt,
temperature=0.2,
timeout_seconds=max(self.timeout_seconds, self.translation_timeout_seconds),
)
try:
parsed = _parse_translation_payload(content, expected_indexes)
except AITutorError:
logger.warning(
"Translation response parse failed: endpoint=%s model=%s "
"expected_indexes=%s prompt=%r raw_content=%r",
self.request_url,
self.model,
expected_indexes,
user_prompt[:2000],
content[:2000],
)
raise
source_by_index = {index: text for index, text in batch}
missing = [
index
for index in expected_indexes
if translation_needs_repair(source_by_index.get(index, ""), parsed.get(index, ""))
]
if missing:
invalid = [
{
"index": index,
"source": source_by_index.get(index, ""),
"translation": parsed.get(index, ""),
}
for index in missing
]
logger.warning(
"Translation response failed validation: endpoint=%s model=%s "
"expected_indexes=%s parsed=%r invalid=%r prompt=%r raw_content=%r",
self.request_url,
self.model,
expected_indexes,
parsed,
invalid,
user_prompt[:2000],
content[:2000],
)
raise AITutorError("AI translation returned an unexpected response.")
return {
index: _apply_translation_terms(source_by_index.get(index, ""), parsed[index], glossary)
for index in expected_indexes
}
def _complete(
self,
system_prompt: str,
user_prompt: str,
*,
temperature: float,
timeout_seconds: float | None = None,
) -> str:
headers = {"Content-Type": "application/json"}
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
timeout = self.timeout_seconds if timeout_seconds is None else timeout_seconds
try:
response = requests.post(
self.request_url,
headers=headers,
json={
"model": self.model,
"messages": [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt},
],
"temperature": temperature,
"stream": False,
},
timeout=timeout,
)
except requests.Timeout as exc:
logger.warning(
"AI request timed out: endpoint=%s model=%s timeout_seconds=%s",
self.request_url,
self.model,
timeout,
)
raise AITutorError(
f"AI service request timed out after {timeout:g} seconds."
) from exc
except requests.RequestException as exc:
logger.exception(
"AI request failed: endpoint=%s model=%s",
self.request_url,
self.model,
)
raise AITutorError("AI service request failed.") from exc
if response.status_code < 200 or response.status_code >= 300:
logger.warning(
"AI service returned HTTP %s: endpoint=%s model=%s body=%s",
response.status_code,
self.request_url,
self.model,
response.text[:1000],
)
raise AITutorError(f"AI service returned HTTP {response.status_code}.")
try:
body = response.json()
content = body["choices"][0]["message"]["content"]
except (KeyError, IndexError, TypeError, ValueError) as exc:
logger.warning(
"AI service returned an unexpected body: endpoint=%s body=%s",
self.request_url,
response.text[:1000],
)
raise AITutorError("AI service returned an unexpected response.") from exc
if not isinstance(content, str) or not content.strip():
logger.warning(
"AI service returned empty content: endpoint=%s body=%s",
self.request_url,
response.text[:1000],
)
raise AITutorError("AI service returned an empty explanation.")
return content.strip()
def translation_needs_repair(source: str | None, zh_text: str | None) -> bool:
"""Return True when a sentence still needs a Chinese translation."""
source_text = (source or "").strip()
if not source_text:
return False
zh = (zh_text or "").strip()
if not zh:
return True
if _CJK_RE.search(zh):
return False
return _CJK_RE.search(source_text) is None
def _translation_batches(
pending: list[tuple[int, str]],
all_sentences: list[tuple[int, str]],
) -> list[list[tuple[int, str]]]:
position = {index: offset for offset, (index, _) in enumerate(all_sentences)}
ordered_pending = sorted(pending, key=lambda item: position[item[0]])
runs: list[list[tuple[int, str]]] = []
current: list[tuple[int, str]] = []
last_pos: int | None = None
for item in ordered_pending:
pos = position[item[0]]
if current and last_pos is not None and pos != last_pos + 1:
runs.append(current)
current = []
current.append(item)
last_pos = pos
if current:
runs.append(current)
batches: list[list[tuple[int, str]]] = []
for run in runs:
batch: list[tuple[int, str]] = []
batch_chars = 0
for item in run:
extra = len(item[1]) + 8
too_many = len(batch) >= _TRANSLATE_BATCH_SENTENCES
too_long = batch and batch_chars + extra > _TRANSLATE_BATCH_CHARS
if too_many or too_long:
batches.append(batch)
batch = []
batch_chars = 0
batch.append(item)
batch_chars += extra
if batch:
batches.append(batch)
return batches
def _neighbor_context(
batch: list[tuple[int, str]],
all_sentences: list[tuple[int, str]],
) -> tuple[list[tuple[int, str]], list[tuple[int, str]]]:
position = {index: offset for offset, (index, _) in enumerate(all_sentences)}
first = position[batch[0][0]]
last = position[batch[-1][0]]
before = all_sentences[max(0, first - _TRANSLATE_CONTEXT_BEFORE):first]
after = all_sentences[last + 1:last + 1 + _TRANSLATE_CONTEXT_AFTER]
return before, after
def _parse_translation_payload(content: str, expected_indexes: list[int] | int) -> dict[int, str]:
if isinstance(expected_indexes, int):
expected = list(range(expected_indexes))
else:
expected = list(expected_indexes)
text = _JSON_FENCE_RE.sub("", content.strip()).strip()
parse_errors: list[str] = []
for candidate in (text, *_json_snippets(text)):
try:
parsed = _translations_from_json(json.loads(candidate), expected)
except (TypeError, ValueError) as exc:
parsed = None
parse_errors.append(str(exc))
if parsed:
return parsed
numbered: dict[int, str] = {}
for line in text.splitlines():
match = _NUMBERED_LINE_RE.match(line)
if match is None:
continue
numbered[int(match.group(1))] = match.group(2).strip()
by_original = {
index: numbered[index]
for index in expected
if numbered.get(index, "").strip()
}
if len(by_original) == len(expected):
return by_original
by_order = {
expected[offset]: numbered[offset + 1]
for offset in range(len(expected))
if numbered.get(offset + 1, "").strip()
}
if len(by_order) == len(expected):
return by_order
logger.warning(
"Translation payload parser exhausted all formats: expected_indexes=%s "
"json_parse_errors=%r numbered_line_keys=%s content=%r",
expected,
parse_errors,
sorted(numbered),
text[:2000],
)
raise AITutorError("AI translation returned an unexpected response.")
def _json_snippets(text: str) -> list[str]:
snippets: list[str] = []
for opener, closer in (("{", "}"), ("[", "]")):
start = text.find(opener)
while start != -1:
depth = 0
in_string = False
escape = False
for offset, char in enumerate(text[start:], start=start):
if in_string:
if escape:
escape = False
elif char == "\\":
escape = True
elif char == '"':
in_string = False
continue
if char == '"':
in_string = True
elif char == opener:
depth += 1
elif char == closer:
depth -= 1
if depth == 0:
snippets.append(text[start:offset + 1])
break
start = text.find(opener, start + 1)
return snippets
def _translations_from_json(data: object, expected: list[int]) -> dict[int, str] | None:
expected_set = set(expected)
if isinstance(data, dict):
nested = None
for key in ("translations", "sentences", "zh", "items"):
if key in data:
nested = data[key]
break
if nested is not None:
parsed = _translations_from_json(nested, expected)
if parsed:
return parsed
values: dict[int, str] = {}
for raw_key, raw_value in data.items():
try:
index = int(raw_key)
except (TypeError, ValueError):
continue
if index not in expected_set or not isinstance(raw_value, str) or not raw_value.strip():
continue
values[index] = raw_value.strip()
if len(values) == len(expected):
return values
return None
if isinstance(data, list) and len(data) == len(expected):
if all(isinstance(item, str) and item.strip() for item in data):
return {
index: item.strip()
for index, item in zip(expected, data)
}
if all(isinstance(item, dict) for item in data):
values = {}
for item in data:
raw_index = item.get("index")
zh = item.get("zh_text") or item.get("zh") or item.get("text")
try:
index = int(raw_index)
except (TypeError, ValueError):
continue
if index not in expected_set or not isinstance(zh, str) or not zh.strip():
continue
values[index] = zh.strip()
if len(values) == len(expected):
return values
return None
def _normalize_translation_terms(terms: list[tuple[str, str]] | None) -> list[tuple[str, str]]:
cleaned: list[tuple[str, str]] = []
seen: dict[str, int] = {}
for item in terms or []:
if not isinstance(item, (tuple, list)) or len(item) < 2:
continue
source = str(item[0] or "").strip()[:120]
zh = str(item[1] or "").strip()[:120]
if not source or not zh:
continue
key = source.casefold()
if key in seen:
cleaned[seen[key]] = (source, zh)
continue
seen[key] = len(cleaned)
cleaned.append((source, zh))
if len(cleaned) >= 50:
break
return cleaned
def _term_pattern(source: str) -> re.Pattern[str]:
escaped = re.escape(source)
if re.fullmatch(r"[A-Za-z][A-Za-z0-9'’.\- ]*", source):
return re.compile(rf"(?<![A-Za-z0-9]){escaped}(?![A-Za-z0-9])", re.IGNORECASE)
return re.compile(escaped)
def _apply_translation_terms(
source_text: str,
zh_text: str,
terms: list[tuple[str, str]] | None,
) -> str:
del source_text
result = zh_text
for source, zh in _normalize_translation_terms(terms):
result = _term_pattern(source).sub(zh, result)
return result