582 lines
21 KiB
Python
582 lines
21 KiB
Python
"""AI-backed sentence explanations and subtitle translation."""
|
||
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from dataclasses import dataclass
|
||
import json
|
||
import logging
|
||
import re
|
||
import threading
|
||
from urllib.parse import urlsplit
|
||
|
||
import requests
|
||
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
_JSON_FENCE_RE = re.compile(r"^```(?:json)?\s*|\s*```$", re.IGNORECASE)
|
||
_NUMBERED_LINE_RE = re.compile(r"^\s*[\[(]?(\d+)[\])]?[\.、:：\)]\s*(.+?)\s*$")
|
||
_TRANSLATE_BATCH_SENTENCES = 12
|
||
_TRANSLATE_BATCH_CHARS = 2_200
|
||
_TRANSLATE_CONTEXT_BEFORE = 8
|
||
_TRANSLATE_CONTEXT_AFTER = 4
|
||
_TRANSLATE_WORKERS = 4
|
||
_TRANSLATE_TIMEOUT_SECONDS = 45.0
|
||
_TRANSLATE_SPLIT_THRESHOLD = 4
|
||
|
||
|
||
class AITutorError(RuntimeError):
|
||
"""Raised when the configured AI service cannot answer a request."""
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class AITutor:
|
||
endpoint: str
|
||
api_key: str
|
||
model: str
|
||
timeout_seconds: float
|
||
|
||
@property
|
||
def available(self) -> bool:
|
||
return bool(self.endpoint and self.model)
|
||
|
||
@property
|
||
def request_url(self) -> str:
|
||
"""Resolve either a full chat URL or an OpenAI-compatible base URL."""
|
||
parsed = urlsplit(self.endpoint)
|
||
path = parsed.path.rstrip("/")
|
||
if parsed.query or path.endswith("/chat/completions"):
|
||
return self.endpoint
|
||
if path.endswith("/v1") or path.endswith("/v1/openai"):
|
||
return self.endpoint + "/chat/completions"
|
||
if not path:
|
||
return self.endpoint + "/v1/chat/completions"
|
||
return self.endpoint
|
||
|
||
def explain(
|
||
self,
|
||
sentence_text: str,
|
||
*,
|
||
course_title: str = "",
|
||
sentence_index: int | None = None,
|
||
language: str = "",
|
||
) -> str:
|
||
if not self.available:
|
||
raise AITutorError("AI explanation is not configured.")
|
||
|
||
context = course_title.strip() or "口语训练课程"
|
||
position = f"第 {sentence_index + 1} 句" if sentence_index is not None else "当前句子"
|
||
language_note = f"句子语言：{language.strip()}。" if language.strip() else ""
|
||
system_prompt = (
|
||
"你是一位耐心的外语口语老师。请用简体中文讲解用户提供的影视/课程句子。"
|
||
"先给出自然的中文翻译，再解释关键词、常用搭配、语法要点，并简要说明这句话适合怎样跟读。"
|
||
"回答使用简明清晰的分段短文，总长度不超过 600 字，不要编造视频里没有的信息。"
|
||
)
|
||
user_prompt = (
|
||
f"课程：{context}\n位置：{position}\n{language_note}句子：{sentence_text}"
|
||
)
|
||
return self._complete(system_prompt, user_prompt, temperature=0.3)
|
||
|
||
def translate_sentences(
|
||
self,
|
||
sentences: list[tuple[int, str]],
|
||
*,
|
||
course_title: str = "",
|
||
existing: dict[int, str] | None = None,
|
||
persist=None,
|
||
terms: list[tuple[str, str]] | None = None,
|
||
) -> dict[int, str]:
|
||
if not self.available:
|
||
raise AITutorError("AI translation is not configured.")
|
||
if not sentences:
|
||
raise AITutorError("No sentences were provided for translation.")
|
||
|
||
ordered = [(index, text.strip()) for index, text in sentences if text.strip()]
|
||
if not ordered:
|
||
raise AITutorError("No sentence text was provided for translation.")
|
||
|
||
glossary = _normalize_translation_terms(terms)
|
||
translations = {
|
||
index: text.strip()
|
||
for index, text in (existing or {}).items()
|
||
if isinstance(text, str) and text.strip()
|
||
}
|
||
pending = [(index, text) for index, text in ordered if index not in translations]
|
||
if not pending:
|
||
return translations
|
||
|
||
persist_lock = threading.Lock()
|
||
|
||
def save_partial(partial: dict[int, str]) -> None:
|
||
if not partial:
|
||
return
|
||
translations.update(partial)
|
||
if persist is None:
|
||
return
|
||
with persist_lock:
|
||
persist(partial)
|
||
|
||
batches = _translation_batches(pending, ordered)
|
||
workers = min(_TRANSLATE_WORKERS, len(batches)) or 1
|
||
failures: list[tuple[list[tuple[int, str]], Exception]] = []
|
||
|
||
def run_batch(batch: list[tuple[int, str]]) -> dict[int, str]:
|
||
return self._translate_batch_resilient(
|
||
batch,
|
||
all_sentences=ordered,
|
||
existing=translations,
|
||
course_title=course_title,
|
||
terms=glossary,
|
||
)
|
||
|
||
if workers == 1:
|
||
for batch in batches:
|
||
try:
|
||
save_partial(run_batch(batch))
|
||
except Exception as exc: # noqa: BLE001 - collect and retry below
|
||
failures.append((batch, exc))
|
||
else:
|
||
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||
futures = {pool.submit(run_batch, batch): batch for batch in batches}
|
||
for future in as_completed(futures):
|
||
batch = futures[future]
|
||
try:
|
||
save_partial(future.result())
|
||
except Exception as exc: # noqa: BLE001
|
||
failures.append((batch, exc))
|
||
|
||
for batch, error in failures:
|
||
logger.warning(
|
||
"Translation batch failed, retrying with smaller windows: size=%s error=%s",
|
||
len(batch),
|
||
error,
|
||
)
|
||
save_partial(
|
||
self._translate_batch_resilient(
|
||
batch,
|
||
all_sentences=ordered,
|
||
existing=translations,
|
||
course_title=course_title,
|
||
terms=glossary,
|
||
force_split=len(batch) > 1,
|
||
)
|
||
)
|
||
|
||
missing = [index for index, _ in pending if not translations.get(index, "").strip()]
|
||
if missing:
|
||
raise AITutorError("AI translation did not cover every sentence.")
|
||
if glossary:
|
||
source_by_index = {index: text for index, text in ordered}
|
||
translations = {
|
||
index: _apply_translation_terms(source_by_index.get(index, ""), zh_text, glossary)
|
||
for index, zh_text in translations.items()
|
||
}
|
||
return translations
|
||
|
||
def _translate_batch_resilient(
|
||
self,
|
||
batch: list[tuple[int, str]],
|
||
*,
|
||
all_sentences: list[tuple[int, str]],
|
||
existing: dict[int, str],
|
||
course_title: str,
|
||
terms: list[tuple[str, str]] | None = None,
|
||
force_split: bool = False,
|
||
) -> dict[int, str]:
|
||
if not batch:
|
||
return {}
|
||
if force_split and len(batch) > 1:
|
||
return self._translate_split_batch(
|
||
batch,
|
||
all_sentences=all_sentences,
|
||
existing=existing,
|
||
course_title=course_title,
|
||
terms=terms,
|
||
)
|
||
|
||
last_error: Exception | None = None
|
||
attempts = 2 if len(batch) <= _TRANSLATE_SPLIT_THRESHOLD else 1
|
||
for _ in range(attempts):
|
||
try:
|
||
return self._translate_batch(
|
||
batch,
|
||
all_sentences=all_sentences,
|
||
existing=existing,
|
||
course_title=course_title,
|
||
terms=terms,
|
||
)
|
||
except AITutorError as exc:
|
||
last_error = exc
|
||
logger.warning(
|
||
"Translation batch attempt failed: size=%s indexes=%s error=%s",
|
||
len(batch),
|
||
[index for index, _ in batch],
|
||
exc,
|
||
)
|
||
|
||
if len(batch) > 1:
|
||
return self._translate_split_batch(
|
||
batch,
|
||
all_sentences=all_sentences,
|
||
existing=existing,
|
||
course_title=course_title,
|
||
terms=terms,
|
||
)
|
||
if last_error is not None:
|
||
raise last_error
|
||
raise AITutorError("AI translation did not cover every sentence.")
|
||
|
||
def _translate_split_batch(
|
||
self,
|
||
batch: list[tuple[int, str]],
|
||
*,
|
||
all_sentences: list[tuple[int, str]],
|
||
existing: dict[int, str],
|
||
course_title: str,
|
||
terms: list[tuple[str, str]] | None = None,
|
||
) -> dict[int, str]:
|
||
mid = max(1, len(batch) // 2)
|
||
merged: dict[int, str] = {}
|
||
for part in (batch[:mid], batch[mid:]):
|
||
merged.update(
|
||
self._translate_batch_resilient(
|
||
part,
|
||
all_sentences=all_sentences,
|
||
existing=existing,
|
||
course_title=course_title,
|
||
terms=terms,
|
||
)
|
||
)
|
||
return merged
|
||
|
||
def _translate_batch(
|
||
self,
|
||
batch: list[tuple[int, str]],
|
||
*,
|
||
all_sentences: list[tuple[int, str]],
|
||
existing: dict[int, str],
|
||
course_title: str,
|
||
terms: list[tuple[str, str]] | None = None,
|
||
) -> dict[int, str]:
|
||
context = course_title.strip() or "口语训练课程"
|
||
existing = dict(existing)
|
||
before, after = _neighbor_context(batch, all_sentences)
|
||
expected_indexes = [index for index, _ in batch]
|
||
glossary = _normalize_translation_terms(terms)
|
||
system_prompt = (
|
||
"你是专业的影视字幕翻译。请结合前后文语境，把指定编号的外语句子翻译成自然、口语化的简体中文。"
|
||
"必须一句对一句，不能合并、拆分、跳过或改写编号。"
|
||
"只输出 JSON 对象，键是句子编号，值是对应中文，不要输出解释或 markdown。"
|
||
)
|
||
if glossary:
|
||
system_prompt += "必须严格使用用户指定的专有名词译法，禁止自行音译成其他写法。"
|
||
user_parts = [
|
||
f"课程：{context}",
|
||
f"本次需要翻译 {len(batch)} 句，编号必须与输入完全一致。",
|
||
]
|
||
if glossary:
|
||
user_parts.append("专有名词译法（必须严格遵守，原文出现时译文必须使用指定中文）：")
|
||
user_parts.extend(f"{source} => {zh}" for source, zh in glossary)
|
||
previous = [
|
||
(index, source, existing[index])
|
||
for index, source in before
|
||
if existing.get(index, "").strip()
|
||
]
|
||
if previous:
|
||
user_parts.append("已有译文（供用词一致，不要重复输出）：")
|
||
user_parts.extend(
|
||
f"[{index}] {source} => {zh}" for index, source, zh in previous[-6:]
|
||
)
|
||
elif before:
|
||
user_parts.append("前文（只供理解语境，不要翻译）：")
|
||
user_parts.extend(f"[{index}] {text}" for index, text in before)
|
||
if after:
|
||
user_parts.append("后文（只供理解语境，不要翻译）：")
|
||
user_parts.extend(f"[{index}] {text}" for index, text in after)
|
||
user_parts.append("待翻译句子：")
|
||
user_parts.extend(f"[{index}] {text}" for index, text in batch)
|
||
user_parts.append(
|
||
"输出示例：{" + ",".join(f'"{index}":"中文"' for index in expected_indexes[:2]) + "}"
|
||
)
|
||
content = self._complete(
|
||
system_prompt,
|
||
"\n".join(user_parts),
|
||
temperature=0.2,
|
||
timeout_seconds=max(self.timeout_seconds, _TRANSLATE_TIMEOUT_SECONDS),
|
||
)
|
||
parsed = _parse_translation_payload(content, expected_indexes)
|
||
missing = [index for index in expected_indexes if not parsed.get(index, "").strip()]
|
||
if missing:
|
||
raise AITutorError("AI translation returned an unexpected response.")
|
||
source_by_index = {index: text for index, text in batch}
|
||
return {
|
||
index: _apply_translation_terms(source_by_index.get(index, ""), parsed[index], glossary)
|
||
for index in expected_indexes
|
||
}
|
||
|
||
def _complete(
|
||
self,
|
||
system_prompt: str,
|
||
user_prompt: str,
|
||
*,
|
||
temperature: float,
|
||
timeout_seconds: float | None = None,
|
||
) -> str:
|
||
headers = {"Content-Type": "application/json"}
|
||
if self.api_key:
|
||
headers["Authorization"] = f"Bearer {self.api_key}"
|
||
timeout = self.timeout_seconds if timeout_seconds is None else timeout_seconds
|
||
try:
|
||
response = requests.post(
|
||
self.request_url,
|
||
headers=headers,
|
||
json={
|
||
"model": self.model,
|
||
"messages": [
|
||
{"role": "system", "content": system_prompt},
|
||
{"role": "user", "content": user_prompt},
|
||
],
|
||
"temperature": temperature,
|
||
"stream": False,
|
||
},
|
||
timeout=timeout,
|
||
)
|
||
except requests.RequestException as exc:
|
||
logger.exception(
|
||
"AI request failed: endpoint=%s model=%s",
|
||
self.request_url,
|
||
self.model,
|
||
)
|
||
raise AITutorError("AI service request failed.") from exc
|
||
|
||
if response.status_code < 200 or response.status_code >= 300:
|
||
logger.warning(
|
||
"AI service returned HTTP %s: endpoint=%s model=%s body=%s",
|
||
response.status_code,
|
||
self.request_url,
|
||
self.model,
|
||
response.text[:1000],
|
||
)
|
||
raise AITutorError(f"AI service returned HTTP {response.status_code}.")
|
||
try:
|
||
body = response.json()
|
||
content = body["choices"][0]["message"]["content"]
|
||
except (KeyError, IndexError, TypeError, ValueError) as exc:
|
||
logger.warning(
|
||
"AI service returned an unexpected body: endpoint=%s body=%s",
|
||
self.request_url,
|
||
response.text[:1000],
|
||
)
|
||
raise AITutorError("AI service returned an unexpected response.") from exc
|
||
if not isinstance(content, str) or not content.strip():
|
||
logger.warning(
|
||
"AI service returned empty content: endpoint=%s body=%s",
|
||
self.request_url,
|
||
response.text[:1000],
|
||
)
|
||
raise AITutorError("AI service returned an empty explanation.")
|
||
return content.strip()
|
||
|
||
|
||
def _translation_batches(
|
||
pending: list[tuple[int, str]],
|
||
all_sentences: list[tuple[int, str]],
|
||
) -> list[list[tuple[int, str]]]:
|
||
position = {index: offset for offset, (index, _) in enumerate(all_sentences)}
|
||
ordered_pending = sorted(pending, key=lambda item: position[item[0]])
|
||
runs: list[list[tuple[int, str]]] = []
|
||
current: list[tuple[int, str]] = []
|
||
last_pos: int | None = None
|
||
for item in ordered_pending:
|
||
pos = position[item[0]]
|
||
if current and last_pos is not None and pos != last_pos + 1:
|
||
runs.append(current)
|
||
current = []
|
||
current.append(item)
|
||
last_pos = pos
|
||
if current:
|
||
runs.append(current)
|
||
|
||
batches: list[list[tuple[int, str]]] = []
|
||
for run in runs:
|
||
batch: list[tuple[int, str]] = []
|
||
batch_chars = 0
|
||
for item in run:
|
||
extra = len(item[1]) + 8
|
||
too_many = len(batch) >= _TRANSLATE_BATCH_SENTENCES
|
||
too_long = batch and batch_chars + extra > _TRANSLATE_BATCH_CHARS
|
||
if too_many or too_long:
|
||
batches.append(batch)
|
||
batch = []
|
||
batch_chars = 0
|
||
batch.append(item)
|
||
batch_chars += extra
|
||
if batch:
|
||
batches.append(batch)
|
||
return batches
|
||
|
||
|
||
def _neighbor_context(
|
||
batch: list[tuple[int, str]],
|
||
all_sentences: list[tuple[int, str]],
|
||
) -> tuple[list[tuple[int, str]], list[tuple[int, str]]]:
|
||
position = {index: offset for offset, (index, _) in enumerate(all_sentences)}
|
||
first = position[batch[0][0]]
|
||
last = position[batch[-1][0]]
|
||
before = all_sentences[max(0, first - _TRANSLATE_CONTEXT_BEFORE):first]
|
||
after = all_sentences[last + 1:last + 1 + _TRANSLATE_CONTEXT_AFTER]
|
||
return before, after
|
||
|
||
|
||
def _parse_translation_payload(content: str, expected_indexes: list[int] | int) -> dict[int, str]:
|
||
if isinstance(expected_indexes, int):
|
||
expected = list(range(expected_indexes))
|
||
else:
|
||
expected = list(expected_indexes)
|
||
text = _JSON_FENCE_RE.sub("", content.strip()).strip()
|
||
for candidate in (text, *_json_snippets(text)):
|
||
try:
|
||
parsed = _translations_from_json(json.loads(candidate), expected)
|
||
except (TypeError, ValueError):
|
||
parsed = None
|
||
if parsed:
|
||
return parsed
|
||
|
||
numbered: dict[int, str] = {}
|
||
for line in text.splitlines():
|
||
match = _NUMBERED_LINE_RE.match(line)
|
||
if match is None:
|
||
continue
|
||
numbered[int(match.group(1))] = match.group(2).strip()
|
||
by_original = {
|
||
index: numbered[index]
|
||
for index in expected
|
||
if numbered.get(index, "").strip()
|
||
}
|
||
if len(by_original) == len(expected):
|
||
return by_original
|
||
by_order = {
|
||
expected[offset]: numbered[offset + 1]
|
||
for offset in range(len(expected))
|
||
if numbered.get(offset + 1, "").strip()
|
||
}
|
||
if len(by_order) == len(expected):
|
||
return by_order
|
||
raise AITutorError("AI translation returned an unexpected response.")
|
||
|
||
|
||
def _json_snippets(text: str) -> list[str]:
|
||
snippets: list[str] = []
|
||
for opener, closer in (("{", "}"), ("[", "]")):
|
||
start = text.find(opener)
|
||
while start != -1:
|
||
depth = 0
|
||
in_string = False
|
||
escape = False
|
||
for offset, char in enumerate(text[start:], start=start):
|
||
if in_string:
|
||
if escape:
|
||
escape = False
|
||
elif char == "\\":
|
||
escape = True
|
||
elif char == '"':
|
||
in_string = False
|
||
continue
|
||
if char == '"':
|
||
in_string = True
|
||
elif char == opener:
|
||
depth += 1
|
||
elif char == closer:
|
||
depth -= 1
|
||
if depth == 0:
|
||
snippets.append(text[start:offset + 1])
|
||
break
|
||
start = text.find(opener, start + 1)
|
||
return snippets
|
||
|
||
|
||
def _translations_from_json(data: object, expected: list[int]) -> dict[int, str] | None:
|
||
expected_set = set(expected)
|
||
if isinstance(data, dict):
|
||
nested = None
|
||
for key in ("translations", "sentences", "zh", "items"):
|
||
if key in data:
|
||
nested = data[key]
|
||
break
|
||
if nested is not None:
|
||
parsed = _translations_from_json(nested, expected)
|
||
if parsed:
|
||
return parsed
|
||
values: dict[int, str] = {}
|
||
for raw_key, raw_value in data.items():
|
||
try:
|
||
index = int(raw_key)
|
||
except (TypeError, ValueError):
|
||
continue
|
||
if index not in expected_set or not isinstance(raw_value, str) or not raw_value.strip():
|
||
continue
|
||
values[index] = raw_value.strip()
|
||
if len(values) == len(expected):
|
||
return values
|
||
return None
|
||
if isinstance(data, list) and len(data) == len(expected):
|
||
if all(isinstance(item, str) and item.strip() for item in data):
|
||
return {
|
||
index: item.strip()
|
||
for index, item in zip(expected, data)
|
||
}
|
||
if all(isinstance(item, dict) for item in data):
|
||
values = {}
|
||
for item in data:
|
||
raw_index = item.get("index")
|
||
zh = item.get("zh_text") or item.get("zh") or item.get("text")
|
||
try:
|
||
index = int(raw_index)
|
||
except (TypeError, ValueError):
|
||
continue
|
||
if index not in expected_set or not isinstance(zh, str) or not zh.strip():
|
||
continue
|
||
values[index] = zh.strip()
|
||
if len(values) == len(expected):
|
||
return values
|
||
return None
|
||
|
||
|
||
def _normalize_translation_terms(terms: list[tuple[str, str]] | None) -> list[tuple[str, str]]:
|
||
cleaned: list[tuple[str, str]] = []
|
||
seen: dict[str, int] = {}
|
||
for item in terms or []:
|
||
if not isinstance(item, (tuple, list)) or len(item) < 2:
|
||
continue
|
||
source = str(item[0] or "").strip()[:120]
|
||
zh = str(item[1] or "").strip()[:120]
|
||
if not source or not zh:
|
||
continue
|
||
key = source.casefold()
|
||
if key in seen:
|
||
cleaned[seen[key]] = (source, zh)
|
||
continue
|
||
seen[key] = len(cleaned)
|
||
cleaned.append((source, zh))
|
||
if len(cleaned) >= 50:
|
||
break
|
||
return cleaned
|
||
|
||
|
||
def _term_pattern(source: str) -> re.Pattern[str]:
|
||
escaped = re.escape(source)
|
||
if re.fullmatch(r"[A-Za-z][A-Za-z0-9'’.\- ]*", source):
|
||
return re.compile(rf"(?<![A-Za-z0-9]){escaped}(?![A-Za-z0-9])", re.IGNORECASE)
|
||
return re.compile(escaped)
|
||
|
||
|
||
def _apply_translation_terms(
|
||
source_text: str,
|
||
zh_text: str,
|
||
terms: list[tuple[str, str]] | None,
|
||
) -> str:
|
||
del source_text
|
||
result = zh_text
|
||
for source, zh in _normalize_translation_terms(terms):
|
||
result = _term_pattern(source).sub(zh, result)
|
||
return result
|