add a retranslate chinese function

This commit is contained in:
2026-10-06 14:47:22 +08:00
parent 4fd0fd36a4
commit b28f809c46
11 changed files with 448 additions and 9 deletions

View File

@@ -2,7 +2,7 @@ import json
import sqlite3
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, Sequence, Tuple
import uuid
from .auth import hash_password, hash_token, new_token, verify_password
@@ -202,6 +202,14 @@ class VideoRepository:
if "zh_text" not in sentence_columns:
connection.execute("ALTER TABLE sentences ADD COLUMN zh_text TEXT")
with self._connect() as connection:
video_columns = {
row["name"]
for row in connection.execute("PRAGMA table_info(videos)")
}
if "translation_glossary" not in video_columns:
connection.execute("ALTER TABLE videos ADD COLUMN translation_glossary TEXT")
def upsert_upload(
self,
*,
@@ -833,6 +841,36 @@ class VideoRepository:
return [_sentence_from_row(row) for row in result_rows]
def get_translation_glossary(self, video_hash: str) -> List[Tuple[str, str]]:
video = self.get_video(video_hash)
if video is None:
return []
return _parse_translation_glossary(video.get("translation_glossary"))
def save_translation_glossary(
self,
video_hash: str,
terms: Sequence[Any],
) -> bool:
normalized_hash = normalize_video_hash(video_hash)
payload = json.dumps(
[
{"source": source, "zh": zh}
for source, zh in _normalize_translation_glossary(terms)
],
ensure_ascii=False,
)
with self._connect() as connection:
cursor = connection.execute(
"""
UPDATE videos
SET translation_glossary = ?, updated_at = ?
WHERE video_hash = ?
""",
(payload, utc_now(), normalized_hash),
)
return cursor.rowcount > 0
def clear_sentence_translations(self, video_hash: str) -> None:
with self._connect() as connection:
connection.execute(
@@ -1345,3 +1383,43 @@ def _moss_segment_row(video_hash: str, index: int, segment: Dict[str, Any]) -> t
str(speaker) if speaker is not None else None,
words_json,
)
def _normalize_translation_glossary(terms: Sequence[Any]) -> List[Tuple[str, str]]:
cleaned: List[Tuple[str, str]] = []
seen: Dict[str, int] = {}
for item in terms or []:
source = ""
zh = ""
if isinstance(item, dict):
source = str(item.get("source") or "")
zh = str(item.get("zh") or "")
elif isinstance(item, (tuple, list)) and len(item) >= 2:
source = str(item[0] or "")
zh = str(item[1] or "")
source = source.strip()[:120]
zh = zh.strip()[:120]
if not source or not zh:
continue
key = source.casefold()
if key in seen:
cleaned[seen[key]] = (source, zh)
continue
seen[key] = len(cleaned)
cleaned.append((source, zh))
if len(cleaned) >= 50:
break
return cleaned
def _parse_translation_glossary(raw: Any) -> List[Tuple[str, str]]:
if not raw:
return []
if isinstance(raw, str):
try:
raw = json.loads(raw)
except json.JSONDecodeError:
return []
if not isinstance(raw, list):
return []
return _normalize_translation_glossary(raw)