From 9980f78cfcb355494754df21a23ab841f6beaa28 Mon Sep 17 00:00:00 2001 From: Shuming Liu Date: Thu, 10 Sep 2026 08:58:33 +0800 Subject: [PATCH] add sentences borders enable in pc player --- README.md | 7 +- media_player.py | 419 +++++++++++++++++++++++++++++++-- requirements.txt | 1 + sentence_api/README.md | 6 + sentence_api/main.py | 45 +++- sentence_api/processing.py | 65 ++++- sentence_api/tests/test_api.py | 36 +++ 7 files changed, 556 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 8f98ceb..472c5d7 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,11 @@ pip 安装依赖后即可直接播放视频。 - 播放 / 暂停 / 停止、上一首 / 下一首 - 前进 / 后退（步长可自定义）与进度条点击拖动定位 - 快进步长可在界面底部直接填写（1-600 秒），自动保存、下次启动自动加载 -- 按句跳转：勾选后 `←` / `→` 自动播放上一句/下一句（基于音频静音自动识别 - 句边界，无需联网，分析结果自动缓存） +- 按句跳转：`, / .` 直接跳转上一句/下一句；勾选“按句跳转”后 `← / →` 也可 + 跳句。句边界通过线上服务生成，PC 仅上传本地提取的音频轨，服务器处理完成 + 后返回边界且不保存音频；结果按文件 SHA-256 缓存在本机。 +- AI 讲解：播放中按 `E` 或点击“AI讲解 (E)”，由服务端 AI 讲解当前句子。 +- 服务配置：点击“服务…”可填写服务器地址，或使用账号密码登录后保存令牌。 - 音量调节、静音，0.5x - 2.0x 变速播放 - 三种循环模式：关 / 单曲循环 / 列表循环 - 播放列表管理（打开文件、打开文件夹、删除条目、双击播放） diff --git a/media_player.py b/media_player.py index 048dcc1..bdda96e 100644 --- a/media_player.py +++ b/media_player.py @@ -13,7 +13,9 @@ Shift+← / → 后退 / 前进 60 秒 ↑ / ↓ 音量 +5 / -5 Ctrl+↑ / ↓ 快进步长 +1 / -1 秒（macOS 亦支持 Command+↑ / ↓） - 勾选“按句跳转”后，←/→ 自动播放上一句/下一句（基于音频静音识别句边界） + , / . 上一句 / 下一句 + 勾选“按句跳转”后，←/→ 也会自动播放上一句/下一句 + E 讲解当前句子（AI） M 静音 / 恢复 Ctrl+← / → 上一首 / 下一首（macOS 也支持 Command+← / →） N / P 下一首 / 上一首 @@ -27,19 +29,21 @@ """ import collections +import hashlib import json import os import queue import sys +import tempfile import threading import time +import requests +from typing import Any, Dict, Optional from pathlib import Path import tkinter as tk from tkinter import filedialog, messagebox, ttk -from sentence_analysis import detect_sentence_boundaries - try: from PIL import Image as PILImage, ImageTk except ImportError: @@ -73,6 +77,41 @@ LOOP_LABELS = {"off": "循环: 关", "one": "循环: 单曲", "list": "循环: SPEED_PRESETS = [0.5, 0.75, 1.0, 1.25, 1.5, 2.0] SEEK_STEP = 10 SEEK_STEP_BIG = 60 +DEFAULT_SERVER_URL = "https://videoservice.d1kt.cn" +CLIENT_NAME = "MediaPlayer/1.0 (Desktop)" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as media_file: + for chunk in iter(lambda: media_file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def extract_audio_to_wav(path: Path, output_path: Path) -> None: + """Extract one mono 16 kHz audio track before uploading it to the server.""" + import av + + with av.open(str(path)) as source: + source_stream = next((s for s in source.streams if s.type == "audio"), None) + if source_stream is None: + raise RuntimeError("文件中没有音频轨") + resampler = av.AudioResampler(format="s16", layout="mono", rate=16_000) + with av.open(str(output_path), "w", format="wav") as target: + target_stream = target.add_stream("pcm_s16le", rate=16_000, layout="mono") + for packet in source.demux(source_stream): + for frame in packet.decode(): + for resampled in resampler.resample(frame): + for encoded in target_stream.encode(resampled): + target.mux(encoded) + for resampled in resampler.resample(None): + for encoded in target_stream.encode(resampled): + target.mux(encoded) + for encoded in target_stream.encode(None): + target.mux(encoded) + + def fmt_time(ms): """把毫秒格式化为 mm:ss 或 h:mm:ss。""" if ms is None or ms < 0: @@ -835,6 +874,8 @@ class MediaPlayerApp(tk.Tk): settings = self._load_settings() self.seek_step = settings["seek_step"] self.sentence_mode = tk.BooleanVar(value=settings["sentence_mode"]) + self.server_url = settings["server_url"].rstrip("/") + self.auth_token = settings["auth_token"] self.loop_mode = "list" self._seeking = False self._switching_until = 0.0 @@ -842,8 +883,12 @@ class MediaPlayerApp(tk.Tk): self._muted_volume = None self._photo = None self._boundaries = {} + self._media_hashes = {} self._analysis_running = set() self._analysis_done = queue.Queue() + self._ui_events = queue.Queue() + self._explanation_window = None + self._explanation_generation = 0 self.engine = self._create_engine() if self.engine is None: @@ -893,6 +938,7 @@ class MediaPlayerApp(tk.Tk): ttk.Button(toolbar, text="打开文件…", command=self.open_files, takefocus=0).pack(side="left", padx=(0, 4)) ttk.Button(toolbar, text="打开文件夹…", command=self.open_folder, takefocus=0).pack(side="left", padx=(0, 8)) + ttk.Button(toolbar, text="服务…", command=self.configure_server, takefocus=0).pack(side="left") self.btn_prev = ttk.Button(toolbar, text="上一首", command=self.prev_track, takefocus=0) self.btn_prev.pack(side="left", padx=2) @@ -996,6 +1042,21 @@ class MediaPlayerApp(tk.Tk): command=self._on_sentence_mode, ) self.sentence_mode_btn.pack(side="left", padx=(16, 0)) + self.btn_ai_explain = ttk.Button( + controls, + text="AI讲解 (E)", + command=self.show_ai_explanation, + takefocus=0, + ) + self.btn_ai_explain.pack(side="left", padx=(12, 0)) + self.sentence_var = tk.StringVar(value="") + self.sentence_label = ttk.Label( + controls, + textvariable=self.sentence_var, + anchor="w", + width=32, + ) + self.sentence_label.pack(side="left", padx=(10, 0)) self.seek_step_spin.bind("", self._on_seek_step_return) self.seek_step_spin.bind("", lambda _e: self._commit_seek_step()) ttk.Label(controls, textvariable=self.status_var, anchor="e").pack(side="right", fill="x", expand=True) @@ -1035,6 +1096,10 @@ class MediaPlayerApp(tk.Tk): ("", lambda _e: self.cycle_loop()), ("", lambda _e: self.step_speed(-1)), ("", lambda _e: self.step_speed(1)), + ("", lambda _e: self.seek_sentence(-1)), + ("", lambda _e: self.seek_sentence(1)), + ("", lambda _e: self.show_ai_explanation()), + ("", lambda _e: self.show_ai_explanation()), ("", lambda _e: self.toggle_fullscreen()), ("", lambda _e: self.toggle_fullscreen()), ("", lambda _e: self.leave_fullscreen()), @@ -1090,17 +1155,21 @@ class MediaPlayerApp(tk.Tk): if self.engine is not None: while True: try: - done_path = self._analysis_done.get_nowait() + done_path, message = self._analysis_done.get_nowait() except queue.Empty: break if done_path == self._current_path() and self.sentence_mode.get(): boundaries = self._boundaries.get(done_path) if boundaries: - self.status_var.set( - f"句边界分析完成：识别到 {len(boundaries)} 句，←/→ 将按句跳转" - ) + self.status_var.set(message or f"句边界已就绪：共 {len(boundaries['sentences'])} 句") else: - self.status_var.set("该文件无法识别句子边界，按句跳转不可用") + self.status_var.set(message or "该文件无法识别句子边界，按句跳转不可用") + while True: + try: + callback = self._ui_events.get_nowait() + except queue.Empty: + break + callback() for event in self.engine.poll_events(): if event == "end": self._on_media_end() @@ -1115,6 +1184,9 @@ class MediaPlayerApp(tk.Tk): self.time_var.set(f"{fmt_time(pos)} / {fmt_time(length)}") if length > 0: self.seekbar.set_frac(pos / length) + sentence = self._current_sentence() + text = str(sentence.get("text") or "").strip() if sentence else "" + self.sentence_var.set(text) playing = self.engine.is_playing() self.btn_play.config(text="暂停" if playing else "播放") self.after(33, self._tick) @@ -1194,7 +1266,17 @@ class MediaPlayerApp(tk.Tk): step = SEEK_STEP if not 1 <= step <= 600: step = SEEK_STEP - return {"seek_step": step, "sentence_mode": bool(data.get("sentence_mode", False))} + server_url = str(data.get("server_url") or DEFAULT_SERVER_URL).strip() + if server_url.lower().startswith("http://") or server_url.lower().startswith("https://"): + server_url = server_url.rstrip("/") + else: + server_url = DEFAULT_SERVER_URL + return { + "seek_step": step, + "sentence_mode": bool(data.get("sentence_mode", False)), + "server_url": server_url, + "auth_token": str(data.get("auth_token") or "").strip(), + } def _save_settings(self): try: @@ -1203,6 +1285,8 @@ class MediaPlayerApp(tk.Tk): { "seek_step": self.seek_step, "sentence_mode": bool(self.sentence_mode.get()), + "server_url": self.server_url, + "auth_token": self.auth_token, }, ensure_ascii=False, indent=2, @@ -1297,6 +1381,7 @@ class MediaPlayerApp(tk.Tk): self.engine.stop() self._seeking = False self.time_var.set("00:00 / --:--") + self.sentence_var.set("") self.seekbar.set_frac(0.0) self.btn_play.config(text="播放") canvas = getattr(self, "video_canvas", None) @@ -1372,46 +1457,186 @@ class MediaPlayerApp(tk.Tk): self._ensure_sentence_boundaries(path) self.status_var.set("正在分析句边界…") elif self._boundaries[path]: - self.status_var.set(f"按句跳转已开启（识别到 {len(self._boundaries[path])} 句）") + count = len(self._boundaries[path]["sentences"]) + self.status_var.set(f"按句跳转已开启（识别到 {count} 句）") else: self.status_var.set("该文件无法识别句子边界，按句跳转不可用") def _ensure_sentence_boundaries(self, path): if path in self._boundaries or path in self._analysis_running: return + if not self.server_url: + self._boundaries[path] = {"sentences": []} + self._analysis_done.put((path, "请先在“服务…”中配置服务器地址")) + return self._analysis_running.add(path) + self.status_var.set("正在从服务器生成句边界…") threading.Thread(target=self._analyze_sentences, args=(path,), daemon=True).start() def _analyze_sentences(self, path): - boundaries = detect_sentence_boundaries(path) - self._boundaries[path] = boundaries + error = "" + document = None + audio_path = None + try: + media_hash = sha256_file(Path(path)) + self._media_hashes[path] = media_hash + document = self._load_cached_boundaries(media_hash) + if document is None: + audio_path = self._make_audio_upload(path) + document = self._generate_remote_boundaries(audio_path) + self._save_cached_boundaries(media_hash, document) + except Exception as exc: + error = str(exc) or exc.__class__.__name__ + finally: + if audio_path is not None: + try: + Path(audio_path).unlink(missing_ok=True) + except OSError: + pass self._analysis_running.discard(path) - self._analysis_done.put(path) + if document is not None and document.get("sentences"): + self._boundaries[path] = document + message = error or f"句边界已生成：共 {len(document['sentences'])} 句，已保存到本地" + self._analysis_done.put((path, message)) + else: + self._boundaries[path] = {"sentences": []} + self._analysis_done.put((path, error or "服务器未能识别句子边界")) + + def _boundary_cache_path(self, media_hash: str) -> Path: + return Path.home() / ".mediaplayer_sentence_boundaries" / f"{media_hash}.json" + + def _load_cached_boundaries(self, media_hash: str) -> Optional[Dict[str, Any]]: + path = self._boundary_cache_path(media_hash) + try: + data = json.loads(path.read_text(encoding="utf-8")) + sentences = data["sentences"] + if not isinstance(sentences, list): + return None + cleaned = [] + previous_end = 0 + for index, item in enumerate(sentences): + start = max(0, int(item["start_ms"])) + end = int(item["end_ms"]) + if end <= start or start < previous_end: + raise ValueError("invalid cached boundary") + cleaned.append({ + "index": index, + "start_ms": start, + "end_ms": end, + "text": (str(item["text"]) if item.get("text") is not None else None), + "language": item.get("language"), + }) + previous_end = end + if not cleaned: + return None + return {"sentences": cleaned, "duration_ms": int(data.get("duration_ms") or previous_end)} + except Exception: + return None + + def _save_cached_boundaries(self, media_hash: str, document: Dict[str, Any]) -> None: + path = self._boundary_cache_path(media_hash) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(".tmp") + payload = { + "version": 1, + "media_sha256": media_hash, + "duration_ms": int(document.get("duration_ms") or 0), + "sentences": document.get("sentences", []), + } + temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + temporary.replace(path) + + def _make_audio_upload(self, path): + source = Path(path) + if source.suffix.lower() in AUDIO_EXTS: + return source + handle = tempfile.NamedTemporaryFile(prefix="mediaplayer-audio-", suffix=".wav", delete=False) + output = Path(handle.name) + handle.close() + extract_audio_to_wav(source, output) + return output + + def _generate_remote_boundaries(self, audio_path) -> Dict[str, Any]: + filename = Path(audio_path).name + content_type = "audio/wav" if filename.lower().endswith(".wav") else "application/octet-stream" + headers = {"User-Agent": CLIENT_NAME} + if self.auth_token: + headers["Authorization"] = f"Bearer {self.auth_token}" + with Path(audio_path).open("rb") as audio_file: + response = requests.post( + self.server_url + "/api/v1/audio/sentence-boundaries", + headers=headers, + files={"file": (filename, audio_file, content_type)}, + data={"language": ""}, + timeout=(15, 3600), + ) + if response.status_code == 401: + raise RuntimeError("请先在“服务…”中登录") + if response.status_code not in range(200, 300): + detail = self._server_error_detail(response) + raise RuntimeError(f"句边界服务返回 {response.status_code}：{detail}") + data = response.json() + if not data.get("sentences"): + raise RuntimeError("服务器未能识别句子边界") + return data + + @staticmethod + def _server_error_detail(response) -> str: + try: + data = response.json() + detail = data.get("detail", response.text) + except Exception: + detail = response.text + detail = str(detail) + return detail if len(detail) <= 300 else detail[:297] + "..." + + def _current_sentence(self) -> Optional[Dict[str, Any]]: + path = self._current_path() + if path is None: + return None + sentences = (self._boundaries.get(path) or {}).get("sentences") or [] + pos = self.engine.get_time() + if pos < 0 or not sentences: + return None + pos_ms = max(0, int(pos)) + current = None + for sentence in sentences: + if pos_ms + 150 < int(sentence["start_ms"]): + break + current = sentence + return current + + def seek_sentence(self, direction): + return self._seek_sentence(direction) def _seek_sentence(self, direction): path = self._current_path() if path is None: return False - boundaries = self._boundaries.get(path) - if not boundaries: + sentences = (self._boundaries.get(path) or {}).get("sentences") or [] + if not sentences: return False pos = self.engine.get_time() if pos < 0: return False pos_sec = pos / 1000.0 if direction > 0: - for boundary in boundaries: + for sentence in sentences: + boundary = int(sentence["start_ms"]) / 1000.0 if boundary > pos_sec + 0.15: - self.seek_to(int(boundary * 1000)) + self.seek_to(int(sentence["start_ms"])) return True return False target = 0.0 - for boundary in reversed(boundaries): + target_ms = 0 + for sentence in reversed(sentences): + boundary = int(sentence["start_ms"]) / 1000.0 if boundary < pos_sec - 0.25: target = boundary + target_ms = int(sentence["start_ms"]) break if target > 0.0 or pos_sec > 0.25: - self.seek_to(int(target * 1000)) + self.seek_to(target_ms) return True return False @@ -1527,6 +1752,162 @@ class MediaPlayerApp(tk.Tk): self._save_playlist() self.status_var.set("播放列表已清空") + def configure_server(self): + window = tk.Toplevel(self) + window.title("服务配置") + window.geometry("420x230") + window.transient(self) + window.resizable(False, False) + frame = ttk.Frame(window, padding=16) + frame.pack(fill="both", expand=True) + frame.columnconfigure(1, weight=1) + + ttk.Label(frame, text="服务器地址").grid(row=0, column=0, sticky="w", pady=5) + server_var = tk.StringVar(value=self.server_url) + ttk.Entry(frame, textvariable=server_var, width=40).grid(row=0, column=1, sticky="ew") + + ttk.Label(frame, text="账号").grid(row=1, column=0, sticky="w", pady=5) + username_var = tk.StringVar() + ttk.Entry(frame, textvariable=username_var).grid(row=1, column=1, sticky="ew") + + ttk.Label(frame, text="密码").grid(row=2, column=0, sticky="w", pady=5) + password_var = tk.StringVar() + ttk.Entry(frame, textvariable=password_var, show="*").grid(row=2, column=1, sticky="ew") + + ttk.Label(frame, text="登录令牌").grid(row=3, column=0, sticky="w", pady=5) + token_var = tk.StringVar(value=self.auth_token) + ttk.Entry(frame, textvariable=token_var, show="*").grid(row=3, column=1, sticky="ew") + + message = tk.StringVar(value="已有令牌可直接保存；没有令牌时输入账号密码登录。") + ttk.Label(frame, textvariable=message, wraplength=360, foreground="#666666").grid( + row=4, column=0, columnspan=2, sticky="ew", pady=(12, 8) + ) + buttons = ttk.Frame(frame) + buttons.grid(row=5, column=0, columnspan=2, sticky="e") + + def save(): + url = server_var.get().strip().rstrip("/") + if not url.lower().startswith(("http://", "https://")): + messagebox.showerror("服务配置", "服务器地址必须以 http:// 或 https:// 开头。", parent=window) + return + self.server_url = url + self.auth_token = token_var.get().strip() + self._save_settings() + window.destroy() + self.status_var.set("服务配置已保存") + + def login_and_save(): + url = server_var.get().strip().rstrip("/") + try: + response = requests.post( + url + "/api/v1/auth/login", + json={"username": username_var.get().strip(), "password": password_var.get()}, + headers={"User-Agent": CLIENT_NAME}, + timeout=(10, 20), + ) + if response.status_code not in range(200, 300): + raise RuntimeError(self._server_error_detail(response)) + token_var.set(response.json()["token"]) + message.set("登录成功，令牌已保存。") + self.server_url = url + self.auth_token = token_var.get().strip() + self._save_settings() + window.after(500, window.destroy) + except Exception as exc: + messagebox.showerror("登录失败", str(exc), parent=window) + + ttk.Button(buttons, text="登录并保存", command=login_and_save, takefocus=0).pack(side="left", padx=(0, 8)) + ttk.Button(buttons, text="保存", command=save, takefocus=0).pack(side="left") + + def show_ai_explanation(self): + if not self.server_url: + self.configure_server() + return + sentence = self._current_sentence() + if not sentence: + self.status_var.set("当前没有可用于讲解的句子边界") + return + sentence_text = str(sentence.get("text") or "").strip() + if not sentence_text: + self.status_var.set("当前句子没有文字内容，AI 讲解不可用") + return + + self._explanation_generation += 1 + generation = self._explanation_generation + if self._explanation_window is not None: + try: + self._explanation_window.destroy() + except tk.TclError: + pass + window = tk.Toplevel(self) + window.title("AI 句子讲解") + window.geometry("480x380") + window.transient(self) + frame = ttk.Frame(window, padding=14) + frame.pack(fill="both", expand=True) + frame.columnconfigure(0, weight=1) + frame.rowconfigure(1, weight=1) + ttk.Label(frame, text=sentence_text, wraplength=440, justify="left", font=("Helvetica", 12, "bold")).grid( + row=0, column=0, sticky="ew", pady=(0, 10) + ) + text = tk.Text(frame, wrap="word", state="disabled", relief="flat", background="#f6f6f6") + text.grid(row=1, column=0, sticky="nsew") + text.insert("1.0", "AI 正在讲解这句话…") + ttk.Button(frame, text="关闭", command=window.destroy, takefocus=0).grid(row=2, column=0, sticky="e", pady=(10, 0)) + self._explanation_window = window + + path = self._current_path() or "" + course_title = Path(path).stem + threading.Thread( + target=self._explain_sentence_worker, + args=(sentence, course_title, generation), + daemon=True, + ).start() + + def _explain_sentence_worker(self, sentence, course_title, generation): + try: + headers = {"Content-Type": "application/json", "User-Agent": CLIENT_NAME} + if self.auth_token: + headers["Authorization"] = f"Bearer {self.auth_token}" + payload = { + "sentence_text": sentence.get("text", ""), + "course_title": course_title, + "sentence_index": sentence.get("index"), + "language": sentence.get("language"), + } + response = requests.post( + self.server_url + "/api/v1/sentences/explain", + headers=headers, + json=payload, + timeout=(15, 90), + ) + if response.status_code == 401: + raise RuntimeError("登录已过期，请在“服务…”中重新登录") + if response.status_code not in range(200, 300): + raise RuntimeError(f"AI 讲解服务返回 {response.status_code}：{self._server_error_detail(response)}") + explanation = response.json()["explanation"] + self._ui_events.put(lambda: self._show_explanation_result(explanation, generation)) + except Exception as exc: + message = str(exc) or exc.__class__.__name__ + self._ui_events.put(lambda: self._show_explanation_result("讲解失败：" + message, generation, error=True)) + + def _show_explanation_result(self, explanation, generation, error=False): + if generation != self._explanation_generation: + return + window = self._explanation_window + if window is None: + return + try: + text = window.nametowidget(window.grid_slaves(row=1, column=0)[0]) + text.configure(state="normal") + text.delete("1.0", "end") + text.insert("1.0", explanation) + text.configure(state="disabled") + if error: + window.title("AI 讲解失败") + except (tk.TclError, IndexError): + pass + def _update_status(self): parts = [f"循环：{LOOP_LABELS[self.loop_mode].split(':')[1].strip()}"] parts.append(f"速度 {self.rate:g}x") diff --git a/requirements.txt b/requirements.txt index 24add0e..c95f337 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ av>=12.0 +requests>=2.32.0 sounddevice>=0.4.6 numpy>=1.26 Pillow>=10.0 diff --git a/sentence_api/README.md b/sentence_api/README.md index 54260c3..3cdeb76 100644 --- a/sentence_api/README.md +++ b/sentence_api/README.md @@ -52,6 +52,7 @@ GET /api/v1/videos GET /api/v1/videos/{sha256} GET /api/v1/videos/{sha256}/content GET /api/v1/videos/{sha256}/sentence-boundaries +POST /api/v1/audio/sentence-boundaries POST /api/v1/sentences/explain POST /api/v1/videos/{sha256}/sentences/{index}/assessments @@ -67,6 +68,11 @@ DELETE /api/v1/admin/videos/{sha256} Admin endpoints use `X-Admin-Key` when `ADMIN_API_KEY` is configured. Assessment requests use `X-Client-Key` when `CLIENT_API_KEY` is configured. +`POST /api/v1/audio/sentence-boundaries` accepts an audio file from an +authenticated client, generates boundaries in memory/temp storage only, and +returns a sentence document. It does not register or archive the uploaded +audio. + ## AI Sentence Explanation Configure an OpenAI-compatible chat-completions endpoint in `sentence_api/.env`. diff --git a/sentence_api/main.py b/sentence_api/main.py index d9d2a3d..6dee872 100644 --- a/sentence_api/main.py +++ b/sentence_api/main.py @@ -2,6 +2,7 @@ import hashlib import hmac import logging import mimetypes +import tempfile import uuid import urllib.request import urllib.error @@ -57,7 +58,7 @@ from .models import ( VideoUploadResponse, ) from .oss import VolcanoOSSUploader -from .processing import VideoProcessor, split_sentence_text +from .processing import VideoProcessor, generate_audio_boundary_document, split_sentence_text from .repository import VideoRepository from .store import BoundaryStore from .transcription import MossTranscriber, Transcriber @@ -72,6 +73,7 @@ SHA256_PATH = ApiPath( ) ALLOWED_VIDEO_EXTENSIONS = {".mp4", ".m4v", ".mov", ".mkv", ".webm"} ALLOWED_AUDIO_EXTENSIONS = {".m4a", ".mp4", ".wav"} +MAX_AUDIO_BOUNDARY_BYTES = 512 * 1024 * 1024 class UploadTooLargeError(ValueError): @@ -329,6 +331,47 @@ def create_app( raise HTTPException(status_code=503, detail=str(exc)) from exc return SentenceExplanationResponse(explanation=explanation) + @application.post( + "/api/v1/audio/sentence-boundaries", + response_model=SentenceBoundaryDocument, + ) + async def generate_sentence_boundaries( + file: UploadFile = File(...), + language: Optional[str] = Form(default=None, max_length=32), + user: AuthenticatedUser = Depends(current_user), + ) -> SentenceBoundaryDocument: + del user + max_bytes = min(MAX_AUDIO_BOUNDARY_BYTES, service_settings.max_upload_bytes) + with tempfile.TemporaryDirectory(prefix="audio-boundaries-") as temp_dir: + audio_path = Path(temp_dir) / "audio.wav" + try: + audio_hash, _size = await run_in_threadpool( + _persist_upload, + file.file, + audio_path, + max_bytes, + ) + document = await run_in_threadpool( + generate_audio_boundary_document, + video_hash=audio_hash, + audio_path=audio_path, + transcriber=moss, + language=language, + ) + except UploadTooLargeError as exc: + raise HTTPException(status_code=413, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + except Exception as exc: + logger.exception("Ad-hoc audio sentence boundary generation failed.") + raise HTTPException( + status_code=502, + detail="Sentence boundary generation failed. Please try again later.", + ) from exc + finally: + await file.close() + return document + @application.delete("/api/v1/courses/{video_hash}/enroll", status_code=204) def unenroll_course( video_hash: str = SHA256_PATH, diff --git a/sentence_api/processing.py b/sentence_api/processing.py index 02a30c4..1c2fe60 100644 --- a/sentence_api/processing.py +++ b/sentence_api/processing.py @@ -14,7 +14,8 @@ from .audio_metrics import ( refine_sentence_start_ms, ) from .config import Settings -from .generate_boundaries import ALGORITHM_VERSION, make_entry +from .generate_boundaries import ALGORITHM_VERSION, make_entry, media_duration_ms +from sentence_analysis import detect_sentence_boundaries from .models import SentenceBoundary, SentenceBoundaryDocument from .oss import VolcanoOSSUploader from .repository import VideoRepository @@ -27,6 +28,7 @@ from .transcription import ( logger = logging.getLogger(__name__) MOSS_ALGORITHM_VERSION = "moss-punctuation-v3" +SILENCE_ALGORITHM_VERSION = "silence-rms-v1" class VideoProcessor: @@ -417,6 +419,67 @@ def document_from_transcript( ) +def generate_audio_boundary_document( + *, + video_hash: str, + audio_path: Path, + transcriber: Transcriber, + language: Optional[str] = None, +) -> SentenceBoundaryDocument: + """Generate sentence boundaries for a client-supplied audio file. + + The caller owns ``audio_path`` and is responsible for deleting it. Nothing + in this function copies or registers the audio in application storage. + """ + duration_ms = media_duration_ms(audio_path) + transcript = None + if transcriber.available: + try: + transcript = transcriber.transcribe(audio_path, language) + except Exception as exc: + logger.warning("Audio boundary transcription failed; using silence fallback: %s", exc) + + if transcript is not None: + document = document_from_transcript( + video_hash=video_hash, + duration_ms=duration_ms, + transcript=transcript, + language=language, + audio_path=audio_path, + ) + if document.sentences: + return document + + starts = detect_sentence_boundaries(audio_path) + if not starts: + raise ValueError("No sentence boundaries could be detected in the audio.") + + starts_ms = sorted({max(0, min(duration_ms - 1, int(round(start * 1000)))) for start in starts}) + sentences: List[SentenceBoundary] = [] + for index, start_ms in enumerate(starts_ms): + end_ms = starts_ms[index + 1] if index + 1 < len(starts_ms) else duration_ms + if end_ms <= start_ms: + continue + sentences.append( + SentenceBoundary( + index=len(sentences), + start_ms=start_ms, + end_ms=end_ms, + text=None, + language=language, + ) + ) + if not sentences: + raise ValueError("Detected audio boundaries do not form valid sentence ranges.") + + return SentenceBoundaryDocument( + video_hash=video_hash, + duration_ms=duration_ms, + algorithm_version=SILENCE_ALGORITHM_VERSION, + sentences=sentences, + ) + + def _media_duration_ms(path: Path) -> int: from .generate_boundaries import media_duration_ms diff --git a/sentence_api/tests/test_api.py b/sentence_api/tests/test_api.py index 5fe445a..a2532f2 100644 --- a/sentence_api/tests/test_api.py +++ b/sentence_api/tests/test_api.py @@ -80,6 +80,42 @@ def test_invalid_hash_is_rejected(tmp_path): assert response.status_code == 422 +def test_adhoc_audio_boundaries_do_not_persist_audio(tmp_path, monkeypatch): + client = make_client(tmp_path) + upload = tmp_path / "audio.wav" + upload.write_bytes(b"not-a-real-audio") + + import sentence_api.processing as processing_module + + monkeypatch.setattr(processing_module, "detect_sentence_boundaries", lambda path: [0.0, 1.0]) + monkeypatch.setattr(processing_module, "media_duration_ms", lambda path: 2000) + + response = client.post( + "/api/v1/audio/sentence-boundaries", + files={"file": ("audio.wav", upload.read_bytes(), "audio/wav")}, + ) + + assert response.status_code == 200, response.text + payload = response.json() + assert payload["algorithm_version"] == "silence-rms-v1" + assert payload["duration_ms"] == 2000 + assert [(item["start_ms"], item["end_ms"]) for item in payload["sentences"]] == [(0, 1000), (1000, 2000)] + assert not (tmp_path / "data" / "work").exists() or not any((tmp_path / "data" / "work").iterdir()) + assert not (tmp_path / "data" / "v").exists() or not any((tmp_path / "data" / "v").iterdir()) + + +def test_adhoc_audio_boundaries_require_login(tmp_path): + client = make_client(tmp_path) + client.headers.clear() + + response = client.post( + "/api/v1/audio/sentence-boundaries", + files={"file": ("audio.wav", b"audio", "audio/wav")}, + ) + + assert response.status_code == 401 + + def test_sentence_explanation_requires_login(tmp_path): index_path = tmp_path / "boundaries.json" index_path.write_text("{}", encoding="utf-8")