From 69d2ba986fbcacea42529ed2129b80417e42368b Mon Sep 17 00:00:00 2001 From: Shuming Liu Date: Fri, 14 Aug 2026 19:06:04 +0800 Subject: [PATCH] add sentence service --- README.md | 8 + android/README.md | 48 + .../oraltrainer/sdk/OralTrainerController.kt | 24 +- .../oraltrainer/sdk/OralTrainerSdk.kt | 2 + .../oraltrainer/sdk/OralTrainerSdkConfig.kt | 1 + .../oraltrainer/sdk/SentenceBoundaryApi.kt | 196 +++ .../oraltrainer/sdk/StreamingCache.kt | 4 +- .../oraltrainer/sample/MainActivity.kt | 61 +- media_player.py | 97 +- sentence_analysis.py | 113 ++ sentence_api/README.md | 92 + sentence_api/__init__.py | 2 + sentence_api/data/sentence_boundaries.json | 1498 +++++++++++++++++ sentence_api/generate_boundaries.py | 116 ++ sentence_api/main.py | 62 + sentence_api/models.py | 48 + sentence_api/requirements.txt | 7 + sentence_api/store.py | 54 + sentence_api/tests/test_api.py | 55 + 19 files changed, 2379 insertions(+), 109 deletions(-) create mode 100644 android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/SentenceBoundaryApi.kt create mode 100644 sentence_analysis.py create mode 100644 sentence_api/README.md create mode 100644 sentence_api/__init__.py create mode 100644 sentence_api/data/sentence_boundaries.json create mode 100644 sentence_api/generate_boundaries.py create mode 100644 sentence_api/main.py create mode 100644 sentence_api/models.py create mode 100644 sentence_api/requirements.txt create mode 100644 sentence_api/store.py create mode 100644 sentence_api/tests/test_api.py diff --git a/README.md b/README.md index 12c37ff..5337648 100644 --- a/README.md +++ b/README.md @@ -121,3 +121,11 @@ SDK 基于 AndroidX Media3 / ExoPlayer，支持在线视频流、本地缓存、 暂停/继续、左划/右划按句或按步长跳转，并预留了学生模仿质量评分算法接口。 详见 [android/README.md](android/README.md)。 + +## 句子边界 API + +仓库新增了 `sentence_api/` 服务。它复用桌面播放器的静音检测算法，提前为视频 +生成句子边界，并根据视频 SHA-256 哈希向 Android 返回毫秒级的 +`start_ms` / `end_ms` 列表。服务不会在移动端请求时临时分析视频。 + +详见 [sentence_api/README.md](sentence_api/README.md)。 diff --git a/android/README.md b/android/README.md index c68e0e5..27a6473 100644 --- a/android/README.md +++ b/android/README.md @@ -17,6 +17,10 @@ and a placeholder interface for future imitation-quality scoring. - Swipe right: next sentence; if there is no sentence data, fast-forward by the configured seek step. +The sample app's previous/next sentence buttons use the same fallback behavior. +Actual sentence navigation requires `SentenceBoundary` timestamps supplied by +the course service or extracted from subtitle/speech analysis data. + These defaults match the desktop player's arrow-key workflow while fitting a tablet touch screen. @@ -62,6 +66,50 @@ OralTrainerSdk.init( ) ``` +## Sentence Boundary API + +The SDK defaults to `https://video_service.d1kt.cn` and requests: + +```text +GET /api/v1/videos/{sha256}/sentence-boundaries +``` + +For a known hash: + +```kotlin +sdk.sentenceBoundaryApi.fetch(videoHash, callback) +``` + +For a local `content://` video, the SDK can hash the file in streaming chunks +before querying the API: + +```kotlin +sdk.sentenceBoundaryApi.fetchForUri(videoUri, contentResolver, callback) +``` + +Override the service only when a staging or private deployment is required: + +```kotlin +OralTrainerSdk.init( + context, + OralTrainerSdkConfig( + sentenceBoundaryApiBaseUrl = "https://video_service.d1kt.cn" + ) +) +``` + +## Local Video Testing + +The computer path `/Users/...` is not visible to an Android device. For a +quick test, copy the video to the device or use the `选择本地视频` button in the +sample app. The Android document picker returns a `content://` URI, which the +SDK supports without requesting broad storage permissions. + +For production courses, keep the video on an HTTPS CDN or object-storage +service and pass its URL as `TrainingMediaItem.uri`. The SDK streams it and +caches downloaded ranges locally. Copying large 4K files to each device is +better reserved for explicitly offline courses. + ## Future Imitation Scoring Provide an implementation of `ImitationQualityAssessor` when the speech diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerController.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerController.kt index 628b50b..1f33577 100644 --- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerController.kt +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerController.kt @@ -242,6 +242,18 @@ class OralTrainerController internal constructor( return true } + fun previousSentenceOrRewind() { + if (!config.sentenceMode || !seekToPreviousSentence()) { + rewind() + } + } + + fun nextSentenceOrForward() { + if (!config.sentenceMode || !seekToNextSentence()) { + fastForward() + } + } + fun performSwipeAction(action: SwipeAction) { when (action) { SwipeAction.NONE -> Unit @@ -249,16 +261,8 @@ class OralTrainerController internal constructor( SwipeAction.FORWARD -> fastForward() SwipeAction.PREVIOUS_SENTENCE -> seekToPreviousSentence() SwipeAction.NEXT_SENTENCE -> seekToNextSentence() - SwipeAction.PREVIOUS_SENTENCE_OR_REWIND -> { - if (!config.sentenceMode || !seekToPreviousSentence()) { - rewind() - } - } - SwipeAction.NEXT_SENTENCE_OR_FORWARD -> { - if (!config.sentenceMode || !seekToNextSentence()) { - fastForward() - } - } + SwipeAction.PREVIOUS_SENTENCE_OR_REWIND -> previousSentenceOrRewind() + SwipeAction.NEXT_SENTENCE_OR_FORWARD -> nextSentenceOrForward() } } diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdk.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdk.kt index 4c199a5..a1f78b0 100644 --- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdk.kt +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdk.kt @@ -9,6 +9,7 @@ class OralTrainerSdk private constructor( private val appContext = context.applicationContext val cache: OralTrainerCache = OralTrainerCache(appContext, config) + val sentenceBoundaryApi: SentenceBoundaryApi = SentenceBoundaryApi(appContext, config) @JvmOverloads fun createController( @@ -24,6 +25,7 @@ class OralTrainerSdk private constructor( } fun release() { + sentenceBoundaryApi.release() StreamingCache.release() } diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdkConfig.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdkConfig.kt index fb8c278..b033443 100644 --- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdkConfig.kt +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/OralTrainerSdkConfig.kt @@ -8,4 +8,5 @@ data class OralTrainerSdkConfig @JvmOverloads constructor( val userAgent: String = "OralTrainerSdk/0.1.0", val connectTimeoutMs: Int = 15_000, val readTimeoutMs: Int = 30_000, + val sentenceBoundaryApiBaseUrl: String = "https://video_service.d1kt.cn", ) diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/SentenceBoundaryApi.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/SentenceBoundaryApi.kt new file mode 100644 index 0000000..56a232f --- /dev/null +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/SentenceBoundaryApi.kt @@ -0,0 +1,196 @@ +package cn.learningpad.oraltrainer.sdk + +import android.content.ContentResolver +import android.content.Context +import android.net.Uri +import android.os.Handler +import android.os.Looper +import org.json.JSONObject +import java.io.BufferedInputStream +import java.io.IOException +import java.io.InterruptedIOException +import java.net.HttpURLConnection +import java.net.URL +import java.security.MessageDigest +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.Future + +data class SentenceBoundaryApiResult( + val videoHash: String, + val durationMs: Long, + val algorithmVersion: String, + val sentences: List, +) + +interface SentenceBoundaryApiCallback { + fun onSuccess(result: SentenceBoundaryApiResult) + + fun onError(error: Throwable) +} + +class CancellableRequest internal constructor( + private val future: Future<*>, +) { + fun cancel() { + future.cancel(true) + } +} + +class SentenceBoundaryApi internal constructor( + context: Context, + private val config: OralTrainerSdkConfig, +) { + private val appContext = context.applicationContext + private val mainHandler = Handler(Looper.getMainLooper()) + private val executor: ExecutorService = Executors.newCachedThreadPool() + private val baseUri = Uri.parse(config.sentenceBoundaryApiBaseUrl.trimEnd('/')) + + init { + require(baseUri.scheme == "https" || baseUri.scheme == "http") { + "sentenceBoundaryApiBaseUrl must use http or https." + } + require(!baseUri.host.isNullOrBlank()) { + "sentenceBoundaryApiBaseUrl must include a host." + } + } + + fun fetch( + videoHash: String, + callback: SentenceBoundaryApiCallback, + ): CancellableRequest { + validateHash(videoHash) + return submit(callback) { fetchBlocking(videoHash.lowercase()) } + } + + fun fetchForUri( + uri: Uri, + contentResolver: ContentResolver = appContext.contentResolver, + callback: SentenceBoundaryApiCallback, + ): CancellableRequest { + return submit(callback) { + val videoHash = sha256(uri, contentResolver) + fetchBlocking(videoHash) + } + } + + fun release() { + executor.shutdownNow() + } + + private fun submit( + callback: SentenceBoundaryApiCallback, + operation: () -> SentenceBoundaryApiResult, + ): CancellableRequest { + val future = executor.submit { + try { + val result = operation() + if (!Thread.currentThread().isInterrupted) { + mainHandler.post { callback.onSuccess(result) } + } + } catch (error: Throwable) { + if (!Thread.currentThread().isInterrupted) { + mainHandler.post { callback.onError(error) } + } + } + } + return CancellableRequest(future) + } + + private fun fetchBlocking(videoHash: String): SentenceBoundaryApiResult { + val endpoint = baseUri.buildUpon() + .appendPath("api") + .appendPath("v1") + .appendPath("videos") + .appendPath(videoHash) + .appendPath("sentence-boundaries") + .build() + val connection = URL(endpoint.toString()).openConnection() as HttpURLConnection + return try { + connection.requestMethod = "GET" + connection.connectTimeout = config.connectTimeoutMs + connection.readTimeout = config.readTimeoutMs + connection.setRequestProperty("Accept", "application/json") + connection.setRequestProperty("User-Agent", config.userAgent) + val statusCode = connection.responseCode + val stream = if (statusCode in 200..299) { + connection.inputStream + } else { + connection.errorStream + } + val body = stream?.bufferedReader(Charsets.UTF_8)?.use { it.readText() }.orEmpty() + if (statusCode !in 200..299) { + throw SentenceBoundaryApiException(statusCode, body) + } + parseResponse(body, videoHash) + } finally { + connection.disconnect() + } + } + + private fun parseResponse( + body: String, + requestedHash: String, + ): SentenceBoundaryApiResult { + val root = JSONObject(body) + val responseHash = root.getString("video_hash").lowercase() + if (responseHash != requestedHash.lowercase()) { + throw IOException("Sentence boundary response hash does not match the request.") + } + val jsonSentences = root.getJSONArray("sentences") + val sentences = buildList(jsonSentences.length()) { + for (index in 0 until jsonSentences.length()) { + val sentence = jsonSentences.getJSONObject(index) + add( + SentenceBoundary( + index = sentence.getInt("index"), + startMs = sentence.getLong("start_ms"), + endMs = sentence.getLong("end_ms"), + text = if (sentence.isNull("text")) null else sentence.getString("text"), + ) + ) + } + } + return SentenceBoundaryApiResult( + videoHash = responseHash, + durationMs = root.getLong("duration_ms"), + algorithmVersion = root.getString("algorithm_version"), + sentences = sentences, + ) + } + + private fun sha256(uri: Uri, contentResolver: ContentResolver): String { + val digest = MessageDigest.getInstance("SHA-256") + contentResolver.openInputStream(uri)?.use { input -> + BufferedInputStream(input).use { buffered -> + val buffer = ByteArray(1024 * 1024) + while (true) { + if (Thread.currentThread().isInterrupted) { + throw InterruptedIOException() + } + val count = buffered.read(buffer) + if (count < 0) { + break + } + digest.update(buffer, 0, count) + } + } + } ?: throw IOException("Cannot open video URI: $uri") + return digest.digest().joinToString("") { byte -> "%02x".format(byte.toInt() and 0xff) } + } + + private fun validateHash(videoHash: String) { + require(SHA256_PATTERN.matches(videoHash)) { + "videoHash must be a 64-character SHA-256 hex digest." + } + } + + companion object { + private val SHA256_PATTERN = Regex("^[0-9a-fA-F]{64}$") + } +} + +class SentenceBoundaryApiException( + val statusCode: Int, + responseBody: String, +) : IOException("Sentence boundary API returned HTTP $statusCode: $responseBody") diff --git a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/StreamingCache.kt b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/StreamingCache.kt index 5fd5e10..fe3a30d 100644 --- a/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/StreamingCache.kt +++ b/android/oral-trainer-sdk/src/main/java/cn/learningpad/oraltrainer/sdk/StreamingCache.kt @@ -5,6 +5,7 @@ package cn.learningpad.oraltrainer.sdk import android.content.Context import androidx.media3.database.StandaloneDatabaseProvider import androidx.media3.datasource.DataSource +import androidx.media3.datasource.DefaultDataSource import androidx.media3.datasource.cache.CacheDataSource import androidx.media3.datasource.cache.LeastRecentlyUsedCacheEvictor import androidx.media3.datasource.cache.SimpleCache @@ -32,11 +33,12 @@ internal object StreamingCache { context: Context, config: OralTrainerSdkConfig, ): DataSource.Factory { - val upstreamFactory = DefaultHttpDataSource.Factory() + val httpDataSourceFactory = DefaultHttpDataSource.Factory() .setUserAgent(config.userAgent) .setConnectTimeoutMs(config.connectTimeoutMs) .setReadTimeoutMs(config.readTimeoutMs) .setAllowCrossProtocolRedirects(true) + val upstreamFactory = DefaultDataSource.Factory(context.applicationContext, httpDataSourceFactory) return CacheDataSource.Factory() .setCache(get(context, config)) diff --git a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt index 041a200..40a00a8 100644 --- a/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt +++ b/android/sample-app/src/main/java/cn/learningpad/oraltrainer/sample/MainActivity.kt @@ -1,6 +1,7 @@ package cn.learningpad.oraltrainer.sample import android.app.Activity +import android.content.Intent import android.graphics.Color import android.net.Uri import android.os.Bundle @@ -19,9 +20,16 @@ import cn.learningpad.oraltrainer.sdk.OralTrainerSdk import cn.learningpad.oraltrainer.sdk.PlaybackSnapshot import cn.learningpad.oraltrainer.sdk.PlayerConfig import cn.learningpad.oraltrainer.sdk.SentenceBoundary +import cn.learningpad.oraltrainer.sdk.SentenceBoundaryApiCallback import cn.learningpad.oraltrainer.sdk.TrainingMediaItem +import cn.learningpad.oraltrainer.sdk.SentenceBoundaryApiResult class MainActivity : Activity() { + private companion object { + const val PICK_VIDEO_REQUEST = 1001 + } + + private lateinit var sdk: OralTrainerSdk private lateinit var controller: OralTrainerController private lateinit var statusText: TextView private lateinit var sentenceText: TextView @@ -29,7 +37,7 @@ class MainActivity : Activity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - val sdk = OralTrainerSdk.init(this) + sdk = OralTrainerSdk.init(this) controller = sdk.createController( playerConfig = PlayerConfig( sentenceMode = true, @@ -66,8 +74,8 @@ class MainActivity : Activity() { addView(commandButton("快退") { controller.rewind() }) addView(commandButton("播放/暂停") { controller.togglePlayPause() }) addView(commandButton("快进") { controller.fastForward() }) - addView(commandButton("上一句") { controller.seekToPreviousSentence() }) - addView(commandButton("下一句") { controller.seekToNextSentence() }) + addView(commandButton("上一句") { controller.previousSentenceOrRewind() }) + addView(commandButton("下一句") { controller.nextSentenceOrForward() }) } val root = LinearLayout(this).apply { @@ -77,6 +85,10 @@ class MainActivity : Activity() { addView(statusText) addView(sentenceText) addView(controls) + addView(Button(this@MainActivity).apply { + text = "选择本地视频" + setOnClickListener { openLocalVideoPicker() } + }) } setContentView(root) @@ -108,6 +120,37 @@ class MainActivity : Activity() { controller.loadItem(sampleOnlineLesson()) } + override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) { + super.onActivityResult(requestCode, resultCode, data) + if (requestCode != PICK_VIDEO_REQUEST || resultCode != RESULT_OK) { + return + } + val uri = data?.data ?: return + val persistableFlags = data.flags and Intent.FLAG_GRANT_READ_URI_PERMISSION + if (persistableFlags != 0) { + runCatching { + contentResolver.takePersistableUriPermission(uri, persistableFlags) + } + } + val item = TrainingMediaItem( + id = uri.toString(), + title = uri.lastPathSegment ?: "本地视频", + uri = uri, + ) + controller.loadItem(item) + statusText.text = "正在获取句子边界..." + sdk.sentenceBoundaryApi.fetchForUri(uri, contentResolver, object : SentenceBoundaryApiCallback { + override fun onSuccess(result: SentenceBoundaryApiResult) { + controller.loadItem(item.copy(sentences = result.sentences)) + statusText.text = "句子边界已加载：${result.sentences.size} 句" + } + + override fun onError(error: Throwable) { + statusText.text = "句子边界获取失败，使用 10 秒跳转：${error.message.orEmpty()}" + } + }) + } + override fun onDestroy() { controller.release() super.onDestroy() @@ -128,6 +171,18 @@ class MainActivity : Activity() { } } + private fun openLocalVideoPicker() { + startActivityForResult( + Intent(Intent.ACTION_OPEN_DOCUMENT).apply { + addCategory(Intent.CATEGORY_OPENABLE) + type = "video/*" + addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION) + addFlags(Intent.FLAG_GRANT_PERSISTABLE_URI_PERMISSION) + }, + PICK_VIDEO_REQUEST, + ) + } + private fun sampleOnlineLesson(): TrainingMediaItem { return TrainingMediaItem( id = "online_sample_01", diff --git a/media_player.py b/media_player.py index e72ec7a..048dcc1 100644 --- a/media_player.py +++ b/media_player.py @@ -38,6 +38,8 @@ from pathlib import Path import tkinter as tk from tkinter import filedialog, messagebox, ttk +from sentence_analysis import detect_sentence_boundaries + try: from PIL import Image as PILImage, ImageTk except ImportError: @@ -71,11 +73,6 @@ LOOP_LABELS = {"off": "循环: 关", "one": "循环: 单曲", "list": "循环: SPEED_PRESETS = [0.5, 0.75, 1.0, 1.25, 1.5, 2.0] SEEK_STEP = 10 SEEK_STEP_BIG = 60 -SILENCE_FLOOR_PERCENTILE = 10 # 用能量低分位估计背景噪声底 -SILENCE_THRESHOLD_FACTOR = 1.5 # 静音阈值 = 噪声底 × 系数 -SILENCE_BRIDGE_GAP = 0.06 # 停顿中 60ms 内的短促残响/噪声也按静音处理 - - def fmt_time(ms): """把毫秒格式化为 mm:ss 或 h:mm:ss。""" if ms is None or ms < 0: @@ -88,96 +85,6 @@ def fmt_time(ms): return f"{minutes:02d}:{seconds:02d}" -def detect_sentence_boundaries(path, min_silence=0.30, min_sentence=0.35): - """基于音频静音自动切分句子，返回每句话开始的时间（秒）列表。 - - 语音之间通常有短暂停顿，把停顿前后的连续声音视为一句话。 - 没有音频轨、无法解码或缺少依赖时返回 None。 - """ - try: - import av - import numpy as np - except ImportError: - return None - try: - container = av.open(str(path)) - audio = next((s for s in container.streams if s.type == "audio"), None) - if audio is None: - container.close() - return None - sample_rate = 16000 - resampler = av.AudioResampler(format="fltp", layout="mono", rate=sample_rate) - window = 480 # 30ms @ 16kHz - frame_seconds = window / sample_rate - energies = [] - buf = [] - - def push(data): - buf.append(data) - total = sum(len(x) for x in buf) - if total < window: - return - arr = np.concatenate(buf) - buf.clear() - n = len(arr) // window * window - win = arr[:n].reshape(-1, window) - energies.extend((np.mean(win * win, axis=1) ** 0.5).tolist()) - if len(arr) > n: - buf.append(arr[n:]) - - for packet in container.demux(audio): - for frame in packet.decode(): - for out in resampler.resample(frame): - push(out.to_ndarray()[0]) - for out in resampler.resample(None): - push(out.to_ndarray()[0]) - if buf: - push(np.zeros(window, dtype=np.float32)) - duration = container.duration - container.close() - if len(energies) < 3: - return None - energies = np.asarray(energies, dtype=np.float64) - floor = float(np.percentile(energies, 95)) - if floor <= 0.0: - return None - noise_floor = float(np.percentile(energies, SILENCE_FLOOR_PERCENTILE)) - threshold = max(0.008, SILENCE_THRESHOLD_FACTOR * noise_floor) - silence = energies < threshold - bridge_frames = int(round(SILENCE_BRIDGE_GAP / frame_seconds)) - if bridge_frames > 0: - bridged = silence.copy() - run_start = None - for idx, is_silent in enumerate(silence): - if not is_silent and run_start is None: - run_start = idx - elif is_silent and run_start is not None: - if run_start > 0 and idx - run_start <= bridge_frames: - bridged[run_start:idx] = True - run_start = None - silence = bridged - boundaries = [0.0] - run_start = None - for idx, is_silent in enumerate(silence): - if is_silent and run_start is None: - run_start = idx - elif not is_silent and run_start is not None: - if (idx - run_start) * frame_seconds >= min_silence: - end_sec = idx * frame_seconds - if end_sec - boundaries[-1] >= min_sentence: - boundaries.append(end_sec) - run_start = None - if run_start is not None and (len(silence) - run_start) * frame_seconds >= min_silence: - end_sec = len(silence) * frame_seconds - if end_sec - boundaries[-1] >= min_sentence: - boundaries.append(end_sec) - if duration and duration > 0: - boundaries = [b for b in boundaries if b < duration / 1e6 - 0.1] - return boundaries - except Exception: - return None - - class _BaseEngine: """播放内核统一接口。""" diff --git a/sentence_analysis.py b/sentence_analysis.py new file mode 100644 index 0000000..257a905 --- /dev/null +++ b/sentence_analysis.py @@ -0,0 +1,113 @@ +"""Shared audio-silence sentence boundary detection. + +This module intentionally has no GUI or web-service dependencies so the +desktop player, the pre-generation CLI, and the API worker use the same +algorithm. +""" + +SILENCE_FLOOR_PERCENTILE = 10 +SILENCE_THRESHOLD_FACTOR = 1.5 +SILENCE_BRIDGE_GAP = 0.06 + + +def detect_sentence_boundaries(path, min_silence=0.30, min_sentence=0.35): + """Return sentence start times in seconds, or None when unavailable. + + Audio is resampled to mono 16 kHz, divided into 30 ms windows, and + silence runs are converted into boundaries. This matches the original + desktop player's behavior. + """ + try: + import av + import numpy as np + except ImportError: + return None + + container = None + try: + container = av.open(str(path)) + audio = next((stream for stream in container.streams if stream.type == "audio"), None) + if audio is None: + return None + + sample_rate = 16_000 + resampler = av.AudioResampler(format="fltp", layout="mono", rate=sample_rate) + window = 480 + frame_seconds = window / sample_rate + energies = [] + buffer = [] + + def push(data): + buffer.append(data) + total = sum(len(chunk) for chunk in buffer) + if total < window: + return + array = np.concatenate(buffer) + buffer.clear() + complete = len(array) // window * window + windows = array[:complete].reshape(-1, window) + energies.extend((np.mean(windows * windows, axis=1) ** 0.5).tolist()) + if len(array) > complete: + buffer.append(array[complete:]) + + for packet in container.demux(audio): + for frame in packet.decode(): + for output in resampler.resample(frame): + push(output.to_ndarray()[0]) + for output in resampler.resample(None): + push(output.to_ndarray()[0]) + if buffer: + push(np.zeros(window, dtype=np.float32)) + + duration = container.duration + if len(energies) < 3: + return None + + energies = np.asarray(energies, dtype=np.float64) + signal_floor = float(np.percentile(energies, 95)) + if signal_floor <= 0.0: + return None + noise_floor = float(np.percentile(energies, SILENCE_FLOOR_PERCENTILE)) + threshold = max(0.008, SILENCE_THRESHOLD_FACTOR * noise_floor) + silence = energies < threshold + + bridge_frames = int(round(SILENCE_BRIDGE_GAP / frame_seconds)) + if bridge_frames > 0: + bridged = silence.copy() + run_start = None + for index, is_silent in enumerate(silence): + if not is_silent and run_start is None: + run_start = index + elif is_silent and run_start is not None: + if run_start > 0 and index - run_start <= bridge_frames: + bridged[run_start:index] = True + run_start = None + silence = bridged + + boundaries = [0.0] + run_start = None + for index, is_silent in enumerate(silence): + if is_silent and run_start is None: + run_start = index + elif not is_silent and run_start is not None: + if (index - run_start) * frame_seconds >= min_silence: + end_seconds = index * frame_seconds + if end_seconds - boundaries[-1] >= min_sentence: + boundaries.append(end_seconds) + run_start = None + if run_start is not None and (len(silence) - run_start) * frame_seconds >= min_silence: + end_seconds = len(silence) * frame_seconds + if end_seconds - boundaries[-1] >= min_sentence: + boundaries.append(end_seconds) + + if duration and duration > 0: + boundaries = [boundary for boundary in boundaries if boundary < duration / 1e6 - 0.1] + return boundaries + except Exception: + return None + finally: + if container is not None: + container.close() + + +__all__ = ["detect_sentence_boundaries"] diff --git a/sentence_api/README.md b/sentence_api/README.md new file mode 100644 index 0000000..afbfc60 --- /dev/null +++ b/sentence_api/README.md @@ -0,0 +1,92 @@ +# Sentence Boundary API + +This service looks up pre-generated sentence boundaries by the SHA-256 hash of +the exact video bytes. It does not analyze media during an API request. + +## Install + +From the repository root: + +```bash +python3 -m venv .venv-sentence-api +. .venv-sentence-api/bin/activate +python -m pip install -r sentence_api/requirements.txt +``` + +## Generate The Index + +Generate boundaries with the same silence detector used by the desktop player: + +```bash +python -m sentence_api.generate_boundaries \ + "/path/to/lesson.mp4" \ + --index sentence_api/data/sentence_boundaries.json +``` + +The command calculates the SHA-256 hash, detects boundaries, converts seconds +to milliseconds, infers each `end_ms` from the next sentence start, and writes +the result atomically into the JSON index. The last sentence ends at the media +duration. + +For a large course library, run this command in an ingestion worker and store +the same document in a database or object storage instead of committing the +JSON file to the application image. + +## Run + +```bash +SENTENCE_BOUNDARIES_FILE=sentence_api/data/sentence_boundaries.json \ +python -m uvicorn sentence_api.main:app --host 0.0.0.0 --port 8000 +``` + +The interactive API documentation is available at `/docs`. + +## Request + +```http +GET /api/v1/videos/{sha256}/sentence-boundaries +``` + +Example using the demo record in the checked-in index: + +```bash +curl http://127.0.0.1:8000/api/v1/videos/468a4d064f6ec49942b45e25ab93c500d31870f978c4f28ff8b3b408852326e0/sentence-boundaries +``` + +The MP4 used during development is also indexed. Its hash is +`b6631d5cf48f37fed0ecc623563dd48b7ed660689b4d25d2ebac7fecab807ddc`, and its +generated index contains 244 boundaries. + +The response is: + +```json +{ + "video_hash": "...", + "duration_ms": 16000, + "algorithm_version": "silence-rms-v1", + "sentences": [ + { + "index": 0, + "start_ms": 0, + "end_ms": 4230, + "text": null + } + ] +} +``` + +Unknown hashes return `404`. A hash must be a 64-character hexadecimal +SHA-256 digest; malformed values return `422`. + +When testing from a physical Android phone, replace `127.0.0.1` with the +computer's LAN IP address. `127.0.0.1` on the phone refers to the phone itself. +For production, expose the API over HTTPS. + +## Android Request Flow + +The mobile app should calculate the hash from the selected `content://` URI in +streaming chunks, request the endpoint, map the returned `sentences` to +`SentenceBoundary`, and then call `controller.loadItem`. The hash must be +calculated from the exact bytes of the same video served to the player. For +HTTPS course videos, the course manifest can carry the hash and avoid hashing +the entire remote file on every device. diff --git a/sentence_api/__init__.py b/sentence_api/__init__.py new file mode 100644 index 0000000..48968a8 --- /dev/null +++ b/sentence_api/__init__.py @@ -0,0 +1,2 @@ +"""Sentence boundary lookup service.""" + diff --git a/sentence_api/data/sentence_boundaries.json b/sentence_api/data/sentence_boundaries.json new file mode 100644 index 0000000..b1e1897 --- /dev/null +++ b/sentence_api/data/sentence_boundaries.json @@ -0,0 +1,1498 @@ +{ + "videos": { + "468a4d064f6ec49942b45e25ab93c500d31870f978c4f28ff8b3b408852326e0": { + "duration_ms": 16000, + "algorithm_version": "demo-v1", + "sentences": [ + { + "index": 0, + "start_ms": 0, + "end_ms": 4000, + "text": "Demo sentence one." + }, + { + "index": 1, + "start_ms": 4000, + "end_ms": 9000, + "text": "Demo sentence two." + }, + { + "index": 2, + "start_ms": 9000, + "end_ms": 16000, + "text": "Demo sentence three." + } + ] + }, + "b6631d5cf48f37fed0ecc623563dd48b7ed660689b4d25d2ebac7fecab807ddc": { + "duration_ms": 1507764, + "algorithm_version": "silence-rms-v1", + "sentences": [ + { + "index": 0, + "start_ms": 0, + "end_ms": 5730, + "text": null + }, + { + "index": 1, + "start_ms": 5730, + "end_ms": 11430, + "text": null + }, + { + "index": 2, + "start_ms": 11430, + "end_ms": 14700, + "text": null + }, + { + "index": 3, + "start_ms": 14700, + "end_ms": 20640, + "text": null + }, + { + "index": 4, + "start_ms": 20640, + "end_ms": 22620, + "text": null + }, + { + "index": 5, + "start_ms": 22620, + "end_ms": 26340, + "text": null + }, + { + "index": 6, + "start_ms": 26340, + "end_ms": 28770, + "text": null + }, + { + "index": 7, + "start_ms": 28770, + "end_ms": 31020, + "text": null + }, + { + "index": 8, + "start_ms": 31020, + "end_ms": 37920, + "text": null + }, + { + "index": 9, + "start_ms": 37920, + "end_ms": 41850, + "text": null + }, + { + "index": 10, + "start_ms": 41850, + "end_ms": 47010, + "text": null + }, + { + "index": 11, + "start_ms": 47010, + "end_ms": 51030, + "text": null + }, + { + "index": 12, + "start_ms": 51030, + "end_ms": 53130, + "text": null + }, + { + "index": 13, + "start_ms": 53130, + "end_ms": 58320, + "text": null + }, + { + "index": 14, + "start_ms": 58320, + "end_ms": 62160, + "text": null + }, + { + "index": 15, + "start_ms": 62160, + "end_ms": 66690, + "text": null + }, + { + "index": 16, + "start_ms": 66690, + "end_ms": 70560, + "text": null + }, + { + "index": 17, + "start_ms": 70560, + "end_ms": 77610, + "text": null + }, + { + "index": 18, + "start_ms": 77610, + "end_ms": 80280, + "text": null + }, + { + "index": 19, + "start_ms": 80280, + "end_ms": 81630, + "text": null + }, + { + "index": 20, + "start_ms": 81630, + "end_ms": 84030, + "text": null + }, + { + "index": 21, + "start_ms": 84030, + "end_ms": 86400, + "text": null + }, + { + "index": 22, + "start_ms": 86400, + "end_ms": 92760, + "text": null + }, + { + "index": 23, + "start_ms": 92760, + "end_ms": 94680, + "text": null + }, + { + "index": 24, + "start_ms": 94680, + "end_ms": 98400, + "text": null + }, + { + "index": 25, + "start_ms": 98400, + "end_ms": 99660, + "text": null + }, + { + "index": 26, + "start_ms": 99660, + "end_ms": 105420, + "text": null + }, + { + "index": 27, + "start_ms": 105420, + "end_ms": 110820, + "text": null + }, + { + "index": 28, + "start_ms": 110820, + "end_ms": 112590, + "text": null + }, + { + "index": 29, + "start_ms": 112590, + "end_ms": 115230, + "text": null + }, + { + "index": 30, + "start_ms": 115230, + "end_ms": 119280, + "text": null + }, + { + "index": 31, + "start_ms": 119280, + "end_ms": 123120, + "text": null + }, + { + "index": 32, + "start_ms": 123120, + "end_ms": 126510, + "text": null + }, + { + "index": 33, + "start_ms": 126510, + "end_ms": 132840, + "text": null + }, + { + "index": 34, + "start_ms": 132840, + "end_ms": 136140, + "text": null + }, + { + "index": 35, + "start_ms": 136140, + "end_ms": 139680, + "text": null + }, + { + "index": 36, + "start_ms": 139680, + "end_ms": 141030, + "text": null + }, + { + "index": 37, + "start_ms": 141030, + "end_ms": 143850, + "text": null + }, + { + "index": 38, + "start_ms": 143850, + "end_ms": 149190, + "text": null + }, + { + "index": 39, + "start_ms": 149190, + "end_ms": 155100, + "text": null + }, + { + "index": 40, + "start_ms": 155100, + "end_ms": 156900, + "text": null + }, + { + "index": 41, + "start_ms": 156900, + "end_ms": 162150, + "text": null + }, + { + "index": 42, + "start_ms": 162150, + "end_ms": 163500, + "text": null + }, + { + "index": 43, + "start_ms": 163500, + "end_ms": 168480, + "text": null + }, + { + "index": 44, + "start_ms": 168480, + "end_ms": 170490, + "text": null + }, + { + "index": 45, + "start_ms": 170490, + "end_ms": 175830, + "text": null + }, + { + "index": 46, + "start_ms": 175830, + "end_ms": 180450, + "text": null + }, + { + "index": 47, + "start_ms": 180450, + "end_ms": 183120, + "text": null + }, + { + "index": 48, + "start_ms": 183120, + "end_ms": 184800, + "text": null + }, + { + "index": 49, + "start_ms": 184800, + "end_ms": 190470, + "text": null + }, + { + "index": 50, + "start_ms": 190470, + "end_ms": 194820, + "text": null + }, + { + "index": 51, + "start_ms": 194820, + "end_ms": 197370, + "text": null + }, + { + "index": 52, + "start_ms": 197370, + "end_ms": 198390, + "text": null + }, + { + "index": 53, + "start_ms": 198390, + "end_ms": 204960, + "text": null + }, + { + "index": 54, + "start_ms": 204960, + "end_ms": 211020, + "text": null + }, + { + "index": 55, + "start_ms": 211020, + "end_ms": 212220, + "text": null + }, + { + "index": 56, + "start_ms": 212220, + "end_ms": 217560, + "text": null + }, + { + "index": 57, + "start_ms": 217560, + "end_ms": 221910, + "text": null + }, + { + "index": 58, + "start_ms": 221910, + "end_ms": 224880, + "text": null + }, + { + "index": 59, + "start_ms": 224880, + "end_ms": 229200, + "text": null + }, + { + "index": 60, + "start_ms": 229200, + "end_ms": 231360, + "text": null + }, + { + "index": 61, + "start_ms": 231360, + "end_ms": 234840, + "text": null + }, + { + "index": 62, + "start_ms": 234840, + "end_ms": 237930, + "text": null + }, + { + "index": 63, + "start_ms": 237930, + "end_ms": 244950, + "text": null + }, + { + "index": 64, + "start_ms": 244950, + "end_ms": 249750, + "text": null + }, + { + "index": 65, + "start_ms": 249750, + "end_ms": 253980, + "text": null + }, + { + "index": 66, + "start_ms": 253980, + "end_ms": 256320, + "text": null + }, + { + "index": 67, + "start_ms": 256320, + "end_ms": 260910, + "text": null + }, + { + "index": 68, + "start_ms": 260910, + "end_ms": 266730, + "text": null + }, + { + "index": 69, + "start_ms": 266730, + "end_ms": 269520, + "text": null + }, + { + "index": 70, + "start_ms": 269520, + "end_ms": 271440, + "text": null + }, + { + "index": 71, + "start_ms": 271440, + "end_ms": 274890, + "text": null + }, + { + "index": 72, + "start_ms": 274890, + "end_ms": 278100, + "text": null + }, + { + "index": 73, + "start_ms": 278100, + "end_ms": 281550, + "text": null + }, + { + "index": 74, + "start_ms": 281550, + "end_ms": 288630, + "text": null + }, + { + "index": 75, + "start_ms": 288630, + "end_ms": 290160, + "text": null + }, + { + "index": 76, + "start_ms": 290160, + "end_ms": 292410, + "text": null + }, + { + "index": 77, + "start_ms": 292410, + "end_ms": 296850, + "text": null + }, + { + "index": 78, + "start_ms": 296850, + "end_ms": 301770, + "text": null + }, + { + "index": 79, + "start_ms": 301770, + "end_ms": 307050, + "text": null + }, + { + "index": 80, + "start_ms": 307050, + "end_ms": 310530, + "text": null + }, + { + "index": 81, + "start_ms": 310530, + "end_ms": 312750, + "text": null + }, + { + "index": 82, + "start_ms": 312750, + "end_ms": 318390, + "text": null + }, + { + "index": 83, + "start_ms": 318390, + "end_ms": 321390, + "text": null + }, + { + "index": 84, + "start_ms": 321390, + "end_ms": 324360, + "text": null + }, + { + "index": 85, + "start_ms": 324360, + "end_ms": 328950, + "text": null + }, + { + "index": 86, + "start_ms": 328950, + "end_ms": 331170, + "text": null + }, + { + "index": 87, + "start_ms": 331170, + "end_ms": 333030, + "text": null + }, + { + "index": 88, + "start_ms": 333030, + "end_ms": 338220, + "text": null + }, + { + "index": 89, + "start_ms": 338220, + "end_ms": 339180, + "text": null + }, + { + "index": 90, + "start_ms": 339180, + "end_ms": 340950, + "text": null + }, + { + "index": 91, + "start_ms": 340950, + "end_ms": 347010, + "text": null + }, + { + "index": 92, + "start_ms": 347010, + "end_ms": 348330, + "text": null + }, + { + "index": 93, + "start_ms": 348330, + "end_ms": 353430, + "text": null + }, + { + "index": 94, + "start_ms": 353430, + "end_ms": 356370, + "text": null + }, + { + "index": 95, + "start_ms": 356370, + "end_ms": 385500, + "text": null + }, + { + "index": 96, + "start_ms": 385500, + "end_ms": 392820, + "text": null + }, + { + "index": 97, + "start_ms": 392820, + "end_ms": 394830, + "text": null + }, + { + "index": 98, + "start_ms": 394830, + "end_ms": 400320, + "text": null + }, + { + "index": 99, + "start_ms": 400320, + "end_ms": 402390, + "text": null + }, + { + "index": 100, + "start_ms": 402390, + "end_ms": 405900, + "text": null + }, + { + "index": 101, + "start_ms": 405900, + "end_ms": 407730, + "text": null + }, + { + "index": 102, + "start_ms": 407730, + "end_ms": 410700, + "text": null + }, + { + "index": 103, + "start_ms": 410700, + "end_ms": 413130, + "text": null + }, + { + "index": 104, + "start_ms": 413130, + "end_ms": 417480, + "text": null + }, + { + "index": 105, + "start_ms": 417480, + "end_ms": 420510, + "text": null + }, + { + "index": 106, + "start_ms": 420510, + "end_ms": 424020, + "text": null + }, + { + "index": 107, + "start_ms": 424020, + "end_ms": 425190, + "text": null + }, + { + "index": 108, + "start_ms": 425190, + "end_ms": 432180, + "text": null + }, + { + "index": 109, + "start_ms": 432180, + "end_ms": 440550, + "text": null + }, + { + "index": 110, + "start_ms": 440550, + "end_ms": 446130, + "text": null + }, + { + "index": 111, + "start_ms": 446130, + "end_ms": 448050, + "text": null + }, + { + "index": 112, + "start_ms": 448050, + "end_ms": 450870, + "text": null + }, + { + "index": 113, + "start_ms": 450870, + "end_ms": 452430, + "text": null + }, + { + "index": 114, + "start_ms": 452430, + "end_ms": 456420, + "text": null + }, + { + "index": 115, + "start_ms": 456420, + "end_ms": 457560, + "text": null + }, + { + "index": 116, + "start_ms": 457560, + "end_ms": 459360, + "text": null + }, + { + "index": 117, + "start_ms": 459360, + "end_ms": 461940, + "text": null + }, + { + "index": 118, + "start_ms": 461940, + "end_ms": 465330, + "text": null + }, + { + "index": 119, + "start_ms": 465330, + "end_ms": 467040, + "text": null + }, + { + "index": 120, + "start_ms": 467040, + "end_ms": 471180, + "text": null + }, + { + "index": 121, + "start_ms": 471180, + "end_ms": 475140, + "text": null + }, + { + "index": 122, + "start_ms": 475140, + "end_ms": 480060, + "text": null + }, + { + "index": 123, + "start_ms": 480060, + "end_ms": 484350, + "text": null + }, + { + "index": 124, + "start_ms": 484350, + "end_ms": 487380, + "text": null + }, + { + "index": 125, + "start_ms": 487380, + "end_ms": 494160, + "text": null + }, + { + "index": 126, + "start_ms": 494160, + "end_ms": 530910, + "text": null + }, + { + "index": 127, + "start_ms": 530910, + "end_ms": 537570, + "text": null + }, + { + "index": 128, + "start_ms": 537570, + "end_ms": 540780, + "text": null + }, + { + "index": 129, + "start_ms": 540780, + "end_ms": 544800, + "text": null + }, + { + "index": 130, + "start_ms": 544800, + "end_ms": 551310, + "text": null + }, + { + "index": 131, + "start_ms": 551310, + "end_ms": 555690, + "text": null + }, + { + "index": 132, + "start_ms": 555690, + "end_ms": 557850, + "text": null + }, + { + "index": 133, + "start_ms": 557850, + "end_ms": 562710, + "text": null + }, + { + "index": 134, + "start_ms": 562710, + "end_ms": 565470, + "text": null + }, + { + "index": 135, + "start_ms": 565470, + "end_ms": 569430, + "text": null + }, + { + "index": 136, + "start_ms": 569430, + "end_ms": 572610, + "text": null + }, + { + "index": 137, + "start_ms": 572610, + "end_ms": 576120, + "text": null + }, + { + "index": 138, + "start_ms": 576120, + "end_ms": 583110, + "text": null + }, + { + "index": 139, + "start_ms": 583110, + "end_ms": 605910, + "text": null + }, + { + "index": 140, + "start_ms": 605910, + "end_ms": 608250, + "text": null + }, + { + "index": 141, + "start_ms": 608250, + "end_ms": 614730, + "text": null + }, + { + "index": 142, + "start_ms": 614730, + "end_ms": 617970, + "text": null + }, + { + "index": 143, + "start_ms": 617970, + "end_ms": 619680, + "text": null + }, + { + "index": 144, + "start_ms": 619680, + "end_ms": 624510, + "text": null + }, + { + "index": 145, + "start_ms": 624510, + "end_ms": 628290, + "text": null + }, + { + "index": 146, + "start_ms": 628290, + "end_ms": 631320, + "text": null + }, + { + "index": 147, + "start_ms": 631320, + "end_ms": 632040, + "text": null + }, + { + "index": 148, + "start_ms": 632040, + "end_ms": 675450, + "text": null + }, + { + "index": 149, + "start_ms": 675450, + "end_ms": 680520, + "text": null + }, + { + "index": 150, + "start_ms": 680520, + "end_ms": 681870, + "text": null + }, + { + "index": 151, + "start_ms": 681870, + "end_ms": 683070, + "text": null + }, + { + "index": 152, + "start_ms": 683070, + "end_ms": 684840, + "text": null + }, + { + "index": 153, + "start_ms": 684840, + "end_ms": 691920, + "text": null + }, + { + "index": 154, + "start_ms": 691920, + "end_ms": 697290, + "text": null + }, + { + "index": 155, + "start_ms": 697290, + "end_ms": 701640, + "text": null + }, + { + "index": 156, + "start_ms": 701640, + "end_ms": 705000, + "text": null + }, + { + "index": 157, + "start_ms": 705000, + "end_ms": 707310, + "text": null + }, + { + "index": 158, + "start_ms": 707310, + "end_ms": 710670, + "text": null + }, + { + "index": 159, + "start_ms": 710670, + "end_ms": 715680, + "text": null + }, + { + "index": 160, + "start_ms": 715680, + "end_ms": 724590, + "text": null + }, + { + "index": 161, + "start_ms": 724590, + "end_ms": 729690, + "text": null + }, + { + "index": 162, + "start_ms": 729690, + "end_ms": 731640, + "text": null + }, + { + "index": 163, + "start_ms": 731640, + "end_ms": 733800, + "text": null + }, + { + "index": 164, + "start_ms": 733800, + "end_ms": 735570, + "text": null + }, + { + "index": 165, + "start_ms": 735570, + "end_ms": 736470, + "text": null + }, + { + "index": 166, + "start_ms": 736470, + "end_ms": 764070, + "text": null + }, + { + "index": 167, + "start_ms": 764070, + "end_ms": 769860, + "text": null + }, + { + "index": 168, + "start_ms": 769860, + "end_ms": 786660, + "text": null + }, + { + "index": 169, + "start_ms": 786660, + "end_ms": 790590, + "text": null + }, + { + "index": 170, + "start_ms": 790590, + "end_ms": 796080, + "text": null + }, + { + "index": 171, + "start_ms": 796080, + "end_ms": 801360, + "text": null + }, + { + "index": 172, + "start_ms": 801360, + "end_ms": 806640, + "text": null + }, + { + "index": 173, + "start_ms": 806640, + "end_ms": 810090, + "text": null + }, + { + "index": 174, + "start_ms": 810090, + "end_ms": 815160, + "text": null + }, + { + "index": 175, + "start_ms": 815160, + "end_ms": 819180, + "text": null + }, + { + "index": 176, + "start_ms": 819180, + "end_ms": 825540, + "text": null + }, + { + "index": 177, + "start_ms": 825540, + "end_ms": 826740, + "text": null + }, + { + "index": 178, + "start_ms": 826740, + "end_ms": 831150, + "text": null + }, + { + "index": 179, + "start_ms": 831150, + "end_ms": 837180, + "text": null + }, + { + "index": 180, + "start_ms": 837180, + "end_ms": 845400, + "text": null + }, + { + "index": 181, + "start_ms": 845400, + "end_ms": 848190, + "text": null + }, + { + "index": 182, + "start_ms": 848190, + "end_ms": 867150, + "text": null + }, + { + "index": 183, + "start_ms": 867150, + "end_ms": 871890, + "text": null + }, + { + "index": 184, + "start_ms": 871890, + "end_ms": 875880, + "text": null + }, + { + "index": 185, + "start_ms": 875880, + "end_ms": 914190, + "text": null + }, + { + "index": 186, + "start_ms": 914190, + "end_ms": 918420, + "text": null + }, + { + "index": 187, + "start_ms": 918420, + "end_ms": 920220, + "text": null + }, + { + "index": 188, + "start_ms": 920220, + "end_ms": 925350, + "text": null + }, + { + "index": 189, + "start_ms": 925350, + "end_ms": 929040, + "text": null + }, + { + "index": 190, + "start_ms": 929040, + "end_ms": 933870, + "text": null + }, + { + "index": 191, + "start_ms": 933870, + "end_ms": 964650, + "text": null + }, + { + "index": 192, + "start_ms": 964650, + "end_ms": 966480, + "text": null + }, + { + "index": 193, + "start_ms": 966480, + "end_ms": 969990, + "text": null + }, + { + "index": 194, + "start_ms": 969990, + "end_ms": 1011090, + "text": null + }, + { + "index": 195, + "start_ms": 1011090, + "end_ms": 1033740, + "text": null + }, + { + "index": 196, + "start_ms": 1033740, + "end_ms": 1042440, + "text": null + }, + { + "index": 197, + "start_ms": 1042440, + "end_ms": 1047390, + "text": null + }, + { + "index": 198, + "start_ms": 1047390, + "end_ms": 1051500, + "text": null + }, + { + "index": 199, + "start_ms": 1051500, + "end_ms": 1056120, + "text": null + }, + { + "index": 200, + "start_ms": 1056120, + "end_ms": 1057890, + "text": null + }, + { + "index": 201, + "start_ms": 1057890, + "end_ms": 1063800, + "text": null + }, + { + "index": 202, + "start_ms": 1063800, + "end_ms": 1072950, + "text": null + }, + { + "index": 203, + "start_ms": 1072950, + "end_ms": 1077360, + "text": null + }, + { + "index": 204, + "start_ms": 1077360, + "end_ms": 1107000, + "text": null + }, + { + "index": 205, + "start_ms": 1107000, + "end_ms": 1129950, + "text": null + }, + { + "index": 206, + "start_ms": 1129950, + "end_ms": 1140480, + "text": null + }, + { + "index": 207, + "start_ms": 1140480, + "end_ms": 1165110, + "text": null + }, + { + "index": 208, + "start_ms": 1165110, + "end_ms": 1168290, + "text": null + }, + { + "index": 209, + "start_ms": 1168290, + "end_ms": 1173930, + "text": null + }, + { + "index": 210, + "start_ms": 1173930, + "end_ms": 1177590, + "text": null + }, + { + "index": 211, + "start_ms": 1177590, + "end_ms": 1189350, + "text": null + }, + { + "index": 212, + "start_ms": 1189350, + "end_ms": 1215870, + "text": null + }, + { + "index": 213, + "start_ms": 1215870, + "end_ms": 1225740, + "text": null + }, + { + "index": 214, + "start_ms": 1225740, + "end_ms": 1229430, + "text": null + }, + { + "index": 215, + "start_ms": 1229430, + "end_ms": 1260870, + "text": null + }, + { + "index": 216, + "start_ms": 1260870, + "end_ms": 1263420, + "text": null + }, + { + "index": 217, + "start_ms": 1263420, + "end_ms": 1265790, + "text": null + }, + { + "index": 218, + "start_ms": 1265790, + "end_ms": 1266810, + "text": null + }, + { + "index": 219, + "start_ms": 1266810, + "end_ms": 1275780, + "text": null + }, + { + "index": 220, + "start_ms": 1275780, + "end_ms": 1279470, + "text": null + }, + { + "index": 221, + "start_ms": 1279470, + "end_ms": 1283040, + "text": null + }, + { + "index": 222, + "start_ms": 1283040, + "end_ms": 1288110, + "text": null + }, + { + "index": 223, + "start_ms": 1288110, + "end_ms": 1317720, + "text": null + }, + { + "index": 224, + "start_ms": 1317720, + "end_ms": 1323570, + "text": null + }, + { + "index": 225, + "start_ms": 1323570, + "end_ms": 1325670, + "text": null + }, + { + "index": 226, + "start_ms": 1325670, + "end_ms": 1341600, + "text": null + }, + { + "index": 227, + "start_ms": 1341600, + "end_ms": 1351560, + "text": null + }, + { + "index": 228, + "start_ms": 1351560, + "end_ms": 1360680, + "text": null + }, + { + "index": 229, + "start_ms": 1360680, + "end_ms": 1362570, + "text": null + }, + { + "index": 230, + "start_ms": 1362570, + "end_ms": 1367070, + "text": null + }, + { + "index": 231, + "start_ms": 1367070, + "end_ms": 1449900, + "text": null + }, + { + "index": 232, + "start_ms": 1449900, + "end_ms": 1454100, + "text": null + }, + { + "index": 233, + "start_ms": 1454100, + "end_ms": 1456800, + "text": null + }, + { + "index": 234, + "start_ms": 1456800, + "end_ms": 1457610, + "text": null + }, + { + "index": 235, + "start_ms": 1457610, + "end_ms": 1461060, + "text": null + }, + { + "index": 236, + "start_ms": 1461060, + "end_ms": 1465470, + "text": null + }, + { + "index": 237, + "start_ms": 1465470, + "end_ms": 1467180, + "text": null + }, + { + "index": 238, + "start_ms": 1467180, + "end_ms": 1483860, + "text": null + }, + { + "index": 239, + "start_ms": 1483860, + "end_ms": 1487820, + "text": null + }, + { + "index": 240, + "start_ms": 1487820, + "end_ms": 1491240, + "text": null + }, + { + "index": 241, + "start_ms": 1491240, + "end_ms": 1495740, + "text": null + }, + { + "index": 242, + "start_ms": 1495740, + "end_ms": 1498770, + "text": null + }, + { + "index": 243, + "start_ms": 1498770, + "end_ms": 1507764, + "text": null + } + ] + } + } +} diff --git a/sentence_api/generate_boundaries.py b/sentence_api/generate_boundaries.py new file mode 100644 index 0000000..cbd3219 --- /dev/null +++ b/sentence_api/generate_boundaries.py @@ -0,0 +1,116 @@ +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from sentence_analysis import detect_sentence_boundaries +from sentence_api.store import normalize_video_hash + + +ALGORITHM_VERSION = "silence-rms-v1" +DEFAULT_INDEX_PATH = Path(__file__).resolve().parent / "data" / "sentence_boundaries.json" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as media_file: + for chunk in iter(lambda: media_file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def media_duration_ms(path: Path) -> int: + try: + import av + except ImportError as exc: + raise RuntimeError("PyAV is required to generate sentence boundaries.") from exc + + container = av.open(str(path)) + try: + if not container.duration or container.duration <= 0: + raise RuntimeError("Could not determine media duration.") + return int(round(container.duration / 1000)) + finally: + container.close() + + +def make_entry(path: Path, video_hash: Optional[str] = None, min_silence: float = 0.30, + min_sentence: float = 0.35) -> Tuple[Dict[str, Any], str]: + duration_ms = media_duration_ms(path) + starts = detect_sentence_boundaries( + path, + min_silence=min_silence, + min_sentence=min_sentence, + ) + if not starts: + raise RuntimeError("No sentence boundaries could be detected.") + + starts_ms = sorted({max(0, int(round(start * 1000))) for start in starts}) + starts_ms = [start for start in starts_ms if start < duration_ms] + sentences: List[Dict[str, Any]] = [] + for index, start_ms in enumerate(starts_ms): + end_ms = starts_ms[index + 1] if index + 1 < len(starts_ms) else duration_ms + if end_ms <= start_ms: + continue + sentences.append({ + "index": len(sentences), + "start_ms": start_ms, + "end_ms": end_ms, + "text": None, + }) + if not sentences: + raise RuntimeError("Detected boundaries do not form valid sentence ranges.") + + entry = { + "duration_ms": duration_ms, + "algorithm_version": ALGORITHM_VERSION, + "sentences": sentences, + } + resolved_hash = normalize_video_hash(video_hash) if video_hash else sha256_file(path) + return entry, resolved_hash + + +def update_index(index_path: Path, video_hash: str, entry: Dict[str, Any]) -> None: + if index_path.exists(): + raw = json.loads(index_path.read_text(encoding="utf-8")) + else: + raw = {"videos": {}} + videos = raw.setdefault("videos", {}) + if not isinstance(videos, dict): + raise ValueError("index JSON must contain a 'videos' object") + videos[normalize_video_hash(video_hash)] = entry + index_path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = index_path.with_suffix(index_path.suffix + ".tmp") + temporary_path.write_text( + json.dumps(raw, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + temporary_path.replace(index_path) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate API sentence boundaries for a video.") + parser.add_argument("video", type=Path) + parser.add_argument("--index", type=Path, default=DEFAULT_INDEX_PATH) + parser.add_argument("--video-hash", help="Override the calculated SHA-256 hash.") + parser.add_argument("--min-silence", type=float, default=0.30) + parser.add_argument("--min-sentence", type=float, default=0.35) + args = parser.parse_args() + + if not args.video.is_file(): + parser.error(f"Video file does not exist: {args.video}") + entry, video_hash = make_entry( + args.video, + video_hash=args.video_hash, + min_silence=args.min_silence, + min_sentence=args.min_sentence, + ) + update_index(args.index, video_hash, entry) + print(f"video_hash={video_hash}") + print(f"sentences={len(entry['sentences'])}") + print(f"index={args.index}") + + +if __name__ == "__main__": + main() diff --git a/sentence_api/main.py b/sentence_api/main.py new file mode 100644 index 0000000..f139172 --- /dev/null +++ b/sentence_api/main.py @@ -0,0 +1,62 @@ +import os +from pathlib import Path +from typing import Any, Dict + +from fastapi import Depends, FastAPI, HTTPException, Path as ApiPath + +from .models import SentenceBoundaryDocument +from .store import BoundaryStore + + +DEFAULT_INDEX_PATH = Path(__file__).resolve().parent / "data" / "sentence_boundaries.json" + + +def create_app(store: BoundaryStore = None) -> FastAPI: + application = FastAPI( + title="Oral Trainer Sentence Boundary API", + version="1.0.0", + description="Looks up pre-generated sentence boundaries by video SHA-256.", + ) + index_path = Path(os.getenv("SENTENCE_BOUNDARIES_FILE", str(DEFAULT_INDEX_PATH))) + application.state.boundary_store = store or BoundaryStore(index_path) + + def get_store() -> BoundaryStore: + return application.state.boundary_store + + @application.get("/healthz") + def healthz(boundary_store: BoundaryStore = Depends(get_store)) -> Dict[str, Any]: + return {"status": "ok", "video_count": boundary_store.count()} + + @application.get( + "/api/v1/videos/{video_hash}/sentence-boundaries", + response_model=SentenceBoundaryDocument, + ) + @application.get( + "/api/v1/sentence-boundaries/{video_hash}", + response_model=SentenceBoundaryDocument, + include_in_schema=False, + ) + def get_sentence_boundaries( + video_hash: str = ApiPath( + min_length=64, + max_length=64, + pattern=r"^[A-Fa-f0-9]{64}$", + description="SHA-256 hex digest of the exact video bytes", + ), + boundary_store: BoundaryStore = Depends(get_store), + ) -> SentenceBoundaryDocument: + document = boundary_store.get(video_hash) + if document is None: + raise HTTPException( + status_code=404, + detail={ + "code": "SENTENCE_BOUNDARIES_NOT_FOUND", + "message": "No sentence boundaries are registered for this video hash.", + }, + ) + return document + + return application + + +app = create_app() diff --git a/sentence_api/models.py b/sentence_api/models.py new file mode 100644 index 0000000..095e738 --- /dev/null +++ b/sentence_api/models.py @@ -0,0 +1,48 @@ +from typing import List, Optional + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +class SentenceBoundary(BaseModel): + model_config = ConfigDict(extra="forbid") + + index: int = Field(ge=0) + start_ms: int = Field(ge=0) + end_ms: int = Field(gt=0) + text: Optional[str] = None + + @model_validator(mode="after") + def validate_range(self): + if self.end_ms <= self.start_ms: + raise ValueError("end_ms must be greater than start_ms") + return self + + +class SentenceBoundaryDocument(BaseModel): + model_config = ConfigDict(extra="forbid") + + video_hash: str + duration_ms: int = Field(gt=0) + algorithm_version: str = Field(min_length=1) + sentences: List[SentenceBoundary] + + @field_validator("video_hash") + @classmethod + def validate_video_hash(cls, value: str) -> str: + normalized = value.strip().lower() + if len(normalized) != 64 or any(char not in "0123456789abcdef" for char in normalized): + raise ValueError("video_hash must be a SHA-256 hex digest") + return normalized + + @model_validator(mode="after") + def validate_sentences(self): + previous_end = 0 + for expected_index, sentence in enumerate(self.sentences): + if sentence.index != expected_index: + raise ValueError("sentence indexes must be contiguous and zero-based") + if sentence.start_ms < previous_end: + raise ValueError("sentences must be sorted and non-overlapping") + if sentence.end_ms > self.duration_ms: + raise ValueError("sentence end_ms cannot exceed duration_ms") + previous_end = sentence.end_ms + return self diff --git a/sentence_api/requirements.txt b/sentence_api/requirements.txt new file mode 100644 index 0000000..6f55850 --- /dev/null +++ b/sentence_api/requirements.txt @@ -0,0 +1,7 @@ +fastapi>=0.115,<1 +uvicorn[standard]>=0.30,<1 +pydantic>=2.7,<3 +pytest>=8,<9 +httpx>=0.27,<1 +av>=12.0 +numpy>=1.26 diff --git a/sentence_api/store.py b/sentence_api/store.py new file mode 100644 index 0000000..b8dc23e --- /dev/null +++ b/sentence_api/store.py @@ -0,0 +1,54 @@ +import json +from pathlib import Path +from threading import RLock +from typing import Dict, Optional + +from .models import SentenceBoundaryDocument + + +def normalize_video_hash(video_hash: str) -> str: + normalized = video_hash.strip().lower() + if len(normalized) != 64 or any(char not in "0123456789abcdef" for char in normalized): + raise ValueError("video_hash must be a SHA-256 hex digest") + return normalized + + +class BoundaryStore: + """Read-only in-memory index loaded from a JSON file.""" + + def __init__(self, index_path: Path): + self.index_path = Path(index_path) + self._lock = RLock() + self._documents: Dict[str, SentenceBoundaryDocument] = {} + self.reload() + + def reload(self) -> None: + raw = json.loads(self.index_path.read_text(encoding="utf-8")) + entries = raw.get("videos", raw) + if not isinstance(entries, dict): + raise ValueError("index JSON must contain a 'videos' object") + + documents: Dict[str, SentenceBoundaryDocument] = {} + for video_hash, entry in entries.items(): + normalized_hash = normalize_video_hash(video_hash) + if not isinstance(entry, dict): + raise ValueError(f"entry for {normalized_hash} must be an object") + document = SentenceBoundaryDocument( + video_hash=normalized_hash, + duration_ms=entry.get("duration_ms"), + algorithm_version=entry.get("algorithm_version", "unknown"), + sentences=entry.get("sentences", []), + ) + documents[normalized_hash] = document + + with self._lock: + self._documents = documents + + def get(self, video_hash: str) -> Optional[SentenceBoundaryDocument]: + normalized_hash = normalize_video_hash(video_hash) + with self._lock: + return self._documents.get(normalized_hash) + + def count(self) -> int: + with self._lock: + return len(self._documents) diff --git a/sentence_api/tests/test_api.py b/sentence_api/tests/test_api.py new file mode 100644 index 0000000..ca45491 --- /dev/null +++ b/sentence_api/tests/test_api.py @@ -0,0 +1,55 @@ +import json + +from fastapi.testclient import TestClient + +from sentence_api.main import create_app +from sentence_api.store import BoundaryStore + + +VIDEO_HASH = "a" * 64 + + +def make_client(tmp_path): + index_path = tmp_path / "boundaries.json" + index_path.write_text( + json.dumps({ + "videos": { + VIDEO_HASH: { + "duration_ms": 9000, + "algorithm_version": "test-v1", + "sentences": [ + {"index": 0, "start_ms": 0, "end_ms": 4000, "text": "One."}, + {"index": 1, "start_ms": 4000, "end_ms": 9000, "text": "Two."}, + ], + } + } + }), + encoding="utf-8", + ) + return TestClient(create_app(BoundaryStore(index_path))) + + +def test_lookup_returns_boundaries(tmp_path): + client = make_client(tmp_path) + + response = client.get(f"/api/v1/videos/{VIDEO_HASH}/sentence-boundaries") + + assert response.status_code == 200 + assert response.json()["sentences"][1]["start_ms"] == 4000 + + +def test_unknown_hash_returns_not_found(tmp_path): + client = make_client(tmp_path) + + response = client.get(f"/api/v1/videos/{'b' * 64}/sentence-boundaries") + + assert response.status_code == 404 + assert response.json()["detail"]["code"] == "SENTENCE_BOUNDARIES_NOT_FOUND" + + +def test_invalid_hash_is_rejected(tmp_path): + client = make_client(tmp_path) + + response = client.get("/api/v1/videos/not-a-sha256/sentence-boundaries") + + assert response.status_code == 422