diff --git a/ios/OralTrainer.xcodeproj/project.xcworkspace/xcuserdata/liushuming.xcuserdatad/UserInterfaceState.xcuserstate b/ios/OralTrainer.xcodeproj/project.xcworkspace/xcuserdata/liushuming.xcuserdatad/UserInterfaceState.xcuserstate index a8b4782..9f45dcd 100644 Binary files a/ios/OralTrainer.xcodeproj/project.xcworkspace/xcuserdata/liushuming.xcuserdatad/UserInterfaceState.xcuserstate and b/ios/OralTrainer.xcodeproj/project.xcworkspace/xcuserdata/liushuming.xcuserdatad/UserInterfaceState.xcuserstate differ diff --git a/ios/OralTrainer/Models/PlaybackModels.swift b/ios/OralTrainer/Models/PlaybackModels.swift index 7bb4584..f320b30 100644 --- a/ios/OralTrainer/Models/PlaybackModels.swift +++ b/ios/OralTrainer/Models/PlaybackModels.swift @@ -64,7 +64,7 @@ enum BoundaryState: Equatable { case failed(message: String) } -/// 云端视频必须先完成整文件下载；下载未完成前不允许创建可播放的 AVPlayerItem。 +/// 云端视频按需流式播放；若本地已有完整缓存文件则优先使用。 enum VideoCacheState: Equatable { case notRequired case downloading @@ -76,9 +76,9 @@ enum VideoCacheState: Equatable { case .notRequired, .ready: return nil case .downloading: - return "正在下载完整视频，下载完成后才能播放" + return "正在缓冲视频…" case .failed(let message): - return "视频缓存失败：\(message)" + return "视频加载失败：\(message)" } } } diff --git a/ios/OralTrainer/Player/PlayerController.swift b/ios/OralTrainer/Player/PlayerController.swift index 2d16d70..86bbb30 100644 --- a/ios/OralTrainer/Player/PlayerController.swift +++ b/ios/OralTrainer/Player/PlayerController.swift @@ -31,8 +31,9 @@ final class PlayerController: ObservableObject { private var swipeBackAnchorTime: TimeInterval = 0 private var speedBeforeLongPress: Float? private var transientTask: Task? - private var cacheSwapTask: Task? + private var cacheFallbackTask: Task? private var activePlaybackURL: URL? + private var didAttemptFullFileFallback = false private var sentenceEndTimeObserver: Any? private var sentenceEndGuardToken = 0 @@ -91,8 +92,9 @@ final class PlayerController: ObservableObject { @MainActor func load(_ item: TrainingMediaItem) { cancelSentenceBoundaryGuard() - cacheSwapTask?.cancel() - cacheSwapTask = nil + cacheFallbackTask?.cancel() + cacheFallbackTask = nil + didAttemptFullFileFallback = false lastSentenceIndex = nil swipeBackAnchorIndex = nil pendingSeekPositionMs = nil @@ -110,12 +112,8 @@ final class PlayerController: ObservableObject { cacheState = .ready preparePlaybackItem(with: cachedURL) } else { - // 关键约束：云端视频没有完整本地文件前，不把 remote URL 交给 AVPlayer。 - activePlaybackURL = nil - cacheState = .downloading - player.replaceCurrentItem(with: nil) - tick() - scheduleFullFileDownload(for: item) + cacheState = .notRequired + preparePlaybackItem(with: item.url) } } @@ -480,39 +478,8 @@ final class PlayerController: ObservableObject { pendingSeekPositionMs ?? currentPositionMs() } - @MainActor - private func scheduleFullFileDownload(for item: TrainingMediaItem) { - let remoteURL = item.url - cacheSwapTask = Task { [weak self] in - let cachedURL: URL - do { - cachedURL = try await StreamingVideoCache.shared.prefetch(remoteURL) - } catch is CancellationError { - return - } catch { - self?.cacheState = .failed(message: error.localizedDescription) - self?.showTransientMessage("视频缓存失败：\(error.localizedDescription)") - return - } - self?.prepareDownloadedFileIfNeeded(remoteURL: remoteURL, cachedURL: cachedURL) - } - } - - @MainActor - private func prepareDownloadedFileIfNeeded(remoteURL: URL, cachedURL: URL) { - guard !Task.isCancelled, - mediaItem?.url == remoteURL, - activePlaybackURL == nil else { - return - } - - cacheState = .ready - preparePlaybackItem(with: cachedURL) - showTransientMessage("视频已完整缓存，可以播放") - } - private func preparePlaybackItem(with playbackURL: URL) { - let newItem = AVPlayerItem(url: playbackURL) + let newItem = makePlayerItem(url: playbackURL) activePlaybackURL = playbackURL attachItemObservers(to: newItem) player.replaceCurrentItem(with: newItem) @@ -524,6 +491,19 @@ final class PlayerController: ObservableObject { tick() } + private func makePlayerItem(url: URL) -> AVPlayerItem { + guard !url.isFileURL else { + return AVPlayerItem(url: url) + } + var options: [String: Any] = [:] + let ext = url.pathExtension.lowercased() + if ext.isEmpty || ["mp4", "m4v", "mov"].contains(ext) { + options["AVURLAssetOverrideMIMETypeKey"] = "video/mp4" + } + let asset = AVURLAsset(url: url, options: options.isEmpty ? nil : options) + return AVPlayerItem(asset: asset, automaticallyLoadedAssetKeys: ["playable", "tracks", "duration"]) + } + /// AVPlayer 的 seek 是异步的；所有 seek 必须统一走这里，避免播放循环、 /// 用户手势和 seek 完成回调交错后把位置拉回旧句子。 private func performSeek( @@ -844,12 +824,57 @@ final class PlayerController: ObservableObject { if item.status == .failed { let message = item.error?.localizedDescription ?? "未知错误" self.showTransientMessage("播放失败：\(message)") + } else if item.status == .readyToPlay { + self.considerFullFileFallback(for: item) } self.tick() } } } + private func considerFullFileFallback(for item: AVPlayerItem) { + guard !didAttemptFullFileFallback, + let media = mediaItem, + !media.isLocalFile, + !(activePlaybackURL?.isFileURL ?? false) else { + return + } + let hasVisibleVideo = item.presentationSize.height >= 2 && item.presentationSize.width >= 2 + if hasVisibleVideo { + return + } + didAttemptFullFileFallback = true + cacheFallbackTask?.cancel() + cacheFallbackTask = Task { @MainActor [weak self] in + try? await Task.sleep(nanoseconds: 800_000_000) + guard let self, !Task.isCancelled else { + return + } + guard self.mediaItem?.id == media.id, + self.player.currentItem === item, + item.presentationSize.height < 2 else { + return + } + do { + let cachedURL = try await StreamingVideoCache.shared.prefetch(media.url) + guard self.mediaItem?.id == media.id else { + return + } + let shouldPlay = self.playbackIntent + let positionMs = self.pendingSeekPositionMs ?? self.currentPositionMs() + self.preparePlaybackItem(with: cachedURL) + self.cacheState = .ready + if positionMs > 0 { + self.performSeek(to: positionMs, resumePlayback: shouldPlay) + } + } catch is CancellationError { + return + } catch { + self.showTransientMessage("视频画面加载失败：\(error.localizedDescription)") + } + } + } + private func configureAudioSession() { let session = AVAudioSession.sharedInstance() try? session.setCategory(.playback, mode: .moviePlayback) diff --git a/ios/OralTrainer/Player/PlayerSurfaceView.swift b/ios/OralTrainer/Player/PlayerSurfaceView.swift index be17a85..37408d2 100644 --- a/ios/OralTrainer/Player/PlayerSurfaceView.swift +++ b/ios/OralTrainer/Player/PlayerSurfaceView.swift @@ -13,15 +13,20 @@ struct PlayerSurfaceView: UIViewRepresentable { func makeUIView(context: Context) -> VideoSurfaceView { let view = VideoSurfaceView() - view.playerLayer.player = controller.player view.controller = controller view.onVerticalSwipe = onVerticalSwipe + view.attach(player: controller.player) + view.setContentHuggingPriority(.defaultLow, for: .vertical) + view.setContentHuggingPriority(.defaultLow, for: .horizontal) + view.setContentCompressionResistancePriority(.defaultLow, for: .vertical) + view.setContentCompressionResistancePriority(.defaultLow, for: .horizontal) return view } func updateUIView(_ uiView: VideoSurfaceView, context: Context) { - uiView.playerLayer.player = controller.player + uiView.controller = controller uiView.onVerticalSwipe = onVerticalSwipe + uiView.attach(player: controller.player) } } @@ -29,13 +34,9 @@ final class VideoSurfaceView: UIView { weak var controller: PlayerController? var onVerticalSwipe: ((Bool) -> Void)? - override class var layerClass: AnyClass { - AVPlayerLayer.self - } + private let videoLayer = AVPlayerLayer() - var playerLayer: AVPlayerLayer { - layer as! AVPlayerLayer - } + var playerLayer: AVPlayerLayer { videoLayer } private let minSwipeDistance: CGFloat = 48 private let minSwipeVelocity: CGFloat = 160 @@ -55,7 +56,11 @@ final class VideoSurfaceView: UIView { override init(frame: CGRect) { super.init(frame: frame) backgroundColor = .black - playerLayer.videoGravity = .resizeAspect + isOpaque = true + clipsToBounds = true + videoLayer.videoGravity = .resizeAspect + videoLayer.backgroundColor = UIColor.black.cgColor + layer.addSublayer(videoLayer) tapRecognizer = UITapGestureRecognizer(target: self, action: #selector(handleTap)) panRecognizer = UIPanGestureRecognizer(target: self, action: #selector(handlePan)) @@ -82,6 +87,25 @@ final class VideoSurfaceView: UIView { fatalError("init(coder:) has not been implemented") } + override func layoutSubviews() { + super.layoutSubviews() + CATransaction.begin() + CATransaction.setDisableActions(true) + videoLayer.frame = bounds + CATransaction.commit() + } + + func attach(player: AVPlayer) { + if videoLayer.player !== player { + videoLayer.player = player + } + if videoLayer.superlayer !== layer { + layer.addSublayer(videoLayer) + } + videoLayer.frame = bounds + videoLayer.isHidden = false + } + @objc private func handleTap() { controller?.handleGesture(.singleTap) } @@ -96,7 +120,6 @@ final class VideoSurfaceView: UIView { case .began: resetSwipeTracking() case .changed: - // 在滑动达到阈值时立即派发，并且同一个手势只允许派发一次。 evaluateSwipe(dx: dx, dy: dy, velocity: velocity) case .ended, .cancelled, .failed: if !didDispatchSwipe { @@ -112,7 +135,6 @@ final class VideoSurfaceView: UIView { guard !didDispatchSwipe else { return } if swipeAxis == nil { - // 只有方向足够明确时锁定轴向，避免轻微斜滑被误判为上下滑。 if abs(dx) > abs(dy) * 1.2 { swipeAxis = .horizontal } else if abs(dy) > abs(dx) * 1.2 { diff --git a/ios/OralTrainer/Services/StreamingVideoCache.swift b/ios/OralTrainer/Services/StreamingVideoCache.swift index 8b7ff75..a941305 100644 --- a/ios/OralTrainer/Services/StreamingVideoCache.swift +++ b/ios/OralTrainer/Services/StreamingVideoCache.swift @@ -1,11 +1,10 @@ import CryptoKit import Foundation -/// iOS 端的持久化整文件视频缓存。 +/// iOS 端的可选整文件视频缓存。 /// -/// AVFoundation 没有公开可持久化的媒体字节缓存 API。这里先把远程视频完整 -/// 下载成原子本地文件；只有文件 -/// 完整落盘后才返回给播放器，避免用部分响应或播放列表创建可播放的 AVPlayerItem。 +/// 播放默认走远程 URL 按需加载（AVPlayer HTTP Range）。若本地已有完整缓存文件， +/// 播放器会优先使用该文件，避免重复拉流。 @MainActor final class StreamingVideoCache { static let shared = StreamingVideoCache() diff --git a/sentence_api/main.py b/sentence_api/main.py index a97d67d..9fe5ea0 100644 --- a/sentence_api/main.py +++ b/sentence_api/main.py @@ -481,9 +481,13 @@ def create_app( media_path = service_settings.videos_dir / row["stored_filename"] if not media_path.is_file(): raise HTTPException(status_code=404, detail="Stored video file is missing.") + media_type = row.get("content_type") or "video/mp4" + if not str(media_type).lower().startswith("video/"): + guessed = mimetypes.guess_type(str(row.get("stored_filename") or ""))[0] + media_type = guessed if guessed and guessed.lower().startswith("video/") else "video/mp4" return FileResponse( media_path, - media_type=row["content_type"], + media_type=media_type, headers={"Accept-Ranges": "bytes", "Cache-Control": "public, max-age=3600"}, ) diff --git a/sentence_api/processing.py b/sentence_api/processing.py index 652e06e..4082597 100644 --- a/sentence_api/processing.py +++ b/sentence_api/processing.py @@ -1,10 +1,11 @@ +import json import logging import shutil import subprocess import tempfile import uuid from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple from .audio_metrics import ( AudioAnalysisError, @@ -85,6 +86,126 @@ def _ensure_webm_cues_front(filepath: Path) -> None: tmp.unlink(missing_ok=True) +_IOS_VIDEO_CODECS = {"h264", "hevc"} +_IOS_AUDIO_CODECS = {"aac", "mp3", "ac3", "eac3", "alac"} + + +def _probe_stream_codecs(filepath: Path) -> Tuple[Optional[str], Optional[str]]: + ffprobe = shutil.which("ffprobe") + if ffprobe is None: + return None, None + completed = subprocess.run( + [ + ffprobe, + "-hide_banner", + "-loglevel", + "error", + "-show_streams", + "-of", + "json", + str(filepath), + ], + capture_output=True, + text=True, + timeout=60, + ) + if completed.returncode != 0: + return None, None + try: + payload = json.loads(completed.stdout or "{}") + except json.JSONDecodeError: + return None, None + video_codec = None + audio_codec = None + for stream in payload.get("streams") or []: + codec_type = stream.get("codec_type") + codec_name = stream.get("codec_name") + if codec_type == "video" and video_codec is None and codec_name not in {"mjpeg", "png"}: + video_codec = codec_name + elif codec_type == "audio" and audio_codec is None: + audio_codec = codec_name + return video_codec, audio_codec + + +def _needs_ios_compatible_transcode(video_codec: Optional[str], audio_codec: Optional[str]) -> bool: + if video_codec is None and audio_codec is None: + return False + video_ok = video_codec is None or video_codec in _IOS_VIDEO_CODECS + audio_ok = audio_codec is None or audio_codec in _IOS_AUDIO_CODECS + return not (video_ok and audio_ok) + + +def _ensure_ios_compatible_media(filepath: Path) -> None: + """Transcode AV1/VP9/Opus etc. to H.264/AAC so iOS AVPlayer can show video.""" + video_codec, audio_codec = _probe_stream_codecs(filepath) + if not _needs_ios_compatible_transcode(video_codec, audio_codec): + return + ffmpeg = shutil.which("ffmpeg") + if ffmpeg is None: + logger.warning( + "ffmpeg not found; skipping iOS-compatible transcode for %s (%s/%s)", + filepath.name, + video_codec, + audio_codec, + ) + return + tmp = filepath.with_suffix(".ios_tmp.mp4") + command = [ + ffmpeg, + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + str(filepath), + "-map", + "0:v:0", + "-map", + "0:a:0?", + "-vf", + "scale='min(1920,iw)':-2", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-b:a", + "128k", + "-ac", + "2", + "-movflags", + "+faststart", + str(tmp), + ] + logger.info( + "Transcoding %s for iOS playback (%s/%s -> h264/aac)", + filepath.name, + video_codec, + audio_codec, + ) + try: + completed = subprocess.run(command, capture_output=True, text=True, timeout=7200) + if completed.returncode == 0 and tmp.is_file() and tmp.stat().st_size > 0: + tmp.replace(filepath) + logger.info("iOS-compatible transcode finished: %s", filepath.name) + return + logger.warning( + "iOS-compatible transcode failed for %s: rc=%d stderr=%s", + filepath.name, + completed.returncode, + (completed.stderr or "")[-400:], + ) + except (subprocess.TimeoutExpired, OSError) as exc: + logger.warning("iOS-compatible transcode error for %s: %s", filepath.name, exc) + finally: + tmp.unlink(missing_ok=True) + + class VideoProcessor: def __init__( self, @@ -110,6 +231,7 @@ class VideoProcessor: work_path: Optional[Path] = None try: _ensure_webm_cues_front(media_path) + _ensure_ios_compatible_media(media_path) if self.transcriber.available: work_path = self.settings.work_dir / f"{video_hash}-{uuid.uuid4().hex}.wav" extract_audio(media_path, work_path)