diff --git a/CHANGELOG.md b/CHANGELOG.md index cadfbc99f..5e1d12223 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ the public-API contract. ### Added +- **Frame-accurate downscaled stills through `FrameExtractor.preview(at:maxWidth:)`.** + Previews decode forward to the requested frame while retaining full reconstruction and high-quality + scaling. They yield to starved playback and keep a small, separate cache. The CLI exposes the same + behavior with `aetherctl extract --preview --width `. + - **Remote MPEG-DASH on-demand manifests play (AE#859, thanks @qoli).** A static, single-Period MPD whose representations are single files addressed by `SegmentBase` failed at open with `Invalid data found when processing input`: the FFmpeg build has no DASH diff --git a/Sources/AetherEngine/FrameExtractor/FrameCache.swift b/Sources/AetherEngine/FrameExtractor/FrameCache.swift index 8a44738b7..745bcb37e 100644 --- a/Sources/AetherEngine/FrameExtractor/FrameCache.swift +++ b/Sources/AetherEngine/FrameExtractor/FrameCache.swift @@ -1,12 +1,20 @@ import Foundation import CoreGraphics -/// Size-bounded LRU cache of rendered frames keyed by `(mode, position bucket)`. -/// Per-mode count limit caps the cache at `thumbnailLimit + snapshotLimit` CGImages. -/// Key omits target size: one extractor serves one fixed-size surface, so a repeat -/// position at a different maxWidth/maxSize returns the first-rendered image. +/// Count-bounded LRU cache of rendered frames with mode-specific position keys. +/// Per-mode count limit caps the cache at `thumbnailLimit + previewLimit + snapshotLimit` CGImages. +/// Preview keys include exact requested time and width. Thumbnail and snapshot stores +/// retain their existing position-bucket keys. /// Not thread-safe; FrameExtractor owns the only instance and touches it actor-isolated. final class FrameCache { + private struct PreviewKey: Hashable { + let seconds: Double + let width: Int + } + private let previewLimit: Int + private var previewOrder: [PreviewKey] = [] + private var previewStore: [PreviewKey: CGImage] = [:] + private let thumbnailLimit: Int private let snapshotLimit: Int private let thumbnailBucketSeconds: Double @@ -21,7 +29,9 @@ final class FrameCache { private var thumbnailStore: [Int: CGImage] = [:] private var snapshotStore: [Int: CGImage] = [:] - init(thumbnailLimit: Int, snapshotLimit: Int, thumbnailBucketSeconds: Double) { + init(thumbnailLimit: Int, snapshotLimit: Int, thumbnailBucketSeconds: Double, previewLimit: Int = 2) { + precondition(previewLimit >= 0, "previewLimit must not be negative") + self.previewLimit = previewLimit precondition(thumbnailBucketSeconds > 0, "thumbnailBucketSeconds must be positive") self.thumbnailLimit = thumbnailLimit self.snapshotLimit = snapshotLimit @@ -38,9 +48,14 @@ final class FrameCache { return rounded < 0x1p62 ? Int(rounded) : 1 << 62 } - func get(mode: FrameMode, seconds: Double) -> CGImage? { + func get(mode: FrameMode, seconds: Double, targetWidth: Int = 0) -> CGImage? { let key = bucket(seconds, mode: mode) switch mode { + case .preview: + let previewKey = PreviewKey(seconds: seconds, width: targetWidth) + guard let img = previewStore[previewKey] else { return nil } + touch(&previewOrder, previewKey) + return img case .thumbnail: guard let img = thumbnailStore[key] else { return nil } touch(&thumbnailOrder, key) @@ -52,9 +67,14 @@ final class FrameCache { } } - func set(_ image: CGImage, mode: FrameMode, seconds: Double) { + func set(_ image: CGImage, mode: FrameMode, seconds: Double, targetWidth: Int = 0) { let key = bucket(seconds, mode: mode) switch mode { + case .preview: + let previewKey = PreviewKey(seconds: seconds, width: targetWidth) + previewStore[previewKey] = image + touch(&previewOrder, previewKey) + evict(&previewOrder, &previewStore, limit: previewLimit) case .thumbnail: thumbnailStore[key] = image touch(&thumbnailOrder, key) @@ -67,20 +87,22 @@ final class FrameCache { } func clear() { + previewOrder.removeAll() + previewStore.removeAll() thumbnailOrder.removeAll() snapshotOrder.removeAll() thumbnailStore.removeAll() snapshotStore.removeAll() } - private func touch(_ order: inout [Int], _ key: Int) { + private func touch(_ order: inout [Key], _ key: Key) { if let idx = order.firstIndex(of: key) { order.remove(at: idx) } order.insert(key, at: 0) } - private func evict(_ order: inout [Int], _ store: inout [Int: CGImage], limit: Int) { + private func evict(_ order: inout [Key], _ store: inout [Key: CGImage], limit: Int) { while store.count > limit, let lru = order.popLast() { store[lru] = nil } diff --git a/Sources/AetherEngine/FrameExtractor/FrameDecodeContext.swift b/Sources/AetherEngine/FrameExtractor/FrameDecodeContext.swift index 4d0201379..f14928b91 100644 --- a/Sources/AetherEngine/FrameExtractor/FrameDecodeContext.swift +++ b/Sources/AetherEngine/FrameExtractor/FrameDecodeContext.swift @@ -357,13 +357,6 @@ final class FrameDecodeContext: @unchecked Sendable { activeProcessorCount: ProcessInfo.processInfo.activeProcessorCount)) ctx.pointee.thread_type = FF_THREAD_FRAME | FF_THREAD_SLICE - // Still extraction needs no deblock or full-rate quality: skip loop filter - // and enable the fast/inaccurate path to cut per-frame CPU on big HEVC/AV1 keyframes. - // AV_CODEC_FLAG2_FAST is a C #define (1 << 0) that does not bridge to Swift. - let AV_CODEC_FLAG2_FAST_VALUE: Int32 = 1 << 0 - ctx.pointee.skip_loop_filter = AVDISCARD_ALL - ctx.pointee.flags2 |= AV_CODEC_FLAG2_FAST_VALUE - var opts: OpaquePointer? if !useHardware { // Belt-and-suspenders with the software get_format callback above. @@ -437,6 +430,7 @@ final class FrameDecodeContext: @unchecked Sendable { /// Decode one frame at/after `seconds`. /// /// - thumbnail: first frame after the seek (keyframe), downscaled to `targetWidth`. + /// - preview: decode forward until pts >= seconds, downscaled to `targetWidth`. /// - snapshot: decode forward until pts >= seconds, return at `maxSize` (aspect-preserved) or native. /// /// `isCancelled` is polled between packets and before conversion so a superseded scrub @@ -462,6 +456,16 @@ final class FrameDecodeContext: @unchecked Sendable { // (the green / magenta still of #103): no still at all, and nothing read or decoded for it. guard Self.yieldsStill(dolbyVisionNoBaseLayer: isDolbyVisionNoBaseLayer) else { return nil } + // Full reconstruction matters for previews and snapshots; only coarse keyframe + // thumbnails opt into skipped loop filtering and the codec's inaccurate fast path. + let codecFastFlag: Int32 = 1 << 0 // AV_CODEC_FLAG2_FAST does not bridge to Swift. + ctx.pointee.skip_loop_filter = mode.isFrameAccurate ? AVDISCARD_DEFAULT : AVDISCARD_ALL + if mode.isFrameAccurate { + ctx.pointee.flags2 &= ~codecFastFlag + } else { + ctx.pointee.flags2 |= codecFastFlag + } + // Bound this decode's HTTP reads so a stalled remote source can't park the // serial decode queue and freeze the scrub preview (issue #27). No-op for // file:// / custom sources. Disarmed on every exit path. @@ -557,7 +561,7 @@ final class FrameDecodeContext: @unchecked Sendable { av_frame_free(&transferredFrame) } } - let width = mode == .thumbnail + let width = mode.usesTargetWidth ? targetWidth : Self.clampedWidth(frame: f, maxSize: maxSize) #if DolbyVisionStillReshaping @@ -577,7 +581,7 @@ final class FrameDecodeContext: @unchecked Sendable { } // tone-map failed: fall through to sws path (degraded but non-nil) } - return convertToCGImage(frame: f, targetWidth: width) + return convertToCGImage(frame: f, targetWidth: width, highQuality: mode.isFrameAccurate) } // Once the demuxer hits EOF, flush the decoder (NULL packet) and keep draining: the @@ -637,9 +641,7 @@ final class FrameDecodeContext: @unchecked Sendable { targetPTS = f.pointee.pts + ticks offsetTicks = nil } - if mode == .snapshot, - f.pointee.pts != Int64.min, - f.pointee.pts < targetPTS { + if !mode.accepts(framePTS: f.pointee.pts, targetPTS: targetPTS) { continue } @@ -770,7 +772,7 @@ final class FrameDecodeContext: @unchecked Sendable { /// sws_scale the frame to RGBA at `targetWidth` (height from source aspect) into an /// owned-buffer CGImage. Mirrors the sws tuple-pointer dance in SoftwareVideoDecoder. - private func convertToCGImage(frame: UnsafeMutablePointer, targetWidth: Int) -> CGImage? { + private func convertToCGImage(frame: UnsafeMutablePointer, targetWidth: Int, highQuality: Bool) -> CGImage? { let srcW = Int(frame.pointee.width) let srcH = Int(frame.pointee.height) guard srcW > 0, srcH > 0, targetWidth > 0 else { return nil } @@ -787,7 +789,7 @@ final class FrameDecodeContext: @unchecked Sendable { swsContext, Int32(srcW), Int32(srcH), srcFmt, Int32(dstW), Int32(dstH), AV_PIX_FMT_RGBA, - Int32(SWS_BILINEAR.rawValue), nil, nil, nil + Int32((highQuality ? SWS_BICUBIC : SWS_BILINEAR).rawValue), nil, nil, nil ) guard swsContext != nil else { return nil } // Without this sws converts every picture with its BT.601 default, which put a 1080p diff --git a/Sources/AetherEngine/FrameExtractor/FrameExtractor.swift b/Sources/AetherEngine/FrameExtractor/FrameExtractor.swift index 29b3ca755..c06d460a0 100644 --- a/Sources/AetherEngine/FrameExtractor/FrameExtractor.swift +++ b/Sources/AetherEngine/FrameExtractor/FrameExtractor.swift @@ -13,15 +13,15 @@ public struct BoundedStillFrame: @unchecked Sendable { } /// Produces still images from a media URL via an isolated FFmpeg decode context, -/// separate from playback. Two modes share one decode core: `snapshot` (frame-accurate, -/// full-res) and `thumbnail` (keyframe-snapped, low-res, fast). Lazy: context opens on +/// separate from playback. Three modes share one decode core: `snapshot` (frame-accurate, +/// full-res), `preview` (frame-accurate, downscaled) and `thumbnail` (keyframe-snapped, fast). Lazy: context opens on /// first use; blocking FFmpeg work runs on a dedicated serial queue, not the cooperative pool. /// Create one per URL; for the playing item prefer `AetherEngine.makeFrameExtractor()`. public actor FrameExtractor { private let context: FrameDecodeContext private let cache: FrameCache private let decodeQueue: DispatchQueue - /// #93 startup: session-coupled extractors yield elective thumbnail decodes while the + /// #93 startup: session-coupled extractors yield elective thumbnail/preview decodes while the /// playback pipeline is starved (see `shouldYield`). nil = never yield (standalone use). private let yieldWhile: (@Sendable () -> Bool)? @@ -137,7 +137,7 @@ public actor FrameExtractor { yieldWhile: nil, residentIdentity: identity) } - /// Yield decision for elective (thumbnail) extraction. The disposable decode's demuxer pulls + /// Yield decision for elective thumbnail/preview extraction. The disposable decode's demuxer pulls /// megabytes over the same link the segment producer needs; during startup and recovery that /// contention tipped the first segment past CoreMedia's ~4 s media timeout and killed the /// AVPlayer loader (-15628, #93 startup). Yield while a producer restart is in flight or the @@ -159,6 +159,12 @@ public actor FrameExtractor { await produce(at: seconds, mode: .thumbnail, targetWidth: maxWidth, maxSize: nil) } + /// Produce a frame-accurate, downscaled still for interactive scrubbing. + /// Like thumbnails, uncached previews yield while the playback pipeline is starved. + public func preview(at seconds: Double, maxWidth: Int = 320) async -> CGImage? { + await produce(at: seconds, mode: .preview, targetWidth: maxWidth, maxSize: nil) + } + public func snapshot(at seconds: Double, maxSize: CGSize? = nil) async -> CGImage? { // targetWidth is inert for snapshot; FrameDecodeContext.clampedWidth governs size. await produce(at: seconds, mode: .snapshot, targetWidth: 0, maxSize: maxSize) @@ -265,7 +271,7 @@ public actor FrameExtractor { /// Permanently tear down the decode context and clear the cache. Awaits teardown on /// the decode queue, so on return the FFmpeg demuxer/codec/sws are fully released. - /// After shutdown() the extractor is dead: thumbnail/snapshot/prewarm return nil/no-op + /// After shutdown() the extractor is dead: thumbnail/preview/snapshot/prewarm return nil/no-op /// and do NOT reopen; create a new FrameExtractor to extract again. public func shutdown() async { idleTask?.cancel() @@ -284,14 +290,14 @@ public actor FrameExtractor { private func produce(at seconds: Double, mode: FrameMode, targetWidth: Int, maxSize: CGSize?) async -> CGImage? { // Audit BIT-105: a 0/0 scrub fraction or `CMTime.indefinite.seconds` is a host bug, not a position. guard !isShutDown, seconds.isFinite, seconds >= 0 else { return nil } - if let hit = cache.get(mode: mode, seconds: seconds) { + if let hit = cache.get(mode: mode, seconds: seconds, targetWidth: targetWidth) { scheduleIdleClose() return hit } - // Elective thumbnails yield to a starved playback pipeline (snapshots are deliberate + // Elective thumbnails/previews yield to a starved playback pipeline (snapshots are deliberate // one-shot user actions and stay ungated). Checked after the cache: hits are free. - if mode == .thumbnail, yieldWhile?() == true { - EngineLog.emit("[FrameExtractor] thumbnail yielded: playback pipeline starved", + if mode.isElective, yieldWhile?() == true { + EngineLog.emit("[FrameExtractor] \(mode) yielded: playback pipeline starved", category: .swPlayback, level: .verbose) return nil } @@ -331,7 +337,7 @@ public actor FrameExtractor { return FrameResult(image: image) } if let image = result.image, !token.isCancelled { - cache.set(image, mode: mode, seconds: seconds) + cache.set(image, mode: mode, seconds: seconds, targetWidth: targetWidth) } scheduleIdleClose() return result.image diff --git a/Sources/AetherEngine/FrameExtractor/FrameTypes.swift b/Sources/AetherEngine/FrameExtractor/FrameTypes.swift index d39e573d9..2cffb6fd3 100644 --- a/Sources/AetherEngine/FrameExtractor/FrameTypes.swift +++ b/Sources/AetherEngine/FrameExtractor/FrameTypes.swift @@ -26,9 +26,19 @@ enum ScrubSegmentTime { /// Which extraction mode produced (or is requested for) a frame. /// -/// - `thumbnail`: nearest keyframe, no forward decode, downscaled. Cheap; scrub/Recents. +/// - `thumbnail`: nearest keyframe, no forward decode, downscaled. Cheap; posters/Recents. +/// - `preview`: frame-accurate forward decode, downscaled. Interactive scrub previews. /// - `snapshot`: frame-accurate (decode forward to exact PTS), full/requested res; stills. public enum FrameMode: Sendable, Hashable { case thumbnail + case preview case snapshot + + var isFrameAccurate: Bool { self != .thumbnail } + var usesTargetWidth: Bool { self != .snapshot } + var isElective: Bool { self != .snapshot } + + func accepts(framePTS: Int64, targetPTS: Int64) -> Bool { + !isFrameAccurate || framePTS == Int64.min || framePTS >= targetPTS + } } diff --git a/Sources/aetherctl/Extract.swift b/Sources/aetherctl/Extract.swift index a9bff6c1a..a801de0cf 100644 --- a/Sources/aetherctl/Extract.swift +++ b/Sources/aetherctl/Extract.swift @@ -43,6 +43,7 @@ func runExtract(url: URL, at seconds: Double, mode: FrameMode, loops: Int, maxWi let image: CGImage? = runBlocking { switch mode { case .thumbnail: return await extractor.thumbnail(at: pos, maxWidth: maxWidth) + case .preview: return await extractor.preview(at: pos, maxWidth: maxWidth) case .snapshot: return await extractor.snapshot(at: pos, maxSize: nil) } } diff --git a/Sources/aetherctl/main.swift b/Sources/aetherctl/main.swift index 0b47e154e..d3b9f7c84 100644 --- a/Sources/aetherctl/main.swift +++ b/Sources/aetherctl/main.swift @@ -113,7 +113,7 @@ func printUsage() { aetherctl dovitest [--out PATH.hevc] aetherctl feltest [--frames N] (DualLayerDolbyVision trait) aetherctl dvsurvey [--frames N] [--list K] - aetherctl extract [--at ] [--snapshot] [--width ] [--loops ] + aetherctl extract [--at ] [--preview|--snapshot] [--width ] [--loops ] aetherctl audio [--seconds N] aetherctl audiotap [--duration S] [--out PATH.wav] [--remote | --software] (#95: decode the loopback audio track to mono 48k WAV, print continuity stats; @@ -206,9 +206,10 @@ func printUsage() { Flags (extract only): --at Seek position in seconds (default 60.0). + --preview Frame-accurate decode downscaled to --width. --snapshot Frame-accurate decode at full resolution instead of nearest-keyframe thumbnail. - --width Max output width for thumbnail mode (default 320). + --width Max output width for thumbnail/preview mode (default 320). --loops Repeat extraction N times, cycling through 8 positions. Useful with `leaks --atExit`. @@ -277,8 +278,8 @@ func printUsage() { extract Extract a still frame from a source. Thumbnail mode (default) seeks to the nearest keyframe and downscales - to --width. Snapshot mode (--snapshot) decodes - frame-accurately at full resolution. Use --loops N + to --width. Preview mode (--preview) decodes frame-accurately + at --width; snapshot mode (--snapshot) uses full resolution. Use --loops N with `leaks --atExit` to detect memory leaks. Writes the first frame to aetherctl-extract-.png in a private per-user temporary directory, printed. @@ -1076,6 +1077,7 @@ if ["probe", "serve", "validate", "swdecode", "extract", "audio", "customio"].co let atSeconds = takeDoubleFlag("--at", from: &rest) ?? 60.0 let extractLoops = takeIntFlag("--loops", from: &rest) ?? 1 let extractWidth = takeIntFlag("--width", from: &rest) ?? 320 + let previewMode = takeFlag("--preview", from: &rest) let snapshotMode = takeFlag("--snapshot", from: &rest) // The opt-in detail passes of `AetherEngine.probe(url:detecting:)`, so both are observable from the CLI // instead of only through a host. Each costs reads past find_stream_info; the bare `probe` does neither. @@ -1196,10 +1198,14 @@ if ["probe", "serve", "validate", "swdecode", "extract", "audio", "customio"].co case "swdecode": exit(runSWDecode(url: url, maxPackets: framesOverride ?? 100)) case "extract": + if previewMode, snapshotMode { + print("ERROR: extract accepts only one of --preview or --snapshot") + exit(64) + } exit(runExtract( url: url, at: atSeconds, - mode: snapshotMode ? .snapshot : .thumbnail, + mode: previewMode ? .preview : (snapshotMode ? .snapshot : .thumbnail), loops: extractLoops, maxWidth: extractWidth )) diff --git a/Tests/AetherEngineTests/FrameCacheTests.swift b/Tests/AetherEngineTests/FrameCacheTests.swift index 263f795b6..2022fec8e 100644 --- a/Tests/AetherEngineTests/FrameCacheTests.swift +++ b/Tests/AetherEngineTests/FrameCacheTests.swift @@ -58,6 +58,24 @@ struct FrameCacheTests { #expect(cache.get(mode: .snapshot, seconds: 10.88) === b) } + @Test("Preview positions and widths are independent and the preview cache stays bounded") + func previewCache() { + let cache = FrameCache(thumbnailLimit: 2, snapshotLimit: 2, thumbnailBucketSeconds: 1.0) + let small = dummyImage(), large = dummyImage(), next = dummyImage() + cache.set(small, mode: .preview, seconds: 1, targetWidth: 160) + cache.set(large, mode: .preview, seconds: 1, targetWidth: 320) + #expect(cache.get(mode: .preview, seconds: 1, targetWidth: 160) === small) + #expect(cache.get(mode: .preview, seconds: 1.0001, targetWidth: 160) == nil) + #expect(cache.get(mode: .thumbnail, seconds: 1) == nil) + #expect(cache.get(mode: .snapshot, seconds: 1) == nil) + cache.set(next, mode: .preview, seconds: 1.0001, targetWidth: 160) + #expect(cache.get(mode: .preview, seconds: 1, targetWidth: 320) == nil) + #expect(cache.get(mode: .preview, seconds: 1, targetWidth: 160) === small) + #expect(cache.get(mode: .preview, seconds: 1.0001, targetWidth: 160) === next) + cache.clear() + #expect(cache.get(mode: .preview, seconds: 1, targetWidth: 160) == nil) + } + @Test("clear empties the cache") func clear() { let cache = FrameCache(thumbnailLimit: 4, snapshotLimit: 2, thumbnailBucketSeconds: 1.0) diff --git a/Tests/AetherEngineTests/FrameDecodeContextHardwareTests.swift b/Tests/AetherEngineTests/FrameDecodeContextHardwareTests.swift index 980f27f3c..6add2b095 100644 --- a/Tests/AetherEngineTests/FrameDecodeContextHardwareTests.swift +++ b/Tests/AetherEngineTests/FrameDecodeContextHardwareTests.swift @@ -71,6 +71,73 @@ struct FrameDecodeContextHardwareTests { #expect(!FrameDecodeContext.stillUsesHardware(allows: false, disabled: false, dvNoBaseLayer: false)) } + @Test("accurate previews decode past the keyframe and match downscaled snapshots") + func accuratePreview() throws { + let data = try #require(Data(base64Encoded: Self.previewFixtureBase64, options: .ignoreUnknownCharacters)) + let context = FrameDecodeContext(reader: DataIOReader(data: data), formatHint: "mp4", allowsHardwareDecode: false) + defer { context.close() } + try context.ensureOpen() + var actual: Double? + let thumbnail = context.decodeFrame(at: 0.75, mode: .thumbnail, targetWidth: 16, maxSize: nil, + isCancelled: { false }, reportDecodedTime: { actual = $0 }) + #expect(thumbnail != nil) + #expect(actual == 0) + let preview = try #require(context.decodeFrame(at: 0.75, mode: .preview, targetWidth: 16, maxSize: nil, + isCancelled: { false }, reportDecodedTime: { actual = $0 })) + #expect(actual == 0.75) + #expect(preview.width == 16 && preview.height == 16) + let snapshot = try #require(context.decodeFrame(at: 0.75, mode: .snapshot, targetWidth: 0, + maxSize: CGSize(width: 16, height: 16), isCancelled: { false })) + let previewData = try #require(preview.dataProvider?.data) + let snapshotData = try #require(snapshot.dataProvider?.data) + #expect(previewData as Data == snapshotData as Data) + #expect(context.decodeFrame(at: 0.75, mode: .preview, targetWidth: 16, maxSize: nil, + isCancelled: { true }) == nil) + } + + @Test("preview API respects dimensions, starvation and shutdown") + func previewLifecycle() async throws { + let data = try #require(Data(base64Encoded: Self.previewFixtureBase64, options: .ignoreUnknownCharacters)) + let extractor = FrameExtractor(reader: DataIOReader(data: data), formatHint: "mp4") + let small = try #require(await extractor.preview(at: 0.75, maxWidth: 8)) + let large = try #require(await extractor.preview(at: 0.75, maxWidth: 16)) + #expect(small.width == 8 && large.width == 16) + #expect(await extractor.preview(at: 0.75, maxWidth: 16) === large) + await extractor.shutdown() + #expect(await extractor.preview(at: 0.75, maxWidth: 16) == nil) + + let starved = FrameExtractor(reader: DataIOReader(data: data), formatHint: "mp4", yieldWhile: { true }) + #expect(await starved.preview(at: 0.75, maxWidth: 16) == nil) + #expect(await starved.snapshot(at: 0.75, maxSize: CGSize(width: 16, height: 16)) != nil) + await starved.shutdown() + } + + /// Eight 32x32 MPEG-4 Part 2 frames at 4 fps; only the first is a keyframe. + /// ffmpeg -f lavfi -i testsrc2=size=32x32:rate=4:duration=2 -c:v mpeg4 -g 100 + /// -sc_threshold 1000000000 -q:v 12 -pix_fmt yuv420p -movflags +faststart preview.mp4 + private static let previewFixtureBase64 = """ + AAAAHGZ0eXBpc29tAAACAGlzb21pc28ybXA0MQAAA3Ntb292AAAAbG12aGQAAAAAAAAAAAAAAAAAAAPoAAAH0AABAAABAAAA + AAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC + AAACnnRyYWsAAABcdGtoZAAAAAMAAAAAAAAAAAAAAAEAAAAAAAAH0AAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAA + AAEAAAAAAAAAAAAAAAAAAEAAAAAAIAAAACAAAAAAACRlZHRzAAAAHGVsc3QAAAAAAAAAAQAAB9AAAAAAAAEAAAAAAhZtZGlh + AAAAIG1kaGQAAAAAAAAAAAAAAAAAAEAAAACAAFXEAAAAAAAtaGRscgAAAAAAAAAAdmlkZQAAAAAAAAAAAAAAAFZpZGVvSGFu + ZGxlcgAAAAHBbWluZgAAABR2bWhkAAAAAQAAAAAAAAAAAAAAJGRpbmYAAAAcZHJlZgAAAAAAAAABAAAADHVybCAAAAABAAAB + gXN0YmwAAADpc3RzZAAAAAAAAAABAAAA2W1wNHYAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAIAAgAEgAAABIAAAAAAAAAAES + TGF2YzYzLjEuMTAyIG1wZWc0AAAAAAAAAAAAAAAAAAAY//8AAABfZXNkcwAAAAADgICATgABAASAgIBAIBEAAAAAAw1AAAAH + YAWAgIAuAAABsAEAAAG1iRMAAAEAAAABIADEjYgAJQEEBBRDAAABskxhdmM2My4xLjEwMgaAgIABAgAAABBwYXNwAAAAAQAA + AAEAAAAUYnRydAAAAAAAAw1AAAAHYAAAABhzdHRzAAAAAAAAAAEAAAAIAAAQAAAAABRzdHNzAAAAAAAAAAEAAAABAAAAHHN0 + c2MAAAAAAAAAAQAAAAEAAAAIAAAAAQAAADRzdHN6AAAAAAAAAAAAAAAIAAABIQAAACYAAAAkAAAAGAAAABUAAAATAAAAGQAA + ABQAAAAUc3RjbwAAAAAAAAABAAADnwAAAGF1ZHRhAAAAWW1ldGEAAAAAAAAAIWhkbHIAAAAAAAAAAG1kaXJhcHBsAAAAAAAA + AAAAAAAALGlsc3QAAAAkqXRvbwAAABxkYXRhAAAAAQAAAABMYXZmNjMuMS4xMDIAAAAIZnJlZQAAAeBtZGF0AAABswAQBwAA + AbYTDGxGBiem/Q3iHttq0ki9pQ4YfSDrZNuIbiNnqkOFHVPSTh8y30HJPjaYIMv0ZZfretJOjInUvQ/EFSHgdB/xZSOCxTw0 + WoQ6FYiXJoqhVvLxe4VKb7s4shqNHauRBj5miAtFvlimDKapq87JahnA3c2KQMM+vF53lRyyyorFrKSPOq0yRa/wki/0dQSS + 9ea9Ogq/3rftxrFMgMjHHlDaiQ3Vpxbo2f2VgRNnWszNnaDIFPpPKYSVH1Y28NMTNT/JICNEY4UqEXVPQ8DoZdAQTP2N+bkD + 2LTUMndzvTXLdG/OIeHXAADMNhf8BaAwL+zL8BaAwL9mRJDSl4yXA4FNhRgOUGGydIjBl0iZkOjAtAZHhQGD3wAAAbZXhhC3 + xRLMfQPj9XgH//qX//5pGxz3PMRpHiSSEF8AAMxvAAABtlsGEKHjFgJvl/1P/4I2/sg65vxwKJ6ZJEKXJH0AAMxvAAABtl+G + EK/H4R7jExCgBLiVKfsAAMxvAAABtmmDCFTfcM9ckZB3CO8AAMxvAAABtleGEL9GOAubAADMCr685QAAAbZbBhCtmwmJrwAA + zAWsfygFesAJOX8AAAG2X4YRNkRyAADMCrvvU9pNfw== + """ + private static let profile5FixtureURL = URL(fileURLWithPath: #filePath) .deletingLastPathComponent() .deletingLastPathComponent() diff --git a/docs/api.md b/docs/api.md index 2a2a4da77..387f20137 100644 --- a/docs/api.md +++ b/docs/api.md @@ -489,7 +489,7 @@ Four exceptions, and each is an exception for a reason: `player.clock` is a separate `@MainActor ObservableObject` on purpose: its ~10 Hz ticks would otherwise fire `objectWillChange` on the engine and re-render every view observing it, which on tvOS rebuilds open `Menu` dropdowns mid-interaction. Time-driven UI observes `clock`, everything else observes the engine. `diagnostics` is split out for the same reason at 1 Hz. -`FrameExtractor` is an `actor`, so its `thumbnail` / `snapshot` / `prewarm` / `shutdown` are `await`ed from anywhere. The audio tap's `AsyncStream` is likewise consumed from any task; only `installAudioTap()` itself is main-actor. +`FrameExtractor` is an `actor`, so its `thumbnail` / `preview` / `snapshot` / `prewarm` / `shutdown` are `await`ed from anywhere. The audio tap's `AsyncStream` is likewise consumed from any task; only `installAudioTap()` itself is main-actor. ### One FFmpeg, and it has to be the engine's @@ -1193,7 +1193,7 @@ say so on the tracker rather than working around it. | `presentationAxisMap` | `PresentationAxisMap`: `sourceSeconds(forItemSeconds:)`, `itemSeconds(forSourceSeconds:)`, `shiftSeconds(atItemSeconds:)`, `seams` (each a `PresentationAxisMap.Seam` of `itemSeconds` and `shiftSeconds`), `isEmpty`. Readable off the main actor. Cue times and `sourceTime` live on the source axis; `AVPlayerItem.currentTime()` lives on the item axis, and they differ by the producer shift. Returns nil rather than a guess where no axis is established. | | `setNativeVideoFrameTimeObserver(_:)` | Takes a `NativeVideoFrameTimeObserver`, called per muxed frame on both axes (`NativeVideoFrameTime`: `source`, `item`, `segmentIndex`, `isKeyframe`, `epoch`). Called on the producer pump thread in **decode** order, so `source` is not monotonic under B-frames. `epoch` rises process-wide. | | `setSoftwareVideoFrameTimeObserver(_:)`, `softwarePresentationTimebase` | The software-path equivalent, a `SoftwareVideoFrameTimeObserver` over `SoftwareVideoFrameTime` (`presentation`, `generation`), in ascending presentation order, on the same axis as the cues. `generation` moves on every renderer flush. | -| `FrameExtractor` | Off-playback stills: `thumbnail(at:maxWidth:)`, `snapshot(at:maxSize:)`, `prewarm()`, `shutdown()`, over a URL or an `IOReader`. Opens its own demuxer, so it needs a source that tolerates a second connection. `boundedSnapshot(at:maxSize:limits:cancellation:)` is the disposable form: one caller-supplied `ProbeLimits` / `ProbeCancellation` budget spans open, stream analysis, seek and decode, a stalled HTTP header included, and the result is a `BoundedStillFrame`. Use a fresh extractor, since an already-open context is refused rather than charged to the budget; the context closes on every outcome. | +| `FrameExtractor` | Off-playback stills: `thumbnail(at:maxWidth:)`, `preview(at:maxWidth:)`, `snapshot(at:maxSize:)`, `prewarm()`, `shutdown()`, over a URL or an `IOReader`. Thumbnail mode returns a coarse keyframe; `FrameMode.preview` decodes forward to the requested time at `maxWidth`, and `FrameMode.snapshot` uses native or bounded dimensions. Preview and snapshot decoding retain full reconstruction and bicubic scaling; thumbnails keep the cheaper decode path. Previews have a separate count-bounded cache keyed by exact request time and width, yield on uncached requests while `yieldWhile` reports starvation, and are cancelled by superseding extraction or `shutdown()`. This API shares ordinary extraction cancellation and HTTP read timeouts; use `boundedSnapshot` when a caller-supplied total deadline is required. Opens its own demuxer, so it needs a source that tolerates a second connection. `boundedSnapshot(at:maxSize:limits:cancellation:)` is the disposable form: one caller-supplied `ProbeLimits` / `ProbeCancellation` budget spans open, stream analysis, seek and decode, a stalled HTTP header included, and the result is a `BoundedStillFrame`. Use a fresh extractor, since an already-open context is refused rather than charged to the budget; the context closes on every outcome. | ## Now Playing and the audio session diff --git a/docs/architecture.md b/docs/architecture.md index fab48e1bc..9ab64c54f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -508,9 +508,9 @@ Sources/AetherEngine/ │ ├── AetherEngine+FrameExtractor.swift makeFrameExtractor() convenience for the currently loaded URL │ ├── DolbyVisionStillConverter.swift `DolbyVisionStillReshaping` trait. Applies the RPU-carried DV colour transform to a DV P5 / P10.0 base-layer frame (IPT-PQ-C2, not YCbCr) so scrub stills lose the green + magenta cast (#103) │ ├── FrameExtractor.swift Off-playback still extraction actor: serial decode queue, cancel-supersede, idle-close -│ ├── FrameDecodeContext.swift Isolated FFmpeg demux + decode + sws_scale → CGImage (thumbnail / snapshot) +│ ├── FrameDecodeContext.swift Isolated FFmpeg demux + decode + sws_scale → CGImage (thumbnail / preview / snapshot) │ ├── FrameCache.swift Bounded LRU: mode-isolated stores, second-bucketed thumbnails -│ ├── FrameTypes.swift FrameMode (.thumbnail / .snapshot) +│ ├── FrameTypes.swift FrameMode (.thumbnail / .preview / .snapshot) │ └── HDRToneMapper.swift zscale + tonemap libavfilter graph: HDR (PQ / HLG, BT.2020) stills → SDR BT.709 ├── IO/ │ ├── IOReader.swift Public custom byte-source protocol + MediaSource (load(source:) input) diff --git a/docs/cli.md b/docs/cli.md index 484664bb0..055ca57de 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -535,10 +535,11 @@ swift run aetherctl scrubpreview --throttle-kbps 20000 --at 2,30,50,65 --keep /t ## extract -Opens a `FrameExtractor` against the source and pulls a still frame. Thumbnail mode (default) snaps to the nearest keyframe and downscales to `--width` (default 320); `--snapshot` decodes frame-accurately at full resolution. `--at ` sets the seek position (default 60.0). The first frame is written to `aetherctl-extract-.png` in aetherctl's private per-user temporary directory (the path is printed). `--loops N` repeats the extraction across eight cycling positions, which pairs with `leaks --atExit` to validate the decode-context teardown is clean: +Opens a `FrameExtractor` against the source and pulls a still frame. Thumbnail mode (default) snaps to the nearest keyframe and downscales to `--width` (default 320); `--preview` decodes frame-accurately at `--width`, while `--snapshot` uses full resolution. The two flags are mutually exclusive. `--at ` sets the seek position (default 60.0). The first frame is written to `aetherctl-extract-.png` in aetherctl's private per-user temporary directory (the path is printed). `--loops N` repeats the extraction across eight cycling positions, which pairs with `leaks --atExit` to validate the decode-context teardown is clean: ```bash swift run aetherctl extract --at 612 --snapshot # frame-accurate still +swift run aetherctl extract --at 612 --preview --width 480 # accurate downscaled preview swift run aetherctl extract --width 480 # keyframe thumbnail leaks --atExit -- .build/debug/aetherctl extract --loops 8 # leak sweep ```