From 79c945fd7cd7b27799ea7761f6b6149042db90ed Mon Sep 17 00:00:00 2001 From: JonLuca DeCaro Date: Sun, 25 Oct 2026 09:25:08 -0700 Subject: [PATCH] feat(lifecycle): retain paused video pipelines on host opt-in --- CHANGELOG.md | 2 ++ Sources/AetherEngine/AetherEngine.swift | 17 ++++++++--- .../Native/SoftwarePlaybackHost.swift | 4 ++- .../Renderer/SampleBufferRenderer.swift | 22 +++++++++++++- .../BackgroundKeepaliveTests.swift | 26 +++++++++++++++++ .../SampleBufferRendererLookaheadTests.swift | 29 +++++++++++++++++++ docs/api.md | 1 + 7 files changed, 95 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cadfbc99f..74b3581c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ the public-API contract. ### Added +- `retainsVideoPipelineWhenBackgrounded` lets hosts retain a paused video pipeline through suspension. It is off by default and leaves playing-session background policy unchanged. Software playback flushes a failed retained video queue before restarting its feeder. + - **Remote MPEG-DASH on-demand manifests play (AE#859, thanks @qoli).** A static, single-Period MPD whose representations are single files addressed by `SegmentBase` failed at open with `Invalid data found when processing input`: the FFmpeg build has no DASH diff --git a/Sources/AetherEngine/AetherEngine.swift b/Sources/AetherEngine/AetherEngine.swift index cea33055d..9a6037ec6 100644 --- a/Sources/AetherEngine/AetherEngine.swift +++ b/Sources/AetherEngine/AetherEngine.swift @@ -917,6 +917,11 @@ public final class AetherEngine: ObservableObject { /// Master enable for background playback (iOS: PiP + background audio; tvOS: PiP keepalive). Default on. public var backgroundPlaybackEnabled = true + + /// Keep a paused video session through background suspension instead of releasing its pipeline. + /// Off by default: the system may invalidate decoders while suspended, requiring a reload. + /// This does not enable background playback or change the policy for a playing session. + public var retainsVideoPipelineWhenBackgrounded = false /// Set by the host from its PiP delegate (iOS: AVKit; tvOS: host-built AVPictureInPictureController); /// the keepalive policy + pause-safety read it. public var pictureInPictureActive = false { @@ -1286,8 +1291,10 @@ public final class AetherEngine: ObservableObject { softwareHostHasAudio: Bool, keepVideoAlive: Bool, pipActive: Bool, - state: PlaybackState + state: PlaybackState, + retainPausedVideoPipeline: Bool = false ) -> BackgroundAction { + if retainPausedVideoPipeline && state == .paused { return .doNothing } if isAudioBackend { return .doNothing } if keepVideoAlive { if hasSoftwareHost { @@ -5905,7 +5912,7 @@ public final class AetherEngine: ObservableObject { // (wedge-safe, mirrors the unconditional background teardown). Audio backends are already spared. // #127: same grace window as the didEnterBackground path, so pause-after-background quick switches // also skip the rebuild. - if isBackgrounded && !pictureInPictureActive && !audioAVPlayerActive && audioHost == nil && softwareHost == nil { + if isBackgrounded && !retainsVideoPipelineWhenBackgrounded && !pictureInPictureActive && !audioAVPlayerActive && audioHost == nil && softwareHost == nil { switch Self.backgroundStep( action: .teardownVideo, state: state, @@ -8512,7 +8519,8 @@ public final class AetherEngine: ObservableObject { softwareHostHasAudio: softwareHost?.deliversAudio == true, keepVideoAlive: keepAlive, pipActive: pictureInPictureActive, - state: state + state: state, + retainPausedVideoPipeline: retainsVideoPipelineWhenBackgrounded ) switch Self.backgroundStep( action: action, @@ -8644,7 +8652,8 @@ public final class AetherEngine: ObservableObject { softwareHostHasAudio: softwareHost?.deliversAudio == true, keepVideoAlive: keepAlive, pipActive: pictureInPictureActive, - state: state + state: state, + retainPausedVideoPipeline: retainsVideoPipelineWhenBackgrounded ) } #endif diff --git a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift index 54fe47192..d846a2e6e 100644 --- a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift +++ b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift @@ -938,12 +938,13 @@ final class SoftwarePlaybackHost { private let reposition: Reposition init(videoGravity: AVLayerVideoGravity = .resizeAspect, + renderer: SampleBufferRenderer? = nil, reposition: @escaping Reposition = { dem, target, budget, queue, superseded in await dem.seekBounded(to: target, timeout: budget, on: queue, isSuperseded: superseded) }) { self.reposition = reposition - self.renderer = SampleBufferRenderer(videoGravity: videoGravity) + self.renderer = renderer ?? SampleBufferRenderer(videoGravity: videoGravity) // Default to the software decoder; load() swaps it for the // VT-backed one when the source's video codec is HEVC. self.videoDecoder = SoftwareVideoDecoder() @@ -1556,6 +1557,7 @@ final class SoftwarePlaybackHost { func play() { guard !stopRequested, failure == nil else { return } + renderer.recoverAfterSuspensionIfNeeded() // Resume after pause(): gate on clockArmed, not demuxLoopStarted. A rate change on the // un-anchored synchronizer (no media at its clock time yet) wedges the delayed-rate-change // machinery permanently frozen; the arming seekClock applies the current lastRate (#107). diff --git a/Sources/AetherEngine/Renderer/SampleBufferRenderer.swift b/Sources/AetherEngine/Renderer/SampleBufferRenderer.swift index c3c4ec9a1..0a673041e 100644 --- a/Sources/AetherEngine/Renderer/SampleBufferRenderer.swift +++ b/Sources/AetherEngine/Renderer/SampleBufferRenderer.swift @@ -12,6 +12,12 @@ import CoreVideo /// (`setArrivalOrderHandover(frameRate:)`). final class SampleBufferRenderer: @unchecked Sendable { + struct ResumeQueue { + let status: @MainActor () -> AVQueuedSampleBufferRenderingStatus + let flush: @MainActor () -> Void + } + + @MainActor private let resumeQueue: ResumeQueue? let displayLayer: AVSampleBufferDisplayLayer /// The layer's queue surface, taken once on the main actor at construction (#351). The 27 SDKs @@ -395,7 +401,8 @@ final class SampleBufferRenderer: @unchecked Sendable { /// Main-actor isolated because the layer is (#351); the only production caller, /// `SoftwarePlaybackHost`, already is. @MainActor - init(videoGravity: AVLayerVideoGravity = .resizeAspect) { + init(resumeQueue: ResumeQueue? = nil, videoGravity: AVLayerVideoGravity = .resizeAspect) { + self.resumeQueue = resumeQueue let layer = Self.makeDisplayLayer(isHDR: false, gravity: videoGravity) displayLayer = layer videoRenderer = layer.sampleBufferRenderer @@ -480,6 +487,19 @@ final class SampleBufferRenderer: @unchecked Sendable { queueTarget.isReadyForMoreMediaData } + /// A suspended process may return with a failed queue that never admits another frame. + /// Recover before the feeder wakes; a healthy full queue must keep its buffered frames. + @MainActor + func recoverAfterSuspensionIfNeeded() { + let status = resumeQueue?.status() ?? queueStatus + guard status == .failed else { return } + if let resumeQueue { + resumeQueue.flush() + } else { + queueTarget.flush() + } + } + private var queueStatus: AVQueuedSampleBufferRenderingStatus { videoRenderer.status } private var queueError: Error? { videoRenderer.error } diff --git a/Tests/AetherEngineTests/BackgroundKeepaliveTests.swift b/Tests/AetherEngineTests/BackgroundKeepaliveTests.swift index 506c76248..aaac628a3 100644 --- a/Tests/AetherEngineTests/BackgroundKeepaliveTests.swift +++ b/Tests/AetherEngineTests/BackgroundKeepaliveTests.swift @@ -103,4 +103,30 @@ struct BackgroundKeepaliveTests { func softwareAudioOnlyWithoutPiP() { #expect(AetherEngine.backgroundAction(isAudioBackend: false, hasSoftwareHost: true, softwareHostHasAudio: true, keepVideoAlive: true, pipActive: false, state: .playing) == .enterSoftwareAudioOnly) } + @MainActor + @Test("retaining paused pipelines is opt-in") + func retentionDefault() throws { + #expect(try AetherEngine().retainsVideoPipelineWhenBackgrounded == false) + } + + @Test("retention keeps paused native and software sessions, without enabling background playback") + func retainedPausedPipelinePolicy() { + for software in [false, true] { + for audio in [false, true] { + #expect(AetherEngine.backgroundAction( + isAudioBackend: false, hasSoftwareHost: software, softwareHostHasAudio: audio, + keepVideoAlive: false, pipActive: false, state: .paused, + retainPausedVideoPipeline: true) == .doNothing) + #expect(AetherEngine.backgroundAction( + isAudioBackend: false, hasSoftwareHost: software, softwareHostHasAudio: audio, + keepVideoAlive: false, pipActive: false, state: .playing, + retainPausedVideoPipeline: true) == .teardownVideo) + } + } + #expect(AetherEngine.backgroundAction( + isAudioBackend: false, hasSoftwareHost: true, softwareHostHasAudio: true, + keepVideoAlive: true, pipActive: false, state: .playing, + retainPausedVideoPipeline: true) == .enterSoftwareAudioOnly) + } + } diff --git a/Tests/AetherEngineTests/SampleBufferRendererLookaheadTests.swift b/Tests/AetherEngineTests/SampleBufferRendererLookaheadTests.swift index 55b004250..98e5bc52d 100644 --- a/Tests/AetherEngineTests/SampleBufferRendererLookaheadTests.swift +++ b/Tests/AetherEngineTests/SampleBufferRendererLookaheadTests.swift @@ -1,3 +1,5 @@ +import AVFoundation +import Combine import Testing @testable import AetherEngine @@ -193,4 +195,31 @@ struct SampleBufferRendererLookaheadTests { rendererReady: ready, felFrameRate: rate, framePTS: pts, clockSeconds: clock, clockArmed: armed) } + @MainActor + @Test("resume flushes a failed queue before publishing play") + func failedQueueRecoversBeforePlay() { + var events: [String] = [] + let renderer = SampleBufferRenderer(resumeQueue: .init( + status: { .failed }, flush: { events.append("flush") })) + let host = SoftwarePlaybackHost(renderer: renderer) + let observation = host.$rate.dropFirst().sink { rate in + if rate > 0 { events.append("play") } + } + host.play() + #expect(events == ["flush", "play"]) + #expect(host.isPlaying) + observation.cancel() + host.stop() + } + + @MainActor + @Test("resume preserves a healthy full queue's frames", arguments: [AVQueuedSampleBufferRenderingStatus.unknown, .rendering]) + func healthyQueueIsNotFlushed(status: AVQueuedSampleBufferRenderingStatus) { + var flushes = 0 + let renderer = SampleBufferRenderer(resumeQueue: .init( + status: { status }, flush: { flushes += 1 })) + renderer.recoverAfterSuspensionIfNeeded() + #expect(flushes == 0) + } + } diff --git a/docs/api.md b/docs/api.md index 2a2a4da77..0a332395a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1189,6 +1189,7 @@ say so on the tracker rather than working around it. | `sourceVideoWidth`, `sourceVideoHeight`, `sourceVideoPixelAspectRatio` | The source's CODED size and the multiplier that turns it into the presented one (`width * ratio`), read once from the probe (on `.remoteBypass`, from AVPlayer's item video track once it resolves: the delivered size). The ratio is 1 on square pixels and on a declared ratio the engine refuses (#290), never a guess; on the paths that draw, prefer what is on screen (`softwareDisplaySize`, `AVPlayerLayer.videoRect`) over recomputing it here. | | `pictureInPictureActive` | Host-set. Keeps the pipeline and the loopback server alive across a background transition, and keeps the software path decoding video for the window. | | `needsForegroundVideoRestore` | True after background policy actually releases the video item. Restore with `reloadAtCurrentPosition(applying:)` before play, choosing the desired transport intent. False when video was retained or after stop/load. | +| `retainsVideoPipelineWhenBackgrounded` | Opt in to retaining a paused video pipeline through background suspension. Default false. The host accepts that system-invalidated decoder sessions can still need `reloadAtCurrentPosition()`; this saves a rebuild when the pipeline survives, but does not enable background playback. Failed software video queues are flushed on play before the feeder wakes; healthy buffered frames remain. Set before backgrounding. | | `backgroundPlaybackEnabled`, `backgroundTeardownGraceSeconds` | Background audio policy; the grace window (15 s default) is what lets a paused session survive a quick app switch. | | `presentationAxisMap` | `PresentationAxisMap`: `sourceSeconds(forItemSeconds:)`, `itemSeconds(forSourceSeconds:)`, `shiftSeconds(atItemSeconds:)`, `seams` (each a `PresentationAxisMap.Seam` of `itemSeconds` and `shiftSeconds`), `isEmpty`. Readable off the main actor. Cue times and `sourceTime` live on the source axis; `AVPlayerItem.currentTime()` lives on the item axis, and they differ by the producer shift. Returns nil rather than a guess where no axis is established. | | `setNativeVideoFrameTimeObserver(_:)` | Takes a `NativeVideoFrameTimeObserver`, called per muxed frame on both axes (`NativeVideoFrameTime`: `source`, `item`, `segmentIndex`, `isKeyframe`, `epoch`). Called on the producer pump thread in **decode** order, so `source` is not monotonic under B-frames. `epoch` rises process-wide. |