From d46a28353443c8dfbdec9aa48505131927e79323 Mon Sep 17 00:00:00 2001 From: Pavel Kajaba <15980400+pkajaba@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:56:34 +0000 Subject: [PATCH 1/4] fix(software-host): recover from system flushes, failed renderers and refused session rebuilds, with three small engine fixes Flush and failure recovery. When the system flushes or fails the host's video queue or its audio renderer, the host flushes the failed queue and the video decoder and resumes at the next keyframe while the audio keeps playing, seeking to the playhead only when that keyframe is further away than the queued audio. A video failure while only audio plays in the background is paid on the foreground return; a live source with no rewind ring is only flushed; a recovery that did not help is not repeated until a frame gets through; the decoder is fed again only from the next keyframe; a failing audio renderer is flushed at most three times in ten seconds. Refused session rebuilds. When VideoToolbox refuses to rebuild an invalidated hardware session, the host retries at each keyframe fed while the session is lost (at least two seconds apart), up to four times, then publishes a playback failure. In-band parameter-set scan budget. The scan that rebuilds an empty hvcC from in-band VPS/SPS/PPS also stops at 16 MiB of input, with the same ceiling armed on the demuxer input. LoadOptions.initialSubtitleTrackID. A load can name the subtitle track to activate by id, ahead of preferredSubtitleLanguages; an id that names no track falls through to the language preference. selectChapter on a disc title no longer applies the container start twice on the software route, and no longer adds the playlist shift on the native route. Tests. Suites that set the origin budget's quiet-period cap hold a counted pin so the cap comes back, and a new test pins that a paused software session and its remote come back after a background teardown. --- CHANGELOG.md | 35 + .../AetherEngine/AetherEngine+Loading.swift | 4 +- Sources/AetherEngine/AetherEngine+Probe.swift | 16 + .../AetherEngine+ReloadWithOptions.swift | 2 +- .../AetherEngine/AetherEngine+Subtitles.swift | 28 +- Sources/AetherEngine/AetherEngine.swift | 37 +- Sources/AetherEngine/Audio/AudioOutput.swift | 9 +- .../Decoder/HardwareVideoDecoder.swift | 30 +- .../Demuxer/OriginRequestBudget.swift | 56 +- .../Native/SoftwareFlushRecovery.swift | 243 ++++ .../Native/SoftwarePlaybackHost.swift | 362 +++++- .../Native/VideoSessionRetryPolicy.swift | 49 + Sources/AetherEngine/PlayerState.swift | 16 + Sources/AetherEngine/PresentationAxis.swift | 39 + .../Renderer/SampleBufferRenderer.swift | 217 +++- .../Video/VideoConfigRecord+InBand.swift | 48 +- .../DiscChapterSeekTargetTests.swift | 171 +++ .../FlushRecoveryTests.swift | 1056 +++++++++++++++++ .../HDR10PlusScanInputBudgetTests.swift | 7 +- .../HLSOriginRelayTests.swift | 8 +- .../InBandParameterSetRebuildTests.swift | 78 ++ .../InitialSubtitleTrackTests.swift | 122 ++ .../Issue410SourceStallVisibilityTests.swift | 5 +- .../OriginBudgetQuietCapPinTests.swift | 65 + .../ResolvedURLInvalidationTests.swift | 5 +- .../SoftwareHostWakeRestoreTests.swift | 99 ++ .../VideoSessionRetryTests.swift | 90 ++ docs/api.md | 1 + docs/architecture.md | 2 + 29 files changed, 2825 insertions(+), 75 deletions(-) create mode 100644 Sources/AetherEngine/Native/SoftwareFlushRecovery.swift create mode 100644 Sources/AetherEngine/Native/VideoSessionRetryPolicy.swift create mode 100644 Tests/AetherEngineTests/DiscChapterSeekTargetTests.swift create mode 100644 Tests/AetherEngineTests/FlushRecoveryTests.swift create mode 100644 Tests/AetherEngineTests/InitialSubtitleTrackTests.swift create mode 100644 Tests/AetherEngineTests/OriginBudgetQuietCapPinTests.swift create mode 100644 Tests/AetherEngineTests/SoftwareHostWakeRestoreTests.swift create mode 100644 Tests/AetherEngineTests/VideoSessionRetryTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bfed7755..bef551791 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,13 +10,48 @@ the public-API contract. ## [Unreleased] +### Added + +- **`LoadOptions.initialSubtitleTrackID` names the subtitle track to activate at load.** + A host that already knows the track it wants, such as a server's default subtitle index, had to + language-match `subtitleTracks` itself or call `selectSubtitleTrack` after the load. The id (an + embedded stream index, or `externalSubtitleTrackIDBase` plus the position in `externalSubtitles`) + now activates on the same post-load path as `preferredSubtitleLanguages` and wins over it. An id + that names no track falls through to the language preference. See `docs/api.md`. + ### Fixed +- **`selectChapter` on a disc title lands on the chapter, not one container start late.** The software route seeks on the session axis and the host adds the clock's session zero back, so adding the whole container start as well applied it twice whenever the start was past the 2 s anchor tolerance. Only the part of the start the zero did not take is added now, and a native disc title no longer adds the playlist shift on top of `seek(to:)`. + - **A 24.000 fps source is programmed as 24 Hz, not 23.976 Hz (AE#884, thanks @akhanwav).** `FrameRateSnap` sent the whole 23.5 to 24.05 window to 23.976, so Match Frame Rate ran 24p content one frame in 1001 fast and the panel repeated a frame about every 42 s. A probe within 0.005 of 24.000 now snaps to 24; a Matroska track with DefaultDuration probes as an exact 24/1 or 24000/1001, so 23.976 sources and noisier probes keep 23.976. +- **A refused video-session rebuild is retried at the next keyframes in the foreground.** When the system + invalidated the hardware session and VideoToolbox refused the rebuild, the decoder dropped every packet + and the picture froze beside live audio until the next foreground return. The host now retries at each + keyframe fed while the session is still lost (at least two seconds apart), up to four times, and then + publishes a playback failure. + +- **The in-band HEVC parameter-set scan stops at 16 MiB of input.** A stream that carries no parameter sets, + or one whose packets are very large, could be read far past what the packet budget implies before the + load gave up. The scan now also stops at a byte budget, the same ceiling is armed on the demuxer input so + one oversized read cannot overshoot it, and the record is left as it was. + +- **The software host recovers when the system flushes or fails its video queue or audio renderer.** The + video renderer reports `requiresFlushToResumeDecoding` (the app lost the right to decode, for instance + after backgrounding) and either renderer can end up failed; nothing observed any of it, so the picture + stayed frozen until the user seeked. The host now flushes the failed queue and the video decoder and the + picture resumes at the next keyframe while the audio plays on. Only when that keyframe lies further away + than the audio already queued, or none has been read yet, does it seek to the playhead instead. A video + failure while only audio plays in the background is paid on the foreground return, a live source with no + rewind ring only flushes, and a queue that is still failing after a recovery is flushed once more and then + left alone until a frame the clock has passed gets through, with a short pause between two seeks to the + playhead. After a flush the video decoder is fed again only from the next keyframe, so no delta frame + decodes against nothing. An audio renderer that keeps failing is flushed at most three times in ten + seconds, and one that flushes itself on a route change keeps the behaviour it had. + ## [7.48.0] - 2026-10-26 ### Added diff --git a/Sources/AetherEngine/AetherEngine+Loading.swift b/Sources/AetherEngine/AetherEngine+Loading.swift index 4e4e7ac53..b2e96ac1e 100644 --- a/Sources/AetherEngine/AetherEngine+Loading.swift +++ b/Sources/AetherEngine/AetherEngine+Loading.swift @@ -2739,8 +2739,8 @@ extension AetherEngine { // stopInternal zeroed sourceStartSeconds; recapture the software-path chapter-seek base from // the reopened demuxer so a DVD chapter seek after an audio switch / custom reload still lands // (the native base self-heals via onPlaylistShiftChanged, this one does not). (#67) - let st = pre.formatStartTime - reopenedStartSeconds = st > 0 ? Double(st) / Double(AV_TIME_BASE) : 0 + // Same resolved origin as the first load (see sourceStartSeconds). + reopenedStartSeconds = pre.resolvedSourceStartOrigin } } diff --git a/Sources/AetherEngine/AetherEngine+Probe.swift b/Sources/AetherEngine/AetherEngine+Probe.swift index 3e264313f..c15d1105f 100644 --- a/Sources/AetherEngine/AetherEngine+Probe.swift +++ b/Sources/AetherEngine/AetherEngine+Probe.swift @@ -775,6 +775,22 @@ extension AetherEngine { return Int32(tracks[index].id) } + /// The subtitle track to activate at the end of a load: `LoadOptions.initialSubtitleTrackID` when it + /// names a track in `tracks`, else the best `preferredSubtitleLanguages` match, else nil (subtitles + /// off). The explicit id wins because a host that names a track has already made the choice the + /// language ranking approximates; an id that matches nothing is ignored rather than treated as an + /// error, so a stale server index cannot fail a load. Pure and nonisolated, like `selectSubtitleIndex`. + nonisolated static func resolveInitialSubtitleIndex( + tracks: [TrackInfo], + initialTrackID: Int?, + preferredLanguages: [String] + ) -> Int32? { + if let initialTrackID, tracks.contains(where: { $0.id == initialTrackID }) { + return Int32(initialTrackID) + } + return selectSubtitleIndex(tracks: tracks, preferredLanguages: preferredLanguages) + } + /// Lower rank wins. The descriptor axis (full > SDH > forced > commentary, from container dispositions) /// dominates; text-vs-bitmap is a tiebreaker (text preferred, since host styling only applies to text /// cues). Sourced from `TrackInfo` disposition flags rather than title strings, so it is locale-robust. diff --git a/Sources/AetherEngine/AetherEngine+ReloadWithOptions.swift b/Sources/AetherEngine/AetherEngine+ReloadWithOptions.swift index 652f0adc9..ad7d2d291 100644 --- a/Sources/AetherEngine/AetherEngine+ReloadWithOptions.swift +++ b/Sources/AetherEngine/AetherEngine+ReloadWithOptions.swift @@ -459,7 +459,7 @@ enum SessionOptionCorrection { "preserveASSMarkup", "prepareNativeSubtitles", "eagerNativeSubtitleReaders", "serveIFramePlaylist", "confirmAtmos", "nativeSubtitlePreferredLanguages", "sequentialOrigin", "maxConcurrentSourceRequests", "heldSourceConnection", "declaredDurationSeconds", "probesize", "maxAnalyzeDuration", "preferredAudioLanguages", - "preferredSubtitleLanguages", "externalSubtitles", "forwardBufferSegments", "sessionCacheByteBudget", "softwarePacketCacheStorage", "progressiveSegmentDelivery", "autoplay", + "preferredSubtitleLanguages", "externalSubtitles", "initialSubtitleTrackID", "forwardBufferSegments", "sessionCacheByteBudget", "softwarePacketCacheStorage", "progressiveSegmentDelivery", "autoplay", "audioDelaySeconds", "teletextPage", "deinterlaceMode", "deinterlaceFieldRate", "preferredDecodePath", "escalatesToSoftwarePath", "sharedOutputRole", "isLiveRejoin", "subtitleSessionCarryover", ] diff --git a/Sources/AetherEngine/AetherEngine+Subtitles.swift b/Sources/AetherEngine/AetherEngine+Subtitles.swift index f5b2c7822..a0aa30823 100644 --- a/Sources/AetherEngine/AetherEngine+Subtitles.swift +++ b/Sources/AetherEngine/AetherEngine+Subtitles.swift @@ -144,22 +144,25 @@ extension AetherEngine { ) } - /// Apply `LoadOptions.preferredSubtitleLanguages` at the end of a successful load: activate the best-ranked - /// subtitle track whose language matches a preference (scanned in order; see `selectSubtitleIndex`), else - /// leave subtitles off (the default). Uses the host-overlay path (equivalent to a `selectSubtitleTrack` - /// call); `startAnchor` is the + /// Apply `LoadOptions.initialSubtitleTrackID`, else `LoadOptions.preferredSubtitleLanguages`, at the end of a + /// successful load: activate the track the id names, else the best-ranked subtitle track whose language + /// matches a preference (scanned in order; see `selectSubtitleIndex`), else leave subtitles off (the default). + /// Uses the host-overlay path (equivalent to a `selectSubtitleTrack` call); `startAnchor` is the /// load's resume position so a mid-file resume seeks the side demuxer to the playhead instead of byte 0. - /// A no-op when the list is empty, no track matches, or the host already activated a subtitle. The resolved + /// A no-op when neither an id nor a language list is set, nothing matches, or the host already activated a subtitle. The resolved /// index is published via `activeSubtitleTrackIndex`. Independent of `prepareNativeSubtitles`. (#73) func applyPreferredSubtitleSelection(startAnchor: Double?, sourceDuration: Double?) { - guard !loadedOptions.preferredSubtitleLanguages.isEmpty, !isSubtitleActive, - !hostExplicitSubtitleAction else { return } - guard let index = Self.selectSubtitleIndex( + guard loadedOptions.initialSubtitleTrackID != nil || !loadedOptions.preferredSubtitleLanguages.isEmpty, + !isSubtitleActive, !hostExplicitSubtitleAction else { return } + guard let index = Self.resolveInitialSubtitleIndex( tracks: subtitleTracks, + initialTrackID: loadedOptions.initialSubtitleTrackID, preferredLanguages: loadedOptions.preferredSubtitleLanguages ) else { return } EngineLog.emit( - "[AetherEngine] preferred-subtitle auto-select stream=\(index) langs=\(loadedOptions.preferredSubtitleLanguages)", + "[AetherEngine] preferred-subtitle auto-select stream=\(index) " + + "initialID=\(loadedOptions.initialSubtitleTrackID.map(String.init) ?? "nil") " + + "langs=\(loadedOptions.preferredSubtitleLanguages)", category: .engine ) // Bound the anchor to the probe duration (synchronously known here; the published `duration` is set @@ -1393,9 +1396,10 @@ extension AetherEngine { /// Register an external subtitle file as a first-class track (AetherEngine#88): it appears in /// `subtitleTracks` with a synthetic id and `isExternal == true` and is selectable via /// `selectSubtitleTrack(index:)`. Overlay-only (no native WebVTT rendition / PiP); declare via - /// `LoadOptions.externalSubtitles` for rendition eligibility. If `preferredSubtitleLanguages` - /// is set, nothing is active, and the host made no explicit choice yet, the preference re-runs - /// so a late-added matching track auto-activates. + /// `LoadOptions.externalSubtitles` for rendition eligibility. If `initialSubtitleTrackID` or + /// `preferredSubtitleLanguages` is set, nothing is active, and the host made no explicit choice + /// yet, the selection re-runs so a late-added track that matches the id or the preference + /// auto-activates. @discardableResult public func addExternalSubtitleTrack(_ track: ExternalSubtitleTrack) -> TrackInfo { let info = registerExternalSubtitleTrack(track) diff --git a/Sources/AetherEngine/AetherEngine.swift b/Sources/AetherEngine/AetherEngine.swift index 792564d47..842e9ccd3 100644 --- a/Sources/AetherEngine/AetherEngine.swift +++ b/Sources/AetherEngine/AetherEngine.swift @@ -760,9 +760,11 @@ public final class AetherEngine: ObservableObject { /// background-resume reopens (a URL-disc reopen with no id would silently revert to the main title). var activeDiscTitleID: Int? - /// Source container start PTS (seconds) from the probe (AVFormatContext.start_time). The software-path - /// playback clock begins here (the native path's content base is `playlistShiftSeconds`), so a DVD - /// chapter seek adds it to the chapter's title-relative (0-based) target. 0 when unknown (#67). + /// Source start PTS (seconds) the software host anchors its clock on: the demuxer's resolved origin + /// (`Demuxer.resolvedSourceStartOrigin`: the container start, backed up by the video stream's own + /// start when the container states none). `PresentationAxis.discChapterSeekTarget` needs the same + /// origin the host derives its session zero from, so the part of it the zero did not take can be + /// added to a DVD chapter's title-relative (0-based) target. 0 when unknown (#67). var sourceStartSeconds: Double = 0 /// Active playback backend: `.native` (AVPlayer) or `.software` (SoftwarePlaybackHost/dav1d/libavcodec). @@ -4963,10 +4965,10 @@ public final class AetherEngine: ObservableObject { republishMediaChapters() activeDiscTitleID = probeOpened ? probe.selectedDiscTitleID : nil selectedDiscTitle = activeDiscTitleID.flatMap { id in discTitles.first { $0.id == id } } - // Content start PTS for the software-path chapter-seek base (see sourceStartSeconds). start_time is - // AV_NOPTS_VALUE (Int64.min) when unknown; only a positive value is a real offset. - let probedStartTime = probeOpened ? probe.formatStartTime : 0 - sourceStartSeconds = probedStartTime > 0 ? Double(probedStartTime) / Double(AV_TIME_BASE) : 0 + // Source start PTS for the software-path chapter-seek base (see sourceStartSeconds): the same + // resolved origin the software host predicts its session zero from, so engine and host agree even + // when the container states no start and only the video stream carries one. + sourceStartSeconds = probeOpened ? probe.resolvedSourceStartOrigin : 0 // Assemble SourceProbe now while the demuxer is open; ownership transfers to loadNative/loadSoftware // after which streams are gone (AetherEngine#28). let sourceProbe: SourceProbe? = probeOpened @@ -7771,6 +7773,10 @@ public final class AetherEngine: ObservableObject { EngineLog.emit("[AetherEngine] selectTitle: scheduling switch to title \(id)", category: .engine) let gen = loadGeneration + // Stream indices are per title, so an initial subtitle track id named for the old title could + // activate an unrelated track on the new one. Drop it here (both reload paths read loadedOptions); + // the language preference decides on the new title. + loadedOptions.initialSubtitleTrackID = nil let options = loadedOptions let custom = isCustomSource Task { @MainActor [weak self] in @@ -7836,15 +7842,18 @@ public final class AetherEngine: ObservableObject { return } // discChapters are title-relative (0-based: chapter 1 = 0), matching the disc timeline and the - // 0-based title duration. The engine clock and seek(to:) run on the source-PTS axis, which begins at - // the title's content start; that base differs by backend (native re-times onto a 0-based playlist - // shifted by playlistShiftSeconds; the software path's raw clock begins at the container start, - // sourceStartSeconds). Add it so the seek lands on the chapter, not the base seconds early. - let base = (playbackBackend == .software) ? sourceStartSeconds : playlistShiftSeconds - let target = chapter.startSeconds + base + // 0-based title duration. `seek(to:)` takes the published axis of the active backend, which differs + // by backend (native re-times onto a 0-based playlist shifted by playlistShiftSeconds; the software + // path's published axis is the clock's session zero off the raw container start, so only the part of + // sourceStartSeconds the zero did not take is added, see `PresentationAxis.discChapterSeekTarget`). + let target = PresentationAxis.discChapterSeekTarget( + chapterStart: chapter.startSeconds, backend: playbackBackend, + sourceStartSeconds: sourceStartSeconds, playlistShiftSeconds: playlistShiftSeconds, + softwareClockZero: softwareClockSessionZero) + let base = target - chapter.startSeconds EngineLog.emit( "[AetherEngine] selectChapter: seeking to chapter \(id) @ title-relative " - + "\(String(format: "%.2f", chapter.startSeconds))s -> source \(String(format: "%.2f", target))s " + + "\(String(format: "%.2f", chapter.startSeconds))s -> seek target \(String(format: "%.2f", target))s " + "(base \(String(format: "%.2f", base))s, backend \(playbackBackend))", category: .engine ) diff --git a/Sources/AetherEngine/Audio/AudioOutput.swift b/Sources/AetherEngine/Audio/AudioOutput.swift index cb88d9182..02eac20d1 100644 --- a/Sources/AetherEngine/Audio/AudioOutput.swift +++ b/Sources/AetherEngine/Audio/AudioOutput.swift @@ -104,9 +104,12 @@ final class AudioOutput: @unchecked Sendable { /// best practice: the notification arrives on an arbitrary thread, so a buffer enqueued /// concurrently with it survives, and a survivor sits in the queue stamped far ahead of the /// timebase, muting the session for as long as it takes the clock to reach it. Re-feeding from - /// the timebase is deliberately NOT attempted: the demuxer stands at the audio lead by then and - /// the sources this happens to are exactly the ones that cannot seek backwards, so the honest - /// outcome is a gap of up to that lead, and then sync as before. + /// the timebase is deliberately NOT attempted, on any source: the demuxer stands at the audio lead + /// by then, so it would take a seek, which cuts the audio that is still playing, to put the dropped + /// lead back. The outcome is a gap of up to that lead, and then sync as before. The video queue is + /// not touched by this flush, so the host does not react to it; it repositions only when its own + /// video queue was flushed and the picture would stay frozen for longer than the audio lead + /// (`FlushRecoveryPolicy`). /// /// The line is also the witness the field log lacked. Across an automatic flush the timebase /// keeps RUNNING at its rate, so a session that froze did not freeze because of this, and only a diff --git a/Sources/AetherEngine/Decoder/HardwareVideoDecoder.swift b/Sources/AetherEngine/Decoder/HardwareVideoDecoder.swift index fb6885445..5abce7611 100644 --- a/Sources/AetherEngine/Decoder/HardwareVideoDecoder.swift +++ b/Sources/AetherEngine/Decoder/HardwareVideoDecoder.swift @@ -171,6 +171,20 @@ final class HardwareVideoDecoder: VideoDecodingPipeline, @unchecked Sendable { } private var _onSessionLost: (@Sendable () -> Void)? + /// A keyframe reached `decode` while the session is lost, so the owner can retry a rebuild that was + /// refused. That keyframe itself is dropped, not decoded: VOD recovers by seeking back, and live + /// with no rewind ring waits one GOP for the next keyframe. Fired on the feed thread, once per + /// such keyframe; the owner decides whether a retry is due. + var onKeyframeWhileLost: (@Sendable () -> Void)? { + get { skipLock.lock(); defer { skipLock.unlock() }; return _onKeyframeWhileLost } + set { skipLock.lock(); _onKeyframeWhileLost = newValue; skipLock.unlock() } + } + private var _onKeyframeWhileLost: (@Sendable () -> Void)? + + /// True from the moment the system invalidates the session (or a rebuild is refused) until a + /// rebuild succeeds. + var isSessionLost: Bool { lock.lock(); defer { lock.unlock() }; return sessionLost } + /// AE#739, all guarded by `lock`. `sessionLost` drops every packet instead of feeding a dead /// session; `awaitingKeyframe` drops packets after a rebuild until one the new session can start /// from. @@ -489,6 +503,12 @@ final class HardwareVideoDecoder: VideoDecodingPipeline, @unchecked Sendable { } #if DEBUG + /// Marks the session lost without touching VideoToolbox, so a test can feed keyframes to a decoder that + /// has no session and see them reported. + func markSessionLostForTesting() { + lock.lock(); sessionLost = true; lock.unlock() + } + /// Invalidate the live session without forgetting it, which is what the system does on an iOS /// screen lock: the next decode returns `kVTInvalidSessionErr`. For the aetherctl harness. func invalidateSessionForTesting() { @@ -610,7 +630,9 @@ final class HardwareVideoDecoder: VideoDecodingPipeline, @unchecked Sendable { // AE#492: see `SoftwareVideoDecoder.decode`. Same rule, same lock as `flush()`. if let epoch, epoch != currentEpoch { lock.unlock(); return } guard session != nil, !sessionLost, let formatDesc = formatDescription else { + let lost = sessionLost lock.unlock() + if lost, (packet.pointee.flags & AV_PKT_FLAG_KEY) != 0 { onKeyframeWhileLost?() } return } lock.unlock() @@ -938,4 +960,10 @@ protocol VideoSessionRebuilding: AnyObject { @discardableResult func rebuildSession() -> Bool } -extension HardwareVideoDecoder: VideoSessionRebuilding {} +/// A decoder that reports each keyframe fed while its session is lost, which is when a refused rebuild can +/// be retried. Separate from `VideoSessionRebuilding` so a decoder that cannot report it still rebuilds. +protocol VideoSessionKeyframeReporting: VideoSessionRebuilding { + var onKeyframeWhileLost: (@Sendable () -> Void)? { get set } +} + +extension HardwareVideoDecoder: VideoSessionKeyframeReporting {} diff --git a/Sources/AetherEngine/Demuxer/OriginRequestBudget.swift b/Sources/AetherEngine/Demuxer/OriginRequestBudget.swift index 16a2f86a7..020aa6552 100644 --- a/Sources/AetherEngine/Demuxer/OriginRequestBudget.swift +++ b/Sources/AetherEngine/Demuxer/OriginRequestBudget.swift @@ -63,13 +63,59 @@ final class OriginRequestBudget: @unchecked Sendable { /// suites of threads on a CI runner. Pinned on `.shared` by those suites; the pacer's own tests /// build their own instances and keep the shipped ladder. /// - /// Those suites pin it in `init` and never put it back, and suites run in parallel, so from the - /// first of them onward `.shared` has no quiet ladder for the rest of the process. A test that - /// asserts `isPaced` on `.shared` therefore passes under `--filter` and fails in a full run, - /// which is not a flake but the reading order. Assert on `snapshot(for:)`, whose refusal count - /// and learned limit this seam does not touch. + /// Setting it directly is for a budget a test owns. On `.shared` use `pinQuietPeriodCapForTesting`: + /// a suite that set it in `init` and never put it back left `.shared` without a quiet ladder for + /// the rest of the process, so a test asserting `isPaced` there passed under `--filter` and failed + /// in a full run, which was the reading order and not a flake. var quietPeriodCapForTesting: TimeInterval? + /// How many pins are alive, and the cap in force before the first of them. Guarded by `lock`. + private var quietPeriodCapPins = 0 + private var quietPeriodCapBeforePins: TimeInterval? + + /// Pins `quietPeriodCapForTesting` for as long as the returned token lives, then puts back what was + /// there before the first live pin. Suites run in parallel, so pins count: the cap is restored when + /// the last one is released, not when the first suite finishes. A test suite keeps the token in a + /// stored property, which lives exactly as long as the suite instance serving one test. + /// All pins on one budget are expected to pin the same cap: the last pin's value stays in force until + /// the last release. + func pinQuietPeriodCapForTesting(_ cap: TimeInterval?) -> QuietPeriodCapPin { + lock.lock(); defer { lock.unlock() } + if quietPeriodCapPins == 0 { quietPeriodCapBeforePins = quietPeriodCapForTesting } + quietPeriodCapPins += 1 + quietPeriodCapForTesting = cap + return QuietPeriodCapPin(budget: self) + } + + /// The token `pinQuietPeriodCapForTesting` hands out. Released exactly once, by `release()` or by + /// `deinit`, whichever comes first. + final class QuietPeriodCapPin: @unchecked Sendable { + private let tokenLock = NSLock() + private var budget: OriginRequestBudget? + + fileprivate init(budget: OriginRequestBudget) { self.budget = budget } + + deinit { release() } + + func release() { + let held = tokenLock.withLock { () -> OriginRequestBudget? in + let held = budget + budget = nil + return held + } + held?.releaseQuietPeriodCapPin() + } + } + + fileprivate func releaseQuietPeriodCapPin() { + lock.lock(); defer { lock.unlock() } + quietPeriodCapPins -= 1 + if quietPeriodCapPins == 0 { + quietPeriodCapForTesting = quietPeriodCapBeforePins + quietPeriodCapBeforePins = nil + } + } + /// Scheme + host + port, matching `SuffixRangeSupport.originKey`. Deliberately NOT the full /// URL: a metered CDN hands out signed URLs with a rotating token, so a per-URL budget would /// start at zero on every refresh and cap nothing. What meters us is the host. diff --git a/Sources/AetherEngine/Native/SoftwareFlushRecovery.swift b/Sources/AetherEngine/Native/SoftwareFlushRecovery.swift new file mode 100644 index 000000000..5de984a23 --- /dev/null +++ b/Sources/AetherEngine/Native/SoftwareFlushRecovery.swift @@ -0,0 +1,243 @@ +import Foundation + +/// What can ask the software host to put a flushed or failed output queue back to work. The system +/// is free to throw a queue away or fail it under the app: the video renderer says it +/// `requiresFlushToResumeDecoding` (and its status turns `.failed`) when the app loses the right to +/// use decoder resources, and either renderer can end up `.failed`. AVPlayer recovers from these on +/// its own; the host owns its queues, so the recovery is its job. +/// +/// The audio renderer flushing itself on a route change (AE#549) is deliberately not a trigger: it +/// leaves the video queue alone, so the picture is not behind anything, and re-feeding the audio it +/// dropped would mean a seek that cuts the audio still playing. +enum FlushRecoveryTrigger: String, Sendable, CaseIterable { + /// `AVSampleBufferVideoRenderer.requiresFlushToResumeDecoding` turned true. + case videoQueueRequiresFlush + /// The video renderer's status was `.failed` when a frame was handed to it, or a decode failure left it so. + case videoQueueFailed + /// The audio renderer's status reads `.failed`. + case audioRendererFailed +} + +/// The rules, pure so they can be checked without a renderer. +/// +/// A flushed video queue is refilled from the decoder, which restarts on the next keyframe. The +/// picture is frozen from the playhead until that keyframe is shown; the audio keeps playing the +/// whole time, as it did before any of this (AE#549). Waiting for the keyframe is enough when the +/// keyframe lies within the audio already queued: the freeze then ends before the audio lead does. +/// When the keyframe lies beyond it, or none has been read yet, the freeze outlasts the audio lead +/// and only a reposition puts a picture back at the playhead: both queues and both decoders +/// flushed, the demuxer put back on the keyframe before the playhead, the skip threshold hiding the +/// preroll and the clock re-anchored there. That is the one case where the audio is cut. +/// +/// On the decoupled route (a VOD source with its own audio decoder) the demuxer reads video up to +/// the audio lead target and parks it, so the keyframe gap is mostly settled by one question: is a +/// keyframe parked at all? With one parked, it lies inside the lead and the rule reduces to +/// "reposition only if no keyframe is parked". The freeze the rule accepts can then run as long as +/// the audio lead target itself (several seconds), which is what a plain flush costs when the next +/// keyframe sits at the far end of the parked packets. +/// +/// Two backstops keep a queue the system keeps failing from freezing the picture or cutting the +/// audio over and over: a flush or reposition that is followed by no presented frame is not repeated +/// (a reposition degrades to a flush), and a reposition is not repeated within +/// `repositionBackoffSeconds` of the last one, whatever the frames did in between. A failed audio +/// renderer is flushed at most `maxAudioFlushesPerWindow` times within `audioFlushWindowSeconds`; one +/// that fails again past that is left alone, since a renderer that refuses the same input again and +/// again is not helped by one more flush. +enum FlushRecoveryPolicy { + enum Action: Equatable, Sendable { + /// Nothing to do: the session is gone, a seek already owns the queues, a recovery is under + /// way, or the last one has not been followed by a presented frame yet. + case ignore + /// Flush everything and put the demuxer back at the playhead (a seek to the current time). + case reposition + /// Flush the video queue and the decoder and carry on from the next keyframe, with the audio + /// playing on. + case flushVideoOnly + /// Flush the audio queue; it is refilled as the feed reaches the clock. + case flushAudioOnly + /// The app cannot decode video right now (background); the recovery is paid on the foreground return. + case deferToForeground + } + + struct State: Sendable { + var stopRequested = false + var hasSource = true + var seekInFlight = false + var recoveryPending = false + var backgroundAudioOnly = false + /// The queue took a frame at some point. Before that it holds nothing worth recovering, and + /// the failed-enqueue path has already flushed and revived it. + var firstFramePresented = true + /// False for a live source with no rewind ring: there is nothing to reposition to. + var canReposition = true + /// Frames the queue accepted since the last video recovery; nil if there has been none. + var framesPresentedSinceLastVideoRecovery: Int? + /// What the last video recovery did. + var lastVideoRecovery: Action? + /// From the playhead to the next keyframe the decoder has not seen, in seconds; nil when no + /// such keyframe has been read yet. + var videoGapSeconds: Double? + /// Audio queued ahead of the playhead, in seconds. + var audioLeadSeconds: Double = 0 + /// Seconds since the last reposition this rule asked for; nil if there has been none. + var secondsSinceLastReposition: Double? + /// The shortest time between two repositions. + var repositionBackoffSeconds = FlushRecoveryPolicy.repositionBackoffSeconds + /// Audio flushes this rule asked for within the last `audioFlushWindowSeconds`. + var audioFlushesInWindow = 0 + } + + /// The shortest time between two repositions, whatever happened in between. + static let repositionBackoffSeconds: Double = 3 + + /// The window an audio flush counts in, and how many it may take: a failed audio renderer that + /// keeps failing is flushed 3 times in 10 s, then left until the window has emptied. + static let audioFlushWindowSeconds: Double = 10 + static let maxAudioFlushesPerWindow = 3 + + /// Whether the picture would stay frozen for longer than the audio keeps playing from its queue. + /// A keyframe that has not been read yet lies beyond everything read, so it counts as longer. + static func videoGapOutlastsAudioLead(gap: Double?, lead: Double) -> Bool { + guard let gap else { return true } + return gap > lead + } + + static func decide(_ trigger: FlushRecoveryTrigger, _ s: State) -> Action { + guard !s.stopRequested, s.hasSource, !s.seekInFlight, !s.recoveryPending else { return .ignore } + switch trigger { + case .audioRendererFailed: + return s.audioFlushesInWindow >= maxAudioFlushesPerWindow ? .ignore : .flushAudioOnly + case .videoQueueRequiresFlush, .videoQueueFailed: + if s.backgroundAudioOnly { return .deferToForeground } + guard s.firstFramePresented else { return .ignore } + // No frame was presented since the last recovery: it did not help, and repeating it cannot. + // A reposition that came to nothing degrades to a plain flush, which the system gets to + // fail once more; a plain flush that came to nothing is not repeated. + if s.framesPresentedSinceLastVideoRecovery == 0 { + return s.lastVideoRecovery == .reposition ? .flushVideoOnly : .ignore + } + guard s.canReposition, + videoGapOutlastsAudioLead(gap: s.videoGapSeconds, lead: s.audioLeadSeconds) + else { return .flushVideoOnly } + if let since = s.secondsSinceLastReposition, since < s.repositionBackoffSeconds { return .flushVideoOnly } + return .reposition + } + } + + /// Whether the video queue needs the flush, from what the renderer reports. + static func videoQueueNeedsFlush(requiresFlush: Bool, failed: Bool) -> Bool { requiresFlush || failed } +} + +/// The video keyframes the demux loop has read and parked for the decoder but not yet handed to it, +/// and the "decode again from a keyframe" gate a recovery flush raises. +/// +/// After a flush the decoder restarts on the first keyframe, so its time is where the picture +/// comes back. Fed by the reading side and the decode hand-off, read by the main actor. +/// +/// A plain recovery flush leaves the demuxer where it stood, so the packets that follow are usually +/// delta frames of a group of pictures whose keyframe went into the flushed decoder. Handed over, +/// they decode against nothing: the software decoder errors, and the hardware one needs a rebuilt +/// session to refuse them. The gate makes the demux thread drop video packets until a keyframe +/// comes, at both decoder hand-offs. +final class SWKeyframeLedger: @unchecked Sendable { + private let lock = NSLock() + private var pending: [Double] = [] + private var awaitingKeyframe = false + private var droppedWhileAwaiting = 0 + + /// A keyframe sits in the parked-packet FIFO of at most `parkedVideoCap` packets (256), so no more + /// than that many can be pending; the bound only guards a decoder that stops taking packets. + static let capacity = 256 + + /// A source whose keyframes carry no flag would otherwise lose its video for good; after this many + /// dropped packets the gate lets the next one through. + static let gateDropLimit = 4096 + + func noteKeyframeRead(pts: Double) { + guard pts.isFinite else { return } + lock.lock() + pending.append(pts) + if pending.count > Self.capacity { pending.removeFirst(pending.count - Self.capacity) } + lock.unlock() + } + + /// A packet went into the decoder. Keyframes are in order, so a keyframe handed over retires + /// every earlier one too. + func noteHandedToDecoder(pts: Double, isKeyframe: Bool) { + guard isKeyframe, pts.isFinite else { return } + lock.lock() + pending.removeAll { $0 <= pts } + lock.unlock() + } + + /// A keyframe that was noted when it was read and then discarded without reaching the decoder. + func noteDropped(pts: Double, isKeyframe: Bool) { + guard isKeyframe, pts.isFinite else { return } + lock.lock() + if let index = pending.firstIndex(of: pts) { pending.remove(at: index) } + lock.unlock() + } + + /// The next keyframe the decoder has not seen, or nil when none has been read. + var nextKeyframe: Double? { + lock.lock() + defer { lock.unlock() } + return pending.first + } + + /// What the gate says about a video packet about to go to the decoder. + enum GateVerdict: Equatable, Sendable { + /// Hand it over; no recovery is waiting for a keyframe. + case pass + /// A recovery is waiting for a keyframe and this is not one: drop it. + case drop + /// The keyframe the recovery was waiting for: hand it over. If the decoder refuses it (a + /// flush landed during the hand-off), the caller raises the gate again. + case passAndOpen + } + + /// A recovery flushed the video decoder: drop video until the next keyframe. Raised before the + /// flush and again after it, so a keyframe handed over in between, and thrown away by the flush, + /// does not count as the one that was waited for. + func requireKeyframe() { + lock.lock() + awaitingKeyframe = true + droppedWhileAwaiting = 0 + lock.unlock() + } + + var isAwaitingKeyframe: Bool { + lock.lock() + defer { lock.unlock() } + return awaitingKeyframe + } + + func gate(isKeyframe: Bool) -> GateVerdict { + lock.lock() + defer { lock.unlock() } + guard awaitingKeyframe else { return .pass } + if isKeyframe || droppedWhileAwaiting >= Self.gateDropLimit { + awaitingKeyframe = false + return .passAndOpen + } + droppedWhileAwaiting += 1 + return .drop + } + + /// The parked packets were thrown away (video not wanted in the background, a seam, a seek + /// noticed by the reading side). The gate is a decoder-side matter and stays as it was. + func discardPending() { + lock.lock() + pending.removeAll() + lock.unlock() + } + + /// A reposition emptied the parked packets and restarts the decoder on a keyframe of its own. + func reset() { + lock.lock() + pending.removeAll() + awaitingKeyframe = false + droppedWhileAwaiting = 0 + lock.unlock() + } +} diff --git a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift index 39a91abde..24d875c6a 100644 --- a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift +++ b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift @@ -942,6 +942,8 @@ final class SoftwarePlaybackHost { /// AE#739: the hardware decoder lost its session while nothing could act on it, or VT refused the /// rebuild. Paid by `exitBackgroundAudioOnly()`. Main actor only. private var videoSessionRecoveryOwed = false + /// How many keyframes have retried a refused rebuild in the foreground. Main actor only. + private var videoSessionRetry = VideoSessionRetryPolicy() nonisolated(unsafe) private var _backgroundAudioOnly = false nonisolated var backgroundAudioOnly: Bool { get { flagsLock.lock(); defer { flagsLock.unlock() }; return _backgroundAudioOnly } @@ -971,6 +973,7 @@ final class SoftwarePlaybackHost { // VT-backed one when the source's video codec is HEVC. self.videoDecoder = SoftwareVideoDecoder() armDisplaySizeObserver() + armQueueFlushObserver() } /// #353: the renderer settles the picture size on the decode thread, where it builds the format @@ -1401,6 +1404,9 @@ final class SoftwarePlaybackHost { (videoDecoder as? any VideoSessionRebuilding)?.onSessionLost = { [weak self] in Task { @MainActor [weak self] in self?.handleVideoSessionLost() } } + (videoDecoder as? any VideoSessionKeyframeReporting)?.onKeyframeWhileLost = { [weak self] in + Task { @MainActor [weak self] in self?.handleKeyframeWhileSessionLost() } + } videoDecoder.onA53Captions = { [weak self] triplets, pts in self?.onA53Captions?(triplets, pts) } @@ -1820,18 +1826,28 @@ final class SoftwarePlaybackHost { /// return, so a recovery owed from a loss seen outside audio-only mode (a PiP session, a refused /// rebuild) is paid here as well. func exitBackgroundAudioOnly() { + // A foreground return is a new episode: the retries spent before it do not count against this one. + videoSessionRetry.reset() if backgroundAudioOnly { - videoDecoder.flush() - renderer.flush() + flushVideoForRecovery(removingDisplayedImage: true) let rebuilt = (videoDecoder as? any VideoSessionRebuilding)?.rebuildSession() if rebuilt == false { videoSessionRecoveryOwed = true } + if rebuilt == true { videoSessionRetry.reset() } backgroundAudioOnly = false EngineLog.emit( "[SWHost] background audio-only left, video resumes at the next keyframe" + (rebuilt.map { ", hardware session rebuilt=\($0)" } ?? ""), category: .swPlayback) } - if videoSessionRecoveryOwed { recoverVideoSession() } + let repositioned = videoSessionRecoveryOwed ? recoverVideoSession() : false + // The video queue failed while video was not wanted. It was flushed above and resumes at the + // next keyframe; whether that is soon enough is the same question as for a failure in the + // foreground, so it goes through the same rule. An ignored answer (a seek already running) drops + // the debt on purpose: that seek repositions the session anyway. + if videoQueueFlushOwed { + videoQueueFlushOwed = false + if !repositioned { handleFlushTrigger(.videoQueueRequiresFlush) } + } } #if DEBUG @@ -1861,24 +1877,56 @@ final class SoftwarePlaybackHost { recoverVideoSession() } + /// A keyframe was fed while the session is lost. A rebuild that VideoToolbox refused in the foreground + /// is retried here, once per keyframe, up to `VideoSessionRetryPolicy.maxAttempts`; after that the + /// failure is surfaced instead of leaving a frozen picture beside live audio. Nothing is owed while the + /// first rebuild after a loss is still in flight, or in audio-only mode, so those keyframes do nothing. + private func handleKeyframeWhileSessionLost() { + guard !stopRequested, !backgroundAudioOnly else { return } + switch videoSessionRetry.onKeyframe(owed: videoSessionRecoveryOwed, now: ProcessInfo.processInfo.systemUptime) { + case .none: + break + case .retry(let attempt): + EngineLog.emit( + "[SWHost] AE#739 video session still lost at a keyframe, rebuild retry \(attempt) of " + + "\(VideoSessionRetryPolicy.maxAttempts)", + category: .swPlayback) + recoverVideoSession() + case .giveUp: + EngineLog.emit( + "[SWHost] AE#739 video session could not be rebuilt after \(VideoSessionRetryPolicy.maxAttempts) " + + "keyframe retries, giving up", + category: .swPlayback) + // `videoSessionRecoveryOwed` stays set on purpose: the failure is published, but a later + // foreground return still gets its own try (and a fresh budget). + if failure == nil { + failure = PlaybackErrorInfo( + kind: .softwarePipelineFailed, + message: "The video decoder could not be restarted after the system reclaimed it.") + } + } + } + /// Rebuild the session, then reposition to the playhead: the demuxer lands on the keyframe before it /// and the seek's skip threshold hides the preroll, so the picture returns on the frame the audio is /// at instead of freezing until the next keyframe. A live source without a rewind ring cannot go /// back and picks up at the next keyframe. A refused rebuild stays owed to the next foreground return. - private func recoverVideoSession() { + @discardableResult + private func recoverVideoSession() -> Bool { guard !stopRequested, let hw = videoDecoder as? any VideoSessionRebuilding else { videoSessionRecoveryOwed = false - return + return false } guard hw.rebuildSession() else { videoSessionRecoveryOwed = true - return + return false } videoSessionRecoveryOwed = false + videoSessionRetry.reset() if isLive && dvrRing == nil { EngineLog.emit("[SWHost] AE#739 video session rebuilt on a live source, resuming at the next keyframe", category: .swPlayback) - return + return false } let target = currentTime EngineLog.emit( @@ -1888,6 +1936,212 @@ final class SoftwarePlaybackHost { Task { @MainActor [weak self] in await self?.seek(to: target) } + return true + } + + // MARK: - Flush recovery + + /// The video queue (or the audio renderer) was flushed or failed by the system and holds none of what + /// the demuxer already handed over. See `FlushRecoveryPolicy` for the rules: audio keeps playing and + /// the picture resumes at the next keyframe, unless that keyframe is further away than the audio + /// lead, in which case the recovery is a seek to the playhead. + /// + /// Main actor only, like every transport call here. The renderers report from arbitrary threads and + /// hop here, so a trigger runs after any seek the user already started has set its in-flight flag. + private var flushRecoveryPending = false + /// AE#739 sibling: a video recovery the background could not pay. Paid by `exitBackgroundAudioOnly()`. + private var videoQueueFlushOwed = false + /// Recovery repositions that have finished, whatever they landed on. Diagnostics and tests. + private(set) var flushRecoveriesCompleted = 0 + /// What the last video recovery was, and how many frames the clock had passed on a healthy queue when it ran. + private var lastVideoRecovery: (action: FlushRecoveryPolicy.Action, presentedFrames: Int)? + /// When the last reposition was started (`ProcessInfo.systemUptime`); the backoff rule's input. + private var lastRepositionUptime: Double? + /// The shortest time between two repositions; tests shorten it. + private var repositionBackoffSeconds = FlushRecoveryPolicy.repositionBackoffSeconds + /// The audio renderer's status as the last poll saw it, so a failure is acted on once, when it begins. + private var audioRendererWasFailed = false + /// When the audio flushes of the last `FlushRecoveryPolicy.audioFlushWindowSeconds` ran (`ProcessInfo.systemUptime`). + private var audioFlushUptimes: [Double] = [] + /// The last few decisions, newest last, for the diagnostic line and the tests. + private(set) var flushRecoveryLog: [(trigger: FlushRecoveryTrigger, action: FlushRecoveryPolicy.Action)] = [] + /// Keyframes read and not yet handed to the decoder; see `SWKeyframeLedger`. + private let keyframeLedger = SWKeyframeLedger() + + private func armQueueFlushObserver() { + renderer.setQueueFlushObserver { [weak self] reason in + Task { @MainActor [weak self] in self?.handleFlushTrigger(reason.trigger) } + } + } + + /// The audio the synchronizer's clock has still to play: what the feed has queued, less the clock. + private func audioLeadSeconds(clock: Double) -> Double { + let fedUntil = Self.diagAudioMarker( + demuxLoopPts: demuxDiag.snapshot.lastAudioPts, + dvrPumpPts: audioLookahead.lastFedAudioPTS, + isDVRSession: isLive && dvrRing != nil) + guard fedUntil.isFinite, clock.isFinite else { return 0 } + return max(0, fedUntil - clock) + } + + private func flushRecoveryState() -> FlushRecoveryPolicy.State { + var state = FlushRecoveryPolicy.State() + state.stopRequested = stopRequested + state.hasSource = demuxer != nil && isReady && failure == nil + state.seekInFlight = seekInFlight + state.recoveryPending = flushRecoveryPending + state.backgroundAudioOnly = backgroundAudioOnly + let clock = audioOutput?.currentTimeSeconds ?? .nan + // Before the first frame, any frame handed to a healthy queue is enough: it holds nothing worth + // recovering otherwise. After a recovery the storm rule wants more than an enqueue the queue + // may still refuse: a frame the clock has passed while the queue stayed healthy. + state.firstFramePresented = renderer.acceptedFrameCount > 0 + state.canReposition = !(isLive && dvrRing == nil) + if let last = lastVideoRecovery { + state.framesPresentedSinceLastVideoRecovery = + renderer.presentedFrameCount(clockSeconds: clock) - last.presentedFrames + state.lastVideoRecovery = last.action + } + state.repositionBackoffSeconds = repositionBackoffSeconds + if let at = lastRepositionUptime { + state.secondsSinceLastReposition = ProcessInfo.processInfo.systemUptime - at + } + let now = ProcessInfo.processInfo.systemUptime + audioFlushUptimes.removeAll { now - $0 >= FlushRecoveryPolicy.audioFlushWindowSeconds } + state.audioFlushesInWindow = audioFlushUptimes.count + state.audioLeadSeconds = audioLeadSeconds(clock: clock) + if clock.isFinite, let keyframe = keyframeLedger.nextKeyframe { + state.videoGapSeconds = max(0, keyframe - clock) + } + return state + } + + /// The position the recovery returns to, on the session axis `seek(to:)` takes. The synchronizer is + /// read directly on VOD (the published `currentTime` lags it by up to a tick); a live DVR session + /// keeps its own offset and uses the published value. + private func recoveryPlayheadSeconds() -> Double { + if !isLive, let raw = audioOutput?.currentTimeSeconds, raw.isFinite, raw >= 0 { + let zero = clockSessionZero + return zero > 0 ? max(0, raw - zero) : raw + } + return currentTime + } + + /// Flushes the video decoder and the video queue, with the audio left alone, and puts the skip + /// threshold at the playhead. The decoder restarts on a keyframe that can sit past the playhead, + /// and the renderer's floor, restarted from nothing, would start at that + /// keyframe: the B-frames that follow it in decode order but precede it in display order would + /// then arrive below it. A floor at the playhead is below every frame still to be shown (the start + /// a seek gives it too), and frames under it are late by definition. + /// + /// The demux thread keeps handing packets to the decoder while this runs, and what follows the flush + /// is usually delta frames of a group of pictures whose keyframe the flush just discarded. The + /// keyframe gate makes the demux thread drop video until the next keyframe; it is raised before the + /// flush, so no delta frame is handed over between the two, and again after it, so a keyframe handed + /// over in between and discarded by the flush does not end the wait. + private func flushVideoForRecovery(removingDisplayedImage: Bool) { + keyframeLedger.requireKeyframe() + let clock = audioOutput?.currentTimeSeconds + let threshold = clock.flatMap { $0.isFinite && $0 >= 0 ? CMTime(seconds: $0, preferredTimescale: 90000) : nil } + videoDecoder.flush() + renderer.flush(removingDisplayedImage: removingDisplayedImage, skipThreshold: threshold) + if let threshold { videoDecoder.skipUntilPTS = threshold } + // The second raise covers a keyframe the demux thread handed over after the first raise and + // before the flush above: the gate opened for it, the flush discarded it, and the delta frames + // behind it must not follow. The cost is that a keyframe decoded under the new epoch in this + // short window is gated too, one extra group of pictures with the old picture held, never garbage. + keyframeLedger.requireKeyframe() + } + + #if DEBUG + /// Tests drive the recovery through the same entry the renderers use, without a session that has + /// decoded a frame. + var videoQueueFlushOwedForTesting: Bool { videoQueueFlushOwed } + var audioRendererForTesting: AVSampleBufferAudioRenderer? { audioOutput?.renderer } + var videoRendererForTesting: SampleBufferRenderer { renderer } + var keyframeLedgerForTesting: SWKeyframeLedger { keyframeLedger } + func setLiveForTesting(_ live: Bool) { isLive = live } + func setVideoDecoderForTesting(_ decoder: any VideoDecodingPipeline) { videoDecoder = decoder } + func seekClockForTesting(to seconds: Double) { + audioOutput?.seekClock(to: CMTime(seconds: seconds, preferredTimescale: 90000), rate: 0) + } + func setAudioFedUntilForTesting(_ pts: Double) { + demuxDiag.update(lastAudioPts: pts, parked: 0, rebuffering: false, generation: .max) + } + /// A feed marker written under the current generation, as the demux loop writes it on every iteration. + func setAudioFedUntilUnderCurrentGenerationForTesting(_ pts: Double) { + demuxDiag.update(lastAudioPts: pts, parked: 0, rebuffering: false, generation: seekGeneration) + } + var audioFedUntilForTesting: Double { demuxDiag.snapshot.lastAudioPts } + var recoveryPendingForTesting: Bool { flushRecoveryPending } + func setRepositionBackoffForTesting(_ seconds: Double) { repositionBackoffSeconds = seconds } + #endif + + @discardableResult + func handleFlushTrigger(_ trigger: FlushRecoveryTrigger) -> FlushRecoveryPolicy.Action { + let state = flushRecoveryState() + let action = FlushRecoveryPolicy.decide(trigger, state) + guard action != .ignore else { return action } + flushRecoveryLog.append((trigger, action)) + if flushRecoveryLog.count > 16 { flushRecoveryLog.removeFirst(flushRecoveryLog.count - 16) } + // A deferral did nothing yet; counting it would turn the recovery it is owed to into a repeat. + switch action { + case .reposition, .flushVideoOnly: + lastVideoRecovery = (action, renderer.presentedFrameCount(clockSeconds: audioOutput?.currentTimeSeconds ?? .nan)) + if action == .reposition { lastRepositionUptime = ProcessInfo.processInfo.systemUptime } + default: + break + } + switch action { + case .ignore: + break + case .deferToForeground: + videoQueueFlushOwed = true + EngineLog.emit("[SWHost] flush recovery (\(trigger.rawValue)): video not wanted now, paid on the foreground return", + category: .swPlayback) + case .flushVideoOnly: + flushVideoForRecovery(removingDisplayedImage: false) + EngineLog.emit("[SWHost] flush recovery (\(trigger.rawValue)): video queue and decoder flushed, resuming at the next keyframe" + + (state.canReposition ? "" : " (live, no rewind)"), + category: .swPlayback) + case .flushAudioOnly: + audioFlushUptimes.append(ProcessInfo.processInfo.systemUptime) + audioOutput?.flush() + // The flush emptied the queue the diagnostic's audio marker describes (as a seek's does), so + // the lead read from it would stay overstated until the feed wrote a new one; the demux + // loop republishes its own marker every iteration, so the stale value itself is retired. + demuxDiag.audioQueueEmptied() + EngineLog.emit("[SWHost] flush recovery (\(trigger.rawValue)): audio queue flushed", + category: .swPlayback) + case .reposition: + let target = recoveryPlayheadSeconds() + flushRecoveryPending = true + EngineLog.emit( + "[SWHost] flush recovery (\(trigger.rawValue)): repositioning to the playhead at " + + "\(String(format: "%.3f", target))s (next keyframe " + + "\(state.videoGapSeconds.map { String(format: "%.1fs", $0) } ?? "not read") ahead, audio lead " + + "\(String(format: "%.1fs", state.audioLeadSeconds)))", + category: .swPlayback) + Task { @MainActor [weak self] in + await self?.seek(to: target) + self?.flushRecoveryPending = false + self?.flushRecoveriesCompleted += 1 + } + } + return action + } + + /// The audio renderer's status, read on the time-update cadence: a failed renderer posts nothing, + /// so its status is the only witness. Acted on when the failure begins, not on every read of it. + func pollAudioRendererStatus(_ status: AVQueuedSampleBufferRenderingStatus) { + guard status == .failed else { + audioRendererWasFailed = false + return + } + guard !audioRendererWasFailed else { return } + // Latched only once the failure was acted on: a trigger the rule ignored (a seek in flight, a + // recovery pending, no source yet) is still owed, and the next poll offers it again. + if handleFlushTrigger(.audioRendererFailed) != .ignore { audioRendererWasFailed = true } } func setRate(_ newRate: Float) { @@ -1972,6 +2226,7 @@ final class SoftwarePlaybackHost { // AE#479: the flush just emptied the queue the diag marker describes, and the pump learns of // this seek only at its next iteration, which a paused landing never reaches until play(). demuxDiag.audioFlushed(generation: generation) + keyframeLedger.reset() // Live source is forward-only; DVR rewind reseeds decoders from the ring without touching the live demuxer's read position. if isLive, let ring = dvrRing { @@ -2319,6 +2574,7 @@ final class SoftwarePlaybackHost { isPlaying = false isRebuffering = false seekInFlight = false + videoQueueFlushOwed = false // A teardown inside a seek's window would otherwise leave it open on this instance. noteSeekSettled(seekGeneration) timeTimer?.cancel() @@ -2619,6 +2875,7 @@ final class SoftwarePlaybackHost { } let diag = demuxDiag + let keyframeLedger = keyframeLedger let readAhead = vodPacketReadAhead demuxQueue.async { Self.runDemuxLoop( @@ -2661,6 +2918,7 @@ final class SoftwarePlaybackHost { subtitleTapSink: getSubtitleTapSink, recordingTap: recordingTap, playedMedia: playedMediaLedger, + keyframeLedger: keyframeLedger, noteRebuffering: noteRebuffering ) } @@ -3373,8 +3631,35 @@ final class SoftwarePlaybackHost { subtitleTapSink: @Sendable () -> ((@Sendable (Int32, UnsafeMutablePointer, AVRational, Bool) -> Void)?) = { nil }, recordingTap: @escaping @Sendable (UnsafeMutablePointer) -> Void = { _ in }, playedMedia: PlayedMediaLedger? = nil, + keyframeLedger: SWKeyframeLedger? = nil, noteRebuffering: @Sendable (Bool) -> Void = { _ in } ) { + // The video packet's time on the clock's axis and whether it is a keyframe, for `keyframeLedger`. + func keyframeTime(of packet: UnsafeMutablePointer) -> (pts: Double, isKey: Bool)? { + let pts = packet.pointee.pts + guard pts != Int64.min, videoTimeBaseSeconds > 0 else { return nil } + return (Double(pts) * videoTimeBaseSeconds, (packet.pointee.flags & AV_PKT_FLAG_KEY) != 0) + } + func noteVideoPacketParked(_ packet: UnsafeMutablePointer) { + if let t = keyframeTime(of: packet), t.isKey { keyframeLedger?.noteKeyframeRead(pts: t.pts) } + } + func noteVideoPacketDecoded(_ packet: UnsafeMutablePointer) { + if let t = keyframeTime(of: packet) { keyframeLedger?.noteHandedToDecoder(pts: t.pts, isKeyframe: t.isKey) } + } + func noteVideoPacketDropped(_ packet: UnsafeMutablePointer) { + if let t = keyframeTime(of: packet) { keyframeLedger?.noteDropped(pts: t.pts, isKeyframe: t.isKey) } + } + // A recovery flush leaves the decoder with no reference chain: until a keyframe comes the packets + // that follow decode against nothing, so they are dropped here, at the one place both routes hand + // video to the decoder. A dropped packet is never a keyframe, so the ledger needs no correction. + func recoveryGate(_ packet: UnsafeMutablePointer) -> SWKeyframeLedger.GateVerdict { + keyframeLedger?.gate(isKeyframe: (packet.pointee.flags & AV_PKT_FLAG_KEY) != 0) ?? .pass + } + // The decoder refuses a packet whose epoch a flush has moved on from. If that was the keyframe + // the gate had just let through, the recovery is still waiting for one. + func reassertGateIfRefused(_ verdict: SWKeyframeLedger.GateVerdict, epoch: UInt64) { + if verdict == .passAndOpen, videoDecoder.feedEpoch != epoch { keyframeLedger?.requireKeyframe() } + } // Clock arming: one-shot latch (seekClock is not idempotent -- re-calling snaps clock back to initialClockTime). Shared with host so a seek before first audio isn't overridden by a late re-arm. // PTS-discontinuity fold (SW): accrue (jumpedPts - expectedContinuation) into discontinuityOffset; subtract from all subsequent PTS so the whole pipeline sees one continuous timeline. Threshold 10s (mirrors native producer); flush decoders at the seam. @@ -3446,6 +3731,7 @@ final class SoftwarePlaybackHost { av_packet_free_safe(p) } parkedVideo.removeAll() + keyframeLedger?.discardPending() } // Decode+enqueue parked video while the renderer will take it. Never blocks. @@ -3474,15 +3760,26 @@ final class SoftwarePlaybackHost { func drainParkedVideoNonblocking() { if seekGeneration() != parkedSeekGeneration { return } - let epoch = videoDecoder.feedEpoch while let head = parkedVideo.first, mayDecodeVideoPacket(head), !stopRequested(), !backgroundAudioOnly() { + // The epoch is read per packet, before the generation check below. A seek bumps the + // generation before it flushes the decoder, so an epoch read ahead of its flush is refused + // by the decoder and one read after it is behind a generation check that fails: either + // way a pre-seek packet is retired (AE#492). Read once per pass, a flush from a recovery + // in the middle of a batch made the decoder refuse the rest of packets that had already + // left the FIFO and the ledger, which the recovery's own gate now accounts for instead. + let epoch = videoDecoder.feedEpoch // Leave the rest standing: the next iteration of the loop frees the FIFO as a batch // once it has read the new generation. if seekGeneration() != parkedSeekGeneration { return } let p = parkedVideo.removeFirst() - setDecodeGeneration(parkedSeekGeneration) - videoDecoder.decode(packet: p, epoch: epoch) + let verdict = recoveryGate(p) + if verdict != .drop { + setDecodeGeneration(parkedSeekGeneration) + noteVideoPacketDecoded(p) + videoDecoder.decode(packet: p, epoch: epoch) + reassertGateIfRefused(verdict, epoch: epoch) + } av_packet_unref(p) av_packet_free_safe(p) } @@ -3911,11 +4208,16 @@ final class SoftwarePlaybackHost { if backgroundAudioOnly() { av_packet_unref(packet) av_packet_free_safe(packet) + // What is parked is behind a hole now: the foreground return flushes the decoder and + // reads on from here. Keeping it would hand the decoder packets that no longer + // follow what it last saw, and leave the ledger describing keyframes that are gone. + freeParkedVideo() return true } if decoupleAudio { // Park; the drains at the iteration top and the bounded-park gate feed the // renderer. Ownership moves to the FIFO - no free here. + noteVideoPacketParked(packet) parkedVideo.append(packet) // Broken-audio clock-arming fallback still applies (decoupleAudio only // requires a DECLARED audio track, not a producing one). The stream that @@ -3927,6 +4229,7 @@ final class SoftwarePlaybackHost { drainParkedVideoNonblocking() return true } + noteVideoPacketParked(packet) // Back-pressure via SampleBufferRenderer.isReadyForMoreMediaData (not the deprecated layer property). Park on condition while paused to avoid 200 Hz CPU spin. // Queue readiness plus the FEL-only look-ahead cap. Hold the compressed packet here, // rather than letting an asynchronous decode allocate another composed buffer. @@ -3968,6 +4271,7 @@ final class SoftwarePlaybackHost { } // Entered background while parked on back-pressure: drop this frame. if backgroundAudioOnly() { + noteVideoPacketDropped(packet) av_packet_unref(packet) av_packet_free_safe(packet) return true @@ -3977,14 +4281,30 @@ final class SoftwarePlaybackHost { av_packet_free_safe(packet) return false } + // A recovery flush can land during the wait. The packet in hand is still current, and the + // epoch read before the read would make the decoder refuse it; if it is the keyframe the + // gate waits for, the picture would wait another group of pictures. Read again here, before + // the generation check: a seek bumps the generation before it flushes the decoder, so an + // epoch read ahead of a seek's flush is refused by the decoder, and one read after it is + // behind a generation check that fails. + epochBeforeRead = videoDecoder.feedEpoch // Re-check generation after the back-pressure wait (seek can land there; pre-seek packet would clear skip thresholds). if seekGeneration() != genBeforeRead { + noteVideoPacketDropped(packet) + av_packet_unref(packet) + av_packet_free_safe(packet) + return true + } + let verdict = recoveryGate(packet) + if verdict == .drop { av_packet_unref(packet) av_packet_free_safe(packet) return true } setDecodeGeneration(genBeforeRead) + noteVideoPacketDecoded(packet) videoDecoder.decode(packet: packet, epoch: epochBeforeRead) + reassertGateIfRefused(verdict, epoch: epochBeforeRead) // Video-only / undecodable audio fallback: arm clock off first video packet (no buffer within the producer's first-buffer budget = decoder not recovering). if !clockArmed(), let aOut = audioOutput, audioDecoder == nil || (audioPacketsSeen >= audioFirstBufferPacketBudget && !audioBuffersProduced) { @@ -4131,6 +4451,10 @@ final class SoftwarePlaybackHost { self.checkSurfaceVisibilityIfDue() self.sampleResidentRanges() guard let aOut = self.audioOutput else { return } + self.pollAudioRendererStatus(aOut.renderer.status) + // Counted while the queue is healthy: by the time a recovery trigger arrives the queue + // has failed or been flushed, and the frames the clock passed would be uncounted. + _ = self.renderer.presentedFrameCount(clockSeconds: aOut.currentTimeSeconds) // #254: a reposition in flight holds `currentTime` at its target; the synchronizer is // still on the pre-seek anchor and would drag the published position backwards. guard !self.seekInFlight, self.failure == nil else { return } @@ -4327,6 +4651,9 @@ final class SWPlaybackDiagState: @unchecked Sendable { private let lock = NSLock() private var _lastAudioPts = Double.nan private var _audioFlushGeneration: UInt64 = 0 + /// The newest PTS that described a queue a recovery has since emptied; a write that does not go + /// beyond it is that same stale value coming round again, and is refused. + private var _emptiedAudioPts = -Double.infinity private var _parked = 0 private var _rebuffering = false private var _sourceExhausted = false @@ -4340,7 +4667,9 @@ final class SWPlaybackDiagState: @unchecked Sendable { /// `parked` and `rebuffering` are the pump's own state and stay unconditional. func update(lastAudioPts: Double, parked: Int, rebuffering: Bool, generation: UInt64) { lock.lock() - if generation >= _audioFlushGeneration { _lastAudioPts = lastAudioPts } + if generation >= _audioFlushGeneration, !(lastAudioPts.isFinite && lastAudioPts <= _emptiedAudioPts) { + _lastAudioPts = lastAudioPts + } _parked = parked _rebuffering = rebuffering lock.unlock() @@ -4353,11 +4682,22 @@ final class SWPlaybackDiagState: @unchecked Sendable { if generation >= _audioFlushGeneration { _audioFlushGeneration = generation _lastAudioPts = .nan + _emptiedAudioPts = -.infinity _sourceExhausted = false } lock.unlock() } + /// A recovery emptied the audio queue without a new generation: the pump is on the same one and its + /// local marker still names the flushed audio, so it is the value rather than the generation that + /// is retired. The next audio the pump enqueues goes beyond it and is taken. + func audioQueueEmptied() { + lock.lock() + if _lastAudioPts.isFinite { _emptiedAudioPts = max(_emptiedAudioPts, _lastAudioPts) } + _lastAudioPts = .nan + lock.unlock() + } + /// AE#374: the producer will yield nothing further. Set from the demux loops' end-of-media exit, /// read by the diagnostic line, which without it describes a finished session as a drifting one. func markSourceExhausted() { diff --git a/Sources/AetherEngine/Native/VideoSessionRetryPolicy.swift b/Sources/AetherEngine/Native/VideoSessionRetryPolicy.swift new file mode 100644 index 000000000..e0ebfcbb8 --- /dev/null +++ b/Sources/AetherEngine/Native/VideoSessionRetryPolicy.swift @@ -0,0 +1,49 @@ +import Foundation + +/// When to retry a video-session rebuild that VideoToolbox refused while the app is in the foreground. +/// +/// A refused rebuild leaves the decoder dropping every packet, with audio still playing. The retry is keyed +/// on the stream, not on a timer: each keyframe fed while the session is still lost is one chance, because +/// from a keyframe on a new session could show a picture again. The attempts are capped, so a session the +/// system will not give back ends in an error instead of a frozen picture that retries for ever. +struct VideoSessionRetryPolicy: Equatable { + /// Keyframes at which a rebuild is tried before the session is given up as unrecoverable. + static let maxAttempts = 4 + /// Wall-clock seconds between attempts. A lost decoder enqueues nothing, so the demux loop is not + /// paced and keyframes can arrive in a burst: without a gap the whole budget would go in milliseconds, + /// leaving a short-lived refusal no time to clear. + static let minSpacing: TimeInterval = 2.0 + + enum Decision: Equatable { + /// Try the rebuild now. + case retry(attempt: Int) + /// The cap was just reached: surface the failure, once. + case giveUp + /// Nothing to do: recovery is not owed, or the failure was already surfaced. + case none + } + + private(set) var attempts = 0 + private(set) var gaveUp = false + private var lastAttempt: TimeInterval? + + /// A keyframe was fed while the session is lost. `owed` is whether the host still owes a recovery. + mutating func onKeyframe(owed: Bool, now: TimeInterval) -> Decision { + guard owed, !gaveUp else { return .none } + if let lastAttempt, now - lastAttempt < Self.minSpacing { return .none } + if attempts >= Self.maxAttempts { + gaveUp = true + return .giveUp + } + attempts += 1 + lastAttempt = now + return .retry(attempt: attempts) + } + + /// A rebuild succeeded, or the app returned to the foreground: the next loss starts a fresh budget. + mutating func reset() { + attempts = 0 + gaveUp = false + lastAttempt = nil + } +} diff --git a/Sources/AetherEngine/PlayerState.swift b/Sources/AetherEngine/PlayerState.swift index 6a4834442..bbd74d898 100644 --- a/Sources/AetherEngine/PlayerState.swift +++ b/Sources/AetherEngine/PlayerState.swift @@ -907,6 +907,20 @@ public struct LoadOptions: Sendable, Equatable { /// are overlay-only until the next load. Default empty. public var externalSubtitles: [ExternalSubtitleTrack] + /// The subtitle track to activate at the end of a successful load, named by `TrackInfo.id`: the + /// container's stream index for an embedded track, or `externalSubtitleTrackIDBase` plus the + /// array index for an entry of `externalSubtitles`. For a host that already knows which track it + /// wants (a server's default subtitle index, a saved per-title choice) and so has no language to + /// rank by. It wins over `preferredSubtitleLanguages`. An id that names no subtitle track falls + /// through to the language preference, so a stale id never fails the load. A later host + /// `selectSubtitleTrack` / `clearSubtitle` overrides it, as it does the language preference. + /// Stream indices belong to one title, so a disc `selectTitle` drops the id for the reload and the + /// language preference decides on the new title; a same-content reload (audio switch, reroute) + /// keeps it. A track added later with `addExternalSubtitleTrack` whose id matches activates too, + /// while nothing is active and the host has made no choice. + /// nil (the default) leaves the choice to `preferredSubtitleLanguages`, else subtitles stay off. + public var initialSubtitleTrackID: Int? = nil + /// Forward-buffer window of the loopback HLS session, in segments (one segment ~ 4 s): how far the /// producer may race ahead of the playhead AND how many forward segments the on-disk cache keeps /// resident (the two are coupled by construction, see `SegmentCache`). Larger values buffer more of @@ -1071,6 +1085,7 @@ public struct LoadOptions: Sendable, Equatable { preferredAudioLanguages: [String] = [], preferredSubtitleLanguages: [String] = [], externalSubtitles: [ExternalSubtitleTrack] = [], + initialSubtitleTrackID: Int? = nil, forwardBufferSegments: Int? = nil, sessionCacheByteBudget: Int? = nil, softwarePacketCacheStorage: SoftwarePacketCacheStorage = .automatic, @@ -1130,6 +1145,7 @@ public struct LoadOptions: Sendable, Equatable { self.preferredAudioLanguages = preferredAudioLanguages self.preferredSubtitleLanguages = preferredSubtitleLanguages self.externalSubtitles = externalSubtitles + self.initialSubtitleTrackID = initialSubtitleTrackID self.forwardBufferSegments = forwardBufferSegments self.sessionCacheByteBudget = sessionCacheByteBudget self.softwarePacketCacheStorage = softwarePacketCacheStorage diff --git a/Sources/AetherEngine/PresentationAxis.swift b/Sources/AetherEngine/PresentationAxis.swift index 964707b6a..d73a4b896 100644 --- a/Sources/AetherEngine/PresentationAxis.swift +++ b/Sources/AetherEngine/PresentationAxis.swift @@ -50,6 +50,45 @@ enum PresentationAxis { return 0 } } + + /// The `seek(to:)` target for a disc chapter. Disc chapters are title-relative (0-based), the chapter's + /// source PTS is `sourceStartSeconds` (the source start origin) plus its start, and `seek(to:)` takes the + /// published axis of the active backend. + /// + /// - `.software`: the published axis is the session axis, source PTS minus the clock's session zero, + /// and the host adds that zero back before the seek reaches the demuxer. A container start past the + /// 2 s tolerance becomes the session zero (`SWClockAnchorPolicy`), so adding the full + /// `sourceStartSeconds` here as well applied the start twice and landed one start late. Only the part + /// of the start the zero did not take is added: all of it inside the tolerance (zero 0), none of it + /// past it. The host reports its zero through a main-actor hop, so until it arrives the zero is + /// predicted the way the host derives it at load (`SWClockAnchorPolicy.resumeSessionZero`), which + /// gives 0 for a start inside the tolerance. + /// - `.native`: the published axis is the 0-based display axis, the same one the disc chapters and the + /// title duration are on (AE#105: a disc title's display origin is the playlist shift itself, and + /// `seek(to:)` adds it back before it subtracts the shift again). Adding the shift here as well put + /// the target a whole clip base past the chapter, beyond the title on a base of 599 s. + /// - Any other backend keeps the base it was written with, `playlistShiftSeconds`. + /// + /// `sourceStartSeconds` must be the origin the host derives its zero from + /// (`Demuxer.resolvedSourceStartOrigin`), so the zero cancels it on the software route. + /// + /// Never negative: the floor only covers a zero that re-anchored past the chapter's source PTS (a + /// first sample later than the chapter), which would otherwise ask for a position before the title. + static func discChapterSeekTarget(chapterStart: Double, backend: PlaybackBackend, + sourceStartSeconds: Double, playlistShiftSeconds: Double, + softwareClockZero: Double?) -> Double { + let base: Double + switch backend { + case .software: + base = sourceStartSeconds - (softwareClockZero + ?? SWClockAnchorPolicy.resumeSessionZero(sourceOriginSeconds: sourceStartSeconds)) + case .native: + base = 0 + default: + base = playlistShiftSeconds + } + return Swift.max(0, chapterStart + base) + } } /// Which source PTS maps to display-0 for a session (AE#105, AE#270). diff --git a/Sources/AetherEngine/Renderer/SampleBufferRenderer.swift b/Sources/AetherEngine/Renderer/SampleBufferRenderer.swift index 099fdddf2..49685219b 100644 --- a/Sources/AetherEngine/Renderer/SampleBufferRenderer.swift +++ b/Sources/AetherEngine/Renderer/SampleBufferRenderer.swift @@ -406,6 +406,160 @@ final class SampleBufferRenderer: @unchecked Sendable { let layer = Self.makeDisplayLayer(isHDR: false, gravity: videoGravity) displayLayer = layer videoRenderer = layer.sampleBufferRenderer + observeQueueFlushRequirements() + } + + deinit { + for token in queueNotificationTokens { NotificationCenter.default.removeObserver(token) } + } + + // MARK: - Flush recovery + + /// Why the queue target needs a flush it did not get from the host. + enum QueueFlushReason: Sendable, Equatable { + /// `requiresFlushToResumeDecoding` turned true (the app lost the right to use decoder resources). + case requiresFlush + /// The queue target's status is `.failed`. + case failed + + var trigger: FlushRecoveryTrigger { + switch self { + case .requiresFlush: .videoQueueRequiresFlush + case .failed: .videoQueueFailed + } + } + } + + /// Told, on whatever thread noticed it and with no lock held, that the queue target needs a flush + /// and has lost what was in it. The host decides between a plain flush and a reposition to the + /// playhead (`FlushRecoveryPolicy`). Guarded by `reorderLock`. + private var _queueFlushObserver: (@Sendable (QueueFlushReason) -> Void)? + func setQueueFlushObserver(_ observer: (@Sendable (QueueFlushReason) -> Void)?) { + reorderLock.lock() + _queueFlushObserver = observer + reorderLock.unlock() + } + + private func reportQueueFlush(_ reason: QueueFlushReason) { + reorderLock.lock() + let observer = _queueFlushObserver + reorderLock.unlock() + observer?(reason) + } + + #if DEBUG + /// Test seam: what a failed enqueue does, without a renderer to fail. + func reportQueueFlushForTesting(_ reason: QueueFlushReason) { reportQueueFlush(reason) } + #endif + + /// Frames the queue target took while it was neither failed nor asking for a flush, counted after + /// the enqueue. `enqueueCount` also counts the frame handed over right after a failed queue was + /// revived, which proves nothing about the queue. The host reads this as "the queue has been given a + /// frame": before the first one there is nothing to recover. Whether a recovery helped is asked of + /// `presentedFrameCount(clockSeconds:)`, which wants more than an enqueue the queue may still + /// refuse. Guarded by `reorderLock`. + private var _acceptedFrameCount = 0 + var acceptedFrameCount: Int { + reorderLock.lock() + defer { reorderLock.unlock() } + return _acceptedFrameCount + } + + private var loggedRequiresFlushAtEnqueue = false + + /// Frames handed to a healthy queue whose time the clock has not passed yet, in source seconds, + /// oldest first. Guarded by `reorderLock`. + private var _pendingPresentation: [Double] = [] + private var _presentedFrameCount = 0 + private static let pendingPresentationCapacity = 256 + + /// Frames the clock has passed while the queue stayed healthy: the stronger evidence the storm rule + /// wants, since a queue can report itself healthy right after an enqueue and refuse the frame later. + /// A frame still ahead of the clock is not counted, and none is while the queue needs a flush; a + /// flush drops the ones not yet counted, as the queue dropped them. + func presentedFrameCount(clockSeconds: Double) -> Int { + let healthy = queueFlushReasonNow() == nil + reorderLock.lock() + defer { reorderLock.unlock() } + if healthy, clockSeconds.isFinite { + let passed = _pendingPresentation.prefix { $0 <= clockSeconds }.count + if passed > 0 { + _pendingPresentation.removeFirst(passed) + _presentedFrameCount += passed + } + } + return _presentedFrameCount + } + + /// The tokens for the notifications below; removed in `deinit`. + private var queueNotificationTokens: [NSObjectProtocol] = [] + + /// The renderer posts from an arbitrary thread, possibly from inside a call that already holds + /// `reorderLock` (`presentationTimeExpectation` is set under it), and the handlers take that lock. + /// They therefore only hop here, the way `AudioOutput` does for its renderer. + private let queueNotificationQueue = DispatchQueue(label: "engine.renderer.queue-flush") + + /// Before this nothing here observed either: a picture that froze after backgrounding stayed frozen + /// until the user seeked, while AVPlayer resumes by itself. + /// + /// The property is read back rather than taken from the payload: the notification says "changed", + /// the property says which way. + private func observeQueueFlushRequirements() { + let center = NotificationCenter.default + let renderer = videoRenderer + queueNotificationTokens.append(center.addObserver( + forName: AVSampleBufferVideoRenderer.requiresFlushToResumeDecodingDidChangeNotification, + object: renderer, queue: nil + ) { [weak self] _ in + self?.queueNotificationQueue.async { [weak self] in + guard let self, self.queueNeedsFlushNow() else { return } + EngineLog.emit("[Renderer] video queue requires a flush to resume decoding (status=\(self.statusName))", + category: .swPlayback) + self.reportQueueFlush(.requiresFlush) + } + }) + queueNotificationTokens.append(center.addObserver( + forName: AVSampleBufferVideoRenderer.didFailToDecodeNotification, + object: renderer, queue: nil + ) { [weak self] note in + let error = note.userInfo?[AVSampleBufferVideoRenderer.didFailToDecodeNotificationErrorKey] as? NSError + self?.queueNotificationQueue.async { [weak self] in + guard let self else { return } + EngineLog.emit("[Renderer] video queue failed to decode: \(error.map { "\($0.domain)/\($0.code)" } ?? "nil") (status=\(self.statusName))", + category: .swPlayback) + // One undecodable frame leaves the queue running; only a queue that is no longer is reported. + guard self.queueNeedsFlushNow() else { return } + self.reportQueueFlush(.failed) + } + }) + } + + #if DEBUG + /// Runs the work the notification handlers queue, then returns. Tests wait on this instead of sleeping. + func drainQueueNotificationsForTesting() { queueNotificationQueue.sync {} } + + /// The test answers below are set on one thread and read on another (the demux loop, the notification + /// queue), so they sit behind a lock. + private let testAnswerLock = NSLock() + private var _queueNeedsFlushForTesting: (@Sendable () -> Bool)? + private var _readyForMoreMediaDataForTesting: (@Sendable () -> Bool)? + + /// A renderer cannot be made to fail on demand; tests answer for it. + var queueNeedsFlushForTesting: (@Sendable () -> Bool)? { + get { testAnswerLock.lock(); defer { testAnswerLock.unlock() }; return _queueNeedsFlushForTesting } + set { testAnswerLock.lock(); _queueNeedsFlushForTesting = newValue; testAnswerLock.unlock() } + } + #endif + + private func queueNeedsFlushNow() -> Bool { queueFlushReasonNow() != nil } + + /// Why the queue target needs a flush right now, nil if it does not. Failed is the stronger of the two. + private func queueFlushReasonNow() -> QueueFlushReason? { + #if DEBUG + if let answer = queueNeedsFlushForTesting { return answer() ? .requiresFlush : nil } + #endif + if videoRenderer.status == .failed { return .failed } + return videoRenderer.requiresFlushToResumeDecoding ? .requiresFlush : nil } /// #303: what the display did with the frames, as the renderer itself counts them. Our own @@ -484,8 +638,24 @@ final class SampleBufferRenderer: @unchecked Sendable { /// isReadyForMoreMediaData stays optimistically true even when the renderer queue is full, causing /// FigVideoQueueRemote -12080 on over-enqueue. var isReadyForMoreMediaData: Bool { - queueTarget.isReadyForMoreMediaData + #if DEBUG + if let answer = readyForMoreMediaDataForTesting { return answer() } + #endif + return queueTarget.isReadyForMoreMediaData + } + + #if DEBUG + /// Holds the demux loop in its back-pressure wait on demand. + var readyForMoreMediaDataForTesting: (@Sendable () -> Bool)? { + get { testAnswerLock.lock(); defer { testAnswerLock.unlock() }; return _readyForMoreMediaDataForTesting } + set { testAnswerLock.lock(); _readyForMoreMediaDataForTesting = newValue; testAnswerLock.unlock() } + } + /// Frames handed to a healthy queue that no sample has counted yet. + var pendingPresentationCountForTesting: Int { + reorderLock.lock(); defer { reorderLock.unlock() } + return _pendingPresentation.count } + #endif /// A suspended process may return with a failed queue that never admits another frame. /// Recover before the feeder wakes; a healthy full queue must keep its buffered frames. @@ -537,6 +707,12 @@ final class SampleBufferRenderer: @unchecked Sendable { func setSkipThreshold(_ time: CMTime?) { reorderLock.lock() + installSkipThresholdLocked(time) + reorderLock.unlock() + } + + /// `reorderLock` held. + private func installSkipThresholdLocked(_ time: CMTime?) { skipUntilPTS = time // Nothing below the threshold will be handed over, and the first frame past it need not be // the lowest one to come, so the floor starts here rather than at that frame. @@ -544,7 +720,6 @@ final class SampleBufferRenderer: @unchecked Sendable { arrivalFloor?.reset(start: CMTimeGetSeconds(time)) arrivalSkipBelow = CMTimeGetSeconds(time) } - reorderLock.unlock() } /// #298: whether a frame's presentation timestamp can be scheduled at all. AV_NOPTS_VALUE reaches @@ -690,9 +865,15 @@ final class SampleBufferRenderer: @unchecked Sendable { /// Discard all buffered frames. `removingDisplayedImage: true` (stop/teardown) also clears the visible /// frame; `false` (seek) holds the last frame on screen until the post-seek frame is enqueued, so a seek /// doesn't flash black between the old and new positions (matches the hardware path's hold-last-frame). - func flush(removingDisplayedImage: Bool = true) { + /// + /// `skipThreshold` is installed under the same lock as the flush, for a recovery that restarts the + /// decoder on a keyframe past the playhead: a frame that arrives between the flush and a separate + /// `setSkipThreshold` call would otherwise be handed over below the playhead (and start the floor + /// at itself). + func flush(removingDisplayedImage: Bool = true, skipThreshold: CMTime? = nil) { reorderLock.lock() reorderBuffer.removeAll() + _pendingPresentation.removeAll() _lastEnqueuedFrame = nil // #407: the next frame handed over will not follow the last one, so the gap between them is // not a cadence measurement. Left standing, every seek would report one enormous interval. @@ -712,6 +893,7 @@ final class SampleBufferRenderer: @unchecked Sendable { // Invalidate the format description cache; the next load() may open a stream with different colorimetry at the same resolution. cachedFormatDesc = nil cachedFormatKey = nil + if let skipThreshold { installSkipThresholdLocked(skipThreshold) } reorderLock.unlock() videoRenderer.flush(removingDisplayedImage: removingDisplayedImage) { } @@ -775,20 +957,45 @@ final class SampleBufferRenderer: @unchecked Sendable { } // Recover from failed queue target (Synchronizer/controlTimebase handoff races can push it here; flush recovers it). let target = queueTarget - if queueStatus == .failed { - if !loggedLayerFailed { + // A queue that requires a flush but is not failed is stalled just the same, and its + // notification does not fire again: if the host ignored it (a seek was in flight, a recovery + // pending) this is the next chance to act on it. Treated like a failed queue: flushed, and + // reported so the host decides how the picture comes back. + if let reason = queueFlushReasonNow() { + if reason == .failed, !loggedLayerFailed { loggedLayerFailed = true EngineLog.emit("[Renderer] queue target failed at enqueue #\(enqueueCount + 1): \(queueError?.localizedDescription ?? "nil"), attempting recovery via flush()", category: .swPlayback) + } else if reason == .requiresFlush, !loggedRequiresFlushAtEnqueue { + loggedRequiresFlushAtEnqueue = true + EngineLog.emit("[Renderer] queue target requires a flush at enqueue #\(enqueueCount + 1) (status=\(statusName)), flushing", category: .swPlayback) } target.flush() + reorderLock.lock() + _pendingPresentation.removeAll() + reorderLock.unlock() + // The flush revives the queue, but what it held is gone and the demuxer stands ahead of + // it: the host decides how the picture comes back. + reportQueueFlush(reason) } if !target.isReadyForMoreMediaData, !loggedNotReady { loggedNotReady = true EngineLog.emit("[Renderer] isReadyForMoreMediaData=false at enqueue #\(enqueueCount + 1) status=\(statusName)", category: .swPlayback) } target.enqueue(sampleBuffer) + let accepted = queueFlushReasonNow() == nil reorderLock.lock() _lastEnqueuedFrame = pixelBuffer + if accepted { + _acceptedFrameCount += 1 + // The queue may still refuse the frame, and says so later than this read. It counts as + // presented only once the clock has passed it with the queue still healthy. + if pts.isNumeric { + _pendingPresentation.append(CMTimeGetSeconds(pts)) + if _pendingPresentation.count > Self.pendingPresentationCapacity { + _pendingPresentation.removeFirst(_pendingPresentation.count - Self.pendingPresentationCapacity) + } + } + } reorderLock.unlock() // #311: reported here rather than at admission, so it describes frames the compositor has diff --git a/Sources/AetherEngine/Video/VideoConfigRecord+InBand.swift b/Sources/AetherEngine/Video/VideoConfigRecord+InBand.swift index c18526e4b..64bd1b68d 100644 --- a/Sources/AetherEngine/Video/VideoConfigRecord+InBand.swift +++ b/Sources/AetherEngine/Video/VideoConfigRecord+InBand.swift @@ -64,6 +64,15 @@ extension VideoConfigRecord { return hvcC } + /// Input the in-band scan may read in total, foreign streams included. Parameter sets sit at the + /// first IRAP, so a few packets find them; this only caps a stream that never carries any. + static let inBandScanByteBudget = 16 * 1024 * 1024 + + static func scanMayContinue(packetsRead: Int, videoPackets: Int, bytesRead: Int, + packetBudget: Int, readBudget: Int, byteBudget: Int) -> Bool { + packetsRead < readBudget && videoPackets < packetBudget && bytesRead < byteBudget + } + /// Replace a video stream's empty hvcC with one rebuilt from the parameter sets at the head of the /// source, so the hardware-decode verdict and `HardwareVideoDecoder` see the record AVPlayer would /// have reconstructed for itself. Returns true when the record was replaced. @@ -71,8 +80,14 @@ extension VideoConfigRecord { /// Seekable sources only: the scan reads from the head and seeks back to 0, which is where every /// caller starts playback from anyway (the start position is applied later). A forward-only /// source is left alone, and keeps today's route. Runs off the main actor; it reads packets. + /// + /// The scan is bounded in packets and in bytes: a stream that carries no parameter sets, or one + /// whose packets are very large, ends it at `inBandScanByteBudget` of input rather than reading + /// until the packet count runs out, and leaves the record as it was. @discardableResult - static func completeHEVCRecordFromInBandParameterSets(demuxer: Demuxer, streamIndex: Int32) -> Bool { + static func completeHEVCRecordFromInBandParameterSets( + demuxer: Demuxer, streamIndex: Int32, byteBudget: Int = inBandScanByteBudget + ) -> Bool { // Stream access goes through `withStream`, never a pointer held across the packet reads: a // stop() closes the demuxer from another thread and frees its streams. let needsRebuild = demuxer.withStream(at: streamIndex) { stream -> Bool in @@ -89,23 +104,34 @@ extension VideoConfigRecord { var sets = HEVCParameterSets() var videoPackets = 0 var packetsRead = 0 + var bytesRead = 0 demuxer.seek(to: 0) - while packetsRead < readBudget, videoPackets < packetBudget, !sets.isComplete { - guard let pkt = try? demuxer.readPacket() else { break } - var toFree: UnsafeMutablePointer? = pkt - defer { trackedPacketFree(&toFree) } - packetsRead += 1 - guard pkt.pointee.stream_index == streamIndex else { continue } - videoPackets += 1 - if let data = pkt.pointee.data { - sets.collect(from: data, size: Int(pkt.pointee.size)) + // The ceiling is also armed on the input, so one oversized read cannot spend more than the + // budget between two checks of the loop. + demuxer.beginInputByteBudget(Int64(byteBudget)) + // Scoped so the ceiling is disarmed before the rewind below, which it would otherwise refuse. + do { + defer { demuxer.endInputByteBudget() } + while scanMayContinue(packetsRead: packetsRead, videoPackets: videoPackets, bytesRead: bytesRead, + packetBudget: packetBudget, readBudget: readBudget, byteBudget: byteBudget), + !sets.isComplete { + guard let pkt = try? demuxer.readPacket() else { break } + var toFree: UnsafeMutablePointer? = pkt + defer { trackedPacketFree(&toFree) } + packetsRead += 1 + bytesRead += Int(max(pkt.pointee.size, 0)) + guard pkt.pointee.stream_index == streamIndex else { continue } + videoPackets += 1 + if let data = pkt.pointee.data { + sets.collect(from: data, size: Int(pkt.pointee.size)) + } } } demuxer.seek(to: 0) guard let vps = sets.vps, let sps = sets.sps, let pps = sets.pps else { EngineLog.emit( - "[VideoConfigRecord] in-band parameter-set scan found none: packets=\(videoPackets) " + "[VideoConfigRecord] in-band parameter-set scan found none: packets=\(videoPackets) bytes=\(bytesRead) " + "vps=\(sets.vps != nil) sps=\(sets.sps != nil) pps=\(sets.pps != nil)", category: .engine ) diff --git a/Tests/AetherEngineTests/DiscChapterSeekTargetTests.swift b/Tests/AetherEngineTests/DiscChapterSeekTargetTests.swift new file mode 100644 index 000000000..b2b0a138a --- /dev/null +++ b/Tests/AetherEngineTests/DiscChapterSeekTargetTests.swift @@ -0,0 +1,171 @@ +import Testing +import Foundation +import Combine +@testable import AetherEngine + +/// `selectChapter(id:)` turns a title-relative disc chapter into a `seek(to:)` target. On the software +/// route the target is on the session axis (source PTS minus the clock's session zero), and the host adds +/// that zero back before the seek reaches the demuxer. A disc title whose container start is past the 2 s +/// anchor tolerance gets that start as its session zero, so the start must not be added a second time. +/// Pure logic: the container start and the session zero are injected, no media is read. +@Suite("Disc chapter seek target") +struct DiscChapterSeekTargetTests { + + /// What the host does with a session-axis target: carry it back onto the source axis. + private func hostSourceSeconds(forSession target: Double, zero: Double) -> Double { + SWClockAnchorPolicy.sourceSeconds(forSession: target, sessionZeroSeconds: zero) + } + + @Test("A disc title that starts at 600 s lands on the chapter, not one start late") + func softwareTitlePastToleranceDoesNotShiftTwice() { + // Start 600 s: the host predicts session zero 600 and reports it. + let zero = SWClockAnchorPolicy.resumeSessionZero(sourceOriginSeconds: 600) + #expect(zero == 600) + let target = PresentationAxis.discChapterSeekTarget( + chapterStart: 120, backend: .software, sourceStartSeconds: 600, + playlistShiftSeconds: 0, softwareClockZero: zero) + #expect(target == 120) + // The chapter's source PTS is start + 120; the host's conversion reaches exactly it. + #expect(hostSourceSeconds(forSession: target, zero: zero) == 720) + } + + @Test("The former rule (add the whole start) overshoots by the start on that title") + func formerRuleOvershoots() { + let zero = 600.0 + let formerTarget = 120 + 600.0 + #expect(hostSourceSeconds(forSession: formerTarget, zero: zero) == 1320) + } + + @Test("A disc title that starts inside the 2 s tolerance keeps the whole start in the target") + func softwareTitleInsideToleranceAddsStart() { + // Start 1.4 s: zero stays 0 (the host never reports one), so the session axis is the raw axis. + let zero = SWClockAnchorPolicy.resumeSessionZero(sourceOriginSeconds: 1.4) + #expect(zero == 0) + let target = PresentationAxis.discChapterSeekTarget( + chapterStart: 120, backend: .software, sourceStartSeconds: 1.4, + playlistShiftSeconds: 0, softwareClockZero: nil) + #expect(abs(target - 121.4) < 1e-9) + #expect(abs(hostSourceSeconds(forSession: target, zero: zero) - 121.4) < 1e-9) + } + + @Test("A zero not reported yet is predicted from the start, so a selection racing the load is right") + func nilZeroIsPredictedFromStart() { + let target = PresentationAxis.discChapterSeekTarget( + chapterStart: 120, backend: .software, sourceStartSeconds: 600, + playlistShiftSeconds: 0, softwareClockZero: nil) + #expect(target == 120) + } + + @Test("A start inside the tolerance: a nil zero and an explicit zero of 0 give the same target") + func nilZeroEqualsZero() { + let nilZero = PresentationAxis.discChapterSeekTarget( + chapterStart: 30, backend: .software, sourceStartSeconds: 0.28, + playlistShiftSeconds: 0, softwareClockZero: nil) + let zeroZero = PresentationAxis.discChapterSeekTarget( + chapterStart: 30, backend: .software, sourceStartSeconds: 0.28, + playlistShiftSeconds: 0, softwareClockZero: 0) + #expect(nilZero == zeroZero) + #expect(abs(nilZero - 30.28) < 1e-9) + } + + @Test("The clock's own zero wins when it re-anchored on a first sample past the start") + func softwareAnchoredZeroPastStart() { + // Container start 600, first sample at 600.4: still inside the tolerance, zero stays the predicted + // 600. A late first sample at 603 re-anchors the zero at 603; the chapter then sits 3 s before + // the zero in session terms and the target stays on the title. + let target = PresentationAxis.discChapterSeekTarget( + chapterStart: 120, backend: .software, sourceStartSeconds: 600, + playlistShiftSeconds: 0, softwareClockZero: 603) + #expect(target == 117) + #expect(hostSourceSeconds(forSession: target, zero: 603) == 720) + } + + @Test("A zero a little past the container start never produces a negative target") + func targetIsNeverNegative() { + // The host takes its zero from the video stream's own start, which can sit just past the + // container start; chapter 1 at 0 would be -0.1 without the floor. + let target = PresentationAxis.discChapterSeekTarget( + chapterStart: 0, backend: .software, sourceStartSeconds: 599.9, + playlistShiftSeconds: 0, softwareClockZero: 600) + #expect(target == 0) + } + + @Test("A container with no stated start uses the video stream's start as the origin, like the host") + func softwareTitleWithNoFormatStart() { + // The reporter's remote ISO: format start NOPTS, video stream start 54000000 in 1/90000 (600 s). + // The host predicts zero 600 from the same origin, so the chapter lands on chapter + 600 in + // source terms, never 600 s early. + let origin = Demuxer.sourceStartOrigin( + formatStartUs: Int64.min, videoStreamStart: 54_000_000, + videoTimeBaseNum: 1, videoTimeBaseDen: 90_000) + #expect(origin == 600) + let reported = PresentationAxis.discChapterSeekTarget( + chapterStart: 120, backend: .software, sourceStartSeconds: origin, + playlistShiftSeconds: 0, softwareClockZero: 600) + #expect(reported == 120) + #expect(hostSourceSeconds(forSession: reported, zero: 600) == 720) + let predicted = PresentationAxis.discChapterSeekTarget( + chapterStart: 120, backend: .software, sourceStartSeconds: origin, + playlistShiftSeconds: 0, softwareClockZero: nil) + #expect(predicted == reported) + } + + @Test("A native disc title is seeked on the 0-based display axis, with no base added") + func nativeTitleUsesDisplayAxis() { + // The playlist shift of a disc title is its clip 0 base (for example 599 s); the display + // axis already has it taken off, so the target is the chapter start itself. + let target = PresentationAxis.discChapterSeekTarget( + chapterStart: 120, backend: .native, sourceStartSeconds: 599, + playlistShiftSeconds: 599, softwareClockZero: nil) + #expect(target == 120) + } + + @Test("Backends without an established axis keep the playlist shift as their base") + func otherBackendsKeepPlaylistShift() { + for backend in [PlaybackBackend.audio, .none] { + let target = PresentationAxis.discChapterSeekTarget( + chapterStart: 120, backend: backend, sourceStartSeconds: 600, + playlistShiftSeconds: 3.5, softwareClockZero: 600) + #expect(target == 123.5, "backend \(backend)") + } + } +} + +/// The engine feeds its own state into the same rule. +@Suite("Disc chapter seek target engine wiring") +@MainActor +struct DiscChapterSeekTargetEngineTests { + + @Test("The engine's software state gives the title-relative start when the zero took the container start") + func engineStateGivesTitleRelativeStart() throws { + let engine = try AetherEngine() + engine.playbackBackend = .software + engine.sourceStartSeconds = 600 + engine.softwareClockSessionZero = 600 + #expect(PresentationAxis.discChapterSeekTarget( + chapterStart: 45, backend: engine.playbackBackend, + sourceStartSeconds: engine.sourceStartSeconds, + playlistShiftSeconds: engine.playlistShiftSeconds, + softwareClockZero: engine.softwareClockSessionZero) == 45) + } + + @Test("selectChapter seeks to the title-relative chapter start on a software title") + func selectChapterDrivesTheHelper() async throws { + let engine = try AetherEngine() + engine.playbackBackend = .software + engine.sourceStartSeconds = 600 + engine.softwareClockSessionZero = 600 + engine.discChapters = [ChapterInfo(id: 3, name: "Chapter 3", startSeconds: 45, durationSeconds: 60)] + var events: [SeekEvent] = [] + let sub = engine.seekEvents.sink { events.append($0) } + defer { sub.cancel() } + + engine.selectChapter(id: 3) + // The seek runs in a main-actor task; the idle engine rejects it with the target it was given. + for _ in 0..<200 where events.isEmpty { try await Task.sleep(nanoseconds: 5_000_000) } + + #expect(events.count == 1) + #expect(events.first?.outcome == .rejected(.noActiveSession)) + #expect(events.first?.target == 45) + } +} diff --git a/Tests/AetherEngineTests/FlushRecoveryTests.swift b/Tests/AetherEngineTests/FlushRecoveryTests.swift new file mode 100644 index 000000000..b3999bda7 --- /dev/null +++ b/Tests/AetherEngineTests/FlushRecoveryTests.swift @@ -0,0 +1,1056 @@ +import Foundation +import AVFoundation +import AetherLibavformat +import AetherLibavcodec +import CoreMedia +import CoreVideo +import Testing +@testable import AetherEngine + +/// The system can flush or fail the video queue and the audio renderer under the software host, and +/// AVPlayer recovers from that by itself. The host keeps the audio playing and lets the picture resume +/// at the next keyframe, and puts the whole session back at the playhead only when that keyframe lies +/// further away than the audio lead. +@Suite("Flush recovery policy") +struct FlushRecoveryPolicyTests { + typealias Policy = FlushRecoveryPolicy + + private static func state(gap: Double?, lead: Double) -> Policy.State { + var s = Policy.State() + s.videoGapSeconds = gap + s.audioLeadSeconds = lead + return s + } + + @Test("a keyframe inside the audio lead is waited for, one beyond it (or none read yet) repositions") + func gapAgainstLead() { + for trigger in [FlushRecoveryTrigger.videoQueueRequiresFlush, .videoQueueFailed] { + #expect(Policy.decide(trigger, Self.state(gap: 1.5, lead: 4)) == .flushVideoOnly) + #expect(Policy.decide(trigger, Self.state(gap: 4, lead: 4)) == .flushVideoOnly) + #expect(Policy.decide(trigger, Self.state(gap: 4.01, lead: 4)) == .reposition) + #expect(Policy.decide(trigger, Self.state(gap: nil, lead: 4)) == .reposition) + #expect(Policy.decide(trigger, Self.state(gap: 0, lead: 0)) == .flushVideoOnly) + #expect(Policy.decide(trigger, Self.state(gap: nil, lead: 0)) == .reposition) + } + #expect(Policy.videoGapOutlastsAudioLead(gap: nil, lead: .infinity)) + } + + @Test("a gone session, a seek in flight or a recovery under way ignore every trigger") + func ignored() { + for trigger in FlushRecoveryTrigger.allCases { + var s = Policy.State(); s.stopRequested = true + #expect(Policy.decide(trigger, s) == .ignore) + s = .init(); s.hasSource = false + #expect(Policy.decide(trigger, s) == .ignore) + s = .init(); s.seekInFlight = true + #expect(Policy.decide(trigger, s) == .ignore) + s = .init(); s.recoveryPending = true + #expect(Policy.decide(trigger, s) == .ignore) + } + } + + @Test("a failed audio renderer is flushed and never repositions, whatever the video gap is") + func audioFailed() { + #expect(Policy.decide(.audioRendererFailed, Self.state(gap: nil, lead: 0)) == .flushAudioOnly) + var s = Self.state(gap: 99, lead: 0); s.backgroundAudioOnly = true + #expect(Policy.decide(.audioRendererFailed, s) == .flushAudioOnly) + s.firstFramePresented = false + #expect(Policy.decide(.audioRendererFailed, s) == .flushAudioOnly) + } + + @Test("a failed audio renderer is flushed a bounded number of times per window, then left alone") + func audioFlushStormCap() { + var s = Self.state(gap: nil, lead: 0) + for n in 0.. CVPixelBuffer? { + var out: CVPixelBuffer? + let status = CVPixelBufferCreate( + kCFAllocatorDefault, width, height, kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, + [kCVPixelBufferIOSurfacePropertiesKey: [:] as CFDictionary] as CFDictionary, &out) + return status == kCVReturnSuccess ? out : nil +} + +@Suite("Flush recovery renderer", .serialized) +@MainActor +struct FlushRecoveryRendererTests { + + final class Box: @unchecked Sendable { + private let lock = NSLock() + private var _reasons: [SampleBufferRenderer.QueueFlushReason] = [] + private var _threads: [Thread] = [] + var reasons: [SampleBufferRenderer.QueueFlushReason] { lock.lock(); defer { lock.unlock() }; return _reasons } + var threads: [Thread] { lock.lock(); defer { lock.unlock() }; return _threads } + func add(_ r: SampleBufferRenderer.QueueFlushReason) { + lock.lock(); _reasons.append(r); _threads.append(.current); lock.unlock() + } + } + + @Test("every report reaches the observer, and none after it is removed") + func rendererReports() { + let renderer = SampleBufferRenderer() + let box = Box() + renderer.setQueueFlushObserver { box.add($0) } + renderer.reportQueueFlushForTesting(.requiresFlush) + renderer.reportQueueFlushForTesting(.failed) + #expect(box.reasons == [.requiresFlush, .failed]) + renderer.setQueueFlushObserver(nil) + renderer.reportQueueFlushForTesting(.failed) + #expect(box.reasons.count == 2) + } + + @Test("the SDK's own notifications are observed, off the posting thread, and a queue that needs nothing reports nothing") + func notificationsUseTheSdkConstants() { + let renderer = SampleBufferRenderer() + let box = Box() + renderer.setQueueFlushObserver { box.add($0) } + let video = renderer.videoRenderer + + // A healthy renderer: the property read back says no flush is needed. + NotificationCenter.default.post( + name: AVSampleBufferVideoRenderer.requiresFlushToResumeDecodingDidChangeNotification, object: video) + NotificationCenter.default.post( + name: AVSampleBufferVideoRenderer.didFailToDecodeNotification, object: video) + renderer.drainQueueNotificationsForTesting() + #expect(box.reasons.isEmpty) + + // A renderer that does need one: both notifications report, from the renderer's own queue. + renderer.queueNeedsFlushForTesting = { true } + NotificationCenter.default.post( + name: AVSampleBufferVideoRenderer.requiresFlushToResumeDecodingDidChangeNotification, object: video) + NotificationCenter.default.post( + name: AVSampleBufferVideoRenderer.didFailToDecodeNotification, object: video, + userInfo: [AVSampleBufferVideoRenderer.didFailToDecodeNotificationErrorKey: + NSError(domain: "test", code: 1)]) + renderer.drainQueueNotificationsForTesting() + #expect(box.reasons == [.requiresFlush, .failed]) + #expect(box.threads.allSatisfy { $0 !== Thread.current }) + + // Another renderer's notification is not ours. + let other = SampleBufferRenderer() + NotificationCenter.default.post( + name: AVSampleBufferVideoRenderer.requiresFlushToResumeDecodingDidChangeNotification, + object: other.videoRenderer) + renderer.drainQueueNotificationsForTesting() + #expect(box.reasons.count == 2) + } + + @Test("a frame the queue took counts as presented") + func acceptedFrames() throws { + let renderer = SampleBufferRenderer() + #expect(renderer.acceptedFrameCount == 0) + renderer.enqueue(pixelBuffer: try #require(makePixelBuffer()), pts: CMTime(seconds: 0, preferredTimescale: 90000)) + renderer.drainReorderBuffer() + #expect(renderer.acceptedFrameCount >= 1) + } + + @Test("a frame counts as presented when the clock has passed it with the queue healthy, not when the enqueue returned") + func presentedNeedsTheClock() throws { + let renderer = SampleBufferRenderer() + renderer.enqueue(pixelBuffer: try #require(makePixelBuffer()), pts: CMTime(seconds: 5, preferredTimescale: 90000)) + renderer.drainReorderBuffer() + #expect(renderer.acceptedFrameCount == 1) + #expect(renderer.presentedFrameCount(clockSeconds: 4.9) == 0) + #expect(renderer.presentedFrameCount(clockSeconds: .nan) == 0) + // The queue turned out to refuse it: no evidence while it needs a flush. + renderer.queueNeedsFlushForTesting = { true } + #expect(renderer.presentedFrameCount(clockSeconds: 6) == 0) + renderer.queueNeedsFlushForTesting = nil + #expect(renderer.presentedFrameCount(clockSeconds: 6) == 1) + #expect(renderer.presentedFrameCount(clockSeconds: 7) == 1) + // A flush drops the frames not yet counted, as the queue dropped them. + renderer.enqueue(pixelBuffer: try #require(makePixelBuffer()), pts: CMTime(seconds: 9, preferredTimescale: 90000)) + renderer.drainReorderBuffer() + renderer.flush(removingDisplayedImage: false) + #expect(renderer.presentedFrameCount(clockSeconds: 10) == 1) + } + + @Test("a queue that requires a flush without being failed is flushed and reported at the next enqueue") + func requiresFlushAtEnqueue() throws { + let renderer = SampleBufferRenderer() + let box = Box() + renderer.setQueueFlushObserver { box.add($0) } + renderer.queueNeedsFlushForTesting = { true } + renderer.enqueue(pixelBuffer: try #require(makePixelBuffer()), pts: CMTime(seconds: 1, preferredTimescale: 90000)) + renderer.drainReorderBuffer() + #expect(box.reasons == [.requiresFlush]) + // The frame handed over right after the flush is not evidence that the queue works. + #expect(renderer.acceptedFrameCount == 0) + renderer.queueNeedsFlushForTesting = nil + renderer.enqueue(pixelBuffer: try #require(makePixelBuffer()), pts: CMTime(seconds: 2, preferredTimescale: 90000)) + renderer.drainReorderBuffer() + #expect(box.reasons == [.requiresFlush]) + #expect(renderer.acceptedFrameCount == 1) + } + + @Test("a flush that carries the skip threshold installs it with the flush: nothing below it slips in between") + func flushWithThreshold() throws { + let renderer = SampleBufferRenderer() + renderer.flush(removingDisplayedImage: false, skipThreshold: CMTime(seconds: 20, preferredTimescale: 90000)) + let before = renderer.enqueueCount + renderer.enqueue(pixelBuffer: try #require(makePixelBuffer()), pts: CMTime(seconds: 19.5, preferredTimescale: 90000)) + renderer.drainReorderBuffer() + #expect(renderer.enqueueCount == before) + renderer.enqueue(pixelBuffer: try #require(makePixelBuffer()), pts: CMTime(seconds: 20.1, preferredTimescale: 90000)) + renderer.drainReorderBuffer() + #expect(renderer.enqueueCount == before + 1) + #expect(renderer.framesBelowFloor == 0) + } +} + +/// A video decoder stand-in that records the packets it accepted, refusing the ones a flush has retired +/// the way the real decoders do. `flushOnDecodeOf` makes a recovery flush land inside the hand-off of +/// the n-th packet it is given. +private final class PacketRecordingDecoder: VideoDecodingPipeline, @unchecked Sendable { + private let lock = NSLock() + private var _epoch: UInt64 = 0 + private var _accepted: [(isKey: Bool, pts: Int64)] = [] + private var _seen = 0 + private var flushDuringPacket: Int? + private var _flushes = 0 + var onFrame: DecodedFrameHandler? + var onFirstHDR10PlusDetected: (@Sendable () -> Void)? + var onA53Captions: (@Sendable ([CCDataParser.CCTriplet], Double) -> Void)? + var onDecodedFormat: (@Sendable (DecodedVideoFormat) -> Void)? + var skipUntilPTS: CMTime? + var feedEpoch: UInt64 { lock.lock(); defer { lock.unlock() }; return _epoch } + var accepted: [(isKey: Bool, pts: Int64)] { lock.lock(); defer { lock.unlock() }; return _accepted } + var seen: Int { lock.lock(); defer { lock.unlock() }; return _seen } + var flushCount: Int { lock.lock(); defer { lock.unlock() }; return _flushes } + func flushOnDecodeOf(packetNumber n: Int) { lock.lock(); flushDuringPacket = n; lock.unlock() } + func open(stream: UnsafeMutablePointer, onFrame: @escaping DecodedFrameHandler) throws {} + func decode(packet: UnsafeMutablePointer, epoch: UInt64?) { + lock.lock(); defer { lock.unlock() } + _seen += 1 + if flushDuringPacket == _seen { _epoch += 1 } + if let epoch, epoch != _epoch { return } + _accepted.append(((packet.pointee.flags & AV_PKT_FLAG_KEY) != 0, packet.pointee.pts)) + } + func flush() { lock.lock(); _epoch += 1; _flushes += 1; lock.unlock() } + func close() {} +} + +/// A reader over in-memory data that blocks, once armed, any read from `blockFrom` bytes on until it is +/// released, so a session cannot reach the end of media while a test looks at the state it is in before +/// it. Before it is armed, or short of `blockFrom`, it behaves as `DataIOReader` does. +private final class GatedIOReader: IOReader, @unchecked Sendable { + private let inner: DataIOReader + private let blockFrom: Int + private let condition = NSCondition() + private var armed = false + private var released = false + private var position = 0 + private var _blockedReads = 0 + init(data: Data, blockFrom: Int) { inner = DataIOReader(data: data); self.blockFrom = blockFrom } + /// Arms the gate; the open of the container reads around the file and must not block. + func arm() { condition.lock(); armed = true; condition.unlock() } + func release() { condition.lock(); released = true; condition.broadcast(); condition.unlock() } + var blockedReads: Int { condition.lock(); defer { condition.unlock() }; return _blockedReads } + func read(_ buffer: UnsafeMutablePointer?, size: Int32) -> Int32 { + condition.lock() + if armed, position >= blockFrom { + _blockedReads += 1 + while !released { condition.wait(until: Date(timeIntervalSinceNow: 0.1)) } + _blockedReads -= 1 + } + condition.unlock() + let n = inner.read(buffer, size: size) + condition.lock() + if n > 0 { position += Int(n) } + condition.unlock() + return n + } + func seek(offset: Int64, whence: Int32) -> Int64 { + let result = inner.seek(offset: offset, whence: whence) + if result >= 0, whence != 65536 { condition.lock(); position = Int(result); condition.unlock() } + return result + } + func close() { inner.close() } + func cancel() { release() } +} + +/// The video fixture's first packet repeated `keyFlags.count` times, each flagged as a keyframe or not, +/// optionally with the audio fixture muxed in, so the demux loop meets delta frames where a test wants them. +private func syntheticVideoFixture(keyFlags: [Bool], withAudio: Bool = false) throws -> Data { + let video = Demuxer() + let audio = Demuxer() + defer { video.close(); audio.close() } + try video.open(reader: DataIOReader(data: ProbeTestFixtures.hdr10Plus()), formatHint: "mp4") + try audio.open(reader: DataIOReader(data: ProbeTestFixtures.eac3()), formatHint: "mp4") + let videoStream = try #require(video.stream(at: video.videoStreamIndex)) + let audioStream = try #require(audio.stream(at: audio.audioStreamIndex)) + let first = try #require(try video.readPacket()) + defer { var owned: UnsafeMutablePointer? = first; trackedPacketFree(&owned) } + + var output: UnsafeMutablePointer? + try #require(avformat_alloc_output_context2(&output, nil, "mp4", nil) >= 0) + let context = try #require(output) + defer { avformat_free_context(context) } + var io: UnsafeMutablePointer? + try #require(avio_open_dyn_buf(&io) >= 0) + let buffer = try #require(io) + context.pointee.pb = buffer + var bytes: UnsafeMutablePointer? + var bufferClosed = false + defer { + if !bufferClosed { _ = avio_close_dyn_buf(buffer, &bytes) } + av_free(bytes) + } + let sources = withAudio ? [videoStream, audioStream] : [videoStream] + for source in sources { + let target = try #require(avformat_new_stream(context, nil)) + try #require(avcodec_parameters_copy(target.pointee.codecpar, source.pointee.codecpar) >= 0) + target.pointee.time_base = source.pointee.time_base + } + try #require(avformat_write_header(context, nil) >= 0) + let videoTarget = try #require(context.pointee.streams[0]) + let tb = videoTarget.pointee.time_base + let step = Int64(max(1, Int(tb.den) / max(1, Int(tb.num)) / 24)) + for (i, isKey) in keyFlags.enumerated() { + let copy = try #require(av_packet_clone(first)) + var owned: UnsafeMutablePointer? = copy + defer { trackedPacketFree(&owned) } + copy.pointee.stream_index = 0 + copy.pointee.pts = Int64(i) * step + copy.pointee.dts = Int64(i) * step + copy.pointee.duration = step + copy.pointee.pos = -1 + if isKey { copy.pointee.flags |= AV_PKT_FLAG_KEY } else { copy.pointee.flags &= ~AV_PKT_FLAG_KEY } + try #require(av_interleaved_write_frame(context, copy) >= 0) + } + if withAudio { + let target = try #require(context.pointee.streams[1]) + while let packet = try audio.readPacket() { + var owned: UnsafeMutablePointer? = packet + defer { trackedPacketFree(&owned) } + av_packet_rescale_ts(packet, audioStream.pointee.time_base, target.pointee.time_base) + packet.pointee.stream_index = 1 + packet.pointee.pos = -1 + try #require(av_interleaved_write_frame(context, packet) >= 0) + } + } + try #require(av_write_trailer(context) >= 0) + let count = avio_close_dyn_buf(buffer, &bytes) + bufferClosed = true + context.pointee.pb = nil + try #require(count > 0) + return Data(bytes: try #require(bytes), count: Int(count)) +} + +@Suite("Flush recovery host", .serialized, .timeLimit(.minutes(2))) +@MainActor +struct FlushRecoveryHostTests { + + final class Targets: @unchecked Sendable { + private let lock = NSLock() + private var _values: [Double] = [] + var values: [Double] { lock.lock(); defer { lock.unlock() }; return _values } + func add(_ v: Double) { lock.lock(); _values.append(v); lock.unlock() } + } + + private func loadedHost(_ targets: Targets) async throws -> SoftwarePlaybackHost { + let host = SoftwarePlaybackHost(reposition: { _, target, _, _, _ in + targets.add(target) + return .landed + }) + let dem = Demuxer() + try dem.open(reader: DataIOReader(data: ProbeTestFixtures.hdr10Plus()), formatHint: "mp4") + do { + try await host.load(demuxer: dem, startPosition: nil, audioSourceStreamIndex: nil) + return host + } catch { host.stop(); throw error } + } + + private func loadedHost(data: Data) async throws -> SoftwarePlaybackHost { + let host = SoftwarePlaybackHost(reposition: { _, _, _, _, _ in .landed }) + let dem = Demuxer() + try dem.open(reader: DataIOReader(data: data), formatHint: "mp4") + do { + try await host.load(demuxer: dem, startPosition: nil, audioSourceStreamIndex: nil) + return host + } catch { host.stop(); throw error } + } + + /// A frame in the video queue, so the host sees a session that has presented something. + private func presentFrame(_ host: SoftwarePlaybackHost, at seconds: Double = 0) throws { + let renderer = host.videoRendererForTesting + renderer.enqueue(pixelBuffer: try #require(makePixelBuffer()), + pts: CMTime(seconds: seconds, preferredTimescale: 90000)) + renderer.drainReorderBuffer() + try #require(renderer.acceptedFrameCount > 0) + } + + /// A decoder in the host that produces no frames and counts nothing: the demux loop would otherwise read the + /// fixture again after a reposition and might present it, and these tests only want what they hand the queue. + @discardableResult + private func installQuietDecoder(_ host: SoftwarePlaybackHost) -> PacketRecordingDecoder { + let decoder = PacketRecordingDecoder() + host.setVideoDecoderForTesting(decoder) + return decoder + } + + @Test("before a frame is presented a video trigger changes nothing") + func notPresented() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .ignore) + #expect(host.flushRecoveryLog.isEmpty) + #expect(targets.values.isEmpty) + } + + @Test("a keyframe inside the audio lead: the audio plays on, the video queue and decoder are flushed, no seek") + func keyframeInsideLeadFlushesOnly() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + let decoder = installQuietDecoder(host) + try presentFrame(host) + host.seekClockForTesting(to: 20) + host.setAudioFedUntilForTesting(30) // lead 10 s + host.keyframeLedgerForTesting.noteKeyframeRead(pts: 24) // gap 4 s + let generation = host.videoRendererForTesting.flushGeneration + + #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .flushVideoOnly) + + #expect(host.videoRendererForTesting.flushGeneration > generation) + // The decoder was flushed, which is what "resumes at the next keyframe" rests on. + #expect(decoder.flushCount == 1) + #expect(decoder.feedEpoch > 0) + // The floor restarts at the playhead, not at whatever frame arrives first. + #expect(decoder.skipUntilPTS.map { abs($0.seconds - 20) < 0.01 } == true) + try await Task.sleep(for: .milliseconds(50)) + #expect(targets.values.isEmpty) + #expect(host.flushRecoveriesCompleted == 0) + #expect(host.flushRecoveryLog.map(\.action) == [.flushVideoOnly]) + } + + @Test("a keyframe beyond the audio lead: the session goes back to the playhead") + func keyframeBeyondLeadRepositions() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + let decoder = installQuietDecoder(host) + try presentFrame(host) + host.seekClockForTesting(to: 20) + host.setAudioFedUntilForTesting(23) // lead 3 s + host.keyframeLedgerForTesting.noteKeyframeRead(pts: 29) // gap 9 s + let generation = host.videoRendererForTesting.flushGeneration + + #expect(host.handleFlushTrigger(.videoQueueFailed) == .reposition) + // Further reports of the same event are ignored while it runs. + #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .ignore) + try await waitFor { host.flushRecoveriesCompleted == 1 } + + let target = try #require(targets.values.first) + #expect(targets.values.count == 1) + #expect(abs(target - 20) < 0.05) + #expect(host.videoRendererForTesting.flushGeneration > generation) + #expect(decoder.flushCount >= 1) + #expect(host.failure == nil) + } + + @Test("no keyframe read yet counts as beyond the lead") + func noKeyframeRepositions() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + try presentFrame(host) + host.seekClockForTesting(to: 20) + host.setAudioFedUntilForTesting(100) + #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .reposition) + try await waitFor { host.flushRecoveriesCompleted == 1 } + #expect(targets.values.count == 1) + } + + @Test("a queue the system keeps failing: the reposition degrades to a flush, which is not repeated, until a frame gets through") + func stormDegrades() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + // The fixture is two keyframes at the start of a short clip; after the reposition to 20 s the + // demux loop would read it again and might present them. Nothing may reach the queue here + // except what the test hands it, so the decoder is a stand-in that produces no frames. + installQuietDecoder(host) + host.setRepositionBackoffForTesting(0) + try presentFrame(host) + host.seekClockForTesting(to: 20) + #expect(host.handleFlushTrigger(.videoQueueFailed) == .reposition) + try await waitFor { host.flushRecoveriesCompleted == 1 } + + // Nothing was presented after it: it did not help. + #expect(host.handleFlushTrigger(.videoQueueFailed) == .flushVideoOnly) + #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .ignore) + #expect(host.handleFlushTrigger(.videoQueueFailed) == .ignore) + #expect(targets.values.count == 1) + + // A frame that was handed over but that the clock has not reached is not evidence yet. + try presentFrame(host, at: 25) + host.seekClockForTesting(to: 21) + #expect(host.handleFlushTrigger(.videoQueueFailed) == .ignore) + + // The clock passes a frame with the queue healthy: the next failure is a new episode. + host.seekClockForTesting(to: 26) + // The reposition to 20 s restarted the demux loop on a clip two seconds long, which hands the + // ledger keyframes far below the clock. They are not what this test is about: no keyframe is + // parked, so the only thing between this trigger and a reposition is the storm rule. + host.keyframeLedgerForTesting.reset() + #expect(host.handleFlushTrigger(.videoQueueFailed) == .reposition) + try await waitFor { host.flushRecoveriesCompleted == 2 } + #expect(host.flushRecoveryLog.map(\.action) == [.reposition, .flushVideoOnly, .reposition]) + } + + @Test("a frame refused by the queue after the enqueue returned does not count as presented") + func refusedFrameIsNotPresented() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + installQuietDecoder(host) + host.setRepositionBackoffForTesting(0) + try presentFrame(host) + host.seekClockForTesting(to: 20) + #expect(host.handleFlushTrigger(.videoQueueFailed) == .reposition) + try await waitFor { host.flushRecoveriesCompleted == 1 } + + // A frame goes in, the queue then reports it needs a flush again: the clock passing the frame + // proves nothing, since the system never showed it. + try presentFrame(host, at: 21) + host.seekClockForTesting(to: 22) + let renderer = host.videoRendererForTesting + renderer.queueNeedsFlushForTesting = { true } + defer { renderer.queueNeedsFlushForTesting = nil } + #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .flushVideoOnly) + #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .ignore) + } + + @Test("a reposition is not repeated inside the backoff even when frames got through in between") + func repositionBackoff() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + installQuietDecoder(host) + try presentFrame(host) + host.seekClockForTesting(to: 20) + #expect(host.handleFlushTrigger(.videoQueueFailed) == .reposition) + try await waitFor { host.flushRecoveriesCompleted == 1 } + try presentFrame(host, at: 21) + host.seekClockForTesting(to: 22) + // A frame got through, so the storm rule is satisfied; with no keyframe parked (the demux loop + // restarted on the short clip and left some far below the clock) only the backoff holds the + // reposition back. + host.keyframeLedgerForTesting.reset() + #expect(host.handleFlushTrigger(.videoQueueFailed) == .flushVideoOnly) + #expect(targets.values.count == 1) + } + + @Test("a live source with no rewind ring only flushes, however far the keyframe is") + func liveOnlyFlushes() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + host.setLiveForTesting(true) + try presentFrame(host) + #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .flushVideoOnly) + try await Task.sleep(for: .milliseconds(50)) + #expect(targets.values.isEmpty) + #expect(host.flushRecoveriesCompleted == 0) + } + + @Test("the renderer's report reaches the host's rule") + func rendererReportReachesHost() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + try presentFrame(host) + host.seekClockForTesting(to: 20) + host.setAudioFedUntilForTesting(25) + host.keyframeLedgerForTesting.noteKeyframeRead(pts: 60) + host.videoRendererForTesting.reportQueueFlushForTesting(.requiresFlush) + try await waitFor { host.flushRecoveriesCompleted == 1 } + #expect(host.flushRecoveryLog.first?.trigger == .videoQueueRequiresFlush) + #expect(targets.values.count == 1) + } + + @Test("a video failure in the background is paid on the foreground return, by the same rule") + func backgroundDefers() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + let decoder = installQuietDecoder(host) + try presentFrame(host) + host.enterBackgroundAudioOnly() + #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .deferToForeground) + #expect(host.videoQueueFlushOwedForTesting) + try await Task.sleep(for: .milliseconds(50)) + #expect(host.flushRecoveriesCompleted == 0) + + // Back in front with no keyframe read yet: the picture would stay frozen past the audio lead. + host.exitBackgroundAudioOnly() + #expect(!host.videoQueueFlushOwedForTesting) + #expect(decoder.flushCount >= 1) + try await waitFor { host.flushRecoveriesCompleted == 1 } + #expect(host.flushRecoveryLog.map(\.action) == [.deferToForeground, .reposition]) + } + + @Test("back in front with a keyframe inside the audio lead the picture just waits for it") + func foregroundReturnInsideLead() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + try presentFrame(host) + host.seekClockForTesting(to: 20) + host.enterBackgroundAudioOnly() + #expect(host.handleFlushTrigger(.videoQueueFailed) == .deferToForeground) + host.setAudioFedUntilForTesting(30) + host.keyframeLedgerForTesting.noteKeyframeRead(pts: 22) + host.exitBackgroundAudioOnly() + try await Task.sleep(for: .milliseconds(50)) + #expect(targets.values.isEmpty) + #expect(host.flushRecoveryLog.map(\.action) == [.deferToForeground, .flushVideoOnly]) + } + + @Test("the audio renderer flushing itself is left to AE#549: no recovery of any kind") + func audioFlushIsNotATrigger() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + try presentFrame(host) + let renderer = try #require(host.audioRendererForTesting) + NotificationCenter.default.post( + name: .AVSampleBufferAudioRendererWasFlushedAutomatically, object: renderer, + userInfo: [AVSampleBufferAudioRendererFlushTimeKey: + NSValue(time: CMTime(seconds: 3, preferredTimescale: 48000))]) + try await Task.sleep(for: .milliseconds(150)) + #expect(host.flushRecoveryLog.isEmpty) + #expect(targets.values.isEmpty) + } + + @Test("the status poll acts once when the audio renderer fails, again only after it recovered and failed anew") + func audioStatusPoll() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + host.pollAudioRendererStatus(.rendering) + #expect(host.flushRecoveryLog.isEmpty) + host.pollAudioRendererStatus(.failed) + host.pollAudioRendererStatus(.failed) + host.pollAudioRendererStatus(.failed) + #expect(host.flushRecoveryLog.map(\.action) == [.flushAudioOnly]) + host.pollAudioRendererStatus(.unknown) + host.pollAudioRendererStatus(.failed) + #expect(host.flushRecoveryLog.map(\.action) == [.flushAudioOnly, .flushAudioOnly]) + #expect(host.flushRecoveryLog.allSatisfy { $0.trigger == .audioRendererFailed }) + try await Task.sleep(for: .milliseconds(50)) + #expect(targets.values.isEmpty) + } + + @Test("a renderer that fails again and again is flushed three times and then left alone") + func audioStatusPollStorm() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + for _ in 0..<(FlushRecoveryPolicy.maxAudioFlushesPerWindow + 3) { + host.pollAudioRendererStatus(.failed) + host.pollAudioRendererStatus(.unknown) + } + #expect(host.flushRecoveryLog.count == FlushRecoveryPolicy.maxAudioFlushesPerWindow) + #expect(host.flushRecoveryLog.allSatisfy { $0.action == .flushAudioOnly }) + } + + @Test("an audio failure that begins while the rule ignores triggers is acted on once the recovery in the way is over") + func audioFailureNotLostWhileIgnored() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + installQuietDecoder(host) + try presentFrame(host) + host.seekClockForTesting(to: 20) + // A video reposition is under way; every trigger is ignored until it ends. + #expect(host.handleFlushTrigger(.videoQueueFailed) == .reposition) + #expect(host.recoveryPendingForTesting) + host.pollAudioRendererStatus(.failed) + host.pollAudioRendererStatus(.failed) + #expect(host.flushRecoveryLog.map(\.action) == [.reposition]) + + try await waitFor { host.flushRecoveriesCompleted == 1 } + // The renderer is still failed: the next poll offers the failure again and it is taken, once. + host.pollAudioRendererStatus(.failed) + host.pollAudioRendererStatus(.failed) + #expect(host.flushRecoveryLog.map(\.action) == [.reposition, .flushAudioOnly]) + } + + @Test("flushing the audio queue retires the diagnostic's audio marker, including the stale value the loop republishes") + func audioFlushResetsLeadMarker() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + host.setAudioFedUntilUnderCurrentGenerationForTesting(30) + #expect(host.audioFedUntilForTesting == 30) + host.pollAudioRendererStatus(.failed) + #expect(host.flushRecoveryLog.map(\.action) == [.flushAudioOnly]) + #expect(host.audioFedUntilForTesting.isNaN) + // The demux loop writes its own marker every iteration; the one it still holds is the flushed audio. + host.setAudioFedUntilUnderCurrentGenerationForTesting(30) + #expect(host.audioFedUntilForTesting.isNaN) + host.setAudioFedUntilUnderCurrentGenerationForTesting(31.5) + #expect(host.audioFedUntilForTesting == 31.5) + } + + @Test("after a plain recovery flush the demux loop hands the decoder no delta frame before the next keyframe") + func noDeltaFramesAfterRecoveryFlush() async throws { + // delta, delta, delta, KEY, delta, delta: the demuxer stands in the middle of a group of pictures. + let data = try syntheticVideoFixture(keyFlags: [false, false, false, true, false, false]) + let host = try await loadedHost(data: data) + defer { host.stop() } + let decoder = PacketRecordingDecoder() + host.setVideoDecoderForTesting(decoder) + try presentFrame(host) + host.seekClockForTesting(to: 0) + host.setAudioFedUntilForTesting(10) + host.keyframeLedgerForTesting.noteKeyframeRead(pts: 0.5) + #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .flushVideoOnly) + #expect(host.keyframeLedgerForTesting.isAwaitingKeyframe) + + host.play() + try await waitFor { decoder.accepted.count == 3 } + // The keyframe and what follows it, and nothing before. + #expect(decoder.accepted.map(\.isKey) == [true, false, false]) + #expect(!host.keyframeLedgerForTesting.isAwaitingKeyframe) + } + + @Test("the same on the decoupled route, where the packets are parked first") + func noDeltaFramesAfterRecoveryFlushDecoupled() async throws { + let data = try syntheticVideoFixture(keyFlags: [false, false, false, true, false, false], withAudio: true) + let host = try await loadedHost(data: data) + defer { host.stop() } + let decoder = PacketRecordingDecoder() + host.setVideoDecoderForTesting(decoder) + try presentFrame(host) + host.seekClockForTesting(to: 0) + host.setAudioFedUntilForTesting(10) + host.keyframeLedgerForTesting.noteKeyframeRead(pts: 0.5) + #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .flushVideoOnly) + + host.play() + try await waitFor { decoder.accepted.count == 3 } + #expect(decoder.accepted.map(\.isKey) == [true, false, false]) + } + + @Test("a flush that lands while the keyframe is being handed over does not end the wait: the next delta frames are dropped") + func flushDuringHandOverKeepsGate() async throws { + // delta, KEY(refused: a flush lands in its hand-off), delta, delta, KEY, delta. + let data = try syntheticVideoFixture(keyFlags: [false, true, false, false, true, false]) + let host = try await loadedHost(data: data) + defer { host.stop() } + let decoder = PacketRecordingDecoder() + decoder.flushOnDecodeOf(packetNumber: 1) // the first packet the gate lets through: packet 2 of the stream + host.setVideoDecoderForTesting(decoder) + try presentFrame(host) + host.seekClockForTesting(to: 0) + host.setAudioFedUntilForTesting(10) + host.keyframeLedgerForTesting.noteKeyframeRead(pts: 0.5) + #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .flushVideoOnly) + + host.play() + try await waitFor { decoder.accepted.count == 2 } + // Only the second keyframe and its follower reached the decoder; the first keyframe was lost to the + // flush and the two delta frames behind it were not handed over. + #expect(decoder.accepted.map(\.isKey) == [true, false]) + #expect(decoder.seen == 3) + } + + @Test("video dropped in the background takes the parked keyframes with it, so the owed recovery sees the hole") + func backgroundDropFreesLedger() async throws { + // A long clip, read through a source that stops answering half way: the end of media also frees what + // is parked, and would pass this test with the background drop taken out. + let data = try syntheticVideoFixture(keyFlags: (0..<8000).map { $0 % 30 == 0 }, withAudio: true) + let reader = GatedIOReader(data: data, blockFrom: data.count / 2) + let dem = Demuxer() + try dem.open(reader: reader, formatHint: "mp4") + reader.arm() + let host = SoftwarePlaybackHost(reposition: { _, _, _, _, _ in .landed }) + defer { reader.release(); host.stop() } + do { try await host.load(demuxer: dem, startPosition: nil, audioSourceStreamIndex: nil) } + catch { host.stop(); throw error } + let decoder = PacketRecordingDecoder() + host.setVideoDecoderForTesting(decoder) + try presentFrame(host) + host.seekClockForTesting(to: 20) + host.setAudioFedUntilForTesting(30) + host.enterBackgroundAudioOnly() + #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .deferToForeground) + // A keyframe that was parked when the background began and sits below the clock: left standing it + // reads as "a keyframe is at hand", a gap of zero. + host.keyframeLedgerForTesting.noteKeyframeRead(pts: 3) + + host.play() + let freed = try await waitFor(upTo: .seconds(10)) { host.keyframeLedgerForTesting.nextKeyframe == nil } + // Judged while the source is held short of its end. + #expect(freed) + let held = try await waitFor(upTo: .seconds(10)) { reader.blockedReads >= 1 } + #expect(held, "the source was not held") + #expect(decoder.seen == 0) + host.exitBackgroundAudioOnly() + #expect(host.flushRecoveryLog.map(\.action) == [.deferToForeground, .reposition]) + } + + /// A recovery, then `frames` frames handed to the healthy queue and passed by the clock, left for the + /// host's own tick to count: nothing here samples the count by hand. + private func healthyPlaybackAfterRecovery(_ host: SoftwarePlaybackHost, from start: Double, frames: Int = 4) async throws { + let renderer = host.videoRendererForTesting + for i in 0.. 0) + host.seekClockForTesting(to: start + 1) + let counted = try await waitFor(upTo: .seconds(10)) { renderer.pendingPresentationCountForTesting == 0 } + try #require(counted, "the host's tick did not count the frames the clock passed") + } + + @Test("after a recovery, healthy playback counts as presented: a later notification with the queue needing a flush recovers again, round after round") + func playbackAfterRecoveryCountsNotification() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + installQuietDecoder(host) + host.setRepositionBackoffForTesting(0) + try presentFrame(host) + host.seekClockForTesting(to: 20) + #expect(host.handleFlushTrigger(.videoQueueFailed) == .reposition) + try await waitFor { host.flushRecoveriesCompleted == 1 } + + let renderer = host.videoRendererForTesting + defer { renderer.queueNeedsFlushForTesting = nil } + for round in 1...2 { + try await healthyPlaybackAfterRecovery(host, from: 21 + Double(round) * 10) + // The queue is now flagged, as it is when the system's notification reaches the host. + renderer.queueNeedsFlushForTesting = { true } + host.keyframeLedgerForTesting.reset() + #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .reposition) + renderer.queueNeedsFlushForTesting = nil + try await waitFor { host.flushRecoveriesCompleted == round + 1 } + } + #expect(host.flushRecoveryLog.map(\.action) == [.reposition, .reposition, .reposition]) + } + + @Test("after a recovery, healthy playback counts as presented: a later failure found at the enqueue recovers again") + func playbackAfterRecoveryCountsEnqueue() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + defer { host.stop() } + installQuietDecoder(host) + host.setRepositionBackoffForTesting(0) + try presentFrame(host) + host.seekClockForTesting(to: 20) + #expect(host.handleFlushTrigger(.videoQueueFailed) == .reposition) + try await waitFor { host.flushRecoveriesCompleted == 1 } + + try await healthyPlaybackAfterRecovery(host, from: 21) + host.keyframeLedgerForTesting.reset() + // The enqueue meets a queue that needs a flush: the renderer flushes it, drops the frames it was + // tracking and reports. The report is handled on the main actor once this test suspends, by which + // time the queue is healthy again. + let renderer = host.videoRendererForTesting + renderer.queueNeedsFlushForTesting = { true } + renderer.enqueue(pixelBuffer: try #require(makePixelBuffer()), pts: CMTime(seconds: 23, preferredTimescale: 90000)) + renderer.drainReorderBuffer() + renderer.queueNeedsFlushForTesting = nil + try await waitFor { host.flushRecoveryLog.count == 2 } + #expect(host.flushRecoveryLog.map(\.action) == [.reposition, .reposition]) + try await waitFor { host.flushRecoveriesCompleted == 2 } + } + + @Test("a keyframe held by back-pressure across a recovery flush is still handed over: the decoder is not refused it, the gate is not raised again") + func keyframeHeldAcrossRecoveryFlush() async throws { + // KEY, delta, delta, KEY, delta, delta. The first keyframe is held in the back-pressure wait. + let data = try syntheticVideoFixture(keyFlags: [true, false, false, true, false, false]) + let host = try await loadedHost(data: data) + defer { host.stop() } + let decoder = PacketRecordingDecoder() + host.setVideoDecoderForTesting(decoder) + try presentFrame(host) + host.seekClockForTesting(to: 0) + let renderer = host.videoRendererForTesting + renderer.readyForMoreMediaDataForTesting = { false } + defer { renderer.readyForMoreMediaDataForTesting = nil } + host.play() + // The packet is parked in the ledger once it has been read, which is after its epoch was. + try await waitFor { host.keyframeLedgerForTesting.nextKeyframe != nil } + // A recovery flush as the host makes it, while the packet is held. + decoder.flush() + host.keyframeLedgerForTesting.requireKeyframe() + renderer.readyForMoreMediaDataForTesting = nil + + let reached = try await waitFor(upTo: .seconds(10)) { decoder.accepted.count == 6 } + #expect(reached, "accepted \(decoder.accepted.map(\.isKey))") + #expect(decoder.accepted.map(\.isKey) == [true, false, false, true, false, false]) + } + + @Test("a stopped session ignores a late report") + func stoppedIgnores() async throws { + let targets = Targets() + let host = try await loadedHost(targets) + try presentFrame(host) + host.stop() + #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .ignore) + #expect(host.flushRecoveryLog.isEmpty) + } +} diff --git a/Tests/AetherEngineTests/HDR10PlusScanInputBudgetTests.swift b/Tests/AetherEngineTests/HDR10PlusScanInputBudgetTests.swift index 903e7e2d2..c7458c7da 100644 --- a/Tests/AetherEngineTests/HDR10PlusScanInputBudgetTests.swift +++ b/Tests/AetherEngineTests/HDR10PlusScanInputBudgetTests.swift @@ -56,9 +56,12 @@ struct HDR10PlusScanInputBudgetTests { /// resyncs through byte by byte inside a single `av_read_frame`. enum TailHeavyMatroska { - static func make(foreignBytes: Int, junkTail: Bool = false) throws -> Data { + /// `bareRecord` leaves the hvcC as its 22-byte header with no parameter-set arrays, the record of a + /// source whose parameter sets are in-band. + static func make(foreignBytes: Int, junkTail: Bool = false, bareRecord: Bool = false) throws -> Data { let mp4 = try ProbeTestFixtures.decode(HDR10PlusProbeIntegrationTests.hdr10Base64) - let (hvcC, frames) = try hevcTrack(mp4) + var (hvcC, frames) = try hevcTrack(mp4) + if bareRecord { hvcC = Array(hvcC.prefix(22)) + [0] } var out = Data() out += element([0x1A, 0x45, 0xDF, 0xA3], diff --git a/Tests/AetherEngineTests/HLSOriginRelayTests.swift b/Tests/AetherEngineTests/HLSOriginRelayTests.swift index 5d217122f..6f90183e4 100644 --- a/Tests/AetherEngineTests/HLSOriginRelayTests.swift +++ b/Tests/AetherEngineTests/HLSOriginRelayTests.swift @@ -336,10 +336,10 @@ struct HLSOriginRelayAddressingTests { // refusal the reader and the subtitle prefetcher would have been metered by. // // Read off the counter and the learned limit rather than `isPaced`. The quiet ladder - // `isPaced` reports is switched off process-wide by `quietPeriodCapForTesting`, which two - // other suites set in their init and never restore, so in a full run this suite's answer - // would be decided by whether one of them happened to have started. The origin's port is - // this test's own, which is what keeps the counter this test's own. + // `isPaced` reports is switched off process-wide while two other suites hold a + // `pinQuietPeriodCapForTesting` pin, and suites run in parallel, so this suite's answer would + // be decided by whether one of them happened to be running. The origin's port is this test's + // own, which is what keeps the counter this test's own. let snapshot = try #require(OriginRequestBudget.shared.snapshot(for: origin)) #expect(snapshot.refusals == 1, "a 429 through the relay was charged \(snapshot.refusals) times") #expect(snapshot.limit == 1, "the refusal did not bring the origin's concurrency down") diff --git a/Tests/AetherEngineTests/InBandParameterSetRebuildTests.swift b/Tests/AetherEngineTests/InBandParameterSetRebuildTests.swift index 713d871bb..5b22a7605 100644 --- a/Tests/AetherEngineTests/InBandParameterSetRebuildTests.swift +++ b/Tests/AetherEngineTests/InBandParameterSetRebuildTests.swift @@ -98,6 +98,84 @@ struct InBandParameterSetRebuildTests { #expect(!VideoConfigRecord.completeHEVCRecordFromInBandParameterSets(demuxer: demuxer, streamIndex: videoIdx)) } + @Test("The scan stops on whichever of the packet, video-packet and byte budgets is spent first") + func scanStopsOnEachBudget() { + func more(read: Int = 0, video: Int = 0, bytes: Int = 0) -> Bool { + VideoConfigRecord.scanMayContinue(packetsRead: read, videoPackets: video, bytesRead: bytes, + packetBudget: 16, readBudget: 512, byteBudget: 1_000_000) + } + #expect(more()) + #expect(!more(read: 512)) + #expect(!more(video: 16)) + #expect(!more(bytes: 1_000_000), "a stream of huge packets with no parameter sets stops on bytes") + #expect(more(read: 511, video: 15, bytes: 999_999)) + #expect(VideoConfigRecord.inBandScanByteBudget > 0) + } + + @Test("A spent byte budget leaves the record as it was and the demuxer at the head", + .enabled(if: fixtureExists("hev1-inband-xps.mp4"), + "run Scripts/fetch-fixtures.sh to generate the in-band xPS clip"), + .timeLimit(.minutes(1))) + func spentByteBudgetLeavesRecordAlone() throws { + let demuxer = Demuxer() + try demuxer.open(url: fixtureURL("hev1-inband-xps.mp4"), profile: .playback) + defer { demuxer.close() } + let videoIdx = demuxer.videoStreamIndex + let codecpar = try #require(demuxer.stream(at: videoIdx)?.pointee.codecpar) + let before = codecpar.pointee.extradata_size + #expect(!VideoConfigRecord.completeHEVCRecordFromInBandParameterSets( + demuxer: demuxer, streamIndex: videoIdx, byteBudget: 0)) + #expect(codecpar.pointee.extradata_size == before) + #expect(VideoConfigRecord.completeHEVCRecordFromInBandParameterSets(demuxer: demuxer, streamIndex: videoIdx)) + } + + /// Two HEVC frames with a bare record, then zeros no Matroska element starts in: libavformat resyncs + /// through them byte by byte inside one `av_read_frame`, so only the ceiling armed on the input + /// stops the scan, and the rewind after it must still be allowed. + private static let junkBytes = 24 * 1024 * 1024 + private static let budget = 64 * 1024 + + private func scanJunkTail(_ demuxer: Demuxer) throws { + let videoIdx = demuxer.videoStreamIndex + let codecpar = try #require(demuxer.stream(at: videoIdx)?.pointee.codecpar) + let before = codecpar.pointee.extradata_size + try #require(HLSVideoEngine.configRecordNeedsInBandRebuild(try #require(codecpar.pointee.extradata), + size: Int(before))) + #expect(!VideoConfigRecord.completeHEVCRecordFromInBandParameterSets( + demuxer: demuxer, streamIndex: Int32(videoIdx), byteBudget: Self.budget)) + #expect(demuxer.inputByteBudgetExhausted, "the input ceiling, not the loop check, ended the scan") + #expect(codecpar.pointee.extradata_size == before) + // The ceiling is disarmed and the demuxer is back at the head: the first packet is the first frame. + let first = try #require(try demuxer.readPacket()) + var toFree: UnsafeMutablePointer? = first + defer { trackedPacketFree(&toFree) } + #expect(first.pointee.pts == 0) + } + + @Test("A junk tail ends the scan on the input byte budget and the demuxer is rewound, on a custom reader", + .timeLimit(.minutes(1))) + func junkTailOnACustomReader() throws { + let data = try TailHeavyMatroska.make(foreignBytes: Self.junkBytes, junkTail: true, bareRecord: true) + try ProbeTestFixtures.withFile(data) { url in + let demuxer = Demuxer() + defer { demuxer.close() } + try demuxer.open(reader: try #require(FileIOReader(url: url)), profile: .playback) + try scanJunkTail(demuxer) + } + } + + @Test("A junk tail ends the scan on the input byte budget and the demuxer is rewound, on a local file", + .timeLimit(.minutes(1))) + func junkTailOnALocalFile() throws { + let data = try TailHeavyMatroska.make(foreignBytes: Self.junkBytes, junkTail: true, bareRecord: true) + try ProbeTestFixtures.withFile(data) { url in + let demuxer = Demuxer() + defer { demuxer.close() } + try demuxer.open(url: url, profile: .playback) + try scanJunkTail(demuxer) + } + } + /// A Profile 7 enhancement layer carries its own VPS / SPS / PPS inside `unspec63` NAL units; /// only the base layer's top-level parameter sets may seed the base layer's record. @Test("The collector keeps top-level parameter sets and skips enhancement-layer ones") diff --git a/Tests/AetherEngineTests/InitialSubtitleTrackTests.swift b/Tests/AetherEngineTests/InitialSubtitleTrackTests.swift new file mode 100644 index 000000000..13a63b5a2 --- /dev/null +++ b/Tests/AetherEngineTests/InitialSubtitleTrackTests.swift @@ -0,0 +1,122 @@ +import Testing +import Foundation +@testable import AetherEngine + +/// `LoadOptions.initialSubtitleTrackID`: a host that already knows which subtitle track it wants (a +/// server's default subtitle index) names it at load instead of ranking by language. Pure selection, +/// no media fixture and no decoder path involved. MainActor only because +/// `externalSubtitleTrackIDBase` is. +@MainActor +struct InitialSubtitleTrackTests { + + private func track(_ id: Int, _ lang: String?, isForced: Bool = false, + isExternal: Bool = false) -> TrackInfo { + TrackInfo(id: id, name: "t\(id)", codec: "subrip", language: lang, isDefault: false, + isForced: isForced, isExternal: isExternal) + } + + @Test("an id that names a track wins over the language preference") + func idWinsOverLanguage() { + let tracks = [track(2, "eng"), track(3, "deu"), track(4, "fra")] + #expect(AetherEngine.resolveInitialSubtitleIndex( + tracks: tracks, initialTrackID: 4, preferredLanguages: ["en"]) == 4) + } + + @Test("an id works without any language preference") + func idWithoutLanguages() { + let tracks = [track(2, "eng"), track(3, "deu")] + #expect(AetherEngine.resolveInitialSubtitleIndex( + tracks: tracks, initialTrackID: 3, preferredLanguages: []) == 3) + } + + @Test("an id that names no track falls through to the language preference") + func staleIdFallsThrough() { + let tracks = [track(2, "eng"), track(3, "deu")] + #expect(AetherEngine.resolveInitialSubtitleIndex( + tracks: tracks, initialTrackID: 99, preferredLanguages: ["de"]) == 3) + } + + @Test("a stale id with no preference leaves subtitles off") + func staleIdNoPreference() { + let tracks = [track(2, "eng")] + #expect(AetherEngine.resolveInitialSubtitleIndex( + tracks: tracks, initialTrackID: 99, preferredLanguages: []) == nil) + } + + @Test("no id keeps the language selection exactly as it was") + func noIdIsTheLanguageSelection() { + let tracks = [track(2, "eng", isForced: true), track(3, "eng")] + let expected = AetherEngine.selectSubtitleIndex(tracks: tracks, preferredLanguages: ["en"]) + #expect(expected == 3) + #expect(AetherEngine.resolveInitialSubtitleIndex( + tracks: tracks, initialTrackID: nil, preferredLanguages: ["en"]) == expected) + } + + @Test("an id can name a forced track the language ranking would not pick") + func idCanNameForcedTrack() { + let tracks = [track(2, "eng", isForced: true), track(3, "eng")] + #expect(AetherEngine.resolveInitialSubtitleIndex( + tracks: tracks, initialTrackID: 2, preferredLanguages: ["en"]) == 2) + } + + @Test("an external track is named by its synthetic id") + func externalTrackById() { + let externalID = AetherEngine.externalSubtitleTrackIDBase + 1 + let tracks = [track(2, "eng"), + track(AetherEngine.externalSubtitleTrackIDBase, "deu", isExternal: true), + track(externalID, "fra", isExternal: true)] + #expect(AetherEngine.resolveInitialSubtitleIndex( + tracks: tracks, initialTrackID: externalID, preferredLanguages: []) == Int32(externalID)) + } + + @Test("the option defaults to nil and takes part in LoadOptions equality") + func optionDefaultsAndEquality() { + #expect(LoadOptions().initialSubtitleTrackID == nil) + #expect(LoadOptions(initialSubtitleTrackID: 3).initialSubtitleTrackID == 3) + #expect(LoadOptions(initialSubtitleTrackID: 3) != LoadOptions()) + } + + @Test("a track id is a tuning field: a reload may change it without being refused") + func notAnIdentityField() { + #expect(!SessionOptionCorrection.loadIdentityFields.contains("initialSubtitleTrackID")) + #expect(SessionOptionCorrection.knownFields.contains("initialSubtitleTrackID")) + } + + @Test("an id with no language preference activates a late-added external track it names") + func idAloneReachesTheApplyGuard() throws { + let engine = try AetherEngine() + let id = AetherEngine.externalSubtitleTrackIDBase + engine.setLoadedOptionsForTesting(LoadOptions(initialSubtitleTrackID: id)) + #expect(engine.loadedOptions.preferredSubtitleLanguages.isEmpty) + let info = engine.addExternalSubtitleTrack( + ExternalSubtitleTrack(url: URL(string: "https://s/x.srt")!, name: "x", language: "de")) + #expect(info.id == id) + #expect(engine.isSubtitleActive) + #expect(engine.activeSubtitleTrackIndex == id) + } + + @Test("a host choice made first still wins over the id") + func hostChoiceBeatsTheId() throws { + let engine = try AetherEngine() + engine.setLoadedOptionsForTesting(LoadOptions( + initialSubtitleTrackID: AetherEngine.externalSubtitleTrackIDBase)) + engine.clearSubtitle() + _ = engine.addExternalSubtitleTrack( + ExternalSubtitleTrack(url: URL(string: "https://s/x.srt")!, name: "x", language: "de")) + #expect(!engine.isSubtitleActive) + } + + @Test("a disc title switch drops the id, since stream indices belong to one title") + func titleSwitchDropsTheId() throws { + let engine = try AetherEngine() + engine.setLoadedOptionsForTesting(LoadOptions(initialSubtitleTrackID: 4)) + engine.loadedURL = URL(fileURLWithPath: "/nonexistent/disc.iso") + engine.discTitles = [TitleInfo(id: 0, name: "Main", durationSeconds: 7200, chapterCount: 12), + TitleInfo(id: 1, name: "Extra", durationSeconds: 600, chapterCount: 1)] + engine.activeDiscTitleID = 0 + engine.selectTitle(id: 0) // already selected: a no-op, the id stays + #expect(engine.loadedOptions.initialSubtitleTrackID == 4) + engine.selectTitle(id: 1) + #expect(engine.loadedOptions.initialSubtitleTrackID == nil) + } +} diff --git a/Tests/AetherEngineTests/Issue410SourceStallVisibilityTests.swift b/Tests/AetherEngineTests/Issue410SourceStallVisibilityTests.swift index de1f3a6e3..80948617c 100644 --- a/Tests/AetherEngineTests/Issue410SourceStallVisibilityTests.swift +++ b/Tests/AetherEngineTests/Issue410SourceStallVisibilityTests.swift @@ -14,8 +14,9 @@ struct Issue410SourceStallVisibilityTests { /// The refusal these tests stage arms the request-rate pacer's wall-clock quiet ladder, which /// added 54 s to `bytes handed back out of memory do not clear the stall` and blocked a reader - /// thread throughout. The subject here is stall visibility, so the ladder is pinned away. - init() { OriginRequestBudget.shared.quietPeriodCapForTesting = 0 } + /// thread throughout. The subject here is stall visibility, so the ladder is pinned away, for the + /// lifetime of this suite instance (one test) and not for the rest of the process. + private let quietCapPin = OriginRequestBudget.shared.pinQuietPeriodCapForTesting(0) private final class PhaseLog: @unchecked Sendable { private let lock = NSLock() diff --git a/Tests/AetherEngineTests/OriginBudgetQuietCapPinTests.swift b/Tests/AetherEngineTests/OriginBudgetQuietCapPinTests.swift new file mode 100644 index 000000000..bb7e8f661 --- /dev/null +++ b/Tests/AetherEngineTests/OriginBudgetQuietCapPinTests.swift @@ -0,0 +1,65 @@ +import Testing +import Foundation +@testable import AetherEngine + +/// `pinQuietPeriodCapForTesting` replaces the suites that set `quietPeriodCapForTesting` on the shared +/// budget in `init` and never put it back. Each test builds its own budget, so none of this touches the +/// process-wide one. +@Suite("Quiet-period cap pin") +struct OriginBudgetQuietCapPinTests { + + @Test("A pin holds the cap while it lives and restores the earlier value on release") + func releaseRestoresPreviousCap() { + let budget = OriginRequestBudget() + budget.quietPeriodCapForTesting = 5 + let pin = budget.pinQuietPeriodCapForTesting(0) + #expect(budget.quietPeriodCapForTesting == 0) + pin.release() + #expect(budget.quietPeriodCapForTesting == 5) + } + + @Test("Dropping the token restores the cap, which is how a suite instance lets go") + func deinitRestoresCap() { + let budget = OriginRequestBudget() + do { + let pin = budget.pinQuietPeriodCapForTesting(0) + withExtendedLifetime(pin) { #expect(budget.quietPeriodCapForTesting == 0) } + } + #expect(budget.quietPeriodCapForTesting == nil) + } + + @Test("With overlapping pins the cap comes back only when the last one is released") + func overlappingPinsCount() { + let budget = OriginRequestBudget() + let first = budget.pinQuietPeriodCapForTesting(0) + let second = budget.pinQuietPeriodCapForTesting(0) + first.release() + #expect(budget.quietPeriodCapForTesting == 0, "the second suite is still running") + second.release() + #expect(budget.quietPeriodCapForTesting == nil) + } + + @Test("Releasing a pin twice gives back only once") + func releaseIsIdempotent() { + let budget = OriginRequestBudget() + let first = budget.pinQuietPeriodCapForTesting(0) + let second = budget.pinQuietPeriodCapForTesting(0) + first.release() + first.release() + #expect(budget.quietPeriodCapForTesting == 0, "a repeated release must not drop the other pin") + second.release() + #expect(budget.quietPeriodCapForTesting == nil) + } + + @Test("A pinned zero leaves the pacer unarmed, and the refusal ladder is back after release") + func pinnedZeroDisarmsPacerUntilReleased() throws { + let budget = OriginRequestBudget() + let url = try #require(URL(string: "http://127.0.0.1:9/movie.bin")) + let pin = budget.pinQuietPeriodCapForTesting(0) + budget.noteRefusal(for: url, status: 429) + #expect(!budget.isPaced(url), "a zero cap must leave the pacer unarmed") + pin.release() + budget.noteRefusal(for: url, status: 429) + #expect(budget.isPaced(url), "without the pin a refusal arms the quiet ladder again") + } +} diff --git a/Tests/AetherEngineTests/ResolvedURLInvalidationTests.swift b/Tests/AetherEngineTests/ResolvedURLInvalidationTests.swift index 613320306..3b132b236 100644 --- a/Tests/AetherEngineTests/ResolvedURLInvalidationTests.swift +++ b/Tests/AetherEngineTests/ResolvedURLInvalidationTests.swift @@ -15,8 +15,9 @@ struct ResolvedURLInvalidationTests { /// These tests drive real 429 and 509 answers, and each one arms the request-rate pacer's /// wall-clock quiet ladder. Their subject is the resolved-URL verdict, not that ladder, so it is /// pinned away: unpinned it added 44 s to a single test here and blocked a reader thread for all - /// of it. `.serialized` above already declares this suite a mutator of process-wide test hooks. - init() { OriginRequestBudget.shared.quietPeriodCapForTesting = 0 } + /// of it. `.serialized` above already declares this suite a mutator of process-wide test hooks. The + /// pin lasts for this suite instance (one test) and then puts the shared budget back. + private let quietCapPin = OriginRequestBudget.shared.pinQuietPeriodCapForTesting(0) private final class AttemptCounter: @unchecked Sendable { private let lock = NSLock() diff --git a/Tests/AetherEngineTests/SoftwareHostWakeRestoreTests.swift b/Tests/AetherEngineTests/SoftwareHostWakeRestoreTests.swift new file mode 100644 index 000000000..6499f1514 --- /dev/null +++ b/Tests/AetherEngineTests/SoftwareHostWakeRestoreTests.swift @@ -0,0 +1,99 @@ +import Foundation +import MediaPlayer +import Testing +@testable import AetherEngine + +/// A software session with the engine-owned Now-Playing card (`ownsSoftwareNowPlaying`), paused, then +/// released by the background teardown the way a tvOS sleep releases it. The teardown takes the +/// software host and its remote-command targets with it, so until the host rebuilds the session +/// nothing answers the remote: on a device this reads as a frozen last frame and dead Play/Pause. +/// The rebuild is the host's call (`needsForegroundVideoRestore`, then `reloadAtCurrentPosition`). +/// This pins that the rebuild alone brings the session back where it was, paused, with the card and +/// the remote commands answering again. +@Suite("Software host: restore after a background teardown", .serialized, .timeLimit(.minutes(2))) +@MainActor +struct SoftwareHostWakeRestoreTests { + + private static let fixture = Task { try await ItemSwapStillTests.fixtureURL() } + + @MainActor + private final class Sink: SoftwareNowPlayingSink { + var handler: (@MainActor (SoftwareNowPlayingCommand) -> Void)? + var deactivated = false + var lastState: MPNowPlayingPlaybackState = .unknown + func activate(skipInterval: Double, handler: @escaping @MainActor (SoftwareNowPlayingCommand) -> Void) { + self.handler = handler + } + func setSeekingEnabled(_ enabled: Bool) {} + func publish(info: [String: Any]?, playbackState: MPNowPlayingPlaybackState) { lastState = playbackState } + func deactivate() { deactivated = true; handler = nil } + } + + private func waitUntil(_ seconds: Double = 10, _ condition: @MainActor () -> Bool) async -> Bool { + let deadline = Date().addingTimeInterval(seconds) + while Date() < deadline { + if condition() { return true } + try? await Task.sleep(nanoseconds: 20_000_000) + } + return condition() + } + + @Test("a paused session torn down for the background comes back paused, with Now-Playing and the remote") + func restoreAfterBackgroundTeardown() async throws { + let file = try await Self.fixture.value + let engine = try AetherEngine() + defer { engine.stop() } + engine.volume = 0 + engine.ownsSoftwareNowPlaying = true + var sinks: [Sink] = [] + engine.makeSoftwareNowPlayingSink = { + let sink = Sink() + sinks.append(sink) + return sink + } + + var options = LoadOptions() + options.preferredDecodePath = .software + _ = try await engine.load(url: file, options: options) + #expect(engine.playbackBackend == .software) + engine.play() + // Past the one-second floor below which a rebuild deliberately restarts at the head + // (`LiveReloadPolicy.resumePosition`), so the restore has a position to come back to. + #expect(await waitUntil { engine.currentTime > 1.5 }) + engine.pause() + let pausedAt = engine.currentTime + #expect(engine.softwareNowPlayingActive) + #expect(sinks.count == 1) + + // What didEnterBackground runs on tvOS for a software session with no PiP. + engine.releaseVideoPipelineForBackground() + #expect(engine.playbackBackend == .none) + #expect(engine.softwareHost == nil) + #expect(engine.state == .paused) + #expect(engine.needsForegroundVideoRestore) + // The symptom: the card and its command targets went with the host, so the remote has + // nothing to reach until the session is rebuilt. + #expect(!engine.softwareNowPlayingActive) + #expect(sinks[0].deactivated) + #expect(sinks[0].handler == nil) + + // The host's half on foreground return: rebuild at the playhead and stay paused. + try await engine.reloadAtCurrentPosition { $0.autoplay = false } + #expect(engine.playbackBackend == .software) + #expect(!engine.needsForegroundVideoRestore) + #expect(engine.state == .paused) + #expect(abs(engine.currentTime - pausedAt) < 0.5, "restored at \(engine.currentTime), paused at \(pausedAt)") + #expect(engine.softwareNowPlayingActive) + #expect(sinks.count == 2) + let restored = try #require(sinks.last) + #expect(!restored.deactivated) + + // A remote Play goes through the rebuilt card's command handler and moves the session again. + let handler = try #require(restored.handler) + handler(.play) + #expect(await waitUntil { engine.state == .playing }) + let resumedFrom = engine.currentTime + #expect(await waitUntil { engine.currentTime > resumedFrom + 0.3 }, + "clock stayed at \(engine.currentTime) after the remote Play") + } +} diff --git a/Tests/AetherEngineTests/VideoSessionRetryTests.swift b/Tests/AetherEngineTests/VideoSessionRetryTests.swift new file mode 100644 index 000000000..861366814 --- /dev/null +++ b/Tests/AetherEngineTests/VideoSessionRetryTests.swift @@ -0,0 +1,90 @@ +import Testing +import CoreMedia +import Foundation +import AetherLibavformat +import AetherLibavcodec +import AetherLibavutil +@testable import AetherEngine + +/// AE#739 follow-up: a rebuild that VideoToolbox refused in the foreground is retried at each following +/// keyframe, a bounded number of times, and ends in a surfaced failure rather than a frozen picture. +@Suite("Video session retry", .serialized, .timeLimit(.minutes(1))) +struct VideoSessionRetryTests { + + // MARK: - Policy + + @Test("each keyframe while a recovery is owed is one retry, up to the cap, then one give-up") + func boundedRetries() { + var policy = VideoSessionRetryPolicy() + var decisions: [VideoSessionRetryPolicy.Decision] = [] + for i in 0..<(VideoSessionRetryPolicy.maxAttempts + 3) { + decisions.append(policy.onKeyframe(owed: true, now: Double(i) * 3)) + } + let expected: [VideoSessionRetryPolicy.Decision] = + (1...VideoSessionRetryPolicy.maxAttempts).map { .retry(attempt: $0) } + [.giveUp, .none, .none] + #expect(decisions == expected) + } + + @Test("a keyframe with nothing owed does nothing and spends no attempt") + func nothingOwed() { + var policy = VideoSessionRetryPolicy() + #expect(policy.onKeyframe(owed: false, now: 0) == .none) + #expect(policy.attempts == 0) + #expect(policy.onKeyframe(owed: true, now: 0) == .retry(attempt: 1)) + } + + @Test("keyframes in a burst spend one attempt, then wait out the spacing") + func spacing() { + var policy = VideoSessionRetryPolicy() + #expect(policy.onKeyframe(owed: true, now: 10) == .retry(attempt: 1)) + #expect(policy.onKeyframe(owed: true, now: 10.1) == .none) + #expect(policy.onKeyframe(owed: true, now: 11.9) == .none) + #expect(policy.attempts == 1) + #expect(policy.onKeyframe(owed: true, now: 12.0) == .retry(attempt: 2)) + } + + @Test("a successful rebuild gives the next loss a fresh budget, even after giving up") + func resetRestoresTheBudget() { + var policy = VideoSessionRetryPolicy() + for i in 0...VideoSessionRetryPolicy.maxAttempts { _ = policy.onKeyframe(owed: true, now: Double(i) * 3) } + #expect(policy.gaveUp) + policy.reset() + #expect(policy.onKeyframe(owed: true, now: 0) == .retry(attempt: 1)) + } + + // MARK: - Decoder reports keyframes while lost + + private final class Counter: @unchecked Sendable { + private let lock = NSLock() + private var n = 0 + func bump() { lock.lock(); n += 1; lock.unlock() } + var value: Int { lock.lock(); defer { lock.unlock() }; return n } + } + + private static func withPacket(key: Bool, _ body: (UnsafeMutablePointer) -> Void) { + var packet = av_packet_alloc() + defer { av_packet_free(&packet) } + guard let p = packet else { return } + p.pointee.flags = key ? AV_PKT_FLAG_KEY : 0 + body(p) + } + + #if DEBUG + @Test("a hardware decoder with a lost session reports each keyframe it is fed, and no other packet") + func hardwareDecoderReportsKeyframes() { + let decoder = HardwareVideoDecoder() + let seen = Counter() + decoder.onKeyframeWhileLost = { seen.bump() } + // Not lost: nothing to report. + Self.withPacket(key: true) { decoder.decode(packet: $0, epoch: nil) } + #expect(seen.value == 0) + decoder.markSessionLostForTesting() + #expect(decoder.isSessionLost) + Self.withPacket(key: false) { decoder.decode(packet: $0, epoch: nil) } + #expect(seen.value == 0) + Self.withPacket(key: true) { decoder.decode(packet: $0, epoch: nil) } + Self.withPacket(key: true) { decoder.decode(packet: $0, epoch: nil) } + #expect(seen.value == 2) + } + #endif +} diff --git a/docs/api.md b/docs/api.md index bb443d941..e5d43d329 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1406,6 +1406,7 @@ All flags default to safe values; the table is the full set. Depth for the media | `preferredAudioLanguages` | empty | First-frame audio pick from the engine's single probe. Ordered BCP-47 / ISO 639 tags; region and script are normalized and rank within one preference (#590). An explicit `audioSourceStreamIndex` still wins. | | `preferredSubtitleLanguages` | empty | Post-load subtitle activation on the host-overlay path. Ranks language specificity (an explicitly opposite script is rejected, not demoted) before the descriptor axis. Pure convenience: no reload and no pre-probe, unlike the audio equivalent. | | `externalSubtitles` | empty | Sidecar files registered at load, so they rank in the language preference and can join the native renditions. | +| `initialSubtitleTrackID` | nil | Post-load subtitle activation by `TrackInfo.id` (an embedded stream index, or `externalSubtitleTrackIDBase` plus the index in `externalSubtitles`), for a host that already knows the track, such as a server's default. Wins over `preferredSubtitleLanguages`; an id that names no track falls through to it instead of failing the load. | | `prepareNativeSubtitles` | false | Declare WebVTT renditions so subtitles survive PiP / AirPlay / external display. | | `eagerNativeSubtitleReaders` | false | Populate those renditions at load instead of on first selection, for playlists AVKit auto-selects. Only meaningful with `prepareNativeSubtitles`. | | `serveIFramePlaylist` | false | Serve an I-frame rendition next to the master, so a stock `AVPlayerViewController` shows scrub thumbnails and scans on I-frames with no host code. One still per segment: an SDR source gets its own keyframes at full resolution, an HDR10, HLG or Dolby Vision source gets tone-mapped 640-pixel Motion JPEG stills (AVKit shows no scrub thumbnail from a PQ or Dolby Vision keyframe). Opens a second reader on the source. Absent (one log line says why) for live, sources without a keyframe index, `sequentialOrigin`, `heldSourceConnection`, single-request origins, discs, media-playlist routing (the usual case: an HDR10, HLG or Dolby Vision title on an output that is not presenting HDR, such as an Apple TV locked to SDR, where AVFoundation accepts no master form for PQ content, AE#682), and a Dolby Vision Profile 5 / 10.0 source in a build without the `DolbyVisionStillReshaping` trait. Hosts with their own transport bar use `scrubThumbnail`. | diff --git a/docs/architecture.md b/docs/architecture.md index f4e1ae7fb..56adc548b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -557,6 +557,8 @@ Sources/AetherEngine/ │ ├── RemoteHLSMediaSelection.swift Remote-HLS bypass (#154): pure loopback→bypass reroute decision for non-live m3u8 sources (FFmpeg has no network) + legible AVMediaSelectionGroup → `subtitleTracks` mapping (synthetic ids from 200000) │ ├── SoftwarePlaybackHost.swift SW path: demux loop + decoders + renderer + synchronizer orchestration │ ├── SoftwarePlaybackHost+AudioProducer.swift SW path: picks and builds the audio producer for the session's track, with every unavailable Atmos route landing on the PCM decode (logged) +│ ├── VideoSessionRetryPolicy.swift SW path: how many keyframes retry a refused video-session rebuild in the foreground before the failure is surfaced +│ ├── SoftwareFlushRecovery.swift SW path: pure rules for putting a flushed or failed video/audio output queue back to work (triggers, seek versus wait for the next keyframe) │ ├── AirPlayPlaylistDecision.swift Pure decisions for the wireless-AirPlay loopback rewrite: which of master / media a receiver is handed, and whether a route change rebuilds the session or swaps its item (a sequential origin, a forward-only live source), kept separate and testable offline like `MasterFallbackDecision` (#86, #227, #526) │ ├── NativeLegibleDeselectPin.swift Keeps AVKit's ready-time legible auto-select from engaging a rendition the host never asked for (Sodalite#38), and publishes the caption requests the system makes on its own as `systemCaptionRequest`, by BCP 47 language rather than by ordinal (Sodalite#65) │ ├── RemoteHLSFormatDetection.swift Dynamic-range classification for the `nativeRemoteHLS` bypass, read back from AVPlayer's own track format instead of a second origin probe, which that path exists to avoid (#168) From 92b8ae861a0a019fc6a57b72a8abe3016b35db22 Mon Sep 17 00:00:00 2001 From: Pavel Kajaba <15980400+pkajaba@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:29:47 +0000 Subject: [PATCH 2/4] test(software-host): wait for the events the tests need instead of giving up on a wall-clock budget The wake-restore, flush-recovery and chapter-seek tests waited for positive events with a deadline of their own (a Date() loop whose try? sleep swallowed the time limit's cancellation, waitFor(upTo: 10 s), a 1 s task poll). They now use the unbounded waitFor and the suite time limit catches a real hang; the chapter-seek wiring suite gets a one-minute limit. --- .../DiscChapterSeekTargetTests.swift | 4 ++-- Tests/AetherEngineTests/FlushRecoveryTests.swift | 13 +++++-------- .../SoftwareHostWakeRestoreTests.swift | 16 +++------------- 3 files changed, 10 insertions(+), 23 deletions(-) diff --git a/Tests/AetherEngineTests/DiscChapterSeekTargetTests.swift b/Tests/AetherEngineTests/DiscChapterSeekTargetTests.swift index b2b0a138a..9bc6b2689 100644 --- a/Tests/AetherEngineTests/DiscChapterSeekTargetTests.swift +++ b/Tests/AetherEngineTests/DiscChapterSeekTargetTests.swift @@ -132,7 +132,7 @@ struct DiscChapterSeekTargetTests { } /// The engine feeds its own state into the same rule. -@Suite("Disc chapter seek target engine wiring") +@Suite("Disc chapter seek target engine wiring", .timeLimit(.minutes(1))) @MainActor struct DiscChapterSeekTargetEngineTests { @@ -162,7 +162,7 @@ struct DiscChapterSeekTargetEngineTests { engine.selectChapter(id: 3) // The seek runs in a main-actor task; the idle engine rejects it with the target it was given. - for _ in 0..<200 where events.isEmpty { try await Task.sleep(nanoseconds: 5_000_000) } + try await waitFor { !events.isEmpty } #expect(events.count == 1) #expect(events.first?.outcome == .rejected(.noActiveSession)) diff --git a/Tests/AetherEngineTests/FlushRecoveryTests.swift b/Tests/AetherEngineTests/FlushRecoveryTests.swift index b3999bda7..2ba485c08 100644 --- a/Tests/AetherEngineTests/FlushRecoveryTests.swift +++ b/Tests/AetherEngineTests/FlushRecoveryTests.swift @@ -944,11 +944,9 @@ struct FlushRecoveryHostTests { host.keyframeLedgerForTesting.noteKeyframeRead(pts: 3) host.play() - let freed = try await waitFor(upTo: .seconds(10)) { host.keyframeLedgerForTesting.nextKeyframe == nil } + try await waitFor { host.keyframeLedgerForTesting.nextKeyframe == nil } // Judged while the source is held short of its end. - #expect(freed) - let held = try await waitFor(upTo: .seconds(10)) { reader.blockedReads >= 1 } - #expect(held, "the source was not held") + try await waitFor { reader.blockedReads >= 1 } #expect(decoder.seen == 0) host.exitBackgroundAudioOnly() #expect(host.flushRecoveryLog.map(\.action) == [.deferToForeground, .reposition]) @@ -961,8 +959,8 @@ struct FlushRecoveryHostTests { for i in 0.. 0) host.seekClockForTesting(to: start + 1) - let counted = try await waitFor(upTo: .seconds(10)) { renderer.pendingPresentationCountForTesting == 0 } - try #require(counted, "the host's tick did not count the frames the clock passed") + // The host's tick counts the frames the clock passed; a hang is the suite's time limit. + try await waitFor { renderer.pendingPresentationCountForTesting == 0 } } @Test("after a recovery, healthy playback counts as presented: a later notification with the queue needing a flush recovers again, round after round") @@ -1039,8 +1037,7 @@ struct FlushRecoveryHostTests { host.keyframeLedgerForTesting.requireKeyframe() renderer.readyForMoreMediaDataForTesting = nil - let reached = try await waitFor(upTo: .seconds(10)) { decoder.accepted.count == 6 } - #expect(reached, "accepted \(decoder.accepted.map(\.isKey))") + try await waitFor { decoder.accepted.count == 6 } #expect(decoder.accepted.map(\.isKey) == [true, false, false, true, false, false]) } diff --git a/Tests/AetherEngineTests/SoftwareHostWakeRestoreTests.swift b/Tests/AetherEngineTests/SoftwareHostWakeRestoreTests.swift index 6499f1514..c282c6ac2 100644 --- a/Tests/AetherEngineTests/SoftwareHostWakeRestoreTests.swift +++ b/Tests/AetherEngineTests/SoftwareHostWakeRestoreTests.swift @@ -29,15 +29,6 @@ struct SoftwareHostWakeRestoreTests { func deactivate() { deactivated = true; handler = nil } } - private func waitUntil(_ seconds: Double = 10, _ condition: @MainActor () -> Bool) async -> Bool { - let deadline = Date().addingTimeInterval(seconds) - while Date() < deadline { - if condition() { return true } - try? await Task.sleep(nanoseconds: 20_000_000) - } - return condition() - } - @Test("a paused session torn down for the background comes back paused, with Now-Playing and the remote") func restoreAfterBackgroundTeardown() async throws { let file = try await Self.fixture.value @@ -59,7 +50,7 @@ struct SoftwareHostWakeRestoreTests { engine.play() // Past the one-second floor below which a rebuild deliberately restarts at the head // (`LiveReloadPolicy.resumePosition`), so the restore has a position to come back to. - #expect(await waitUntil { engine.currentTime > 1.5 }) + try await waitFor { engine.currentTime > 1.5 } engine.pause() let pausedAt = engine.currentTime #expect(engine.softwareNowPlayingActive) @@ -91,9 +82,8 @@ struct SoftwareHostWakeRestoreTests { // A remote Play goes through the rebuilt card's command handler and moves the session again. let handler = try #require(restored.handler) handler(.play) - #expect(await waitUntil { engine.state == .playing }) + try await waitFor { engine.state == .playing } let resumedFrom = engine.currentTime - #expect(await waitUntil { engine.currentTime > resumedFrom + 0.3 }, - "clock stayed at \(engine.currentTime) after the remote Play") + try await waitFor { engine.currentTime > resumedFrom + 0.3 } } } From 95b5d796e550e11d3b7f1dd569b516530633d09e Mon Sep 17 00:00:00 2001 From: Pavel Kajaba <15980400+pkajaba@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:55:36 +0000 Subject: [PATCH 3/4] fix(software-host): keep a refused video-session rebuild from ending the session in error A rebuild VideoToolbox refuses is retried at each keyframe in the foreground. After the fourth refusal the host published a playback failure, which moves the engine to .error. The engine keeps that state until the next load, so a rebuild that succeeded later left the session errored, and flush recovery stayed off while the failure was set. The retries also ran while the app was in the background (a retained paused session), where a refusal is expected. The host no longer publishes that failure: after the budget is spent it waits for the next foreground return, which starts a fresh budget. Nothing is spent while the app is in the background. The engine reports background and foreground to the host. Also: the flush recovery tests check exact state after each synchronous trigger instead of sleeping, the AE#549 case waits on the audio output's own count, FlushRecoveryPolicy says why a DVR session always repositions, and the selectChapter changelog entry is wrapped. --- CHANGELOG.md | 10 +++- Sources/AetherEngine/AetherEngine.swift | 2 + .../Native/SoftwareFlushRecovery.swift | 4 +- .../Native/SoftwarePlaybackHost.swift | 48 ++++++++++++++----- .../Native/VideoSessionRetryPolicy.swift | 2 +- .../FlushRecoveryTests.swift | 20 ++++---- .../VideoSessionRetryTests.swift | 44 ++++++++++++++++- docs/architecture.md | 2 +- 8 files changed, 106 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bef551791..4ba979458 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,11 @@ the public-API contract. ### Fixed -- **`selectChapter` on a disc title lands on the chapter, not one container start late.** The software route seeks on the session axis and the host adds the clock's session zero back, so adding the whole container start as well applied it twice whenever the start was past the 2 s anchor tolerance. Only the part of the start the zero did not take is added now, and a native disc title no longer adds the playlist shift on top of `seek(to:)`. +- **`selectChapter` on a disc title lands on the chapter, not one container start late.** The software route + seeks on the session axis and the host adds the clock's session zero back, so adding the whole container + start as well applied it twice whenever the start was past the 2 s anchor tolerance. Only the part of the + start the zero did not take is added now, and a native disc title no longer adds the playlist shift on top + of `seek(to:)`. - **A 24.000 fps source is programmed as 24 Hz, not 23.976 Hz (AE#884, thanks @akhanwav).** `FrameRateSnap` sent the whole 23.5 to 24.05 window to 23.976, so Match Frame Rate ran 24p content one frame in 1001 fast and the @@ -32,7 +36,9 @@ the public-API contract. invalidated the hardware session and VideoToolbox refused the rebuild, the decoder dropped every packet and the picture froze beside live audio until the next foreground return. The host now retries at each keyframe fed while the session is still lost (at least two seconds apart), up to four times, and then - publishes a playback failure. + waits for the next foreground return, which starts a fresh budget. Nothing is retried in the background, + where VideoToolbox refuses a rebuild by design. The session is not moved to `.error` for this: that state + is terminal until the next load, so a rebuild that succeeds later could not clear it. - **The in-band HEVC parameter-set scan stops at 16 MiB of input.** A stream that carries no parameter sets, or one whose packets are very large, could be read far past what the packet budget implies before the diff --git a/Sources/AetherEngine/AetherEngine.swift b/Sources/AetherEngine/AetherEngine.swift index 842e9ccd3..5804cf540 100644 --- a/Sources/AetherEngine/AetherEngine.swift +++ b/Sources/AetherEngine/AetherEngine.swift @@ -8307,6 +8307,7 @@ public final class AetherEngine: ObservableObject { ) { [weak self] _ in Task { @MainActor in guard let self else { return } + self.softwareHost?.noteAppInBackground(false) #if os(iOS) self.cancelBackgroundGraceWindow() self.softwareHost?.exitBackgroundAudioOnly() @@ -8466,6 +8467,7 @@ public final class AetherEngine: ObservableObject { func noteDidEnterBackground() { isBackgrounded = true backgroundActionOwed = backgroundDecisionMustWait + softwareHost?.noteAppInBackground(true) } /// Audit LIF-105: the PiP window closed while backgrounded, so nothing keeps the app running and diff --git a/Sources/AetherEngine/Native/SoftwareFlushRecovery.swift b/Sources/AetherEngine/Native/SoftwareFlushRecovery.swift index 5de984a23..5ce815d7c 100644 --- a/Sources/AetherEngine/Native/SoftwareFlushRecovery.swift +++ b/Sources/AetherEngine/Native/SoftwareFlushRecovery.swift @@ -27,7 +27,9 @@ enum FlushRecoveryTrigger: String, Sendable, CaseIterable { /// When the keyframe lies beyond it, or none has been read yet, the freeze outlasts the audio lead /// and only a reposition puts a picture back at the playhead: both queues and both decoders /// flushed, the demuxer put back on the keyframe before the playhead, the skip threshold hiding the -/// preroll and the clock re-anchored there. That is the one case where the audio is cut. +/// preroll and the clock re-anchored there. That is the one case where the audio is cut. A DVR session's +/// live feeder keeps no keyframe ledger, so its gap is always unknown, and the rule reads an unknown gap +/// as beyond the audio lead: a video recovery there repositions, the conservative choice. /// /// On the decoupled route (a VOD source with its own audio decoder) the demuxer reads video up to /// the audio lead target and parks it, so the keyframe gap is mostly settled by one question: is a diff --git a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift index 24d875c6a..d466e153c 100644 --- a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift +++ b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift @@ -944,6 +944,10 @@ final class SoftwarePlaybackHost { private var videoSessionRecoveryOwed = false /// How many keyframes have retried a refused rebuild in the foreground. Main actor only. private var videoSessionRetry = VideoSessionRetryPolicy() + /// The app is in the background. That includes a retained paused session, which keeps its video + /// pipeline without audio-only mode, and where VideoToolbox refuses a rebuild by design. Set by the + /// engine's lifecycle observers through `noteAppInBackground(_:)`. Main actor only. + private var appInBackground = false nonisolated(unsafe) private var _backgroundAudioOnly = false nonisolated var backgroundAudioOnly: Bool { get { flagsLock.lock(); defer { flagsLock.unlock() }; return _backgroundAudioOnly } @@ -1878,11 +1882,12 @@ final class SoftwarePlaybackHost { } /// A keyframe was fed while the session is lost. A rebuild that VideoToolbox refused in the foreground - /// is retried here, once per keyframe, up to `VideoSessionRetryPolicy.maxAttempts`; after that the - /// failure is surfaced instead of leaving a frozen picture beside live audio. Nothing is owed while the - /// first rebuild after a loss is still in flight, or in audio-only mode, so those keyframes do nothing. + /// is retried here, once per keyframe, up to `VideoSessionRetryPolicy.maxAttempts`. Nothing is spent + /// while the app is in the background (audio-only mode, or a retained paused session), where a rebuild + /// is refused by design, nor while the first rebuild after a loss is still in flight. Once the budget is + /// spent the host waits for the next foreground return, which starts a fresh one. private func handleKeyframeWhileSessionLost() { - guard !stopRequested, !backgroundAudioOnly else { return } + guard !stopRequested, !backgroundAudioOnly, !appInBackground else { return } switch videoSessionRetry.onKeyframe(owed: videoSessionRecoveryOwed, now: ProcessInfo.processInfo.systemUptime) { case .none: break @@ -1895,18 +1900,23 @@ final class SoftwarePlaybackHost { case .giveUp: EngineLog.emit( "[SWHost] AE#739 video session could not be rebuilt after \(VideoSessionRetryPolicy.maxAttempts) " - + "keyframe retries, giving up", + + "keyframe retries, waiting for the next foreground return", category: .swPlayback) - // `videoSessionRecoveryOwed` stays set on purpose: the failure is published, but a later - // foreground return still gets its own try (and a fresh budget). - if failure == nil { - failure = PlaybackErrorInfo( - kind: .softwarePipelineFailed, - message: "The video decoder could not be restarted after the system reclaimed it.") - } + // `videoSessionRecoveryOwed` stays set on purpose: the next foreground return gets its own try + // with a fresh budget. No `failure` is published. The engine's `.error` state is terminal until + // the next load (its status sinks return early on it), so a rebuild that succeeds later could + // not clear a failure published here, and the session would stay errored after it recovered. } } + /// The engine reports the app going to the background or back to the foreground, with or without + /// audio-only mode. A foreground return is a new episode for the keyframe retries, as it is for + /// `exitBackgroundAudioOnly()`, so the budget spent before it does not count against the next loss. + func noteAppInBackground(_ inBackground: Bool) { + appInBackground = inBackground + if !inBackground { videoSessionRetry.reset() } + } + /// Rebuild the session, then reposition to the playhead: the demuxer lands on the keyframe before it /// and the seek's skip threshold hides the preroll, so the picture returns on the frame the audio is /// at instead of freezing until the next keyframe. A live source without a rewind ring cannot go @@ -2075,6 +2085,20 @@ final class SoftwarePlaybackHost { var audioFedUntilForTesting: Double { demuxDiag.snapshot.lastAudioPts } var recoveryPendingForTesting: Bool { flushRecoveryPending } func setRepositionBackoffForTesting(_ seconds: Double) { repositionBackoffSeconds = seconds } + var audioAutomaticFlushCountForTesting: Int? { audioOutput?.automaticFlushCount } + /// A session the system has invalidated and whose rebuild is owed, as `handleVideoSessionLost()` leaves it. + func setVideoSessionRecoveryOwedForTesting() { videoSessionRecoveryOwed = true } + func keyframeWhileSessionLostForTesting() { handleKeyframeWhileSessionLost() } + /// A budget with every foreground retry spent, so the next keyframe gives up. + func spendVideoSessionRetriesForTesting() { + var policy = VideoSessionRetryPolicy() + for i in 0.. generation) // The decoder was flushed, which is what "resumes at the next keyframe" rests on. @@ -579,9 +583,6 @@ struct FlushRecoveryHostTests { #expect(decoder.feedEpoch > 0) // The floor restarts at the playhead, not at whatever frame arrives first. #expect(decoder.skipUntilPTS.map { abs($0.seconds - 20) < 0.01 } == true) - try await Task.sleep(for: .milliseconds(50)) - #expect(targets.values.isEmpty) - #expect(host.flushRecoveriesCompleted == 0) #expect(host.flushRecoveryLog.map(\.action) == [.flushVideoOnly]) } @@ -598,6 +599,7 @@ struct FlushRecoveryHostTests { let generation = host.videoRendererForTesting.flushGeneration #expect(host.handleFlushTrigger(.videoQueueFailed) == .reposition) + #expect(host.recoveryPendingForTesting) // Further reports of the same event are ignored while it runs. #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .ignore) try await waitFor { host.flushRecoveriesCompleted == 1 } @@ -711,7 +713,7 @@ struct FlushRecoveryHostTests { host.setLiveForTesting(true) try presentFrame(host) #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .flushVideoOnly) - try await Task.sleep(for: .milliseconds(50)) + #expect(!host.recoveryPendingForTesting) #expect(targets.values.isEmpty) #expect(host.flushRecoveriesCompleted == 0) } @@ -741,7 +743,7 @@ struct FlushRecoveryHostTests { host.enterBackgroundAudioOnly() #expect(host.handleFlushTrigger(.videoQueueRequiresFlush) == .deferToForeground) #expect(host.videoQueueFlushOwedForTesting) - try await Task.sleep(for: .milliseconds(50)) + #expect(!host.recoveryPendingForTesting) #expect(host.flushRecoveriesCompleted == 0) // Back in front with no keyframe read yet: the picture would stay frozen past the audio lead. @@ -764,7 +766,7 @@ struct FlushRecoveryHostTests { host.setAudioFedUntilForTesting(30) host.keyframeLedgerForTesting.noteKeyframeRead(pts: 22) host.exitBackgroundAudioOnly() - try await Task.sleep(for: .milliseconds(50)) + #expect(!host.recoveryPendingForTesting) #expect(targets.values.isEmpty) #expect(host.flushRecoveryLog.map(\.action) == [.deferToForeground, .flushVideoOnly]) } @@ -780,7 +782,9 @@ struct FlushRecoveryHostTests { name: .AVSampleBufferAudioRendererWasFlushedAutomatically, object: renderer, userInfo: [AVSampleBufferAudioRendererFlushTimeKey: NSValue(time: CMTime(seconds: 3, preferredTimescale: 48000))]) - try await Task.sleep(for: .milliseconds(150)) + // The AudioOutput's count is the witness that the renderer's notification was handled; the host + // hears nothing from it, so its log has to stay empty once the witness has landed. + try await waitFor { host.audioAutomaticFlushCountForTesting == 1 } #expect(host.flushRecoveryLog.isEmpty) #expect(targets.values.isEmpty) } @@ -800,7 +804,7 @@ struct FlushRecoveryHostTests { host.pollAudioRendererStatus(.failed) #expect(host.flushRecoveryLog.map(\.action) == [.flushAudioOnly, .flushAudioOnly]) #expect(host.flushRecoveryLog.allSatisfy { $0.trigger == .audioRendererFailed }) - try await Task.sleep(for: .milliseconds(50)) + #expect(!host.recoveryPendingForTesting) #expect(targets.values.isEmpty) } diff --git a/Tests/AetherEngineTests/VideoSessionRetryTests.swift b/Tests/AetherEngineTests/VideoSessionRetryTests.swift index 861366814..d92e8f358 100644 --- a/Tests/AetherEngineTests/VideoSessionRetryTests.swift +++ b/Tests/AetherEngineTests/VideoSessionRetryTests.swift @@ -7,7 +7,7 @@ import AetherLibavutil @testable import AetherEngine /// AE#739 follow-up: a rebuild that VideoToolbox refused in the foreground is retried at each following -/// keyframe, a bounded number of times, and ends in a surfaced failure rather than a frozen picture. +/// keyframe, a bounded number of times, then waits for the next foreground return instead of retrying for ever. @Suite("Video session retry", .serialized, .timeLimit(.minutes(1))) struct VideoSessionRetryTests { @@ -52,6 +52,48 @@ struct VideoSessionRetryTests { #expect(policy.onKeyframe(owed: true, now: 0) == .retry(attempt: 1)) } + // MARK: - Host: background, give-up and the foreground return + + @Test("a keyframe in the background spends no retry, whether audio-only or a retained paused session") + func backgroundSpendsNoRetry() { + let host = SoftwarePlaybackHost(reposition: { _, _, _, _, _ in .landed }) + defer { host.stop() } + host.noteAppInBackground(true) + host.setVideoSessionRecoveryOwedForTesting() + for _ in 0..<(VideoSessionRetryPolicy.maxAttempts + 2) { + host.keyframeWhileSessionLostForTesting() + } + #expect(host.videoSessionRetryAttemptsForTesting == 0) + #expect(!host.videoSessionRetryGaveUpForTesting) + #expect(host.failure == nil) + } + + @Test("a spent budget gives up without publishing a failure, so the session is not left in the error state") + func giveUpPublishesNoFailure() { + let host = SoftwarePlaybackHost(reposition: { _, _, _, _, _ in .landed }) + defer { host.stop() } + host.spendVideoSessionRetriesForTesting() + host.setVideoSessionRecoveryOwedForTesting() + host.keyframeWhileSessionLostForTesting() + #expect(host.videoSessionRetryGaveUpForTesting) + #expect(host.failure == nil) + } + + @Test("the foreground return starts a fresh budget after a give-up") + func foregroundReturnStartsFreshBudget() { + let host = SoftwarePlaybackHost(reposition: { _, _, _, _, _ in .landed }) + defer { host.stop() } + host.spendVideoSessionRetriesForTesting() + host.setVideoSessionRecoveryOwedForTesting() + host.keyframeWhileSessionLostForTesting() + #expect(host.videoSessionRetryGaveUpForTesting) + host.noteAppInBackground(true) + #expect(host.videoSessionRetryGaveUpForTesting) + host.noteAppInBackground(false) + #expect(!host.videoSessionRetryGaveUpForTesting) + #expect(host.videoSessionRetryAttemptsForTesting == 0) + } + // MARK: - Decoder reports keyframes while lost private final class Counter: @unchecked Sendable { diff --git a/docs/architecture.md b/docs/architecture.md index 56adc548b..6d30dc782 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -557,7 +557,7 @@ Sources/AetherEngine/ │ ├── RemoteHLSMediaSelection.swift Remote-HLS bypass (#154): pure loopback→bypass reroute decision for non-live m3u8 sources (FFmpeg has no network) + legible AVMediaSelectionGroup → `subtitleTracks` mapping (synthetic ids from 200000) │ ├── SoftwarePlaybackHost.swift SW path: demux loop + decoders + renderer + synchronizer orchestration │ ├── SoftwarePlaybackHost+AudioProducer.swift SW path: picks and builds the audio producer for the session's track, with every unavailable Atmos route landing on the PCM decode (logged) -│ ├── VideoSessionRetryPolicy.swift SW path: how many keyframes retry a refused video-session rebuild in the foreground before the failure is surfaced +│ ├── VideoSessionRetryPolicy.swift SW path: how many keyframes retry a refused video-session rebuild in the foreground before the host waits for the next foreground return │ ├── SoftwareFlushRecovery.swift SW path: pure rules for putting a flushed or failed video/audio output queue back to work (triggers, seek versus wait for the next keyframe) │ ├── AirPlayPlaylistDecision.swift Pure decisions for the wireless-AirPlay loopback rewrite: which of master / media a receiver is handed, and whether a route change rebuilds the session or swaps its item (a sequential origin, a forward-only live source), kept separate and testable offline like `MasterFallbackDecision` (#86, #227, #526) │ ├── NativeLegibleDeselectPin.swift Keeps AVKit's ready-time legible auto-select from engaging a rendition the host never asked for (Sodalite#38), and publishes the caption requests the system makes on its own as `systemCaptionRequest`, by BCP 47 language rather than by ordinal (Sodalite#65) From fe8b453c1b0a9152cf8f24382ba03dc415724c54 Mon Sep 17 00:00:00 2001 From: Pavel Kajaba <15980400+pkajaba@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:07:25 +0000 Subject: [PATCH 4/4] fix(software-host): retry a refused video-session rebuild slowly instead of giving up A refused rebuild now keeps being retried at keyframes: four quick attempts, then one every 30 seconds, so a session the system gives back later recovers without a reload. Nothing is retried while the app is in the background without picture-in-picture, where VideoToolbox refuses by design. The engine tells the host when the picture is off screen, and a foreground return starts a fresh quick budget. No failure is published for a refused rebuild. The give-up that stopped the retries froze the picture for good on platforms that never return to the foreground (macOS), with no error shown. The give-up also did not publish a failure: the engine's .error state is terminal until the next load. The DVR flush-recovery doc now names the rules that can degrade a reposition to a plain flush. --- CHANGELOG.md | 11 ++-- Sources/AetherEngine/AetherEngine.swift | 12 +++- .../Native/SoftwareFlushRecovery.swift | 3 +- .../Native/SoftwarePlaybackHost.swift | 40 +++++-------- .../Native/VideoSessionRetryPolicy.swift | 33 +++++----- .../VideoSessionRetryTests.swift | 60 ++++++++++++------- docs/architecture.md | 2 +- 7 files changed, 88 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ba979458..293c82eaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,11 +34,12 @@ the public-API contract. - **A refused video-session rebuild is retried at the next keyframes in the foreground.** When the system invalidated the hardware session and VideoToolbox refused the rebuild, the decoder dropped every packet - and the picture froze beside live audio until the next foreground return. The host now retries at each - keyframe fed while the session is still lost (at least two seconds apart), up to four times, and then - waits for the next foreground return, which starts a fresh budget. Nothing is retried in the background, - where VideoToolbox refuses a rebuild by design. The session is not moved to `.error` for this: that state - is terminal until the next load, so a rebuild that succeeds later could not clear it. + and the picture froze beside live audio. The host now retries at each keyframe fed while the session is + still lost: four attempts at least two seconds apart, then one attempt every 30 seconds, so a session the + system gives back later recovers without a reload. Nothing is retried while the app is in the background + without picture-in-picture, where VideoToolbox refuses a rebuild by design; a foreground return starts a + fresh quick budget. The session is not moved to `.error` for this: that state is terminal until the next + load, so a rebuild that succeeds later could not clear it. - **The in-band HEVC parameter-set scan stops at 16 MiB of input.** A stream that carries no parameter sets, or one whose packets are very large, could be read far past what the packet budget implies before the diff --git a/Sources/AetherEngine/AetherEngine.swift b/Sources/AetherEngine/AetherEngine.swift index 5804cf540..3a3555857 100644 --- a/Sources/AetherEngine/AetherEngine.swift +++ b/Sources/AetherEngine/AetherEngine.swift @@ -946,6 +946,7 @@ public final class AetherEngine: ObservableObject { // SW-PiP Phase C: flip the frame compositor with the PiP state so subtitles appear in the // window and never double-draw under the fullscreen host overlay. softwareHost?.updateSubtitleCompositor(cues: subtitleCues + secondarySubtitleCues, enabled: pictureInPictureActive, delaySeconds: softwareSubtitleDelaySeconds) + noteVideoPresentationToHost() #if os(tvOS) // PiP window closed while backgrounded: nothing keeps the app running anymore, so the // wedge-safe teardown is due before idle suspension (mirrors the iOS pause-while-backgrounded path). @@ -8307,7 +8308,6 @@ public final class AetherEngine: ObservableObject { ) { [weak self] _ in Task { @MainActor in guard let self else { return } - self.softwareHost?.noteAppInBackground(false) #if os(iOS) self.cancelBackgroundGraceWindow() self.softwareHost?.exitBackgroundAudioOnly() @@ -8319,6 +8319,7 @@ public final class AetherEngine: ObservableObject { self.isBackgrounded = false if returnedFromBackground { Self.clearPanelRefusalOnForegroundReturn() } self.backgroundActionOwed = false + self.noteVideoPresentationToHost() } } lifecycleObservers.append(fgObserver) @@ -8467,7 +8468,14 @@ public final class AetherEngine: ObservableObject { func noteDidEnterBackground() { isBackgrounded = true backgroundActionOwed = backgroundDecisionMustWait - softwareHost?.noteAppInBackground(true) + noteVideoPresentationToHost() + } + + /// Tells the software host whether a refused video rebuild may be retried now. A backgrounded app with + /// picture-in-picture on still shows the picture, and the software host keeps decoding for it, so only + /// a background with no picture window counts as "nothing is showing video". + private func noteVideoPresentationToHost() { + softwareHost?.noteAppInBackground(isBackgrounded && !pictureInPictureActive) } /// Audit LIF-105: the PiP window closed while backgrounded, so nothing keeps the app running and diff --git a/Sources/AetherEngine/Native/SoftwareFlushRecovery.swift b/Sources/AetherEngine/Native/SoftwareFlushRecovery.swift index 5ce815d7c..fc02d696c 100644 --- a/Sources/AetherEngine/Native/SoftwareFlushRecovery.swift +++ b/Sources/AetherEngine/Native/SoftwareFlushRecovery.swift @@ -29,7 +29,8 @@ enum FlushRecoveryTrigger: String, Sendable, CaseIterable { /// flushed, the demuxer put back on the keyframe before the playhead, the skip threshold hiding the /// preroll and the clock re-anchored there. That is the one case where the audio is cut. A DVR session's /// live feeder keeps no keyframe ledger, so its gap is always unknown, and the rule reads an unknown gap -/// as beyond the audio lead: a video recovery there repositions, the conservative choice. +/// as beyond the audio lead: a video recovery there repositions, the conservative choice (the backoff and +/// no-presented-frame rules below can still degrade that to a plain flush). /// /// On the decoupled route (a VOD source with its own audio decoder) the demuxer reads video up to /// the audio lead target and parks it, so the keyframe gap is mostly settled by one question: is a diff --git a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift index d466e153c..116bbf875 100644 --- a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift +++ b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift @@ -944,9 +944,9 @@ final class SoftwarePlaybackHost { private var videoSessionRecoveryOwed = false /// How many keyframes have retried a refused rebuild in the foreground. Main actor only. private var videoSessionRetry = VideoSessionRetryPolicy() - /// The app is in the background. That includes a retained paused session, which keeps its video - /// pipeline without audio-only mode, and where VideoToolbox refuses a rebuild by design. Set by the - /// engine's lifecycle observers through `noteAppInBackground(_:)`. Main actor only. + /// Nothing shows the picture: the app is in the background with no picture-in-picture window. That covers + /// audio-only mode and a retained paused session, where VideoToolbox refuses a rebuild by design. The engine + /// reports it through `noteAppInBackground(_:)`. Main actor only. private var appInBackground = false nonisolated(unsafe) private var _backgroundAudioOnly = false nonisolated var backgroundAudioOnly: Bool { @@ -1881,11 +1881,12 @@ final class SoftwarePlaybackHost { recoverVideoSession() } - /// A keyframe was fed while the session is lost. A rebuild that VideoToolbox refused in the foreground - /// is retried here, once per keyframe, up to `VideoSessionRetryPolicy.maxAttempts`. Nothing is spent - /// while the app is in the background (audio-only mode, or a retained paused session), where a rebuild - /// is refused by design, nor while the first rebuild after a loss is still in flight. Once the budget is - /// spent the host waits for the next foreground return, which starts a fresh one. + /// A keyframe was fed while the session is lost. A rebuild that VideoToolbox refused is retried here, once + /// per keyframe: quickly for the first `VideoSessionRetryPolicy.maxAttempts`, then every + /// `VideoSessionRetryPolicy.slowSpacing` seconds, so a session the system gives back later still recovers + /// without a load. Nothing is spent while the picture is off screen (see `appInBackground`), nor while the + /// first rebuild after a loss is still in flight. No failure is published for a refused rebuild: the + /// engine's `.error` state is terminal until the next load, so a later rebuild could not clear it. private func handleKeyframeWhileSessionLost() { guard !stopRequested, !backgroundAudioOnly, !appInBackground else { return } switch videoSessionRetry.onKeyframe(owed: videoSessionRecoveryOwed, now: ProcessInfo.processInfo.systemUptime) { @@ -1893,28 +1894,18 @@ final class SoftwarePlaybackHost { break case .retry(let attempt): EngineLog.emit( - "[SWHost] AE#739 video session still lost at a keyframe, rebuild retry \(attempt) of " - + "\(VideoSessionRetryPolicy.maxAttempts)", + "[SWHost] AE#739 video session still lost at a keyframe, rebuild retry \(attempt)", category: .swPlayback) recoverVideoSession() - case .giveUp: - EngineLog.emit( - "[SWHost] AE#739 video session could not be rebuilt after \(VideoSessionRetryPolicy.maxAttempts) " - + "keyframe retries, waiting for the next foreground return", - category: .swPlayback) - // `videoSessionRecoveryOwed` stays set on purpose: the next foreground return gets its own try - // with a fresh budget. No `failure` is published. The engine's `.error` state is terminal until - // the next load (its status sinks return early on it), so a rebuild that succeeds later could - // not clear a failure published here, and the session would stay errored after it recovered. } } - /// The engine reports the app going to the background or back to the foreground, with or without - /// audio-only mode. A foreground return is a new episode for the keyframe retries, as it is for - /// `exitBackgroundAudioOnly()`, so the budget spent before it does not count against the next loss. + /// The engine reports whether the picture is off screen (backgrounded with no picture-in-picture window). + /// Coming back on screen is a new episode for the keyframe retries, as `exitBackgroundAudioOnly()` says, + /// so the budget spent before it does not count against the next loss. func noteAppInBackground(_ inBackground: Bool) { + if appInBackground && !inBackground { videoSessionRetry.reset() } appInBackground = inBackground - if !inBackground { videoSessionRetry.reset() } } /// Rebuild the session, then reposition to the playhead: the demuxer lands on the keyframe before it @@ -2089,7 +2080,7 @@ final class SoftwarePlaybackHost { /// A session the system has invalidated and whose rebuild is owed, as `handleVideoSessionLost()` leaves it. func setVideoSessionRecoveryOwedForTesting() { videoSessionRecoveryOwed = true } func keyframeWhileSessionLostForTesting() { handleKeyframeWhileSessionLost() } - /// A budget with every foreground retry spent, so the next keyframe gives up. + /// A budget with every quick retry spent, so the next keyframe waits out the slow spacing. func spendVideoSessionRetriesForTesting() { var policy = VideoSessionRetryPolicy() for i in 0.. Decision { - guard owed, !gaveUp else { return .none } - if let lastAttempt, now - lastAttempt < Self.minSpacing { return .none } - if attempts >= Self.maxAttempts { - gaveUp = true - return .giveUp - } + guard owed else { return .none } + let spacing = attempts < Self.maxAttempts ? Self.minSpacing : Self.slowSpacing + if let lastAttempt, now - lastAttempt < spacing { return .none } attempts += 1 lastAttempt = now return .retry(attempt: attempts) } - /// A rebuild succeeded, or the app returned to the foreground: the next loss starts a fresh budget. + /// A rebuild succeeded, or the picture came back on screen: the next loss starts a fresh quick budget. mutating func reset() { attempts = 0 - gaveUp = false lastAttempt = nil } } diff --git a/Tests/AetherEngineTests/VideoSessionRetryTests.swift b/Tests/AetherEngineTests/VideoSessionRetryTests.swift index d92e8f358..9ca14f95c 100644 --- a/Tests/AetherEngineTests/VideoSessionRetryTests.swift +++ b/Tests/AetherEngineTests/VideoSessionRetryTests.swift @@ -6,22 +6,26 @@ import AetherLibavcodec import AetherLibavutil @testable import AetherEngine -/// AE#739 follow-up: a rebuild that VideoToolbox refused in the foreground is retried at each following -/// keyframe, a bounded number of times, then waits for the next foreground return instead of retrying for ever. +/// AE#739 follow-up: a rebuild that VideoToolbox refused while the picture is on screen is retried at each +/// following keyframe: quickly at first, then at a slower spacing, so a session the system gives back recovers. @Suite("Video session retry", .serialized, .timeLimit(.minutes(1))) struct VideoSessionRetryTests { // MARK: - Policy - @Test("each keyframe while a recovery is owed is one retry, up to the cap, then one give-up") - func boundedRetries() { + @Test("the first retries are quick, then the retries slow down instead of stopping") + func quickThenSlow() { var policy = VideoSessionRetryPolicy() var decisions: [VideoSessionRetryPolicy.Decision] = [] - for i in 0..<(VideoSessionRetryPolicy.maxAttempts + 3) { + // Four keyframes three seconds apart are all quick attempts; the slow spacing then holds the next back. + for i in 0..