From 77e6deb2afd750d6032407751f4c7a56a18c9ed5 Mon Sep 17 00:00:00 2001 From: qoli <2818670+qoli@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:51:59 +0800 Subject: [PATCH 1/2] perf(subtitles): retain bitmap images with deferred PNG backing Keep authored subtitle history lossless while avoiding expanded RGBA storage for every retained cue. Document the drawing contract and encoded backing accounting, with pixel, provider lifetime and subtitle regressions. --- CHANGELOG.md | 8 ++ .../Decoder/EmbeddedSubtitleDecoder.swift | 41 +++++++++- Sources/AetherEngine/PlayerState.swift | 11 ++- .../Issue628PGSOffMainDecodeTests.swift | 14 ++-- .../SubtitleBitmapStorageTests.swift | 76 +++++++++++++++++++ docs/api.md | 2 +- 6 files changed, 141 insertions(+), 11 deletions(-) create mode 100644 Tests/AetherEngineTests/SubtitleBitmapStorageTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index cadfbc99f..7ab7208e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -112,6 +112,14 @@ the public-API contract. ### Fixed +- **Retained bitmap subtitles use lossless encoded backing instead of keeping every image + expanded to RGBA.** PGS, DVB and DVD subtitle history retains PNG-backed `CGImage` providers + and expands pixels when drawn, preserving the authored crop, placement, alpha, forced flag + and cue lifetime for subtitle offsets and backward seeks. Encoding runs on the existing + subtitle worker. `SubtitleImage.encodedBackingBytes` reports the known compressed backing + size separately from process memory; callers draw the image rather than assuming its provider + has a particular raw pixel layout. + - **Source failures keep their cause and request ownership.** Initial live probes preserve HTTP, URL and TLS failures without changing the live-start `liveSourceUnavailable` kind (HTTP and certificate classifications stay specific). A transport failure at open (a reset or a timeout from diff --git a/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift b/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift index 6cb4293fb..0adcc4e98 100644 --- a/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift +++ b/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift @@ -1,5 +1,7 @@ import Foundation import CoreGraphics +import ImageIO +import UniformTypeIdentifiers import Compression import AetherLibavcodec import AetherLibavformat @@ -622,7 +624,8 @@ final class EmbeddedSubtitleDecoder { /// Render a PGS/DVB/HDMV bitmap rect into a CGImage with normalised position. /// Palette from libavcodec is 32-bit with alpha in the high byte and BGR below ([B,G,R,A] on little-endian); - /// rewritten to RGBA for CGImage's premultipliedLast. Internal for unit tests (#146). + /// rewritten to premultiplied RGBA, then retained losslessly with deferred pixel decoding. + /// Internal for unit tests (#146). static func imageForSubtitleRect( _ rect: UnsafeMutablePointer, videoWidth: Int, @@ -724,8 +727,40 @@ final class EmbeddedSubtitleDecoder { position = CGRect(x: 0.1, y: 0.8, width: 0.8, height: 0.15) } - return SubtitleImage(cgImage: cgImage, position: position, + guard let retained = retainedBitmap(cgImage) else { + EngineLog.emit("[subtitle] lossless bitmap storage failed size=\(cropW)x\(cropH)") + return nil + } + return SubtitleImage(cgImage: retained.image, position: position, canvasSize: CGSize(width: Int(videoWidth), height: Int(videoHeight)), - isForced: (r.flags & AV_SUBTITLE_FLAG_FORCED) != 0) + isForced: (r.flags & AV_SUBTITLE_FLAG_FORCED) != 0, + encodedBackingBytes: retained.encodedByteCount) + } + + /// The subtitle history needs authored images for offsets and backward seeks, but does not + /// need every image expanded to RGBA at once. Retain PNG bytes through ImageIO's provider; + /// drawing decodes the same pixels without caching them for the whole cue lifetime. This + /// runs on the existing subtitle decode worker, not on the presentation actor. + static func retainedBitmap(_ image: CGImage) -> (image: CGImage, encodedByteCount: Int)? { + autoreleasepool { + let data = NSMutableData() + guard let destination = CGImageDestinationCreateWithData( + data, UTType.png.identifier as CFString, 1, nil + ) else { return nil } + CGImageDestinationAddImage(destination, image, nil) + guard CGImageDestinationFinalize(destination) else { return nil } + let options = [kCGImageSourceShouldCache: false] as CFDictionary + guard let source = CGImageSourceCreateWithData(data, options), + let decoded = CGImageSourceCreateImageAtIndex(source, 0, options), + let colorSpace = decoded.colorSpace, let provider = decoded.dataProvider, + let retained = CGImage( + width: decoded.width, height: decoded.height, + bitsPerComponent: decoded.bitsPerComponent, bitsPerPixel: decoded.bitsPerPixel, + bytesPerRow: decoded.bytesPerRow, space: colorSpace, + bitmapInfo: decoded.bitmapInfo, provider: provider, decode: nil, + shouldInterpolate: image.shouldInterpolate, intent: image.renderingIntent + ) else { return nil } + return (retained, data.length) + } } } diff --git a/Sources/AetherEngine/PlayerState.swift b/Sources/AetherEngine/PlayerState.swift index 03f70f361..c6a3b2744 100644 --- a/Sources/AetherEngine/PlayerState.swift +++ b/Sources/AetherEngine/PlayerState.swift @@ -1619,7 +1619,9 @@ extension SubtitleCue: Equatable { } } -/// Decoded PGS / HDMV PGS / DVB / DVD bitmap subtitle. CGImage is fully rendered (RGBA, premultiplied alpha). Position is [0, 1] against the source video frame; multiply by the on-screen video rect to place it. +/// Decoded PGS / HDMV PGS / DVB / DVD bitmap subtitle. CGImage carries the rendered image and +/// alpha metadata; its provider may defer pixel expansion. Draw it rather than assuming a raw +/// premultiplied RGBA provider layout. Position is [0, 1] against the source video frame. public struct SubtitleImage: @unchecked Sendable { public let cgImage: CGImage public let position: CGRect @@ -1632,12 +1634,17 @@ public struct SubtitleImage: @unchecked Sendable { /// AV_SUBTITLE_FLAG_FORCED from the decoded rect (#146): the disc authored this object as a /// forced caption (sign / foreign-dialogue overlay shown even with subtitles off). public let isForced: Bool + /// Known compressed provider storage, when the decoder retains a lossless encoded image. + /// This excludes ImageIO/CGImage overhead and temporary pixels used while drawing. + public let encodedBackingBytes: Int? - public init(cgImage: CGImage, position: CGRect, canvasSize: CGSize = .zero, isForced: Bool = false) { + public init(cgImage: CGImage, position: CGRect, canvasSize: CGSize = .zero, isForced: Bool = false, + encodedBackingBytes: Int? = nil) { self.cgImage = cgImage self.position = position self.canvasSize = canvasSize self.isForced = isForced + self.encodedBackingBytes = encodedBackingBytes } } diff --git a/Tests/AetherEngineTests/Issue628PGSOffMainDecodeTests.swift b/Tests/AetherEngineTests/Issue628PGSOffMainDecodeTests.swift index f167fea5c..f59e3db1f 100644 --- a/Tests/AetherEngineTests/Issue628PGSOffMainDecodeTests.swift +++ b/Tests/AetherEngineTests/Issue628PGSOffMainDecodeTests.swift @@ -47,11 +47,15 @@ struct Issue628PGSOffMainDecodeTests { } private static func bytes(of image: CGImage) -> [UInt8] { - let data = image.dataProvider!.data! as Data - var out: [UInt8] = [] - for row in 0.. 0) + #expect(retained.image.width == width) + #expect(retained.image.height == height) + #expect(!retained.image.shouldInterpolate) + #expect(render(retained.image) == render(original)) + // Repeated presentation and backward seek use the same retained provider. + #expect(render(retained.image) == render(original)) + } + + @Test("flat subtitle pixels retain compressed backing rather than a full RGBA plane") + func flatSubtitleStorageIsSmaller() throws { + let width = 1024, height = 96 + var pixels = Data(count: width * height * 4) + pixels.withUnsafeMutableBytes { (bytes: UnsafeMutableRawBufferPointer) in + for y in 8..<88 { + for x in 16..<1008 where (x + y) % 12 < 4 { + let offset = (y * width + x) * 4 + bytes[offset] = 128; bytes[offset + 1] = 64 + bytes[offset + 2] = 32; bytes[offset + 3] = 128 + } + } + } + let original = try #require(makeImage(pixels, width: width, height: height)) + let retained = try #require(EmbeddedSubtitleDecoder.retainedBitmap(original)) + #expect(retained.encodedByteCount < pixels.count) + #expect(render(retained.image) == render(original)) + let image = SubtitleImage(cgImage: retained.image, position: .zero, + encodedBackingBytes: retained.encodedByteCount) + #expect(image.encodedBackingBytes == retained.encodedByteCount) + } + + private func makeImage(_ data: Data, width: Int, height: Int) -> CGImage? { + guard let provider = CGDataProvider(data: data as CFData), + let space = CGColorSpace(name: CGColorSpace.sRGB) else { return nil } + return CGImage(width: width, height: height, bitsPerComponent: 8, bitsPerPixel: 32, + bytesPerRow: width * 4, space: space, + bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue), + provider: provider, decode: nil, shouldInterpolate: false, intent: .defaultIntent) + } + + private func render(_ image: CGImage) -> Data { + var data = Data(count: image.width * image.height * 4) + data.withUnsafeMutableBytes { bytes in + let context = CGContext(data: bytes.baseAddress, width: image.width, height: image.height, + bitsPerComponent: 8, bytesPerRow: image.width * 4, + space: CGColorSpace(name: CGColorSpace.sRGB)!, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)! + context.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height)) + } + return data + } +} diff --git a/docs/api.md b/docs/api.md index 2a2a4da77..771a3d8f8 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1450,7 +1450,7 @@ All flags default to safe values; the table is the full set. Depth for the media | `SubtitleCue` | `id`, `startTime`, `endTime`, `body` (a `SubtitleCue.Body`: `.text`, `.richText`, `.image`), `placement`, plus `text` and `isForced` conveniences. | | `SubtitleTextRun` | `text`, `color`, `isBold`, `isItalic`, `isUnderlined`, `isStruckThrough`, `fontName`, `fontSize`, `isStyled`. | | `SubtitleTextPlacement` | `alignment` (numpad), `position` (a [0, 1] anchor). | -| `SubtitleImage` | `cgImage`, `position`, `canvasSize`, `isForced`. | +| `SubtitleImage` | `cgImage`, `position`, `canvasSize`, `isForced`, `encodedBackingBytes`. Bitmap images retain lossless PNG backing and expand pixels when drawn. Draw the `CGImage` rather than assuming its provider exposes premultiplied RGBA bytes. `encodedBackingBytes` is the known compressed backing size, or nil for an image without that accounting; it excludes ImageIO/CGImage overhead and temporary drawing pixels, so it is not a process-memory measurement. | | `ExternalSubtitleTrack` | `url`, `name`, `language`, `isForced`, `isHearingImpaired`, `isDefault`, `httpHeaders` (nil forwards `LoadOptions.httpHeaders`), `formatHint` for URLs whose path hides the format, and `sourceStreamIndex` for a container holding several subtitle streams. That index addresses the container at `url`, not the played media. | | `NativeSubtitleTrack` | `ordinal`, `language`, `displayName`, plus `sameLanguageRank(of:in:)` for disambiguating same-language options (eng Full against eng SDH). | | `RecordingState` | `.idle`, `.recording(RecordingProgress)`, `.ended(RecordingEndReason)`, `.failed(RecordingFailure)`. What the session's live recording is doing. | From d95d2d6edd8ae3115b8b9d55da8e32fe2bebc718 Mon Sep 17 00:00:00 2001 From: qoli <2818670+qoli@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:04:28 +0800 Subject: [PATCH 2/2] docs(subtitles): clarify PNG presentation and provider contracts Document renderer-retained pixels and main-thread decode risks. Separate the host-visible provider contract change from the added diagnostic count. --- CHANGELOG.md | 21 ++++++++++++------- .../Decoder/EmbeddedSubtitleDecoder.swift | 9 ++++---- Sources/AetherEngine/PlayerState.swift | 3 ++- docs/api.md | 2 +- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ab7208e4..a5939eac7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ the public-API contract. ### Added +- **`SubtitleImage.encodedBackingBytes` reports known compressed provider storage.** + The optional count excludes ImageIO/CGImage overhead, temporary drawing pixels and decoded + pixels a renderer retains. It is a diagnostic count, not a process-footprint measurement or + a memory budget. + - **Remote MPEG-DASH on-demand manifests play (AE#859, thanks @qoli).** A static, single-Period MPD whose representations are single files addressed by `SegmentBase` failed at open with `Invalid data found when processing input`: the FFmpeg build has no DASH @@ -76,6 +81,14 @@ the public-API contract. ### Changed +- **`SubtitleImage.cgImage` no longer guarantees a premultiplied RGBA data-provider layout.** + Bitmap subtitle history retains lossless PNG-backed images. Hosts that inspect provider bytes + must draw into an explicitly configured bitmap context instead of interpreting them as raw + RGBA. Authored pixels, alpha, crop, placement, forced flags and cue lifetime are preserved. + PNG encoding runs on the subtitle worker; pixel decoding follows the consumer's rendering + path. Core Animation may decode during a main-thread commit and retain decoded pixels for + the image's lifetime, so encoded backing alone does not bound post-presentation memory. + - **A Profile 7 FEL source keeps the native route while the host reports an AirPlay video receiver on the audio route (iOS).** The dual-layer route draws into a local layer that no receiver picks up, so a TV the viewer sent the title to played sound only. When the host says the receiver shows video @@ -112,14 +125,6 @@ the public-API contract. ### Fixed -- **Retained bitmap subtitles use lossless encoded backing instead of keeping every image - expanded to RGBA.** PGS, DVB and DVD subtitle history retains PNG-backed `CGImage` providers - and expands pixels when drawn, preserving the authored crop, placement, alpha, forced flag - and cue lifetime for subtitle offsets and backward seeks. Encoding runs on the existing - subtitle worker. `SubtitleImage.encodedBackingBytes` reports the known compressed backing - size separately from process memory; callers draw the image rather than assuming its provider - has a particular raw pixel layout. - - **Source failures keep their cause and request ownership.** Initial live probes preserve HTTP, URL and TLS failures without changing the live-start `liveSourceUnavailable` kind (HTTP and certificate classifications stay specific). A transport failure at open (a reset or a timeout from diff --git a/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift b/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift index 0adcc4e98..c120ff7b3 100644 --- a/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift +++ b/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift @@ -737,10 +737,11 @@ final class EmbeddedSubtitleDecoder { encodedBackingBytes: retained.encodedByteCount) } - /// The subtitle history needs authored images for offsets and backward seeks, but does not - /// need every image expanded to RGBA at once. Retain PNG bytes through ImageIO's provider; - /// drawing decodes the same pixels without caching them for the whole cue lifetime. This - /// runs on the existing subtitle decode worker, not on the presentation actor. + /// Retain authored bitmap history as lossless PNG for offsets and backward seeks. + /// Encoding runs on the subtitle decode worker; pixel decoding follows the consumer's + /// rendering path. Core Animation may decode during a main-thread commit and retain the + /// decoded pixels for the CGImage's lifetime. Disabling ImageIO caching does not prevent + /// renderer-side caching or bound the process footprint after presentation. static func retainedBitmap(_ image: CGImage) -> (image: CGImage, encodedByteCount: Int)? { autoreleasepool { let data = NSMutableData() diff --git a/Sources/AetherEngine/PlayerState.swift b/Sources/AetherEngine/PlayerState.swift index c6a3b2744..77e1fd738 100644 --- a/Sources/AetherEngine/PlayerState.swift +++ b/Sources/AetherEngine/PlayerState.swift @@ -1635,7 +1635,8 @@ public struct SubtitleImage: @unchecked Sendable { /// forced caption (sign / foreign-dialogue overlay shown even with subtitles off). public let isForced: Bool /// Known compressed provider storage, when the decoder retains a lossless encoded image. - /// This excludes ImageIO/CGImage overhead and temporary pixels used while drawing. + /// Excludes ImageIO/CGImage overhead, temporary drawing pixels and decoded pixels retained + /// by a renderer. This diagnostic count is not a process-footprint measurement or memory bound. public let encodedBackingBytes: Int? public init(cgImage: CGImage, position: CGRect, canvasSize: CGSize = .zero, isForced: Bool = false, diff --git a/docs/api.md b/docs/api.md index 771a3d8f8..f66fbfcb3 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1450,7 +1450,7 @@ All flags default to safe values; the table is the full set. Depth for the media | `SubtitleCue` | `id`, `startTime`, `endTime`, `body` (a `SubtitleCue.Body`: `.text`, `.richText`, `.image`), `placement`, plus `text` and `isForced` conveniences. | | `SubtitleTextRun` | `text`, `color`, `isBold`, `isItalic`, `isUnderlined`, `isStruckThrough`, `fontName`, `fontSize`, `isStyled`. | | `SubtitleTextPlacement` | `alignment` (numpad), `position` (a [0, 1] anchor). | -| `SubtitleImage` | `cgImage`, `position`, `canvasSize`, `isForced`, `encodedBackingBytes`. Bitmap images retain lossless PNG backing and expand pixels when drawn. Draw the `CGImage` rather than assuming its provider exposes premultiplied RGBA bytes. `encodedBackingBytes` is the known compressed backing size, or nil for an image without that accounting; it excludes ImageIO/CGImage overhead and temporary drawing pixels, so it is not a process-memory measurement. | +| `SubtitleImage` | `cgImage`, `position`, `canvasSize`, `isForced`, `encodedBackingBytes`. Bitmap images retain lossless PNG backing and expand pixels through the consumer's rendering path. The provider no longer guarantees a premultiplied RGBA layout: hosts that need raw pixels must draw into an explicitly configured bitmap context. Core Animation may decode during a main-thread commit and retain decoded pixels for the image's lifetime. `encodedBackingBytes` is the known compressed backing size, or nil for an image without that accounting; it excludes ImageIO/CGImage overhead, temporary drawing pixels and renderer-retained decoded pixels, so it is neither a process-footprint measurement nor a memory bound. | | `ExternalSubtitleTrack` | `url`, `name`, `language`, `isForced`, `isHearingImpaired`, `isDefault`, `httpHeaders` (nil forwards `LoadOptions.httpHeaders`), `formatHint` for URLs whose path hides the format, and `sourceStreamIndex` for a container holding several subtitle streams. That index addresses the container at `url`, not the played media. | | `NativeSubtitleTrack` | `ordinal`, `language`, `displayName`, plus `sameLanguageRank(of:in:)` for disambiguating same-language options (eng Full against eng SDH). | | `RecordingState` | `.idle`, `.recording(RecordingProgress)`, `.ended(RecordingEndReason)`, `.failed(RecordingFailure)`. What the session's live recording is doing. |