perf(subtitles): retain bitmap images with deferred PNG backing

CHANGELOG.md

@@ -112,6 +112,14 @@ the public-API contract.
 
 ### Fixed
 
+- **Retained bitmap subtitles use lossless encoded backing instead of keeping every image
+  expanded to RGBA.** PGS, DVB and DVD subtitle history retains PNG-backed `CGImage` providers
+  and expands pixels when drawn, preserving the authored crop, placement, alpha, forced flag
+  and cue lifetime for subtitle offsets and backward seeks. Encoding runs on the existing
+  subtitle worker. `SubtitleImage.encodedBackingBytes` reports the known compressed backing
+  size separately from process memory; callers draw the image rather than assuming its provider
+  has a particular raw pixel layout.
+
 - **Source failures keep their cause and request ownership.** Initial live probes preserve HTTP,
   URL and TLS failures without changing the live-start `liveSourceUnavailable` kind (HTTP and
   certificate classifications stay specific). A transport failure at open (a reset or a timeout from

Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift

@@ -1,5 +1,7 @@
 import Foundation
 import CoreGraphics
+import ImageIO
+import UniformTypeIdentifiers
 import Compression
 import AetherLibavcodec
 import AetherLibavformat
@@ -622,7 +624,8 @@ final class EmbeddedSubtitleDecoder {
 
     /// Render a PGS/DVB/HDMV bitmap rect into a CGImage with normalised position.
     /// Palette from libavcodec is 32-bit with alpha in the high byte and BGR below ([B,G,R,A] on little-endian);
-    /// rewritten to RGBA for CGImage's premultipliedLast. Internal for unit tests (#146).
+    /// rewritten to premultiplied RGBA, then retained losslessly with deferred pixel decoding.
+    /// Internal for unit tests (#146).
     static func imageForSubtitleRect(
         _ rect: UnsafeMutablePointer<AVSubtitleRect>,
         videoWidth: Int,
@@ -724,8 +727,40 @@ final class EmbeddedSubtitleDecoder {
             position = CGRect(x: 0.1, y: 0.8, width: 0.8, height: 0.15)
         }
 
-        return SubtitleImage(cgImage: cgImage, position: position,
+        guard let retained = retainedBitmap(cgImage) else {
+            EngineLog.emit("[subtitle] lossless bitmap storage failed size=\(cropW)x\(cropH)")
+            return nil
+        }
+        return SubtitleImage(cgImage: retained.image, position: position,
                              canvasSize: CGSize(width: Int(videoWidth), height: Int(videoHeight)),
-                             isForced: (r.flags & AV_SUBTITLE_FLAG_FORCED) != 0)
+                             isForced: (r.flags & AV_SUBTITLE_FLAG_FORCED) != 0,
+                             encodedBackingBytes: retained.encodedByteCount)
+    }
+
+    /// The subtitle history needs authored images for offsets and backward seeks, but does not
+    /// need every image expanded to RGBA at once. Retain PNG bytes through ImageIO's provider;
+    /// drawing decodes the same pixels without caching them for the whole cue lifetime. This
+    /// runs on the existing subtitle decode worker, not on the presentation actor.
+    static func retainedBitmap(_ image: CGImage) -> (image: CGImage, encodedByteCount: Int)? {
+        autoreleasepool {
+            let data = NSMutableData()
+            guard let destination = CGImageDestinationCreateWithData(
+                data, UTType.png.identifier as CFString, 1, nil
+            ) else { return nil }
+            CGImageDestinationAddImage(destination, image, nil)
+            guard CGImageDestinationFinalize(destination) else { return nil }
+            let options = [kCGImageSourceShouldCache: false] as CFDictionary
+            guard let source = CGImageSourceCreateWithData(data, options),
+                  let decoded = CGImageSourceCreateImageAtIndex(source, 0, options),
+                  let colorSpace = decoded.colorSpace, let provider = decoded.dataProvider,
+                  let retained = CGImage(
+                    width: decoded.width, height: decoded.height,
+                    bitsPerComponent: decoded.bitsPerComponent, bitsPerPixel: decoded.bitsPerPixel,
+                    bytesPerRow: decoded.bytesPerRow, space: colorSpace,
+                    bitmapInfo: decoded.bitmapInfo, provider: provider, decode: nil,
+                    shouldInterpolate: image.shouldInterpolate, intent: image.renderingIntent
+                  ) else { return nil }
+            return (retained, data.length)
+        }
     }
 }

Sources/AetherEngine/PlayerState.swift

@@ -1619,7 +1619,9 @@ extension SubtitleCue: Equatable {
     }
 }
 
-/// Decoded PGS / HDMV PGS / DVB / DVD bitmap subtitle. CGImage is fully rendered (RGBA, premultiplied alpha). Position is [0, 1] against the source video frame; multiply by the on-screen video rect to place it.
+/// Decoded PGS / HDMV PGS / DVB / DVD bitmap subtitle. CGImage carries the rendered image and
+/// alpha metadata; its provider may defer pixel expansion. Draw it rather than assuming a raw
+/// premultiplied RGBA provider layout. Position is [0, 1] against the source video frame.
 public struct SubtitleImage: @unchecked Sendable {
     public let cgImage: CGImage
     public let position: CGRect
@@ -1632,12 +1634,17 @@ public struct SubtitleImage: @unchecked Sendable {
     /// AV_SUBTITLE_FLAG_FORCED from the decoded rect (#146): the disc authored this object as a
     /// forced caption (sign / foreign-dialogue overlay shown even with subtitles off).
     public let isForced: Bool
+    /// Known compressed provider storage, when the decoder retains a lossless encoded image.
+    /// This excludes ImageIO/CGImage overhead and temporary pixels used while drawing.
+    public let encodedBackingBytes: Int?
 
-    public init(cgImage: CGImage, position: CGRect, canvasSize: CGSize = .zero, isForced: Bool = false) {
+    public init(cgImage: CGImage, position: CGRect, canvasSize: CGSize = .zero, isForced: Bool = false,
+                encodedBackingBytes: Int? = nil) {
         self.cgImage = cgImage
         self.position = position
         self.canvasSize = canvasSize
         self.isForced = isForced
+        self.encodedBackingBytes = encodedBackingBytes
     }
 }
 

Tests/AetherEngineTests/Issue628PGSOffMainDecodeTests.swift

@@ -47,11 +47,15 @@ struct Issue628PGSOffMainDecodeTests {
     }
 
     private static func bytes(of image: CGImage) -> [UInt8] {
-        let data = image.dataProvider!.data! as Data
-        var out: [UInt8] = []
-        for row in 0..<image.height {
-            let start = row * image.bytesPerRow
-            out.append(contentsOf: data[start..<(start + image.width * 4)])
+        // A deferred ImageIO provider can expose straight-alpha pixels. Compare the actual
+        // rendered premultiplied RGBA contract against the original palette algorithm.
+        var out = [UInt8](repeating: 0, count: image.width * image.height * 4)
+        out.withUnsafeMutableBytes { bytes in
+            let context = CGContext(data: bytes.baseAddress, width: image.width, height: image.height,
+                                    bitsPerComponent: 8, bytesPerRow: image.width * 4,
+                                    space: CGColorSpace(name: CGColorSpace.sRGB)!,
+                                    bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)!
+            context.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height))
         }
         return out
     }

Tests/AetherEngineTests/SubtitleBitmapStorageTests.swift

@@ -0,0 +1,76 @@
+import CoreGraphics
+import Foundation
+import Testing
+@testable import AetherEngine
+
+struct SubtitleBitmapStorageTests {
+    @Test("retained subtitle images preserve rendered alpha after temporary storage is released")
+    func losslessPixelsAndProviderLifetime() throws {
+        let width = 256, height = 32
+        var pixels = Data(count: width * height * 4)
+        pixels.withUnsafeMutableBytes { (bytes: UnsafeMutableRawBufferPointer) in
+            for y in 0..<height {
+                for x in 0..<width {
+                    let offset = (y * width + x) * 4
+                    let alpha = x
+                    bytes[offset] = UInt8(((y * 7 % 256) * alpha + 127) / 255)
+                    bytes[offset + 1] = UInt8(((x * 13 % 256) * alpha + 127) / 255)
+                    bytes[offset + 2] = UInt8(((x * 23 % 256) * alpha + 127) / 255)
+                    bytes[offset + 3] = UInt8(alpha)
+                }
+            }
+        }
+        let original = try #require(makeImage(pixels, width: width, height: height))
+        let retained = try #require(autoreleasepool { EmbeddedSubtitleDecoder.retainedBitmap(original) })
+        #expect(retained.encodedByteCount > 0)
+        #expect(retained.image.width == width)
+        #expect(retained.image.height == height)
+        #expect(!retained.image.shouldInterpolate)
+        #expect(render(retained.image) == render(original))
+        // Repeated presentation and backward seek use the same retained provider.
+        #expect(render(retained.image) == render(original))
+    }
+
+    @Test("flat subtitle pixels retain compressed backing rather than a full RGBA plane")
+    func flatSubtitleStorageIsSmaller() throws {
+        let width = 1024, height = 96
+        var pixels = Data(count: width * height * 4)
+        pixels.withUnsafeMutableBytes { (bytes: UnsafeMutableRawBufferPointer) in
+            for y in 8..<88 {
+                for x in 16..<1008 where (x + y) % 12 < 4 {
+                    let offset = (y * width + x) * 4
+                    bytes[offset] = 128; bytes[offset + 1] = 64
+                    bytes[offset + 2] = 32; bytes[offset + 3] = 128
+                }
+            }
+        }
+        let original = try #require(makeImage(pixels, width: width, height: height))
+        let retained = try #require(EmbeddedSubtitleDecoder.retainedBitmap(original))
+        #expect(retained.encodedByteCount < pixels.count)
+        #expect(render(retained.image) == render(original))
+        let image = SubtitleImage(cgImage: retained.image, position: .zero,
+                                  encodedBackingBytes: retained.encodedByteCount)
+        #expect(image.encodedBackingBytes == retained.encodedByteCount)
+    }
+
+    private func makeImage(_ data: Data, width: Int, height: Int) -> CGImage? {
+        guard let provider = CGDataProvider(data: data as CFData),
+              let space = CGColorSpace(name: CGColorSpace.sRGB) else { return nil }
+        return CGImage(width: width, height: height, bitsPerComponent: 8, bitsPerPixel: 32,
+                       bytesPerRow: width * 4, space: space,
+                       bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue),
+                       provider: provider, decode: nil, shouldInterpolate: false, intent: .defaultIntent)
+    }
+
+    private func render(_ image: CGImage) -> Data {
+        var data = Data(count: image.width * image.height * 4)
+        data.withUnsafeMutableBytes { bytes in
+            let context = CGContext(data: bytes.baseAddress, width: image.width, height: image.height,
+                                    bitsPerComponent: 8, bytesPerRow: image.width * 4,
+                                    space: CGColorSpace(name: CGColorSpace.sRGB)!,
+                                    bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)!
+            context.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height))
+        }
+        return data
+    }
+}

docs/api.md

@@ -1450,7 +1450,7 @@ All flags default to safe values; the table is the full set. Depth for the media
 | `SubtitleCue` | `id`, `startTime`, `endTime`, `body` (a `SubtitleCue.Body`: `.text`, `.richText`, `.image`), `placement`, plus `text` and `isForced` conveniences. |
 | `SubtitleTextRun` | `text`, `color`, `isBold`, `isItalic`, `isUnderlined`, `isStruckThrough`, `fontName`, `fontSize`, `isStyled`. |
 | `SubtitleTextPlacement` | `alignment` (numpad), `position` (a [0, 1] anchor). |
-| `SubtitleImage` | `cgImage`, `position`, `canvasSize`, `isForced`. |
+| `SubtitleImage` | `cgImage`, `position`, `canvasSize`, `isForced`, `encodedBackingBytes`. Bitmap images retain lossless PNG backing and expand pixels when drawn. Draw the `CGImage` rather than assuming its provider exposes premultiplied RGBA bytes. `encodedBackingBytes` is the known compressed backing size, or nil for an image without that accounting; it excludes ImageIO/CGImage overhead and temporary drawing pixels, so it is not a process-memory measurement. |
 | `ExternalSubtitleTrack` | `url`, `name`, `language`, `isForced`, `isHearingImpaired`, `isDefault`, `httpHeaders` (nil forwards `LoadOptions.httpHeaders`), `formatHint` for URLs whose path hides the format, and `sourceStreamIndex` for a container holding several subtitle streams. That index addresses the container at `url`, not the played media. |
 | `NativeSubtitleTrack` | `ordinal`, `language`, `displayName`, plus `sameLanguageRank(of:in:)` for disambiguating same-language options (eng Full against eng SDH). |
 | `RecordingState` | `.idle`, `.recording(RecordingProgress)`, `.ended(RecordingEndReason)`, `.failed(RecordingFailure)`. What the session's live recording is doing. |