diff --git a/CHANGELOG.md b/CHANGELOG.md index cadfbc99f..7ab7208e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -112,6 +112,14 @@ the public-API contract. ### Fixed +- **Retained bitmap subtitles use lossless encoded backing instead of keeping every image + expanded to RGBA.** PGS, DVB and DVD subtitle history retains PNG-backed `CGImage` providers + and expands pixels when drawn, preserving the authored crop, placement, alpha, forced flag + and cue lifetime for subtitle offsets and backward seeks. Encoding runs on the existing + subtitle worker. `SubtitleImage.encodedBackingBytes` reports the known compressed backing + size separately from process memory; callers draw the image rather than assuming its provider + has a particular raw pixel layout. + - **Source failures keep their cause and request ownership.** Initial live probes preserve HTTP, URL and TLS failures without changing the live-start `liveSourceUnavailable` kind (HTTP and certificate classifications stay specific). A transport failure at open (a reset or a timeout from diff --git a/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift b/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift index 6cb4293fb..0adcc4e98 100644 --- a/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift +++ b/Sources/AetherEngine/Decoder/EmbeddedSubtitleDecoder.swift @@ -1,5 +1,7 @@ import Foundation import CoreGraphics +import ImageIO +import UniformTypeIdentifiers import Compression import AetherLibavcodec import AetherLibavformat @@ -622,7 +624,8 @@ final class EmbeddedSubtitleDecoder { /// Render a PGS/DVB/HDMV bitmap rect into a CGImage with normalised position. /// Palette from libavcodec is 32-bit with alpha in the high byte and BGR below ([B,G,R,A] on little-endian); - /// rewritten to RGBA for CGImage's premultipliedLast. Internal for unit tests (#146). + /// rewritten to premultiplied RGBA, then retained losslessly with deferred pixel decoding. + /// Internal for unit tests (#146). static func imageForSubtitleRect( _ rect: UnsafeMutablePointer, videoWidth: Int, @@ -724,8 +727,40 @@ final class EmbeddedSubtitleDecoder { position = CGRect(x: 0.1, y: 0.8, width: 0.8, height: 0.15) } - return SubtitleImage(cgImage: cgImage, position: position, + guard let retained = retainedBitmap(cgImage) else { + EngineLog.emit("[subtitle] lossless bitmap storage failed size=\(cropW)x\(cropH)") + return nil + } + return SubtitleImage(cgImage: retained.image, position: position, canvasSize: CGSize(width: Int(videoWidth), height: Int(videoHeight)), - isForced: (r.flags & AV_SUBTITLE_FLAG_FORCED) != 0) + isForced: (r.flags & AV_SUBTITLE_FLAG_FORCED) != 0, + encodedBackingBytes: retained.encodedByteCount) + } + + /// The subtitle history needs authored images for offsets and backward seeks, but does not + /// need every image expanded to RGBA at once. Retain PNG bytes through ImageIO's provider; + /// drawing decodes the same pixels without caching them for the whole cue lifetime. This + /// runs on the existing subtitle decode worker, not on the presentation actor. + static func retainedBitmap(_ image: CGImage) -> (image: CGImage, encodedByteCount: Int)? { + autoreleasepool { + let data = NSMutableData() + guard let destination = CGImageDestinationCreateWithData( + data, UTType.png.identifier as CFString, 1, nil + ) else { return nil } + CGImageDestinationAddImage(destination, image, nil) + guard CGImageDestinationFinalize(destination) else { return nil } + let options = [kCGImageSourceShouldCache: false] as CFDictionary + guard let source = CGImageSourceCreateWithData(data, options), + let decoded = CGImageSourceCreateImageAtIndex(source, 0, options), + let colorSpace = decoded.colorSpace, let provider = decoded.dataProvider, + let retained = CGImage( + width: decoded.width, height: decoded.height, + bitsPerComponent: decoded.bitsPerComponent, bitsPerPixel: decoded.bitsPerPixel, + bytesPerRow: decoded.bytesPerRow, space: colorSpace, + bitmapInfo: decoded.bitmapInfo, provider: provider, decode: nil, + shouldInterpolate: image.shouldInterpolate, intent: image.renderingIntent + ) else { return nil } + return (retained, data.length) + } } } diff --git a/Sources/AetherEngine/PlayerState.swift b/Sources/AetherEngine/PlayerState.swift index 03f70f361..c6a3b2744 100644 --- a/Sources/AetherEngine/PlayerState.swift +++ b/Sources/AetherEngine/PlayerState.swift @@ -1619,7 +1619,9 @@ extension SubtitleCue: Equatable { } } -/// Decoded PGS / HDMV PGS / DVB / DVD bitmap subtitle. CGImage is fully rendered (RGBA, premultiplied alpha). Position is [0, 1] against the source video frame; multiply by the on-screen video rect to place it. +/// Decoded PGS / HDMV PGS / DVB / DVD bitmap subtitle. CGImage carries the rendered image and +/// alpha metadata; its provider may defer pixel expansion. Draw it rather than assuming a raw +/// premultiplied RGBA provider layout. Position is [0, 1] against the source video frame. public struct SubtitleImage: @unchecked Sendable { public let cgImage: CGImage public let position: CGRect @@ -1632,12 +1634,17 @@ public struct SubtitleImage: @unchecked Sendable { /// AV_SUBTITLE_FLAG_FORCED from the decoded rect (#146): the disc authored this object as a /// forced caption (sign / foreign-dialogue overlay shown even with subtitles off). public let isForced: Bool + /// Known compressed provider storage, when the decoder retains a lossless encoded image. + /// This excludes ImageIO/CGImage overhead and temporary pixels used while drawing. + public let encodedBackingBytes: Int? - public init(cgImage: CGImage, position: CGRect, canvasSize: CGSize = .zero, isForced: Bool = false) { + public init(cgImage: CGImage, position: CGRect, canvasSize: CGSize = .zero, isForced: Bool = false, + encodedBackingBytes: Int? = nil) { self.cgImage = cgImage self.position = position self.canvasSize = canvasSize self.isForced = isForced + self.encodedBackingBytes = encodedBackingBytes } } diff --git a/Tests/AetherEngineTests/Issue628PGSOffMainDecodeTests.swift b/Tests/AetherEngineTests/Issue628PGSOffMainDecodeTests.swift index f167fea5c..f59e3db1f 100644 --- a/Tests/AetherEngineTests/Issue628PGSOffMainDecodeTests.swift +++ b/Tests/AetherEngineTests/Issue628PGSOffMainDecodeTests.swift @@ -47,11 +47,15 @@ struct Issue628PGSOffMainDecodeTests { } private static func bytes(of image: CGImage) -> [UInt8] { - let data = image.dataProvider!.data! as Data - var out: [UInt8] = [] - for row in 0.. 0) + #expect(retained.image.width == width) + #expect(retained.image.height == height) + #expect(!retained.image.shouldInterpolate) + #expect(render(retained.image) == render(original)) + // Repeated presentation and backward seek use the same retained provider. + #expect(render(retained.image) == render(original)) + } + + @Test("flat subtitle pixels retain compressed backing rather than a full RGBA plane") + func flatSubtitleStorageIsSmaller() throws { + let width = 1024, height = 96 + var pixels = Data(count: width * height * 4) + pixels.withUnsafeMutableBytes { (bytes: UnsafeMutableRawBufferPointer) in + for y in 8..<88 { + for x in 16..<1008 where (x + y) % 12 < 4 { + let offset = (y * width + x) * 4 + bytes[offset] = 128; bytes[offset + 1] = 64 + bytes[offset + 2] = 32; bytes[offset + 3] = 128 + } + } + } + let original = try #require(makeImage(pixels, width: width, height: height)) + let retained = try #require(EmbeddedSubtitleDecoder.retainedBitmap(original)) + #expect(retained.encodedByteCount < pixels.count) + #expect(render(retained.image) == render(original)) + let image = SubtitleImage(cgImage: retained.image, position: .zero, + encodedBackingBytes: retained.encodedByteCount) + #expect(image.encodedBackingBytes == retained.encodedByteCount) + } + + private func makeImage(_ data: Data, width: Int, height: Int) -> CGImage? { + guard let provider = CGDataProvider(data: data as CFData), + let space = CGColorSpace(name: CGColorSpace.sRGB) else { return nil } + return CGImage(width: width, height: height, bitsPerComponent: 8, bitsPerPixel: 32, + bytesPerRow: width * 4, space: space, + bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue), + provider: provider, decode: nil, shouldInterpolate: false, intent: .defaultIntent) + } + + private func render(_ image: CGImage) -> Data { + var data = Data(count: image.width * image.height * 4) + data.withUnsafeMutableBytes { bytes in + let context = CGContext(data: bytes.baseAddress, width: image.width, height: image.height, + bitsPerComponent: 8, bytesPerRow: image.width * 4, + space: CGColorSpace(name: CGColorSpace.sRGB)!, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)! + context.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height)) + } + return data + } +} diff --git a/docs/api.md b/docs/api.md index 2a2a4da77..771a3d8f8 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1450,7 +1450,7 @@ All flags default to safe values; the table is the full set. Depth for the media | `SubtitleCue` | `id`, `startTime`, `endTime`, `body` (a `SubtitleCue.Body`: `.text`, `.richText`, `.image`), `placement`, plus `text` and `isForced` conveniences. | | `SubtitleTextRun` | `text`, `color`, `isBold`, `isItalic`, `isUnderlined`, `isStruckThrough`, `fontName`, `fontSize`, `isStyled`. | | `SubtitleTextPlacement` | `alignment` (numpad), `position` (a [0, 1] anchor). | -| `SubtitleImage` | `cgImage`, `position`, `canvasSize`, `isForced`. | +| `SubtitleImage` | `cgImage`, `position`, `canvasSize`, `isForced`, `encodedBackingBytes`. Bitmap images retain lossless PNG backing and expand pixels when drawn. Draw the `CGImage` rather than assuming its provider exposes premultiplied RGBA bytes. `encodedBackingBytes` is the known compressed backing size, or nil for an image without that accounting; it excludes ImageIO/CGImage overhead and temporary drawing pixels, so it is not a process-memory measurement. | | `ExternalSubtitleTrack` | `url`, `name`, `language`, `isForced`, `isHearingImpaired`, `isDefault`, `httpHeaders` (nil forwards `LoadOptions.httpHeaders`), `formatHint` for URLs whose path hides the format, and `sourceStreamIndex` for a container holding several subtitle streams. That index addresses the container at `url`, not the played media. | | `NativeSubtitleTrack` | `ordinal`, `language`, `displayName`, plus `sameLanguageRank(of:in:)` for disambiguating same-language options (eng Full against eng SDH). | | `RecordingState` | `.idle`, `.recording(RecordingProgress)`, `.ended(RecordingEndReason)`, `.failed(RecordingFailure)`. What the session's live recording is doing. |