From 815a51ae53c74c561c7b578c9573203563f639e5 Mon Sep 17 00:00:00 2001 From: "Sam.Richards@taurich.org" Date: Wed, 28 Oct 2026 13:41:23 +0000 Subject: [PATCH] media_reader/ffmpeg: decode H.264, HEVC and ProRes with VideoToolbox on macOS Adds a VideoToolbox decode backend inside FFMpegStream, so on macOS H.264, HEVC and ProRes video go through Apple's licensed (and mostly hardware) decoders while libavformat still handles demuxing, seeking, timecode and audio. AAC uses FFmpeg's AudioToolbox decoder (aac_at) where available. - New preference /plugin/media_reader/FFMPEG/videotoolbox: off / prefer (default) / only. 'prefer' falls back to FFmpeg's decoder if a session can't be created; 'only' reports an error on the frame instead. - VtDecoder follows avcodec's send/receive/flush contract, so FFMpegDecoder, thumbnails and caching are unchanged. Frames are reordered into presentation order using the highest DTS seen. - Output stays native YCbCr. VideoToolbox's biplanar formats are repacked by small vectorised loops (vt_repack) into planar formats the existing shader reads: 8/10 bit 4:2:0/4:2:2/4:4:4, 12 bit 4:2:2/4:4:4, and ProRes 4444 with alpha ('s4as'). 12 bit 4:2:0 HEVC stays on FFmpeg: VideoToolbox's chroma for it is inaccurate. - A video stream VideoToolbox decodes no longer needs an FFmpeg decoder to exist, ready for builds without the patent-encumbered decoders. - Changing the preference at runtime drops frames decoded under the old mode. Measured on an M2 Max: H.264/HEVC output bit-exact against FFmpeg, ProRes within 4 code values; 5-10x less CPU for 4K playback, which now plays smoothly where software decode stutters. Also gives gtest targets the __apple__ define the headers rely on to undo MacTypes.h's 'nil' macro; ffmpeg_test didn't compile on macOS without it. Tests: ffmpeg_test covers repack loops, decode order, format/colour and random-access parity with FFmpeg's decoder, fallback, thumbnails, AAC timing and preference changes, using media from test/videotoolbox_media/generate.sh (skipped when not generated). Co-Authored-By: Claude Opus 5.5 --- cmake/macros.cmake | 4 + .../plugin_media_reader_ffmpeg.json | 11 + .../media_reader/ffmpeg/src/CMakeLists.txt | 14 + src/plugin/media_reader/ffmpeg/src/ffmpeg.cpp | 39 +- src/plugin/media_reader/ffmpeg/src/ffmpeg.hpp | 2 + .../ffmpeg/src/ffmpeg_decoder.cpp | 9 +- .../ffmpeg/src/ffmpeg_decoder.hpp | 5 +- .../media_reader/ffmpeg/src/ffmpeg_stream.cpp | 316 +++++- .../media_reader/ffmpeg/src/ffmpeg_stream.hpp | 55 +- .../media_reader/ffmpeg/src/vt_decoder.cpp | 810 ++++++++++++++ .../media_reader/ffmpeg/src/vt_decoder.hpp | 71 ++ .../media_reader/ffmpeg/src/vt_repack.cpp | 178 ++++ .../media_reader/ffmpeg/src/vt_repack.hpp | 113 ++ .../media_reader/ffmpeg/test/CMakeLists.txt | 6 + .../media_reader/ffmpeg/test/ffmpeg_test.cpp | 993 ++++++++++++++++++ .../ffmpeg/test/videotoolbox_media/README.md | 56 + .../test/videotoolbox_media/generate.sh | 138 +++ 17 files changed, 2770 insertions(+), 50 deletions(-) create mode 100644 src/plugin/media_reader/ffmpeg/src/vt_decoder.cpp create mode 100644 src/plugin/media_reader/ffmpeg/src/vt_decoder.hpp create mode 100644 src/plugin/media_reader/ffmpeg/src/vt_repack.cpp create mode 100644 src/plugin/media_reader/ffmpeg/src/vt_repack.hpp create mode 100644 src/plugin/media_reader/ffmpeg/test/videotoolbox_media/README.md create mode 100755 src/plugin/media_reader/ffmpeg/test/videotoolbox_media/generate.sh diff --git a/cmake/macros.cmake b/cmake/macros.cmake index d9a059fc8..65e317b10 100644 --- a/cmake/macros.cmake +++ b/cmake/macros.cmake @@ -77,6 +77,10 @@ if (BUILD_TESTING) PRIVATE TEST_RESOURCE=\"${TEST_RESOURCE}\" PRIVATE ROOT_DIR=\"${ROOT_DIR}\" PRIVATE $<$:XSTUDIO_DEBUG=1> + # Same platform defines as default_compile_options: the headers rely on + # __apple__ to #undef the 'nil' macro that MacTypes.h defines. + $<$:__apple__> + $<$:__OPENGL_4_1__> ) endmacro() endif (BUILD_TESTING) diff --git a/share/preference/plugin_media_reader_ffmpeg.json b/share/preference/plugin_media_reader_ffmpeg.json index 0281ac183..de33dfe1f 100644 --- a/share/preference/plugin_media_reader_ffmpeg.json +++ b/share/preference/plugin_media_reader_ffmpeg.json @@ -12,6 +12,17 @@ "datatype": "int", "context": ["APPLICATION"] }, + "videotoolbox": { + "path": "/plugin/media_reader/FFMPEG/videotoolbox", + "default_value": "prefer", + "description": "macOS only. Decode H.264, HEVC and ProRes video with Apple's VideoToolbox. 'prefer' falls back to FFmpeg's own decoder when VideoToolbox can't decode a stream; 'only' reports an error instead; 'off' always uses FFmpeg's decoder. Applies to media loaded after the change.", + "value": "prefer", + "datatype": "multichoice string", + "options": ["off", "prefer", "only"], + "context": ["APPLICATION"], + "category": "Playback", + "display_name": "VideoToolbox Decoding (macOS)" + }, "supported": { "path": "/plugin/media_reader/FFMPEG/supported", "description": "Control plugin format support", diff --git a/src/plugin/media_reader/ffmpeg/src/CMakeLists.txt b/src/plugin/media_reader/ffmpeg/src/CMakeLists.txt index 76a228264..add976ade 100644 --- a/src/plugin/media_reader/ffmpeg/src/CMakeLists.txt +++ b/src/plugin/media_reader/ffmpeg/src/CMakeLists.txt @@ -4,6 +4,7 @@ set(SOURCES ffmpeg_stream.cpp ffmpeg_decoder.cpp ffmpeg.cpp + vt_repack.cpp ) add_library(${PROJECT_NAME} SHARED ${SOURCES}) @@ -46,4 +47,17 @@ else() endif() +if(APPLE) + # VideoToolbox decode backend (see vt_decoder.hpp) + target_sources(${PROJECT_NAME} PRIVATE vt_decoder.cpp) + target_compile_definitions(${PROJECT_NAME} PRIVATE XSTUDIO_HAS_VIDEOTOOLBOX=1) + target_link_libraries(${PROJECT_NAME} + PRIVATE + "-framework VideoToolbox" + "-framework CoreMedia" + "-framework CoreVideo" + "-framework CoreFoundation" + ) +endif() + set_target_properties(${PROJECT_NAME} PROPERTIES LINK_DEPENDS_NO_SHARED true) diff --git a/src/plugin/media_reader/ffmpeg/src/ffmpeg.cpp b/src/plugin/media_reader/ffmpeg/src/ffmpeg.cpp index 0b00cfb30..37ad6e7c6 100644 --- a/src/plugin/media_reader/ffmpeg/src/ffmpeg.cpp +++ b/src/plugin/media_reader/ffmpeg/src/ffmpeg.cpp @@ -384,6 +384,7 @@ plugin_manager::PluginFactoryCollection *plugin_factory_collection_ptr() { FFMpegMediaReader::FFMpegMediaReader(const utility::JsonStore &prefs) : MediaReader("FFMPEG", prefs) { readers_per_source_ = 1; + videotoolbox_mode_ = VideoToolboxMode::Prefer; update_preferences(prefs); } @@ -398,6 +399,21 @@ void FFMpegMediaReader::update_preferences(const utility::JsonStore &prefs) { spdlog::warn("{} {}", __PRETTY_FUNCTION__, e.what()); } + try { + const auto mode = videotoolbox_mode_from_string( + preference_value(prefs, "/plugin/media_reader/FFMPEG/videotoolbox")); + if (mode != videotoolbox_mode_) { + // Don't hand out a frame decoded under the previous mode, and decode + // with the new one from the next request on. + last_decoded_image_.reset(); + decoder.reset(); + thumbnail_decoder.reset(); + } + videotoolbox_mode_ = mode; + } catch (const std::exception &e) { + spdlog::warn("{} {}", __PRETTY_FUNCTION__, e.what()); + } + try { readers_per_source_ = @@ -439,9 +455,14 @@ ImageBufPtr FFMpegMediaReader::image(const media::AVFrameID &mptr) { return last_decoded_image_; } - if (!decoder || decoder->path() != path || decoder->stream_id() != mptr.stream_id()) { + if (!decoder || decoder->path() != path || decoder->stream_id() != mptr.stream_id() || + decoder->videotoolbox_mode() != videotoolbox_mode_) { decoder.reset(new FFMpegDecoder( - path, soundcard_sample_rate_, default_rate_, mptr.stream_id())); + path, + soundcard_sample_rate_, + default_rate_, + mptr.stream_id(), + videotoolbox_mode_)); } decoder->decode_video_frame(mptr.frame(), rt); @@ -455,6 +476,11 @@ ImageBufPtr FFMpegMediaReader::image(const media::AVFrameID &mptr) { last_decoded_image_ = rt; } + } catch (const videotoolbox_unavailable_error &e) { + // Always shown on the frame: the user needs to know why there's no picture + // (e.g. the 'only' preference, or a build without FFmpeg's decoder). + rt = make_blank_image(); + rt->set_error(e.what()); } catch (std::exception &e) { rt = make_blank_image(); if (mptr.error() != "") { @@ -513,7 +539,8 @@ AudioBufPtr FFMpegMediaReader::audio(const media::AVFrameID &mptr) { xstudio::media::MediaDetail FFMpegMediaReader::detail(const caf::uri &uri) const { - FFMpegDecoder t_decoder(uri_convert(uri), soundcard_sample_rate_, default_rate_); + FFMpegDecoder t_decoder( + uri_convert(uri), soundcard_sample_rate_, default_rate_, "", videotoolbox_mode_); // N.B. MediaDetail needs frame duration, so invert frame rate std::vector streams; @@ -626,7 +653,11 @@ FFMpegMediaReader::thumbnail(const media::AVFrameID &mptr, const size_t thumb_si // DebugTimer d(path, mptr.frame()); if (!thumbnail_decoder || thumbnail_decoder->path() != path) { thumbnail_decoder.reset(new FFMpegDecoder( - path, soundcard_sample_rate_, default_rate_, mptr.stream_id())); + path, + soundcard_sample_rate_, + default_rate_, + mptr.stream_id(), + videotoolbox_mode_)); } std::shared_ptr rt = diff --git a/src/plugin/media_reader/ffmpeg/src/ffmpeg.hpp b/src/plugin/media_reader/ffmpeg/src/ffmpeg.hpp index cdd766458..529853e7e 100644 --- a/src/plugin/media_reader/ffmpeg/src/ffmpeg.hpp +++ b/src/plugin/media_reader/ffmpeg/src/ffmpeg.hpp @@ -10,6 +10,7 @@ namespace xstudio::media_reader { namespace ffmpeg { class FFMpegDecoder; + enum class VideoToolboxMode; } class FFMpegMediaReader : public MediaReader { @@ -53,6 +54,7 @@ class FFMpegMediaReader : public MediaReader { std::shared_ptr thumbnail_decoder; int readers_per_source_; + ffmpeg::VideoToolboxMode videotoolbox_mode_; int soundcard_sample_rate_ = {48000}; int channels_ = 2; utility::FrameRate default_rate_ = {utility::FrameRate(timebase::k_flicks_24fps)}; diff --git a/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.cpp b/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.cpp index 610dc03a8..a7851f901 100644 --- a/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.cpp +++ b/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.cpp @@ -30,7 +30,8 @@ FFMpegDecoder::FFMpegDecoder( std::string path, const int soundcard_sample_rate, const utility::FrameRate default_rate, - std::string stream_id) + std::string stream_id, + VideoToolboxMode videotoolbox_mode) : movie_file_path_(std::move(path)), last_requested_frame_(-100), last_decoded_frame_(-100), @@ -38,7 +39,8 @@ FFMpegDecoder::FFMpegDecoder( av_format_ctx_(nullptr), soundcard_sample_rate_(soundcard_sample_rate), stream_id_(std::move(stream_id)), - default_rate_(default_rate) + default_rate_(default_rate), + videotoolbox_mode_(videotoolbox_mode) { open_handles(); @@ -68,7 +70,8 @@ void FFMpegDecoder::open_handles() { av_format_ctx_->streams[i], i, ffmpeg_threads, - movie_file_path_); + movie_file_path_, + videotoolbox_mode_); } catch ([[maybe_unused]] std::exception &e) { } diff --git a/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.hpp b/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.hpp index 3d094e1a8..ab9cff376 100644 --- a/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.hpp +++ b/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.hpp @@ -13,7 +13,8 @@ class FFMpegDecoder { std::string path, const int soundcard_sample_rate, const utility::FrameRate default_rate, - std::string stream_id = ""); + std::string stream_id = "", + VideoToolboxMode videotoolbox_mode = VideoToolboxMode::Off); ~FFMpegDecoder(); @@ -32,6 +33,7 @@ class FFMpegDecoder { const std::string &path() const { return movie_file_path_; } const std::string &stream_id() const { return stream_id_; } + VideoToolboxMode videotoolbox_mode() const { return videotoolbox_mode_; } int64_t duration_frames() const { return duration_frames_; } utility::FrameRate frame_rate(unsigned int stream_idx = UINT_MAX) const; utility::Timecode first_frame_timecode(); @@ -102,5 +104,6 @@ class FFMpegDecoder { int64_t duration_frames_; const std::string stream_id_; const utility::FrameRate default_rate_; + const VideoToolboxMode videotoolbox_mode_; }; } // namespace xstudio::media_reader::ffmpeg diff --git a/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.cpp b/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.cpp index 22b7892ec..b45f50ca6 100644 --- a/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.cpp +++ b/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.cpp @@ -6,6 +6,12 @@ #include "ffmpeg_stream.hpp" #include "xstudio/media/media_error.hpp" +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX +#include "vt_decoder.hpp" +#endif + +#include +#include #ifdef __GNUC__ // Check if GCC compiler is being used #pragma GCC diagnostic ignored "-Wdeprecated-declarations" @@ -259,6 +265,37 @@ static int setup_video_buffer(AVCodecContext *ctx, AVFrame *ffmpeg_frame_, int / namespace xstudio::media_reader::ffmpeg { +namespace { + // Each FFMpegDecoder (playback, thumbnails, media detail) opens its own streams, + // so to log a backend choice once per source rather than once per decoder, + // remember what has been logged. + bool first_time_for(const std::string &key) { + static std::mutex m; + static std::set seen; + std::lock_guard l(m); + return seen.insert(key).second; + } +} // namespace + +VideoToolboxMode videotoolbox_mode_from_string(const std::string &value) { + if (value == "off") + return VideoToolboxMode::Off; + if (value == "only") + return VideoToolboxMode::Only; + return VideoToolboxMode::Prefer; +} + +const char *to_string(const VideoToolboxMode mode) { + switch (mode) { + case VideoToolboxMode::Off: + return "off"; + case VideoToolboxMode::Only: + return "only"; + default: + return "prefer"; + } +} + // The SWS_CS_* coefficient set matching a stream's colourspace. Only needed where // sws does the YUV to RGB conversion itself - everywhere else our own shader does // it, from the matrix built in set_shader_pix_format_info. @@ -564,13 +601,13 @@ ImageBufPtr FFMpegStream::get_ffmpeg_frame_as_xstudio_image() { // See http://stash/projects/RND/repos/showsetup_data/pull-requests/794/overview const bool is_dneg_mov = utility::ends_with(format_context_->url, ".dneg.mov"); AVColorRange color_range = ffmpeg_frame_->color_range; - if (is_dneg_mov && codec_->id == AV_CODEC_ID_DNXHD) { + if (is_dneg_mov && avc_stream_->codecpar->codec_id == AV_CODEC_ID_DNXHD) { color_range = AVCOL_RANGE_MPEG; } set_shader_pix_format_info( jsn, - codec_->id, + avc_stream_->codecpar->codec_id, (AVPixelFormat)ffmpeg_pixel_format, color_range, ffmpeg_frame_->colorspace); @@ -771,15 +808,39 @@ AudioBufPtr FFMpegStream::get_ffmpeg_frame_as_xstudio_audio() { } FFMpegStream::FFMpegStream( - AVFormatContext *fmt_ctx, AVStream *stream, int index, int thread_count, std::string path) + AVFormatContext *fmt_ctx, + AVStream *stream, + int index, + int thread_count, + std::string path, + VideoToolboxMode videotoolbox_mode) : stream_index_(index), format_context_(fmt_ctx), avc_stream_(stream), - source_path(std::move(path)) { + source_path(std::move(path)), + videotoolbox_mode_(videotoolbox_mode) { codec_type_ = avc_stream_->codecpar->codec_type; ffmpeg_frame_ = av_frame_alloc(); - codec_ = avcodec_find_decoder(avc_stream_->codecpar->codec_id); + thread_count_ = thread_count; + + // VideoToolbox gets first refusal on video streams (design D7). FFmpeg's decoder + // is only looked for if VideoToolbox doesn't take the stream, so a build without + // one (e.g. no h264 decoder) still opens streams VideoToolbox decodes. + if (codec_type_ == AVMEDIA_TYPE_VIDEO) + try_videotoolbox(); + + if (!vt_decoder_ && videotoolbox_error_.empty()) + codec_ = avcodec_find_decoder(avc_stream_->codecpar->codec_id); + +#ifdef __APPLE__ + // AAC through AudioToolbox when FFmpeg was built with it (design D10) + if (codec_type_ == AVMEDIA_TYPE_AUDIO && + avc_stream_->codecpar->codec_id == AV_CODEC_ID_AAC) { + if (const AVCodec *aac_at = avcodec_find_decoder_by_name("aac_at")) + codec_ = aac_at; + } +#endif codec_context_ = avcodec_alloc_context3(codec_); @@ -808,28 +869,38 @@ FFMpegStream::FFMpegStream( #endif } - } else if (codec_type_ == AVMEDIA_TYPE_VIDEO && codec_) { + } else if ( + codec_type_ == AVMEDIA_TYPE_VIDEO && + (codec_ || vt_decoder_ || !videotoolbox_error_.empty())) { stream_type_ = VIDEO_STREAM; - if (codec_->capabilities & AV_CODEC_CAP_SLICE_THREADS) { - codec_context_->thread_count = thread_count; - codec_context_->thread_type = FF_THREAD_SLICE; - } - if (codec_->capabilities & AV_CODEC_CAP_FRAME_THREADS) { - codec_context_->thread_count = thread_count; - codec_context_->thread_type |= FF_THREAD_FRAME; - } - - /** initialize the stream parameters with demuxer information */ - AVC_CHECK_THROW( - avcodec_parameters_to_context(codec_context_, avc_stream_->codecpar), - "avcodec_parameters_to_context"); - AVC_CHECK_THROW(avcodec_open2(codec_context_, codec_, nullptr), "avcodec_open2"); + spdlog::debug( + "{} video stream {}: videotoolbox mode {}", + source_path, + stream_index_, + to_string(videotoolbox_mode_)); - ffmpeg_frame_->width = avc_stream_->codecpar->width; - ffmpeg_frame_->height = avc_stream_->codecpar->height; - ffmpeg_frame_->format = codec_context_->pix_fmt; + if (codec_) { + is_mjpeg = open_software_video_decoder(); + } else { + // VideoToolbox decodes the stream (or should have, and frame requests + // will report why it can't). The context only carries the stream + // parameters and the pixel format frames come out in. + AVC_CHECK_THROW( + avcodec_parameters_to_context(codec_context_, avc_stream_->codecpar), + "avcodec_parameters_to_context"); +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX + if (vt_decoder_) { + codec_context_->pix_fmt = vt_decoder_->output_pix_fmt(); + // frames arrive in an ImageBuffer, as with setup_video_buffer + using_own_frame_allocation = true; + } +#endif + ffmpeg_frame_->width = avc_stream_->codecpar->width; + ffmpeg_frame_->height = avc_stream_->codecpar->height; + ffmpeg_frame_->format = codec_context_->pix_fmt; + } // store resolution and pixel aspect resolution_ = Imath::V2f(avc_stream_->codecpar->width, avc_stream_->codecpar->height); @@ -838,21 +909,6 @@ FFMpegStream::FFMpegStream( pixel_aspect_ = float(sar.num) / float(sar.den); } - // Note - gif decoder has AV_CODEC_CAP_DR1 flag, but it doesn't appear to work, i.e. - // if xstudio takes over ffmpeg_frame_ buffer mem allocation decoding doesn't work - // correctly and we get repeated frames in the stream decode - if (strcmp(codec_->name, "gif") && (codec_->capabilities & AV_CODEC_CAP_DR1)) { - - // See Note 1 below - // codec_ allows us to allocated AVFrame buffers - codec_context_->get_buffer2 = setup_video_buffer; - using_own_frame_allocation = true; - if (!strcmp(codec_->name, "mjpeg")) { - is_mjpeg = true; - } - } - codec_context_->opaque = this; - } else if (codec_type_ == AVMEDIA_TYPE_AUDIO && codec_) { stream_type_ = AUDIO_STREAM; @@ -907,6 +963,130 @@ FFMpegStream::FFMpegStream( } } +bool FFMpegStream::open_software_video_decoder() { + + bool is_mjpeg = false; + + if (codec_->capabilities & AV_CODEC_CAP_SLICE_THREADS) { + codec_context_->thread_count = thread_count_; + codec_context_->thread_type = FF_THREAD_SLICE; + } + if (codec_->capabilities & AV_CODEC_CAP_FRAME_THREADS) { + codec_context_->thread_count = thread_count_; + codec_context_->thread_type |= FF_THREAD_FRAME; + } + + /** initialize the stream parameters with demuxer information */ + AVC_CHECK_THROW( + avcodec_parameters_to_context(codec_context_, avc_stream_->codecpar), + "avcodec_parameters_to_context"); + AVC_CHECK_THROW(avcodec_open2(codec_context_, codec_, nullptr), "avcodec_open2"); + + ffmpeg_frame_->width = avc_stream_->codecpar->width; + ffmpeg_frame_->height = avc_stream_->codecpar->height; + ffmpeg_frame_->format = codec_context_->pix_fmt; + + // Note - gif decoder has AV_CODEC_CAP_DR1 flag, but it doesn't appear to work, i.e. + // if xstudio takes over ffmpeg_frame_ buffer mem allocation decoding doesn't work + // correctly and we get repeated frames in the stream decode + if (strcmp(codec_->name, "gif") && (codec_->capabilities & AV_CODEC_CAP_DR1)) { + + // See Note 1 below + // codec_ allows us to allocated AVFrame buffers + codec_context_->get_buffer2 = setup_video_buffer; + using_own_frame_allocation = true; + if (!strcmp(codec_->name, "mjpeg")) { + is_mjpeg = true; + } + } else { + using_own_frame_allocation = false; + } + codec_context_->opaque = this; + + return is_mjpeg; +} + +void FFMpegStream::try_videotoolbox() { +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX + const char *codec_name = avcodec_get_name(avc_stream_->codecpar->codec_id); + + if (videotoolbox_mode_ == VideoToolboxMode::Off) { + if (first_time_for(source_path + "|ffmpeg|off")) + spdlog::info( + "{}: {} video decoded by FFmpeg (VideoToolbox is off)", + source_path, + codec_name); + return; + } + + std::string reason; + if (!VtDecoder::eligible(avc_stream_, reason)) { + if (first_time_for(source_path + "|ffmpeg")) + spdlog::info( + "{}: {} video decoded by FFmpeg ({})", source_path, codec_name, reason); + return; + } + + std::unique_ptr vt = VtDecoder::create(avc_stream_, reason); + if (vt) { + if (first_time_for(source_path + "|videotoolbox")) + spdlog::info( + "{}: {} video decoded by VideoToolbox ({}, {})", + source_path, + codec_name, + vt->output_format_name(), + vt->using_hardware() ? "hardware" : "software"); + vt_decoder_ = std::move(vt); + return; + } + + // VideoToolbox should have taken the stream but couldn't + const bool can_fall_back = videotoolbox_mode_ == VideoToolboxMode::Prefer && + avcodec_find_decoder(avc_stream_->codecpar->codec_id); + if (can_fall_back) { + if (first_time_for(source_path + "|fallback")) + spdlog::warn( + "{}: VideoToolbox can't decode this {} video ({}); using FFmpeg's decoder", + source_path, + codec_name, + reason); + } else { + videotoolbox_error_ = + fmt::format("VideoToolbox decoding unavailable for {}: {}", codec_name, reason); + if (first_time_for(source_path + "|failed")) + spdlog::warn("{}: {}", source_path, videotoolbox_error_); + } +#endif +} + +void FFMpegStream::fall_back_to_software(const std::string &reason) { + + vt_decoder_.reset(); + const char *codec_name = avcodec_get_name(avc_stream_->codecpar->codec_id); + const AVCodec *codec = videotoolbox_mode_ == VideoToolboxMode::Prefer + ? avcodec_find_decoder(avc_stream_->codecpar->codec_id) + : nullptr; + if (!codec) { + videotoolbox_error_ = + fmt::format("VideoToolbox decoding unavailable for {}: {}", codec_name, reason); + if (first_time_for(source_path + "|failed")) + spdlog::warn("{}: {}", source_path, videotoolbox_error_); + throw videotoolbox_unavailable_error(videotoolbox_error_); + } + + if (first_time_for(source_path + "|fallback")) + spdlog::warn( + "{}: VideoToolbox can't decode this {} video ({}); using FFmpeg's decoder", + source_path, + codec_name, + reason); + + codec_ = codec; + avcodec_free_context(&codec_context_); + codec_context_ = avcodec_alloc_context3(codec_); + open_software_video_decoder(); +} + FFMpegStream::~FFMpegStream() { if (audio_resampler_ctx_) @@ -1150,7 +1330,19 @@ int64_t FFMpegStream::receive_frame() { av_frame_unref(ffmpeg_frame_); - int rt = avcodec_receive_frame(codec_context_, ffmpeg_frame_); + if (!videotoolbox_error_.empty()) + throw videotoolbox_unavailable_error(videotoolbox_error_); + + int rt = 0; +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX + if (vt_decoder_) { + rt = vt_decoder_->receive_frame(ffmpeg_frame_); + // ProRes 4444 alpha is only known once the first frame header is seen + if (!rt) + codec_context_->pix_fmt = AVPixelFormat(ffmpeg_frame_->format); + } else +#endif + rt = avcodec_receive_frame(codec_context_, ffmpeg_frame_); // std::cerr << source_path << " WAS " << ffmpeg_frame_->pts << "\n"; @@ -1172,15 +1364,57 @@ int64_t FFMpegStream::receive_frame() { return rt; } -void FFMpegStream::send_flush_packet() { avcodec_send_packet(codec_context_, nullptr); } +void FFMpegStream::send_flush_packet() { +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX + if (vt_decoder_) { + vt_decoder_->send_packet(nullptr); + return; + } +#endif + if (videotoolbox_error_.empty()) + avcodec_send_packet(codec_context_, nullptr); +} int FFMpegStream::send_packet(AVPacket *avc_packet_) { + + if (!videotoolbox_error_.empty()) { + av_packet_unref(avc_packet_); + throw videotoolbox_unavailable_error(videotoolbox_error_); + } + +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX + if (vt_decoder_) { + int rt = vt_decoder_->send_packet(avc_packet_); + if (rt == AVERROR(ENOTSUP)) { + // e.g. packets without timestamps (design D8) + try { + fall_back_to_software("stream packets have no timestamps"); + } catch (...) { + av_packet_unref(avc_packet_); + throw; + } + } else { + av_packet_unref(avc_packet_); + return rt; + } + } +#endif + int rt = avcodec_send_packet(codec_context_, avc_packet_); av_packet_unref(avc_packet_); return rt; } -void FFMpegStream::flush_buffers() { avcodec_flush_buffers(codec_context_); } +void FFMpegStream::flush_buffers() { +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX + if (vt_decoder_) { + vt_decoder_->flush(); + return; + } +#endif + if (videotoolbox_error_.empty()) + avcodec_flush_buffers(codec_context_); +} double FFMpegStream::duration_seconds() const { diff --git a/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.hpp b/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.hpp index 4552476bc..a964b8255 100644 --- a/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.hpp +++ b/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.hpp @@ -59,6 +59,26 @@ AVPixelFormat shader_friendly_pix_format(AVPixelFormat src); where sws does the YUV to RGB conversion rather than our own shader. */ int sws_colourspace_from_av(AVColorSpace colorspace); +/* Value of the /plugin/media_reader/FFMPEG/videotoolbox preference. Has no effect +on platforms without VideoToolbox. */ +enum class VideoToolboxMode { Off, Prefer, Only }; + +/* Parses the preference value ("off", "prefer", "only"). Unrecognised values give +Prefer, the default. */ +VideoToolboxMode videotoolbox_mode_from_string(const std::string &value); + +const char *to_string(VideoToolboxMode mode); + +/* Thrown when a stream should have been decoded by VideoToolbox but can't be, and +falling back to FFmpeg's decoder isn't allowed or possible. The reader reports it +on the frame as an error. */ +class videotoolbox_unavailable_error : public std::runtime_error { + public: + using std::runtime_error::runtime_error; +}; + +class VtDecoder; + typedef enum { VIDEO_STREAM = 1, AUDIO_STREAM = 2, @@ -76,7 +96,8 @@ class FFMpegStream { AVStream *stream, int index, int thread_count, - std::string path); + std::string path, + VideoToolboxMode videotoolbox_mode = VideoToolboxMode::Off); virtual ~FFMpegStream(); @@ -126,6 +147,19 @@ class FFMpegStream { [[nodiscard]] bool is_attached_pic() const { return is_attached_pic_; } + /* True if VideoToolbox decodes this stream rather than FFmpeg. */ + [[nodiscard]] bool decoded_by_videotoolbox() const { return bool(vt_decoder_); } + + /* True if an FFmpeg decoder was opened for this stream. */ + [[nodiscard]] bool uses_ffmpeg_decoder() const { return codec_ != nullptr; } + + /* Name of the FFmpeg decoder opened for this stream, empty if none. */ + [[nodiscard]] std::string ffmpeg_decoder_name() const { return codec_ ? codec_->name : ""; } + + /* True once a frame of this stream has needed a CPU pixel format conversion + (swscale) before the shader could use it. */ + [[nodiscard]] bool converted_on_cpu() const { return format_conversion_warning_issued; } + [[nodiscard]] int stream_index() const { return stream_index_; } [[nodiscard]] bool is_drop_frame_timecode() const { return is_drop_frame_timecode_; } @@ -154,6 +188,18 @@ class FFMpegStream { void decode_attached_pic(); + // Sets up vt_decoder_, or videotoolbox_error_ if VideoToolbox should decode the + // stream but can't and FFmpeg mustn't (design D7). No-op without VideoToolbox. + void try_videotoolbox(); + + // Opens codec_ (FFmpeg's decoder) on codec_context_ for a video stream. + // Returns true for mjpeg. + bool open_software_video_decoder(); + + // VideoToolbox rejected the stream's packets: carry on with FFmpeg's decoder + // if allowed, otherwise throw videotoolbox_unavailable_error. + void fall_back_to_software(const std::string &reason); + // void setup_frame(ImageStorePtr & video_frame); int stream_index_; AVCodecContext *codec_context_{nullptr}; @@ -190,5 +236,12 @@ class FFMpegStream { utility::FrameRate frame_rate_; ImageBufPtr attached_pic_; + + VideoToolboxMode videotoolbox_mode_ = {VideoToolboxMode::Off}; + int thread_count_ = {0}; + // shared_ptr so that this header needs no complete VtDecoder (and no + // VideoToolbox) on any platform + std::shared_ptr vt_decoder_; + std::string videotoolbox_error_; }; } // namespace xstudio::media_reader::ffmpeg diff --git a/src/plugin/media_reader/ffmpeg/src/vt_decoder.cpp b/src/plugin/media_reader/ffmpeg/src/vt_decoder.cpp new file mode 100644 index 000000000..817e79bb9 --- /dev/null +++ b/src/plugin/media_reader/ffmpeg/src/vt_decoder.cpp @@ -0,0 +1,810 @@ +// SPDX-License-Identifier: Apache-2.0 + +// xstudio headers first: the Apple headers further down define a 'nil' macro that +// breaks caf headers included after them. +#include "vt_decoder.hpp" +#include "vt_repack.hpp" +#include "xstudio/media_reader/image_buffer.hpp" +#include "xstudio/utility/logging.hpp" + +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +} + +#include +#include +#include +#include + +namespace xstudio::media_reader::ffmpeg { + +namespace { + + // VideoToolbox output formats used here (design D4), and the planar format each + // is repacked to. + constexpr OSType ks4as = kCVPixelFormatType_444YpCbCr16VideoRange_16A_TriPlanar; // 's4as' + + enum class Packing { Bits8, Bits10InMsbs, Bits16, S4as }; + + struct OutputFormat { + OSType vt; + AVPixelFormat target; + Packing packing; + vt_repack::Chroma chroma; + bool full_range; + }; + + using vt_repack::Chroma; + const OutputFormat k_output_formats[] = { + {kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + AV_PIX_FMT_YUV420P, + Packing::Bits8, + Chroma::C420, + false}, + {kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + AV_PIX_FMT_YUV420P, + Packing::Bits8, + Chroma::C420, + true}, + {kCVPixelFormatType_422YpCbCr8BiPlanarVideoRange, + AV_PIX_FMT_YUV422P, + Packing::Bits8, + Chroma::C422, + false}, + {kCVPixelFormatType_422YpCbCr8BiPlanarFullRange, + AV_PIX_FMT_YUV422P, + Packing::Bits8, + Chroma::C422, + true}, + {kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange, + AV_PIX_FMT_YUV444P, + Packing::Bits8, + Chroma::C444, + false}, + {kCVPixelFormatType_444YpCbCr8BiPlanarFullRange, + AV_PIX_FMT_YUV444P, + Packing::Bits8, + Chroma::C444, + true}, + {kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, + AV_PIX_FMT_YUV420P10LE, + Packing::Bits10InMsbs, + Chroma::C420, + false}, + {kCVPixelFormatType_420YpCbCr10BiPlanarFullRange, + AV_PIX_FMT_YUV420P10LE, + Packing::Bits10InMsbs, + Chroma::C420, + true}, + {kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange, + AV_PIX_FMT_YUV422P10LE, + Packing::Bits10InMsbs, + Chroma::C422, + false}, + {kCVPixelFormatType_422YpCbCr10BiPlanarFullRange, + AV_PIX_FMT_YUV422P10LE, + Packing::Bits10InMsbs, + Chroma::C422, + true}, + {kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange, + AV_PIX_FMT_YUV444P10LE, + Packing::Bits10InMsbs, + Chroma::C444, + false}, + {kCVPixelFormatType_444YpCbCr10BiPlanarFullRange, + AV_PIX_FMT_YUV444P10LE, + Packing::Bits10InMsbs, + Chroma::C444, + true}, + {kCVPixelFormatType_422YpCbCr16BiPlanarVideoRange, + AV_PIX_FMT_YUV422P12LE, + Packing::Bits16, + Chroma::C422, + false}, + {kCVPixelFormatType_444YpCbCr16BiPlanarVideoRange, + AV_PIX_FMT_YUV444P12LE, + Packing::Bits16, + Chroma::C444, + false}, + // the target here is for 4444 with alpha; without, it's YUV444P12LE + {ks4as, AV_PIX_FMT_YUVA444P12LE, Packing::S4as, Chroma::C444, false}, + }; + + const OutputFormat *output_format(const OSType vt) { + for (const auto &f : k_output_formats) + if (f.vt == vt) + return &f; + return nullptr; + } + + // The VideoToolbox format for a bit depth, chroma format and range, 0 if none. + OSType + output_format_for(const Packing packing, const Chroma chroma, const bool full_range) { + for (const auto &f : k_output_formats) + if (f.packing == packing && f.chroma == chroma && f.full_range == full_range) + return f.vt; + return 0; + } + + std::string fourcc_string(const uint32_t f) { + std::string s; + for (int shift = 24; shift >= 0; shift -= 8) { + const char c = char((f >> shift) & 0xff); + s += (c >= 32 && c < 127) ? c : '?'; + } + return s; + } + + // ProRes codec_tag ('apch' etc., stored little-endian by FFmpeg) to the + // big-endian CMVideoCodecType VideoToolbox wants. 0 if it isn't a ProRes tag + // VideoToolbox decodes. + CMVideoCodecType prores_codec_type(const uint32_t tag) { + const uint32_t t = + ((tag & 0xff) << 24) | ((tag & 0xff00) << 8) | ((tag >> 8) & 0xff00) | (tag >> 24); + switch (t) { + case kCMVideoCodecType_AppleProRes422Proxy: + case kCMVideoCodecType_AppleProRes422LT: + case kCMVideoCodecType_AppleProRes422: + case kCMVideoCodecType_AppleProRes422HQ: + case kCMVideoCodecType_AppleProRes4444: + case kCMVideoCodecType_AppleProRes4444XQ: + return t; + default: + return 0; + } + } + + bool is_prores_4444(const CMVideoCodecType t) { + return t == kCMVideoCodecType_AppleProRes4444 || + t == kCMVideoCodecType_AppleProRes4444XQ; + } + + struct PictureFormat { + int bit_depth = 0; + int chroma = 0; // 420, 422 or 444 + }; + + // Bit depth and chroma format of an H.264 stream, from the avcC record. The + // High profile extension that carries them is optional; without it the + // profile tells us whether it can be anything but 8 bit 4:2:0. + bool avcc_picture_format(const uint8_t *d, const int size, PictureFormat &fmt) { + if (size < 7 || d[0] != 1) + return false; + const int profile = d[1]; + int pos = 5; + const int num_sps = d[pos++] & 0x1f; + for (int i = 0; i < num_sps; ++i) { + if (pos + 2 > size) + return false; + pos += 2 + ((d[pos] << 8) | d[pos + 1]); + } + if (pos + 1 > size) + return false; + const int num_pps = d[pos++]; + for (int i = 0; i < num_pps; ++i) { + if (pos + 2 > size) + return false; + pos += 2 + ((d[pos] << 8) | d[pos + 1]); + } + if (pos > size) + return false; + + const bool high_profile = profile == 100 || profile == 110 || profile == 122 || + profile == 144 || profile == 244; + if (high_profile && pos + 3 <= size) { + static const int chroma_formats[] = {400, 420, 422, 444}; + fmt.chroma = chroma_formats[d[pos] & 3]; + fmt.bit_depth = 8 + (d[pos + 1] & 7); + return true; + } + if (profile == 100 || !high_profile) { + // Baseline/Main/Extended, and High (not High 10/4:2:2/4:4:4), are + // always 8 bit 4:2:0. + fmt = {8, 420}; + return true; + } + return false; + } + + // Bit depth and chroma format of an HEVC stream, from the hvcC record header. + bool hvcc_picture_format(const uint8_t *d, const int size, PictureFormat &fmt) { + if (size < 23 || d[0] != 1) + return false; + static const int chroma_formats[] = {400, 420, 422, 444}; + fmt.chroma = chroma_formats[d[16] & 3]; + fmt.bit_depth = 8 + (d[17] & 7); + return true; + } + + PictureFormat picture_format_from_pix_fmt(const int format) { + const AVPixFmtDescriptor *d = av_pix_fmt_desc_get(AVPixelFormat(format)); + if (!d || (d->flags & AV_PIX_FMT_FLAG_RGB)) + return {}; + PictureFormat fmt; + fmt.bit_depth = d->comp[0].depth; + fmt.chroma = d->log2_chroma_w == 0 ? 444 : (d->log2_chroma_h ? 420 : 422); + return fmt; + } + + struct Choice { + CMVideoCodecType codec_type = 0; + OSType output_format = 0; + }; + + // Decides whether VideoToolbox should decode the stream, and with which output + // format. Returns false with 'reason' set if not. + bool choose(const AVStream *stream, Choice &choice, std::string &reason) { + + const AVCodecParameters *par = stream->codecpar; + if (par->codec_type != AVMEDIA_TYPE_VIDEO) { + reason = "not a video stream"; + return false; + } + if (par->width <= 0 || par->height <= 0) { + reason = "unknown picture size"; + return false; + } + if (stream->disposition & AV_DISPOSITION_ATTACHED_PIC) { + reason = "attached picture"; + return false; + } + + const bool full_range = par->color_range == AVCOL_RANGE_JPEG; + + if (par->codec_id == AV_CODEC_ID_PRORES) { + choice.codec_type = prores_codec_type(par->codec_tag); + if (!choice.codec_type) { + reason = "unrecognised ProRes codec tag"; + return false; + } + choice.output_format = is_prores_4444(choice.codec_type) + ? ks4as + : kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange; + return true; + } + + if (par->codec_id != AV_CODEC_ID_H264 && par->codec_id != AV_CODEC_ID_HEVC) { + reason = std::string("codec ") + avcodec_get_name(par->codec_id) + + " is not decoded with VideoToolbox"; + return false; + } + + // A length-prefixed stream carries an avcC/hvcC record (first byte 1). An + // Annex-B stream (raw .h264/.ts) has start codes instead, or nothing. + if (par->extradata_size < 7 || par->extradata[0] != 1) { + reason = "no avcC/hvcC decoder configuration (Annex-B stream)"; + return false; + } + + PictureFormat fmt; + const bool parsed = par->codec_id == AV_CODEC_ID_H264 + ? avcc_picture_format(par->extradata, par->extradata_size, fmt) + : hvcc_picture_format(par->extradata, par->extradata_size, fmt); + if (!parsed) + fmt = picture_format_from_pix_fmt(par->format); + + choice.codec_type = + par->codec_id == AV_CODEC_ID_H264 ? kCMVideoCodecType_H264 : kCMVideoCodecType_HEVC; + const char *codec = par->codec_id == AV_CODEC_ID_H264 ? "H.264" : "HEVC"; + + Chroma chroma; + switch (fmt.chroma) { + case 420: + chroma = Chroma::C420; + break; + case 422: + chroma = Chroma::C422; + break; + case 444: + chroma = Chroma::C444; + break; + default: + reason = + fmt::format("{} with chroma format {} (e.g. monochrome)", codec, fmt.chroma); + return false; + } + + if (fmt.bit_depth == 8) + choice.output_format = output_format_for(Packing::Bits8, chroma, full_range); + else if (fmt.bit_depth == 10) + choice.output_format = output_format_for(Packing::Bits10InMsbs, chroma, full_range); + else if (fmt.bit_depth == 12 && par->codec_id == AV_CODEC_ID_HEVC) { + // VideoToolbox decodes these in software. Its 12 bit 4:2:0 chroma doesn't + // match FFmpeg's and is further from the source (design D4, spike 1.8), + // and there are no full range 16 bit formats to ask for. + if (chroma == Chroma::C420) { + reason = "12 bit 4:2:0 HEVC (VideoToolbox's chroma is inaccurate)"; + return false; + } + if (full_range) { + reason = "full range 12 bit HEVC"; + return false; + } + choice.output_format = output_format_for(Packing::Bits16, chroma, false); + } + + if (!choice.output_format) { + reason = fmt::format("{} at {} bit {}", codec, fmt.bit_depth, fmt.chroma); + return false; + } + return true; + } + + // Does the first ProRes frame carry alpha? From the frame header, the same + // way FFmpeg's decoder decides between yuv444p12 and yuva444p12. + bool prores_frame_has_alpha(const AVPacket *pkt) { + // 4 byte frame size, 'icpf', then the frame header; alpha_info is the low + // nibble of header byte 17. + if (!pkt || pkt->size < 8 + 18) + return false; + if (memcmp(pkt->data + 4, "icpf", 4)) + return false; + return (pkt->data[8 + 17] & 0x0f) != 0; + } + + AVPixelFormat target_pix_fmt(const OSType vt, const bool alpha) { + const OutputFormat *f = output_format(vt); + if (!f) + return AV_PIX_FMT_NONE; + if (f->packing == Packing::S4as && !alpha) + return AV_PIX_FMT_YUV444P12LE; + return f->target; + } + + bool cf_equal(CFTypeRef value, CFStringRef key) { + return value && CFGetTypeID(value) == CFStringGetTypeID() && + CFStringCompare((CFStringRef)value, key, 0) == kCFCompareEqualTo; + } + + // Colour metadata VideoToolbox attached to a decoded buffer, for fields the + // container left unspecified (design D6). + void colour_from_attachments(CVPixelBufferRef pb, AVFrame *frame) { + if (frame->colorspace == AVCOL_SPC_UNSPECIFIED) { + CFTypeRef m = CVBufferCopyAttachment(pb, kCVImageBufferYCbCrMatrixKey, nullptr); + if (cf_equal(m, kCVImageBufferYCbCrMatrix_ITU_R_709_2)) + frame->colorspace = AVCOL_SPC_BT709; + else if (cf_equal(m, kCVImageBufferYCbCrMatrix_ITU_R_601_4)) + frame->colorspace = AVCOL_SPC_SMPTE170M; + else if (cf_equal(m, kCVImageBufferYCbCrMatrix_ITU_R_2020)) + frame->colorspace = AVCOL_SPC_BT2020_NCL; + else if (cf_equal(m, kCVImageBufferYCbCrMatrix_SMPTE_240M_1995)) + frame->colorspace = AVCOL_SPC_SMPTE240M; + if (m) + CFRelease(m); + } + if (frame->color_primaries == AVCOL_PRI_UNSPECIFIED) { + CFTypeRef p = CVBufferCopyAttachment(pb, kCVImageBufferColorPrimariesKey, nullptr); + if (cf_equal(p, kCVImageBufferColorPrimaries_ITU_R_709_2)) + frame->color_primaries = AVCOL_PRI_BT709; + else if (cf_equal(p, kCVImageBufferColorPrimaries_ITU_R_2020)) + frame->color_primaries = AVCOL_PRI_BT2020; + else if (cf_equal(p, kCVImageBufferColorPrimaries_P3_D65)) + frame->color_primaries = AVCOL_PRI_SMPTE432; + else if (cf_equal(p, kCVImageBufferColorPrimaries_DCI_P3)) + frame->color_primaries = AVCOL_PRI_SMPTE431; + else if (cf_equal(p, kCVImageBufferColorPrimaries_EBU_3213)) + frame->color_primaries = AVCOL_PRI_BT470BG; + else if (cf_equal(p, kCVImageBufferColorPrimaries_SMPTE_C)) + frame->color_primaries = AVCOL_PRI_SMPTE170M; + if (p) + CFRelease(p); + } + if (frame->color_trc == AVCOL_TRC_UNSPECIFIED) { + CFTypeRef t = + CVBufferCopyAttachment(pb, kCVImageBufferTransferFunctionKey, nullptr); + if (cf_equal(t, kCVImageBufferTransferFunction_ITU_R_709_2)) + frame->color_trc = AVCOL_TRC_BT709; + else if (cf_equal(t, kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ)) + frame->color_trc = AVCOL_TRC_SMPTE2084; + else if (cf_equal(t, kCVImageBufferTransferFunction_ITU_R_2100_HLG)) + frame->color_trc = AVCOL_TRC_ARIB_STD_B67; + else if (cf_equal(t, kCVImageBufferTransferFunction_ITU_R_2020)) + frame->color_trc = AVCOL_TRC_BT2020_10; + else if (cf_equal(t, kCVImageBufferTransferFunction_SMPTE_240M_1995)) + frame->color_trc = AVCOL_TRC_SMPTE240M; + if (t) + CFRelease(t); + } + } + + // Same ownership trick as setup_video_buffer() in ffmpeg_stream.cpp: the + // AVBufferRef in frame->buf[0] owns a heap-allocated ImageBufPtr that keeps the + // pixel memory alive and lets FFMpegStream take the ImageBuffer over. + void image_buf_ptr_deleter(void *, uint8_t *data) { delete (ImageBufPtr *)data; } + +} // namespace + +struct VtDecoder::Impl { + VTDecompressionSessionRef session = nullptr; + CMVideoFormatDescriptionRef format = nullptr; + Choice choice; + AVRational time_base = {1, 1}; + int width = 0; + int height = 0; + bool full_range = false; + bool has_alpha = false; + bool first_packet = true; + bool hardware = false; + AVColorSpace colorspace; + AVColorPrimaries primaries; + AVColorTransferCharacteristic trc; + + std::mutex mutex; + // Presentation timestamps of packets submitted that VideoToolbox hasn't + // called back for yet + std::set pending; + // Frames VideoToolbox has returned, by presentation timestamp + std::map decoded; + // Highest decode timestamp submitted. Every later packet has pts >= its + // dts >= this, so a decoded frame with pts <= max_dts can't be preceded by + // anything still to come (design D8). + int64_t max_dts = std::numeric_limits::min(); + bool draining = false; + + ~Impl() { + clear_decoded(); + if (session) { + VTDecompressionSessionWaitForAsynchronousFrames(session); + VTDecompressionSessionInvalidate(session); + CFRelease(session); + } + if (format) + CFRelease(format); + } + + void clear_decoded() { + std::lock_guard l(mutex); + for (auto &p : decoded) + CVPixelBufferRelease(p.second); + decoded.clear(); + pending.clear(); + max_dts = std::numeric_limits::min(); + } + + static void output_callback( + void *refcon, + void *source_frame_refcon, + OSStatus status, + VTDecodeInfoFlags flags, + CVImageBufferRef image, + CMTime, + CMTime) { + auto *self = static_cast(refcon); + const int64_t pts = int64_t(reinterpret_cast(source_frame_refcon)); + std::lock_guard l(self->mutex); + self->pending.erase(pts); + if (status != noErr || (flags & kVTDecodeInfo_FrameDropped) || !image) { + // Undecodable (e.g. an open-GOP leading picture after a seek); it is + // simply never output. + return; + } + auto it = self->decoded.find(pts); + if (it != self->decoded.end()) + CVPixelBufferRelease(it->second); + self->decoded[pts] = CVPixelBufferRetain(image); + } +}; + +VtDecoder::VtDecoder(std::unique_ptr impl) : impl_(std::move(impl)) {} + +VtDecoder::~VtDecoder() = default; + +bool VtDecoder::eligible(const AVStream *stream, std::string &reason) { + Choice choice; + return choose(stream, choice, reason); +} + +std::unique_ptr VtDecoder::create(const AVStream *stream, std::string &error) { + + auto impl = std::make_unique(); + if (!choose(stream, impl->choice, error)) + return nullptr; + + // Debugging aid, and how the fallback/failure paths are tested: pretend + // VideoToolbox can't create a session. + if (getenv("XSTUDIO_VIDEOTOOLBOX_FORCE_FAILURE")) { + error = "session creation disabled by XSTUDIO_VIDEOTOOLBOX_FORCE_FAILURE"; + return nullptr; + } + + const AVCodecParameters *par = stream->codecpar; + impl->time_base = stream->time_base; + impl->width = par->width; + impl->height = par->height; + impl->full_range = par->color_range == AVCOL_RANGE_JPEG; + impl->colorspace = par->color_space; + impl->primaries = par->color_primaries; + impl->trc = par->color_trc; + // Until the first ProRes frame header says otherwise, 4444 is assumed to have alpha. + impl->has_alpha = impl->choice.output_format == ks4as; + + CFMutableDictionaryRef extensions = CFDictionaryCreateMutable( + nullptr, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + if (par->codec_id == AV_CODEC_ID_H264 || par->codec_id == AV_CODEC_ID_HEVC) { + CFMutableDictionaryRef atoms = CFDictionaryCreateMutable( + nullptr, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + CFDataRef data = CFDataCreate(nullptr, par->extradata, par->extradata_size); + CFDictionarySetValue( + atoms, par->codec_id == AV_CODEC_ID_H264 ? CFSTR("avcC") : CFSTR("hvcC"), data); + CFDictionarySetValue( + extensions, kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms, atoms); + CFRelease(data); + CFRelease(atoms); + } + OSStatus status = CMVideoFormatDescriptionCreate( + nullptr, impl->choice.codec_type, par->width, par->height, extensions, &impl->format); + CFRelease(extensions); + if (status != noErr) { + error = fmt::format("CMVideoFormatDescriptionCreate failed ({})", int(status)); + return nullptr; + } + + CFMutableDictionaryRef spec = CFDictionaryCreateMutable( + nullptr, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + // Use hardware when there is some; VideoToolbox falls back to its own software + // decoder (e.g. ProRes on Intel) otherwise. + CFDictionarySetValue( + spec, + kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder, + kCFBooleanTrue); + + CFMutableDictionaryRef attributes = CFDictionaryCreateMutable( + nullptr, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + const SInt32 pixel_format = SInt32(impl->choice.output_format); + CFNumberRef number = CFNumberCreate(nullptr, kCFNumberSInt32Type, &pixel_format); + CFDictionarySetValue(attributes, kCVPixelBufferPixelFormatTypeKey, number); + CFRelease(number); + + VTDecompressionOutputCallbackRecord callback{&Impl::output_callback, impl.get()}; + status = VTDecompressionSessionCreate( + nullptr, impl->format, spec, attributes, &callback, &impl->session); + CFRelease(spec); + CFRelease(attributes); + if (status != noErr) { + impl->session = nullptr; + error = fmt::format("VTDecompressionSessionCreate failed ({})", int(status)); + return nullptr; + } + + CFBooleanRef hw = nullptr; + if (VTSessionCopyProperty( + impl->session, + kVTDecompressionPropertyKey_UsingHardwareAcceleratedVideoDecoder, + nullptr, + &hw) == noErr && + hw) { + impl->hardware = hw == kCFBooleanTrue; + CFRelease(hw); + } + + return std::unique_ptr(new VtDecoder(std::move(impl))); +} + +int VtDecoder::send_packet(const AVPacket *packet) { + + Impl &d = *impl_; + + if (!packet || !packet->size) { + // Start draining: everything submitted so far comes out of receive_frame, + // then AVERROR_EOF. + VTDecompressionSessionFinishDelayedFrames(d.session); + VTDecompressionSessionWaitForAsynchronousFrames(d.session); + std::lock_guard l(d.mutex); + d.draining = true; + return 0; + } + + { + std::lock_guard l(d.mutex); + if (d.draining) + return AVERROR_EOF; + } + + // Reordering relies on every packet having both timestamps (D8). + if (packet->pts == AV_NOPTS_VALUE || packet->dts == AV_NOPTS_VALUE) + return AVERROR(ENOTSUP); + + if (d.first_packet) { + d.first_packet = false; + if (d.choice.output_format == ks4as) + d.has_alpha = prores_frame_has_alpha(packet); + } + + // Wrap the packet data without copying: decoding is synchronous (D9), so it is + // finished with before this returns. + CMBlockBufferRef block = nullptr; + OSStatus status = CMBlockBufferCreateWithMemoryBlock( + nullptr, + packet->data, + size_t(packet->size), + kCFAllocatorNull, + nullptr, + 0, + size_t(packet->size), + 0, + &block); + if (status != noErr) + return AVERROR(ENOMEM); + + const AVRational tb = d.time_base; + CMSampleTimingInfo timing; + timing.duration = + packet->duration > 0 ? CMTimeMake(packet->duration * tb.num, tb.den) : kCMTimeInvalid; + timing.presentationTimeStamp = CMTimeMake(packet->pts * tb.num, tb.den); + timing.decodeTimeStamp = CMTimeMake(packet->dts * tb.num, tb.den); + const size_t sample_size = size_t(packet->size); + + CMSampleBufferRef sample = nullptr; + status = CMSampleBufferCreateReady( + nullptr, block, d.format, 1, 1, &timing, 1, &sample_size, &sample); + CFRelease(block); + if (status != noErr) + return AVERROR(ENOMEM); + + { + std::lock_guard l(d.mutex); + d.pending.insert(packet->pts); + d.max_dts = std::max(d.max_dts, packet->dts); + } + + VTDecodeInfoFlags info = 0; + status = VTDecompressionSessionDecodeFrame( + d.session, + sample, + 0, // synchronous + reinterpret_cast(intptr_t(packet->pts)), + &info); + CFRelease(sample); + + if (status != noErr) { + // VideoToolbox may or may not have called back with the error; either way + // this frame won't be output. + std::lock_guard l(d.mutex); + d.pending.erase(packet->pts); + if (status == kVTInvalidSessionErr) + return AVERROR_EXTERNAL; + spdlog::debug( + "VideoToolbox could not decode packet at pts {}: {}", packet->pts, int(status)); + } + return 0; +} + +int VtDecoder::receive_frame(AVFrame *frame) { + + Impl &d = *impl_; + CVPixelBufferRef pb = nullptr; + int64_t pts = 0; + { + std::lock_guard l(d.mutex); + if (d.decoded.empty()) + return d.draining ? AVERROR_EOF : AVERROR(EAGAIN); + + // The earliest decoded frame can go out once no frame with an earlier + // timestamp can still turn up: none is in flight, and no packet yet to be + // sent can have one (pts <= max_dts). + auto first = d.decoded.begin(); + if (!d.draining && (first->first > d.max_dts || + (!d.pending.empty() && first->first > *d.pending.begin()))) + return AVERROR(EAGAIN); + + pts = first->first; + pb = first->second; + d.decoded.erase(first); + } + + const AVPixelFormat target = target_pix_fmt(d.choice.output_format, d.has_alpha); + const int w = int(CVPixelBufferGetWidth(pb)); + const int h = int(CVPixelBufferGetHeight(pb)); + const vt_repack::Layout layout = vt_repack::planar_layout(target, w, h); + + ImageBufPtr image(new ImageBuffer()); + auto *base = reinterpret_cast(image->allocate(layout.total_size + 4096)); + + auto dst = [&](const int i) -> vt_repack::DstPlane { + return {base + layout.offsets[i], layout.linesizes[i]}; + }; + + CVPixelBufferLockBaseAddress(pb, kCVPixelBufferLock_ReadOnly); + auto src = [&](const size_t i) -> vt_repack::SrcPlane { + return { + static_cast(CVPixelBufferGetBaseAddressOfPlane(pb, i)), + CVPixelBufferGetBytesPerRowOfPlane(pb, i)}; + }; + + const OSType got = CVPixelBufferGetPixelFormatType(pb); + const OutputFormat *of = got == d.choice.output_format ? output_format(got) : nullptr; + const bool ok = of != nullptr; + if (of) { + switch (of->packing) { + case Packing::Bits8: + vt_repack::biplanar8_to_planar8( + src(0), src(1), dst(0), dst(1), dst(2), w, h, of->chroma); + break; + case Packing::Bits10InMsbs: + vt_repack::biplanar10_to_planar10( + src(0), src(1), dst(0), dst(1), dst(2), w, h, of->chroma); + break; + case Packing::Bits16: + vt_repack::biplanar16_to_planar12( + src(0), src(1), dst(0), dst(1), dst(2), w, h, of->chroma); + break; + case Packing::S4as: + vt_repack::s4as_to_yuva444p12( + src(0), + src(1), + src(2), + dst(0), + dst(1), + dst(2), + d.has_alpha ? dst(3) : vt_repack::DstPlane{nullptr, 0}, + w, + h); + break; + } + } + + if (ok) { + av_frame_unref(frame); + frame->format = target; + frame->width = w; + frame->height = h; + frame->pts = pts; + frame->best_effort_timestamp = pts; + frame->color_range = of->full_range ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG; + frame->colorspace = d.colorspace; + frame->color_primaries = d.primaries; + frame->color_trc = d.trc; + colour_from_attachments(pb, frame); + + for (int i = 0; i < layout.planes; ++i) { + frame->data[i] = base + layout.offsets[i]; + frame->linesize[i] = int(layout.linesizes[i]); + } + frame->extended_data = frame->data; + frame->buf[0] = av_buffer_create( + reinterpret_cast(new ImageBufPtr(image)), + sizeof(ImageBufPtr), + image_buf_ptr_deleter, + nullptr, + 0); + } + + CVPixelBufferUnlockBaseAddress(pb, kCVPixelBufferLock_ReadOnly); + CVPixelBufferRelease(pb); + + if (!ok) { + spdlog::warn( + "VideoToolbox returned pixel format {}, expected {}", + fourcc_string(got), + fourcc_string(d.choice.output_format)); + return AVERROR_EXTERNAL; + } + return 0; +} + +void VtDecoder::flush() { + if (impl_->session) + VTDecompressionSessionWaitForAsynchronousFrames(impl_->session); + impl_->clear_decoded(); + std::lock_guard l(impl_->mutex); + impl_->draining = false; +} + +AVPixelFormat VtDecoder::output_pix_fmt() const { + return target_pix_fmt(impl_->choice.output_format, impl_->has_alpha); +} + +std::string VtDecoder::output_format_name() const { + return fourcc_string(impl_->choice.output_format); +} + +bool VtDecoder::using_hardware() const { return impl_->hardware; } + +} // namespace xstudio::media_reader::ffmpeg diff --git a/src/plugin/media_reader/ffmpeg/src/vt_decoder.hpp b/src/plugin/media_reader/ffmpeg/src/vt_decoder.hpp new file mode 100644 index 000000000..5797fcbc7 --- /dev/null +++ b/src/plugin/media_reader/ffmpeg/src/vt_decoder.hpp @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +// VideoToolbox decode backend for FFMpegStream (macOS only). +// +// This header deliberately includes no Apple headers: MacTypes.h defines a 'nil' +// macro that breaks caf, and this header is included alongside caf ones. + +#include +#include + +extern "C" { +#include +#include +} + +namespace xstudio::media_reader::ffmpeg { + +/* Decodes one video stream with a VideoToolbox decompression session. + +The send_packet / receive_frame / flush methods follow the same rules as +avcodec_send_packet / avcodec_receive_frame / avcodec_flush_buffers, return codes +included, so FFMpegStream can use it in place of an AVCodecContext. + +Frames come out in presentation order, in a planar YUV(A) format the xstudio +shader reads directly. Their pixel data lives in an xstudio ImageBuffer, which is +hidden in frame->buf[0] the same way setup_video_buffer() in ffmpeg_stream.cpp +does it. */ +class VtDecoder { + public: + /* Whether the stream's codec and decoder configuration can go to VideoToolbox. + If not, 'reason' says why. */ + static bool eligible(const AVStream *stream, std::string &reason); + + /* Creates a decoder for the stream. Returns null and sets 'error' if + VideoToolbox can't create a session for it. Setting the environment variable + XSTUDIO_VIDEOTOOLBOX_FORCE_FAILURE makes it always fail, for testing the + fallback paths. */ + static std::unique_ptr create(const AVStream *stream, std::string &error); + + ~VtDecoder(); + + VtDecoder(const VtDecoder &) = delete; + VtDecoder &operator=(const VtDecoder &) = delete; + + /* A null packet starts draining, as with avcodec_send_packet. Returns 0 or a + negative AVERROR. The packet isn't unreferenced. */ + int send_packet(const AVPacket *packet); + + /* Returns 0 with 'frame' filled, AVERROR(EAGAIN) if more packets are needed, + or AVERROR_EOF once fully drained. */ + int receive_frame(AVFrame *frame); + + /* Drops every frame in flight, e.g. before a seek. */ + void flush(); + + /* The planar pixel format receive_frame() produces. */ + [[nodiscard]] AVPixelFormat output_pix_fmt() const; + + /* VideoToolbox's name for the session's output format, for logging. */ + [[nodiscard]] std::string output_format_name() const; + + [[nodiscard]] bool using_hardware() const; + + private: + struct Impl; + explicit VtDecoder(std::unique_ptr impl); + std::unique_ptr impl_; +}; + +} // namespace xstudio::media_reader::ffmpeg diff --git a/src/plugin/media_reader/ffmpeg/src/vt_repack.cpp b/src/plugin/media_reader/ffmpeg/src/vt_repack.cpp new file mode 100644 index 000000000..f8ca53891 --- /dev/null +++ b/src/plugin/media_reader/ffmpeg/src/vt_repack.cpp @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "vt_repack.hpp" + +#include + +extern "C" { +#include +#include +} + +namespace xstudio::media_reader::ffmpeg::vt_repack { + +namespace { + // The loops below are kept simple (contiguous rows, no branches) so the compiler + // can vectorise them. + + template inline const T *src_row(const SrcPlane &p, const int y) { + return reinterpret_cast(p.data + p.stride * size_t(y)); + } + + template inline T *dst_row(const DstPlane &p, const int y) { + return reinterpret_cast(p.data + p.stride * size_t(y)); + } +} // namespace + +Layout planar_layout(const AVPixelFormat format, const int width, const int height) { + + Layout layout; + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(format); + if (!desc || !(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) + return layout; + + std::array linesizes = {0, 0, 0, 0}; + if (av_image_fill_linesizes(linesizes.data(), format, width) < 0) + return layout; + + std::array aligned = {0, 0, 0, 0}; + for (int i = 0; i < 4; ++i) + aligned[i] = FFALIGN(linesizes[i], 64); + + std::array plane_sizes = {0, 0, 0, 0}; + if (av_image_fill_plane_sizes(plane_sizes.data(), format, height, aligned.data()) < 0) + return layout; + + layout.format = format; + for (int i = 0; i < 4; ++i) { + if (!plane_sizes[i]) + break; + layout.offsets[i] = layout.total_size; + layout.linesizes[i] = size_t(aligned[i]); + layout.total_size += plane_sizes[i]; + layout.planes++; + } + return layout; +} + +void biplanar8_to_planar8( + const SrcPlane y, + const SrcPlane cbcr, + const DstPlane dst_y, + const DstPlane dst_u, + const DstPlane dst_v, + const int width, + const int height, + const Chroma chroma) { + + for (int row = 0; row < height; ++row) + std::memcpy(dst_y.data + dst_y.stride * row, y.data + y.stride * row, size_t(width)); + + const int cw = chroma_width(width, chroma); + const int ch = chroma_height(height, chroma); + for (int row = 0; row < ch; ++row) { + const uint8_t *__restrict in = src_row(cbcr, row); + uint8_t *__restrict u = dst_row(dst_u, row); + uint8_t *__restrict v = dst_row(dst_v, row); + for (int x = 0; x < cw; ++x) { + u[x] = in[2 * x]; + v[x] = in[2 * x + 1]; + } + } +} + +void biplanar10_to_planar10( + const SrcPlane y, + const SrcPlane cbcr, + const DstPlane dst_y, + const DstPlane dst_u, + const DstPlane dst_v, + const int width, + const int height, + const Chroma chroma) { + + for (int row = 0; row < height; ++row) { + const uint16_t *__restrict in = src_row(y, row); + uint16_t *__restrict out = dst_row(dst_y, row); + for (int x = 0; x < width; ++x) + out[x] = in[x] >> 6; + } + + const int cw = chroma_width(width, chroma); + const int ch = chroma_height(height, chroma); + for (int row = 0; row < ch; ++row) { + const uint16_t *__restrict in = src_row(cbcr, row); + uint16_t *__restrict u = dst_row(dst_u, row); + uint16_t *__restrict v = dst_row(dst_v, row); + for (int x = 0; x < cw; ++x) { + u[x] = in[2 * x] >> 6; + v[x] = in[2 * x + 1] >> 6; + } + } +} + +void biplanar16_to_planar12( + const SrcPlane y, + const SrcPlane cbcr, + const DstPlane dst_y, + const DstPlane dst_u, + const DstPlane dst_v, + const int width, + const int height, + const Chroma chroma) { + + for (int row = 0; row < height; ++row) { + const uint16_t *__restrict in = src_row(y, row); + uint16_t *__restrict out = dst_row(dst_y, row); + for (int x = 0; x < width; ++x) + out[x] = round_16_to_12(in[x]); + } + + const int cw = chroma_width(width, chroma); + const int ch = chroma_height(height, chroma); + for (int row = 0; row < ch; ++row) { + const uint16_t *__restrict in = src_row(cbcr, row); + uint16_t *__restrict u = dst_row(dst_u, row); + uint16_t *__restrict v = dst_row(dst_v, row); + for (int x = 0; x < cw; ++x) { + u[x] = round_16_to_12(in[2 * x]); + v[x] = round_16_to_12(in[2 * x + 1]); + } + } +} + +void s4as_to_yuva444p12( + const SrcPlane y, + const SrcPlane cbcr, + const SrcPlane a, + const DstPlane dst_y, + const DstPlane dst_u, + const DstPlane dst_v, + const DstPlane dst_a, + const int width, + const int height) { + + if (!dst_a.data) { + // no alpha wanted: the same as sv44 + biplanar16_to_planar12(y, cbcr, dst_y, dst_u, dst_v, width, height, Chroma::C444); + return; + } + + // One pass writing all four planes measured faster than a pass per plane. + for (int row = 0; row < height; ++row) { + const uint16_t *__restrict in_y = src_row(y, row); + const uint16_t *__restrict in_uv = src_row(cbcr, row); + const uint16_t *__restrict in_a = src_row(a, row); + uint16_t *__restrict out_y = dst_row(dst_y, row); + uint16_t *__restrict out_u = dst_row(dst_u, row); + uint16_t *__restrict out_v = dst_row(dst_v, row); + uint16_t *__restrict out_a = dst_row(dst_a, row); + for (int x = 0; x < width; ++x) { + out_y[x] = round_16_to_12(in_y[x]); + out_u[x] = round_16_to_12(in_uv[2 * x]); + out_v[x] = round_16_to_12(in_uv[2 * x + 1]); + out_a[x] = round_16_to_12(in_a[x]); + } + } +} + +} // namespace xstudio::media_reader::ffmpeg::vt_repack diff --git a/src/plugin/media_reader/ffmpeg/src/vt_repack.hpp b/src/plugin/media_reader/ffmpeg/src/vt_repack.hpp new file mode 100644 index 000000000..fab38eb26 --- /dev/null +++ b/src/plugin/media_reader/ffmpeg/src/vt_repack.hpp @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +// Repacks the pixel layouts VideoToolbox decodes to into planar formats the xstudio +// YUV shader reads directly. Platform-neutral (just pointers and strides) so the +// loops can be unit tested anywhere. See design D5 in +// openspec/changes/add-videotoolbox-decode-backend. + +#include +#include +#include + +extern "C" { +#include +} + +namespace xstudio::media_reader::ffmpeg::vt_repack { + +struct SrcPlane { + const uint8_t *data; + size_t stride; // bytes per row +}; + +struct DstPlane { + uint8_t *data; + size_t stride; // bytes per row +}; + +/* Plane layout of a planar destination image in one contiguous buffer. */ +struct Layout { + AVPixelFormat format = AV_PIX_FMT_NONE; + std::array offsets = {0, 0, 0, 0}; + std::array linesizes = {0, 0, 0, 0}; + size_t total_size = 0; + int planes = 0; +}; + +/* Layout for a planar 'format' image of width x height. Line sizes are 64 byte +aligned, matching what setup_video_buffer() gives ffmpeg's own decoders. Returns a +layout with format AV_PIX_FMT_NONE for packed formats (AV_PIX_FMT_FLAG_PLANAR unset). */ +Layout planar_layout(AVPixelFormat format, int width, int height); + +/* Chroma subsampling of a source and its planar destination. */ +enum class Chroma { C420, C422, C444 }; + +inline int chroma_width(const int width, const Chroma c) { + return c == Chroma::C444 ? width : (width + 1) / 2; +} + +inline int chroma_height(const int height, const Chroma c) { + return c == Chroma::C420 ? (height + 1) / 2 : height; +} + +/* 8 bit Y + interleaved CbCr (VideoToolbox '420v' / '422v' / '444v' and the +full range 'f' variants; ffmpeg nv12 / nv16 / nv24) to yuv420p / yuv422p / +yuv444p. */ +void biplanar8_to_planar8( + SrcPlane y, + SrcPlane cbcr, + DstPlane dst_y, + DstPlane dst_u, + DstPlane dst_v, + int width, + int height, + Chroma chroma); + +/* 10 bit samples in the top bits of 16 bit words, Y + interleaved CbCr +(VideoToolbox 'x420' / 'x422' / 'x444' and the full range variants; ffmpeg +p010le / p210le / p410le) to yuv420p10le / yuv422p10le / yuv444p10le. */ +void biplanar10_to_planar10( + SrcPlane y, + SrcPlane cbcr, + DstPlane dst_y, + DstPlane dst_u, + DstPlane dst_v, + int width, + int height, + Chroma chroma); + +/* 16 bit Y + interleaved CbCr (VideoToolbox 'sv20' / 'sv22' / 'sv44'; ffmpeg +p016le / p216le / p416le) to yuv420p12le / yuv422p12le / yuv444p12le. Samples are +rounded to 12 bits. */ +void biplanar16_to_planar12( + SrcPlane y, + SrcPlane cbcr, + DstPlane dst_y, + DstPlane dst_u, + DstPlane dst_v, + int width, + int height, + Chroma chroma); + +/* 16 bit 4:4:4 Y + interleaved CbCr + A (VideoToolbox 's4as') to yuva444p12le, or +to yuv444p12le if dst_a.data is null. Samples are rounded to 12 bits. */ +void s4as_to_yuva444p12( + SrcPlane y, + SrcPlane cbcr, + SrcPlane a, + DstPlane dst_y, + DstPlane dst_u, + DstPlane dst_v, + DstPlane dst_a, + int width, + int height); + +/* 16 bit value rounded to 12 bits, clamped. Clamping first keeps every step within +16 bits ((65527 + 8) >> 4 == 4095), so loops vectorise at full width. */ +inline uint16_t round_16_to_12(const uint16_t v) { + const uint16_t clamped = v < 65527 ? v : uint16_t(65527); + return uint16_t(uint16_t(clamped + 8) >> 4); +} + +} // namespace xstudio::media_reader::ffmpeg::vt_repack diff --git a/src/plugin/media_reader/ffmpeg/test/CMakeLists.txt b/src/plugin/media_reader/ffmpeg/test/CMakeLists.txt index 902c030fe..03cc7a4ad 100644 --- a/src/plugin/media_reader/ffmpeg/test/CMakeLists.txt +++ b/src/plugin/media_reader/ffmpeg/test/CMakeLists.txt @@ -26,6 +26,12 @@ target_include_directories(ffmpeg_test ) target_link_directories(ffmpeg_test PRIVATE ${FFMPEG_LIBRARY_DIRS}) +# Media written by videotoolbox_media/generate.sh (tests needing it skip when absent) +target_compile_definitions(ffmpeg_test + PRIVATE VT_TEST_MEDIA_DIR="${CMAKE_BINARY_DIR}/test_media/videotoolbox" + PRIVATE $<$:XSTUDIO_HAS_VIDEOTOOLBOX=1> +) + add_test(ffmpeg_tests ffmpeg_test) set_target_properties(ffmpeg_test PROPERTIES LINK_DEPENDS_NO_SHARED true) diff --git a/src/plugin/media_reader/ffmpeg/test/ffmpeg_test.cpp b/src/plugin/media_reader/ffmpeg/test/ffmpeg_test.cpp index 01bc041c2..5bfb8a320 100644 --- a/src/plugin/media_reader/ffmpeg/test/ffmpeg_test.cpp +++ b/src/plugin/media_reader/ffmpeg/test/ffmpeg_test.cpp @@ -1,8 +1,19 @@ // SPDX-License-Identifier: Apache-2.0 #include +#include +#include +#include +#include +#include + #include "ffmpeg_decoder.hpp" #include "ffmpeg.hpp" +#include "vt_repack.hpp" +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX +#include "vt_decoder.hpp" +#endif +#include "xstudio/global_store/global_store.hpp" #include "xstudio/media/media.hpp" #include "xstudio/media_reader/media_reader.hpp" #include "xstudio/utility/helpers.hpp" @@ -117,3 +128,985 @@ TEST(FFMpegSwsColourspace, MapsStreamColourspaceToCoefficients) { EXPECT_EQ(sws_colourspace_from_av(AVCOL_SPC_UNSPECIFIED), SWS_CS_ITU601); } + +TEST(FFMpegVideoToolboxMode, ParsesPreferenceValues) { + EXPECT_EQ(videotoolbox_mode_from_string("off"), VideoToolboxMode::Off); + EXPECT_EQ(videotoolbox_mode_from_string("prefer"), VideoToolboxMode::Prefer); + EXPECT_EQ(videotoolbox_mode_from_string("only"), VideoToolboxMode::Only); + // Anything else gets the default. + EXPECT_EQ(videotoolbox_mode_from_string(""), VideoToolboxMode::Prefer); + EXPECT_EQ(videotoolbox_mode_from_string("Only"), VideoToolboxMode::Prefer); + + for (auto mode : {VideoToolboxMode::Off, VideoToolboxMode::Prefer, VideoToolboxMode::Only}) + EXPECT_EQ(videotoolbox_mode_from_string(to_string(mode)), mode); +} + +// Media from test/videotoolbox_media/generate.sh. Tests that need it are skipped when +// it hasn't been generated. +static std::string vt_media(const std::string &name) { + const std::string path = std::string(VT_TEST_MEDIA_DIR) + "/" + name; + return std::filesystem::exists(path) ? path : std::string(); +} + +#define VT_MEDIA_OR_SKIP(var, name) \ + const std::string var = vt_media(name); \ + if (var.empty()) \ + GTEST_SKIP() << name << " not found in " VT_TEST_MEDIA_DIR \ + << " - run test/videotoolbox_media/generate.sh" + +TEST(FFMpegVideoToolboxMode, DecoderCarriesMode) { + VT_MEDIA_OR_SKIP(path, "h264_8bit_bframes.mov"); + for (auto mode : + {VideoToolboxMode::Off, VideoToolboxMode::Prefer, VideoToolboxMode::Only}) { + FFMpegDecoder decoder( + path, 48000, utility::FrameRate(timebase::k_flicks_24fps), "", mode); + EXPECT_EQ(decoder.videotoolbox_mode(), mode); + } +} + +// Decodes one frame through the full reader path (stream, shader params). +static ImageBufPtr decode_frame( + const std::string &path, + const int frame, + const VideoToolboxMode mode = VideoToolboxMode::Off) { + // Stream 0 is the video stream in all the test media. + FFMpegDecoder decoder( + path, 48000, utility::FrameRate(timebase::k_flicks_24fps), "stream 0", mode); + ImageBufPtr buf; + decoder.decode_video_frame(frame, buf); + return buf; +} + +// Y offset the shader subtracts: 0 for full range, 16 (8 bit) for video range. +static int luma_offset(const ImageBufPtr &buf) { + return buf->shader_params()["yuv_offsets"][2].get(); +} + +// DNxHD/HR in a .dneg.mov is always treated as video range, whatever it is tagged. +TEST(FFMpegColourRange, DnegMovForcesVideoRangeForDNxHD) { + VT_MEDIA_OR_SKIP(plain, "dnxhr_fullrange.mov"); + VT_MEDIA_OR_SKIP(dneg, "dnxhr_fullrange.dneg.mov"); + + auto plain_buf = decode_frame(plain, 0); + auto dneg_buf = decode_frame(dneg, 0); + ASSERT_TRUE(plain_buf && dneg_buf); + EXPECT_EQ(luma_offset(plain_buf), 0); + EXPECT_EQ(luma_offset(dneg_buf), 16); +} + +// --------------------------------------------------------------------------------- +// vt_repack: VideoToolbox pixel layouts to planar formats the shader reads. +// Checked against av_read_image_line2 reading the same bytes as the matching +// FFmpeg format, so FFmpeg's own pixel descriptors are the reference. + +namespace { + +using namespace xstudio::media_reader::ffmpeg::vt_repack; + +// A source plane with padded rows, filled with pseudo-random values plus the +// extremes 0 and 0xFFFF (or 0xFF) at the start of every row. +struct TestPlane { + std::vector bytes; + size_t stride; + TestPlane(const int row_bytes, const int rows, const int bytes_per_sample, uint32_t seed) + : stride(size_t(row_bytes) + 40) { + bytes.resize(stride * rows); + for (auto &b : bytes) { + seed = seed * 1664525u + 1013904223u; + b = uint8_t(seed >> 24); + } + for (int r = 0; r < rows; ++r) { + uint8_t *row = bytes.data() + stride * r; + std::memset(row, 0, bytes_per_sample); + std::memset(row + bytes_per_sample, 0xff, bytes_per_sample); + } + } + SrcPlane src() const { return {bytes.data(), stride}; } +}; + +struct TestImage { + Layout layout; + std::vector bytes; + TestImage(const AVPixelFormat fmt, const int w, const int h) + : layout(planar_layout(fmt, w, h)), bytes(layout.total_size) {} + DstPlane dst(const int i) { + return {bytes.data() + layout.offsets[i], layout.linesizes[i]}; + } + // component c at (x, y) via FFmpeg's pixel descriptor + int value(const int c, const int x, const int y) const { + const uint8_t *data[4]; + int linesize[4]; + for (int i = 0; i < 4; ++i) { + data[i] = bytes.data() + layout.offsets[i]; + linesize[i] = int(layout.linesizes[i]); + } + uint16_t v = 0; + av_read_image_line2( + &v, data, linesize, av_pix_fmt_desc_get(layout.format), x, y, c, 1, 0, 2); + return v; + } +}; + +// component c at (x, y) of source planes interpreted as FFmpeg format 'fmt' +int reference_value( + const AVPixelFormat fmt, + const std::vector &planes, + const int c, + const int x, + const int y) { + const uint8_t *data[4] = {nullptr, nullptr, nullptr, nullptr}; + int linesize[4] = {0, 0, 0, 0}; + for (size_t i = 0; i < planes.size(); ++i) { + data[i] = planes[i]->bytes.data(); + linesize[i] = int(planes[i]->stride); + } + uint16_t v = 0; + av_read_image_line2(&v, data, linesize, av_pix_fmt_desc_get(fmt), x, y, c, 1, 0, 2); + return v; +} + +} // namespace + +TEST(VtRepack, PlanarLayoutIsAlignedAndContiguous) { + const auto l = planar_layout(AV_PIX_FMT_YUV420P10LE, 1921, 1081); + ASSERT_EQ(l.format, AV_PIX_FMT_YUV420P10LE); + ASSERT_EQ(l.planes, 3); + for (int i = 0; i < 3; ++i) + EXPECT_EQ(l.linesizes[i] % 64, 0u); + EXPECT_GE(l.linesizes[0], 1921u * 2); + EXPECT_GE(l.linesizes[1], 961u * 2); + EXPECT_EQ(l.offsets[1], l.linesizes[0] * 1081); + EXPECT_EQ(l.offsets[2], l.offsets[1] + l.linesizes[1] * 541); + EXPECT_EQ(l.total_size, l.offsets[2] + l.linesizes[2] * 541); + + // packed formats have no plane layout + EXPECT_EQ(planar_layout(AV_PIX_FMT_UYVY422, 64, 64).format, AV_PIX_FMT_NONE); +} + +// Every VideoToolbox biplanar layout against FFmpeg's description of it: 8 bit +// (nv12/16/24), 10 bit in the top bits (p010/210/410) and 16 bit (p016/216/416, +// rounded to 12 bits). +TEST(VtRepack, BiplanarToPlanarAllChromaFormats) { + enum Kind { Bits8, Bits10, Bits16 }; + struct Case { + Kind kind; + Chroma chroma; + AVPixelFormat src, dst; + }; + const Case cases[] = { + {Bits8, Chroma::C420, AV_PIX_FMT_NV12, AV_PIX_FMT_YUV420P}, + {Bits8, Chroma::C422, AV_PIX_FMT_NV16, AV_PIX_FMT_YUV422P}, + {Bits8, Chroma::C444, AV_PIX_FMT_NV24, AV_PIX_FMT_YUV444P}, + {Bits10, Chroma::C420, AV_PIX_FMT_P010LE, AV_PIX_FMT_YUV420P10LE}, + {Bits10, Chroma::C422, AV_PIX_FMT_P210LE, AV_PIX_FMT_YUV422P10LE}, + {Bits10, Chroma::C444, AV_PIX_FMT_P410LE, AV_PIX_FMT_YUV444P10LE}, + {Bits16, Chroma::C420, AV_PIX_FMT_P016LE, AV_PIX_FMT_YUV420P12LE}, + {Bits16, Chroma::C422, AV_PIX_FMT_P216LE, AV_PIX_FMT_YUV422P12LE}, + {Bits16, Chroma::C444, AV_PIX_FMT_P416LE, AV_PIX_FMT_YUV444P12LE}, + }; + for (const auto &cs : cases) { + for (auto [w, h] : {std::pair{64, 32}, std::pair{33, 17}}) { + const int bps = cs.kind == Bits8 ? 1 : 2; + const int cw = chroma_width(w, cs.chroma), ch = chroma_height(h, cs.chroma); + TestPlane y(w * bps, h, bps, 3), uv(cw * 2 * bps, ch, bps, 4); + if (cs.kind == Bits10) { + // VideoToolbox leaves the low 6 bits zero; so does a real P010 encoder. + for (auto *pl : {&y, &uv}) + for (size_t i = 0; i + 1 < pl->bytes.size(); i += 2) + pl->bytes[i] &= 0xc0; + } + + TestImage out(cs.dst, w, h); + switch (cs.kind) { + case Bits8: + biplanar8_to_planar8( + y.src(), uv.src(), out.dst(0), out.dst(1), out.dst(2), w, h, cs.chroma); + break; + case Bits10: + biplanar10_to_planar10( + y.src(), uv.src(), out.dst(0), out.dst(1), out.dst(2), w, h, cs.chroma); + break; + case Bits16: + biplanar16_to_planar12( + y.src(), uv.src(), out.dst(0), out.dst(1), out.dst(2), w, h, cs.chroma); + break; + } + + auto expected = [&](const int c, const int x, const int yy) { + const int ref = reference_value(cs.src, {&y, &uv}, c, x, yy); + return cs.kind == Bits16 ? int(round_16_to_12(uint16_t(ref))) : ref; + }; + for (int c = 0; c < 3; ++c) { + const int pw = c ? cw : w, ph = c ? ch : h; + for (int yy = 0; yy < ph; ++yy) + for (int x = 0; x < pw; ++x) + ASSERT_EQ(out.value(c, x, yy), expected(c, x, yy)) + << av_get_pix_fmt_name(cs.src) << " " << w << "x" << h << " c" << c + << " " << x << "," << yy; + } + } + } +} + +// s4as = p416le (16 bit Y + interleaved CbCr) plus a 16 bit alpha plane. +TEST(VtRepack, S4asToYuva444p12RoundsAndKeepsAlpha) { + for (auto [w, h] : {std::pair{64, 32}, std::pair{33, 17}}) { + TestPlane y(w * 2, h, 2, 5), uv(w * 4, h, 2, 6), a(w * 2, h, 2, 7); + TestImage out(AV_PIX_FMT_YUVA444P12LE, w, h); + s4as_to_yuva444p12( + y.src(), uv.src(), a.src(), out.dst(0), out.dst(1), out.dst(2), out.dst(3), w, h); + + for (int yy = 0; yy < h; ++yy) + for (int x = 0; x < w; ++x) { + for (int c = 0; c < 3; ++c) { + const int ref = reference_value(AV_PIX_FMT_P416LE, {&y, &uv}, c, x, yy); + ASSERT_EQ(out.value(c, x, yy), round_16_to_12(uint16_t(ref))) + << "c" << c << " " << x << "," << yy; + } + const uint16_t alpha = + *reinterpret_cast(a.bytes.data() + a.stride * yy + x * 2); + ASSERT_EQ(out.value(3, x, yy), round_16_to_12(alpha)) << "A " << x << "," << yy; + } + } + EXPECT_EQ(round_16_to_12(0), 0); + EXPECT_EQ(round_16_to_12(0xffff), 4095); + EXPECT_EQ(round_16_to_12(7), 0); + EXPECT_EQ(round_16_to_12(8), 1); + EXPECT_EQ(round_16_to_12(0xfff0), 4095); +} + +// Not run by default (timing-dependent). Run with --gtest_also_run_disabled_tests. +// Budget (design D5, measured on an M2 Max): at most 3 ms per 4K frame, single +// threaded, and at most 2.5x a plain copy of the same source bytes. swscale, which +// this replaced, took 7.7-9.6 ms. +TEST(VtRepack, DISABLED_BenchmarkAgainstMemcpy4K) { + const int w = 3840, h = 2160, reps = 30; + using clock = std::chrono::steady_clock; + auto time_ms = [&](auto &&fn) { + fn(); // warm up + const auto t0 = clock::now(); + for (int i = 0; i < reps; ++i) + fn(); + return std::chrono::duration(clock::now() - t0).count() / reps; + }; + auto memcpy_ms = [&](const std::vector &planes) { + size_t total = 0; + for (auto *p : planes) + total += p->bytes.size(); + std::vector dst(total); + return time_ms([&]() { + size_t off = 0; + for (auto *p : planes) { + std::memcpy(dst.data() + off, p->bytes.data(), p->bytes.size()); + off += p->bytes.size(); + } + }); + }; + auto report = [](const char *name, const double ms, const double copy_ms) { + std::cout << name << ": " << ms << " ms, memcpy " << copy_ms << " ms, ratio " + << ms / copy_ms << "\n"; + EXPECT_LE(ms, 3.0) << name; + EXPECT_LE(ms / copy_ms, 2.5) << name; + }; + + // every biplanar loop: name, bytes per sample, chroma, output format + enum Kind { Bits8, Bits10, Bits16 }; + struct Case { + const char *name; + Kind kind; + Chroma chroma; + AVPixelFormat dst; + }; + const Case cases[] = { + {"420v -> yuv420p", Bits8, Chroma::C420, AV_PIX_FMT_YUV420P}, + {"422v -> yuv422p", Bits8, Chroma::C422, AV_PIX_FMT_YUV422P}, + {"444v -> yuv444p", Bits8, Chroma::C444, AV_PIX_FMT_YUV444P}, + {"x420 -> yuv420p10le", Bits10, Chroma::C420, AV_PIX_FMT_YUV420P10LE}, + {"x422 -> yuv422p10le", Bits10, Chroma::C422, AV_PIX_FMT_YUV422P10LE}, + {"x444 -> yuv444p10le", Bits10, Chroma::C444, AV_PIX_FMT_YUV444P10LE}, + {"sv22 -> yuv422p12le", Bits16, Chroma::C422, AV_PIX_FMT_YUV422P12LE}, + {"sv44 -> yuv444p12le", Bits16, Chroma::C444, AV_PIX_FMT_YUV444P12LE}, + }; + for (const auto &cs : cases) { + const int bps = cs.kind == Bits8 ? 1 : 2; + const int cw = chroma_width(w, cs.chroma), ch = chroma_height(h, cs.chroma); + TestPlane y(w * bps, h, bps, 1), uv(cw * 2 * bps, ch, bps, 2); + TestImage out(cs.dst, w, h); + const double ms = time_ms([&]() { + switch (cs.kind) { + case Bits8: + biplanar8_to_planar8( + y.src(), uv.src(), out.dst(0), out.dst(1), out.dst(2), w, h, cs.chroma); + break; + case Bits10: + biplanar10_to_planar10( + y.src(), uv.src(), out.dst(0), out.dst(1), out.dst(2), w, h, cs.chroma); + break; + case Bits16: + biplanar16_to_planar12( + y.src(), uv.src(), out.dst(0), out.dst(1), out.dst(2), w, h, cs.chroma); + break; + } + }); + report(cs.name, ms, memcpy_ms({&y, &uv})); + } + { + TestPlane y(w * 2, h, 2, 1), uv(w * 4, h, 2, 2), a(w * 2, h, 2, 3); + TestImage out(AV_PIX_FMT_YUVA444P12LE, w, h); + const double ms = time_ms([&]() { + s4as_to_yuva444p12( + y.src(), + uv.src(), + a.src(), + out.dst(0), + out.dst(1), + out.dst(2), + out.dst(3), + w, + h); + }); + report("s4as -> yuva444p12le", ms, memcpy_ms({&y, &uv, &a})); + } +} + +// --------------------------------------------------------------------------------- +// VtDecoder, driven directly with packets from libavformat. +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX + +namespace { + +struct Demuxer { + AVFormatContext *ctx = nullptr; + AVStream *stream = nullptr; + AVPacket *pkt = av_packet_alloc(); + explicit Demuxer(const std::string &path) { + if (avformat_open_input(&ctx, path.c_str(), nullptr, nullptr) == 0 && + avformat_find_stream_info(ctx, nullptr) >= 0) { + const int i = av_find_best_stream(ctx, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0); + if (i >= 0) + stream = ctx->streams[i]; + } + } + ~Demuxer() { + av_packet_free(&pkt); + avformat_close_input(&ctx); + } + // next video packet, or null at the end + AVPacket *next() { + av_packet_unref(pkt); + while (av_read_frame(ctx, pkt) >= 0) { + if (pkt->stream_index == stream->index) + return pkt; + av_packet_unref(pkt); + } + return nullptr; + } +}; + +struct DecodeResult { + std::vector packet_pts; + std::vector frame_pts; + int send_errors = 0; + AVPixelFormat format = AV_PIX_FMT_NONE; + AVColorSpace colorspace = AVCOL_SPC_UNSPECIFIED; + AVColorPrimaries primaries = AVCOL_PRI_UNSPECIFIED; + AVColorTransferCharacteristic trc = AVCOL_TRC_UNSPECIFIED; + AVColorRange range = AVCOL_RANGE_UNSPECIFIED; +}; + +// Decodes the whole stream with VideoToolbox, avcodec-style send/receive loop. +DecodeResult vt_decode_all(const std::string &path, const int max_frames = 1 << 30) { + DecodeResult r; + Demuxer dm(path); + std::string error; + auto vt = VtDecoder::create(dm.stream, error); + EXPECT_TRUE(vt) << error; + if (!vt) + return r; + AVFrame *frame = av_frame_alloc(); + auto take = [&]() { + if (r.frame_pts.empty()) { + r.format = AVPixelFormat(frame->format); + r.colorspace = frame->colorspace; + r.primaries = frame->color_primaries; + r.trc = frame->color_trc; + r.range = frame->color_range; + } + r.frame_pts.push_back(frame->pts); + }; + while (AVPacket *pkt = dm.next()) { + if (int(r.frame_pts.size()) >= max_frames) + break; + r.packet_pts.push_back(pkt->pts); + if (vt->send_packet(pkt) < 0) + r.send_errors++; + while (vt->receive_frame(frame) == 0) + take(); + } + vt->send_packet(nullptr); + while (vt->receive_frame(frame) == 0) + take(); + av_frame_free(&frame); + return r; +} + +// Same, with FFmpeg's software decoder - the reference. +DecodeResult sw_decode_all(const std::string &path) { + DecodeResult r; + Demuxer dm(path); + const AVCodec *codec = avcodec_find_decoder(dm.stream->codecpar->codec_id); + AVCodecContext *ctx = avcodec_alloc_context3(codec); + avcodec_parameters_to_context(ctx, dm.stream->codecpar); + avcodec_open2(ctx, codec, nullptr); + AVFrame *frame = av_frame_alloc(); + auto take = [&]() { + if (r.frame_pts.empty()) { + r.format = AVPixelFormat(frame->format); + r.colorspace = frame->colorspace; + r.primaries = frame->color_primaries; + r.trc = frame->color_trc; + r.range = frame->color_range; + } + r.frame_pts.push_back(frame->pts); + }; + while (AVPacket *pkt = dm.next()) { + r.packet_pts.push_back(pkt->pts); + avcodec_send_packet(ctx, pkt); + while (avcodec_receive_frame(ctx, frame) == 0) + take(); + } + avcodec_send_packet(ctx, nullptr); + while (avcodec_receive_frame(ctx, frame) == 0) + take(); + av_frame_free(&frame); + avcodec_free_context(&ctx); + return r; +} + +const std::vector> k_vt_media_and_format = { + {"h264_8bit_bframes.mov", "420v"}, + {"hevc_8bit_opengop.mov", "420v"}, + {"hevc_main10_bframes.mov", "x420"}, + {"hevc_422_10bit_bframes.mov", "x422"}, + {"hevc_main10_bt2020_pq.mov", "x420"}, + {"prores_422_proxy.mov", "x422"}, + {"prores_422_lt.mov", "x422"}, + {"prores_422_standard.mov", "x422"}, + {"prores_422_hq.mov", "x422"}, + {"prores_4444_alpha.mov", "s4as"}, + {"prores_4444xq.mov", "s4as"}, + // variants (spike 1.8); the 12 bit ones need FFMPEG_HIGHBIT when generating + {"variant_h264_10bit_420.mov", "x420"}, + {"variant_h264_8bit_422.mov", "422v"}, + {"variant_h264_10bit_422.mov", "x422"}, + {"variant_h264_8bit_444.mov", "444v"}, + {"variant_h264_10bit_444.mov", "x444"}, + {"variant_hevc_8bit_422.mov", "422v"}, + {"variant_hevc_8bit_444.mov", "444v"}, + {"variant_hevc_10bit_444.mov", "x444"}, + {"variant_hevc_12bit_422.mov", "sv22"}, + {"variant_hevc_12bit_444.mov", "sv44"}, +}; + +// Is the clip there? Variant clips are optional; the core set is required. +bool have_clip(const std::string &name) { + if (!vt_media(name).empty()) + return true; + EXPECT_EQ(name.rfind("variant_", 0), 0u) << name << " missing"; + return false; +} + +} // namespace + +// Task 4.1 / 4.2 +TEST(VtDecoder, EligibilityAndOutputFormat) { + VT_MEDIA_OR_SKIP(first, "h264_8bit_bframes.mov"); + for (const auto &[name, expected] : k_vt_media_and_format) { + if (!have_clip(name)) + continue; + Demuxer dm(vt_media(name)); + ASSERT_TRUE(dm.stream) << name; + std::string reason; + EXPECT_TRUE(VtDecoder::eligible(dm.stream, reason)) << name << ": " << reason; + auto vt = VtDecoder::create(dm.stream, reason); + ASSERT_TRUE(vt) << name << ": " << reason; + EXPECT_EQ(vt->output_format_name(), expected) << name; + } + + for (const auto *name : + {"h264_annexb.h264", "dnxhr_fullrange.mov", "variant_hevc_12bit_420.mov"}) { + if (vt_media(name).empty()) + continue; // the 12 bit clip is optional + Demuxer dm(vt_media(name)); + ASSERT_TRUE(dm.stream) << name; + std::string reason; + EXPECT_FALSE(VtDecoder::eligible(dm.stream, reason)) << name; + EXPECT_FALSE(reason.empty()) << name; + EXPECT_FALSE(VtDecoder::create(dm.stream, reason)) << name; + } +} + +// Task 4.3 / 4.4: every packet decodes, frames come out once each in presentation +// order, and the last frames aren't lost in the reorder delay. +TEST(VtDecoder, DecodesEveryPacketInPresentationOrder) { + VT_MEDIA_OR_SKIP(first, "h264_8bit_bframes.mov"); + for (const auto &[name, fmt] : k_vt_media_and_format) { + if (!have_clip(name)) + continue; + const auto r = vt_decode_all(vt_media(name)); + EXPECT_EQ(r.send_errors, 0) << name; + ASSERT_EQ(r.frame_pts.size(), r.packet_pts.size()) << name; + EXPECT_TRUE(std::is_sorted(r.frame_pts.begin(), r.frame_pts.end())) << name; + EXPECT_EQ(std::adjacent_find(r.frame_pts.begin(), r.frame_pts.end()), r.frame_pts.end()) + << name << " repeats a frame"; + auto sorted_packets = r.packet_pts; + std::sort(sorted_packets.begin(), sorted_packets.end()); + EXPECT_EQ(r.frame_pts, sorted_packets) << name; + } +} + +// Task 4.5 / 4.6: planar output format and colour metadata match FFmpeg's decoder. +TEST(VtDecoder, FormatAndColourMatchSoftwareDecoder) { + VT_MEDIA_OR_SKIP(first, "h264_8bit_bframes.mov"); + for (const auto &[name, fmt] : k_vt_media_and_format) { + if (!have_clip(name)) + continue; + const auto vt = vt_decode_all(vt_media(name), 1); + const auto sw = sw_decode_all(vt_media(name)); + ASSERT_FALSE(vt.frame_pts.empty()) << name; + EXPECT_EQ(vt.format, sw.format) << name << ": " << av_get_pix_fmt_name(vt.format) + << " vs " << av_get_pix_fmt_name(sw.format); + EXPECT_EQ(vt.colorspace, sw.colorspace) << name; + EXPECT_EQ(vt.primaries, sw.primaries) << name; + EXPECT_EQ(vt.trc, sw.trc) << name; + // FFmpeg reports MPEG range for ProRes; unspecified elsewhere means MPEG too. + auto norm = [](AVColorRange r) { + return r == AVCOL_RANGE_UNSPECIFIED ? AVCOL_RANGE_MPEG : r; + }; + EXPECT_EQ(norm(vt.range), norm(sw.range)) << name; + } +} + +// Task 4.5: decode 1000+ frames; run under 'leaks' to check no CVPixelBuffer or +// session objects are left behind. +TEST(VtDecoder, LongDecodeForLeakCheck) { + VT_MEDIA_OR_SKIP(path, "h264_8bit_bframes.mov"); + size_t frames = 0; + for (int i = 0; i < 21; ++i) + frames += vt_decode_all(path).frame_pts.size(); + EXPECT_GE(frames, 1000u); +} + + +// --------------------------------------------------------------------------------- +// Through FFMpegDecoder / FFMpegStream: the path the reader uses. + +namespace { + +// Planar YUV(A) samples of a decoded image, read via the shader parameters the +// reader produced (so this also checks those are right). +struct Samples { + int bits = 0; + std::array, 4> planes; +}; + +Samples samples_of(const ImageBufPtr &buf) { + Samples out; + const auto &p = buf->shader_params(); + out.bits = p.value("bits_per_channel", 8); + const int bps = out.bits > 8 ? 2 : 1; + const auto size = buf->image_size_in_pixels(); + const int cw = p.value("half_scale_uvx", 0) ? (size.x + 1) / 2 : size.x; + const int ch = p.value("half_scale_uvy", 0) ? (size.y + 1) / 2 : size.y; + const char *names[4] = {"y", "u", "v", "a"}; + for (int i = 0; i < 4; ++i) { + const int linesize = p.value(std::string(names[i]) + "_linesize", 0); + if (!linesize) + continue; + const size_t offset = p.value(std::string(names[i]) + "_plane_bytes_offset", size_t(0)); + const int w = (i == 1 || i == 2) ? cw : size.x; + const int h = (i == 1 || i == 2) ? ch : size.y; + auto &plane = out.planes[i]; + plane.reserve(size_t(w) * h); + const auto *base = reinterpret_cast(buf->buffer()) + offset; + for (int y = 0; y < h; ++y) { + const uint8_t *row = base + size_t(linesize) * y; + for (int x = 0; x < w; ++x) + plane.push_back( + bps == 2 ? reinterpret_cast(row)[x] : uint16_t(row[x])); + } + } + return out; +} + +// Largest absolute sample difference between two images, -1 if their layouts differ. +int max_difference(const ImageBufPtr &a, const ImageBufPtr &b) { + const Samples sa = samples_of(a), sb = samples_of(b); + if (sa.bits != sb.bits) + return -1; + int worst = 0; + for (int i = 0; i < 4; ++i) { + if (sa.planes[i].size() != sb.planes[i].size()) + return -1; + for (size_t k = 0; k < sa.planes[i].size(); ++k) + worst = std::max(worst, std::abs(int(sa.planes[i][k]) - int(sb.planes[i][k]))); + } + return worst; +} + +// Scoped environment variable +struct ScopedEnv { + std::string name; + ScopedEnv(const char *n, const char *v) : name(n) { setenv(n, v, 1); } + ~ScopedEnv() { unsetenv(name.c_str()); } +}; + +FFMpegDecoder make_decoder(const std::string &path, const VideoToolboxMode mode) { + return FFMpegDecoder( + path, 48000, utility::FrameRate(timebase::k_flicks_24fps), "stream 0", mode); +} + +} // namespace + +// Task 5.1 (and the second half of 4.5): frames come from VideoToolbox, as planar +// YUV the shader reads directly - no CPU conversion on the way. +TEST(VtStream, PlaysThroughVideoToolboxWithoutCpuConversion) { + VT_MEDIA_OR_SKIP(first, "h264_8bit_bframes.mov"); + for (const auto &[name, fmt] : k_vt_media_and_format) { + if (!have_clip(name)) + continue; + auto decoder = make_decoder(vt_media(name), VideoToolboxMode::Prefer); + auto stream = decoder.stream(0); + ASSERT_TRUE(stream) << name; + EXPECT_TRUE(stream->decoded_by_videotoolbox()) << name; + // Task 3.2: FFmpeg's decoder is never even looked up, so builds without + // one for this codec work the same. + EXPECT_FALSE(stream->uses_ffmpeg_decoder()) << name; + for (int f : {0, 1, 2, 10}) { + ImageBufPtr buf; + decoder.decode_video_frame(f, buf); + ASSERT_TRUE(buf) << name << " frame " << f; + EXPECT_EQ(buf->shader_params().value("pix_fmt", -1), 0) + << name << ": YUV shader path"; + EXPECT_EQ(buf->decoder_frame_number(), f) << name; + } + EXPECT_FALSE(stream->converted_on_cpu()) << name; + } +} + +// Spec: frame-accurate presentation order (task 7.1). A fixed jumbled sequence of +// frame requests - forwards, backwards, first and last - must give the same +// pictures as FFmpeg's decoder: bit-exact for H.264/HEVC, within the ProRes +// tolerance measured in the spike (max 4 code values). +TEST(VtStream, RandomAccessMatchesSoftwareDecoder) { + VT_MEDIA_OR_SKIP(first, "h264_8bit_bframes.mov"); + const std::vector sequence = { + 0, 1, 2, 47, 12, 13, 5, 30, 29, 28, 46, 23, 24, 0, 37, 11}; + for (const auto &[name, fmt] : k_vt_media_and_format) { + if (!have_clip(name)) + continue; + const bool prores = name.rfind("prores", 0) == 0; + auto vt = make_decoder(vt_media(name), VideoToolboxMode::Prefer); + auto sw = make_decoder(vt_media(name), VideoToolboxMode::Off); + ASSERT_TRUE(vt.stream(0)->decoded_by_videotoolbox()) << name; + ASSERT_FALSE(sw.stream(0)->decoded_by_videotoolbox()) << name; + for (const int f : sequence) { + ImageBufPtr a, b; + std::string vt_error, sw_error; + try { + vt.decode_video_frame(f, a); + } catch (const std::exception &e) { + vt_error = e.what(); + } + try { + sw.decode_video_frame(f, b); + } catch (const std::exception &e) { + sw_error = e.what(); + } + // FFMpegDecoder can't reach the frame just before an open-GOP keyframe + // after a backward jump (frame 23 of the open-GOP clips here) with FFmpeg's + // decoder - a seek issue predating VideoToolbox support. VideoToolbox may do + // better, but must never fail where FFmpeg's decoder succeeds. + if (vt_error.empty() && !sw_error.empty()) { + ASSERT_TRUE(a) << name << " frame " << f; + EXPECT_EQ(a->decoder_frame_number(), f) << name; + continue; + } + EXPECT_EQ(vt_error, sw_error) << name << " frame " << f; + if (!vt_error.empty() || !sw_error.empty()) + continue; + ASSERT_TRUE(a && b) << name << " frame " << f; + EXPECT_EQ(a->decoder_frame_number(), f) << name; + EXPECT_EQ(a->has_alpha(), b->has_alpha()) << name; + const int diff = max_difference(a, b); + if (name.find("12bit") != std::string::npos) { + EXPECT_GE(diff, 0) << name << " frame " << f << ": layouts differ"; + EXPECT_LE(diff, 1) << name << " frame " << f; + } else if (prores) { + EXPECT_GE(diff, 0) << name << " frame " << f << ": layouts differ"; + EXPECT_LE(diff, 4) << name << " frame " << f; + } else { + EXPECT_EQ(diff, 0) << name << " frame " << f; + } + } + } +} + +// Task 5.2: when VideoToolbox can't create a session, 'prefer' falls back to +// FFmpeg's decoder and 'only' fails each frame with a message naming the codec. +TEST(VtStream, SessionFailureFallsBackOrFails) { + VT_MEDIA_OR_SKIP(path, "hevc_main10_bframes.mov"); + ScopedEnv force("XSTUDIO_VIDEOTOOLBOX_FORCE_FAILURE", "1"); + + { + auto decoder = make_decoder(path, VideoToolboxMode::Prefer); + EXPECT_FALSE(decoder.stream(0)->decoded_by_videotoolbox()); + ImageBufPtr buf; + decoder.decode_video_frame(3, buf); + ASSERT_TRUE(buf); + EXPECT_EQ(buf->decoder_frame_number(), 3); + } + { + auto decoder = make_decoder(path, VideoToolboxMode::Only); + ImageBufPtr buf; + try { + decoder.decode_video_frame(3, buf); + ADD_FAILURE() << "expected videotoolbox_unavailable_error"; + } catch (const videotoolbox_unavailable_error &e) { + EXPECT_NE(std::string(e.what()).find("hevc"), std::string::npos) << e.what(); + EXPECT_NE(std::string(e.what()).find("VideoToolbox"), std::string::npos) + << e.what(); + } + // and keeps failing cleanly rather than crashing + EXPECT_THROW(decoder.decode_video_frame(4, buf), videotoolbox_unavailable_error); + } +} + +// Task 5.3: streams VideoToolbox doesn't take are decoded by FFmpeg, in any mode, +// exactly as with VideoToolbox off. (A raw Annex-B stream has no container timing, +// so the reader gives it no duration or frame numbers either way.) +TEST(VtStream, IneligibleStreamsUseSoftware) { + VT_MEDIA_OR_SKIP(annexb, "h264_annexb.h264"); + VT_MEDIA_OR_SKIP(dnx, "dnxhr_fullrange.mov"); + for (const auto &path : {annexb, dnx}) { + auto reference = make_decoder(path, VideoToolboxMode::Off); + ImageBufPtr ref; + reference.decode_video_frame(5, ref); + ASSERT_TRUE(ref) << path; + for (auto mode : {VideoToolboxMode::Prefer, VideoToolboxMode::Only}) { + auto decoder = make_decoder(path, mode); + EXPECT_FALSE(decoder.stream(0)->decoded_by_videotoolbox()) << path; + ImageBufPtr buf; + decoder.decode_video_frame(5, buf); + ASSERT_TRUE(buf) << path; + EXPECT_EQ(buf->decoder_frame_number(), ref->decoder_frame_number()) << path; + EXPECT_EQ(max_difference(buf, ref), 0) << path; + } + } +} + +// Task 5.4: thumbnails of VideoToolbox-decoded media. +TEST(VtStream, Thumbnails) { + VT_MEDIA_OR_SKIP(first, "h264_8bit_bframes.mov"); + for (const auto &[name, fmt] : k_vt_media_and_format) { + if (!have_clip(name)) + continue; + auto decoder = make_decoder(vt_media(name), VideoToolboxMode::Prefer); + ASSERT_TRUE(decoder.stream(0)->decoded_by_videotoolbox()) << name; + auto thumb = decoder.decode_thumbnail_frame(24, 256); + ASSERT_TRUE(thumb) << name; + EXPECT_GT(thumb->width(), 0u) << name; + EXPECT_GT(thumb->height(), 0u) << name; + } +} + +// Changing the preference at runtime must not hand back a frame decoded under the +// previous mode (FFMpegMediaReader keeps the last decoded image to answer repeat +// requests). +TEST(VtStream, PreferenceChangeDropsFramesDecodedUnderOldMode) { + VT_MEDIA_OR_SKIP(path, "prores_422_hq.mov"); + + utility::JsonStore prefs; + global_store::set_preference_value( + prefs, std::string("prefer"), "/plugin/media_reader/FFMPEG/videotoolbox"); + FFMpegMediaReader reader(prefs); + + const media::AVFrameID frame( + posix_path_to_uri(path), + 10, + 0, + media::FS_ON_DISK, + 0, + 1.0f, + utility::FrameRate(timebase::k_flicks_24fps), + "stream 0"); + + const ImageBufPtr vt = reader.image(frame); + ASSERT_TRUE(vt); + vt->set_media_key(frame.key()); // as the reader actor does + EXPECT_EQ(reader.image(frame), vt) << "repeat request is answered from memory"; + + global_store::set_preference_value( + prefs, std::string("off"), "/plugin/media_reader/FFMPEG/videotoolbox"); + reader.update_preferences(prefs); + + const ImageBufPtr sw = reader.image(frame); + ASSERT_TRUE(sw); + EXPECT_NE(sw, vt); + // Apple's and FFmpeg's ProRes decoders differ slightly, so a fresh decode shows it + EXPECT_GT(max_difference(sw, vt), 0); + EXPECT_LE(max_difference(sw, vt), 4); +} + +// --------------------------------------------------------------------------------- +// Task 6.1: AAC through AudioToolbox (aac_at) + +namespace { + +struct DecodedAudio { + int64_t first_pts = AV_NOPTS_VALUE; + std::vector samples; // first channel +}; + +DecodedAudio decode_audio_with(const std::string &path, const char *decoder_name) { + DecodedAudio out; + AVFormatContext *ctx = nullptr; + if (avformat_open_input(&ctx, path.c_str(), nullptr, nullptr) < 0) + return out; + avformat_find_stream_info(ctx, nullptr); + const int index = av_find_best_stream(ctx, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0); + const AVCodec *codec = avcodec_find_decoder_by_name(decoder_name); + AVCodecContext *dec = avcodec_alloc_context3(codec); + avcodec_parameters_to_context(dec, ctx->streams[index]->codecpar); + avcodec_open2(dec, codec, nullptr); + AVPacket *pkt = av_packet_alloc(); + AVFrame *frame = av_frame_alloc(); + auto take = [&]() { + if (out.first_pts == AV_NOPTS_VALUE) + out.first_pts = frame->pts; + for (int i = 0; i < frame->nb_samples; ++i) { + switch (frame->format) { + case AV_SAMPLE_FMT_FLTP: + case AV_SAMPLE_FMT_FLT: + out.samples.push_back(reinterpret_cast(frame->data[0])[i]); + break; + case AV_SAMPLE_FMT_S16P: + case AV_SAMPLE_FMT_S16: + out.samples.push_back( + reinterpret_cast(frame->data[0])[i] / 32768.0f); + break; + default: + break; + } + } + }; + while (av_read_frame(ctx, pkt) >= 0) { + if (pkt->stream_index == index) { + avcodec_send_packet(dec, pkt); + while (avcodec_receive_frame(dec, frame) == 0) + take(); + } + av_packet_unref(pkt); + } + avcodec_send_packet(dec, nullptr); + while (avcodec_receive_frame(dec, frame) == 0) + take(); + av_frame_free(&frame); + av_packet_free(&pkt); + avcodec_free_context(&dec); + avformat_close_input(&ctx); + return out; +} + +} // namespace + +TEST(AacAudioToolbox, StreamUsesAacAt) { + VT_MEDIA_OR_SKIP(path, "h264_aac.mov"); + ASSERT_TRUE(avcodec_find_decoder_by_name("aac_at")) << "FFmpeg built without AudioToolbox"; + for (auto mode : {VideoToolboxMode::Off, VideoToolboxMode::Prefer}) { + FFMpegDecoder decoder( + path, 48000, utility::FrameRate(timebase::k_flicks_24fps), "stream 1", mode); + auto stream = decoder.stream(1); + ASSERT_TRUE(stream); + EXPECT_EQ(stream->ffmpeg_decoder_name(), "aac_at") << to_string(mode); + AudioBufPtr audio; + decoder.decode_audio_frame(10, audio); + ASSERT_TRUE(audio); + EXPECT_GT(audio->num_samples(), 0u); + } +} + +// aac_at must line up sample for sample with FFmpeg's own AAC decoder - same first +// timestamp, same length, no priming offset - or audio sync would change. +TEST(AacAudioToolbox, MatchesFFmpegAacTiming) { + VT_MEDIA_OR_SKIP(path, "h264_aac.mov"); + const auto ref = decode_audio_with(path, "aac"); + const auto at = decode_audio_with(path, "aac_at"); + ASSERT_FALSE(ref.samples.empty()); + ASSERT_FALSE(at.samples.empty()); + EXPECT_EQ(at.first_pts, ref.first_pts); + EXPECT_EQ(at.samples.size(), ref.samples.size()); + + // best alignment within +-4096 samples, on the chirp (non-periodic) + const size_t n = std::min(at.samples.size(), ref.samples.size()); + int best_lag = 0; + double best_err = 1e30; + for (int lag = -4096; lag <= 4096; lag += 1) { + double err = 0; + size_t used = 0; + for (size_t i = 8192; i + 8192 < n; i += 7) { + const long j = long(i) + lag; + const double d = at.samples[size_t(j)] - ref.samples[i]; + err += d * d; + used++; + } + err /= double(used); + if (err < best_err) { + best_err = err; + best_lag = lag; + } + } + EXPECT_EQ(best_lag, 0); + EXPECT_LT(std::sqrt(best_err), 1e-3) << "RMS difference between AAC decoders"; +} + +#endif // XSTUDIO_HAS_VIDEOTOOLBOX + + +// Not run by default (slow). Run with --gtest_also_run_disabled_tests. +// Memory must stay flat however many frames are decoded: VideoToolbox's frames live +// in IOSurfaces, which 'leaks' can't see but the process footprint includes. +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX +#include + +static double footprint_mb() { + task_vm_info_data_t info; + mach_msg_type_number_t count = TASK_VM_INFO_COUNT; + if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&info, &count) != KERN_SUCCESS) + return -1; + return double(info.phys_footprint) / (1024.0 * 1024.0); +} + +TEST(VtStream, DISABLED_MemoryStaysFlatOverLongDecode) { + VT_MEDIA_OR_SKIP(path, "bench_uhd_prores_4444_grain.mov"); + for (auto mode : {VideoToolboxMode::Off, VideoToolboxMode::Prefer}) { + std::vector mb; + for (int pass = 0; pass < 6; ++pass) { + // a new decoder each pass, like the reader for thumbnails / reopened media + auto decoder = make_decoder(path, mode); + for (int f = 0; f < 120; ++f) { + ImageBufPtr buf; + decoder.decode_video_frame(f, buf); + } + mb.push_back(footprint_mb()); + } + std::cout << to_string(mode) << " footprint MB after each 120 frame pass:"; + for (auto v : mb) + std::cout << " " << int(v); + std::cout << "\n"; + // allow some slack for allocator pools; a leaked 4K 4444 frame is ~66MB + EXPECT_LT(mb.back() - mb[1], 200.0) << to_string(mode); + } +} + +#endif diff --git a/src/plugin/media_reader/ffmpeg/test/videotoolbox_media/README.md b/src/plugin/media_reader/ffmpeg/test/videotoolbox_media/README.md new file mode 100644 index 000000000..715d8a8e4 --- /dev/null +++ b/src/plugin/media_reader/ffmpeg/test/videotoolbox_media/README.md @@ -0,0 +1,56 @@ +# VideoToolbox decode test media + +Media used to test the VideoToolbox decode backend of the FFmpeg reader +(`openspec/changes/add-videotoolbox-decode-backend`). The files aren't +committed. Generate them with: + +```sh +./generate.sh [path/to/ffmpeg] [output dir] + +# also write the 12 bit / 10 bit 4:4:4 HEVC variants, which need a libx265 built +# for high bit depth (the vcpkg one is 8 bit only) +FFMPEG_HIGHBIT="rez-env ffmpeg-9.0 -- ffmpeg" ./generate.sh +``` + +By default the script uses the vcpkg-built FFmpeg at +`build/vcpkg_installed/arm-osx/tools/ffmpeg/ffmpeg` and writes to +`build/test_media/videotoolbox/`. It needs an FFmpeg with libx264, libx265 and +the VideoToolbox encoders, so it has to run on macOS. Generation takes about +a minute on an M2 Max, and the output is about 1.7 GB, most of it the 4K +ProRes benchmark files. + +Every clip is `testsrc2` at 24 fps, so each frame shows a running counter for +checking frame accuracy by eye. The HD clips are 1920x1080 and 2 s long +(48 frames). The `bench_` clips are 3840x2160 and 5 s long (120 frames). Every +clip is tagged BT.709 limited range unless listed otherwise. + +| File | Codec / profile | Depth, subsampling | GOP / structure | Tests | +|---|---|---|---|---| +| `h264_8bit_bframes.mov` | H.264 High (libx264), avcC | 8-bit 4:2:0 | 3 B-frames, closed GOP of 24 | Reorder (D8), random access, bit-exact parity | +| `hevc_main10_bframes.mov` | HEVC Main 10 (VideoToolbox), hvcC | 10-bit 4:2:0 | 2 B-frames, GOP 24 | `x420`/P010 output path, reorder | +| `hevc_422_10bit_bframes.mov` | HEVC Main 4:2:2 10 (VideoToolbox), hvcC | 10-bit 4:2:2 | 2 B-frames, GOP 24 | `x422`/P210 output path | +| `hevc_8bit_opengop.mov` | HEVC Main (libx265), hvcC | 8-bit 4:2:0 | 4 B-frames, **open GOP**, keyint 24 | Leading pictures after a seek | +| `hevc_main10_bt2020_pq.mov` | HEVC Main 10 (VideoToolbox), hvcC | 10-bit 4:2:0 | 2 B-frames, GOP 24 | Colour metadata: BT.2020 non-constant-luminance matrix and primaries, PQ transfer | +| `prores_422_proxy.mov` | ProRes 422 Proxy (`apco`) | 10-bit 4:2:2 | Intra | ProRes 422 decode | +| `prores_422_lt.mov` | ProRes 422 LT (`apcs`) | 10-bit 4:2:2 | Intra | " | +| `prores_422_standard.mov` | ProRes 422 (`apcn`) | 10-bit 4:2:2 | Intra | " | +| `prores_422_hq.mov` | ProRes 422 HQ (`apch`) | 10-bit 4:2:2 | Intra | " | +| `prores_4444_alpha.mov` | ProRes 4444 (`ap4h`) | 12-bit 4:4:4 + alpha | Intra | Alpha preserved. Alpha is a horizontal ramp from 0 (left) to 1 (right) | +| `prores_4444xq.mov` | ProRes 4444 XQ (`ap4x`) | 12-bit 4:4:4 | Intra | 4444 XQ decode, 12-bit precision | +| `h264_annexb.h264` | H.264 High, raw Annex-B elementary stream | 8-bit 4:2:0 | 3 B-frames | Not eligible for VideoToolbox; must fall back to software. No container, so FFmpeg guesses 25 fps | +| `h264_aac.mov` | H.264 High + AAC-LC mono 48 kHz, rising chirp | 8-bit 4:2:0 | 3 B-frames | AAC through `aac_at` (D10), A/V sync. The chirp never repeats, so any sample offset between AAC decoders shows up | +| `dnxhr_fullrange.mov`, `dnxhr_fullrange.dneg.mov` | DNxHR HQ tagged full range (the same file twice) | 8-bit 4:2:2 | Intra | Not VideoToolbox. Regression test for the `.dneg.mov` video-range override | +| `variant_h264_{10bit_420,8bit_422,10bit_422,8bit_444,10bit_444}.mov` | H.264 High 10 / High 4:2:2 / High 4:4:4 Predictive | as named | 3 B-frames | Spike 1.8 variants | +| `variant_hevc_{8bit_422,8bit_444}.mov` | HEVC Rext (libx265) | as named | 4 B-frames, open GOP | Spike 1.8 variants | +| `variant_hevc_{10bit_444,12bit_420,12bit_422,12bit_444}.mov` | HEVC Rext, only with `FFMPEG_HIGHBIT` | as named | 4 B-frames, open GOP | Spike 1.8 variants. 12-bit 4:2:0 must stay on FFmpeg | +| `bench_uhd_hevc_main10.mov` | HEVC Main 10 (VideoToolbox), 40 Mb/s | 10-bit 4:2:0 | 2 B-frames, GOP 24 | Throughput benchmark (task 1.5) | +| `bench_uhd_prores_4444.mov` | ProRes 4444 (`ap4h`) | 12-bit 4:4:4 | Intra | Throughput benchmark (task 1.5) | +| `bench_uhd_{hevc_main10,h264,prores_422hq,prores_4444}_grain.mov` | As above, with temporal grain | | | Realistic decode cost for benchmarks: `testsrc2` alone compresses unrealistically well | + +Notes: + +- The vcpkg `libx265` is an 8-bit build, so the 10-bit HEVC clips come from + Apple's `hevc_videotoolbox` encoder. `libx265` is still used for the open-GOP + clip, because `hevc_videotoolbox` has no open-GOP control. +- FFmpeg 8 takes colour tags from the frames, so the script tags them with a + `setparams` filter as well as with the `-color_*` output options. diff --git a/src/plugin/media_reader/ffmpeg/test/videotoolbox_media/generate.sh b/src/plugin/media_reader/ffmpeg/test/videotoolbox_media/generate.sh new file mode 100755 index 000000000..ffc4a8be2 --- /dev/null +++ b/src/plugin/media_reader/ffmpeg/test/videotoolbox_media/generate.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# +# Generates the media set used to test the VideoToolbox decode backend. +# See README.md in this directory for what each file exercises. +# +# usage: generate.sh [ffmpeg binary] [output dir] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../../../.." && pwd)" + +FFMPEG="${1:-${REPO_ROOT}/build/vcpkg_installed/arm-osx/tools/ffmpeg/ffmpeg}" +OUT="${2:-${REPO_ROOT}/build/test_media/videotoolbox}" + +mkdir -p "${OUT}" + +FF=("${FFMPEG}" -hide_banner -loglevel error -y) + +# 2 seconds at 24fps; frame number burnt in so frame accuracy can be checked by eye +HD="testsrc2=size=1920x1080:rate=24:duration=2" +UHD="testsrc2=size=3840x2160:rate=24:duration=5" +# FFmpeg 8 takes colour tags from the frames, so set them with setparams rather than +# -color_* output options alone. +P709="setparams=colorspace=bt709:color_primaries=bt709:color_trc=bt709:range=tv" +P2020PQ="setparams=colorspace=bt2020nc:color_primaries=bt2020:color_trc=smpte2084:range=tv" +TAGS_709=(-colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range tv) + +echo "Writing test media to ${OUT}" + +# H.264 8-bit 4:2:0, 3 B-frames, closed GOP of 24 +"${FF[@]}" -f lavfi -i "${HD}" -c:v libx264 -pix_fmt yuv420p -bf 3 -g 24 \ + -x264-params open-gop=0 -vf "${P709}" "${TAGS_709[@]}" "${OUT}/h264_8bit_bframes.mov" + +# The vcpkg libx265 is an 8 bit build, so 10 bit HEVC comes from Apple's encoder. +# libx265 is still used where we need GOP control that hevc_videotoolbox doesn't expose. + +# HEVC Main10 4:2:0, B-frames +"${FF[@]}" -f lavfi -i "${HD}" -c:v hevc_videotoolbox -profile:v main10 -pix_fmt p010le \ + -bf 2 -g 24 -b:v 8M -tag:v hvc1 -vf "${P709}" "${TAGS_709[@]}" "${OUT}/hevc_main10_bframes.mov" + +# HEVC Main 4:2:2 10, B-frames +"${FF[@]}" -f lavfi -i "${HD}" -c:v hevc_videotoolbox -profile:v main42210 -pix_fmt p210le \ + -bf 2 -g 24 -b:v 10M -tag:v hvc1 -vf "${P709}" "${TAGS_709[@]}" "${OUT}/hevc_422_10bit_bframes.mov" + +# HEVC Main (8 bit), open GOP with B-frames - leading pictures after each I frame +"${FF[@]}" -f lavfi -i "${HD}" -c:v libx265 -pix_fmt yuv420p -tag:v hvc1 \ + -x265-params "bframes=4:keyint=24:open-gop=1:log-level=error" -vf "${P709}" "${TAGS_709[@]}" \ + "${OUT}/hevc_8bit_opengop.mov" + +# HEVC Main10 tagged BT.2020 / PQ +"${FF[@]}" -f lavfi -i "${HD}" -c:v hevc_videotoolbox -profile:v main10 -pix_fmt p010le \ + -bf 2 -g 24 -b:v 8M -tag:v hvc1 -vf "${P2020PQ}" \ + -colorspace bt2020nc -color_primaries bt2020 -color_trc smpte2084 -color_range tv \ + "${OUT}/hevc_main10_bt2020_pq.mov" + +# ProRes 422 family, via Apple's encoder +for profile in proxy lt standard hq; do + "${FF[@]}" -f lavfi -i "${HD}" -c:v prores_videotoolbox -profile:v "${profile}" \ + -pix_fmt p210le -vf "${P709}" "${TAGS_709[@]}" "${OUT}/prores_422_${profile}.mov" +done + +# ProRes 4444 with an alpha gradient, and 4444 XQ +"${FF[@]}" -f lavfi -i "${HD}" -vf "format=yuva444p10le,geq=lum='lum(X,Y)':cb='cb(X,Y)':cr='cr(X,Y)':a='X*1023/W',${P709}" \ + -c:v prores_videotoolbox -profile:v 4444 -pix_fmt ayuv64le "${TAGS_709[@]}" \ + "${OUT}/prores_4444_alpha.mov" +"${FF[@]}" -f lavfi -i "${HD}" -c:v prores_videotoolbox -profile:v xq -pix_fmt ayuv64le \ + -vf "${P709}" "${TAGS_709[@]}" "${OUT}/prores_4444xq.mov" + +# Raw Annex-B H.264 elementary stream (no avcC) - must fall back to software +"${FF[@]}" -f lavfi -i "${HD}" -c:v libx264 -pix_fmt yuv420p -bf 3 -g 24 \ + -vf "${P709}" "${TAGS_709[@]}" -f h264 "${OUT}/h264_annexb.h264" + +# H.264 with an AAC audio track: a rising chirp, so that any offset between AAC +# decoders shows up (a steady tone would hide one that is a whole number of cycles, +# e.g. the usual 2112 sample priming at 1kHz/48kHz) +"${FF[@]}" -f lavfi -i "${HD}" -f lavfi -i "aevalsrc='0.5*sin(2*PI*(200+1500*t)*t)':s=48000:d=2" \ + -c:v libx264 -pix_fmt yuv420p -bf 3 -g 24 -vf "${P709}" "${TAGS_709[@]}" -c:a aac -b:a 192k \ + -shortest "${OUT}/h264_aac.mov" + +# DNxHR HQ tagged full range, and the same file named .dneg.mov - the reader forces +# DNxHD/HR in .dneg.mov files to video range. Not VideoToolbox media, but the +# regression test for that override lives with these tests. +"${FF[@]}" -f lavfi -i "${HD}" -c:v dnxhd -profile:v dnxhr_hq -pix_fmt yuv422p \ + -vf "setparams=colorspace=bt709:color_primaries=bt709:color_trc=bt709:range=pc" \ + -colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range pc \ + "${OUT}/dnxhr_fullrange.mov" +cp "${OUT}/dnxhr_fullrange.mov" "${OUT}/dnxhr_fullrange.dneg.mov" + +# Variants VideoToolbox isn't used for yet (design D4), to see what it does with them. +# libx265 here is 8 bit only, so there is no 12 bit or 10 bit 4:4:4 HEVC. +for v in "yuv420p10le h264_10bit_420" "yuv422p h264_8bit_422" "yuv422p10le h264_10bit_422" \ + "yuv444p h264_8bit_444" "yuv444p10le h264_10bit_444"; do + set -- $v + "${FF[@]}" -f lavfi -i "${HD}" -c:v libx264 -pix_fmt "$1" -bf 3 -g 24 \ + -vf "${P709}" "${TAGS_709[@]}" "${OUT}/variant_$2.mov" +done +for v in "yuv422p hevc_8bit_422" "yuv444p hevc_8bit_444"; do + set -- $v + "${FF[@]}" -f lavfi -i "${HD}" -c:v libx265 -pix_fmt "$1" -tag:v hvc1 \ + -x265-params "bframes=4:keyint=24:log-level=error" -vf "${P709}" "${TAGS_709[@]}" \ + "${OUT}/variant_$2.mov" +done + +# High bit depth HEVC needs a libx265 built for it, which the vcpkg one isn't. Point +# FFMPEG_HIGHBIT at an ffmpeg that has one (e.g. "rez-env ffmpeg-9.0 -- ffmpeg") to +# also write these. +if [ -n "${FFMPEG_HIGHBIT:-}" ]; then + for v in "yuv420p12le hevc_12bit_420" "yuv422p12le hevc_12bit_422" \ + "yuv444p12le hevc_12bit_444" "yuv444p10le hevc_10bit_444"; do + set -- $v + ${FFMPEG_HIGHBIT} -hide_banner -loglevel error -y -f lavfi -i "${HD}" -c:v libx265 \ + -pix_fmt "$1" -tag:v hvc1 -x265-params "bframes=4:keyint=24:log-level=error" \ + -vf "${P709}" "${TAGS_709[@]}" "${OUT}/variant_$2.mov" + done +fi + +# 4K benchmark material +"${FF[@]}" -f lavfi -i "${UHD}" -c:v hevc_videotoolbox -profile:v main10 -pix_fmt p010le \ + -bf 2 -g 24 -b:v 40M -tag:v hvc1 -vf "${P709}" "${TAGS_709[@]}" "${OUT}/bench_uhd_hevc_main10.mov" +"${FF[@]}" -f lavfi -i "${UHD}" -c:v prores_videotoolbox -profile:v 4444 -pix_fmt ayuv64le \ + -vf "${P709}" "${TAGS_709[@]}" "${OUT}/bench_uhd_prores_4444.mov" + +# testsrc2 compresses unrealistically well, so these add temporal grain to get bitrates and +# decode cost closer to real camera footage. +GRAIN="noise=alls=12:allf=t+u" +"${FF[@]}" -f lavfi -i "${UHD}" -c:v hevc_videotoolbox -profile:v main10 -pix_fmt p010le \ + -bf 2 -g 24 -b:v 80M -tag:v hvc1 -vf "${GRAIN},${P709}" "${TAGS_709[@]}" \ + "${OUT}/bench_uhd_hevc_main10_grain.mov" +"${FF[@]}" -f lavfi -i "${UHD}" -c:v prores_videotoolbox -profile:v hq -pix_fmt p210le \ + -vf "${GRAIN},${P709}" "${TAGS_709[@]}" "${OUT}/bench_uhd_prores_422hq_grain.mov" +"${FF[@]}" -f lavfi -i "${UHD}" -c:v prores_videotoolbox -profile:v 4444 -pix_fmt ayuv64le \ + -vf "${GRAIN},${P709}" "${TAGS_709[@]}" "${OUT}/bench_uhd_prores_4444_grain.mov" +"${FF[@]}" -f lavfi -i "${UHD}" -c:v libx264 -pix_fmt yuv420p -bf 3 -g 24 -b:v 60M \ + -vf "${GRAIN},${P709}" "${TAGS_709[@]}" "${OUT}/bench_uhd_h264_grain.mov" + +echo "Done."