diff --git a/cmake/macros.cmake b/cmake/macros.cmake index d9a059fc8..65e317b10 100644 --- a/cmake/macros.cmake +++ b/cmake/macros.cmake @@ -77,6 +77,10 @@ if (BUILD_TESTING) PRIVATE TEST_RESOURCE=\"${TEST_RESOURCE}\" PRIVATE ROOT_DIR=\"${ROOT_DIR}\" PRIVATE $<$:XSTUDIO_DEBUG=1> + # Same platform defines as default_compile_options: the headers rely on + # __apple__ to #undef the 'nil' macro that MacTypes.h defines. + $<$:__apple__> + $<$:__OPENGL_4_1__> ) endmacro() endif (BUILD_TESTING) diff --git a/share/preference/plugin_media_reader_ffmpeg.json b/share/preference/plugin_media_reader_ffmpeg.json index 0281ac183..de33dfe1f 100644 --- a/share/preference/plugin_media_reader_ffmpeg.json +++ b/share/preference/plugin_media_reader_ffmpeg.json @@ -12,6 +12,17 @@ "datatype": "int", "context": ["APPLICATION"] }, + "videotoolbox": { + "path": "/plugin/media_reader/FFMPEG/videotoolbox", + "default_value": "prefer", + "description": "macOS only. Decode H.264, HEVC and ProRes video with Apple's VideoToolbox. 'prefer' falls back to FFmpeg's own decoder when VideoToolbox can't decode a stream; 'only' reports an error instead; 'off' always uses FFmpeg's decoder. Applies to media loaded after the change.", + "value": "prefer", + "datatype": "multichoice string", + "options": ["off", "prefer", "only"], + "context": ["APPLICATION"], + "category": "Playback", + "display_name": "VideoToolbox Decoding (macOS)" + }, "supported": { "path": "/plugin/media_reader/FFMPEG/supported", "description": "Control plugin format support", diff --git a/src/plugin/media_reader/ffmpeg/src/CMakeLists.txt b/src/plugin/media_reader/ffmpeg/src/CMakeLists.txt index 76a228264..add976ade 100644 --- a/src/plugin/media_reader/ffmpeg/src/CMakeLists.txt +++ b/src/plugin/media_reader/ffmpeg/src/CMakeLists.txt @@ -4,6 +4,7 @@ set(SOURCES ffmpeg_stream.cpp ffmpeg_decoder.cpp ffmpeg.cpp + vt_repack.cpp ) add_library(${PROJECT_NAME} SHARED ${SOURCES}) @@ -46,4 +47,17 @@ else() endif() +if(APPLE) + # VideoToolbox decode backend (see vt_decoder.hpp) + target_sources(${PROJECT_NAME} PRIVATE vt_decoder.cpp) + target_compile_definitions(${PROJECT_NAME} PRIVATE XSTUDIO_HAS_VIDEOTOOLBOX=1) + target_link_libraries(${PROJECT_NAME} + PRIVATE + "-framework VideoToolbox" + "-framework CoreMedia" + "-framework CoreVideo" + "-framework CoreFoundation" + ) +endif() + set_target_properties(${PROJECT_NAME} PROPERTIES LINK_DEPENDS_NO_SHARED true) diff --git a/src/plugin/media_reader/ffmpeg/src/ffmpeg.cpp b/src/plugin/media_reader/ffmpeg/src/ffmpeg.cpp index 0b00cfb30..37ad6e7c6 100644 --- a/src/plugin/media_reader/ffmpeg/src/ffmpeg.cpp +++ b/src/plugin/media_reader/ffmpeg/src/ffmpeg.cpp @@ -384,6 +384,7 @@ plugin_manager::PluginFactoryCollection *plugin_factory_collection_ptr() { FFMpegMediaReader::FFMpegMediaReader(const utility::JsonStore &prefs) : MediaReader("FFMPEG", prefs) { readers_per_source_ = 1; + videotoolbox_mode_ = VideoToolboxMode::Prefer; update_preferences(prefs); } @@ -398,6 +399,21 @@ void FFMpegMediaReader::update_preferences(const utility::JsonStore &prefs) { spdlog::warn("{} {}", __PRETTY_FUNCTION__, e.what()); } + try { + const auto mode = videotoolbox_mode_from_string( + preference_value(prefs, "/plugin/media_reader/FFMPEG/videotoolbox")); + if (mode != videotoolbox_mode_) { + // Don't hand out a frame decoded under the previous mode, and decode + // with the new one from the next request on. + last_decoded_image_.reset(); + decoder.reset(); + thumbnail_decoder.reset(); + } + videotoolbox_mode_ = mode; + } catch (const std::exception &e) { + spdlog::warn("{} {}", __PRETTY_FUNCTION__, e.what()); + } + try { readers_per_source_ = @@ -439,9 +455,14 @@ ImageBufPtr FFMpegMediaReader::image(const media::AVFrameID &mptr) { return last_decoded_image_; } - if (!decoder || decoder->path() != path || decoder->stream_id() != mptr.stream_id()) { + if (!decoder || decoder->path() != path || decoder->stream_id() != mptr.stream_id() || + decoder->videotoolbox_mode() != videotoolbox_mode_) { decoder.reset(new FFMpegDecoder( - path, soundcard_sample_rate_, default_rate_, mptr.stream_id())); + path, + soundcard_sample_rate_, + default_rate_, + mptr.stream_id(), + videotoolbox_mode_)); } decoder->decode_video_frame(mptr.frame(), rt); @@ -455,6 +476,11 @@ ImageBufPtr FFMpegMediaReader::image(const media::AVFrameID &mptr) { last_decoded_image_ = rt; } + } catch (const videotoolbox_unavailable_error &e) { + // Always shown on the frame: the user needs to know why there's no picture + // (e.g. the 'only' preference, or a build without FFmpeg's decoder). + rt = make_blank_image(); + rt->set_error(e.what()); } catch (std::exception &e) { rt = make_blank_image(); if (mptr.error() != "") { @@ -513,7 +539,8 @@ AudioBufPtr FFMpegMediaReader::audio(const media::AVFrameID &mptr) { xstudio::media::MediaDetail FFMpegMediaReader::detail(const caf::uri &uri) const { - FFMpegDecoder t_decoder(uri_convert(uri), soundcard_sample_rate_, default_rate_); + FFMpegDecoder t_decoder( + uri_convert(uri), soundcard_sample_rate_, default_rate_, "", videotoolbox_mode_); // N.B. MediaDetail needs frame duration, so invert frame rate std::vector streams; @@ -626,7 +653,11 @@ FFMpegMediaReader::thumbnail(const media::AVFrameID &mptr, const size_t thumb_si // DebugTimer d(path, mptr.frame()); if (!thumbnail_decoder || thumbnail_decoder->path() != path) { thumbnail_decoder.reset(new FFMpegDecoder( - path, soundcard_sample_rate_, default_rate_, mptr.stream_id())); + path, + soundcard_sample_rate_, + default_rate_, + mptr.stream_id(), + videotoolbox_mode_)); } std::shared_ptr rt = diff --git a/src/plugin/media_reader/ffmpeg/src/ffmpeg.hpp b/src/plugin/media_reader/ffmpeg/src/ffmpeg.hpp index cdd766458..529853e7e 100644 --- a/src/plugin/media_reader/ffmpeg/src/ffmpeg.hpp +++ b/src/plugin/media_reader/ffmpeg/src/ffmpeg.hpp @@ -10,6 +10,7 @@ namespace xstudio::media_reader { namespace ffmpeg { class FFMpegDecoder; + enum class VideoToolboxMode; } class FFMpegMediaReader : public MediaReader { @@ -53,6 +54,7 @@ class FFMpegMediaReader : public MediaReader { std::shared_ptr thumbnail_decoder; int readers_per_source_; + ffmpeg::VideoToolboxMode videotoolbox_mode_; int soundcard_sample_rate_ = {48000}; int channels_ = 2; utility::FrameRate default_rate_ = {utility::FrameRate(timebase::k_flicks_24fps)}; diff --git a/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.cpp b/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.cpp index 610dc03a8..a7851f901 100644 --- a/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.cpp +++ b/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.cpp @@ -30,7 +30,8 @@ FFMpegDecoder::FFMpegDecoder( std::string path, const int soundcard_sample_rate, const utility::FrameRate default_rate, - std::string stream_id) + std::string stream_id, + VideoToolboxMode videotoolbox_mode) : movie_file_path_(std::move(path)), last_requested_frame_(-100), last_decoded_frame_(-100), @@ -38,7 +39,8 @@ FFMpegDecoder::FFMpegDecoder( av_format_ctx_(nullptr), soundcard_sample_rate_(soundcard_sample_rate), stream_id_(std::move(stream_id)), - default_rate_(default_rate) + default_rate_(default_rate), + videotoolbox_mode_(videotoolbox_mode) { open_handles(); @@ -68,7 +70,8 @@ void FFMpegDecoder::open_handles() { av_format_ctx_->streams[i], i, ffmpeg_threads, - movie_file_path_); + movie_file_path_, + videotoolbox_mode_); } catch ([[maybe_unused]] std::exception &e) { } diff --git a/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.hpp b/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.hpp index 3d094e1a8..ab9cff376 100644 --- a/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.hpp +++ b/src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.hpp @@ -13,7 +13,8 @@ class FFMpegDecoder { std::string path, const int soundcard_sample_rate, const utility::FrameRate default_rate, - std::string stream_id = ""); + std::string stream_id = "", + VideoToolboxMode videotoolbox_mode = VideoToolboxMode::Off); ~FFMpegDecoder(); @@ -32,6 +33,7 @@ class FFMpegDecoder { const std::string &path() const { return movie_file_path_; } const std::string &stream_id() const { return stream_id_; } + VideoToolboxMode videotoolbox_mode() const { return videotoolbox_mode_; } int64_t duration_frames() const { return duration_frames_; } utility::FrameRate frame_rate(unsigned int stream_idx = UINT_MAX) const; utility::Timecode first_frame_timecode(); @@ -102,5 +104,6 @@ class FFMpegDecoder { int64_t duration_frames_; const std::string stream_id_; const utility::FrameRate default_rate_; + const VideoToolboxMode videotoolbox_mode_; }; } // namespace xstudio::media_reader::ffmpeg diff --git a/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.cpp b/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.cpp index 22b7892ec..b45f50ca6 100644 --- a/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.cpp +++ b/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.cpp @@ -6,6 +6,12 @@ #include "ffmpeg_stream.hpp" #include "xstudio/media/media_error.hpp" +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX +#include "vt_decoder.hpp" +#endif + +#include +#include #ifdef __GNUC__ // Check if GCC compiler is being used #pragma GCC diagnostic ignored "-Wdeprecated-declarations" @@ -259,6 +265,37 @@ static int setup_video_buffer(AVCodecContext *ctx, AVFrame *ffmpeg_frame_, int / namespace xstudio::media_reader::ffmpeg { +namespace { + // Each FFMpegDecoder (playback, thumbnails, media detail) opens its own streams, + // so to log a backend choice once per source rather than once per decoder, + // remember what has been logged. + bool first_time_for(const std::string &key) { + static std::mutex m; + static std::set seen; + std::lock_guard l(m); + return seen.insert(key).second; + } +} // namespace + +VideoToolboxMode videotoolbox_mode_from_string(const std::string &value) { + if (value == "off") + return VideoToolboxMode::Off; + if (value == "only") + return VideoToolboxMode::Only; + return VideoToolboxMode::Prefer; +} + +const char *to_string(const VideoToolboxMode mode) { + switch (mode) { + case VideoToolboxMode::Off: + return "off"; + case VideoToolboxMode::Only: + return "only"; + default: + return "prefer"; + } +} + // The SWS_CS_* coefficient set matching a stream's colourspace. Only needed where // sws does the YUV to RGB conversion itself - everywhere else our own shader does // it, from the matrix built in set_shader_pix_format_info. @@ -564,13 +601,13 @@ ImageBufPtr FFMpegStream::get_ffmpeg_frame_as_xstudio_image() { // See http://stash/projects/RND/repos/showsetup_data/pull-requests/794/overview const bool is_dneg_mov = utility::ends_with(format_context_->url, ".dneg.mov"); AVColorRange color_range = ffmpeg_frame_->color_range; - if (is_dneg_mov && codec_->id == AV_CODEC_ID_DNXHD) { + if (is_dneg_mov && avc_stream_->codecpar->codec_id == AV_CODEC_ID_DNXHD) { color_range = AVCOL_RANGE_MPEG; } set_shader_pix_format_info( jsn, - codec_->id, + avc_stream_->codecpar->codec_id, (AVPixelFormat)ffmpeg_pixel_format, color_range, ffmpeg_frame_->colorspace); @@ -771,15 +808,39 @@ AudioBufPtr FFMpegStream::get_ffmpeg_frame_as_xstudio_audio() { } FFMpegStream::FFMpegStream( - AVFormatContext *fmt_ctx, AVStream *stream, int index, int thread_count, std::string path) + AVFormatContext *fmt_ctx, + AVStream *stream, + int index, + int thread_count, + std::string path, + VideoToolboxMode videotoolbox_mode) : stream_index_(index), format_context_(fmt_ctx), avc_stream_(stream), - source_path(std::move(path)) { + source_path(std::move(path)), + videotoolbox_mode_(videotoolbox_mode) { codec_type_ = avc_stream_->codecpar->codec_type; ffmpeg_frame_ = av_frame_alloc(); - codec_ = avcodec_find_decoder(avc_stream_->codecpar->codec_id); + thread_count_ = thread_count; + + // VideoToolbox gets first refusal on video streams (design D7). FFmpeg's decoder + // is only looked for if VideoToolbox doesn't take the stream, so a build without + // one (e.g. no h264 decoder) still opens streams VideoToolbox decodes. + if (codec_type_ == AVMEDIA_TYPE_VIDEO) + try_videotoolbox(); + + if (!vt_decoder_ && videotoolbox_error_.empty()) + codec_ = avcodec_find_decoder(avc_stream_->codecpar->codec_id); + +#ifdef __APPLE__ + // AAC through AudioToolbox when FFmpeg was built with it (design D10) + if (codec_type_ == AVMEDIA_TYPE_AUDIO && + avc_stream_->codecpar->codec_id == AV_CODEC_ID_AAC) { + if (const AVCodec *aac_at = avcodec_find_decoder_by_name("aac_at")) + codec_ = aac_at; + } +#endif codec_context_ = avcodec_alloc_context3(codec_); @@ -808,28 +869,38 @@ FFMpegStream::FFMpegStream( #endif } - } else if (codec_type_ == AVMEDIA_TYPE_VIDEO && codec_) { + } else if ( + codec_type_ == AVMEDIA_TYPE_VIDEO && + (codec_ || vt_decoder_ || !videotoolbox_error_.empty())) { stream_type_ = VIDEO_STREAM; - if (codec_->capabilities & AV_CODEC_CAP_SLICE_THREADS) { - codec_context_->thread_count = thread_count; - codec_context_->thread_type = FF_THREAD_SLICE; - } - if (codec_->capabilities & AV_CODEC_CAP_FRAME_THREADS) { - codec_context_->thread_count = thread_count; - codec_context_->thread_type |= FF_THREAD_FRAME; - } - - /** initialize the stream parameters with demuxer information */ - AVC_CHECK_THROW( - avcodec_parameters_to_context(codec_context_, avc_stream_->codecpar), - "avcodec_parameters_to_context"); - AVC_CHECK_THROW(avcodec_open2(codec_context_, codec_, nullptr), "avcodec_open2"); + spdlog::debug( + "{} video stream {}: videotoolbox mode {}", + source_path, + stream_index_, + to_string(videotoolbox_mode_)); - ffmpeg_frame_->width = avc_stream_->codecpar->width; - ffmpeg_frame_->height = avc_stream_->codecpar->height; - ffmpeg_frame_->format = codec_context_->pix_fmt; + if (codec_) { + is_mjpeg = open_software_video_decoder(); + } else { + // VideoToolbox decodes the stream (or should have, and frame requests + // will report why it can't). The context only carries the stream + // parameters and the pixel format frames come out in. + AVC_CHECK_THROW( + avcodec_parameters_to_context(codec_context_, avc_stream_->codecpar), + "avcodec_parameters_to_context"); +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX + if (vt_decoder_) { + codec_context_->pix_fmt = vt_decoder_->output_pix_fmt(); + // frames arrive in an ImageBuffer, as with setup_video_buffer + using_own_frame_allocation = true; + } +#endif + ffmpeg_frame_->width = avc_stream_->codecpar->width; + ffmpeg_frame_->height = avc_stream_->codecpar->height; + ffmpeg_frame_->format = codec_context_->pix_fmt; + } // store resolution and pixel aspect resolution_ = Imath::V2f(avc_stream_->codecpar->width, avc_stream_->codecpar->height); @@ -838,21 +909,6 @@ FFMpegStream::FFMpegStream( pixel_aspect_ = float(sar.num) / float(sar.den); } - // Note - gif decoder has AV_CODEC_CAP_DR1 flag, but it doesn't appear to work, i.e. - // if xstudio takes over ffmpeg_frame_ buffer mem allocation decoding doesn't work - // correctly and we get repeated frames in the stream decode - if (strcmp(codec_->name, "gif") && (codec_->capabilities & AV_CODEC_CAP_DR1)) { - - // See Note 1 below - // codec_ allows us to allocated AVFrame buffers - codec_context_->get_buffer2 = setup_video_buffer; - using_own_frame_allocation = true; - if (!strcmp(codec_->name, "mjpeg")) { - is_mjpeg = true; - } - } - codec_context_->opaque = this; - } else if (codec_type_ == AVMEDIA_TYPE_AUDIO && codec_) { stream_type_ = AUDIO_STREAM; @@ -907,6 +963,130 @@ FFMpegStream::FFMpegStream( } } +bool FFMpegStream::open_software_video_decoder() { + + bool is_mjpeg = false; + + if (codec_->capabilities & AV_CODEC_CAP_SLICE_THREADS) { + codec_context_->thread_count = thread_count_; + codec_context_->thread_type = FF_THREAD_SLICE; + } + if (codec_->capabilities & AV_CODEC_CAP_FRAME_THREADS) { + codec_context_->thread_count = thread_count_; + codec_context_->thread_type |= FF_THREAD_FRAME; + } + + /** initialize the stream parameters with demuxer information */ + AVC_CHECK_THROW( + avcodec_parameters_to_context(codec_context_, avc_stream_->codecpar), + "avcodec_parameters_to_context"); + AVC_CHECK_THROW(avcodec_open2(codec_context_, codec_, nullptr), "avcodec_open2"); + + ffmpeg_frame_->width = avc_stream_->codecpar->width; + ffmpeg_frame_->height = avc_stream_->codecpar->height; + ffmpeg_frame_->format = codec_context_->pix_fmt; + + // Note - gif decoder has AV_CODEC_CAP_DR1 flag, but it doesn't appear to work, i.e. + // if xstudio takes over ffmpeg_frame_ buffer mem allocation decoding doesn't work + // correctly and we get repeated frames in the stream decode + if (strcmp(codec_->name, "gif") && (codec_->capabilities & AV_CODEC_CAP_DR1)) { + + // See Note 1 below + // codec_ allows us to allocated AVFrame buffers + codec_context_->get_buffer2 = setup_video_buffer; + using_own_frame_allocation = true; + if (!strcmp(codec_->name, "mjpeg")) { + is_mjpeg = true; + } + } else { + using_own_frame_allocation = false; + } + codec_context_->opaque = this; + + return is_mjpeg; +} + +void FFMpegStream::try_videotoolbox() { +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX + const char *codec_name = avcodec_get_name(avc_stream_->codecpar->codec_id); + + if (videotoolbox_mode_ == VideoToolboxMode::Off) { + if (first_time_for(source_path + "|ffmpeg|off")) + spdlog::info( + "{}: {} video decoded by FFmpeg (VideoToolbox is off)", + source_path, + codec_name); + return; + } + + std::string reason; + if (!VtDecoder::eligible(avc_stream_, reason)) { + if (first_time_for(source_path + "|ffmpeg")) + spdlog::info( + "{}: {} video decoded by FFmpeg ({})", source_path, codec_name, reason); + return; + } + + std::unique_ptr vt = VtDecoder::create(avc_stream_, reason); + if (vt) { + if (first_time_for(source_path + "|videotoolbox")) + spdlog::info( + "{}: {} video decoded by VideoToolbox ({}, {})", + source_path, + codec_name, + vt->output_format_name(), + vt->using_hardware() ? "hardware" : "software"); + vt_decoder_ = std::move(vt); + return; + } + + // VideoToolbox should have taken the stream but couldn't + const bool can_fall_back = videotoolbox_mode_ == VideoToolboxMode::Prefer && + avcodec_find_decoder(avc_stream_->codecpar->codec_id); + if (can_fall_back) { + if (first_time_for(source_path + "|fallback")) + spdlog::warn( + "{}: VideoToolbox can't decode this {} video ({}); using FFmpeg's decoder", + source_path, + codec_name, + reason); + } else { + videotoolbox_error_ = + fmt::format("VideoToolbox decoding unavailable for {}: {}", codec_name, reason); + if (first_time_for(source_path + "|failed")) + spdlog::warn("{}: {}", source_path, videotoolbox_error_); + } +#endif +} + +void FFMpegStream::fall_back_to_software(const std::string &reason) { + + vt_decoder_.reset(); + const char *codec_name = avcodec_get_name(avc_stream_->codecpar->codec_id); + const AVCodec *codec = videotoolbox_mode_ == VideoToolboxMode::Prefer + ? avcodec_find_decoder(avc_stream_->codecpar->codec_id) + : nullptr; + if (!codec) { + videotoolbox_error_ = + fmt::format("VideoToolbox decoding unavailable for {}: {}", codec_name, reason); + if (first_time_for(source_path + "|failed")) + spdlog::warn("{}: {}", source_path, videotoolbox_error_); + throw videotoolbox_unavailable_error(videotoolbox_error_); + } + + if (first_time_for(source_path + "|fallback")) + spdlog::warn( + "{}: VideoToolbox can't decode this {} video ({}); using FFmpeg's decoder", + source_path, + codec_name, + reason); + + codec_ = codec; + avcodec_free_context(&codec_context_); + codec_context_ = avcodec_alloc_context3(codec_); + open_software_video_decoder(); +} + FFMpegStream::~FFMpegStream() { if (audio_resampler_ctx_) @@ -1150,7 +1330,19 @@ int64_t FFMpegStream::receive_frame() { av_frame_unref(ffmpeg_frame_); - int rt = avcodec_receive_frame(codec_context_, ffmpeg_frame_); + if (!videotoolbox_error_.empty()) + throw videotoolbox_unavailable_error(videotoolbox_error_); + + int rt = 0; +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX + if (vt_decoder_) { + rt = vt_decoder_->receive_frame(ffmpeg_frame_); + // ProRes 4444 alpha is only known once the first frame header is seen + if (!rt) + codec_context_->pix_fmt = AVPixelFormat(ffmpeg_frame_->format); + } else +#endif + rt = avcodec_receive_frame(codec_context_, ffmpeg_frame_); // std::cerr << source_path << " WAS " << ffmpeg_frame_->pts << "\n"; @@ -1172,15 +1364,57 @@ int64_t FFMpegStream::receive_frame() { return rt; } -void FFMpegStream::send_flush_packet() { avcodec_send_packet(codec_context_, nullptr); } +void FFMpegStream::send_flush_packet() { +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX + if (vt_decoder_) { + vt_decoder_->send_packet(nullptr); + return; + } +#endif + if (videotoolbox_error_.empty()) + avcodec_send_packet(codec_context_, nullptr); +} int FFMpegStream::send_packet(AVPacket *avc_packet_) { + + if (!videotoolbox_error_.empty()) { + av_packet_unref(avc_packet_); + throw videotoolbox_unavailable_error(videotoolbox_error_); + } + +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX + if (vt_decoder_) { + int rt = vt_decoder_->send_packet(avc_packet_); + if (rt == AVERROR(ENOTSUP)) { + // e.g. packets without timestamps (design D8) + try { + fall_back_to_software("stream packets have no timestamps"); + } catch (...) { + av_packet_unref(avc_packet_); + throw; + } + } else { + av_packet_unref(avc_packet_); + return rt; + } + } +#endif + int rt = avcodec_send_packet(codec_context_, avc_packet_); av_packet_unref(avc_packet_); return rt; } -void FFMpegStream::flush_buffers() { avcodec_flush_buffers(codec_context_); } +void FFMpegStream::flush_buffers() { +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX + if (vt_decoder_) { + vt_decoder_->flush(); + return; + } +#endif + if (videotoolbox_error_.empty()) + avcodec_flush_buffers(codec_context_); +} double FFMpegStream::duration_seconds() const { diff --git a/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.hpp b/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.hpp index 4552476bc..a964b8255 100644 --- a/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.hpp +++ b/src/plugin/media_reader/ffmpeg/src/ffmpeg_stream.hpp @@ -59,6 +59,26 @@ AVPixelFormat shader_friendly_pix_format(AVPixelFormat src); where sws does the YUV to RGB conversion rather than our own shader. */ int sws_colourspace_from_av(AVColorSpace colorspace); +/* Value of the /plugin/media_reader/FFMPEG/videotoolbox preference. Has no effect +on platforms without VideoToolbox. */ +enum class VideoToolboxMode { Off, Prefer, Only }; + +/* Parses the preference value ("off", "prefer", "only"). Unrecognised values give +Prefer, the default. */ +VideoToolboxMode videotoolbox_mode_from_string(const std::string &value); + +const char *to_string(VideoToolboxMode mode); + +/* Thrown when a stream should have been decoded by VideoToolbox but can't be, and +falling back to FFmpeg's decoder isn't allowed or possible. The reader reports it +on the frame as an error. */ +class videotoolbox_unavailable_error : public std::runtime_error { + public: + using std::runtime_error::runtime_error; +}; + +class VtDecoder; + typedef enum { VIDEO_STREAM = 1, AUDIO_STREAM = 2, @@ -76,7 +96,8 @@ class FFMpegStream { AVStream *stream, int index, int thread_count, - std::string path); + std::string path, + VideoToolboxMode videotoolbox_mode = VideoToolboxMode::Off); virtual ~FFMpegStream(); @@ -126,6 +147,19 @@ class FFMpegStream { [[nodiscard]] bool is_attached_pic() const { return is_attached_pic_; } + /* True if VideoToolbox decodes this stream rather than FFmpeg. */ + [[nodiscard]] bool decoded_by_videotoolbox() const { return bool(vt_decoder_); } + + /* True if an FFmpeg decoder was opened for this stream. */ + [[nodiscard]] bool uses_ffmpeg_decoder() const { return codec_ != nullptr; } + + /* Name of the FFmpeg decoder opened for this stream, empty if none. */ + [[nodiscard]] std::string ffmpeg_decoder_name() const { return codec_ ? codec_->name : ""; } + + /* True once a frame of this stream has needed a CPU pixel format conversion + (swscale) before the shader could use it. */ + [[nodiscard]] bool converted_on_cpu() const { return format_conversion_warning_issued; } + [[nodiscard]] int stream_index() const { return stream_index_; } [[nodiscard]] bool is_drop_frame_timecode() const { return is_drop_frame_timecode_; } @@ -154,6 +188,18 @@ class FFMpegStream { void decode_attached_pic(); + // Sets up vt_decoder_, or videotoolbox_error_ if VideoToolbox should decode the + // stream but can't and FFmpeg mustn't (design D7). No-op without VideoToolbox. + void try_videotoolbox(); + + // Opens codec_ (FFmpeg's decoder) on codec_context_ for a video stream. + // Returns true for mjpeg. + bool open_software_video_decoder(); + + // VideoToolbox rejected the stream's packets: carry on with FFmpeg's decoder + // if allowed, otherwise throw videotoolbox_unavailable_error. + void fall_back_to_software(const std::string &reason); + // void setup_frame(ImageStorePtr & video_frame); int stream_index_; AVCodecContext *codec_context_{nullptr}; @@ -190,5 +236,12 @@ class FFMpegStream { utility::FrameRate frame_rate_; ImageBufPtr attached_pic_; + + VideoToolboxMode videotoolbox_mode_ = {VideoToolboxMode::Off}; + int thread_count_ = {0}; + // shared_ptr so that this header needs no complete VtDecoder (and no + // VideoToolbox) on any platform + std::shared_ptr vt_decoder_; + std::string videotoolbox_error_; }; } // namespace xstudio::media_reader::ffmpeg diff --git a/src/plugin/media_reader/ffmpeg/src/vt_decoder.cpp b/src/plugin/media_reader/ffmpeg/src/vt_decoder.cpp new file mode 100644 index 000000000..817e79bb9 --- /dev/null +++ b/src/plugin/media_reader/ffmpeg/src/vt_decoder.cpp @@ -0,0 +1,810 @@ +// SPDX-License-Identifier: Apache-2.0 + +// xstudio headers first: the Apple headers further down define a 'nil' macro that +// breaks caf headers included after them. +#include "vt_decoder.hpp" +#include "vt_repack.hpp" +#include "xstudio/media_reader/image_buffer.hpp" +#include "xstudio/utility/logging.hpp" + +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +} + +#include +#include +#include +#include + +namespace xstudio::media_reader::ffmpeg { + +namespace { + + // VideoToolbox output formats used here (design D4), and the planar format each + // is repacked to. + constexpr OSType ks4as = kCVPixelFormatType_444YpCbCr16VideoRange_16A_TriPlanar; // 's4as' + + enum class Packing { Bits8, Bits10InMsbs, Bits16, S4as }; + + struct OutputFormat { + OSType vt; + AVPixelFormat target; + Packing packing; + vt_repack::Chroma chroma; + bool full_range; + }; + + using vt_repack::Chroma; + const OutputFormat k_output_formats[] = { + {kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + AV_PIX_FMT_YUV420P, + Packing::Bits8, + Chroma::C420, + false}, + {kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + AV_PIX_FMT_YUV420P, + Packing::Bits8, + Chroma::C420, + true}, + {kCVPixelFormatType_422YpCbCr8BiPlanarVideoRange, + AV_PIX_FMT_YUV422P, + Packing::Bits8, + Chroma::C422, + false}, + {kCVPixelFormatType_422YpCbCr8BiPlanarFullRange, + AV_PIX_FMT_YUV422P, + Packing::Bits8, + Chroma::C422, + true}, + {kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange, + AV_PIX_FMT_YUV444P, + Packing::Bits8, + Chroma::C444, + false}, + {kCVPixelFormatType_444YpCbCr8BiPlanarFullRange, + AV_PIX_FMT_YUV444P, + Packing::Bits8, + Chroma::C444, + true}, + {kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, + AV_PIX_FMT_YUV420P10LE, + Packing::Bits10InMsbs, + Chroma::C420, + false}, + {kCVPixelFormatType_420YpCbCr10BiPlanarFullRange, + AV_PIX_FMT_YUV420P10LE, + Packing::Bits10InMsbs, + Chroma::C420, + true}, + {kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange, + AV_PIX_FMT_YUV422P10LE, + Packing::Bits10InMsbs, + Chroma::C422, + false}, + {kCVPixelFormatType_422YpCbCr10BiPlanarFullRange, + AV_PIX_FMT_YUV422P10LE, + Packing::Bits10InMsbs, + Chroma::C422, + true}, + {kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange, + AV_PIX_FMT_YUV444P10LE, + Packing::Bits10InMsbs, + Chroma::C444, + false}, + {kCVPixelFormatType_444YpCbCr10BiPlanarFullRange, + AV_PIX_FMT_YUV444P10LE, + Packing::Bits10InMsbs, + Chroma::C444, + true}, + {kCVPixelFormatType_422YpCbCr16BiPlanarVideoRange, + AV_PIX_FMT_YUV422P12LE, + Packing::Bits16, + Chroma::C422, + false}, + {kCVPixelFormatType_444YpCbCr16BiPlanarVideoRange, + AV_PIX_FMT_YUV444P12LE, + Packing::Bits16, + Chroma::C444, + false}, + // the target here is for 4444 with alpha; without, it's YUV444P12LE + {ks4as, AV_PIX_FMT_YUVA444P12LE, Packing::S4as, Chroma::C444, false}, + }; + + const OutputFormat *output_format(const OSType vt) { + for (const auto &f : k_output_formats) + if (f.vt == vt) + return &f; + return nullptr; + } + + // The VideoToolbox format for a bit depth, chroma format and range, 0 if none. + OSType + output_format_for(const Packing packing, const Chroma chroma, const bool full_range) { + for (const auto &f : k_output_formats) + if (f.packing == packing && f.chroma == chroma && f.full_range == full_range) + return f.vt; + return 0; + } + + std::string fourcc_string(const uint32_t f) { + std::string s; + for (int shift = 24; shift >= 0; shift -= 8) { + const char c = char((f >> shift) & 0xff); + s += (c >= 32 && c < 127) ? c : '?'; + } + return s; + } + + // ProRes codec_tag ('apch' etc., stored little-endian by FFmpeg) to the + // big-endian CMVideoCodecType VideoToolbox wants. 0 if it isn't a ProRes tag + // VideoToolbox decodes. + CMVideoCodecType prores_codec_type(const uint32_t tag) { + const uint32_t t = + ((tag & 0xff) << 24) | ((tag & 0xff00) << 8) | ((tag >> 8) & 0xff00) | (tag >> 24); + switch (t) { + case kCMVideoCodecType_AppleProRes422Proxy: + case kCMVideoCodecType_AppleProRes422LT: + case kCMVideoCodecType_AppleProRes422: + case kCMVideoCodecType_AppleProRes422HQ: + case kCMVideoCodecType_AppleProRes4444: + case kCMVideoCodecType_AppleProRes4444XQ: + return t; + default: + return 0; + } + } + + bool is_prores_4444(const CMVideoCodecType t) { + return t == kCMVideoCodecType_AppleProRes4444 || + t == kCMVideoCodecType_AppleProRes4444XQ; + } + + struct PictureFormat { + int bit_depth = 0; + int chroma = 0; // 420, 422 or 444 + }; + + // Bit depth and chroma format of an H.264 stream, from the avcC record. The + // High profile extension that carries them is optional; without it the + // profile tells us whether it can be anything but 8 bit 4:2:0. + bool avcc_picture_format(const uint8_t *d, const int size, PictureFormat &fmt) { + if (size < 7 || d[0] != 1) + return false; + const int profile = d[1]; + int pos = 5; + const int num_sps = d[pos++] & 0x1f; + for (int i = 0; i < num_sps; ++i) { + if (pos + 2 > size) + return false; + pos += 2 + ((d[pos] << 8) | d[pos + 1]); + } + if (pos + 1 > size) + return false; + const int num_pps = d[pos++]; + for (int i = 0; i < num_pps; ++i) { + if (pos + 2 > size) + return false; + pos += 2 + ((d[pos] << 8) | d[pos + 1]); + } + if (pos > size) + return false; + + const bool high_profile = profile == 100 || profile == 110 || profile == 122 || + profile == 144 || profile == 244; + if (high_profile && pos + 3 <= size) { + static const int chroma_formats[] = {400, 420, 422, 444}; + fmt.chroma = chroma_formats[d[pos] & 3]; + fmt.bit_depth = 8 + (d[pos + 1] & 7); + return true; + } + if (profile == 100 || !high_profile) { + // Baseline/Main/Extended, and High (not High 10/4:2:2/4:4:4), are + // always 8 bit 4:2:0. + fmt = {8, 420}; + return true; + } + return false; + } + + // Bit depth and chroma format of an HEVC stream, from the hvcC record header. + bool hvcc_picture_format(const uint8_t *d, const int size, PictureFormat &fmt) { + if (size < 23 || d[0] != 1) + return false; + static const int chroma_formats[] = {400, 420, 422, 444}; + fmt.chroma = chroma_formats[d[16] & 3]; + fmt.bit_depth = 8 + (d[17] & 7); + return true; + } + + PictureFormat picture_format_from_pix_fmt(const int format) { + const AVPixFmtDescriptor *d = av_pix_fmt_desc_get(AVPixelFormat(format)); + if (!d || (d->flags & AV_PIX_FMT_FLAG_RGB)) + return {}; + PictureFormat fmt; + fmt.bit_depth = d->comp[0].depth; + fmt.chroma = d->log2_chroma_w == 0 ? 444 : (d->log2_chroma_h ? 420 : 422); + return fmt; + } + + struct Choice { + CMVideoCodecType codec_type = 0; + OSType output_format = 0; + }; + + // Decides whether VideoToolbox should decode the stream, and with which output + // format. Returns false with 'reason' set if not. + bool choose(const AVStream *stream, Choice &choice, std::string &reason) { + + const AVCodecParameters *par = stream->codecpar; + if (par->codec_type != AVMEDIA_TYPE_VIDEO) { + reason = "not a video stream"; + return false; + } + if (par->width <= 0 || par->height <= 0) { + reason = "unknown picture size"; + return false; + } + if (stream->disposition & AV_DISPOSITION_ATTACHED_PIC) { + reason = "attached picture"; + return false; + } + + const bool full_range = par->color_range == AVCOL_RANGE_JPEG; + + if (par->codec_id == AV_CODEC_ID_PRORES) { + choice.codec_type = prores_codec_type(par->codec_tag); + if (!choice.codec_type) { + reason = "unrecognised ProRes codec tag"; + return false; + } + choice.output_format = is_prores_4444(choice.codec_type) + ? ks4as + : kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange; + return true; + } + + if (par->codec_id != AV_CODEC_ID_H264 && par->codec_id != AV_CODEC_ID_HEVC) { + reason = std::string("codec ") + avcodec_get_name(par->codec_id) + + " is not decoded with VideoToolbox"; + return false; + } + + // A length-prefixed stream carries an avcC/hvcC record (first byte 1). An + // Annex-B stream (raw .h264/.ts) has start codes instead, or nothing. + if (par->extradata_size < 7 || par->extradata[0] != 1) { + reason = "no avcC/hvcC decoder configuration (Annex-B stream)"; + return false; + } + + PictureFormat fmt; + const bool parsed = par->codec_id == AV_CODEC_ID_H264 + ? avcc_picture_format(par->extradata, par->extradata_size, fmt) + : hvcc_picture_format(par->extradata, par->extradata_size, fmt); + if (!parsed) + fmt = picture_format_from_pix_fmt(par->format); + + choice.codec_type = + par->codec_id == AV_CODEC_ID_H264 ? kCMVideoCodecType_H264 : kCMVideoCodecType_HEVC; + const char *codec = par->codec_id == AV_CODEC_ID_H264 ? "H.264" : "HEVC"; + + Chroma chroma; + switch (fmt.chroma) { + case 420: + chroma = Chroma::C420; + break; + case 422: + chroma = Chroma::C422; + break; + case 444: + chroma = Chroma::C444; + break; + default: + reason = + fmt::format("{} with chroma format {} (e.g. monochrome)", codec, fmt.chroma); + return false; + } + + if (fmt.bit_depth == 8) + choice.output_format = output_format_for(Packing::Bits8, chroma, full_range); + else if (fmt.bit_depth == 10) + choice.output_format = output_format_for(Packing::Bits10InMsbs, chroma, full_range); + else if (fmt.bit_depth == 12 && par->codec_id == AV_CODEC_ID_HEVC) { + // VideoToolbox decodes these in software. Its 12 bit 4:2:0 chroma doesn't + // match FFmpeg's and is further from the source (design D4, spike 1.8), + // and there are no full range 16 bit formats to ask for. + if (chroma == Chroma::C420) { + reason = "12 bit 4:2:0 HEVC (VideoToolbox's chroma is inaccurate)"; + return false; + } + if (full_range) { + reason = "full range 12 bit HEVC"; + return false; + } + choice.output_format = output_format_for(Packing::Bits16, chroma, false); + } + + if (!choice.output_format) { + reason = fmt::format("{} at {} bit {}", codec, fmt.bit_depth, fmt.chroma); + return false; + } + return true; + } + + // Does the first ProRes frame carry alpha? From the frame header, the same + // way FFmpeg's decoder decides between yuv444p12 and yuva444p12. + bool prores_frame_has_alpha(const AVPacket *pkt) { + // 4 byte frame size, 'icpf', then the frame header; alpha_info is the low + // nibble of header byte 17. + if (!pkt || pkt->size < 8 + 18) + return false; + if (memcmp(pkt->data + 4, "icpf", 4)) + return false; + return (pkt->data[8 + 17] & 0x0f) != 0; + } + + AVPixelFormat target_pix_fmt(const OSType vt, const bool alpha) { + const OutputFormat *f = output_format(vt); + if (!f) + return AV_PIX_FMT_NONE; + if (f->packing == Packing::S4as && !alpha) + return AV_PIX_FMT_YUV444P12LE; + return f->target; + } + + bool cf_equal(CFTypeRef value, CFStringRef key) { + return value && CFGetTypeID(value) == CFStringGetTypeID() && + CFStringCompare((CFStringRef)value, key, 0) == kCFCompareEqualTo; + } + + // Colour metadata VideoToolbox attached to a decoded buffer, for fields the + // container left unspecified (design D6). + void colour_from_attachments(CVPixelBufferRef pb, AVFrame *frame) { + if (frame->colorspace == AVCOL_SPC_UNSPECIFIED) { + CFTypeRef m = CVBufferCopyAttachment(pb, kCVImageBufferYCbCrMatrixKey, nullptr); + if (cf_equal(m, kCVImageBufferYCbCrMatrix_ITU_R_709_2)) + frame->colorspace = AVCOL_SPC_BT709; + else if (cf_equal(m, kCVImageBufferYCbCrMatrix_ITU_R_601_4)) + frame->colorspace = AVCOL_SPC_SMPTE170M; + else if (cf_equal(m, kCVImageBufferYCbCrMatrix_ITU_R_2020)) + frame->colorspace = AVCOL_SPC_BT2020_NCL; + else if (cf_equal(m, kCVImageBufferYCbCrMatrix_SMPTE_240M_1995)) + frame->colorspace = AVCOL_SPC_SMPTE240M; + if (m) + CFRelease(m); + } + if (frame->color_primaries == AVCOL_PRI_UNSPECIFIED) { + CFTypeRef p = CVBufferCopyAttachment(pb, kCVImageBufferColorPrimariesKey, nullptr); + if (cf_equal(p, kCVImageBufferColorPrimaries_ITU_R_709_2)) + frame->color_primaries = AVCOL_PRI_BT709; + else if (cf_equal(p, kCVImageBufferColorPrimaries_ITU_R_2020)) + frame->color_primaries = AVCOL_PRI_BT2020; + else if (cf_equal(p, kCVImageBufferColorPrimaries_P3_D65)) + frame->color_primaries = AVCOL_PRI_SMPTE432; + else if (cf_equal(p, kCVImageBufferColorPrimaries_DCI_P3)) + frame->color_primaries = AVCOL_PRI_SMPTE431; + else if (cf_equal(p, kCVImageBufferColorPrimaries_EBU_3213)) + frame->color_primaries = AVCOL_PRI_BT470BG; + else if (cf_equal(p, kCVImageBufferColorPrimaries_SMPTE_C)) + frame->color_primaries = AVCOL_PRI_SMPTE170M; + if (p) + CFRelease(p); + } + if (frame->color_trc == AVCOL_TRC_UNSPECIFIED) { + CFTypeRef t = + CVBufferCopyAttachment(pb, kCVImageBufferTransferFunctionKey, nullptr); + if (cf_equal(t, kCVImageBufferTransferFunction_ITU_R_709_2)) + frame->color_trc = AVCOL_TRC_BT709; + else if (cf_equal(t, kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ)) + frame->color_trc = AVCOL_TRC_SMPTE2084; + else if (cf_equal(t, kCVImageBufferTransferFunction_ITU_R_2100_HLG)) + frame->color_trc = AVCOL_TRC_ARIB_STD_B67; + else if (cf_equal(t, kCVImageBufferTransferFunction_ITU_R_2020)) + frame->color_trc = AVCOL_TRC_BT2020_10; + else if (cf_equal(t, kCVImageBufferTransferFunction_SMPTE_240M_1995)) + frame->color_trc = AVCOL_TRC_SMPTE240M; + if (t) + CFRelease(t); + } + } + + // Same ownership trick as setup_video_buffer() in ffmpeg_stream.cpp: the + // AVBufferRef in frame->buf[0] owns a heap-allocated ImageBufPtr that keeps the + // pixel memory alive and lets FFMpegStream take the ImageBuffer over. + void image_buf_ptr_deleter(void *, uint8_t *data) { delete (ImageBufPtr *)data; } + +} // namespace + +struct VtDecoder::Impl { + VTDecompressionSessionRef session = nullptr; + CMVideoFormatDescriptionRef format = nullptr; + Choice choice; + AVRational time_base = {1, 1}; + int width = 0; + int height = 0; + bool full_range = false; + bool has_alpha = false; + bool first_packet = true; + bool hardware = false; + AVColorSpace colorspace; + AVColorPrimaries primaries; + AVColorTransferCharacteristic trc; + + std::mutex mutex; + // Presentation timestamps of packets submitted that VideoToolbox hasn't + // called back for yet + std::set pending; + // Frames VideoToolbox has returned, by presentation timestamp + std::map decoded; + // Highest decode timestamp submitted. Every later packet has pts >= its + // dts >= this, so a decoded frame with pts <= max_dts can't be preceded by + // anything still to come (design D8). + int64_t max_dts = std::numeric_limits::min(); + bool draining = false; + + ~Impl() { + clear_decoded(); + if (session) { + VTDecompressionSessionWaitForAsynchronousFrames(session); + VTDecompressionSessionInvalidate(session); + CFRelease(session); + } + if (format) + CFRelease(format); + } + + void clear_decoded() { + std::lock_guard l(mutex); + for (auto &p : decoded) + CVPixelBufferRelease(p.second); + decoded.clear(); + pending.clear(); + max_dts = std::numeric_limits::min(); + } + + static void output_callback( + void *refcon, + void *source_frame_refcon, + OSStatus status, + VTDecodeInfoFlags flags, + CVImageBufferRef image, + CMTime, + CMTime) { + auto *self = static_cast(refcon); + const int64_t pts = int64_t(reinterpret_cast(source_frame_refcon)); + std::lock_guard l(self->mutex); + self->pending.erase(pts); + if (status != noErr || (flags & kVTDecodeInfo_FrameDropped) || !image) { + // Undecodable (e.g. an open-GOP leading picture after a seek); it is + // simply never output. + return; + } + auto it = self->decoded.find(pts); + if (it != self->decoded.end()) + CVPixelBufferRelease(it->second); + self->decoded[pts] = CVPixelBufferRetain(image); + } +}; + +VtDecoder::VtDecoder(std::unique_ptr impl) : impl_(std::move(impl)) {} + +VtDecoder::~VtDecoder() = default; + +bool VtDecoder::eligible(const AVStream *stream, std::string &reason) { + Choice choice; + return choose(stream, choice, reason); +} + +std::unique_ptr VtDecoder::create(const AVStream *stream, std::string &error) { + + auto impl = std::make_unique(); + if (!choose(stream, impl->choice, error)) + return nullptr; + + // Debugging aid, and how the fallback/failure paths are tested: pretend + // VideoToolbox can't create a session. + if (getenv("XSTUDIO_VIDEOTOOLBOX_FORCE_FAILURE")) { + error = "session creation disabled by XSTUDIO_VIDEOTOOLBOX_FORCE_FAILURE"; + return nullptr; + } + + const AVCodecParameters *par = stream->codecpar; + impl->time_base = stream->time_base; + impl->width = par->width; + impl->height = par->height; + impl->full_range = par->color_range == AVCOL_RANGE_JPEG; + impl->colorspace = par->color_space; + impl->primaries = par->color_primaries; + impl->trc = par->color_trc; + // Until the first ProRes frame header says otherwise, 4444 is assumed to have alpha. + impl->has_alpha = impl->choice.output_format == ks4as; + + CFMutableDictionaryRef extensions = CFDictionaryCreateMutable( + nullptr, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + if (par->codec_id == AV_CODEC_ID_H264 || par->codec_id == AV_CODEC_ID_HEVC) { + CFMutableDictionaryRef atoms = CFDictionaryCreateMutable( + nullptr, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + CFDataRef data = CFDataCreate(nullptr, par->extradata, par->extradata_size); + CFDictionarySetValue( + atoms, par->codec_id == AV_CODEC_ID_H264 ? CFSTR("avcC") : CFSTR("hvcC"), data); + CFDictionarySetValue( + extensions, kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms, atoms); + CFRelease(data); + CFRelease(atoms); + } + OSStatus status = CMVideoFormatDescriptionCreate( + nullptr, impl->choice.codec_type, par->width, par->height, extensions, &impl->format); + CFRelease(extensions); + if (status != noErr) { + error = fmt::format("CMVideoFormatDescriptionCreate failed ({})", int(status)); + return nullptr; + } + + CFMutableDictionaryRef spec = CFDictionaryCreateMutable( + nullptr, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + // Use hardware when there is some; VideoToolbox falls back to its own software + // decoder (e.g. ProRes on Intel) otherwise. + CFDictionarySetValue( + spec, + kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder, + kCFBooleanTrue); + + CFMutableDictionaryRef attributes = CFDictionaryCreateMutable( + nullptr, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + const SInt32 pixel_format = SInt32(impl->choice.output_format); + CFNumberRef number = CFNumberCreate(nullptr, kCFNumberSInt32Type, &pixel_format); + CFDictionarySetValue(attributes, kCVPixelBufferPixelFormatTypeKey, number); + CFRelease(number); + + VTDecompressionOutputCallbackRecord callback{&Impl::output_callback, impl.get()}; + status = VTDecompressionSessionCreate( + nullptr, impl->format, spec, attributes, &callback, &impl->session); + CFRelease(spec); + CFRelease(attributes); + if (status != noErr) { + impl->session = nullptr; + error = fmt::format("VTDecompressionSessionCreate failed ({})", int(status)); + return nullptr; + } + + CFBooleanRef hw = nullptr; + if (VTSessionCopyProperty( + impl->session, + kVTDecompressionPropertyKey_UsingHardwareAcceleratedVideoDecoder, + nullptr, + &hw) == noErr && + hw) { + impl->hardware = hw == kCFBooleanTrue; + CFRelease(hw); + } + + return std::unique_ptr(new VtDecoder(std::move(impl))); +} + +int VtDecoder::send_packet(const AVPacket *packet) { + + Impl &d = *impl_; + + if (!packet || !packet->size) { + // Start draining: everything submitted so far comes out of receive_frame, + // then AVERROR_EOF. + VTDecompressionSessionFinishDelayedFrames(d.session); + VTDecompressionSessionWaitForAsynchronousFrames(d.session); + std::lock_guard l(d.mutex); + d.draining = true; + return 0; + } + + { + std::lock_guard l(d.mutex); + if (d.draining) + return AVERROR_EOF; + } + + // Reordering relies on every packet having both timestamps (D8). + if (packet->pts == AV_NOPTS_VALUE || packet->dts == AV_NOPTS_VALUE) + return AVERROR(ENOTSUP); + + if (d.first_packet) { + d.first_packet = false; + if (d.choice.output_format == ks4as) + d.has_alpha = prores_frame_has_alpha(packet); + } + + // Wrap the packet data without copying: decoding is synchronous (D9), so it is + // finished with before this returns. + CMBlockBufferRef block = nullptr; + OSStatus status = CMBlockBufferCreateWithMemoryBlock( + nullptr, + packet->data, + size_t(packet->size), + kCFAllocatorNull, + nullptr, + 0, + size_t(packet->size), + 0, + &block); + if (status != noErr) + return AVERROR(ENOMEM); + + const AVRational tb = d.time_base; + CMSampleTimingInfo timing; + timing.duration = + packet->duration > 0 ? CMTimeMake(packet->duration * tb.num, tb.den) : kCMTimeInvalid; + timing.presentationTimeStamp = CMTimeMake(packet->pts * tb.num, tb.den); + timing.decodeTimeStamp = CMTimeMake(packet->dts * tb.num, tb.den); + const size_t sample_size = size_t(packet->size); + + CMSampleBufferRef sample = nullptr; + status = CMSampleBufferCreateReady( + nullptr, block, d.format, 1, 1, &timing, 1, &sample_size, &sample); + CFRelease(block); + if (status != noErr) + return AVERROR(ENOMEM); + + { + std::lock_guard l(d.mutex); + d.pending.insert(packet->pts); + d.max_dts = std::max(d.max_dts, packet->dts); + } + + VTDecodeInfoFlags info = 0; + status = VTDecompressionSessionDecodeFrame( + d.session, + sample, + 0, // synchronous + reinterpret_cast(intptr_t(packet->pts)), + &info); + CFRelease(sample); + + if (status != noErr) { + // VideoToolbox may or may not have called back with the error; either way + // this frame won't be output. + std::lock_guard l(d.mutex); + d.pending.erase(packet->pts); + if (status == kVTInvalidSessionErr) + return AVERROR_EXTERNAL; + spdlog::debug( + "VideoToolbox could not decode packet at pts {}: {}", packet->pts, int(status)); + } + return 0; +} + +int VtDecoder::receive_frame(AVFrame *frame) { + + Impl &d = *impl_; + CVPixelBufferRef pb = nullptr; + int64_t pts = 0; + { + std::lock_guard l(d.mutex); + if (d.decoded.empty()) + return d.draining ? AVERROR_EOF : AVERROR(EAGAIN); + + // The earliest decoded frame can go out once no frame with an earlier + // timestamp can still turn up: none is in flight, and no packet yet to be + // sent can have one (pts <= max_dts). + auto first = d.decoded.begin(); + if (!d.draining && (first->first > d.max_dts || + (!d.pending.empty() && first->first > *d.pending.begin()))) + return AVERROR(EAGAIN); + + pts = first->first; + pb = first->second; + d.decoded.erase(first); + } + + const AVPixelFormat target = target_pix_fmt(d.choice.output_format, d.has_alpha); + const int w = int(CVPixelBufferGetWidth(pb)); + const int h = int(CVPixelBufferGetHeight(pb)); + const vt_repack::Layout layout = vt_repack::planar_layout(target, w, h); + + ImageBufPtr image(new ImageBuffer()); + auto *base = reinterpret_cast(image->allocate(layout.total_size + 4096)); + + auto dst = [&](const int i) -> vt_repack::DstPlane { + return {base + layout.offsets[i], layout.linesizes[i]}; + }; + + CVPixelBufferLockBaseAddress(pb, kCVPixelBufferLock_ReadOnly); + auto src = [&](const size_t i) -> vt_repack::SrcPlane { + return { + static_cast(CVPixelBufferGetBaseAddressOfPlane(pb, i)), + CVPixelBufferGetBytesPerRowOfPlane(pb, i)}; + }; + + const OSType got = CVPixelBufferGetPixelFormatType(pb); + const OutputFormat *of = got == d.choice.output_format ? output_format(got) : nullptr; + const bool ok = of != nullptr; + if (of) { + switch (of->packing) { + case Packing::Bits8: + vt_repack::biplanar8_to_planar8( + src(0), src(1), dst(0), dst(1), dst(2), w, h, of->chroma); + break; + case Packing::Bits10InMsbs: + vt_repack::biplanar10_to_planar10( + src(0), src(1), dst(0), dst(1), dst(2), w, h, of->chroma); + break; + case Packing::Bits16: + vt_repack::biplanar16_to_planar12( + src(0), src(1), dst(0), dst(1), dst(2), w, h, of->chroma); + break; + case Packing::S4as: + vt_repack::s4as_to_yuva444p12( + src(0), + src(1), + src(2), + dst(0), + dst(1), + dst(2), + d.has_alpha ? dst(3) : vt_repack::DstPlane{nullptr, 0}, + w, + h); + break; + } + } + + if (ok) { + av_frame_unref(frame); + frame->format = target; + frame->width = w; + frame->height = h; + frame->pts = pts; + frame->best_effort_timestamp = pts; + frame->color_range = of->full_range ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG; + frame->colorspace = d.colorspace; + frame->color_primaries = d.primaries; + frame->color_trc = d.trc; + colour_from_attachments(pb, frame); + + for (int i = 0; i < layout.planes; ++i) { + frame->data[i] = base + layout.offsets[i]; + frame->linesize[i] = int(layout.linesizes[i]); + } + frame->extended_data = frame->data; + frame->buf[0] = av_buffer_create( + reinterpret_cast(new ImageBufPtr(image)), + sizeof(ImageBufPtr), + image_buf_ptr_deleter, + nullptr, + 0); + } + + CVPixelBufferUnlockBaseAddress(pb, kCVPixelBufferLock_ReadOnly); + CVPixelBufferRelease(pb); + + if (!ok) { + spdlog::warn( + "VideoToolbox returned pixel format {}, expected {}", + fourcc_string(got), + fourcc_string(d.choice.output_format)); + return AVERROR_EXTERNAL; + } + return 0; +} + +void VtDecoder::flush() { + if (impl_->session) + VTDecompressionSessionWaitForAsynchronousFrames(impl_->session); + impl_->clear_decoded(); + std::lock_guard l(impl_->mutex); + impl_->draining = false; +} + +AVPixelFormat VtDecoder::output_pix_fmt() const { + return target_pix_fmt(impl_->choice.output_format, impl_->has_alpha); +} + +std::string VtDecoder::output_format_name() const { + return fourcc_string(impl_->choice.output_format); +} + +bool VtDecoder::using_hardware() const { return impl_->hardware; } + +} // namespace xstudio::media_reader::ffmpeg diff --git a/src/plugin/media_reader/ffmpeg/src/vt_decoder.hpp b/src/plugin/media_reader/ffmpeg/src/vt_decoder.hpp new file mode 100644 index 000000000..5797fcbc7 --- /dev/null +++ b/src/plugin/media_reader/ffmpeg/src/vt_decoder.hpp @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +// VideoToolbox decode backend for FFMpegStream (macOS only). +// +// This header deliberately includes no Apple headers: MacTypes.h defines a 'nil' +// macro that breaks caf, and this header is included alongside caf ones. + +#include +#include + +extern "C" { +#include +#include +} + +namespace xstudio::media_reader::ffmpeg { + +/* Decodes one video stream with a VideoToolbox decompression session. + +The send_packet / receive_frame / flush methods follow the same rules as +avcodec_send_packet / avcodec_receive_frame / avcodec_flush_buffers, return codes +included, so FFMpegStream can use it in place of an AVCodecContext. + +Frames come out in presentation order, in a planar YUV(A) format the xstudio +shader reads directly. Their pixel data lives in an xstudio ImageBuffer, which is +hidden in frame->buf[0] the same way setup_video_buffer() in ffmpeg_stream.cpp +does it. */ +class VtDecoder { + public: + /* Whether the stream's codec and decoder configuration can go to VideoToolbox. + If not, 'reason' says why. */ + static bool eligible(const AVStream *stream, std::string &reason); + + /* Creates a decoder for the stream. Returns null and sets 'error' if + VideoToolbox can't create a session for it. Setting the environment variable + XSTUDIO_VIDEOTOOLBOX_FORCE_FAILURE makes it always fail, for testing the + fallback paths. */ + static std::unique_ptr create(const AVStream *stream, std::string &error); + + ~VtDecoder(); + + VtDecoder(const VtDecoder &) = delete; + VtDecoder &operator=(const VtDecoder &) = delete; + + /* A null packet starts draining, as with avcodec_send_packet. Returns 0 or a + negative AVERROR. The packet isn't unreferenced. */ + int send_packet(const AVPacket *packet); + + /* Returns 0 with 'frame' filled, AVERROR(EAGAIN) if more packets are needed, + or AVERROR_EOF once fully drained. */ + int receive_frame(AVFrame *frame); + + /* Drops every frame in flight, e.g. before a seek. */ + void flush(); + + /* The planar pixel format receive_frame() produces. */ + [[nodiscard]] AVPixelFormat output_pix_fmt() const; + + /* VideoToolbox's name for the session's output format, for logging. */ + [[nodiscard]] std::string output_format_name() const; + + [[nodiscard]] bool using_hardware() const; + + private: + struct Impl; + explicit VtDecoder(std::unique_ptr impl); + std::unique_ptr impl_; +}; + +} // namespace xstudio::media_reader::ffmpeg diff --git a/src/plugin/media_reader/ffmpeg/src/vt_repack.cpp b/src/plugin/media_reader/ffmpeg/src/vt_repack.cpp new file mode 100644 index 000000000..f8ca53891 --- /dev/null +++ b/src/plugin/media_reader/ffmpeg/src/vt_repack.cpp @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: Apache-2.0 +#include "vt_repack.hpp" + +#include + +extern "C" { +#include +#include +} + +namespace xstudio::media_reader::ffmpeg::vt_repack { + +namespace { + // The loops below are kept simple (contiguous rows, no branches) so the compiler + // can vectorise them. + + template inline const T *src_row(const SrcPlane &p, const int y) { + return reinterpret_cast(p.data + p.stride * size_t(y)); + } + + template inline T *dst_row(const DstPlane &p, const int y) { + return reinterpret_cast(p.data + p.stride * size_t(y)); + } +} // namespace + +Layout planar_layout(const AVPixelFormat format, const int width, const int height) { + + Layout layout; + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(format); + if (!desc || !(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) + return layout; + + std::array linesizes = {0, 0, 0, 0}; + if (av_image_fill_linesizes(linesizes.data(), format, width) < 0) + return layout; + + std::array aligned = {0, 0, 0, 0}; + for (int i = 0; i < 4; ++i) + aligned[i] = FFALIGN(linesizes[i], 64); + + std::array plane_sizes = {0, 0, 0, 0}; + if (av_image_fill_plane_sizes(plane_sizes.data(), format, height, aligned.data()) < 0) + return layout; + + layout.format = format; + for (int i = 0; i < 4; ++i) { + if (!plane_sizes[i]) + break; + layout.offsets[i] = layout.total_size; + layout.linesizes[i] = size_t(aligned[i]); + layout.total_size += plane_sizes[i]; + layout.planes++; + } + return layout; +} + +void biplanar8_to_planar8( + const SrcPlane y, + const SrcPlane cbcr, + const DstPlane dst_y, + const DstPlane dst_u, + const DstPlane dst_v, + const int width, + const int height, + const Chroma chroma) { + + for (int row = 0; row < height; ++row) + std::memcpy(dst_y.data + dst_y.stride * row, y.data + y.stride * row, size_t(width)); + + const int cw = chroma_width(width, chroma); + const int ch = chroma_height(height, chroma); + for (int row = 0; row < ch; ++row) { + const uint8_t *__restrict in = src_row(cbcr, row); + uint8_t *__restrict u = dst_row(dst_u, row); + uint8_t *__restrict v = dst_row(dst_v, row); + for (int x = 0; x < cw; ++x) { + u[x] = in[2 * x]; + v[x] = in[2 * x + 1]; + } + } +} + +void biplanar10_to_planar10( + const SrcPlane y, + const SrcPlane cbcr, + const DstPlane dst_y, + const DstPlane dst_u, + const DstPlane dst_v, + const int width, + const int height, + const Chroma chroma) { + + for (int row = 0; row < height; ++row) { + const uint16_t *__restrict in = src_row(y, row); + uint16_t *__restrict out = dst_row(dst_y, row); + for (int x = 0; x < width; ++x) + out[x] = in[x] >> 6; + } + + const int cw = chroma_width(width, chroma); + const int ch = chroma_height(height, chroma); + for (int row = 0; row < ch; ++row) { + const uint16_t *__restrict in = src_row(cbcr, row); + uint16_t *__restrict u = dst_row(dst_u, row); + uint16_t *__restrict v = dst_row(dst_v, row); + for (int x = 0; x < cw; ++x) { + u[x] = in[2 * x] >> 6; + v[x] = in[2 * x + 1] >> 6; + } + } +} + +void biplanar16_to_planar12( + const SrcPlane y, + const SrcPlane cbcr, + const DstPlane dst_y, + const DstPlane dst_u, + const DstPlane dst_v, + const int width, + const int height, + const Chroma chroma) { + + for (int row = 0; row < height; ++row) { + const uint16_t *__restrict in = src_row(y, row); + uint16_t *__restrict out = dst_row(dst_y, row); + for (int x = 0; x < width; ++x) + out[x] = round_16_to_12(in[x]); + } + + const int cw = chroma_width(width, chroma); + const int ch = chroma_height(height, chroma); + for (int row = 0; row < ch; ++row) { + const uint16_t *__restrict in = src_row(cbcr, row); + uint16_t *__restrict u = dst_row(dst_u, row); + uint16_t *__restrict v = dst_row(dst_v, row); + for (int x = 0; x < cw; ++x) { + u[x] = round_16_to_12(in[2 * x]); + v[x] = round_16_to_12(in[2 * x + 1]); + } + } +} + +void s4as_to_yuva444p12( + const SrcPlane y, + const SrcPlane cbcr, + const SrcPlane a, + const DstPlane dst_y, + const DstPlane dst_u, + const DstPlane dst_v, + const DstPlane dst_a, + const int width, + const int height) { + + if (!dst_a.data) { + // no alpha wanted: the same as sv44 + biplanar16_to_planar12(y, cbcr, dst_y, dst_u, dst_v, width, height, Chroma::C444); + return; + } + + // One pass writing all four planes measured faster than a pass per plane. + for (int row = 0; row < height; ++row) { + const uint16_t *__restrict in_y = src_row(y, row); + const uint16_t *__restrict in_uv = src_row(cbcr, row); + const uint16_t *__restrict in_a = src_row(a, row); + uint16_t *__restrict out_y = dst_row(dst_y, row); + uint16_t *__restrict out_u = dst_row(dst_u, row); + uint16_t *__restrict out_v = dst_row(dst_v, row); + uint16_t *__restrict out_a = dst_row(dst_a, row); + for (int x = 0; x < width; ++x) { + out_y[x] = round_16_to_12(in_y[x]); + out_u[x] = round_16_to_12(in_uv[2 * x]); + out_v[x] = round_16_to_12(in_uv[2 * x + 1]); + out_a[x] = round_16_to_12(in_a[x]); + } + } +} + +} // namespace xstudio::media_reader::ffmpeg::vt_repack diff --git a/src/plugin/media_reader/ffmpeg/src/vt_repack.hpp b/src/plugin/media_reader/ffmpeg/src/vt_repack.hpp new file mode 100644 index 000000000..fab38eb26 --- /dev/null +++ b/src/plugin/media_reader/ffmpeg/src/vt_repack.hpp @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +// Repacks the pixel layouts VideoToolbox decodes to into planar formats the xstudio +// YUV shader reads directly. Platform-neutral (just pointers and strides) so the +// loops can be unit tested anywhere. See design D5 in +// openspec/changes/add-videotoolbox-decode-backend. + +#include +#include +#include + +extern "C" { +#include +} + +namespace xstudio::media_reader::ffmpeg::vt_repack { + +struct SrcPlane { + const uint8_t *data; + size_t stride; // bytes per row +}; + +struct DstPlane { + uint8_t *data; + size_t stride; // bytes per row +}; + +/* Plane layout of a planar destination image in one contiguous buffer. */ +struct Layout { + AVPixelFormat format = AV_PIX_FMT_NONE; + std::array offsets = {0, 0, 0, 0}; + std::array linesizes = {0, 0, 0, 0}; + size_t total_size = 0; + int planes = 0; +}; + +/* Layout for a planar 'format' image of width x height. Line sizes are 64 byte +aligned, matching what setup_video_buffer() gives ffmpeg's own decoders. Returns a +layout with format AV_PIX_FMT_NONE for packed formats (AV_PIX_FMT_FLAG_PLANAR unset). */ +Layout planar_layout(AVPixelFormat format, int width, int height); + +/* Chroma subsampling of a source and its planar destination. */ +enum class Chroma { C420, C422, C444 }; + +inline int chroma_width(const int width, const Chroma c) { + return c == Chroma::C444 ? width : (width + 1) / 2; +} + +inline int chroma_height(const int height, const Chroma c) { + return c == Chroma::C420 ? (height + 1) / 2 : height; +} + +/* 8 bit Y + interleaved CbCr (VideoToolbox '420v' / '422v' / '444v' and the +full range 'f' variants; ffmpeg nv12 / nv16 / nv24) to yuv420p / yuv422p / +yuv444p. */ +void biplanar8_to_planar8( + SrcPlane y, + SrcPlane cbcr, + DstPlane dst_y, + DstPlane dst_u, + DstPlane dst_v, + int width, + int height, + Chroma chroma); + +/* 10 bit samples in the top bits of 16 bit words, Y + interleaved CbCr +(VideoToolbox 'x420' / 'x422' / 'x444' and the full range variants; ffmpeg +p010le / p210le / p410le) to yuv420p10le / yuv422p10le / yuv444p10le. */ +void biplanar10_to_planar10( + SrcPlane y, + SrcPlane cbcr, + DstPlane dst_y, + DstPlane dst_u, + DstPlane dst_v, + int width, + int height, + Chroma chroma); + +/* 16 bit Y + interleaved CbCr (VideoToolbox 'sv20' / 'sv22' / 'sv44'; ffmpeg +p016le / p216le / p416le) to yuv420p12le / yuv422p12le / yuv444p12le. Samples are +rounded to 12 bits. */ +void biplanar16_to_planar12( + SrcPlane y, + SrcPlane cbcr, + DstPlane dst_y, + DstPlane dst_u, + DstPlane dst_v, + int width, + int height, + Chroma chroma); + +/* 16 bit 4:4:4 Y + interleaved CbCr + A (VideoToolbox 's4as') to yuva444p12le, or +to yuv444p12le if dst_a.data is null. Samples are rounded to 12 bits. */ +void s4as_to_yuva444p12( + SrcPlane y, + SrcPlane cbcr, + SrcPlane a, + DstPlane dst_y, + DstPlane dst_u, + DstPlane dst_v, + DstPlane dst_a, + int width, + int height); + +/* 16 bit value rounded to 12 bits, clamped. Clamping first keeps every step within +16 bits ((65527 + 8) >> 4 == 4095), so loops vectorise at full width. */ +inline uint16_t round_16_to_12(const uint16_t v) { + const uint16_t clamped = v < 65527 ? v : uint16_t(65527); + return uint16_t(uint16_t(clamped + 8) >> 4); +} + +} // namespace xstudio::media_reader::ffmpeg::vt_repack diff --git a/src/plugin/media_reader/ffmpeg/test/CMakeLists.txt b/src/plugin/media_reader/ffmpeg/test/CMakeLists.txt index 902c030fe..03cc7a4ad 100644 --- a/src/plugin/media_reader/ffmpeg/test/CMakeLists.txt +++ b/src/plugin/media_reader/ffmpeg/test/CMakeLists.txt @@ -26,6 +26,12 @@ target_include_directories(ffmpeg_test ) target_link_directories(ffmpeg_test PRIVATE ${FFMPEG_LIBRARY_DIRS}) +# Media written by videotoolbox_media/generate.sh (tests needing it skip when absent) +target_compile_definitions(ffmpeg_test + PRIVATE VT_TEST_MEDIA_DIR="${CMAKE_BINARY_DIR}/test_media/videotoolbox" + PRIVATE $<$:XSTUDIO_HAS_VIDEOTOOLBOX=1> +) + add_test(ffmpeg_tests ffmpeg_test) set_target_properties(ffmpeg_test PROPERTIES LINK_DEPENDS_NO_SHARED true) diff --git a/src/plugin/media_reader/ffmpeg/test/ffmpeg_test.cpp b/src/plugin/media_reader/ffmpeg/test/ffmpeg_test.cpp index 01bc041c2..5bfb8a320 100644 --- a/src/plugin/media_reader/ffmpeg/test/ffmpeg_test.cpp +++ b/src/plugin/media_reader/ffmpeg/test/ffmpeg_test.cpp @@ -1,8 +1,19 @@ // SPDX-License-Identifier: Apache-2.0 #include +#include +#include +#include +#include +#include + #include "ffmpeg_decoder.hpp" #include "ffmpeg.hpp" +#include "vt_repack.hpp" +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX +#include "vt_decoder.hpp" +#endif +#include "xstudio/global_store/global_store.hpp" #include "xstudio/media/media.hpp" #include "xstudio/media_reader/media_reader.hpp" #include "xstudio/utility/helpers.hpp" @@ -117,3 +128,985 @@ TEST(FFMpegSwsColourspace, MapsStreamColourspaceToCoefficients) { EXPECT_EQ(sws_colourspace_from_av(AVCOL_SPC_UNSPECIFIED), SWS_CS_ITU601); } + +TEST(FFMpegVideoToolboxMode, ParsesPreferenceValues) { + EXPECT_EQ(videotoolbox_mode_from_string("off"), VideoToolboxMode::Off); + EXPECT_EQ(videotoolbox_mode_from_string("prefer"), VideoToolboxMode::Prefer); + EXPECT_EQ(videotoolbox_mode_from_string("only"), VideoToolboxMode::Only); + // Anything else gets the default. + EXPECT_EQ(videotoolbox_mode_from_string(""), VideoToolboxMode::Prefer); + EXPECT_EQ(videotoolbox_mode_from_string("Only"), VideoToolboxMode::Prefer); + + for (auto mode : {VideoToolboxMode::Off, VideoToolboxMode::Prefer, VideoToolboxMode::Only}) + EXPECT_EQ(videotoolbox_mode_from_string(to_string(mode)), mode); +} + +// Media from test/videotoolbox_media/generate.sh. Tests that need it are skipped when +// it hasn't been generated. +static std::string vt_media(const std::string &name) { + const std::string path = std::string(VT_TEST_MEDIA_DIR) + "/" + name; + return std::filesystem::exists(path) ? path : std::string(); +} + +#define VT_MEDIA_OR_SKIP(var, name) \ + const std::string var = vt_media(name); \ + if (var.empty()) \ + GTEST_SKIP() << name << " not found in " VT_TEST_MEDIA_DIR \ + << " - run test/videotoolbox_media/generate.sh" + +TEST(FFMpegVideoToolboxMode, DecoderCarriesMode) { + VT_MEDIA_OR_SKIP(path, "h264_8bit_bframes.mov"); + for (auto mode : + {VideoToolboxMode::Off, VideoToolboxMode::Prefer, VideoToolboxMode::Only}) { + FFMpegDecoder decoder( + path, 48000, utility::FrameRate(timebase::k_flicks_24fps), "", mode); + EXPECT_EQ(decoder.videotoolbox_mode(), mode); + } +} + +// Decodes one frame through the full reader path (stream, shader params). +static ImageBufPtr decode_frame( + const std::string &path, + const int frame, + const VideoToolboxMode mode = VideoToolboxMode::Off) { + // Stream 0 is the video stream in all the test media. + FFMpegDecoder decoder( + path, 48000, utility::FrameRate(timebase::k_flicks_24fps), "stream 0", mode); + ImageBufPtr buf; + decoder.decode_video_frame(frame, buf); + return buf; +} + +// Y offset the shader subtracts: 0 for full range, 16 (8 bit) for video range. +static int luma_offset(const ImageBufPtr &buf) { + return buf->shader_params()["yuv_offsets"][2].get(); +} + +// DNxHD/HR in a .dneg.mov is always treated as video range, whatever it is tagged. +TEST(FFMpegColourRange, DnegMovForcesVideoRangeForDNxHD) { + VT_MEDIA_OR_SKIP(plain, "dnxhr_fullrange.mov"); + VT_MEDIA_OR_SKIP(dneg, "dnxhr_fullrange.dneg.mov"); + + auto plain_buf = decode_frame(plain, 0); + auto dneg_buf = decode_frame(dneg, 0); + ASSERT_TRUE(plain_buf && dneg_buf); + EXPECT_EQ(luma_offset(plain_buf), 0); + EXPECT_EQ(luma_offset(dneg_buf), 16); +} + +// --------------------------------------------------------------------------------- +// vt_repack: VideoToolbox pixel layouts to planar formats the shader reads. +// Checked against av_read_image_line2 reading the same bytes as the matching +// FFmpeg format, so FFmpeg's own pixel descriptors are the reference. + +namespace { + +using namespace xstudio::media_reader::ffmpeg::vt_repack; + +// A source plane with padded rows, filled with pseudo-random values plus the +// extremes 0 and 0xFFFF (or 0xFF) at the start of every row. +struct TestPlane { + std::vector bytes; + size_t stride; + TestPlane(const int row_bytes, const int rows, const int bytes_per_sample, uint32_t seed) + : stride(size_t(row_bytes) + 40) { + bytes.resize(stride * rows); + for (auto &b : bytes) { + seed = seed * 1664525u + 1013904223u; + b = uint8_t(seed >> 24); + } + for (int r = 0; r < rows; ++r) { + uint8_t *row = bytes.data() + stride * r; + std::memset(row, 0, bytes_per_sample); + std::memset(row + bytes_per_sample, 0xff, bytes_per_sample); + } + } + SrcPlane src() const { return {bytes.data(), stride}; } +}; + +struct TestImage { + Layout layout; + std::vector bytes; + TestImage(const AVPixelFormat fmt, const int w, const int h) + : layout(planar_layout(fmt, w, h)), bytes(layout.total_size) {} + DstPlane dst(const int i) { + return {bytes.data() + layout.offsets[i], layout.linesizes[i]}; + } + // component c at (x, y) via FFmpeg's pixel descriptor + int value(const int c, const int x, const int y) const { + const uint8_t *data[4]; + int linesize[4]; + for (int i = 0; i < 4; ++i) { + data[i] = bytes.data() + layout.offsets[i]; + linesize[i] = int(layout.linesizes[i]); + } + uint16_t v = 0; + av_read_image_line2( + &v, data, linesize, av_pix_fmt_desc_get(layout.format), x, y, c, 1, 0, 2); + return v; + } +}; + +// component c at (x, y) of source planes interpreted as FFmpeg format 'fmt' +int reference_value( + const AVPixelFormat fmt, + const std::vector &planes, + const int c, + const int x, + const int y) { + const uint8_t *data[4] = {nullptr, nullptr, nullptr, nullptr}; + int linesize[4] = {0, 0, 0, 0}; + for (size_t i = 0; i < planes.size(); ++i) { + data[i] = planes[i]->bytes.data(); + linesize[i] = int(planes[i]->stride); + } + uint16_t v = 0; + av_read_image_line2(&v, data, linesize, av_pix_fmt_desc_get(fmt), x, y, c, 1, 0, 2); + return v; +} + +} // namespace + +TEST(VtRepack, PlanarLayoutIsAlignedAndContiguous) { + const auto l = planar_layout(AV_PIX_FMT_YUV420P10LE, 1921, 1081); + ASSERT_EQ(l.format, AV_PIX_FMT_YUV420P10LE); + ASSERT_EQ(l.planes, 3); + for (int i = 0; i < 3; ++i) + EXPECT_EQ(l.linesizes[i] % 64, 0u); + EXPECT_GE(l.linesizes[0], 1921u * 2); + EXPECT_GE(l.linesizes[1], 961u * 2); + EXPECT_EQ(l.offsets[1], l.linesizes[0] * 1081); + EXPECT_EQ(l.offsets[2], l.offsets[1] + l.linesizes[1] * 541); + EXPECT_EQ(l.total_size, l.offsets[2] + l.linesizes[2] * 541); + + // packed formats have no plane layout + EXPECT_EQ(planar_layout(AV_PIX_FMT_UYVY422, 64, 64).format, AV_PIX_FMT_NONE); +} + +// Every VideoToolbox biplanar layout against FFmpeg's description of it: 8 bit +// (nv12/16/24), 10 bit in the top bits (p010/210/410) and 16 bit (p016/216/416, +// rounded to 12 bits). +TEST(VtRepack, BiplanarToPlanarAllChromaFormats) { + enum Kind { Bits8, Bits10, Bits16 }; + struct Case { + Kind kind; + Chroma chroma; + AVPixelFormat src, dst; + }; + const Case cases[] = { + {Bits8, Chroma::C420, AV_PIX_FMT_NV12, AV_PIX_FMT_YUV420P}, + {Bits8, Chroma::C422, AV_PIX_FMT_NV16, AV_PIX_FMT_YUV422P}, + {Bits8, Chroma::C444, AV_PIX_FMT_NV24, AV_PIX_FMT_YUV444P}, + {Bits10, Chroma::C420, AV_PIX_FMT_P010LE, AV_PIX_FMT_YUV420P10LE}, + {Bits10, Chroma::C422, AV_PIX_FMT_P210LE, AV_PIX_FMT_YUV422P10LE}, + {Bits10, Chroma::C444, AV_PIX_FMT_P410LE, AV_PIX_FMT_YUV444P10LE}, + {Bits16, Chroma::C420, AV_PIX_FMT_P016LE, AV_PIX_FMT_YUV420P12LE}, + {Bits16, Chroma::C422, AV_PIX_FMT_P216LE, AV_PIX_FMT_YUV422P12LE}, + {Bits16, Chroma::C444, AV_PIX_FMT_P416LE, AV_PIX_FMT_YUV444P12LE}, + }; + for (const auto &cs : cases) { + for (auto [w, h] : {std::pair{64, 32}, std::pair{33, 17}}) { + const int bps = cs.kind == Bits8 ? 1 : 2; + const int cw = chroma_width(w, cs.chroma), ch = chroma_height(h, cs.chroma); + TestPlane y(w * bps, h, bps, 3), uv(cw * 2 * bps, ch, bps, 4); + if (cs.kind == Bits10) { + // VideoToolbox leaves the low 6 bits zero; so does a real P010 encoder. + for (auto *pl : {&y, &uv}) + for (size_t i = 0; i + 1 < pl->bytes.size(); i += 2) + pl->bytes[i] &= 0xc0; + } + + TestImage out(cs.dst, w, h); + switch (cs.kind) { + case Bits8: + biplanar8_to_planar8( + y.src(), uv.src(), out.dst(0), out.dst(1), out.dst(2), w, h, cs.chroma); + break; + case Bits10: + biplanar10_to_planar10( + y.src(), uv.src(), out.dst(0), out.dst(1), out.dst(2), w, h, cs.chroma); + break; + case Bits16: + biplanar16_to_planar12( + y.src(), uv.src(), out.dst(0), out.dst(1), out.dst(2), w, h, cs.chroma); + break; + } + + auto expected = [&](const int c, const int x, const int yy) { + const int ref = reference_value(cs.src, {&y, &uv}, c, x, yy); + return cs.kind == Bits16 ? int(round_16_to_12(uint16_t(ref))) : ref; + }; + for (int c = 0; c < 3; ++c) { + const int pw = c ? cw : w, ph = c ? ch : h; + for (int yy = 0; yy < ph; ++yy) + for (int x = 0; x < pw; ++x) + ASSERT_EQ(out.value(c, x, yy), expected(c, x, yy)) + << av_get_pix_fmt_name(cs.src) << " " << w << "x" << h << " c" << c + << " " << x << "," << yy; + } + } + } +} + +// s4as = p416le (16 bit Y + interleaved CbCr) plus a 16 bit alpha plane. +TEST(VtRepack, S4asToYuva444p12RoundsAndKeepsAlpha) { + for (auto [w, h] : {std::pair{64, 32}, std::pair{33, 17}}) { + TestPlane y(w * 2, h, 2, 5), uv(w * 4, h, 2, 6), a(w * 2, h, 2, 7); + TestImage out(AV_PIX_FMT_YUVA444P12LE, w, h); + s4as_to_yuva444p12( + y.src(), uv.src(), a.src(), out.dst(0), out.dst(1), out.dst(2), out.dst(3), w, h); + + for (int yy = 0; yy < h; ++yy) + for (int x = 0; x < w; ++x) { + for (int c = 0; c < 3; ++c) { + const int ref = reference_value(AV_PIX_FMT_P416LE, {&y, &uv}, c, x, yy); + ASSERT_EQ(out.value(c, x, yy), round_16_to_12(uint16_t(ref))) + << "c" << c << " " << x << "," << yy; + } + const uint16_t alpha = + *reinterpret_cast(a.bytes.data() + a.stride * yy + x * 2); + ASSERT_EQ(out.value(3, x, yy), round_16_to_12(alpha)) << "A " << x << "," << yy; + } + } + EXPECT_EQ(round_16_to_12(0), 0); + EXPECT_EQ(round_16_to_12(0xffff), 4095); + EXPECT_EQ(round_16_to_12(7), 0); + EXPECT_EQ(round_16_to_12(8), 1); + EXPECT_EQ(round_16_to_12(0xfff0), 4095); +} + +// Not run by default (timing-dependent). Run with --gtest_also_run_disabled_tests. +// Budget (design D5, measured on an M2 Max): at most 3 ms per 4K frame, single +// threaded, and at most 2.5x a plain copy of the same source bytes. swscale, which +// this replaced, took 7.7-9.6 ms. +TEST(VtRepack, DISABLED_BenchmarkAgainstMemcpy4K) { + const int w = 3840, h = 2160, reps = 30; + using clock = std::chrono::steady_clock; + auto time_ms = [&](auto &&fn) { + fn(); // warm up + const auto t0 = clock::now(); + for (int i = 0; i < reps; ++i) + fn(); + return std::chrono::duration(clock::now() - t0).count() / reps; + }; + auto memcpy_ms = [&](const std::vector &planes) { + size_t total = 0; + for (auto *p : planes) + total += p->bytes.size(); + std::vector dst(total); + return time_ms([&]() { + size_t off = 0; + for (auto *p : planes) { + std::memcpy(dst.data() + off, p->bytes.data(), p->bytes.size()); + off += p->bytes.size(); + } + }); + }; + auto report = [](const char *name, const double ms, const double copy_ms) { + std::cout << name << ": " << ms << " ms, memcpy " << copy_ms << " ms, ratio " + << ms / copy_ms << "\n"; + EXPECT_LE(ms, 3.0) << name; + EXPECT_LE(ms / copy_ms, 2.5) << name; + }; + + // every biplanar loop: name, bytes per sample, chroma, output format + enum Kind { Bits8, Bits10, Bits16 }; + struct Case { + const char *name; + Kind kind; + Chroma chroma; + AVPixelFormat dst; + }; + const Case cases[] = { + {"420v -> yuv420p", Bits8, Chroma::C420, AV_PIX_FMT_YUV420P}, + {"422v -> yuv422p", Bits8, Chroma::C422, AV_PIX_FMT_YUV422P}, + {"444v -> yuv444p", Bits8, Chroma::C444, AV_PIX_FMT_YUV444P}, + {"x420 -> yuv420p10le", Bits10, Chroma::C420, AV_PIX_FMT_YUV420P10LE}, + {"x422 -> yuv422p10le", Bits10, Chroma::C422, AV_PIX_FMT_YUV422P10LE}, + {"x444 -> yuv444p10le", Bits10, Chroma::C444, AV_PIX_FMT_YUV444P10LE}, + {"sv22 -> yuv422p12le", Bits16, Chroma::C422, AV_PIX_FMT_YUV422P12LE}, + {"sv44 -> yuv444p12le", Bits16, Chroma::C444, AV_PIX_FMT_YUV444P12LE}, + }; + for (const auto &cs : cases) { + const int bps = cs.kind == Bits8 ? 1 : 2; + const int cw = chroma_width(w, cs.chroma), ch = chroma_height(h, cs.chroma); + TestPlane y(w * bps, h, bps, 1), uv(cw * 2 * bps, ch, bps, 2); + TestImage out(cs.dst, w, h); + const double ms = time_ms([&]() { + switch (cs.kind) { + case Bits8: + biplanar8_to_planar8( + y.src(), uv.src(), out.dst(0), out.dst(1), out.dst(2), w, h, cs.chroma); + break; + case Bits10: + biplanar10_to_planar10( + y.src(), uv.src(), out.dst(0), out.dst(1), out.dst(2), w, h, cs.chroma); + break; + case Bits16: + biplanar16_to_planar12( + y.src(), uv.src(), out.dst(0), out.dst(1), out.dst(2), w, h, cs.chroma); + break; + } + }); + report(cs.name, ms, memcpy_ms({&y, &uv})); + } + { + TestPlane y(w * 2, h, 2, 1), uv(w * 4, h, 2, 2), a(w * 2, h, 2, 3); + TestImage out(AV_PIX_FMT_YUVA444P12LE, w, h); + const double ms = time_ms([&]() { + s4as_to_yuva444p12( + y.src(), + uv.src(), + a.src(), + out.dst(0), + out.dst(1), + out.dst(2), + out.dst(3), + w, + h); + }); + report("s4as -> yuva444p12le", ms, memcpy_ms({&y, &uv, &a})); + } +} + +// --------------------------------------------------------------------------------- +// VtDecoder, driven directly with packets from libavformat. +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX + +namespace { + +struct Demuxer { + AVFormatContext *ctx = nullptr; + AVStream *stream = nullptr; + AVPacket *pkt = av_packet_alloc(); + explicit Demuxer(const std::string &path) { + if (avformat_open_input(&ctx, path.c_str(), nullptr, nullptr) == 0 && + avformat_find_stream_info(ctx, nullptr) >= 0) { + const int i = av_find_best_stream(ctx, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0); + if (i >= 0) + stream = ctx->streams[i]; + } + } + ~Demuxer() { + av_packet_free(&pkt); + avformat_close_input(&ctx); + } + // next video packet, or null at the end + AVPacket *next() { + av_packet_unref(pkt); + while (av_read_frame(ctx, pkt) >= 0) { + if (pkt->stream_index == stream->index) + return pkt; + av_packet_unref(pkt); + } + return nullptr; + } +}; + +struct DecodeResult { + std::vector packet_pts; + std::vector frame_pts; + int send_errors = 0; + AVPixelFormat format = AV_PIX_FMT_NONE; + AVColorSpace colorspace = AVCOL_SPC_UNSPECIFIED; + AVColorPrimaries primaries = AVCOL_PRI_UNSPECIFIED; + AVColorTransferCharacteristic trc = AVCOL_TRC_UNSPECIFIED; + AVColorRange range = AVCOL_RANGE_UNSPECIFIED; +}; + +// Decodes the whole stream with VideoToolbox, avcodec-style send/receive loop. +DecodeResult vt_decode_all(const std::string &path, const int max_frames = 1 << 30) { + DecodeResult r; + Demuxer dm(path); + std::string error; + auto vt = VtDecoder::create(dm.stream, error); + EXPECT_TRUE(vt) << error; + if (!vt) + return r; + AVFrame *frame = av_frame_alloc(); + auto take = [&]() { + if (r.frame_pts.empty()) { + r.format = AVPixelFormat(frame->format); + r.colorspace = frame->colorspace; + r.primaries = frame->color_primaries; + r.trc = frame->color_trc; + r.range = frame->color_range; + } + r.frame_pts.push_back(frame->pts); + }; + while (AVPacket *pkt = dm.next()) { + if (int(r.frame_pts.size()) >= max_frames) + break; + r.packet_pts.push_back(pkt->pts); + if (vt->send_packet(pkt) < 0) + r.send_errors++; + while (vt->receive_frame(frame) == 0) + take(); + } + vt->send_packet(nullptr); + while (vt->receive_frame(frame) == 0) + take(); + av_frame_free(&frame); + return r; +} + +// Same, with FFmpeg's software decoder - the reference. +DecodeResult sw_decode_all(const std::string &path) { + DecodeResult r; + Demuxer dm(path); + const AVCodec *codec = avcodec_find_decoder(dm.stream->codecpar->codec_id); + AVCodecContext *ctx = avcodec_alloc_context3(codec); + avcodec_parameters_to_context(ctx, dm.stream->codecpar); + avcodec_open2(ctx, codec, nullptr); + AVFrame *frame = av_frame_alloc(); + auto take = [&]() { + if (r.frame_pts.empty()) { + r.format = AVPixelFormat(frame->format); + r.colorspace = frame->colorspace; + r.primaries = frame->color_primaries; + r.trc = frame->color_trc; + r.range = frame->color_range; + } + r.frame_pts.push_back(frame->pts); + }; + while (AVPacket *pkt = dm.next()) { + r.packet_pts.push_back(pkt->pts); + avcodec_send_packet(ctx, pkt); + while (avcodec_receive_frame(ctx, frame) == 0) + take(); + } + avcodec_send_packet(ctx, nullptr); + while (avcodec_receive_frame(ctx, frame) == 0) + take(); + av_frame_free(&frame); + avcodec_free_context(&ctx); + return r; +} + +const std::vector> k_vt_media_and_format = { + {"h264_8bit_bframes.mov", "420v"}, + {"hevc_8bit_opengop.mov", "420v"}, + {"hevc_main10_bframes.mov", "x420"}, + {"hevc_422_10bit_bframes.mov", "x422"}, + {"hevc_main10_bt2020_pq.mov", "x420"}, + {"prores_422_proxy.mov", "x422"}, + {"prores_422_lt.mov", "x422"}, + {"prores_422_standard.mov", "x422"}, + {"prores_422_hq.mov", "x422"}, + {"prores_4444_alpha.mov", "s4as"}, + {"prores_4444xq.mov", "s4as"}, + // variants (spike 1.8); the 12 bit ones need FFMPEG_HIGHBIT when generating + {"variant_h264_10bit_420.mov", "x420"}, + {"variant_h264_8bit_422.mov", "422v"}, + {"variant_h264_10bit_422.mov", "x422"}, + {"variant_h264_8bit_444.mov", "444v"}, + {"variant_h264_10bit_444.mov", "x444"}, + {"variant_hevc_8bit_422.mov", "422v"}, + {"variant_hevc_8bit_444.mov", "444v"}, + {"variant_hevc_10bit_444.mov", "x444"}, + {"variant_hevc_12bit_422.mov", "sv22"}, + {"variant_hevc_12bit_444.mov", "sv44"}, +}; + +// Is the clip there? Variant clips are optional; the core set is required. +bool have_clip(const std::string &name) { + if (!vt_media(name).empty()) + return true; + EXPECT_EQ(name.rfind("variant_", 0), 0u) << name << " missing"; + return false; +} + +} // namespace + +// Task 4.1 / 4.2 +TEST(VtDecoder, EligibilityAndOutputFormat) { + VT_MEDIA_OR_SKIP(first, "h264_8bit_bframes.mov"); + for (const auto &[name, expected] : k_vt_media_and_format) { + if (!have_clip(name)) + continue; + Demuxer dm(vt_media(name)); + ASSERT_TRUE(dm.stream) << name; + std::string reason; + EXPECT_TRUE(VtDecoder::eligible(dm.stream, reason)) << name << ": " << reason; + auto vt = VtDecoder::create(dm.stream, reason); + ASSERT_TRUE(vt) << name << ": " << reason; + EXPECT_EQ(vt->output_format_name(), expected) << name; + } + + for (const auto *name : + {"h264_annexb.h264", "dnxhr_fullrange.mov", "variant_hevc_12bit_420.mov"}) { + if (vt_media(name).empty()) + continue; // the 12 bit clip is optional + Demuxer dm(vt_media(name)); + ASSERT_TRUE(dm.stream) << name; + std::string reason; + EXPECT_FALSE(VtDecoder::eligible(dm.stream, reason)) << name; + EXPECT_FALSE(reason.empty()) << name; + EXPECT_FALSE(VtDecoder::create(dm.stream, reason)) << name; + } +} + +// Task 4.3 / 4.4: every packet decodes, frames come out once each in presentation +// order, and the last frames aren't lost in the reorder delay. +TEST(VtDecoder, DecodesEveryPacketInPresentationOrder) { + VT_MEDIA_OR_SKIP(first, "h264_8bit_bframes.mov"); + for (const auto &[name, fmt] : k_vt_media_and_format) { + if (!have_clip(name)) + continue; + const auto r = vt_decode_all(vt_media(name)); + EXPECT_EQ(r.send_errors, 0) << name; + ASSERT_EQ(r.frame_pts.size(), r.packet_pts.size()) << name; + EXPECT_TRUE(std::is_sorted(r.frame_pts.begin(), r.frame_pts.end())) << name; + EXPECT_EQ(std::adjacent_find(r.frame_pts.begin(), r.frame_pts.end()), r.frame_pts.end()) + << name << " repeats a frame"; + auto sorted_packets = r.packet_pts; + std::sort(sorted_packets.begin(), sorted_packets.end()); + EXPECT_EQ(r.frame_pts, sorted_packets) << name; + } +} + +// Task 4.5 / 4.6: planar output format and colour metadata match FFmpeg's decoder. +TEST(VtDecoder, FormatAndColourMatchSoftwareDecoder) { + VT_MEDIA_OR_SKIP(first, "h264_8bit_bframes.mov"); + for (const auto &[name, fmt] : k_vt_media_and_format) { + if (!have_clip(name)) + continue; + const auto vt = vt_decode_all(vt_media(name), 1); + const auto sw = sw_decode_all(vt_media(name)); + ASSERT_FALSE(vt.frame_pts.empty()) << name; + EXPECT_EQ(vt.format, sw.format) << name << ": " << av_get_pix_fmt_name(vt.format) + << " vs " << av_get_pix_fmt_name(sw.format); + EXPECT_EQ(vt.colorspace, sw.colorspace) << name; + EXPECT_EQ(vt.primaries, sw.primaries) << name; + EXPECT_EQ(vt.trc, sw.trc) << name; + // FFmpeg reports MPEG range for ProRes; unspecified elsewhere means MPEG too. + auto norm = [](AVColorRange r) { + return r == AVCOL_RANGE_UNSPECIFIED ? AVCOL_RANGE_MPEG : r; + }; + EXPECT_EQ(norm(vt.range), norm(sw.range)) << name; + } +} + +// Task 4.5: decode 1000+ frames; run under 'leaks' to check no CVPixelBuffer or +// session objects are left behind. +TEST(VtDecoder, LongDecodeForLeakCheck) { + VT_MEDIA_OR_SKIP(path, "h264_8bit_bframes.mov"); + size_t frames = 0; + for (int i = 0; i < 21; ++i) + frames += vt_decode_all(path).frame_pts.size(); + EXPECT_GE(frames, 1000u); +} + + +// --------------------------------------------------------------------------------- +// Through FFMpegDecoder / FFMpegStream: the path the reader uses. + +namespace { + +// Planar YUV(A) samples of a decoded image, read via the shader parameters the +// reader produced (so this also checks those are right). +struct Samples { + int bits = 0; + std::array, 4> planes; +}; + +Samples samples_of(const ImageBufPtr &buf) { + Samples out; + const auto &p = buf->shader_params(); + out.bits = p.value("bits_per_channel", 8); + const int bps = out.bits > 8 ? 2 : 1; + const auto size = buf->image_size_in_pixels(); + const int cw = p.value("half_scale_uvx", 0) ? (size.x + 1) / 2 : size.x; + const int ch = p.value("half_scale_uvy", 0) ? (size.y + 1) / 2 : size.y; + const char *names[4] = {"y", "u", "v", "a"}; + for (int i = 0; i < 4; ++i) { + const int linesize = p.value(std::string(names[i]) + "_linesize", 0); + if (!linesize) + continue; + const size_t offset = p.value(std::string(names[i]) + "_plane_bytes_offset", size_t(0)); + const int w = (i == 1 || i == 2) ? cw : size.x; + const int h = (i == 1 || i == 2) ? ch : size.y; + auto &plane = out.planes[i]; + plane.reserve(size_t(w) * h); + const auto *base = reinterpret_cast(buf->buffer()) + offset; + for (int y = 0; y < h; ++y) { + const uint8_t *row = base + size_t(linesize) * y; + for (int x = 0; x < w; ++x) + plane.push_back( + bps == 2 ? reinterpret_cast(row)[x] : uint16_t(row[x])); + } + } + return out; +} + +// Largest absolute sample difference between two images, -1 if their layouts differ. +int max_difference(const ImageBufPtr &a, const ImageBufPtr &b) { + const Samples sa = samples_of(a), sb = samples_of(b); + if (sa.bits != sb.bits) + return -1; + int worst = 0; + for (int i = 0; i < 4; ++i) { + if (sa.planes[i].size() != sb.planes[i].size()) + return -1; + for (size_t k = 0; k < sa.planes[i].size(); ++k) + worst = std::max(worst, std::abs(int(sa.planes[i][k]) - int(sb.planes[i][k]))); + } + return worst; +} + +// Scoped environment variable +struct ScopedEnv { + std::string name; + ScopedEnv(const char *n, const char *v) : name(n) { setenv(n, v, 1); } + ~ScopedEnv() { unsetenv(name.c_str()); } +}; + +FFMpegDecoder make_decoder(const std::string &path, const VideoToolboxMode mode) { + return FFMpegDecoder( + path, 48000, utility::FrameRate(timebase::k_flicks_24fps), "stream 0", mode); +} + +} // namespace + +// Task 5.1 (and the second half of 4.5): frames come from VideoToolbox, as planar +// YUV the shader reads directly - no CPU conversion on the way. +TEST(VtStream, PlaysThroughVideoToolboxWithoutCpuConversion) { + VT_MEDIA_OR_SKIP(first, "h264_8bit_bframes.mov"); + for (const auto &[name, fmt] : k_vt_media_and_format) { + if (!have_clip(name)) + continue; + auto decoder = make_decoder(vt_media(name), VideoToolboxMode::Prefer); + auto stream = decoder.stream(0); + ASSERT_TRUE(stream) << name; + EXPECT_TRUE(stream->decoded_by_videotoolbox()) << name; + // Task 3.2: FFmpeg's decoder is never even looked up, so builds without + // one for this codec work the same. + EXPECT_FALSE(stream->uses_ffmpeg_decoder()) << name; + for (int f : {0, 1, 2, 10}) { + ImageBufPtr buf; + decoder.decode_video_frame(f, buf); + ASSERT_TRUE(buf) << name << " frame " << f; + EXPECT_EQ(buf->shader_params().value("pix_fmt", -1), 0) + << name << ": YUV shader path"; + EXPECT_EQ(buf->decoder_frame_number(), f) << name; + } + EXPECT_FALSE(stream->converted_on_cpu()) << name; + } +} + +// Spec: frame-accurate presentation order (task 7.1). A fixed jumbled sequence of +// frame requests - forwards, backwards, first and last - must give the same +// pictures as FFmpeg's decoder: bit-exact for H.264/HEVC, within the ProRes +// tolerance measured in the spike (max 4 code values). +TEST(VtStream, RandomAccessMatchesSoftwareDecoder) { + VT_MEDIA_OR_SKIP(first, "h264_8bit_bframes.mov"); + const std::vector sequence = { + 0, 1, 2, 47, 12, 13, 5, 30, 29, 28, 46, 23, 24, 0, 37, 11}; + for (const auto &[name, fmt] : k_vt_media_and_format) { + if (!have_clip(name)) + continue; + const bool prores = name.rfind("prores", 0) == 0; + auto vt = make_decoder(vt_media(name), VideoToolboxMode::Prefer); + auto sw = make_decoder(vt_media(name), VideoToolboxMode::Off); + ASSERT_TRUE(vt.stream(0)->decoded_by_videotoolbox()) << name; + ASSERT_FALSE(sw.stream(0)->decoded_by_videotoolbox()) << name; + for (const int f : sequence) { + ImageBufPtr a, b; + std::string vt_error, sw_error; + try { + vt.decode_video_frame(f, a); + } catch (const std::exception &e) { + vt_error = e.what(); + } + try { + sw.decode_video_frame(f, b); + } catch (const std::exception &e) { + sw_error = e.what(); + } + // FFMpegDecoder can't reach the frame just before an open-GOP keyframe + // after a backward jump (frame 23 of the open-GOP clips here) with FFmpeg's + // decoder - a seek issue predating VideoToolbox support. VideoToolbox may do + // better, but must never fail where FFmpeg's decoder succeeds. + if (vt_error.empty() && !sw_error.empty()) { + ASSERT_TRUE(a) << name << " frame " << f; + EXPECT_EQ(a->decoder_frame_number(), f) << name; + continue; + } + EXPECT_EQ(vt_error, sw_error) << name << " frame " << f; + if (!vt_error.empty() || !sw_error.empty()) + continue; + ASSERT_TRUE(a && b) << name << " frame " << f; + EXPECT_EQ(a->decoder_frame_number(), f) << name; + EXPECT_EQ(a->has_alpha(), b->has_alpha()) << name; + const int diff = max_difference(a, b); + if (name.find("12bit") != std::string::npos) { + EXPECT_GE(diff, 0) << name << " frame " << f << ": layouts differ"; + EXPECT_LE(diff, 1) << name << " frame " << f; + } else if (prores) { + EXPECT_GE(diff, 0) << name << " frame " << f << ": layouts differ"; + EXPECT_LE(diff, 4) << name << " frame " << f; + } else { + EXPECT_EQ(diff, 0) << name << " frame " << f; + } + } + } +} + +// Task 5.2: when VideoToolbox can't create a session, 'prefer' falls back to +// FFmpeg's decoder and 'only' fails each frame with a message naming the codec. +TEST(VtStream, SessionFailureFallsBackOrFails) { + VT_MEDIA_OR_SKIP(path, "hevc_main10_bframes.mov"); + ScopedEnv force("XSTUDIO_VIDEOTOOLBOX_FORCE_FAILURE", "1"); + + { + auto decoder = make_decoder(path, VideoToolboxMode::Prefer); + EXPECT_FALSE(decoder.stream(0)->decoded_by_videotoolbox()); + ImageBufPtr buf; + decoder.decode_video_frame(3, buf); + ASSERT_TRUE(buf); + EXPECT_EQ(buf->decoder_frame_number(), 3); + } + { + auto decoder = make_decoder(path, VideoToolboxMode::Only); + ImageBufPtr buf; + try { + decoder.decode_video_frame(3, buf); + ADD_FAILURE() << "expected videotoolbox_unavailable_error"; + } catch (const videotoolbox_unavailable_error &e) { + EXPECT_NE(std::string(e.what()).find("hevc"), std::string::npos) << e.what(); + EXPECT_NE(std::string(e.what()).find("VideoToolbox"), std::string::npos) + << e.what(); + } + // and keeps failing cleanly rather than crashing + EXPECT_THROW(decoder.decode_video_frame(4, buf), videotoolbox_unavailable_error); + } +} + +// Task 5.3: streams VideoToolbox doesn't take are decoded by FFmpeg, in any mode, +// exactly as with VideoToolbox off. (A raw Annex-B stream has no container timing, +// so the reader gives it no duration or frame numbers either way.) +TEST(VtStream, IneligibleStreamsUseSoftware) { + VT_MEDIA_OR_SKIP(annexb, "h264_annexb.h264"); + VT_MEDIA_OR_SKIP(dnx, "dnxhr_fullrange.mov"); + for (const auto &path : {annexb, dnx}) { + auto reference = make_decoder(path, VideoToolboxMode::Off); + ImageBufPtr ref; + reference.decode_video_frame(5, ref); + ASSERT_TRUE(ref) << path; + for (auto mode : {VideoToolboxMode::Prefer, VideoToolboxMode::Only}) { + auto decoder = make_decoder(path, mode); + EXPECT_FALSE(decoder.stream(0)->decoded_by_videotoolbox()) << path; + ImageBufPtr buf; + decoder.decode_video_frame(5, buf); + ASSERT_TRUE(buf) << path; + EXPECT_EQ(buf->decoder_frame_number(), ref->decoder_frame_number()) << path; + EXPECT_EQ(max_difference(buf, ref), 0) << path; + } + } +} + +// Task 5.4: thumbnails of VideoToolbox-decoded media. +TEST(VtStream, Thumbnails) { + VT_MEDIA_OR_SKIP(first, "h264_8bit_bframes.mov"); + for (const auto &[name, fmt] : k_vt_media_and_format) { + if (!have_clip(name)) + continue; + auto decoder = make_decoder(vt_media(name), VideoToolboxMode::Prefer); + ASSERT_TRUE(decoder.stream(0)->decoded_by_videotoolbox()) << name; + auto thumb = decoder.decode_thumbnail_frame(24, 256); + ASSERT_TRUE(thumb) << name; + EXPECT_GT(thumb->width(), 0u) << name; + EXPECT_GT(thumb->height(), 0u) << name; + } +} + +// Changing the preference at runtime must not hand back a frame decoded under the +// previous mode (FFMpegMediaReader keeps the last decoded image to answer repeat +// requests). +TEST(VtStream, PreferenceChangeDropsFramesDecodedUnderOldMode) { + VT_MEDIA_OR_SKIP(path, "prores_422_hq.mov"); + + utility::JsonStore prefs; + global_store::set_preference_value( + prefs, std::string("prefer"), "/plugin/media_reader/FFMPEG/videotoolbox"); + FFMpegMediaReader reader(prefs); + + const media::AVFrameID frame( + posix_path_to_uri(path), + 10, + 0, + media::FS_ON_DISK, + 0, + 1.0f, + utility::FrameRate(timebase::k_flicks_24fps), + "stream 0"); + + const ImageBufPtr vt = reader.image(frame); + ASSERT_TRUE(vt); + vt->set_media_key(frame.key()); // as the reader actor does + EXPECT_EQ(reader.image(frame), vt) << "repeat request is answered from memory"; + + global_store::set_preference_value( + prefs, std::string("off"), "/plugin/media_reader/FFMPEG/videotoolbox"); + reader.update_preferences(prefs); + + const ImageBufPtr sw = reader.image(frame); + ASSERT_TRUE(sw); + EXPECT_NE(sw, vt); + // Apple's and FFmpeg's ProRes decoders differ slightly, so a fresh decode shows it + EXPECT_GT(max_difference(sw, vt), 0); + EXPECT_LE(max_difference(sw, vt), 4); +} + +// --------------------------------------------------------------------------------- +// Task 6.1: AAC through AudioToolbox (aac_at) + +namespace { + +struct DecodedAudio { + int64_t first_pts = AV_NOPTS_VALUE; + std::vector samples; // first channel +}; + +DecodedAudio decode_audio_with(const std::string &path, const char *decoder_name) { + DecodedAudio out; + AVFormatContext *ctx = nullptr; + if (avformat_open_input(&ctx, path.c_str(), nullptr, nullptr) < 0) + return out; + avformat_find_stream_info(ctx, nullptr); + const int index = av_find_best_stream(ctx, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0); + const AVCodec *codec = avcodec_find_decoder_by_name(decoder_name); + AVCodecContext *dec = avcodec_alloc_context3(codec); + avcodec_parameters_to_context(dec, ctx->streams[index]->codecpar); + avcodec_open2(dec, codec, nullptr); + AVPacket *pkt = av_packet_alloc(); + AVFrame *frame = av_frame_alloc(); + auto take = [&]() { + if (out.first_pts == AV_NOPTS_VALUE) + out.first_pts = frame->pts; + for (int i = 0; i < frame->nb_samples; ++i) { + switch (frame->format) { + case AV_SAMPLE_FMT_FLTP: + case AV_SAMPLE_FMT_FLT: + out.samples.push_back(reinterpret_cast(frame->data[0])[i]); + break; + case AV_SAMPLE_FMT_S16P: + case AV_SAMPLE_FMT_S16: + out.samples.push_back( + reinterpret_cast(frame->data[0])[i] / 32768.0f); + break; + default: + break; + } + } + }; + while (av_read_frame(ctx, pkt) >= 0) { + if (pkt->stream_index == index) { + avcodec_send_packet(dec, pkt); + while (avcodec_receive_frame(dec, frame) == 0) + take(); + } + av_packet_unref(pkt); + } + avcodec_send_packet(dec, nullptr); + while (avcodec_receive_frame(dec, frame) == 0) + take(); + av_frame_free(&frame); + av_packet_free(&pkt); + avcodec_free_context(&dec); + avformat_close_input(&ctx); + return out; +} + +} // namespace + +TEST(AacAudioToolbox, StreamUsesAacAt) { + VT_MEDIA_OR_SKIP(path, "h264_aac.mov"); + ASSERT_TRUE(avcodec_find_decoder_by_name("aac_at")) << "FFmpeg built without AudioToolbox"; + for (auto mode : {VideoToolboxMode::Off, VideoToolboxMode::Prefer}) { + FFMpegDecoder decoder( + path, 48000, utility::FrameRate(timebase::k_flicks_24fps), "stream 1", mode); + auto stream = decoder.stream(1); + ASSERT_TRUE(stream); + EXPECT_EQ(stream->ffmpeg_decoder_name(), "aac_at") << to_string(mode); + AudioBufPtr audio; + decoder.decode_audio_frame(10, audio); + ASSERT_TRUE(audio); + EXPECT_GT(audio->num_samples(), 0u); + } +} + +// aac_at must line up sample for sample with FFmpeg's own AAC decoder - same first +// timestamp, same length, no priming offset - or audio sync would change. +TEST(AacAudioToolbox, MatchesFFmpegAacTiming) { + VT_MEDIA_OR_SKIP(path, "h264_aac.mov"); + const auto ref = decode_audio_with(path, "aac"); + const auto at = decode_audio_with(path, "aac_at"); + ASSERT_FALSE(ref.samples.empty()); + ASSERT_FALSE(at.samples.empty()); + EXPECT_EQ(at.first_pts, ref.first_pts); + EXPECT_EQ(at.samples.size(), ref.samples.size()); + + // best alignment within +-4096 samples, on the chirp (non-periodic) + const size_t n = std::min(at.samples.size(), ref.samples.size()); + int best_lag = 0; + double best_err = 1e30; + for (int lag = -4096; lag <= 4096; lag += 1) { + double err = 0; + size_t used = 0; + for (size_t i = 8192; i + 8192 < n; i += 7) { + const long j = long(i) + lag; + const double d = at.samples[size_t(j)] - ref.samples[i]; + err += d * d; + used++; + } + err /= double(used); + if (err < best_err) { + best_err = err; + best_lag = lag; + } + } + EXPECT_EQ(best_lag, 0); + EXPECT_LT(std::sqrt(best_err), 1e-3) << "RMS difference between AAC decoders"; +} + +#endif // XSTUDIO_HAS_VIDEOTOOLBOX + + +// Not run by default (slow). Run with --gtest_also_run_disabled_tests. +// Memory must stay flat however many frames are decoded: VideoToolbox's frames live +// in IOSurfaces, which 'leaks' can't see but the process footprint includes. +#ifdef XSTUDIO_HAS_VIDEOTOOLBOX +#include + +static double footprint_mb() { + task_vm_info_data_t info; + mach_msg_type_number_t count = TASK_VM_INFO_COUNT; + if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&info, &count) != KERN_SUCCESS) + return -1; + return double(info.phys_footprint) / (1024.0 * 1024.0); +} + +TEST(VtStream, DISABLED_MemoryStaysFlatOverLongDecode) { + VT_MEDIA_OR_SKIP(path, "bench_uhd_prores_4444_grain.mov"); + for (auto mode : {VideoToolboxMode::Off, VideoToolboxMode::Prefer}) { + std::vector mb; + for (int pass = 0; pass < 6; ++pass) { + // a new decoder each pass, like the reader for thumbnails / reopened media + auto decoder = make_decoder(path, mode); + for (int f = 0; f < 120; ++f) { + ImageBufPtr buf; + decoder.decode_video_frame(f, buf); + } + mb.push_back(footprint_mb()); + } + std::cout << to_string(mode) << " footprint MB after each 120 frame pass:"; + for (auto v : mb) + std::cout << " " << int(v); + std::cout << "\n"; + // allow some slack for allocator pools; a leaked 4K 4444 frame is ~66MB + EXPECT_LT(mb.back() - mb[1], 200.0) << to_string(mode); + } +} + +#endif diff --git a/src/plugin/media_reader/ffmpeg/test/videotoolbox_media/README.md b/src/plugin/media_reader/ffmpeg/test/videotoolbox_media/README.md new file mode 100644 index 000000000..715d8a8e4 --- /dev/null +++ b/src/plugin/media_reader/ffmpeg/test/videotoolbox_media/README.md @@ -0,0 +1,56 @@ +# VideoToolbox decode test media + +Media used to test the VideoToolbox decode backend of the FFmpeg reader +(`openspec/changes/add-videotoolbox-decode-backend`). The files aren't +committed. Generate them with: + +```sh +./generate.sh [path/to/ffmpeg] [output dir] + +# also write the 12 bit / 10 bit 4:4:4 HEVC variants, which need a libx265 built +# for high bit depth (the vcpkg one is 8 bit only) +FFMPEG_HIGHBIT="rez-env ffmpeg-9.0 -- ffmpeg" ./generate.sh +``` + +By default the script uses the vcpkg-built FFmpeg at +`build/vcpkg_installed/arm-osx/tools/ffmpeg/ffmpeg` and writes to +`build/test_media/videotoolbox/`. It needs an FFmpeg with libx264, libx265 and +the VideoToolbox encoders, so it has to run on macOS. Generation takes about +a minute on an M2 Max, and the output is about 1.7 GB, most of it the 4K +ProRes benchmark files. + +Every clip is `testsrc2` at 24 fps, so each frame shows a running counter for +checking frame accuracy by eye. The HD clips are 1920x1080 and 2 s long +(48 frames). The `bench_` clips are 3840x2160 and 5 s long (120 frames). Every +clip is tagged BT.709 limited range unless listed otherwise. + +| File | Codec / profile | Depth, subsampling | GOP / structure | Tests | +|---|---|---|---|---| +| `h264_8bit_bframes.mov` | H.264 High (libx264), avcC | 8-bit 4:2:0 | 3 B-frames, closed GOP of 24 | Reorder (D8), random access, bit-exact parity | +| `hevc_main10_bframes.mov` | HEVC Main 10 (VideoToolbox), hvcC | 10-bit 4:2:0 | 2 B-frames, GOP 24 | `x420`/P010 output path, reorder | +| `hevc_422_10bit_bframes.mov` | HEVC Main 4:2:2 10 (VideoToolbox), hvcC | 10-bit 4:2:2 | 2 B-frames, GOP 24 | `x422`/P210 output path | +| `hevc_8bit_opengop.mov` | HEVC Main (libx265), hvcC | 8-bit 4:2:0 | 4 B-frames, **open GOP**, keyint 24 | Leading pictures after a seek | +| `hevc_main10_bt2020_pq.mov` | HEVC Main 10 (VideoToolbox), hvcC | 10-bit 4:2:0 | 2 B-frames, GOP 24 | Colour metadata: BT.2020 non-constant-luminance matrix and primaries, PQ transfer | +| `prores_422_proxy.mov` | ProRes 422 Proxy (`apco`) | 10-bit 4:2:2 | Intra | ProRes 422 decode | +| `prores_422_lt.mov` | ProRes 422 LT (`apcs`) | 10-bit 4:2:2 | Intra | " | +| `prores_422_standard.mov` | ProRes 422 (`apcn`) | 10-bit 4:2:2 | Intra | " | +| `prores_422_hq.mov` | ProRes 422 HQ (`apch`) | 10-bit 4:2:2 | Intra | " | +| `prores_4444_alpha.mov` | ProRes 4444 (`ap4h`) | 12-bit 4:4:4 + alpha | Intra | Alpha preserved. Alpha is a horizontal ramp from 0 (left) to 1 (right) | +| `prores_4444xq.mov` | ProRes 4444 XQ (`ap4x`) | 12-bit 4:4:4 | Intra | 4444 XQ decode, 12-bit precision | +| `h264_annexb.h264` | H.264 High, raw Annex-B elementary stream | 8-bit 4:2:0 | 3 B-frames | Not eligible for VideoToolbox; must fall back to software. No container, so FFmpeg guesses 25 fps | +| `h264_aac.mov` | H.264 High + AAC-LC mono 48 kHz, rising chirp | 8-bit 4:2:0 | 3 B-frames | AAC through `aac_at` (D10), A/V sync. The chirp never repeats, so any sample offset between AAC decoders shows up | +| `dnxhr_fullrange.mov`, `dnxhr_fullrange.dneg.mov` | DNxHR HQ tagged full range (the same file twice) | 8-bit 4:2:2 | Intra | Not VideoToolbox. Regression test for the `.dneg.mov` video-range override | +| `variant_h264_{10bit_420,8bit_422,10bit_422,8bit_444,10bit_444}.mov` | H.264 High 10 / High 4:2:2 / High 4:4:4 Predictive | as named | 3 B-frames | Spike 1.8 variants | +| `variant_hevc_{8bit_422,8bit_444}.mov` | HEVC Rext (libx265) | as named | 4 B-frames, open GOP | Spike 1.8 variants | +| `variant_hevc_{10bit_444,12bit_420,12bit_422,12bit_444}.mov` | HEVC Rext, only with `FFMPEG_HIGHBIT` | as named | 4 B-frames, open GOP | Spike 1.8 variants. 12-bit 4:2:0 must stay on FFmpeg | +| `bench_uhd_hevc_main10.mov` | HEVC Main 10 (VideoToolbox), 40 Mb/s | 10-bit 4:2:0 | 2 B-frames, GOP 24 | Throughput benchmark (task 1.5) | +| `bench_uhd_prores_4444.mov` | ProRes 4444 (`ap4h`) | 12-bit 4:4:4 | Intra | Throughput benchmark (task 1.5) | +| `bench_uhd_{hevc_main10,h264,prores_422hq,prores_4444}_grain.mov` | As above, with temporal grain | | | Realistic decode cost for benchmarks: `testsrc2` alone compresses unrealistically well | + +Notes: + +- The vcpkg `libx265` is an 8-bit build, so the 10-bit HEVC clips come from + Apple's `hevc_videotoolbox` encoder. `libx265` is still used for the open-GOP + clip, because `hevc_videotoolbox` has no open-GOP control. +- FFmpeg 8 takes colour tags from the frames, so the script tags them with a + `setparams` filter as well as with the `-color_*` output options. diff --git a/src/plugin/media_reader/ffmpeg/test/videotoolbox_media/generate.sh b/src/plugin/media_reader/ffmpeg/test/videotoolbox_media/generate.sh new file mode 100755 index 000000000..ffc4a8be2 --- /dev/null +++ b/src/plugin/media_reader/ffmpeg/test/videotoolbox_media/generate.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# +# Generates the media set used to test the VideoToolbox decode backend. +# See README.md in this directory for what each file exercises. +# +# usage: generate.sh [ffmpeg binary] [output dir] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../../../.." && pwd)" + +FFMPEG="${1:-${REPO_ROOT}/build/vcpkg_installed/arm-osx/tools/ffmpeg/ffmpeg}" +OUT="${2:-${REPO_ROOT}/build/test_media/videotoolbox}" + +mkdir -p "${OUT}" + +FF=("${FFMPEG}" -hide_banner -loglevel error -y) + +# 2 seconds at 24fps; frame number burnt in so frame accuracy can be checked by eye +HD="testsrc2=size=1920x1080:rate=24:duration=2" +UHD="testsrc2=size=3840x2160:rate=24:duration=5" +# FFmpeg 8 takes colour tags from the frames, so set them with setparams rather than +# -color_* output options alone. +P709="setparams=colorspace=bt709:color_primaries=bt709:color_trc=bt709:range=tv" +P2020PQ="setparams=colorspace=bt2020nc:color_primaries=bt2020:color_trc=smpte2084:range=tv" +TAGS_709=(-colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range tv) + +echo "Writing test media to ${OUT}" + +# H.264 8-bit 4:2:0, 3 B-frames, closed GOP of 24 +"${FF[@]}" -f lavfi -i "${HD}" -c:v libx264 -pix_fmt yuv420p -bf 3 -g 24 \ + -x264-params open-gop=0 -vf "${P709}" "${TAGS_709[@]}" "${OUT}/h264_8bit_bframes.mov" + +# The vcpkg libx265 is an 8 bit build, so 10 bit HEVC comes from Apple's encoder. +# libx265 is still used where we need GOP control that hevc_videotoolbox doesn't expose. + +# HEVC Main10 4:2:0, B-frames +"${FF[@]}" -f lavfi -i "${HD}" -c:v hevc_videotoolbox -profile:v main10 -pix_fmt p010le \ + -bf 2 -g 24 -b:v 8M -tag:v hvc1 -vf "${P709}" "${TAGS_709[@]}" "${OUT}/hevc_main10_bframes.mov" + +# HEVC Main 4:2:2 10, B-frames +"${FF[@]}" -f lavfi -i "${HD}" -c:v hevc_videotoolbox -profile:v main42210 -pix_fmt p210le \ + -bf 2 -g 24 -b:v 10M -tag:v hvc1 -vf "${P709}" "${TAGS_709[@]}" "${OUT}/hevc_422_10bit_bframes.mov" + +# HEVC Main (8 bit), open GOP with B-frames - leading pictures after each I frame +"${FF[@]}" -f lavfi -i "${HD}" -c:v libx265 -pix_fmt yuv420p -tag:v hvc1 \ + -x265-params "bframes=4:keyint=24:open-gop=1:log-level=error" -vf "${P709}" "${TAGS_709[@]}" \ + "${OUT}/hevc_8bit_opengop.mov" + +# HEVC Main10 tagged BT.2020 / PQ +"${FF[@]}" -f lavfi -i "${HD}" -c:v hevc_videotoolbox -profile:v main10 -pix_fmt p010le \ + -bf 2 -g 24 -b:v 8M -tag:v hvc1 -vf "${P2020PQ}" \ + -colorspace bt2020nc -color_primaries bt2020 -color_trc smpte2084 -color_range tv \ + "${OUT}/hevc_main10_bt2020_pq.mov" + +# ProRes 422 family, via Apple's encoder +for profile in proxy lt standard hq; do + "${FF[@]}" -f lavfi -i "${HD}" -c:v prores_videotoolbox -profile:v "${profile}" \ + -pix_fmt p210le -vf "${P709}" "${TAGS_709[@]}" "${OUT}/prores_422_${profile}.mov" +done + +# ProRes 4444 with an alpha gradient, and 4444 XQ +"${FF[@]}" -f lavfi -i "${HD}" -vf "format=yuva444p10le,geq=lum='lum(X,Y)':cb='cb(X,Y)':cr='cr(X,Y)':a='X*1023/W',${P709}" \ + -c:v prores_videotoolbox -profile:v 4444 -pix_fmt ayuv64le "${TAGS_709[@]}" \ + "${OUT}/prores_4444_alpha.mov" +"${FF[@]}" -f lavfi -i "${HD}" -c:v prores_videotoolbox -profile:v xq -pix_fmt ayuv64le \ + -vf "${P709}" "${TAGS_709[@]}" "${OUT}/prores_4444xq.mov" + +# Raw Annex-B H.264 elementary stream (no avcC) - must fall back to software +"${FF[@]}" -f lavfi -i "${HD}" -c:v libx264 -pix_fmt yuv420p -bf 3 -g 24 \ + -vf "${P709}" "${TAGS_709[@]}" -f h264 "${OUT}/h264_annexb.h264" + +# H.264 with an AAC audio track: a rising chirp, so that any offset between AAC +# decoders shows up (a steady tone would hide one that is a whole number of cycles, +# e.g. the usual 2112 sample priming at 1kHz/48kHz) +"${FF[@]}" -f lavfi -i "${HD}" -f lavfi -i "aevalsrc='0.5*sin(2*PI*(200+1500*t)*t)':s=48000:d=2" \ + -c:v libx264 -pix_fmt yuv420p -bf 3 -g 24 -vf "${P709}" "${TAGS_709[@]}" -c:a aac -b:a 192k \ + -shortest "${OUT}/h264_aac.mov" + +# DNxHR HQ tagged full range, and the same file named .dneg.mov - the reader forces +# DNxHD/HR in .dneg.mov files to video range. Not VideoToolbox media, but the +# regression test for that override lives with these tests. +"${FF[@]}" -f lavfi -i "${HD}" -c:v dnxhd -profile:v dnxhr_hq -pix_fmt yuv422p \ + -vf "setparams=colorspace=bt709:color_primaries=bt709:color_trc=bt709:range=pc" \ + -colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range pc \ + "${OUT}/dnxhr_fullrange.mov" +cp "${OUT}/dnxhr_fullrange.mov" "${OUT}/dnxhr_fullrange.dneg.mov" + +# Variants VideoToolbox isn't used for yet (design D4), to see what it does with them. +# libx265 here is 8 bit only, so there is no 12 bit or 10 bit 4:4:4 HEVC. +for v in "yuv420p10le h264_10bit_420" "yuv422p h264_8bit_422" "yuv422p10le h264_10bit_422" \ + "yuv444p h264_8bit_444" "yuv444p10le h264_10bit_444"; do + set -- $v + "${FF[@]}" -f lavfi -i "${HD}" -c:v libx264 -pix_fmt "$1" -bf 3 -g 24 \ + -vf "${P709}" "${TAGS_709[@]}" "${OUT}/variant_$2.mov" +done +for v in "yuv422p hevc_8bit_422" "yuv444p hevc_8bit_444"; do + set -- $v + "${FF[@]}" -f lavfi -i "${HD}" -c:v libx265 -pix_fmt "$1" -tag:v hvc1 \ + -x265-params "bframes=4:keyint=24:log-level=error" -vf "${P709}" "${TAGS_709[@]}" \ + "${OUT}/variant_$2.mov" +done + +# High bit depth HEVC needs a libx265 built for it, which the vcpkg one isn't. Point +# FFMPEG_HIGHBIT at an ffmpeg that has one (e.g. "rez-env ffmpeg-9.0 -- ffmpeg") to +# also write these. +if [ -n "${FFMPEG_HIGHBIT:-}" ]; then + for v in "yuv420p12le hevc_12bit_420" "yuv422p12le hevc_12bit_422" \ + "yuv444p12le hevc_12bit_444" "yuv444p10le hevc_10bit_444"; do + set -- $v + ${FFMPEG_HIGHBIT} -hide_banner -loglevel error -y -f lavfi -i "${HD}" -c:v libx265 \ + -pix_fmt "$1" -tag:v hvc1 -x265-params "bframes=4:keyint=24:log-level=error" \ + -vf "${P709}" "${TAGS_709[@]}" "${OUT}/variant_$2.mov" + done +fi + +# 4K benchmark material +"${FF[@]}" -f lavfi -i "${UHD}" -c:v hevc_videotoolbox -profile:v main10 -pix_fmt p010le \ + -bf 2 -g 24 -b:v 40M -tag:v hvc1 -vf "${P709}" "${TAGS_709[@]}" "${OUT}/bench_uhd_hevc_main10.mov" +"${FF[@]}" -f lavfi -i "${UHD}" -c:v prores_videotoolbox -profile:v 4444 -pix_fmt ayuv64le \ + -vf "${P709}" "${TAGS_709[@]}" "${OUT}/bench_uhd_prores_4444.mov" + +# testsrc2 compresses unrealistically well, so these add temporal grain to get bitrates and +# decode cost closer to real camera footage. +GRAIN="noise=alls=12:allf=t+u" +"${FF[@]}" -f lavfi -i "${UHD}" -c:v hevc_videotoolbox -profile:v main10 -pix_fmt p010le \ + -bf 2 -g 24 -b:v 80M -tag:v hvc1 -vf "${GRAIN},${P709}" "${TAGS_709[@]}" \ + "${OUT}/bench_uhd_hevc_main10_grain.mov" +"${FF[@]}" -f lavfi -i "${UHD}" -c:v prores_videotoolbox -profile:v hq -pix_fmt p210le \ + -vf "${GRAIN},${P709}" "${TAGS_709[@]}" "${OUT}/bench_uhd_prores_422hq_grain.mov" +"${FF[@]}" -f lavfi -i "${UHD}" -c:v prores_videotoolbox -profile:v 4444 -pix_fmt ayuv64le \ + -vf "${GRAIN},${P709}" "${TAGS_709[@]}" "${OUT}/bench_uhd_prores_4444_grain.mov" +"${FF[@]}" -f lavfi -i "${UHD}" -c:v libx264 -pix_fmt yuv420p -bf 3 -g 24 -b:v 60M \ + -vf "${GRAIN},${P709}" "${TAGS_709[@]}" "${OUT}/bench_uhd_h264_grain.mov" + +echo "Done."