(feature): Decode H.264, HEVC and ProRes with VideoToolbox on MacOs.

cmake/macros.cmake

@@ -77,6 +77,10 @@ if (BUILD_TESTING)
 			PRIVATE TEST_RESOURCE=\"${TEST_RESOURCE}\"
 			PRIVATE ROOT_DIR=\"${ROOT_DIR}\"
 			PRIVATE $<$<CONFIG:Debug>:XSTUDIO_DEBUG=1>
+			# Same platform defines as default_compile_options: the headers rely on
+			# __apple__ to #undef the 'nil' macro that MacTypes.h defines.
+			$<$<PLATFORM_ID:Darwin>:__apple__>
+			$<$<PLATFORM_ID:Darwin>:__OPENGL_4_1__>
 		)
 	endmacro()
 endif (BUILD_TESTING)

share/preference/plugin_media_reader_ffmpeg.json

@@ -12,6 +12,17 @@
 					"datatype": "int",
 					"context": ["APPLICATION"]
 				},
+				"videotoolbox": {
+					"path": "/plugin/media_reader/FFMPEG/videotoolbox",
+					"default_value": "prefer",
+					"description": "macOS only. Decode H.264, HEVC and ProRes video with Apple's VideoToolbox. 'prefer' falls back to FFmpeg's own decoder when VideoToolbox can't decode a stream; 'only' reports an error instead; 'off' always uses FFmpeg's decoder. Applies to media loaded after the change.",
+					"value": "prefer",
+					"datatype": "multichoice string",
+					"options": ["off", "prefer", "only"],
+					"context": ["APPLICATION"],
+					"category": "Playback",
+					"display_name": "VideoToolbox Decoding (macOS)"
+				},
 				"supported": {
 					"path": "/plugin/media_reader/FFMPEG/supported",
 					"description": "Control plugin format support",

src/plugin/media_reader/ffmpeg/src/CMakeLists.txt

@@ -4,6 +4,7 @@ set(SOURCES
 	ffmpeg_stream.cpp
 	ffmpeg_decoder.cpp
 	ffmpeg.cpp
+	vt_repack.cpp
 )
 
 add_library(${PROJECT_NAME} SHARED ${SOURCES})
@@ -46,4 +47,17 @@ else()
 
 endif()
 
+if(APPLE)
+	# VideoToolbox decode backend (see vt_decoder.hpp)
+	target_sources(${PROJECT_NAME} PRIVATE vt_decoder.cpp)
+	target_compile_definitions(${PROJECT_NAME} PRIVATE XSTUDIO_HAS_VIDEOTOOLBOX=1)
+	target_link_libraries(${PROJECT_NAME}
+		PRIVATE
+			"-framework VideoToolbox"
+			"-framework CoreMedia"
+			"-framework CoreVideo"
+			"-framework CoreFoundation"
+	)
+endif()
+
 set_target_properties(${PROJECT_NAME} PROPERTIES LINK_DEPENDS_NO_SHARED true)

src/plugin/media_reader/ffmpeg/src/ffmpeg.cpp

@@ -384,6 +384,7 @@ plugin_manager::PluginFactoryCollection *plugin_factory_collection_ptr() {
 FFMpegMediaReader::FFMpegMediaReader(const utility::JsonStore &prefs)
     : MediaReader("FFMPEG", prefs) {
     readers_per_source_ = 1;
+    videotoolbox_mode_  = VideoToolboxMode::Prefer;
     update_preferences(prefs);
 }
 
@@ -398,6 +399,21 @@ void FFMpegMediaReader::update_preferences(const utility::JsonStore &prefs) {
         spdlog::warn("{} {}", __PRETTY_FUNCTION__, e.what());
     }
 
+    try {
+        const auto mode = videotoolbox_mode_from_string(
+            preference_value<std::string>(prefs, "/plugin/media_reader/FFMPEG/videotoolbox"));
+        if (mode != videotoolbox_mode_) {
+            // Don't hand out a frame decoded under the previous mode, and decode
+            // with the new one from the next request on.
+            last_decoded_image_.reset();
+            decoder.reset();
+            thumbnail_decoder.reset();
+        }
+        videotoolbox_mode_ = mode;
+    } catch (const std::exception &e) {
+        spdlog::warn("{} {}", __PRETTY_FUNCTION__, e.what());
+    }
+
     try {
 
         readers_per_source_ =
@@ -439,9 +455,14 @@ ImageBufPtr FFMpegMediaReader::image(const media::AVFrameID &mptr) {
             return last_decoded_image_;
         }
 
-        if (!decoder || decoder->path() != path || decoder->stream_id() != mptr.stream_id()) {
+        if (!decoder || decoder->path() != path || decoder->stream_id() != mptr.stream_id() ||
+            decoder->videotoolbox_mode() != videotoolbox_mode_) {
             decoder.reset(new FFMpegDecoder(
-                path, soundcard_sample_rate_, default_rate_, mptr.stream_id()));
+                path,
+                soundcard_sample_rate_,
+                default_rate_,
+                mptr.stream_id(),
+                videotoolbox_mode_));
         }
 
         decoder->decode_video_frame(mptr.frame(), rt);
@@ -455,6 +476,11 @@ ImageBufPtr FFMpegMediaReader::image(const media::AVFrameID &mptr) {
             last_decoded_image_ = rt;
         }
 
+    } catch (const videotoolbox_unavailable_error &e) {
+        // Always shown on the frame: the user needs to know why there's no picture
+        // (e.g. the 'only' preference, or a build without FFmpeg's decoder).
+        rt = make_blank_image();
+        rt->set_error(e.what());
     } catch (std::exception &e) {
         rt = make_blank_image();
         if (mptr.error() != "") {
@@ -513,7 +539,8 @@ AudioBufPtr FFMpegMediaReader::audio(const media::AVFrameID &mptr) {
 
 xstudio::media::MediaDetail FFMpegMediaReader::detail(const caf::uri &uri) const {
 
-    FFMpegDecoder t_decoder(uri_convert(uri), soundcard_sample_rate_, default_rate_);
+    FFMpegDecoder t_decoder(
+        uri_convert(uri), soundcard_sample_rate_, default_rate_, "", videotoolbox_mode_);
     // N.B. MediaDetail needs frame duration, so invert frame rate
     std::vector<media::StreamDetail> streams;
 
@@ -626,7 +653,11 @@ FFMpegMediaReader::thumbnail(const media::AVFrameID &mptr, const size_t thumb_si
         // DebugTimer d(path, mptr.frame());
         if (!thumbnail_decoder || thumbnail_decoder->path() != path) {
             thumbnail_decoder.reset(new FFMpegDecoder(
-                path, soundcard_sample_rate_, default_rate_, mptr.stream_id()));
+                path,
+                soundcard_sample_rate_,
+                default_rate_,
+                mptr.stream_id(),
+                videotoolbox_mode_));
         }
 
         std::shared_ptr<thumbnail::ThumbnailBuffer> rt =

src/plugin/media_reader/ffmpeg/src/ffmpeg.hpp

@@ -10,6 +10,7 @@ namespace xstudio::media_reader {
 
 namespace ffmpeg {
     class FFMpegDecoder;
+    enum class VideoToolboxMode;
 }
 
 class FFMpegMediaReader : public MediaReader {
@@ -53,6 +54,7 @@ class FFMpegMediaReader : public MediaReader {
     std::shared_ptr<ffmpeg::FFMpegDecoder> thumbnail_decoder;
 
     int readers_per_source_;
+    ffmpeg::VideoToolboxMode videotoolbox_mode_;
     int soundcard_sample_rate_       = {48000};
     int channels_                    = 2;
     utility::FrameRate default_rate_ = {utility::FrameRate(timebase::k_flicks_24fps)};

src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.cpp

@@ -30,15 +30,17 @@ FFMpegDecoder::FFMpegDecoder(
     std::string path,
     const int soundcard_sample_rate,
     const utility::FrameRate default_rate,
-    std::string stream_id)
+    std::string stream_id,
+    VideoToolboxMode videotoolbox_mode)
     : movie_file_path_(std::move(path)),
       last_requested_frame_(-100),
       last_decoded_frame_(-100),
       avc_packet_(nullptr),
       av_format_ctx_(nullptr),
       soundcard_sample_rate_(soundcard_sample_rate),
       stream_id_(std::move(stream_id)),
-      default_rate_(default_rate)
+      default_rate_(default_rate),
+      videotoolbox_mode_(videotoolbox_mode)
 
 {
     open_handles();
@@ -68,7 +70,8 @@ void FFMpegDecoder::open_handles() {
                 av_format_ctx_->streams[i],
                 i,
                 ffmpeg_threads,
-                movie_file_path_);
+                movie_file_path_,
+                videotoolbox_mode_);
 
         } catch ([[maybe_unused]] std::exception &e) {
         }

src/plugin/media_reader/ffmpeg/src/ffmpeg_decoder.hpp

@@ -13,7 +13,8 @@ class FFMpegDecoder {
         std::string path,
         const int soundcard_sample_rate,
         const utility::FrameRate default_rate,
-        std::string stream_id = "");
+        std::string stream_id              = "",
+        VideoToolboxMode videotoolbox_mode = VideoToolboxMode::Off);
 
     ~FFMpegDecoder();
 
@@ -32,6 +33,7 @@ class FFMpegDecoder {
 
     const std::string &path() const { return movie_file_path_; }
     const std::string &stream_id() const { return stream_id_; }
+    VideoToolboxMode videotoolbox_mode() const { return videotoolbox_mode_; }
     int64_t duration_frames() const { return duration_frames_; }
     utility::FrameRate frame_rate(unsigned int stream_idx = UINT_MAX) const;
     utility::Timecode first_frame_timecode();
@@ -102,5 +104,6 @@ class FFMpegDecoder {
     int64_t duration_frames_;
     const std::string stream_id_;
     const utility::FrameRate default_rate_;
+    const VideoToolboxMode videotoolbox_mode_;
 };
 } // namespace xstudio::media_reader::ffmpeg