diff --git a/src/plugin/python_plugins/CMakeLists.txt b/src/plugin/python_plugins/CMakeLists.txt index 5cc0206c8..710e038ff 100644 --- a/src/plugin/python_plugins/CMakeLists.txt +++ b/src/plugin/python_plugins/CMakeLists.txt @@ -4,4 +4,5 @@ add_python_plugin(on_screen_version_name) add_python_plugin(viewport_field_chart) add_python_plugin(annotations_exporter) add_python_plugin(basic_viewport_mask) -add_python_plugin(utility) \ No newline at end of file +add_python_plugin(utility) +add_python_plugin(scene_cut_detector) \ No newline at end of file diff --git a/src/plugin/python_plugins/scene_cut_detector/README.md b/src/plugin/python_plugins/scene_cut_detector/README.md new file mode 100644 index 000000000..2e9d9ae6d --- /dev/null +++ b/src/plugin/python_plugins/scene_cut_detector/README.md @@ -0,0 +1,50 @@ +# Scene Cut Detector + +This experimental Python plugin detects hard scene cuts in a selected video and creates a new xSTUDIO Timeline with one sub-clip per detected scene. + +## Requirements + +- A Windows xSTUDIO build with the Python plugin support enabled. +- `ffmpeg.exe` available on `PATH`. +- `numpy` available to xSTUDIO's bundled Python environment. + +The plugin intentionally uses an `ffmpeg` subprocess instead of `opencv-python` or PyAV. This keeps the runtime dependency small and avoids relying on compiled Python video-decoding wheels. + +## Usage + +1. Start the development build with `build/run_xstudio.bat`. +2. Add a video to a Playlist and select it. +3. Right-click the media item and choose **Detect Scene Cuts...**. +4. Choose a detector mode, threshold, and minimum scene length. +5. Click **Detect Scene Cuts**. + +The result is a new Timeline in the inspected Playlist. The source media is registered on that Timeline and split at the detected frame numbers. + +## Parameters + +- **Content (fast cuts)** compares each frame with the previous frame using a fixed mean-pixel-difference threshold. +- **Adaptive (camera motion)** compares a frame against a rolling average of recent differences, which can help when the source contains camera movement. +- **Cut Threshold** controls how large a frame change must be. Lower values detect more changes; higher values reduce false positives. +- **Minimum Scene Length (frames)** prevents cuts from being placed too close together. + +Detection currently runs synchronously, so the xSTUDIO UI may be unresponsive while a long source is scanned. This is a known limitation of the first version. + +## Development + +The pure detection and URI parsing helpers are in `detector.py`. They can be tested without a running xSTUDIO session: + +```powershell +python -m pytest src/plugin/python_plugins/scene_cut_detector/test/test_scene_cut_detector.py +``` + +For a development build, redeploy the Python plugin after editing: + +```powershell +cmake --build build --target COPY_PY_PLUGIN_scene_cut_detector +``` + +Then restart `build/run_xstudio.bat -n`. + +## Review status + +This is an initial contribution under review. Known follow-up work includes background processing with a thread-safe xSTUDIO API boundary, richer detector options, timeline/audio handling, and packaging/runtime discovery of `ffmpeg`. diff --git a/src/plugin/python_plugins/scene_cut_detector/__init__.py b/src/plugin/python_plugins/scene_cut_detector/__init__.py new file mode 100644 index 000000000..d0e20b23e --- /dev/null +++ b/src/plugin/python_plugins/scene_cut_detector/__init__.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 +try: + from .scene_cut_detector_plugin import create_plugin_instance +except ImportError: + # The 'xstudio' package is only available inside xSTUDIO's embedded + # Python environment, not when running the unit tests in test/ + # standalone. Importing this package must not fail in that case. + create_plugin_instance = None + diff --git a/src/plugin/python_plugins/scene_cut_detector/detector.py b/src/plugin/python_plugins/scene_cut_detector/detector.py new file mode 100644 index 000000000..6df72d65e --- /dev/null +++ b/src/plugin/python_plugins/scene_cut_detector/detector.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Pure helpers for Scene Cut Detector.""" + +from urllib.parse import unquote + +import numpy as np + +DETECTOR_OPTIONS = ["Content (fast cuts)", "Adaptive (camera motion)"] + + +def uri_to_local_path(uri_str): + """Convert an xSTUDIO file URI to a local filesystem path.""" + path_part = uri_str + if path_part.startswith("file://"): + path_part = path_part[len("file://"):] + if path_part.startswith("localhost"): + path_part = path_part[len("localhost"):] + elif path_part.startswith("file:"): + path_part = path_part[len("file:"):] + + path_part = unquote(path_part).lstrip("/") + if len(path_part) > 1 and path_part[1] == ":": + return path_part + return "/" + path_part + + +def compute_cut_frames(frames, detector_name, threshold, min_scene_len): + """Return frame numbers where consecutive frames indicate a scene cut.""" + cut_frames = [] + recent_diffs = [] + previous_frame = None + frame_index = 0 + last_cut_frame = -min_scene_len + + for frame in frames: + frame = np.asarray(frame, dtype=np.float32) + + if previous_frame is not None: + diff = float(np.abs(frame - previous_frame).mean()) + + if detector_name == DETECTOR_OPTIONS[1] and recent_diffs: + rolling_mean = sum(recent_diffs) / len(recent_diffs) + is_cut = diff > threshold and diff > 3.0 * max(rolling_mean, 1.0) + else: + is_cut = diff > threshold + + if is_cut and (frame_index - last_cut_frame) >= min_scene_len: + cut_frames.append(frame_index) + last_cut_frame = frame_index + + recent_diffs.append(diff) + if len(recent_diffs) > 15: + recent_diffs.pop(0) + + previous_frame = frame + frame_index += 1 + + return cut_frames diff --git a/src/plugin/python_plugins/scene_cut_detector/qml/SceneCutDetector.1/SceneCutDetectorDialog.qml b/src/plugin/python_plugins/scene_cut_detector/qml/SceneCutDetector.1/SceneCutDetectorDialog.qml new file mode 100644 index 000000000..fff61194e --- /dev/null +++ b/src/plugin/python_plugins/scene_cut_detector/qml/SceneCutDetector.1/SceneCutDetectorDialog.qml @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: Apache-2.0 +import QtQuick +import QtQuick.Layouts + +import xstudio.qml.models 1.0 +import xStudio 1.0 + +XsWindow { + + id: dialog + width: 420 + title: "Detect Scene Cuts" + minimumWidth: 400 + minimumHeight: 260 + + flags: Qt.platform.os === "windows" ? Qt.Window : Qt.Dialog + + property var widgetHeight: 24 + + XsModuleData { + id: plugin_attrs + modelDataName: "scene_cut_detector_attrs" + } + XsAttributeValue { + id: threshold + attributeTitle: "Cut Threshold" + model: plugin_attrs + } + XsAttributeValue { + id: min_scene_len + attributeTitle: "Minimum Scene Length (frames)" + model: plugin_attrs + } + + ColumnLayout { + + width: parent.width + + GridLayout { + + Layout.fillWidth: true + Layout.margins: 20 + columnSpacing: 12 + rowSpacing: 14 + columns: 2 + + XsText { + text: "Detector" + Layout.alignment: Qt.AlignRight + } + + XsAttrComboBox { + id: detectorChoice + Layout.alignment: Qt.AlignLeft + Layout.minimumWidth: 220 + Layout.preferredHeight: widgetHeight + attr_title: "Detector" + attr_model_name: "scene_cut_detector_attrs" + } + + XsText { + text: "Cut Threshold" + Layout.alignment: Qt.AlignRight + } + + XsTextField { + id: thresholdField + Layout.preferredWidth: 220 + Layout.preferredHeight: widgetHeight + text: threshold.value != undefined ? threshold.value.toString() : "" + onEditingFinished: threshold.value = parseFloat(text) + onAccepted: threshold.value = parseFloat(text) + } + + XsText { + text: "Min Scene Length (frames)" + Layout.alignment: Qt.AlignRight + } + + XsTextField { + id: minSceneLenField + Layout.preferredWidth: 220 + Layout.preferredHeight: widgetHeight + text: min_scene_len.value != undefined ? min_scene_len.value.toString() : "" + onEditingFinished: min_scene_len.value = parseInt(text) + onAccepted: min_scene_len.value = parseInt(text) + } + } + } + + RowLayout { + + spacing: 0 + anchors.left: parent.left + anchors.right: parent.right + anchors.bottom: parent.bottom + anchors.margins: 10 + + Item { + Layout.fillWidth: true + } + + XsSimpleButton { + Layout.alignment: Qt.AlignRight + text: qsTr("Cancel") + width: XsStyleSheet.primaryButtonStdWidth*2 + onClicked: { + dialog.hide() + dialog.destroy() + } + } + + XsSimpleButton { + Layout.alignment: Qt.AlignRight + Layout.leftMargin: 5 + text: qsTr("Detect Scene Cuts") + width: XsStyleSheet.primaryButtonStdWidth*3 + onClicked: { + + var return_val = python_callback("run_from_dialog") + + if (Array.isArray(return_val)) { + // run_from_dialog returns [True, message] + dialogHelpers.messageDialogFunc("Detect Scene Cuts", return_val[1], "Ok") + if (return_val[0] == true) { + dialog.visible = false + } + } else { + // report (likely) error of some sort + dialogHelpers.errorDialogFunc("Detect Scene Cuts", return_val) + } + } + } + } +} diff --git a/src/plugin/python_plugins/scene_cut_detector/scene_cut_detector_plugin.py b/src/plugin/python_plugins/scene_cut_detector/scene_cut_detector_plugin.py new file mode 100644 index 000000000..5bff0a3f4 --- /dev/null +++ b/src/plugin/python_plugins/scene_cut_detector/scene_cut_detector_plugin.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: Apache-2.0 +"""Scene Cut Detector plugin. + +Adds a "Detect Scene Cuts" command to the playlist media-item context menu +(and, as a fallback, to the currently selected Clip on an open Timeline). +It analyses the source video for hard scene cuts and builds a new Timeline +in the current Playlist where the source has been split into one sub-clip +per detected scene. + +Detection is done by piping downscaled grayscale frames out of ffmpeg and +looking for large frame-to-frame differences (numpy). This avoids relying +on compiled Python video-decoding extensions (e.g. opencv-python, PyAV): +those wheels' native DLLs failed to load in testing on this machine, while +a plain ffmpeg.exe subprocess + numpy worked fine. + +Requires 'ffmpeg' (and 'ffprobe') to be available on PATH, e.g.: + winget install Gyan.FFmpeg +""" + +import shutil +import subprocess +from pathlib import Path + +import numpy as np + +from xstudio.plugin import PluginBase +from .detector import DETECTOR_OPTIONS, compute_cut_frames, uri_to_local_path + +# path to QML resources relative to this .py file +qml_folder_name = "qml/SceneCutDetector.1" + +dialog_qml = """ +SceneCutDetectorDialog { +} +""" + +# Frames are downscaled to this size before comparison - big enough to +# catch real cuts, small enough to decode/compare quickly. +_ANALYSIS_WIDTH = 64 +_ANALYSIS_HEIGHT = 36 +_ANALYSIS_FRAME_BYTES = _ANALYSIS_WIDTH * _ANALYSIS_HEIGHT + + +class MissingFFmpegError(Exception): + """Raised when the 'ffmpeg' executable can't be found on PATH.""" + + +class SceneCutDetector(PluginBase): + """Split a media item into sub-clips at each detected scene cut.""" + + def __init__(self, connection): + + PluginBase.__init__( + self, + connection, + name="SceneCutDetector", + qml_folder=qml_folder_name, + ) + + # User-facing settings, remembered as preferences between sessions, + # and exposed as a group so our settings dialog QML can bind to them + # (same mechanism AnnotationsExporter uses for its export dialog). + self.detector_type = self.add_attribute( + attribute_name="Detector", + attribute_value=DETECTOR_OPTIONS[0], + attribute_role_data={ + "combo_box_options": DETECTOR_OPTIONS, + "combo_box_options_enabled": [True, True], + }, + register_as_preference=True, + ) + + self.threshold = self.add_attribute( + attribute_name="Cut Threshold", + attribute_value=27.0, + register_as_preference=True, + ) + + self.min_scene_len = self.add_attribute( + attribute_name="Minimum Scene Length (frames)", + attribute_value=15, + register_as_preference=True, + ) + + for attr in (self.detector_type, self.threshold, self.min_scene_len): + attr.expose_in_ui_attrs_group("scene_cut_detector_attrs") + + # Single menu item - opens the settings dialog, which has its own + # "Detect Scene Cuts" button to trigger the run. + self.menu_id = self.insert_menu_item( + menu_model_name="media_list_menu_", + menu_text="Detect Scene Cuts...", + menu_path="", + menu_item_position=20.0, + callback=self.menu_callback, + ) + + self.connect_to_ui() + + # ------------------------------------------------------------------ + # Menu handling + # ------------------------------------------------------------------ + + def menu_callback(self): + """Open the settings dialog for the currently selected media.""" + + media = self._selected_media() + if media is None: + self.popup_message_box( + "Detect Scene Cuts", + "Select a single video media item (in a Playlist, or a Clip " + "on a Timeline) before running scene cut detection.", + ) + return + + self.create_qml_item(dialog_qml) + + def run_from_dialog(self): + """Called from the settings dialog's "Detect Scene Cuts" button. + + Runs synchronously (blocking the UI) - see NOTE in + _detect_and_build_timeline for why this can't use a background + thread. Returns [True, message] on success, or an error string, + matching the convention AnnotationsExportDialog.qml expects. + """ + + media = self._selected_media() + if media is None: + return "No media selected." + + try: + return self._detect_and_build_timeline( + media, + self.detector_type.value(), + self.threshold.value(), + int(self.min_scene_len.value()), + ) + except Exception as exc: # noqa: BLE001 - surfaced to the dialog + return str(exc) + + def _selected_media(self): + """Resolve the media the user wants to analyse. + + Looks first at the playlist media-list selection, then falls back to + the selected Clip on the currently inspected Timeline. + """ + + selected = self.connection.api.session.selected_media + if selected: + return selected[0] + + container = self.connection.api.session.inspected_container + selection = getattr(container, "selection", None) + if selection: + clip = selection[0] + return getattr(clip, "media", None) + + return None + + # ------------------------------------------------------------------ + # Detection + timeline construction + # ------------------------------------------------------------------ + + def _detect_and_build_timeline(self, media, detector_name, threshold, min_scene_len): + """Returns [True, message] on success. Raises on failure - the + caller (run_from_dialog) turns exceptions into an error string for + the settings dialog to display. + """ + try: + file_path = self._resolve_media_path(media) + cut_frames = self._detect_cuts(file_path, detector_name, threshold, min_scene_len) + + if not cut_frames: + return [True, "No scene cuts were detected in \"{}\".".format(media.name)] + + scene_count = self._build_timeline(media, cut_frames) + + return [ + True, + "Created a new Timeline with {} scene(s) for \"{}\".".format( + scene_count, media.name + ), + ] + except MissingFFmpegError: + raise RuntimeError( + "'ffmpeg' (and 'ffprobe') could not be found on PATH. " + "Install it with, e.g.: winget install Gyan.FFmpeg" + ) + except Exception: # noqa: BLE001 - log full traceback, then re-raise + import traceback + traceback.print_exc() + raise + + def _resolve_media_path(self, media): + """Get a local filesystem path for the selected media's current source.""" + source = media.media_source() + media_ref = source.media_reference + if media_ref is None: + raise RuntimeError("Selected media has no resolvable media reference.") + + file_path = uri_to_local_path(str(media_ref.uri())) + if not Path(file_path).is_file(): + raise RuntimeError("Media file not found on disk: {}".format(file_path)) + + return file_path + + def _detect_cuts(self, file_path, detector_name, threshold, min_scene_len): + """Decode downscaled grayscale frames via ffmpeg and delegate to + compute_cut_frames() for the actual difference/threshold logic. + + Returns a sorted list of 0-based frame numbers (in source decode + order) where a new scene starts. + """ + if shutil.which("ffmpeg") is None: + raise MissingFFmpegError() + + cmd = [ + "ffmpeg", + "-v", "error", + "-i", file_path, + "-vf", "scale={}:{}".format(_ANALYSIS_WIDTH, _ANALYSIS_HEIGHT), + "-pix_fmt", "gray", + "-f", "rawvideo", + "-", + ] + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + frame_count = 0 + + def read_frames(): + nonlocal frame_count + while True: + raw = proc.stdout.read(_ANALYSIS_FRAME_BYTES) + if len(raw) < _ANALYSIS_FRAME_BYTES: + break + frame_count += 1 + yield np.frombuffer(raw, dtype=np.uint8) + + try: + cut_frames = compute_cut_frames(read_frames(), detector_name, threshold, min_scene_len) + finally: + proc.stdout.close() + stderr = proc.stderr.read() + proc.wait() + + if proc.returncode != 0 and frame_count == 0: + raise RuntimeError( + "ffmpeg failed to decode \"{}\":\n{}".format( + file_path, stderr.decode(errors="replace") + ) + ) + + return cut_frames + + def _build_timeline(self, media, cut_frames): + # NOTE: track.insert_clip() (and other helpers like create_video_track, + # insert_gap) call Connection.remote_spawn(), which only works over a + # genuine external/remote Connection - it fails with "Failed to spawn + # remote actor" when called via a resident plugin's own in-process + # connection (confirmed by testing: identical code succeeds from an + # external script using its own Connection(auto_connect=True), but + # fails via self.connection here). So we open our own loopback + # connection to do the actor-spawning parts of timeline construction. + from xstudio.connection import Connection as _Connection + + helper = _Connection(auto_connect=True) + + helper_media = helper.api.session.selected_media + if not helper_media: + raise RuntimeError("Media selection was lost before the timeline could be built.") + helper_media = helper_media[0] + + playlist = helper.api.session.inspected_container + # Fall back to the first playlist if the inspected container isn't + # a playlist (e.g. user already had a Timeline open). + if not hasattr(playlist, "create_timeline"): + playlist = helper.api.session.playlists[0] + + # Build the timeline with its default track(s) (with_tracks=True) + # rather than manually spawning a Track and inserting it into the + # stack ourselves - the latter reliably hung/crashed xstudio when + # tested directly (a Clip couldn't be inserted into a manually + # constructed track). A Timeline also keeps its own internal media + # registry, so the source media must be added to the timeline (via + # add_media) before a Clip referencing it can be inserted. + _uuid, timeline = playlist.create_timeline( + name="{} - Scene Cuts".format(media.name), + with_tracks=True, + ) + + timeline.add_media(helper_media) + + track = timeline.video_tracks[0] + clip = track.insert_clip(helper_media) + + # NOTE: use track.split(frame) (absolute track-position split), not + # track.split_child(clip, frame). After the first split, the + # original `clip` object only spans the first sub-range, so a later + # (larger) absolute frame number is no longer valid relative to it - + # that raised "Invalid frame to split on" when tested. track.split() + # looks up the item at that absolute frame itself, so it stays + # correct across repeated splits. + for frame in sorted(cut_frames): + track.split(frame) + + for index, scene_clip in enumerate(track.clips, start=1): + scene_clip.item_name = "Scene {:03d}".format(index) + + return len(track.clips) + + +def create_plugin_instance(connection): + """Create and return a plugin instance.""" + return SceneCutDetector(connection) diff --git a/src/plugin/python_plugins/scene_cut_detector/test/test_scene_cut_detector.py b/src/plugin/python_plugins/scene_cut_detector/test/test_scene_cut_detector.py new file mode 100644 index 000000000..38f121075 --- /dev/null +++ b/src/plugin/python_plugins/scene_cut_detector/test/test_scene_cut_detector.py @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +import sys +from pathlib import Path + +# detector.py has no xstudio dependency, but it lives inside the plugin +# package (whose __init__.py does import xstudio). Importing it as a +# top-level module (rather than via the scene_cut_detector package) avoids +# pulling in that unrelated, app-only dependency just to run these tests. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import numpy as np + +from detector import DETECTOR_OPTIONS, compute_cut_frames, uri_to_local_path + + +def test_uri_to_local_path_handles_xstudio_windows_uri(): + assert uri_to_local_path("file://localhost//C:/Users/test/clip.mov") == "C:/Users/test/clip.mov" + + +def test_uri_to_local_path_handles_posix_uri(): + assert uri_to_local_path("file:///tmp/clip.mov") == "/tmp/clip.mov" + + +def test_content_detector_finds_large_frame_change(): + frames = [ + np.zeros((2, 2), dtype=np.uint8), + np.zeros((2, 2), dtype=np.uint8), + np.full((2, 2), 255, dtype=np.uint8), + np.full((2, 2), 255, dtype=np.uint8), + ] + + assert compute_cut_frames(frames, DETECTOR_OPTIONS[0], 27.0, 1) == [2] + + +def test_minimum_scene_length_suppresses_nearby_cuts(): + frames = [ + np.zeros((2, 2), dtype=np.uint8), + np.full((2, 2), 255, dtype=np.uint8), + np.zeros((2, 2), dtype=np.uint8), + np.full((2, 2), 255, dtype=np.uint8), + ] + + assert compute_cut_frames(frames, DETECTOR_OPTIONS[0], 27.0, 3) == [1] + + +def test_constant_frames_have_no_cuts(): + frames = [np.full((2, 2), 42, dtype=np.uint8) for _ in range(10)] + + assert compute_cut_frames(frames, DETECTOR_OPTIONS[0], 27.0, 1) == []