ffmpeg: ffmpeg_dec: core module framework and audio decoders (FLAC, MP3, AAC, Opus)

posix/include/rtos/idc.h

@@ -28,6 +28,9 @@
 /** \brief IDC send non-blocking flag. */
 #define IDC_NON_BLOCKING	1
 
+/** \brief IDC send blocking flag, wait for completion without timing out. */
+#define IDC_BLOCKING_NO_TIMEOUT	4
+
 /** \brief IDC send core power up flag. */
 #define IDC_POWER_UP		2
 

src/audio/CMakeLists.txt

@@ -41,6 +41,9 @@ if(NOT CONFIG_COMP_MODULE_SHARED_LIBRARY_BUILD)
 	if(CONFIG_COMP_DRC)
 		add_subdirectory(drc)
 	endif()
+	if(CONFIG_COMP_FFMPEG_DEC)
+		add_subdirectory(ffmpeg_dec)
+	endif()
 	if(CONFIG_COMP_FIR)
 		add_subdirectory(eq_fir)
 	endif()

src/audio/Kconfig

@@ -196,6 +196,7 @@ rsource "dcblock/Kconfig"
 rsource "drc/Kconfig"
 rsource "eq_fir/Kconfig"
 rsource "eq_iir/Kconfig"
+rsource "ffmpeg_dec/Kconfig"
 rsource "google/Kconfig"
 rsource "igo_nr/Kconfig"
 rsource "mfcc/Kconfig"

src/audio/base_fw.c

@@ -108,6 +108,34 @@ static void get_codec_info(struct sof_tlv **tuple)
 	codec_info.items[codec_info.count++] =
 		SET_CODEC_INFO_ITEM(SND_AUDIOCODEC_PCM, SOF_IPC_STREAM_PLAYBACK);
 #endif
+#ifdef CONFIG_FFMPEG_DEC_FLAC
+	/* FLAC decode is provided by the ffmpeg_dec (libavcodec) module. */
+	codec_info.items[codec_info.count++] =
+		SET_CODEC_INFO_ITEM(SND_AUDIOCODEC_FLAC, SOF_IPC_STREAM_PLAYBACK);
+#endif
+#if defined(CONFIG_FFMPEG_DEC_AAC) && !defined(CONFIG_CADENCE_CODEC_AAC_DEC)
+	/* AAC-LC decode is provided by the ffmpeg_dec (libavcodec) module (guarded so
+	 * it is not advertised twice when the Cadence AAC decoder is also present).
+	 */
+	codec_info.items[codec_info.count++] =
+		SET_CODEC_INFO_ITEM(SND_AUDIOCODEC_AAC, SOF_IPC_STREAM_PLAYBACK);
+#endif
+#if defined(CONFIG_FFMPEG_DEC_MP3) && !defined(CONFIG_CADENCE_CODEC_MP3_DEC)
+	/* MP3 decode is provided by the ffmpeg_dec (libavcodec) module (guarded so
+	 * it is not advertised twice when the Cadence MP3 decoder is also present).
+	 */
+	codec_info.items[codec_info.count++] =
+		SET_CODEC_INFO_ITEM(SND_AUDIOCODEC_MP3, SOF_IPC_STREAM_PLAYBACK);
+#endif
+#ifdef CONFIG_FFMPEG_DEC_OPUS
+	/* Opus decode is provided by the ffmpeg_dec (libavcodec) module.
+	 * Advertise OPUS_RAW and BESPOKE.
+	 */
+	codec_info.items[codec_info.count++] =
+		SET_CODEC_INFO_ITEM(SND_AUDIOCODEC_OPUS_RAW, SOF_IPC_STREAM_PLAYBACK);
+	codec_info.items[codec_info.count++] =
+		SET_CODEC_INFO_ITEM(SND_AUDIOCODEC_BESPOKE, SOF_IPC_STREAM_PLAYBACK);
+#endif
 
 	if (!codec_info.count)
 		return;

src/audio/copier/copier.c

@@ -1054,6 +1054,7 @@ __cold static int copier_set_configuration(struct processing_module *mod,
 					   size_t response_size)
 {
 	struct comp_dev *dev = mod->dev;
+	struct copier_data *cd = module_get_private_data(mod);
 
 	assert_can_be_cold();
 
@@ -1066,6 +1067,13 @@ __cold static int copier_set_configuration(struct processing_module *mod,
 		return set_attenuation(dev, fragment_size, (const char *)fragment);
 	case IPC4_COPIER_MODULE_CFG_PARAM_CHANNEL_MAP:
 		return set_chmap(dev, fragment, fragment_size);
+	case IPC4_COPIER_MODULE_CFG_PARAM_DRAIN_BYTES:
+		if (!cd->hd || fragment_size < sizeof(uint64_t))
+			return -EINVAL;
+		memcpy_s(&cd->hd->drain_bytes, sizeof(cd->hd->drain_bytes),
+			 fragment, sizeof(uint64_t));
+		comp_dbg(dev, "copier drain_bytes=%u", (uint32_t)cd->hd->drain_bytes);
+		return 0;
 	default:
 		return -EINVAL;
 	}

src/audio/copier/copier.h

@@ -189,7 +189,14 @@ enum ipc4_copier_module_config_params {
 	 * from gateway buffer to copier with any order.
 	 * Same mapping will be applied for all copier sinks.
 	 */
-	IPC4_COPIER_MODULE_CFG_PARAM_CHANNEL_MAP = 7
+	IPC4_COPIER_MODULE_CFG_PARAM_CHANNEL_MAP = 7,
+	/*
+	 * SOF extension (compress drain): total committed gateway bytes. The host
+	 * sends this on DRAIN so the host copier can flag end-of-stream once it has
+	 * delivered exactly this many bytes downstream - the free-running HDA
+	 * gateway write pointer cannot serve as an EOS boundary. Payload: u64.
+	 */
+	IPC4_COPIER_MODULE_CFG_PARAM_DRAIN_BYTES = 100
 };
 
 struct ipc4_copier_config_timestamp_init_data {

src/audio/copier/host_copier.h

@@ -75,6 +75,7 @@ struct host_data {
 	uint16_t stream_tag;
 	uint16_t no_stream_position; /**< 1 means don't send stream position */
 	uint64_t total_data_processed;
+	uint64_t drain_bytes;		/**< compress-drain committed byte total; 0 = inactive */
 	uint64_t nobytes_last_logged;	/**< uptime, when "no bytes" was last logged */
 	unsigned int n_skipped;		/**< number of "no bytes" skipped */
 	uint8_t cont_update_posn;	/**< 1 means continuous update stream position */

src/audio/ffmpeg_dec/CMakeLists.txt

@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+if(CONFIG_COMP_FFMPEG_DEC STREQUAL "m" AND DEFINED CONFIG_LLEXT)
+  add_subdirectory(llext ${PROJECT_BINARY_DIR}/ffmpeg_dec_llext)
+  add_dependencies(app ffmpeg_dec)
+else()
+  add_local_sources(sof ffmpeg_dec.c)
+
+  if(CONFIG_COMP_FFMPEG_DEC_STUB)
+    add_local_sources(sof ffmpeg_dec-stub.c)
+  else()
+    add_local_sources(sof ffmpeg_dec-ffmpeg.c)
+    add_local_sources(sof ffmpeg_dec-convert.c)
+    # Scalar soft-float for the SOF glue that touches float. core-isa.h reports
+    # XCHAL_HAVE_FP=1 so LLVM would lower the FLT/FLTP -> Q1.31 sample conversion
+    # in ffmpeg_dec_emit_frame() to hardware FP0 ops (lsi/mul.s/trunc.s), which
+    # fault EXCCAUSE 0 (illegal instruction) on this CONFIG_FPU=n image -- the FP0
+    # coprocessor is never enabled. Disable the fp feature so LLVM emits
+    # soft-float libcalls, exactly as ffmpeg.cmake does for FFmpeg/libm. FLAC/MP3
+    # never hit this (their emit path is fixed-point); AAC/Opus (FLT/FLTP) do.
+    # NB: modules_sof (the library that compiles these) is created in
+    # zephyr/CMakeLists.txt, a different directory scope -- so the property
+    # must target THAT scope via TARGET_DIRECTORY, else CMake silently
+    # ignores it and the glue keeps hardware FP0.
+    set(_ffmpeg_dec_sources
+        ${CMAKE_CURRENT_SOURCE_DIR}/ffmpeg_dec.c
+        ${CMAKE_CURRENT_SOURCE_DIR}/ffmpeg_dec-ffmpeg.c
+        ${CMAKE_CURRENT_SOURCE_DIR}/ffmpeg_dec-convert.c)
+    if(CMAKE_C_COMPILER_ID MATCHES "Clang")
+        set_source_files_properties(
+            ${_ffmpeg_dec_sources}
+            TARGET_DIRECTORY modules_sof
+            PROPERTIES COMPILE_OPTIONS "-Xclang;-target-feature;-Xclang;-fp")
+    endif()
+    # NOTE: ffmpeg_dec-shims.c / ffmpeg_dec-alloc.c are LLEXT-only (see
+    # llext/CMakeLists.txt). They provide a full private libc (malloc, str*,
+    # snprintf, ...) the isolated LLEXT cannot import; in a built-in (=y) image
+    # Zephyr's common libc already provides those (whole-archived), so including
+    # them here would multiply-define malloc/str*/snprintf. Instead
+    # ffmpeg_dec-builtin-libc.c supplies ONLY the handful of libc symbols common
+    # libc does NOT export that libav* still references (stderr/fread/getenv/
+    # sscanf/..., z_errno_wrap, and the __errno bridge the newlib math objects
+    # linked below use on error paths) plus a no-op ffmpeg_dec_libc_bind (=y
+    # allocates from common-libc malloc, no module-heap binding needed). Double
+    # libm itself is real newlib math, not a stub -- see FFMPEG_TARGET_LIBM below.
+    add_local_sources(sof ffmpeg_dec-builtin-libc.c)
+
+    # =y heap routing: --wrap malloc/free/realloc/calloc (below) onto the SOF DSP
+    # heap so FFmpeg av_malloc() draws from the NOBITS .heap_mem instead of the
+    # image_size-capped common-libc arena (AAC-LC needs a single ~541 KiB alloc
+    # that the arena cannot hold). Also provides ffmpeg_dec_libc_bind().
+    add_local_sources(sof ffmpeg_dec-builtin-alloc.c)
+
+    # Built-in (=y) real backend: cross-build the FFmpeg archive via ffmpeg.cmake
+    # and statically link it into the base firmware. Mirrors the google_rtc =y
+    # pattern (src/audio/google/CMakeLists.txt): headers via
+    # zephyr_include_directories, archive via the SOF interface library which
+    # propagates to zephyr.elf. LLD resolves all relocations at link time, so
+    # this avoids the LLEXT runtime-relocation path entirely.
+    is_zephyr(zephyr)
+    if(zephyr)
+      include(${CMAKE_CURRENT_LIST_DIR}/ffmpeg.cmake)
+      zephyr_include_directories(${FFMPEG_INSTALL_DIR}/include)
+      target_link_directories(SOF INTERFACE ${FFMPEG_INSTALL_DIR}/lib)
+      # avcodec must precede the libs it depends on (swresample/avutil), and the
+      # newlib math archive (FFMPEG_TARGET_LIBM) must come LAST so the linker
+      # resolves libav*'s double libm references (sin/cos/exp/log/pow/...) against
+      # it -- as an archive it only pulls the math objects actually referenced.
+      # Do NOT add -lc: the base
+      # image links bare-metal (-nostdlib) and the remaining libc symbols libav*
+      # needs resolve from Zephyr's minimal libc already in the link (plus the
+      # stubs in ffmpeg_dec-builtin-libc.c). FLAC is integer-only so it never
+      # calls libm at runtime, but the references still resolve to real kernels
+      # now, and lossy decoders (AAC/Opus/Vorbis) need them to decode correctly.
+      target_link_libraries(SOF INTERFACE avcodec swresample avutil)
+      # libm (newlib double math) LAST so libav*/libshine references
+      # (modf/cos/sin/log/sqrt/exp2) resolve against it.
+      target_link_libraries(SOF INTERFACE ${FFMPEG_TARGET_LIBM})
+      # Redirect libav* malloc/free/realloc/calloc to __wrap_* in
+      # ffmpeg_dec-builtin-alloc.c (SOF DSP heap). --wrap resolves references to
+      # the symbol -> __wrap_<sym>, and __real_<sym> -> the original common-libc
+      # allocator (used pre-bind). Propagates to the zephyr.elf link via SOF.
+      target_link_options(SOF INTERFACE
+        -Wl,--wrap=malloc -Wl,--wrap=free -Wl,--wrap=realloc -Wl,--wrap=calloc)
+      # Both archives must be built before modules_sof links against them:
+      # ffmpeg_ext (libav*) and newlib_m (libnewlib_m.a, the double math compiled
+      # from vendored newlib source with -mlongcalls -- see ffmpeg.cmake 3c).
+      add_dependencies(modules_sof ffmpeg_ext newlib_m)
+    endif()
+  endif()
+endif()