Skip to content

Rework linking of libduckdb.a AND lib_<name>_extension.a, moving away from building duckdb in-tree #635

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
carlopi wants to merge 2 commits into
base: v2.0-cyanoptera
Choose a base branch
from
Open

Rework linking of libduckdb.a AND lib_<name>_extension.a, moving away from building duckdb in-tree #635

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4ab8375
Separate building of libraries from using libraries
carlopi Sep 24, 2026
138b6a7
Merge branch 'v2.0-cyanoptera' into rework_linking
carlopi Sep 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
334 changes: 330 additions & 4 deletions cmake/duckdb_loader.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ endmacro()
_duckdb_set_default(DUCKDB_SOURCE_PATH
"${CMAKE_CURRENT_SOURCE_DIR}/external/duckdb")

# Prebuilt archives: a folder with libduckdb_static.a and lib<ext>_extension.a.
# When set, DuckDB is not built: headers come from DUCKDB_SOURCE_PATH and the
# archives from this folder.
_duckdb_set_default(DUCKDB_LIBRARY_DIR "")

# Extension list - commonly used extensions for Python
_duckdb_set_default(BUILD_EXTENSIONS "core_functions;parquet;icu;json")

Expand Down Expand Up @@ -63,6 +68,9 @@ _duckdb_set_default(DEBUG_STACKTRACE OFF)
set(DUCKDB_SOURCE_PATH
"${DUCKDB_SOURCE_PATH}"
CACHE PATH "Path to DuckDB source directory")
set(DUCKDB_LIBRARY_DIR
"${DUCKDB_LIBRARY_DIR}"
CACHE PATH "Folder with prebuilt DuckDB archives (empty: build DuckDB)")
set(BUILD_EXTENSIONS
"${BUILD_EXTENSIONS}"
CACHE STRING "Semicolon-separated list of extensions to enable")
Expand Down Expand Up @@ -196,6 +204,315 @@ function(_duckdb_print_summary)
endif()
endfunction()

function(_duckdb_python_source_commit out_var)
# The commit of the DuckDB source tree the headers come from. The build
# backend passes it as GIT_COMMIT_HASH (also for sdists, which have no git
# history); a plain CMake run asks git.
if(GIT_COMMIT_HASH)
set(${out_var} "${GIT_COMMIT_HASH}" PARENT_SCOPE)
return()
endif()
find_package(Git QUIET)
if(GIT_FOUND)
execute_process(
COMMAND "${GIT_EXECUTABLE}" -C "${DUCKDB_SOURCE_PATH}" rev-parse HEAD
OUTPUT_VARIABLE commit
OUTPUT_STRIP_TRAILING_WHITESPACE
RESULT_VARIABLE result
ERROR_QUIET)
if(result EQUAL 0)
set(${out_var} "${commit}" PARENT_SCOPE)
return()
endif()
endif()
message(
FATAL_ERROR
"Cannot determine the commit of ${DUCKDB_SOURCE_PATH}, pass GIT_COMMIT_HASH"
)
endfunction()

function(_duckdb_python_architecture out_var)
# CMAKE_OSX_ARCHITECTURES is what is built when cross-compiling on macOS
if(CMAKE_OSX_ARCHITECTURES)
set(${out_var} "${CMAKE_OSX_ARCHITECTURES}" PARENT_SCOPE)
else()
set(${out_var} "${CMAKE_SYSTEM_PROCESSOR}" PARENT_SCOPE)
endif()
endfunction()

function(_duckdb_python_read_archive_manifest library_dir out_extensions)
# Reads library_dir/duckdb_archives.json and checks that the archives are
# usable with this build. DuckDB's C++ API has no ABI guarantee, so the
# archives must be built from the same commit as the headers, and with the
# same configuration where it changes how DuckDB's headers compile.
set(manifest "${library_dir}/duckdb_archives.json")
if(NOT EXISTS "${manifest}")
message(FATAL_ERROR "DUCKDB_LIBRARY_DIR has no duckdb_archives.json: ${library_dir}")
endif()
file(READ "${manifest}" json)
foreach(field duckdb_commit build_type force_assert system architecture)
string(JSON ${field} ERROR_VARIABLE error GET "${json}" ${field})
if(error)
message(FATAL_ERROR "Invalid ${manifest}: ${error}")
endif()
endforeach()

set(mismatches "")
_duckdb_python_source_commit(source_commit)
if(NOT duckdb_commit STREQUAL source_commit)
string(APPEND mismatches
"\n commit: archives ${duckdb_commit}, headers ${source_commit}")
endif()
# Our code is compiled with DUCKDB_DEBUG_MODE in Debug builds only
set(archives_debug OFF)
if(build_type STREQUAL "Debug")
set(archives_debug ON)
endif()
set(this_debug OFF)
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
set(this_debug ON)
endif()
if(NOT archives_debug STREQUAL this_debug)
string(APPEND mismatches
"\n build type: archives ${build_type}, this build ${CMAKE_BUILD_TYPE}")
endif()
set(archives_force_assert OFF)
if(force_assert)
set(archives_force_assert ON)
endif()
set(this_force_assert OFF)
if(FORCE_ASSERT)
set(this_force_assert ON)
endif()
if(NOT archives_force_assert STREQUAL this_force_assert)
string(APPEND mismatches
"\n force_assert: archives ${force_assert}, this build ${FORCE_ASSERT}")
endif()
if(NOT system STREQUAL CMAKE_SYSTEM_NAME)
string(APPEND mismatches
"\n system: archives ${system}, this build ${CMAKE_SYSTEM_NAME}")
endif()
_duckdb_python_architecture(this_architecture)
if(NOT architecture STREQUAL this_architecture)
string(APPEND mismatches
"\n architecture: archives ${architecture}, this build ${this_architecture}")
endif()
if(mismatches)
message(FATAL_ERROR "The archives in ${library_dir} do not match this build:${mismatches}")
endif()

set(extensions "")
string(JSON count ERROR_VARIABLE error LENGTH "${json}" extensions)
if(error)
message(FATAL_ERROR "Invalid ${manifest}: ${error}")
endif()
if(count GREATER 0)
math(EXPR last "${count} - 1")
foreach(index RANGE ${last})
string(JSON ext GET "${json}" extensions ${index})
list(APPEND extensions ${ext})
endforeach()
endif()
set(${out_extensions} "${extensions}" PARENT_SCOPE)
endfunction()

function(_duckdb_python_add_archive_targets library_dir)
# Creates the targets that DuckDB's own build would provide, as imported
# static libraries from library_dir: duckdb_static, and an <ext>_extension for
# each extension in ARGN. libduckdb_static already bundles DuckDB's
# third-party libraries, so it only needs the system libraries DuckDB links
# (see DUCKDB_SYSTEM_LIBS in DuckDB's src/CMakeLists.txt)
set(prefix "${CMAKE_STATIC_LIBRARY_PREFIX}")
set(suffix "${CMAKE_STATIC_LIBRARY_SUFFIX}")

find_package(Threads REQUIRED)
set(system_libs Threads::Threads ${CMAKE_DL_LIBS})
if(MSVC OR MINGW)
list(APPEND system_libs ws2_32 rstrtmgr)
endif()
if(MSVC)
list(APPEND system_libs bcrypt)
endif()

add_library(duckdb_static STATIC IMPORTED)
set_target_properties(
duckdb_static
PROPERTIES IMPORTED_LOCATION
"${library_dir}/${prefix}duckdb_static${suffix}"
INTERFACE_LINK_LIBRARIES "${system_libs}")

foreach(ext IN LISTS ARGN)
message(STATUS "- ${ext}_extension")
add_library(${ext}_extension STATIC IMPORTED)
# The engine archive must come after the extension archives on the link line
set_target_properties(
${ext}_extension
PROPERTIES IMPORTED_LOCATION
"${library_dir}/${prefix}${ext}_extension${suffix}"
INTERFACE_LINK_LIBRARIES duckdb_static)
endforeach()
endfunction()

function(_duckdb_python_import_archives library_dir)
# Uses the prebuilt archives in library_dir, after checking its manifest
_duckdb_python_read_archive_manifest("${library_dir}" extensions)
set(prefix "${CMAKE_STATIC_LIBRARY_PREFIX}")
set(suffix "${CMAKE_STATIC_LIBRARY_SUFFIX}")
if(NOT EXISTS "${library_dir}/${prefix}duckdb_static${suffix}")
message(
FATAL_ERROR
"DUCKDB_LIBRARY_DIR does not contain ${prefix}duckdb_static${suffix}: ${library_dir}"
)
endif()
foreach(ext IN LISTS extensions)
if(NOT EXISTS "${library_dir}/${prefix}${ext}_extension${suffix}")
message(
FATAL_ERROR
"Extension '${ext}' is in the manifest of ${library_dir} but its archive is missing"
)
endif()
endforeach()
message(STATUS "Using prebuilt DuckDB archives from ${library_dir}")
_duckdb_python_add_archive_targets("${library_dir}" ${extensions})
endfunction()

function(_duckdb_python_build_archives library_dir)
# Builds DuckDB from DUCKDB_SOURCE_PATH as a separate CMake project. It writes
# its archives straight into library_dir, and a manifest once the build
# succeeded, so library_dir has the same layout as a prebuilt folder and can
# be reused as DUCKDB_LIBRARY_DIR by other builds. Only duckdb_static and the
# extensions to link are built.
include(ExternalProject)
if(LINK_EXTENSIONS)
set(extensions ${LINK_EXTENSIONS})
else()
set(extensions ${BUILD_EXTENSIONS})
endif()
set(targets duckdb_static)
foreach(ext IN LISTS extensions)
if(NOT ext IN_LIST BUILD_EXTENSIONS)
message(
FATAL_ERROR
"Cannot link extension '${ext}': DuckDB did not configure it, add it to BUILD_EXTENSIONS"
)
endif()
list(APPEND targets ${ext}_extension)
endforeach()

# Everything DuckDB's build needs from this one. Lists are passed with | as
# separator, see LIST_SEPARATOR below.
set(cmake_args "")
foreach(
var IN
ITEMS CMAKE_BUILD_TYPE
CMAKE_CXX_STANDARD
CMAKE_MSVC_RUNTIME_LIBRARY
CMAKE_MAKE_PROGRAM
CMAKE_TOOLCHAIN_FILE
CMAKE_C_COMPILER
CMAKE_CXX_COMPILER
CMAKE_C_COMPILER_LAUNCHER
CMAKE_CXX_COMPILER_LAUNCHER
CMAKE_C_FLAGS
CMAKE_CXX_FLAGS
CMAKE_OSX_ARCHITECTURES
CMAKE_OSX_DEPLOYMENT_TARGET
CMAKE_OSX_SYSROOT
BUILD_EXTENSIONS
BUILD_SHELL
BUILD_UNITTESTS
BUILD_BENCHMARKS
DISABLE_UNITY
DISABLE_BUILTIN_EXTENSIONS
ENABLE_EXTENSION_AUTOINSTALL
ENABLE_EXTENSION_AUTOLOADING
NATIVE_ARCH
ENABLE_SANITIZER
ENABLE_UBSAN
FORCE_ASSERT
DEBUG_STACKTRACE
OVERRIDE_GIT_DESCRIBE
GIT_COMMIT_HASH)
if(DEFINED ${var} AND NOT "${${var}}" STREQUAL "")
string(REPLACE ";" "|" value "${${var}}")
list(APPEND cmake_args "-D${var}=${value}")
endif()
endforeach()

# Only the archives we link go to library_dir, DuckDB's third-party archives
# (already bundled in libduckdb_static) stay in its build tree. The
# properties are set from a file DuckDB includes in its project() call,
# deferred until DuckDB defined its targets. $<1:...> keeps multi-config
# generators from appending a per-config directory.
list(JOIN targets " " target_list)
set(archive_output "${CMAKE_CURRENT_BINARY_DIR}/duckdb_archive_output.cmake")
file(
CONFIGURE
OUTPUT "${archive_output}"
CONTENT
"# Generated by _duckdb_python_build_archives in cmake/duckdb_loader.cmake. Do not edit.
cmake_language(DEFER CALL set_target_properties @target_list@
PROPERTIES ARCHIVE_OUTPUT_DIRECTORY \"$<1:@library_dir@>\")
"
@ONLY)
list(APPEND cmake_args "-DCMAKE_PROJECT_DuckDB_INCLUDE=${archive_output}")

# The manifest describing the archives, installed next to them after a
# successful build
_duckdb_python_source_commit(commit)
_duckdb_python_architecture(architecture)
set(force_assert false)
if(FORCE_ASSERT)
set(force_assert true)
endif()
list(TRANSFORM extensions PREPEND "\"" OUTPUT_VARIABLE quoted)
list(TRANSFORM quoted APPEND "\"")
list(JOIN quoted ", " extension_array)
set(manifest "${CMAKE_CURRENT_BINARY_DIR}/duckdb_archives.json")
file(
CONFIGURE
OUTPUT "${manifest}"
CONTENT
"{
\"duckdb_commit\": \"${commit}\",
\"build_type\": \"${CMAKE_BUILD_TYPE}\",
\"force_assert\": ${force_assert},
\"system\": \"${CMAKE_SYSTEM_NAME}\",
\"architecture\": \"${architecture}\",
\"extensions\": [${extension_array}]
}
"
@ONLY)

set(prefix "${CMAKE_STATIC_LIBRARY_PREFIX}")
set(suffix "${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(byproducts "${library_dir}/${prefix}duckdb_static${suffix}")
foreach(ext IN LISTS extensions)
list(APPEND byproducts "${library_dir}/${prefix}${ext}_extension${suffix}")
endforeach()

message(STATUS "Building DuckDB archives into ${library_dir}")
ExternalProject_Add(
duckdb_build
SOURCE_DIR "${DUCKDB_SOURCE_PATH}"
PREFIX "${CMAKE_CURRENT_BINARY_DIR}/duckdb-prefix"
BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/duckdb-build"
LIST_SEPARATOR "|"
CMAKE_ARGS ${cmake_args}
BUILD_COMMAND "${CMAKE_COMMAND}" --build <BINARY_DIR> --config $<CONFIG>
--target ${targets}
# Let DuckDB's own build decide what is out of date
BUILD_ALWAYS ON
BUILD_BYPRODUCTS ${byproducts}
INSTALL_COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${manifest}"
"${library_dir}/duckdb_archives.json"
USES_TERMINAL_BUILD ON)

_duckdb_python_add_archive_targets("${library_dir}" ${extensions})
# Dependencies of imported targets are followed by the targets linking them
add_dependencies(duckdb_static duckdb_build)
endfunction()

function(_duckdb_python_write_static_extension_loader output)
# Writes a C++ source defining duckdb_register_static_extensions(), which
# registers each extension given in ARGN. This only relies on DuckDB's public
Expand Down Expand Up @@ -256,11 +573,15 @@ endfunction()
# ════════════════════════════════════════════════════════════════════════════════

function(duckdb_add_library target_name)
# The headers always come from the source tree
_duckdb_validate_source_path()
_duckdb_print_summary()

# Add DuckDB subdirectory - it will use our variables
add_subdirectory("${DUCKDB_SOURCE_PATH}" duckdb EXCLUDE_FROM_ALL)
if(DUCKDB_LIBRARY_DIR)
_duckdb_python_import_archives("${DUCKDB_LIBRARY_DIR}")
else()
_duckdb_print_summary()
_duckdb_python_build_archives("${CMAKE_CURRENT_BINARY_DIR}/duckdb_archives")
endif()

# Create clean interface target
_duckdb_create_interface_target(${target_name})
Expand Down Expand Up @@ -288,7 +609,12 @@ function(duckdb_python_link_extensions target_name)
if(link_extensions)
message(STATUS "Linking DuckDB extensions:")
foreach(ext IN LISTS link_extensions)
if(NOT TARGET ${ext}_extension)
if(NOT TARGET ${ext}_extension AND DUCKDB_LIBRARY_DIR)
message(
FATAL_ERROR
"Cannot link extension '${ext}': it is not in the manifest of ${DUCKDB_LIBRARY_DIR}"
)
elseif(NOT TARGET ${ext}_extension)
message(
FATAL_ERROR
"Cannot link extension '${ext}': DuckDB did not configure it, add it to BUILD_EXTENSIONS"
Expand Down
Loading