Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 106 additions & 24 deletions cmake/CheckAVX.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -50,46 +50,122 @@ set(AVX512IFMA_CODE "
}
")

macro(check_avx_lang lang type flags)
set(__FLAG_I 1)
set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS})
set(CMAKE_REQUIRED_QUIET_SAVE ${CMAKE_REQUIRED_QUIET})
foreach(__FLAG ${flags})
if(NOT ${lang}_${type}_FOUND)
set(CMAKE_REQUIRED_QUIET TRUE)
set(CMAKE_REQUIRED_FLAGS ${__FLAG})
set(SMALL_PRIME_FFT_AVX2_CODE "
#include <immintrin.h>
#if !defined(__x86_64__) || !(defined(__GNUC__) || defined(__clang__)) || \
!defined(__AVX2__) || !defined(__FMA__)
#error Small-prime AVX2/FMA kernels are not enabled
#endif
int main() {
__m256d a = _mm256_set1_pd(1.0);
a = _mm256_fmadd_pd(a, a, a);
return _mm256_cvtsd_f64(a) != 2.0;
}
")

set(SMALL_PRIME_FFT_AVX512_CODE "
#include <immintrin.h>
#if !defined(__x86_64__) || !(defined(__GNUC__) || defined(__clang__)) || \
!defined(__AVX2__) || !defined(__FMA__) || !defined(__AVX512F__) || !defined(__AVX512DQ__)
#error Small-prime AVX-512 kernels are not enabled
#endif
int main() {
__m512d a = _mm512_set1_pd(1.0);
a = _mm512_fmadd_pd(a, a, a);
a = _mm512_and_pd(a, a);
__m256d high = _mm512_extractf64x4_pd(a, 1);
return _mm256_cvtsd_f64(high) != 2.0;
}
")

function(check_avx_lang lang type flags)
set(found FALSE)
set(selected_flags "")
set(required_flags_save "${CMAKE_REQUIRED_FLAGS}")
set(CMAKE_REQUIRED_QUIET TRUE)
foreach(flag ${flags})
if(NOT found)
set(CMAKE_REQUIRED_FLAGS "${required_flags_save} ${flag}")
# Cache the probe, not a previous selection. Source/flag/toolchain changes must be
# checked again when an existing build directory is reconfigured.
string(CONCAT probe_context
"${${type}_CODE}|${CMAKE_REQUIRED_FLAGS}|${CMAKE_REQUIRED_DEFINITIONS}|"
"${CMAKE_REQUIRED_INCLUDES}|${CMAKE_REQUIRED_LIBRARIES}|"
"${CMAKE_REQUIRED_LINK_OPTIONS}|${CMAKE_REQUIRED_LINK_DIRECTORIES}|"
"${CMAKE_${lang}_COMPILER}|${CMAKE_${lang}_COMPILER_ID}|${CMAKE_${lang}_COMPILER_VERSION}|"
"${CMAKE_${lang}_COMPILER_TARGET}|${CMAKE_${lang}_FLAGS}|${CMAKE_${lang}_FLAGS_DEBUG}|"
"${CMAKE_${lang}_FLAGS_RELEASE}|${CMAKE_${lang}_FLAGS_RELWITHDEBINFO}|"
"${CMAKE_${lang}_FLAGS_MINSIZEREL}|${CMAKE_BUILD_TYPE}|${CMAKE_TRY_COMPILE_CONFIGURATION}|"
"${CMAKE_TRY_COMPILE_TARGET_TYPE}|${CMAKE_SYSROOT}|${CMAKE_OSX_ARCHITECTURES}|"
"${CMAKE_OSX_SYSROOT}|${CMAKE_OSX_DEPLOYMENT_TARGET}")
string(SHA256 probe_key "${probe_context}")
set(probe_var "${lang}_HAS_${type}_${probe_key}")
get_property(probe_checked GLOBAL PROPERTY "${probe_var}_CHECKED")
if(NOT probe_checked)
# Recheck once per configuration, including after linker/toolchain
# settings outside the probe's immediate context have changed.
unset(${probe_var} CACHE)
endif()
if("${lang}" STREQUAL "CXX")
check_cxx_source_compiles("${${type}_CODE}" ${lang}_HAS_${type}_${__FLAG_I})
check_cxx_source_compiles("${${type}_CODE}" ${probe_var})
else()
check_c_source_compiles("${${type}_CODE}" ${lang}_HAS_${type}_${__FLAG_I})
check_c_source_compiles("${${type}_CODE}" ${probe_var})
endif()
if(${lang}_HAS_${type}_${__FLAG_I})
set(${lang}_${type}_FOUND TRUE CACHE BOOL "${lang} ${type} support")
set(${lang}_${type}_FLAGS "${__FLAG}" CACHE STRING "${lang} ${type} flags")
set_property(GLOBAL PROPERTY "${probe_var}_CHECKED" TRUE)
if(${probe_var})
set(found TRUE)
set(selected_flags "${flag}")
endif()
math(EXPR __FLAG_I "${__FLAG_I}+1")
endif()
endforeach()
set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE})
set(CMAKE_REQUIRED_QUIET ${CMAKE_REQUIRED_QUIET_SAVE})

if(NOT ${lang}_${type}_FOUND)
set(${lang}_${type}_FOUND FALSE CACHE BOOL "${lang} ${type} support")
set(${lang}_${type}_FLAGS "" CACHE STRING "${lang} ${type} flags")
endif()

set(${lang}_${type}_FOUND ${found} CACHE BOOL "${lang} ${type} support" FORCE)
set(${lang}_${type}_FLAGS "${selected_flags}" CACHE STRING "${lang} ${type} flags" FORCE)
set(${lang}_${type}_FOUND ${found} PARENT_SCOPE)
set(${lang}_${type}_FLAGS "${selected_flags}" PARENT_SCOPE)
mark_as_advanced(${lang}_${type}_FOUND ${lang}_${type}_FLAGS)
endfunction()

# These bundles match the guards in small_prime_fft_arithmetic.hpp. IFMA is a
# separate algebra capability; neither IFMA nor AVX512F alone establishes them.
macro(check_small_prime_fft)
check_host_cpu_feature(AVX2 "avx2")
check_host_cpu_feature(FMA "fma")
check_host_cpu_feature(AVX512F "avx512f")
check_host_cpu_feature(AVX512DQ "avx512dq")

foreach(feature SMALL_PRIME_FFT_AVX2 SMALL_PRIME_FFT_AVX512)
set(CXX_${feature}_FOUND FALSE)
set(CXX_${feature}_FLAGS "")
endforeach()
if(HOST_CPU_AVX2_FOUND AND HOST_CPU_FMA_FOUND)
# Always export explicit flags, even if the configuring project already supplies them.
check_avx_lang(CXX SMALL_PRIME_FFT_AVX2 "-mavx2 -mfma")
if(CXX_SMALL_PRIME_FFT_AVX2_FOUND AND HOST_CPU_AVX512F_FOUND AND HOST_CPU_AVX512DQ_FOUND)
check_avx_lang(CXX SMALL_PRIME_FFT_AVX512 "-mavx2 -mfma -mavx512f -mavx512dq")
endif()
endif()
foreach(feature SMALL_PRIME_FFT_AVX2 SMALL_PRIME_FFT_AVX512)
set(CXX_${feature}_FOUND ${CXX_${feature}_FOUND} CACHE BOOL "CXX ${feature} support" FORCE)
set(CXX_${feature}_FLAGS "${CXX_${feature}_FLAGS}" CACHE STRING "CXX ${feature} flags" FORCE)
mark_as_advanced(CXX_${feature}_FOUND CXX_${feature}_FLAGS)
endforeach()
endmacro()

# Both widths use the same host contract; VL is required by the four-lane kernel.
macro(check_avx512ifma)
# Compiler AVX-512 options can also enable AVX2/FMA implicitly. Check that
# baseline without depending on the FFT probes or requiring AVX512DQ.
check_host_cpu_feature(AVX2 "avx2")
check_host_cpu_feature(FMA "fma")
check_host_cpu_feature(AVX512F "avx512f")
check_host_cpu_feature(AVX512IFMA "avx512ifma")
check_host_cpu_feature(AVX512VL "avx512vl")
if(HOST_CPU_AVX512F_FOUND AND HOST_CPU_AVX512IFMA_FOUND AND HOST_CPU_AVX512VL_FOUND)
check_avx_lang(CXX "AVX512IFMA" " ;-mavx512f -mavx512vl -mavx512ifma;/arch:AVX512")
if(HOST_CPU_AVX2_FOUND AND HOST_CPU_FMA_FOUND AND HOST_CPU_AVX512F_FOUND AND
HOST_CPU_AVX512IFMA_FOUND AND HOST_CPU_AVX512VL_FOUND)
check_avx_lang(CXX "AVX512IFMA" "-mavx512f -mavx512vl -mavx512ifma;/arch:AVX512")
else()
set(CXX_AVX512IFMA_FOUND FALSE)
set(CXX_AVX512IFMA_FLAGS "")
set(CXX_AVX512IFMA_FOUND FALSE CACHE BOOL "CXX AVX512IFMA support" FORCE)
set(CXX_AVX512IFMA_FLAGS "" CACHE STRING "CXX AVX512IFMA flags" FORCE)
endif()
Expand All @@ -106,6 +182,8 @@ macro(check_avx)
else()
set(C_AVX_FOUND FALSE CACHE BOOL "C AVX support" FORCE)
set(CXX_AVX_FOUND FALSE CACHE BOOL "CXX AVX support" FORCE)
set(C_AVX_FLAGS "" CACHE STRING "C AVX flags" FORCE)
set(CXX_AVX_FLAGS "" CACHE STRING "CXX AVX flags" FORCE)
endif()

if(HOST_CPU_AVX2_FOUND)
Expand All @@ -114,6 +192,8 @@ macro(check_avx)
else()
set(C_AVX2_FOUND FALSE CACHE BOOL "C AVX2 support" FORCE)
set(CXX_AVX2_FOUND FALSE CACHE BOOL "CXX AVX2 support" FORCE)
set(C_AVX2_FLAGS "" CACHE STRING "C AVX2 flags" FORCE)
set(CXX_AVX2_FLAGS "" CACHE STRING "CXX AVX2 flags" FORCE)
endif()

if(HOST_CPU_AVX512F_FOUND)
Expand All @@ -122,5 +202,7 @@ macro(check_avx)
else()
set(C_AVX512_FOUND FALSE CACHE BOOL "C AVX512 support" FORCE)
set(CXX_AVX512_FOUND FALSE CACHE BOOL "CXX AVX512 support" FORCE)
set(C_AVX512_FLAGS "" CACHE STRING "C AVX512 flags" FORCE)
set(CXX_AVX512_FLAGS "" CACHE STRING "CXX AVX512 flags" FORCE)
endif()
endmacro()
23 changes: 23 additions & 0 deletions cmake/CheckHostCPUFeature.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,25 @@ function(check_host_cpu_feature feature builtin_name)
string(TOUPPER "${feature}" feature_upper)
set(result_var "HOST_CPU_${feature_upper}_FOUND")

# Recheck the current host once per configuration. A cached result from a
# different host or toolchain is not evidence for this configuration.
string(CONCAT probe_context
"${CMAKE_CROSSCOMPILING}|${CMAKE_HOST_SYSTEM_PROCESSOR}|${CMAKE_CXX_COMPILER}|"
"${CMAKE_CXX_COMPILER_ID}|${CMAKE_CXX_COMPILER_VERSION}|${CMAKE_CXX_COMPILER_TARGET}|"
"${CMAKE_CXX_FLAGS}|${CMAKE_CXX_FLAGS_DEBUG}|${CMAKE_CXX_FLAGS_RELEASE}|"
"${CMAKE_CXX_FLAGS_RELWITHDEBINFO}|${CMAKE_CXX_FLAGS_MINSIZEREL}|${CMAKE_BUILD_TYPE}|"
"${CMAKE_TRY_COMPILE_CONFIGURATION}|${CMAKE_SYSROOT}|${CMAKE_OSX_ARCHITECTURES}|"
"${CMAKE_OSX_SYSROOT}|${CMAKE_OSX_DEPLOYMENT_TARGET}|${CMAKE_REQUIRED_FLAGS}|${builtin_name}")
string(SHA256 context "${probe_context}")
get_property(checked_context GLOBAL PROPERTY "${result_var}_CONTEXT")
if(NOT "${checked_context}" STREQUAL "${context}")
unset(${result_var})
unset(${result_var} CACHE)
unset(${result_var}_COMPILED CACHE)
unset(${result_var}_EXITCODE CACHE)
endif()
set_property(GLOBAL PROPERTY "${result_var}_CONTEXT" "${context}")

if(CMAKE_CROSSCOMPILING OR NOT CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(AMD64|amd64|x86_64|i[3-6]86)$")
set(${result_var} FALSE CACHE BOOL "${feature} available on host CPU" FORCE)
set(${result_var} FALSE PARENT_SCOPE)
Expand All @@ -28,8 +47,12 @@ function(check_host_cpu_feature feature builtin_name)
set(msvc_test "max_leaf >= 1 && (leaf1[2] & (1 << 27)) && (leaf1[2] & (1 << 28)) && ((_xgetbv(0) & 0x6) == 0x6)")
elseif(feature_upper STREQUAL "AVX2")
set(msvc_test "max_leaf >= 7 && (leaf1[2] & (1 << 27)) && (leaf1[2] & (1 << 28)) && ((_xgetbv(0) & 0x6) == 0x6) && (leaf7[1] & (1 << 5))")
elseif(feature_upper STREQUAL "FMA")
set(msvc_test "max_leaf >= 1 && (leaf1[2] & (1 << 12)) && (leaf1[2] & (1 << 27)) && (leaf1[2] & (1 << 28)) && ((_xgetbv(0) & 0x6) == 0x6)")
elseif(feature_upper STREQUAL "AVX512F")
set(msvc_test "max_leaf >= 7 && (leaf1[2] & (1 << 27)) && ((_xgetbv(0) & 0xE6) == 0xE6) && (leaf7[1] & (1 << 16))")
elseif(feature_upper STREQUAL "AVX512DQ")
set(msvc_test "max_leaf >= 7 && (leaf1[2] & (1 << 27)) && ((_xgetbv(0) & 0xE6) == 0xE6) && (leaf7[1] & (1 << 16)) && (leaf7[1] & (1 << 17))")
elseif(feature_upper STREQUAL "AVX512IFMA")
set(msvc_test "max_leaf >= 7 && (leaf1[2] & (1 << 27)) && ((_xgetbv(0) & 0xE6) == 0xE6) && (leaf7[1] & (1 << 16)) && (leaf7[1] & (1 << 21))")
elseif(feature_upper STREQUAL "AVX512VL")
Expand Down
74 changes: 74 additions & 0 deletions cmake/tests/small_prime_features/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
cmake_minimum_required(VERSION 3.22)
project(small_prime_feature_consumer LANGUAGES C CXX)

get_filename_component(CRYPTO3_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../../.." ABSOLUTE)
option(FEATURE_CHECK_COMPILE_ONLY "Build an object without running target code" OFF)
set(FEATURE_CHECK_HOST "native" CACHE STRING "native, full, avx2, no-fma, no-dq, no-ifma, or unsupported")

if(NOT FEATURE_CHECK_HOST STREQUAL "native")
if(NOT FEATURE_CHECK_COMPILE_ONLY OR CMAKE_CROSSCOMPILING)
message(FATAL_ERROR "Simulated host checks require compile-only mode and a native compiler configuration")
endif()
# Isolate host-policy simulation here. The compiler probes and Crypto3 targets
# below are real; a simulated CPU never authorizes executing the built code.
list(APPEND CMAKE_MODULE_PATH "${CRYPTO3_SOURCE_DIR}/cmake")
include(CheckHostCPUFeature)
set(test_features AVX AVX2 FMA AVX512F AVX512DQ AVX512VL AVX512IFMA)
if(FEATURE_CHECK_HOST STREQUAL "avx2")
set(test_features AVX AVX2 FMA)
elseif(FEATURE_CHECK_HOST STREQUAL "no-fma")
list(REMOVE_ITEM test_features FMA)
elseif(FEATURE_CHECK_HOST STREQUAL "no-dq")
list(REMOVE_ITEM test_features AVX512DQ)
elseif(FEATURE_CHECK_HOST STREQUAL "no-ifma")
list(REMOVE_ITEM test_features AVX512IFMA)
elseif(FEATURE_CHECK_HOST STREQUAL "unsupported")
set(test_features "")
elseif(NOT FEATURE_CHECK_HOST STREQUAL "full")
message(FATAL_ERROR "Unknown simulated host: ${FEATURE_CHECK_HOST}")
endif()
function(check_host_cpu_feature feature builtin_name)
if(feature IN_LIST test_features)
set(supported TRUE)
else()
set(supported FALSE)
endif()
set(HOST_CPU_${feature}_FOUND ${supported} CACHE BOOL "Simulated host feature" FORCE)
set(HOST_CPU_${feature}_FOUND ${supported} PARENT_SCOPE)
endfunction()
endif()

set(BUILD_TESTS OFF CACHE BOOL "Build only requested checks")
set(BUILD_BENCH_TESTS OFF CACHE BOOL "No profiling runs")
enable_testing()
add_subdirectory("${CRYPTO3_SOURCE_DIR}" crypto3 EXCLUDE_FROM_ALL)

if(FEATURE_CHECK_COMPILE_ONLY)
add_library(feature_consumer OBJECT feature_consumer.cpp)
else()
add_executable(feature_consumer feature_consumer.cpp)
add_test(NAME small_prime_feature_consumer COMMAND feature_consumer)
endif()
target_compile_features(feature_consumer PRIVATE cxx_std_23)
# No instruction flags or feature definitions are supplied by this consumer.
target_link_libraries(feature_consumer PRIVATE crypto3::math)

foreach(feature AVX2 AVX512)
if(CXX_SMALL_PRIME_FFT_${feature}_FOUND)
set(expect_${feature} 1)
else()
set(expect_${feature} 0)
endif()
endforeach()
set(expect_IFMA 0)
if(expect_AVX512 AND CXX_AVX512IFMA_FOUND)
set(expect_IFMA 1)
endif()
target_compile_definitions(feature_consumer PRIVATE
EXPECT_FFT_AVX2=${expect_AVX2} EXPECT_FFT_AVX512=${expect_AVX512} EXPECT_CRT_IFMA=${expect_IFMA})
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/selection.txt"
"avx2=${expect_AVX2}\navx512=${expect_AVX512}\ncrt_ifma=${expect_IFMA}\n"
"avx2_flags=${CXX_SMALL_PRIME_FFT_AVX2_FLAGS}\navx512_flags=${CXX_SMALL_PRIME_FFT_AVX512_FLAGS}\n"
"ifma_found=${CXX_AVX512IFMA_FOUND}\nifma_flags=${CXX_AVX512IFMA_FLAGS}\n"
"cross=${CMAKE_CROSSCOMPILING}\ncompiler=${CMAKE_CXX_COMPILER_ID}\n"
"frontend=${CMAKE_CXX_COMPILER_FRONTEND_VARIANT}\n")
27 changes: 27 additions & 0 deletions cmake/tests/small_prime_features/feature_consumer.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
// Copyright (c) 2026. Distributed under the MIT License.

#include <nil/crypto3/math/polynomial/backends/detail/small_prime_fft/bn254_small_prime_fft_crt.hpp>

#ifdef CRYPTO3_SMALL_PRIME_FFT_AVX2_SELECTED
constexpr bool fft_avx2 = true;
#else
constexpr bool fft_avx2 = false;
#endif
#ifdef CRYPTO3_SMALL_PRIME_FFT_AVX512_SELECTED
constexpr bool fft_avx512 = true;
#else
constexpr bool fft_avx512 = false;
#endif
#ifdef CRYPTO3_SMALL_PRIME_CRT_IFMA_SELECTED
constexpr bool crt_ifma = true;
#else
constexpr bool crt_ifma = false;
#endif

static_assert(fft_avx2 == bool(EXPECT_FFT_AVX2), "AVX2/FMA FFT selection does not match configuration");
static_assert(fft_avx512 == bool(EXPECT_FFT_AVX512), "AVX512F/DQ FFT selection does not match configuration");
static_assert(crt_ifma == bool(EXPECT_CRT_IFMA), "IFMA CRT selection does not match configuration");

int main() {
return 0;
}
Loading
Loading