diff --git a/CMakeLists.txt b/CMakeLists.txt index 69b6b8e..3a75b98 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,7 +40,6 @@ if(PROJECT_IS_TOP_LEVEL AND BUILD_TESTING) target_compile_options(${target} PRIVATE "$<$:/permissive-;/W4;/wd4324;/w14242>" "$<$:-Wall;-Wextra;-Wpedantic;-Wshadow;-Wconversion;-Wsign-conversion;-Wold-style-cast;-Wundef;-Wcast-align;-Wzero-as-null-pointer-constant;-Wformat=2>" - "$<$,$,12>>:-Wno-interference-size>" ) endfunction() diff --git a/README.md b/README.md index b17d329..83d526c 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,7 @@ t2.join(); - `template void push(P &&v);` Enqueue an item using move construction. Participates in overload - resolution only if `std::is_nothrow_constructible::value == - true`. Blocks if queue is full. + resolution only if `std::is_nothrow_constructible_v`. Blocks if queue is full. - `bool try_push(const T &v);` @@ -76,8 +75,7 @@ t2.join(); - `template bool try_push(P &&v);` Try to enqueue an item using move construction. Participates in - overload resolution only if `std::is_nothrow_constructible::value == true`. Returns `true` on success and `false` if queue + overload resolution only if `std::is_nothrow_constructible_v`. Returns `true` on success and `false` if queue is full. - `void pop(T &v);` @@ -90,6 +88,11 @@ t2.join(); Try to dequeue an item by copying or moving the item into `v`. Return `true` on sucess and `false` if the queue is empty. +- `template bool try_consume(Consumer &&consumer);` + + Tries to dequeue an item and pass it to a non-throwing callable as an rvalue. + Returns `true` on success and `false` if the queue is empty. + - `ssize_t size();` Returns the number of elements in the queue. @@ -149,10 +152,7 @@ implementation: - [X] Add allocator supports so that the queue could be used with huge pages and shared memory - [ ] Add benchmarks and compare to `boost::lockfree::queue` and others -- [ ] Use C++20 concepts instead of `static_assert` if available -- [X] Use `std::hardware_destructive_interference_size` if available - [ ] Add API for zero-copy deqeue and batch dequeue operations -- [ ] Add `[[nodiscard]]` attributes ## About diff --git a/include/rigtorp/MPMCQueue.h b/include/rigtorp/MPMCQueue.h index 5bc5ab2..367fd30 100644 --- a/include/rigtorp/MPMCQueue.h +++ b/include/rigtorp/MPMCQueue.h @@ -22,81 +22,32 @@ SOFTWARE. #pragma once -#include #include -#include -#include // offsetof +#include +#include #include #include -#include // std::hardware_destructive_interference_size +#include #include #include - -#ifndef __cpp_aligned_new -#ifdef _WIN32 -#include // _aligned_malloc -#else -#include // posix_memalign -#endif -#endif +#include namespace rigtorp { namespace mpmc { -#if defined(__cpp_lib_hardware_interference_size) && !defined(__APPLE__) -#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Winterference-size" -#endif -static constexpr size_t hardwareInterferenceSize = - std::hardware_destructive_interference_size; -#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12 -#pragma GCC diagnostic pop -#endif -#else -static constexpr size_t hardwareInterferenceSize = 64; -#endif -#if defined(__cpp_aligned_new) -template using AlignedAllocator = std::allocator; -#else -template struct AlignedAllocator { - using value_type = T; - - T *allocate(std::size_t n) { - if (n > std::numeric_limits::max() / sizeof(T)) { - throw std::bad_array_new_length(); - } -#ifdef _WIN32 - auto *p = static_cast(_aligned_malloc(sizeof(T) * n, alignof(T))); - if (p == nullptr) { - throw std::bad_alloc(); - } +#if defined(__s390x__) +inline constexpr std::size_t kFalseSharingRange = 256; +#elif defined(__APPLE__) && defined(__aarch64__) +inline constexpr std::size_t kFalseSharingRange = 128; #else - T *p; - if (posix_memalign(reinterpret_cast(&p), alignof(T), - sizeof(T) * n) != 0) { - throw std::bad_alloc(); - } +inline constexpr std::size_t kFalseSharingRange = 64; #endif - return p; - } - void deallocate(T *p, std::size_t) { -#ifdef _WIN32 - _aligned_free(p); -#else - free(p); -#endif - } -}; -#endif -#if defined(__cpp_lib_byte) && !defined(__APPLE__) -using Byte = std::byte; -#else -using Byte = unsigned char; -#endif +template using AlignedAllocator = std::allocator; template struct Slot { + Slot() noexcept : turn(0) {} + ~Slot() noexcept { if (turn & 1) { destroy(); @@ -104,71 +55,62 @@ template struct Slot { } template void construct(Args &&...args) noexcept { - static_assert(std::is_nothrow_constructible::value, - "T must be nothrow constructible with Args&&..."); - new (&storage) T(std::forward(args)...); + std::construct_at(reinterpret_cast(storage), + std::forward(args)...); } - void destroy() noexcept { - static_assert(std::is_nothrow_destructible::value, - "T must be nothrow destructible"); - reinterpret_cast(&storage)->~T(); - } + void destroy() noexcept { std::destroy_at(value()); } - T &&move() noexcept { return reinterpret_cast(storage); } + T &&move() noexcept { return std::move(*value()); } - // Align to avoid false sharing between adjacent slots - alignas(hardwareInterferenceSize) std::atomic turn = {0}; - alignas(T) std::array storage; -}; + // Align to avoid false sharing between adjacent slots. + alignas(kFalseSharingRange) std::atomic turn; -template >> -class Queue { private: - static_assert(std::is_nothrow_copy_assignable::value || - std::is_nothrow_move_assignable::value, - "T must be nothrow copy or move assignable"); + T *value() noexcept { return std::launder(reinterpret_cast(storage)); } - static_assert(std::is_nothrow_destructible::value, - "T must be nothrow destructible"); + alignas(T) std::byte storage[sizeof(T)]; +}; +template >> + requires std::is_nothrow_destructible_v +class Queue { public: - explicit Queue(const size_t capacity, + explicit Queue(const std::size_t capacity, const Allocator &allocator = Allocator()) : capacity_(capacity), allocator_(allocator), head_(0), tail_(0) { - if (capacity_ < 1) { + if (capacity_ == 0) [[unlikely]] { throw std::invalid_argument("capacity < 1"); } + if (capacity_ == std::numeric_limits::max()) [[unlikely]] { + throw std::bad_array_new_length(); + } // Allocate one extra slot to prevent false sharing on the last slot slots_ = allocator_.allocate(capacity_ + 1); // Allocators are not required to honor alignment for over-aligned types // (see http://eel.is/c++draft/allocator.requirements#10) so we verify // alignment here - if (reinterpret_cast(slots_) % alignof(Slot) != 0) { + if (reinterpret_cast(slots_) % alignof(Slot) != 0) [[unlikely]] { allocator_.deallocate(slots_, capacity_ + 1); throw std::bad_alloc(); } - for (size_t i = 0; i < capacity_; ++i) { - new (&slots_[i]) Slot(); + for (std::size_t i = 0; i < capacity_; ++i) { + std::construct_at(&slots_[i]); } static_assert( - alignof(Slot) == hardwareInterferenceSize, + alignof(Slot) >= kFalseSharingRange, "Slot must be aligned to cache line boundary to prevent false sharing"); - static_assert(sizeof(Slot) % hardwareInterferenceSize == 0, + static_assert(sizeof(Slot) % kFalseSharingRange == 0, "Slot size must be a multiple of cache line size to prevent " "false sharing between adjacent slots"); - static_assert(sizeof(Queue) % hardwareInterferenceSize == 0, + static_assert(sizeof(Queue) % kFalseSharingRange == 0, "Queue size must be a multiple of cache line size to " "prevent false sharing between adjacent queues"); - static_assert( - offsetof(Queue, tail_) - offsetof(Queue, head_) == - static_cast(hardwareInterferenceSize), - "head and tail must be a cache line apart to prevent false sharing"); } ~Queue() noexcept { - for (size_t i = 0; i < capacity_; ++i) { - slots_[i].~Slot(); + for (std::size_t i = 0; i < capacity_; ++i) { + std::destroy_at(&slots_[i]); } allocator_.deallocate(slots_, capacity_ + 1); } @@ -177,9 +119,9 @@ class Queue { Queue(const Queue &) = delete; Queue &operator=(const Queue &) = delete; - template void emplace(Args &&...args) noexcept { - static_assert(std::is_nothrow_constructible::value, - "T must be nothrow constructible with Args&&..."); + template + requires std::is_nothrow_constructible_v + void emplace(Args &&...args) noexcept { auto const head = head_.fetch_add(1); auto &slot = slots_[idx(head)]; auto const t = turn(head); @@ -189,9 +131,9 @@ class Queue { slot.turn.store(t * 2 + 1, std::memory_order_release); } - template bool try_emplace(Args &&...args) noexcept { - static_assert(std::is_nothrow_constructible::value, - "T must be nothrow constructible with Args&&..."); + template + requires std::is_nothrow_constructible_v + [[nodiscard]] bool try_emplace(Args &&...args) noexcept { auto head = head_.load(std::memory_order_acquire); for (;;) { auto &slot = slots_[idx(head)]; @@ -212,79 +154,63 @@ class Queue { } } - void push(const T &v) noexcept { - static_assert(std::is_nothrow_copy_constructible::value, - "T must be nothrow copy constructible"); + void push(const T &v) noexcept + requires std::is_nothrow_copy_constructible_v + { emplace(v); } - template ::value>::type> + template + requires std::is_nothrow_constructible_v void push(P &&v) noexcept { emplace(std::forward

(v)); } - bool try_push(const T &v) noexcept { - static_assert(std::is_nothrow_copy_constructible::value, - "T must be nothrow copy constructible"); + [[nodiscard]] bool try_push(const T &v) noexcept + requires std::is_nothrow_copy_constructible_v + { return try_emplace(v); } - template ::value>::type> - bool try_push(P &&v) noexcept { + template + requires std::is_nothrow_constructible_v + [[nodiscard]] bool try_push(P &&v) noexcept { return try_emplace(std::forward

(v)); } - void pop(T &v) noexcept { + void pop(T &v) noexcept + requires(std::is_nothrow_move_assignable_v || + std::is_nothrow_copy_assignable_v) + { auto const tail = tail_.fetch_add(1); auto &slot = slots_[idx(tail)]; auto const t = turn(tail); while (t * 2 + 1 != slot.turn.load(std::memory_order_acquire)) ; - v = slot.move(); + assign(v, slot.move()); slot.destroy(); slot.turn.store(t * 2 + 2, std::memory_order_release); } - bool try_pop(T &v) noexcept { - auto tail = tail_.load(std::memory_order_acquire); - for (;;) { - auto &slot = slots_[idx(tail)]; - auto const t = turn(tail); - if (t * 2 + 1 == slot.turn.load(std::memory_order_acquire)) { - if (tail_.compare_exchange_strong(tail, tail + 1)) { - v = slot.move(); - slot.destroy(); - slot.turn.store(t * 2 + 2, std::memory_order_release); - return true; - } - } else { - auto const prevTail = tail; - tail = tail_.load(std::memory_order_acquire); - if (tail == prevTail) { - return false; - } - } - } + [[nodiscard]] bool try_pop(T &v) noexcept + requires(std::is_nothrow_move_assignable_v || + std::is_nothrow_copy_assignable_v) + { + return try_consume([&v](T &&item) noexcept { assign(v, std::move(item)); }); } - template bool try_consume(Consumer &&c) noexcept { -#ifdef __cpp_lib_is_invocable - static_assert(std::is_nothrow_invocable_v, - "Consumer must be noexcept invocable with T&&"); -#endif + template + requires requires(Consumer &&c, T &&item) { + { std::forward(c)(std::move(item)) } noexcept; + } + [[nodiscard]] bool try_consume(Consumer &&c) noexcept { auto tail = tail_.load(std::memory_order_acquire); for (;;) { auto &slot = slots_[idx(tail)]; auto const t = turn(tail); if (t * 2 + 1 == slot.turn.load(std::memory_order_acquire)) { if (tail_.compare_exchange_strong(tail, tail + 1)) { - static_assert(noexcept(c(slot.move())), - "Consumer must be noexcept invocable with T&&"); - c(slot.move()); + std::forward(c)(slot.move()); slot.destroy(); slot.turn.store(t * 2 + 2, std::memory_order_release); return true; @@ -303,33 +229,38 @@ class Queue { /// The size can be negative when the queue is empty and there is at least one /// reader waiting. Since this is a concurrent queue the size is only a best /// effort guess until all reader and writer threads have been joined. - ptrdiff_t size() const noexcept { + [[nodiscard]] std::ptrdiff_t size() const noexcept { // TODO: How can we deal with wrapped queue on 32bit? - return static_cast(head_.load(std::memory_order_relaxed) - + return static_cast(head_.load(std::memory_order_relaxed) - tail_.load(std::memory_order_relaxed)); } /// Returns true if the queue is empty. /// Since this is a concurrent queue this is only a best effort guess /// until all reader and writer threads have been joined. - bool empty() const noexcept { return size() <= 0; } + [[nodiscard]] bool empty() const noexcept { return size() <= 0; } + private: - constexpr size_t idx(size_t i) const noexcept { return i % capacity_; } + static void assign(T &destination, T &&item) noexcept { + if constexpr (std::is_nothrow_move_assignable_v) { + destination = std::move(item); + } else { + destination = std::as_const(item); + } + } - constexpr size_t turn(size_t i) const noexcept { return i / capacity_; } + constexpr std::size_t idx(std::size_t i) const noexcept { return i % capacity_; } + + constexpr std::size_t turn(std::size_t i) const noexcept { return i / capacity_; } private: - const size_t capacity_; + const std::size_t capacity_; Slot *slots_; -#if defined(__has_cpp_attribute) && __has_cpp_attribute(no_unique_address) - Allocator allocator_ [[no_unique_address]]; -#else - Allocator allocator_; -#endif + [[no_unique_address]] Allocator allocator_; // Align to avoid false sharing between head_ and tail_ - alignas(hardwareInterferenceSize) std::atomic head_; - alignas(hardwareInterferenceSize) std::atomic tail_; + alignas(kFalseSharingRange) std::atomic head_; + alignas(kFalseSharingRange) std::atomic tail_; }; } // namespace mpmc diff --git a/src/MPMCQueueExample.cpp b/src/MPMCQueueExample.cpp index 0c6dc44..b23cd9a 100644 --- a/src/MPMCQueueExample.cpp +++ b/src/MPMCQueueExample.cpp @@ -3,7 +3,8 @@ #include int main(int argc, char *argv[]) { - (void)argc, (void)argv; + (void)argc; + (void)argv; using namespace rigtorp; diff --git a/src/MPMCQueueTest.cpp b/src/MPMCQueueTest.cpp index a226c38..ec46dc3 100644 --- a/src/MPMCQueueTest.cpp +++ b/src/MPMCQueueTest.cpp @@ -68,7 +68,8 @@ struct TestType { std::set TestType::constructed; int main(int argc, char *argv[]) { - (void)argc, (void)argv; + (void)argc; + (void)argv; using namespace rigtorp; @@ -108,44 +109,38 @@ int main(int argc, char *argv[]) { // Copyable only type { - struct Test { - Test() {} + struct alignas(512) Test { + Test() = default; Test(const Test &) noexcept {} Test &operator=(const Test &) noexcept { return *this; } Test(Test &&) = delete; + Test &operator=(Test &&) = delete; }; MPMCQueue q(16); // lvalue Test v; q.emplace(v); - q.try_emplace(v); + assert(q.try_emplace(v)); q.push(v); - q.try_push(v); + assert(q.try_push(v)); // xvalue q.push(Test()); - q.try_push(Test()); + assert(q.try_push(Test())); + q.pop(v); + assert(q.try_pop(v)); } // Movable only type { MPMCQueue> q(16); - // lvalue - // auto v = std::unique_ptr(new int(1)); - // q.emplace(v); - // q.try_emplace(v); - // q.push(v); - // q.try_push(v); - // xvalue - q.emplace(std::unique_ptr(new int(1))); - q.try_emplace(std::unique_ptr(new int(2))); - q.push(std::unique_ptr(new int(3))); - q.try_push(std::unique_ptr(new int(4))); - + q.emplace(std::make_unique(1)); + assert(q.try_emplace(std::make_unique(2))); + q.push(std::make_unique(3)); + assert(q.try_push(std::make_unique(4))); std::unique_ptr v; q.pop(v); - assert(*v == 1); - assert(q.try_pop(v) == true); - assert(*v == 2); + assert(v && *v == 1); + assert(q.try_pop(v) && *v == 2); assert(q.try_consume([&v](std::unique_ptr &&w) noexcept { v = std::move(w); }) == true); @@ -156,7 +151,7 @@ int main(int argc, char *argv[]) { { MPMCQueue> q(1); { - auto up = std::unique_ptr(new int(1)); + auto up = std::make_unique(1); auto *rp = up.get(); std::unique_ptr up2; assert(q.try_emplace(std::move(up)) == true); @@ -168,7 +163,7 @@ int main(int argc, char *argv[]) { } { - auto up = std::unique_ptr(new int(1)); + auto up = std::make_unique(1); auto *rp = up.get(); std::unique_ptr up2; q.emplace(std::move(up)); @@ -180,7 +175,7 @@ int main(int argc, char *argv[]) { } { - auto up = std::unique_ptr(new int(1)); + auto up = std::make_unique(1); auto *rp = up.get(); std::unique_ptr up2; q.push(std::move(up)); @@ -192,7 +187,7 @@ int main(int argc, char *argv[]) { } { - auto up = std::unique_ptr(new int(1)); + auto up = std::make_unique(1); auto *rp = up.get(); std::unique_ptr up2; assert(q.try_push(std::move(up)) == true);