From 97294babc34e94df1effd2645c05afb33b093d65 Mon Sep 17 00:00:00 2001 From: Erik Rigtorp Date: Tue, 17 Nov 2026 22:20:00 -0500 Subject: [PATCH] Hide queue slots and rebind allocators internally --- README.md | 15 ++++-- include/rigtorp/MPMCQueue.h | 94 +++++++++++++++++++------------------ src/MPMCQueueTest.cpp | 61 ++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 83d526c..3264a5b 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,18 @@ t2.join(); ## Usage -- `MPMCQueue(size_t capacity);` +- `MPMCQueue(std::size_t capacity, const Allocator &allocator = Allocator());` + + Constructs a queue holding items of type `T` with the given capacity. + The default allocator is `std::allocator`. Custom allocators must rebind + to the queue's private slot type, support its alignment, and use raw + pointers after rebinding. The queue uses the allocator for backing storage + and manages element construction and destruction directly. + + Allocators that previously named `rigtorp::mpmc::Slot` should now be + supplied as a rebindable allocator for `T`. Use `rigtorp::MPMCQueue` + as the queue type. - Constructs a new `MPMCQueue` holding items of type `T` with capacity - `capacity`. - - `void emplace(Args &&... args);` Enqueue an item using inplace construction. Blocks if queue is full. diff --git a/include/rigtorp/MPMCQueue.h b/include/rigtorp/MPMCQueue.h index 367fd30..c7181cc 100644 --- a/include/rigtorp/MPMCQueue.h +++ b/include/rigtorp/MPMCQueue.h @@ -33,51 +33,55 @@ SOFTWARE. #include namespace rigtorp { -namespace mpmc { +template > + requires std::is_nothrow_destructible_v +class MPMCQueue { #if defined(__s390x__) -inline constexpr std::size_t kFalseSharingRange = 256; + static constexpr std::size_t kFalseSharingRange = 256; #elif defined(__APPLE__) && defined(__aarch64__) -inline constexpr std::size_t kFalseSharingRange = 128; + static constexpr std::size_t kFalseSharingRange = 128; #else -inline constexpr std::size_t kFalseSharingRange = 64; + static constexpr std::size_t kFalseSharingRange = 64; #endif -template using AlignedAllocator = std::allocator; + struct Slot { + Slot() noexcept : turn(0) {} -template struct Slot { - Slot() noexcept : turn(0) {} + ~Slot() noexcept { + if (turn & 1) { + destroy(); + } + } - ~Slot() noexcept { - if (turn & 1) { - destroy(); + template void construct(Args &&...args) noexcept { + std::construct_at(reinterpret_cast(storage), + std::forward(args)...); } - } - template void construct(Args &&...args) noexcept { - std::construct_at(reinterpret_cast(storage), - std::forward(args)...); - } + void destroy() noexcept { std::destroy_at(value()); } - void destroy() noexcept { std::destroy_at(value()); } + T &&move() noexcept { return std::move(*value()); } - T &&move() noexcept { return std::move(*value()); } + // Align to avoid false sharing between adjacent slots. + alignas(kFalseSharingRange) std::atomic turn; - // Align to avoid false sharing between adjacent slots. - alignas(kFalseSharingRange) std::atomic turn; + private: + T *value() noexcept { return std::launder(reinterpret_cast(storage)); } -private: - T *value() noexcept { return std::launder(reinterpret_cast(storage)); } + alignas(T) std::byte storage[sizeof(T)]; + }; - alignas(T) std::byte storage[sizeof(T)]; -}; + using SlotAllocator = + typename std::allocator_traits::template rebind_alloc; + using SlotAllocatorTraits = std::allocator_traits; + + static_assert(std::is_same_v, + "The rebound allocator must use raw pointers"); -template >> - requires std::is_nothrow_destructible_v -class Queue { public: - explicit Queue(const std::size_t capacity, - const Allocator &allocator = Allocator()) + explicit MPMCQueue(const std::size_t capacity, + const Allocator &allocator = Allocator()) : capacity_(capacity), allocator_(allocator), head_(0), tail_(0) { if (capacity_ == 0) [[unlikely]] { throw std::invalid_argument("capacity < 1"); @@ -86,38 +90,39 @@ class Queue { throw std::bad_array_new_length(); } // Allocate one extra slot to prevent false sharing on the last slot - slots_ = allocator_.allocate(capacity_ + 1); + slots_ = SlotAllocatorTraits::allocate(allocator_, capacity_ + 1); // Allocators are not required to honor alignment for over-aligned types // (see http://eel.is/c++draft/allocator.requirements#10) so we verify // alignment here - if (reinterpret_cast(slots_) % alignof(Slot) != 0) [[unlikely]] { - allocator_.deallocate(slots_, capacity_ + 1); + if (reinterpret_cast(slots_) % alignof(Slot) != 0) + [[unlikely]] { + deallocate_slots(); throw std::bad_alloc(); } for (std::size_t i = 0; i < capacity_; ++i) { std::construct_at(&slots_[i]); } static_assert( - alignof(Slot) >= kFalseSharingRange, + alignof(Slot) >= kFalseSharingRange, "Slot must be aligned to cache line boundary to prevent false sharing"); - static_assert(sizeof(Slot) % kFalseSharingRange == 0, + static_assert(sizeof(Slot) % kFalseSharingRange == 0, "Slot size must be a multiple of cache line size to prevent " "false sharing between adjacent slots"); - static_assert(sizeof(Queue) % kFalseSharingRange == 0, + static_assert(sizeof(MPMCQueue) % kFalseSharingRange == 0, "Queue size must be a multiple of cache line size to " "prevent false sharing between adjacent queues"); } - ~Queue() noexcept { + ~MPMCQueue() noexcept { for (std::size_t i = 0; i < capacity_; ++i) { std::destroy_at(&slots_[i]); } - allocator_.deallocate(slots_, capacity_ + 1); + deallocate_slots(); } // non-copyable and non-movable - Queue(const Queue &) = delete; - Queue &operator=(const Queue &) = delete; + MPMCQueue(const MPMCQueue &) = delete; + MPMCQueue &operator=(const MPMCQueue &) = delete; template requires std::is_nothrow_constructible_v @@ -254,18 +259,17 @@ class Queue { constexpr std::size_t turn(std::size_t i) const noexcept { return i / capacity_; } private: + void deallocate_slots() noexcept { + SlotAllocatorTraits::deallocate(allocator_, slots_, capacity_ + 1); + } + const std::size_t capacity_; - Slot *slots_; - [[no_unique_address]] Allocator allocator_; + Slot *slots_; + [[no_unique_address]] SlotAllocator allocator_; // Align to avoid false sharing between head_ and tail_ alignas(kFalseSharingRange) std::atomic head_; alignas(kFalseSharingRange) std::atomic tail_; }; -} // namespace mpmc - -template >> -using MPMCQueue = mpmc::Queue; } // namespace rigtorp diff --git a/src/MPMCQueueTest.cpp b/src/MPMCQueueTest.cpp index ec46dc3..f5219b3 100644 --- a/src/MPMCQueueTest.cpp +++ b/src/MPMCQueueTest.cpp @@ -25,6 +25,8 @@ SOFTWARE. #include #include #include +#include +#include #include #include #include @@ -67,6 +69,44 @@ struct TestType { std::set TestType::constructed; +struct AllocationState { + std::size_t count = 0; + bool misalign = false; +}; + +template struct TestAllocator { + using value_type = T; + + explicit TestAllocator(AllocationState &allocationState) noexcept + : state(allocationState) {} + + template + TestAllocator(const TestAllocator &other) noexcept : state(other.state) {} + + T *allocate(std::size_t count) { + assert(state.count == 0); + auto *allocation = std::allocator{}.allocate(count); + state.count = count; + if (state.misalign) { + auto *bytes = reinterpret_cast(allocation); + return static_cast(static_cast(bytes + 1)); + } + return allocation; + } + + void deallocate(T *allocation, std::size_t count) noexcept { + assert(state.count == count); + if (state.misalign) { + auto *bytes = reinterpret_cast(allocation); + allocation = static_cast(static_cast(bytes - 1)); + } + std::allocator{}.deallocate(allocation, count); + state.count = 0; + } + + AllocationState &state; +}; + int main(int argc, char *argv[]) { (void)argc; (void)argv; @@ -209,6 +249,27 @@ int main(int argc, char *argv[]) { assert(throws == true); } + { + AllocationState state; + { + MPMCQueue> q(1, TestAllocator(state)); + assert(state.count == 2); + assert(q.try_push(42)); + int value = 0; + assert(q.try_pop(value) && value == 42); + } + assert(state.count == 0); + + state.misalign = true; + bool throws = false; + try { + MPMCQueue> q(1, TestAllocator(state)); + } catch (const std::bad_alloc &) { + throws = true; + } + assert(throws && state.count == 0); + } + // Fuzz test { const std::size_t numOps = 1000;