Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,18 @@ t2.join();

## Usage

- `MPMCQueue<T>(size_t capacity);`
- `MPMCQueue<T, Allocator>(std::size_t capacity, const Allocator &allocator = Allocator());`

Constructs a queue holding items of type `T` with the given capacity.
The default allocator is `std::allocator<T>`. Custom allocators must rebind
to the queue's private slot type, support its alignment, and use raw
pointers after rebinding. The queue uses the allocator for backing storage
and manages element construction and destruction directly.

Allocators that previously named `rigtorp::mpmc::Slot<T>` should now be
supplied as a rebindable allocator for `T`. Use `rigtorp::MPMCQueue`
as the queue type.

Constructs a new `MPMCQueue` holding items of type `T` with capacity
`capacity`.

- `void emplace(Args &&... args);`

Enqueue an item using inplace construction. Blocks if queue is full.
Expand Down
94 changes: 49 additions & 45 deletions include/rigtorp/MPMCQueue.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,51 +33,55 @@ SOFTWARE.
#include <utility>

namespace rigtorp {
namespace mpmc {

template <typename T, typename Allocator = std::allocator<T>>
requires std::is_nothrow_destructible_v<T>
class MPMCQueue {
#if defined(__s390x__)
inline constexpr std::size_t kFalseSharingRange = 256;
static constexpr std::size_t kFalseSharingRange = 256;
#elif defined(__APPLE__) && defined(__aarch64__)
inline constexpr std::size_t kFalseSharingRange = 128;
static constexpr std::size_t kFalseSharingRange = 128;
#else
inline constexpr std::size_t kFalseSharingRange = 64;
static constexpr std::size_t kFalseSharingRange = 64;
#endif

template <typename T> using AlignedAllocator = std::allocator<T>;
struct Slot {
Slot() noexcept : turn(0) {}

template <typename T> struct Slot {
Slot() noexcept : turn(0) {}
~Slot() noexcept {
if (turn & 1) {
destroy();
}
}

~Slot() noexcept {
if (turn & 1) {
destroy();
template <typename... Args> void construct(Args &&...args) noexcept {
std::construct_at(reinterpret_cast<T *>(storage),
std::forward<Args>(args)...);
}
}

template <typename... Args> void construct(Args &&...args) noexcept {
std::construct_at(reinterpret_cast<T *>(storage),
std::forward<Args>(args)...);
}
void destroy() noexcept { std::destroy_at(value()); }

void destroy() noexcept { std::destroy_at(value()); }
T &&move() noexcept { return std::move(*value()); }

T &&move() noexcept { return std::move(*value()); }
// Align to avoid false sharing between adjacent slots.
alignas(kFalseSharingRange) std::atomic<std::size_t> turn;

// Align to avoid false sharing between adjacent slots.
alignas(kFalseSharingRange) std::atomic<std::size_t> turn;
private:
T *value() noexcept { return std::launder(reinterpret_cast<T *>(storage)); }

private:
T *value() noexcept { return std::launder(reinterpret_cast<T *>(storage)); }
alignas(T) std::byte storage[sizeof(T)];
};

alignas(T) std::byte storage[sizeof(T)];
};
using SlotAllocator =
typename std::allocator_traits<Allocator>::template rebind_alloc<Slot>;
using SlotAllocatorTraits = std::allocator_traits<SlotAllocator>;

static_assert(std::is_same_v<typename SlotAllocatorTraits::pointer, Slot *>,
"The rebound allocator must use raw pointers");

template <typename T, typename Allocator = AlignedAllocator<Slot<T>>>
requires std::is_nothrow_destructible_v<T>
class Queue {
public:
explicit Queue(const std::size_t capacity,
const Allocator &allocator = Allocator())
explicit MPMCQueue(const std::size_t capacity,
const Allocator &allocator = Allocator())
: capacity_(capacity), allocator_(allocator), head_(0), tail_(0) {
if (capacity_ == 0) [[unlikely]] {
throw std::invalid_argument("capacity < 1");
Expand All @@ -86,38 +90,39 @@ class Queue {
throw std::bad_array_new_length();
}
// Allocate one extra slot to prevent false sharing on the last slot
slots_ = allocator_.allocate(capacity_ + 1);
slots_ = SlotAllocatorTraits::allocate(allocator_, capacity_ + 1);
// Allocators are not required to honor alignment for over-aligned types
// (see http://eel.is/c++draft/allocator.requirements#10) so we verify
// alignment here
if (reinterpret_cast<std::uintptr_t>(slots_) % alignof(Slot<T>) != 0) [[unlikely]] {
allocator_.deallocate(slots_, capacity_ + 1);
if (reinterpret_cast<std::uintptr_t>(slots_) % alignof(Slot) != 0)
[[unlikely]] {
deallocate_slots();
throw std::bad_alloc();
}
for (std::size_t i = 0; i < capacity_; ++i) {
std::construct_at(&slots_[i]);
}
static_assert(
alignof(Slot<T>) >= kFalseSharingRange,
alignof(Slot) >= kFalseSharingRange,
"Slot must be aligned to cache line boundary to prevent false sharing");
static_assert(sizeof(Slot<T>) % kFalseSharingRange == 0,
static_assert(sizeof(Slot) % kFalseSharingRange == 0,
"Slot size must be a multiple of cache line size to prevent "
"false sharing between adjacent slots");
static_assert(sizeof(Queue) % kFalseSharingRange == 0,
static_assert(sizeof(MPMCQueue) % kFalseSharingRange == 0,
"Queue size must be a multiple of cache line size to "
"prevent false sharing between adjacent queues");
}

~Queue() noexcept {
~MPMCQueue() noexcept {
for (std::size_t i = 0; i < capacity_; ++i) {
std::destroy_at(&slots_[i]);
}
allocator_.deallocate(slots_, capacity_ + 1);
deallocate_slots();
}

// non-copyable and non-movable
Queue(const Queue &) = delete;
Queue &operator=(const Queue &) = delete;
MPMCQueue(const MPMCQueue &) = delete;
MPMCQueue &operator=(const MPMCQueue &) = delete;

template <typename... Args>
requires std::is_nothrow_constructible_v<T, Args &&...>
Expand Down Expand Up @@ -254,18 +259,17 @@ class Queue {
constexpr std::size_t turn(std::size_t i) const noexcept { return i / capacity_; }

private:
void deallocate_slots() noexcept {
SlotAllocatorTraits::deallocate(allocator_, slots_, capacity_ + 1);
}

const std::size_t capacity_;
Slot<T> *slots_;
[[no_unique_address]] Allocator allocator_;
Slot *slots_;
[[no_unique_address]] SlotAllocator allocator_;

// Align to avoid false sharing between head_ and tail_
alignas(kFalseSharingRange) std::atomic<std::size_t> head_;
alignas(kFalseSharingRange) std::atomic<std::size_t> tail_;
};
} // namespace mpmc

template <typename T,
typename Allocator = mpmc::AlignedAllocator<mpmc::Slot<T>>>
using MPMCQueue = mpmc::Queue<T, Allocator>;

} // namespace rigtorp
61 changes: 61 additions & 0 deletions src/MPMCQueueTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ SOFTWARE.
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <memory>
#include <new>
#include <rigtorp/MPMCQueue.h>
#include <set>
#include <thread>
Expand Down Expand Up @@ -67,6 +69,44 @@ struct TestType {

std::set<const TestType *> TestType::constructed;

struct AllocationState {
std::size_t count = 0;
bool misalign = false;
};

template <typename T> struct TestAllocator {
using value_type = T;

explicit TestAllocator(AllocationState &allocationState) noexcept
: state(allocationState) {}

template <typename U>
TestAllocator(const TestAllocator<U> &other) noexcept : state(other.state) {}

T *allocate(std::size_t count) {
assert(state.count == 0);
auto *allocation = std::allocator<T>{}.allocate(count);
state.count = count;
if (state.misalign) {
auto *bytes = reinterpret_cast<std::byte *>(allocation);
return static_cast<T *>(static_cast<void *>(bytes + 1));
}
return allocation;
}

void deallocate(T *allocation, std::size_t count) noexcept {
assert(state.count == count);
if (state.misalign) {
auto *bytes = reinterpret_cast<std::byte *>(allocation);
allocation = static_cast<T *>(static_cast<void *>(bytes - 1));
}
std::allocator<T>{}.deallocate(allocation, count);
state.count = 0;
}

AllocationState &state;
};

int main(int argc, char *argv[]) {
(void)argc;
(void)argv;
Expand Down Expand Up @@ -209,6 +249,27 @@ int main(int argc, char *argv[]) {
assert(throws == true);
}

{
AllocationState state;
{
MPMCQueue<int, TestAllocator<int>> q(1, TestAllocator<int>(state));
assert(state.count == 2);
assert(q.try_push(42));
int value = 0;
assert(q.try_pop(value) && value == 42);
}
assert(state.count == 0);

state.misalign = true;
bool throws = false;
try {
MPMCQueue<int, TestAllocator<int>> q(1, TestAllocator<int>(state));
} catch (const std::bad_alloc &) {
throws = true;
}
assert(throws && state.count == 0);
}

// Fuzz test
{
const std::size_t numOps = 1000;
Expand Down
Loading