diff --git a/CMakeLists.txt b/CMakeLists.txt index 1dc8325..ba34c86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,6 +45,10 @@ if(PROJECT_IS_TOP_LEVEL AND BUILD_TESTING) if(UNIX AND NOT APPLE) spscqueue_executable(SPSCQueueBenchmark src/SPSCQueueBenchmark.cpp) + find_library(RTE_RING_LIBRARY rte_ring) + if(RTE_RING_LIBRARY) + target_link_libraries(SPSCQueueBenchmark PRIVATE ${RTE_RING_LIBRARY}) + endif() spscqueue_executable(SPSCQueueExampleHugepages src/SPSCQueueExampleHugepages.cpp) endif() diff --git a/README.md b/README.md index 38f8bb3..3ca8406 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,21 @@ # SPSCQueue.h -A single-producer single-consumer queue written in C++20. It has a fixed capacity -and supports wait-free operations. - -In the benchmarks below, it is faster than both -[*boost::lockfree::spsc*](https://www.boost.org/doc/libs/1_76_0/doc/html/boost/lockfree/spsc_queue.html) -and [*folly::ProducerConsumerQueue*](https://github.com/facebook/folly/blob/master/folly/docs/ProducerConsumerQueue.md). +A single-header, wait-free single-producer single-consumer queue written in +C++20. + +Why choose `SPSCQueue`: + +- **Fast under real workloads**: leads all tested queues in bounded throughput + (`219.3 Mops/s`) with `86–91 ns` warm round-trip latency (faster than + `boost::lockfree::spsc_queue`, `folly::ProducerConsumerQueue`, + `moodycamel::ReaderWriterQueue`, `dro::SPSCQueue`, and `DPDK rte_ring`). +- **No artificial type or capacity restrictions**: supports + non-default-constructible and immovable types, arbitrary non-power-of-two + capacities, custom allocators (including `allocate_at_least`), and zero-copy + consumption (`front()`/`pop()`, `try_consume()`). +- **Drop-in and rigorously verified**: single header with no external + dependencies, 100% MC/DC test coverage, and bounded RC11 weak-memory + verification with GenMC. ## Example @@ -247,28 +257,77 @@ verification layers: ## Benchmarks -Throughput benchmark measures throughput between 2 threads for a queue of `int` -items. - -Latency benchmark measures round trip time between 2 threads communicating using -2 queues of `int` items. - -Benchmark results for a AMD Ryzen 9 3900X 12-Core Processor, the 2 threads are -running on different cores on the same chiplet: - -| Queue | Throughput (ops/ms) | Latency RTT (ns) | -| ---------------------------- | ------------------: | ---------------: | -| SPSCQueue | 362723 | 133 | -| boost::lockfree::spsc | 209877 | 222 | -| folly::ProducerConsumerQueue | 148818 | 147 | - -## Cited by +`SPSCQueueBenchmark` interleaves all queues in a Latin-square order across +multiple trials and reports the median for four workloads using `int` items: + +- **Bounded throughput** (`Mops/s`): 50M operations on a 65,536-entry queue. + This is the primary throughput metric: real queues run under backpressure + where producer and consumer repeatedly catch up, wrap around, and exchange + cache lines, rather than simply buffering millions of items uncontested. +- **Decoupled throughput** (`Mops/s`): 2M operations on a 2M-entry queue. + Measures uncontended per-operation overhead when the ring is large enough + that producer and consumer rarely stall on each other. +- **Warm latency** (`ns RTT`): 2M round trips between 2 threads via 2 queues. + Measures best-case ping-pong handoff latency with queue metadata hot in CPU + caches, isolating atomic synchronization and false-sharing overhead. +- **Cold latency** (`ns RTT`): round trips striped across 262,144 queue pairs. + Evicts queue state from CPU caches between accesses, measuring how compact + and cache-friendly the queue layout is under bursty or many-queue workloads. + +To build and run the benchmark with all optional comparison queues using Nix: + +```sh +nix develop --command bash -c "cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build --target SPSCQueueBenchmark" +./build/SPSCQueueBenchmark 0 1 +``` -SPSCQueue have been cited by the following papers: +Median results on an AMD Ryzen 7 8745HS (2 threads pinned to separate physical +cores 0 and 1 sharing L3 cache): + +| Queue | Decoupled (Mops/s) | Bounded (Mops/s) | Warm RTT (ns) | Cold RTT (ns) | +| --- | ---: | ---: | ---: | ---: | +| `rigtorp::SPSCQueue` | 717.3 | 219.3 | 91 | 142 | +| `rigtorp::SPSCQueue` (`try_pop`) | 683.4 | 204.2 | 86 | 161 | +| `rigtorp::SPSCQueue` (`try_consume`) | 696.8 | 200.0 | 86 | 125 | +| `boost::lockfree::spsc_queue` | 207.5 | 186.9 | 155 | 163 | +| `folly::ProducerConsumerQueue` | 226.5 | 149.0 | 136 | 146 | +| `rigtorp::MPMCQueue` | 65.7 | 61.4 | 118 | 294 | +| `moodycamel::ReaderWriterQueue` | 218.1 | 164.9 | 139 | 155 | +| `dro::SPSCQueue` | 866.4 | 172.1 | 154 | 147 | +| `atomic_queue::AtomicQueueB` (`SPSC`) | 739.5 | 204.2 | 95 | 97 | +| `atomic_queue::AtomicQueueB2` (`SPSC`) | 654.1 | 163.7 | 100 | 106 | +| `DPDK rte_ring` (`SP/SC`) | 180.0 | 133.1 | 146 | 145 | + +## Citation + +If you use SPSCQueue in a paper, please cite it. Include the version or commit +used in your work. + +```bibtex +@misc{rigtorp_spscqueue, + author = {Erik Rigtorp}, + title = {{SPSCQueue}}, + howpublished = {\url{https://github.com/rigtorp/SPSCQueue}} +} +``` -- Peizhao Ou and Brian Demsky. 2018. Towards understanding the costs of avoiding - out-of-thin-air results. Proc. ACM Program. Lang. 2, OOPSLA, Article 136 - (October 2018), 29 pages. DOI: +Publications that cite SPSCQueue: + +- Peizhao Ou and Brian Demsky. 2018. + [Towards Understanding the Costs of Avoiding Out-of-Thin-Air Results](https://doi.org/10.1145/3276506). + OOPSLA. +- Thibaut Carpentier. 2018. + [Synchronisation de données inter-processus dans les applications audio temps réel : qu’est-ce qui débloque ?](https://hal.science/hal-01791422). + JIM, pp. 35–44. +- Jörg Christian Kirchhof. 2022. + [Model-Driven Development, Deployment, and Analysis of Internet of Things Applications](https://www.se-rwth.de/phdtheses/Diss-Kirchhof-Model-Driven-Development-Deployment-and-Analysis-of-Internet-of-Things-Applications.pdf). + Doctoral thesis. +- Clément Allain. 2025. + [Verification of fine-grained concurrent OCaml 5 algorithms using separation logic](https://iris-project.org/pdfs/2025-phd-allain.pdf). + Doctoral thesis. +- Ziye Xie et al. 2026. + [Toward Accurate and Accessible Markerless Neuronavigation](https://arxiv.org/abs/2602.07052v1). + Preprint, version 1. ## About diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..de8887f --- /dev/null +++ b/flake.lock @@ -0,0 +1,78 @@ +{ + "nodes": { + "atomic-queue": { + "flake": false, + "locked": { + "lastModified": 1792723865, + "narHash": "sha256-Bp4NUoiTdVW9CMQqhMzH3fm/pL6DrPuwxx2mR+7KaiM=", + "owner": "max0x7ba", + "repo": "atomic_queue", + "rev": "b7f1befe89446752767243c5e5550779a9a5eed2", + "type": "github" + }, + "original": { + "owner": "max0x7ba", + "repo": "atomic_queue", + "type": "github" + } + }, + "dro-spsc-queue": { + "flake": false, + "locked": { + "lastModified": 1764518466, + "narHash": "sha256-/bGBhsDuU/ItBgl15y56KASeQxLRr6Nt4u8XgEKOas4=", + "owner": "drogalis", + "repo": "SPSC-Queue", + "rev": "c959bd4f7204dd73c4e75a2b00c3459c3a5e9a96", + "type": "github" + }, + "original": { + "owner": "drogalis", + "repo": "SPSC-Queue", + "type": "github" + } + }, + "mpmcqueue": { + "flake": false, + "locked": { + "lastModified": 1694997842, + "narHash": "sha256-oHrXvCVE9zd6mlT5gJzp/7Eb0/WZ0uXOeOHvucVlMD4=", + "owner": "rigtorp", + "repo": "MPMCQueue", + "rev": "b9808ede08f26fa9df4df4e081d19cace8f6c6ea", + "type": "github" + }, + "original": { + "owner": "rigtorp", + "repo": "MPMCQueue", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1794386569, + "narHash": "sha256-x9sADJvoVG0L6RejZB+/BB4YiGZFBZrfTlhF4uBq1tQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "d4f25a3e562008648d45bcf2d70bcc7422cad84e", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "atomic-queue": "atomic-queue", + "dro-spsc-queue": "dro-spsc-queue", + "mpmcqueue": "mpmcqueue", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..b1f1329 --- /dev/null +++ b/flake.nix @@ -0,0 +1,94 @@ +{ + description = "SPSCQueue benchmark environment with optional comparison queues"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + mpmcqueue = { + url = "github:rigtorp/MPMCQueue"; + flake = false; + }; + dro-spsc-queue = { + url = "github:drogalis/SPSC-Queue"; + flake = false; + }; + atomic-queue = { + url = "github:max0x7ba/atomic_queue"; + flake = false; + }; + }; + + outputs = { + self, + nixpkgs, + mpmcqueue, + dro-spsc-queue, + atomic-queue, + }: let + supportedSystems = ["x86_64-linux" "aarch64-linux"]; + forAllSystems = nixpkgs.lib.genAttrs supportedSystems; + in { + packages = forAllSystems (system: let + pkgs = nixpkgs.legacyPackages.${system}; + benchmark = pkgs.stdenv.mkDerivation { + pname = "spscqueue-benchmark"; + version = "2.0.0"; + src = ./.; + + hardeningDisable = ["all"]; + + nativeBuildInputs = [pkgs.cmake]; + buildInputs = [ + pkgs.boost.dev + pkgs.dpdk + pkgs.folly.dev + pkgs.readerwriterqueue + ]; + + cmakeFlags = [ + "-DCMAKE_BUILD_TYPE=Release" + "-DBUILD_TESTING=ON" + ]; + + env.NIX_CFLAGS_COMPILE = "-isystem ${mpmcqueue}/include -isystem ${dro-spsc-queue}/include -isystem ${atomic-queue}/include"; + + buildFlags = ["SPSCQueueBenchmark"]; + + installPhase = '' + runHook preInstall + mkdir -p $out/bin + cp SPSCQueueBenchmark $out/bin/SPSCQueueBenchmark + runHook postInstall + ''; + }; + in { + inherit benchmark; + default = benchmark; + }); + + apps = forAllSystems (system: { + benchmark = { + type = "app"; + program = "${self.packages.${system}.benchmark}/bin/SPSCQueueBenchmark"; + }; + default = self.apps.${system}.benchmark; + }); + + devShells = forAllSystems (system: let + pkgs = nixpkgs.legacyPackages.${system}; + in { + default = pkgs.mkShell { + packages = [ + pkgs.cmake + pkgs.ninja + pkgs.boost.dev + pkgs.dpdk + pkgs.folly.dev + pkgs.readerwriterqueue + ]; + shellHook = '' + export NIX_CFLAGS_COMPILE="''${NIX_CFLAGS_COMPILE:-} -isystem ${mpmcqueue}/include -isystem ${dro-spsc-queue}/include -isystem ${atomic-queue}/include" + ''; + }; + }); + }; +} diff --git a/src/SPSCQueueBenchmark.cpp b/src/SPSCQueueBenchmark.cpp index 87385d2..9faaf9e 100644 --- a/src/SPSCQueueBenchmark.cpp +++ b/src/SPSCQueueBenchmark.cpp @@ -1,13 +1,20 @@ // SPDX-FileCopyrightText: 2018 Erik Rigtorp // SPDX-License-Identifier: MIT +#include +#include #include #include #include +#include +#include #include +#include #include #include +#include #include +#include #if __has_include( ) #include @@ -17,6 +24,44 @@ #include #endif +#if __has_include() +#include +#endif + +#if __has_include() +#include +#elif __has_include() +#include +#endif + +#if __has_include() +#include +#endif + +#if __has_include() +#include +#endif + +#if __has_include() && __has_include() +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wold-style-cast" +#pragma GCC diagnostic ignored "-Wconversion" +#pragma GCC diagnostic ignored "-Wsign-conversion" +#pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" +#pragma GCC diagnostic ignored "-Wpedantic" +#pragma GCC diagnostic ignored "-Wcast-align" +#pragma GCC diagnostic ignored "-Wshadow" +#endif +extern "C" { +#include +#include +} +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif +#endif + void pinThread(int cpu) { if (cpu < 0) { return; @@ -33,6 +78,143 @@ void pinThread(int cpu) { } } +template +static int64_t runSynced(int cpu1, int cpu2, Consumer &&consumer, + Producer &&producer) { + std::atomic consumerReady{false}; + std::atomic go{false}; + auto t = std::thread([&] { + pinThread(cpu1); + consumerReady.store(true, std::memory_order_release); + while (!go.load(std::memory_order_acquire)) + ; + consumer(); + }); + + pinThread(cpu2); + while (!consumerReady.load(std::memory_order_acquire)) + ; + + auto start = std::chrono::steady_clock::now(); + go.store(true, std::memory_order_release); + producer(); + t.join(); + auto stop = std::chrono::steady_clock::now(); + return std::chrono::duration_cast(stop - start) + .count(); +} + +template +static int64_t runTPSingle(int cpu1, int cpu2, int64_t iters, + Consumer &&consumer, Producer &&producer) { + constexpr int64_t prefill = 4096; + auto run = [&] { + for (int64_t i = 0; i < prefill; ++i) { + producer(i); + } + auto ns = runSynced( + cpu1, cpu2, + [&] { + for (int64_t i = 0; i < iters; ++i) { + consumer(i); + } + }, + [&] { + for (int64_t i = prefill; i < iters + prefill; ++i) { + producer(i); + } + }); + for (int64_t i = iters; i < iters + prefill; ++i) { + consumer(i); + } + return ns; + }; + + run(); // warmup + return iters * 1000000 / run(); +} + +template +static int64_t runRTTSingle(int cpu1, int cpu2, int64_t iters, + Consumer &&consumer, Producer &&producer) { + auto run = [&] { + return runSynced( + cpu1, cpu2, + [&] { + for (int64_t i = 0; i < iters; ++i) { + consumer(); + } + }, + [&] { + for (int64_t i = 0; i < iters; ++i) { + producer(i); + } + }); + }; + + run(); // warmup + return run() / iters; +} + +template +static int64_t runColdSingle(int cpu1, int cpu2, size_t numQueues, + int64_t coldRounds, MakeQ &&makeQ, Pop &&pop, + Push &&push) { + std::vector qs1, qs2; + qs1.reserve(numQueues); + qs2.reserve(numQueues); + for (size_t i = 0; i < numQueues; ++i) { + qs1.emplace_back(makeQ()); + qs2.emplace_back(makeQ()); + } + + auto run = [&] { + return runSynced( + cpu1, cpu2, + [&] { + for (int64_t r = 0; r < coldRounds; ++r) { + for (size_t q = 0; q < numQueues; ++q) { + pop(*qs1[q], static_cast(q)); + push(*qs2[q], static_cast(q)); + } + } + }, + [&] { + for (int64_t r = 0; r < coldRounds; ++r) { + for (size_t q = 0; q < numQueues; ++q) { + push(*qs1[q], static_cast(q)); + pop(*qs2[q], static_cast(q)); + } + } + }); + }; + + run(); // warmup + return run() / (coldRounds * static_cast(numQueues)); +} + +struct BenchEntry { + std::string name; + std::function runTP; + std::function runRTT; + std::function runCold; + std::vector tpDecoupledResults{}; + std::vector tpBoundedResults{}; + std::vector rttResults{}; + std::vector coldResults{}; +}; + +static void printSummary(std::vector &v, const char *label, + const char *unit, double scale = 1.0, + int precision = 0) { + std::sort(v.begin(), v.end()); + std::cout << std::fixed << std::setprecision(precision) << " " << label + << ": min=" << static_cast(v.front()) * scale + << " median=" << static_cast(v[v.size() / 2]) * scale + << " max=" << static_cast(v.back()) * scale << " " << unit + << std::endl; +} + int main(int argc, char *argv[]) { (void)argc, (void)argv; @@ -46,199 +228,632 @@ int main(int argc, char *argv[]) { cpu2 = std::stoi(argv[2]); } - const size_t queueSize = 10000000; - const int64_t iters = 10000000; + const size_t decoupledCap = 1u << 21; // 2,097,152 (8 MiB for 4B slots) + const int64_t decoupledIters = 2000000; + const size_t boundedCap = 1u << 16; // 65,536 + const int64_t boundedIters = 50000000; + const size_t queueSize = boundedCap; + const int64_t rttIters = 2000000; + const size_t coldQueues = 262144; + const int64_t coldRounds = 4; - std::cout << "SPSCQueue:" << std::endl; + std::vector entries; - { - SPSCQueue q(queueSize); - auto t = std::thread([&] { - pinThread(cpu1); - for (int64_t i = 0; i < iters; ++i) { - while (!q.front()) - ; - if (*q.front() != i) { - throw std::runtime_error(""); - } - q.pop(); - } - }); + entries.push_back({"SPSCQueue:", + [&](size_t cap, int64_t iters) { + SPSCQueue q(cap); + return runTPSingle( + cpu1, cpu2, iters, + [&](int64_t i) { + int *p; + while (!(p = q.front())) + ; + if (*p != i) { + throw std::runtime_error(""); + } + q.pop(); + }, + [&](int64_t i) { q.emplace(static_cast(i)); }); + }, + [&] { + SPSCQueue q1(queueSize), q2(queueSize); + return runRTTSingle( + cpu1, cpu2, rttIters, + [&] { + int *p; + while (!(p = q1.front())) + ; + q2.emplace(*p); + q1.pop(); + }, + [&](int64_t i) { + q1.emplace(static_cast(i)); + while (!q2.front()) + ; + q2.pop(); + }); + }, + [&] { + return runColdSingle( + cpu1, cpu2, coldQueues, coldRounds, + [] { return std::make_unique>(8); }, + [](SPSCQueue &q, int expected) { + int *p; + while (!(p = q.front())) + ; + if (*p != expected) { + throw std::runtime_error(""); + } + q.pop(); + }, + [](SPSCQueue &q, int val) { q.emplace(val); }); + }}); - pinThread(cpu2); + entries.push_back({"SPSCQueue (try_pop):", + [&](size_t cap, int64_t iters) { + SPSCQueue q(cap); + return runTPSingle( + cpu1, cpu2, iters, + [&](int64_t i) { + int val; + while (!q.try_pop(val)) + ; + if (val != i) { + throw std::runtime_error(""); + } + }, + [&](int64_t i) { q.emplace(static_cast(i)); }); + }, + [&] { + SPSCQueue q1(queueSize), q2(queueSize); + return runRTTSingle( + cpu1, cpu2, rttIters, + [&] { + int val; + while (!q1.try_pop(val)) + ; + q2.emplace(val); + }, + [&](int64_t i) { + q1.emplace(static_cast(i)); + int val; + while (!q2.try_pop(val)) + ; + }); + }, + [&] { + return runColdSingle( + cpu1, cpu2, coldQueues, coldRounds, + [] { return std::make_unique>(8); }, + [](SPSCQueue &q, int expected) { + int val; + while (!q.try_pop(val)) + ; + if (val != expected) { + throw std::runtime_error(""); + } + }, + [](SPSCQueue &q, int val) { q.emplace(val); }); + }}); - auto start = std::chrono::steady_clock::now(); - for (int64_t i = 0; i < iters; ++i) { - q.emplace(static_cast(i)); - } - t.join(); - auto stop = std::chrono::steady_clock::now(); - std::cout << iters * 1000000 / - std::chrono::duration_cast(stop - - start) - .count() - << " ops/ms" << std::endl; - } - - { - SPSCQueue q1(queueSize), q2(queueSize); - auto t = std::thread([&] { - pinThread(cpu1); - for (int64_t i = 0; i < iters; ++i) { - while (!q1.front()) - ; - q2.emplace(*q1.front()); - q1.pop(); - } - }); - - pinThread(cpu2); - - auto start = std::chrono::steady_clock::now(); - for (int64_t i = 0; i < iters; ++i) { - q1.emplace(static_cast(i)); - while (!q2.front()) - ; - q2.pop(); - } - auto stop = std::chrono::steady_clock::now(); - t.join(); - std::cout << std::chrono::duration_cast(stop - - start) - .count() / - iters - << " ns RTT" << std::endl; - } + entries.push_back( + {"SPSCQueue (try_consume):", + [&](size_t cap, int64_t iters) { + SPSCQueue q(cap); + return runTPSingle( + cpu1, cpu2, iters, + [&](int64_t i) { + while (!q.try_consume([&](int &val) { + if (val != i) { + throw std::runtime_error(""); + } + })) + ; + }, + [&](int64_t i) { q.emplace(static_cast(i)); }); + }, + [&] { + SPSCQueue q1(queueSize), q2(queueSize); + return runRTTSingle( + cpu1, cpu2, rttIters, + [&] { + while (!q1.try_consume([&](int &val) { q2.emplace(val); })) + ; + }, + [&](int64_t i) { + q1.emplace(static_cast(i)); + while (!q2.try_consume([](int &) {})) + ; + }); + }, + [&] { + return runColdSingle( + cpu1, cpu2, coldQueues, coldRounds, + [] { return std::make_unique>(8); }, + [](SPSCQueue &q, int expected) { + while (!q.try_consume([&](int &val) { + if (val != expected) { + throw std::runtime_error(""); + } + })) + ; + }, + [](SPSCQueue &q, int val) { q.emplace(val); }); + }}); #if __has_include( ) - std::cout << "boost::lockfree::spsc:" << std::endl; - { - boost::lockfree::spsc_queue q(queueSize); - auto t = std::thread([&] { - pinThread(cpu1); - for (int64_t i = 0; i < iters; ++i) { - int val; - while (q.pop(&val, 1) != 1) - ; - if (val != i) { - throw std::runtime_error(""); - } - } - }); - - pinThread(cpu2); + entries.push_back( + {"boost::lockfree::spsc:", + [&](size_t cap, int64_t iters) { + boost::lockfree::spsc_queue q(cap); + return runTPSingle( + cpu1, cpu2, iters, + [&](int64_t i) { + int val; + while (!q.pop(val)) + ; + if (val != i) { + throw std::runtime_error(""); + } + }, + [&](int64_t i) { + while (!q.push(static_cast(i))) + ; + }); + }, + [&] { + boost::lockfree::spsc_queue q1(queueSize), q2(queueSize); + return runRTTSingle( + cpu1, cpu2, rttIters, + [&] { + int val; + while (!q1.pop(val)) + ; + while (!q2.push(val)) + ; + }, + [&](int64_t i) { + while (!q1.push(static_cast(i))) + ; + int val; + while (!q2.pop(val)) + ; + }); + }, + [&] { + return runColdSingle( + cpu1, cpu2, coldQueues, coldRounds, + [] { + return std::make_unique>(8); + }, + [](boost::lockfree::spsc_queue &q, int expected) { + int val; + while (!q.pop(val)) + ; + if (val != expected) { + throw std::runtime_error(""); + } + }, + [](boost::lockfree::spsc_queue &q, int val) { + while (!q.push(val)) + ; + }); + }}); +#endif - auto start = std::chrono::steady_clock::now(); - for (int64_t i = 0; i < iters; ++i) { - while (!q.push(static_cast(i))) - ; - } - t.join(); - auto stop = std::chrono::steady_clock::now(); - std::cout << iters * 1000000 / - std::chrono::duration_cast(stop - - start) - .count() - << " ops/ms" << std::endl; - } +#if __has_include() + entries.push_back( + {"folly::ProducerConsumerQueue:", + [&](size_t cap, int64_t iters) { + folly::ProducerConsumerQueue q(static_cast(cap)); + return runTPSingle( + cpu1, cpu2, iters, + [&](int64_t i) { + int val; + while (!q.read(val)) + ; + if (val != i) { + throw std::runtime_error(""); + } + }, + [&](int64_t i) { + while (!q.write(static_cast(i))) + ; + }); + }, + [&] { + folly::ProducerConsumerQueue q1(queueSize), q2(queueSize); + return runRTTSingle( + cpu1, cpu2, rttIters, + [&] { + int val; + while (!q1.read(val)) + ; + q2.write(val); + }, + [&](int64_t i) { + while (!q1.write(static_cast(i))) + ; + int val; + while (!q2.read(val)) + ; + }); + }, + [&] { + return runColdSingle( + cpu1, cpu2, coldQueues, coldRounds, + [] { + return std::make_unique>(8); + }, + [](folly::ProducerConsumerQueue &q, int expected) { + int val; + while (!q.read(val)) + ; + if (val != expected) { + throw std::runtime_error(""); + } + }, + [](folly::ProducerConsumerQueue &q, int val) { + while (!q.write(val)) + ; + }); + }}); +#endif - { - boost::lockfree::spsc_queue q1(queueSize), q2(queueSize); - auto t = std::thread([&] { - pinThread(cpu1); - for (int64_t i = 0; i < iters; ++i) { - int val; - while (q1.pop(&val, 1) != 1) - ; - while (!q2.push(val)) - ; - } - }); +#if __has_include() + entries.push_back({"rigtorp::MPMCQueue:", + [&](size_t cap, int64_t iters) { + MPMCQueue q(cap); + return runTPSingle( + cpu1, cpu2, iters, + [&](int64_t i) { + int val; + q.pop(val); + if (val != i) { + throw std::runtime_error(""); + } + }, + [&](int64_t i) { q.push(static_cast(i)); }); + }, + [&] { + MPMCQueue q1(queueSize), q2(queueSize); + return runRTTSingle( + cpu1, cpu2, rttIters, + [&] { + int val; + q1.pop(val); + q2.push(val); + }, + [&](int64_t i) { + q1.push(static_cast(i)); + int val; + q2.pop(val); + }); + }, + [&] { + return runColdSingle( + cpu1, cpu2, coldQueues, coldRounds, + [] { return std::make_unique>(8); }, + [](MPMCQueue &q, int expected) { + int val; + q.pop(val); + if (val != expected) { + throw std::runtime_error(""); + } + }, + [](MPMCQueue &q, int val) { q.push(val); }); + }}); +#endif - pinThread(cpu2); +#if __has_include() || \ + __has_include() + entries.push_back( + {"moodycamel::ReaderWriterQueue:", + [&](size_t cap, int64_t iters) { + moodycamel::ReaderWriterQueue q(cap); + return runTPSingle( + cpu1, cpu2, iters, + [&](int64_t i) { + int val; + while (!q.try_dequeue(val)) + ; + if (val != i) { + throw std::runtime_error(""); + } + }, + [&](int64_t i) { + while (!q.try_enqueue(static_cast(i))) + ; + }); + }, + [&] { + moodycamel::ReaderWriterQueue q1(queueSize), q2(queueSize); + return runRTTSingle( + cpu1, cpu2, rttIters, + [&] { + int val; + while (!q1.try_dequeue(val)) + ; + q2.enqueue(val); + }, + [&](int64_t i) { + while (!q1.try_enqueue(static_cast(i))) + ; + int val; + while (!q2.try_dequeue(val)) + ; + }); + }, + [&] { + return runColdSingle( + cpu1, cpu2, coldQueues, coldRounds, + [] { + return std::make_unique>(8); + }, + [](moodycamel::ReaderWriterQueue &q, int expected) { + int val; + while (!q.try_dequeue(val)) + ; + if (val != expected) { + throw std::runtime_error(""); + } + }, + [](moodycamel::ReaderWriterQueue &q, int val) { + while (!q.try_enqueue(val)) + ; + }); + }}); +#endif - auto start = std::chrono::steady_clock::now(); - for (int64_t i = 0; i < iters; ++i) { - while (!q1.push(static_cast(i))) - ; - int val; - while (q2.pop(&val, 1) != 1) - ; - } - auto stop = std::chrono::steady_clock::now(); - t.join(); - std::cout << std::chrono::duration_cast(stop - - start) - .count() / - iters - << " ns RTT" << std::endl; - } +#if __has_include() + entries.push_back( + {"dro::SPSCQueue:", + [&](size_t cap, int64_t iters) { + dro::SPSCQueue q(cap); + return runTPSingle( + cpu1, cpu2, iters, + [&](int64_t i) { + int val; + q.pop(val); + if (val != i) { + throw std::runtime_error(""); + } + }, + [&](int64_t i) { q.emplace(static_cast(i)); }); + }, + [&] { + dro::SPSCQueue q1(queueSize), q2(queueSize); + return runRTTSingle( + cpu1, cpu2, rttIters, + [&] { + int val; + q1.pop(val); + q2.emplace(val); + }, + [&](int64_t i) { + q1.emplace(static_cast(i)); + int val; + q2.pop(val); + }); + }, + [&] { + return runColdSingle( + cpu1, cpu2, coldQueues, coldRounds, + [] { return std::make_unique>(8); }, + [](dro::SPSCQueue &q, int expected) { + int val; + q.pop(val); + if (val != expected) { + throw std::runtime_error(""); + } + }, + [](dro::SPSCQueue &q, int val) { q.emplace(val); }); + }}); #endif -#if __has_include() - std::cout << "folly::ProducerConsumerQueue:" << std::endl; +#if __has_include() + entries.push_back({"atomic_queue::AtomicQueueB (SPSC):", + [&](size_t cap, int64_t iters) { + atomic_queue::AtomicQueueB, -1, + true, false, true> + q(static_cast(cap)); + return runTPSingle( + cpu1, cpu2, iters, + [&](int64_t i) { + if (q.pop() != i) { + throw std::runtime_error(""); + } + }, + [&](int64_t i) { q.push(static_cast(i)); }); + }, + [&] { + atomic_queue::AtomicQueueB, -1, + true, false, true> + q1(static_cast(queueSize)), + q2(static_cast(queueSize)); + return runRTTSingle( + cpu1, cpu2, rttIters, [&] { q2.push(q1.pop()); }, + [&](int64_t i) { + q1.push(static_cast(i)); + (void)q2.pop(); + }); + }, + [&] { + using AQB = + atomic_queue::AtomicQueueB, + -1, true, false, true>; + return runColdSingle( + cpu1, cpu2, coldQueues, coldRounds, + [] { return std::make_unique(8); }, + [](AQB &q, int expected) { + if (q.pop() != expected) { + throw std::runtime_error(""); + } + }, + [](AQB &q, int val) { q.push(val); }); + }}); + entries.push_back({"atomic_queue::AtomicQueueB2 (SPSC):", + [&](size_t cap, int64_t iters) { + atomic_queue::AtomicQueueB2, + true, false, true> + q(static_cast(cap)); + return runTPSingle( + cpu1, cpu2, iters, + [&](int64_t i) { + if (q.pop() != i) { + throw std::runtime_error(""); + } + }, + [&](int64_t i) { q.push(static_cast(i)); }); + }, + [&] { + atomic_queue::AtomicQueueB2, + true, false, true> + q1(static_cast(queueSize)), + q2(static_cast(queueSize)); + return runRTTSingle( + cpu1, cpu2, rttIters, [&] { q2.push(q1.pop()); }, + [&](int64_t i) { + q1.push(static_cast(i)); + (void)q2.pop(); + }); + }, + [&] { + using AQB2 = + atomic_queue::AtomicQueueB2, + true, false, true>; + return runColdSingle( + cpu1, cpu2, coldQueues, coldRounds, + [] { return std::make_unique(8); }, + [](AQB2 &q, int expected) { + if (q.pop() != expected) { + throw std::runtime_error(""); + } + }, + [](AQB2 &q, int val) { q.push(val); }); + }}); +#endif + +#if __has_include() && __has_include() { - folly::ProducerConsumerQueue q(queueSize); - auto t = std::thread([&] { - pinThread(cpu1); - for (int64_t i = 0; i < iters; ++i) { - int val; - while (!q.read(val)) - ; - if (val != i) { - throw std::runtime_error(""); - } + auto makeRing = [](const char *name, unsigned int count) { + ssize_t sz = rte_ring_get_memsize_elem(sizeof(int), count); + if (sz < 0) { + throw std::runtime_error("rte_ring_get_memsize_elem failed"); + } + void *mem = std::aligned_alloc(64, static_cast(sz)); + if (!mem) { + throw std::bad_alloc(); } - }); + auto *r = static_cast(mem); + if (rte_ring_init(r, name, count, RING_F_SP_ENQ | RING_F_SC_DEQ) != 0) { + std::free(mem); + throw std::runtime_error("rte_ring_init failed"); + } + return r; + }; + const unsigned int ringCount = 1u << 16; - pinThread(cpu2); + entries.push_back( + {"DPDK rte_ring (SP/SC):", + [&, makeRing](size_t cap, int64_t iters) { + struct rte_ring *r = makeRing("tp", static_cast(cap)); + auto res = runTPSingle( + cpu1, cpu2, iters, + [&](int64_t i) { + int val; + while (rte_ring_sc_dequeue_elem(r, &val, sizeof(int)) != 0) + ; + if (val != i) { + throw std::runtime_error(""); + } + }, + [&](int64_t i) { + int val = static_cast(i); + while (rte_ring_sp_enqueue_elem(r, &val, sizeof(int)) != 0) + ; + }); + std::free(r); + return res; + }, + [&, makeRing] { + struct rte_ring *r1 = makeRing("rtt1", ringCount); + struct rte_ring *r2 = makeRing("rtt2", ringCount); + auto res = runRTTSingle( + cpu1, cpu2, rttIters, + [&] { + int val; + while (rte_ring_sc_dequeue_elem(r1, &val, sizeof(int)) != 0) + ; + while (rte_ring_sp_enqueue_elem(r2, &val, sizeof(int)) != 0) + ; + }, + [&](int64_t i) { + int val = static_cast(i); + while (rte_ring_sp_enqueue_elem(r1, &val, sizeof(int)) != 0) + ; + while (rte_ring_sc_dequeue_elem(r2, &val, sizeof(int)) != 0) + ; + }); + std::free(r1); + std::free(r2); + return res; + }, + [&, makeRing] { + return runColdSingle( + cpu1, cpu2, coldQueues, coldRounds, + [&] { + return std::unique_ptr( + makeRing("c", 8), std::free); + }, + [](struct rte_ring &r, int expected) { + int val; + while (rte_ring_sc_dequeue_elem(&r, &val, sizeof(int)) != 0) + ; + if (val != expected) { + throw std::runtime_error(""); + } + }, + [](struct rte_ring &r, int val) { + while (rte_ring_sp_enqueue_elem(&r, &val, sizeof(int)) != 0) + ; + }); + }}); + } +#endif - auto start = std::chrono::steady_clock::now(); - for (int64_t i = 0; i < iters; ++i) { - while (!q.write(i)) - ; + // Balanced Latin square: each entry runs once in every position 0..n-1. + const size_t n = entries.size(); + std::vector baseOrder(n); + for (size_t j = 0; j < n; ++j) { + if (j == 0) { + baseOrder[j] = 0; + } else if (j % 2 == 1) { + baseOrder[j] = (j + 1) / 2; + } else { + baseOrder[j] = n - j / 2; } - t.join(); - auto stop = std::chrono::steady_clock::now(); - std::cout << iters * 1000000 / - std::chrono::duration_cast(stop - - start) - .count() - << " ops/ms" << std::endl; } - - { - folly::ProducerConsumerQueue q1(queueSize), q2(queueSize); - auto t = std::thread([&] { - pinThread(cpu1); - for (int64_t i = 0; i < iters; ++i) { - int val; - while (!q1.read(val)) - ; - q2.write(val); - } - }); - - pinThread(cpu2); - - auto start = std::chrono::steady_clock::now(); - for (int64_t i = 0; i < iters; ++i) { - while (!q1.write(i)) - ; - int val; - while (!q2.read(val)) - ; + for (size_t r = 0; r < n; ++r) { + for (size_t idx = 0; idx < n; ++idx) { + size_t i = (baseOrder[idx] + r) % n; + entries[i].tpDecoupledResults.push_back( + entries[i].runTP(decoupledCap, decoupledIters)); + entries[i].tpBoundedResults.push_back( + entries[i].runTP(boundedCap, boundedIters)); + entries[i].rttResults.push_back(entries[i].runRTT()); + entries[i].coldResults.push_back(entries[i].runCold()); } - auto stop = std::chrono::steady_clock::now(); - t.join(); - std::cout << std::chrono::duration_cast(stop - - start) - .count() / - iters - << " ns RTT" << std::endl; } -#endif + + for (auto &entry : entries) { + std::cout << entry.name << std::endl; + printSummary(entry.tpDecoupledResults, "decoupled", "Mops/s", 1e-3, 1); + printSummary(entry.tpBoundedResults, "bounded ", "Mops/s", 1e-3, 1); + printSummary(entry.rttResults, "latency ", "ns RTT"); + printSummary(entry.coldResults, "cold ", "ns RTT"); + } return 0; }