diff --git a/.github/workflows/root-ci-config/buildconfig/fedora43.txt b/.github/workflows/root-ci-config/buildconfig/fedora43.txt index a65637d48d557..80e2a1473e3ff 100644 --- a/.github/workflows/root-ci-config/buildconfig/fedora43.txt +++ b/.github/workflows/root-ci-config/buildconfig/fedora43.txt @@ -2,7 +2,6 @@ CMAKE_CXX_STANDARD=23 builtin_civetweb=ON experimental_adaptivecpp=OFF pythia8=ON -roofit_multiprocess=ON test_distrdf_dask=OFF test_distrdf_pyspark=OFF vdt=OFF diff --git a/.github/workflows/root-ci-config/buildconfig/fedora44.txt b/.github/workflows/root-ci-config/buildconfig/fedora44.txt index c3b55fb254d94..8917b6298e66a 100644 --- a/.github/workflows/root-ci-config/buildconfig/fedora44.txt +++ b/.github/workflows/root-ci-config/buildconfig/fedora44.txt @@ -1,7 +1,6 @@ CMAKE_CXX_STANDARD=23 experimental_adaptivecpp=OFF pythia8=ON -roofit_multiprocess=ON test_distrdf_pyspark=OFF vdt=OFF cefweb=ON diff --git a/.github/workflows/root-ci-config/buildconfig/global.txt b/.github/workflows/root-ci-config/buildconfig/global.txt index 0d83e5d28d4e6..aa4f0756f5cdb 100644 --- a/.github/workflows/root-ci-config/buildconfig/global.txt +++ b/.github/workflows/root-ci-config/buildconfig/global.txt @@ -67,7 +67,6 @@ pyroot=ON pythia8=OFF qt6web=OFF roofit=ON -roofit_multiprocess=OFF root7=ON rootbench=OFF roottest=ON diff --git a/.github/workflows/root-ci-config/buildconfig/rawhide.txt b/.github/workflows/root-ci-config/buildconfig/rawhide.txt index d28eb9aca7c5d..4fff3403a6b58 100644 --- a/.github/workflows/root-ci-config/buildconfig/rawhide.txt +++ b/.github/workflows/root-ci-config/buildconfig/rawhide.txt @@ -3,4 +3,3 @@ pythia8=ON test_distrdf_dask=OFF test_distrdf_pyspark=OFF vdt=OFF -roofit_multiprocess=ON diff --git a/README/ReleaseNotes/v642/index.md b/README/ReleaseNotes/v642/index.md index 99b74f18edbef..4665f31c34509 100644 --- a/README/ReleaseNotes/v642/index.md +++ b/README/ReleaseNotes/v642/index.md @@ -50,6 +50,7 @@ The following people have contributed to this new version: * The overloads of `RooAbsReal::createChi2()` and `RooAbsReal::chi2FitTo()` that take unbinned **RooDataSet** data objects were deprecated in ROOT 6.40 and are now removed. * The **RooStats::HybridPlot** class and the related **HybridResult::GetPlot** method were deprecated in ROOT 6.40 and are now removed. * The `builtin_zeromq` and `builtin_cppzmq` build options that were deprecated in ROOT 6.40 are now removed. +* The `roofit_multiprocess` build option is deprecated and will be removed in ROOT 6.44. It has no effect anymore: RooFit's multi-process test statistics no longer depend on ZeroMQ and are now always built on non-Windows platforms, so there is no reason for an opt-in build option anymore (see the RooFit section below). * The ROOT **auth** package together with `TVirtualAuth` and `TROOT::GetListOfSecContexts()`, and the **authenticated sockets** (`TSocket::CreateAuthSocket()`) feature are now removed following deprecation in ROOT 6.40. * The `TSSLSocket` class is now removed following deprecation in ROOT 6.40. * The bindings to the R programming language that are enabled with the `r=ON` or `tmva-rmva=ON` build options (`TRInterface`, RMVA, and friends) are removed, following deprecation in ROOT 6.40. Their maintenance is no longer justified, given the broader adoption of the scientific Python ecosystem. Users who still rely on R from C++ are encouraged to call R directly via https://cran.r-project.org/package=RInside, which is what the ROOT bindings were using internally. @@ -154,6 +155,13 @@ the cut instead of being selected based on `sqrt(abs(x))`. ## RooFit +### RooFit::MultiProcess without ZeroMQ, now enabled by default + +The `RooFit::MultiProcess` package that implements the parallel gradient minimization with `fitTo(..., RooFit::Parallelize(n))` previously communicated between the forked processes with ZeroMQ sockets, which required building ROOT with `roofit_multiprocess=ON` and the ZeroMQ (with draft API) and cppzmq dependencies. +The interprocess communication is now implemented directly on top of plain `socketpair()` pipes that are inherited by the forked worker processes, so the ZeroMQ and cppzmq dependencies and the `RooFitZMQ` library are removed entirely. +Since the feature no longer needs extra dependencies, it is now always built on non-Windows platforms and the `roofit_multiprocess` build option has no effect anymore; it is deprecated and will be removed in ROOT 6.44. +For implementers of custom `RooFit::MultiProcess::Job` subclasses, the message type in the `Job` interface changed from `zmq::message_t` to the new `RooFit::MultiProcess::Message` byte-buffer class, which supports the same usage patterns. + ### Small changes * The `RooMinimizer::Strategy` enum has been removed. It named the Minuit strategies that are usually referred to just by integers, but caused confusion because it didn't include the unnamed "Strategy 3". Since people usually set the strategy with integer values anyway, it was decided that the simplest solution to avoid the confusion was simply to remove the `RooMinimizer::Strategy` enum diff --git a/cmake/modules/FindZeroMQ.cmake b/cmake/modules/FindZeroMQ.cmake deleted file mode 100644 index e8c2889962e3b..0000000000000 --- a/cmake/modules/FindZeroMQ.cmake +++ /dev/null @@ -1,67 +0,0 @@ -# Based on https://github.com/zeromq/cppzmq/blob/a98fa4a91d868a3844e5456741d6782cc1a8d98b/libzmq-pkg-config/FindZeroMQ.cmake -# MIT Licensed: -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH ON) -find_package(PkgConfig) -pkg_check_modules(PC_LIBZMQ QUIET libzmq) - -set(ZeroMQ_VERSION ${PC_LIBZMQ_VERSION}) - -find_path(ZeroMQ_INCLUDE_DIR zmq.h - PATHS ${ZeroMQ_DIR}/include - ${PC_LIBZMQ_INCLUDE_DIRS}) - -find_library(ZeroMQ_LIBRARY - NAMES zmq - PATHS ${ZeroMQ_DIR}/lib - ${PC_LIBZMQ_LIBDIR} - ${PC_LIBZMQ_LIBRARY_DIRS}) - -set ( ZeroMQ_LIBRARIES ${ZeroMQ_LIBRARY} ) -set ( ZeroMQ_INCLUDE_DIRS ${ZeroMQ_INCLUDE_DIR} ) - -# check for zmq_ppoll -if(ZeroMQ_LIBRARIES) - include(CheckCXXSymbolExists) - set(CMAKE_REQUIRED_LIBRARIES ${ZeroMQ_LIBRARIES}) - set(CMAKE_REQUIRED_INCLUDES ${ZeroMQ_INCLUDE_DIRS}) - set(CMAKE_REQUIRED_DEFINITIONS "-DZMQ_BUILD_DRAFT_API") - check_cxx_symbol_exists(zmq_ppoll zmq.h ZeroMQ_HAS_PPOLL) - if(NOT ZeroMQ_HAS_PPOLL) - message(SEND_ERROR "ZeroMQ library was compiled without draft API support (-DENABLE_DRAFTS).") - endif() - unset(CMAKE_REQUIRED_LIBRARIES) - unset(CMAKE_REQUIRED_INCLUDES) - unset(CMAKE_REQUIRED_DEFINITIONS) -endif() - -include ( FindPackageHandleStandardArgs ) -# handle the QUIETLY and REQUIRED arguments and set ZeroMQ_FOUND to TRUE -# if all listed variables are TRUE -find_package_handle_standard_args ( ZeroMQ DEFAULT_MSG ZeroMQ_LIBRARIES ZeroMQ_INCLUDE_DIRS ) - -if(ZeroMQ_FOUND) - if(NOT TARGET libzmq) - add_library(libzmq UNKNOWN IMPORTED) - set_target_properties(libzmq PROPERTIES - IMPORTED_LOCATION ${ZeroMQ_LIBRARIES} - INTERFACE_INCLUDE_DIRECTORIES ${ZeroMQ_INCLUDE_DIRS}) - endif() -endif() diff --git a/cmake/modules/Findcppzmq.cmake b/cmake/modules/Findcppzmq.cmake deleted file mode 100644 index 8b6d5f6cddae6..0000000000000 --- a/cmake/modules/Findcppzmq.cmake +++ /dev/null @@ -1,21 +0,0 @@ -if (NOT (ZeroMQ_FOUND OR TARGET libzmq)) - message(FATAL_ERROR "Search for libzmq first!") -endif() - -find_path(cppzmq_INCLUDE_DIRS "zmq.hpp" - HINTS "${ZeroMQ_INCLUDE_DIR}" - PATH_SUFFIXES "include" "cppzmq" - ) -mark_as_advanced(cppzmq_INCLUDE_DIRS) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args (cppzmq DEFAULT_MSG cppzmq_INCLUDE_DIRS) - -if(cppzmq_FOUND) - add_library(cppzmq INTERFACE IMPORTED) - set_target_properties(cppzmq PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${CPPZMQ_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES libzmq - ) - set(CPPZMQ_LIBRARIES cppzmq) -endif() \ No newline at end of file diff --git a/cmake/modules/RootBuildOptions.cmake b/cmake/modules/RootBuildOptions.cmake index be3b19988076d..6278e8bf5cb5e 100644 --- a/cmake/modules/RootBuildOptions.cmake +++ b/cmake/modules/RootBuildOptions.cmake @@ -153,7 +153,6 @@ ROOT_BUILD_OPTION(pyroot ON "Enable support for automatic Python bindings (PyROO ROOT_BUILD_OPTION(pythia8 OFF "Enable support for Pythia 8.x [GPL]") ROOT_BUILD_OPTION(qt6web OFF "Enable support for Qt6 web-based display (requires Qt6::WebEngineCore and Qt6::WebEngineWidgets)") ROOT_BUILD_OPTION(roofit ON "Build the advanced fitting package RooFit, and RooStats for statistical tests. If xml is available, also build HistFactory.") -ROOT_BUILD_OPTION(roofit_multiprocess OFF "Build RooFit::MultiProcess and multi-process RooFit::TestStatistics classes (requires ZeroMQ >= 4.3.5 built with -DENABLE_DRAFTS and cppzmq).") ROOT_BUILD_OPTION(root7 ON "Build ROOT 7 experimental components of ROOT") ROOT_BUILD_OPTION(runtime_cxxmodules ON "Enable runtime support for C++ modules") ROOT_BUILD_OPTION(shadowpw OFF "Enable support for shadow passwords") @@ -304,11 +303,6 @@ if(builtin_openssl AND NOT APPLE) message(FATAL_ERROR ">>> Option 'builtin_openssl' is only supported on macOS.") endif() -# MultiProcess is not possible on Windows, so fail if it is manually set: -if(roofit_multiprocess AND WIN32) - message(FATAL_ERROR ">>> Option 'roofit_multiprocess' is not supported on Windows.") -endif() - #---Options depending of CMake Generator------------------------------------------------------- if( CMAKE_GENERATOR STREQUAL Ninja) set(fortran_defvalue OFF) @@ -392,6 +386,10 @@ foreach(opt afdsmgrd afs alien bonjour builtin_afterimage builtin_davix builtin_ endforeach() #---Deprecated options------------------------------------------------------------------------ +if(DEFINED roofit_multiprocess) + message(DEPRECATION ">>> Option 'roofit_multiprocess' has no effect anymore and will be removed in the next release of ROOT: RooFit::MultiProcess no longer needs ZeroMQ and is now always built on all platforms except Windows.") +endif() + foreach(opt mpi r tmva-pymva) if(${opt}) message(DEPRECATION ">>> Option '${opt}' is deprecated and will be removed in the next release of ROOT. Please contact root-dev@cern.ch should you still need it.") diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index 67aa8431fcaf4..d5e7c6c70fcc3 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -1067,32 +1067,6 @@ if (mpi) endif() endif() -#---Check for ZeroMQ when building RooFit::MultiProcess-------------------------------------------- - -if (roofit_multiprocess) - message(STATUS "Looking for ZeroMQ (libzmq)") - - # Temporarily prefer config mode over module mode, so that a CMake-installed system version - # gets detected before looking for an autotools-installed system version (which the - # FindZeroMQ.cmake module does). - set(CMAKE_FIND_PACKAGE_PREFER_CONFIG_ORIGINAL_VALUE ${CMAKE_FIND_PACKAGE_PREFER_CONFIG}) - set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE) - - # The fail-on-missing branching is not implemented, and we always look for - # ZeroMQ and cppzmq with REQUIRED to fail configuration if not available. - # That's because the roofit_multiprocess option can only be deliberately - # enabled by the user with roofit_multiprocess=ON, in which case it would - # be frustrating to get it auto-disabled on missing dependencies. - find_package(ZeroMQ 4.3.5 REQUIRED) - - # Reset default find_package mode - set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ${CMAKE_FIND_PACKAGE_PREFER_CONFIG_ORIGINAL_VALUE}) - unset(CMAKE_FIND_PACKAGE_PREFER_CONFIG_ORIGINAL_VALUE) - - message(STATUS "Looking for ZeroMQ C++ bindings (cppzmq)") - find_package(cppzmq REQUIRED) -endif (roofit_multiprocess) - #---Check for googletest--------------------------------------------------------------- if (testing OR testsupport) if (builtin_gtest) diff --git a/roofit/CMakeLists.txt b/roofit/CMakeLists.txt index 309e6cf0a0060..7357c4e531f93 100644 --- a/roofit/CMakeLists.txt +++ b/roofit/CMakeLists.txt @@ -8,8 +8,7 @@ set(roofit_legacy_eval_backend ON CACHE BOOL "" FORCE) add_subdirectory(batchcompute) add_subdirectory(codegen) -if (roofit_multiprocess) - add_subdirectory(roofitZMQ) +if(NOT WIN32) add_subdirectory(multiprocess) endif() add_subdirectory(roofitcore) diff --git a/roofit/multiprocess/CMakeLists.txt b/roofit/multiprocess/CMakeLists.txt index 880ebd2860617..b6b02a92bade6 100644 --- a/roofit/multiprocess/CMakeLists.txt +++ b/roofit/multiprocess/CMakeLists.txt @@ -5,6 +5,7 @@ ROOT_LINKER_LIBRARY(RooFitMultiProcess src/worker.cxx + src/Channel.cxx src/Messenger.cxx src/ProcessManager.cxx src/util.cxx @@ -18,11 +19,9 @@ ROOT_LINKER_LIBRARY(RooFitMultiProcess src/HeatmapAnalyzer.cxx LIBRARIES Core - DEPENDENCIES - RooFitZMQ ) -target_link_libraries(RooFitMultiProcess PUBLIC Hist RooFitZMQ) +target_link_libraries(RooFitMultiProcess PUBLIC Hist) set(RooFitMultiProcess_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/res") target_include_directories(RooFitMultiProcess PRIVATE ${RooFitMultiProcess_INCLUDE_DIR} @@ -41,10 +40,13 @@ if(NOT CMAKE_VERSION VERSION_LESS "3.23.0") inc/RooFit/MultiProcess/Config.h inc/RooFit/MultiProcess/HeatmapAnalyzer.h inc/RooFit/MultiProcess/types.h + res/RooFit/MultiProcess/Channel.h res/RooFit/MultiProcess/JobManager.h res/RooFit/MultiProcess/Job.h + res/RooFit/MultiProcess/Message.h res/RooFit/MultiProcess/Messenger.h res/RooFit/MultiProcess/Messenger_decl.h + res/RooFit/MultiProcess/Poller.h res/RooFit/MultiProcess/ProcessManager.h res/RooFit/MultiProcess/ProcessTimer.h res/RooFit/MultiProcess/Queue.h diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/Channel.h b/roofit/multiprocess/res/RooFit/MultiProcess/Channel.h new file mode 100644 index 0000000000000..b55fd2923c164 --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/Channel.h @@ -0,0 +1,163 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_Channel +#define ROOT_ROOFIT_MultiProcess_Channel + +#include "RooFit/MultiProcess/Message.h" + +#include +#include +#include +#include +#include +#include + +namespace RooFit { +namespace MultiProcess { + +/// Thrown when a blocking wait on a Channel is interrupted, e.g. by a signal. +/// The errno-style number is available through num(), mirroring the interface +/// of the zmq error types that were used here before, so the error handling +/// logic in util.cxx could stay the same. +class ppoll_error_t : public std::runtime_error { +public: + explicit ppoll_error_t(int errnum, const std::string &what) : std::runtime_error(what), errnum_(errnum) {} + int num() const { return errnum_; } + +private: + int errnum_; +}; + +/// \class Channel +/// \brief One endpoint of a full-duplex interprocess message pipe +/// +/// A Channel wraps one end of an AF_UNIX socketpair() created before forking +/// the child processes, and provides framed, whole-message send and receive +/// operations on top of the byte stream. Each frame is preceded by an 8-byte +/// header containing the payload size and a "more" bit that marks all but the +/// last frame of a multipart message. +/// +/// Sends never block: bytes that the kernel socket buffer does not accept +/// immediately are stored in a per-channel pending-output buffer, which is +/// flushed opportunistically whenever any Channel in the process waits for +/// input (see wait()). This mimics the previous ZeroMQ setup with an +/// unlimited high-water mark and avoids send-send deadlocks between +/// processes. +class Channel { +public: + Channel() = default; + /// Takes ownership of fd (one end of a socketpair) and makes it non-blocking. + explicit Channel(int fd); + ~Channel(); + + Channel(const Channel &) = delete; + Channel &operator=(const Channel &) = delete; + Channel(Channel &&other) noexcept; + Channel &operator=(Channel &&other) noexcept; + + bool valid() const { return fd_ >= 0; } + int fd() const { return fd_; } + + /// Queue one frame for sending and write out as much as the socket accepts. + void send_frame(const void *data, std::size_t size, bool more); + + /// Non-blocking receive attempt. Returns true and fills msg/more when a + /// complete frame was received; returns false if more bytes are needed. + bool try_recv_frame(Message &msg, bool *more); + + /// Blocking receive of one complete frame, interruptible by SIGTERM + /// (throws ppoll_error_t, like the poll functions). + Message recv_frame(bool *more = nullptr); + + bool has_pending_output() const { return out_pos_ < out_buf_.size(); } + /// Write out pending output; returns true when all of it has been written. + bool try_flush(); + + /// Wait until at least one of read_channels has input available, flushing + /// the pending output of all live Channels in this process meanwhile. + /// Returns the indices into read_channels that are readable. A negative + /// timeout means wait forever; otherwise the result may be empty after + /// timeout_ms milliseconds. Throws ppoll_error_t with num() == EINTR when + /// interrupted by a signal (including the SIGTERM self-pipe wake-up). + static std::vector wait(const std::vector &read_channels, int timeout_ms); + +private: + void close_fd(); + /// Handle end-of-stream / closed-connection conditions; never returns. + [[noreturn]] static void throw_connection_closed(); + + int fd_ = -1; + + // outgoing bytes not yet accepted by the kernel socket buffer + std::vector out_buf_; + std::size_t out_pos_ = 0; + + // incoming frame in progress + std::uint64_t in_header_ = 0; + std::size_t in_header_bytes_ = 0; + bool in_have_header_ = false; + Message in_msg_; + std::size_t in_msg_bytes_ = 0; +}; + +// Helper functions to send/receive single typed items over a Channel. These +// implement the same wire conventions as the old ZeroMQSvc encode/decode: +// trivially copyable types are sent as their raw bytes, strings as their +// character contents, and Message objects pass through as-is. + +template ::type>::value && + !std::is_pointer::type>::value, + bool>::type = true> +void send_item(Channel &channel, const T &item, bool more) +{ + channel.send_frame(&item, sizeof(T), more); +} + +inline void send_item(Channel &channel, const std::string &item, bool more) +{ + channel.send_frame(item.data(), item.size(), more); +} + +inline void send_item(Channel &channel, const char *item, bool more) +{ + channel.send_frame(item, std::strlen(item), more); +} + +inline void send_item(Channel &channel, const Message &item, bool more) +{ + channel.send_frame(item.data(), item.size(), more); +} + +template +value_t receive_item(Channel &channel, bool *more = nullptr) +{ + Message msg = channel.recv_frame(more); + if constexpr (std::is_same::value) { + return msg; + } else if constexpr (std::is_same::value) { + return std::string(msg.data(), msg.size()); + } else { + static_assert(std::is_trivially_copyable::value, + "only trivially copyable types, std::string and Message can be received"); + if (msg.size() != sizeof(value_t)) { + throw std::runtime_error("MultiProcess::receive_item: message size does not match receive type"); + } + value_t value; + std::memcpy(&value, msg.data(), sizeof(value_t)); + return value; + } +} + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Channel diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/Job.h b/roofit/multiprocess/res/RooFit/MultiProcess/Job.h index 33d4492ba93d1..6b2582344eddc 100644 --- a/roofit/multiprocess/res/RooFit/MultiProcess/Job.h +++ b/roofit/multiprocess/res/RooFit/MultiProcess/Job.h @@ -13,8 +13,9 @@ #ifndef ROOT_ROOFIT_MultiProcess_Job_decl #define ROOT_ROOFIT_MultiProcess_Job_decl +#include "RooFit/MultiProcess/Message.h" + #include -#include namespace RooFit { namespace MultiProcess { @@ -33,7 +34,7 @@ class Job { virtual void update_state(); virtual void send_back_task_result_from_worker(std::size_t task) = 0; - virtual bool receive_task_result_on_master(const zmq::message_t &message) = 0; + virtual bool receive_task_result_on_master(const Message &message) = 0; void gather_worker_results(); diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/Message.h b/roofit/multiprocess/res/RooFit/MultiProcess/Message.h new file mode 100644 index 0000000000000..d120c69570c32 --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/Message.h @@ -0,0 +1,80 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_Message +#define ROOT_ROOFIT_MultiProcess_Message + +#include +#include +#include +#include +#include + +namespace RooFit { +namespace MultiProcess { + +/// A contiguous byte buffer used as the unit of interprocess communication. +/// +/// This is the plain replacement for zmq::message_t: Job implementations +/// build a Message on the sending side (e.g. from a struct or an array of +/// doubles) and read it out via the typed data() accessors on the +/// receiving side. +class Message { +public: + Message() = default; + explicit Message(std::size_t size) : buf_(size) {} + + /// Create a message by copying the bytes of the elements in the range + /// [first, last), like the equivalent zmq::message_t constructor. + template + Message(ForwardIt first, ForwardIt last) + { + using value_t = typename std::iterator_traits::value_type; + buf_.resize(sizeof(value_t) * std::distance(first, last)); + char *out = buf_.data(); + for (ForwardIt it = first; it != last; ++it) { + value_t const &item = *it; + std::memcpy(out, &item, sizeof(value_t)); + out += sizeof(value_t); + } + } + + void *data() { return buf_.data(); } + const void *data() const { return buf_.data(); } + + template + T *data() + { + return reinterpret_cast(buf_.data()); + } + template + const T *data() const + { + return reinterpret_cast(buf_.data()); + } + + /// Size of the message in bytes. + std::size_t size() const { return buf_.size(); } + +private: + std::vector buf_; +}; + +// for debug printing in the Messenger +inline std::ostream &operator<<(std::ostream &out, const Message &msg) +{ + return out << "Message(" << msg.size() << " bytes)"; +} + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Message diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/Messenger.h b/roofit/multiprocess/res/RooFit/MultiProcess/Messenger.h index eeb3acdf61952..bc5a678c2cba4 100644 --- a/roofit/multiprocess/res/RooFit/MultiProcess/Messenger.h +++ b/roofit/multiprocess/res/RooFit/MultiProcess/Messenger.h @@ -15,6 +15,9 @@ #include "RooFit/MultiProcess/Messenger_decl.h" +#include +#include // getpid + #ifdef NDEBUG #undef NDEBUG #define turn_NDEBUG_back_on @@ -34,16 +37,14 @@ void Messenger::send_from_worker_to_queue(T item, Ts... items) debug_print(ss.str()); #endif - zmqSvc().send(*this_worker_qw_push_, item, send_flag_); - // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_item(this_worker_qw_, item, /*more=*/false); send_from_worker_to_queue(items...); } template value_t Messenger::receive_from_worker_on_queue(std::size_t this_worker_id) { - qw_pull_poller_[this_worker_id].ppoll(-1, &ppoll_sigmask); - auto value = zmqSvc().receive(*qw_pull_[this_worker_id], zmq::recv_flags::dontwait); + auto value = receive_item(qw_[this_worker_id]); #ifndef NDEBUG std::stringstream ss; @@ -63,16 +64,14 @@ void Messenger::send_from_queue_to_worker(std::size_t this_worker_id, T item, Ts debug_print(ss.str()); #endif - zmqSvc().send(*qw_push_[this_worker_id], item, send_flag_); - // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_item(qw_[this_worker_id], item, /*more=*/false); send_from_queue_to_worker(this_worker_id, items...); } template value_t Messenger::receive_from_queue_on_worker() { - qw_pull_poller_[0].ppoll(-1, &ppoll_sigmask); - auto value = zmqSvc().receive(*this_worker_qw_pull_, zmq::recv_flags::dontwait); + auto value = receive_item(this_worker_qw_); #ifndef NDEBUG std::stringstream ss; @@ -94,16 +93,14 @@ void Messenger::send_from_queue_to_master(T item, Ts... items) debug_print(ss.str()); #endif - zmqSvc().send(*mq_push_, item, send_flag_); - // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_item(mq_, item, /*more=*/false); send_from_queue_to_master(items...); } template value_t Messenger::receive_from_queue_on_master() { - mq_pull_poller_.ppoll(-1, &ppoll_sigmask); - auto value = zmqSvc().receive(*mq_pull_, zmq::recv_flags::dontwait); + auto value = receive_item(mq_); #ifndef NDEBUG std::stringstream ss; @@ -123,16 +120,14 @@ void Messenger::send_from_master_to_queue(T item, Ts... items) debug_print(ss.str()); #endif - zmqSvc().send(*mq_push_, item, send_flag_); - // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_item(mq_, item, /*more=*/false); send_from_master_to_queue(items...); } template value_t Messenger::receive_from_master_on_queue() { - mq_pull_poller_.ppoll(-1, &ppoll_sigmask); - auto value = zmqSvc().receive(*mq_pull_, zmq::recv_flags::dontwait); + auto value = receive_item(mq_); #ifndef NDEBUG std::stringstream ss; @@ -145,7 +140,7 @@ value_t Messenger::receive_from_master_on_queue() // -- MASTER - WORKER COMMUNICATION -- -/// specialization that sends the final message +/// specialization that sends the final part of a message template void Messenger::publish_from_master_to_workers(T &&item) { @@ -155,10 +150,12 @@ void Messenger::publish_from_master_to_workers(T &&item) debug_print(ss.str()); #endif - zmqSvc().send(*mw_pub_, std::forward(item), send_flag_); + for (auto &channel : mw_) { + send_item(channel, item, /*more=*/false); + } } -/// specialization that queues first parts of multipart messages +/// specialization that sends the first parts of multipart messages template void Messenger::publish_from_master_to_workers(T &&item, T2 &&item2, Ts &&...items) { @@ -168,15 +165,16 @@ void Messenger::publish_from_master_to_workers(T &&item, T2 &&item2, Ts &&...ite debug_print(ss.str()); #endif - zmqSvc().send(*mw_pub_, std::forward(item), send_flag_ | zmq::send_flags::sndmore); + for (auto &channel : mw_) { + send_item(channel, item, /*more=*/true); + } publish_from_master_to_workers(std::forward(item2), std::forward(items)...); } template value_t Messenger::receive_from_master_on_worker(bool *more) { - mw_sub_poller_.ppoll(-1, &ppoll_sigmask); - auto value = zmqSvc().receive(*mw_sub_, zmq::recv_flags::dontwait, more); + auto value = receive_item(this_worker_mw_, more); #ifndef NDEBUG std::stringstream ss; @@ -187,7 +185,7 @@ value_t Messenger::receive_from_master_on_worker(bool *more) return value; } -/// specialization that sends the final message +/// specialization that sends the final part of a message template void Messenger::send_from_worker_to_master(T &&item) { @@ -197,10 +195,10 @@ void Messenger::send_from_worker_to_master(T &&item) debug_print(ss.str()); #endif - zmqSvc().send(*wm_push_, std::forward(item), send_flag_); + send_item(this_worker_mw_, item, /*more=*/false); } -/// specialization that queues first parts of multipart messages +/// specialization that sends the first parts of multipart messages template void Messenger::send_from_worker_to_master(T &&item, T2 &&item2, Ts &&...items) { @@ -210,20 +208,24 @@ void Messenger::send_from_worker_to_master(T &&item, T2 &&item2, Ts &&...items) debug_print(ss.str()); #endif - zmqSvc().send(*wm_push_, std::forward(item), send_flag_ | zmq::send_flags::sndmore); - // if (sizeof...(items) > 0) { // this will only work with if constexpr, c++17 + send_item(this_worker_mw_, item, /*more=*/true); send_from_worker_to_master(std::forward(item2), std::forward(items)...); } template value_t Messenger::receive_from_worker_on_master(bool *more) { - wm_pull_poller_.ppoll(-1, &ppoll_sigmask); - auto value = zmqSvc().receive(*wm_pull_, zmq::recv_flags::dontwait, more); + Channel &channel = select_worker_channel_on_master(); + bool more_parts = false; + auto value = receive_item(channel, &more_parts); + update_worker_channel_on_master(channel, more_parts); + if (more) { + *more = more_parts; + } #ifndef NDEBUG std::stringstream ss; - ss << "PID " << getpid() << " receives M2W " << value; + ss << "PID " << getpid() << " receives W2M " << value; debug_print(ss.str()); #endif diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/Messenger_decl.h b/roofit/multiprocess/res/RooFit/MultiProcess/Messenger_decl.h index bd101181d3ec5..0cca5b30f8acc 100644 --- a/roofit/multiprocess/res/RooFit/MultiProcess/Messenger_decl.h +++ b/roofit/multiprocess/res/RooFit/MultiProcess/Messenger_decl.h @@ -14,25 +14,22 @@ #define ROOT_ROOFIT_MultiProcess_Messenger_decl #include "RooFit/MultiProcess/ProcessManager.h" -#include "RooFit_ZMQ/ZeroMQSvc.h" -#include "RooFit_ZMQ/ZeroMQPoller.h" +#include "RooFit/MultiProcess/Channel.h" +#include "RooFit/MultiProcess/Poller.h" #include -#include -#include // sigprocmask, sigset_t, etc #include +#include namespace RooFit { namespace MultiProcess { -void set_socket_immediate(ZmqLingeringSocketPtr<> &socket); - // test messages enum class X2X : int { ping = -1, pong = -2, initial_value = 0 }; class Messenger { public: - explicit Messenger(const ProcessManager &process_manager); + explicit Messenger(ProcessManager &process_manager); ~Messenger(); void test_connections(const ProcessManager &process_manager); @@ -51,8 +48,8 @@ class Messenger { fromQonW, }; - std::pair create_queue_poller(); - std::pair create_worker_poller(); + std::pair create_queue_poller(); + std::pair create_worker_poller(); // -- WORKER - QUEUE COMMUNICATION -- @@ -101,52 +98,31 @@ class Messenger { void test_receive(X2X expected_ping_value, test_rcv_pipes rcv_pipe, std::size_t worker_id); void test_send(X2X ping_value, test_snd_pipes snd_pipe, std::size_t worker_id); - sigset_t ppoll_sigmask; - - void set_send_flag(zmq::send_flags flag); - private: void debug_print(std::string s); - template - void bindAddr(T &socket, std::string &&addr) - { - bound_ipc_addresses_.emplace_back(addr); - socket->bind(bound_ipc_addresses_.back()); - } - - // push - std::vector> qw_push_; - ZmqLingeringSocketPtr<> this_worker_qw_push_; - ZmqLingeringSocketPtr<> mq_push_; - // pollers for all push sockets - std::vector qw_push_poller_; - ZeroMQPoller mq_push_poller_; - // pull - std::vector> qw_pull_; - ZmqLingeringSocketPtr<> this_worker_qw_pull_; - ZmqLingeringSocketPtr<> mq_pull_; - // pollers for all pull sockets - std::vector qw_pull_poller_; - ZeroMQPoller mq_pull_poller_; - - // publish/subscribe sockets for parameter updating from master to workers - ZmqLingeringSocketPtr<> mw_pub_; - ZmqLingeringSocketPtr<> mw_sub_; - ZeroMQPoller mw_sub_poller_; - // push/pull sockets for result retrieving from workers on master - ZmqLingeringSocketPtr<> wm_push_; - ZmqLingeringSocketPtr<> wm_pull_; - ZeroMQPoller wm_pull_poller_; - - // destruction flags to distinguish between different process-type setups: - bool close_MQ_on_destruct_ = false; - bool close_this_QW_on_destruct_ = false; - bool close_QW_container_on_destruct_ = false; - - zmq::send_flags send_flag_ = zmq::send_flags::none; - - std::vector bound_ipc_addresses_; + /// On master: pick the worker channel to receive the next message from. + /// Continues an in-progress multipart message from the same worker; + /// otherwise waits for any worker and picks one round-robin. + Channel &select_worker_channel_on_master(); + void update_worker_channel_on_master(Channel &channel, bool more); + + // master-queue channel (on master and queue processes) + Channel mq_; + // queue-worker channels (all workers on the queue process, only the own + // one on worker processes) + std::vector qw_; + Channel this_worker_qw_; + // master-worker channels, carrying both the state updates that were + // previously published over PUB-SUB and the task results (all workers on + // the master process, only the own one on worker processes) + std::vector mw_; + Channel this_worker_mw_; + + // on master: bookkeeping for receiving from any worker + Poller mw_poller_; + Channel *mw_current_source_ = nullptr; + std::size_t mw_next_poll_position_ = 0; }; // Messages from master to queue diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/Poller.h b/roofit/multiprocess/res/RooFit/MultiProcess/Poller.h new file mode 100644 index 0000000000000..6592a3d5345a7 --- /dev/null +++ b/roofit/multiprocess/res/RooFit/MultiProcess/Poller.h @@ -0,0 +1,83 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ +#ifndef ROOT_ROOFIT_MultiProcess_Poller +#define ROOT_ROOFIT_MultiProcess_Poller + +#include "RooFit/MultiProcess/Channel.h" + +#include +#include +#include + +namespace RooFit { +namespace MultiProcess { + +/// \class Poller +/// \brief Waits for input on a set of registered Channels +/// +/// Replacement for the ZeroMQPoller: channels get a stable index in +/// registration order, poll() returns the indices of the channels that have +/// input available, and channels can be unregistered without changing the +/// indices of the others. +/// +/// The Poller stores plain pointers, so registered Channel objects must stay +/// at their memory location while the Poller is in use. +class Poller { +public: + /// Register a channel for input polling; returns its stable index. + std::size_t register_channel(const Channel &channel) + { + entries_.emplace_back(next_index_++, &channel); + return entries_.back().first; + } + + void unregister_channel(const Channel &channel) + { + for (auto it = entries_.begin(); it != entries_.end(); ++it) { + if (it->second == &channel) { + entries_.erase(it); + return; + } + } + throw std::runtime_error("Poller::unregister_channel: channel not registered"); + } + + std::size_t size() const { return entries_.size(); } + + /// Wait for input; returns the registration indices of readable channels. + /// Throws ppoll_error_t with num() == EINTR when a SIGTERM was received. + std::vector poll(int timeout_ms = -1) const + { + if (entries_.empty() && timeout_ms < 0) { + throw std::logic_error("Poller::poll: waiting without timeout on a poller with no registered channels"); + } + std::vector channels; + channels.reserve(entries_.size()); + for (auto &entry : entries_) { + channels.push_back(entry.second); + } + std::vector result; + for (std::size_t pos : Channel::wait(channels, timeout_ms)) { + result.push_back(entries_[pos].first); + } + return result; + } + +private: + std::vector> entries_; + std::size_t next_index_ = 0; +}; + +} // namespace MultiProcess +} // namespace RooFit + +#endif // ROOT_ROOFIT_MultiProcess_Poller diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/ProcessManager.h b/roofit/multiprocess/res/RooFit/MultiProcess/ProcessManager.h index 39927d48bafcd..90e5cba2f5fa0 100644 --- a/roofit/multiprocess/res/RooFit/MultiProcess/ProcessManager.h +++ b/roofit/multiprocess/res/RooFit/MultiProcess/ProcessManager.h @@ -14,7 +14,8 @@ #define ROOT_ROOFIT_MultiProcess_ProcessManager #include // pid_t -#include // sig_atomic_t and for sigterm handling on child processes (in ProcessManager.cxx) +#include +#include // sig_atomic_t and for sigterm handling on child processes (in ProcessManager.cxx) #include // forward declaration @@ -45,6 +46,17 @@ class ProcessManager { static void handle_sigterm(int signum); static bool sigterm_received(); + /// Read end of the self-pipe that the SIGTERM handler writes to (or -1 on + /// the master process, which installs no handler); used by Channel::wait. + static int sigterm_wake_fd(); + + // Interprocess channel file descriptors, created with socketpair() before + // forking. The Messenger claims the ends belonging to the current process + // and takes over their ownership; unclaimed descriptors are closed when + // this ProcessManager is destroyed. + int claim_mq_fd(); + int claim_qw_fd(std::size_t worker_ix); + int claim_mw_fd(std::size_t worker_ix); // for debugging/testing: pid_t get_queue_pid() const { return queue_pid_; } @@ -53,6 +65,9 @@ class ProcessManager { private: void initialize_processes(bool cpu_pinning = true); void shutdown_processes(); + void create_channel_fds(); + void close_unused_channel_fds(); + void close_channel_fds(); bool is_master_ = false; bool is_queue_ = false; @@ -66,7 +81,16 @@ class ProcessManager { bool initialized_ = false; + // socketpair ends for the interprocess channels; in each array, index 0 is + // the end used by the process listed first in the member name (m: master, + // q: queue, w: worker), index 1 the other end + std::array mq_fds_{{-1, -1}}; + std::vector> qw_fds_; + std::vector> mw_fds_; + static volatile sig_atomic_t sigterm_received_; + static int sigterm_wake_read_fd_; + static int sigterm_wake_write_fd_; }; } // namespace MultiProcess diff --git a/roofit/multiprocess/res/RooFit/MultiProcess/util.h b/roofit/multiprocess/res/RooFit/MultiProcess/util.h index 2893d39eea37f..98c1396c341bb 100644 --- a/roofit/multiprocess/res/RooFit/MultiProcess/util.h +++ b/roofit/multiprocess/res/RooFit/MultiProcess/util.h @@ -14,20 +14,19 @@ #ifndef ROOT_ROOFIT_MultiProcess_util #define ROOT_ROOFIT_MultiProcess_util -#include "RooFit_ZMQ/ppoll.h" // for ZMQ::ppoll_error_t -#include "RooFit_ZMQ/ZeroMQPoller.h" +#include "RooFit/MultiProcess/Channel.h" // ppoll_error_t +#include "RooFit/MultiProcess/Poller.h" +#include #include // getpid, pid_t +#include namespace RooFit { namespace MultiProcess { int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing); -enum class zmq_ppoll_error_response { abort, unknown_eintr, retry }; -zmq_ppoll_error_response handle_zmq_ppoll_error(ZMQ::ppoll_error_t &e); -std::tuple>, bool> -careful_ppoll(ZeroMQPoller &poller, const sigset_t &ppoll_sigmask, std::size_t max_tries = 2); +std::tuple, bool> careful_poll(Poller &poller); } // namespace MultiProcess } // namespace RooFit diff --git a/roofit/multiprocess/src/Channel.cxx b/roofit/multiprocess/src/Channel.cxx new file mode 100644 index 0000000000000..180a495ec6924 --- /dev/null +++ b/roofit/multiprocess/src/Channel.cxx @@ -0,0 +1,336 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/Channel.h" +#include "RooFit/MultiProcess/ProcessManager.h" + +#include +#include +#include + +#include +#include +#include +#include + +namespace RooFit { +namespace MultiProcess { + +namespace { + +constexpr std::uint64_t moreBit = std::uint64_t(1) << 63; +constexpr std::uint64_t sizeMask = moreBit - 1; + +/// Per-process registry of all live channels, so that any blocking wait can +/// flush the pending output of every channel (also the ones not being read +/// from) and no send can be starved. The processes are single-threaded, so a +/// plain static is fine here. The vector is intentionally leaked: Channels +/// held by the static JobManager instance are destroyed during static +/// destruction, which can happen after a function-local static vector would +/// have been destroyed. +std::vector &liveChannels() +{ + static auto *channels = new std::vector; + return *channels; +} + +void registerChannel(Channel *channel) +{ + liveChannels().push_back(channel); +} + +void unregisterChannel(Channel *channel) +{ + auto &channels = liveChannels(); + channels.erase(std::remove(channels.begin(), channels.end(), channel), channels.end()); +} + +ssize_t send_some(int fd, const void *buf, std::size_t n) +{ +#ifdef MSG_NOSIGNAL + return ::send(fd, buf, n, MSG_NOSIGNAL); +#else + return ::send(fd, buf, n, 0); +#endif +} + +} // namespace + +Channel::Channel(int fd) : fd_(fd) +{ + int flags = fcntl(fd_, F_GETFL, 0); + if (flags == -1 || fcntl(fd_, F_SETFL, flags | O_NONBLOCK) == -1) { + throw std::runtime_error(std::string("MultiProcess::Channel: could not set O_NONBLOCK: ") + strerror(errno)); + } +#ifdef SO_NOSIGPIPE + // on platforms without MSG_NOSIGNAL (macOS), prevent SIGPIPE on writes to a closed peer + int optval = 1; + setsockopt(fd_, SOL_SOCKET, SO_NOSIGPIPE, &optval, sizeof(optval)); +#endif + registerChannel(this); +} + +Channel::~Channel() +{ + if (valid()) { + unregisterChannel(this); + } + close_fd(); +} + +Channel::Channel(Channel &&other) noexcept + : fd_(other.fd_), + out_buf_(std::move(other.out_buf_)), + out_pos_(other.out_pos_), + in_header_(other.in_header_), + in_header_bytes_(other.in_header_bytes_), + in_have_header_(other.in_have_header_), + in_msg_(std::move(other.in_msg_)), + in_msg_bytes_(other.in_msg_bytes_) +{ + other.fd_ = -1; + if (valid()) { + unregisterChannel(&other); + registerChannel(this); + } +} + +Channel &Channel::operator=(Channel &&other) noexcept +{ + if (this != &other) { + if (valid()) { + unregisterChannel(this); + } + close_fd(); + fd_ = other.fd_; + out_buf_ = std::move(other.out_buf_); + out_pos_ = other.out_pos_; + in_header_ = other.in_header_; + in_header_bytes_ = other.in_header_bytes_; + in_have_header_ = other.in_have_header_; + in_msg_ = std::move(other.in_msg_); + in_msg_bytes_ = other.in_msg_bytes_; + other.fd_ = -1; + if (valid()) { + unregisterChannel(&other); + registerChannel(this); + } + } + return *this; +} + +void Channel::close_fd() +{ + if (fd_ >= 0) { + ::close(fd_); + fd_ = -1; + } +} + +void Channel::throw_connection_closed() +{ + // A closed connection during shutdown just means the other process was + // terminated a moment before this one noticed; in that case exit the event + // loops through the regular SIGTERM path. The SIGTERM may still be in + // flight, so give it a moment to arrive. + if (!ProcessManager::sigterm_received()) { + int wake_fd = ProcessManager::sigterm_wake_fd(); + if (wake_fd >= 0) { + pollfd pfd{wake_fd, POLLIN, 0}; + ::poll(&pfd, 1, 500); + } + } + if (ProcessManager::sigterm_received()) { + throw ppoll_error_t(EINTR, "MultiProcess::Channel: connection closed while terminating"); + } + throw std::runtime_error("MultiProcess::Channel: connection closed by peer process (did it die unexpectedly?)"); +} + +void Channel::send_frame(const void *data, std::size_t size, bool more) +{ + std::uint64_t header = (std::uint64_t(size) & sizeMask) | (more ? moreBit : 0); + // Append to the pending-output buffer and then write out as much as the + // socket accepts. Appending first keeps this simple and correct also when + // there already is pending output; the extra copy is negligible for the + // message sizes used here. For multipart messages, the flush is deferred + // to the final frame, so a k-frame message costs one send() system call + // instead of k. Deferring is safe: any blocking wait() in this process + // also flushes the pending output of all channels. + const char *headerBytes = reinterpret_cast(&header); + out_buf_.insert(out_buf_.end(), headerBytes, headerBytes + sizeof(header)); + const char *dataBytes = static_cast(data); + out_buf_.insert(out_buf_.end(), dataBytes, dataBytes + size); + if (!more) { + try_flush(); + } +} + +bool Channel::try_flush() +{ + while (out_pos_ < out_buf_.size()) { + ssize_t n = send_some(fd_, out_buf_.data() + out_pos_, out_buf_.size() - out_pos_); + if (n >= 0) { + out_pos_ += n; + } else if (errno == EINTR) { + continue; + } else if (errno == EAGAIN || errno == EWOULDBLOCK) { + return false; + } else if (errno == EPIPE || errno == ECONNRESET) { + throw_connection_closed(); + } else { + throw std::runtime_error(std::string("MultiProcess::Channel: send failed: ") + strerror(errno)); + } + } + out_buf_.clear(); + out_pos_ = 0; + return true; +} + +bool Channel::try_recv_frame(Message &msg, bool *more) +{ + if (!in_have_header_) { + char *headerBytes = reinterpret_cast(&in_header_); + while (in_header_bytes_ < sizeof(in_header_)) { + ssize_t n = ::read(fd_, headerBytes + in_header_bytes_, sizeof(in_header_) - in_header_bytes_); + if (n > 0) { + in_header_bytes_ += n; + } else if (n == 0) { + throw_connection_closed(); + } else if (errno == EINTR) { + continue; + } else if (errno == EAGAIN || errno == EWOULDBLOCK) { + return false; + } else if (errno == ECONNRESET) { + throw_connection_closed(); + } else { + throw std::runtime_error(std::string("MultiProcess::Channel: receive failed: ") + strerror(errno)); + } + } + in_have_header_ = true; + in_msg_ = Message(in_header_ & sizeMask); + in_msg_bytes_ = 0; + } + + // Read exactly the payload of the current frame, so that any following + // frames stay in the kernel buffer and poll() remains accurate. + char *payload = in_msg_.data(); + while (in_msg_bytes_ < in_msg_.size()) { + ssize_t n = ::read(fd_, payload + in_msg_bytes_, in_msg_.size() - in_msg_bytes_); + if (n > 0) { + in_msg_bytes_ += n; + } else if (n == 0) { + throw_connection_closed(); + } else if (errno == EINTR) { + continue; + } else if (errno == EAGAIN || errno == EWOULDBLOCK) { + return false; + } else if (errno == ECONNRESET) { + throw_connection_closed(); + } else { + throw std::runtime_error(std::string("MultiProcess::Channel: receive failed: ") + strerror(errno)); + } + } + + msg = std::move(in_msg_); + if (more) { + *more = (in_header_ & moreBit) != 0; + } + in_have_header_ = false; + in_header_ = 0; + in_header_bytes_ = 0; + in_msg_ = Message{}; + in_msg_bytes_ = 0; + return true; +} + +Message Channel::recv_frame(bool *more) +{ + Message msg; + while (!try_recv_frame(msg, more)) { + wait({this}, -1); + } + return msg; +} + +std::vector Channel::wait(const std::vector &read_channels, int timeout_ms) +{ + while (true) { + std::vector pollfds; + pollfds.reserve(read_channels.size() + liveChannels().size() + 1); + + int wake_fd = ProcessManager::sigterm_wake_fd(); + if (wake_fd >= 0) { + pollfds.push_back({wake_fd, POLLIN, 0}); + } + const std::size_t first_read_item = pollfds.size(); + for (const Channel *channel : read_channels) { + pollfds.push_back({channel->fd(), POLLIN, 0}); + } + // also watch all channels that still have pending output, so their + // sends make progress while we wait and no two processes can deadlock + // each other with full socket buffers + std::vector flush_channels; + for (Channel *channel : liveChannels()) { + if (channel->has_pending_output()) { + flush_channels.push_back(channel); + pollfds.push_back({channel->fd(), POLLOUT, 0}); + } + } + + int rc = ::poll(pollfds.data(), pollfds.size(), timeout_ms); + if (rc < 0) { + if (errno == EINTR) { + // Retry on benign signal interruptions (profilers, debuggers, + // SIGCHLD, ...). This is essential for protocol integrity: a + // multi-frame message is received with one blocking receive per + // frame, and surfacing a benign EINTR mid-sequence to the event + // loops would make them restart the loop and desynchronize the + // wire protocol. Only termination requests leave this function + // exceptionally. There is no lost-wakeup race with SIGTERM: the + // handler also writes to the self-pipe, which the next poll + // reports as readable. + if (ProcessManager::sigterm_received()) { + throw ppoll_error_t(EINTR, "poll interrupted by SIGTERM"); + } + continue; + } + throw std::runtime_error(std::string("MultiProcess::Channel::wait: poll failed: ") + strerror(errno)); + } + + // a byte on the self-pipe means a SIGTERM arrived (possibly before we + // entered poll, which is exactly the race the self-pipe closes) + if (wake_fd >= 0 && (pollfds[0].revents & POLLIN)) { + throw ppoll_error_t(EINTR, "poll interrupted by SIGTERM"); + } + + for (std::size_t fx = 0; fx < flush_channels.size(); ++fx) { + std::size_t item = first_read_item + read_channels.size() + fx; + if (pollfds[item].revents & (POLLOUT | POLLERR | POLLHUP)) { + flush_channels[fx]->try_flush(); + } + } + + std::vector readable; + for (std::size_t ix = 0; ix < read_channels.size(); ++ix) { + if (pollfds[first_read_item + ix].revents & (POLLIN | POLLHUP | POLLERR)) { + readable.push_back(ix); + } + } + if (!readable.empty() || timeout_ms >= 0) { + return readable; + } + // infinite timeout, but we only woke up to flush output: wait again + } +} + +} // namespace MultiProcess +} // namespace RooFit diff --git a/roofit/multiprocess/src/Job.cxx b/roofit/multiprocess/src/Job.cxx index b5024d044355a..ffa0957662e41 100644 --- a/roofit/multiprocess/src/Job.cxx +++ b/roofit/multiprocess/src/Job.cxx @@ -29,13 +29,13 @@ namespace MultiProcess { * Classes inheriting from Job must implement the pure virtual methods: * - void evaluate_task(std::size_t task) * - void send_back_task_result_from_worker(std::size_t task) - * - void receive_task_result_on_master(const zmq::message_t & message) + * - void receive_task_result_on_master(const Message & message) * * An example/reference implementation can be found in test_Job.cxx. * * Most Jobs will also want to override the virtual update_state() function. * This function can be used to send and receive state from master to worker. - * In the worker loop, when something is received over the ZeroMQ "SUB" socket, + * In the worker loop, when a state update is received from the master process, * update_state() is called to put the received data into the right places, * thus updating for instance parameter values on the worker that were updated * since the last call on the master side. @@ -130,10 +130,10 @@ void Job::gather_worker_results() /// \brief Virtual function to update any necessary state on workers /// -/// This function is called from the worker loop when something is received -/// over the ZeroMQ "SUB" socket. The master process sends messages to workers -/// on its "PUB" socket. Thus, we can update, for instance, parameter values -/// on the worker that were updated since the last call on the master side. +/// This function is called from the worker loop when a state update message +/// from the master process is received. Thus, we can update, for instance, +/// parameter values on the worker that were updated since the last call on +/// the master side. /// \note Implementers: make sure to also update the state_id_ member. void Job::update_state() {} diff --git a/roofit/multiprocess/src/JobManager.cxx b/roofit/multiprocess/src/JobManager.cxx index c84a40d41aebf..c031af8e62764 100644 --- a/roofit/multiprocess/src/JobManager.cxx +++ b/roofit/multiprocess/src/JobManager.cxx @@ -51,8 +51,6 @@ JobManager *JobManager::instance() if (!JobManager::is_instantiated()) { instance_.reset(new JobManager(Config::getDefaultNWorkers())); // can't use make_unique, because ctor is private instance_->messenger().test_connections(instance_->process_manager()); - // set send to non blocking on all processes after checking the connections are working: - instance_->messenger().set_send_flag(zmq::send_flags::dontwait); } return instance_.get(); } @@ -100,10 +98,14 @@ JobManager::~JobManager() // There used to be an assert statement that checked whether the job_objects // map was empty at destruction time, but that neglected the second possibility // and led to assertion failures, which left the Messenger and ProcessManager - // objects intact, leading to the forked processes and their ZeroMQ resources - // to remain after exiting the main/master/parent process. - messenger_ptr_.reset(); + // objects intact, leading to the forked processes and their communication + // resources to remain after exiting the main/master/parent process. + // Note the destruction order: the ProcessManager first terminates the child + // processes (SIGTERM) while all communication channels are still open, so + // that no process sees a closed connection during a normal shutdown; only + // then the Messenger closes the channels. process_manager_ptr_.reset(); + messenger_ptr_.reset(); queue_ptr_.reset(); } @@ -129,7 +131,12 @@ std::size_t JobManager::add_job_object(Job *job_object) // static function Job *JobManager::get_job_object(std::size_t job_object_id) { - return job_objects_[job_object_id]; + auto found = job_objects_.find(job_object_id); + if (found == job_objects_.end()) { + throw std::runtime_error("JobManager::get_job_object: unknown job ID " + std::to_string(job_object_id) + + ", the interprocess message stream may be corrupted"); + } + return found->second; } // static function @@ -167,35 +174,20 @@ void JobManager::retrieve(std::size_t requesting_job_id) bool job_fully_retrieved = false; while (not job_fully_retrieved) { try { - auto task_result_message = messenger().receive_from_worker_on_master(); - auto job_object_id = *reinterpret_cast( - task_result_message.data()); // job_id must always be the first element of the result message! + auto task_result_message = messenger().receive_from_worker_on_master(); + if (task_result_message.size() < sizeof(std::size_t)) { + throw std::runtime_error("JobManager::retrieve: received a task result message that is too short to " + "contain a job ID, the interprocess message stream may be corrupted"); + } + auto job_object_id = *task_result_message.data(); // job_id must always be the first element of + // the result message! bool this_job_fully_retrieved = JobManager::get_job_object(job_object_id)->receive_task_result_on_master(task_result_message); if (requesting_job_id == job_object_id) { job_fully_retrieved = this_job_fully_retrieved; } - } catch (ZMQ::ppoll_error_t &e) { - zmq_ppoll_error_response response; - try { - response = handle_zmq_ppoll_error(e); - } catch (std::logic_error &) { - printf("JobManager::retrieve got unhandleable ZMQ::ppoll_error_t\n"); - throw; - } - if (response == zmq_ppoll_error_response::abort) { - throw std::logic_error("in JobManager::retrieve: master received a SIGTERM, aborting"); - } else if (response == zmq_ppoll_error_response::unknown_eintr) { - printf("EINTR in JobManager::retrieve, continuing\n"); - continue; - } else if (response == zmq_ppoll_error_response::retry) { - printf("EAGAIN from ppoll in JobManager::retrieve, continuing\n"); - continue; - } - } catch (zmq::error_t &e) { - printf("unhandled zmq::error_t (not a ppoll_error_t) in JobManager::retrieve with errno %d: %s\n", e.num(), - e.what()); - throw; + } catch (ppoll_error_t &) { + throw std::logic_error("in JobManager::retrieve: master received a SIGTERM, aborting"); } } } @@ -215,13 +207,29 @@ void JobManager::activate() { activated_ = true; + // Note on error handling: the queue and worker processes are forked from + // the master, so the stack below this function belongs to the master-side + // caller. An exception escaping the event loops (e.g. from a closed + // connection when another process died unexpectedly) must therefore never + // propagate out of this function on a child process: it would unwind into + // code that was never meant to run on this process. Report it and exit. if (process_manager().is_queue()) { - queue()->loop(); + try { + queue()->loop(); + } catch (const std::exception &e) { + fprintf(stderr, "queue process (PID %d) exits after exception: %s\n", getpid(), e.what()); + std::_Exit(1); + } std::_Exit(0); } if (!is_worker_loop_running() && process_manager().is_worker()) { - RooFit::MultiProcess::worker_loop(); + try { + RooFit::MultiProcess::worker_loop(); + } catch (const std::exception &e) { + fprintf(stderr, "worker process (PID %d) exits after exception: %s\n", getpid(), e.what()); + std::_Exit(1); + } std::_Exit(0); } } diff --git a/roofit/multiprocess/src/Messenger.cxx b/roofit/multiprocess/src/Messenger.cxx index 06dd2665efd58..d33bdc1a4cd3b 100644 --- a/roofit/multiprocess/src/Messenger.cxx +++ b/roofit/multiprocess/src/Messenger.cxx @@ -14,356 +14,129 @@ #include "RooFit/MultiProcess/Messenger.h" #include "RooFit/MultiProcess/util.h" -#include - -#include // sigprocmask etc +#include +#include namespace RooFit { namespace MultiProcess { -void set_socket_immediate(ZmqLingeringSocketPtr<> &socket) -{ - int optval = 1; - socket->set(zmq::sockopt::immediate, optval); -} - /** \class Messenger * - * \brief Manages ZeroMQ sockets and wraps send and receive calls + * \brief Manages the interprocess communication channels and wraps send and receive calls * * This class is used for all interprocess communication between the master, - * queue and worker processes. It sets up ZeroMQ sockets between all processes - * over IPC socket files stored in /tmp on the filesystem. + * queue and worker processes. The communication runs over pipes built on + * socketpair(), which are created in the ProcessManager before forking, so + * that all processes inherit their ends of the connected channels; see + * Channel for the wire format. * - * Several sockets are used for communication between different places for - * different purposes: - * - Master and queue processes each have a PUSH-PULL socket pair to directly - * send/receive data between only the master and queue processes. This is - * currently used mainly for sending tasks to the queue from master. The - * socket from queue back to master is used only to test connections and may - * be removed in the future. - * - The queue process also has a PUSH-PULL socket pair with each worker - * process. These are used by the workers to obtain tasks from the queue. - * - The master has a PUB socket that the workers subscribe to with SUB - * sockets. These are used to update state. Note that to ensure robust - * reception of all messages on the SUB socket, it's important to send over - * state in as little messages as possible. For instance, it's best to send - * arrays over in a single big message instead of sending over each element - * separately. This also improves performance, since each message has some - * fixed overhead. - * - Each worker has a PUSH socket connected to a PULL socket on master that - * is used to send back task results from workers to master in - * 'JobManager::retrieve()'. + * Several channels connect the processes for different purposes: + * - The master and queue processes share a channel that is mainly used for + * sending tasks to the queue from master. + * - The queue process shares a channel with each worker process. These are + * used by the workers to obtain tasks from the queue. + * - The master shares a channel with each worker process. The master -> worker + * direction carries state updates (previously published over a ZeroMQ + * PUB-SUB socket) and the worker -> master direction carries back task + * results, which the master receives in 'JobManager::retrieve()'. * * @param process_manager ProcessManager instance which manages the master, * queue and worker processes that we want to set up * communication for in this Messenger. */ -Messenger::Messenger(const ProcessManager &process_manager) +Messenger::Messenger(ProcessManager &process_manager) { - sigemptyset(&ppoll_sigmask); - - auto makeAddrPrefix = [](pid_t pid) -> std::string { - std::string tmpPath = gSystem->TempDirectory(); - return "ipc://" + tmpPath + "/roofit_" + std::to_string(pid) + "_roofitMP"; - }; - - // high water mark for master-queue sending, which can be quite a busy channel, especially at the start of a run - int hwm = 0; - // create zmq connections and pollers where necessary - // Note: zmq context is automatically created in the ZeroMQSvc class and maintained as singleton. - // It is reset in the ProcessManager, if necessary. Do not do that here, see comments in ProcessManager - // constructor. - try { - if (process_manager.is_master()) { - auto addrBase = makeAddrPrefix(getpid()); - - mq_push_.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); - mq_push_->set(zmq::sockopt::sndhwm, hwm); - bindAddr(mq_push_, addrBase + "_from_master_to_queue"); - - mq_push_poller_.register_socket(*mq_push_, zmq::event_flags::pollout); - - mq_pull_.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); - mq_pull_->set(zmq::sockopt::rcvhwm, hwm); - bindAddr(mq_pull_, addrBase + "_from_queue_to_master"); - - mq_pull_poller_.register_socket(*mq_pull_, zmq::event_flags::pollin); - - mw_pub_.reset(zmqSvc().socket_ptr(zmq::socket_type::pub)); - mw_pub_->set(zmq::sockopt::sndhwm, hwm); - bindAddr(mw_pub_, addrBase + "_from_master_to_workers"); - - wm_pull_.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); - wm_pull_->set(zmq::sockopt::rcvhwm, hwm); - bindAddr(wm_pull_, addrBase + "_from_workers_to_master"); - wm_pull_poller_.register_socket(*wm_pull_, zmq::event_flags::pollin); - - close_MQ_on_destruct_ = true; - - // make sure all subscribers are connected - ZmqLingeringSocketPtr<> subscriber_ping_socket{zmqSvc().socket_ptr(zmq::socket_type::pull)}; - bindAddr(subscriber_ping_socket, addrBase + "_subscriber_ping_socket"); - ZeroMQPoller subscriber_ping_poller; - subscriber_ping_poller.register_socket(*subscriber_ping_socket, zmq::event_flags::pollin); - std::size_t N_subscribers_confirmed = 0; - while (N_subscribers_confirmed < process_manager.N_workers()) { - zmqSvc().send(*mw_pub_, false); - auto poll_results = subscriber_ping_poller.poll(0); - for (std::size_t ix = 0; ix < poll_results.size(); ++ix) { - auto request = zmqSvc().receive(*subscriber_ping_socket, zmq::recv_flags::dontwait); - assert(request == "present"); - ++N_subscribers_confirmed; - } - } - zmqSvc().send(*mw_pub_, true); - - } else if (process_manager.is_queue()) { - auto addrBase = makeAddrPrefix(getppid()); - - // first the queue-worker sockets - // do resize instead of reserve so that the unique_ptrs are initialized - // (to nullptr) so that we can do reset below, alternatively you can do - // push/emplace_back with move or something - qw_push_.resize(process_manager.N_workers()); - qw_pull_.resize(process_manager.N_workers()); - qw_push_poller_.resize(process_manager.N_workers()); - qw_pull_poller_.resize(process_manager.N_workers()); - for (std::size_t ix = 0; ix < process_manager.N_workers(); ++ix) { - // push - qw_push_[ix].reset(zmqSvc().socket_ptr(zmq::socket_type::push)); - bindAddr(qw_push_[ix], addrBase + "_from_queue_to_worker_" + std::to_string(ix)); - - qw_push_poller_[ix].register_socket(*qw_push_[ix], zmq::event_flags::pollout); - - // pull - qw_pull_[ix].reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); - bindAddr(qw_pull_[ix], addrBase + "_from_worker_" + std::to_string(ix) + "_to_queue"); - - qw_pull_poller_[ix].register_socket(*qw_pull_[ix], zmq::event_flags::pollin); - } - - // then the master-queue sockets - mq_push_.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); - mq_push_->set(zmq::sockopt::sndhwm, hwm); - mq_push_->connect(addrBase + "_from_queue_to_master"); - - mq_push_poller_.register_socket(*mq_push_, zmq::event_flags::pollout); - - mq_pull_.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); - mq_pull_->set(zmq::sockopt::rcvhwm, hwm); - mq_pull_->connect(addrBase + "_from_master_to_queue"); - - mq_pull_poller_.register_socket(*mq_pull_, zmq::event_flags::pollin); - - close_MQ_on_destruct_ = true; - close_QW_container_on_destruct_ = true; - } else if (process_manager.is_worker()) { - auto addrBase = makeAddrPrefix(getppid()); - - // we only need one queue-worker pipe on the worker - qw_push_poller_.resize(1); - qw_pull_poller_.resize(1); - - // push - this_worker_qw_push_.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); - auto addr = addrBase + "_from_worker_" + std::to_string(process_manager.worker_id()) + "_to_queue"; - this_worker_qw_push_->connect(addr); - - qw_push_poller_[0].register_socket(*this_worker_qw_push_, zmq::event_flags::pollout); - - // pull - this_worker_qw_pull_.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); - addr = addrBase + "_from_queue_to_worker_" + std::to_string(process_manager.worker_id()); - this_worker_qw_pull_->connect(addr); - - qw_pull_poller_[0].register_socket(*this_worker_qw_pull_, zmq::event_flags::pollin); - - mw_sub_.reset(zmqSvc().socket_ptr(zmq::socket_type::sub)); - mw_sub_->set(zmq::sockopt::rcvhwm, hwm); - mw_sub_->set(zmq::sockopt::subscribe, ""); - mw_sub_->connect(addrBase + "_from_master_to_workers"); - mw_sub_poller_.register_socket(*mw_sub_, zmq::event_flags::pollin); - - wm_push_.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); - wm_push_->set(zmq::sockopt::sndhwm, hwm); - wm_push_->connect(addrBase + "_from_workers_to_master"); - - // check publisher connection and then wait until all subscribers are connected - ZmqLingeringSocketPtr<> subscriber_ping_socket{zmqSvc().socket_ptr(zmq::socket_type::push)}; - subscriber_ping_socket->connect(addrBase + "_subscriber_ping_socket"); - auto all_connected = zmqSvc().receive(*mw_sub_); - zmqSvc().send(*subscriber_ping_socket, "present"); - - while (!all_connected) { - all_connected = zmqSvc().receive(*mw_sub_); - } - - close_this_QW_on_destruct_ = true; - } else { - // should never get here - throw std::runtime_error("Messenger ctor: I'm neither master, nor queue, nor a worker"); + // Claim the channel ends for this process type from the ProcessManager, + // which created them before forking. The channels are connected from + // birth, so no connection handshake is necessary. + if (process_manager.is_master()) { + mq_ = Channel{process_manager.claim_mq_fd()}; + mw_.reserve(process_manager.N_workers()); + for (std::size_t ix = 0; ix < process_manager.N_workers(); ++ix) { + mw_.emplace_back(process_manager.claim_mw_fd(ix)); + } + for (auto &channel : mw_) { + mw_poller_.register_channel(channel); } - } catch (zmq::error_t &e) { - std::cerr << e.what() << " -- errnum: " << e.num() << std::endl; - throw; - }; + } else if (process_manager.is_queue()) { + mq_ = Channel{process_manager.claim_mq_fd()}; + qw_.reserve(process_manager.N_workers()); + for (std::size_t ix = 0; ix < process_manager.N_workers(); ++ix) { + qw_.emplace_back(process_manager.claim_qw_fd(ix)); + } + } else if (process_manager.is_worker()) { + this_worker_qw_ = Channel{process_manager.claim_qw_fd(process_manager.worker_id())}; + this_worker_mw_ = Channel{process_manager.claim_mw_fd(process_manager.worker_id())}; + } else { + // should never get here + throw std::runtime_error("Messenger ctor: I'm neither master, nor queue, nor a worker"); + } } -Messenger::~Messenger() +Messenger::~Messenger() = default; + +void Messenger::test_send(X2X ping_value, test_snd_pipes snd_pipe, std::size_t worker_id) { - if (close_MQ_on_destruct_) { - try { - mq_push_.reset(); - mq_pull_.reset(); - mw_pub_.reset(); - wm_pull_.reset(); - // remove bound files - for (const auto &address : bound_ipc_addresses_) { - // no need to check return value, they are only zero byte /tmp files, the OS should eventually clean them up - remove(address.substr(6).c_str()); - } - } catch (const std::exception &e) { - std::cerr << "WARNING: something in Messenger dtor threw an exception! Original exception message:\n" - << e.what() << std::endl; - } + switch (snd_pipe) { + case test_snd_pipes::M2Q: { + send_from_master_to_queue(ping_value); + break; } - if (close_this_QW_on_destruct_) { - this_worker_qw_push_.reset(); - this_worker_qw_pull_.reset(); - mw_sub_.reset(); - wm_push_.reset(); + case test_snd_pipes::Q2M: { + send_from_queue_to_master(ping_value); + break; + } + case test_snd_pipes::Q2W: { + send_from_queue_to_worker(worker_id, ping_value); + break; + } + case test_snd_pipes::W2Q: { + send_from_worker_to_queue(ping_value); + break; } - if (close_QW_container_on_destruct_) { - for (auto &socket : qw_push_) { - socket.reset(); - } - for (auto &socket : qw_pull_) { - socket.reset(); - } } - // Dev note: do not call zmqSvc()::close_context from here! The Messenger - // is (a member of) a static variable (JobManager) and ZeroMQSvc is static - // as well (the singleton returned by zmqSvc()). Because of the "static - // destruction order fiasco", it is not guaranteed that ZeroMQSvc singleton - // state is still available at time of destruction of the Messenger. Instead - // of a compile time error, this will lead to segfaults at runtime when - // exiting the program (on some platforms), because even though the ZeroMQSvc - // singleton pointer may be overwritten with random data, it will usually - // not randomly become nullptr, which means the nullptr check in the getter - // will still pass and the randomized pointer will be dereferenced. - // Instead, we close context in any new ProcessManager that may be created, - // which means the Messenger will get a fresh context anyway. } -void Messenger::test_send(X2X ping_value, test_snd_pipes snd_pipe, std::size_t worker_id) +void Messenger::test_receive(X2X expected_ping_value, test_rcv_pipes rcv_pipe, std::size_t worker_id) { + X2X handshake = X2X::initial_value; + try { - switch (snd_pipe) { - case test_snd_pipes::M2Q: { - send_from_master_to_queue(ping_value); + switch (rcv_pipe) { + case test_rcv_pipes::fromMonQ: { + handshake = receive_from_master_on_queue(); break; } - case test_snd_pipes::Q2M: { - send_from_queue_to_master(ping_value); + case test_rcv_pipes::fromQonM: { + handshake = receive_from_queue_on_master(); break; } - case test_snd_pipes::Q2W: { - send_from_queue_to_worker(worker_id, ping_value); + case test_rcv_pipes::fromQonW: { + handshake = receive_from_queue_on_worker(); break; } - case test_snd_pipes::W2Q: { - send_from_worker_to_queue(ping_value); + case test_rcv_pipes::fromWonQ: { + handshake = receive_from_worker_on_queue(worker_id); break; } } - } catch (zmq::error_t &e) { - if (e.num() == EAGAIN) { - throw std::runtime_error("Messenger::test_connections: SEND over master-queue connection timed out!"); - } else { - throw; - } - } -} - -void Messenger::test_receive(X2X expected_ping_value, test_rcv_pipes rcv_pipe, std::size_t worker_id) -{ - X2X handshake = X2X::initial_value; - - std::size_t max_tries = 3; - std::size_t tries = 0; - bool carry_on = true; - while (carry_on && (tries++ < max_tries)) { - try { - switch (rcv_pipe) { - case test_rcv_pipes::fromMonQ: { - handshake = receive_from_master_on_queue(); - break; - } - case test_rcv_pipes::fromQonM: { - handshake = receive_from_queue_on_master(); - break; - } - case test_rcv_pipes::fromQonW: { - handshake = receive_from_queue_on_worker(); - break; - } - case test_rcv_pipes::fromWonQ: { - handshake = receive_from_worker_on_queue(worker_id); - break; - } - } - carry_on = false; - } catch (ZMQ::ppoll_error_t &e) { - auto response = handle_zmq_ppoll_error(e); - if (response == zmq_ppoll_error_response::abort) { - throw std::runtime_error("EINTR in test_receive and SIGTERM received, aborting\n"); - } else if (response == zmq_ppoll_error_response::unknown_eintr) { - printf("EINTR in test_receive but no SIGTERM received, try %zu\n", tries); - continue; - } else if (response == zmq_ppoll_error_response::retry) { - printf("EAGAIN in test_receive, try %zu\n", tries); - continue; - } - } catch (zmq::error_t &e) { - if (e.num() == EAGAIN) { - throw std::runtime_error("Messenger::test_connections: RECEIVE over master-queue connection timed out!"); - } else { - printf("unhandled zmq::error_t (not a ppoll_error_t) in Messenger::test_receive with errno %d: %s\n", - e.num(), e.what()); - throw; - } - } + } catch (ppoll_error_t &) { + throw std::runtime_error("SIGTERM received in test_receive, aborting\n"); } if (handshake != expected_ping_value) { throw std::runtime_error( - "Messenger::test_connections: RECEIVE over master-queue connection failed, did not receive expected value!"); + "Messenger::test_connections: RECEIVE over connection failed, did not receive expected value!"); } } -/// \brief Test whether push-pull sockets are working -/// -/// \note This function tests the PUSH-PULL socket pairs only. The PUB-SUB sockets are already tested in the -/// constructor. +/// \brief Test whether the channels between all processes are working /// /// \param process_manager ProcessManager object used to instantiate this object. Used to identify which process we are -/// running on and hence which sockets need to be tested. +/// running on and hence which channels need to be tested. void Messenger::test_connections(const ProcessManager &process_manager) { - if (process_manager.is_queue() || process_manager.is_worker()) { - // Before blocking SIGTERM, set the signal handler, so we can also check after blocking whether a signal occurred - // In our case, we already set it in the ProcessManager after forking to the queue and worker processes. - sigset_t sigmask; - sigemptyset(&sigmask); - sigaddset(&sigmask, SIGTERM); - int rc = sigprocmask(SIG_BLOCK, &sigmask, &ppoll_sigmask); - if (rc < 0) { - throw std::runtime_error("sigprocmask failed in test_connections"); - } - } - if (process_manager.is_master()) { test_receive(X2X::ping, test_rcv_pipes::fromQonM, -1); test_send(X2X::pong, test_snd_pipes::M2Q, -1); @@ -372,7 +145,7 @@ void Messenger::test_connections(const ProcessManager &process_manager) // which means workers are done as well, so if master is done everything is done: test_receive(X2X::pong, test_rcv_pipes::fromQonM, -1); } else if (process_manager.is_queue()) { - ZeroMQPoller poller; + Poller poller; std::size_t mq_index; std::tie(poller, mq_index) = create_queue_poller(); @@ -383,29 +156,26 @@ void Messenger::test_connections(const ProcessManager &process_manager) while (!process_manager.sigterm_received() && (poller.size() > 0)) { // poll: wait until status change (-1: infinite timeout) - std::vector> poll_result; + std::vector poll_result; bool abort; - std::tie(poll_result, abort) = careful_ppoll(poller, ppoll_sigmask); + std::tie(poll_result, abort) = careful_poll(poller); if (abort) break; - // then process incoming messages from sockets - for (auto readable_socket : poll_result) { - // message comes from the master/queue socket (first element): - if (readable_socket.first == mq_index) { + // then process incoming messages from the channels + for (auto readable_index : poll_result) { + // message comes from the master/queue channel (first element): + if (readable_index == mq_index) { test_receive(X2X::pong, test_rcv_pipes::fromMonQ, -1); test_receive(X2X::ping, test_rcv_pipes::fromMonQ, -1); - poller.unregister_socket(*mq_pull_); - } else { // from a worker socket - // TODO: dangerous assumption for this_worker_id, may become invalid if we allow multiple queue_loops on - // the same process! - auto this_worker_id = readable_socket.first - 1; // TODO: replace with a more reliable lookup - + poller.unregister_channel(mq_); + } else { // from a worker channel + auto this_worker_id = readable_index - 1; // by construction of the queue poller test_receive(X2X::pong, test_rcv_pipes::fromWonQ, this_worker_id); test_receive(X2X::ping, test_rcv_pipes::fromWonQ, this_worker_id); test_send(X2X::pong, test_snd_pipes::Q2W, this_worker_id); - poller.unregister_socket(*qw_pull_[this_worker_id]); + poller.unregister_channel(qw_[this_worker_id]); } } } @@ -420,31 +190,54 @@ void Messenger::test_connections(const ProcessManager &process_manager) // should never get here throw std::runtime_error("Messenger::test_connections: I'm neither master, nor queue, nor a worker"); } - - if (process_manager.is_queue() || process_manager.is_worker()) { - // clean up signal management modifications - sigprocmask(SIG_SETMASK, &ppoll_sigmask, nullptr); - } } /// Helper function that creates a poller for Queue::loop() -std::pair Messenger::create_queue_poller() +std::pair Messenger::create_queue_poller() { - ZeroMQPoller poller; - std::size_t mq_index = poller.register_socket(*mq_pull_, zmq::event_flags::pollin); - for (auto &s : qw_pull_) { - poller.register_socket(*s, zmq::event_flags::pollin); + Poller poller; + std::size_t mq_index = poller.register_channel(mq_); + for (auto &channel : qw_) { + poller.register_channel(channel); } return {std::move(poller), mq_index}; } /// Helper function that creates a poller for worker_loop() -std::pair Messenger::create_worker_poller() +std::pair Messenger::create_worker_poller() { - ZeroMQPoller poller; - poller.register_socket(*this_worker_qw_pull_, zmq::event_flags::pollin); - std::size_t mw_sub_index = poller.register_socket(*mw_sub_, zmq::event_flags::pollin); - return {std::move(poller), mw_sub_index}; + Poller poller; + poller.register_channel(this_worker_qw_); + std::size_t mw_index = poller.register_channel(this_worker_mw_); + return {std::move(poller), mw_index}; +} + +Channel &Messenger::select_worker_channel_on_master() +{ + // continue receiving the parts of an in-progress multipart message from + // the same worker (multipart messages must arrive as one unit, like with + // the ZeroMQ sockets used before) + if (mw_current_source_ != nullptr) { + return *mw_current_source_; + } + auto readable = mw_poller_.poll(-1); + // rotate over the workers for fairness, like a ZeroMQ PULL socket would + for (std::size_t offset = 0; offset < mw_.size(); ++offset) { + std::size_t candidate = (mw_next_poll_position_ + offset) % mw_.size(); + for (std::size_t index : readable) { + if (index == candidate) { + mw_next_poll_position_ = (candidate + 1) % mw_.size(); + return mw_[candidate]; + } + } + } + // cannot happen: poll(-1) always returns at least one readable channel + throw std::logic_error("Messenger::select_worker_channel_on_master: poll returned no readable channels"); +} + +void Messenger::update_worker_channel_on_master(Channel &channel, bool more) +{ + mw_current_source_ = more ? &channel : nullptr; } // -- WORKER - QUEUE COMMUNICATION -- @@ -459,12 +252,6 @@ void Messenger::send_from_queue_to_master() {} void Messenger::send_from_master_to_queue() {} -/// Set the flag used in all send functions; 0, ZMQ_DONTWAIT, ZMQ_SNDMORE or bitwise combination -void Messenger::set_send_flag(zmq::send_flags flag) -{ - send_flag_ = flag; -} - // for debugging #define PROCESS_VAL(p) \ case (p): s = #p; break; diff --git a/roofit/multiprocess/src/PriorityQueue.h b/roofit/multiprocess/src/PriorityQueue.h index ae816b8e3abb5..a011a56cf39fb 100644 --- a/roofit/multiprocess/src/PriorityQueue.h +++ b/roofit/multiprocess/src/PriorityQueue.h @@ -15,6 +15,8 @@ #include "RooFit/MultiProcess/Queue.h" #include +#include +#include namespace RooFit { namespace MultiProcess { diff --git a/roofit/multiprocess/src/ProcessManager.cxx b/roofit/multiprocess/src/ProcessManager.cxx index 81f8ce7d8df92..f820c2e03dde6 100644 --- a/roofit/multiprocess/src/ProcessManager.cxx +++ b/roofit/multiprocess/src/ProcessManager.cxx @@ -18,8 +18,10 @@ #include "RooFit/MultiProcess/Config.h" #include -#include // for strsignal -#include // for wait +#include // for strsignal +#include // for fcntl, O_NONBLOCK +#include // for socketpair +#include // for wait #include #include @@ -40,18 +42,10 @@ namespace MultiProcess { /// \param N_workers Number of worker processes to spawn. ProcessManager::ProcessManager(std::size_t N_workers) : N_workers_(N_workers) { - // Note: zmq context is automatically created in the ZeroMQSvc class and maintained as singleton, - // but we must close any possibly existing state before reusing it. This assumes that our Messenger - // is the only user of ZeroMQSvc and that there is only one Messenger at a time. Beware that - // this must be designed more carefully if either of these assumptions change! Note also that this - // call must be done before the ProcessManager forks new processes, otherwise the master process' - // context that will be cloned to all forked processes will be closed multiple times, which will - // hang, because the ZeroMQ context creates threads and these will not be cloned along with the - // fork. See the ZeroMQ documentation for more details on this. In principle, one could design this - // in a more finegrained way by keeping the context on the master process and only recreating it - // on child processes (while avoiding calling the destructor on the child processes!). This - // approach may offer more flexibility if this is needed in the future. - zmqSvc().close_context(); + // The socketpairs used for interprocess communication must be created + // before forking, so that all processes inherit the file descriptors of + // the connected channels. + create_channel_fds(); initialize_processes(); } @@ -62,19 +56,36 @@ ProcessManager::~ProcessManager() } else { wait_for_sigterm_then_exit(); } + close_channel_fds(); } // static member initialization volatile sig_atomic_t ProcessManager::sigterm_received_ = 0; +int ProcessManager::sigterm_wake_read_fd_ = -1; +int ProcessManager::sigterm_wake_write_fd_ = -1; // static function /// We need this to tell the children to die, because we can't talk /// to them anymore during JobManager destruction, because that kills /// the Messenger first. We do that with SIGTERMs. The sigterm_received() /// should be checked in message loops to stop them when it's true. +/// The handler also writes to a self-pipe, so that a poll that is entered +/// after the flag check but before signal delivery still wakes up. void ProcessManager::handle_sigterm(int /*signum*/) { sigterm_received_ = 1; + if (sigterm_wake_write_fd_ >= 0) { + char byte = 't'; + // write is async-signal-safe; a full pipe just means a wake-up is already pending + ssize_t unused = write(sigterm_wake_write_fd_, &byte, 1); + (void)unused; + } +} + +// static function +int ProcessManager::sigterm_wake_fd() +{ + return sigterm_wake_read_fd_; } // static function @@ -105,6 +116,120 @@ pid_t fork_and_handle_errors() return child_pid; } +namespace { + +/// Set FD_CLOEXEC so that the descriptor is not leaked into programs that a +/// process executes (e.g. with gSystem->Exec). Leaked duplicates of the +/// channel descriptors would keep the connections open after the owning +/// process dies, defeating the closed-connection detection in Channel. +void set_close_on_exec(int fd) +{ + int flags = fcntl(fd, F_GETFD, 0); + if (flags == -1 || fcntl(fd, F_SETFD, flags | FD_CLOEXEC) == -1) { + throw std::runtime_error(std::string("ProcessManager: could not set FD_CLOEXEC: ") + strerror(errno)); + } +} + +void make_socketpair(std::array &fds) +{ + if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds.data()) != 0) { + throw std::runtime_error(std::string("ProcessManager: socketpair failed: ") + strerror(errno)); + } + set_close_on_exec(fds[0]); + set_close_on_exec(fds[1]); +} + +void close_fd_pair(std::array &fds, int keep = -1) +{ + for (int &fd : fds) { + if (fd >= 0 && fd != keep) { + close(fd); + fd = -1; + } + } +} + +int claim_fd(int &fd) +{ + if (fd < 0) { + throw std::logic_error("ProcessManager: channel file descriptor already claimed or not owned by this process"); + } + int result = fd; + fd = -1; + return result; +} + +} // namespace + +/// Create the socketpairs that connect the processes. Must be called before +/// forking; every process then keeps only the ends it needs (see +/// close_unused_channel_fds). +void ProcessManager::create_channel_fds() +{ + make_socketpair(mq_fds_); + qw_fds_.resize(N_workers_, {{-1, -1}}); + mw_fds_.resize(N_workers_, {{-1, -1}}); + for (std::size_t ix = 0; ix < N_workers_; ++ix) { + make_socketpair(qw_fds_[ix]); + make_socketpair(mw_fds_[ix]); + } +} + +/// Close the channel ends that do not belong to the current process type. +void ProcessManager::close_unused_channel_fds() +{ + for (std::size_t ix = 0; ix < N_workers_; ++ix) { + if (is_master_) { + close_fd_pair(qw_fds_[ix]); + close_fd_pair(mw_fds_[ix], mw_fds_[ix][0]); + } else if (is_queue_) { + close_fd_pair(qw_fds_[ix], qw_fds_[ix][0]); + close_fd_pair(mw_fds_[ix]); + } else { // worker + close_fd_pair(qw_fds_[ix], ix == worker_id_ ? qw_fds_[ix][1] : -1); + close_fd_pair(mw_fds_[ix], ix == worker_id_ ? mw_fds_[ix][1] : -1); + } + } + if (is_master_) { + close_fd_pair(mq_fds_, mq_fds_[0]); + } else if (is_queue_) { + close_fd_pair(mq_fds_, mq_fds_[1]); + } else { + close_fd_pair(mq_fds_); + } +} + +/// Close all channel ends still owned by this ProcessManager (i.e. not +/// claimed by a Messenger). +void ProcessManager::close_channel_fds() +{ + close_fd_pair(mq_fds_); + for (auto &fds : qw_fds_) { + close_fd_pair(fds); + } + for (auto &fds : mw_fds_) { + close_fd_pair(fds); + } +} + +/// Hand over the master-queue channel end for the current process type. +int ProcessManager::claim_mq_fd() +{ + return claim_fd(is_master_ ? mq_fds_[0] : mq_fds_[1]); +} + +/// Hand over the queue-worker channel end for the current process type. +int ProcessManager::claim_qw_fd(std::size_t worker_ix) +{ + return claim_fd(is_queue_ ? qw_fds_[worker_ix][0] : qw_fds_[worker_ix][1]); +} + +/// Hand over the master-worker channel end for the current process type. +int ProcessManager::claim_mw_fd(std::size_t worker_ix) +{ + return claim_fd(is_master_ ? mw_fds_[worker_ix][0] : mw_fds_[worker_ix][1]); +} + /// \brief Fork processes and activate CPU pinning /// /// \param cpu_pinning Activate CPU pinning if true. Effective on Linux only. @@ -143,8 +268,35 @@ void ProcessManager::initialize_processes(bool cpu_pinning) } } + close_unused_channel_fds(); + // set the sigterm handler on the child processes if (!is_master_) { + // Create the self-pipe that the handler writes to before installing the + // handler. The pipe wakes up any poll on the channels, also when the + // signal arrived just before the poll was entered (see Channel::wait). + if (sigterm_wake_read_fd_ < 0) { + int pipe_fds[2]; + if (pipe(pipe_fds) != 0) { + std::perror("pipe failed"); + std::exit(1); + } + for (int fd : pipe_fds) { + int flags = fcntl(fd, F_GETFL, 0); + if (flags == -1 || fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1) { + std::perror("fcntl failed"); + std::exit(1); + } + int fd_flags = fcntl(fd, F_GETFD, 0); + if (fd_flags == -1 || fcntl(fd, F_SETFD, fd_flags | FD_CLOEXEC) == -1) { + std::perror("fcntl failed"); + std::exit(1); + } + } + sigterm_wake_read_fd_ = pipe_fds[0]; + sigterm_wake_write_fd_ = pipe_fds[1]; + } + struct sigaction sa; memset(&sa, '\0', sizeof(sa)); sa.sa_handler = ProcessManager::handle_sigterm; diff --git a/roofit/multiprocess/src/Queue.cxx b/roofit/multiprocess/src/Queue.cxx index aaea4d378ebab..ae748820ce361 100644 --- a/roofit/multiprocess/src/Queue.cxx +++ b/roofit/multiprocess/src/Queue.cxx @@ -16,6 +16,8 @@ #include "RooFit/MultiProcess/ProcessManager.h" #include "RooFit/MultiProcess/util.h" +#include + namespace RooFit { namespace MultiProcess { @@ -83,61 +85,37 @@ void Queue::process_worker_message(std::size_t this_worker_id, W2Q message) void Queue::loop() { assert(JobManager::instance()->process_manager().is_queue()); - ZeroMQPoller poller; + Poller poller; std::size_t mq_index; std::tie(poller, mq_index) = JobManager::instance()->messenger().create_queue_poller(); - // Before blocking SIGTERM, set the signal handler, so we can also check after blocking whether a signal occurred - // In our case, we already set it in the ProcessManager after forking to the queue and worker processes. - - sigset_t sigmask; - sigemptyset(&sigmask); - sigaddset(&sigmask, SIGTERM); - sigprocmask(SIG_BLOCK, &sigmask, &JobManager::instance()->messenger().ppoll_sigmask); - - // Before doing anything, check whether we have received a terminate signal while blocking signals! - // In this case, we also do that in the while condition. + // The SIGTERM handler was set in the ProcessManager after forking to the queue and worker + // processes; it wakes up any poll through the self-pipe, so no signal blocking is needed here. while (!ProcessManager::sigterm_received()) { - try { // watch for zmq_error from ppoll caused by SIGTERM from master + try { // watch for poll interruption caused by SIGTERM from master // poll: wait until status change (-1: infinite timeout) - auto poll_result = poller.ppoll(-1, &JobManager::instance()->messenger().ppoll_sigmask); - // then process incoming messages from sockets - for (auto readable_socket : poll_result) { - // message comes from the master/queue socket (first element): - if (readable_socket.first == mq_index) { + auto poll_result = poller.poll(-1); + // then process incoming messages from the channels + for (auto readable_index : poll_result) { + // message comes from the master/queue channel (first element): + if (readable_index == mq_index) { auto message = JobManager::instance()->messenger().receive_from_master_on_queue(); process_master_message(message); - } else { // from a worker socket - auto this_worker_id = readable_socket.first - 1; + } else { // from a worker channel + // by construction of the queue poller: the master-queue channel is + // registered first (index 0), followed by the worker channels in + // worker-ID order + auto this_worker_id = readable_index - 1; auto message = JobManager::instance()->messenger().receive_from_worker_on_queue(this_worker_id); process_worker_message(this_worker_id, message); } } - } catch (ZMQ::ppoll_error_t &e) { - zmq_ppoll_error_response response; - try { - response = handle_zmq_ppoll_error(e); - } catch (std::logic_error &) { - printf("queue loop got unhandleable ZMQ::ppoll_error_t\n"); - throw; - } - if (response == zmq_ppoll_error_response::abort) { - break; - } else if (response == zmq_ppoll_error_response::unknown_eintr) { - printf("EINTR in queue loop but no SIGTERM received, continuing\n"); - continue; - } else if (response == zmq_ppoll_error_response::retry) { - printf("EAGAIN from ppoll in queue loop, continuing\n"); - continue; - } - } catch (zmq::error_t &e) { - printf("unhandled zmq::error_t (not a ppoll_error_t) in queue loop with errno %d: %s\n", e.num(), e.what()); - throw; + } catch (ppoll_error_t &) { + // SIGTERM received (benign signal interruptions are retried inside + // Channel::wait), so exit the loop + break; } } - - // clean up signal management modifications - sigprocmask(SIG_SETMASK, &JobManager::instance()->messenger().ppoll_sigmask, nullptr); } } // namespace MultiProcess diff --git a/roofit/multiprocess/src/util.cxx b/roofit/multiprocess/src/util.cxx index 8a361ba688229..be934130fe8a5 100644 --- a/roofit/multiprocess/src/util.cxx +++ b/roofit/multiprocess/src/util.cxx @@ -61,67 +61,18 @@ int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing) return status; } -zmq_ppoll_error_response handle_zmq_ppoll_error(ZMQ::ppoll_error_t &e) -{ - if ((e.num() == EINTR) && (ProcessManager::sigterm_received())) { - // valid EINTR, because we want to exit and kill the processes on SIGTERM - return zmq_ppoll_error_response::abort; - } else if (e.num() == EINTR) { - // on other EINTRs, we retry (mostly this happens in debuggers) - return zmq_ppoll_error_response::unknown_eintr; - } else if (e.num() == EAGAIN) { - // This can happen from recv if ppoll initially gets a read-ready signal for a socket, - // but the received data does not pass the checksum test, so the socket becomes unreadable - // again or from non-blocking send if the socket becomes unwritable either due to the HWM - // being reached or the socket not being connected (anymore). The latter case usually means - // the connection has been severed from the other side, meaning it has probably been killed - // and in that case the next ppoll call will probably also receive a SIGTERM, ending the - // loop. In case something else is wrong, this message will print multiple times, which - // should be taken as a cue for writing a bug report :) - return zmq_ppoll_error_response::retry; - } else { - char buffer[512]; - snprintf(buffer, 512, - "handle_zmq_ppoll_error is out of options to handle exception, caught ZMQ::ppoll_error_t had errno %d " - "and text: %s\n", - e.num(), e.what()); - throw std::logic_error(buffer); - } -} - // returns a tuple containing first the poll result and second a boolean flag that tells the caller whether it should -// abort the enclosing loop -std::tuple>, bool> -careful_ppoll(ZeroMQPoller &poller, const sigset_t &ppoll_sigmask, std::size_t max_tries) +// abort the enclosing loop because a SIGTERM was received +std::tuple, bool> careful_poll(Poller &poller) { - std::size_t tries = 0; - std::vector> poll_result; + // Benign signal interruptions are already retried inside Channel::wait, so + // an exception here means a termination request. + std::vector poll_result; bool abort = true; - bool carry_on = true; - while (carry_on && (tries++ < max_tries)) { - if (tries > 1) { - printf("careful_ppoll try %zu\n", tries); - } - try { // watch for zmq_error from ppoll caused by SIGTERM from master - poll_result = poller.ppoll(-1, &ppoll_sigmask); - abort = false; - carry_on = false; - } catch (ZMQ::ppoll_error_t &e) { - auto response = handle_zmq_ppoll_error(e); - if (response == zmq_ppoll_error_response::abort) { - break; - } else if (response == zmq_ppoll_error_response::unknown_eintr) { - printf("EINTR in careful_ppoll but no SIGTERM received, try %zu\n", tries); - continue; - } else if (response == zmq_ppoll_error_response::retry) { - printf("EAGAIN in careful_ppoll (from either send or receive), try %zu\n", tries); - continue; - } - } - } - - if (tries == max_tries) { - printf("careful_ppoll reached maximum number of tries, %zu, please report as a bug\n", tries); + try { + poll_result = poller.poll(-1); + abort = false; + } catch (ppoll_error_t &) { } return std::make_tuple(poll_result, abort); } diff --git a/roofit/multiprocess/src/worker.cxx b/roofit/multiprocess/src/worker.cxx index 230174fd7d6db..25cee3b4b346e 100644 --- a/roofit/multiprocess/src/worker.cxx +++ b/roofit/multiprocess/src/worker.cxx @@ -23,6 +23,7 @@ #include #include // getpid, pid_t +#include #include // EINTR #include // sigprocmask etc @@ -49,23 +50,15 @@ void worker_loop() // use a flag to not ask twice bool dequeue_acknowledged = true; - ZeroMQPoller poller; + Poller poller; std::size_t mw_sub_index; std::tie(poller, mw_sub_index) = JobManager::instance()->messenger().create_worker_poller(); - // Before blocking SIGTERM, set the signal handler, so we can also check after blocking whether a signal occurred - // In our case, we already set it in the ProcessManager after forking to the queue and worker processes. - - sigset_t sigmask; - sigemptyset(&sigmask); - sigaddset(&sigmask, SIGTERM); - sigprocmask(SIG_BLOCK, &sigmask, &JobManager::instance()->messenger().ppoll_sigmask); - - // Before doing anything, check whether we have received a terminate signal while blocking signals! - // In this case, we also do that in the while condition. + // The SIGTERM handler was set in the ProcessManager after forking to the queue and worker + // processes; it wakes up any poll through the self-pipe, so no signal blocking is needed here. while (!ProcessManager::sigterm_received()) { - try { // watch for error from ppoll (which is called inside receive functions) caused by SIGTERM from master + try { // watch for error from poll (which is called inside receive functions) caused by SIGTERM from master // try to dequeue a task if (dequeue_acknowledged) { // don't ask twice @@ -73,24 +66,24 @@ void worker_loop() dequeue_acknowledged = false; } - // wait for handshake from queue or update from SUB socket - auto poll_result = poller.ppoll(-1, &JobManager::instance()->messenger().ppoll_sigmask); - // because the poller may now have a waiting update from master over the SUB socket, - // but the queue socket could be first in the poll_result vector, and during handling - // of a new task it is possible we need to already receive the updated state over SUB, - // we have to then flip this boolean so that in the for loop when we reach the SUB - // socket's result, we can skip it (otherwise we will hang there, because no more - // updated state will be coming): + // wait for handshake from queue or update from the master-worker channel + auto poll_result = poller.poll(-1); + // because the poller may now have a waiting update from master over the master-worker + // channel, but the queue channel could be first in the poll_result vector, and during + // handling of a new task it is possible we need to already receive the updated state, + // we have to then flip this boolean so that in the for loop when we reach the + // master-worker channel's result, we can skip it (otherwise we will hang there, + // because no more updated state will be coming): bool skip_sub = false; - // then process incoming messages from sockets - for (auto readable_socket : poll_result) { - // message comes from the master-worker SUB socket (first element): - if (readable_socket.first == mw_sub_index) { + // then process incoming messages from the channels + for (auto readable_index : poll_result) { + // message comes from the master-worker channel (first element): + if (readable_index == mw_sub_index) { if (!skip_sub) { auto job_id = JobManager::instance()->messenger().receive_from_master_on_worker(); JobManager::get_job_object(job_id)->update_state(); } - } else { // from queue socket + } else { // from queue channel message_q2w = JobManager::instance()->messenger().receive_from_queue_on_worker(); switch (message_q2w) { case Q2W::dequeue_rejected: { @@ -121,34 +114,15 @@ void worker_loop() } } - } catch (ZMQ::ppoll_error_t &e) { - zmq_ppoll_error_response response; - try { - response = handle_zmq_ppoll_error(e); - } catch (std::logic_error &) { - printf("worker loop at PID %d got unhandleable ZMQ::ppoll_error_t\n", getpid()); - throw; - } - if (response == zmq_ppoll_error_response::abort) { - break; - } else if (response == zmq_ppoll_error_response::unknown_eintr) { - printf("EINTR in worker loop at PID %d but no SIGTERM received, continuing\n", getpid()); - continue; - } else if (response == zmq_ppoll_error_response::retry) { - printf("EAGAIN from ppoll in worker loop at PID %d, continuing\n", getpid()); - continue; - } - } catch (zmq::error_t &e) { - printf("unhandled zmq::error_t (not a ppoll_error_t) in worker loop at PID %d with errno %d: %s\n", getpid(), - e.num(), e.what()); - throw; + } catch (ppoll_error_t &) { + // SIGTERM received (benign signal interruptions are retried inside + // Channel::wait), so exit the loop + break; } } - if (RooFit::MultiProcess::Config::getTimingAnalysis()) ProcessTimer::write_file(); - - // clean up signal management modifications - sigprocmask(SIG_SETMASK, &JobManager::instance()->messenger().ppoll_sigmask, nullptr); + if (RooFit::MultiProcess::Config::getTimingAnalysis()) + ProcessTimer::write_file(); worker_loop_running = false; } diff --git a/roofit/multiprocess/test/CMakeLists.txt b/roofit/multiprocess/test/CMakeLists.txt index c191d56346d22..1c080fc00d94f 100644 --- a/roofit/multiprocess/test/CMakeLists.txt +++ b/roofit/multiprocess/test/CMakeLists.txt @@ -8,6 +8,7 @@ ROOT_ADD_GTEST(test_RooFit_MultiProcess_Job test_Job.cxx LIBRARIES RooFitMultiPr # link to the INTERFACE library separately, ROOT_EXECUTABLE cannot handle INTERFACE library properties: target_link_libraries(test_RooFit_MultiProcess_Job PUBLIC RooFit_multiprocess_testing_utils) +ROOT_ADD_GTEST(test_RooFit_MultiProcess_Channel test_Channel.cxx LIBRARIES RooFitMultiProcess) ROOT_ADD_GTEST(test_RooFit_MultiProcess_ProcessManager test_ProcessManager.cxx LIBRARIES RooFitMultiProcess) ROOT_ADD_GTEST(test_RooFit_MultiProcess_Messenger test_Messenger.cxx LIBRARIES RooFitMultiProcess) diff --git a/roofit/multiprocess/test/NoopJob.h b/roofit/multiprocess/test/NoopJob.h index ca404a473aa69..98939afcb63b0 100644 --- a/roofit/multiprocess/test/NoopJob.h +++ b/roofit/multiprocess/test/NoopJob.h @@ -14,6 +14,8 @@ #define ROOT_NOOPJOB_H #include "RooFit/MultiProcess/Job.h" + +#include // needed to complete type returned from... #include "RooFit/MultiProcess/JobManager.h" // ... Job::get_manager() #include "RooFit/MultiProcess/ProcessManager.h" // ... JobManager::process_manager() @@ -48,12 +50,12 @@ class NoopJob : public RooFit::MultiProcess::Job { void send_back_task_result_from_worker(std::size_t task) override { task_result_t task_result{id_, task}; - zmq::message_t message(sizeof(task_result_t)); + RooFit::MultiProcess::Message message(sizeof(task_result_t)); memcpy(message.data(), &task_result, sizeof(task_result_t)); get_manager()->messenger().send_from_worker_to_master(std::move(message)); } - bool receive_task_result_on_master(const zmq::message_t &message) override + bool receive_task_result_on_master(const RooFit::MultiProcess::Message &message) override { /*auto result =*/ message.data(); --N_tasks_at_workers_; diff --git a/roofit/multiprocess/test/OrderTrackingJob.h b/roofit/multiprocess/test/OrderTrackingJob.h index e2ad34665c368..acdbd65c3268c 100644 --- a/roofit/multiprocess/test/OrderTrackingJob.h +++ b/roofit/multiprocess/test/OrderTrackingJob.h @@ -15,6 +15,7 @@ #include "RooFit/MultiProcess/types.h" #include "NoopJob.h" +#include #include class OrderTrackingJob : public NoopJob { @@ -24,7 +25,7 @@ class OrderTrackingJob : public NoopJob { { } - bool receive_task_result_on_master(const zmq::message_t &message) override + bool receive_task_result_on_master(const RooFit::MultiProcess::Message &message) override { auto result = message.data(); received_task_order[id_].push_back(result->task_id); diff --git a/roofit/multiprocess/test/test_Channel.cxx b/roofit/multiprocess/test/test_Channel.cxx new file mode 100644 index 0000000000000..ffa9c3679a2d8 --- /dev/null +++ b/roofit/multiprocess/test/test_Channel.cxx @@ -0,0 +1,200 @@ +/* + * Project: RooFit + * Authors: + * Jonas Rembser, CERN 2026 + * + * Copyright (c) 2026, CERN + * + * Redistribution and use in source and binary forms, + * with or without modification, are permitted according to the terms + * listed in LICENSE (http://roofit.sourceforge.net/license.txt) + */ + +#include "RooFit/MultiProcess/Channel.h" + +#include "gtest/gtest.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace { + +std::pair makeChannelPair() +{ + int fds[2]; + if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) != 0) { + throw std::runtime_error("socketpair failed"); + } + return {RooFit::MultiProcess::Channel{fds[0]}, RooFit::MultiProcess::Channel{fds[1]}}; +} + +std::vector testPattern(std::size_t n, double offset) +{ + std::vector values(n); + std::iota(values.begin(), values.end(), offset); + return values; +} + +} // namespace + +TEST(TestMPChannel, SmallFramesRoundTrip) +{ + auto channels = makeChannelPair(); + + RooFit::MultiProcess::send_item(channels.first, std::size_t{42}, false); + RooFit::MultiProcess::send_item(channels.first, 3.14, false); + RooFit::MultiProcess::send_item(channels.first, std::string("hello"), false); + + EXPECT_EQ(RooFit::MultiProcess::receive_item(channels.second), 42u); + EXPECT_EQ(RooFit::MultiProcess::receive_item(channels.second), 3.14); + EXPECT_EQ(RooFit::MultiProcess::receive_item(channels.second), "hello"); +} + +TEST(TestMPChannel, MultipartMoreFlag) +{ + auto channels = makeChannelPair(); + + RooFit::MultiProcess::send_item(channels.first, 1, true); + RooFit::MultiProcess::send_item(channels.first, 2, true); + RooFit::MultiProcess::send_item(channels.first, 3, false); + + bool more = false; + EXPECT_EQ(RooFit::MultiProcess::receive_item(channels.second, &more), 1); + EXPECT_TRUE(more); + EXPECT_EQ(RooFit::MultiProcess::receive_item(channels.second, &more), 2); + EXPECT_TRUE(more); + EXPECT_EQ(RooFit::MultiProcess::receive_item(channels.second, &more), 3); + EXPECT_FALSE(more); +} + +// A frame far larger than the kernel socket buffer must be retained in the +// channel's pending-output buffer and be flushed while the receiving side +// waits for input. This exercises the partial-write machinery that production +// fits hit with large state-update and result messages. +TEST(TestMPChannel, LargeFrameExceedsSocketBuffer) +{ + auto channels = makeChannelPair(); + + // 4 MB payload, well above the default AF_UNIX buffer size + auto values = testPattern(512 * 1024, 0.); + RooFit::MultiProcess::Message msg(values.begin(), values.end()); + RooFit::MultiProcess::send_item(channels.first, msg, false); + + // the socket cannot have accepted everything yet + EXPECT_TRUE(channels.first.has_pending_output()); + + // receiving drains the sender's pending output: Channel::wait flushes the + // pending output of all channels in the process while waiting for input + auto received = RooFit::MultiProcess::receive_item(channels.second); + ASSERT_EQ(received.size(), values.size() * sizeof(double)); + const double *data = received.data(); + for (std::size_t ix = 0; ix < values.size(); ++ix) { + ASSERT_EQ(data[ix], values[ix]) << "at index " << ix; + } + EXPECT_FALSE(channels.first.has_pending_output()); +} + +// Both directions blocked at the same time: each side first queues a frame +// larger than the socket buffer, then receives the other side's frame. With +// blocking sends this would deadlock two processes; the pending-output +// buffers plus the flush-during-wait must resolve it. +TEST(TestMPChannel, BidirectionalPendingOutput) +{ + auto channels = makeChannelPair(); + + auto valuesA = testPattern(512 * 1024, 0.); + auto valuesB = testPattern(512 * 1024, 1000000.); + RooFit::MultiProcess::Message msgA(valuesA.begin(), valuesA.end()); + RooFit::MultiProcess::Message msgB(valuesB.begin(), valuesB.end()); + + RooFit::MultiProcess::send_item(channels.first, msgA, false); + RooFit::MultiProcess::send_item(channels.second, msgB, false); + EXPECT_TRUE(channels.first.has_pending_output()); + EXPECT_TRUE(channels.second.has_pending_output()); + + auto receivedB = RooFit::MultiProcess::receive_item(channels.first); + auto receivedA = RooFit::MultiProcess::receive_item(channels.second); + + ASSERT_EQ(receivedA.size(), valuesA.size() * sizeof(double)); + ASSERT_EQ(receivedB.size(), valuesB.size() * sizeof(double)); + EXPECT_EQ(receivedA.data()[valuesA.size() - 1], valuesA.back()); + EXPECT_EQ(receivedB.data()[valuesB.size() - 1], valuesB.back()); + EXPECT_FALSE(channels.first.has_pending_output()); + EXPECT_FALSE(channels.second.has_pending_output()); +} + +// A blocking receive must survive benign signal interruptions (profilers, +// SIGCHLD, debuggers): Channel::wait retries on EINTR instead of surfacing +// it, which is what keeps multi-frame message sequences from desynchronizing +// the wire protocol in the event loops. +TEST(TestMPChannel, BenignSignalsDoNotInterruptReceive) +{ + int fds[2]; + ASSERT_EQ(socketpair(AF_UNIX, SOCK_STREAM, 0, fds), 0); + + pid_t child_pid = fork(); + ASSERT_NE(child_pid, -1); + if (child_pid == 0) { // child: wait a moment, then send one frame + close(fds[0]); + { + RooFit::MultiProcess::Channel channel{fds[1]}; + usleep(200000); + RooFit::MultiProcess::send_item(channel, std::size_t{1234}, false); + } + std::_Exit(0); + } + + // parent: bombard itself with SIGALRM every 10 ms while blocking in receive + close(fds[1]); + struct sigaction sa; + memset(&sa, '\0', sizeof(sa)); + sa.sa_handler = [](int) {}; + ASSERT_EQ(sigaction(SIGALRM, &sa, nullptr), 0); + itimerval timer{{0, 10000}, {0, 10000}}; + ASSERT_EQ(setitimer(ITIMER_REAL, &timer, nullptr), 0); + + { + RooFit::MultiProcess::Channel channel{fds[0]}; + EXPECT_EQ(RooFit::MultiProcess::receive_item(channel), 1234u); + } + + itimerval stop_timer{{0, 0}, {0, 0}}; + setitimer(ITIMER_REAL, &stop_timer, nullptr); + sa.sa_handler = SIG_DFL; + sigaction(SIGALRM, &sa, nullptr); + + int status = -1; + ASSERT_EQ(waitpid(child_pid, &status, 0), child_pid); + EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0); +} + +// Multipart frames are only flushed on the last part, and several messages +// queued back-to-back must come out with intact boundaries and "more" flags. +TEST(TestMPChannel, QueuedMultipartMessages) +{ + auto channels = makeChannelPair(); + + auto values = testPattern(128 * 1024, 0.); + for (int repeat = 0; repeat < 3; ++repeat) { + RooFit::MultiProcess::Message msg(values.begin(), values.end()); + RooFit::MultiProcess::send_item(channels.first, std::size_t(repeat), true); + RooFit::MultiProcess::send_item(channels.first, msg, false); + } + + for (int repeat = 0; repeat < 3; ++repeat) { + bool more = false; + auto id = RooFit::MultiProcess::receive_item(channels.second, &more); + EXPECT_EQ(id, static_cast(repeat)); + EXPECT_TRUE(more); + auto msg = RooFit::MultiProcess::receive_item(channels.second, &more); + EXPECT_EQ(msg.size(), values.size() * sizeof(double)); + EXPECT_FALSE(more); + } +} diff --git a/roofit/multiprocess/test/test_Job.cxx b/roofit/multiprocess/test/test_Job.cxx index fc3fc600a7573..3537ab7d2ac5c 100644 --- a/roofit/multiprocess/test/test_Job.cxx +++ b/roofit/multiprocess/test/test_Job.cxx @@ -117,12 +117,12 @@ class xSquaredPlusBVectorParallel : public RooFit::MultiProcess::Job { void send_back_task_result_from_worker(std::size_t task) override { task_result_t task_result{id_, task, serial_->result_[task]}; - zmq::message_t message(sizeof(task_result_t)); + RooFit::MultiProcess::Message message(sizeof(task_result_t)); memcpy(message.data(), &task_result, sizeof(task_result_t)); get_manager()->messenger().send_from_worker_to_master(std::move(message)); } - bool receive_task_result_on_master(const zmq::message_t &message) override + bool receive_task_result_on_master(const RooFit::MultiProcess::Message &message) override { auto result = message.data(); serial_->result_[result->task_id] = result->value; diff --git a/roofit/multiprocess/test/test_Messenger.cxx b/roofit/multiprocess/test/test_Messenger.cxx index 23a451612d276..615556f131eb9 100644 --- a/roofit/multiprocess/test/test_Messenger.cxx +++ b/roofit/multiprocess/test/test_Messenger.cxx @@ -34,20 +34,8 @@ TEST(TestMPMessenger, Connections) } } RooFit::MultiProcess::Messenger messenger(pm); - if (pm.is_master()) { - // more SIGCHLD handling - sigset_t sigmask; - sigemptyset(&sigmask); - sigaddset(&sigmask, SIGCHLD); - int rc = sigprocmask(SIG_BLOCK, &sigmask, &messenger.ppoll_sigmask); - if (rc < 0) { - throw std::runtime_error("sigprocmask failed in TestMPMessenger.Connections"); - } - } messenger.test_connections(pm); if (pm.is_master()) { - // clean up signal management modifications - sigprocmask(SIG_SETMASK, &messenger.ppoll_sigmask, nullptr); sa.sa_handler = SIG_DFL; if (sigaction(SIGCHLD, &sa, nullptr) < 0) { std::perror("sigaction failed"); @@ -58,9 +46,9 @@ TEST(TestMPMessenger, Connections) TEST(TestMPMessenger, ConnectionsManualExit) { - // the point of this test is to see whether clean-up of ZeroMQ resources is done properly without calling any - // destructors (which is what happens when you call _Exit() instead of regularly ending the program by reaching the - // end of main()). + // the point of this test is to see whether clean-up of the communication resources is done properly without calling + // any destructors (which is what happens when you call _Exit() instead of regularly ending the program by reaching + // the end of main()). struct sigaction sa; @@ -75,16 +63,6 @@ TEST(TestMPMessenger, ConnectionsManualExit) } } RooFit::MultiProcess::Messenger messenger(pm); - if (pm.is_master()) { - // more SIGCHLD handling - sigset_t sigmask; - sigemptyset(&sigmask); - sigaddset(&sigmask, SIGCHLD); - int rc = sigprocmask(SIG_BLOCK, &sigmask, &messenger.ppoll_sigmask); - if (rc < 0) { - throw std::runtime_error("sigprocmask failed in TestMPMessenger.Connections"); - } - } messenger.test_connections(pm); if (!pm.is_master()) { // just wait until we get terminated @@ -95,8 +73,6 @@ TEST(TestMPMessenger, ConnectionsManualExit) pm.terminate(); } if (pm.is_master()) { - // clean up signal management modifications - sigprocmask(SIG_SETMASK, &messenger.ppoll_sigmask, nullptr); sa.sa_handler = SIG_DFL; if (sigaction(SIGCHLD, &sa, nullptr) < 0) { std::perror("sigaction failed"); diff --git a/roofit/roofitZMQ/CMakeLists.txt b/roofit/roofitZMQ/CMakeLists.txt deleted file mode 100644 index 1c1e3279cf517..0000000000000 --- a/roofit/roofitZMQ/CMakeLists.txt +++ /dev/null @@ -1,41 +0,0 @@ -############################################################################ -# CMakeLists.txt file for building ROOT roofitcore/ZMQ package -# @author Patrick Bos, Netherlands eScience Center -############################################################################ - -ROOT_LINKER_LIBRARY(RooFitZMQ - src/ZeroMQSvc.cpp - src/ZeroMQPoller.cpp - src/functions.cpp - src/ppoll.cpp - ) - -target_link_libraries(RooFitZMQ PUBLIC libzmq cppzmq) -set(RooFitZMQ_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/res) -target_include_directories(RooFitZMQ - PRIVATE ${RooFitZMQ_INCLUDE_DIR} - INTERFACE $) - -# zmq_ppoll is still in the draft API, and RooFitZMQ relies on it -target_compile_definitions(RooFitZMQ PUBLIC ZMQ_BUILD_DRAFT_API) -# to avoid leaking symbols -target_compile_definitions(RooFitZMQ PUBLIC ZMQ_NO_EXPORT) - -if(NOT CMAKE_VERSION VERSION_LESS "3.23.0") - target_sources( - RooFitZMQ - PRIVATE - FILE_SET private_header_files - TYPE HEADERS - BASE_DIRS res/ - FILES - res/RooFit_ZMQ/ppoll.h - res/RooFit_ZMQ/Utility.h - res/RooFit_ZMQ/ZeroMQSvc.h - res/RooFit_ZMQ/ZeroMQPoller.h - res/RooFit_ZMQ/functions.h - ) -endif() - - -ROOT_ADD_TEST_SUBDIRECTORY(test) diff --git a/roofit/roofitZMQ/res/RooFit_ZMQ/Utility.h b/roofit/roofitZMQ/res/RooFit_ZMQ/Utility.h deleted file mode 100644 index c2196865cbd75..0000000000000 --- a/roofit/roofitZMQ/res/RooFit_ZMQ/Utility.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Project: RooFit - * Authors: - * RA, Roel Aaij, NIKHEF - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2021, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#ifndef SERIALIZE_UTILITY_H -#define SERIALIZE_UTILITY_H 1 - -#include - -namespace ZMQ { -namespace Detail { - -template -using simple_object = std::is_trivially_copyable; - -// is trivial -template -struct is_trivial - : std::conditional::type>::value, std::true_type, std::false_type>::type { -}; - -} // namespace Detail -} // namespace ZMQ - -#endif // SERIALIZE_UTILITY_H diff --git a/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQPoller.h b/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQPoller.h deleted file mode 100644 index fda44dc3b913a..0000000000000 --- a/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQPoller.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Project: RooFit - * Authors: - * RA, Roel Aaij, NIKHEF - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2021, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#ifndef ZEROMQPOLLER_H -#define ZEROMQPOLLER_H 1 - -#include "RooFit_ZMQ/ZeroMQSvc.h" -#include "RooFit_ZMQ/functions.h" -#include - -#include -#include -#include -#include - -class ZeroMQPoller { -public: - using entry_t = std::tuple; - // The key is what zmq::socket_t stores inside, and what goes into - // pollitem_t through zmq::socket_t's conversion to void* operator - using sockets_t = std::unordered_map; - - using fd_entry_t = std::tuple; - using fds_t = std::unordered_map; - - using free_t = std::deque; - - ZeroMQPoller() = default; - - std::vector> poll(int timeo = -1); - std::vector> ppoll(int timeo, const sigset_t *sigmask); - - size_t size() const; - - size_t register_socket(zmq::socket_t &socket, zmq::event_flags type); - size_t register_socket(int fd, zmq::event_flags type); - - size_t unregister_socket(zmq::socket_t &socket); - size_t unregister_socket(int fd); - -private: - // Vector of (socket, flags) - std::vector m_items; - sockets_t m_sockets; - fds_t m_fds; - - // free slots in items - free_t m_free; -}; - -#endif // ZEROMQPOLLER_H diff --git a/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQSvc.h b/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQSvc.h deleted file mode 100644 index 4a6b0f6e48bb5..0000000000000 --- a/roofit/roofitZMQ/res/RooFit_ZMQ/ZeroMQSvc.h +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Project: RooFit - * Authors: - * RA, Roel Aaij, NIKHEF - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2021, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#ifndef ZEROMQ_IZEROMQSVC_H -#define ZEROMQ_IZEROMQSVC_H 1 - -#include -#include "RooFit_ZMQ/Utility.h" -#include "RooFit_ZMQ/functions.h" - -#include -#include -#include -#include -#include -#include // std::cerr -#include -#include - -// debugging -#include // getpid - -namespace ZMQ { - -struct TimeOutException : std::exception { - TimeOutException() = default; -}; - -struct MoreException : std::exception { - MoreException() = default; -}; - -} // namespace ZMQ - -template -struct ZmqLingeringSocketPtrDeleter { - void operator()(zmq::socket_t *socket) - { - int tries = 0; - int max_tries = 3; - while (true) { - try { - // the actual work this function should do, plus the delete socket below: - if (socket) - socket->set(zmq::sockopt::linger, PERIOD); - break; - } catch (zmq::error_t &e) { - if (++tries == max_tries || e.num() == EINVAL || e.num() == ETERM || - e.num() == ENOTSOCK // not recoverable from here - ) { - std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; - throw; - } - std::cerr << "RETRY " << tries << "/" << (max_tries - 1) - << " in ZmqLingeringSocketPtrDeleter: call interrupted (errno: " << e.num() << ")\n"; - } - } - - delete socket; - } -}; - -template -using ZmqLingeringSocketPtr = std::unique_ptr>; - -// We retry send and receive only on EINTR, all other errors are either fatal, or can only -// be handled at the caller. -template -auto retry_send(zmq::socket_t &socket, int max_tries, args_t... args) -> decltype(socket.send(args...)) -{ - int tries = 0; - while (true) { - try { - // the actual work this function should do, all the rest is error handling: - return socket.send(args...); - } catch (zmq::error_t &e) { - if (++tries == max_tries || e.num() != EINTR // only recoverable error - ) { - throw; - } - std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::send (retry_send) on pid " - << getpid() << ": " << e.what() << ")\n"; - } - } -} - -template -auto retry_recv(zmq::socket_t &socket, int max_tries, args_t... args) -> decltype(socket.recv(args...)) -{ - int tries = 0; - while (true) { - try { - // the actual work this function should do, all the rest is error handling: - return socket.recv(args...); - } catch (zmq::error_t &e) { - if (++tries == max_tries || e.num() != EINTR // only recoverable error - ) { - throw; - } - std::cerr << "RETRY " << tries << "/" << (max_tries - 1) << " in ZeroMQSvc::recv (retry_recv) on pid " - << getpid() << ": " << e.what() << ")\n"; - } - } -} - -class ZeroMQSvc { - // Note on error handling: - // Creating message_t can throw, but only when memory ran out (errno ENOMEM), - // and that is something only the caller can fix, so we don't catch it here. - -public: - enum Encoding { Text = 0, Binary }; - - Encoding encoding() const; - void setEncoding(const Encoding &e); - zmq::context_t &context() const; - zmq::socket_t socket(zmq::socket_type type) const; - zmq::socket_t *socket_ptr(zmq::socket_type type) const; - void close_context() const; - - /// decode message with ZMQ, POD version - template ::value && ZMQ::Detail::is_trivial::value, T>::type - * = nullptr> - T decode(const zmq::message_t &msg) const - { - T object; - memcpy(&object, msg.data(), msg.size()); - return object; - } - - /// decode ZMQ message, string version - template ::value, T>::type * = nullptr> - std::string decode(const zmq::message_t &msg) const - { - std::string r(msg.size() + 1, char{}); - r.assign(static_cast(msg.data()), msg.size()); - return r; - } - - /// receive message with ZMQ, general version - // FIXME: what to do with flags=nullptr.... more is a pointer, that might prevent conversion - template ::value), T>::type * = nullptr> - T receive(zmq::socket_t &socket, zmq::recv_flags flags = zmq::recv_flags::none, bool *more = nullptr) const - { - // receive message - zmq::message_t msg; - auto recv_result = retry_recv(socket, 2, std::ref(msg), flags); - if (!recv_result) { - throw ZMQ::TimeOutException{}; - } - if (more) - *more = msg.more(); - - // decode message - return decode(msg); - } - - /// receive message with ZMQ - template ::value, T>::type * = nullptr> - T receive(zmq::socket_t &socket, zmq::recv_flags flags = zmq::recv_flags::none, bool *more = nullptr) const - { - // receive message - zmq::message_t msg; - auto recv_result = retry_recv(socket, 2, std::ref(msg), flags); - if (!recv_result) { - throw ZMQ::TimeOutException{}; - } - if (more) - *more = msg.more(); - return msg; - } - - /// encode message to ZMQ - template ::value && ZMQ::Detail::is_trivial::value, T>::type - * = nullptr> - zmq::message_t encode(const T &item, std::function sizeFun = ZMQ::defaultSizeOf) const - { - size_t s = sizeFun(item); - zmq::message_t msg{s}; - memcpy((void *)msg.data(), &item, s); - return msg; - } - - zmq::message_t encode(const char *item) const; - zmq::message_t encode(const std::string &item) const; - - /// Send message with ZMQ - template ::value, T>::type * = nullptr> - zmq::send_result_t send(zmq::socket_t &socket, const T &item, zmq::send_flags flags = zmq::send_flags::none) const - { - return retry_send(socket, 1, encode(item), flags); - } - - zmq::send_result_t - send(zmq::socket_t &socket, const char *item, zmq::send_flags flags = zmq::send_flags::none) const; - zmq::send_result_t - send(zmq::socket_t &socket, zmq::message_t &msg, zmq::send_flags flags = zmq::send_flags::none) const; - zmq::send_result_t - send(zmq::socket_t &socket, zmq::message_t &&msg, zmq::send_flags flags = zmq::send_flags::none) const; - -private: - Encoding m_enc = Text; - mutable zmq::context_t *m_context = nullptr; -}; - -ZeroMQSvc &zmqSvc(); - -#endif // ZEROMQ_IZEROMQSVC_H diff --git a/roofit/roofitZMQ/res/RooFit_ZMQ/functions.h b/roofit/roofitZMQ/res/RooFit_ZMQ/functions.h deleted file mode 100644 index 2c4175ca27be1..0000000000000 --- a/roofit/roofitZMQ/res/RooFit_ZMQ/functions.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Project: RooFit - * Authors: - * RA, Roel Aaij, NIKHEF - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2021, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#ifndef ZEROMQ_FUNCTIONS_H -#define ZEROMQ_FUNCTIONS_H 1 - -#include // std::size_t - -namespace ZMQ { - -template -std::size_t defaultSizeOf(const T &) -{ - return sizeof(T); -} - -std::size_t stringLength(const char &cs); - -} // namespace ZMQ - -#endif // ZEROMQ_FUNCTIONS_H diff --git a/roofit/roofitZMQ/res/RooFit_ZMQ/ppoll.h b/roofit/roofitZMQ/res/RooFit_ZMQ/ppoll.h deleted file mode 100644 index 4eef9da3c9039..0000000000000 --- a/roofit/roofitZMQ/res/RooFit_ZMQ/ppoll.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Project: RooFit - * Authors: - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2021, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#ifndef ROOT_ROOFIT_ZMQ_ppoll -#define ROOT_ROOFIT_ZMQ_ppoll - -#include -#include - -namespace ZMQ { - -int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t *sigmask_); -int ppoll(std::vector &items, long timeout_, const sigset_t *sigmask_); -class ppoll_error_t : public zmq::error_t { -}; - -} // namespace ZMQ - -#endif // ROOT_ROOFIT_ZMQ_ppoll diff --git a/roofit/roofitZMQ/src/ZeroMQPoller.cpp b/roofit/roofitZMQ/src/ZeroMQPoller.cpp deleted file mode 100644 index 5d5e550c9e33e..0000000000000 --- a/roofit/roofitZMQ/src/ZeroMQPoller.cpp +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Project: RooFit - * Authors: - * RA, Roel Aaij, NIKHEF - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2021, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#include "RooFit_ZMQ/ZeroMQPoller.h" - -#include "RooFit_ZMQ/ppoll.h" -#include - -/** \class ZeroMQPoller - * \brief Wrapper class for polling ZeroMQ sockets - * - * This class simplifies calls to poll or ppoll ZeroMQ sockets. It stores the - * list of sockets to be polled, which means they don't have to be separately - * carried around by the user. It also parses output and returns an easily - * digestible vector of events. - */ - -/** - * \brief Poll the sockets - * - * \param[in] timeo Timeout in milliseconds. 0 means return immediately. -1 means wait for an event indefinitely. - * \return A vector of pairs of index and flags; index is the index of the registered fd or socket and flags are 0 (no - * events), ZMQ_POLLIN or ZMQ_POLLOUT. - * - * \note This function can throw (from inside zmq::poll), so wrap in try-catch! - */ -std::vector> ZeroMQPoller::poll(int timeo) -{ - std::vector> r; - if (m_items.empty()) { - throw std::runtime_error("No sockets registered"); - } - int n = 0; - while (true) { - try { - n = zmq::poll(m_items, std::chrono::milliseconds{timeo}); - if (n == 0) - return r; - break; - } catch (const zmq::error_t &e) { - std::cerr << "in ZeroMQPoller::poll on PID " << getpid() << ": " << e.what() << std::endl; - if (e.num() != EINTR) { - throw; - } - } - } - // TODO: replace this with ranges::v3::zip - for (size_t i = 0; i < m_items.size(); ++i) { - void *socket = m_items[i].socket; - size_t index = 0; - zmq::event_flags flags = zmq::event_flags::none; - if (socket == nullptr) { - // an fd was registered - std::tie(index, flags) = m_fds[m_items[i].fd]; - } else { - // a socket was registered - const zmq::socket_t *s; - std::tie(index, flags, s) = m_sockets[socket]; - } - if (m_items[i].revents & short(flags)) { - r.emplace_back(index, flags); - } - } - return r; -} - -/** - * \brief Poll the sockets with ppoll - * - * By polling with ppoll instead of poll, one can pass along a signal mask to - * handle POSIX signals properly. See the zmq_ppoll documentation for examples - * of when this is useful: http://api.zeromq.org/ - * - * \param[in] timeo Timeout in milliseconds. 0 means return immediately. -1 means wait for an event indefinitely. - * \param[in] sigmask A non-nullptr pointer to a signal mask must be constructed and passed to 'sigmask'. See the man page - * of sigprocmask(2) for more details on this. \return A vector of pairs of index and flags; index is the index of the - * registered fd or socket and flags are 0 (no events), ZMQ_POLLIN or ZMQ_POLLOUT. - * - * \note This function can throw (from inside ZMQ::ppoll), so wrap in try-catch! - */ -std::vector> ZeroMQPoller::ppoll(int timeo, const sigset_t *sigmask) -{ - if (m_items.empty()) { - throw std::runtime_error("No sockets registered"); - } - - std::vector> r; - - auto n = ZMQ::ppoll(m_items, timeo, sigmask); - if (n == 0) - return r; - - for (auto &m_item : m_items) { - size_t index = 0; - zmq::event_flags flags = zmq::event_flags::none; - if (m_item.socket == nullptr) { - // an fd was registered - std::tie(index, flags) = m_fds[m_item.fd]; - } else { - // a socket was registered - const zmq::socket_t *s; - std::tie(index, flags, s) = m_sockets[m_item.socket]; - } - if (m_item.revents & short(flags)) { - r.emplace_back(index, flags); - } - } - return r; -} - -size_t ZeroMQPoller::size() const -{ - return m_items.size(); -} - -/** - * \brief Register socket to poll - * - * Adds the socket to the internal list of sockets to poll. - * - * \param[in] socket Socket to register. - * \param[in] type Type of events to poll for. Can be ZMQ_POLLIN, ZMQ_POLLOUT or a bit-wise combination of the two. - * \return The index of the socket in the poller's internal list. Can be used to match with indices returned from - * (p)poll. - */ -size_t ZeroMQPoller::register_socket(zmq::socket_t &socket, zmq::event_flags type) -{ - zmq::socket_t *s = &socket; - auto it = m_sockets.find(s); - if (it != m_sockets.end()) { - return std::get<0>(it->second); - } - size_t index = m_free.empty() ? m_items.size() : m_free.front(); - if (!m_free.empty()) - m_free.pop_front(); - // NOTE: this uses the conversion-to-void* operator of - // zmq::socket_t, which returns the wrapped object - m_items.push_back({socket, 0, static_cast(type), 0}); - - // We need to lookup by the pointer to the object wrapped by zmq::socket_t - m_sockets.emplace(m_items.back().socket, std::make_tuple(index, type, s)); - return index; -} - -/** - * \brief Register socket to poll - * - * Adds the socket to the internal list of sockets to poll. - * - * \param[in] fd File descriptor of socket to register. - * \param[in] type Type of events to poll for. Can be ZMQ_POLLIN, ZMQ_POLLOUT or a bit-wise combination of the two. - * \return The index of the socket in the poller's internal list. Can be used to match with indices returned from - * (p)poll. - */ -size_t ZeroMQPoller::register_socket(int fd, zmq::event_flags type) -{ - auto it = m_fds.find(fd); - if (it != m_fds.end()) { - return std::get<0>(it->second); - } - size_t index = m_free.empty() ? m_items.size() : m_free.front(); - if (!m_free.empty()) - m_free.pop_front(); - // NOTE: this uses the conversion-to-void* operator of - // zmq::socket_t, which returns the wrapped object - m_items.push_back({nullptr, fd, static_cast(type), 0}); - - // We need to lookup by the pointer to the object wrapped by zmq::socket_t - m_fds.emplace(fd, std::make_tuple(index, type)); - return index; -} - -/** - * \brief Unregister socket from poller - * - * Removes the socket from the internal list of sockets to poll. - * - * \param[in] socket Socket to unregister. - * \return The index of the socket in the poller's internal list before removal. - */ -size_t ZeroMQPoller::unregister_socket(zmq::socket_t &socket) -{ - if (!m_sockets.count(socket.operator void *())) { - throw std::out_of_range("Socket is not registered"); - } - // Remove from m_sockets - // Can't search by the key of m_sockets, as that is the wrapped - // object, but have to use the pointer to the wrapper - // (zmq::socket_t) - auto it = std::find_if(begin(m_sockets), end(m_sockets), [&socket](const decltype(m_sockets)::value_type &entry) { - return &socket == std::get<2>(entry.second); - }); - auto index = std::get<0>(it->second); - m_free.push_back(index); - void *it_first = it->first; - m_sockets.erase(it); - - // Remove from m_items - auto found = std::find_if(begin(m_items), end(m_items), - [&it_first](const zmq::pollitem_t &item) { return it_first == item.socket; }); - assert(found != end(m_items)); - m_items.erase(found); - - return index; -} - -/** - * \brief Unregister socket from poller - * - * Removes the socket from the internal list of sockets to poll. - * - * \param[in] fd File descriptor of socket to unregister. - * \return The index of the socket in the poller's internal list before removal. - */ -size_t ZeroMQPoller::unregister_socket(int fd) -{ - if (!m_fds.count(fd)) { - throw std::out_of_range("fileno is not registered"); - } - // Remove from m_fds - auto it = m_fds.find(fd); - auto index = std::get<0>(it->second); - m_free.push_back(index); - int it_first = it->first; - m_fds.erase(it); - - // Remove from m_items - auto found = std::find_if(begin(m_items), end(m_items), - [&it_first](const zmq::pollitem_t &item) { return it_first == item.fd; }); - assert(found != end(m_items)); - m_items.erase(found); - - return index; -} diff --git a/roofit/roofitZMQ/src/ZeroMQSvc.cpp b/roofit/roofitZMQ/src/ZeroMQSvc.cpp deleted file mode 100644 index 63c66bb0a6c60..0000000000000 --- a/roofit/roofitZMQ/src/ZeroMQSvc.cpp +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Project: RooFit - * Authors: - * RA, Roel Aaij, NIKHEF - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2021, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#include "RooFit_ZMQ/ZeroMQSvc.h" - -#include // std::ref - -/** \class ZeroMQSvc - * \brief Wrapper class for basic ZeroMQ context and socket management - * - * This singleton class wraps a couple of basic ZeroMQ tasks: - * - * 1. Creating, storing and eventually closing a ZeroMQ context. - * 2. Creating new sockets in the context. - * 3. Sending, receiving, encoding and decoding messages over sockets. - * - * For convenience, it offers a number of template overloads that automatically - * encode all kinds of data types in ZeroMQ message objects. - */ - -/** - * \brief Get singleton object of this class - */ -ZeroMQSvc &zmqSvc() -{ - static std::unique_ptr svc; - if (!svc) { - svc = std::make_unique(); - } - return *svc; -} - -ZeroMQSvc::Encoding ZeroMQSvc::encoding() const -{ - return m_enc; -} - -/** - * \brief Set encoding mode - * - * \param[in] e Encoding mode; either Text or Binary. - */ -void ZeroMQSvc::setEncoding(const ZeroMQSvc::Encoding &e) -{ - m_enc = e; -} - -/** - * \brief Get context - * - * Creates a context if it has not yet been created and returns a reference to it. - */ -zmq::context_t &ZeroMQSvc::context() const -{ - if (!m_context) { - try { - m_context = new zmq::context_t; - } catch (zmq::error_t &e) { - std::cerr << "ERROR: Creating ZeroMQ context failed. This only happens when PGM initialization failed or when " - "a nullptr was returned from zmq_ctx_new because the created context was invalid. Contact ZMQ " - "experts when this happens, because it shouldn't.\n"; - throw; - } - } - return *m_context; -} - -/** - * \brief Create and return a new socket - * - * \param[in] type Type of the socket. See http://api.zeromq.org/master:zmq-socket for possible values. - * \return The socket object. - */ -zmq::socket_t ZeroMQSvc::socket(zmq::socket_type type) const -{ - try { - // the actual work this function should do, all the rest is error handling: - return zmq::socket_t{context(), type}; - } catch (zmq::error_t &e) { - // all zmq errors not recoverable from here, only at call site - std::cerr << "ERROR in ZeroMQSvc::socket: " << e.what() << " (errno: " << e.num() << ")\n"; - throw; - } -} - -/** - * \brief Create and return a new socket by pointer - * - * \param[in] type Type of the socket. See http://api.zeromq.org/master:zmq-socket for possible values. - * \return A raw pointer to the socket object. Note: the caller must take ownership! - */ -zmq::socket_t *ZeroMQSvc::socket_ptr(zmq::socket_type type) const -{ - try { - // the actual work this function should do, all the rest is error handling: - return new zmq::socket_t(context(), type); - } catch (zmq::error_t &e) { - // all zmq errors not recoverable from here, only at call site - std::cerr << "ERROR in ZeroMQSvc::socket_ptr: " << e.what() << " (errno: " << e.num() << ")\n"; - throw; - } -} - -void ZeroMQSvc::close_context() const -{ - if (m_context) { - delete m_context; - m_context = nullptr; - } -} - -/** - * \fn zmq::message_t ZeroMQSvc::encode(const char *item) const - * \brief Encode string as a ZeroMQ message object - * - * \param[in] item String. - */ -zmq::message_t ZeroMQSvc::encode(const char *item) const -{ - std::function fun = ZMQ::stringLength; - return encode(*item, fun); -} - -/** - * \overload zmq::message_t ZeroMQSvc::encode(const std::string &item) const - */ -zmq::message_t ZeroMQSvc::encode(const std::string &item) const -{ - return encode(item.c_str()); -} - -/** - * \fn bool ZeroMQSvc::send(zmq::socket_t &socket, const char *item, zmq::send_flags flags) const - * \brief Send message over a socket - * - * \param[in] socket Socket. - * \param[in] item Message to send over. - * \param[in] flags Flags to send. See http://api.zeromq.org/master:zmq-send for possible flags and the cppzmq API for - * the type-safe equivalents in the zmq::send_flags enum class. - * \return An optional of type zmq::send_result_t that contains the number of bytes sent if successful, and is empty if - * EAGAIN was received, which probably means you should try again. - */ -zmq::send_result_t ZeroMQSvc::send(zmq::socket_t &socket, const char *item, zmq::send_flags flags) const -{ - return retry_send(socket, 2, encode(item), flags); -} - -/** - * \overload zmq::send_result_t ZeroMQSvc::send(zmq::socket_t &socket, zmq::message_t &msg, zmq::send_flags flags) const - */ -zmq::send_result_t ZeroMQSvc::send(zmq::socket_t &socket, zmq::message_t &msg, zmq::send_flags flags) const -{ - return retry_send(socket, 2, std::ref(msg), flags); -} - -/** - * \overload zmq::send_result_t ZeroMQSvc::send(zmq::socket_t &socket, zmq::message_t &&msg, zmq::send_flags flags) - * const - */ -zmq::send_result_t ZeroMQSvc::send(zmq::socket_t &socket, zmq::message_t &&msg, zmq::send_flags flags) const -{ - return retry_send(socket, 2, std::move(msg), flags); -} diff --git a/roofit/roofitZMQ/src/functions.cpp b/roofit/roofitZMQ/src/functions.cpp deleted file mode 100644 index 6f7a0f0d72f23..0000000000000 --- a/roofit/roofitZMQ/src/functions.cpp +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Project: RooFit - * Authors: - * RA, Roel Aaij, NIKHEF - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2021, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#include "RooFit_ZMQ/functions.h" - -#include - -namespace ZMQ { -std::size_t stringLength(const char &cs) -{ - return strlen(&cs); -} -} // namespace ZMQ diff --git a/roofit/roofitZMQ/src/ppoll.cpp b/roofit/roofitZMQ/src/ppoll.cpp deleted file mode 100644 index c6b4c75d346b3..0000000000000 --- a/roofit/roofitZMQ/src/ppoll.cpp +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Project: RooFit - * Authors: - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2021, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#include "RooFit_ZMQ/ppoll.h" - -namespace ZMQ { - -/// Wrapper around zmq_ppoll -/// This function can throw, so wrap in try-catch! -int ppoll(zmq_pollitem_t *items_, size_t nitems_, long timeout_, const sigset_t *sigmask_) -{ - int rc = zmq_ppoll(items_, static_cast(nitems_), timeout_, sigmask_); - if (rc < 0) - throw ppoll_error_t(); - return rc; -} - -/// Wrapper around zmq_ppoll -/// This function can throw, so wrap in try-catch! -int ppoll(std::vector &items, long timeout_, const sigset_t *sigmask_) -{ - return ppoll(items.data(), items.size(), timeout_, sigmask_); -} - -} // namespace ZMQ diff --git a/roofit/roofitZMQ/test/CMakeLists.txt b/roofit/roofitZMQ/test/CMakeLists.txt deleted file mode 100644 index ade7488ea80a0..0000000000000 --- a/roofit/roofitZMQ/test/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -ROOT_ADD_GTEST(test_RooFitZMQ test_ZMQ.cpp LIBRARIES RooFitZMQ Core) -ROOT_ADD_GTEST(test_RooFitZMQ_polling test_polling.cxx LIBRARIES RooFitZMQ Core) -ROOT_ADD_GTEST(test_RooFitZMQ_HWM test_HWM.cxx LIBRARIES RooFitZMQ Core) -ROOT_ADD_GTEST(test_RooFitZMQ_load_balancing test_ZMQ_load_balancing.cxx LIBRARIES RooFitZMQ Core) -ROOT_ADD_GTEST(test_RooFitZMQ_mkstemp test_mkstemp.cxx LIBRARIES RooFitZMQ Core) diff --git a/roofit/roofitZMQ/test/test_HWM.cxx b/roofit/roofitZMQ/test/test_HWM.cxx deleted file mode 100644 index a84d56e3d3f23..0000000000000 --- a/roofit/roofitZMQ/test/test_HWM.cxx +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Project: RooFit - * Authors: - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2021, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#include "RooFit_ZMQ/ZeroMQSvc.h" - -#include - -#include "gtest/gtest.h" - -#include -#include // fork, usleep - -class HighWaterMarkTest : public ::testing::Test { -protected: - void SetUp() override - { - do { - child_pid = fork(); - } while (child_pid == -1); // retry if fork fails - - std::string tmpPath = gSystem->TempDirectory(); - const std::string ipc = "ipc://" + std::string{tmpPath} + "/roofit_ZMQ_test_fork_polling_P2C.ipc"; - - if (child_pid > 0) { // parent - pusher.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); - if (set_hwm) { - EXPECT_EQ(zmq_setsockopt(*pusher, ZMQ_SNDHWM, &hwm, sizeof hwm), 0); - } - pusher->bind(ipc); - } else { // child - puller.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); - if (set_hwm) { - EXPECT_EQ(zmq_setsockopt(*puller, ZMQ_RCVHWM, &hwm, sizeof hwm), 0); - } - puller->connect(ipc); - } - } - - void TearDown() override - { - if (child_pid > 0) { // parent - // wait for child - int status = 0; - pid_t pid; - do { - pid = waitpid(child_pid, &status, 0); - } while (-1 == pid && EINTR == errno); // retry on interrupted system call - if (0 != status) { - if (WIFEXITED(status)) { - printf("exited, status=%d\n", WEXITSTATUS(status)); - } else if (WIFSIGNALED(status)) { - printf("killed by signal %d\n", WTERMSIG(status)); - } else if (WIFSTOPPED(status)) { - printf("stopped by signal %d\n", WSTOPSIG(status)); - } else if (WIFCONTINUED(status)) { - printf("continued\n"); - } - } - if (-1 == pid) { - throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); - } - pusher.reset(); - zmqSvc().close_context(); - } else { // child - puller.reset(); - zmqSvc().close_context(); - _Exit(0); - } - } - - void run_test() - { - std::size_t max_sends = 2000; - - if (child_pid > 0) { // parent - // start test - for (std::size_t ix = 0; ix < max_sends; ++ix) { - zmqSvc().send(*pusher, 0.1f); - zmqSvc().send(*pusher, 1); - zmqSvc().send(*pusher, true); - if (ix % 100 == 0) { - printf("parent at ix = %zu\n", ix); - } - } - } else { // child - // wait a few seconds before reading to allow the parent to overflow the HWM - printf("child waiting for 2 seconds...\n"); - sleep(2); - printf("child starts receiving\n"); - - for (std::size_t ix = 0; ix < max_sends; ++ix) { - zmqSvc().receive(*puller); - zmqSvc().receive(*puller); - zmqSvc().receive(*puller); - if (ix % 100 == 0) { - printf("child at ix = %zu\n", ix); - } - } - } - } - - pid_t child_pid{0}; - ZmqLingeringSocketPtr<> pusher, puller; - bool set_hwm = false; - int hwm = 0; -}; - -TEST_F(HighWaterMarkTest, demonstrateHittingDefaultHWM) -{ - run_test(); -} - -class HighWaterMarkZeroTest : public HighWaterMarkTest { - void SetUp() override - { - set_hwm = true; - hwm = 0; - HighWaterMarkTest::SetUp(); - } -}; - -TEST_F(HighWaterMarkZeroTest, zeroHWM) -{ - run_test(); -} - -class HighWaterMark2kTest : public HighWaterMarkTest { - void SetUp() override - { - set_hwm = true; - hwm = 2000; - HighWaterMarkTest::SetUp(); - } -}; - -TEST_F(HighWaterMark2kTest, HWM2k) -{ - run_test(); -} diff --git a/roofit/roofitZMQ/test/test_ZMQ.cpp b/roofit/roofitZMQ/test/test_ZMQ.cpp deleted file mode 100644 index 6cf764f797b1d..0000000000000 --- a/roofit/roofitZMQ/test/test_ZMQ.cpp +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Project: RooFit - * Authors: - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2021, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#include "RooFit_ZMQ/ZeroMQSvc.h" - -#include - -#include - -#include "gtest/gtest.h" - -#if ROOT_VERSION_CODE < ROOT_VERSION(6, 32, 99) -// Backward compatibility for gtest version < 1.10.0 -#ifndef INSTANTIATE_TEST_SUITE_P -#define INSTANTIATE_TEST_SUITE_P INSTANTIATE_TEST_CASE_P -#endif -#endif - -#include // fork, usleep - -#include - -// N.B.: wait_for_child is identically defined in RooFit::MultiProcess, but we copy it here to reduce module -// interdependencies. It also requires an extra include: -#include // kill, SIGKILL - -int wait_for_child(pid_t child_pid, bool may_throw, int retries_before_killing) -{ - int status = 0; - int patience = retries_before_killing; - pid_t tmp; - do { - if (patience-- < 1) { - ::kill(child_pid, SIGKILL); - } - tmp = waitpid(child_pid, &status, WNOHANG); - } while (tmp == 0 // child has not yet changed state, try again - || (-1 == tmp && EINTR == errno) // retry on interrupted system call - ); - - if (0 != status) { - if (WIFEXITED(status)) { - printf("exited, status=%d\n", WEXITSTATUS(status)); - } else if (WIFSIGNALED(status)) { - if (WTERMSIG(status) != SIGKILL) { // SIGKILL is expected, so needn't be printed - printf("killed by signal %d\n", WTERMSIG(status)); - } - } else if (WIFSTOPPED(status)) { - printf("stopped by signal %d\n", WSTOPSIG(status)); - } else if (WIFCONTINUED(status)) { - printf("continued\n"); - } - } - - if (-1 == tmp && may_throw) { - throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); - } - - return status; -} - -void elaborate_bind(const ZmqLingeringSocketPtr<> &socket, std::string name) -{ - try { - socket->bind(name); - } catch (const zmq::error_t &e) { - if (e.num() == EADDRINUSE) { - std::cerr << "address already in use, retrying bind in 500ms\n"; - usleep(500000); - try { - socket->bind(name); - } catch (const zmq::error_t &e2) { - if (e2.num() == EADDRINUSE) { - std::cerr - << "again: address already in use, aborting; please check whether there are any remaining improperly " - "exited processes (zombies) around or whether some other program is using port 6660\n"; - } - throw; - } - // Sometimes, the socket from the previous test needs some time to close, so - // we introduce a latency here. A more robust and fast approach might be to - // do the following on the bind side: - // 1. first try another port, e.g. increase by one - // 2. if that doesn't work, do the latency and retry the original port - // The connect side then also needs to change, because it doesn't know which - // port the bind side will bind to. The connect side could try connecting to - // both options asynchronously, and then in a loop check both for signs of - // life. If one comes alive, transfer ownership of that pointer to the pointer - // you want to eventually use (`socket`) and that's it. - } else { - throw; - } - } -} - -class AllSocketTypes - : public ::testing::TestWithParam, - std::pair /* socket_names */>> {}; - -TEST_P(AllSocketTypes, forkHandshake) -{ - auto socket_names = std::get<2>(GetParam()); - pid_t child_pid{0}; - do { - child_pid = fork(); - } while (child_pid == -1); // retry if fork fails - - if (child_pid > 0) { // master - ZmqLingeringSocketPtr<> socket; - socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); - elaborate_bind(socket, socket_names.second); - // bind is on the master process to avoid zombie children to hold on to binds - - // start test - zmqSvc().send(*socket, std::string("breaker breaker")); - - auto receipt = zmqSvc().receive(*socket); - - EXPECT_EQ(receipt, 1212); - - socket.reset(); - zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang - - wait_for_child(child_pid, true, 5); - } else { // child - ZmqLingeringSocketPtr<> socket; - socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); - socket->connect(socket_names.first); - - // start test - auto receipt = zmqSvc().receive(*socket); - if (receipt == "breaker breaker") { - zmqSvc().send(*socket, 1212); - } - // take care, don't just use _exit, it will not cleanly destroy context etc! - // if you really need to, at least close and destroy everything properly - socket.reset(); - zmqSvc().close_context(); - _Exit(0); - } -} - -std::string tmpPath = gSystem->TempDirectory(); -std::string ipc{"ipc://" + tmpPath + "/roofit_ZMQ_test_fork.ipc"}; -std::string tcp_server{"tcp://127.0.0.1:6660"}; -std::string tcp_client{"tcp://*:6660"}; -auto socket_name_options = ::testing::Values(std::make_pair(tcp_server, tcp_client), std::make_pair(ipc, ipc)); - -INSTANTIATE_TEST_SUITE_P(REQREP, AllSocketTypes, - ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability - ::testing::Values(std::make_pair(zmq::socket_type::req, - zmq::socket_type::rep)), - socket_name_options)); -INSTANTIATE_TEST_SUITE_P(PAIRPAIR, AllSocketTypes, - ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability - ::testing::Values(std::make_pair(zmq::socket_type::pair, - zmq::socket_type::pair)), - socket_name_options)); - -class AsyncSocketTypes - : public ::testing::TestWithParam< - std::tuple, - std::pair /* socket_names */, bool /* expect_throw */>> {}; - -TEST_P(AsyncSocketTypes, forkMultiSendReceive) -{ - bool expect_throw = std::get<3>(GetParam()); - ZmqLingeringSocketPtr<> socket; - auto socket_names = std::get<2>(GetParam()); - pid_t child_pid{0}; - do { - child_pid = fork(); - } while (child_pid == -1); // retry if fork fails - - if (child_pid > 0) { // master - socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); - elaborate_bind(socket, socket_names.second); - // bind is on the master process to avoid zombie children to hold on to binds - - // start test: send 2 things, receive 1, send 1 more, finish - zmqSvc().send(*socket, std::string("breaker breaker")); - - if (expect_throw) { - EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); - // NOTE: also in case of a throw, be sure to properly close down the connection! - // Otherwise, you may get zombies waiting for a reply. - socket.reset(); - zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang - wait_for_child(child_pid, true, 5); - return; - } else { - EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); - } - - auto receipt = zmqSvc().receive(*socket); - - EXPECT_EQ(receipt, 1212); - - zmqSvc().send(*socket, std::string("kthxbye")); - - socket.reset(); - zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang - - wait_for_child(child_pid, true, 5); - } else { // child - socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); - socket->connect(socket_names.first); - - // start test, receive something - auto receipt1 = zmqSvc().receive(*socket); - std::string receipt2; - if (expect_throw) { - EXPECT_ANY_THROW(zmqSvc().receive(*socket)); - // NOTE: also in case of a throw, be sure to properly close down the connection! - // Otherwise, you may get zombies waiting for a reply. - socket.reset(); - zmqSvc().close_context(); - _Exit(0); - } else { - receipt2 = zmqSvc().receive(*socket); - } - - if (receipt1 == "breaker breaker" && receipt2 == "anybody out there?") { - zmqSvc().send(*socket, 1212); - } - auto receipt3 = zmqSvc().receive(*socket); - if (receipt3 != "kthxbye") { - std::cerr << "did not receive final reply correctly\n"; - } - - // take care, don't just use _exit, it will not cleanly destroy context etc! - // if you really need to, at least close and destroy everything properly - socket.reset(); - zmqSvc().close_context(); - _Exit(0); - } -} - -TEST_P(AsyncSocketTypes, forkIgnoreSomeMessages) -{ - bool expect_throw = std::get<3>(GetParam()); - ZmqLingeringSocketPtr<> socket; - auto socket_names = std::get<2>(GetParam()); - pid_t child_pid{0}; - do { - child_pid = fork(); - } while (child_pid == -1); // retry if fork fails - - if (child_pid > 0) { // master - socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).first)); - elaborate_bind(socket, socket_names.second); - // bind is on the master process to avoid zombie children to hold on to binds - - // start test: send 2 things, receive 1, send 1 more, finish - zmqSvc().send(*socket, std::string("breaker breaker")); - - if (expect_throw) { - EXPECT_ANY_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); - // NOTE: also in case of a throw, be sure to properly close down the connection! - // Otherwise, you may get zombies waiting for a reply. - socket.reset(); - zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang - wait_for_child(child_pid, true, 5); - return; - } else { - EXPECT_NO_THROW(zmqSvc().send(*socket, std::string("anybody out there?"))); - } - - auto receipt = zmqSvc().receive(*socket); - - EXPECT_EQ(receipt, 1212); - - zmqSvc().send(*socket, std::string("kthxbye")); - - socket.reset(); - zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang - - wait_for_child(child_pid, true, 5); - } else { // child - socket.reset(zmqSvc().socket_ptr(std::get<1>(GetParam()).second)); - socket->connect(socket_names.first); - - // start test, receive first thing - auto receipt = zmqSvc().receive(*socket); - if (receipt == "breaker breaker") { - zmqSvc().send(*socket, 1212); - } - - // ignore the rest of the sent messages, but give the other end a second to - // actually send its stuff, instead of hanging in retry_send because the - // connection has died; a better solution would be if retry_send (in - // ZeroMQSvc::send) had a callback mechanism that could be used to break - // out when a child has died, but ok - sleep(1); - - // take care, don't just use _exit, it will not cleanly destroy context etc! - // if you really need to, at least close and destroy everything properly - socket.reset(); - zmqSvc().close_context(); - _Exit(0); - } -} - -INSTANTIATE_TEST_SUITE_P(PAIRPAIR, AsyncSocketTypes, - ::testing::Combine(::testing::Range(0, 10), // repeat to probe connection stability - ::testing::Values(std::make_pair(zmq::socket_type::pair, - zmq::socket_type::pair)), - socket_name_options, - ::testing::Values(false) // don't expect throw - )); - -INSTANTIATE_TEST_SUITE_P(REQREP, AsyncSocketTypes, - ::testing::Combine(::testing::Values(0), // no repeats, we only care about the throw - ::testing::Values(std::make_pair(zmq::socket_type::req, - zmq::socket_type::rep)), - socket_name_options, - ::testing::Values(true) // expect throw - )); diff --git a/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx b/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx deleted file mode 100644 index e75ea22a83139..0000000000000 --- a/roofit/roofitZMQ/test/test_ZMQ_load_balancing.cxx +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Project: RooFit - * Authors: - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2021, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#include "RooFit_ZMQ/ZeroMQSvc.h" -#include "RooFit_ZMQ/ZeroMQPoller.h" - -#include - -#include "gtest/gtest.h" - -#include // fork, usleep - -class ZMQPushPullTest : public ::testing::Test { -protected: - std::size_t N_children = 4; - std::size_t max_sends = 20; - - void SetUp() override - { - for (std::size_t i = 0; i < N_children; ++i) { - do { - child_pid = fork(); - } while (child_pid == -1); // retry if fork fails - if (child_pid == 0) { // child - child_id = i; - break; - } else { - child_pids.push_back(child_pid); - } - } - - std::string tmpPath = gSystem->TempDirectory(); - - if (child_pid > 0) { // parent - pusher.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); - pusher->bind("ipc://" + tmpPath + "/roofit_ZMQ_test_push_pull_P2C.ipc"); - } else { // child - puller.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); - puller->connect("ipc://" + tmpPath + "/roofit_ZMQ_test_push_pull_P2C.ipc"); - - poller.register_socket(*puller, zmq::event_flags::pollin); - } - } - - void TearDown() override - { - if (child_pid > 0) { // parent - // wait for children - int status = 0; - pid_t pid; - for (pid_t child_pid_i : child_pids) { - do { - pid = waitpid(child_pid_i, &status, 0); - } while (-1 == pid && EINTR == errno); // retry on interrupted system call - if (0 != status) { - if (WIFEXITED(status)) { - printf("exited, status=%d\n", WEXITSTATUS(status)); - } else if (WIFSIGNALED(status)) { - printf("killed by signal %d\n", WTERMSIG(status)); - } else if (WIFSTOPPED(status)) { - printf("stopped by signal %d\n", WSTOPSIG(status)); - } else if (WIFCONTINUED(status)) { - printf("continued\n"); - } - } - if (-1 == pid) { - throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); - } - } - pusher.reset(); - zmqSvc().close_context(); - } else { // child - puller.reset(); - zmqSvc().close_context(); - _Exit(0); - } - } - - void run_parent() - { - // start test - usleep(1000); // wait a second so that all pull sockets are connected for round-robin distribution - // if you don't wait a second above, the push socket will "round-robin" all the messages to just one or two - // connected sockets - for (std::size_t ix = 0; ix < max_sends; ++ix) { - zmqSvc().send(*pusher, 0); - } - for (std::size_t ix = 0; ix < N_children; ++ix) { - // end by sending some 1's to all children, to let them know the sending is over - zmqSvc().send(*pusher, 1); - } - } - - void run_child() - { - std::size_t count = 0; - for (std::size_t ix = 0; ix < max_sends; ++ix) { - auto r = poller.poll(2000); - if (r.empty()) { - printf("poller of child %d timed out after 2 seconds\n", child_id); - break; - } - auto value = zmqSvc().receive(*puller, zmq::recv_flags::dontwait); - usleep(200); // "do some work" - printf("value on child %d: %d\n", child_id, value); - if (value == 1) { - printf("child %d got value %d, done here\n", child_id, value); - break; - } - ++count; - } - printf("child %d got %zu values\n", child_id, count); - } - - pid_t child_pid{0}; - int child_id = -1; - std::vector child_pids; - ZmqLingeringSocketPtr<> pusher, puller; - ZeroMQPoller poller; -}; - -/// This test shows how push-pull is unsuited for load balancing; messages are just sent to the first available pull -/// socket without any dynamic load balancing -TEST_F(ZMQPushPullTest, demoRoundRobin) -{ - if (child_pid > 0) { - run_parent(); - } else { - run_child(); - } -} - -/// This test tries to see whether push-pull can be made to work as a bit of a load balancer, using a low HWM at the -/// receiver -TEST_F(ZMQPushPullTest, demoHWM1LoadBalancing) -{ - if (child_pid > 0) { - run_parent(); - } else { - puller->set(zmq::sockopt::rcvhwm, 1); - run_child(); - } -} diff --git a/roofit/roofitZMQ/test/test_mkstemp.cxx b/roofit/roofitZMQ/test/test_mkstemp.cxx deleted file mode 100644 index 6e2a10947d3cf..0000000000000 --- a/roofit/roofitZMQ/test/test_mkstemp.cxx +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Project: RooFit - * Authors: - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2021, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#include "RooFit_ZMQ/ZeroMQSvc.h" - -#include - -#include "gtest/gtest.h" - -#include // mkstemp -#include // for "legacy" systems - -TEST(BindToTmpFile, mkstemp) -{ - std::string tmpPath = gSystem->TempDirectory(); - - std::string filename = tmpPath + "/roofit_MP_XXXXXX"; - while (mkstemp(const_cast(filename.c_str())) < 0) { - } - EXPECT_NE(filename, tmpPath + "/roofit_MP_XXXXXX"); - auto socket = zmqSvc().socket(zmq::socket_type::push); - std::string address = "ipc://" + filename; - try { - socket.bind(address); - } catch (const zmq::error_t &) { - printf("caught an exception\n"); - } -} diff --git a/roofit/roofitZMQ/test/test_polling.cxx b/roofit/roofitZMQ/test/test_polling.cxx deleted file mode 100644 index 196b60031f43f..0000000000000 --- a/roofit/roofitZMQ/test/test_polling.cxx +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Project: RooFit - * Authors: - * PB, Patrick Bos, Netherlands eScience Center, p.bos@esciencecenter.nl - * - * Copyright (c) 2021, CERN - * - * Redistribution and use in source and binary forms, - * with or without modification, are permitted according to the terms - * listed in LICENSE (http://roofit.sourceforge.net/license.txt) - */ - -#include "RooFit_ZMQ/ZeroMQSvc.h" -#include "RooFit_ZMQ/ZeroMQPoller.h" - -#include - -#include "gtest/gtest.h" - -#include // fork, usleep -#include // signal blocking -#include // strsignal() - -#include - -static volatile sig_atomic_t terminated = 0; - -void handle_sigterm(int signum) -{ - terminated = 1; - std::cout << "handled signal " << strsignal(signum) << " on PID " << getpid() << std::endl; -} - -std::string unique_tmp_ipc_address(const char *filename_template) -{ - assert(strlen(filename_template) < 256); - char filename_template_mutable[256]; - strcpy(filename_template_mutable, filename_template); - while (mkstemp(filename_template_mutable) >= 0) { - } - std::string tmpPath = gSystem->TempDirectory(); - std::stringstream ss; - ss << "ipc://" << tmpPath << "/roofit_" << filename_template_mutable << ".ipc"; - return ss.str(); -} - -TEST(Polling, doublePoll) -{ - auto M2C_address = unique_tmp_ipc_address("ZMQ_test_fork_polling_M2C_XXXXXX"); - auto C2M_address = unique_tmp_ipc_address("ZMQ_test_fork_polling_C2M_XXXXXX"); - pid_t child_pid{0}; - do { - child_pid = fork(); - } while (child_pid == -1); // retry if fork fails - - if (child_pid > 0) { // master - sigset_t sigmask; - sigset_t sigmask_old; - sigemptyset(&sigmask); - sigaddset(&sigmask, SIGCHLD); - sigprocmask(SIG_BLOCK, &sigmask, &sigmask_old); - - ZmqLingeringSocketPtr<> pusher; - ZmqLingeringSocketPtr<> puller; - pusher.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); - pusher->bind(M2C_address); - puller.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); - puller->bind(C2M_address); - - ZeroMQPoller poller1; - ZeroMQPoller poller2; - poller1.register_socket(*puller, zmq::event_flags::pollin); - poller2.register_socket(*puller, zmq::event_flags::pollin); - - // start test - zmqSvc().send(*pusher, std::string("breaker breaker")); - - auto result1a = poller1.poll(-1); - auto result1b = poller1.poll(-1); - auto result2 = poller2.poll(-1); - EXPECT_EQ(result1a.size(), result1b.size()); - EXPECT_EQ(result1a.size(), result2.size()); - - auto receipt = zmqSvc().receive(*puller, zmq::recv_flags::dontwait); - - EXPECT_EQ(receipt, 1212); - - kill(child_pid, SIGTERM); - - pusher.reset(); - puller.reset(); - zmqSvc().close_context(); // if you don't close context in parent process as well, the next repeat will hang - - // wait for child - int status = 0; - pid_t pid; - do { - pid = waitpid(child_pid, &status, 0); - } while (-1 == pid && EINTR == errno); // retry on interrupted system call - - if (0 != status) { - if (WIFEXITED(status)) { - printf("exited, status=%d\n", WEXITSTATUS(status)); - } else if (WIFSIGNALED(status)) { - printf("killed by signal %d\n", WTERMSIG(status)); - } else if (WIFSTOPPED(status)) { - printf("stopped by signal %d\n", WSTOPSIG(status)); - } else if (WIFCONTINUED(status)) { - printf("continued\n"); - } - } - - if (-1 == pid) { - throw std::runtime_error(std::string("waitpid, errno ") + std::to_string(errno)); - } - - sigprocmask(SIG_SETMASK, &sigmask_old, nullptr); - } else { // child - sigset_t sigmask; - sigset_t sigmask_old; - sigemptyset(&sigmask); - sigaddset(&sigmask, SIGTERM); - sigprocmask(SIG_BLOCK, &sigmask, &sigmask_old); - - struct sigaction sa; - memset(&sa, '\0', sizeof(sa)); - sa.sa_handler = handle_sigterm; - - if (sigaction(SIGTERM, &sa, nullptr) < 0) { - std::perror("sigaction failed"); - std::exit(1); - } - - ZmqLingeringSocketPtr<> puller; - ZmqLingeringSocketPtr<> pusher; - puller.reset(zmqSvc().socket_ptr(zmq::socket_type::pull)); - puller->connect(M2C_address); - pusher.reset(zmqSvc().socket_ptr(zmq::socket_type::push)); - pusher->connect(C2M_address); - - ZeroMQPoller poller1; - ZeroMQPoller poller2; - poller1.register_socket(*puller, zmq::event_flags::pollin); - poller2.register_socket(*puller, zmq::event_flags::pollin); - - // start test - auto result1a = poller1.poll(-1); - auto result1b = poller1.poll(-1); - auto result2 = poller2.poll(-1); - EXPECT_EQ(result1a.size(), result1b.size()); - EXPECT_EQ(result1a.size(), result2.size()); - - auto receipt = zmqSvc().receive(*puller, zmq::recv_flags::dontwait); - if (receipt == "breaker breaker") { - zmqSvc().send(*pusher, 1212); - } - // take care, don't just use _exit, it will not cleanly destroy context etc! - // if you really need to, at least close and destroy everything properly - - sigprocmask(SIG_SETMASK, &sigmask_old, nullptr); - - while (!terminated) { - } - - puller.reset(); - pusher.reset(); - zmqSvc().close_context(); - _Exit(0); - } -} diff --git a/roofit/roofitcore/CMakeLists.txt b/roofit/roofitcore/CMakeLists.txt index fba900f3c3025..4e2fe10fcd4b5 100644 --- a/roofit/roofitcore/CMakeLists.txt +++ b/roofit/roofitcore/CMakeLists.txt @@ -9,7 +9,7 @@ # @author Pere Mato, CERN ############################################################################ -if(roofit_multiprocess) +if(NOT WIN32) set(RooFitMPTestStatisticsSources src/TestStatistics/LikelihoodGradientJob.cxx src/TestStatistics/LikelihoodJob.cxx @@ -496,7 +496,7 @@ if(roofit_legacy_eval_backend) target_compile_definitions(RooFitCore PUBLIC ROOFIT_LEGACY_EVAL_BACKEND) endif() -if(roofit_multiprocess) +if(NOT WIN32) target_compile_definitions(RooFitCore PUBLIC ROOFIT_MULTIPROCESS) endif() diff --git a/roofit/roofitcore/src/RooMinimizer.cxx b/roofit/roofitcore/src/RooMinimizer.cxx index 2246a8f3d3529..133b70f7ffc77 100644 --- a/roofit/roofitcore/src/RooMinimizer.cxx +++ b/roofit/roofitcore/src/RooMinimizer.cxx @@ -181,8 +181,7 @@ RooMinimizer::RooMinimizer(RooAbsReal &function, Config const &cfg) : _function{ RooFit::TestStatistics::LikelihoodGradientMode::multiprocess); #else throw std::logic_error( - "Parallel minimization requested, but ROOT was not compiled with multiprocessing enabled, " - "please recompile with -Droofit_multiprocess=ON for parallel evaluation"); + "Parallel minimization requested, but multiprocessing is not supported on this platform"); #endif } else { // modular test statistic non parallel coutW(InputArguments) @@ -355,9 +354,7 @@ int RooMinimizer::minimize(const char *type, const char *alg) #ifdef ROOFIT_MULTIPROCESS addParamsToProcessTimer(); #else - throw std::logic_error("ProcessTimer, but ROOT was not compiled with multiprocessing enabled, " - "please recompile with -Droofit_multiprocess=ON for logging with the " - "ProcessTimer."); + throw std::logic_error("ProcessTimer requested, but multiprocessing is not supported on this platform."); #endif } _fcn->Synchronize(_config.ParamsSettings()); diff --git a/roofit/roofitcore/src/TestStatistics/LikelihoodGradientJob.cxx b/roofit/roofitcore/src/TestStatistics/LikelihoodGradientJob.cxx index 58ae110730382..0e887f1244d22 100644 --- a/roofit/roofitcore/src/TestStatistics/LikelihoodGradientJob.cxx +++ b/roofit/roofitcore/src/TestStatistics/LikelihoodGradientJob.cxx @@ -94,12 +94,12 @@ void LikelihoodGradientJob::evaluate_task(std::size_t task) void LikelihoodGradientJob::send_back_task_result_from_worker(std::size_t task) { task_result_t task_result{id_, task, grad_[task]}; - zmq::message_t message(sizeof(task_result_t)); + RooFit::MultiProcess::Message message(sizeof(task_result_t)); memcpy(message.data(), &task_result, sizeof(task_result_t)); get_manager()->messenger().send_from_worker_to_master(std::move(message)); } -bool LikelihoodGradientJob::receive_task_result_on_master(const zmq::message_t &message) +bool LikelihoodGradientJob::receive_task_result_on_master(const RooFit::MultiProcess::Message &message) { auto result = message.data(); grad_[result->task_id] = result->grad; @@ -115,14 +115,14 @@ bool LikelihoodGradientJob::receive_task_result_on_master(const zmq::message_t & void LikelihoodGradientJob::update_workers_state() { // TODO optimization: only send changed parameters (now sending all) - zmq::message_t gradient_message(grad_.begin(), grad_.end()); - zmq::message_t minuit_internal_x_message(minuit_internal_x_.begin(), minuit_internal_x_.end()); + RooFit::MultiProcess::Message gradient_message(grad_.begin(), grad_.end()); + RooFit::MultiProcess::Message minuit_internal_x_message(minuit_internal_x_.begin(), minuit_internal_x_.end()); double maxFCN = minimizer_->maxFCN(); double fcnOffset = minimizer_->fcnOffset(); ++state_id_; if (shared_offset_.offsets() != offsets_previous_) { - zmq::message_t offsets_message(shared_offset_.offsets().begin(), shared_offset_.offsets().end()); + RooFit::MultiProcess::Message offsets_message(shared_offset_.offsets().begin(), shared_offset_.offsets().end()); get_manager()->messenger().publish_from_master_to_workers( id_, state_id_, isCalculating_, maxFCN, fcnOffset, std::move(gradient_message), std::move(minuit_internal_x_message), std::move(offsets_message)); @@ -157,14 +157,16 @@ void LikelihoodGradientJob::update_state() minimizer_->fcnOffset() = fcnOffset; assert(more); - auto gradient_message = get_manager()->messenger().receive_from_master_on_worker(&more); + auto gradient_message = + get_manager()->messenger().receive_from_master_on_worker(&more); assert(more); auto gradient_message_begin = gradient_message.data(); auto gradient_message_end = gradient_message_begin + gradient_message.size() / sizeof(ROOT::Minuit2::DerivatorElement); std::copy(gradient_message_begin, gradient_message_end, grad_.begin()); - auto minuit_internal_x_message = get_manager()->messenger().receive_from_master_on_worker(&more); + auto minuit_internal_x_message = + get_manager()->messenger().receive_from_master_on_worker(&more); auto minuit_internal_x_message_begin = minuit_internal_x_message.data(); auto minuit_internal_x_message_end = minuit_internal_x_message_begin + minuit_internal_x_message.size() / sizeof(double); @@ -172,11 +174,12 @@ void LikelihoodGradientJob::update_state() if (more) { // offsets also incoming - auto offsets_message = get_manager()->messenger().receive_from_master_on_worker(&more); + auto offsets_message = + get_manager()->messenger().receive_from_master_on_worker(&more); assert(!more); auto offsets_message_begin = offsets_message.data>(); std::size_t N_offsets = offsets_message.size() / sizeof(ROOT::Math::KahanSum); - shared_offset_.offsets().reserve(N_offsets); + shared_offset_.offsets().resize(N_offsets); auto offsets_message_end = offsets_message_begin + N_offsets; std::copy(offsets_message_begin, offsets_message_end, shared_offset_.offsets().begin()); } diff --git a/roofit/roofitcore/src/TestStatistics/LikelihoodGradientJob.h b/roofit/roofitcore/src/TestStatistics/LikelihoodGradientJob.h index b54f61606cdeb..64c0682f996ca 100644 --- a/roofit/roofitcore/src/TestStatistics/LikelihoodGradientJob.h +++ b/roofit/roofitcore/src/TestStatistics/LikelihoodGradientJob.h @@ -65,7 +65,7 @@ class LikelihoodGradientJob : public MultiProcess::Job, public LikelihoodGradien ROOT::Minuit2::DerivatorElement grad; }; void send_back_task_result_from_worker(std::size_t task) override; - bool receive_task_result_on_master(const zmq::message_t &message) override; + bool receive_task_result_on_master(const RooFit::MultiProcess::Message &message) override; void update_workers_state(); void update_workers_state_isCalculating(); diff --git a/roofit/roofitcore/src/TestStatistics/LikelihoodJob.cxx b/roofit/roofitcore/src/TestStatistics/LikelihoodJob.cxx index 6efe297299934..41ccae76283cf 100644 --- a/roofit/roofitcore/src/TestStatistics/LikelihoodJob.cxx +++ b/roofit/roofitcore/src/TestStatistics/LikelihoodJob.cxx @@ -74,7 +74,7 @@ void LikelihoodJob::update_state() case update_state_mode::parameters: { state_id_ = get_manager()->messenger().receive_from_master_on_worker(&more); assert(more); - auto message = get_manager()->messenger().receive_from_master_on_worker(&more); + auto message = get_manager()->messenger().receive_from_master_on_worker(&more); auto message_begin = message.data(); auto message_end = message_begin + message.size() / sizeof(update_state_t); std::vector to_update(message_begin, message_end); @@ -88,7 +88,8 @@ void LikelihoodJob::update_state() if (more) { // offsets also incoming - auto offsets_message = get_manager()->messenger().receive_from_master_on_worker(&more); + auto offsets_message = + get_manager()->messenger().receive_from_master_on_worker(&more); assert(!more); auto offsets_message_begin = offsets_message.data>(); std::size_t N_offsets = offsets_message.size() / sizeof(ROOT::Math::KahanSum); @@ -166,11 +167,12 @@ void LikelihoodJob::updateWorkersParameters() bool update_offsets = isOffsetting() && shared_offset_.offsets() != offsets_previous_; if (!to_update.empty() || update_offsets) { ++state_id_; - zmq::message_t message(to_update.begin(), to_update.end()); + RooFit::MultiProcess::Message message(to_update.begin(), to_update.end()); // always send Job id first! This is used in worker_loop to route the // update_state call to the correct Job. if (update_offsets) { - zmq::message_t offsets_message(shared_offset_.offsets().begin(), shared_offset_.offsets().end()); + RooFit::MultiProcess::Message offsets_message(shared_offset_.offsets().begin(), + shared_offset_.offsets().end()); get_manager()->messenger().publish_from_master_to_workers(id_, update_state_mode::parameters, state_id_, std::move(message), std::move(offsets_message)); offsets_previous_ = shared_offset_.offsets(); @@ -246,12 +248,12 @@ void LikelihoodJob::send_back_task_result_from_worker(std::size_t /*task*/) } task_result_t task_result{id_, result_.Result(), result_.Carry(), numErrors > 0}; - zmq::message_t message(sizeof(task_result_t)); + RooFit::MultiProcess::Message message(sizeof(task_result_t)); memcpy(message.data(), &task_result, sizeof(task_result_t)); get_manager()->messenger().send_from_worker_to_master(std::move(message)); } -bool LikelihoodJob::receive_task_result_on_master(const zmq::message_t &message) +bool LikelihoodJob::receive_task_result_on_master(const RooFit::MultiProcess::Message &message) { auto task_result = message.data(); results_.emplace_back(task_result->value, task_result->carry); diff --git a/roofit/roofitcore/src/TestStatistics/LikelihoodJob.h b/roofit/roofitcore/src/TestStatistics/LikelihoodJob.h index 0a54c543e8a44..5c49510eae834 100644 --- a/roofit/roofitcore/src/TestStatistics/LikelihoodJob.h +++ b/roofit/roofitcore/src/TestStatistics/LikelihoodJob.h @@ -61,7 +61,7 @@ class LikelihoodJob : public MultiProcess::Job, public LikelihoodWrapper { }; void send_back_task_result_from_worker(std::size_t task) override; - bool receive_task_result_on_master(const zmq::message_t &message) override; + bool receive_task_result_on_master(const RooFit::MultiProcess::Message &message) override; void enableOffsetting(bool flag) override; diff --git a/roofit/roofitcore/test/CMakeLists.txt b/roofit/roofitcore/test/CMakeLists.txt index 428d3861d2f4e..21204155cf6e8 100644 --- a/roofit/roofitcore/test/CMakeLists.txt +++ b/roofit/roofitcore/test/CMakeLists.txt @@ -94,7 +94,7 @@ ROOT_ADD_GTEST(testRooSimultaneous testRooSimultaneous.cxx LIBRARIES RooFitCore ROOT_ADD_GTEST(testRooTruthModel testRooTruthModel.cxx LIBRARIES RooFitCore RooFit COPY_TO_BUILDDIR ${CMAKE_CURRENT_SOURCE_DIR}/rooAbsAnaConvPdf_classV3.root) -if (roofit_multiprocess) +if(NOT WIN32) ROOT_ADD_GTEST(testTestStatisticsPlot TestStatistics/testPlot.cxx LIBRARIES RooFitMultiProcess RooFitCore RooFit) ROOT_ADD_GTEST(testLikelihoodGradientJob TestStatistics/testLikelihoodGradientJob.cxx LIBRARIES RooFitMultiProcess RooFitCore m ROOT::TestSupport) target_include_directories(testLikelihoodGradientJob PRIVATE ${RooFitCore_MultiProcess_TestStatistics_INCLUDE_DIR})