diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 01882122e82c13e104979cdb48a7560cc2000d73..8d806a974479a42869b5bcdcecb7b89f51ae9e2e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -213,9 +213,13 @@ clang-tidy: variables: EMPER_LOG_TIMESTAMP: "none" -.locked-ws-queues: +.ws-queue-scheduler-locked: variables: - EMPER_LOCKED_WS_QUEUE: "true" + EMPER_WS_QUEUE_SCHEDULER: "locked" + +.ws-queue-scheduler-cl2: + variables: + EMPER_WS_QUEUE_SCHEDULER: "cl2" .waitfree-ws: variables: @@ -335,10 +339,15 @@ test-do-not-log-timestamp: - .test - .do-not-log-timestamp -test-locked-ws-queues: +test-ws-queue-scheduler-locked: + extends: + - .test + - .ws-queue-scheduler-locked + +test-ws-queue-scheduler-cl2: extends: - .test - - .locked-ws-queues + - .ws-queue-scheduler-cl2 test-waitfree-ws: extends: @@ -467,9 +476,9 @@ build-each-poller: continuation-stealing-locked: extends: - .test + - .ws-queue-scheduler-locked variables: EMPER_CONTINUATION_STEALING_MODE: 'locked' - EMPER_LOCKED_WS_QUEUE: 'true' continuation-stealing-madv-free: extends: diff --git a/Makefile b/Makefile index 733ef52bad27d66e90658a96da3ba120d349694a..9b9ecb705d98d87f78fdbacea20ab4bf3ef7f5a5 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ fibril-locked: rm -f build $(MAKE) build \ EMPER_CONTINUATION_STEALING_MODE=locked \ - EMPER_LOCKED_WS_QUEUE=true \ + EMPER_WS_QUEUE_SCHEDULER=locked \ EMPER_IO=false \ BUILDDIR="build-$@" diff --git a/emper/Fibril.cpp b/emper/Fibril.cpp index 9cfe87a55be9a522a714d5a451b9bf9db266ee7d..6c76992ccb81599b977cc1591804b8c9ff01eaca 100644 --- a/emper/Fibril.cpp +++ b/emper/Fibril.cpp @@ -51,3 +51,11 @@ void Fibril::printTo(std::ostream& strm, bool withPtr) const { << "]"; // clang-format-off } + +void emper::maybeLockFibril(void* queueItem) { + auto* abstractFiber = static_cast<AbstractFiber*>(queueItem); + Fibril* fibril = abstractFiber->asFibrilIfPossible(); + if (fibril) { + fibril->fibrilMutex.lock(); + } +} diff --git a/emper/Fibril.hpp b/emper/Fibril.hpp index b12150c1d5250be6c177f21a34cc8e8316659e04..606ebd5484a056a8e955a5d5aef8c9b6494323c2 100644 --- a/emper/Fibril.hpp +++ b/emper/Fibril.hpp @@ -24,6 +24,10 @@ #define emper_fibril __attribute__((optimize("no-omit-frame-pointer"))) +namespace emper { +void maybeLockFibril(void* queueItem); +} + namespace adt { template <typename I, const uintptr_t SIZE> class LockedQueue; // IWYU pragma: keep @@ -38,6 +42,7 @@ class Fibril : public AbstractFiber, public Logger<LogSubsystem::FIBRIL> { // Members for locked continuation stealing. std::mutex fibrilMutex; + friend void emper::maybeLockFibril(void* queueItem); // Members for wait-free continuation stealing. diff --git a/emper/MemoryManager.hpp b/emper/MemoryManager.hpp index 5447ccc3a1f4307510f181af000f8779a7c273ad..14bd2e1fd66c4313de285600613cdef9dd5676a6 100644 --- a/emper/MemoryManager.hpp +++ b/emper/MemoryManager.hpp @@ -7,7 +7,7 @@ #include "Runtime.hpp" #include "emper-common.h" #include "lib/adt/BoundedBumpArray.hpp" -#include "lib/adt/WsClQueue.hpp" +#include "lib/adt/WsQueue.hpp" template <typename T, intptr_t WS_QUEUE_SIZE, size_t WORKER_EXCLUSIVE_QUEUE_SIZE> class MemoryManager { @@ -16,11 +16,12 @@ class MemoryManager { adt::BoundedBumpArray<void, WORKER_EXCLUSIVE_QUEUE_SIZE>** workerExclusiveQueues; - adt::WsClQueue<void*, WS_QUEUE_SIZE>** queues; + using WsQueue = lib::adt::wsqueue::Default<void*, WS_QUEUE_SIZE>; + WsQueue** queues; static thread_local adt::BoundedBumpArray<void, WORKER_EXCLUSIVE_QUEUE_SIZE> workerExclusiveQueue; - static thread_local adt::WsClQueue<void*, WS_QUEUE_SIZE> queue; + static thread_local WsQueue queue; static auto mallocMemory() -> void* { void* memory; diff --git a/emper/lib/adt/LockedQueue.hpp b/emper/lib/adt/LockedQueue.hpp index eda6e980169bac8e0754dec5458f293ae85288e0..1fcd6bd1c4ab1fae72d6cc0fec42b1db5f405dc5 100644 --- a/emper/lib/adt/LockedQueue.hpp +++ b/emper/lib/adt/LockedQueue.hpp @@ -7,9 +7,13 @@ #include <type_traits> #include "Common.hpp" -#include "Fibril.hpp" #include "StealingResult.hpp" +// Forward declaration, declared and defined in Fibril.hpp. +namespace emper { +void maybeLockFibril(void* queueItem); +} + namespace adt { template <typename I, const uintptr_t SIZE> @@ -62,11 +66,7 @@ class LockedQueue { // by Schmaus et al. [schmaus2021nowa], Figure 6. and Listing 2. if constexpr (emper::CONTINUATION_STEALING_MODE_LOCKED && std::is_same<AbstractFiber*, I>::value) { - AbstractFiber* abstractFiber = static_cast<AbstractFiber*>(*itemPtr); - Fibril* fibril = abstractFiber->asFibrilIfPossible(); - if (fibril) { - fibril->fibrilMutex.lock(); - } + emper::maybeLockFibril(*itemPtr); } deque.pop_front(); diff --git a/emper/lib/adt/WsClV2Queue.hpp b/emper/lib/adt/WsClV2Queue.hpp index 8e4d88a2c46e15da6045fdd78d312e5117e69fab..943f41c8bbc6c434c01ce25cb13e60fb76224053 100644 --- a/emper/lib/adt/WsClV2Queue.hpp +++ b/emper/lib/adt/WsClV2Queue.hpp @@ -137,7 +137,7 @@ loop: if constexpr (maxRetries < 0) goto loop; // Loop maxRetries times if constexpr (maxRetries > 0) { - if (retires == maxRetries) return emper::StealingResult::LostRace; + if (retries == maxRetries) return emper::StealingResult::LostRace; ++retries; } diff --git a/emper/lib/adt/WsQueue.hpp b/emper/lib/adt/WsQueue.hpp new file mode 100644 index 0000000000000000000000000000000000000000..44edaebe0bd58b9878f6ab4199430a9f1bfb0801 --- /dev/null +++ b/emper/lib/adt/WsQueue.hpp @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright © 2022 Florian Schmaus +#pragma once + +#include "emper-config.h" + +#ifdef EMPER_LIB_ADT_WSQUEUE_INCLUDE_LOCKED_QUEUE +#include "lib/adt/LockedQueue.hpp" +#endif + +#ifdef EMPER_LIB_ADT_WSQUEUE_INCLUDE_CL_QUEUE +#include "lib/adt/WsClQueue.hpp" +#endif + +#ifdef EMPER_LIB_ADT_WSQUEUE_INCLUDE_CL2_QUEUE +#include "lib/adt/WsClV2Queue.hpp" +#endif + +namespace lib::adt::wsqueue { + +template <typename PAYLOAD, const uintptr_t CAPACITY> +using Default = EMPER_WS_QUEUE_DEFAULT_TYPE<PAYLOAD, CAPACITY>; + +template <typename PAYLOAD, const uintptr_t CAPACITY> +using Scheduler = EMPER_WS_QUEUE_SCHEDULER_TYPE<PAYLOAD, CAPACITY>; + +} // namespace lib::adt::wsqueue diff --git a/emper/lib/adt/meson.build b/emper/lib/adt/meson.build new file mode 100644 index 0000000000000000000000000000000000000000..e0b853fe2a18472d25d80130820da51f7866832e --- /dev/null +++ b/emper/lib/adt/meson.build @@ -0,0 +1,43 @@ +lib_adt_wsqueue_include_locked_queue = false +lib_adt_wsqueue_include_cl_queue = false +lib_adt_wsqueue_include_cl2_queue = false + + +if ws_queue_default == 'locked' + lib_adt_wsqueue_include_locked_queue = true + ws_queue_default_type = '::adt::LockedQueue' +elif ws_queue_default == 'cl' + lib_adt_wsqueue_include_cl_queue = true + ws_queue_default_type = '::adt::WsClQueue' +elif ws_queue_default == 'cl2' + lib_adt_wsqueue_include_cl2_queue = true + ws_queue_default_type = '::adt::WsClV2Queue' +else + error('Unknown setting for ws_queue_default: ' + ws_queue_default) +endif + + +if ws_queue_scheduler == 'default' + ws_queue_scheduler = ws_queue_default +endif + +if ws_queue_scheduler == 'locked' + lib_adt_wsqueue_include_locked_queue = true + ws_queue_scheduler_type = '::adt::LockedQueue' +elif ws_queue_scheduler == 'cl' + lib_adt_wsqueue_include_cl_queue = true + ws_queue_scheduler_type = '::adt::WsClQueue' +elif ws_queue_scheduler == 'cl2' + lib_adt_wsqueue_include_cl2_queue = true + ws_queue_scheduler_type = '::adt::WsClV2Queue' +else + error('Unknown setting for ws_queue_scheduler: ' + ws_queue_scheduler) +endif + + +conf_data.set('EMPER_LIB_ADT_WSQUEUE_INCLUDE_LOCKED_QUEUE', lib_adt_wsqueue_include_locked_queue) +conf_data.set('EMPER_LIB_ADT_WSQUEUE_INCLUDE_CL_QUEUE', lib_adt_wsqueue_include_cl_queue) +conf_data.set('EMPER_LIB_ADT_WSQUEUE_INCLUDE_CL2_QUEUE', lib_adt_wsqueue_include_cl2_queue) + +conf_data.set('EMPER_WS_QUEUE_DEFAULT_TYPE', ws_queue_default_type) +conf_data.set('EMPER_WS_QUEUE_SCHEDULER_TYPE', ws_queue_scheduler_type) diff --git a/emper/lib/meson.build b/emper/lib/meson.build index 2ee763b692c54979677ffea7bc96faa7c8c1b883..3aa7f1952fceb1111e39cb3324c2a389ae6300e1 100644 --- a/emper/lib/meson.build +++ b/emper/lib/meson.build @@ -5,4 +5,5 @@ emper_cpp_sources += files( 'util.cpp', ) +subdir('adt') subdir('sync') diff --git a/emper/strategies/AbstractWorkStealingScheduler.hpp b/emper/strategies/AbstractWorkStealingScheduler.hpp index b38489ad8314551d30649be236cfc9c08b248535..f6006cb241c7959c30b9b8ce653f15e54fca6ab7 100644 --- a/emper/strategies/AbstractWorkStealingScheduler.hpp +++ b/emper/strategies/AbstractWorkStealingScheduler.hpp @@ -12,12 +12,7 @@ #include "Scheduler.hpp" #include "emper-common.h" #include "lib/adt/MpscQueue.hpp" - -#ifdef EMPER_LOCKED_WS_QUEUE -#include "lib/adt/LockedQueue.hpp" -#else -#include "lib/adt/WsClQueue.hpp" -#endif +#include "lib/adt/WsQueue.hpp" struct NextFiberResult; class AbstractFiber; @@ -26,11 +21,7 @@ class RuntimeStrategy; class AbstractWorkStealingScheduler : public Scheduler { template <size_t SIZE> -#ifdef EMPER_LOCKED_WS_QUEUE - using WsQueue = adt::LockedQueue<AbstractFiber*, SIZE>; -#else - using WsQueue = adt::WsClQueue<AbstractFiber*, SIZE>; -#endif + using WsQueue = lib::adt::wsqueue::Scheduler<AbstractFiber*, SIZE>; using MpscQueue = adt::MpscQueue<Fiber>; public: diff --git a/meson.build b/meson.build index f77b9cdd119a579a92fb887aea44e16ab8d9ecc2..6d86de05b83b0b0687abe50cacfb9ce3112690ba 100644 --- a/meson.build +++ b/meson.build @@ -53,14 +53,14 @@ if cpp_is_clang endif continuation_stealing_mode = get_option('continuation_stealing_mode') -locked_ws_queue = get_option('locked_ws_queue') +ws_queue_default = get_option('ws_queue_default') +ws_queue_scheduler = get_option('ws_queue_scheduler') assume_page_size = get_option('assume_page_size') assume_cache_line_size = get_option('assume_cache_line_size') conf_data.set('EMPER_WORKER_SLEEP', get_option('worker_sleep')) conf_data.set('EMPER_WORKER_WAKEUP_STRATEGY', get_option('worker_wakeup_strategy')) conf_data.set('EMPER_WORKER_IGNORE_WAKEUP_HINT', get_option('worker_ignore_wakeup_hint')) -conf_data.set('EMPER_LOCKED_WS_QUEUE', locked_ws_queue) conf_data.set('EMPER_LOCKED_MPSC_QUEUE', get_option('locked_mpsc_queue')) conf_data.set('EMPER_OVERFLOW_QUEUE', get_option('overflow_queue')) conf_data.set('EMPER_STATS', get_option('stats')) @@ -91,8 +91,14 @@ else endif conf_data.set('EMPER_CONTEXT_ALIGNAS', context_alignas) -if continuation_stealing_mode == 'locked' and not locked_ws_queue - error('*Locked* continuation stealing only works with locked work-stealing queues (locked_ws_queue=true)') +if ws_queue_scheduler == 'locked' or (ws_queue_scheduler == 'default' and ws_queue_default == 'locked') + ws_queue_scheduler_locked = true +else + ws_queue_scheduler_locked = false +endif + +if continuation_stealing_mode == 'locked' and not ws_queue_scheduler_locked + error('*Locked* continuation stealing only works with locked work-stealing queues (ws_queue_scheduler=locked)') endif if continuation_stealing_mode == 'waitfree' conf_data.set('EMPER_CONTINUATION_STEALING_MODE_WAITFREE', true) diff --git a/meson_options.txt b/meson_options.txt index b3e4b3ccfad1a54f3f4811c6821563bc8ee5e6eb..49ad8d4fcf82166c57190629734499a964059a50 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -72,10 +72,27 @@ option( value: false, ) option( - 'locked_ws_queue', - type: 'boolean', - value: false, - description: 'Use a fully locked queue for work-stealing', + 'ws_queue_default', + type: 'combo', + choices: [ + 'locked', + 'cl', + 'cl2', + ], + value: 'cl', + description: 'The default work-stealing queue to use in the runtime-system', +) +option( + 'ws_queue_scheduler', + type: 'combo', + choices: [ + 'default', + 'locked', + 'cl', + 'cl2', + ], + value: 'default', + description: 'The work-stealing queue to use for scheduling', ) option( 'locked_mpsc_queue',