diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8d806a974479a42869b5bcdcecb7b89f51ae9e2e..d4f0cf392d36822bb238fc6cff1f290f3935ff2a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -221,6 +221,14 @@ clang-tidy:
variables:
EMPER_WS_QUEUE_SCHEDULER: "cl2"
+.ws-queue-scheduler-cl3:
+ variables:
+ EMPER_WS_QUEUE_SCHEDULER: "cl3"
+
+.ws-queue-scheduler-cl4:
+ variables:
+ EMPER_WS_QUEUE_SCHEDULER: "cl4"
+
.waitfree-ws:
variables:
EMPER_WAITFREE_WORK_STEALING: "true"
@@ -349,6 +357,16 @@ test-ws-queue-scheduler-cl2:
- .test
- .ws-queue-scheduler-cl2
+test-ws-queue-scheduler-cl3:
+ extends:
+ - .test
+ - .ws-queue-scheduler-cl3
+
+test-ws-queue-scheduler-cl3:
+ extends:
+ - .test
+ - .ws-queue-scheduler-cl3
+
test-waitfree-ws:
extends:
- .test
diff --git a/README.md b/README.md
index ccdd3ad739d9a25bc6036741005fe7badded083d..aa7414bd86b7e9ad76b60b080d5feba92d80703f 100644
--- a/README.md
+++ b/README.md
@@ -75,4 +75,36 @@ url: https://www4.cs.fau.de/~flow/papers/schmaus2021nowa.pdf
[pfeiffer2020cactus]
Pfeiffer, Nicolas. A Wait-Free Cactus Stack Implementation for a Microparalelism
Runtime. Master's thesis. MA-I4-2020-02. Mar. 2, 2020.
-url: https://www4.cs.fau.de/~flow/papers/pfeiffer2020cactus.pdf
+url: https://www4.cs.fau.de/~flow/papers/pfeiffer2020cactus.pdf
+
+# Literature
+
+> The dwarf sees farther than the giant, when he has the giant's shoulder to mount on.
+ - Samuel Taylor Coleridge, The Friend (1828)
+
+EMPER uses concepts, ideas and algorithms from the following
+publications. You will find the key of a publication, e.g.,
+[chase2005dynamic] sometimes mentioned in EMPER's source code.
+
+[chase2005dynamic]
+Chase, David and Yossi Lev. “Dynamic Circular Work-Stealing Dequeâ€. In: Pro-
+ceedings of the Seventeenth Annual ACM Symposium on Parallelism in Algorithms
+and Architectures. SPAA ’05. Las Vegas, Nevada, USA: Association
+for Computing Machinery, 2005, pp. 21–28. isbn: 1581139861. doi: [10.1145/1073970.1073974](https://doi.org/10.1145/1073970.1073974).
+
+[le2013correct]
+Lê, Nhat Minh, Antoniu Pop, Albert Cohen, and Francesco Zappa Nardelli.
+“Correct and Efficient Work-Stealing for Weak Memory Modelsâ€. In: Proceedings
+of the 18th ACM SIGPLAN Symposium on Principles and Practice of
+Parallel Programming. PPoPP ’13. Shenzhen, China: Association for Computing
+Machinery, 2013, pp. 69–80. isbn: 9781450319225. doi: 10.1145/2442516.2442524.
+url: https://hal.inria.fr/hal-00802885/document
+
+[norris2013cdschecker]
+Norris, Brian and Brian Demsky. “CDSchecker: Checking Concurrent Data Structures
+Written with C/C++ Atomicsâ€. In: Proceedings of the 2013 ACM SIGPLAN
+International Conference on Object Oriented Programming Systems
+Languages Applications. OOPSLA ’13. Indianapolis, Indiana, USA: Association
+for Computing Machinery, 2013, pp. 131–150. isbn: 9781450323741.
+doi: 10.1145/2509136.2509514.
+url: http://plrg.eecs.uci.edu/publications/c11modelcheck.pdf
diff --git a/emper/lib/adt/PushBottomResult.hpp b/emper/lib/adt/PushBottomResult.hpp
index b65236ce624f3b41d807c774787d81a0bcdc27bb..c61eddcff6de8c41bc79a2d6ae15c9ac6d7905e0 100644
--- a/emper/lib/adt/PushBottomResult.hpp
+++ b/emper/lib/adt/PushBottomResult.hpp
@@ -2,6 +2,8 @@
// Copyright © 2022 Florian Schmaus
#pragma once
+#include <cstdint>
+
namespace adt {
struct PushBottomResult {
diff --git a/emper/lib/adt/WsClv3Queue.hpp b/emper/lib/adt/WsClv3Queue.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..33e1a38982575b92e365139fce9258e50fd390a0
--- /dev/null
+++ b/emper/lib/adt/WsClv3Queue.hpp
@@ -0,0 +1,154 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright © 2020-2022 Florian Schmaus
+#pragma once
+
+#include <atomic>
+#include <cassert>
+#include <cstdint>
+
+#include "Common.hpp"
+#include "StealingResult.hpp"
+#include "emper-common.h"
+#include "lib/adt/PushBottomResult.hpp"
+
+namespace adt {
+
+/**
+ * @file
+ *
+ * A CL queue used for work stealing.
+ *
+ * Implements the partially concurrent queue for the work stealing
+ * scheduling algorithm as defined in "Dynamic Circular Work-Stealing Deque" by Chase and Lev
+ * [chase2005dynamic]. The worker uses pushBottom to push new items on the stack, and retrieves
+ * items via popBottom. Eventually, when its own queue is empty, it will use popTop to retrieve
+ * items from a different Worker's queue.
+ *
+ * Only pushBottom/popBottom and popTop may be used concurrently,
+ * i.e. concurrent use of pushBottom with popBottom is not allowed.
+ *
+ * Unlike the ABP queue, the CL queue uses a circular array and
+ * only-incrementing uint64_t bottom and top fields. According to the
+ * authors "A 64-bit integer is large enough to accommodate 64 years
+ * of pushes, pops and steals executing at a rate of 4 billtion
+ * operations per second", so overflows should be no problem.
+ */
+template <typename PAYLOAD, const uintptr_t CAPACITY>
+class WsClv3Queue {
+ ALIGN_TO_CACHE_LINE std::atomic<uint64_t> bottom;
+ ALIGN_TO_CACHE_LINE std::atomic<uint64_t> top;
+ // NOLINTNEXTLINE(modernize-avoid-c-arrays)
+ PAYLOAD queue[CAPACITY];
+
+ public:
+ // 'bottom' and 'top' are initialized to '1', instead of '0' as
+ // it's done in the "Dynamic Circular Work-Stealing Deque" paper
+ // because popBottom will decrement bottom, which will result in
+ // an underflow if bottom is '0'. The paper's queue uses Java
+ // 'long' for bottom and top and is thus safe since it's signed.
+ WsClv3Queue() : bottom(1), top(1) {}
+ // TODO: Decide what to do regarding the following suppressed lint.
+ // NOLINTNEXTLINE(readability-avoid-const-params-in-decls)
+ auto pushBottom(const PAYLOAD item) -> PushBottomResult;
+ template <const int maxRetries>
+ auto popTop(PAYLOAD *item) -> emper::StealingResult;
+ auto popBottom(PAYLOAD *item) -> bool;
+ [[nodiscard]] auto isFull() const -> bool;
+ [[nodiscard]] auto isEmpty() const -> bool;
+ void print() const;
+ [[nodiscard]] inline auto capacity() const -> uintptr_t { return CAPACITY; }
+ [[nodiscard]] inline auto usedSlots() const -> uint64_t { return bottom - top; }
+ [[nodiscard]] inline auto freeSlots() const -> uintptr_t { return capacity() - usedSlots(); }
+};
+
+template <typename PAYLOAD, const uintptr_t CAPACITY>
+auto WsClv3Queue<PAYLOAD, CAPACITY>::pushBottom(const PAYLOAD item) -> PushBottomResult {
+ bool pushed = true;
+ uint64_t localBottom = bottom.load(std::memory_order_relaxed);
+ uint64_t localTop = top.load(std::memory_order_relaxed);
+
+ uint64_t currentSize = localBottom - localTop;
+ if (currentSize >= CAPACITY) {
+ assert(localBottom == localTop);
+ pushed = false;
+ goto out;
+ }
+
+ queue[bottom % CAPACITY] = item;
+ bottom.store(localBottom + 1, std::memory_order_release);
+
+out:
+ return PushBottomResult{pushed, pushed ? currentSize + 1 : currentSize};
+}
+
+template <typename PAYLOAD, const uintptr_t CAPACITY>
+template <const int maxRetries>
+auto WsClv3Queue<PAYLOAD, CAPACITY>::popTop(PAYLOAD *item) -> emper::StealingResult {
+ int retries = 0;
+
+ uint64_t localTop = top.load(std::memory_order_relaxed);
+
+loop:
+ ATTR_UNUSED; // to suppress unused label 'loop' warning if maxRetries == 0.
+ uint64_t localBottom = bottom.load(std::memory_order_acquire);
+
+ if (localBottom <= localTop) return emper::StealingResult::Empty;
+
+ *item = queue[localTop % CAPACITY];
+
+ // If cas fails, then this popTop() lost the race against another
+ // popTop().
+ if (top.compare_exchange_strong(localTop, localTop + 1, std::memory_order_release,
+ std::memory_order_acquire))
+ return emper::StealingResult::Stolen;
+
+ // Loop indefinitely
+ if constexpr (maxRetries < 0) goto loop;
+ // Loop maxRetries times
+ if constexpr (maxRetries > 0) {
+ if (retries == maxRetries) return emper::StealingResult::LostRace;
+
+ ++retries;
+ }
+
+ return emper::StealingResult::LostRace;
+}
+
+template <typename PAYLOAD, const uintptr_t CAPACITY>
+auto WsClv3Queue<PAYLOAD, CAPACITY>::popBottom(PAYLOAD *item) -> bool {
+ uint64_t localBottom = bottom.fetch_sub(1, std::memory_order_acq_rel) - 1;
+ uint64_t localTop = top.load(std::memory_order_acquire);
+
+ if (localTop > localBottom) {
+ // The queue is empty.
+ bottom = localTop;
+ return false;
+ }
+
+ *item = queue[localBottom % CAPACITY];
+ if (localBottom > localTop) return true;
+
+ bool res = top.compare_exchange_weak(localTop, localTop + 1, std::memory_order_release,
+ std::memory_order_relaxed);
+
+ // Either a popTop() removed the element ('res' is false) or we
+ // removed the element ('res' is true), but we need to increment
+ // the 'bottom' value, since the element bottom pointed at is now
+ // gone. N.B. bottom does point to the next free slot, the actual
+ // element we remove is bottom-1.
+ bottom.store(localBottom + 1, std::memory_order_relaxed);
+
+ return res;
+}
+
+template <typename PAYLOAD, const uintptr_t CAPACITY>
+auto WsClv3Queue<PAYLOAD, CAPACITY>::isFull() const -> bool {
+ return usedSlots() >= CAPACITY;
+}
+
+template <typename PAYLOAD, const uintptr_t CAPACITY>
+auto WsClv3Queue<PAYLOAD, CAPACITY>::isEmpty() const -> bool {
+ return top >= bottom;
+}
+
+} // namespace adt
diff --git a/emper/lib/adt/WsClv4Queue.hpp b/emper/lib/adt/WsClv4Queue.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..987c5b49ec1e0d0999dd98da495ad8ffd2e540e8
--- /dev/null
+++ b/emper/lib/adt/WsClv4Queue.hpp
@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright © 2020-2022 Florian Schmaus
+#pragma once
+
+#include <atomic>
+#include <cassert>
+#include <cstdint>
+
+#include "Common.hpp"
+#include "StealingResult.hpp"
+#include "emper-common.h"
+#include "lib/adt/PushBottomResult.hpp"
+
+namespace adt {
+
+/**
+ * @file
+ *
+ * A CL queue used for work stealing.
+ *
+ * Implements the partially concurrent queue for the work stealing
+ * scheduling algorithm as defined in "Dynamic Circular Work-Stealing Deque" by Chase and Lev
+ * [chase2005dynamic]. The worker uses pushBottom to push new items on the stack, and retrieves
+ * items via popBottom. Eventually, when its own queue is empty, it will use popTop to retrieve
+ * items from a different Worker's queue.
+ *
+ * Only pushBottom/popBottom and popTop may be used concurrently,
+ * i.e. concurrent use of pushBottom with popBottom is not allowed.
+ *
+ * Unlike the ABP queue, the CL queue uses a circular array and
+ * only-incrementing uint64_t bottom and top fields. According to the
+ * authors "A 64-bit integer is large enough to accommodate 64 years
+ * of pushes, pops and steals executing at a rate of 4 billtion
+ * operations per second", so overflows should be no problem.
+ *
+ * This also incoroperates the findings from the paper "Correct and
+ * Efficient Work-Stealing for Weak Memory Models" by Lê et
+ * al. [le2013correct]. Since this queue is not dynamically resized
+ * (yet), the bug in the implementation of this paper, found by Norris
+ * and Demsky [norris2013cdschecker], does not exist.
+ */
+template <typename PAYLOAD, const uintptr_t CAPACITY>
+class WsClv4Queue {
+ ALIGN_TO_CACHE_LINE std::atomic<uint64_t> bottom;
+ ALIGN_TO_CACHE_LINE std::atomic<uint64_t> top;
+ // NOLINTNEXTLINE(modernize-avoid-c-arrays)
+ PAYLOAD queue[CAPACITY];
+
+ public:
+ // 'bottom' and 'top' are initialized to '1', instead of '0' as
+ // it's done in the "Dynamic Circular Work-Stealing Deque" paper
+ // because popBottom will decrement bottom, which will result in
+ // an underflow if bottom is '0'. The paper's queue uses Java
+ // 'long' for bottom and top and is thus safe since it's signed.
+ WsClv4Queue() : bottom(1), top(1) {}
+ // TODO: Decide what to do regarding the following suppressed lint.
+ // NOLINTNEXTLINE(readability-avoid-const-params-in-decls)
+ auto pushBottom(const PAYLOAD item) -> PushBottomResult;
+ template <const int maxRetries>
+ auto popTop(PAYLOAD *item) -> emper::StealingResult;
+ auto popBottom(PAYLOAD *item) -> bool;
+ [[nodiscard]] auto isFull() const -> bool;
+ [[nodiscard]] auto isEmpty() const -> bool;
+ void print() const;
+ [[nodiscard]] inline auto capacity() const -> uintptr_t { return CAPACITY; }
+ [[nodiscard]] inline auto usedSlots() const -> uint64_t { return bottom - top; }
+ [[nodiscard]] inline auto freeSlots() const -> uintptr_t { return capacity() - usedSlots(); }
+};
+
+template <typename PAYLOAD, const uintptr_t CAPACITY>
+auto WsClv4Queue<PAYLOAD, CAPACITY>::pushBottom(const PAYLOAD item) -> PushBottomResult {
+ bool pushed = true;
+ uint64_t localTop = top.load(std::memory_order_relaxed);
+ std::atomic_thread_fence(std::memory_order_seq_cst);
+ uint64_t localBottom = bottom.load(std::memory_order_relaxed);
+
+ uint64_t currentSize = localBottom - localTop;
+ if (currentSize >= CAPACITY) {
+ assert(localBottom == localTop);
+ pushed = false;
+ goto out;
+ }
+
+ queue[bottom % CAPACITY] = item;
+ std::atomic_thread_fence(std::memory_order_release);
+ bottom.store(localBottom + 1, std::memory_order_relaxed);
+
+out:
+ return PushBottomResult{pushed, pushed ? currentSize + 1 : currentSize};
+}
+
+template <typename PAYLOAD, const uintptr_t CAPACITY>
+template <const int maxRetries>
+auto WsClv4Queue<PAYLOAD, CAPACITY>::popTop(PAYLOAD *item) -> emper::StealingResult {
+ int retries = 0;
+
+loop:
+ ATTR_UNUSED; // to suppress unused label 'loop' warning if maxRetries == 0.
+ uint64_t localTop = top.load(std::memory_order_acquire);
+ std::atomic_thread_fence(std::memory_order_seq_cst);
+ uint64_t localBottom = bottom.load(std::memory_order_acquire);
+
+ if (localBottom <= localTop) return emper::StealingResult::Empty;
+
+ *item = queue[localTop % CAPACITY];
+
+ // If cas fails, then this popTop() lost the race against another
+ // popTop().
+ if (top.compare_exchange_strong(localTop, localTop + 1, std::memory_order_seq_cst,
+ std::memory_order_relaxed))
+ return emper::StealingResult::Stolen;
+
+ // Loop indefinitely
+ if constexpr (maxRetries < 0) goto loop;
+ // Loop maxRetries times
+ if constexpr (maxRetries > 0) {
+ if (retries == maxRetries) return emper::StealingResult::LostRace;
+
+ ++retries;
+ }
+
+ return emper::StealingResult::LostRace;
+}
+
+template <typename PAYLOAD, const uintptr_t CAPACITY>
+auto WsClv4Queue<PAYLOAD, CAPACITY>::popBottom(PAYLOAD *item) -> bool {
+ uint64_t localBottom = bottom.load(std::memory_order_relaxed) - 1;
+ bottom.store(localBottom, std::memory_order_relaxed);
+ std::atomic_thread_fence(std::memory_order_seq_cst);
+ uint64_t localTop = top.load(std::memory_order_acquire);
+
+ if (localTop > localBottom) {
+ // The queue is empty.
+ bottom.store(localBottom + 1, std::memory_order_relaxed);
+ return false;
+ }
+
+ *item = queue[localBottom % CAPACITY];
+ if (localBottom > localTop) return true;
+
+ bool res = top.compare_exchange_weak(localTop, localTop + 1, std::memory_order_seq_cst,
+ std::memory_order_relaxed);
+
+ // Either a popTop() removed the element ('res' is false) or we
+ // removed the element ('res' is true), but we need to increment
+ // the 'bottom' value, since the element bottom pointed at is now
+ // gone. N.B. bottom does point to the next free slot, the actual
+ // element we remove is bottom-1.
+ bottom.store(localBottom + 1, std::memory_order_relaxed);
+
+ return res;
+}
+
+template <typename PAYLOAD, const uintptr_t CAPACITY>
+auto WsClv4Queue<PAYLOAD, CAPACITY>::isFull() const -> bool {
+ return usedSlots() >= CAPACITY;
+}
+
+template <typename PAYLOAD, const uintptr_t CAPACITY>
+auto WsClv4Queue<PAYLOAD, CAPACITY>::isEmpty() const -> bool {
+ return top >= bottom;
+}
+
+} // namespace adt
diff --git a/emper/lib/adt/WsQueue.hpp b/emper/lib/adt/WsQueue.hpp
index 44edaebe0bd58b9878f6ab4199430a9f1bfb0801..53daafdc3115c457ac3b271af3216e1b18499866 100644
--- a/emper/lib/adt/WsQueue.hpp
+++ b/emper/lib/adt/WsQueue.hpp
@@ -16,6 +16,14 @@
#include "lib/adt/WsClV2Queue.hpp"
#endif
+#ifdef EMPER_LIB_ADT_WSQUEUE_INCLUDE_CL3_QUEUE
+#include "lib/adt/WsClv3Queue.hpp"
+#endif
+
+#ifdef EMPER_LIB_ADT_WSQUEUE_INCLUDE_CL4_QUEUE
+#include "lib/adt/WsClv4Queue.hpp"
+#endif
+
namespace lib::adt::wsqueue {
template <typename PAYLOAD, const uintptr_t CAPACITY>
diff --git a/emper/lib/adt/meson.build b/emper/lib/adt/meson.build
index e0b853fe2a18472d25d80130820da51f7866832e..c96975ce9be62d301e39a62585a6cfb472efe0c3 100644
--- a/emper/lib/adt/meson.build
+++ b/emper/lib/adt/meson.build
@@ -1,6 +1,8 @@
lib_adt_wsqueue_include_locked_queue = false
lib_adt_wsqueue_include_cl_queue = false
lib_adt_wsqueue_include_cl2_queue = false
+lib_adt_wsqueue_include_cl3_queue = false
+lib_adt_wsqueue_include_cl4_queue = false
if ws_queue_default == 'locked'
@@ -12,6 +14,12 @@ elif ws_queue_default == 'cl'
elif ws_queue_default == 'cl2'
lib_adt_wsqueue_include_cl2_queue = true
ws_queue_default_type = '::adt::WsClV2Queue'
+elif ws_queue_default == 'cl3'
+ lib_adt_wsqueue_include_cl3_queue = true
+ ws_queue_default_type = '::adt::WsClv3Queue'
+elif ws_queue_default == 'cl4'
+ lib_adt_wsqueue_include_cl4_queue = true
+ ws_queue_default_type = '::adt::WsClv4Queue'
else
error('Unknown setting for ws_queue_default: ' + ws_queue_default)
endif
@@ -30,6 +38,12 @@ elif ws_queue_scheduler == 'cl'
elif ws_queue_scheduler == 'cl2'
lib_adt_wsqueue_include_cl2_queue = true
ws_queue_scheduler_type = '::adt::WsClV2Queue'
+elif ws_queue_scheduler == 'cl3'
+ lib_adt_wsqueue_include_cl3_queue = true
+ ws_queue_scheduler_type = '::adt::WsClv3Queue'
+elif ws_queue_scheduler == 'cl4'
+ lib_adt_wsqueue_include_cl4_queue = true
+ ws_queue_scheduler_type = '::adt::WsClv4Queue'
else
error('Unknown setting for ws_queue_scheduler: ' + ws_queue_scheduler)
endif
@@ -38,6 +52,8 @@ endif
conf_data.set('EMPER_LIB_ADT_WSQUEUE_INCLUDE_LOCKED_QUEUE', lib_adt_wsqueue_include_locked_queue)
conf_data.set('EMPER_LIB_ADT_WSQUEUE_INCLUDE_CL_QUEUE', lib_adt_wsqueue_include_cl_queue)
conf_data.set('EMPER_LIB_ADT_WSQUEUE_INCLUDE_CL2_QUEUE', lib_adt_wsqueue_include_cl2_queue)
+conf_data.set('EMPER_LIB_ADT_WSQUEUE_INCLUDE_CL3_QUEUE', lib_adt_wsqueue_include_cl3_queue)
+conf_data.set('EMPER_LIB_ADT_WSQUEUE_INCLUDE_CL4_QUEUE', lib_adt_wsqueue_include_cl4_queue)
conf_data.set('EMPER_WS_QUEUE_DEFAULT_TYPE', ws_queue_default_type)
conf_data.set('EMPER_WS_QUEUE_SCHEDULER_TYPE', ws_queue_scheduler_type)
diff --git a/extra-libs/meson.build b/extra-libs/meson.build
new file mode 100644
index 0000000000000000000000000000000000000000..72019537126c9c354fa8909b4c0a1abda528c9c6
--- /dev/null
+++ b/extra-libs/meson.build
@@ -0,0 +1 @@
+subdir('ws-queue')
diff --git a/extra-libs/ws-queue/WorkStealingQueueLib.cpp b/extra-libs/ws-queue/WorkStealingQueueLib.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c39bb20649bf3439d1f56fc1b2f70498b109f4e6
--- /dev/null
+++ b/extra-libs/ws-queue/WorkStealingQueueLib.cpp
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright © 2022 Florian Schmaus
+#include "WorkStealingQueueLib.hpp"
+
+#include "StealingResult.hpp"
+#include "lib/adt/PushBottomResult.hpp"
+#include "lib/adt/WsClv3Queue.hpp"
+#include "lib/adt/WsClv4Queue.hpp"
+
+namespace emper::extralibs::wsqueue {
+
+constexpr int maxRetries = 0;
+
+namespace cl3 {
+static adt::WsClv3Queue<int, 16> queue;
+
+auto pushBottom(int value) -> adt::PushBottomResult { return queue.pushBottom(value); }
+
+auto popBottom(int* value) -> bool { return queue.popBottom(value); }
+
+auto popTop(int* value) -> StealingResult { return queue.popTop<maxRetries>(value); }
+} // namespace cl3
+
+namespace cl4 {
+static adt::WsClv4Queue<int, 16> queue;
+
+auto pushBottom(int value) -> adt::PushBottomResult { return queue.pushBottom(value); }
+
+auto popBottom(int* value) -> bool { return queue.popBottom(value); }
+
+auto popTop(int* value) -> StealingResult { return queue.popTop<maxRetries>(value); }
+} // namespace cl4
+
+} // namespace emper::extralibs::wsqueue
diff --git a/extra-libs/ws-queue/WorkStealingQueueLib.hpp b/extra-libs/ws-queue/WorkStealingQueueLib.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..8ffe24e0b5e129efcf4a05a859c1ba03b3f439c2
--- /dev/null
+++ b/extra-libs/ws-queue/WorkStealingQueueLib.hpp
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright © 2022 Florian Schmaus
+#pragma once
+
+#include "StealingResult.hpp"
+#include "lib/adt/PushBottomResult.hpp"
+
+namespace emper::extralibs::wsqueue {
+
+namespace cl3 {
+auto pushBottom(int value) -> adt::PushBottomResult;
+
+auto popBottom(int* value) -> bool;
+
+auto popTop(int* value) -> StealingResult;
+} // namespace cl3
+
+namespace cl4 {
+auto pushBottom(int value) -> adt::PushBottomResult;
+
+auto popBottom(int* value) -> bool;
+
+auto popTop(int* value) -> StealingResult;
+} // namespace cl4
+
+} // namespace emper::extralibs::wsqueue
diff --git a/extra-libs/ws-queue/meson.build b/extra-libs/ws-queue/meson.build
new file mode 100644
index 0000000000000000000000000000000000000000..3196606c2bdbb4186bc4987aaa82b2b97d5ac1dc
--- /dev/null
+++ b/extra-libs/ws-queue/meson.build
@@ -0,0 +1,5 @@
+ws_queue_lib = library(
+ 'ws-queue',
+ ['WorkStealingQueueLib.cpp'],
+ include_directories: emper_all_include,
+)
diff --git a/meson.build b/meson.build
index 6d86de05b83b0b0687abe50cacfb9ce3112690ba..6ef7c078f701a26f69c6b4310a70c7bbe209f3b6 100644
--- a/meson.build
+++ b/meson.build
@@ -231,6 +231,7 @@ if get_option('io_single_uring')
endif
subdir('emper')
+subdir('extra-libs')
subdir('tests')
subdir('apps')
subdir('eval')
diff --git a/meson_options.txt b/meson_options.txt
index 49ad8d4fcf82166c57190629734499a964059a50..9a5dfb73daee0df4b031e92711eb11de7e890551 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -78,6 +78,8 @@ option(
'locked',
'cl',
'cl2',
+ 'cl3',
+ 'cl4',
],
value: 'cl',
description: 'The default work-stealing queue to use in the runtime-system',
@@ -90,6 +92,8 @@ option(
'locked',
'cl',
'cl2',
+ 'cl3',
+ 'cl4',
],
value: 'default',
description: 'The work-stealing queue to use for scheduling',