diff --git a/emper/Dispatcher.hpp b/emper/Dispatcher.hpp
index 4aca48c416e84ec37313986723ae880201e5e88c..9b2d390ee9b297d3770e26c52bee4dd1142f90ac 100644
--- a/emper/Dispatcher.hpp
+++ b/emper/Dispatcher.hpp
@@ -57,4 +57,5 @@ class Dispatcher : public Logger<LogSubsystem::DISP> {
static auto isDispatchedControlFlow() -> bool { return getCurrentFiberPtr() != nullptr; }
friend ContextManager;
+ friend class Scheduler;
};
diff --git a/emper/Fiber.hpp b/emper/Fiber.hpp
index 7caf6080b50f17aa05747465529f03e829356992..9f6965f9e2526c26e637bdb9fb2e378ac54e3e61 100644
--- a/emper/Fiber.hpp
+++ b/emper/Fiber.hpp
@@ -112,6 +112,8 @@ class ALIGN_TO_CACHE_LINE Fiber : public Logger<LogSubsystem::F> {
void print() const;
+ [[nodiscard]] auto isRunnable() const -> bool { return runnable; }
+
friend auto operator<<(std::ostream& strm, const Fiber& fiber) -> std::ostream&;
static inline auto from(fiber_fun_t function, void* arg) -> Fiber* {
diff --git a/emper/RuntimeStrategy.hpp b/emper/RuntimeStrategy.hpp
index 2cbe48bdc9a1a49f9a680dcadfdce0c4f52da49f..6fc7723517701bb166568802d95390703be213d1 100644
--- a/emper/RuntimeStrategy.hpp
+++ b/emper/RuntimeStrategy.hpp
@@ -11,6 +11,7 @@ class RuntimeStrategyStats;
class RuntimeStrategy {
friend class Runtime;
+ friend class Scheduler;
private:
virtual auto getScheduler() -> Scheduler& = 0;
diff --git a/emper/Scheduler.cpp b/emper/Scheduler.cpp
index eb4dde0b187e9d719b8ca1c412affbecc2fcc1bf..534edfbe8dac6d4eb4b5da096bedcc7232bef845 100644
--- a/emper/Scheduler.cpp
+++ b/emper/Scheduler.cpp
@@ -4,8 +4,10 @@
#include "CallerEnvironment.hpp"
#include "Runtime.hpp"
+#include "RuntimeStrategy.hpp"
-Scheduler::Scheduler(Runtime& runtime) : runtime(runtime) {}
+Scheduler::Scheduler(Runtime& runtime, RuntimeStrategy& strategy)
+ : runtime(runtime), dispatcher(strategy.getDispatcher()) {}
void Scheduler::addNewWorkerHook(const std::function<void(workerid_t)>& hook) {
runtime.addNewWorkerHook(hook);
diff --git a/emper/Scheduler.hpp b/emper/Scheduler.hpp
index d71ba18b813d1be89113d6bff27520315366b1aa..bb1d34bfed6dbf92d228c75265ec0c25ff986788 100644
--- a/emper/Scheduler.hpp
+++ b/emper/Scheduler.hpp
@@ -7,12 +7,14 @@
#include "CallerEnvironment.hpp"
#include "Debug.hpp" // for LogSubsystem, LogSubsystem::SCHED, Logger
+#include "Dispatcher.hpp"
#include "Emper.hpp"
#include "Fiber.hpp" // for Fiber
#include "emper-common.h" // for workeraffinity_t
#include "lib/adt/LockedUnboundedQueue.hpp"
class Runtime;
+class RuntimeStrategy;
struct NextFiberResult;
class Scheduler : public Logger<LogSubsystem::SCHED> {
@@ -24,7 +26,9 @@ class Scheduler : public Logger<LogSubsystem::SCHED> {
protected:
Runtime& runtime;
- Scheduler(Runtime& runtime);
+ Dispatcher& dispatcher;
+
+ Scheduler(Runtime& runtime, RuntimeStrategy& strategy);
virtual ~Scheduler() = default;
@@ -55,6 +59,8 @@ class Scheduler : public Logger<LogSubsystem::SCHED> {
virtual void scheduleFromAnywhereInternal(Fiber& fiber) = 0;
virtual void scheduleFromAnywhereInternal(Fiber** fibers, unsigned count) = 0;
+ void recycle(Fiber* fiber) { dispatcher.recycle(fiber); };
+
public:
void schedule(Fiber& fiber) {
LOGD("Scheduling fiber " << &fiber);
diff --git a/emper/strategies/AbstractWorkStealingScheduler.cpp b/emper/strategies/AbstractWorkStealingScheduler.cpp
index 7ca3722796e8608d64b33f41a2bbd59ce4457c32..62ecb93cc08a6dd3512a68181f2de127adc57c5f 100644
--- a/emper/strategies/AbstractWorkStealingScheduler.cpp
+++ b/emper/strategies/AbstractWorkStealingScheduler.cpp
@@ -7,21 +7,21 @@
#include "Common.hpp" // for unlikely, likely
#include "Debug.hpp" // for ABORT
#include "Emper.hpp" // for OVERFLOW_QUEUE
+#include "Fiber.hpp"
#include "NextFiberResult.hpp"
#include "Runtime.hpp" // for Runtime
#include "emper-common.h" // for workerid_t
#include "strategies/AbstractWorkStealingStrategy.hpp"
#include "strategies/AbstractWorkStealingWorkerStats.hpp"
-class Fiber;
-
using awss = AbstractWorkStealingStrategy;
thread_local AbstractWorkStealingScheduler::WsQueue<AbstractWorkStealingScheduler::QUEUE_SIZE>
AbstractWorkStealingScheduler::queue;
-AbstractWorkStealingScheduler::AbstractWorkStealingScheduler(Runtime& runtime)
- : Scheduler(runtime) {
+AbstractWorkStealingScheduler::AbstractWorkStealingScheduler(Runtime& runtime,
+ RuntimeStrategy& strategy)
+ : Scheduler(runtime, strategy) {
const workerid_t workerCount = runtime.getWorkerCount();
queues = new AbstractWorkStealingScheduler::WsQueue<QUEUE_SIZE>*[workerCount];
@@ -48,14 +48,24 @@ void AbstractWorkStealingScheduler::scheduleViaWorkStealing(Fiber& fiber) {
onNewWork();
}
+auto AbstractWorkStealingScheduler::maybeRecycle(Fiber* fiber) -> bool {
+ if (fiber->isRunnable()) return false;
+
+ recycle(fiber);
+ return true;
+}
+
auto AbstractWorkStealingScheduler::nextFiberViaWorkStealing() -> std::pair<Fiber*, FiberSource> {
FiberSource fiberSource = FiberSource::local;
Fiber* fiber;
+popBottom:
bool poped = queue.popBottom(&fiber);
if (likely(poped)) {
emper::statsIncr(awss::stats.nextFiberFromLocal);
+ if (maybeRecycle(fiber)) goto popBottom;
+
goto out;
}
@@ -70,21 +80,27 @@ auto AbstractWorkStealingScheduler::nextFiberViaWorkStealing() -> std::pair<Fibe
// Don't steal from ourselves.
if (unlikely(victim == myWorkerId)) continue;
+ popTop:
poped = queues[victim]->popTop(&fiber);
if (poped) {
emper::statsIncr(awss::stats.nextFiberStolen);
+ if (maybeRecycle(fiber)) goto popTop;
+
fiberSource = FiberSource::stolen;
goto out;
}
}
}
+dequeueFromAnywhere:
// Try the "scheduled from anywhere" queue to get work as last resort.
fiber = dequeueFiberFromAnywhereQueue();
if (fiber) {
emper::statsIncr(awss::stats.nextFiberFromAnywhereQueue);
+ if (maybeRecycle(fiber)) goto dequeueFromAnywhere;
+
fiberSource = FiberSource::anywhereQueue;
}
diff --git a/emper/strategies/AbstractWorkStealingScheduler.hpp b/emper/strategies/AbstractWorkStealingScheduler.hpp
index 64071b71c28f8572aac2c7165bbd18711f50b5f1..e482243b50e4cbced073810220b028427248afbb 100644
--- a/emper/strategies/AbstractWorkStealingScheduler.hpp
+++ b/emper/strategies/AbstractWorkStealingScheduler.hpp
@@ -17,6 +17,7 @@
class Fiber;
class Runtime;
+class RuntimeStrategy;
class AbstractWorkStealingScheduler : public Scheduler {
template <size_t SIZE>
@@ -41,10 +42,12 @@ class AbstractWorkStealingScheduler : public Scheduler {
void scheduleViaWorkStealing(Fiber& fiber);
+ auto maybeRecycle(Fiber* fiber) -> bool;
+
auto nextFiberViaWorkStealing() -> std::pair<Fiber*, FiberSource>;
auto nextFiberResultViaWorkStealing() -> NextFiberResult;
public:
- AbstractWorkStealingScheduler(Runtime& runtime);
+ AbstractWorkStealingScheduler(Runtime& runtime, RuntimeStrategy& strategy);
};
diff --git a/emper/strategies/laws/LawsScheduler.cpp b/emper/strategies/laws/LawsScheduler.cpp
index 138d9ef47a7f6c3016e44a02781bcbab66def4dc..8066c5cb03c5a3efb05ac8ab43efefcb7f4e5dd9 100644
--- a/emper/strategies/laws/LawsScheduler.cpp
+++ b/emper/strategies/laws/LawsScheduler.cpp
@@ -14,7 +14,8 @@
thread_local LawsScheduler::LawsMpscQueue LawsScheduler::priorityQueue;
-LawsScheduler::LawsScheduler(Runtime& runtime) : AbstractWorkStealingScheduler(runtime) {
+LawsScheduler::LawsScheduler(Runtime& runtime, RuntimeStrategy& strategy)
+ : AbstractWorkStealingScheduler(runtime, strategy) {
const workerid_t workerCount = runtime.getWorkerCount();
priorityQueues = new LawsScheduler::LawsMpscQueue*[workerCount];
diff --git a/emper/strategies/laws/LawsScheduler.hpp b/emper/strategies/laws/LawsScheduler.hpp
index 8f9c168288c1029435ccd936e7b4cc392f0b40a7..f4b0f242418e48ae92825736456979666913a6c5 100644
--- a/emper/strategies/laws/LawsScheduler.hpp
+++ b/emper/strategies/laws/LawsScheduler.hpp
@@ -8,6 +8,7 @@
#include "strategies/AbstractWorkStealingScheduler.hpp"
class Runtime;
+class RuntimeStrategy;
struct NextFiberResult;
class LawsScheduler : public AbstractWorkStealingScheduler {
@@ -27,7 +28,7 @@ class LawsScheduler : public AbstractWorkStealingScheduler {
void scheduleFromAnywhereInternal(Fiber** fibers, unsigned count) override;
public:
- LawsScheduler(Runtime& runtime);
+ LawsScheduler(Runtime& runtime, RuntimeStrategy& strategy);
auto nextFiber() -> NextFiberResult override;
};
diff --git a/emper/strategies/laws/LawsStrategy.cpp b/emper/strategies/laws/LawsStrategy.cpp
index 57116344cd124bab6f6a48d604cc7f97aacc6683..603391b469f8b1273dafccf0142dc37b1670c4b6 100644
--- a/emper/strategies/laws/LawsStrategy.cpp
+++ b/emper/strategies/laws/LawsStrategy.cpp
@@ -13,7 +13,7 @@ thread_local LawsWorkerStats LawsStrategy::stats;
LawsStrategy::LawsStrategy(Runtime& runtime)
: AbstractWorkStealingStrategy(runtime),
- scheduler(runtime),
+ scheduler(runtime, *this),
dispatcher(runtime),
allWorkerStats([] { return &stats; }, runtime) {}
diff --git a/emper/strategies/ws/WsScheduler.cpp b/emper/strategies/ws/WsScheduler.cpp
index b57510cb6ec60bc7704a4c86d792d6e23c904a2d..ae98ce217b45cfafff75f2397bf0da215c976a1d 100644
--- a/emper/strategies/ws/WsScheduler.cpp
+++ b/emper/strategies/ws/WsScheduler.cpp
@@ -2,4 +2,5 @@
// Copyright © 2020-2021 Florian Schmaus
#include "WsScheduler.hpp"
-WsScheduler::WsScheduler(Runtime& runtime) : AbstractWorkStealingScheduler(runtime) {}
+WsScheduler::WsScheduler(Runtime& runtime, RuntimeStrategy& strategy)
+ : AbstractWorkStealingScheduler(runtime, strategy) {}
diff --git a/emper/strategies/ws/WsScheduler.hpp b/emper/strategies/ws/WsScheduler.hpp
index c122f301a6cc817a9fe567154bde5b8ac7ca1707..c0ce9c3b8766f7b673e71a34f04ac24974e1e27c 100644
--- a/emper/strategies/ws/WsScheduler.hpp
+++ b/emper/strategies/ws/WsScheduler.hpp
@@ -8,6 +8,7 @@
class Fiber;
class Runtime;
+class RuntimeStrategy;
class WsScheduler : public AbstractWorkStealingScheduler {
protected:
@@ -23,7 +24,7 @@ class WsScheduler : public AbstractWorkStealingScheduler {
}
public:
- WsScheduler(Runtime& runtime);
+ WsScheduler(Runtime& runtime, RuntimeStrategy& strategy);
auto nextFiber() -> NextFiberResult override { return nextFiberResultViaWorkStealing(); };
};
diff --git a/emper/strategies/ws/WsStrategy.cpp b/emper/strategies/ws/WsStrategy.cpp
index 11fcd3d02223f41f793f0703dfc8c8f7822596a4..b6eb19e0c03ed9b2821ee7b68eeda6d9f1d1b36d 100644
--- a/emper/strategies/ws/WsStrategy.cpp
+++ b/emper/strategies/ws/WsStrategy.cpp
@@ -10,7 +10,7 @@ class Runtime;
class RuntimeStrategyStats;
WsStrategy::WsStrategy(Runtime& runtime)
- : AbstractWorkStealingStrategy(runtime), scheduler(runtime), dispatcher(runtime) {}
+ : AbstractWorkStealingStrategy(runtime), scheduler(runtime, *this), dispatcher(runtime) {}
auto WsStrategy::getScheduler() -> WsScheduler& { return scheduler; }