From 919e6b3f19439efa5d0e6c0fd395724c27f40e04 Mon Sep 17 00:00:00 2001
From: Florian Schmaus <flow@cs.fau.de>
Date: Fri, 9 Apr 2021 17:36:58 +0200
Subject: [PATCH] Add WorkerLocalData and use it for stats

---
 emper/Emper.hpp                               |  9 +++-
 emper/Runtime.cpp                             | 41 ++++++++++++++++---
 emper/Runtime.hpp                             | 11 ++++-
 emper/WorkerLocalData.hpp                     | 39 ++++++++++++++++++
 .../AbstractWorkStealingScheduler.cpp         | 31 +++++---------
 .../AbstractWorkStealingScheduler.hpp         | 15 +------
 .../strategies/AbstractWorkStealingStats.cpp  | 29 +++++++------
 .../strategies/AbstractWorkStealingStats.hpp  | 10 ++---
 .../AbstractWorkStealingStrategy.cpp          |  7 ++++
 .../AbstractWorkStealingStrategy.hpp          | 19 +++++----
 .../AbstractWorkStealingWorkerStats.hpp       | 23 +++++++++++
 emper/strategies/laws/LawsDispatcher.cpp      |  1 +
 emper/strategies/laws/LawsDispatcher.hpp      |  5 ++-
 emper/strategies/laws/LawsScheduler.cpp       |  2 +-
 emper/strategies/laws/LawsScheduler.hpp       |  3 +-
 emper/strategies/laws/LawsStrategy.cpp        |  3 +-
 emper/strategies/meson.build                  |  1 +
 emper/strategies/ws/WsScheduler.cpp           |  7 +---
 emper/strategies/ws/WsScheduler.hpp           | 13 +-----
 emper/strategies/ws/WsStrategy.cpp            |  3 +-
 20 files changed, 179 insertions(+), 93 deletions(-)
 create mode 100644 emper/WorkerLocalData.hpp
 create mode 100644 emper/strategies/AbstractWorkStealingStrategy.cpp
 create mode 100644 emper/strategies/AbstractWorkStealingWorkerStats.hpp

diff --git a/emper/Emper.hpp b/emper/Emper.hpp
index 2bfbf49f..9c7e0664 100644
--- a/emper/Emper.hpp
+++ b/emper/Emper.hpp
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
-// Copyright © 2020 Florian Schmaus
+// Copyright © 2020-2021 Florian Schmaus
 #pragma once
 
 #include <string>
@@ -16,6 +16,13 @@ static const bool STATS =
 #endif
 		;
 
+template <typename C>
+void statsIncr(C& counter) {
+	if constexpr (STATS) {
+		counter++;
+	}
+}
+
 static const bool WORKER_SLEEP =
 #ifdef EMPER_WORKER_SLEEP
 		true
diff --git a/emper/Runtime.cpp b/emper/Runtime.cpp
index 9969fcc0..d604ceb8 100644
--- a/emper/Runtime.cpp
+++ b/emper/Runtime.cpp
@@ -11,8 +11,8 @@
 
 #include <cstdlib>	// for rand, srand, abort
 #include <cstring>
+#include <iostream>
 #include <memory>	 // for __shared_ptr_access, shared_ptr
-#include <ostream>
 #include <string>	 // for string
 #include <thread>
 
@@ -67,7 +67,8 @@ using emper::io::IoContext;
 Runtime::Runtime(workerid_t workerCount, RuntimeStrategyFactory& strategyFactory, unsigned int seed)
 		: workerCount(workerCount),
 			workerLatch(workerCount),
-			workerThreadExitLatch(workerCount),
+			firstWorkerThreadExitLatch(workerCount),
+			secondWorkerThreadExitLatch(workerCount),
 			strategy(strategyFactory.constructRuntimeStrategy(*this)),
 			scheduler(strategy->getScheduler()),
 			dispatcher(strategy->getDispatcher()),
@@ -236,7 +237,14 @@ auto Runtime::workerLoop(Worker* worker) -> void* {
 	// it would be possible that one thread is work-stealing,
 	// potentially accessing a work stealing queue of an worker thread
 	// that already exited, causing an invalid access.
-	workerThreadExitLatch.count_down_and_wait();
+	firstWorkerThreadExitLatch.count_down_and_wait();
+
+	if (worker->getWorkerId() == 0) {
+		// Obtain stats when all workers reached the first latch but before they exit.
+		strategyLastStats = strategy->getStats();
+	}
+
+	secondWorkerThreadExitLatch.count_down_and_wait();
 
 	return nullptr;
 }
@@ -312,8 +320,31 @@ void Runtime::initiateAndWaitUntilTermination() {
 }
 
 void Runtime::printStats() {
-	auto runtimeStrategyStats = strategy->getStats();
-	runtimeStrategyStats->print();
+	if constexpr (!emper::STATS) {
+		std::cout << "printStats(): EMPER stats disabled by static configuration" << std::endl;
+		return;
+	}
+
+	std::shared_ptr<RuntimeStrategyStats> stats;
+	if (terminateWorkers) {
+		// The runtime was already requested to shut down. We can not
+		// obtain stats as it is not guranteed that the worker threads are
+		// still running in this case. We can however check
+		// strategyLastStats and print that.
+
+		if (strategyLastStats) {
+			stats = strategyLastStats;
+		} else {
+			std::cerr << "printStats(): worker threads have already been scheduled for termination, but "
+									 "not last stats recored yet."
+								<< std::endl;
+			return;
+		}
+	} else {
+		stats = strategy->getStats();
+	}
+
+	stats->print();
 }
 
 void Runtime::printLastRuntimeStats() {
diff --git a/emper/Runtime.hpp b/emper/Runtime.hpp
index 621eda41..993bd24f 100644
--- a/emper/Runtime.hpp
+++ b/emper/Runtime.hpp
@@ -9,7 +9,8 @@
 #include <cstdint>		 // for intptr_t
 #include <cstdlib>		 // for abort
 #include <functional>	 // for function
-#include <mutex>			 // for mutex, lock_guard, unique_lock
+#include <memory>
+#include <mutex>
 #include <random>
 #include <vector>	 // for vector
 
@@ -31,6 +32,7 @@ class Dispatcher;
 class Fiber;
 class RuntimeStrategy;
 class RuntimeStrategyFactory;
+class RuntimeStrategyStats;
 
 namespace emper::io {
 class GlobalIoContext;
@@ -51,7 +53,8 @@ class Runtime : public Logger<LogSubsystem::RUNTI> {
 	std::vector<std::function<void(workerid_t)>> newWorkerHooks;
 
 	Latch workerLatch;
-	Latch workerThreadExitLatch;
+	Latch firstWorkerThreadExitLatch;
+	Latch secondWorkerThreadExitLatch;
 
 	RuntimeStrategy* const strategy;
 	Scheduler& scheduler;
@@ -70,6 +73,8 @@ class Runtime : public Logger<LogSubsystem::RUNTI> {
 
 	bool threadsRunning = false;
 
+	std::shared_ptr<RuntimeStrategyStats> strategyLastStats;
+
 	auto workerLoop(Worker* worker) -> void*;
 
 	ALIGN_TO_CACHE_LINE WorkerWakeupSemaphore wakeupSem;
@@ -221,4 +226,6 @@ class Runtime : public Logger<LogSubsystem::RUNTI> {
 	friend IoContext;
 	template <typename T, intptr_t WS_QUEUE_SIZE, size_t WORKER_EXCLUSIVE_QUEUE_SIZE>
 	friend class MemoryManager;
+	template <typename>
+	friend class WorkerLocalData;
 };
diff --git a/emper/WorkerLocalData.hpp b/emper/WorkerLocalData.hpp
new file mode 100644
index 00000000..a1adbb60
--- /dev/null
+++ b/emper/WorkerLocalData.hpp
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright © 2021 Florian Schmaus
+#pragma once
+
+#include <functional>
+
+#include "Runtime.hpp"
+
+template <typename D>
+class WorkerLocalData {
+ private:
+	std::function<D*(void)> getWorkerLocalPtrCallback;
+	std::vector<D*> workerLocalData;
+
+ public:
+	WorkerLocalData(std::function<D*(void)> getWorkerLocalPtrCallback, Runtime& runtime)
+			: getWorkerLocalPtrCallback(getWorkerLocalPtrCallback) {
+		auto workerCount = runtime.getWorkerCount();
+		workerLocalData = std::vector<D*>(workerCount);
+		runtime.addNewWorkerHook([this, getWorkerLocalPtrCallback](workerid_t workerId) {
+			D* workerLocalDataPtr = getWorkerLocalPtrCallback();
+			this->workerLocalData.at(workerId) = workerLocalDataPtr;
+		});
+	}
+
+	auto getSnapshot() -> std::vector<D> {
+		std::vector<D> res;
+		for (D* data : workerLocalData) {
+			res.emplace_back(*data);
+		}
+		return res;
+	}
+
+	void forEach(std::function<void(D*)> callback) {
+		for (D* data : workerLocalData) {
+			callback(data);
+		}
+	}
+};
diff --git a/emper/strategies/AbstractWorkStealingScheduler.cpp b/emper/strategies/AbstractWorkStealingScheduler.cpp
index d78614a6..7ca37227 100644
--- a/emper/strategies/AbstractWorkStealingScheduler.cpp
+++ b/emper/strategies/AbstractWorkStealingScheduler.cpp
@@ -2,7 +2,6 @@
 // Copyright © 2021 Florian Schmaus
 #include "AbstractWorkStealingScheduler.hpp"
 
-#include <atomic>
 #include <ostream>	// for operator<<, basic_ostream<>::__ostream_type
 
 #include "Common.hpp"	 // for unlikely, likely
@@ -12,15 +11,17 @@
 #include "Runtime.hpp"		 // for Runtime
 #include "emper-common.h"	 // for workerid_t
 #include "strategies/AbstractWorkStealingStrategy.hpp"
+#include "strategies/AbstractWorkStealingWorkerStats.hpp"
 
 class Fiber;
 
+using awss = AbstractWorkStealingStrategy;
+
 thread_local AbstractWorkStealingScheduler::WsQueue<AbstractWorkStealingScheduler::QUEUE_SIZE>
 		AbstractWorkStealingScheduler::queue;
 
-AbstractWorkStealingScheduler::AbstractWorkStealingScheduler(
-		Runtime& runtime, AbstractWorkStealingStrategy& abstractWorkStealingStrategy)
-		: Scheduler(runtime), abstractWorkStealingStrategy(abstractWorkStealingStrategy) {
+AbstractWorkStealingScheduler::AbstractWorkStealingScheduler(Runtime& runtime)
+		: Scheduler(runtime) {
 	const workerid_t workerCount = runtime.getWorkerCount();
 	queues = new AbstractWorkStealingScheduler::WsQueue<QUEUE_SIZE>*[workerCount];
 
@@ -34,17 +35,12 @@ void AbstractWorkStealingScheduler::scheduleViaWorkStealing(Fiber& fiber) {
 		if constexpr (emper::OVERFLOW_QUEUE) {
 			enqueueInAnywhereQueue(fiber);
 
-			if constexpr (emper::STATS) {
-				// TODO: Use template magic so that this becomes
-				// incrementRelaxed(abstractWorkStealingStrategy.scheduledFibersToLocal)
-				abstractWorkStealingStrategy.scheduledFibersToOverflowQueue.fetch_add(
-						1, std::memory_order_relaxed);
-			}
+			emper::statsIncr(awss::stats.scheduledFibersToOverflowQueue);
 		} else {
 			ABORT("Could not push fiber " << &fiber << " into queue");
 		}
 	} else if constexpr (emper::STATS) {
-		abstractWorkStealingStrategy.scheduledFibersToLocal.fetch_add(1, std::memory_order_relaxed);
+		awss::stats.scheduledFibersToLocal++;
 	}
 
 	// Classes using this method are supposed to always invoke this
@@ -58,9 +54,7 @@ auto AbstractWorkStealingScheduler::nextFiberViaWorkStealing() -> std::pair<Fibe
 
 	bool poped = queue.popBottom(&fiber);
 	if (likely(poped)) {
-		if constexpr (emper::STATS) {
-			abstractWorkStealingStrategy.nextFiberFromLocal.fetch_add(1, std::memory_order_relaxed);
-		}
+		emper::statsIncr(awss::stats.nextFiberFromLocal);
 
 		goto out;
 	}
@@ -78,9 +72,7 @@ auto AbstractWorkStealingScheduler::nextFiberViaWorkStealing() -> std::pair<Fibe
 
 			poped = queues[victim]->popTop(&fiber);
 			if (poped) {
-				if constexpr (emper::STATS) {
-					abstractWorkStealingStrategy.nextFiberStolen.fetch_add(1, std::memory_order_relaxed);
-				}
+				emper::statsIncr(awss::stats.nextFiberStolen);
 
 				fiberSource = FiberSource::stolen;
 				goto out;
@@ -91,10 +83,7 @@ auto AbstractWorkStealingScheduler::nextFiberViaWorkStealing() -> std::pair<Fibe
 	// Try the "scheduled from anywhere" queue to get work as last resort.
 	fiber = dequeueFiberFromAnywhereQueue();
 	if (fiber) {
-		if constexpr (emper::STATS) {
-			abstractWorkStealingStrategy.nextFiberFromAnywhereQueue.fetch_add(1,
-																																				std::memory_order_relaxed);
-		}
+		emper::statsIncr(awss::stats.nextFiberFromAnywhereQueue);
 
 		fiberSource = FiberSource::anywhereQueue;
 	}
diff --git a/emper/strategies/AbstractWorkStealingScheduler.hpp b/emper/strategies/AbstractWorkStealingScheduler.hpp
index ce0d378d..64071b71 100644
--- a/emper/strategies/AbstractWorkStealingScheduler.hpp
+++ b/emper/strategies/AbstractWorkStealingScheduler.hpp
@@ -8,7 +8,6 @@
 
 #include "NextFiberResult.hpp"
 #include "Scheduler.hpp"
-#include "emper-common.h"
 
 #ifdef EMPER_LOCKED_WS_QUEUE
 #include "lib/adt/LockedQueue.hpp"
@@ -18,7 +17,6 @@
 
 class Fiber;
 class Runtime;
-class AbstractWorkStealingStrategy;
 
 class AbstractWorkStealingScheduler : public Scheduler {
 	template <size_t SIZE>
@@ -28,16 +26,6 @@ class AbstractWorkStealingScheduler : public Scheduler {
 	using WsQueue = adt::WsClQueue<Fiber*, SIZE>;
 #endif
 
- private:
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wattributes"
-	AbstractWorkStealingStrategy& abstractWorkStealingStrategy
-#ifndef EMPER_STATS
-			ATTR_UNUSED
-#endif
-			;
-#pragma GCC diagnostic pop
-
  public:
 	static const int QUEUE_SIZE = 1024;
 
@@ -58,6 +46,5 @@ class AbstractWorkStealingScheduler : public Scheduler {
 	auto nextFiberResultViaWorkStealing() -> NextFiberResult;
 
  public:
-	AbstractWorkStealingScheduler(Runtime& runtime,
-																AbstractWorkStealingStrategy& abstractWorkStealingStrategy);
+	AbstractWorkStealingScheduler(Runtime& runtime);
 };
diff --git a/emper/strategies/AbstractWorkStealingStats.cpp b/emper/strategies/AbstractWorkStealingStats.cpp
index 7d6c5860..a8e833ee 100644
--- a/emper/strategies/AbstractWorkStealingStats.cpp
+++ b/emper/strategies/AbstractWorkStealingStats.cpp
@@ -2,23 +2,28 @@
 // Copyright © 2021 Florian Schmaus
 #include "strategies/AbstractWorkStealingStats.hpp"
 
-#include <atomic>
 #include <iostream>
+#include <string>
 
+#include "WorkerLocalData.hpp"
 #include "strategies/AbstractWorkStealingStrategy.hpp"
 
 AbstractWorkStealingStats::AbstractWorkStealingStats(AbstractWorkStealingStrategy& strategy)
-		: scheduledFibersToLocal(strategy.scheduledFibersToLocal),
-			scheduledFibersToOverflowQueue(strategy.scheduledFibersToOverflowQueue),
-			nextFiberFromLocal(strategy.nextFiberFromLocal),
-			nextFiberStolen(strategy.nextFiberStolen),
-			nextFiberFromAnywhereQueue(strategy.nextFiberFromAnywhereQueue) {}
+		: workerStats(strategy.allWorkerStats.getSnapshot()) {
+	for (auto& workerStat : workerStats) {
+		comulatedWorkerStats += workerStat;
+	}
+}
 
 void AbstractWorkStealingStats::print() {
-	std::cout << "AbstractWorkStealingStats"
-						<< " scheduledFibersToLocal:" << scheduledFibersToLocal
-						<< " scheduledFibersToOverflowQueue:" << scheduledFibersToOverflowQueue
-						<< " nextFiberFromLocal:" << nextFiberFromLocal
-						<< " nextFiberStolen:" << nextFiberStolen
-						<< " nextFiberFromAnywhereQueue:" << nextFiberFromAnywhereQueue << std::endl;
+	std::cout << "total-scheduled-fibers-to-local: "
+						<< std::to_string(comulatedWorkerStats.scheduledFibersToLocal) << std::endl
+						<< "total-scheduled-fibers-to-overflow-queue: "
+						<< std::to_string(comulatedWorkerStats.scheduledFibersToOverflowQueue) << std::endl
+						<< "total-next-fiber-from-local: "
+						<< std::to_string(comulatedWorkerStats.nextFiberFromLocal) << std::endl
+						<< "total-next-fiber-stolen: " << std::to_string(comulatedWorkerStats.nextFiberStolen)
+						<< std::endl
+						<< "total-next-fiber-from-anywhere-queue: "
+						<< std::to_string(comulatedWorkerStats.nextFiberFromAnywhereQueue) << std::endl;
 }
diff --git a/emper/strategies/AbstractWorkStealingStats.hpp b/emper/strategies/AbstractWorkStealingStats.hpp
index a5f86150..f8d19002 100644
--- a/emper/strategies/AbstractWorkStealingStats.hpp
+++ b/emper/strategies/AbstractWorkStealingStats.hpp
@@ -2,19 +2,17 @@
 // Copyright © 2021 Florian Schmaus
 #pragma once
 
-#include <cstdint>
+#include <vector>
 
 #include "RuntimeStrategyStats.hpp"
+#include "strategies/AbstractWorkStealingWorkerStats.hpp"
 
 class AbstractWorkStealingStrategy;
 
 class AbstractWorkStealingStats : public RuntimeStrategyStats {
  public:
-	const uint64_t scheduledFibersToLocal;
-	const uint64_t scheduledFibersToOverflowQueue;
-	const uint64_t nextFiberFromLocal;
-	const uint64_t nextFiberStolen;
-	const uint64_t nextFiberFromAnywhereQueue;
+	std::vector<AbstractWorkStealingWorkerStats> workerStats;
+	AbstractWorkStealingWorkerStats comulatedWorkerStats;
 
 	AbstractWorkStealingStats(AbstractWorkStealingStrategy &strategy);
 
diff --git a/emper/strategies/AbstractWorkStealingStrategy.cpp b/emper/strategies/AbstractWorkStealingStrategy.cpp
new file mode 100644
index 00000000..ee866d13
--- /dev/null
+++ b/emper/strategies/AbstractWorkStealingStrategy.cpp
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright © 2021 Florian Schmaus
+#include "strategies/AbstractWorkStealingStrategy.hpp"
+
+#include "strategies/AbstractWorkStealingWorkerStats.hpp"
+
+thread_local AbstractWorkStealingWorkerStats AbstractWorkStealingStrategy::stats;
diff --git a/emper/strategies/AbstractWorkStealingStrategy.hpp b/emper/strategies/AbstractWorkStealingStrategy.hpp
index 090b7ffb..597ab09b 100644
--- a/emper/strategies/AbstractWorkStealingStrategy.hpp
+++ b/emper/strategies/AbstractWorkStealingStrategy.hpp
@@ -1,25 +1,26 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
-// Copyright © 2020 Florian Schmaus
+// Copyright © 2020-2021 Florian Schmaus
 #pragma once
 
-#include <atomic>
-#include <cstdint>
-#include <memory>
+#include <functional>
 
 #include "RuntimeStrategy.hpp"
+#include "WorkerLocalData.hpp"
 
 class AbstractWorkStealingScheduler;
 class AbstractWorkStealingStats;
+class Runtime;
+struct AbstractWorkStealingWorkerStats;
 
 class AbstractWorkStealingStrategy : public RuntimeStrategy {
+ public:
  private:
-	std::atomic<std::uint64_t> scheduledFibersToLocal;
-	std::atomic<std::uint64_t> scheduledFibersToOverflowQueue;
-	std::atomic<std::uint64_t> nextFiberFromLocal;
-	std::atomic<std::uint64_t> nextFiberStolen;
-	std::atomic<std::uint64_t> nextFiberFromAnywhereQueue;
+	static thread_local AbstractWorkStealingWorkerStats stats;
+
+	WorkerLocalData<AbstractWorkStealingWorkerStats> allWorkerStats;
 
  protected:
+	AbstractWorkStealingStrategy(Runtime& runtime) : allWorkerStats([] { return &stats; }, runtime) {}
 	~AbstractWorkStealingStrategy() override = default;
 
 	friend AbstractWorkStealingScheduler;
diff --git a/emper/strategies/AbstractWorkStealingWorkerStats.hpp b/emper/strategies/AbstractWorkStealingWorkerStats.hpp
new file mode 100644
index 00000000..19394a5a
--- /dev/null
+++ b/emper/strategies/AbstractWorkStealingWorkerStats.hpp
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright © 2020-2021 Florian Schmaus
+#pragma once
+
+#include <cstdint>
+
+struct AbstractWorkStealingWorkerStats {
+	uint64_t scheduledFibersToLocal = 0;
+	uint64_t scheduledFibersToOverflowQueue = 0;
+	uint64_t nextFiberFromLocal = 0;
+	uint64_t nextFiberStolen = 0;
+	uint64_t nextFiberFromAnywhereQueue = 0;
+
+	auto operator+=(const AbstractWorkStealingWorkerStats& other)
+			-> AbstractWorkStealingWorkerStats& {
+		scheduledFibersToLocal += other.scheduledFibersToLocal;
+		scheduledFibersToOverflowQueue += other.scheduledFibersToOverflowQueue;
+		nextFiberFromLocal += other.nextFiberFromLocal;
+		nextFiberStolen += other.nextFiberStolen;
+		nextFiberFromAnywhereQueue = other.nextFiberFromAnywhereQueue;
+		return *this;
+	}
+};
diff --git a/emper/strategies/laws/LawsDispatcher.cpp b/emper/strategies/laws/LawsDispatcher.cpp
index 0edaa153..698b7d27 100644
--- a/emper/strategies/laws/LawsDispatcher.cpp
+++ b/emper/strategies/laws/LawsDispatcher.cpp
@@ -9,6 +9,7 @@
 #include "LawsStrategy.hpp"	 // for LawsStrategy, LawsStrategy::FiberSource
 #include "NextFiberResult.hpp"
 #include "Runtime.hpp"
+#include "emper-common.h"
 
 void LawsDispatcher::recycle(Fiber* fiber) {
 	// If the ref count has not reached zero yet, do not recycle the
diff --git a/emper/strategies/laws/LawsDispatcher.hpp b/emper/strategies/laws/LawsDispatcher.hpp
index fec816a1..e18f4ead 100644
--- a/emper/strategies/laws/LawsDispatcher.hpp
+++ b/emper/strategies/laws/LawsDispatcher.hpp
@@ -1,9 +1,10 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
-// Copyright © 2020 Florian Schmaus
+// Copyright © 2020-2021 Florian Schmaus
 #pragma once
 
 #include "Dispatcher.hpp"
-#include "emper-common.h"
+#include "emper-common.h"	 // IWYU pragma: keep
+#include "emper-config.h"	 // IWYU pragma: keep
 
 class Fiber;
 class LawsStrategy;
diff --git a/emper/strategies/laws/LawsScheduler.cpp b/emper/strategies/laws/LawsScheduler.cpp
index d05f9c62..04647b97 100644
--- a/emper/strategies/laws/LawsScheduler.cpp
+++ b/emper/strategies/laws/LawsScheduler.cpp
@@ -14,7 +14,7 @@
 thread_local LawsScheduler::LawsMpscQueue LawsScheduler::priorityQueue;
 
 LawsScheduler::LawsScheduler(Runtime& runtime, LawsStrategy& lawsStrategy)
-		: AbstractWorkStealingScheduler(runtime, lawsStrategy), lawsStrategy(lawsStrategy) {
+		: AbstractWorkStealingScheduler(runtime), lawsStrategy(lawsStrategy) {
 	const workerid_t workerCount = runtime.getWorkerCount();
 	priorityQueues = new LawsScheduler::LawsMpscQueue*[workerCount];
 
diff --git a/emper/strategies/laws/LawsScheduler.hpp b/emper/strategies/laws/LawsScheduler.hpp
index a2fd6184..a1edcaf9 100644
--- a/emper/strategies/laws/LawsScheduler.hpp
+++ b/emper/strategies/laws/LawsScheduler.hpp
@@ -3,7 +3,8 @@
 #pragma once
 
 #include "Fiber.hpp"
-#include "emper-common.h"
+#include "emper-common.h"	 // IWYU pragma: keep
+#include "emper-config.h"	 // IWYU pragma: keep
 #include "lib/adt/MpscQueue.hpp"
 #include "strategies/AbstractWorkStealingScheduler.hpp"
 
diff --git a/emper/strategies/laws/LawsStrategy.cpp b/emper/strategies/laws/LawsStrategy.cpp
index 68bb821e..a12a5ad0 100644
--- a/emper/strategies/laws/LawsStrategy.cpp
+++ b/emper/strategies/laws/LawsStrategy.cpp
@@ -7,7 +7,8 @@
 #include "strategies/laws/LawsStrategyStats.hpp"	// for LawsStrategyStats
 
 LawsStrategy::LawsStrategy(Runtime& runtime)
-		: scheduler(runtime, *this),
+		: AbstractWorkStealingStrategy(runtime),
+			scheduler(runtime, *this),
 			dispatcher(runtime, *this),
 			scheduledFibersToRemotePriority(0),
 			dispatchedFiberFromPriority(0),
diff --git a/emper/strategies/meson.build b/emper/strategies/meson.build
index 2586f72a..6858f902 100644
--- a/emper/strategies/meson.build
+++ b/emper/strategies/meson.build
@@ -1,6 +1,7 @@
 emper_cpp_sources += files(
   'AbstractWorkStealingScheduler.cpp',
   'AbstractWorkStealingStats.cpp',
+  'AbstractWorkStealingStrategy.cpp',
 )
 
 subdir('ws')
diff --git a/emper/strategies/ws/WsScheduler.cpp b/emper/strategies/ws/WsScheduler.cpp
index 41c1b0a5..b57510cb 100644
--- a/emper/strategies/ws/WsScheduler.cpp
+++ b/emper/strategies/ws/WsScheduler.cpp
@@ -1,8 +1,5 @@
 // SPDX-License-Identifier: LGPL-3.0-or-later
-// Copyright © 2020 Florian Schmaus
+// Copyright © 2020-2021 Florian Schmaus
 #include "WsScheduler.hpp"
 
-#include "strategies/ws/WsStrategy.hpp"
-
-WsScheduler::WsScheduler(Runtime& runtime, WsStrategy& wsStrategy)
-		: AbstractWorkStealingScheduler(runtime, wsStrategy), wsStrategy(wsStrategy) {}
+WsScheduler::WsScheduler(Runtime& runtime) : AbstractWorkStealingScheduler(runtime) {}
diff --git a/emper/strategies/ws/WsScheduler.hpp b/emper/strategies/ws/WsScheduler.hpp
index 63a634d3..392b0aff 100644
--- a/emper/strategies/ws/WsScheduler.hpp
+++ b/emper/strategies/ws/WsScheduler.hpp
@@ -3,28 +3,17 @@
 #pragma once
 
 #include "NextFiberResult.hpp"
-#include "emper-common.h"	 // for ATTR_UNUSED
 #include "strategies/AbstractWorkStealingScheduler.hpp"
 
 class Fiber;
 class Runtime;
-class WsStrategy;
 
 class WsScheduler : public AbstractWorkStealingScheduler {
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wattributes"
-	WsStrategy& wsStrategy
-#ifndef EMPER_STATS
-			ATTR_UNUSED
-#endif
-			;
-#pragma GCC diagnostic pop
-
  protected:
 	void scheduleInternal(Fiber& fiber) override { scheduleViaWorkStealing(fiber); }
 
  public:
-	WsScheduler(Runtime& runtime, WsStrategy& wsStrategy);
+	WsScheduler(Runtime& runtime);
 
 	auto nextFiber() -> NextFiberResult override { return nextFiberResultViaWorkStealing(); };
 };
diff --git a/emper/strategies/ws/WsStrategy.cpp b/emper/strategies/ws/WsStrategy.cpp
index 72bc6fe4..11fcd3d0 100644
--- a/emper/strategies/ws/WsStrategy.cpp
+++ b/emper/strategies/ws/WsStrategy.cpp
@@ -9,7 +9,8 @@
 class Runtime;
 class RuntimeStrategyStats;
 
-WsStrategy::WsStrategy(Runtime& runtime) : scheduler(runtime, *this), dispatcher(runtime) {}
+WsStrategy::WsStrategy(Runtime& runtime)
+		: AbstractWorkStealingStrategy(runtime), scheduler(runtime), dispatcher(runtime) {}
 
 auto WsStrategy::getScheduler() -> WsScheduler& { return scheduler; }
 
-- 
GitLab