add scheduler support for IO hints

emper/FiberSource.hpp

@@ -20,6 +20,7 @@ enum class FiberSource : uintptr_t {
 	ioStolen,			 /*!< A other worker's io_uring completion queue */
 	anywhereQueue, /*!< The anywhere queue */
 	hintWsq,			 /*!< A known other worker's work-stealing queue */
+	hintIo,				 /*!< A known worker's io_uring completion queue */
 	hintAq,				 /*!< Straight from the anywhere queue */
 };
 

emper/strategies/AbstractWorkStealingScheduler.cpp

@@ -149,11 +149,24 @@ auto AbstractWorkStealingScheduler::nextFiberViaAnywhereQueue() -> std::optional
 	return std::nullopt;
 }
 
-auto AbstractWorkStealingScheduler::tryStealFiberFrom(workerid_t victim)
+auto AbstractWorkStealingScheduler::tryStealIoCompletionFrom(workerid_t victim)
+		-> std::optional<NextFiberResult> {
+	auto* victimIo = runtime.ioContexts[victim];
+	Fiber* fiber = victimIo->reapSingleCompletion<CallerEnvironment::EMPER>();
+	if (fiber) {
+		emper::statsIncr(awss::stats.nextIoFiberStolen);
+		return NextFiberResult{fiber, emper::FiberSource::ioStolen};
+	}
+
+	return std::nullopt;
+}
+
+auto AbstractWorkStealingScheduler::tryStealWorkFrom(workerid_t victim)
 		-> std::optional<NextFiberResult> {
 	constexpr int maxRetries = emper::WAITFREE_WORK_STEALING ? 0 : -1;
 	AbstractFiber* fiber;
 popTop:
+
 	StealingResult res = queues[victim]->popTop<maxRetries>(&fiber);
 	if (res == StealingResult::Stolen) {
 		emper::statsIncr(awss::stats.nextFiberStolen);
@@ -163,13 +176,16 @@ popTop:
 		return NextFiberResult{fiber, emper::FiberSource::stolen};
 	}
 
+	return std::nullopt;
+}
+
+auto AbstractWorkStealingScheduler::tryStealFiberFrom(workerid_t victim)
+		-> std::optional<NextFiberResult> {
+	auto result = tryStealWorkFrom(victim);
+	if (result) return result;
+
 	if constexpr (emper::IO_STEALING) {
-		auto* victimIo = runtime.ioContexts[victim];
-		fiber = victimIo->reapSingleCompletion<CallerEnvironment::EMPER>();
-		if (fiber) {
-			emper::statsIncr(awss::stats.nextIoFiberStolen);
-			return NextFiberResult{fiber, emper::FiberSource::ioStolen};
-		}
+		return tryStealIoCompletionFrom(victim);
 	}
 
 	return std::nullopt;
@@ -220,6 +236,15 @@ popBottom:
 				}
 				break;
 			}
+			case emper::FiberSource::hintIo: {
+				const auto victim = dispatchHint.getWorker();
+				const auto stolenIo = tryStealIoCompletionFrom(victim);
+				if (stolenIo) {
+					emper::statsIncr(awss::stats.nextFiberFromHintIo);
+					return NextFiberResult{(*stolenIo).fiber, emper::FiberSource::hintIo};
+				}
+				break;
+			}
 			default:
 				DIE_MSG("invalid dispatch hint");
 		}

emper/strategies/AbstractWorkStealingScheduler.hpp

@@ -26,6 +26,8 @@ class AbstractWorkStealingScheduler : public Scheduler {
 
  private:
 	auto nextFiberViaAnywhereQueue() -> std::optional<NextFiberResult>;
+	auto tryStealWorkFrom(workerid_t victim) -> std::optional<NextFiberResult>;
+	auto tryStealIoCompletionFrom(workerid_t victim) -> std::optional<NextFiberResult>;
 	auto tryStealFiberFrom(workerid_t victim) -> std::optional<NextFiberResult>;
 
 	bool queueFullWarningEmitted = false;

emper/strategies/AbstractWorkStealingStats.cpp

@@ -31,6 +31,8 @@ void AbstractWorkStealingStats::print(std::ostream& out) {
 			<< std::to_string(comulatedWorkerStats.nextFiberFromHintLocal) << std::endl
 			<< "total-next-fiber-hint-anywherequeue: "
 			<< std::to_string(comulatedWorkerStats.nextFiberFromHintAnywhere) << std::endl
+			<< "total-next-fiber-hint-io: " << std::to_string(comulatedWorkerStats.nextFiberFromHintIo)
+			<< std::endl
 			<< "total-next-fiber-stolen: " << std::to_string(comulatedWorkerStats.nextFiberStolen)
 			<< std::endl
 			<< "total-next-io-fiber-stolen: " << std::to_string(comulatedWorkerStats.nextIoFiberStolen)

emper/strategies/AbstractWorkStealingWorkerStats.cpp

@@ -13,6 +13,7 @@ auto AbstractWorkStealingWorkerStats::operator+=(const AbstractWorkStealingWorke
 	nextFiberFromLocal += other.nextFiberFromLocal;
 	nextFiberFromHintLocal += other.nextFiberFromHintLocal;
 	nextFiberFromHintAnywhere += other.nextFiberFromHintAnywhere;
+	nextFiberFromHintIo += other.nextFiberFromHintIo;
 	nextFiberStolen += other.nextFiberStolen;
 	nextIoFiberStolen += other.nextIoFiberStolen;
 	nextFiberFromAnywhereQueue += other.nextFiberFromAnywhereQueue;

emper/strategies/AbstractWorkStealingWorkerStats.hpp

@@ -20,6 +20,7 @@ class AbstractWorkStealingWorkerStats {
 	uint64_t nextFiberFromLocal = 0;
 	uint64_t nextFiberFromHintLocal = 0;
 	uint64_t nextFiberFromHintAnywhere = 0;
+	uint64_t nextFiberFromHintIo = 0;
 	uint64_t nextFiberStolen = 0;
 	uint64_t nextIoFiberStolen = 0;
 	uint64_t nextFiberFromAnywhereQueue = 0;