diff --git a/emper/FiberSource.hpp b/emper/FiberSource.hpp index 75412f41d81fbca90683e33b60840d67cf4de1fe..39d5d7294cec286c013790215279429248c5e29b 100644 --- a/emper/FiberSource.hpp +++ b/emper/FiberSource.hpp @@ -20,6 +20,7 @@ enum class FiberSource : uintptr_t { ioStolen, /*!< A other worker's io_uring completion queue */ anywhereQueue, /*!< The anywhere queue */ hintWsq, /*!< A known other worker's work-stealing queue */ + hintIo, /*!< A known worker's io_uring completion queue */ hintAq, /*!< Straight from the anywhere queue */ }; diff --git a/emper/strategies/AbstractWorkStealingScheduler.cpp b/emper/strategies/AbstractWorkStealingScheduler.cpp index 6045e06df12f4cd9f1f08e58bd178885e05d3702..97f0cbae116cd3c97d9ca31facbdcf8958ce0782 100644 --- a/emper/strategies/AbstractWorkStealingScheduler.cpp +++ b/emper/strategies/AbstractWorkStealingScheduler.cpp @@ -149,11 +149,24 @@ auto AbstractWorkStealingScheduler::nextFiberViaAnywhereQueue() -> std::optional return std::nullopt; } -auto AbstractWorkStealingScheduler::tryStealFiberFrom(workerid_t victim) +auto AbstractWorkStealingScheduler::tryStealIoCompletionFrom(workerid_t victim) + -> std::optional<NextFiberResult> { + auto* victimIo = runtime.ioContexts[victim]; + Fiber* fiber = victimIo->reapSingleCompletion<CallerEnvironment::EMPER>(); + if (fiber) { + emper::statsIncr(awss::stats.nextIoFiberStolen); + return NextFiberResult{fiber, emper::FiberSource::ioStolen}; + } + + return std::nullopt; +} + +auto AbstractWorkStealingScheduler::tryStealWorkFrom(workerid_t victim) -> std::optional<NextFiberResult> { constexpr int maxRetries = emper::WAITFREE_WORK_STEALING ? 0 : -1; AbstractFiber* fiber; popTop: + StealingResult res = queues[victim]->popTop<maxRetries>(&fiber); if (res == StealingResult::Stolen) { emper::statsIncr(awss::stats.nextFiberStolen); @@ -163,13 +176,16 @@ popTop: return NextFiberResult{fiber, emper::FiberSource::stolen}; } + return std::nullopt; +} + +auto AbstractWorkStealingScheduler::tryStealFiberFrom(workerid_t victim) + -> std::optional<NextFiberResult> { + auto result = tryStealWorkFrom(victim); + if (result) return result; + if constexpr (emper::IO_STEALING) { - auto* victimIo = runtime.ioContexts[victim]; - fiber = victimIo->reapSingleCompletion<CallerEnvironment::EMPER>(); - if (fiber) { - emper::statsIncr(awss::stats.nextIoFiberStolen); - return NextFiberResult{fiber, emper::FiberSource::ioStolen}; - } + return tryStealIoCompletionFrom(victim); } return std::nullopt; @@ -220,6 +236,15 @@ popBottom: } break; } + case emper::FiberSource::hintIo: { + const auto victim = dispatchHint.getWorker(); + const auto stolenIo = tryStealIoCompletionFrom(victim); + if (stolenIo) { + emper::statsIncr(awss::stats.nextFiberFromHintIo); + return NextFiberResult{(*stolenIo).fiber, emper::FiberSource::hintIo}; + } + break; + } default: DIE_MSG("invalid dispatch hint"); } diff --git a/emper/strategies/AbstractWorkStealingScheduler.hpp b/emper/strategies/AbstractWorkStealingScheduler.hpp index 9afead16bce5c3356b5b20bcad64c5363b6b6a99..face284fef3fe3996085267b56c0ebaa0640487b 100644 --- a/emper/strategies/AbstractWorkStealingScheduler.hpp +++ b/emper/strategies/AbstractWorkStealingScheduler.hpp @@ -26,6 +26,8 @@ class AbstractWorkStealingScheduler : public Scheduler { private: auto nextFiberViaAnywhereQueue() -> std::optional<NextFiberResult>; + auto tryStealWorkFrom(workerid_t victim) -> std::optional<NextFiberResult>; + auto tryStealIoCompletionFrom(workerid_t victim) -> std::optional<NextFiberResult>; auto tryStealFiberFrom(workerid_t victim) -> std::optional<NextFiberResult>; bool queueFullWarningEmitted = false; diff --git a/emper/strategies/AbstractWorkStealingStats.cpp b/emper/strategies/AbstractWorkStealingStats.cpp index af7b74ee83e34238c09833be21638f993d4c2428..0fb4f536fa89566e2dbb282738dc4854e412c3ef 100644 --- a/emper/strategies/AbstractWorkStealingStats.cpp +++ b/emper/strategies/AbstractWorkStealingStats.cpp @@ -31,6 +31,8 @@ void AbstractWorkStealingStats::print(std::ostream& out) { << std::to_string(comulatedWorkerStats.nextFiberFromHintLocal) << std::endl << "total-next-fiber-hint-anywherequeue: " << std::to_string(comulatedWorkerStats.nextFiberFromHintAnywhere) << std::endl + << "total-next-fiber-hint-io: " << std::to_string(comulatedWorkerStats.nextFiberFromHintIo) + << std::endl << "total-next-fiber-stolen: " << std::to_string(comulatedWorkerStats.nextFiberStolen) << std::endl << "total-next-io-fiber-stolen: " << std::to_string(comulatedWorkerStats.nextIoFiberStolen) diff --git a/emper/strategies/AbstractWorkStealingWorkerStats.cpp b/emper/strategies/AbstractWorkStealingWorkerStats.cpp index ca3ef250b10adbb8a38dd09e9bf851b990b8c9ae..44b62d32da90816abc8646c67fb3b1ff7c8e4723 100644 --- a/emper/strategies/AbstractWorkStealingWorkerStats.cpp +++ b/emper/strategies/AbstractWorkStealingWorkerStats.cpp @@ -13,6 +13,7 @@ auto AbstractWorkStealingWorkerStats::operator+=(const AbstractWorkStealingWorke nextFiberFromLocal += other.nextFiberFromLocal; nextFiberFromHintLocal += other.nextFiberFromHintLocal; nextFiberFromHintAnywhere += other.nextFiberFromHintAnywhere; + nextFiberFromHintIo += other.nextFiberFromHintIo; nextFiberStolen += other.nextFiberStolen; nextIoFiberStolen += other.nextIoFiberStolen; nextFiberFromAnywhereQueue += other.nextFiberFromAnywhereQueue; diff --git a/emper/strategies/AbstractWorkStealingWorkerStats.hpp b/emper/strategies/AbstractWorkStealingWorkerStats.hpp index e8e25bef9e692d550ec21a722cbf23adbb67d835..3616c374626db55b96417d2a3891e44fa3fe0929 100644 --- a/emper/strategies/AbstractWorkStealingWorkerStats.hpp +++ b/emper/strategies/AbstractWorkStealingWorkerStats.hpp @@ -20,6 +20,7 @@ class AbstractWorkStealingWorkerStats { uint64_t nextFiberFromLocal = 0; uint64_t nextFiberFromHintLocal = 0; uint64_t nextFiberFromHintAnywhere = 0; + uint64_t nextFiberFromHintIo = 0; uint64_t nextFiberStolen = 0; uint64_t nextIoFiberStolen = 0; uint64_t nextFiberFromAnywhereQueue = 0;