diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d4f0cf392d36822bb238fc6cff1f290f3935ff2a..3591f70cf746c4b9d0928b664419650cad9046ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -159,6 +159,10 @@ clang-tidy: variables: EMPER_IO_LOCKLESS_CQ: 'true' +.emper-io-waitfree-stealing: + variables: + EMPER_IO_WAITFREE_STEALING: 'true' + .default-library-static: variables: EMPER_DEFAULT_LIBRARY: 'static' @@ -435,6 +439,13 @@ test-lockless-io-stealing: - .emper-io-stealing - .emper-lockless-cq +test-waitfree-io-stealing: + extends: + - .test + - .emper-io-stealing + - .emper-lockless-cq + - .emper-io-waitfree-stealing + test-io-stealing-pipe-no-completer: extends: - .test diff --git a/emper/io/IoContext.cpp b/emper/io/IoContext.cpp index b183abf03dacf14729c6693566f1e991d0e68983..10ecca47b9a4dbad1b7c109e2099569e624c5361 100644 --- a/emper/io/IoContext.cpp +++ b/emper/io/IoContext.cpp @@ -233,7 +233,7 @@ static constexpr auto LL_WRITE_MEM_ORDER = : std::memory_order_seq_cst; template <CallerEnvironment callerEnvironment> -auto IoContext::tryReapCompletionWaitFree(Fiber *continuation) -> emper::StealingResult { +auto IoContext::tryReapCompletionWaitFree(Fiber **continuation) -> emper::StealingResult { Completion completion; struct io_uring_cq *cq = &ring.cq; @@ -270,7 +270,7 @@ auto IoContext::tryReapCompletionWaitFree(Fiber *continuation) -> emper::Stealin completion.first = cqe->res; completion.second = cqe_data; - if (!ahead->compare_exchange_weak(head, head + 1, LL_WRITE_MEM_ORDER, LL_READ_MEM_ORDER)) + if (!ahead->compare_exchange_strong(head, head + 1, LL_WRITE_MEM_ORDER, LL_READ_MEM_ORDER)) return emper::StealingResult::LostRace; trackReqsInUring(-1); @@ -283,6 +283,14 @@ auto IoContext::tryReapCompletionWaitFree(Fiber *continuation) -> emper::Stealin return emper::StealingResult::Stolen; } +// show the compiler our template incarnations +template auto IoContext::tryReapCompletionWaitFree<CallerEnvironment::OWNER>(Fiber **continuation) + -> emper::StealingResult; +template auto IoContext::tryReapCompletionWaitFree<CallerEnvironment::EMPER>(Fiber **continuation) + -> emper::StealingResult; +template auto IoContext::tryReapCompletionWaitFree<CallerEnvironment::ANYWHERE>( + Fiber **continuation) -> emper::StealingResult; + template <CallerEnvironment callerEnvironment, unsigned toReap> auto IoContext::reapCompletionsLockless(Fiber **continuations) -> unsigned { Completion reapedCompletions[toReap]; // NOLINT(modernize-avoid-c-arrays) diff --git a/emper/io/IoContext.hpp b/emper/io/IoContext.hpp index a877a7488b1a6f393f21b825b62a352cc5ecaef4..2a0ba954455203db6dd337c031701e3a47c1b220 100644 --- a/emper/io/IoContext.hpp +++ b/emper/io/IoContext.hpp @@ -270,7 +270,7 @@ class IoContext : public Logger<LogSubsystem::IO> { } template <CallerEnvironment callerEnvironment> - [[nodiscard]] auto tryReapCompletionWaitFree(Fiber *continuation) -> emper::StealingResult; + [[nodiscard]] auto tryReapCompletionWaitFree(Fiber **continuation) -> emper::StealingResult; template <CallerEnvironment callerEnvironment, unsigned toReap> [[nodiscard]] auto reapCompletionsLockless(Fiber **continuations) -> unsigned;