From 81c453663326005ca5d68c2e63e5e00b97098eb4 Mon Sep 17 00:00:00 2001
From: Florian Fischer <florian.fischer@muhq.space>
Date: Thu, 17 Feb 2022 13:59:42 +0100
Subject: [PATCH] io: make max unbounded io worker configurable
Since Linux 5.15 io_uring can limit the number of iow threads created
using IORING_REGISTER_IOWQ_MAX_WORKERS.
Bump liburing wrap to version 2.1 to use
io_uring_register_iowq_max_workers.
Expose this via the meson variable io_unbounded_iow_max and the
environment variable EMPER_IO_UNBOUNDED_IOW_MAX.
See for an detailed explanation:
https://blog.cloudflare.com/missing-manuals-io_uring-worker-pool
---
emper/io/GlobalIoContext.hpp | 5 ++++-
emper/io/IoContext.cpp | 39 +++++++++++++++++++++++++++++++++++-
emper/io/IoContext.hpp | 16 +++++++++++++--
emper/lib/env.cpp | 31 ++++++++++++++++++++++++++++
emper/lib/env.hpp | 28 +++++++++-----------------
emper/lib/meson.build | 1 +
meson.build | 3 ++-
meson_options.txt | 6 ++++++
subprojects/liburing.wrap | 15 +++++++-------
9 files changed, 113 insertions(+), 31 deletions(-)
create mode 100644 emper/lib/env.cpp
diff --git a/emper/io/GlobalIoContext.hpp b/emper/io/GlobalIoContext.hpp
index 7dbf3114..d819ba8e 100644
--- a/emper/io/GlobalIoContext.hpp
+++ b/emper/io/GlobalIoContext.hpp
@@ -20,9 +20,12 @@ class GlobalIoContext : public IoContext {
friend class RecvFuture;
private:
- GlobalIoContext(Runtime& runtime, workerid_t workerCount) : IoContext(runtime, workerCount) {
+ GlobalIoContext(Runtime& runtime, workerid_t workerCount, unsigned unboundedIowMax)
+ : IoContext(runtime, workerCount, unboundedIowMax) {
worker = nullptr;
}
+ GlobalIoContext(Runtime& runtime, workerid_t workerCount)
+ : GlobalIoContext(runtime, workerCount, IoContext::getDefaultUnboundedIowMax()) {}
enum class PointerTags : uint16_t { Future, Callback, IoContext, TerminationEvent };
diff --git a/emper/io/IoContext.cpp b/emper/io/IoContext.cpp
index 56c4c1e9..b183abf0 100644
--- a/emper/io/IoContext.cpp
+++ b/emper/io/IoContext.cpp
@@ -10,10 +10,12 @@
#include <unistd.h>
#include <algorithm>
+#include <array>
#include <atomic> // for atomic, __atomic_base
#include <cassert> // for assert
#include <cerrno> // for errno, ECANCELED, EBUSY, EAGAIN, EINTR
#include <cstring> // for memset
+#include <optional>
#include <ostream> // for basic_osteram::operator<<, operator<<
#include <string>
#include <vector>
@@ -25,11 +27,14 @@
#include "Emper.hpp" // for DEBUG, IO_URING_SQPOLL
#include "Runtime.hpp"
#include "emper-common.h"
+#include "emper-config.h"
#include "io/Future.hpp"
#include "io/GlobalIoContext.hpp"
#include "io/Stats.hpp" // for Stats, nanoseconds
#include "io/SubmitActor.hpp"
+#include "lib/LinuxVersion.hpp"
#include "lib/TaggedPtr.hpp"
+#include "lib/env.hpp"
using emper::lib::TaggedPtr;
@@ -495,7 +500,8 @@ template auto IoContext::reapCompletionsLocked<CallerEnvironment::ANYWHERE,
IoContext::CQE_BATCH_COUNT>(Fiber **contiunations)
-> unsigned;
-IoContext::IoContext(Runtime &runtime, size_t uring_entries) : runtime(runtime) {
+IoContext::IoContext(Runtime &runtime, size_t uring_entries, unsigned unboundedIowMax)
+ : runtime(runtime) {
struct io_uring_params params;
memset(¶ms, 0, sizeof(params));
@@ -579,6 +585,26 @@ IoContext::IoContext(Runtime &runtime, size_t uring_entries) : runtime(runtime)
}
}
+ // Limit the number of unbounded iow thread created by this io_uring
+ // By default the number of unbounded iow worker threads is RLIMIT_NPROC
+ // which may be excessive.
+ // IORING_REGISTER_IOWQ_MAX_WORKERS sets a maximum value for bounded and unbounded
+ // worker creation per NUMA node and thread.
+ // See:
+ // https://blog.cloudflare.com/missing-manuals-io_uring-worker-pool/
+ if (unboundedIowMax) {
+ if (EMPER_LINUX_LT("5.15")) DIE_MSG("can not set unbounded iow max on linux < 5.15");
+
+ std::array<unsigned int, 2> maxWorkerConfig = {
+ 0, // do not change bounded worker count
+ unboundedIowMax,
+ };
+
+ if (unlikely(io_uring_register_iowq_max_workers(&ring, maxWorkerConfig.data()) < 0))
+ DIE_MSG_ERRNO("setting the max unbounded io worker count to " << unboundedIowMax
+ << " failed");
+ }
+
// reserve space for a full SQ
preparedSqes.reserve(*this->ring.sq.kring_entries);
@@ -613,6 +639,17 @@ IoContext::~IoContext() {
delete submitter;
}
+auto IoContext::getDefaultUnboundedIowMax() -> unsigned {
+ unsigned unboundedIowMax = EMPER_IO_UNBOUNDED_IOW_MAX;
+
+ static const std::string unboundedIowMaxEnvVarName = "EMPER_UNBOUNDED_IOW_MAX";
+ auto unboundedIowMaxEnv =
+ emper::lib::env::getUnsignedFromEnv<unsigned>(unboundedIowMaxEnvVarName);
+ if (unboundedIowMaxEnv) return unboundedIowMaxEnv.value();
+
+ return unboundedIowMax;
+}
+
auto IoContext::getSqHead() const -> unsigned { return *ring.sq.khead; }
auto IoContext::getSqTail() const -> unsigned { return *ring.sq.ktail; }
auto IoContext::getSqEntries() const -> unsigned { return *ring.sq.kring_entries; }
diff --git a/emper/io/IoContext.hpp b/emper/io/IoContext.hpp
index 53e2a05b..a877a748 100644
--- a/emper/io/IoContext.hpp
+++ b/emper/io/IoContext.hpp
@@ -302,10 +302,22 @@ class IoContext : public Logger<LogSubsystem::IO> {
}
public:
- IoContext(Runtime &runtime, size_t uring_entries);
- IoContext(Runtime &runtime) : IoContext(runtime, EMPER_IO_WORKER_URING_ENTRIES){};
+ IoContext(Runtime &runtime, size_t uring_entries, unsigned unboundedIowMax);
+ IoContext(Runtime &runtime)
+ : IoContext(runtime, EMPER_IO_WORKER_URING_ENTRIES, IoContext::getDefaultUnboundedIowMax()){};
~IoContext();
+ /**
+ * @brief get the default maximum count of unbounded io worker threads
+ *
+ * The default is 0 (unbounded), which can be overwritten during
+ * compile time (-Dio_unbouned_iow_max) or runtime using the
+ * environment variable EMPER_UNBOUNDED_IOW_MAX.
+ *
+ * @return the default maximum count of unbounded io worker threads
+ */
+ static auto getDefaultUnboundedIowMax() -> unsigned;
+
/**
* @brief get IoContext of the current worker
*
diff --git a/emper/lib/env.cpp b/emper/lib/env.cpp
new file mode 100644
index 00000000..a6c065f6
--- /dev/null
+++ b/emper/lib/env.cpp
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright © 2021-2022 Florian Fischer
+#include "lib/env.hpp"
+
+#include <ostream>
+#include <string>
+
+#include "Common.hpp"
+
+namespace emper::lib::env {
+
+auto getBoolFromEnv(const std::string& key) -> std::optional<bool> {
+ char* envVar = std::getenv(key.c_str());
+ if (!envVar) {
+ return std::nullopt;
+ }
+
+ std::string envStr(envVar);
+ if (envStr == "true") {
+ return true;
+ }
+
+ if (envStr == "false") {
+ return false;
+ }
+
+ DIE_MSG(key << " has invalid value: " << envStr << " (expected true or false)");
+ // Should never be reached.
+ return false;
+}
+} // namespace emper::lib::env
diff --git a/emper/lib/env.hpp b/emper/lib/env.hpp
index d03784be..1179253d 100644
--- a/emper/lib/env.hpp
+++ b/emper/lib/env.hpp
@@ -1,31 +1,21 @@
// SPDX-License-Identifier: LGPL-3.0-or-later
-// Copyright © 2021 Florian Fischer
+// Copyright © 2021-2022 Florian Fischer
#pragma once
#include <cinttypes>
+#include <cstdlib>
+#include <iterator>
+#include <memory>
+#include <optional>
+#include <ostream>
#include <string>
+#include <typeinfo>
-#include "Debug.hpp"
+#include "Common.hpp"
namespace emper::lib::env {
-static auto getBoolFromEnv(const std::string& key) -> std::optional<bool> {
- char* envVar = std::getenv(key.c_str());
- if (!envVar) {
- return std::nullopt;
- }
-
- std::string envStr(envVar);
- if (envStr == "true") {
- return true;
- }
-
- if (envStr == "false") {
- return false;
- }
-
- DIE_MSG(key << " has invalid value: " << envStr << " (expected true or false)");
-}
+auto getBoolFromEnv(const std::string& key) -> std::optional<bool>;
template <typename unsigned_type>
static auto getUnsignedFromEnv(const std::string& key) -> std::optional<unsigned_type> {
diff --git a/emper/lib/meson.build b/emper/lib/meson.build
index 0fd76504..2ee763b6 100644
--- a/emper/lib/meson.build
+++ b/emper/lib/meson.build
@@ -1,5 +1,6 @@
emper_cpp_sources += files(
'DebugUtil.cpp',
+ 'env.cpp',
'LinuxVersion.cpp',
'util.cpp',
)
diff --git a/meson.build b/meson.build
index 8928df39..18d27999 100644
--- a/meson.build
+++ b/meson.build
@@ -13,7 +13,7 @@ thread_dep = dependency('threads')
conf_data = configuration_data()
use_bundled_deps = get_option('use_bundled_deps')
-liburing_version = '2.0'
+liburing_version = '2.1'
uring_dep = dependency('liburing', version: liburing_version, required: use_bundled_deps == 'never')
if not uring_dep.found() or use_bundled_deps == 'always'
message('liburing ' + liburing_version + ' not found in system, using liburing subproject')
@@ -157,6 +157,7 @@ io_bool_options = [
io_raw_options = [
'worker_uring_entries',
+ 'unbounded_iow_max',
'lockless_memory_order',
]
diff --git a/meson_options.txt b/meson_options.txt
index ecca5993..9b3a0d15 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -176,6 +176,12 @@ option(
value: 16,
description: 'Number of entries in each worker io_uring'
)
+option(
+ 'io_unbounded_iow_max',
+ type: 'integer',
+ value: 0,
+ description: 'Set the maximum number of unbounded io worker threads created per io_uring'
+)
option(
'io_uring_sq_poller',
type: 'combo',
diff --git a/subprojects/liburing.wrap b/subprojects/liburing.wrap
index 3fc0f9b2..b3c66d52 100644
--- a/subprojects/liburing.wrap
+++ b/subprojects/liburing.wrap
@@ -1,11 +1,12 @@
[wrap-file]
-directory = liburing-liburing-2.0
-source_url = https://github.com/axboe/liburing/archive/refs/tags/liburing-2.0.tar.gz
-source_filename = liburing-2.0.tar.gz
-source_hash = ca069ecc4aa1baf1031bd772e4e97f7e26dfb6bb733d79f70159589b22ab4dc0
-patch_filename = liburing_2.0-1_patch.zip
-patch_url = https://wrapdb.mesonbuild.com/v2/liburing_2.0-1/get_patch
-patch_hash = c63ac1a5b5ffbc088dd772878f694c5c0e502d4e4c58d262f0272ae40e648d3b
+directory = liburing-liburing-2.1
+source_url = https://github.com/axboe/liburing/archive/refs/tags/liburing-2.1.tar.gz
+source_filename = liburing-2.1.tar.gz
+source_hash = f1e0500cb3934b0b61c5020c3999a973c9c93b618faff1eba75aadc95bb03e07
+patch_filename = liburing_2.1-1_patch.zip
+patch_url = https://wrapdb.mesonbuild.com/v2/liburing_2.1-1/get_patch
+patch_hash = aaeb2be1f5eacf4f41e0537aa02154486e0729a3395d2e832de7657ab0b56290
[provide]
uring = uring
+
--
GitLab