.clang-format

+---
+BasedOnStyle:  Google
+ColumnLimit:     100
+TabWidth: 2
+UseTab: Always

.clang-tidy

----
-Checks:          'clang-diagnostic-*,clang-analyzer-*,-clang-analyzer-alpha*'
-HeaderFilterRegex: ''
-AnalyzeTemporaryDtors: false
-User:            flo
-CheckOptions:    
-  - key:             google-readability-braces-around-statements.ShortStatementLines
-    value:           '1'
-  - key:             google-readability-function-size.StatementThreshold
-    value:           '800'
-  - key:             google-readability-namespace-comments.ShortNamespaceLines
-    value:           '10'
-  - key:             google-readability-namespace-comments.SpacesBeforeComments
-    value:           '2'
-...
+Checks: >
+  bugprone-*,
+  cert-*,
+  linuxkernel-*,
+  modernize-*,
+  performance-*,
+  portability-*,
+  readability-*,
+  -bugprone-assignment-in-if-condition,
+  -bugprone-easily-swappable-parameters,
+  -cert-err58-cpp,
+  -clang-diagnostic-empty-translation-unit,
+  -performance-avoid-endl,
+  -readability-braces-around-statements,
+  -readability-function-cognitive-complexity,
+  -readability-identifier-length,
+  -readability-implicit-bool-conversion,
+  -readability-isolate-declaration,
+  -readability-magic-numbers,
+  -readability-simplify-boolean-expr,
 
+WarningsAsErrors: >
+  bugprone-*,
+  modernize-*,
+  clang-*,
+  readability-*,
+  performance-*,
+
+HeaderFilterRegex: '(?!subprojects).*'

.editorconfig

+# top-most EditorConfig file
+root = true
+
+[*]
+charset = utf-8
+trim_trailing_whitespace = true
+end_of_line = lf
+insert_final_newline = true
+
+[*.{c,h,cpp,hpp}]
+indent_style = tab
+indent_size = 2
+
+[*.yml]
+indent_style = space
+indent_size = 2
+
+[meson_options.txt]
+indent_style = space
+indent_size = 2

.git-blame-ignore-revs

+# [codestyle] Add check-format target and apply codestyle to codebase
+f48f3822146083ba78c28c5928388dd4d8901029
\ No newline at end of file

.gitignore

@@ -2,3 +2,14 @@
 
 /build
 /build-*/
+
+/.cache/
+/.clangd/
+
+/clang-tidy-report
+
+tools/gdb/__pycache__/
+
+subprojects/packagecache/
+subprojects/googletest*
+subprojects/liburing*

.gitlab-ci-local/.gitignore

+*
+!.gitignore

.gitlab-ci.yml

+image: "flowdalic/gentoo-dev:20241216"
+
+before_script:
+  - ulimit -a
+  - nproc
+  - |
+    readarray TOOLS <<EOF
+    c++
+    cc
+    clang
+    clang++
+    clang-tidy
+    doxygen
+    g++
+    gcc
+    include-what-you-use
+    meson
+    nasm
+    valgrind
+    EOF
+    for tool in ${TOOLS[@]}; do
+      if ! command -v $tool; then
+        echo "No $tool binary found"
+        continue
+      fi
+      echo -n "$tool version: "
+      $tool --version
+    done
+
+cache:
+  paths:
+    - subprojects/packagecache
+
+variables:
+  BUILDDTYPE: debugoptimized
+  GCC: gcc
+  GXX: g++
+  CC: $GCC
+  CXX: $GXX
+  EXTRA_NINJA_ARGS: -v
+
+stages:
+  - smoke-test
+  - test
+
+.meson-test:
+  artifacts:
+    paths:
+      - build-*/meson-logs
+    when: on_failure
+    reports:
+      junit: build-*/meson-logs/testlog.junit.xml
+
+smoke-test-suite:
+  extends:
+    - .meson-test
+  stage: smoke-test
+  script: make smoke-test-suite
+
+fast-static-analysis:
+   stage: smoke-test
+   script: make fast-static-analysis
+   variables:
+       EMPER_IO: "true"
+
+.fast-variant-check:
+   stage: test
+   script: make fast-static-analysis smoke-test-suite
+   variables:
+       EMPER_IO: "true"
+
+iwyu:
+   script: IWYU_TOOL="${CI_PROJECT_DIR}/tools/iwyu_tool.py" make iwyu
+   variables:
+       EMPER_IO: "true"
+
+clang-tidy:
+   script: make tidy
+   variables:
+       EMPER_IO: "true"
+
+.build:
+  stage: test
+  script:
+    - make
+
+.test:
+  extends:
+    - .meson-test
+  stage: test
+  script:
+    - make test
+
+.gcc:
+  variables:
+    CC: $GCC
+    CXX: $GXX
+
+.clang:
+  variables:
+    CC: clang
+    CXX: clang++
+
+.libc++:
+  extends:
+    - .clang
+  variables:
+    EMPER_USE_BUNDLED_DEPS: "always"
+    EMPER_CPP_ARGS: "-stdlib=libc++"
+    EMPER_CPP_LINK_ARGS: "-stdlib=libc++"
+
+.lto:
+  variables:
+    EMPER_B_LTO: "true"
+
+.emper-ws-scheduling:
+  variables:
+    EMPER_DEFAULT_SCHEDULING_STRATEGY: "work_stealing"
+
+.emper-laws-scheduling:
+  variables:
+    EMPER_DEFAULT_SCHEDULING_STRATEGY: "locality_aware_work_stealing"
+
+.emper-worker-no-sleep:
+  variables:
+    EMPER_WORKER_SLEEP: 'false'
+
+.emper-stats:
+  variables:
+    EMPER_STATS_ALL: 'true'
+
+.emper-userspace-rcu:
+  variables:
+    EMPER_USERSPACE_RCU: 'true'
+
+.emper-no-io:
+  variables:
+    EMPER_IO: 'false'
+
+.emper-pipe-sleep-strategy:
+  variables:
+    EMPER_WORKER_SLEEP_STRATEGY: 'pipe'
+
+.emper-no-completer:
+  variables:
+    EMPER_IO_COMPLETER_BEHAVIOR: 'none'
+
+.emper-single-poller:
+  variables:
+    EMPER_IO_URING_SQ_POLLER: 'one'
+
+.emper-numa-poller:
+  variables:
+    EMPER_IO_URING_SQ_POLLER: 'numa'
+
+.emper-each-poller:
+  variables:
+    EMPER_IO_URING_SQ_POLLER: 'each'
+
+.emper-single-uring:
+  variables:
+    EMPER_IO_SINGLE_URING: 'true'
+
+.emper-synchronous-io:
+  variables:
+    EMPER_IO_SYNCHRONOUS: 'true'
+
+.emper-io-stealing:
+  variables:
+    EMPER_IO_STEALING: 'true'
+
+.emper-lockless-cq:
+  variables:
+    EMPER_IO_LOCKLESS_CQ: 'true'
+
+.emper-io-waitfree-stealing:
+  variables:
+    EMPER_IO_WAITFREE_STEALING: 'true'
+
+.default-library-static:
+  variables:
+    EMPER_DEFAULT_LIBRARY: 'static'
+
+.clang-sanitizer:
+  extends:
+    - .clang
+    - .default-library-static
+  variables:
+    # Clang's sanitizers require undefined symbols to be allowed when
+    # linking. See also:
+    # - https://github.com/mesonbuild/meson/issues/764
+    # - https://mesonbuild.com/Builtin-options.html
+    EMPER_B_LUNDEF: 'false'
+
+
+.clang-sanitizer-undefined:
+  extends:
+    - .clang-sanitizer
+  variables:
+    EMPER_B_SANITIZE: 'undefined'
+
+#.clang-sanitizer-memory:
+#  extends:
+#    - .clang-sanitizer
+#  variables:
+#    EMPER_B_SANITIZE: 'memory'
+
+#.sanitizer-address:
+#  extends:
+#    - .default-library-static
+#  variables:
+#    EMPER_B_SANITIZE: 'address'
+
+.release-build:
+  variables:
+    BUILDTYPE: release
+
+.debug-build:
+  variables:
+    BUILDTYPE: debug
+
+.worker-wakeup-strategy-all:
+  variables:
+    EMPER_WORKER_WAKEUP_STRATEGY: "all"
+
+.emper-worker-wakeup-strategy-throttle:
+  variables:
+    EMPER_WORKER_WAKEUP_STRATEGY: "throttle"
+
+.do-not-log-timestamp:
+  variables:
+    EMPER_LOG_TIMESTAMP: "none"
+
+.ws-queue-scheduler-locked:
+  variables:
+    EMPER_WS_QUEUE_SCHEDULER: "locked"
+
+.ws-queue-scheduler-cl2:
+  variables:
+    EMPER_WS_QUEUE_SCHEDULER: "cl2"
+
+.ws-queue-scheduler-cl3:
+  variables:
+    EMPER_WS_QUEUE_SCHEDULER: "cl3"
+
+.ws-queue-scheduler-cl4:
+  variables:
+    EMPER_WS_QUEUE_SCHEDULER: "cl4"
+
+.waitfree-ws:
+  variables:
+    EMPER_WAITFREE_WORK_STEALING: "true"
+
+.futex-wakeup-semaphore:
+  variables:
+    EMPER_WAKEUP_SEMAPHORE_IMPLEMENTATION: "futex"
+
+.futex2-wakeup-semaphore:
+  variables:
+    EMPER_WAKEUP_SEMAPHORE_IMPLEMENTATION: "futex2"
+
+.locked-wakeup-semaphore:
+  variables:
+    EMPER_WAKEUP_SEMAPHORE_IMPLEMENTATION: "locked"
+
+.set-affinity-on-block:
+  variables:
+    EMPER_SET_AFFINITY_ON_BLOCK: 'true'
+
+test-gcc:
+  extends:
+    - .test
+    - .gcc
+
+test-gcc-release:
+  extends:
+    - test-gcc
+    - .release-build
+
+test-gcc-debug:
+  extends:
+    - test-gcc
+    - .debug-build
+
+test-clang:
+  extends:
+    - .test
+    - .clang
+
+test-clang-release:
+  extends:
+    - test-clang
+    - .release-build
+
+test-clang-debug:
+  extends:
+    - test-clang
+    - .debug-build
+
+smoke-test-libc++:
+  stage: smoke-test
+  extends:
+    - .fast-variant-check
+    - .libc++
+
+test-lto:
+  extends:
+    - .test
+    - .lto
+
+test-libc++:
+  extends:
+    - .test
+    - .libc++
+
+test-worker-no-sleep:
+  extends:
+    - .test
+    - .emper-worker-no-sleep
+
+test-with-stats:
+  extends:
+    - .test
+    - .emper-stats
+
+test-with-userspace-rcu:
+  extends:
+    - .test
+    - .emper-userspace-rcu
+
+test-clang-sanitizer-undefined:
+  extends:
+    - .test
+    - .clang-sanitizer-undefined
+
+#test-clang-sanitizer-memory:
+#  extends:
+#    - .test
+#    - .clang-sanitizer-memory
+
+#test-clang-sanitizer-address:
+#  extends:
+#    - .test
+#    - .clang
+#    - .sanitizer-address
+
+test-laws:
+  extends:
+    - .test
+    - .emper-laws-scheduling
+
+test-laws-release:
+  extends:
+    - test-laws
+    - .release-build
+
+test-worker-wakeup-strategy-all:
+  extends:
+    - .test
+    - .worker-wakeup-strategy-all
+
+# Disable throttle test till throttle works with notifySpecific
+#test-worker-wakeup-strategy-throttle:
+# extends:
+#   - .test
+#   - .emper-worker-wakeup-strategy-throttle
+
+test-do-not-log-timestamp:
+  extends:
+    - .test
+    - .do-not-log-timestamp
+
+test-ws-queue-scheduler-locked:
+  extends:
+    - .test
+    - .ws-queue-scheduler-locked
+
+test-ws-queue-scheduler-cl2:
+  extends:
+    - .test
+    - .ws-queue-scheduler-cl2
+
+test-ws-queue-scheduler-cl3:
+  extends:
+    - .test
+    - .ws-queue-scheduler-cl3
+
+test-ws-queue-scheduler-cl3:
+  extends:
+    - .test
+    - .ws-queue-scheduler-cl3
+
+test-waitfree-ws:
+  extends:
+    - .test
+    - .waitfree-ws
+
+test-futex-wakeup-semaphore:
+  extends:
+    - .test
+    - .futex-wakeup-semaphore
+
+# TODO: enable this if the CI has linux >= 5.16
+build-futex-wakeup-semaphore:
+  extends:
+    - .build
+    - .futex2-wakeup-semaphore
+
+test-locked-wakeup-semaphore:
+  extends:
+    - .test
+    - .locked-wakeup-semaphore
+
+test-set-affinity-on-block:
+  extends:
+    - .test
+    - .set-affinity-on-block
+
+test-mmapped-log:
+  extends:
+    - .meson-test
+  script: make && EMPER_LOG_FILE=emper.log make test
+
+test-no-io:
+  extends:
+    - .test
+    - .emper-no-io
+
+test-single-uring:
+  extends:
+    - .test
+    - .emper-single-uring
+
+test-synchronous-io:
+  extends:
+    - .test
+    - .emper-synchronous-io
+
+test-pipe-sleep-strategy:
+  extends:
+    - .test
+    - .emper-pipe-sleep-strategy
+
+test-pipe-sleep-strategy-no-completer:
+  extends:
+    - .test
+    - .emper-pipe-sleep-strategy
+    - .emper-no-completer
+
+test-lockless-cq:
+  extends:
+    - .test
+    - .emper-lockless-cq
+
+test-io-stealing:
+  extends:
+    - .test
+    - .emper-io-stealing
+
+test-lockless-io-stealing:
+  extends:
+    - .test
+    - .emper-io-stealing
+    - .emper-lockless-cq
+
+test-waitfree-io-stealing:
+  extends:
+    - .test
+    - .emper-io-stealing
+    - .emper-lockless-cq
+    - .emper-io-waitfree-stealing
+
+test-io-stealing-pipe-no-completer:
+  extends:
+    - .test
+    - .emper-pipe-sleep-strategy
+    - .emper-no-completer
+    - .emper-io-stealing
+
+test-io-stealing-pipe-no-completer-lockless:
+  extends:
+    - .test
+    - .emper-pipe-sleep-strategy
+    - .emper-no-completer
+    - .emper-io-stealing
+    - .emper-lockless-cq
+
+smoke-test-locked-queue-rwlock:
+  extends:
+    - .fast-variant-check
+  variables:
+    EMPER_LOCKED_UNBOUNDED_QUEUE_IMPLEMENTATION: "rwlock"
+
+smoke-test-locked-queue-shared-mutex:
+  extends:
+    - .fast-variant-check
+  variables:
+    EMPER_LOCKED_UNBOUNDED_QUEUE_IMPLEMENTATION: "shared_mutex"
+
+smoke-test-locked-queue-boost-shared-mutex:
+  extends:
+    - .fast-variant-check
+  variables:
+    EMPER_LOCKED_UNBOUNDED_QUEUE_IMPLEMENTATION: "boost_shared_mutex"
+
+smoke-test-locked-queue-boost-userspace-rcu:
+  extends:
+    - .fast-variant-check
+  variables:
+    EMPER_USERSPACE_RCU: "true"
+
+# Only build the poller variants because sqpoll needs linux >= 5.15
+# TODO: also test those variants if the CI uses linux >= 5.15
+build-single-poller:
+  extends:
+    - .build
+    - .emper-single-poller
+
+build-numa-poller:
+  extends:
+    - .build
+    - .emper-single-poller
+
+build-each-poller:
+  extends:
+    - .build
+    - .emper-each-poller
+
+continuation-stealing-locked:
+  extends:
+    - .test
+    - .ws-queue-scheduler-locked
+  variables:
+    EMPER_CONTINUATION_STEALING_MODE: 'locked'
+
+continuation-stealing-madv-free:
+  extends:
+    - .test
+  variables:
+    EMPER_CONTINUATION_STEALING_MADVISE_STACK: 'free'
+
+stack-guard-page:
+  extends:
+    - .test
+  variables:
+    EMPER_STACK_GUARD_PAGE: 'true'
+
+build-only-emper-dep:
+  extends:
+    - .build
+  variables:
+    EMPER_BUILD_ONLY_EMPER_DEP: 'true'

CMakeLists.txt

-cmake_minimum_required(VERSION 3.3)
-
-project(EMPER)
-
-set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
-message("CMAKE_MODULE_PATH: ${CMAKE_MODULE_PATH}")
-include(doxygen)
-
-execute_process(COMMAND ${PROJECT_SOURCE_DIR}/scripts/dependencyManager.sh -v install)
-set(EIGEN3_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/external-libs/eigen)
-find_package(Eigen3 REQUIRED)
-
-set(CMAKE_CXX_STANDARD 14)
-set(CMAKE_CXX_STANDARD_REQUIRED on)
-set(CMAKE_EXPORT_COMPILE_COMMANDS on)
-
-set(COMMON_FLAGS "-Wall -pedantic -Wextra -Werror")
-set(COMMON_FLAGS "${COMMON_FLAGS} -fno-exceptions")
-
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON_FLAGS}")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS}")
-
-message("CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}")
-message("CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}")
-
-set(COMMON_DEBUG_FLAGS "-ggdb3")
-set(COMMON_BUILD_DEBUG_FLAGS "${COMMON_DEBUG_FLAGS} -O0")
-
-set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${COMMON_BUILD_DEBUG_FLAGS}")
-set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMMON_BUILD_DEBUG_FLAGS}")
-
-set(COMMON_BUILD_RELWITHDEBUG_FLAGS "${COMMON_DEBUG_FLAGS} -O3")
-set(CMAKE_C_FLAGS_RELWITHDEBUG "${CMAKE_C_FLAGS_RELWITHDEBUG} ${COMMON_BUILD_RELWITHDEBUG_FLAGS}")
-set(CMAKE_CXX_FLAGS_RELWITHDEBUG "${CMAKE_CXX_FLAGS_RELWITHDEBUG} ${COMMON_BUILD_RELWITHDEBUG_FLAGS}")
-
-# For "-rdynamic" see https://stackoverflow.com/a/77336/194894 and "man gcc"
-# We probably don't need it since EMPER is mostly statically linked,
-# but it can't hurt to have it (hopefully).
-set(CMAKE_EXEC_LINKER_FLAGS_DEBUG "${CMAKE_EXEC_LINKER_FLAGS_DEBUG} -rdynamic")
-
-add_definitions(-D_GNU_SOURCE)
-
-set(THREADS_PREFER_PTHREAD_FLAG ON)
-find_package(Threads REQUIRED)
-
-enable_testing()
-
-# Custom options, enable with "cmake -DEMPER_WORKER_SLEEP=ON"
-# Source: https://stackoverflow.com/a/10364240/194894
-macro(emper_option option_name option_description)
-    option(EMPER_${option_name} ${option_description})
-    if(EMPER_${option_name})
-        add_definitions(-DEMPER_${option_name})
-    endif(EMPER_${option_name})
-endmacro()
-
-emper_option(WORKER_SLEEP "Enable sleeping worker support")
-emper_option(LOCKED_WS_QUEUE "Use a fully locked queue for work-stealing")
-emper_option(OVERFLOW_QUEUE "Use a overflow queue in case the primary queue is full")
-emper_option(LOCKED_MPSC_QUEUE "Use the locked variant for the MPSC queue")
-emper_option(STATS "Collect stats and print them at the end of the execution")
-
-# Macro to add files to a var. Can even be used in subdirectories.
-# Source: http://stackoverflow.com/a/7049380/194894
-macro (add_files var)
-    file (RELATIVE_PATH _relPath "${PROJECT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}")
-    foreach (_src ${ARGN})
-        if (_relPath)
-            list (APPEND ${var} "${_relPath}/${_src}")
-        else()
-            list (APPEND ${var} "${_src}")
-        endif()
-    endforeach()
-    if (_relPath)
-        # propagate ${var} to parent directory
-        set (${var} ${${var}} PARENT_SCOPE)
-    endif()
-endmacro()
-
-enable_language(ASM_NASM)
-
-add_subdirectory("emper")
-
-message("EMPER_SOURCE: ${EMPER_SOURCE}")
-message("EMPER_INCLUDE: ${EMPER_INCLUDE}")
-message("CMAKE_CXX_FLAGS_RELEASE: ${CMAKE_CXX_FLAGS_RELEASE}")
-message("CMAKE_CXX_FLAGS_RELWITHDEBUG: ${CMAKE_CXX_FLAGS_RELWITHDEBUG}")
-message("CMAKE_CXX_FLAGS_DEBUG: ${CMAKE_CXX_FLAGS_DEBUG}")
-
-add_library(emper_asm_source STATIC ${EMPER_ASM_SOURCE})
-
-add_library(emper STATIC ${EMPER_SOURCE})
-foreach(include_dir ${EMPER_INCLUDE})
-  message("Adding include directory: ${include_dir}")
-  target_include_directories(emper PUBLIC
-	$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/${include_dir}>
-	$<INSTALL_INTERFACE:${include_dir}>
-  )
-endforeach()
-# The following property is supposed to enable something like LTO. But
-# CMake appearantly only supports it for the icc. See
-# http://stackoverflow.com/a/31522585/194894
-# set_property(TARGET emper PROPERTY INTERPROCEDURAL_OPTIMIZATION True)
-target_link_libraries(emper Threads::Threads emper_asm_source)
-
-add_library(c_emper STATIC ${C_EMPER_SOURCE})
-# See comment above regarding INTERPROCEDURAL_OPTIMIZATION
-# set_property(TARGET c_emper PROPERTY INTERPROCEDURAL_OPTIMIZATION True)
-target_link_libraries(c_emper emper)
-
-add_subdirectory("lib")
-
-add_subdirectory("apps")
-
-add_subdirectory("tests")
-
-add_subdirectory("eval")
-
-file(GLOB ALL_SOURCE_FILES *.cpp)
-file(GLOB ALL_HEADER_FILES *.hpp)
-
-add_custom_target(clang-tidy
-  COMMAND clang-tidy
-  -config=''
-  -p ${CMAKE_BINARY_DIR}
-  ${ALL_SOURCE_FILES}
-  )
-add_dependencies(clang-tidy Main)
-
-add_custom_target(CheckEmperVersionHeader
-  COMMAND ${PROJECT_SOURCE_DIR}/scripts/versionManager.sh -v check
-  COMMENT "Checking if EMPER version header is up-to-date"
-  )
-add_dependencies(emper CheckEmperVersionHeader)

LICENSE

+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

Makefile

-.PHONY: all clean distclean doc full release relwithdebug reldebug debug stresstest test
 SHELL = bash
 
+# https://stackoverflow.com/a/39124162/194894
+word-dot = $(word $2,$(subst ., ,$1))
+
+MESON_VERSION=$(shell meson --version)
+MESON_MAJOR_VERSION=$(call word-dot, $(MESON_VERSION), 1)
+MESON_MINOR_VERSION=$(call word-dot, $(MESON_VERSION), 2)
+
+.PHONY: all build check check-format clean distclean\
+	doc release debug stresstest test
+
+all: build
+
+export BUILDTYPE ?= debugoptimized
+export BUILDDIR = build-$(BUILDTYPE)
+
 NPROC := $(shell nproc)
-COMMON_MAKE_ARGS := -j $(NPROC) -l $(NPROC) $(MAKEFLAGS)
+JOBS := $(shell echo $$(( $(NPROC) + 6)))
+LOAD := $(shell echo $$(( $(NPROC) * 2)))
+
+NINJA_BIN ?= ninja
+NINJA := $(NINJA_BIN) -j $(JOBS) -l $(LOAD) $(EXTRA_NINJA_ARGS)
+
+build:
+	[[ -L build ]] || ./tools/prepare-build-dir
+	$(NINJA) -C $@
+
+release:
+	rm -f build
+	$(MAKE) build BUILDTYPE=$@
+
+debug:
+	rm -f build
+	$(MAKE) build BUILDTYPE=$@
+
+libc++:
+	rm -f build
+	$(MAKE) build \
+		CC=clang CXX=clang++ \
+		EMPER_CPP_ARGS="-stdlib=libc++" \
+		EMPER_CPP_LINK_ARGS="-stdlib=libc++" \
+		EMPER_USE_BUNDLED_DEPS="always" \
+		BUILDDIR="build-libc++"
+
+.PHONY: clang
+clang:
+	rm -f build
+	$(MAKE) build \
+		CC=clang CXX=clang++ \
+		BUILDDIR="build-$@"
+
+.PHONY: clang-release
+clang-release:
+	rm -f build
+	$(MAKE) build \
+		CC=clang CXX=clang++ \
+		BUILDTYPE=release \
+		BUILDDIR="build-$@"
+
+.PHONY: gcc-%
+gcc-%: GCC_VER=$(subst gcc-,,$@)
+gcc-%:
+	rm -f build
+	$(MAKE) build \
+		CC=gcc-$(GCC_VER) CXX=g++-$(GCC_VER) \
+		BUILDDIR="build-$@"
+
+.PHONY: test-%
+test-%: TEST_BUILD=$(subst test-,,$@)
+test-%:
+	$(MAKE) $(TEST_BUILD)
+	meson test -C build-$(TEST_BUILD)
+
+.PHONY: fibril-locked
+fibril-locked:
+	rm -f build
+	$(MAKE) build \
+		EMPER_CONTINUATION_STEALING_MODE=locked \
+		EMPER_WS_QUEUE_SCHEDULER=locked \
+		EMPER_IO=false \
+		BUILDDIR="build-$@"
+
+.PHONY: fibril-unmap
+fibril-unmap:
+	rm -f build
+	$(MAKE) build \
+		EMPER_CONTINUATION_STEALING_MADVISE_STACK=free \
+		EMPER_IO=false \
+		BUILDDIR="build-$@"
 
+.PHONY: stats
+stats:
+	rm -f build
+	$(MAKE) build \
+		EMPER_LOG_LEVEL="Info" \
+		EMPER_STATS=true \
+		BUILDDIR="build-$@"
+
+.PHONY: lto
+lto:
+	rm -f build
+	$(MAKE) build \
+		BUILDTYPE=release \
+		EMPER_B_LTO=true \
+		BUILDDIR="build-$@"
+
+.PHONY: asan
+asan:
+	rm -f build
+	$(MAKE) build \
+		EMPER_LOG_LEVEL="Info" \
+		EMPER_B_SANITIZE=address \
+		BUILDDIR="build-$@"
+
+.PHONY: asan-test
+asan-test: asan
+	meson test -C build-asan
+
+.PHONY: waitfd
+waitfd:
+	rm -f build
+	$(MAKE) build \
+		BUILDTYPE=release \
+		BUILDDIR="build-$@" \
+		EMPER_WORKER_SLEEP_STRATEGY=waitfd
+
+
+.PHONY: fast-static-analysis
+fast-static-analysis: all check-format check-license doc
+
+.PHONY: static-analysis
+static-analysis: fast-static-analysis iwyu tidy
+
+.PHONY: smoke-test-suite
+smoke-test-suite: all
+	cd build && meson test --suite smoke
 
-all: release
+.PHONY: smoke-test
+smoke-test: smoke-test-suite static-analysis
 
-full: release test doc
+TEST_NINJA_TARGETS += test
+
+doc: all
+	$(NINJA) -C build doc/html
+
+check: smoke-test test
+
+test: all
+	$(NINJA) -C build $(TEST_NINJA_TARGETS)
 
 clean:
 	rm -f build
 	rm -rf build-*
 
 distclean: clean
-	./scripts/dependencyManager.sh -v clean
-	./scripts/versionManager.sh -v clean
 	git clean -x -d -f
 
-debug release relwithdebug:
-	[[ -d build-$@ ]] || mkdir build-$@
-	rm -f build
-	ln -rs build-$@ build
-	cd build-$@; \
-		[[ -f CMakeCache.txt ]] || cmake -DCMAKE_BUILD_TYPE=$@ .. \
-		&& make $(COMMON_MAKE_ARGS)
+check-format:
+	./tools/check-format
+
+.PHONY: check-license
+check-license:
+	./tools/check-license
+
+.PHONY: format
+format: all
+	$(NINJA) -C build clang-format
 
-reldebug: relwithdebug
+.PHONY: tidy
+tidy: compile_commands_wo_subprojects/compile_commands.json
+	./tools/run-clang-tidy
 
-test doc: release
-	cd build \
-		&& make $(COMMON_MAKE_ARGS) $@
+PHONY: iwyu
+iwyu: compile_commands_wo_subprojects/compile_commands.json
+	$(NINJA) -C build $@
+
+build/compile_commands.json: all
+
+compile_commands_wo_subprojects/compile_commands.json: all build/compile_commands.json
+	./tools/gen-compile-commands-wo-subprojects
+
+PHONY: fix-includes
+fix-includes: all
+	./tools/fix-includes
 
 stresstest: test
 	./stresstest/stresstest.sh build/tests/simplest_fib_test
+
+# TODO: Determine how we can run also jobs from the 'test' stage,
+# e.g. test-gcc.
+.PHONY: gitlab-runner
+gitlab-runner:
+	gitlab-ci-local smoke-test-suite
+
+.PHONY: subprojects
+subprojects:
+	meson subprojects update --reset
+
+.PHONY: external-deps
+external-deps: subprojects

README.md

+# EMPER
+
+The Efficient Massive-Parallelism Execution Realm (EMPER) is a concurrency platform to develop and execute parallel applications.
+
+EMPER's primary objective is to support the research of different aspects of parallel execution of a potentially massive amount of concurrent strands of execution on current and future many-core systems.
+Those aspects include, for example, scheduling strategies, synchronization primitives, and ultimately the overall coordination of concurrent strands of execution concerning optimal resource usage, including locality and energy efficiency.
+We believe that a concurrency platform can only achieve outstanding results if the runtime system and operating system cooperate closely.
+Hence another research focus of EMPER is the interface between the user-space runtime system and the operating system.
+For example, EMPER uses modern asynchronous system request techniques and a novel approach called "I/O-stealing" to dispatch the operating system's responses back into the user-space runtime system.
+
+The research with EMPER follows a focused approach to demonstrate and prove the viability of researched concepts in a concrete implementation.
+Furthermore, EMPER strives for correctness so that the results of evaluations involving EMPER are reliable.
+While developed as part of academic activity, we develop EMPER to produce a usable end-product in mind, wherever possible and sensible.
+
+To enable every efficient fork/join parallelism, the runtime system of EMPER implements the *wait-free* "Nowa" continuation-stealing approach as described in the IPDPS 2021 paper by Schmaus et al.
+This implementation allows for very efficient fork/join parallelism for two reasons. First, continuation-stealing enables dynamic task parallelism: The concurrency expressed at the programming-language layer is only lifted into parallelism by the runtime system if there are available workers.
+Secondly, the wait-free Nowa approach allows for scalability on systems with many cores.
+
+
+```c++
+emper_fibril auto fib(int n) -> int {
+	if (n < 2) return n;
+
+	Fibril fibril;
+	int a, b;
+
+	fibril.spawn(&a, fib, n - 1);
+
+	b = fib(n - 2);
+
+	fibril.sync();
+
+	return a + b;
+}
+```
+
+## Scheduling
+
+EMPER provides a modular architecture allowing for different scheduling strategies. Currently two scheduling strategies are implemented:
+- Work stealing scheduling (WS)
+- Locality aware and work stealing scheduling (LAWS)
+
+
+# License
+
+EMPER is licensed under the GNU Lesser General Public License, version 3 (or any later version). See the file 'LGPL-3' for the full text of this license.
+
+
+# Acknowledgements
+
+Nicolas Pfeiffer wrote the first prototypical implementation of continuation-stealing for EMPER.
+Much of his code was re-used for the current implementation.
+
+Florian Fischer wrote the EMPER interface for "pseudo-blocking system calls" based on Linux's io_uring.
+We would also like to thank Jens Axboe and Pavel Begunkov for creating and constantly improving io_uring.
+
+This work was partially funded by the Deutsche Forschungsgemeinschaft (DFG, German Research Foundation) – project number 146371743 – TRR 89 "Invasive Computing".
+
+# Publications
+
+[schmaus2021modern]
+Schmaus, Florian, Florian Fischer, Timo Hönig, and Wolfgang Schröder-Preikschat.
+Modern Concurrency Platforms Require Modern System-Call Techniques.
+Tech. rep. CS-2021-02. Friedrich-Alexander Universität Erlangen-Nürnberg,
+Technische Fakultät, Nov. 23, 2021. doi: 10.25593/issn.2191-5008/CS-2021-02.
+url: https://www4.cs.fau.de/~flow/papers/schmaus2021modern.pdf
+
+[schmaus2021nowa]
+Schmaus, Florian, Nicolas Pfeiffer, Timo Hönig, Jörg Nolte, and Wolfgang Schröder-
+Preikschat. “Nowa: A Wait-Free Continuation-Stealing Concurrency Platform”.
+In: 2021 IEEE International Parallel and Distributed Processing Symposium (IPDPS).
+May 2021, pp. 360–371. doi: 10.1109/IPDPS49936.2021.00044.
+url: https://www4.cs.fau.de/~flow/papers/schmaus2021nowa.pdf
+
+[pfeiffer2020cactus]
+Pfeiffer, Nicolas. A Wait-Free Cactus Stack Implementation for a Microparalelism
+Runtime. Master's thesis. MA-I4-2020-02. Mar. 2, 2020.
+url: https://www4.cs.fau.de/~flow/papers/pfeiffer2020cactus.pdf
+
+# Literature
+
+> The dwarf sees farther than the giant, when he has the giant's shoulder to mount on.
+  - Samuel Taylor Coleridge, The Friend (1828)
+
+EMPER uses concepts, ideas and algorithms from the following
+publications. You will find the key of a publication, e.g.,
+[chase2005dynamic] sometimes mentioned in EMPER's source code.
+
+[chase2005dynamic]
+Chase, David and Yossi Lev. “Dynamic Circular Work-Stealing Deque”. In: Pro-
+ceedings of the Seventeenth Annual ACM Symposium on Parallelism in Algorithms
+and Architectures. SPAA ’05. Las Vegas, Nevada, USA: Association
+for Computing Machinery, 2005, pp. 21–28. isbn: 1581139861. doi: [10.1145/1073970.1073974](https://doi.org/10.1145/1073970.1073974).
+
+[le2013correct]
+Lê, Nhat Minh, Antoniu Pop, Albert Cohen, and Francesco Zappa Nardelli.
+“Correct and Efficient Work-Stealing for Weak Memory Models”. In: Proceedings
+of the 18th ACM SIGPLAN Symposium on Principles and Practice of
+Parallel Programming. PPoPP ’13. Shenzhen, China: Association for Computing
+Machinery, 2013, pp. 69–80. isbn: 9781450319225. doi: 10.1145/2442516.2442524.
+url: https://hal.inria.fr/hal-00802885/document
+
+[norris2013cdschecker]
+Norris, Brian and Brian Demsky. “CDSchecker: Checking Concurrent Data Structures
+Written with C/C++ Atomics”. In: Proceedings of the 2013 ACM SIGPLAN
+International Conference on Object Oriented Programming Systems
+Languages Applications. OOPSLA ’13. Indianapolis, Indiana, USA: Association
+for Computing Machinery, 2013, pp. 131–150. isbn: 9781450323741.
+doi: 10.1145/2509136.2509514.
+url: http://plrg.eecs.uci.edu/publications/c11modelcheck.pdf

apps/CMakeLists.txt

-function(emper_app_single_file emper_app_name)
-  set(emper_app_filename "${emper_app_name}.cpp")
-  message("EMPER single file app: ${emper_app_name} (${emper_app_filename})")
-  add_executable(${emper_app_name} ${emper_app_filename})
-  target_link_libraries(${emper_app_name} Threads::Threads emper)
-endfunction()
-
-emper_app_single_file(Main)
-emper_app_single_file(WorkerSleepExample)

apps/Coordinator.cpp

+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright © 2021 Florian Fischer
+#include <sys/types.h>
+
+#include <array>
+#include <atomic>
+#include <cstdlib>
+#include <functional>
+#include <iostream>
+#include <mutex>
+#include <string>
+#include <vector>
+
+#include "Common.hpp"
+#include "Runtime.hpp"
+#include "io.hpp"
+#include "io/Future.hpp"
+#include "io/IoContext.hpp"
+
+std::string HOST = "::";
+std::string PORT = "12346";
+
+std::atomic<unsigned> ready;
+std::mutex lock;
+std::vector<int> conns;
+
+void notify() {
+	std::vector<emper::io::SendFuture*> futures;
+	std::string msg = "Go";
+	for (int conn : conns) {
+		auto* sf = new emper::io::SendFuture(conn, msg.c_str(), msg.size(), 0);
+		futures.push_back(sf);
+	}
+
+	IoContext::getIo()->submit(futures.begin(), futures.end());
+	for (auto* f : futures) {
+		int res = f->wait();
+		if (res < 0) DIE_MSG_ERRNO("send failed");
+		delete f;
+	}
+
+	for (int conn : conns) emper::io::closeAndForget(conn);
+}
+
+// NOLINTNEXTLINE(bugprone-exception-escape)
+auto main(int argc, char* argv[]) -> int {
+	if (argc != 2) {
+		std::cerr << "Usage: " << argv[0] << " <count>" << std::endl;
+		exit(EXIT_FAILURE);
+	}
+
+	const int scount = std::stoi(argv[1]);
+	if (scount < 0) DIE_MSG("count must be positiv");
+	const auto count = static_cast<unsigned>(scount);
+
+	std::cout << "Network barrier listening on " << HOST << ":" << PORT << " for " << count
+						<< " connections" << std::endl;
+
+	Runtime runtime;
+	auto coordinator_func = [&](int socket) {
+		{
+			std::lock_guard<std::mutex> l(lock);
+			conns.push_back(socket);
+		}
+
+		std::array<char, 16> buf;
+		ssize_t recv = emper::io::recvAndWait(socket, buf.data(), buf.size(), 0);
+		if (recv < 0) DIE_MSG_ERRNO("recv failed");
+
+		unsigned r = ready.fetch_add(1) + 1;
+		if (r == count) {
+			notify();
+			runtime.initiateTermination();
+		}
+	};
+
+	auto* listener = emper::io::tcp_listener(HOST, PORT, coordinator_func, scount,
+																					 {emper::io::SockOpt::ReusePort});
+
+	if (!listener) {
+		exit(EXIT_FAILURE);
+	}
+
+	runtime.scheduleFromAnywhere(*listener);
+
+	runtime.waitUntilFinished();
+
+	return EXIT_SUCCESS;
+}

apps/EchoServer.cpp

+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright © 2020-2021 Florian Fischer
+#include <sys/socket.h>
+#include <sys/types.h>
+
+#include <atomic>
+#include <cerrno>
+#include <chrono>
+#include <cstdlib>
+#include <cstring>
+#include <functional>
+#include <iostream>
+#include <random>
+#include <string>
+#include <vector>
+
+#include "Common.hpp"
+#include "Debug.hpp"
+#include "Runtime.hpp"
+#include "RuntimeBuilder.hpp"
+#include "emper-common.h"
+#include "emper-config.h"
+#include "io.hpp"
+
+#ifdef EMPER_HAS_COMPARE_H
+#include <compare>
+#endif
+
+static const std::string HOST = "::";
+static const std::string PORT = "12345";
+static const int BACKLOG = 1024;
+
+static const size_t BUF_SIZE = 1024;
+
+static unsigned int computations_us = 0;
+static unsigned int max_computations_us = 0;
+static float max_computations_probability = -1;
+
+static std::atomic<bool> quit = false;
+
+// NOLINTNEXTLINE(cert-msc32-c,cert-msc51-cpp)
+static thread_local std::mt19937 randGenerator;
+static auto getComputation() -> unsigned {
+	// fixed computation is computations_us
+	if (!max_computations_us) return computations_us;
+
+	// computation is in range [computations_us, max_computations_us]
+	if (max_computations_probability == -1) {
+		std::uniform_int_distribution<unsigned int> distribution(computations_us, max_computations_us);
+		return computations_us += distribution(randGenerator);
+	}
+
+	// computation is either computations_us or max_computations_us with probability
+	// max_computations_probability
+	std::uniform_real_distribution<float> distribution(0, 1);
+	float p = distribution(randGenerator);
+	return p >= max_computations_probability ? max_computations_us : computations_us;
+}
+
+auto main(int argc, char* argv[]) -> int {
+	std::string host = HOST;
+	std::string port = PORT;
+
+	if (argc > 5) {
+		std::cerr << "Usage: " << argv[0] << " [port] [computations_us]"
+							<< " [max_computations_us] [max_computations_probability]" << std::endl;
+		exit(EXIT_FAILURE);
+	}
+
+	if (argc > 1) {
+		port = std::string(argv[1]);
+	}
+
+	if (argc > 2) {
+		computations_us = std::stoi(argv[2]);
+	}
+
+	if (argc > 3) {
+		max_computations_us = std::stoi(argv[3]);
+		if (max_computations_us < computations_us)
+			DIE_MSG("max_computations_us must be bigger than computations_us");
+	}
+
+	if (argc > 4) {
+		max_computations_probability = std::stof(argv[4]);
+		if (max_computations_probability < 0 || max_computations_probability > 1)
+			DIE_MSG("max_computations_probability must be in [0,1]");
+	}
+
+	std::cout << "Echoserver listening on " << host << ":" << port;
+	if (computations_us) {
+		std::cout << " with " << computations_us;
+		if (max_computations_us) std::cout << " - " << max_computations_us;
+		std::cout << " us computations";
+	}
+	std::cout << std::endl;
+
+	RuntimeBuilder runtimeBuilder;
+	if (max_computations_us) {
+		runtimeBuilder.newWorkerHook([](workerid_t id) { randGenerator.seed(id); });
+	}
+
+	auto runtime = runtimeBuilder.build();
+
+	auto serverFunc = [](int socket) {
+		// NOLINTNEXTLINE(modernize-avoid-c-arrays)
+		char buf[BUF_SIZE];
+		while (!quit.load(std::memory_order_consume)) {
+			ssize_t bytes_recv = emper::io::recvAndWait(socket, buf, sizeof(buf), 0);
+			if (unlikely(bytes_recv <= 0)) {
+				// socket was shutdown
+				if (bytes_recv < 0) {
+					LOGE("server read failed:" << strerror(errno));
+				}
+				break;
+			}
+
+			if (unlikely(bytes_recv == 5 && strncmp("quit\n", buf, bytes_recv) == 0)) {
+				quit = true;
+				Runtime::getRuntime()->initiateTermination();
+				break;
+			}
+
+			if (computations_us) {
+				unsigned int computation = getComputation();
+				const auto start = std::chrono::steady_clock::now();
+				const auto deadline = start + std::chrono::microseconds(computation);
+				// TODO: The suppressed linter error below may be a false positive
+				// reported by clang-tidy.
+				// NOLINTNEXTLINE(modernize-use-nullptr)
+				while (std::chrono::steady_clock::now() < deadline) {
+				}
+			}
+
+			ssize_t bytes_send = emper::io::sendAndWait(socket, buf, bytes_recv, MSG_NOSIGNAL, true);
+			if (unlikely(bytes_recv != bytes_send)) {
+				LOGE("server send failed: " << strerror(errno));
+				break;
+			}
+		}
+
+		emper::io::closeAndForget(socket);
+	};
+
+	auto* listener =
+			emper::io::tcp_listener(host, port, serverFunc, BACKLOG, {emper::io::SockOpt::ReusePort});
+
+	if (!listener) {
+		exit(EXIT_FAILURE);
+	}
+
+	runtime.scheduleFromAnywhere(*listener);
+
+	runtime.waitUntilFinished();
+
+	return EXIT_SUCCESS;
+}

apps/EchoServerCallback.cpp

+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright © 2020-2021 Florian Fischer
+#include <sys/socket.h>
+
+#include <array>
+#include <atomic>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <functional>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "Common.hpp"
+#include "Debug.hpp"
+#include "Runtime.hpp"
+#include "io.hpp"
+#include "io/Future.hpp"
+
+using SendFuture = emper::io::SendFuture;
+using RecvFuture = emper::io::RecvFuture;
+
+const std::string HOST = "::";
+const std::string PORT = "12345";
+static const int BACKLOG = 1024;
+
+const size_t BUF_SIZE = 1024;
+
+static std::atomic<bool> quit = false;
+
+class Client {
+	int sockfd;
+	std::array<char, BUF_SIZE> buf;
+
+	void terminate() {
+		emper::io::closeAndForget(sockfd);
+		delete this;
+	}
+
+	void submitSend(int32_t bytes_recv) {
+		SendFuture sf(sockfd, buf.data(), bytes_recv, MSG_NOSIGNAL);
+		sf.setCallback(
+				[this, bytes_recv](int32_t bytes_send) { this->onSend(bytes_send, bytes_recv); });
+		sf.submit();
+	}
+
+	void onSend(int32_t bytes_send, int32_t bytes_recv) {
+		if (unlikely(bytes_send != bytes_recv)) {
+			LOGE("server send failed: " << strerror(-bytes_send));
+			this->terminate();
+			return;
+		}
+
+		if (!quit.load(std::memory_order_consume)) {
+			submitRecv();
+		} else {
+			this->terminate();
+		}
+	}
+
+	void onRecv(int32_t bytes_recv) {
+		if (unlikely(bytes_recv <= 0)) {
+			// socket was shutdown
+			if (bytes_recv < 0) {
+				LOGE("server read failed:" << strerror(-bytes_recv));
+			}
+
+			this->terminate();
+			return;
+		}
+
+		if (unlikely(bytes_recv == 5 && strncmp("quit\n", buf.data(), bytes_recv) == 0)) {
+			quit = true;
+			Runtime::getRuntime()->initiateTermination();
+			this->terminate();
+			return;
+		}
+
+		submitSend(bytes_recv);
+	}
+
+ public:
+	Client(int socket) : sockfd(socket) {}
+
+	void submitRecv() {
+		RecvFuture rf(sockfd, buf.data(), BUF_SIZE, 0);
+		rf.setCallback([this](int32_t bytes_recv) { this->onRecv(bytes_recv); });
+		rf.submit();
+	}
+};
+
+auto main(int argc, char* argv[]) -> int {
+	std::string host = HOST;
+	std::string port = PORT;
+
+	if (argc > 2) {
+		std::cerr << "Usage: " << argv[0] << " [port]" << std::endl;
+		exit(EXIT_FAILURE);
+	}
+
+	if (argc > 1) {
+		port = std::string(argv[1]);
+	}
+
+	std::cout << "Echoserver listening on " << host << ":" << port << std::endl;
+
+	Runtime runtime;
+	auto* listener = emper::io::tcp_listener(host, port,
+																					 [](int socket) {
+																						 auto* client = new Client(socket);
+																						 client->submitRecv();
+																					 },
+																					 BACKLOG, {emper::io::SockOpt::ReusePort});
+
+	if (!listener) {
+		exit(EXIT_FAILURE);
+	}
+
+	runtime.scheduleFromAnywhere(*listener);
+
+	runtime.waitUntilFinished();
+
+	return EXIT_FAILURE;
+}

apps/EmperInfo.cpp

+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright © 2022 Florian Schmaus
+#include <cstdlib>
+
+#include "Emper.hpp"
+#include "emper-common.h"
+
+auto main(UNUSED_ARG int argc, UNUSED_ARG char* argv[]) -> int {
+	emper::printInfo();
+	return EXIT_SUCCESS;
+}

apps/FibChildStealing.cpp

+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright © 2020-2022 Florian Schmaus
+#include <boost/program_options.hpp>
+#include <cstdint>
+#include <iostream>	 // for basic_ostream::operator<<
+#include <string>
+#include <thread>
+
+#include "BinaryPrivateSemaphore.hpp"		 // for BPS
+#include "CountingPrivateSemaphore.hpp"	 // for CPS
+#include "Debug.hpp"										 // for DBG
+#include "Fiber.hpp"										 // for Fiber
+#include "PrivateSemaphore.hpp"					 // for PS
+#include "Runtime.hpp"									 // for Runtime
+#include "lib/sync/Semaphore.hpp"
+
+namespace po = boost::program_options;
+
+using fibParams = struct {
+	uint64_t n;
+	uint64_t* result;
+	PS* sem;
+};
+
+static void fib(void* voidParams) {
+	auto* params = static_cast<fibParams*>(voidParams);
+	uint64_t n = params->n;
+	auto* result = params->result;
+	PS* sem = params->sem;
+
+	if (n < 2) {
+		*result = n;
+	} else {
+		CPS newSem(2);
+
+		uint64_t a, b;
+
+		fibParams newParams1;
+		newParams1.n = n - 1;
+		newParams1.result = &a;
+		newParams1.sem = &newSem;
+
+		// Note that this is the inefficient spawn/sync variant, we
+		// usually would compute one previous fib number without spawning.
+		fibParams newParams2;
+		newParams2.n = n - 2;
+		newParams2.result = &b;
+		newParams2.sem = &newSem;
+
+		Fiber* f1 = Fiber::from(&fib, &newParams1);
+		Fiber* f2 = Fiber::from(&fib, &newParams2);
+
+		Runtime* runtime = Runtime::getRuntime();
+		runtime->schedule(*f1);
+		runtime->schedule(*f2);
+
+		DBG("fib: Calling wait for n=" << n);
+		newSem.wait();
+
+		*result = a + b;
+	}
+
+	DBG("fib: Calling signalAndExit for n=" << n);
+	sem->signalAndExit();
+}
+
+// NOLINTNEXTLINE(bugprone-exception-escape)
+auto main(int argc, char* argv[]) -> int {
+	uint64_t fibNum = 12;
+	po::options_description desc("Allowed options");
+	// clang-format off
+	desc.add_options()
+		("help", "Show help")
+		("nthreads", po::value<unsigned int>()->default_value(std::thread::hardware_concurrency()), "Number of worker threads used by EMPER's runtime system")
+		("fibnum", po::value<uint64_t>(&fibNum)->default_value(fibNum), "The Fibonacci number to compute")
+		;
+	// clang-format on
+
+	// Make 'fibnum' a positional option.
+	po::positional_options_description pos_desc;
+	pos_desc.add("fibnum", -1);
+
+	// clang-format off
+	auto parse_result = po::command_line_parser(argc, argv)
+		.options(desc)
+		.positional(pos_desc)
+		.run()
+		;
+	// clang-format on
+
+	po::variables_map vm;
+	po::store(parse_result, vm);
+	po::notify(vm);
+
+	const unsigned nthreads = vm["nthreads"].as<unsigned int>();
+
+	std::cout << "Number of threads: " << nthreads << std::endl;
+
+	Runtime runtime(nthreads);
+
+	emper::lib::sync::Semaphore semaphore;
+
+	Fiber* fibFiber = Fiber::from([&] {
+		uint64_t result;
+		BPS sem;
+		fibParams params = {fibNum, &result, &sem};
+
+		fib(&params);
+
+		sem.wait();
+
+		std::cout << "fib(" << fibNum << ") = " << result << std::endl;
+
+		semaphore.notify();
+	});
+
+	runtime.scheduleFromAnywhere(*fibFiber);
+
+	semaphore.wait();
+
+	return 0;
+}

apps/MatMul.cpp

+// SPDX-License-Identifier: LGPL-3.0-or-later
+// Copyright © 2024 Florian Schmaus
+#include <array>
+#include <boost/numeric/ublas/io.hpp>
+#include <boost/numeric/ublas/matrix.hpp>
+#include <boost/program_options.hpp>
+#include <chrono>
+#include <cstdint>
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+#include <random>
+#include <string>
+#include <thread>
+#include <type_traits>
+#include <vector>
+
+#include "Common.hpp"
+#include "CountingPrivateSemaphore.hpp"
+#include "Fiber.hpp"
+#include "Runtime.hpp"
+#include "Semaphore.hpp"
+#include "emper.hpp"
+
+namespace po = boost::program_options;
+
+template <class T>
+using matrix = boost::numeric::ublas::matrix<T>;
+
+static auto sequentialMatMul(matrix<double> &a, matrix<double> &b) -> matrix<double> {
+	const auto m = a.size1();	 // number of rows of a
+	const auto n = a.size2();	 // number of columns of a
+	if (n != b.size1()) DIE_MSG("Invalid argument");
+	const auto l = b.size2();	 // number of rows of b
+
+	matrix<double> res(m, l);
+
+	for (unsigned int i = 0; i < m; ++i) {
+		for (unsigned int j = 0; j < l; ++j) {
+			res(i, j) = 0;
+			for (unsigned int k = 0; k < n; ++k) {
+				res(i, j) = res(i, j) + a(i, k) * b(k, j);
+			}
+		}
+	}
+
+	return res;
+}
+
+static auto naiveParallelMatMul(matrix<double> &a, matrix<double> &b) -> matrix<double> {
+	const auto m = a.size1();	 // number of rows of a
+	const auto n = a.size2();	 // number of columns of a
+	if (n != b.size1()) DIE_MSG("Invalid argument");
+	const auto l = b.size2();	 // number of rows of b
+
+	matrix<double> res(m, l);
+
+	CPS cps;
+
+	for (unsigned int i = 0; i < m; ++i) {
+		for (unsigned int j = 0; j < l; ++j) {
+			spawn(
+					[i, j, n, &a, &b, &res] {
+						res(i, j) = 0;
+						for (unsigned int k = 0; k < n; ++k) {
+							res(i, j) = res(i, j) + a(i, k) * b(k, j);
+						}
+					},
+					cps);
+		}
+	}
+
+	cps.wait();
+
+	return res;
+}
+
+static auto stupidParallelMatMul(matrix<double> &a, matrix<double> &b) -> matrix<double> {
+	const auto m = a.size1();	 // number of rows of a
+	const auto n = a.size2();	 // number of columns of a
+	if (n != b.size1()) DIE_MSG("Invalid argument");
+	const auto l = b.size2();	 // number of rows of b
+
+	matrix<double> res(m, l);
+
+	CPS outerCps;
+
+	for (unsigned int i = 0; i < m; ++i) {
+		for (unsigned int j = 0; j < l; ++j) {
+			outerCps.incrementCounterByOne();
+
+			auto *innerCps = new CPS();
+			auto *sem = new emper::Semaphore(1);
+			auto *scalprod = new double;
+
+			for (unsigned int k = 0; k < n; ++k) {
+				spawn(
+						[i, j, k, sem, scalprod, &a, &b] {
+							double product = a(i, k) * b(k, j);
+
+							sem->acquire();
+							*scalprod += product;
+							sem->release();
+						},
+						*innerCps);
+			}
+
+			async([i, j, innerCps, sem, scalprod, &res, &outerCps] {
+				innerCps->wait();
+
+				res(i, j) = *scalprod;
+
+				outerCps.signal();
+
+				delete innerCps;
+				delete sem;
+				delete scalprod;
+			});
+		}
+	}
+
+	outerCps.wait();
+
+	return res;
+}
+
+template <class G>
+static auto createRandomMatrix(unsigned int m_dim, unsigned int n_dim, float zero_probability, G &g)
+		-> matrix<double> {
+	double min = 0;
+	double max = 1'000'000'000'000'000;
+	std::uniform_real_distribution<> value_distribution(min, max);
+
+	std::bernoulli_distribution zero_distribution(zero_probability);
+
+	matrix<double> res(m_dim, n_dim);
+	for (unsigned int m = 0; m < m_dim; ++m) {
+		for (unsigned int n = 0; n < n_dim; ++n) {
+			double value;
+
+			if (zero_distribution(g))
+				value = 0;
+			else
+				value = value_distribution(g);
+
+			res(m, n) = value;
+		}
+	}
+
+	return res;
+}
+
+using ParMatMul = struct {
+	std::string name;
+	matrix<double> (*mul)(matrix<double> &a, matrix<double> &b);
+};
+
+namespace chrn = std::chrono;
+using Clock = std::chrono::high_resolution_clock;
+
+static auto matmul(const po::variables_map &vm) -> int {
+	const auto nthreads = vm["nthreads"].as<unsigned int>();
+	const auto seed = vm["seed"].as<std::uint_fast64_t>();
+	const auto m = vm["m"].as<unsigned int>();
+	const auto n = vm["n"].as<unsigned int>();
+	const auto l = vm["l"].as<unsigned int>();
+	const auto par_mat_muls = vm["parMatMuls"].as<unsigned int>();
+	const auto iters = vm["iters"].as<unsigned int>();
+	const auto zero_probability = vm["zeroProb"].as<float>();
+	const auto verify = vm["verify"].as<bool>();
+
+	Runtime runtime(nthreads);
+	std::mt19937_64 rng(seed);
+
+	std::vector<matrix<double>> a_mtxs, b_mtxs, seq_results;
+
+	for (unsigned int i = 0; i < par_mat_muls; ++i) {
+		auto a = createRandomMatrix(m, n, zero_probability, rng);
+		a_mtxs.push_back(a);
+
+		auto b = createRandomMatrix(n, l, zero_probability, rng);
+		b_mtxs.push_back(b);
+
+		auto seq_res = sequentialMatMul(a_mtxs[i], b_mtxs[i]);
+		seq_results.push_back(seq_res);
+	}
+
+	auto par_mat_mul_impls = std::to_array<ParMatMul>({
+			{"naive", naiveParallelMatMul},
+			{"stupid", stupidParallelMatMul},
+	});
+
+	int exitStatus = EXIT_SUCCESS;
+	Fiber *alphaFiber = Fiber::from([&] {
+		for (auto impl : par_mat_mul_impls) {
+			std::vector<chrn::milliseconds> durations(iters);
+			std::cout << "Start " << impl.name << " with " << iters << " iterations: " << std::flush;
+
+			for (unsigned int iter = 0; iter < iters; ++iter) {
+				std::vector<matrix<double>> results(par_mat_muls);
+				CPS cps(par_mat_muls);
+
+				chrn::time_point<Clock> start = Clock::now();
+				for (unsigned int i = 0; i < par_mat_muls; ++i) {
+					async([i, &cps, &impl, &results, &a_mtxs, &b_mtxs] {
+						auto a = a_mtxs[i];
+						auto b = b_mtxs[i];
+
+						auto res = impl.mul(a, b);
+
+						results[i] = res;
+						cps.signal();
+					});
+				}
+
+				cps.wait();
+				chrn::time_point<Clock> end = Clock::now();
+
+				std::cout << "." << std::flush;
+
+				auto duration = chrn::duration_cast<chrn::milliseconds>(end - start);
+				durations[iter] = duration;
+
+				if (!verify) continue;
+				for (unsigned int i = 0; i < par_mat_muls; ++i) {
+					if (boost::numeric::ublas::detail::equals(seq_results[i], results[i], 1.e-6, 0.))
+						continue;
+
+					// clang-format off
+					std::cerr
+						<< std::endl
+						<< "ERROR: Matrix " << i << " produced by " << impl.name << " incorrect" << std::endl
+						<< "Expected matrix: " << std::endl << seq_results[i] << std::endl
+						<< "Provided matrix: " << std::endl << results[i] << std::endl
+						;
+					// clang-format on
+
+					exitStatus = EXIT_FAILURE;
+					runtime.initiateTermination();
+					return;
+				}
+			}
+
+			std::cout << std::endl << "Implementation '" << impl.name << "' took ";
+			for (unsigned int i = 0; i < iters; ++i) {
+				std::cout << durations[i];
+				if (i + 1 != iters) std::cout << ", ";
+			}
+			std::cout << ";" << std::endl;
+		}
+
+		runtime.initiateTermination();
+	});
+	runtime.scheduleFromAnywhere(*alphaFiber);
+	runtime.waitUntilFinished();
+
+	return exitStatus;
+}
+
+auto main(int argc, char *argv[]) -> int {
+	po::options_description desc("Allowed options");
+	// clang-format off
+	desc.add_options()
+		("help", "Show help")
+		("nthreads", po::value<unsigned int>()->default_value(std::thread::hardware_concurrency()), "Number of worker threads used by EMPER's runtime system")
+		("seed", po::value<std::uint_fast64_t>()->default_value(20180922), "Initial seed of the random number generator")
+		("m", po::value<unsigned int>()->default_value(10), "Number of rows of the first matrix and of the result matrix")
+		("n", po::value<unsigned int>()->default_value(10), "Number of columns of the first matrix and of rows of the second matrix")
+		("l", po::value<unsigned int>()->default_value(10), "Number of columns of the second matrix and of the result matrix")
+		("parMatMuls", po::value<unsigned int>()->default_value(5), "Number of matrix multiplications to perform in parallel")
+		("iters", po::value<unsigned int>()->default_value(3), "Number of benchmark iterations")
+		("zeroProb", po::value<float>()->default_value(0.01), "Probaility of generating a zero when initializing the matrices")
+		("verify", po::value<bool>()->default_value(true), "Verify the result of the matrix multiplication")
+		;
+	// clang-format on
+
+	auto parse_result = po::command_line_parser(argc, argv).options(desc).run();
+
+	po::variables_map vm;
+	po::store(parse_result, vm);
+	po::notify(vm);
+
+	if (vm.count("help")) {
+		std::cout << desc << std::endl;
+		return EXIT_SUCCESS;
+	}
+
+	try {
+		return matmul(vm);
+	} catch (std::exception &e) {
+		std::cerr << e.what() << std::endl;
+		return EXIT_FAILURE;
+	}
+}