Commit 2fbeb921 authored by Florian Fischer's avatar Florian Fischer
Browse files

add io_uring client build tooling

parent 8dca1494
.cache/
build/
/build
/build-*/
/.cache/
/.clangd/
subprojects/packagecache/
subprojects/liburing*
SHELL = bash
.PHONY: all build check check-format clean distclean\
doc release debug stresstest test
all: build
export BUILDTYPE ?= release
export BUILDDIR = build-$(BUILDTYPE)
NPROC := $(shell nproc)
JOBS := $(shell echo $$(( $(NPROC) + 6)))
LOAD := $(shell echo $$(( $(NPROC) * 2)))
NINJA_BIN ?= ninja
NINJA := $(NINJA_BIN) -j $(JOBS) -l $(LOAD) $(EXTRA_NINJA_ARGS)
build:
[[ -L build ]] || ./tools/prepare-build-dir
$(NINJA) -C $@
release:
rm -f build
$(MAKE) build BUILDTYPE=$@
debug:
rm -f build
$(MAKE) build BUILDTYPE=$@
debugoptimized:
rm -f build
$(MAKE) build BUILDTYPE=$@
.PHONY: fast-static-analysis
fast-static-analysis: all check-format check-license
.PHONY: static-analysis
static-analysis: fast-static-analysis iwyu tidy
.PHONY: check-format
check-format:
./tools/check-format
.PHONY: check-license
check-license:
./tools/check-license
check: static-analysis test
test: all
$(NINJA) -C build test
clean:
rm -f build
rm -rf build-*
distclean: clean
git clean -x -d -f
.PHONY: format
format: all
$(NINJA) -C build clang-format
.PHONY: tidy
tidy: compile_commands_wo_subprojects/compile_commands.json
./tools/run-clang-tidy
PHONY: iwyu
iwyu: compile_commands_wo_subprojects/compile_commands.json
$(NINJA) -C build $@
build/compile_commands.json: all
compile_commands_wo_subprojects/compile_commands.json: all build/compile_commands.json
./tools/gen-compile-commands-wo-subprojects
PHONY: fix-includes
fix-includes: all
./tools/fix-includes
// Copyright Florian Fischer 2021
// SPDX-License-Identifier: GPL-3.0-or-later
// Copyright © 2021 Florian Fischer
#include <liburing.h>
#include <netdb.h>
#include <sys/socket.h>
......@@ -121,7 +122,6 @@ Client::Client(const std::string& host, const std::string& port, unsigned connec
workerCount(std::min(_workerCount, connections)),
echoSize(echoSize),
connectBarrier(workerCount + 1) {
std::cout << "[global]" << std::endl;
std::cout << "clients = " << connections << std::endl;
std::cout << "size = " << echoSize << std::endl;
......@@ -222,8 +222,7 @@ Worker::~Worker() {
auto Worker::getEchos() -> unsigned {
unsigned echos = 0;
for (unsigned i = 0; i < connectionCount; ++i)
echos += connections[i]->echos;
for (unsigned i = 0; i < connectionCount; ++i) echos += connections[i]->echos;
return echos;
}
......
[
{
"directory": "/home/muhq/code/emper-io-eval/clients/io_uring/build-debugoptimized",
"command": "ccache c++ -Iclient.p -I. -I.. -fdiagnostics-color=always -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -Wnon-virtual-dtor -Wextra -Wpedantic -Werror -std=c++2a -O2 -g -pthread -MD -MQ client.p/client.cpp.o -MF client.p/client.cpp.o.d -o client.p/client.cpp.o -c ../client.cpp",
"file": "../client.cpp",
"output": "client.p/client.cpp.o"
}
]
\ No newline at end of file
#!/usr/bin/env bash
set -euo pipefail
# Pretty fancy method to get reliable the absolute path of a shell
# script, *even if it is sourced*. Credits go to GreenFox on
# stackoverflow: http://stackoverflow.com/a/12197518/194894
pushd . > /dev/null
SCRIPTDIR="${BASH_SOURCE[0]}";
while([ -h "${SCRIPTDIR}" ]); do
cd "`dirname "${SCRIPTDIR}"`"
SCRIPTDIR="$(readlink "`basename "${SCRIPTDIR}"`")";
done
cd "`dirname "${SCRIPTDIR}"`" > /dev/null
SCRIPTDIR="`pwd`";
popd > /dev/null
DEBUG=false
VERBOSE=false
while getopts dv OPT; do
case $OPT in
d)
set -x
DEBUG=true
VERBOSE=true
;;
v)
VERBOSE=true
;;
*)
echo "usage: ${0##*/} [-dq} [--] ARGS..."
exit 2
esac
done
shift $(( OPTIND - 1 ))
OPTIND=1
ROOTDIR=$(readlink -f "${SCRIPTDIR}/..")
MAX_PROCS=$(nproc)
FILES_TO_CHECK_FILE=$(mktemp)
cleanup() {
rm "${FILES_TO_CHECK_FILE}"
}
if ! $DEBUG; then
trap cleanup EXIT
fi
cd "${ROOTDIR}"
PRUNE_PATHS=()
PRUNE_PATHS+=(./build*) # Generated files
PRUNE_PATHS+=(./subprojects) # Subprojects, since are under different licenses
PRUNE_PATH_ARG=""
# https://stackoverflow.com/a/12298615/194894
for path in "${PRUNE_PATHS[@]::${#PRUNE_PATHS[@]}-1}"; do
PRUNE_PATH_ARG+="-path ${path} -o "
done
PRUNE_PATH_ARG+="-path ${PRUNE_PATHS[-1]}"
# shellcheck disable=SC2086
find . \( ${PRUNE_PATH_ARG} \) -prune -o \
-type f -regextype posix-extended -regex '.*\.(c|h|cpp|hpp)' -print0 \
> "${FILES_TO_CHECK_FILE}"
if $VERBOSE; then
echo "About to check the following files for correct formatting via clang-format"
tr '\0' '\n' < "${FILES_TO_CHECK_FILE}"
fi
# Note that the --dry-run and --Werror clang-format arguments require
# clang-format 10 or higher. See https://reviews.llvm.org/D68554
xargs --null --max-args=3 --max-procs="${MAX_PROCS}" \
clang-format --style=file --dry-run -Werror \
< "${FILES_TO_CHECK_FILE}"
FILE_COUNT=$(<"${FILES_TO_CHECK_FILE}" tr -cd '\0' | wc -c)
echo "Checked ${FILE_COUNT} files for format violations"
#!/usr/bin/env bash
set -euo pipefail
# Meson issue regarding iwyu integration: https://github.com/mesonbuild/meson/issues/2637
echoerr() {
echo "${@}" 1>&2
}
# Pretty fancy method to get reliable the absolute path of a shell
# script, *even if it is sourced*. Credits go to GreenFox on
# stackoverflow: http://stackoverflow.com/a/12197518/194894
pushd . > /dev/null
SCRIPTDIR="${BASH_SOURCE[0]}";
while([ -h "${SCRIPTDIR}" ]); do
cd "`dirname "${SCRIPTDIR}"`"
SCRIPTDIR="$(readlink "`basename "${SCRIPTDIR}"`")";
done
cd "`dirname "${SCRIPTDIR}"`" > /dev/null
SCRIPTDIR="`pwd`";
popd > /dev/null
if [[ -v MESON_BUILD_ROOT ]]; then
MESON_BUILD_ROOT_SET=true
else
MESON_BUILD_ROOT_SET=false
fi
if ! ${MESON_BUILD_ROOT_SET}; then
ROOTDIR=$(readlink -f "${SCRIPTDIR}/..")
make -C "${ROOTDIR}"
MESON_BUILD_ROOT="${ROOTDIR}/build"
exec ninja -C "${MESON_BUILD_ROOT}" iwyu
fi
readonly IWYU_LOG="${MESON_BUILD_ROOT}/iwyu.log"
if [[ ! -v IWYU_TOOL ]]; then
for POSSIBLE_CMD in iwyu_tool iwyu-tool iwyu_tool.py; do
if command -v ${POSSIBLE_CMD} >/dev/null; then
IWYU_TOOL=${POSSIBLE_CMD}
break
fi
done
if [[ ! -v IWYU_TOOL ]]; then
echoerr "iwyu_tool not found"
exit 1
fi
fi
NPROC=$(nproc)
LOAD=$(python -c "print(${NPROC} * 1.5)")
IWYU_TOOL_ARGS=(
-p "${MESON_SOURCE_ROOT}/compile_commands_wo_subprojects"
--jobs "${NPROC}"
)
if "${IWYU_TOOL}" --help | grep -Fq -- '--load LOAD'; then
IWYU_TOOL_ARGS+=(--load "${LOAD}")
else
# See https://github.com/include-what-you-use/include-what-you-use/pull/891
echo "WARNING: ${IWYU_TOOL} does not support --load"
fi
set +e
${IWYU_TOOL} ${IWYU_TOOL_ARGS[@]} \
-- \
-Xiwyu --mapping_file="${MESON_SOURCE_ROOT}/iwyu-mappings.imp" \
> "${IWYU_LOG}"
set -e
# Only newer versions of IWYU_TOOL fail with an exit code (IWYU >
# 0.15, since commit
# https://github.com/include-what-you-use/include-what-you-use/commit/a7499e4a2b416592777cc4c33fca746d091af738). Therefore
# we have to manually see if this IWYU_TOOL run failed with an error
# exit code.
# shellcheck: disable=SC2181
if [[ ${?} -gt 0 ]]; then
echoerr "IWYU exited with ${?}"
cat "${IWYU_LOG}"
exit $?
fi
if [[ ! -s "${IWYU_LOG}" ]]; then
echoerr "${IWYU_LOG} is empty (or non existent)"
exit 1
fi
# Sadly, older iwyu_tool.py version do not return an non-zero exit value if
# there are include issues, so we have to check the output manually.
# See https://github.com/include-what-you-use/include-what-you-use/issues/790
# Also note that the output contains "error: no such file or
# directory: 'cc'" if ccache is used (which meson does by default if
# ccache is available).
# See
# - https://github.com/include-what-you-use/include-what-you-use/issues/789
# - https://github.com/include-what-you-use/include-what-you-use/commit/a7499e4a2b416592777cc4c33fca746d091af738
ERROR_STRINGS=()
ERROR_STRINGS+=("should add these lines:")
ERROR_STRINGS+=("fatal error:")
for ERROR_STRING in "${ERROR_STRINGS[@]}"; do
if grep -q "${ERROR_STRING}" "${IWYU_LOG}"; then
echoerr "IWYU found errors!"
cat "${IWYU_LOG}"
exit 1
fi
done
#!/usr/bin/env bash
set -euo pipefail
# Pretty fancy method to get reliable the absolute path of a shell
# script, *even if it is sourced*. Credits go to GreenFox on
# stackoverflow: http://stackoverflow.com/a/12197518/194894
pushd . > /dev/null
SCRIPTDIR="${BASH_SOURCE[0]}";
while([ -h "${SCRIPTDIR}" ]); do
cd "`dirname "${SCRIPTDIR}"`"
SCRIPTDIR="$(readlink "`basename "${SCRIPTDIR}"`")";
done
cd "`dirname "${SCRIPTDIR}"`" > /dev/null
SCRIPTDIR="`pwd`";
popd > /dev/null
echoerr() { echo "$@" 1>&2; }
AUTOFIX=false
VERBOSE=false
while getopts :dfv OPT; do
case $OPT in
d)
set -x
;;
f)
AUTOFIX=true
;;
v)
VERBOSE=true
;;
*)
echo "usage: ${0##*/} [-d]"
exit 2
esac
done
shift $(( OPTIND - 1 ))
OPTIND=1
export ROOTDIR
ROOTDIR=$(readlink -f "${SCRIPTDIR}/..")
export CHECKED_FILES_FILE
CHECKED_FILES_FILE=$(mktemp)
export INVALID_LICENSE_FILES_FILE
INVALID_LICENSE_FILES_FILE=$(mktemp)
function cleanup() {
rm "${CHECKED_FILES_FILE}"
rm "${INVALID_LICENSE_FILES_FILE}"
}
trap cleanup EXIT
# Safely call bash function from xargs
# https://stackoverflow.com/questions/11003418/calling-shell-functions-with-xargs/11003457#11003457
check_license() {
echo "${1}" >> "${CHECKED_FILES_FILE}"
local license_identifier=".*SPDX-License-Identifier: .*GPL-3.0-or-later"
local copyright_regex=".*Copyright (© )?[0-9]{4}(-[0-9]{4})?( \w)+.*"
local sed_cmd="{N; N; /^${license_identifier}\n${copyright_regex}\n.+$/!{q1}}"
local full_path="${ROOTDIR}/${1}"
if ! head --lines=3 "${full_path}" |\
sed --regexp-extended --quiet "${sed_cmd}" >/dev/null
then
echo "${1}" >> "${INVALID_LICENSE_FILES_FILE}"
return 1
fi
}
export -f check_license
MAX_PROCS=$(nproc)
cd "${ROOTDIR}"
set +e
find . \( -path '*/\.*' -o -path './subprojects*' -o -path "./build*" -o -path "./test/3rd-party*" \) -prune -o \
-type f -regextype posix-extended -regex '.*\.(c|h|cpp|hpp)' -print0 |\
xargs --null --max-procs="${MAX_PROCS}" -I {} \
bash -c 'check_license "$@"' _ "{}"
readonly CHECK_RETURN_VALUE="$?"
set -e
if [[ $CHECK_RETURN_VALUE -ne 0 ]]; then
mapfile -t < "${INVALID_LICENSE_FILES_FILE}"
echoerr "The following ${#MAPFILE[@]} files have a missing or invalid license header:"
for INVALID_LICENSE_FILE in "${MAPFILE[@]}"; do
echoerr "- ${INVALID_LICENSE_FILE}"
done
if $AUTOFIX; then
echo "Autofixing enabled, prepending default header to those files"
for INVALID_LICENSE_FILE in "${MAPFILE[@]}"; do
TMPFILE=$(mktemp)
cat <<EOF > "${TMPFILE}"
// SPDX-License-Identifier: LGPL-3.0-or-later
// Copyright © $(date +%Y) $(git config user.name)
EOF
cat "${INVALID_LICENSE_FILE}" >> "${TMPFILE}"
mv "${TMPFILE}" "${INVALID_LICENSE_FILE}"
done
fi
exit "${CHECK_RETURN_VALUE}"
fi
FILE_COUNT=$(<"${CHECKED_FILES_FILE}" wc -l)
echo "Checked ${FILE_COUNT} files for license headers"
if $VERBOSE; then
mapfile -t < "${CHECKED_FILES_FILE}"
for FILE in "${MAPFILE[@]}"; do
echo "- ${FILE}"
done
fi
#!/usr/bin/env bash
# Pretty fancy method to get reliable the absolute path of a shell
# script, *even if it is sourced*. Credits go to GreenFox on
# stackoverflow: http://stackoverflow.com/a/12197518/194894
pushd . > /dev/null
SCRIPTDIR="${BASH_SOURCE[0]}";
while([ -h "${SCRIPTDIR}" ]); do
cd "`dirname "${SCRIPTDIR}"`"
SCRIPTDIR="$(readlink "`basename "${SCRIPTDIR}"`")";
done
cd "`dirname "${SCRIPTDIR}"`" > /dev/null
SCRIPTDIR="`pwd`";
popd > /dev/null
echoerr() { echo "$@" 1>&2; }
CLANG_TIDY_BIN="clang-tidy"
ATTEMPT_TO_AUTOFIX=true
DEBUG=false
export DEBUG
VERBOSE=false
export VERBOSE
while getopts :c:dnv OPT; do
case $OPT in
d)
set -x
DEBUG=true
;;
c)
CLANG_TIDY_BIN="${OPTARG}"
;;
n)
ATTEMPT_TO_AUTOFIX=false
;;
v)
VERBOSE=true
;;
*)
echo "usage: ${0##*/} [-d]"
exit 2
esac
done
shift $(( OPTIND - 1 ))
OPTIND=1
export ROOTDIR
ROOTDIR=$(readlink -f "${SCRIPTDIR}/..")
readonly TEMP_DIR="/var/tmp"
CLANG_TIDY_FILES=$(mktemp --tmpdir="${TEMP_DIR}")
export CLANG_TIDY_FILES
UNFIXABLE_CLANG_TIDY_FILES=$(mktemp --tmpdir="${TEMP_DIR}")
export UNFIXABLE_CLANG_TIDY_FILES
cleanup() {
if $DEBUG; then
echo "Debug activated, preserving temprorary files"
echo "CLANG_TIDY_FILES=${CLANG_TIDY_FILES}"
echo "UNFIXABLE_CLANG_TIDY_FILES=${UNFIXABLE_CLANG_TIDY_FILES}"
else
rm "${CLANG_TIDY_FILES}"
rm "${UNFIXABLE_CLANG_TIDY_FILES}"
fi
}
trap cleanup exit
check_needs_fixing() {
if $DEBUG; then
set -x
fi
local file="${1}"
if ! ${CLANG_TIDY} "${file}"; then
echo "${file}" >> "${CLANG_TIDY_FILES}"
fi
}
export -f check_needs_fixing
attempt_to_fix() {
if $DEBUG; then
set -x
fi
local file="${1}"
for attempt in {1..10}; do
if $VERBOSE; then
echo "Performing ${attempt}. attempt to fix ${file}"
fi
if ${CLANG_TIDY} --fix --format-style=file "${file}"; then
exit
fi
done
echo "${file}" >> "${UNFIXABLE_CLANG_TIDY_FILES}"
exit 1
}
export -f attempt_to_fix
if $DEBUG; then
MAX_PROCS=1
else
MAX_PROCS=$(nproc)
fi
readonly BUILDDIR="${ROOTDIR}/build"
if [[ ! -d "${BUILDDIR}" ]]; then
echoerr "${BUILDDIR} does not exists, run make first"
exit 1
fi
CLANG_TIDY_OPTS=()
CLANG_TIDY_OPTS=("-p=${BUILDDIR}")
if ! $VERBOSE; then
CLANG_TIDY_OPTS+=("--quiet")
fi
# shellcheck disable=SC2124
readonly CLANG_TIDY="${CLANG_TIDY_BIN} ${CLANG_TIDY_OPTS[@]}"
export CLANG_TIDY
set +e
find "${ROOTDIR}" \( -path '*/\.*' -o -path "./build*" -o -path "./build-*" -o -path "./test/3rd-party*" \) -prune -o \
-type f -regextype posix-extended -regex '.*\.(c|h|cpp|hpp)' -print0 |\
xargs --null --max-args=1 --max-procs="${MAX_PROCS}" -I {} \
bash -c 'check_needs_fixing "$@"' _ "{}"
set -e
readonly CHECK_NEEDS_FIXING_RETURN_VALUE="${?}"
if [[ ! -s "${CLANG_TIDY_FILES}" ]]; then
exit "${CHECK_NEEDS_FIXING_RETURN_VALUE}"
fi
CLANG_TIDY_FILES_NUM=$(<"${CLANG_TIDY_FILES}" wc -l)
echo "The following ${CLANG_TIDY_FILES_NUM} files have issues according to clang-tidy"
cat "${CLANG_TIDY_FILES}"
if ! $ATTEMPT_TO_AUTOFIX; then
echo "Attempt to autofix disabled"
exit 1
fi
set +e
<"${CLANG_TIDY_FILES}" xargs --max-args=1 --max-procs="${MAX_PROCS}" -I {} \
bash -c 'attempt_to_fix "$@"' _ "{}"
set -e
readonly ATTEMP_TO_FIX_RETURN_VALUE="${?}"
if [[ "${ATTEMP_TO_FIX_RETURN_VALUE}" -ne 0 ]]; then
UNFIXABLE_CLANG_TIDY_FILES_NUM=$(<"${UNFIXABLE_CLANG_TIDY_FILES}" wc -l)
echoerr "The following ${UNFIXABLE_CLANG_TIDY_FILES_NUM} files could not be fixed"
cat "${UNFIXABLE_CLANG_TIDY_FILES}" >&2
exit "${ATTEMP_TO_FIX_RETURN_VALUE}"
fi
#!/usr/bin/env bash
set -euo pipefail
useradd -u "${USER_ID}" -o -m user
groupmod -g "${GROUP_ID}" user
OUTSIDE_EMPER_ROOT="${1}"
shift
cd "${OUTSIDE_EMPER_ROOT}"