diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..b45f55965f50c43afac65ff257774856c4d1d31c --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +results/ +__pycache__/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..f6e30cb7051658b8a8f274188545f65cab7e6323 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,42 @@ +image: "flowdalic/debian-testing-dev:1.20" + +before_script: +- | + readarray TOOLS <<EOF + mypy + pdoc + pyflakes + pylint + pytest + python + yapf + EOF + for tool in ${TOOLS[@]}; do + echo -e "$tool version: " + $tool --version + done + +cache: + paths: + - haystack + +stages: + - check + - test + +check-format: + stage: check + script: + - make check-format + +check-pylint: + stage: check + script: + - make check-pylint + +search-emper: + stage: test + script: + - rm -rf emper + - make check-and-reinit-submodules emper-vanilla + - ./eval.py -i 'emper-vanilla' diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..7ec8b8e966667211dd46b0b8cb1c92bd93461847 --- /dev/null +++ b/Makefile @@ -0,0 +1,130 @@ +PYTHONFILES := eval.py summarize.py gen_boxplots.py + +MAKEFILES = $(shell dirname $(shell find . -name Makefile \ + -not -path "./Makefile" \ + -not -path "./emper/*")) + +EMPER_ROOT := emper +EMPER_VARIANTS := vanilla no-sleep \ + pipe pipe-no-hint pipe-no-completer \ + io-stealing io-stealing-lockless \ + io-stealing-pipe io-stealing-lockless-pipe \ + io-stealing-pipe-no-comp io-stealing-lockless-pipe-no-comp \ + single-uring +EMPER_BUILD_TARGETS := $(addprefix emper-,$(EMPER_VARIANTS)) + +.PHONY: $(EMPER_BUILD_TARGETS) + +.PHONY: eval +eval: all + ./eval.py + +.PHONY: stats +stats: all + ./eval.py --desc-stats=desc-stats.yml + +.PHONY: docker-eval +docker-eval: + ./docker.sh $(MAKE) eval + +.PHONY: docker-stats +docker-stats: + ./docker.sh $(MAKE) stats + +.PHONY: all +all: $(MAKEFILES) $(EMPER_BUILD_TARGETS) +#all: check-and-reinit-submodules $(MAKEFILES) $(EMPER_BUILD_TARGETS) + +# git submodule target taken from here: +# https://stackoverflow.com/questions/52337010/automatic-initialization-and-update-of-submodules-in-makefile +.PHONY: check-and-reinit-submodules +check-and-reinit-submodules: + @if git submodule status | egrep -q '^[-]|^[+]' ; then \ + echo "INFO: Need to reinitialize git submodules"; \ + git submodule update --init; \ + fi + +.PHONY: $(MAKEFILES) +$(MAKEFILES): + $(MAKE) -C $@ + +MESON_SETUP := CFLAGS=-g CXXFLAGS=-g meson setup +COMMON_EMPER_MESON_ARGS := --buildtype=release --fatal-meson-warnings -Dstats=true +EMPER_PIPE := -Dworker_sleep_strategy=pipe +EMPER_NO_SLEEP := -Dworker_sleep=false +EMPER_NO_COMPLETER := -Dio_completer_behavior=none +EMPER_IO_STEALING := -Dio_stealing=true +EMPER_IO_LOCKLESS_CQ := -Dio_lockless_cq=true +EMPER_SINGLE_URING := -Dio_single_uring=true +EMPER_IGNORE_HINTS := -Dworker_ignore_wakeup_hint=true + +$(EMPER_ROOT)/build-vanilla: + $(MESON_SETUP) $(COMMON_EMPER_MESON_ARGS) $@ $(@D) + +$(EMPER_ROOT)/build-no-sleep: + $(MESON_SETUP) $(COMMON_EMPER_MESON_ARGS) $(EMPER_NO_SLEEP) $@ $(@D) + +$(EMPER_ROOT)/build-pipe: + $(MESON_SETUP) $(COMMON_EMPER_MESON_ARGS) $(EMPER_PIPE) $@ $(@D) + +$(EMPER_ROOT)/build-pipe-no-hint: + $(MESON_SETUP) $(COMMON_EMPER_MESON_ARGS) $(EMPER_PIPE) $(EMPER_IGNORE_HINTS) $@ $(@D) + +$(EMPER_ROOT)/build-pipe-no-completer: + $(MESON_SETUP) $(COMMON_EMPER_MESON_ARGS) $(EMPER_NO_COMPLETER) $(EMPER_PIPE) $@ $(@D) + +$(EMPER_ROOT)/build-io-stealing: + $(MESON_SETUP) $(COMMON_EMPER_MESON_ARGS) $(EMPER_IO_STEALING) $@ $(@D) + +$(EMPER_ROOT)/build-io-stealing-lockless: + $(MESON_SETUP) $(COMMON_EMPER_MESON_ARGS) $(EMPER_IO_STEALING) $(EMPER_IO_LOCKLESS_CQ) $@ $(@D) + +$(EMPER_ROOT)/build-io-stealing-pipe: + $(MESON_SETUP) $(COMMON_EMPER_MESON_ARGS) $(EMPER_IO_STEALING) $(EMPER_PIPE) $@ $(@D) + +$(EMPER_ROOT)/build-io-stealing-lockless-pipe: + $(MESON_SETUP) $(COMMON_EMPER_MESON_ARGS) $(EMPER_IO_STEALING) \ + $(EMPER_IO_LOCKLESS_CQ) $(EMPER_PIPE) $@ $(@D) + +$(EMPER_ROOT)/build-io-stealing-pipe-no-comp: + $(MESON_SETUP) $(COMMON_EMPER_MESON_ARGS) $(EMPER_IO_STEALING) \ + $(EMPER_NO_COMPLETER) $(EMPER_PIPE) $@ $(@D) + +$(EMPER_ROOT)/build-io-stealing-lockless-pipe-no-comp: + $(MESON_SETUP) $(COMMON_EMPER_MESON_ARGS) $(EMPER_IO_STEALING) \ + $(EMPER_NO_COMPLETER) $(EMPER_IO_LOCKLESS_CQ) $(EMPER_PIPE) $@ $(@D) + +$(EMPER_ROOT)/build-single-uring: + $(MESON_SETUP) $(COMMON_EMPER_MESON_ARGS) $(EMPER_SINGLE_URING) $@ $(@D) + + +define buildEmper +emper-$(1): $(EMPER_ROOT)/build-$(1) + ninja -C $(EMPER_ROOT)/build-$(1) +endef + +# generate emper build targets +$(foreach variant,$(EMPER_VARIANTS),$(eval $(call buildEmper,$(variant)))) + +.PHONY: check +check: check-format check-pylint + +.PHONY: format +format: + yapf -i $(PYTHONFILES) + +.PHONY: check-format +check-format: + yapf -d $(PYTHONFILES) + +.PHONY: check-pylint +check-pylint: + pylint -j 0 $(PYTHONFILES) || ./tools/check-pylint $$? + +.PHONY: clean-emper +clean-emper: + $(MAKE) -C emper clean + +.PHONY: clean +clean: clean-emper + for mkf in $(MAKEFILES); do $(MAKE) -C "$${mkf}" clean; done diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1bb0d8e2fe34af54411c8b103398df97eb3b3c6a --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +# EMPER io latency evaluation artifact + +To run the evaluation using the installed tools and toolchains simply run: +``` +make +``` + +To run the evaluation using a fitting docker image run: +``` +make docker-eval +``` + + +## Dependencies + +* python3 +* native execution: + * git + * meson + * golang + * a C++17 toolchain + * linux >= 5.1 +* docker execution: + * docker diff --git a/docker.sh b/docker.sh new file mode 100755 index 0000000000000000000000000000000000000000..a5f047190e5d980a148cd69d9db1719aa1a6e8c0 --- /dev/null +++ b/docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# The directory of this script is also the project's root directory. +ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +IMAGE=$(sed --regexp-extended --quiet 's;^image: "([^"]*)"$;\1;p' "${ROOT}/.gitlab-ci.yml") + +docker run \ + --volume="${ROOT}:${ROOT}" \ + --interactive \ + --tty \ + --security-opt=seccomp:unconfined \ + --env USER_ID="${UID}" \ + --env GROUP_ID="$(id -g ${USER})" \ + "${IMAGE}" \ + "${ROOT}/tools/docker-prepare" "${ROOT}" $@ diff --git a/eval.py b/eval.py new file mode 100755 index 0000000000000000000000000000000000000000..4fd587457c49ce5b4faa3d5a1e2e9a29fe253c1d --- /dev/null +++ b/eval.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Evaluate different emper variants and their io latency""" + +import argparse +import copy +import datetime +import fnmatch +import os +import subprocess +import sys +from pathlib import Path +import platform +import typing as T + +import yaml + +from summarize import collect, summarize, calc_stats, calc_avgs + +ROOT_DIR = Path(os.path.dirname(os.path.realpath(__file__))) +EMPER_ROOT = ROOT_DIR / 'emper' + +ARTIFACT_DESC = subprocess.check_output( + 'git describe --dirty --always'.split(), cwd=ROOT_DIR, text=True)[:-1] + +TARGETS = {} + +EMPER_VARIANTS = { + 'vanilla': {}, + 'pipe': {}, + 'pipe-no-hint': {}, + 'pipe-no-completer': {}, + 'io-stealing': {}, + 'io-stealing-lockless': {}, + 'io-stealing-pipe': {}, + 'io-stealing-lockless-pipe': {}, + 'io-stealing-pipe-no-comp': {}, + 'io-stealing-lockless-pipe-no-comp': {}, + 'single-uring': {}, +} + +for variant, _desc in EMPER_VARIANTS.items(): + desc = dict(_desc) + if 'cmd' not in desc: + desc['cmd'] = f'{EMPER_ROOT}/build-{variant}/eval/io_latency --data' + TARGETS[f'emper-{variant}'] = desc + + +def filter_targets(include, exclude): + """Apply an include and exclude filter to the targets + + The filters use POSIX globbing""" + targets = copy.copy(TARGETS) + + if include: + filtered_targets = {} + for inc in include: + filtered_targets.update( + {t: targets[t] + for t in fnmatch.filter(targets.keys(), inc)}) + targets = filtered_targets + + if exclude: + filtered_targets = {} + for exp in exclude: + filtered_targets.update({ + t: targets[t] + for t in targets + if t not in fnmatch.filter(targets.keys(), exp) + }) + targets = filtered_targets + return targets + + +RESULTS_ROOT = ROOT_DIR / 'results' + + +def prepare_env(update_env: T.MutableMapping) -> T.Dict: + """Update and return the a copy of os.environ with a new mapping""" + current_env = dict(os.environ) + current_env.update(update_env) + return current_env + + +def main(args): + """Run an evaluation""" + for target, target_conf in TARGETS.items(): + cmd = target_conf['cmd'] + + print(f"measuring {target} ...\u001b[K\r", end='') + stats_file = RESULT_DIR / f'{target}.stats' + + if args.verbose: + print(f'Measure {target} using: {cmd}') + + target_env = target_conf.get('env', {}) + target_env['EMPER_STATS_FILE'] = stats_file + + out_path = RESULT_DIR / f'{target}.out' + err_path = RESULT_DIR / f'{target}.err' + with open(out_path, 'w', encoding='utf-8') as out_file, open( + err_path, 'w', encoding='utf-8') as err_file: + subprocess.run(cmd.split(), + check=True, + stdout=out_file, + stderr=err_file, + env=prepare_env(target_env)) + + # delete empty files + if not os.path.getsize(err_path): + os.remove(err_path) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-v', + '--verbose', + help='show build output', + action='store_true') + parser.add_argument("-i", + "--implementations", + help="implementations to plot", + nargs='+') + parser.add_argument("-ix", + "--exclude-implementations", + help="implementations to exclude", + nargs='+') + parser.add_argument('--desc-stats', + help='descriptive statistics', + type=str) + + _args = parser.parse_args() + + RESULT_DIR = (RESULTS_ROOT / f'{ARTIFACT_DESC}-{platform.uname().node}' / + datetime.datetime.now().strftime("%Y-%m-%dT%H_%M_%S")) + os.makedirs(RESULT_DIR) + print(f'Save results at: {RESULT_DIR}') + + TARGETS = filter_targets(_args.implementations, + _args.exclude_implementations) + main(_args) + + _data = collect(result_dir=RESULT_DIR) + + if _args.desc_stats: + stats = calc_stats(_data) + print(yaml.safe_dump(stats)) + else: + print('### Summary ###') + avgs = calc_avgs(_data) + sys.exit(summarize(avgs)) diff --git a/gen_boxplots.py b/gen_boxplots.py new file mode 100755 index 0000000000000000000000000000000000000000..f077ffabdac4d114d5bede8ca3ab82f48b220f92 --- /dev/null +++ b/gen_boxplots.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""generate boxplots from descriptive stats in yaml""" + +import sys + +import yaml + + +def parse_data(stream): + """Convert yamls string to dict""" + return yaml.safe_load(stream) + +TIKZ_TEMPLATE = \ +"""\\documentclass[]{standalone} +\\usepackage{pgfplots} +\\pgfplotsset{compat=1.17} +\\usepgfplotslibrary{statistics} +\\begin{document} +\\begin{tikzpicture} +\\begin{axis}[ + boxplot/draw direction=y, + legend columns=2, + area legend, + legend style={ + draw=none, + fill=none, + at={(0.5,-0.1)}, + anchor=north + }, +] +$PLOTS$ +\\end{axis} +\\end{tikzpicture} +\\end{document} +""" + +PLOT_TEMPLATE = \ +"""\\addplot+[ + boxplot prepared={{ + lower whisker={lower_whisker}, + lower quartile={lower_quartile}, + median={median}, + average={mean}, + upper quartile={upper_quartile}, + upper whisker={upper_whisker}, + }}, +] coordinates{{{outlier_cords}}}; +\\addlegendentry{{{variant}}} +""" + + +def generate_boxplot(data) -> str: + """generate a tikzboxplot""" + plots = '' + for variant, stats in data.items(): + outlier_cords = '' + for outlier in stats['outliers']: + outlier_cords += f'(0, {outlier}) ' + plots += PLOT_TEMPLATE.format(variant=variant.replace('_', '-'), + outlier_cords=outlier_cords, + **stats) + + return TIKZ_TEMPLATE.replace('$PLOTS$', plots) + + +def main(): + """read yaml and print generated boxplots""" + if len(sys.argv) == 1: + data = parse_data(sys.stdin) + else: + with open(sys.argv[1], 'r', encoding='utf-8') as yaml_file: + data = parse_data(yaml_file) + + tikz = generate_boxplot(data) + + if len(sys.argv) > 2: + with open(sys.argv[2], 'w', encoding='utf-8') as saveto: + print(tikz, file=saveto) + else: + print(tikz) + + +if __name__ == '__main__': + main() diff --git a/summarize.py b/summarize.py new file mode 100755 index 0000000000000000000000000000000000000000..870900d460a1b136cb33ed9adc33c39ccdbdeb16 --- /dev/null +++ b/summarize.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Collect and summarize evaluation results""" + +import argparse +import os +import sys +import typing as T +from pathlib import Path + +import numpy +import yaml + +Measurements = T.Sequence[int] +Data = T.Mapping[str, Measurements] + + +def collect(result_dir) -> Data: + """Collect the data in result_dir and return calculated averages""" + if not result_dir or not os.path.exists(result_dir) or not os.path.isdir( + result_dir): + print(f'{result_dir} is not a directory') + return None + + result_dir = Path(result_dir) + + data = {} + for result_file_path in result_dir.iterdir(): + if result_file_path.suffix != '.out': + continue + + variant = result_file_path.name.split('.')[0] + with open(result_file_path, 'r', encoding='utf-8') as result_file: + times = [int(line) for line in result_file.readlines()] + + data[variant] = times + + return data + + +def calc_avgs(data): + """Calculate only averages from data""" + avgs = {t: numpy.mean(ms) for t, ms in data.items()} + + return avgs + + +DescriptiveStats = T.Mapping[str, float] +Stats = T.Mapping[str, DescriptiveStats] + + +def calc_stats(data: Data) -> Stats: + """Calculate and return descriptive stats of all measurements in data""" + stats = {} + for variant, measurements in data.items(): + variant_stats = {} + stats[variant] = variant_stats + + variant_stats['mean'] = numpy.mean(measurements) + variant_stats['std'] = numpy.std(measurements) + + measurements.sort() + variant_stats['min'] = measurements[0] + variant_stats['max'] = measurements[-1] + variant_stats['median'] = numpy.median(measurements) + upper_quartile = numpy.percentile(measurements, 75) + variant_stats['upper_quartile'] = upper_quartile + lower_quartile = numpy.percentile(measurements, 25) + variant_stats['lower_quartile'] = lower_quartile + iqr = upper_quartile - lower_quartile + + # find whiskers + i = 0 + while measurements[i] < lower_quartile - 1.5 * iqr: + i += 1 + variant_stats['lower_whisker'] = measurements[i] + variant_stats['outliers'] = measurements[:i] + + i = len(measurements) - 1 + while measurements[i] > upper_quartile + 1.5 * iqr: + i -= 1 + variant_stats['upper_whisker'] = measurements[i] + variant_stats['outliers'] += measurements[i + 1:] + + # convert everything to float to easily dump it using pyyaml + for key, value in variant_stats.items(): + if isinstance(value, list): + continue + variant_stats[key] = float(value) + return stats + + +def summarize(avgs=None, stats=None, desc_stats=None) -> int: + """Print a summary for each selected key of the collected stats""" + if not avgs and not stats: + print('no data to summarize') + return 1 + + for variant in avgs or stats: + if avgs: + print(f'\t{variant}: {avgs[variant]}') + else: + for stat in desc_stats or stats[variant].keys(): + print(f'\t{variant}-{stat}: {stats[variant][stat]}') + + return 0 + + +def collect_and_summarize(result_dir=None, desc_stats=None) -> int: + """Collect data and print a summary of the collected data""" + data = collect(result_dir) + stats = calc_stats(data) + + if not stats: + return 1 + + if not summarize(stats=stats, desc_stats=desc_stats): + return 1 + + return 0 + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-s', + '--desc-stats', + help='print all stats not only means', + nargs='*') + parser.add_argument('--yaml', + help='dump statistics as yaml', + action='store_true') + parser.add_argument('result_dir', + help='directory containing the results to summarize') + + _args = parser.parse_args() + + if _args.yaml: + print(yaml.safe_dump(calc_stats(collect(_args.result_dir)))) + sys.exit(0) + + print('### Summary ###') + sys.exit(collect_and_summarize(**vars(_args))) diff --git a/tools/check-pylint b/tools/check-pylint new file mode 100755 index 0000000000000000000000000000000000000000..9b7c7e5c1b5e6d1665d90afec575869030809074 --- /dev/null +++ b/tools/check-pylint @@ -0,0 +1,22 @@ +#!/bin/bash + +PYLINT_EXIT=${1} + +# pylint error masks +FATAL_MASK=1 +ERROR_MASK=2 +WARNING_MASK=4 +REFACTOR_MASK=8 +CONVENTION_MASK=16 +USAGE_ERROR_MASK=32 + +# fail on fatal +[[ $(( $PYLINT_EXIT & $FATAL_MASK )) -gt 0 ]] && exit $PYLINT_EXIT + +# fail on error +[[ $(( $PYLINT_EXIT & $ERROR_MASK )) -gt 0 ]] && exit $PYLINT_EXIT + +# fail on warning +[[ $(( $PYLINT_EXIT & $WARNING_MASK )) -gt 0 ]] && exit $PYLINT_EXIT + +exit 0 diff --git a/tools/docker-prepare b/tools/docker-prepare new file mode 100755 index 0000000000000000000000000000000000000000..c23f19a232e8ba28d6c135b6c92e6ac547951db0 --- /dev/null +++ b/tools/docker-prepare @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +useradd -u "${USER_ID}" -o -m user +groupmod -g "${GROUP_ID}" user + +OUTSIDE_ROOT="${1}" +shift + +cd "${OUTSIDE_ROOT}" + +# shellcheck disable=SC2068 +exec sudo -u user $@