Select Git revision
eval.py 7.45 KiB
#!/usr/bin/env python3
"""Evaluate different emper sleep strategies their latency and power savings"""
import argparse
import datetime
import os
import signal
import subprocess
import sys
from pathlib import Path
import platform
import typing as T
from summarize import collect, summarize, calc_avgs
ROOT_DIR = Path(os.path.dirname(os.path.realpath(__file__)))
EMPER_ROOT = ROOT_DIR / 'emper'
ARTIFACT_DESC = subprocess.check_output(
'git describe --dirty --always'.split(), cwd=ROOT_DIR, text=True)[:-1]
TARGETS = {
v: f'{EMPER_ROOT}/build-{v}/eval/pulse'
for v in ['vanilla', 'greedy-sem', 'no-sleep', 'pipe', 'pipe-no-hint']
}
RESULTS_ROOT = ROOT_DIR / 'results'
def prepare_env(update_env: T.MutableMapping) -> T.Dict:
"""Update and return the a copy of os.environ with a new mapping"""
current_env = dict(os.environ)
current_env.update(update_env)
return current_env
PERF_EXE = 'perf'
PERF_EVENT_SELECTION = '-dd'
def perf_get_energy_events() -> str:
"""Return the energy events supported by this system"""
if not perf_get_energy_events.events: # type: ignore
perf_get_energy_events.events = ','.join([ # type: ignore
l.split()[0] for l in subprocess.check_output(
'perf list'.split(), text=True).splitlines() if 'energy' in l
])
return perf_get_energy_events.events # type: ignore
perf_get_energy_events.events = '' # type: ignore
MEASURE_ENERGY_CMD = f'{PERF_EXE} stat -a -x; -e {{energy_events}} -o {{out}}'
def main(args):
"""Run an evaluation"""
for target, cmd in TARGETS.items():
out_path = RESULT_DIR / f'{target}.out'
err_path = RESULT_DIR / f'{target}.err'
latency_file_path = RESULT_DIR / f'{target}.latencies.csv'
cmd += (
f' --iterations {args.iterations} --pulse {args.pulse}'
f' --utilization {args.utilization} --latencies-file {latency_file_path}'
)
if args.flamegraph:
perf_out = RESULT_DIR / f'{target}.perf.data'
cmd = f'{PERF_EXE} record --call-graph dwarf -o {perf_out} {cmd}'
elif args.perf_stats or args.perf_record:
perf_event_selection = ','.join(
args.perf_stats) if args.perf_stats else PERF_EVENT_SELECTION
if args.perf_record:
perf_out = RESULT_DIR / f'{target}.perf.data'
cmd = f'{PERF_EXE} record -g {perf_event_selection} -o {perf_out} {cmd}'
else:
perf_out = RESULT_DIR / f'{target}.perf.stats'
cmd = f'{PERF_EXE} stat {perf_event_selection} -x, -o {perf_out} {cmd}'
print(f"measuring {target} ...\u001b[K\r", end='')
stats_file = RESULT_DIR / f'{target}.stats'
if args.verbose:
print(f'Measure {target} using: {cmd}')
target_env = {'EMPER_STATS_FILE': stats_file}
if args.worker_count:
target_env['EMPER_WORKER_COUNT'] = str(args.worker_count)
measure_energy_proc = None
if args.measure_energy:
measure_energy_out = RESULT_DIR / f'{target}.energy.stats'
measure_energy_cmd = MEASURE_ENERGY_CMD.format(
energy_events=perf_get_energy_events(), out=measure_energy_out)
measure_energy_proc = subprocess.Popen(measure_energy_cmd.split())
with open(out_path, 'w', encoding='utf-8') as out_file, open(
err_path, 'w', encoding='utf-8') as err_file:
subprocess.run(cmd.split(),
check=True,
stdout=out_file,
stderr=err_file,
env=prepare_env(target_env))
if measure_energy_proc:
measure_energy_proc.send_signal(signal.SIGINT)
measure_energy_proc.wait()
# delete empty files
if not os.path.getsize(err_path):
os.remove(err_path)
def generate_flamegraphs(result_dir):
"""generate flamegraphs from recorded perf data files"""
for path in result_dir.iterdir():
if path.suffix != '.data':
continue
print(f'\rGenerating flamgraph from {path.name} ...\u001b[K', end='')
cmd = f'{ROOT_DIR/"tools"/"generate-flamegraph.sh"} {path}'
subprocess.run(cmd.split(), check=True)
print()
def write_desc(args):
"""Write a YAML description of the experiment"""
desc_file_path = RESULT_DIR / 'desc.yml'
with open(desc_file_path, 'w', encoding='utf-8') as desc_file:
print(f'cmd: {" ".join(sys.argv)}', file=desc_file)
uname = os.uname()
print(
(f'uname_client: {uname.sysname} {uname.nodename} {uname.release} '
f'{uname.version} {uname.machine}'),
file=desc_file)
print('args:', file=desc_file)
print(f' iterations: {args.iterations}', file=desc_file)
print(f' pulse: {args.pulse}', file=desc_file)
print(f' utilization: {args.utilization}', file=desc_file)
print('targets:', file=desc_file)
for target in TARGETS:
print(f' name: {target}', file=desc_file)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-v',
'--verbose',
help='show build output',
action='store_true')
parser.add_argument("-w",
"--worker-count",
help="number of EMPER worker threads",
type=int)
parser.add_argument("-u",
"--utilization",
help="target runtime utilization",
default=80,
type=int)
parser.add_argument("-p",
"--pulse",
help="the work pulse frequency in us",
default=1000000,
type=int)
parser.add_argument("-i",
"--iterations",
help="the number of pulses",
default=30,
type=int)
parser.add_argument('--data-root',
help='root path where results should be saved',
default=RESULTS_ROOT,
type=str)
parser.add_argument('--perf-stats',
help='use perf to collect performance counter stats',
nargs='*')
parser.add_argument('--perf-record',
help='use perf to record a profile',
action='store_true')
parser.add_argument('--measure-energy',
help='Use perf to measure the used energy',
action='store_true')
parser.add_argument('--flamegraph',
help='generate flamegraphs',
action='store_true')
_args = parser.parse_args()
RESULT_DIR = (_args.data_root /
f'{ARTIFACT_DESC}-{platform.uname().node}' /
datetime.datetime.now().strftime("%Y-%m-%dT%H_%M_%S"))
os.makedirs(RESULT_DIR)
print(f'Save results at: {RESULT_DIR}')
write_desc(_args)
main(_args)
if _args.flamegraph:
print()
generate_flamegraphs(RESULT_DIR)
sys.exit(0)
_data = collect(result_dir=RESULT_DIR)
if _data is None:
print(f'Error: no data was collected from {RESULT_DIR}',
file=sys.stderr)
sys.exit(1)
print('\n### Summary ###')
avgs = calc_avgs(_data)
sys.exit(summarize(avgs=avgs))