Skip to content
Snippets Groups Projects
Select Git revision
  • 04388969353dcc0f287d71b1db99ebd9eb0585e5
  • master default
  • develop
  • borders
  • tests
  • 16
  • 15
  • gaps
  • v1.6.1
  • v1.6
  • v1.5
  • v1.4
  • v1.3.3
  • v1.3.2
  • v1.3.2rc2
  • v1.3.1rc1
  • v1.3.1.2
  • v1.3.1.1
  • v1.3.1
  • v1.3
  • v1.3rc2
  • v1.3rc1
  • v1.2.1
  • v1.2
  • v1.1.4
  • v1.1.3
  • v1.1.2
  • v1.1.1
28 results

tiling.js

Blame
  • eval.py 7.45 KiB
    #!/usr/bin/env python3
    """Evaluate different emper sleep strategies their latency and power savings"""
    
    import argparse
    import datetime
    import os
    import signal
    import subprocess
    import sys
    from pathlib import Path
    import platform
    import typing as T
    
    from summarize import collect, summarize, calc_avgs
    
    ROOT_DIR = Path(os.path.dirname(os.path.realpath(__file__)))
    EMPER_ROOT = ROOT_DIR / 'emper'
    
    ARTIFACT_DESC = subprocess.check_output(
        'git describe --dirty --always'.split(), cwd=ROOT_DIR, text=True)[:-1]
    
    TARGETS = {
        v: f'{EMPER_ROOT}/build-{v}/eval/pulse'
        for v in ['vanilla', 'greedy-sem', 'no-sleep', 'pipe', 'pipe-no-hint']
    }
    
    RESULTS_ROOT = ROOT_DIR / 'results'
    
    
    def prepare_env(update_env: T.MutableMapping) -> T.Dict:
        """Update and return the a copy of os.environ with a new mapping"""
        current_env = dict(os.environ)
        current_env.update(update_env)
        return current_env
    
    
    PERF_EXE = 'perf'
    PERF_EVENT_SELECTION = '-dd'
    
    
    def perf_get_energy_events() -> str:
        """Return the energy events supported by this system"""
        if not perf_get_energy_events.events:  # type: ignore
            perf_get_energy_events.events = ','.join([  # type: ignore
                l.split()[0] for l in subprocess.check_output(
                    'perf list'.split(), text=True).splitlines() if 'energy' in l
            ])
        return perf_get_energy_events.events  # type: ignore
    
    
    perf_get_energy_events.events = ''  # type: ignore
    
    MEASURE_ENERGY_CMD = f'{PERF_EXE} stat -a -x; -e {{energy_events}} -o {{out}}'
    
    
    def main(args):
        """Run an evaluation"""
        for target, cmd in TARGETS.items():
            out_path = RESULT_DIR / f'{target}.out'
            err_path = RESULT_DIR / f'{target}.err'
            latency_file_path = RESULT_DIR / f'{target}.latencies.csv'
    
            cmd += (
                f' --iterations {args.iterations} --pulse {args.pulse}'
                f' --utilization {args.utilization} --latencies-file {latency_file_path}'
            )
    
            if args.flamegraph:
                perf_out = RESULT_DIR / f'{target}.perf.data'
                cmd = f'{PERF_EXE} record --call-graph dwarf -o {perf_out} {cmd}'
    
            elif args.perf_stats or args.perf_record:
                perf_event_selection = ','.join(
                    args.perf_stats) if args.perf_stats else PERF_EVENT_SELECTION
    
                if args.perf_record:
                    perf_out = RESULT_DIR / f'{target}.perf.data'
                    cmd = f'{PERF_EXE} record -g {perf_event_selection} -o {perf_out} {cmd}'
                else:
                    perf_out = RESULT_DIR / f'{target}.perf.stats'
                    cmd = f'{PERF_EXE} stat {perf_event_selection} -x, -o {perf_out} {cmd}'
    
            print(f"measuring {target} ...\u001b[K\r", end='')
            stats_file = RESULT_DIR / f'{target}.stats'
    
            if args.verbose:
                print(f'Measure {target} using: {cmd}')
    
            target_env = {'EMPER_STATS_FILE': stats_file}
            if args.worker_count:
                target_env['EMPER_WORKER_COUNT'] = str(args.worker_count)
    
            measure_energy_proc = None
            if args.measure_energy:
                measure_energy_out = RESULT_DIR / f'{target}.energy.stats'
                measure_energy_cmd = MEASURE_ENERGY_CMD.format(
                    energy_events=perf_get_energy_events(), out=measure_energy_out)
                measure_energy_proc = subprocess.Popen(measure_energy_cmd.split())
    
            with open(out_path, 'w', encoding='utf-8') as out_file, open(
                    err_path, 'w', encoding='utf-8') as err_file:
                subprocess.run(cmd.split(),
                               check=True,
                               stdout=out_file,
                               stderr=err_file,
                               env=prepare_env(target_env))
    
            if measure_energy_proc:
                measure_energy_proc.send_signal(signal.SIGINT)
                measure_energy_proc.wait()
    
            # delete empty files
            if not os.path.getsize(err_path):
                os.remove(err_path)
    
    
    def generate_flamegraphs(result_dir):
        """generate flamegraphs from recorded perf data files"""
        for path in result_dir.iterdir():
            if path.suffix != '.data':
                continue
    
            print(f'\rGenerating flamgraph from {path.name} ...\u001b[K', end='')
            cmd = f'{ROOT_DIR/"tools"/"generate-flamegraph.sh"} {path}'
            subprocess.run(cmd.split(), check=True)
        print()
    
    
    def write_desc(args):
        """Write a YAML description of the experiment"""
        desc_file_path = RESULT_DIR / 'desc.yml'
    
        with open(desc_file_path, 'w', encoding='utf-8') as desc_file:
            print(f'cmd: {" ".join(sys.argv)}', file=desc_file)
    
            uname = os.uname()
            print(
                (f'uname_client: {uname.sysname} {uname.nodename} {uname.release} '
                 f'{uname.version} {uname.machine}'),
                file=desc_file)
    
            print('args:', file=desc_file)
            print(f'  iterations: {args.iterations}', file=desc_file)
            print(f'  pulse: {args.pulse}', file=desc_file)
            print(f'  utilization: {args.utilization}', file=desc_file)
            print('targets:', file=desc_file)
            for target in TARGETS:
                print(f'  name: {target}', file=desc_file)
    
    
    if __name__ == '__main__':
        parser = argparse.ArgumentParser()
        parser.add_argument('-v',
                            '--verbose',
                            help='show build output',
                            action='store_true')
        parser.add_argument("-w",
                            "--worker-count",
                            help="number of EMPER worker threads",
                            type=int)
        parser.add_argument("-u",
                            "--utilization",
                            help="target runtime utilization",
                            default=80,
                            type=int)
        parser.add_argument("-p",
                            "--pulse",
                            help="the work pulse frequency in us",
                            default=1000000,
                            type=int)
        parser.add_argument("-i",
                            "--iterations",
                            help="the number of pulses",
                            default=30,
                            type=int)
        parser.add_argument('--data-root',
                            help='root path where results should be saved',
                            default=RESULTS_ROOT,
                            type=str)
        parser.add_argument('--perf-stats',
                            help='use perf to collect performance counter stats',
                            nargs='*')
        parser.add_argument('--perf-record',
                            help='use perf to record a profile',
                            action='store_true')
        parser.add_argument('--measure-energy',
                            help='Use perf to measure the used energy',
                            action='store_true')
        parser.add_argument('--flamegraph',
                            help='generate flamegraphs',
                            action='store_true')
    
        _args = parser.parse_args()
    
        RESULT_DIR = (_args.data_root /
                      f'{ARTIFACT_DESC}-{platform.uname().node}' /
                      datetime.datetime.now().strftime("%Y-%m-%dT%H_%M_%S"))
        os.makedirs(RESULT_DIR)
        print(f'Save results at: {RESULT_DIR}')
    
        write_desc(_args)
    
        main(_args)
    
        if _args.flamegraph:
            print()
            generate_flamegraphs(RESULT_DIR)
            sys.exit(0)
    
        _data = collect(result_dir=RESULT_DIR)
        if _data is None:
            print(f'Error: no data was collected from {RESULT_DIR}',
                  file=sys.stderr)
            sys.exit(1)
    
        print('\n### Summary ###')
        avgs = calc_avgs(_data)
        sys.exit(summarize(avgs=avgs))