Skip to content
Snippets Groups Projects
Select Git revision
  • 00f74bd822dda4b39ed439f53e0767283f85c5e5
  • master default
  • glibc-cacheline-exclusive
  • cmd_wrapper
  • bumpptrs
  • wait4_rusage
  • cache_exclusive_alloc
  • loop_keep_allocs
8 results

bench.py

Blame
  • bench.py 8.08 KiB
    #!/usr/bin/env python3
    
    # Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
    #
    # This file is part of allocbench.
    #
    # allocbench is free software: you can redistribute it and/or modify
    # it under the terms of the GNU General Public License as published by
    # the Free Software Foundation, either version 3 of the License, or
    # (at your option) any later version.
    #
    # allocbench is distributed in the hope that it will be useful,
    # but WITHOUT ANY WARRANTY; without even the implied warranty of
    # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    # GNU General Public License for more details.
    #
    # You should have received a copy of the GNU General Public License
    # along with allocbench.  If not, see <http://www.gnu.org/licenses/>.
    
    """Start an allocbench run"""
    
    import argparse
    import atexit
    import datetime
    import importlib
    import os
    import subprocess
    import sys
    import traceback
    
    from src.allocator import collect_allocators
    import src.facter
    import src.globalvars
    from src.util import find_cmd
    from src.util import print_status, print_warn, print_error
    from src.util import print_info, print_info2, print_debug
    from src.util import print_license_and_exit, print_version_and_exit
    
    
    def epilog():
        """Run tasks on exit"""
        # After early errors resdir may not be set
        if src.globalvars.resdir is not None:
            if os.listdir(src.globalvars.resdir) == []:
                print_warn("Remove empty resultdir")
                os.removedirs(src.globalvars.resdir)
            else:
                endtime = datetime.datetime.now().isoformat()
                endtime = endtime[:endtime.rfind(':')]
                src.globalvars.facts["endtime"] = endtime
                src.facter.store_facts(src.globalvars.resdir)
    
        # remove a left over status file if some is present
        if os.path.exists("status"):
            os.remove("status")
    
    
    def check_dependencies():
        """Check if known requirements of allocbench are met"""
        # used python 3.6 features: f-strings
        if sys.version_info[0] < 3 or sys.version_info[1] < 6:
            print_error("At least python version 3.6 is required.")
            exit(1)
    
        # matplotlib is needed by Benchmark.plot_*
        try:
            importlib.import_module("matplotlib")
        except ModuleNotFoundError:
            print_error("matplotlib not found.")
            exit(1)
        # TODO mariadb
    
    
    def main():
        check_dependencies()
    
        parser = argparse.ArgumentParser(description="benchmark memory allocators")
        parser.add_argument("--analyze", help="analyze benchmark behavior using malt", action="store_true")
        parser.add_argument("-r", "--runs", help="how often the benchmarks run", default=3, type=int)
        parser.add_argument("-v", "--verbose", help="more output", action='count')
        parser.add_argument("-b", "--benchmarks", help="benchmarks to run", nargs='+')
        parser.add_argument("-xb", "--exclude-benchmarks", help="explicitly excluded benchmarks", nargs='+')
        parser.add_argument("-a", "--allocators", help="allocators to test", type=str, nargs='+',
                            default=["all"])
        parser.add_argument("-rd", "--resultdir", help="directory where all results go", type=str)
        parser.add_argument("--license", help="print license info and exit", action='store_true')
        parser.add_argument("--version", help="print version info and exit", action='store_true')
    
        args = parser.parse_args()
        if args.license:
            print_license_and_exit()
    
        if args.version:
            print_version_and_exit()
    
        atexit.register(epilog)
    
        # Set global verbosity
        # quiet | -1: Don't output to stdout
        # default | 0: Only print status and errors
        # 1: Print warnings and some infos
        # 2: Print all infos
        # 3: Print all awailable infos
        if args.verbose:
            src.globalvars.verbosity = args.verbose
    
        print_info2("Arguments:", args)
    
        # Prepare allocbench
        print_status("Building allocbench ...")
        make_cmd = ["make"]
        if src.globalvars.verbosity < 1:
            make_cmd.append("-s")
        else:
            # Flush stdout so the color reset from print_status works
            print("", end="", flush=True)
        subprocess.run(make_cmd)
    
        # allocators to benchmark
        src.globalvars.allocators = collect_allocators(args.allocators)
    
        print_info("Allocators:", *src.globalvars.allocators.keys())
        print_debug("Allocators:", *src.globalvars.allocators.items())
    
        if src.globalvars.allocators == {}:
            print_error("Abort because there are no allocators to benchmark")
            exit(1)
    
        # collect facts about benchmark environment
        src.facter.collect_facts()
    
        # Create result directory
        if args.resultdir:
            src.globalvars.resdir = os.path.join(args.resultdir)
        else:
            src.globalvars.resdir = os.path.join("results",
                                                 src.globalvars.facts["hostname"],
                                                 src.globalvars.facts["starttime"])
    
        print_status("Writing results to:", src.globalvars.resdir)
        os.makedirs(src.globalvars.resdir, exist_ok=True)
    
        cwd = os.getcwd()
    
        # Run actual benchmarks
        for bench in src.globalvars.benchmarks:
            if args.benchmarks and bench not in args.benchmarks:
                continue
    
            if args.exclude_benchmarks and bench in args.exclude_benchmarks:
                continue
    
            bench_module = importlib.import_module(f"src.benchmarks.{bench}")
    
            if not hasattr(bench_module, bench):
                print_error(f"{bench_module} has no member {bench}.")
                print_error(f"Skipping {bench_module}")
                continue
    
            bench = getattr(bench_module, bench)
    
            print_status("Preparing", bench.name, "...")
            try:
                bench.prepare()
            except Exception:
                print_error(traceback.format_exc())
                print_error(f"Skipping {bench}! Preparing failed.")
                continue
    
    
            if args.analyze:
                print_status("Analysing {} ...".format(bench))
    
                # Create benchmark result directory
                if not os.path.isdir(bench.result_dir):
                    print_info2("Creating benchmark result dir:", bench.result_dir)
                    os.makedirs(bench.result_dir, exist_ok=True)
    
                if find_cmd("malt") is not None:
                    analyze_alloc = "malt"
                else:
                    print_warn("malt not found. Using chattymalloc.")
                    analyze_alloc = "chattymalloc"
    
                old_allocs = bench.allocators
                old_measure_cmd = bench.measure_cmd
                bench.measure_cmd = ""
                analyze_alloc_module = importlib.import_module(f"src.allocators.{analyze_alloc}")
                bench.allocators = {analyze_alloc: getattr(analyze_alloc_module, analyze_alloc).build()}
    
                try:
                    bench.run(runs=1)
                except Exception:
                    print_error(traceback.format_exc())
                    print_error("Skipping analysis of", bench, "!")
    
                bench.measure_cmd = old_measure_cmd
    
                # Remove results for analyze_alloc
                if analyze_alloc in bench.results:
                    del bench.results[analyze_alloc]
                if "stats" in bench.results and analyze_alloc in bench.results["stats"]:
                    del bench.results["stats"][analyze_alloc]
    
                # restore allocs
                bench.allocators = old_allocs
    
            if args.runs > 0:
                print_status("Running", bench.name, "...")
                start_time = datetime.datetime.now()
                bench.results['facts']['start-time'] = start_time.isoformat()
                try:
                    bench.run(runs=args.runs)
                except Exception:
                    # Reset cwd
                    os.chdir(cwd)
    
                    print_error(traceback.format_exc())
                    print_error("Skipping", bench, "!")
    
                    continue
                end_time = datetime.datetime.now()
                bench.results['facts']['end-time'] = end_time.isoformat()
                bench.results['facts']['duration'] = (end_time - start_time).total_seconds()
    
            # Save results in resultdir
            bench.save(src.globalvars.resdir)
    
            if hasattr(bench, "cleanup"):
                print_status("Cleaning up", bench.name, "...")
                bench.cleanup()
    
    
    if __name__ == "__main__":
        main()