Commit 8174a918 authored by Florian Fischer's avatar Florian Fischer
Browse files

make the whole project more python idiomatic

* rename src directory to allocbench
* make global variable names UPPERCASE
* format a lot of code using yapf
* use lowercase ld_preload and ld_library_path as Allocator members
* name expected Errors 'err' and don't raise a new Exception
* disable some pylint messages
parent 8f52e8fc
PYTHONFILES = $(shell find src/ -name "*.py")
SRCDIR = allocbench
MAKEFILES = $(shell dirname $(shell find src/ -name Makefile))
CMAKELISTS = $(shell dirname $(shell find src/ -name CMakeLists.txt))
PYTHONFILES = $(shell find $(SRCDIR)/ -name "*.py")
MAKEFILES = $(shell dirname $(shell find $(SRCDIR)/ -name Makefile))
CMAKELISTS = $(shell dirname $(shell find $(SRCDIR)/ -name CMakeLists.txt))
OBJDIR = $(PWD)/build
......@@ -22,7 +24,7 @@ export LDXXFLAGS = $(LDFLAGS) -static-libstdc++
all: $(OBJDIR)/ccinfo $(MAKEFILES) $(CMAKELISTS)
$(CMAKELISTS):
$(eval BENCHDIR=$(OBJDIR)$(shell echo $@ | sed s/src//))
$(eval BENCHDIR=$(OBJDIR)$(shell echo $@ | sed s/$(SRCDIR)//))
@if test \( ! \( -d $(BENCHDIR) \) \) ;then mkdir -p $(BENCHDIR);fi
ifneq (,$(findstring s,$(MAKEFLAGS)))
cd $(BENCHDIR); cmake $(PWD)/$@ >/dev/null
......@@ -32,9 +34,9 @@ endif
$(MAKE) -C $(BENCHDIR)
$(MAKEFILES):
$(eval BENCHDIR=$(OBJDIR)$(shell echo $@ | sed s/src//))
$(eval BENCHDIR=$(OBJDIR)$(shell echo $@ | sed s/$(SRCDIR)//))
@if test \( ! \( -d $(BENCHDIR) \) \) ;then mkdir -p $(BENCHDIR);fi
$(MAKE) -C $@ OBJDIR=$(OBJDIR)$(shell echo $@ | sed s/src//)
$(MAKE) -C $@ OBJDIR=$(OBJDIR)$(shell echo $@ | sed s/$(SRCDIR)//)
$(OBJDIR)/ccinfo: | $(OBJDIR)
$(CC) -v 2> $@
......
......@@ -99,7 +99,7 @@ It groups the included allocators into categories to produce readable and not ex
runs only the loop benchmark for all included allocators and will put its
results in `$PWD/results/$HOSTNAME/<time>/loop`.
./bench.py -a BA_allocators
./bench.py -a ba_allocators
builds all allocators used in Florian Fischer's [BA thesis](https://muhq.space/ba.html)
and runs all benchmarks.
......
......@@ -26,9 +26,9 @@ import shutil
from subprocess import CalledProcessError
import sys
from src.artifact import ArchiveArtifact, GitArtifact
import src.globalvars
from src.util import print_status, print_debug, print_error, print_info2, run_cmd
from allocbench.artifact import ArchiveArtifact, GitArtifact
from allocbench.globalvars import ALLOCBUILDDIR, ALLOCSRCDIR
from allocbench.util import print_status, print_debug, print_error, print_info2, run_cmd
LIBRARY_PATH = ""
for line in run_cmd(["ldconfig", "-v", "-N"],
......@@ -37,9 +37,8 @@ for line in run_cmd(["ldconfig", "-v", "-N"],
if not line.startswith('\t'):
LIBRARY_PATH += line
BUILDDIR = Path(src.globalvars.allocbuilddir)
BUILDDIR = Path(ALLOCBUILDDIR)
SRCDIR = BUILDDIR / "src"
ALLOCDEFDIR = Path('src/allocators')
SRCDIR.mkdir(parents=True, exist_ok=True)
......@@ -52,14 +51,14 @@ class Allocator:
Allocator.build will compile the allocator and produce a for allocbench usable
allocator dict"""
allowed_attributes = [
"binary_suffix", "cmd_prefix", "LD_PRELOAD", "LD_LIBRARY_PATH",
"binary_suffix", "cmd_prefix", "ld_preload", "ld_library_path",
"color", "sources", "version", "patches", "prepare_cmds", "build_cmds"
]
binary_suffix = None
cmd_prefix = None
LD_PRELOAD = None
LD_LIBRARY_PATH = None
ld_preload = None
ld_library_path = None
color = None
sources = None
version = None
......@@ -107,20 +106,20 @@ class Allocator:
if not_patched:
try:
run_cmd(["patch", "-p0", "--verbose"], cwd=cwd, input=patch_content)
except CalledProcessError as e:
print_debug(e.stderr, file=sys.stderr)
except CalledProcessError as err:
print_debug(err.stderr, file=sys.stderr)
print_error(f"Patching of {self.name} failed.")
raise e
raise
if self.prepare_cmds:
print_status(f"Run prepare commands {self.name} ...")
for cmd in self.prepare_cmds:
try:
run_cmd(cmd, shell=True, cwd=self.srcdir)
except CalledProcessError as e:
print_debug(e.stderr, file=sys.stderr)
except CalledProcessError as err:
print_debug(err.stderr, file=sys.stderr)
print_error(f"Prepare {self.name} failed")
raise e
raise
def build(self):
"""Build the allocator if needed and produce an allocator dict"""
......@@ -153,27 +152,27 @@ class Allocator:
try:
run_cmd(cmd, cwd=BUILDDIR, shell=True)
except CalledProcessError as e:
print_debug(e.stderr, file=sys.stderr)
except CalledProcessError as err:
print_debug(err.stderr, file=sys.stderr)
print_error(f"Builing {self.name} failed")
shutil.rmtree(self.dir, ignore_errors=True)
raise e
raise
with open(buildtimestamp_path, "w") as buildtimestamp_file:
print_info2("Save build time to:", buildtimestamp_path)
buildtimestamp_file.write(str(datetime.now().timestamp()))
with open(buildtimestamp_path, "w") as buildtimestamp_file:
print_info2("Save build time to:", buildtimestamp_path)
buildtimestamp_file.write(str(datetime.now().timestamp()))
print_info2("Create allocator dictionary")
res_dict = {
"cmd_prefix": self.cmd_prefix or "",
"binary_suffix": self.binary_suffix or "",
"LD_PRELOAD": self.LD_PRELOAD or "",
"LD_LIBRARY_PATH": self.LD_LIBRARY_PATH or "",
"LD_PRELOAD": self.ld_preload or "",
"LD_LIBRARY_PATH": self.ld_library_path or "",
"color": self.color
}
for attr in ["LD_PRELOAD", "LD_LIBRARY_PATH", "cmd_prefix"]:
value = getattr(self, attr, "") or ""
value = getattr(self, attr.lower(), "") or ""
if value != "":
value = value.format(dir=self.dir, srcdir=self.srcdir)
res_dict[attr] = value
......@@ -221,11 +220,11 @@ def collect_available_allocators():
available_allocators = {}
for alloc_def_path in Path(ALLOCDEFDIR).glob('*.py'):
alloc_module_name = '.'.join(alloc_def_path.parts[:-1] + (alloc_def_path.stem,))
for alloc_def_path in Path(ALLOCSRCDIR).glob('*.py'):
alloc_module_name = f'allocbench.allocators.{alloc_def_path.stem}'
module = importlib.import_module(alloc_module_name)
for name, obj in module.__dict__.items():
if issubclass(obj.__class__, src.allocator.Allocator):
if issubclass(obj.__class__, Allocator):
available_allocators[name] = obj
return available_allocators
......@@ -253,7 +252,7 @@ def collect_allocators(allocators):
Otherwise allocators is interpreted as a list of names or files. If an entry in
allocators is a file it is handled as a allocator collection file exporting
a allocators variable. If the entry is no file it is interpreted as an allocator
name and is searched for in our allocator definitions located at src/allocators.
name and is searched for in our allocator definitions located at allocbench/allocators.
"""
# Default allocators definition file
......@@ -262,8 +261,12 @@ def collect_allocators(allocators):
if allocators is None and os.path.isfile(default_allocators_file):
return read_allocators_collection_file(default_allocators_file)
available_allocators = collect_available_allocators()
ret = {}
for name in allocators:
if name == "all":
return available_allocators
if name == "installed":
print_status("Using system-wide installed allocators ...")
ret.update(collect_installed_allocators())
......@@ -274,7 +277,6 @@ def collect_allocators(allocators):
# interpret name as allocator name or wildcard
else:
available_allocators = collect_available_allocators()
matched_allocators = fnmatch.filter(available_allocators.keys(), name)
if matched_allocators:
ret.update({a: available_allocators[a].build() for a in matched_allocators})
......
......@@ -16,9 +16,10 @@
# along with allocbench. If not, see <http://www.gnu.org/licenses/>.
"""Collection containing all allocators used in Florian's BA thesis"""
from src.allocators.glibc import glibc, glibc_notc
from src.allocators.tcmalloc import tcmalloc
from src.allocators.jemalloc import jemalloc
from src.allocators.hoard import hoard
from allocbench.allocators.glibc import glibc, glibc_notc
from allocbench.allocators.tcmalloc import tcmalloc
from allocbench.allocators.jemalloc import jemalloc
from allocbench.allocators.hoard import hoard
# pylint: disable=invalid-name
allocators = [glibc, glibc_notc, tcmalloc, jemalloc, hoard]
# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
# Copyright 2018-2020 Florian Fischer <florian.fl.fischer@fau.de>
#
# This file is part of allocbench.
#
......@@ -18,12 +18,13 @@
The bumpptr allocator makes the biggest possible tradeoff between speed and
memory in speeds favor. Memory is mmapped per thread and never freed.
See src/bumpptr.c for the actual implementation.
See allocbench/bumpptr.c for the actual implementation.
"""
import os
from src.allocator import Allocator, BUILDDIR
from allocbench.allocator import Allocator, BUILDDIR
# pylint: disable=invalid-name
bumpptr = Allocator("bumpptr",
LD_PRELOAD=os.path.join(BUILDDIR, "bumpptr_alloc.so"),
ld_preload=os.path.join(BUILDDIR, "bumpptr_alloc.so"),
color="xkcd:black")
......@@ -21,13 +21,12 @@ the allocator usage and the executed program. It overrides the malloc API
and saves each call and its result to a memory mapped output file.
"""
import os
from src.artifact import GitArtifact
from src.allocator import Allocator
from allocbench.artifact import GitArtifact
from allocbench.allocator import Allocator
VERSION = "1a09b144eb18919014ecf86da3442344b0eaa5b2"
class Chattymalloc(Allocator):
"""Chattymalloc definition for allocbench"""
......@@ -44,9 +43,10 @@ class Chattymalloc(Allocator):
f"meson {{srcdir}} {{dir}} {configuration}", "ninja -C {dir}"
]
self.LD_PRELOAD = "{dir}/libchattymalloc.so"
self.cmd_prefix="env CHATTYMALLOC_FILE={{result_dir}}/{{perm}}.trace"
self.ld_preload = "{dir}/libchattymalloc.so"
self.cmd_prefix = "env CHATTYMALLOC_FILE={{result_dir}}/{{perm}}.trace"
super().__init__(name, **kwargs)
# pylint: disable=invalid-name
chattymalloc = Chattymalloc("chattymalloc", version=VERSION)
# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
# Copyright 2018-2020 Florian Fischer <florian.fl.fischer@fau.de>
#
# This file is part of allocbench.
#
......@@ -18,8 +18,8 @@
from multiprocessing import cpu_count
from src.allocator import Allocator, LIBRARY_PATH
from src.artifact import GitArtifact
from allocbench.allocator import Allocator, LIBRARY_PATH
from allocbench.artifact import GitArtifact
class Glibc(Allocator):
......@@ -47,6 +47,7 @@ class Glibc(Allocator):
super().__init__(name, **kwargs)
# pylint: disable=invalid-name
glibc = Glibc("glibc", version="glibc-2.29", color="xkcd:red")
glibc_notc = Glibc("glibc-noThreadCache",
......
# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
# Copyright 2018-2020 Florian Fischer <florian.fl.fischer@fau.de>
#
# This file is part of allocbench.
#
......@@ -16,8 +16,8 @@
# along with allocbench.
"""Hoard allocator definition for allocbench"""
from src.allocator import Allocator
from src.artifact import GitArtifact
from allocbench.allocator import Allocator
from allocbench.artifact import GitArtifact
class Hoard(Allocator):
......@@ -26,7 +26,7 @@ class Hoard(Allocator):
sources = GitArtifact("Hoard", "https://github.com/emeryberger/Hoard.git")
def __init__(self, name, **kwargs):
self.LD_PRELOAD = "{dir}/libhoard.so"
self.ld_preload = "{dir}/libhoard.so"
self.build_cmds = [
"cd {srcdir}/src; make", "mkdir -p {dir}",
"ln -f -s {srcdir}/src/libhoard.so {dir}/libhoard.so"
......@@ -36,6 +36,7 @@ class Hoard(Allocator):
super().__init__(name, **kwargs)
# pylint: disable=invalid-name
hoard = Hoard("Hoard",
version="aa6d31700d5368a9f5ede3d62731247c8d9f0ebb",
color="xkcd:brown")
# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
# Copyright 2018-2020 Florian Fischer <florian.fl.fischer@fau.de>
#
# This file is part of allocbench.
#
......@@ -16,8 +16,8 @@
# along with allocbench.
"""jemalloc definition for allocbench"""
from src.allocator import Allocator
from src.artifact import GitArtifact
from allocbench.allocator import Allocator
from allocbench.artifact import GitArtifact
class Jemalloc(Allocator):
......@@ -27,7 +27,7 @@ class Jemalloc(Allocator):
"https://github.com/jemalloc/jemalloc.git")
def __init__(self, name, **kwargs):
self.LD_PRELOAD = "{dir}/libjemalloc.so"
self.ld_preload = "{dir}/libjemalloc.so"
self.prepare_cmds = ["./autogen.sh"]
self.build_cmds = [
"cd {srcdir}; ./configure --prefix={dir}", "cd {srcdir}; make -j4",
......@@ -38,4 +38,5 @@ class Jemalloc(Allocator):
super().__init__(name, **kwargs)
# pylint: disable=invalid-name
jemalloc = Jemalloc("jemalloc", version="5.1.0", color="xkcd:yellow")
# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
# Copyright 2018-2020 Florian Fischer <florian.fl.fischer@fau.de>
#
# This file is part of allocbench.
#
......@@ -16,8 +16,8 @@
# along with allocbench.
"""Lockless allocator definition for allocbench"""
from src.allocator import Allocator
from src.artifact import ArchiveArtifact
from allocbench.allocator import Allocator
from allocbench.artifact import ArchiveArtifact
class LocklessAllocator(Allocator):
......@@ -34,9 +34,10 @@ class LocklessAllocator(Allocator):
"ln -f -s {srcdir}/lockless_allocator/libllalloc.so.1.3 {dir}/libllalloc.so"
]
self.LD_PRELOAD = "{dir}/libllalloc.so"
self.ld_preload = "{dir}/libllalloc.so"
super().__init__(name, **kwargs)
# pylint: disable=invalid-name
llalloc = LocklessAllocator("llalloc", color="purple")
# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
# Copyright 2018-2020 Florian Fischer <florian.fl.fischer@fau.de>
#
# This file is part of allocbench.
#
......@@ -20,9 +20,10 @@ Malt is a malloc tracker and used to capture and analyse the allocation profile
of a program. See https://github.com/memtt/malt for more details
"""
from src.allocator import Allocator
from allocbench.allocator import Allocator
# result_dir and perm are substituted during Benchmark.run
# pylint: disable=invalid-name
malt = Allocator(
"malt",
cmd_prefix="malt -q -o output:name={{result_dir}}/malt.{{perm}}.%3")
# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
# Copyright 2018-2020 Florian Fischer <florian.fl.fischer@fau.de>
#
# This file is part of allocbench.
#
......@@ -16,8 +16,8 @@
# along with allocbench.
"""Mesh definition for allocbench"""
from src.allocator import Allocator
from src.artifact import GitArtifact
from allocbench.allocator import Allocator
from allocbench.artifact import GitArtifact
class Mesh(Allocator):
......@@ -26,7 +26,7 @@ class Mesh(Allocator):
sources = GitArtifact("Mesh", "https://github.com/plasma-umass/Mesh")
def __init__(self, name, **kwargs):
self.LD_PRELOAD = "{dir}/libmesh.so"
self.ld_preload = "{dir}/libmesh.so"
self.build_cmds = [
"cd {srcdir}; ./configure", "cd {srcdir}; make -j 4",
"mkdir -p {dir}", "ln -f -s {srcdir}/libmesh.so {dir}/libmesh.so"
......@@ -35,6 +35,7 @@ class Mesh(Allocator):
super().__init__(name, **kwargs)
# pylint: disable=invalid-name
mesh = Mesh("Mesh",
version="4a1012cee990cb98cc1ea0294a836f467b29be02",
color="xkcd:mint")
# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
# Copyright 2018-2020 Florian Fischer <florian.fl.fischer@fau.de>
#
# This file is part of allocbench.
#
......@@ -16,8 +16,8 @@
# along with allocbench.
"""mimalloc definition for allocbench"""
from src.allocator import Allocator
from src.artifact import GitArtifact
from allocbench.allocator import Allocator
from allocbench.artifact import GitArtifact
class Mimalloc(Allocator):
......@@ -26,7 +26,7 @@ class Mimalloc(Allocator):
sources = GitArtifact("mimalloc", "https://github.com/microsoft/mimalloc")
def __init__(self, name, **kwargs):
self.LD_PRELOAD = "{dir}/libmimalloc.so"
self.ld_preload = "{dir}/libmimalloc.so"
self.build_cmds = [
"mkdir -p {dir}", "cd {dir}; cmake {srcdir}", "cd {dir}; make"
]
......@@ -34,5 +34,5 @@ class Mimalloc(Allocator):
super().__init__(name, **kwargs)
# pylint: disable=invalid-name
mimalloc = Mimalloc("mimalloc", version="v1.6.0")
# Copyright 2018-2019 Florian Fischer <florian.fl.fischer@fau.de>
# Copyright 2018-2020 Florian Fischer <florian.fl.fischer@fau.de>
#
# This file is part of allocbench.
#
......@@ -16,9 +16,10 @@
# along with allocbench.
"""Collection containing all no falsesahring patches"""
from src.allocators.tcmalloc import tcmalloc_gperftools, tcmalloc_gperftools_nofs
from src.allocators.glibc import glibc, glibc_nofs, glibc_nofs_fancy
from allocbench.allocators.tcmalloc import tcmalloc_gperftools, tcmalloc_gperftools_nofs
from allocbench.allocators.glibc import glibc, glibc_nofs, glibc_nofs_fancy
# pylint: disable=invalid-name
allocators = [
glibc, glibc_nofs, glibc_nofs_fancy, tcmalloc_gperftools,
tcmalloc_gperftools_nofs
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment