Skip to content
GitLab
Commit 81828c8b authored by Christian Dietrich's avatar Christian Dietrich
Browse files

incremental_rebuild: use whole-build wrappers 

Since we want to measure an end to end time, we have to use the
bare-mininum wrappers that only call clang with {nothing, ccache, clang-hash}
parent ea7e4206
Pipeline #2983 passed with stage
in 0 seconds
Hide whitespace changes
Inline Side-by-side
experiments/Makefile
View file @ 81828c8b
JOBS := $(shell nproc)
CLANG_HASH=${HOME}/w/clang-hash
MUSL = ${CLANG_HASH}/hash-projects/musl
POSTGRES = ${CLANG_HASH}/hash-projects/postgresql
POSTGRESQL = ${CLANG_HASH}/hash-projects/postgresql
all:
@echo "Look at the makefile"
incremental_rebuild_postgres_normal:
./incremental_rebuild.py -s -d ../versuchung-data --mode normal \
--jobs $(JOBS) -v \
--project-clone-url ${POSTGRES} \
--clang_hash-clone-url ${CLANG_HASH}
incremental_rebuild_postgres_hash:
./incremental_rebuild.py -s -d ../versuchung-data --mode clang-hash \
--jobs $(JOBS) -v \
--project-clone-url ${POSTGRES} \
--clang_hash-clone-url ${CLANG_HASH}
incremental_rebuild_musl_normal:
./incremental_rebuild.py -s -d ../versuchung-data --mode normal \
--jobs $(JOBS) -v \
--project-clone-url ${MUSL} \
--clang_hash-clone-url ${CLANG_HASH}
incremental_rebuild_musl_hash:
./incremental_rebuild.py -s -d ../versuchung-data --mode clang-hash \
--jobs $(JOBS) -v \
--project-clone-url ${MUSL} \
--clang_hash-clone-url ${CLANG_HASH}
INC = ./incremental_rebuild.py -s -d ../versuchung-data --jobs $(JOBS) \
-v --clang_hash-clone-url ${CLANG_HASH}
inc_postgres_normal:
${INC} --project-clone-url ${POSTGRESQL} --mode normal
inc_postgres_ccache:
${INC} --project-clone-url ${POSTGRESQL} --mode ccache
inc_postgres_hash:
${INC} --project-clone-url ${POSTGRESQL} --mode clang-hash
inc_musl_normal:
${INC} --project-clone-url ${MUSL} --mode normal
inc_musl_ccache:
${INC} --project-clone-url ${MUSL} --mode ccache
inc_musl_hash:
${INC} --project-clone-url ${MUSL} --mode clang-hash
experiments/incremental_rebuild.py
View file @ 81828c8b
......@@ -21,7 +21,7 @@ class IncrementalCompilation(Experiment):
"clang_hash": GitArchive("/home/stettberger/w/clang-hash/"),
"project": GitArchive("/home/stettberger/w/clang-hash/hash-projects/musl",
shallow=True),
"touch-only": Bool(True),
"touch-only": Bool(False),
"mode": String("normal"),
"jobs": Integer(4),
}
......@@ -30,12 +30,22 @@ class IncrementalCompilation(Experiment):
"stats": File("summary.dict"),
}
def setup_compiler_paths(self, clang):
os.environ['CC'] = clang
def setup_compiler_paths(self, clang_path):
if self.mode.value == "normal":
CC = os.path.join(clang_path, "build/wrappers/clang-normal")
elif self.mode.value == "clang-hash":
CC = os.path.join(clang_path, "build/wrappers/clang-hash-stop")
elif self.mode.value == "ccache":
cache_dir = os.path.join(self.tmp_directory.path, "ccache")
os.mkdir(cache_dir)
os.environ["CCACHE_DIR"] = cache_dir
CC = os.path.join(clang_path, "build/wrappers/clang-ccache")
else:
raise RuntimeError("Not a valid mode")
os.environ['CC'] = CC
def call_configure(self, path):
if 'HASH_VERBOSE' in os.environ:
del os.environ['HASH_VERBOSE']
if self.project_name() == "postgresql":
shell("cd %s; ./configure --enable-depend", path)
elif self.project_name() == "musl":
......@@ -43,17 +53,8 @@ class IncrementalCompilation(Experiment):
else:
raise RuntimeError("Not a valid project")
os.environ['HASH_VERBOSE'] = '1'
if self.mode.value == "normal":
if 'STOP_IF_SAME_HASH' in os.environ:
del os.environ['STOP_IF_SAME_HASH']
elif self.mode.value == "clang-hash":
os.environ['STOP_IF_SAME_HASH'] = '1'
else:
raise RuntimeError("Invalid operation mode")
def call_make(self, path):
shell("cd %s; make -j %s", path, str(self.jobs.value))
return shell("cd %s; make -j %s", path, str(self.jobs.value))
def get_sources(self, path):
for root, dirnames, filenames in os.walk(path):
......@@ -69,17 +70,15 @@ class IncrementalCompilation(Experiment):
fd.write("\n")
def rebuild(self, path, cause):
run_id = len(self.build_info['builds'])
info = {'id': run_id, 'filename': cause}
info = {'filename': cause}
self.build_info['builds'].append(info)
# We export the RUN ID to the clang wrapper script, so it can
# include it into the object file records
os.environ['RUN_ID'] = "%d" % run_id
# Recompile!
start_time = time.time()
self.call_make(path)
ret = self.call_make(path)
end_time = time.time()
# Account only nano seconds, everywhere
......@@ -87,12 +86,12 @@ class IncrementalCompilation(Experiment):
info['build-time'] = build_time
logging.info("Rebuild done[%s]: %s s", cause,
build_time / 1e9)
return ret
def run(self):
self.suspend_on_error = True
# Determine the mode
modes = ('normal', 'clang-hash')
modes = ('normal', 'ccache', 'clang-hash')
if not self.mode.value in modes:
raise RuntimeError("Mode can only be one of: %s"%modes)
......@@ -100,10 +99,8 @@ class IncrementalCompilation(Experiment):
logging.info("Build the Clang-Hash Plugin")
with self.clang_hash as cl_path:
shell("cd %s; mkdir build; cd build; cmake ..; make -j 4", cl_path)
clanghash_wrapper = os.path.join(cl_path, "build/wrappers/clang")
# Project name
os.environ["PROJECT"] = self.project_name()
logging.info("Cloning project... %s", self.project_name())
self.build_info = {"project-name": self.project_name(),
"commit-hash": self.metadata["project-hash"],
......@@ -111,9 +108,7 @@ class IncrementalCompilation(Experiment):
with self.project as src_path:
# First, we redirect all calls to the compiler to our
# clang hash wrapper
self.setup_compiler_paths(clanghash_wrapper)
os.environ["COMMIT_HASH"] = self.metadata["project-hash"]
os.environ["CLANG_HASH_OUTPUT_DIR"] = self.hashes.path
self.setup_compiler_paths(cl_path)
nr_files = len(list(self.get_sources(src_path)))
logging.info("#files: %d", nr_files)
......@@ -134,6 +129,9 @@ class IncrementalCompilation(Experiment):
def project_name(self):
return os.path.basename(self.metadata['project-clone-url'])
def variant_name(self):
return "%s-%s"%(self.project_name(), self.metadata['mode'])
def symlink_name(self):
return "%s-%s-%s"%(self.title, self.project_name(), self.metadata['mode'])
......
experiments/plots.py
View file @ 81828c8b
......@@ -20,6 +20,7 @@ import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from incremental_rebuild import IncrementalCompilation
from lib import read_hash_directory
class IncrementalCompilationPlots(Experiment):
inputs = {'results': List(IncrementalCompilation(), [])}
......@@ -44,16 +45,26 @@ class IncrementalCompilationPlots(Experiment):
build_times_all = []
build_times_headers = []
build_times_sources = []
#hash_info = read_hash_directory(result.hashes.path)
#print(hash_info)
for build in records['builds']:
t = build['build-time'] / 1e9
if build['filename'] == "FRESH_BUILD":
print(result.variant_name(), "FB", t)
continue
# Get a float in seconds
t = build['build-time'] / 1e9
build_times_all.append(t)
if build['filename'].endswith('.h'):
build_times_headers.append(t)
else:
build_times_sources.append(t)
if "alltypes" in build['filename']:
print(result.variant_name(), t)
#print(build['id'])
self.save([result.variant_name(), 'rebuild', 'avg'],
np.average(build_times_all))
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment