Commit 4c715b34 authored by Christian Dietrich's avatar Christian Dietrich
Browse files

experimental: restructure historical build

Since we have problem we heavily merging in build histories (cpython),
We now try a more robust variant for each commit:

1. Build the parent
   1.a) Last Build was the parent, and that was good -> Do nothing
   1.b) Last Build not the parent. git reset and build
      A) was ok? fine
      B) rebuild failed -> git clean, reset, try again
2. Build the commit itself

We always try to build children first to avoid heavy jumping between
different banches and commits. But that is only an optimization. Lets
give it a try.
parent 7d6a5543
Pipeline #3056 passed with stage
in 0 seconds
...@@ -29,6 +29,48 @@ class HistoricalCompilation(Experiment, ClangHashHelper): ...@@ -29,6 +29,48 @@ class HistoricalCompilation(Experiment, ClangHashHelper):
"stats": File("summary.dict"), "stats": File("summary.dict"),
} }
def build_parent(self, commit, from_scratch = False):
def eq_hash(a, b):
if len(a) == 0 or len(b) == 0:
return
if len(a) > len(b):
return a.startswith(b)
else:
return b.startswith(a)
src_path = self.project.path
if from_scratch:
shell("cd %s; git clean -dfx", src_path)
logging.info("Parent [%s^]: clean build", commit)
shell("cd %s; git reset --hard %s^", src_path, commit)
info = {"commit": commit + "^"}
self.call_configure(src_path)
self.rebuild(src_path, info, True)
# Did initial commit fail? Try again
if info.get("failed"):
logging.info("Parent[%s^]: failed" commit)
return False
return True
else:
(lines, _) = shell("cd %s; git rev-parse %s^",
src_path, commit)
parent_revision = lines[0].strip()
if self.current_revision and eq_hash(self.current_revision, parent_revision):
logging.info("Parent[%s^]: resuse good parent", commit)
return True
else:
logging.info("Parent[%s^]: resuse similar build directory", commit)
shell("cd %s; git reset --hard %s^", src_path, commit)
info = {"commit": commit +"^"}
self.call_reconfigure(src_path)
self.rebuild(src_path, info, True)
# Did initial commit fail? Try again
if info.get("failed"):
return self.build_parent(commit, from_scratch=True)
return True
def run(self): def run(self):
# Determine the mode # Determine the mode
modes = ('normal', 'ccache', 'clang-hash', 'ccache-clang-hash') modes = ('normal', 'ccache', 'clang-hash', 'ccache-clang-hash')
...@@ -47,37 +89,51 @@ class HistoricalCompilation(Experiment, ClangHashHelper): ...@@ -47,37 +89,51 @@ class HistoricalCompilation(Experiment, ClangHashHelper):
'builds': []} 'builds': []}
with self.project as src_path: with self.project as src_path:
(commits, _) = shell("cd %s; git log --oneline --topo-order", src_path) (commits, _) = shell("cd %s; git log --no-merges --oneline --topo-order --format='%%H %%P %%s'", src_path)
commits = [x.split(" ", 1) for x in reversed(commits)] # [0] is hash. [1] is parent, [2] rest
commits = [x.split(" ", 2) for x in reversed(commits)]
commits = commits[-self.commits.value:] commits = commits[-self.commits.value:]
self.current_revision = None
# First, we redirect all calls to the compiler to our # First, we redirect all calls to the compiler to our
# clang hash wrapper # clang hash wrapper
self.setup_compiler_paths(cl_path) self.setup_compiler_paths(cl_path)
while True:
commit = commits.pop(0)
logging.info("Build: %s", commit)
shell("cd %s; git clean -dfx; git reset --hard %s", src_path, commit[0])
info = {"commit": "FRESH_BUILD"}
# Initial build of the given project
self.call_configure(src_path)
self.rebuild(src_path, info, True)
# Did initial commit fail? Try again
if "failed" in info:
continue
self.build_info["builds"].append(info)
break
time = 0 time = 0
for commit in commits: while commits:
# Search for a child of the current revision
commit = None
if self.current_revision:
for idx in range(0, len(commits)):
if commits[idx][1] == self.current_revision:
commit = commits[idx]
del commits[idx]
break
# No Child found -> Take the first one.
if not commit:
commit = commits.pop(0)
info = {"commit": commit[0], "summary": commit[2]}
# First, we build the parent. In a total linear
# history, this is a NOP. Otherwise, we try to reset
# to the actual parent, and rebuild the project. This
# may fail, since the current commit might fix this.
self.build_parent(commit[0])
shell("cd %s; git reset --hard %s", src_path, commit[0]) shell("cd %s; git reset --hard %s", src_path, commit[0])
info = {"commit": commit[0], "summary": commit[1]}
self.call_reconfigure(src_path) self.call_reconfigure(src_path)
self.rebuild(src_path, info, fail_ok=True) self.rebuild(src_path, info, fail_ok=True)
self.build_info["builds"].append(info) self.build_info["builds"].append(info)
if not info.get("failed"): if not info.get("failed"):
time += info['build-time'] / 1e9 time += info['build-time'] / 1e9
# Build was good. Remember that.
self.current_revision = commit[0]
else:
self.current_revision = None
logging.info("Rebuild for %d commits takes %f minutes", logging.info("Rebuild for %d commits takes %f minutes",
self.commits.value, time/60.) self.commits.value, time/60.)
......
...@@ -51,7 +51,10 @@ class ClangHashHelper: ...@@ -51,7 +51,10 @@ class ClangHashHelper:
elif self.project_name() in ("musl", "bash", "samba"): elif self.project_name() in ("musl", "bash", "samba"):
shell("cd %s; ./configure", path) shell("cd %s; ./configure", path)
elif self.project_name() in ("cpython",): elif self.project_name() in ("cpython",):
shell("cd %s; mkdir build; cd build; ../configure", path) shell("cd %s; mkdir -p build build/Modules;", path)
shell("cd %s; cp -u Modules/Setup.dist build/Modules/Setup", path)
shell("cd %s; cd build; ../configure", path)
elif self.project_name() in ('mbedtls'): elif self.project_name() in ('mbedtls'):
shell("cd %s; mkdir build; cd build; cmake .. -DCMAKE_C_COMPILER=$CC -DENABLE_PROGRAMS=OFF", path) shell("cd %s; mkdir build; cd build; cmake .. -DCMAKE_C_COMPILER=$CC -DENABLE_PROGRAMS=OFF", path)
elif self.project_name() in ('lua',): elif self.project_name() in ('lua',):
...@@ -72,7 +75,8 @@ class ClangHashHelper: ...@@ -72,7 +75,8 @@ class ClangHashHelper:
if self.project_name() in ('lua',): if self.project_name() in ('lua',):
self.call_configure(path) self.call_configure(path)
if self.project_name() in ('cpython',): if self.project_name() in ('cpython',):
shell("cd %s/build; make config.status", path) shell("cd %s; mkdir -p build/Modules; cp -u Modules/Setup.dist build/Modules/Setup", path)
def call_make(self, path): def call_make(self, path):
if self.project_name() in ("mbedtls", "cpython"): if self.project_name() in ("mbedtls", "cpython"):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment