experiments: measure historical commits

experiments/Makefile

@@ -22,7 +22,7 @@ inc_$3_$2_append:
 inc_$3_$2_touch:
 	./incremental_rebuild.py -s \
 		-d ../versuchung-data --jobs $(JOBS) \
-		-v --clang_hash-clone-url ${CLANG_HASH} \
+		-vv --clang_hash-clone-url ${CLANG_HASH} \
 		--project-clone-url ${$1} --mode $2 --touch-only true
 
 inc_$3: inc_$3_$2_touch inc_$3_$2_append

experiments/historical_build.py

+#!/usr/bin/env python2
+
+import os
+import sys
+import logging
+import time
+
+tmp_path = "%s/git/versuchung/src"% os.environ["HOME"]
+if os.path.exists(tmp_path):
+    sys.path.append(tmp_path)
+
+from versuchung.experiment import Experiment
+from versuchung.types import String, Bool,Integer
+from versuchung.files import File, Directory
+from versuchung.archives import GitArchive
+from versuchung.execute import shell
+from versuchung.tex import DatarefDict
+from lib import ClangHashHelper
+
+class HistoricalCompilation(Experiment, ClangHashHelper):
+    inputs = {
+        "clang_hash": GitArchive("/home/stettberger/w/clang-hash/"),
+        "project": GitArchive("/home/stettberger/w/clang-hash/hash-projects/lua"),
+        "mode": String("normal"),
+        "commits": Integer(50),
+        "jobs": Integer(4),
+    }
+    outputs = {
+        "stats": File("summary.dict"),
+    }
+
+    def run(self):
+        # Determine the mode
+        modes = ('normal', 'ccache', 'clang-hash')
+        if not self.mode.value in modes:
+            raise RuntimeError("Mode can only be one of: %s"%modes)
+
+        logging.info("Build the Clang-Hash Plugin")
+        with self.clang_hash as cl_path:
+            shell("cd %s; mkdir build; cd build; cmake ..; make -j 4", cl_path)
+
+        # Project name
+        logging.info("Cloning project... %s", self.project_name())
+        self.build_info = {"project-name": self.project_name(),
+                           "commit-hash": self.metadata["project-hash"],
+                           'builds': []}
+
+        with self.project as src_path:
+            (commits, _) = shell("cd %s; git log --oneline", src_path)
+            commits = [x.split(" ", 1) for x in reversed(commits)]
+            commits = commits[-self.commits.value:]
+
+            # First, we redirect all calls to the compiler to our
+            # clang hash wrapper
+            self.setup_compiler_paths(cl_path)
+
+
+            while True:
+                commit = commits.pop(0)
+                logging.info("Build: %s", commit)
+                shell("cd %s; git clean -dfx; git reset --hard %s", src_path, commit[0])
+                info = {"commit": "FRESH_BUILD"}
+                            # Initial build of the given project
+                self.call_configure(src_path)
+                self.rebuild(src_path, info, True)
+                # Did initial commit fail? Try again
+                if "failed" in info:
+                    continue
+                self.build_info["builds"].append(info)
+                break
+
+            time = 0
+            for commit in commits:
+                shell("cd %s; git reset --hard %s", src_path, commit[0])
+                info = {"commit": commit[0], "summary": commit[1]}
+                if self.project_name() == "lua":
+                    self.call_configure(src_path)
+                self.rebuild(src_path, info, fail_ok=True)
+                self.build_info["builds"].append(info)
+                if not info.get("failed"):
+                    time += info['build-time'] / 1e9
+            logging.info("Rebuild for %d commits takes %f minutes",
+                         self.commits.value, time/60.)
+
+        # Output the summary of this build into the statistics file.
+        with open(self.stats.path, "w+") as fd:
+            fd.write(repr(self.build_info))
+
+    def variant_name(self):
+        return "%s-%s"%(self.project_name(), self.metadata['mode'])
+
+    def symlink_name(self):
+        return "%s-%s"%(self.title, self.variant_name())
+
+
+if __name__ == "__main__":
+    experiment = HistoricalCompilation()
+    dirname = experiment(sys.argv)

experiments/incremental_rebuild.py

@@ -15,8 +15,9 @@ from versuchung.files import File, Directory
 from versuchung.archives import GitArchive
 from versuchung.execute import shell
 from versuchung.tex import DatarefDict
+from lib import ClangHashHelper
 
-class IncrementalCompilation(Experiment):
+class IncrementalCompilation(Experiment, ClangHashHelper):
     inputs = {
         "clang_hash": GitArchive("/home/stettberger/w/clang-hash/"),
         "project": GitArchive("/home/stettberger/w/clang-hash/hash-projects/musl",
@@ -29,37 +30,6 @@ class IncrementalCompilation(Experiment):
         "stats": File("summary.dict"),
     }
 
-    def setup_compiler_paths(self, clang_path):
-        if self.mode.value == "normal":
-            CC = os.path.join(clang_path, "build/wrappers/clang-normal")
-        elif self.mode.value == "clang-hash":
-            CC = os.path.join(clang_path, "build/wrappers/clang-hash-stop")
-        elif self.mode.value == "ccache":
-            cache_dir = os.path.join(self.tmp_directory.path, "ccache")
-            os.mkdir(cache_dir)
-            os.environ["CCACHE_DIR"] = cache_dir
-            CC = os.path.join(clang_path, "build/wrappers/clang-ccache")
-        else:
-            raise RuntimeError("Not a valid mode")
-
-        os.environ['CC'] = CC
-        self.CC = CC
-
-    def call_configure(self, path):
-        if self.project_name() == "postgresql":
-            shell("cd %s; ./configure --enable-depend", path)
-        elif self.project_name() in ("musl", "cpython", "bash", "waf"):
-            shell("cd %s; ./configure", path)
-        elif self.project_name() in ('mbedtls'):
-            shell("cd %s; cmake . -DCMAKE_C_COMPILER=$CC", path)
-        elif self.project_name() in ('lua',):
-            pass
-        else:
-            raise RuntimeError("Not a valid project")
-
-    def call_make(self, path, cause = ""):
-        return shell("cd %s; make -j %s", path, str(self.jobs.value))
-
     def get_sources(self, path):
         ret = []
         for root, dirnames, filenames in os.walk(path):
@@ -82,30 +52,6 @@ class IncrementalCompilation(Experiment):
             with open(path, "w") as fd:
                 fd.write(content)
 
-    def rebuild(self, path, cause):
-        info = {'filename': cause}
-        self.build_info['builds'].append(info)
-
-        # We export the RUN ID to the clang wrapper script, so it can
-        # include it into the object file records
-
-        # Recompile!
-        start_time = time.time()
-        ret = self.call_make(path, cause)
-        end_time = time.time()
-
-        # Call the lines that include the full compiler path. This
-        # number is not useful, if -j N was done....
-        # Therefore, we do not record it.
-        compiler_calls = len([1 for x in ret[0]
-                              if x.startswith(self.CC) and '-c' in x])
-        # Account only nano seconds, everywhere
-        build_time = int((end_time - start_time) * 1e9)
-        info['build-time'] = build_time
-        logging.info("Rebuild done[%s]: %s s; CC() = %d ", cause,
-                     build_time / 1e9,
-                     compiler_calls)
-        return ret
 
     def run(self):
         # Determine the mode
@@ -136,20 +82,21 @@ class IncrementalCompilation(Experiment):
 
             # Initial build of the given project
             self.call_configure(src_path)
-            self.rebuild(src_path, "FRESH_BUILD")
+            info = {"filename": "FRESH_BUILD"}
+            self.rebuild(src_path, info)
+            self.build_info["builds"].append(info)
 
             # Iterate over all files
             for fn in sources:
                 self.touch(fn)
-                self.rebuild(src_path, fn)
+                info = {"filename": fn}
+                self.rebuild(src_path, info)
+                self.build_info["builds"].append(info)
 
         # Output the summary of this build into the statistics file.
         with open(self.stats.path, "w+") as fd:
             fd.write(repr(self.build_info))
 
-    def project_name(self):
-        return os.path.basename(self.metadata['project-clone-url'])
-
     def variant_name(self):
         mod = "append"
         if self.metadata['touch-only']:

experiments/plots.py

@@ -29,9 +29,6 @@ class IncrementalCompilationPlots(Experiment):
         self.tex['/'.join(path)] = value
         logging.info("%s = %s", '/'.join(path), value)
 
-    def symlink_name(self):
-        return "%s-%s" %(self.title, self.project_name)
-
     def run(self):
         self.project_name = ""
         for result in sorted(self.results, key=lambda x: (x.variant_name())):
@@ -44,7 +41,7 @@ class IncrementalCompilationPlots(Experiment):
                 t = build['build-time'] / 1e9
 
                 if build['filename'] == "FRESH_BUILD":
-                    print(result.variant_name(), "FB", t)
+                    self.save([result.variant_name(), "fresh build"], t)
                     continue
                 # Get a float in seconds
                 build_times_all.append(t)
@@ -53,20 +50,16 @@ class IncrementalCompilationPlots(Experiment):
                 else:
                     build_times_sources.append(t)
 
-                #if "alltypes" in build['filename']:
-                #    print(build['filename'], t, build['compiler-calls'])
 
                 #print(build['id'])
+            def seq(key, seq):
+                self.save(key +["count"], len(seq))
+                self.save(key +["avg"],   np.average(seq))
 
+            seq([result.variant_name(), 'rebuild'], build_times_all)
+            seq([result.variant_name(), 'rebuild', "sources"], build_times_sources)
+            seq([result.variant_name(), 'rebuild', "headers"], build_times_headers)
 
-            self.save([result.variant_name(), 'rebuild', 'avg'],
-                      np.average(build_times_all))
-            if build_times_sources:
-                self.save([result.variant_name(), 'rebuild', 'sources', 'avg'],
-                          np.average(build_times_sources))
-            if build_times_headers:
-                self.save([result.variant_name(), 'rebuild', 'headers', 'avg'],
-                          np.average(build_times_headers))
 
 
 if __name__ == "__main__":