Merge branch 'master' of gitlab.cs.fau.de:stettberger/clang-hash

experiments/analyze_results.py

@@ -88,12 +88,18 @@ class AnalyzeResults(Experiment):
         ################################################################
         x = sorted(self.historical, key=lambda x:x.project_name())
         hist = defaultdict(lambda: 0)
+        method_stats = defaultdict(lambda: defaultdict(lambda: 0))
+
         for (project, results) in groupby(x, key=lambda x:x.project_name()):
             times = defaultdict(lambda: dict())
 
             for result in sorted(results, key=lambda x:x.variant_name()):
+                key = [result.variant_name(), 'historical']
                 records = eval(result.stats.value)
 
+                # How Many Hits were produced by clang-hash/ccache
+                stats = defaultdict(lambda : 0)
+
                 build_times = []
                 failed = 0
                 for build in records['builds']:
@@ -105,15 +111,29 @@ class AnalyzeResults(Experiment):
                     times[build['commit']][result.metadata['mode']] = t
                     hist[int(t)] += 1
 
-
+                    stats['misses/clang-hash'] += build.get('clang-hash-misses',0)
+                    stats['hits/clang-hash'] += build.get('clang-hash-hits',0)
+                    stats['misses/ccache'] += build.get('ccache-misses',0)
+                    stats['hits/ccache'] += build.get('ccache-hits',0)
+                    stats['hits'] += build.get('ccache-hits',0) \
+                                     + build.get('clang-hash-hits',0)
+                    stats['misses'] += build.get('ccache-misses',0) \
+                                       + build.get('clang-hash-misses',0)
+                    if result.mode.value == "ccache-clang-hash":
+                        stats["misses"] -= build.get('clang-hash-hits',0)
+
+                # Over all builds of an experiment
                 def seq(key, seq):
                     self.save(key +["sum"], sum(seq))
                     self.save(key +["count"], len(seq))
                     self.save(key +["avg"],   np.average(seq))
 
-                key = [result.variant_name(), 'historical']
                 self.save(key + ["failed"], failed)
                 seq(key, build_times)
+                for k in stats:
+                    self.save(key + [k], stats[k])
+                    # Aggregate hits and misses per method
+                    method_stats[result.metadata["mode"]][k] += stats[k]
 
             try:
                 x = sorted(times, key=lambda x: times[x]['clang-hash']/times[x]['normal'])
@@ -128,6 +148,10 @@ class AnalyzeResults(Experiment):
             except:
                 pass
 
+        # Output method statistics
+        for method in method_stats:
+            for k in method_stats[method]:
+                self.save([method, "historical", k], method_stats[method][k])
 
 if __name__ == "__main__":
     experiment = AnalyzeResults()

experiments/historical_build.py

@@ -90,10 +90,6 @@ class HistoricalCompilation(Experiment, ClangHashHelper):
                            "commit-hash": self.metadata["project-hash"],
                            'builds': []}
 
-        hash_log = self.tmp_directory.new_file("clang-hash.log")
-        if "clang-hash" in self.mode.value:
-            os.environ["CLANG_HASH_LOGFILE"] = hash_log.path
-
         with self.project as src_path:
             (commits, _) = shell("cd %s; git log --no-merges --oneline --topo-order --format='%%H %%P %%s'", src_path)
             # [0] is hash. [1] is parent, [2] rest
@@ -151,13 +147,6 @@ class HistoricalCompilation(Experiment, ClangHashHelper):
         with open(self.stats.path, "w+") as fd:
             fd.write(repr(self.build_info))
 
-        if "ccache" in self.mode.value:
-            shell("ccache -s > %s", self.ccache_stats.path)
-
-        # Copy Log to the result directory
-        with open(hash_log.path) as fd:
-            self.clang_hash_stats.value = fd.read()
-
     def variant_name(self):
         return "%s-%s"%(self.project_name(), self.metadata['mode'])
 

experiments/lib.py

@@ -3,6 +3,7 @@ import fnmatch
 import time
 from versuchung.execute import shell, CommandFailed, shell_failok
 import logging
+import tempfile
 
 def read_hash_directory(hash_dir, remove_keys = []):
     """Read in all records from a hash dir
@@ -79,7 +80,8 @@ class ClangHashHelper:
         if self.project_name() in ('cpython',):
             shell("cd %s; mkdir -p build/Modules; cp -u Modules/Setup.dist build/Modules/Setup", path)
             shell_failok("cd %s/build; make config.status;", path)
-
+        if self.project_name() in ('postgresql', 'bash'):
+            shell_failok("cd %s; make config.status", path)
 
 
     def call_make(self, path):
@@ -88,7 +90,25 @@ class ClangHashHelper:
         else:
             return shell("cd %s; make -j %s", path, str(self.jobs.value))
 
+    def ccache_hits(self):
+        (lines, _) = shell("ccache -s")
+        ccache_hits = 0
+        ccache_misses = 0
+        for line in lines:
+            if "cache hit" in line and "rate" not in line:
+                ccache_hits += int(line[line.index(")")+1:].strip())
+            if "cache miss" in line:
+                ccache_misses += int(line[line.index("miss")+4:].strip())
+        return ccache_hits, ccache_misses
+
     def rebuild(self, path, info, fail_ok=False):
+        if "ccache" in self.mode.value:
+            old_ccache_hits, old_ccache_misses = self.ccache_hits()
+
+        if "clang-hash" in self.mode.value:
+            hash_log = tempfile.NamedTemporaryFile()
+            os.environ["CLANG_HASH_LOGFILE"] = hash_log.name
+
         # Recompile!
         start_time = time.time()
         try:
@@ -101,9 +121,20 @@ class ClangHashHelper:
                 ret = ("", 1)
         end_time = time.time()
 
-        # Account only nano seconds, everywhere
         build_time = int((end_time - start_time) * 1e9)
         info['build-time'] = build_time
+
+        # Record Cache misses and hits
+        if "ccache" in self.mode.value:
+            ccache_hits, ccache_misses = self.ccache_hits()
+            info['ccache-hits'] = ccache_hits - old_ccache_hits
+            info['ccache-misses'] = ccache_misses - old_ccache_misses
+
+        if "clang-hash" in self.mode.value:
+            log = hash_log.read()
+            info['clang-hash-hits'] = log.count("H")
+            info['clang-hash-misses'] = log.count("M")
+
         logging.info("Rebuild done[%s]: %s s; failed=%s",
                      info.get("filename") or info.get("commit"),
                      build_time / 1e9,