compare-logs: tool for comparing different log files

wrappers/compare-logs

+#!/usr/bin/python
+
+import sys
+import collections
+import logging
+
+logging.basicConfig(level=logging.DEBUG)
+
+def read_log(fn, prefix="INFO:root:clang-hash"):
+    """Reads a clang-hash gcc wrapper log file and returns a dict of records"""
+    with open(fn) as fd:
+        result = {}
+        for line in fd.readlines():
+            if not line.startswith(prefix):
+                continue
+            line = line[len(prefix):].strip()
+            d = {}
+            for field in line.split():
+                if not ":" in field:
+                    continue
+                key, value = field.split(":", 1)
+                d[key] = value
+            if "fn" not in d or "ast" not in d or not "obj" in d:
+                logging.warn("Ignored line: %s", line)
+                continue
+            if d["ast"] is "None":
+                logging.warn("Ignored line (AST is None): %s", line)
+                continue
+
+            fn = d["fn"]
+            result[fn] = d
+    return result
+
+def compare_logs_seq(logs):
+    last = None
+    this = None
+    for this in logs:
+        if last is None:
+            last = this
+            continue
+        compare_logs(last, logs[last], this, logs[this])
+        last = this
+
+
+minimal_hash_length = 9
+def shorten(HASH):
+    return HASH[:minimal_hash_length]
+
+def compare_logs(name_a, a, name_b, b):
+    # Follow all files where the filename stayed equal
+    common_fns = list(set(a.keys()) & set(b.keys()))
+    for fn in common_fns:
+        # When the object hash has changed, the ast hash *MUST* also change
+        if a[fn]["obj"] != b[fn]["obj"] and a[fn]["ast"] == b[fn]["ast"]:
+            logging.error("[%s -> %s] Hash of %s (%s, %s) has changed; but AST hash remained equal (%s)",
+                          name_a, name_b, fn,
+                          shorten(a[fn]["obj"]),
+                          shorten(b[fn]["obj"]),
+                          shorten(a[fn]["ast"]))
+
+    # For all AST hashes that are equal, all obj hashes must also be equal
+    by_ast_hash = collections.defaultdict(list)
+    for fn, data in a.items():
+        by_ast_hash[data["ast"]].append((name_a, data["fn"], data["obj"]))
+    for fn, data in b.items():
+        by_ast_hash[data["ast"]].append((name_b, data["fn"], data["obj"]))
+
+    for ast_hash, records in by_ast_hash.items():
+        first_obj_hash = records[0][2]
+        first_obj_fn = records[0][1]
+
+        # This is also true, when the filename did not change
+        if any([x[2] != first_obj_hash for x in records]):
+            if any([x[1] != first_obj_fn for x in records]):
+                   logging.error("[%s -> %s] got AST hash %s, but objects differ: \n %s",
+                                 name_a, name_b, shorten(ast_hash), "\n  ".join([str(x) for x in records]))
+
+    # Give warnings for files where the object hash did not differ, but different AST hashes where produced
+    by_obj_hash = collections.defaultdict(list)
+    for fn, data in a.items():
+        by_obj_hash[data["obj"]].append((name_a, data["fn"], data["ast"]))
+    for fn, data in b.items():
+        by_obj_hash[data["obj"]].append((name_b, data["fn"], data["ast"]))
+
+    for obj_hash, records in by_obj_hash.items():
+        first_ast_hash = records[0][2]
+        first_ast_fn = records[0][1]
+
+        # This is also true, when the filename did not change
+        if any([x[2] != first_ast_hash for x in records]):
+            logging.warn("[%s -> %s] got OBJ hash %s, but AST hashes differ: \n %s",
+                         name_a, name_b, shorten(obj_hash), "\n  ".join([str(x) for x in records]))
+
+if __name__ == "__main__":
+    logs = collections.OrderedDict()
+    for fn in sys.argv[1:]:
+        logs[fn] = read_log(fn)
+
+    compare_logs_seq(logs)

wrappers/compare-logs-example-A.log

+INFO:root:clang-hash fn:shell/ash.o obj:5cca7fd959b1e615c149c49c413cf5513145f156 ast:e2f6c0158dc52df245c6264758ef06931ecf3689
+INFO:root:clang-hash fn:shell/ash_ptr_hack.o obj:330bf5a6629f24efc349195931e942b0fccdbe79 ast:1add7e99804e9929db7084bcd6048945c7d92a7c
+INFO:root:clang-hash fn:shell/cttyhack.o obj:7a7e952ad11f51393b422c44f5f838350dafe322 ast:74ccec1c3e49844bcdd990d7ffd8ca66ea95275a
+INFO:root:clang-hash fn:shell/hush.o obj:f1e972190428fdc9e10b8a7ff75d29776c340207 ast:388378548cf1bec3f1d7235fa8ba4be6ef116b10
+INFO:root:clang-hash fn:shell/match.o obj:789bd39f9007b495e796d2fa882dda7457af2a73 ast:9b0deba904e6493073f62c03027ac5b4ecdcaa81
+INFO:root:clang-hash fn:shell/math.o obj:25e58d048e188cdebe5a880167192eaed2bc4a4e ast:b2b00e77fca9659f086918ed766969ccbdbbe224
+INFO:root:clang-hash fn:shell/random.o obj:f074f8c93734a80d7b6f37d872c4f538acc81b0d ast:24065db32337c24b3b4154ee019a690905168b1a
+INFO:root:clang-hash fn:shell/shell_common.o obj:1333fb2d160e3772d59b7fba31b5b4a4b2ce3d5e ast:71f1543fae3b6cd330ce3f684b53ac15184bef75
+

wrappers/compare-logs-example-B.log

+# AST Hash changed unneccessary
+INFO:root:clang-hash fn:shell/ash.o obj:5cca7fd959b1e615c149c49c413cf5513145f156 ast:a2f6c0158dc52df245c6264758ef06931ecf3689
+
+# OBJECT hash did change, AST NOT (no rename)
+INFO:root:clang-hash fn:shell/ash_ptr_hack.o obj:230bf5a6629f24efc349195931e942b0fccdbe79 ast:1add7e99804e9929db7084bcd6048945c7d92a7c
+
+# OBJECT hash did change, AST NOT (rename)
+INFO:root:clang-hash fn:shell/xxxhack.o obj:2a7e952ad11f51393b422c44f5f838350dafe322 ast:74ccec1c3e49844bcdd990d7ffd8ca66ea95275a
+
+INFO:root:clang-hash fn:shell/hush.o obj:f1e972190428fdc9e10b8a7ff75d29776c340207 ast:388378548cf1bec3f1d7235fa8ba4be6ef116b10
+INFO:root:clang-hash fn:shell/match.o obj:789bd39f9007b495e796d2fa882dda7457af2a73 ast:9b0deba904e6493073f62c03027ac5b4ecdcaa81
+INFO:root:clang-hash fn:shell/math.o obj:25e58d048e188cdebe5a880167192eaed2bc4a4e ast:b2b00e77fca9659f086918ed766969ccbdbbe224
+INFO:root:clang-hash fn:shell/random.o obj:f074f8c93734a80d7b6f37d872c4f538acc81b0d ast:24065db32337c24b3b4154ee019a690905168b1a
+INFO:root:clang-hash fn:shell/shell_common.o obj:1333fb2d160e3772d59b7fba31b5b4a4b2ce3d5e ast:71f1543fae3b6cd330ce3f684b53ac15184bef75
+

wrappers/gcc

+#!/usr/bin/python
+
+import sys
+import os
+import hashlib
+import logging
+import re
+from subprocess import *
+
+
+if __name__ == "__main__":
+   args = sys.argv[1:]
+   level = logging.INFO
+   if "V" in os.environ:
+      level = logging.DEBUG
+   logging.basicConfig(filename = "clang-hash.log", level=level)
+   logging.debug(str(sys.argv))
+
+   x = Popen(["/usr/bin/gcc"] + args, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr)
+   retcode = x.wait()
+
+   filter_flags = (
+      "-fno-guess-branch-probability",
+      "-finline-limit=",
+      "-falign-functions=",
+      "-falign-labels=",
+      "-falign-loops=",
+      "-falign-jumps="
+      )
+
+   kconfig = "kconfig" in "".join(args)
+
+   if "-c" in args and "-o" in args and "/dev/null" not in args and not kconfig:
+      # Save Objectfile data
+      objectfile = args[args.index("-o")+1]
+      objectfile_hash = None
+      if os.path.exists(objectfile):
+         with open(objectfile, "rb") as fd:
+            objectfile_data = fd.read()
+            objectfile_hash = hashlib.sha1()
+            objectfile_hash.update(objectfile_data)
+            objectfile_hash = objectfile_hash.hexdigest()
+      else:
+         objectfile_data = None
+
+      A = []
+      for x in args:
+         ignore = False
+         for pattern in filter_flags:
+            if x.startswith(pattern):
+               ignore = True
+
+         if not ignore:
+            if x.startswith("-D"):
+               if "KBUILD_STR" in x:
+                  (x, _) = re.subn("=KBUILD_STR\(([^)]*)\)", '=', x)
+                  logging.info(x)
+            A.append(x)
+
+      logging.info("%s", A)
+      stderr_fd = open("%s.clang-hash-stderr" % objectfile, "w+")
+      p = Popen(["clang-hash"] + A, stdin=PIPE, stderr = stderr_fd)
+      p.stdin.close()
+      p.communicate()
+      p.wait()
+
+      if objectfile_data:
+         with open(objectfile, "wb+") as fd:
+            fd.write(objectfile_data)
+
+      if os.path.exists(objectfile +  ".hash"):
+         with open(objectfile +  ".hash") as fd:
+            ast_hash = fd.read()
+            if ast_hash == "":
+               ast_hash = None
+      else:
+         ast_hash = None
+      logging.info("clang-hash fn:%s obj:%s ast:%s", objectfile, objectfile_hash, ast_hash)
+
+
+   sys.exit(retcode)