Commit 5e2add65 authored by Christian Dietrich's avatar Christian Dietrich
Browse files

compare-logs: tool for comparing different log files

parent 7d0470ba
#!/usr/bin/python
import sys
import collections
import logging
logging.basicConfig(level=logging.DEBUG)
def read_log(fn, prefix="INFO:root:clang-hash"):
"""Reads a clang-hash gcc wrapper log file and returns a dict of records"""
with open(fn) as fd:
result = {}
for line in fd.readlines():
if not line.startswith(prefix):
continue
line = line[len(prefix):].strip()
d = {}
for field in line.split():
if not ":" in field:
continue
key, value = field.split(":", 1)
d[key] = value
if "fn" not in d or "ast" not in d or not "obj" in d:
logging.warn("Ignored line: %s", line)
continue
if d["ast"] is "None":
logging.warn("Ignored line (AST is None): %s", line)
continue
fn = d["fn"]
result[fn] = d
return result
def compare_logs_seq(logs):
last = None
this = None
for this in logs:
if last is None:
last = this
continue
compare_logs(last, logs[last], this, logs[this])
last = this
minimal_hash_length = 9
def shorten(HASH):
return HASH[:minimal_hash_length]
def compare_logs(name_a, a, name_b, b):
# Follow all files where the filename stayed equal
common_fns = list(set(a.keys()) & set(b.keys()))
for fn in common_fns:
# When the object hash has changed, the ast hash *MUST* also change
if a[fn]["obj"] != b[fn]["obj"] and a[fn]["ast"] == b[fn]["ast"]:
logging.error("[%s -> %s] Hash of %s (%s, %s) has changed; but AST hash remained equal (%s)",
name_a, name_b, fn,
shorten(a[fn]["obj"]),
shorten(b[fn]["obj"]),
shorten(a[fn]["ast"]))
# For all AST hashes that are equal, all obj hashes must also be equal
by_ast_hash = collections.defaultdict(list)
for fn, data in a.items():
by_ast_hash[data["ast"]].append((name_a, data["fn"], data["obj"]))
for fn, data in b.items():
by_ast_hash[data["ast"]].append((name_b, data["fn"], data["obj"]))
for ast_hash, records in by_ast_hash.items():
first_obj_hash = records[0][2]
first_obj_fn = records[0][1]
# This is also true, when the filename did not change
if any([x[2] != first_obj_hash for x in records]):
if any([x[1] != first_obj_fn for x in records]):
logging.error("[%s -> %s] got AST hash %s, but objects differ: \n %s",
name_a, name_b, shorten(ast_hash), "\n ".join([str(x) for x in records]))
# Give warnings for files where the object hash did not differ, but different AST hashes where produced
by_obj_hash = collections.defaultdict(list)
for fn, data in a.items():
by_obj_hash[data["obj"]].append((name_a, data["fn"], data["ast"]))
for fn, data in b.items():
by_obj_hash[data["obj"]].append((name_b, data["fn"], data["ast"]))
for obj_hash, records in by_obj_hash.items():
first_ast_hash = records[0][2]
first_ast_fn = records[0][1]
# This is also true, when the filename did not change
if any([x[2] != first_ast_hash for x in records]):
logging.warn("[%s -> %s] got OBJ hash %s, but AST hashes differ: \n %s",
name_a, name_b, shorten(obj_hash), "\n ".join([str(x) for x in records]))
if __name__ == "__main__":
logs = collections.OrderedDict()
for fn in sys.argv[1:]:
logs[fn] = read_log(fn)
compare_logs_seq(logs)
INFO:root:clang-hash fn:shell/ash.o obj:5cca7fd959b1e615c149c49c413cf5513145f156 ast:e2f6c0158dc52df245c6264758ef06931ecf3689
INFO:root:clang-hash fn:shell/ash_ptr_hack.o obj:330bf5a6629f24efc349195931e942b0fccdbe79 ast:1add7e99804e9929db7084bcd6048945c7d92a7c
INFO:root:clang-hash fn:shell/cttyhack.o obj:7a7e952ad11f51393b422c44f5f838350dafe322 ast:74ccec1c3e49844bcdd990d7ffd8ca66ea95275a
INFO:root:clang-hash fn:shell/hush.o obj:f1e972190428fdc9e10b8a7ff75d29776c340207 ast:388378548cf1bec3f1d7235fa8ba4be6ef116b10
INFO:root:clang-hash fn:shell/match.o obj:789bd39f9007b495e796d2fa882dda7457af2a73 ast:9b0deba904e6493073f62c03027ac5b4ecdcaa81
INFO:root:clang-hash fn:shell/math.o obj:25e58d048e188cdebe5a880167192eaed2bc4a4e ast:b2b00e77fca9659f086918ed766969ccbdbbe224
INFO:root:clang-hash fn:shell/random.o obj:f074f8c93734a80d7b6f37d872c4f538acc81b0d ast:24065db32337c24b3b4154ee019a690905168b1a
INFO:root:clang-hash fn:shell/shell_common.o obj:1333fb2d160e3772d59b7fba31b5b4a4b2ce3d5e ast:71f1543fae3b6cd330ce3f684b53ac15184bef75
# AST Hash changed unneccessary
INFO:root:clang-hash fn:shell/ash.o obj:5cca7fd959b1e615c149c49c413cf5513145f156 ast:a2f6c0158dc52df245c6264758ef06931ecf3689
# OBJECT hash did change, AST NOT (no rename)
INFO:root:clang-hash fn:shell/ash_ptr_hack.o obj:230bf5a6629f24efc349195931e942b0fccdbe79 ast:1add7e99804e9929db7084bcd6048945c7d92a7c
# OBJECT hash did change, AST NOT (rename)
INFO:root:clang-hash fn:shell/xxxhack.o obj:2a7e952ad11f51393b422c44f5f838350dafe322 ast:74ccec1c3e49844bcdd990d7ffd8ca66ea95275a
INFO:root:clang-hash fn:shell/hush.o obj:f1e972190428fdc9e10b8a7ff75d29776c340207 ast:388378548cf1bec3f1d7235fa8ba4be6ef116b10
INFO:root:clang-hash fn:shell/match.o obj:789bd39f9007b495e796d2fa882dda7457af2a73 ast:9b0deba904e6493073f62c03027ac5b4ecdcaa81
INFO:root:clang-hash fn:shell/math.o obj:25e58d048e188cdebe5a880167192eaed2bc4a4e ast:b2b00e77fca9659f086918ed766969ccbdbbe224
INFO:root:clang-hash fn:shell/random.o obj:f074f8c93734a80d7b6f37d872c4f538acc81b0d ast:24065db32337c24b3b4154ee019a690905168b1a
INFO:root:clang-hash fn:shell/shell_common.o obj:1333fb2d160e3772d59b7fba31b5b4a4b2ce3d5e ast:71f1543fae3b6cd330ce3f684b53ac15184bef75
#!/usr/bin/python
import sys
import os
import hashlib
import logging
import re
from subprocess import *
if __name__ == "__main__":
args = sys.argv[1:]
level = logging.INFO
if "V" in os.environ:
level = logging.DEBUG
logging.basicConfig(filename = "clang-hash.log", level=level)
logging.debug(str(sys.argv))
x = Popen(["/usr/bin/gcc"] + args, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr)
retcode = x.wait()
filter_flags = (
"-fno-guess-branch-probability",
"-finline-limit=",
"-falign-functions=",
"-falign-labels=",
"-falign-loops=",
"-falign-jumps="
)
kconfig = "kconfig" in "".join(args)
if "-c" in args and "-o" in args and "/dev/null" not in args and not kconfig:
# Save Objectfile data
objectfile = args[args.index("-o")+1]
objectfile_hash = None
if os.path.exists(objectfile):
with open(objectfile, "rb") as fd:
objectfile_data = fd.read()
objectfile_hash = hashlib.sha1()
objectfile_hash.update(objectfile_data)
objectfile_hash = objectfile_hash.hexdigest()
else:
objectfile_data = None
A = []
for x in args:
ignore = False
for pattern in filter_flags:
if x.startswith(pattern):
ignore = True
if not ignore:
if x.startswith("-D"):
if "KBUILD_STR" in x:
(x, _) = re.subn("=KBUILD_STR\(([^)]*)\)", '=', x)
logging.info(x)
A.append(x)
logging.info("%s", A)
stderr_fd = open("%s.clang-hash-stderr" % objectfile, "w+")
p = Popen(["clang-hash"] + A, stdin=PIPE, stderr = stderr_fd)
p.stdin.close()
p.communicate()
p.wait()
if objectfile_data:
with open(objectfile, "wb+") as fd:
fd.write(objectfile_data)
if os.path.exists(objectfile + ".hash"):
with open(objectfile + ".hash") as fd:
ast_hash = fd.read()
if ast_hash == "":
ast_hash = None
else:
ast_hash = None
logging.info("clang-hash fn:%s obj:%s ast:%s", objectfile, objectfile_hash, ast_hash)
sys.exit(retcode)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment