lib.py 5.99 KB
Newer Older
Christian Dietrich's avatar
Christian Dietrich committed
1
2
3
import os
import fnmatch
import time
4
from versuchung.execute import shell, CommandFailed, shell_failok
Christian Dietrich's avatar
Christian Dietrich committed
5
import logging
6
import tempfile
Christian Dietrich's avatar
Christian Dietrich committed
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

def read_hash_directory(hash_dir, remove_keys = []):
    """Read in all records from a hash dir
    """
    ret = []
    for root, dirnames, filenames in os.walk(hash_dir):
        for filename in fnmatch.filter(filenames, '*.info'):
            with open(os.path.join(root, filename)) as fd:
                data = "[%s]" % (",".join(fd.readlines()))
                data = eval(data)
                for record in data:
                    for key in remove_keys:
                        del record[key]
                ret.extend(data)
                print len(ret)
    return ret


class ClangHashHelper:
    def project_name(self):
        return os.path.basename(self.metadata['project-clone-url'])

    def setup_compiler_paths(self, clang_path):
        if "ccache" in self.mode.value:
            cache_dir = os.path.join(self.tmp_directory.path, "ccache")
            os.mkdir(cache_dir)
            os.environ["CCACHE_DIR"] = cache_dir
34
        if "clang-hash" in self.mode.value:
35
            cache_dir = os.path.join(self.tmp_directory.path, "clang-hash-cache")
36
37
            os.mkdir(cache_dir)
            os.environ["CLANG_HASH_CACHE"] = cache_dir
Christian Dietrich's avatar
Christian Dietrich committed
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

        if self.mode.value == "normal":
            CC = os.path.join(clang_path, "build/wrappers/clang-normal")
        elif self.mode.value == "clang-hash":
            CC = os.path.join(clang_path, "build/wrappers/clang-hash-stop")
        elif self.mode.value == "ccache-clang-hash":
            CC = os.path.join(clang_path, "build/wrappers/clang-ccache-hash-stop")
        elif self.mode.value == "ccache":
            CC = os.path.join(clang_path, "build/wrappers/clang-ccache")
        else:
            raise RuntimeError("Not a valid mode")

        os.environ['CC'] = CC
        self.CC = CC

    def call_configure(self, path):
        if self.project_name() == "postgresql":
55
            shell("cd %s; mkdir -p build; cd build; ../configure --enable-depend", path)
Christian Dietrich's avatar
Christian Dietrich committed
56
        elif self.project_name() in ("musl", "bash"):
57
            shell_failok("cd %s; ./configure", path)
Christian Dietrich's avatar
Christian Dietrich committed
58
59
60
        elif self.project_name() in ("samba"):
            # Samba does not do optimizations if the argv[0] of the compiler is unknown. The default is -O2 for gcc. Therefore, we also use that.
            shell_failok("cd %s; ADDITIONAL_CFLAGS=-O2 ./configure", path)
61
        elif self.project_name() in ("cpython",):
62
63
64
65
            shell("cd %s; mkdir -p build build/Modules;", path)
            shell("cd %s; cp -u Modules/Setup.dist build/Modules/Setup", path)
            shell("cd %s; cd build; ../configure", path)

Christian Dietrich's avatar
Christian Dietrich committed
66
        elif self.project_name() in ('mbedtls'):
67
            shell("cd %s; mkdir -p build; cd build; cmake .. -DCMAKE_C_COMPILER=$CC -DENABLE_PROGRAMS=OFF", path)
Christian Dietrich's avatar
Christian Dietrich committed
68
69
70
71
72
        elif self.project_name() in ('lua',):
            # This is an ugly hack to make it possible to override the
            # CC variable from the outsite.
            with open("%s/makefile" % path) as fd:
                content = fd.readlines()
73
74
            content += "\nCC=%s\n" % (os.environ["CC"])

Christian Dietrich's avatar
Christian Dietrich committed
75
76
77
78
79
80
            with open("%s/makefile" % path, "w") as fd:
                fd.write("".join(content))

        else:
            raise RuntimeError("Not a valid project")

81
    def call_reconfigure(self, path):
82
        if self.project_name() in ('lua','mbedtls'):
83
84
            self.call_configure(path)
        if self.project_name() in ('cpython',):
85
            shell("cd %s; mkdir -p build/Modules; cp -u Modules/Setup.dist build/Modules/Setup", path)
86
            shell_failok("cd %s/build; make config.status;", path)
87
88
89
        if self.project_name() == "postgresql":
            shell_failok("cd %s/build; make config.status", path)
        if self.project_name() == "bash":
90
            shell_failok("cd %s; make config.status", path)
91

92

Christian Dietrich's avatar
Christian Dietrich committed
93
    def call_make(self, path):
94
        if self.project_name() in ("mbedtls", "cpython", "postgresql"):
95
            return shell("cd %s/build; make -k -j %s", path, str(self.jobs.value))
96
        else:
97
            return shell("cd %s; make -k -j %s", path, str(self.jobs.value))
Christian Dietrich's avatar
Christian Dietrich committed
98

99
100
101
102
103
104
105
106
107
108
109
    def ccache_hits(self):
        (lines, _) = shell("ccache -s")
        ccache_hits = 0
        ccache_misses = 0
        for line in lines:
            if "cache hit" in line and "rate" not in line:
                ccache_hits += int(line[line.index(")")+1:].strip())
            if "cache miss" in line:
                ccache_misses += int(line[line.index("miss")+4:].strip())
        return ccache_hits, ccache_misses

Christian Dietrich's avatar
Christian Dietrich committed
110
    def rebuild(self, path, info, fail_ok=False):
111
112
113
114
115
116
117
        if "ccache" in self.mode.value:
            old_ccache_hits, old_ccache_misses = self.ccache_hits()

        if "clang-hash" in self.mode.value:
            hash_log = tempfile.NamedTemporaryFile()
            os.environ["CLANG_HASH_LOGFILE"] = hash_log.name

Christian Dietrich's avatar
Christian Dietrich committed
118
119
120
121
122
123
124
125
126
127
128
129
130
131
        # Recompile!
        start_time = time.time()
        try:
            ret = self.call_make(path)
        except CommandFailed as e:
            if not fail_ok:
                raise
            else:
                info['failed'] = True
                ret = ("", 1)
        end_time = time.time()

        build_time = int((end_time - start_time) * 1e9)
        info['build-time'] = build_time
132
        info['build-log'] = ret[0]
133
134
135
136
137
138
139
140
141

        # Record Cache misses and hits
        if "ccache" in self.mode.value:
            ccache_hits, ccache_misses = self.ccache_hits()
            info['ccache-hits'] = ccache_hits - old_ccache_hits
            info['ccache-misses'] = ccache_misses - old_ccache_misses

        if "clang-hash" in self.mode.value:
            log = hash_log.read()
142
            #info['clang-hash-log'] = log
143
144
            info['clang-hash-hits'] = log.count("H")
            info['clang-hash-misses'] = log.count("M")
145
            hash_log.close()
146

Christian Dietrich's avatar
Christian Dietrich committed
147
148
149
150
        logging.info("Rebuild done[%s]: %s s; failed=%s",
                     info.get("filename") or info.get("commit"),
                     build_time / 1e9,
                     info.get("failed", False))
151

Christian Dietrich's avatar
Christian Dietrich committed
152
        return info