lib.py 5.38 KB
Newer Older
Christian Dietrich's avatar
Christian Dietrich committed
1
2
3
import os
import fnmatch
import time
4
from versuchung.execute import shell, CommandFailed, shell_failok
Christian Dietrich's avatar
Christian Dietrich committed
5
import logging
6
import tempfile
Christian Dietrich's avatar
Christian Dietrich committed
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

def read_hash_directory(hash_dir, remove_keys = []):
    """Read in all records from a hash dir
    """
    ret = []
    for root, dirnames, filenames in os.walk(hash_dir):
        for filename in fnmatch.filter(filenames, '*.info'):
            with open(os.path.join(root, filename)) as fd:
                data = "[%s]" % (",".join(fd.readlines()))
                data = eval(data)
                for record in data:
                    for key in remove_keys:
                        del record[key]
                ret.extend(data)
                print len(ret)
    return ret


class ClangHashHelper:
    def project_name(self):
        return os.path.basename(self.metadata['project-clone-url'])

    def setup_compiler_paths(self, clang_path):
        if "ccache" in self.mode.value:
            cache_dir = os.path.join(self.tmp_directory.path, "ccache")
            os.mkdir(cache_dir)
            os.environ["CCACHE_DIR"] = cache_dir
34
35
            os.environ["CCACHE_LOGFILE"] = "/tmp/lua.ccache.log"

Christian Dietrich's avatar
Christian Dietrich committed
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

        if self.mode.value == "normal":
            CC = os.path.join(clang_path, "build/wrappers/clang-normal")
        elif self.mode.value == "clang-hash":
            CC = os.path.join(clang_path, "build/wrappers/clang-hash-stop")
        elif self.mode.value == "ccache-clang-hash":
            CC = os.path.join(clang_path, "build/wrappers/clang-ccache-hash-stop")
        elif self.mode.value == "ccache":
            CC = os.path.join(clang_path, "build/wrappers/clang-ccache")
        else:
            raise RuntimeError("Not a valid mode")

        os.environ['CC'] = CC
        self.CC = CC

    def call_configure(self, path):
        if self.project_name() == "postgresql":
            shell("cd %s; ./configure --enable-depend", path)
54
        elif self.project_name() in ("musl", "bash", "samba"):
55
            shell_failok("cd %s; ./configure", path)
56
        elif self.project_name() in ("cpython",):
57
58
59
60
            shell("cd %s; mkdir -p build build/Modules;", path)
            shell("cd %s; cp -u Modules/Setup.dist build/Modules/Setup", path)
            shell("cd %s; cd build; ../configure", path)

Christian Dietrich's avatar
Christian Dietrich committed
61
        elif self.project_name() in ('mbedtls'):
62
            shell("cd %s; mkdir -p build; cd build; cmake .. -DCMAKE_C_COMPILER=$CC -DENABLE_PROGRAMS=OFF", path)
Christian Dietrich's avatar
Christian Dietrich committed
63
64
65
66
67
        elif self.project_name() in ('lua',):
            # This is an ugly hack to make it possible to override the
            # CC variable from the outsite.
            with open("%s/makefile" % path) as fd:
                content = fd.readlines()
68
69
            content += "\nCC=%s\n" % (os.environ["CC"])

Christian Dietrich's avatar
Christian Dietrich committed
70
71
72
73
74
75
76
            with open("%s/makefile" % path, "w") as fd:
                fd.write("".join(content))


        else:
            raise RuntimeError("Not a valid project")

77
    def call_reconfigure(self, path):
78
        if self.project_name() in ('lua','mbedtls'):
79
80
            self.call_configure(path)
        if self.project_name() in ('cpython',):
81
            shell("cd %s; mkdir -p build/Modules; cp -u Modules/Setup.dist build/Modules/Setup", path)
82
            shell_failok("cd %s/build; make config.status;", path)
83
84
        if self.project_name() in ('postgresql', 'bash'):
            shell_failok("cd %s; make config.status", path)
85

86

Christian Dietrich's avatar
Christian Dietrich committed
87
    def call_make(self, path):
88
        if self.project_name() in ("mbedtls", "cpython"):
89
90
91
            return shell("cd %s/build; make -j %s", path, str(self.jobs.value))
        else:
            return shell("cd %s; make -j %s", path, str(self.jobs.value))
Christian Dietrich's avatar
Christian Dietrich committed
92

93
94
95
96
97
98
99
100
101
102
103
    def ccache_hits(self):
        (lines, _) = shell("ccache -s")
        ccache_hits = 0
        ccache_misses = 0
        for line in lines:
            if "cache hit" in line and "rate" not in line:
                ccache_hits += int(line[line.index(")")+1:].strip())
            if "cache miss" in line:
                ccache_misses += int(line[line.index("miss")+4:].strip())
        return ccache_hits, ccache_misses

Christian Dietrich's avatar
Christian Dietrich committed
104
    def rebuild(self, path, info, fail_ok=False):
105
106
107
108
109
110
111
        if "ccache" in self.mode.value:
            old_ccache_hits, old_ccache_misses = self.ccache_hits()

        if "clang-hash" in self.mode.value:
            hash_log = tempfile.NamedTemporaryFile()
            os.environ["CLANG_HASH_LOGFILE"] = hash_log.name

Christian Dietrich's avatar
Christian Dietrich committed
112
113
114
115
116
117
118
119
120
121
122
123
124
125
        # Recompile!
        start_time = time.time()
        try:
            ret = self.call_make(path)
        except CommandFailed as e:
            if not fail_ok:
                raise
            else:
                info['failed'] = True
                ret = ("", 1)
        end_time = time.time()

        build_time = int((end_time - start_time) * 1e9)
        info['build-time'] = build_time
126
127
128
129
130
131
132
133
134
135
136

        # Record Cache misses and hits
        if "ccache" in self.mode.value:
            ccache_hits, ccache_misses = self.ccache_hits()
            info['ccache-hits'] = ccache_hits - old_ccache_hits
            info['ccache-misses'] = ccache_misses - old_ccache_misses

        if "clang-hash" in self.mode.value:
            log = hash_log.read()
            info['clang-hash-hits'] = log.count("H")
            info['clang-hash-misses'] = log.count("M")
137
            hash_log.close()
138

Christian Dietrich's avatar
Christian Dietrich committed
139
140
141
142
        logging.info("Rebuild done[%s]: %s s; failed=%s",
                     info.get("filename") or info.get("commit"),
                     build_time / 1e9,
                     info.get("failed", False))
143

Christian Dietrich's avatar
Christian Dietrich committed
144
        return info