Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Christian Dietrich
clang-hash
Commits
9defe951
Commit
9defe951
authored
Jan 31, 2017
by
Ludwig Fueracker
Browse files
Merge branch 'master' of gitlab.cs.fau.de:stettberger/clang-hash
parents
1e89fc34
52945b9b
Pipeline
#3074
passed with stage
in 0 seconds
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
experiments/analyze_results.py
View file @
9defe951
...
...
@@ -88,12 +88,18 @@ class AnalyzeResults(Experiment):
################################################################
x
=
sorted
(
self
.
historical
,
key
=
lambda
x
:
x
.
project_name
())
hist
=
defaultdict
(
lambda
:
0
)
method_stats
=
defaultdict
(
lambda
:
defaultdict
(
lambda
:
0
))
for
(
project
,
results
)
in
groupby
(
x
,
key
=
lambda
x
:
x
.
project_name
()):
times
=
defaultdict
(
lambda
:
dict
())
for
result
in
sorted
(
results
,
key
=
lambda
x
:
x
.
variant_name
()):
key
=
[
result
.
variant_name
(),
'historical'
]
records
=
eval
(
result
.
stats
.
value
)
# How Many Hits were produced by clang-hash/ccache
stats
=
defaultdict
(
lambda
:
0
)
build_times
=
[]
failed
=
0
for
build
in
records
[
'builds'
]:
...
...
@@ -105,15 +111,29 @@ class AnalyzeResults(Experiment):
times
[
build
[
'commit'
]][
result
.
metadata
[
'mode'
]]
=
t
hist
[
int
(
t
)]
+=
1
stats
[
'misses/clang-hash'
]
+=
build
.
get
(
'clang-hash-misses'
,
0
)
stats
[
'hits/clang-hash'
]
+=
build
.
get
(
'clang-hash-hits'
,
0
)
stats
[
'misses/ccache'
]
+=
build
.
get
(
'ccache-misses'
,
0
)
stats
[
'hits/ccache'
]
+=
build
.
get
(
'ccache-hits'
,
0
)
stats
[
'hits'
]
+=
build
.
get
(
'ccache-hits'
,
0
)
\
+
build
.
get
(
'clang-hash-hits'
,
0
)
stats
[
'misses'
]
+=
build
.
get
(
'ccache-misses'
,
0
)
\
+
build
.
get
(
'clang-hash-misses'
,
0
)
if
result
.
mode
.
value
==
"ccache-clang-hash"
:
stats
[
"misses"
]
-=
build
.
get
(
'clang-hash-hits'
,
0
)
# Over all builds of an experiment
def
seq
(
key
,
seq
):
self
.
save
(
key
+
[
"sum"
],
sum
(
seq
))
self
.
save
(
key
+
[
"count"
],
len
(
seq
))
self
.
save
(
key
+
[
"avg"
],
np
.
average
(
seq
))
key
=
[
result
.
variant_name
(),
'historical'
]
self
.
save
(
key
+
[
"failed"
],
failed
)
seq
(
key
,
build_times
)
for
k
in
stats
:
self
.
save
(
key
+
[
k
],
stats
[
k
])
# Aggregate hits and misses per method
method_stats
[
result
.
metadata
[
"mode"
]][
k
]
+=
stats
[
k
]
try
:
x
=
sorted
(
times
,
key
=
lambda
x
:
times
[
x
][
'clang-hash'
]
/
times
[
x
][
'normal'
])
...
...
@@ -128,6 +148,10 @@ class AnalyzeResults(Experiment):
except
:
pass
# Output method statistics
for
method
in
method_stats
:
for
k
in
method_stats
[
method
]:
self
.
save
([
method
,
"historical"
,
k
],
method_stats
[
method
][
k
])
if
__name__
==
"__main__"
:
experiment
=
AnalyzeResults
()
...
...
experiments/historical_build.py
View file @
9defe951
...
...
@@ -90,10 +90,6 @@ class HistoricalCompilation(Experiment, ClangHashHelper):
"commit-hash"
:
self
.
metadata
[
"project-hash"
],
'builds'
:
[]}
hash_log
=
self
.
tmp_directory
.
new_file
(
"clang-hash.log"
)
if
"clang-hash"
in
self
.
mode
.
value
:
os
.
environ
[
"CLANG_HASH_LOGFILE"
]
=
hash_log
.
path
with
self
.
project
as
src_path
:
(
commits
,
_
)
=
shell
(
"cd %s; git log --no-merges --oneline --topo-order --format='%%H %%P %%s'"
,
src_path
)
# [0] is hash. [1] is parent, [2] rest
...
...
@@ -151,13 +147,6 @@ class HistoricalCompilation(Experiment, ClangHashHelper):
with
open
(
self
.
stats
.
path
,
"w+"
)
as
fd
:
fd
.
write
(
repr
(
self
.
build_info
))
if
"ccache"
in
self
.
mode
.
value
:
shell
(
"ccache -s > %s"
,
self
.
ccache_stats
.
path
)
# Copy Log to the result directory
with
open
(
hash_log
.
path
)
as
fd
:
self
.
clang_hash_stats
.
value
=
fd
.
read
()
def
variant_name
(
self
):
return
"%s-%s"
%
(
self
.
project_name
(),
self
.
metadata
[
'mode'
])
...
...
experiments/lib.py
View file @
9defe951
...
...
@@ -3,6 +3,7 @@ import fnmatch
import
time
from
versuchung.execute
import
shell
,
CommandFailed
,
shell_failok
import
logging
import
tempfile
def
read_hash_directory
(
hash_dir
,
remove_keys
=
[]):
"""Read in all records from a hash dir
...
...
@@ -79,7 +80,8 @@ class ClangHashHelper:
if
self
.
project_name
()
in
(
'cpython'
,):
shell
(
"cd %s; mkdir -p build/Modules; cp -u Modules/Setup.dist build/Modules/Setup"
,
path
)
shell_failok
(
"cd %s/build; make config.status;"
,
path
)
if
self
.
project_name
()
in
(
'postgresql'
,
'bash'
):
shell_failok
(
"cd %s; make config.status"
,
path
)
def
call_make
(
self
,
path
):
...
...
@@ -88,7 +90,25 @@ class ClangHashHelper:
else
:
return
shell
(
"cd %s; make -j %s"
,
path
,
str
(
self
.
jobs
.
value
))
def
ccache_hits
(
self
):
(
lines
,
_
)
=
shell
(
"ccache -s"
)
ccache_hits
=
0
ccache_misses
=
0
for
line
in
lines
:
if
"cache hit"
in
line
and
"rate"
not
in
line
:
ccache_hits
+=
int
(
line
[
line
.
index
(
")"
)
+
1
:].
strip
())
if
"cache miss"
in
line
:
ccache_misses
+=
int
(
line
[
line
.
index
(
"miss"
)
+
4
:].
strip
())
return
ccache_hits
,
ccache_misses
def
rebuild
(
self
,
path
,
info
,
fail_ok
=
False
):
if
"ccache"
in
self
.
mode
.
value
:
old_ccache_hits
,
old_ccache_misses
=
self
.
ccache_hits
()
if
"clang-hash"
in
self
.
mode
.
value
:
hash_log
=
tempfile
.
NamedTemporaryFile
()
os
.
environ
[
"CLANG_HASH_LOGFILE"
]
=
hash_log
.
name
# Recompile!
start_time
=
time
.
time
()
try
:
...
...
@@ -101,9 +121,20 @@ class ClangHashHelper:
ret
=
(
""
,
1
)
end_time
=
time
.
time
()
# Account only nano seconds, everywhere
build_time
=
int
((
end_time
-
start_time
)
*
1e9
)
info
[
'build-time'
]
=
build_time
# Record Cache misses and hits
if
"ccache"
in
self
.
mode
.
value
:
ccache_hits
,
ccache_misses
=
self
.
ccache_hits
()
info
[
'ccache-hits'
]
=
ccache_hits
-
old_ccache_hits
info
[
'ccache-misses'
]
=
ccache_misses
-
old_ccache_misses
if
"clang-hash"
in
self
.
mode
.
value
:
log
=
hash_log
.
read
()
info
[
'clang-hash-hits'
]
=
log
.
count
(
"H"
)
info
[
'clang-hash-misses'
]
=
log
.
count
(
"M"
)
logging
.
info
(
"Rebuild done[%s]: %s s; failed=%s"
,
info
.
get
(
"filename"
)
or
info
.
get
(
"commit"
),
build_time
/
1e9
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment