Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
A
allocbench
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container registry
Model registry
Operate
Environments
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Florian Fischer
allocbench
Commits
31a14856
Commit
31a14856
authored
5 years ago
by
Florian Fischer
Browse files
Options
Downloads
Patches
Plain Diff
fmt chattyparser using yapf
parent
438c90a2
No related branches found
No related tags found
No related merge requests found
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
src/chattyparser.py
+68
-26
68 additions, 26 deletions
src/chattyparser.py
with
68 additions
and
26 deletions
src/chattyparser.py
+
68
−
26
View file @
31a14856
...
@@ -16,7 +16,6 @@
...
@@ -16,7 +16,6 @@
#
#
# You should have received a copy of the GNU General Public License
# You should have received a copy of the GNU General Public License
# along with allocbench. If not, see <http://www.gnu.org/licenses/>.
# along with allocbench. If not, see <http://www.gnu.org/licenses/>.
"""
Parser and Plotter for the traces produced by chattymalloc
"""
"""
Parser and Plotter for the traces produced by chattymalloc
"""
import
re
import
re
...
@@ -34,19 +33,35 @@ FREE_RE = re.compile(f"^f {PTR}$")
...
@@ -34,19 +33,35 @@ FREE_RE = re.compile(f"^f {PTR}$")
CALLOC_RE
=
re
.
compile
(
f
"
^c (?P<nmemb>
\\
d+)
{
SIZE
}
{
PTR
}
$
"
)
CALLOC_RE
=
re
.
compile
(
f
"
^c (?P<nmemb>
\\
d+)
{
SIZE
}
{
PTR
}
$
"
)
REALLOC_RE
=
re
.
compile
(
f
"
^r
{
PTR
}
{
SIZE
}
{
PTR
.
replace
(
'
ptr
'
,
'
nptr
'
)
}
$
"
)
REALLOC_RE
=
re
.
compile
(
f
"
^r
{
PTR
}
{
SIZE
}
{
PTR
.
replace
(
'
ptr
'
,
'
nptr
'
)
}
$
"
)
MEMALIGN_RE
=
re
.
compile
(
f
"
^ma
{
ALIGNMENT
}
{
SIZE
}
{
PTR
}
$
"
)
MEMALIGN_RE
=
re
.
compile
(
f
"
^ma
{
ALIGNMENT
}
{
SIZE
}
{
PTR
}
$
"
)
POSIX_MEMALIGN_RE
=
re
.
compile
(
f
"
^p_ma
{
PTR
}
{
ALIGNMENT
}
{
SIZE
}
(?P<ret>
\\
d+)$
"
)
POSIX_MEMALIGN_RE
=
re
.
compile
(
f
"
^p_ma
{
PTR
}
{
ALIGNMENT
}
{
SIZE
}
(?P<ret>
\\
d+)$
"
)
VALLOC_RE
=
re
.
compile
(
f
"
^v
{
SIZE
}
{
PTR
}
$
"
)
VALLOC_RE
=
re
.
compile
(
f
"
^v
{
SIZE
}
{
PTR
}
$
"
)
PVALLOC_RE
=
re
.
compile
(
f
"
^pv
{
SIZE
}
{
PTR
}
$
"
)
PVALLOC_RE
=
re
.
compile
(
f
"
^pv
{
SIZE
}
{
PTR
}
$
"
)
ALIGNED_ALLOC_RE
=
re
.
compile
(
f
"
^a_m
{
ALIGNMENT
}
{
SIZE
}
{
PTR
}
$
"
)
ALIGNED_ALLOC_RE
=
re
.
compile
(
f
"
^a_m
{
ALIGNMENT
}
{
SIZE
}
{
PTR
}
$
"
)
TRACE_REGEX
=
{
"
malloc
"
:
MALLOC_RE
,
"
free
"
:
FREE_RE
,
"
calloc
"
:
CALLOC_RE
,
TRACE_REGEX
=
{
"
realloc
"
:
REALLOC_RE
,
"
memalign
"
:
MEMALIGN_RE
,
"
malloc
"
:
MALLOC_RE
,
"
posix_memalign
"
:
POSIX_MEMALIGN_RE
,
"
valloc
"
:
VALLOC_RE
,
"
free
"
:
FREE_RE
,
"
pvalloc
"
:
PVALLOC_RE
,
"
aligned_alloc
"
:
ALIGNED_ALLOC_RE
}
"
calloc
"
:
CALLOC_RE
,
"
realloc
"
:
REALLOC_RE
,
"
memalign
"
:
MEMALIGN_RE
,
def
record_allocation
(
hist
,
total_size
,
allocations
,
ptr
,
size
,
coll_size
,
"
posix_memalign
"
:
POSIX_MEMALIGN_RE
,
req_size
,
nohist
,
optr
=
None
,
free
=
False
):
"
valloc
"
:
VALLOC_RE
,
"
pvalloc
"
:
PVALLOC_RE
,
"
aligned_alloc
"
:
ALIGNED_ALLOC_RE
}
def
record_allocation
(
hist
,
total_size
,
allocations
,
ptr
,
size
,
coll_size
,
req_size
,
nohist
,
optr
=
None
,
free
=
False
):
"""
add allocation to histogram or total requested memory
"""
add allocation to histogram or total requested memory
hist - dict mapping allocation sizes to their occurrence
hist - dict mapping allocation sizes to their occurrence
...
@@ -97,14 +112,26 @@ def record_allocation(hist, total_size, allocations, ptr, size, coll_size,
...
@@ -97,14 +112,26 @@ def record_allocation(hist, total_size, allocations, ptr, size, coll_size,
total_size
.
append
(
total_size
[
-
1
])
total_size
.
append
(
total_size
[
-
1
])
def
parse
(
path
=
"
chattymalloc.txt
"
,
coll_size
=
True
,
req_size
=
None
,
nohist
=
False
):
def
parse
(
path
=
"
chattymalloc.txt
"
,
coll_size
=
True
,
req_size
=
None
,
nohist
=
False
):
"""
parse a chattymalloc trace
"""
parse a chattymalloc trace
:returns: a histogram dict, a dict of occurred function calls, list of total requested memory
:returns: a histogram dict, a dict of occurred function calls, list of total requested memory
"""
"""
# count function calls
# count function calls
calls
=
{
"
malloc
"
:
0
,
"
free
"
:
0
,
"
calloc
"
:
0
,
"
realloc
"
:
0
,
"
memalign
"
:
0
,
calls
=
{
"
posix_memalign
"
:
0
,
"
valloc
"
:
0
,
"
pvalloc
"
:
0
,
"
aligned_alloc
"
:
0
}
"
malloc
"
:
0
,
"
free
"
:
0
,
"
calloc
"
:
0
,
"
realloc
"
:
0
,
"
memalign
"
:
0
,
"
posix_memalign
"
:
0
,
"
valloc
"
:
0
,
"
pvalloc
"
:
0
,
"
aligned_alloc
"
:
0
}
# Dictionary to track all live allocations
# Dictionary to track all live allocations
allocations
=
{}
allocations
=
{}
# List of total live memory per operation
# List of total live memory per operation
...
@@ -129,7 +156,8 @@ def parse(path="chattymalloc.txt", coll_size=True, req_size=None, nohist=False):
...
@@ -129,7 +156,8 @@ def parse(path="chattymalloc.txt", coll_size=True, req_size=None, nohist=False):
if
func
==
"
free
"
:
if
func
==
"
free
"
:
record
(
res
[
"
ptr
"
],
0
,
free
=
True
)
record
(
res
[
"
ptr
"
],
0
,
free
=
True
)
elif
func
==
"
calloc
"
:
elif
func
==
"
calloc
"
:
record
(
res
[
"
ptr
"
],
int
(
res
[
"
nmemb
"
])
*
int
(
res
[
"
size
"
]))
record
(
res
[
"
ptr
"
],
int
(
res
[
"
nmemb
"
])
*
int
(
res
[
"
size
"
]))
elif
func
==
"
realloc
"
:
elif
func
==
"
realloc
"
:
record
(
res
[
"
nptr
"
],
res
[
"
size
"
],
optr
=
res
[
"
ptr
"
])
record
(
res
[
"
nptr
"
],
res
[
"
size
"
],
optr
=
res
[
"
ptr
"
])
else
:
else
:
...
@@ -161,8 +189,11 @@ def plot_profile(total_sizes, trace_path, plot_path, sizes):
...
@@ -161,8 +189,11 @@ def plot_profile(total_sizes, trace_path, plot_path, sizes):
"""
Plot a memory profile of the total memory and the top 5 sizes
"""
"""
Plot a memory profile of the total memory and the top 5 sizes
"""
x_vals
=
range
(
0
,
len
(
total_sizes
))
x_vals
=
range
(
0
,
len
(
total_sizes
))
plt
.
plot
(
x_vals
,
total_sizes
/
1000
,
marker
=
''
,
plt
.
plot
(
x_vals
,
linestyle
=
'
-
'
,
label
=
"
Total requested
"
)
total_sizes
/
1000
,
marker
=
''
,
linestyle
=
'
-
'
,
label
=
"
Total requested
"
)
for
size
in
sizes
:
for
size
in
sizes
:
_
,
_
,
total_size
=
parse
(
path
=
trace_path
,
nohist
=
True
,
req_size
=
size
)
_
,
_
,
total_size
=
parse
(
path
=
trace_path
,
nohist
=
True
,
req_size
=
size
)
...
@@ -175,10 +206,11 @@ def plot_profile(total_sizes, trace_path, plot_path, sizes):
...
@@ -175,10 +206,11 @@ def plot_profile(total_sizes, trace_path, plot_path, sizes):
plt
.
savefig
(
plot_path
)
plt
.
savefig
(
plot_path
)
plt
.
clf
()
plt
.
clf
()
def
plot_hist_ascii
(
path
,
hist
,
calls
):
def
plot_hist_ascii
(
path
,
hist
,
calls
):
"""
Plot an ascii histogram
"""
"""
Plot an ascii histogram
"""
# make sure all sizes are stored as ints
# make sure all sizes are stored as ints
for
nonint_key
in
[
key
for
key
in
hist
.
keys
()
if
type
(
key
)
is
not
int
]:
for
nonint_key
in
[
key
for
key
in
hist
.
keys
()
if
not
isinstance
(
key
,
int
)
]:
hist
[
int
(
nonint_key
)]
=
hist
[
nonint_key
]
hist
[
int
(
nonint_key
)]
=
hist
[
nonint_key
]
del
hist
[
nonint_key
]
del
hist
[
nonint_key
]
...
@@ -187,7 +219,6 @@ def plot_hist_ascii(path, hist, calls):
...
@@ -187,7 +219,6 @@ def plot_hist_ascii(path, hist, calls):
size_class
=
size
//
16
size_class
=
size
//
16
bins
[
size_class
]
=
bins
.
get
(
size_class
,
0
)
+
hist
[
size
]
bins
[
size_class
]
=
bins
.
get
(
size_class
,
0
)
+
hist
[
size
]
with
open
(
path
,
"
w
"
)
as
hist_file
:
with
open
(
path
,
"
w
"
)
as
hist_file
:
print
(
"
Total function calls:
"
,
sum
(
calls
.
values
()),
file
=
hist_file
)
print
(
"
Total function calls:
"
,
sum
(
calls
.
values
()),
file
=
hist_file
)
for
func
,
func_calls
in
calls
.
items
():
for
func
,
func_calls
in
calls
.
items
():
...
@@ -199,14 +230,19 @@ def plot_hist_ascii(path, hist, calls):
...
@@ -199,14 +230,19 @@ def plot_hist_ascii(path, hist, calls):
top10
=
[
t
[
1
]
for
t
in
sorted
([(
n
,
s
)
for
s
,
n
in
hist
.
items
()])[
-
10
:]]
top10
=
[
t
[
1
]
for
t
in
sorted
([(
n
,
s
)
for
s
,
n
in
hist
.
items
()])[
-
10
:]]
top10_total
=
sum
([
hist
[
size
]
for
size
in
top10
])
top10_total
=
sum
([
hist
[
size
]
for
size
in
top10
])
print
(
f
"
Top 10 allocation sizes
{
(
top10_total
/
total
)
*
100
:
.
2
f
}
% of all allocations
"
,
file
=
hist_file
)
print
(
f
"
Top 10 allocation sizes
{
(
top10_total
/
total
)
*
100
:
.
2
f
}
% of all allocations
"
,
file
=
hist_file
)
for
i
,
size
in
enumerate
(
reversed
(
top10
)):
for
i
,
size
in
enumerate
(
reversed
(
top10
)):
print
(
f
"
{
i
+
1
}
.
{
size
}
B occurred
{
hist
[
size
]
}
times
"
,
file
=
hist_file
)
print
(
f
"
{
i
+
1
}
.
{
size
}
B occurred
{
hist
[
size
]
}
times
"
,
file
=
hist_file
)
print
(
file
=
hist_file
)
print
(
file
=
hist_file
)
for
i
in
[
64
,
1024
,
4096
]:
for
i
in
[
64
,
1024
,
4096
]:
allocations
=
sum
([
n
for
s
,
n
in
hist
.
items
()
if
s
<=
i
])
allocations
=
sum
([
n
for
s
,
n
in
hist
.
items
()
if
s
<=
i
])
print
(
f
"
allocations <=
{
i
}
:
{
allocations
}
{
(
allocations
/
total
)
*
100
:
.
2
f
}
%
"
,
file
=
hist_file
)
print
(
f
"
allocations <=
{
i
}
:
{
allocations
}
{
(
allocations
/
total
)
*
100
:
.
2
f
}
%
"
,
file
=
hist_file
)
print
(
file
=
hist_file
)
print
(
file
=
hist_file
)
print
(
"
Histogram of sizes:
"
,
file
=
hist_file
)
print
(
"
Histogram of sizes:
"
,
file
=
hist_file
)
...
@@ -216,9 +252,12 @@ def plot_hist_ascii(path, hist, calls):
...
@@ -216,9 +252,12 @@ def plot_hist_ascii(path, hist, calls):
for
current_bin
in
sbins
:
for
current_bin
in
sbins
:
perc
=
bins
[
current_bin
]
/
total
*
100
perc
=
bins
[
current_bin
]
/
total
*
100
binsize
=
f
"
{{:<
{
binmaxlength
}
}} - {{:>
{
binmaxlength
}
}}
"
binsize
=
f
"
{{:<
{
binmaxlength
}
}} - {{:>
{
binmaxlength
}
}}
"
print
(
binsize
.
format
(
current_bin
*
16
,
(
current_bin
+
1
)
*
16
-
1
),
end
=
"
"
,
file
=
hist_file
)
print
(
binsize
.
format
(
current_bin
*
16
,
(
current_bin
+
1
)
*
16
-
1
),
end
=
"
"
,
file
=
hist_file
)
amount
=
"
{:<
"
+
amountmaxlength
+
"
} {:.2f}% {}
"
amount
=
"
{:<
"
+
amountmaxlength
+
"
} {:.2f}% {}
"
print
(
amount
.
format
(
bins
[
current_bin
],
perc
,
'
*
'
*
int
(
perc
/
2
)),
file
=
hist_file
)
print
(
amount
.
format
(
bins
[
current_bin
],
perc
,
'
*
'
*
int
(
perc
/
2
)),
file
=
hist_file
)
if
__name__
==
"
__main__
"
:
if
__name__
==
"
__main__
"
:
...
@@ -226,12 +265,15 @@ if __name__ == "__main__":
...
@@ -226,12 +265,15 @@ if __name__ == "__main__":
# to keep chattyparser independent from allocbench
# to keep chattyparser independent from allocbench
if
"
--license
"
in
sys
.
argv
:
if
"
--license
"
in
sys
.
argv
:
print
(
"
Copyright (C) 2018-2019 Florian Fischer
"
)
print
(
"
Copyright (C) 2018-2019 Florian Fischer
"
)
print
(
"
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
"
)
print
(
"
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
"
)
sys
.
exit
(
0
)
sys
.
exit
(
0
)
if
len
(
sys
.
argv
)
!=
2
or
sys
.
argv
[
1
]
in
[
"
-h
"
,
"
--help
"
]:
if
len
(
sys
.
argv
)
!=
2
or
sys
.
argv
[
1
]
in
[
"
-h
"
,
"
--help
"
]:
print
(
"
chattyparser: parse chattymalloc output and
"
,
print
(
"
chattyparser: parse chattymalloc output and
"
,
"
create size histogram and memory profile
"
,
file
=
sys
.
stderr
)
"
create size histogram and memory profile
"
,
file
=
sys
.
stderr
)
print
(
f
"
Usage:
{
sys
.
argv
[
0
]
}
chattymalloc-file
"
,
file
=
sys
.
stderr
)
print
(
f
"
Usage:
{
sys
.
argv
[
0
]
}
chattymalloc-file
"
,
file
=
sys
.
stderr
)
sys
.
exit
(
1
)
sys
.
exit
(
1
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment