Commit b8209948 authored by Florian Fischer's avatar Florian Fischer
Browse files

fix perf parsing and add new plot functions and use them

parent ede9aaa8
......@@ -56,7 +56,7 @@ def main():
if not args.nosum and not (args.runs < 1 and not args.load):
print("Summarizing", bench.name, "...")
bench.summary(args.summarydir)
bench.summary(args.summarydir or "")
if (args.runs > 0 or args.analyse) and hasattr(bench, "cleanup"):
print("Cleaning up", bench.name, "...")
......
......@@ -2,6 +2,8 @@ from collections import namedtuple
import copy
import csv
import itertools
import matplotlib.pyplot as plt
import numpy as np
import os
import pickle
import shutil
......@@ -14,7 +16,7 @@ class Benchmark (object):
defaults = {
"name" : "default_benchmark",
"description" : "This is the default benchmark description please add your own useful one.",
"measure_cmd" : "perf stat -x, -dd ",
"analyse_cmd" : "memusage -p {} -t ",
"cmd" : "true",
......@@ -32,7 +34,7 @@ class Benchmark (object):
self.args = {}
self.Perm = namedtuple("Perm", self.args.keys())
if not hasattr(self, "results"):
self.results = {}
self.results["args"] = self.args
......@@ -41,7 +43,7 @@ class Benchmark (object):
if not hasattr(self, "requirements"):
self.requirements = []
def save(self, path=None, verbose=False):
f = path if path else self.name + ".save"
if verbose:
......@@ -78,7 +80,7 @@ class Benchmark (object):
for r in self.requirements:
fpath, fname = os.path.split(r)
if fpath:
if not is_exe(r):
return False
......@@ -113,7 +115,7 @@ class Benchmark (object):
break
if is_fixed:
yield p
def analyse(self, verbose=False, nolibmemusage=True):
if not nolibmemusage and not shutil.which("memusage"):
......@@ -140,7 +142,7 @@ class Benchmark (object):
if "binary_suffix" in self.cmd:
perm["binary_suffix"] = ""
actual_cmd += self.cmd.format(**perm)
res = subprocess.run(actual_cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
......@@ -191,7 +193,7 @@ class Benchmark (object):
for perm in self.iterate_args():
i += 1
print(i, "of", n,"\r", end='')
actual_cmd = self.measure_cmd
perm_dict = perm._asdict()
......@@ -217,7 +219,7 @@ class Benchmark (object):
return False
result = {}
# Read VmHWM from status file. If our benchmark didn't fork
# the first occurance of VmHWM is from our benchmark
with open("status", "r") as f:
......@@ -226,13 +228,13 @@ class Benchmark (object):
result["VmHWM"] = l.split()[1]
break
os.remove("status")
if hasattr(self, "process_output"):
self.process_output(result, res.stdout, res.stderr,
tname, perm, verbose)
# Parse perf output if available
if self.measure_cmd != self.defaults["measure_cmd"]:
if self.measure_cmd == self.defaults["measure_cmd"]:
csvreader = csv.reader(res.stderr.splitlines(), delimiter=',')
for row in csvreader:
# Split of the user/kernel space info to be better portable
......@@ -251,3 +253,58 @@ class Benchmark (object):
print()
return True
def plot_single_arg(self, yval, ylabel="'y-label'", xlabel="'x-label'",
title="default title", filepostfix="", sumdir="", arg=""):
args = self.results["args"]
targets = self.results["targets"]
arg = arg or list(args.keys())[0]
for target in targets:
y_vals = []
for perm in self.iterate_args():
d = []
for m in self.results[target][perm]:
d.append(eval(yval.format(**m)))
y_vals.append(np.mean(d))
x_vals = list(range(1, len(y_vals) + 1))
plt.plot(x_vals, y_vals, marker='.', linestyle='-',
label=target, color=targets[target]["color"])
plt.legend()
plt.xticks(x_vals, args[arg])
plt.xlabel(eval(xlabel))
plt.ylabel(eval(ylabel))
plt.title(eval(title))
plt.savefig(os.path.join(sumdir, ".".join([self.name, filepostfix, "png"])))
plt.clf()
def plot_fixed_arg(self, yval, ylabel="'y-label'", xlabel="loose_arg",
title="default title", filepostfix="", sumdir="", fixed=[]):
args = self.results["args"]
targets = self.results["targets"]
for arg in fixed or args:
loose_arg = [a for a in args if a != arg][0]
for arg_value in args[arg]:
for target in targets:
y_vals = []
for perm in self.iterate_args_fixed({arg : arg_value}, args=args):
d = []
for m in self.results[target][perm]:
d.append(eval(yval.format(**m)))
y_vals.append(np.mean(d))
x_vals = list(range(1, len(y_vals) + 1))
plt.plot(x_vals, y_vals, marker='.', linestyle='-',
label=target, color=targets[target]["color"])
plt.legend()
plt.xticks(x_vals, args[loose_arg])
plt.xlabel(eval(xlabel))
plt.ylabel(eval(ylabel))
plt.title(eval(title))
plt.savefig(os.path.join(sumdir, ".".join([self.name, arg,
str(arg_value), filepostfix, "png"])))
plt.clf()
import matplotlib.pyplot as plt
import multiprocessing
import numpy as np
import os
import re
from benchmark import Benchmark
......@@ -32,35 +29,12 @@ class Benchmark_Larson( Benchmark ):
if res:
result["throughput"] = int(res.group("throughput"))
return
print(stdout)
print("no match")
def summary(self, sd=None):
# Speedup thrash
args = self.results["args"]
nthreads = args["threads"]
targets = self.results["targets"]
sd = sd or ""
for arg in args:
loose_arg = [a for a in args if a != arg][0]
for arg_value in args[arg]:
for target in targets:
y_vals = []
for perm in self.iterate_args_fixed({arg : arg_value}, args=args):
d = [m["throughput"] for m in self.results[target][perm]]
y_vals.append(np.mean(d))
x_vals = list(range(1, len(y_vals) + 1))
plt.plot(x_vals, y_vals, marker='.', linestyle='-',
label=target, color=targets[target]["color"])
plt.legend()
plt.xticks(x_vals, args[loose_arg])
plt.xlabel(loose_arg)
plt.ylabel("OPS/s")
plt.title("Larson: " + arg + " " + str(arg_value))
plt.savefig(os.path.join(sd, ".".join([self.name, arg, str(arg_value), "png"])))
plt.clf()
def summary(self, sumdir):
# Plot threads->throughput and maxsize->throughput
self.plot_fixed_args("{throughput}",
ylabel="'OPS/s'",
title = "'Larson: ' + arg + ' ' + str(arg_value)",
sumdir=sumdir)
larson = Benchmark_Larson()
......@@ -22,63 +22,22 @@ class Benchmark_Loop( Benchmark ):
self.requirements = ["build/bench_loop"]
super().__init__()
def summary(self, sd=None):
def summary(self, sumdir):
args = self.results["args"]
targets = self.results["targets"]
sd = sd or ""
# Speed
for arg in args:
loose_arg = [a for a in args if a != arg][0]
for arg_value in args[arg]:
for target in targets:
y_vals = []
for perm in self.iterate_args_fixed({arg : arg_value}, args=args):
d = []
for measure in self.results[target][perm]:
# nthreads/time = MOPS/s
for e in measure:
if "task-clock" in e:
d.append(perm.nthreads/float(measure[e]))
y_vals.append(np.mean(d))
x_vals = list(range(1, len(y_vals) + 1))
plt.plot(x_vals, y_vals, marker='.', linestyle='-',
label=target, color=targets[target]["color"])
plt.legend()
plt.xticks(x_vals, args[loose_arg])
plt.xlabel(loose_arg)
plt.ylabel("MOPS/s")
plt.title("Loop: " + arg + " " + str(arg_value))
plt.savefig(os.path.join(sd, ".".join([self.name, arg, str(arg_value), "png"])))
plt.clf()
self.plot_fixed_arg("perm.nthreads / float({task-clock})",
ylabel = '"MOPS/s"',
title = '"Loop: " + arg + " " + str(arg_value)',
filepostfix="tclock",
sumdir=sumdir)
# Memusage
for arg in args:
loose_arg = [a for a in args if a != arg][0]
for arg_value in args[arg]:
for target in targets:
y_vals = []
for perm in self.iterate_args_fixed({arg : arg_value}, args=args):
d = []
for measure in self.results[target][perm]:
d.append(int(measure["VmHWM"]))
y_vals.append(np.mean(d))
x_vals = list(range(1, len(y_vals) + 1))
plt.plot(x_vals, y_vals, marker='.', linestyle='-',
label=target, color=targets[target]["color"])
plt.legend()
plt.xticks(x_vals, args[loose_arg])
plt.xlabel(loose_arg)
plt.ylabel("VmHWM")
plt.title("Loop Memusage: " + arg + " " + str(arg_value))
plt.savefig(os.path.join(sd, ".".join([self.name, arg, str(arg_value), "mem", "png"])))
plt.clf()
self.plot_fixed_arg("int({VmHWM})",
ylabel='"VmHWM in kB"',
title= '"Loop Memusage: " + arg + " " + str(arg_value)',
filepostfix="memusage",
sumdir=sumdir)
loop = Benchmark_Loop()
......@@ -34,7 +34,8 @@ class Benchmark_MYSQL( Benchmark ):
if "hoard" in self.targets:
del(self.targets["hoard"])
self.args = {"nthreads" : range(1, multiprocessing.cpu_count() * 4 + 1, 2)}
# self.args = {"nthreads" : range(1, multiprocessing.cpu_count() * 4 + 1, 2)}
self.args = {"nthreads" : range(1, 2)}
self.cmd = cmd
self.measure_cmd = ""
......@@ -138,15 +139,15 @@ class Benchmark_MYSQL( Benchmark ):
with open("/proc/"+str(self.server.pid)+"/status", "r") as f:
for l in f.readlines():
if l.startswith("VmHWM:"):
result["rssmax"] = l.replace("VmHWM:", "").strip().split()[0]
result["rssmax"] = int(l.replace("VmHWM:", "").strip().split()[0])
break
def analyse(self, verbose=False):
def analyse(self, verbose=False, nolibmemusage=""):
import chattyparser
nthreads = [0] + list(self.args["nthreads"])
failed = False
self.results["hist"] = {}
os.environ["LD_PRELOAD"] = "build/chattymalloc.so"
runs = len(nthreads)
......@@ -160,7 +161,7 @@ class Benchmark_MYSQL( Benchmark ):
if t != 0:
target_cmd = self.cmd.format(nthreads=t).split(" ")
p = subprocess.run(target_cmd,
p = subrocess.run(target_cmd,
stderr=PIPE,
stdout=PIPE,
universal_newlines=True)
......@@ -174,7 +175,9 @@ class Benchmark_MYSQL( Benchmark ):
self.server.kill()
self.server.wait()
self.results["hist"][t] = self.parse_chattymalloc_data()
hist, calls, reqsize, top5reqsize = chattyparser.parse()
chattyparser.plot_hist_ascii(hist, calls, ".".join([self.name, str(t),
"memusage", "hist"]))
if failed:
print(self.server.stdout.read())
......@@ -182,91 +185,42 @@ class Benchmark_MYSQL( Benchmark ):
return False
print()
def summary(self, sd=None):
sd = sd or ""
def summary(self, sd):
targets = self.results["targets"]
args = self.results["args"]
nthreads = list(self.results["args"]["nthreads"])
# linear plot
for target in targets:
y_vals = []
for perm in self.iterate_args(args=args):
d = [int(m["transactions"]) for m in self.results[target][perm]]
y_vals.append(np.mean(d))
plt.plot(nthreads, y_vals, label=target, linestyle='-',
marker='.', color=targets[target]["color"])
plt.legend()
plt.xlabel("threads")
plt.ylabel("transactions")
plt.title("sysbench oltp read only")
plt.savefig(os.path.join(sd,self.name + ".l.ro.png"))
plt.clf()
self.plot_single_arg("{transactions}",
xlabel = '"threads"',
ylabel = '"transactions"',
title = '"sysbench oltp read only"',
filepostfix = "l.ro",
sumdir = sd)
# bar plot
for i, target in enumerate(targets):
y_vals = []
for perm in self.iterate_args(args=args):
for perm in self.iterate_args(args=self.results["args"]):
d = [int(m["transactions"]) for m in self.results[target][perm]]
y_vals.append(np.mean(d))
x_vals = [x-i/8 for x in range(1, len(nthreads) + 1)]
x_vals = [x-i/8 for x in range(1, len(y_vals) + 1)]
plt.bar(x_vals, y_vals, width=0.2, label=target, align="center",
color=targets[target]["color"])
plt.legend()
plt.xlabel("threads")
plt.xticks(range(1, len(nthreads) + 1), nthreads)
plt.xticks(range(1, len(y_vals) + 1), self.results["args"]["nthreads"])
plt.ylabel("transactions")
plt.title("sysbench oltp read only")
plt.savefig(os.path.join(sd, self.name + ".b.ro.png"))
plt.clf()
# Histogram
if "hist" in self.results:
for t, h in self.results["hist"].items():
self.plot_hist_ascii(h, os.path.join(sd, self.name+"."+str(t)+".hist"))
#Build up data
print(t)
d = []
num_discarded = 0
total = h["total"]
del(h["total"])
for size, freq in h.items():
if freq > 5 and size <= 10000:
d += [size] * freq
else:
num_discarded += freq
print("in hist")
print(len(d), max(d), min(d))
n, bins, patches = plt.hist(x=d, bins="auto")
plt.xlabel("allocation sizes in byte")
plt.ylabel("number of allocation")
plt.title("Histogram for " + str(t) + " threads\n"
+ str(num_discarded) + " not between 8 and 10000 byte")
plt.savefig(os.path.join(sd, self.name + ".hist." + str(t) + ".png"))
plt.clf()
h["total"] = total
# Memusage
for target in targets:
y_vals = []
for perm in self.iterate_args(args=args):
d = [int(m["rssmax"]) for m in self.results[target][perm]]
y_vals.append(np.mean(d))
plt.plot(nthreads, y_vals, marker='.', linestyle='-', label=target,
color=targets[target]["color"])
plt.legend()
plt.xlabel("threads")
plt.ylabel("kb")
plt.title("Memusage mysqld")
plt.savefig(os.path.join(sd, self.name + ".ro.mem.png"))
plt.clf()
self.plot_single_arg("{rssmax}",
xlabel = '"threads"',
ylabel = '"VmHWM in kB"',
title = '"Memusage sysbench oltp read only"',
filepostfix = "ro.mem",
sumdir = sd)
mysql = Benchmark_MYSQL()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment