Skip to content
GitLab
Verified Commit c0f8f34a authored by Sebastian Endres's avatar Sebastian Endres
Browse files

Add compare script to compare results

parent 8ebed322
Hide whitespace changes
Inline Side-by-side
.gitignore
View file @ c0f8f34a
__pycache__
.ipynb_checkpoints
result.json
.vscode
compare_results.py 0 → 100755
View file @ c0f8f34a
#!/usr/bin/env python3
import argparse
from functools import cached_property
from pathlib import Path
import requests
from matplotlib import pyplot as plt
from termcolor import colored, cprint
from result_parser import Result
from utils import Subplot, existing_file_path_or_url
SAME_AVG_THRESH_PERC = 0.05
SAME_VAR_THRESH_PERC = 0.10
HIGH_AVG_DEVIATION_PERC = 0.2
HIGH_VAR_DEVIATION_PERC = 0.2
def parse_args():
"""Parse command line args."""
parser = argparse.ArgumentParser()
parser.add_argument(
"--plot",
action="store_true",
help="Plot a box plot.",
)
parser.add_argument(
"--output",
action="store",
default=None,
type=Path,
help="Store the plot into this file",
)
parser.add_argument(
"result1",
type=existing_file_path_or_url,
help="Result.",
)
parser.add_argument(
"result2",
type=existing_file_path_or_url,
help="Result.",
)
parser.add_argument(
"measurement",
type=str,
help="The measurement abbr to compare.",
)
return parser.parse_args()
def fetch_result(url: str) -> Result:
result = requests.get(url)
result.raise_for_status()
data = result.json()
return Result(url, data)
class CompareCli:
def __init__(
self,
result1: Result,
result2: Result,
measurement: str,
labels: tuple[str, str],
plot=False,
output=None,
):
self.result1 = result1
self.result2 = result2
self.measurement = measurement
self.labels = labels
self.plot = plot
self.output = output
self._unit = ""
@cached_property
def result_comparison(self):
"""
Compare 2 results.
"""
measurements1 = self.result1.get_all_measuements_of_type(self.measurement)
measurements2 = self.result2.get_all_measuements_of_type(self.measurement)
compare_result = {
"missing in 1": list[str](),
"missing in 2": list[str](),
"failed in 1": list[str](),
"failed in 2": list[str](),
"same avg and var": list[tuple[str, float, float, float]](),
"same avg different var": list[
tuple[str, tuple[float, float, float], tuple[float, float, float], bool]
](),
"different avg same var": list[
tuple[str, tuple[float, float, float], tuple[float, float, float], bool]
](),
"different avg and var": list[
tuple[str, tuple[float, float, float], tuple[float, float, float], bool]
](),
"tldr": "",
}
lookup1 = {
meas_result.combination: meas_result for meas_result in measurements1
}
num_missing_or_failed = 0
num_almost_equal = 0
num_different_meas_results = 0
avgs1 = list[float]()
avgs2 = list[float]()
for meas_result2 in measurements2:
combi: str = meas_result2.combination
meas_result1 = lookup1.pop(combi, None)
if not meas_result1 or (
meas_result1.result == "unsupported"
and meas_result2.result != "unsupported"
):
compare_result["missing in 1"].append(combi)
num_missing_or_failed += 1
elif meas_result1.result == "failed" and meas_result2.result == "succeeded":
compare_result["failed in 1"].append(combi)
num_missing_or_failed += 1
elif meas_result1.result == "succeeded" and meas_result2.result == "failed":
compare_result["failed in 2"].append(combi)
num_missing_or_failed += 1
elif (
meas_result1.result == "succeeded"
and meas_result2.result == "succeeded"
):
assert meas_result1.unit == meas_result2.unit
self._unit = meas_result1.unit
# compare avg
assert meas_result2.avg and meas_result1.avg
avg_dev = meas_result2.avg / meas_result1.avg - 1
same_avg = abs(avg_dev) < SAME_AVG_THRESH_PERC
high_avg_dev = abs(avg_dev) > HIGH_AVG_DEVIATION_PERC
# compare var
diff_var = meas_result1.var - meas_result2.var
var_dev = diff_var / meas_result1.avg
same_var = abs(var_dev) < SAME_VAR_THRESH_PERC
high_var_dev = abs(var_dev) > HIGH_VAR_DEVIATION_PERC
data: tuple[
str, tuple[float, float, float], tuple[float, float, float], bool
] = (
combi,
(meas_result1.avg, meas_result2.avg, avg_dev),
(meas_result1.var, meas_result2.var, var_dev),
high_avg_dev or high_var_dev,
)
if same_avg and same_var:
key = "same avg and var"
num_almost_equal += 1
elif same_avg and not same_var:
key = "same avg different var"
num_different_meas_results += 1
elif not same_avg and same_var:
key = "different avg same var"
num_different_meas_results += 1
else:
key = "different avg and var"
num_different_meas_results += 1
compare_result[key].append(data)
avgs1.append(meas_result1.avg)
avgs2.append(meas_result2.avg)
compare_result["missing in 2"].extend(
meas_result1.combination for meas_result1 in lookup1.values()
)
num_missing_or_failed += len(lookup1)
compare_result["tldr"] = "\n".join(
(
"There are "
+ colored(
f"{num_missing_or_failed or 'no'} missing or failing results",
color="red",
)
+ " in either of the two result files.",
colored(
f"{num_almost_equal} have (almost) equal results.",
color="green",
),
colored(
f"{num_different_meas_results} have different results.",
color="yellow",
),
colored(
f"The average of the average values of result1 is {sum(avgs1) / len(avgs1):.0f}.",
color="cyan",
),
colored(
f"The average of the average values of result2 is {sum(avgs2) / len(avgs2):.0f}.",
color="cyan",
),
)
)
return compare_result
def pretty_print_compare_result(self):
"""
Pretty print it.
"""
def error_helper(prop: str):
lst = self.result_comparison[prop]
cprint(f"{prop} ({len(lst)}):", color="red", attrs=["bold"])
for entry in lst:
cprint(f" - {entry}", color="red")
print()
def detailed_helper(prop: str, color: str):
lst = self.result_comparison[prop]
cprint(f"{prop} ({len(lst)}):", color=color, attrs=["bold"])
for entry in lst:
cprint(
f" - {entry[0]}\t ({entry[1][0]} / {entry[1][1]} ± {entry[2][0]} / {entry[2][1]} | deviation: {entry[1][2] * 100:.0f} % ± {entry[2][2] * 100:.0f} %)",
color=color,
attrs=["bold"] if entry[3] else None,
)
print()
error_helper("missing in 1")
error_helper("missing in 2")
error_helper("failed in 1")
error_helper("failed in 2")
detailed_helper("different avg and var", color="yellow")
detailed_helper("different avg same var", color="yellow")
detailed_helper("same avg different var", color="green")
detailed_helper("same avg and var", color="green")
cprint("TL;DR;", attrs=["bold"])
print()
print(self.result_comparison["tldr"])
def plot_deviation(self):
"""
Plot something.
"""
avgs1 = [
*(x[1][0] for x in self.result_comparison["same avg and var"]),
*(x[1][0] for x in self.result_comparison["same avg different var"]),
*(x[1][0] for x in self.result_comparison["different avg same var"]),
*(x[1][0] for x in self.result_comparison["different avg and var"]),
]
avgs2 = [
*(x[1][1] for x in self.result_comparison["same avg and var"]),
*(x[1][1] for x in self.result_comparison["same avg different var"]),
*(x[1][1] for x in self.result_comparison["different avg same var"]),
*(x[1][1] for x in self.result_comparison["different avg and var"]),
]
with Subplot() as (fig, ax):
ax.set_ylabel("Average Data Rate of Implementation Combination")
ax.set_title(f"Comparison of Results of Measurement {self.measurement}")
ax.yaxis.set_major_formatter(lambda val, _pos: f"{int(val)} {self._unit}")
ax.boxplot([avgs1, avgs2], labels=self.labels)
if self.output:
fig.savefig(self.output, bbox_inches="tight")
else:
plt.show()
def run(self):
"""docstring for main"""
self.pretty_print_compare_result()
if self.plot:
self.plot_deviation()
def main():
args = parse_args()
result1 = (
Result(args.result1)
if isinstance(args.result1, Path)
else fetch_result(args.result1.geturl())
)
result2 = (
Result(args.result2)
if isinstance(args.result2, Path)
else fetch_result(args.result2.geturl())
)
cli = CompareCli(
result1=result1,
result2=result2,
measurement=args.measurement,
labels=(
(
args.result1.name
if isinstance(args.result1, Path)
else args.result1.geturl()
),
(
args.result2.name
if isinstance(args.result2, Path)
else args.result2.geturl()
),
),
plot=args.plot,
output=args.output,
)
cli.run()
if __name__ == "__main__":
main()
poetry.lock
View file @ c0f8f34a
......@@ -68,6 +68,14 @@ typing-extensions = ">=3.7.4"
colorama = ["colorama (>=0.4.3)"]
d = ["aiohttp (>=3.3.2)", "aiohttp-cors"]
[[package]]
name = "certifi"
version = "2021.5.30"
description = "Python package for providing Mozilla's CA Bundle."
category = "main"
optional = false
python-versions = "*"
[[package]]
name = "cfgv"
version = "3.3.1"
......@@ -76,6 +84,17 @@ category = "dev"
optional = false
python-versions = ">=3.6.1"
[[package]]
name = "charset-normalizer"
version = "2.0.6"
description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
category = "main"
optional = false
python-versions = ">=3.5.0"
[package.extras]
unicode_backport = ["unicodedata2"]
[[package]]
name = "click"
version = "8.0.1"
......@@ -163,6 +182,14 @@ python-versions = ">=3.6.1"
[package.extras]
license = ["editdistance-s"]
[[package]]
name = "idna"
version = "3.2"
description = "Internationalized Domain Names in Applications (IDNA)"
category = "main"
optional = false
python-versions = ">=3.5"
[[package]]
name = "ipdb"
version = "0.13.9"
......@@ -552,6 +579,35 @@ category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "requests"
version = "2.26.0"
description = "Python HTTP for Humans."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
[package.dependencies]
certifi = ">=2017.4.17"
charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""}
idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""}
urllib3 = ">=1.21.1,<1.27"
[package.extras]
socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"]
[[package]]
name = "rope"
version = "0.20.1"
description = "a python refactoring library..."
category = "dev"
optional = false
python-versions = "*"
[package.extras]
dev = ["pytest", "pytest-timeout"]
[[package]]
name = "six"
version = "1.16.0"
......@@ -603,6 +659,19 @@ category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "urllib3"
version = "1.26.7"
description = "HTTP library with thread-safe connection pooling, file post, and more."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
[package.extras]
brotli = ["brotlipy (>=0.6.0)"]
secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[[package]]
name = "virtualenv"
version = "20.7.2"
......@@ -652,7 +721,7 @@ termcolor = ">=1.1.0,<2.0.0"
[metadata]
lock-version = "1.1"
python-versions = ">=3.9,<3.11"
content-hash = "842665f1d2e794d0551896c8ab3c016e4d9686d9d0056a5e0264427effe25eda"
content-hash = "007b6a19c94611d8194b0adbe382ad4a0496d6a76a9571ff4e57e1d6de818c83"
[metadata.files]
appdirs = [
......@@ -678,10 +747,18 @@ backcall = [
black = [
{file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"},
]
certifi = [
{file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"},
{file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"},
]
cfgv = [
{file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"},
{file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"},
]
charset-normalizer = [
{file = "charset-normalizer-2.0.6.tar.gz", hash = "sha256:5ec46d183433dcbd0ab716f2d7f29d8dee50505b3fdb40c6b985c7c4f5a3591f"},
{file = "charset_normalizer-2.0.6-py3-none-any.whl", hash = "sha256:5d209c0a931f215cee683b6445e2d77677e7e75e159f78def0db09d68fafcaa6"},
]
click = [
{file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"},
{file = "click-8.0.1.tar.gz", hash = "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a"},
......@@ -718,6 +795,10 @@ identify = [
{file = "identify-2.2.14-py2.py3-none-any.whl", hash = "sha256:113a76a6ba614d2a3dd408b3504446bcfac0370da5995aa6a17fd7c6dffde02d"},
{file = "identify-2.2.14.tar.gz", hash = "sha256:32f465f3c48083f345ad29a9df8419a4ce0674bf4a8c3245191d65c83634bdbf"},
]
idna = [
{file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"},
{file = "idna-3.2.tar.gz", hash = "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"},
]
ipdb = [
{file = "ipdb-0.13.9.tar.gz", hash = "sha256:951bd9a64731c444fd907a5ce268543020086a697f6be08f7cc2c9a752a278c5"},
]
......@@ -1166,6 +1247,13 @@ regex = [
{file = "regex-2021.8.28-cp39-cp39-win_amd64.whl", hash = "sha256:610b690b406653c84b7cb6091facb3033500ee81089867ee7d59e675f9ca2b73"},
{file = "regex-2021.8.28.tar.gz", hash = "sha256:f585cbbeecb35f35609edccb95efd95a3e35824cd7752b586503f7e6087303f1"},
]
requests = [
{file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"},
{file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"},
]
rope = [
{file = "rope-0.20.1.tar.gz", hash = "sha256:505a2f6b4ac7b18e0429be179f4d8712243a194da5c866b0731f9d91ce7590bb"},
]
six = [
{file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
{file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
......@@ -1218,6 +1306,10 @@ typing-extensions = [
{file = "typing_extensions-3.10.0.2-py3-none-any.whl", hash = "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"},
{file = "typing_extensions-3.10.0.2.tar.gz", hash = "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e"},
]
urllib3 = [
{file = "urllib3-1.26.7-py2.py3-none-any.whl", hash = "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"},
{file = "urllib3-1.26.7.tar.gz", hash = "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece"},
]
virtualenv = [
{file = "virtualenv-20.7.2-py2.py3-none-any.whl", hash = "sha256:e4670891b3a03eb071748c569a87cceaefbf643c5bac46d996c5a45c34aa0f06"},
{file = "virtualenv-20.7.2.tar.gz", hash = "sha256:9ef4e8ee4710826e98ff3075c9a4739e2cb1040de6a2a8d35db0055840dc96a0"},
......
pyproject.toml
View file @ c0f8f34a
......@@ -17,6 +17,7 @@ yaspin = "^2.1.0"
humanize = "^3.11.0"
Jinja2 = "^3.0.1"
prettytable = "^2.2.0"
requests = "^2.26.0"
[tool.poetry.dev-dependencies]
pylint = "^2.6.0"
......@@ -26,6 +27,7 @@ data-science-types = "^0.2.22"
pre-commit = "^2.9.3"
ipython = "^7.27.0"
ipdb = "^0.13.9"
rope = "^0.20.1"
[build-system]
requires = ["poetry-core>=1.0.0"]
......
utils.py
View file @ c0f8f34a
......@@ -6,9 +6,11 @@ import os
import statistics
import sys
import typing
from dataclasses import dataclass
from pathlib import Path
from typing import Callable, NamedTuple, Optional, TypeVar, Union
from dataclasses import dataclass
from urllib.parse import ParseResult as URL
from urllib.parse import urlparse
import termcolor
from matplotlib import pyplot as plt
......@@ -242,6 +244,24 @@ def existing_file_path(value: str, allow_none=False) -> Optional[Path]:
return path
def existing_file_path_or_url(value: str) -> Union[Path, URL]:
"""
An existing file or an URL for argparse.
"""
try:
path = existing_file_path(value)
assert path
return path
except argparse.ArgumentTypeError as err:
url = urlparse(value)
if not url.scheme or not url.netloc:
raise argparse.ArgumentTypeError(
"Argument seems neither to be a existing file nor an URL."
) from err
return url
@dataclass
class TraceTriple:
left_pcap_path: Path
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment