Verified Commit 07579c58 authored by Sebastian Endres's avatar Sebastian Endres
Browse files

Plot packet size plots, add tango colors, enhance spinner

parent 52f8b400
......@@ -2,9 +2,6 @@
"""Plot time packet-number plots and more."""
# TODO
# - plot packet sizes
import argparse
import sys
import typing
......@@ -13,16 +10,19 @@ from pathlib import Path
from typing import Optional, Union
import numpy as np
from humanize.filesize import naturalsize
from matplotlib import colors
from matplotlib import pyplot as plt
from termcolor import colored, cprint
from tango_colors import Tango
from tracer import ParsingError, Trace
from utils import (
Statistics,
Subplot,
YaspinWrapper,
create_relpath,
existing_file_path,
format_file_size,
map2d,
map3d,
)
......@@ -39,12 +39,16 @@ class PlotMode(Enum):
OFFSET_NUMBER = "offset-number"
PACKET_NUMBER = "packet-number"
FILE_SIZE = "file-size"
PACKET_SIZE = "packet-size"
SIZE_HIST = "size-hist"
DEFAULT_TITLES = {
PlotMode.OFFSET_NUMBER: "Time vs. Offset-Number",
PlotMode.PACKET_NUMBER: "Time vs. Packet-Number",
PlotMode.FILE_SIZE: "Time vs. Transmitted File Size",
PlotMode.PACKET_SIZE: "Time vs. Packet Size",
PlotMode.SIZE_HIST: "Size Histogram",
}
......@@ -145,6 +149,7 @@ class PlotCli:
annotate=annotate,
mode=mode,
)
self._colors = Tango(model="HTML")
if not keylog_files:
keylog_files = [None] * len(pcap_files)
......@@ -186,7 +191,7 @@ class PlotCli:
label_side="right",
):
"""Annotate with vline."""
ax.axvline(x=x, color="red", alpha=0.5) # , linestyle="--"
ax.axvline(x=x, color=self._colors.ScarletRed, alpha=0.5) # , linestyle="--"
xoffset = 10 if label_side == "right" else -20
ax.annotate(
text,
......@@ -200,7 +205,7 @@ class PlotCli:
"alpha": 0.5,
},
rotation=90,
color="red",
color=self._colors.ScarletRed,
alpha=0.5,
)
......@@ -220,10 +225,10 @@ class PlotCli:
textcoords=ax.transData,
arrowprops=dict(
arrowstyle="<->",
color="red",
color=self._colors.ScarletRed,
alpha=0.5,
),
color="red",
color=self._colors.ScarletRed,
alpha=0.5,
)
ax.annotate(
......@@ -233,10 +238,10 @@ class PlotCli:
textcoords=ax.transData,
arrowprops=dict(
arrowstyle="|-|",
color="red",
color=self._colors.ScarletRed,
alpha=0.5,
),
color="red",
color=self._colors.ScarletRed,
alpha=0.5,
)
ax.text(
......@@ -245,7 +250,7 @@ class PlotCli:
text,
ha="center",
va="center",
color="red",
color=self._colors.ScarletRed,
alpha=0.5,
bbox=dict(fc="white", ec="none"),
)
......@@ -300,7 +305,9 @@ class PlotCli:
ax.set_ylabel("Offset")
assert self.title
ax.set_title(self.title)
ax.yaxis.set_major_formatter(lambda val, _pos: format_file_size(val))
ax.yaxis.set_major_formatter(
lambda val, _pos: naturalsize(val, binary=True)
)
with YaspinWrapper(
debug=self.debug, text="processing...", color="cyan"
......@@ -360,8 +367,6 @@ class PlotCli:
ax.set_xlim(left=min(0, min_timestamp), right=max_timestamp)
ax.set_ylim(bottom=min(0, min_offset), top=max_offset)
spinner.ok("✔")
with YaspinWrapper(
debug=self.debug, text="plotting...", color="cyan"
) as spinner:
......@@ -375,7 +380,7 @@ class PlotCli:
trace_offsets,
marker="o",
linestyle="",
color="#CCC",
color=self._colors.aluminium4,
)
for (
......@@ -394,7 +399,7 @@ class PlotCli:
(*trace_first_offsets, *trace_retrans_offsets),
marker="o",
linestyle="",
color="#CCC",
color=self._colors.aluminium4,
)
# plot main trace (request and response separated)
......@@ -404,21 +409,21 @@ class PlotCli:
request_offsets[0],
marker="o",
linestyle="",
color="#73d216",
color=self._colors.Chameleon,
)
ax.plot(
response_first_timestamps[0],
response_first_offsets[0],
marker="o",
linestyle="",
color="#3465A4",
color=self._colors.SkyBlue,
)
ax.plot(
response_retrans_timestamps[0],
response_retrans_offsets[0],
marker="o",
linestyle="",
color="#b60000",
color=self._colors.ScarletRed,
)
self._annotate_time_plot(ax, height=max_offset)
......@@ -462,8 +467,6 @@ class PlotCli:
ax.set_xlim(left=min(0, min_timestamp), right=max_timestamp)
ax.set_ylim(bottom=min(0, min_packet_number), top=max_packet_number)
spinner.ok("✔")
with YaspinWrapper(
debug=self.debug, text="plotting...", color="cyan"
) as spinner:
......@@ -477,7 +480,7 @@ class PlotCli:
trace_packet_numbers,
marker="o",
linestyle="",
color="#CCC",
color=self._colors.aluminium4,
)
for trace_timestamps, trace_packet_numbers in zip(
......@@ -488,7 +491,7 @@ class PlotCli:
trace_packet_numbers,
marker="o",
linestyle="",
color="#CCC",
color=self._colors.aluminium4,
)
# plot main trace (request and response separated)
......@@ -498,14 +501,14 @@ class PlotCli:
request_packet_numbers[0],
marker="o",
linestyle="",
color="#729fcf",
color=self._colors.skyblue1,
)
ax.plot(
response_timestamps[0],
response_packet_numbers[0],
marker="o",
linestyle="",
color="#3465A4",
color=self._colors.plum1,
)
self._annotate_time_plot(ax, height=max_packet_number)
......@@ -519,7 +522,9 @@ class PlotCli:
ax.set_ylabel("Transmitted File Size")
assert self.title
ax.set_title(self.title)
ax.yaxis.set_major_formatter(lambda val, _pos: format_file_size(val))
ax.yaxis.set_major_formatter(
lambda val, _pos: naturalsize(val, binary=True)
)
with YaspinWrapper(
debug=self.debug, text="processing...", color="cyan"
......@@ -552,8 +557,6 @@ class PlotCli:
ax.set_ylim(bottom=min(0, min_file_size), top=max_file_size)
ax.set_yticks(np.arange(0, max_file_size * 1.1, 1024 * 1024))
spinner.ok("✔")
with YaspinWrapper(
debug=self.debug, text="plotting...", color="cyan"
) as spinner:
......@@ -567,7 +570,7 @@ class PlotCli:
trace_file_sizes,
marker="o",
linestyle="",
color="#CCC",
color=self._colors.aluminium4,
)
# plot main trace
......@@ -577,12 +580,239 @@ class PlotCli:
accumulated_transmitted_file_size[0],
marker="o",
linestyle="",
color="#3465A4",
color=self._colors.SkyBlue,
)
self._annotate_time_plot(ax, height=max_file_size)
self._save(output_file, spinner)
def _process_packet_sizes(self):
"""Helper function."""
# only response
# only the first trace
packet_sizes = list[int]()
overhead_sizes = list[int]()
stream_data_sizes = list[int]()
timestamps = list[float]()
max_timestamp = float("-inf")
min_timestamp = float("inf")
for frame in self.traces[0].response_stream_frames:
packet_size = int(frame.packet_length)
packet_sizes.append(packet_size)
stream_data_size = self.traces[0].get_stream_length(frame)
stream_data_sizes.append(stream_data_size)
overhead_sizes.append(packet_size - stream_data_size)
timestamps.append(frame.norm_time)
max_timestamp = max(max_timestamp, frame.norm_time)
min_timestamp = max(min_timestamp, frame.norm_time)
packet_stats = Statistics.calc(packet_sizes)
overhead_stats = Statistics.calc(overhead_sizes)
stream_data_stats = Statistics.calc(stream_data_sizes)
return (
packet_sizes,
overhead_sizes,
stream_data_sizes,
timestamps,
max_timestamp,
min_timestamp,
packet_stats,
overhead_stats,
stream_data_stats,
)
def plot_packet_size(self, output_file: Optional[Path]):
"""Plot the packet size diagram."""
with Subplot() as (_fig, ax):
ax.grid(True)
ax.set_xlabel("Time (s)")
ax.set_ylabel("Packet Size")
assert self.title
ax.set_title(self.title)
ax.yaxis.set_major_formatter(
lambda val, _pos: naturalsize(val, binary=True)
)
with YaspinWrapper(
debug=self.debug, text="processing...", color="cyan"
) as spinner:
(
_packet_sizes,
overhead_sizes,
stream_data_sizes,
timestamps,
max_timestamp,
min_timestamp,
packet_stats,
overhead_stats,
stream_data_stats,
) = self._process_packet_sizes()
ax.set_xlim(left=min(0, min_timestamp), right=max_timestamp)
# ax.set_ylim(bottom=0, top=packet_stats.max)
# ax.set_yticks(np.arange(0, packet_stats.max * 1.1, 1024))
with YaspinWrapper(
debug=self.debug, text="plotting...", color="cyan"
) as spinner:
# no shadow traces here
# plot main trace
ax.stackplot(
timestamps,
(
stream_data_sizes,
overhead_sizes,
),
colors=(
self._colors.skyblue1,
self._colors.plum1,
),
edgecolor=(
self._colors.skyblue3,
self._colors.plum3,
),
labels=(
"Stream Data Size",
"Overhead Size",
),
baseline="zero",
step="pre",
)
ax.legend(loc="upper left")
ax.text(
0.95,
0.05,
"\n".join(
(
"Packet Statistics",
packet_stats.mpl_label_short(naturalsize),
"\n Stream Data Statistics",
stream_data_stats.mpl_label_short(naturalsize),
"\n Overhead Statistics",
overhead_stats.mpl_label_short(naturalsize),
)
),
transform=ax.transAxes,
fontsize=12,
verticalalignment="bottom",
horizontalalignment="right",
bbox=dict(
boxstyle="round",
facecolor=self._colors.chocolate1,
edgecolor=self._colors.chocolate3,
alpha=0.5,
),
)
self._annotate_time_plot(ax, height=packet_stats.max)
self._save(output_file, spinner)
def plot_packet_hist(self, output_file: Optional[Path]):
"""Plot the packet size histogram."""
with Subplot(ncols=3) as (fig, axs):
assert self.title
for ax in axs:
ax.grid(True)
ax.set_xlabel("Size")
ax.set_ylabel("Amount of Packets")
ax.xaxis.set_major_formatter(
lambda val, _pos: naturalsize(val, binary=True)
)
with YaspinWrapper(
debug=self.debug, text="processing...", color="cyan"
) as spinner:
(
packet_sizes,
overhead_sizes,
stream_data_sizes,
_timestamps,
_max_timestamp,
_min_timestamp,
packet_stats,
overhead_stats,
stream_data_stats,
) = self._process_packet_sizes()
with YaspinWrapper(
debug=self.debug, text="plotting...", color="cyan"
) as spinner:
fig.suptitle(f"{self.title}\n{packet_stats.num} Packets")
n_bins = 100
axs[0].set_title("Overall Packet Size")
axs[0].hist(
packet_sizes,
bins=n_bins,
color=self._colors.Plum,
)
axs[0].text(
0.5,
0.5,
packet_stats.mpl_label(naturalsize),
transform=axs[0].transAxes,
fontsize=10,
verticalalignment="center",
horizontalalignment="center",
bbox=dict(
boxstyle="round",
facecolor=self._colors.chocolate1,
edgecolor=self._colors.chocolate3,
alpha=0.5,
),
)
axs[1].set_title("Overhead Size")
axs[1].hist(
overhead_sizes,
bins=n_bins,
color=self._colors.ScarletRed,
)
axs[1].text(
0.5,
0.5,
stream_data_stats.mpl_label(naturalsize),
transform=axs[1].transAxes,
fontsize=10,
verticalalignment="center",
horizontalalignment="center",
bbox=dict(
boxstyle="round",
facecolor=self._colors.chocolate1,
edgecolor=self._colors.chocolate3,
alpha=0.5,
),
)
axs[2].set_title("Stream Data Size")
axs[2].hist(
stream_data_sizes,
bins=n_bins,
color=self._colors.Chameleon,
)
axs[2].text(
0.5,
0.5,
overhead_stats.mpl_label(naturalsize),
transform=axs[2].transAxes,
fontsize=10,
verticalalignment="center",
horizontalalignment="center",
bbox=dict(
boxstyle="round",
facecolor=self._colors.chocolate1,
edgecolor=self._colors.chocolate3,
alpha=0.5,
),
)
self._save(output_file, spinner)
def _save(self, output_file: Optional[Path], spinner: YaspinWrapper):
"""Save or show the plot."""
......@@ -591,8 +821,10 @@ class PlotCli:
spinner.text = colored(
f"{create_relpath(output_file)} written.", color="green"
)
spinner.ok("✔")
else:
spinner.write(f"✔ {spinner.text}")
spinner.text = "Showing plot"
spinner.ok("✔")
plt.show()
def run(self):
......@@ -600,36 +832,38 @@ class PlotCli:
mapping = {
PlotMode.OFFSET_NUMBER: {
"callback": self.plot_offset_number,
"desc": "time vs. offset number",
},
PlotMode.PACKET_NUMBER: {
"callback": self.plot_packet_number,
"desc": "time vs. packet number",
},
PlotMode.FILE_SIZE: {
"callback": self.plot_file_size,
"desc": "time vs. transmitted file size",
},
PlotMode.PACKET_SIZE: {
"callback": self.plot_packet_size,
"single": True,
},
PlotMode.SIZE_HIST: {
"callback": self.plot_packet_hist,
"single": True,
},
}
# avoid lazy result parsing:
cfg = mapping[self.mode]
single: Optional[bool] = cfg.get("single")
callback = cfg["callback"]
desc = DEFAULT_TITLES[self.mode]
num_traces = 1 if single else len(self.traces)
for trace in self.traces:
assert (
trace.packets
), "Trace {trace} contains no filtered packets! Are the secrets injected?"
# avoid lazy result parsing:
if self.annotate:
# raise errors early
if single:
self.traces[0].parse()
else:
for trace in self.traces:
trace.parse()
cfg = mapping[self.mode]
callback = cfg["callback"]
desc = cfg["desc"]
cprint(
f"Plotting {len(self.traces)} traces into a {desc} plot...", color="cyan"
)
cprint(f"⚒ Plotting {num_traces} traces into a {desc} plot...", color="cyan")
callback(self.output_file)
......@@ -653,7 +887,7 @@ def main():
try:
cli.run()
except ParsingError as err:
sys.exit(colored(err, color="red"))
sys.exit(colored(str(err), color="red"))
if __name__ == "__main__":
......
......@@ -117,9 +117,20 @@ category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "humanize"
version = "3.11.0"
description = "Python humanize utilities"
category = "main"
optional = false
python-versions = ">=3.6"
[package.extras]
tests = ["freezegun", "pytest", "pytest-cov"]
[[package]]
name = "identify"
version = "2.2.13"
version = "2.2.14"
description = "File identification library for Python"
category = "dev"
optional = false
......@@ -452,7 +463,7 @@ termcolor = ">=1.1.0,<2.0.0"
[metadata]
lock-version = "1.1"
python-versions = ">=3.9,<3.11"
content-hash = "0962aaedaf0cd5f7b7fb4cb79fe0eb526e6adc9eddad896b18ff32e7686b8937"
content-hash = "3df3d2b86614dbf3530b64458e1fd4d8e7c8170d08f6f2f4f158227d6e4673b4"
[metadata.files]
appdirs = [
......@@ -498,9 +509,13 @@ filelock = [
{file = "filelock-3.0.12-py3-none-any.whl", hash = "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836"},
{file = "filelock-3.0.12.tar.gz", hash = "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59"},
]
humanize = [
{file = "humanize-3.11.0-py3-none-any.whl", hash = "sha256:06c79af7873473c47477840010ccb9b11b0d431f37f4a81b71edd653211936be"},
{file = "humanize-3.11.0.tar.gz", hash = "sha256:4160cdc63fcd0daac27d2e1e218a31bb396fc3fe5712d153675d89432a03778f"},
]
identify = [
{file = "identify-2.2.13-py2.py3-none-any.whl", hash = "sha256:7199679b5be13a6b40e6e19ea473e789b11b4e3b60986499b1f589ffb03c217c"},
{file = "identify-2.2.13.tar.gz", hash = "sha256:7bc6e829392bd017236531963d2d937d66fc27cadc643ac0aba2ce9f26157c79"},
{file = "identify-2.2.14-py2.py3-none-any.whl", hash = "sha256:113a76a6ba614d2a3dd408b3504446bcfac0370da5995aa6a17fd7c6dffde02d"},
{file = "identify-2.2.14.tar.gz", hash = "sha256:32f465f3c48083f345ad29a9df8419a4ce0674bf4a8c3245191d65c83634bdbf"},
]
isort = [
{file = "isort-5.9.3-py3-none-any.whl", hash = "sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2"},
......
[MASTER]
ignore-paths=tango_colors.py
[MESSAGES CONTROL]
disable=old-octal-literal,unichr-builtin,standarderror-builtin,round-builtin,setslice-method,hex-method,file-builtin,dict-iter-method,raising-string,range-builtin-not-iterating,next-method-called,print-statement,coerce-method,reload-builtin,raw_input-builtin,xrange-builtin,intern-builtin,using-cmp-argument,cmp-builtin,unicode-builtin,long-suffix,zip-builtin-not-iterating,getslice-method,reduce-builtin,filter-builtin-not-iterating,input-builtin,apply-builtin,long-builtin,indexing-exception,cmp-method,buffer-builtin,coerce-builtin,old-division,parameter-unpacking,import-star-module-level,suppressed-message,unpacking-in-except,old-ne-operator,oct-method,backtick,old-raise-syntax,useless-suppression,metaclass-assignment,dict-view-method,delslice-method,basestring-builtin,map-builtin-not-iterating,execfile-builtin,nonzero-method,no-absolute-import,no-else-return,no-else-raise,arguments-differ,too-many-arguments
......
......@@ -14,6 +14,7 @@ prompt-toolkit = "^3.0.19"
termcolor = "^1.1.0"
numpy = "^1.21.2"
yaspin = "^2.1.0"
humanize = "^3.11.0"
[tool.poetry.dev-dependencies]
pylint = "^2.6.0"
......
#!/usr/bin/env python
"""
#Tango Colors module
This module provides tango colors in several model.
[tango palette](http://tango.freedesktop.org/Tango_Icon_Theme_Guidelines#Color_Palette)
"""
import re
import unittest
import matplotlib.colors as mplcolors
import