Verified Commit cc5f3e17 authored by Sebastian Endres's avatar Sebastian Endres
Browse files

Fix many things...

parent 1056f91a
......@@ -9,10 +9,10 @@ from typing import Literal, Optional, Union
from termcolor import colored, cprint
from plot_diagram import DEFAULT_TITLES, PlotMode, PlotCli
from plot_diagram import DEFAULT_TITLES, PlotCli, PlotMode
from result_parser import ExtendedMeasurementResult, Result
from tracer import ParsingError
from utils import existing_file_path
from utils import create_relpath, existing_file_path
DEFAULT_TITLE = "{mode_default_title} (server: {server}, client: {client})"
......@@ -33,7 +33,7 @@ class PlotModeAll(Enum):
return {
PlotModeAll.PACKET_NUMBER: [PlotMode.PACKET_NUMBER],
PlotModeAll.FILE_SIZE: [PlotMode.FILE_SIZE],
PlotModeAll.ALL: [PlotMode.PACKET_NUMBER, PlotMode.FILE_SIZE],
PlotModeAll.ALL: [mode for mode in PlotMode],
}[self]
......@@ -192,8 +192,8 @@ class PlotAllCli:
cprint(
(
f"[i] Plotting in {test_case_dir.relative_to(self._current_log_dir)} "
f"({len(pcaps)} traces) -> {output_file}"
f"[i] Plotting in {create_relpath(test_case_dir)} "
f"({len(pcaps)} traces) -> {create_relpath(output_file)}"
),
attrs=["bold"],
)
......@@ -288,7 +288,6 @@ class PlotAllCli:
for measurement_result in measurement_results:
print(f" - {measurement_result.combination}")
breakpoint()
def run(self):
for result_file in self.result_files:
......
......@@ -12,13 +12,13 @@ import typing
from enum import Enum
from pathlib import Path
from typing import Callable, Literal, Optional, TypeVar, Union
from termcolor import cprint
import numpy as np
from matplotlib import pyplot as plt
from yaspin import yaspin
from tracer import ParsingError, Trace, get_quic_payload_size
from utils import existing_file_path
from utils import YaspinWrapper, existing_file_path
if typing.TYPE_CHECKING:
from collections.abc import Iterable
......@@ -45,6 +45,16 @@ def map2d(func: Callable[["Iterable[T]"], T], arrays: "Iterable[Iterable[T]]") -
return func(map(func, arrays))
def map3d(
func: Callable[["Iterable[T]"], T],
arrays: "Iterable[Iterable[Iterable[T]]]",
) -> T:
def inner_func(arr):
return map2d(func, arr)
return func(map(inner_func, arrays))
def format_file_size(val: Union[int, float]) -> str:
"""Format bytes."""
......@@ -127,7 +137,7 @@ def parse_args():
parser.add_argument(
"--mode",
action="store",
choices=[mode.value for mode in PlotMode],
choices=PlotMode,
type=PlotMode,
default=PlotMode.PACKET_NUMBER,
help="The mode of plotting (time vs. packet-number or time vs. file-size",
......@@ -137,6 +147,11 @@ def parse_args():
action="store_true",
help="Cache parsed trace",
)
parser.add_argument(
"--debug",
action="store_true",
help="Debug mode.",
)
args = parser.parse_args()
......@@ -162,11 +177,12 @@ class PlotCli:
annotate=True,
mode: PlotMode = PlotMode.PACKET_NUMBER,
cache=False,
debug=False,
):
if not keylog_files:
keylog_files = [None] * len(pcap_files)
print("Loading traces...")
cprint("Loading traces...", color="grey")
self.traces = [
Trace(
file=pcap_file,
......@@ -180,6 +196,7 @@ class PlotCli:
self.output_file = output_file
self.annotate = annotate
self.mode = mode
self.debug = debug
def vline_annotate(
self,
......@@ -237,10 +254,6 @@ class PlotCli:
def plot_packet_number(self):
"""Plot the packet number diagram."""
print(
f"Plotting {len(self.traces)} traces into a time vs. packet-number plot..."
)
with Subplot(nrows=1, ncols=1) as (_fig, ax):
ax.grid(True)
ax.set_xlabel("Time (s)")
......@@ -256,36 +269,72 @@ class PlotCli:
if self.annotate:
# raise errors early
_ = trace.get_facts()
trace.parse()
with yaspin(text="processing...", color="cyan") as spinner:
timestamps = [
with YaspinWrapper(
debug=self.debug, text="processing...", color="cyan"
) as spinner:
request_timestamps = [
np.array(
[
float(packet.sniff_timestamp)
for packet in trace.request_packets
]
)
for trace in self.traces
]
response_timestamps = [
np.array(
[
float(packet.sniff_timestamp)
for packet in trace.response_packets
]
)
for trace in self.traces
]
request_packet_numbers = [
np.array(
[float(packet.sniff_timestamp) for packet in trace.packets]
[
int(packet.quic.packet_number)
for packet in trace.request_packets
]
)
for trace in self.traces
]
packet_numbers = [
response_packet_numbers = [
np.array(
[int(packet.quic.packet_number) for packet in trace.packets]
[
int(packet.quic.packet_number)
for packet in trace.response_packets
]
)
for trace in self.traces
]
min_packet_number: int = map2d(min, packet_numbers)
max_packet_number: int = map2d(max, packet_numbers)
min_timestamp: np.float64 = map2d(min, timestamps)
max_timestamp: np.float64 = map2d(max, timestamps)
min_packet_number: int = map3d(
min, [request_packet_numbers, response_packet_numbers]
)
max_packet_number: int = map3d(
max, [request_packet_numbers, response_packet_numbers]
)
min_timestamp: np.float64 = map3d(
min, [request_timestamps, response_timestamps]
)
max_timestamp: np.float64 = map3d(
max, [request_timestamps, response_timestamps]
)
ax.set_xlim(left=min(0, min_timestamp), right=max_timestamp)
ax.set_ylim(bottom=min(0, min_packet_number), top=max_packet_number)
spinner.ok("✅")
with yaspin(text="plotting...", color="cyan") as spinner:
# plot shadow traces
with YaspinWrapper(
debug=self.debug, text="plotting...", color="cyan"
) as spinner:
# plot shadow traces (request and response separated)
for trace_timestamps, trace_packet_numbers in zip(
timestamps[1:], packet_numbers[1:]
request_timestamps, request_packet_numbers
):
ax.plot(
trace_timestamps,
......@@ -295,11 +344,29 @@ class PlotCli:
color="#CCC",
)
# plot main trace
for trace_timestamps, trace_packet_numbers in zip(
response_timestamps, response_packet_numbers
):
ax.plot(
trace_timestamps,
trace_packet_numbers,
marker="o",
linestyle="",
color="#CCC",
)
# plot main trace (request and response separated)
ax.plot(
timestamps[0],
packet_numbers[0],
request_timestamps[0],
request_packet_numbers[0],
marker="o",
linestyle="",
color="#729fcf",
)
ax.plot(
response_timestamps[0],
response_packet_numbers[0],
marker="o",
linestyle="",
color="#3465A4",
......@@ -327,19 +394,27 @@ class PlotCli:
if self.annotate:
# raise errors early
_ = trace.get_facts()
trace.parse()
with yaspin(text="processing...", color="cyan") as spinner:
# drop GET request
with YaspinWrapper(
debug=self.debug, text="processing...", color="cyan"
) as spinner:
# only response
timestamps = [
np.array(
[float(packet.sniff_timestamp) for packet in trace.packets[1:]]
[
float(packet.sniff_timestamp)
for packet in trace.response_packets
]
)
for trace in self.traces
]
file_sizes = [
np.array(
[get_quic_payload_size(packet) for packet in trace.packets[1:]]
[
get_quic_payload_size(packet)
for packet in trace.response_packets
]
)
for trace in self.traces
]
......@@ -357,7 +432,9 @@ class PlotCli:
spinner.ok("✅")
with yaspin(text="plotting...", color="cyan") as spinner:
with YaspinWrapper(
debug=self.debug, text="plotting...", color="cyan"
) as spinner:
# plot shadow traces
for trace_timestamps, trace_file_sizes in zip(
......@@ -390,7 +467,7 @@ class PlotCli:
if self.output_file:
plt.savefig(self.output_file, dpi=300, transparent=True)
print(f"{self.output_file} written.")
cprint(f"{self.output_file} written.", color="green")
else:
plt.show()
......@@ -410,7 +487,9 @@ class PlotCli:
callback = cfg["callback"]
desc = cfg["desc"]
print(f"Plotting {len(self.traces)} traces into a {desc} plot...")
cprint(
f"Plotting {len(self.traces)} traces into a {desc} plot...", color="cyan"
)
callback()
......@@ -429,6 +508,7 @@ def main():
annotate=not args.no_annotation,
mode=args.mode,
cache=args.cache,
debug=args.debug,
)
try:
cli.run()
......
......@@ -2,14 +2,17 @@
import pickle
import subprocess
import sys
import typing
from functools import cached_property
from pathlib import Path
from typing import Any, Optional, Union, cast
from typing import Any, Callable, Iterator, Optional, TypedDict
import pyshark
from prompt_toolkit.shortcuts import ProgressBar
from yaspin import yaspin
from termcolor import cprint
from utils import YaspinWrapper, create_relpath
if typing.TYPE_CHECKING:
from pyshark.packet.packet import Packet
......@@ -24,17 +27,23 @@ class ParsingError(Exception):
self.trace = trace
def iter_stream_frames(packet: "Packet") -> Iterator:
for quic_layer in packet.get_multiple_layers("quic"):
if hasattr(quic_layer, "stream_stream_id"):
yield quic_layer
def get_quic_payload_size(packet: "Packet") -> int:
"""Return the stream_data payload size of this packet."""
payload_size = 0
for quic_layer in packet.get_multiple_layers("quic"):
if not hasattr(quic_layer, "stream_data"):
# we are only intereseted in stream frames
continue
for quic_layer in iter_stream_frames(packet):
stream_data = quic_layer.stream_data
stream_data_len = len(quic_layer.stream_data.binary_value)
if stream_data.raw_value is None:
stream_data_len = 0
else:
stream_data_len = len(quic_layer.stream_data.binary_value)
if hasattr(quic_layer, "stream_length"):
assert int(quic_layer.stream_length) == stream_data_len, (
......@@ -47,6 +56,148 @@ def get_quic_payload_size(packet: "Packet") -> int:
return payload_size
def follow_stream(stream_frames: list[Any]) -> bytes:
buf = list[int]()
for frame in stream_frames:
offset = get_stream_offset(frame)
assert offset is not None
extend_buf = offset - len(buf)
if extend_buf > 0:
buf += [0] * extend_buf
buf[offset:] = frame.stream_data.binary_value
return bytes(buf)
def get_frame_prop_from_all_frames(
packet: "Packet",
prop_name: str,
include_none: bool = False,
callback: Optional[Callable[[Any], Any]] = None,
) -> Any:
ret = list[Any]()
for quic_layer in iter_stream_frames(packet):
if not hasattr(quic_layer, prop_name):
if include_none:
ret.append(None)
else:
raw_value = getattr(quic_layer, prop_name)
if callback:
ret.append(callback(raw_value))
else:
ret.append(raw_value)
return ret
def get_stream_offset(quic_layer) -> Optional[int]:
if not hasattr(quic_layer, "stream_off"):
# not a stream_frame
return None
if quic_layer.stream_off.int_value:
# try:
return int(quic_layer.stream_offset)
# except AttributeError:
# breakpoint()
else:
return 0
def get_stream_offsets(packet: "Packet") -> list[int]:
offsets = [
get_stream_offset(quic_layer) for quic_layer in iter_stream_frames(packet)
]
return [off for off in offsets if off is not None]
def get_stream_fin_packet_number(packets: list["Packet"], trace: "Trace") -> list[int]:
"""
Check if the last packet and only the last packet of this stream ends the stream.
:Return: A list of packet numbers, in which this stream was closed.
This may be multiple packets, if the fin packet was re-sent.
All other cases should be validated and an error should be raised.
"""
class StreamFinEntry(TypedDict):
packet_number: int
stream_id: int
offset: int
stream_fins = list[StreamFinEntry]()
# packets may be out of order:
max_offset = float("-inf")
pkn_with_max_offset = float("-inf")
for packet in packets:
for quic_layer in iter_stream_frames(packet):
layer_offset = get_stream_offset(quic_layer)
if layer_offset is None:
breakpoint()
if layer_offset is not None and layer_offset > max_offset:
max_offset = layer_offset
pkn_with_max_offset = int(quic_layer.packet_number)
if hasattr(quic_layer, "stream_fin") and quic_layer.stream_fin.int_value:
stream_fins.append(
{
"packet_number": int(quic_layer.packet_number),
"stream_id": int(quic_layer.stream_stream_id, base=16),
"offset": layer_offset,
}
)
if not stream_fins:
raise ParsingError(
"Last packet that contains a stream frame does not close stream.",
trace=trace,
)
elif len(stream_fins) == 1:
fin_pkn: int = stream_fins[0]["packet_number"]
if pkn_with_max_offset < 0 or fin_pkn == pkn_with_max_offset:
return [fin_pkn]
else:
raise ParsingError(
(
f"Stream {stream_fins[0]['stream_id']} was closed before "
"the last packet was sent. "
f"(closed with #{fin_pkn}, max offset in #{pkn_with_max_offset})"
),
trace=trace,
)
else:
# this may happen if the packet was re-sent -> all must have same stream_id (check) and same offset
if not all(
fin_pkg["stream_id"] == stream_fins[0]["stream_id"]
for fin_pkg in stream_fins
):
raise ParsingError(
"There are multiple stream ids in this list. Is it HTTP/3?", trace=trace
)
if not all(
fin_pkg["offset"] == stream_fins[0]["offset"] for fin_pkg in stream_fins
):
raise ParsingError("Stream was closed multiple times.", trace=trace)
assert max_offset == stream_fins[0]["offset"]
return [fin_pkg["packet_number"] for fin_pkg in stream_fins]
class Trace:
"""A pcap trace."""
......@@ -56,7 +207,9 @@ class Trace:
display_filter="",
keylog_file: Optional[Path] = None,
cache=False,
debug=False,
):
self.debug = debug
self._display_filter = display_filter
self._keylog_file = keylog_file
override_prefs = {}
......@@ -69,6 +222,12 @@ class Trace:
else:
self._cache_file = None
if file.suffix not in (".pcap", ".pcapng"):
cprint(
f"Warning! Are you sure, that {file} is a pcap/pcapng file?",
color="yellow",
)
self._cap = pyshark.FileCapture(
str(file.absolute()),
display_filter=display_filter,
......@@ -77,6 +236,9 @@ class Trace:
disable_protocol="http3",
decode_as={"udp.port==443": "quic"},
)
self._facts = dict[str, Any]()
self._request_packets = list["Packet"]()
self._response_packets = list["Packet"]()
def __str__(self):
trace_file_name = Path(self._cap.input_filename).name
......@@ -106,8 +268,9 @@ class Trace:
if self._cache_file and self._cache_file.is_file():
with self._cache_file.open("rb") as cache_file:
with yaspin(
text=f"Loading cache from {self._cache_file}",
with YaspinWrapper(
debug=self.debug,
text=f"Loading cache from {create_relpath(self._cache_file)}",
color="green",
) as spinner:
cached_packets = pickle.load(cache_file)
......@@ -129,8 +292,9 @@ class Trace:
if self._cache_file:
with self._cache_file.open("wb") as cache_file:
with yaspin(
text=f"Saving parsed packets to {self._cache_file}",
with YaspinWrapper(
debug=self.debug,
text=f"Saving parsed packets to {create_relpath(self._cache_file)}",
color="green",
) as spinner:
pickle.dump(obj=packets, file=cache_file)
......@@ -138,92 +302,144 @@ class Trace:
return packets
def get_facts(self) -> dict[str, Any]:
"""Ensure facts are populated. Used to raise exceptions early. TODO this is hacky."""
@property
def request_packets(self) -> list["Packet"]:
if not self._request_packets:
self.parse()
return self.facts
return self._request_packets
@cached_property
@property
def response_packets(self) -> list["Packet"]:
if not self._response_packets:
self.parse()
return self._response_packets
@property
def facts(self) -> dict[str, Any]:
if not self._facts:
self.parse()
return self._facts
def parse(self) -> None:
"""
Analyze the packets and return a "fact" dict containing:
- is_http
- ttfb
- request_start
- client/server IP/port
- packet number of stream.fin == 1
"""
if len(self.packets) < 2:
raise ParsingError(
"There are less than 2 quic packets in this trace. Did you provide the SSLKEYLOG?",
"There are less than 2 quic stream packets in this trace. Did you provide the SSLKEYLOG?",
trace=self,
)
# TODO check if pacing was used?
results = dict[str, Any]()
self._facts = dict[str, Any]()
# check if first packet is an HTTP (0.9) request
first_packet = self.packets[0]
first_quic_layers = [
quic
for quic in first_packet.get_multiple_layers("quic")
if hasattr(quic, "stream_data")
# get client and server IP addresses and UDP ports
server_ip = first_packet.ip.dst
client_ip = first_packet.ip.src
self._facts["client_ip"] = client_ip
self._facts["server_ip"] = server_ip
server_port = first_packet.udp.dstport
client_port = first_packet.udp.srcport
self._facts["client_port"] = client_port
self._facts["server_port"] = server_port
client_tuple = (client_ip, client_port)
server_tuple = (server_ip, server_port)