Verified Commit 0ece43e4 authored by Sebastian Endres's avatar Sebastian Endres
Browse files

Fix things, more beautiful unicodes, prevent validation errors from stopping the plotting

parent 7cb036cd
......@@ -47,7 +47,7 @@ class SecretsInjector:
if len(results) > 1:
self.log(
f"[i] found more than one keylog file for {test_case_path}. Using the first one.",
f" found more than one keylog file for {test_case_path}. Using the first one.",
color="yellow",
)
......@@ -56,7 +56,7 @@ class SecretsInjector:
def inject(self, pcap_path: Path, pcap_ng_path: Path, keylog_file: Path):
if not pcap_path.is_file():
self.log(
f"[!] Raw pcap file {pcap_path} not found. Skipping.",
f" Raw pcap file {pcap_path} not found. Skipping.",
color="red",
)
......@@ -64,7 +64,7 @@ class SecretsInjector:
if pcap_ng_path.is_file() and pcap_ng_path.stat().st_size > 0:
self.log(
f"[i] {pcap_ng_path} already exists. Skipping.",
f" {pcap_ng_path} already exists. Skipping.",
color="cyan",
)
self.num_already_injected += 1
......@@ -72,7 +72,7 @@ class SecretsInjector:
return
try:
self.log(f"[i] Injecting {keylog_file} into {pcap_path} -> {pcap_ng_path}.")
self.log(f" Injecting {keylog_file} into {pcap_path} -> {pcap_ng_path}.")
subprocess.check_call(
[
"editcap",
......@@ -85,7 +85,7 @@ class SecretsInjector:
self.num_injected += 1
except subprocess.CalledProcessError:
self.log(
f"[!] Failed to inject secrets {keylog_file} into {pcap_path}.",
f" Failed to inject secrets {keylog_file} into {pcap_path}.",
color="red",
)
self.num_failed += 1
......@@ -99,7 +99,7 @@ class SecretsInjector:
if not keylog_file:
self.log(
f"[!] no keylog file found in {test_run_dir}",
f" no keylog file found in {test_run_dir}",
color="red",
)
self.num_no_secret_found += 1
......@@ -163,7 +163,7 @@ class SecretsInjector:
cprint(msg, **kwargs, end="\n")
log_str = (
f"[i] injected: {self.num_injected}, "
f" injected: {self.num_injected}, "
f"failed: {self.num_failed}, "
f"already injected: {self.num_already_injected}, "
f"no keylog file found: {self.num_no_secret_found}"
......@@ -179,7 +179,7 @@ class SecretsInjector:
else:
self.inject_in_log_dir(spec)
self.log("[i] Done", color="green", attrs=["bold"])
self.log(" Done", color="green", attrs=["bold"])
def main():
......
......@@ -55,7 +55,7 @@ def parse_args():
default=DEFAULT_TITLE,
help=(
f"The title for the diagram (default='{DEFAULT_TITLE}'). "
"WATCH OUT! THIS TTILE WILL BE FORMATTED -> WE TRUST THE FORMAT STRING!"
"WATCH OUT! THIS TTILE WILL BE FORMATTED WE TRUST THE FORMAT STRING!"
),
)
parser.add_argument(
......@@ -122,11 +122,11 @@ class PlotAllCli:
self._current_log_dir: Optional[Path] = None
self.debug = debug
def plot_in_testcase_dir(
def plot_in_meas_run_dir(
self,
measurement_result: ExtendedMeasurementResult,
modes: list[PlotMode],
) -> Optional[str]:
) -> list[str]:
"""Generate plot for for this test case."""
assert self._current_log_dir
test_case_dir = measurement_result.log_dir_for_test
......@@ -134,7 +134,7 @@ class PlotAllCli:
if not measurement_result.succeeded and not self.include_failed:
cprint(
(
"[i] Measurement "
" Measurement "
f"{measurement_result.log_dir_for_test.relative_to(self._current_log_dir)} "
"Failed. Skipping. Use --include-failed to include it anyway."
),
......@@ -142,19 +142,19 @@ class PlotAllCli:
color="cyan",
)
return "testcase failed"
return ["testcase failed"]
pcaps: list[Path] = []
for iteration_dir in measurement_result.iteration_log_dirs:
for repetition_dir in measurement_result.repetition_log_dirs:
pcapng = (
iteration_dir
repetition_dir
/ "sim"
/ f"trace_node_{self.side.value}_with_secrets.pcapng"
)
if not pcapng.is_file():
print(f"[!] {pcapng} does not exist", file=sys.stderr)
print(f" {pcapng} does not exist", file=sys.stderr)
continue
......@@ -162,12 +162,12 @@ class PlotAllCli:
if not pcaps:
cprint(
f"[!] no pcapng files found for {test_case_dir}. Skipping...",
f" no pcapng files found for {test_case_dir}. Skipping...",
file=sys.stderr,
color="red",
)
return "no traces files found"
return ["no traces files found"]
cli = PlotCli(
pcap_files=pcaps,
......@@ -176,25 +176,29 @@ class PlotAllCli:
debug=self.debug,
)
rets = list[str]()
for mode in modes:
output_file = Path(test_case_dir / f"time_{mode.value}_plot.{self.format}")
if not self.force and output_file.is_file():
cprint(
(
f"[i] {output_file.relative_to(self._current_log_dir)} already exists. "
f" {output_file.relative_to(self._current_log_dir)} already exists. "
"Skipping. Use --force to overwrite."
),
file=sys.stderr,
color="cyan",
)
return "already exists"
rets.append("already exists")
continue
cprint(
(
f"[i] Plotting in {create_relpath(test_case_dir)} "
f"({len(pcaps)} traces) -> {create_relpath(output_file)}"
f" Plotting in {create_relpath(test_case_dir)} "
f"({len(pcaps)} traces) {create_relpath(output_file)}"
),
attrs=["bold"],
)
......@@ -213,29 +217,31 @@ class PlotAllCli:
except ParsingError as err:
cprint(
(
f"[!] Could not parse {err.trace} in "
f" Could not parse {err.trace} in "
f"{test_case_dir.relative_to(self._current_log_dir)}. "
"Skipping..."
),
file=sys.stderr,
color="red",
)
cprint(f"[!] {err}", file=sys.stderr, color="red")
cprint(f" {err}", file=sys.stderr, color="red")
return str(err)
rets.append(str(err))
return None
continue
return rets
def plot_in_log_dir(self, result: Result):
"""Generate plots for result file."""
cprint(
f"[i] Plotting results {result} (log dir: {result.log_dir})",
f" Plotting results {result} (log dir: {result.log_dir})",
attrs=["bold"],
)
self._current_log_dir = result.log_dir
plot_results = dict[ExtendedMeasurementResult, Optional[str]]()
plot_results = defaultdict[str, set[str]](set[str])
for tmp1 in result.measurement_results.values():
for tmp2 in tmp1.values():
......@@ -247,45 +253,33 @@ class PlotAllCli:
measurement_results = list(tmp2.values())
for measurement_result in measurement_results:
plot_result = self.plot_in_testcase_dir(
results = self.plot_in_meas_run_dir(
measurement_result,
modes=self.modes,
)
plot_results[measurement_result] = plot_result
self.evaluate_plot_results(plot_results)
def evaluate_plot_results(
self, plot_results: dict[ExtendedMeasurementResult, Optional[str]]
):
"""Print a summary."""
grouped = defaultdict(list)
for plot_result in results:
if plot_result == "already_exists":
plot_result = colored(plot_result, color="green")
plot_results[plot_result].add(measurement_result.combination)
for measurement_result, plot_result in plot_results.items():
msg: str
if not plot_result:
msg = colored("success", "green")
elif plot_result in ("already exists"):
msg = colored(plot_result, "green")
else:
msg = colored(plot_result, "red")
grouped[msg].append(measurement_result)
if not plot_results:
plot_results[colored("success", color="green")].add(
measurement_result.combination
)
# Print a summary.
print()
print("#### Results:")
for msg, measurement_results in grouped.items():
print(f" - {msg}: {len(measurement_results)}")
print()
for msg, measurement_results in grouped.items():
print(f"- {msg}")
for msg, combinations in plot_results.items():
print(f"- {msg}: {len(combinations)}")
for combination in combinations:
print(f" - `{combination}`")
for measurement_result in measurement_results:
print(f" - `{measurement_result.combination}`")
print()
def run(self):
for result_file in self.result_files:
......@@ -313,7 +307,7 @@ def main():
except KeyboardInterrupt:
sys.exit("\nQuit")
cprint("Done", color="green", attrs=["bold"])
cprint("Done", color="green", attrs=["bold"])
if __name__ == "__main__":
......
......@@ -255,10 +255,21 @@ class PlotCli:
bbox=dict(fc="white", ec="none"),
)
def _annotate_time_plot(self, ax: plt.Axes, height: Union[float, int]):
def _annotate_time_plot(
self, ax: plt.Axes, height: Union[float, int], spinner: YaspinWrapper
):
if not self.annotate:
return
if not self.traces[0].facts["is_http09"]:
spinner.write(
colored(
f"⨯ Can't annotate plot, because facts are missing.", color="red"
)
)
return
ttfb = self.traces[0].facts["ttfb"]
req_start = self.traces[0].facts["request_start"]
pglt = self.traces[0].facts["plt"]
......@@ -423,10 +434,10 @@ class PlotCli:
response_retrans_offsets[0],
marker="o",
linestyle="",
color=self._colors.ScarletRed,
color=self._colors.Orange,
)
self._annotate_time_plot(ax, height=max_offset)
self._annotate_time_plot(ax, height=max_offset, spinner=spinner)
self._save(output_file, spinner)
def plot_packet_number(self, output_file: Optional[Path]):
......@@ -501,17 +512,17 @@ class PlotCli:
request_packet_numbers[0],
marker="o",
linestyle="",
color=self._colors.skyblue1,
color=self._colors.Plum,
)
ax.plot(
response_timestamps[0],
response_packet_numbers[0],
marker="o",
linestyle="",
color=self._colors.plum1,
color=self._colors.SkyBlue,
)
self._annotate_time_plot(ax, height=max_packet_number)
self._annotate_time_plot(ax, height=max_packet_number, spinner=spinner)
self._save(output_file, spinner)
def plot_file_size(self, output_file: Optional[Path]):
......@@ -583,7 +594,7 @@ class PlotCli:
color=self._colors.SkyBlue,
)
self._annotate_time_plot(ax, height=max_file_size)
self._annotate_time_plot(ax, height=max_file_size, spinner=spinner)
self._save(output_file, spinner)
def _process_packet_sizes(self):
......@@ -707,7 +718,7 @@ class PlotCli:
),
)
self._annotate_time_plot(ax, height=packet_stats.max)
self._annotate_time_plot(ax, height=packet_stats.max, spinner=spinner)
self._save(output_file, spinner)
......
......@@ -31,6 +31,18 @@ class ParsingError(Exception):
self.trace = trace
class FinError(ParsingError):
"""Error with closing streams detected."""
class HTTP09Error(ParsingError):
"""Error with HTTP/0.9 detected."""
class CryptoError(ParsingError):
"""Error with crypto detected."""
# def get_frame_prop_from_all_frames(
# packet: "Packet",
# prop_name: str,
......@@ -80,7 +92,7 @@ class Trace:
if file.suffix not in (".pcap", ".pcapng"):
cprint(
f"[!] Warning! Are you sure, that {file} is a pcap/pcapng file?",
f" Warning! Are you sure, that {file} is a pcap/pcapng file?",
color="yellow",
)
......@@ -100,6 +112,10 @@ class Trace:
self._request_stream_frames = list[QuicStreamLayer]()
self._response_stream_frames = list[QuicStreamLayer]()
self._parsed = False
self._error_cfg = {
HTTP09Error: "warning",
FinError: "warning",
}
def __str__(self):
trace_file_name = Path(self._cap.input_filename).name
......@@ -360,62 +376,81 @@ class Trace:
trace=self,
)
self._facts["request_stream_fin_pkns"] = self.get_stream_fin_packet_number(
self._request_stream_frames
)
try:
self._facts["request_stream_fin_pkns"] = self.get_stream_fin_packet_number(
self._request_stream_frames
)
except FinError as err:
if self._error_cfg[FinError] == "error":
raise err
else:
cprint(f"⨯ Validation error: {err}", file=sys.stderr, color="red")
request_raw = self.follow_stream(self._request_stream_frames)
try:
request = request_raw.decode("utf-8")
except UnicodeDecodeError as err:
raise ParsingError(
(
"Request seems not to be a HTTP/0.9 GET request. Maybe it is HTTP/3? "
f"{request_raw[:16]!r}"
),
trace=self,
) from err
request_raw = self.follow_stream(self._request_stream_frames)
try:
request = request_raw.decode("utf-8")
except UnicodeDecodeError as err:
raise HTTP09Error(
(
"Request seems not to be a HTTP/0.9 GET request. Maybe it is HTTP/3? "
f"{request_raw[:16]!r}"
),
trace=self,
) from err
if not request.startswith("GET /"):
raise ParsingError(
"First packet is not a HTTP/0.9 GET request.",
trace=self,
)
if not request.startswith("GET /"):
raise HTTP09Error(
"First packet is not a HTTP/0.9 GET request.",
trace=self,
)
self._facts["request"] = request
self._facts["is_http"] = True
self._facts["request"] = request
self._facts["is_http09"] = True
# check if all other packets are in direction from server to client:
# check if all other packets are in direction from server to client:
if float(self._request_stream_frames[-1].norm_time) >= float(
self._response_stream_frames[0].norm_time
):
raise ParsingError(
"Request packets appear after first response packet. Is this really HTTP/0.9?",
trace=self,
)
if float(self._request_stream_frames[-1].norm_time) >= float(
self._response_stream_frames[0].norm_time
):
raise HTTP09Error(
"Request packets appear after first response packet. Is this really HTTP/0.9?",
trace=self,
)
# calculate times
ttfb = self._response_stream_frames[0].norm_time
pglt = self._response_stream_frames[-1].norm_time
req_start = self._request_stream_frames[0].norm_time
resp_delay = ttfb - req_start
self._facts["ttfb"] = ttfb
self._facts["plt"] = pglt
self._facts["request_start"] = req_start
self._facts["response_delay"] = resp_delay
# check fin bit
self._facts["response_stream_fin_pkns"] = self.get_stream_fin_packet_number(
self._response_stream_frames
)
# calculate times
ttfb = self._response_stream_frames[0].norm_time
pglt = self._response_stream_frames[-1].norm_time
req_start = self._request_stream_frames[0].norm_time
resp_delay = ttfb - req_start
self._facts["ttfb"] = ttfb
self._facts["plt"] = pglt
self._facts["request_start"] = req_start
self._facts["response_delay"] = resp_delay
except HTTP09Error as err:
if self._error_cfg[HTTP09Error] == "error":
raise err
else:
self._facts["is_http09"] = False
cprint(f"⨯ Validation error: {err}", file=sys.stderr, color="red")
try:
# check fin bit
self._facts["response_stream_fin_pkns"] = self.get_stream_fin_packet_number(
self._response_stream_frames
)
except FinError as err:
if self._error_cfg[FinError] == "error":
raise err
else:
cprint(f"⨯ Validation error: {err}", file=sys.stderr, color="red")
self._parsed = True
def iter_stream_frames(self, packet: "Packet") -> Iterator:
for quic_layer in packet.get_multiple_layers("quic"):
if hasattr(quic_layer, "decryption_failed"):
raise ParsingError(
raise CryptoError(
f"Decryption of QUIC crypto failed: {quic_layer.decryption_failed}",
trace=self,
)
......@@ -469,7 +504,7 @@ class Trace:
if not all(byte is not None for byte in buf):
cprint(
"[!] Warning! Did not receive all bytes in follow_stream.",
" Warning! Did not receive all bytes in follow_stream.",
color="yellow",
)
......@@ -545,11 +580,11 @@ class Trace:
msg = "Last packet that contains a stream frame does not close stream."
if warn_only:
cprint(f"[!] {msg}", color="red", file=sys.stderr)
cprint(f" {msg}", color="red", file=sys.stderr)
return []
else:
raise ParsingError(msg, trace=self)
raise FinError(msg, trace=self)
elif len(stream_fins) == 1:
fin_pkn: int = stream_fins[0]["packet_number"]
......@@ -563,11 +598,11 @@ class Trace:
)
if warn_only:
cprint(f"[!] {msg}", color="red", file=sys.stderr)
cprint(f" {msg}", color="red", file=sys.stderr)
return []
else:
raise ParsingError(
raise FinError(
msg,
trace=self,
)
......@@ -582,11 +617,11 @@ class Trace:
msg = "There are multiple stream ids in this list. Is it HTTP/3?"
if warn_only:
cprint(f"[!] {msg}", color="red", file=sys.stderr)
cprint(f" {msg}", color="red", file=sys.stderr)
return []
else:
raise ParsingError(msg, trace=self)
raise FinError(msg, trace=self)
if not all(
fin_pkg["offset"] == stream_fins[0]["offset"] for fin_pkg in stream_fins
......@@ -594,11 +629,11 @@ class Trace:
msg = "Stream was closed multiple times."
if warn_only:
cprint(f"[!] {msg}", color="red", file=sys.stderr)
cprint(f" {msg}", color="red", file=sys.stderr)
return []
else:
raise ParsingError(msg, trace=self)
raise FinError(msg, trace=self)
assert max_offset == stream_fins[0]["offset"]
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment