Fix things, more beautiful unicodes, prevent validation errors from stopping the plotting

inject_secrets.py

@@ -47,7 +47,7 @@ class SecretsInjector:
 
         if len(results) > 1:
             self.log(
-                f"[i] found more than one keylog file for {test_case_path}. Using the first one.",
+                f"⚒ found more than one keylog file for {test_case_path}. Using the first one.",
                 color="yellow",
             )
 
@@ -56,7 +56,7 @@ class SecretsInjector:
     def inject(self, pcap_path: Path, pcap_ng_path: Path, keylog_file: Path):
         if not pcap_path.is_file():
             self.log(
-                f"[!] Raw pcap file {pcap_path} not found. Skipping.",
+                f"⨯ Raw pcap file {pcap_path} not found. Skipping.",
                 color="red",
             )
 
@@ -64,7 +64,7 @@ class SecretsInjector:
 
         if pcap_ng_path.is_file() and pcap_ng_path.stat().st_size > 0:
             self.log(
-                f"[i] {pcap_ng_path} already exists. Skipping.",
+                f"⚒ {pcap_ng_path} already exists. Skipping.",
                 color="cyan",
             )
             self.num_already_injected += 1
@@ -72,7 +72,7 @@ class SecretsInjector:
             return
 
         try:
-            self.log(f"[i] Injecting {keylog_file} into {pcap_path} -> {pcap_ng_path}.")
+            self.log(f"⚒ Injecting {keylog_file} into {pcap_path} -> {pcap_ng_path}.")
             subprocess.check_call(
                 [
                     "editcap",
@@ -85,7 +85,7 @@ class SecretsInjector:
             self.num_injected += 1
         except subprocess.CalledProcessError:
             self.log(
-                f"[!] Failed to inject secrets {keylog_file} into {pcap_path}.",
+                f"⨯ Failed to inject secrets {keylog_file} into {pcap_path}.",
                 color="red",
             )
             self.num_failed += 1
@@ -99,7 +99,7 @@ class SecretsInjector:
 
         if not keylog_file:
             self.log(
-                f"[!] no keylog file found in {test_run_dir}",
+                f"⨯ no keylog file found in {test_run_dir}",
                 color="red",
             )
             self.num_no_secret_found += 1
@@ -163,7 +163,7 @@ class SecretsInjector:
             cprint(msg, **kwargs, end="\n")
 
         log_str = (
-            f"[i] injected: {self.num_injected}, "
+            f"⚒ injected: {self.num_injected}, "
             f"failed: {self.num_failed}, "
             f"already injected: {self.num_already_injected}, "
             f"no keylog file found: {self.num_no_secret_found}"
@@ -179,7 +179,7 @@ class SecretsInjector:
             else:
                 self.inject_in_log_dir(spec)
 
-        self.log("[i] Done", color="green", attrs=["bold"])
+        self.log("⚒ Done", color="green", attrs=["bold"])
 
 
 def main():

plot_all.py

@@ -55,7 +55,7 @@ def parse_args():
         default=DEFAULT_TITLE,
         help=(
             f"The title for the diagram (default='{DEFAULT_TITLE}'). "
-            "WATCH OUT! THIS TTILE WILL BE FORMATTED -> WE TRUST THE FORMAT STRING!"
+            "WATCH OUT! THIS TTILE WILL BE FORMATTED → WE TRUST THE FORMAT STRING!"
         ),
     )
     parser.add_argument(
@@ -122,11 +122,11 @@ class PlotAllCli:
         self._current_log_dir: Optional[Path] = None
         self.debug = debug
 
-    def plot_in_testcase_dir(
+    def plot_in_meas_run_dir(
         self,
         measurement_result: ExtendedMeasurementResult,
         modes: list[PlotMode],
-    ) -> Optional[str]:
+    ) -> list[str]:
         """Generate plot for for this test case."""
         assert self._current_log_dir
         test_case_dir = measurement_result.log_dir_for_test
@@ -134,7 +134,7 @@ class PlotAllCli:
         if not measurement_result.succeeded and not self.include_failed:
             cprint(
                 (
-                    "[i] Measurement "
+                    "⚒ Measurement "
                     f"{measurement_result.log_dir_for_test.relative_to(self._current_log_dir)} "
                     "Failed. Skipping. Use --include-failed to include it anyway."
                 ),
@@ -142,19 +142,19 @@ class PlotAllCli:
                 color="cyan",
             )
 
-            return "testcase failed"
+            return ["testcase failed"]
 
         pcaps: list[Path] = []
 
-        for iteration_dir in measurement_result.iteration_log_dirs:
+        for repetition_dir in measurement_result.repetition_log_dirs:
             pcapng = (
-                iteration_dir
+                repetition_dir
                 / "sim"
                 / f"trace_node_{self.side.value}_with_secrets.pcapng"
             )
 
             if not pcapng.is_file():
-                print(f"[!] {pcapng} does not exist", file=sys.stderr)
+                print(f"⨯ {pcapng} does not exist", file=sys.stderr)
 
                 continue
 
@@ -162,12 +162,12 @@ class PlotAllCli:
 
         if not pcaps:
             cprint(
-                f"[!] no pcapng files found for {test_case_dir}. Skipping...",
+                f"⨯ no pcapng files found for {test_case_dir}. Skipping...",
                 file=sys.stderr,
                 color="red",
             )
 
-            return "no traces files found"
+            return ["no traces files found"]
 
         cli = PlotCli(
             pcap_files=pcaps,
@@ -176,25 +176,29 @@ class PlotAllCli:
             debug=self.debug,
         )
 
+        rets = list[str]()
+
         for mode in modes:
             output_file = Path(test_case_dir / f"time_{mode.value}_plot.{self.format}")
 
             if not self.force and output_file.is_file():
                 cprint(
                     (
-                        f"[i] {output_file.relative_to(self._current_log_dir)} already exists. "
+                        f"⚒ {output_file.relative_to(self._current_log_dir)} already exists. "
                         "Skipping. Use --force to overwrite."
                     ),
                     file=sys.stderr,
                     color="cyan",
                 )
 
-                return "already exists"
+                rets.append("already exists")
+
+                continue
 
             cprint(
                 (
-                    f"[i] Plotting in {create_relpath(test_case_dir)} "
-                    f"({len(pcaps)} traces) -> {create_relpath(output_file)}"
+                    f"⚒ Plotting in {create_relpath(test_case_dir)} "
+                    f"({len(pcaps)} traces) → {create_relpath(output_file)}"
                 ),
                 attrs=["bold"],
             )
@@ -213,29 +217,31 @@ class PlotAllCli:
             except ParsingError as err:
                 cprint(
                     (
-                        f"[!] Could not parse {err.trace} in "
+                        f"⨯ Could not parse {err.trace} in "
                         f"{test_case_dir.relative_to(self._current_log_dir)}. "
                         "Skipping..."
                     ),
                     file=sys.stderr,
                     color="red",
                 )
-                cprint(f"[!] {err}", file=sys.stderr, color="red")
+                cprint(f"⨯ {err}", file=sys.stderr, color="red")
 
-                return str(err)
+                rets.append(str(err))
 
-        return None
+                continue
+
+        return rets
 
     def plot_in_log_dir(self, result: Result):
         """Generate plots for result file."""
         cprint(
-            f"[i] Plotting results {result} (log dir: {result.log_dir})",
+            f"⚒ Plotting results {result} (log dir: {result.log_dir})",
             attrs=["bold"],
         )
 
         self._current_log_dir = result.log_dir
 
-        plot_results = dict[ExtendedMeasurementResult, Optional[str]]()
+        plot_results = defaultdict[str, set[str]](set[str])
 
         for tmp1 in result.measurement_results.values():
             for tmp2 in tmp1.values():
@@ -247,45 +253,33 @@ class PlotAllCli:
                     measurement_results = list(tmp2.values())
 
                 for measurement_result in measurement_results:
-                    plot_result = self.plot_in_testcase_dir(
+                    results = self.plot_in_meas_run_dir(
                         measurement_result,
                         modes=self.modes,
                     )
-                    plot_results[measurement_result] = plot_result
-
-        self.evaluate_plot_results(plot_results)
 
-    def evaluate_plot_results(
-        self, plot_results: dict[ExtendedMeasurementResult, Optional[str]]
-    ):
-        """Print a summary."""
-        grouped = defaultdict(list)
+                    for plot_result in results:
+                        if plot_result == "already_exists":
+                            plot_result = colored(plot_result, color="green")
+                        plot_results[plot_result].add(measurement_result.combination)
 
-        for measurement_result, plot_result in plot_results.items():
-            msg: str
-
-            if not plot_result:
-                msg = colored("success", "green")
-            elif plot_result in ("already exists"):
-                msg = colored(plot_result, "green")
-            else:
-                msg = colored(plot_result, "red")
-
-            grouped[msg].append(measurement_result)
+                    if not plot_results:
+                        plot_results[colored("success", color="green")].add(
+                            measurement_result.combination
+                        )
 
+        # Print a summary.
         print()
         print("#### Results:")
-
-        for msg, measurement_results in grouped.items():
-            print(f"  - {msg}: {len(measurement_results)}")
-
         print()
 
-        for msg, measurement_results in grouped.items():
-            print(f"- {msg}")
+        for msg, combinations in plot_results.items():
+            print(f"- {msg}: {len(combinations)}")
+
+            for combination in combinations:
+                print(f"  - `{combination}`")
 
-            for measurement_result in measurement_results:
-                print(f"  - `{measurement_result.combination}`")
+        print()
 
     def run(self):
         for result_file in self.result_files:
@@ -313,7 +307,7 @@ def main():
     except KeyboardInterrupt:
         sys.exit("\nQuit")
 
-    cprint("Done", color="green", attrs=["bold"])
+    cprint("✔ Done", color="green", attrs=["bold"])
 
 
 if __name__ == "__main__":

plot_diagram.py

@@ -255,10 +255,21 @@ class PlotCli:
             bbox=dict(fc="white", ec="none"),
         )
 
-    def _annotate_time_plot(self, ax: plt.Axes, height: Union[float, int]):
+    def _annotate_time_plot(
+        self, ax: plt.Axes, height: Union[float, int], spinner: YaspinWrapper
+    ):
         if not self.annotate:
             return
 
+        if not self.traces[0].facts["is_http09"]:
+            spinner.write(
+                colored(
+                    f"⨯ Can't annotate plot, because facts are missing.", color="red"
+                )
+            )
+
+            return
+
         ttfb = self.traces[0].facts["ttfb"]
         req_start = self.traces[0].facts["request_start"]
         pglt = self.traces[0].facts["plt"]
@@ -423,10 +434,10 @@ class PlotCli:
                     response_retrans_offsets[0],
                     marker="o",
                     linestyle="",
-                    color=self._colors.ScarletRed,
+                    color=self._colors.Orange,
                 )
 
-                self._annotate_time_plot(ax, height=max_offset)
+                self._annotate_time_plot(ax, height=max_offset, spinner=spinner)
                 self._save(output_file, spinner)
 
     def plot_packet_number(self, output_file: Optional[Path]):
@@ -501,17 +512,17 @@ class PlotCli:
                     request_packet_numbers[0],
                     marker="o",
                     linestyle="",
-                    color=self._colors.skyblue1,
+                    color=self._colors.Plum,
                 )
                 ax.plot(
                     response_timestamps[0],
                     response_packet_numbers[0],
                     marker="o",
                     linestyle="",
-                    color=self._colors.plum1,
+                    color=self._colors.SkyBlue,
                 )
 
-                self._annotate_time_plot(ax, height=max_packet_number)
+                self._annotate_time_plot(ax, height=max_packet_number, spinner=spinner)
                 self._save(output_file, spinner)
 
     def plot_file_size(self, output_file: Optional[Path]):
@@ -583,7 +594,7 @@ class PlotCli:
                     color=self._colors.SkyBlue,
                 )
 
-                self._annotate_time_plot(ax, height=max_file_size)
+                self._annotate_time_plot(ax, height=max_file_size, spinner=spinner)
                 self._save(output_file, spinner)
 
     def _process_packet_sizes(self):
@@ -707,7 +718,7 @@ class PlotCli:
                     ),
                 )
 
-                self._annotate_time_plot(ax, height=packet_stats.max)
+                self._annotate_time_plot(ax, height=packet_stats.max, spinner=spinner)
 
                 self._save(output_file, spinner)
 

tracer.py

@@ -31,6 +31,18 @@ class ParsingError(Exception):
         self.trace = trace
 
 
+class FinError(ParsingError):
+    """Error with closing streams detected."""
+
+
+class HTTP09Error(ParsingError):
+    """Error with HTTP/0.9 detected."""
+
+
+class CryptoError(ParsingError):
+    """Error with crypto detected."""
+
+
 #  def get_frame_prop_from_all_frames(
 #      packet: "Packet",
 #      prop_name: str,
@@ -80,7 +92,7 @@ class Trace:
 
         if file.suffix not in (".pcap", ".pcapng"):
             cprint(
-                f"[!] Warning! Are you sure, that {file} is a pcap/pcapng file?",
+                f"⨯ Warning! Are you sure, that {file} is a pcap/pcapng file?",
                 color="yellow",
             )
 
@@ -100,6 +112,10 @@ class Trace:
         self._request_stream_frames = list[QuicStreamLayer]()
         self._response_stream_frames = list[QuicStreamLayer]()
         self._parsed = False
+        self._error_cfg = {
+            HTTP09Error: "warning",
+            FinError: "warning",
+        }
 
     def __str__(self):
         trace_file_name = Path(self._cap.input_filename).name
@@ -360,62 +376,81 @@ class Trace:
                 trace=self,
             )
 
-        self._facts["request_stream_fin_pkns"] = self.get_stream_fin_packet_number(
-            self._request_stream_frames
-        )
+        try:
+            self._facts["request_stream_fin_pkns"] = self.get_stream_fin_packet_number(
+                self._request_stream_frames
+            )
+        except FinError as err:
+            if self._error_cfg[FinError] == "error":
+                raise err
+            else:
+                cprint(f"⨯ Validation error: {err}", file=sys.stderr, color="red")
 
-        request_raw = self.follow_stream(self._request_stream_frames)
         try:
-            request = request_raw.decode("utf-8")
-        except UnicodeDecodeError as err:
-            raise ParsingError(
-                (
-                    "Request seems not to be a HTTP/0.9 GET request. Maybe it is HTTP/3? "
-                    f"{request_raw[:16]!r}"
-                ),
-                trace=self,
-            ) from err
+            request_raw = self.follow_stream(self._request_stream_frames)
+            try:
+                request = request_raw.decode("utf-8")
+            except UnicodeDecodeError as err:
+                raise HTTP09Error(
+                    (
+                        "Request seems not to be a HTTP/0.9 GET request. Maybe it is HTTP/3? "
+                        f"{request_raw[:16]!r}"
+                    ),
+                    trace=self,
+                ) from err
 
-        if not request.startswith("GET /"):
-            raise ParsingError(
-                "First packet is not a HTTP/0.9 GET request.",
-                trace=self,
-            )
+            if not request.startswith("GET /"):
+                raise HTTP09Error(
+                    "First packet is not a HTTP/0.9 GET request.",
+                    trace=self,
+                )
 
-        self._facts["request"] = request
-        self._facts["is_http"] = True
+            self._facts["request"] = request
+            self._facts["is_http09"] = True
 
-        # check if all other packets are in direction from server to client:
+            # check if all other packets are in direction from server to client:
 
-        if float(self._request_stream_frames[-1].norm_time) >= float(
-            self._response_stream_frames[0].norm_time
-        ):
-            raise ParsingError(
-                "Request packets appear after first response packet. Is this really HTTP/0.9?",
-                trace=self,
-            )
+            if float(self._request_stream_frames[-1].norm_time) >= float(
+                self._response_stream_frames[0].norm_time
+            ):
+                raise HTTP09Error(
+                    "Request packets appear after first response packet. Is this really HTTP/0.9?",
+                    trace=self,
+                )
 
-        # calculate times
-        ttfb = self._response_stream_frames[0].norm_time
-        pglt = self._response_stream_frames[-1].norm_time
-        req_start = self._request_stream_frames[0].norm_time
-        resp_delay = ttfb - req_start
-        self._facts["ttfb"] = ttfb
-        self._facts["plt"] = pglt
-        self._facts["request_start"] = req_start
-        self._facts["response_delay"] = resp_delay
-
-        # check fin bit
-        self._facts["response_stream_fin_pkns"] = self.get_stream_fin_packet_number(
-            self._response_stream_frames
-        )
+            # calculate times
+            ttfb = self._response_stream_frames[0].norm_time
+            pglt = self._response_stream_frames[-1].norm_time
+            req_start = self._request_stream_frames[0].norm_time
+            resp_delay = ttfb - req_start
+            self._facts["ttfb"] = ttfb
+            self._facts["plt"] = pglt
+            self._facts["request_start"] = req_start
+            self._facts["response_delay"] = resp_delay
+        except HTTP09Error as err:
+            if self._error_cfg[HTTP09Error] == "error":
+                raise err
+            else:
+                self._facts["is_http09"] = False
+                cprint(f"⨯ Validation error: {err}", file=sys.stderr, color="red")
+
+        try:
+            # check fin bit
+            self._facts["response_stream_fin_pkns"] = self.get_stream_fin_packet_number(
+                self._response_stream_frames
+            )
+        except FinError as err:
+            if self._error_cfg[FinError] == "error":
+                raise err
+            else:
+                cprint(f"⨯ Validation error: {err}", file=sys.stderr, color="red")
 
         self._parsed = True
 
     def iter_stream_frames(self, packet: "Packet") -> Iterator:
         for quic_layer in packet.get_multiple_layers("quic"):
             if hasattr(quic_layer, "decryption_failed"):
-                raise ParsingError(
+                raise CryptoError(
                     f"Decryption of QUIC crypto failed: {quic_layer.decryption_failed}",
                     trace=self,
                 )
@@ -469,7 +504,7 @@ class Trace:
 
         if not all(byte is not None for byte in buf):
             cprint(
-                "[!] Warning! Did not receive all bytes in follow_stream.",
+                "⨯ Warning! Did not receive all bytes in follow_stream.",
                 color="yellow",
             )
 
@@ -545,11 +580,11 @@ class Trace:
             msg = "Last packet that contains a stream frame does not close stream."
 
             if warn_only:
-                cprint(f"[!] {msg}", color="red", file=sys.stderr)
+                cprint(f"⨯ {msg}", color="red", file=sys.stderr)
 
                 return []
             else:
-                raise ParsingError(msg, trace=self)
+                raise FinError(msg, trace=self)
         elif len(stream_fins) == 1:
             fin_pkn: int = stream_fins[0]["packet_number"]
 
@@ -563,11 +598,11 @@ class Trace:
                 )
 
                 if warn_only:
-                    cprint(f"[!] {msg}", color="red", file=sys.stderr)
+                    cprint(f"⨯ {msg}", color="red", file=sys.stderr)
 
                     return []
                 else:
-                    raise ParsingError(
+                    raise FinError(
                         msg,
                         trace=self,
                     )
@@ -582,11 +617,11 @@ class Trace:
                 msg = "There are multiple stream ids in this list. Is it HTTP/3?"
 
                 if warn_only:
-                    cprint(f"[!] {msg}", color="red", file=sys.stderr)
+                    cprint(f"⨯ {msg}", color="red", file=sys.stderr)
 
                     return []
                 else:
-                    raise ParsingError(msg, trace=self)
+                    raise FinError(msg, trace=self)
 
             if not all(
                 fin_pkg["offset"] == stream_fins[0]["offset"] for fin_pkg in stream_fins
@@ -594,11 +629,11 @@ class Trace:
                 msg = "Stream was closed multiple times."
 
                 if warn_only:
-                    cprint(f"[!] {msg}", color="red", file=sys.stderr)
+                    cprint(f"⨯ {msg}", color="red", file=sys.stderr)
 
                     return []
                 else:
-                    raise ParsingError(msg, trace=self)
+                    raise FinError(msg, trace=self)
 
             assert max_offset == stream_fins[0]["offset"]