From 286c56e7b60fc61ed356080c108037b2b20ce8e6 Mon Sep 17 00:00:00 2001
From: Tim Rheinfels <tim.rheinfels@fau.de>
Date: Wed, 24 May 2023 08:46:47 +0200
Subject: [PATCH] scripts: add benchmark class
---
scripts/benchmark.py | 124 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 124 insertions(+)
create mode 100644 scripts/benchmark.py
diff --git a/scripts/benchmark.py b/scripts/benchmark.py
new file mode 100644
index 0000000..a187ed7
--- /dev/null
+++ b/scripts/benchmark.py
@@ -0,0 +1,124 @@
+## This file is part of the execution-time evaluation for the qronos observer abstractions.
+## Copyright (C) 2022-2023 Tim Rheinfels <tim.rheinfels@fau.de>
+## See https://gitlab.cs.fau.de/qronos-state-abstractions/execution-time
+##
+## This program is free software: you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation, either version 3 of the License, or
+## (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+###
+### @file benchmark.py
+###
+### @brief Provides the @ref benchmark.Benchmark class used for all execution time benchmarks
+###
+### @author Tim Rheinfels <tim.rheinfels@fau.de>
+###
+
+import re
+import numpy as np
+
+###
+### @brief Encapsulates an execution time benchmark, i.e., a collection of related campaigns parametrized by a key
+###
+class Benchmark(object):
+
+ ###
+ ### @brief Validates the additional data supplied by a @p campaign
+ ###
+ ### @param campaign Dictionary associated with the campaign to validate
+ ### @param key Key describing the campaign
+ ###
+ ### @note This function must be overloaded by all child classes
+ ###
+ def _validate(self, campaign, key):
+ raise NotImplementedError('Benchmark may not be used directly!')
+
+ ###
+ ### @brief Constructor
+ ###
+ ### @param data Dictionarized JSON data as per @ref data.load_json_data
+ ### @param name Name for the benchmark
+ ### @param regex Regex used to match all relevant campaigns
+ ### @param key_function Function transforming an re.match object into a string identifying the matched campaign
+ ###
+ def __init__(self, data, name, regex, key_function):
+ assert(isinstance(name, str))
+ assert(isinstance(regex, str))
+
+ self._name = name
+ self._execution_times = {}
+
+ # Gather and validate campaigns
+ regex = re.compile(regex)
+ for campaign in data['execution_time_measurements']:
+
+ # Skip campaign's not matching the given regex
+ match = regex.match(campaign['name'])
+ if match is None:
+ continue
+
+ # Determine key
+ key = key_function(match)
+
+ # Delegate to child class for validation
+ self._validate(campaign, key)
+
+ # Gather execution times over all runs into an array
+ execution_times = []
+ for run in campaign['tests']:
+ execution_times.append(np.array(run['repetitions'], dtype=int))
+
+ # Don't overwrite data
+ assert(key not in self._execution_times)
+
+ # Actually store execution times
+ self._execution_times[key] = np.array(execution_times)
+
+ ###
+ ### @brief Getter for the benchmark's name
+ ###
+ ### @returns Benchmark's name
+ ###
+ @property
+ def name(self):
+ return self._name
+
+ ###
+ ### @brief Getter for the benchmark's raw execution times in CPU cycles
+ ###
+ ### @returns Dictionary mapping the campaigns' keys to their execution time data.
+ ### The latter are two-dimensional np.ndarray instances with the first
+ ### dimension enumerating the runs and the second one the repetitions.
+ ###
+ @property
+ def execution_times(self):
+ return self._execution_times
+
+ ###
+ ### @brief Computes some statistics from the linearized (i.e., treate runs and repetitions equally) execution time data in CPU cycles
+ ###
+ ### @returns Dictionary mapping the campaigns' keys to the statistics.
+ ### The latter are again dictionaries containing the keys
+ ### - mean: empirical mean over all samples,
+ ### - std: empirical standard deviation over all samples,
+ ### - min: minimum over all samples,
+ ### - max: maximum over all samples and
+ ### - cov: empirical coefficient of variation (standard deviation over mean) over all samples.
+ ###
+ def compute_statistics(self):
+ return {k: {
+ 'mean': np.mean(v),
+ 'std': np.std(v),
+ 'min': np.min(v),
+ 'max': np.max(v),
+ 'cov': np.std(v) / np.mean(v),
+ } for k, v in self.execution_times.items()}
--
GitLab