Skip to content
Snippets Groups Projects
Commit 0c5b3223 authored by Tim Rheinfels's avatar Tim Rheinfels
Browse files

scripts: add evaluation script

parent 21e8c928
No related branches found
No related tags found
No related merge requests found
## This file is part of the execution-time evaluation for the qronos observer abstractions.
## Copyright (C) 2022-2023 Tim Rheinfels <tim.rheinfels@fau.de>
## See https://gitlab.cs.fau.de/qronos-state-abstractions/execution-time
##
## This program is free software: you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program. If not, see <https://www.gnu.org/licenses/>.
###
### @file evaluate.py
###
### @brief This script uses the data obtained from the STM32
### and runs the evaluation by computing statistics and
### the execution time bar chart.
###
### @author Tim Rheinfels <tim.rheinfels@fau.de>
###
import logging
import matplotlib.pyplot as plot
import os
import seaborn as sb
import sys
import tabulate
from benchmarks.blind_abstraction import BlindAbstraction
from benchmarks.observer_abstraction import ObserverAbstraction
from benchmarks.luenberger_observer import LuenbergerObserver
from benchmarks.kalman_filter import KalmanFilter
from data import load_json_data
# ===== Configuration =====
# Measurement sizes n_y to be plotted in the bar graph
measurement_sizes = (1, 10, 20, 30, 40, 50, 60)
# ===== Configuration End =====
# Taken from
# https://gist.github.com/SeanSyue/8c8ff717681e9ecffc8e43a686e68fd9
"""
Since os.listdir returns filenames in an arbitary order,
this function is very handy for generating well-ordered filenames list.
Credit: https://stackoverflow.com/questions/4813061/non-alphanumeric-list-order-from-os-listdir/48030307#48030307
"""
import re
def sorted_alphanumeric(data):
convert = lambda text: int(text) if text.isdigit() else text.lower()
alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
return sorted(data, key=alphanum_key)
if len(sys.argv) != 3:
print('Usage: %s <input_file> <result_dir>' % sys.argv[0])
print('')
print('Parameters:')
print(' input_file: File to read the data from')
print(' result_dir: Diretory to store the results in')
print('')
print()
sys.exit(1)
data = load_json_data(sys.argv[1])
result_dir = sys.argv[2]
# Load and process benchmark results
benchmarks = [
BlindAbstraction(data),
ObserverAbstraction(data),
LuenbergerObserver(data),
KalmanFilter(data),
]
# Create result dirs
os.makedirs(result_dir, exist_ok = True)
os.makedirs(os.path.join(result_dir, 'statistics'), exist_ok = True)
# Tabularise execution times, print, and store
for benchmark in benchmarks:
statistics = benchmark.compute_statistics()
headers = ('Benchmark \ Execution Time / cycles', 'mean', 'std', 'cov', 'min', 'max')
rows = []
for key in sorted_alphanumeric(statistics.keys()):
rows.append((key, statistics[key]['mean'], statistics[key]['std'], statistics[key]['cov'], statistics[key]['min'], statistics[key]['max']))
rows.append(('', '', '', '', '', ''))
table = tabulate.tabulate(rows, headers)
# To stdout
print('Statistics for Benchmark "%s":' % benchmark.name)
print(table)
print()
# To file
with open(os.path.join(result_dir, 'statistics', '%s.txt' % benchmark.name), 'w') as f:
f.write(table)
# Create bar graph figure comparing the execution times
_, ax = plot.subplots(1, figsize=(10, 3.33))
x = []
y = []
hue = []
for benchmark in benchmarks:
statistics = benchmark.compute_statistics()
for n_y in measurement_sizes:
key = str(n_y)
if key not in statistics.keys():
logging.warn('Measurement size n_y=%d not contained in benchmark %s' % (n_y, benchmark.name))
execution_times = benchmark.execution_times[key].flatten()
n = len(execution_times)
x.extend([n_y] * n)
y.extend(execution_times)
hue.extend([benchmark.name] * n)
bars = sb.barplot(x=x, y=y, hue=hue, errorbar=None, linewidth=8, ax=ax, alpha=0.99)
# Set hatches
hatches = (None, '\\\\', 'xx', '//')
for i, patch in enumerate(bars.patches):
patch.set_hatch(hatches[int(i / len(measurement_sizes))])
ax.set_yscale('log')
ax.set_title('Execution Time Measurements on Cortex-M4')
ax.set_xlabel(r'Measurement Dimension $n_y$')
ax.set_ylabel(r'Execution Time in CPU Cycles')
ax.grid()
ax.legend(loc='upper left')
# Save
plot.savefig(os.path.join(result_dir, 'execution_times.pdf'), bbox_inches='tight')
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment