diff --git a/dfa/benchmark.org b/dfa/benchmark.org
index aa98aec7c205d1d8721ef4d2115ac2f44a0e5e92..ce5c89fbdff37395feee9a9dc8a6d734266f2e3b 100644
--- a/dfa/benchmark.org
+++ b/dfa/benchmark.org
@@ -31,14 +31,14 @@ The fields in the following table are:
- Edges :: Number of edges in the coalgebra
- I :: Size of initial partition
- Q :: Size of final partition
-- Q_s1 :: Size of final partition restricted to sort 1
+- Qs1 :: Size of final partition restricted to sort 1
- t :: Overall time for CoPaR in seconds
-- t_p :: Time to parse the input in seconds
-- t_a :: Overall time to execute the refinement algorithm
-- t_i :: Time for the initialization procedure
-- t_r :: Time for the actual partition refinement
-- t_v :: Overall time for Valmari in seconds
-- Q_v :: Size of final partition that valmari's tool computes
+- tp :: Time to parse the input in seconds
+- ta :: Overall time to execute the refinement algorithm
+- ti :: Time for the initialization procedure
+- tr :: Time for the actual partition refinement
+- tv :: Overall time for Valmari in seconds
+- Qv :: Size of final partition that valmari's tool computes
For aggregated benchmarks, each timing result also has an additional column with
the standard derivation of the samples
@@ -99,12 +99,12 @@ return(filename)
** TODO Test
-#+begin_src sh :hlines yes :colnames '("N" "M" "i" "States" "Edges" "I" "Q" "Q_s1" "Opt" "t (s)" "t_p (s)" "t_a (s)" "t_i (s)" "t_r (s)" "t_v (s)" "Q_v")
+#+begin_src sh :hlines yes :colnames '("N" "M" "i" "States" "Edges" "I" "Q" "Qs1" "Opt" "t (s)" "tp (s)" "ta (s)" "ti (s)" "tr (s)" "tv (s)" "Qv")
python bench.py run ../../copar/bin/copar ../../valmari_cpp/mdpmin 10 10 --indiv
#+end_src
#+RESULTS:
-| N | M | i | States | Edges | I | Q | Q_s1 | Opt | t (s) | t_p (s) | t_a (s) | t_i (s) | t_r (s) | t_v (s) | Q_v |
+| N | M | i | States | Edges | I | Q | Qs1 | Opt | t (s) | tp (s) | ta (s) | ti (s) | tr (s) | tv (s) | Qv |
|----+----+---+--------+-------+---+----+------+-----+-------------+-------------+-------------+------------+------------+-------------+-----|
| 10 | 10 | 0 | 10 | 100 | 2 | 10 | 10 | 83 | 0.001046352 | 0.00092647 | 8.813e-05 | 2.5603e-05 | 4.8467e-05 | 0.0013978 | 10 |
| 10 | 10 | 1 | 10 | 100 | 2 | 10 | 10 | 81 | 0.001332556 | 0.00117165 | 0.000121423 | 4.205e-05 | 6.1448e-05 | 0.001185295 | 10 |
@@ -128,7 +128,7 @@ done
*** And running them
-#+begin_src sh :hlines yes :colnames '("N" "M" "i" "States" "Edges" "I" "Q" "Q_s1" "Opts" "t (s)" "σ" "t_p (s)" "σ" "t_a (s)" "σ" "t_i (s)" "σ" "t_r (s)" "σ" "t_v (s)" "σ" "Q_v")
+#+begin_src sh :hlines yes :colnames '("N" "M" "i" "States" "Edges" "I" "Q" "Qs1" "Opts" "t (s)" "σ" "tp (s)" "σ" "ta (s)" "σ" "ti (s)" "σ" "tr (s)" "σ" "tv (s)" "σ" "Qv")
for n in 100000 110000 120000 130000 140000 150000 160000 170000 180000 190000 200000 210000 220000 230000 240000 250000 260000 270000 280000 290000 300000; do
python bench.py run ../../copar/bin/copar ../../valmari_cpp/mdpmin $n 1 --stddev
done
@@ -136,7 +136,7 @@ done
#+NAME: results1
#+RESULTS:
-| N | M | i | States | Edges | I | Q | Q_s1 | Opts | t (s) | σ | t_p (s) | σ | t_a (s) | σ | t_i (s) | σ | t_r (s) | σ | t_v (s) | σ | Q_v |
+| N | M | i | States | Edges | I | Q | Qs1 | Opts | t (s) | σ | tp (s) | σ | ta (s) | σ | ti (s) | σ | tr (s) | σ | tv (s) | σ | Qv |
|--------+---+----+--------+--------+---+--------+--------+-------+--------------------+---------------------+--------------------+----------------------+--------------------+---------------------+--------------------+----------------------+--------------------+---------------------+---------------------+----------------------+--------|
| 100000 | 1 | 10 | 100000 | 100000 | 2 | 65407 | 65407 | 8066 | 4.5290850337 | 0.11864934189220436 | 1.2670609352 | 0.008980379315232729 | 2.8846108683000002 | 0.10597994596851053 | 0.523533524 | 0.012535445952974214 | 2.3008021567999997 | 0.1055769208893637 | 0.13519612100000003 | 0.013846578756400505 | 65407 |
| 110000 | 1 | 10 | 110000 | 110000 | 2 | 72036 | 72036 | 8623 | 5.0699149869 | 0.07183568383526712 | 1.550735742 | 0.012781696128692933 | 3.2605090468 | 0.07796323372066957 | 0.5889638794 | 0.010617361192314432 | 2.5968446309 | 0.06931852161488808 | 0.1568809575 | 0.00901868889061136 | 72036 |
@@ -178,7 +178,7 @@ done
*** And running them
-#+begin_src sh :hlines yes :colnames '("N" "M" "i" "States" "Edges" "I" "Q" "Q_s1" "Opts" "t (s)" "σ" "t_p (s)" "σ" "t_a (s)" "σ" "t_i (s)" "σ" "t_r (s)" "σ" "t_v (s)" "σ" "Q_v")
+#+begin_src sh :hlines yes :colnames '("N" "M" "i" "States" "Edges" "I" "Q" "Qs1" "Opts" "t (s)" "σ" "tp (s)" "σ" "ta (s)" "σ" "ti (s)" "σ" "tr (s)" "σ" "tv (s)" "σ" "Qv")
for n in 100000 110000 120000 130000 140000 150000 160000 170000 180000 190000 200000 210000 220000 230000 240000 250000 260000 270000 280000 290000 300000; do
python bench.py run ../../copar/bin/copar ../../valmari_cpp/mdpmin $n 2 --stddev
done
@@ -186,7 +186,7 @@ done
#+NAME: results2
#+RESULTS:
-| N | M | i | States | Edges | I | Q | Q_s1 | Opts | t (s) | σ | t_p (s) | σ | t_a (s) | σ | t_i (s) | σ | t_r (s) | σ | t_v (s) | σ | Q_v |
+| N | M | i | States | Edges | I | Q | Qs1 | Opts | t (s) | σ | tp (s) | σ | ta (s) | σ | ti (s) | σ | tr (s) | σ | tv (s) | σ | Qv |
|--------+---+----+--------+--------+---+--------+--------+--------+--------------------+----------------------+--------------------+----------------------+--------------------+----------------------+--------------------+----------------------+--------------------+----------------------+---------------------+-----------------------+--------|
| 100000 | 2 | 10 | 100000 | 200000 | 2 | 100000 | 100000 | 181160 | 4.6682598973 | 0.08297578258883191 | 1.7466071907 | 0.046301398085246505 | 2.6960031217 | 0.05678813781082613 | 0.6481005733 | 0.014042033012002532 | 1.9517191918 | 0.06471030997185793 | 0.26132676630000007 | 0.0069860096592970315 | 100000 |
| 110000 | 2 | 10 | 110000 | 220000 | 2 | 110000 | 110000 | 193895 | 5.4192957357000004 | 0.04358637576199741 | 2.0576445523 | 0.011675123216284503 | 3.1132264053 | 0.04179021345785444 | 0.7028545399 | 0.013187307245525014 | 2.3127195961 | 0.03735404238917737 | 0.3032304626 | 0.007008853475927248 | 110000 |
@@ -225,7 +225,7 @@ for n in 1000 1100 1200 1300 1400 1500 1600 1700 1800 1900 2000 2000 2100 2200 2
done
#+END_SRC
- #+BEGIN_SRC sh :hlines yes :colnames '("N" "M" "i" "States" "Edges" "I" "Q" "Q_s1" "t (s)" "σ" "t_p (s)" "σ" "t_a (s)" "σ" "t_i (s)" "σ" "t_r (s)" "σ" "t_v (s)" "σ" "Q_v")
+ #+BEGIN_SRC sh :hlines yes :colnames '("N" "M" "i" "States" "Edges" "I" "Q" "Qs1" "t (s)" "σ" "tp (s)" "σ" "ta (s)" "σ" "ti (s)" "σ" "tr (s)" "σ" "tv (s)" "σ" "Qv")
python bench.py run --stddev ../../copar/bin/copar ../../valmari_cpp/mdpmin 1000 1000
python bench.py run --stddev ../../copar/bin/copar ../../valmari_cpp/mdpmin 1100 1000
python bench.py run --stddev ../../copar/bin/copar ../../valmari_cpp/mdpmin 1200 1000
@@ -241,7 +241,7 @@ done
#+NAME: results1000
#+RESULTS:
- | N | M | i | States | Edges | I | Q | Q_s1 | t (s) | σ | t_p (s) | σ | t_a (s) | σ | t_i (s) | σ | t_r (s) | σ | t_v (s) | σ | Q_v |
+ | N | M | i | States | Edges | I | Q | Qs1 | t (s) | σ | tp (s) | σ | ta (s) | σ | ti (s) | σ | tr (s) | σ | tv (s) | σ | Qv |
|------+------+----+--------+---------+---+------+------+--------------------+----------------------+--------------------+----------------------+--------------------+----------------------+--------------------+-----------------------+--------------------+----------------------+--------------------+----------------------+------|
| 1000 | 1000 | 10 | 1000 | 1000000 | 2 | 1000 | 1000 | 3.9310659747 | 0.08814522097255945 | 2.6619831237 | 0.055274922278595574 | 1.2615084707 | 0.03296793653488977 | 0.6944962514 | 0.01687407887910384 | 0.4457377868 | 0.013997283312452657 | 1.0652697109 | 0.03344509703654639 | 1000 |
| 1100 | 1000 | 10 | 1100 | 1100000 | 2 | 1100 | 1100 | 4.415969773900001 | 0.014060774244470062 | 2.9951420233 | 0.010950712046698105 | 1.4126832495000001 | 0.006914149208955129 | 0.7695829744999999 | 0.005935086168028538 | 0.5062192168999999 | 0.005052405925752206 | 1.2477546128 | 0.01071710076218558 | 1100 |
@@ -302,7 +302,7 @@ done
** For m = 10000
- #+BEGIN_SRC sh :hlines yes :colnames '("N" "M" "States" "Edges" "I" "Q" "Q_s1" "t (s)" "t_p (s)" "t_a (s)" "t_i (s)" "t_r (s)" "t_v (s)" "Q_v")
+ #+BEGIN_SRC sh :hlines yes :colnames '("N" "M" "States" "Edges" "I" "Q" "Qs1" "t (s)" "tp (s)" "ta (s)" "ti (s)" "tr (s)" "tv (s)" "Qv")
python bench.py run ../../copar/bin/copar ../../valmari_cpp/mdpmin 500 10000
python bench.py run ../../copar/bin/copar ../../valmari_cpp/mdpmin 600 10000
python bench.py run ../../copar/bin/copar ../../valmari_cpp/mdpmin 700 10000
@@ -314,7 +314,7 @@ done
#+NAME: results10000
#+RESULTS:
- | N | M | States | Edges | I | Q | Q_s1 | t (s) | t_p (s) | t_a (s) | t_i (s) | t_r (s) | t_v (s) | Q_v |
+ | N | M | States | Edges | I | Q | Qs1 | t (s) | tp (s) | ta (s) | ti (s) | tr (s) | tv (s) | Qv |
|------+-------+--------+----------+---+------+------+--------------+--------------+-------------+-------------+-------------+--------------+------|
| 500 | 10000 | 500 | 5000000 | 2 | 500 | 500 | 42.104668963 | 37.582293662 | 4.493314329 | 1.518858584 | 2.386916363 | 6.366641481 | 500 |
| 600 | 10000 | 600 | 6000000 | 2 | 600 | 600 | 50.191903227 | 44.747513385 | 5.40920263 | 1.820207355 | 2.88154887 | 7.835340678 | 600 |