Add benchmarks for random DFAs

.gitignore

@@ -6,3 +6,4 @@ valmari_fig4/benchmark.tex
 valmari_fig4/benchmark.tmp
 valmari_fig4/benchmarks.png
 prism/bench/
+dfa/bench/

dfa/bench.py

+#!/usr/bin/env python
+
+import sys
+import json
+import subprocess
+import os
+
+if len(sys.argv) != 5:
+    print("Usage: %s GENERATOR MA N M" % sys.argv[0])
+    sys.exit(1)
+
+converter = sys.argv[1]
+ma = sys.argv[2]
+n = sys.argv[3]
+m = sys.argv[4]
+
+# Generate random dfa
+os.makedirs("bench", exist_ok=True)
+filename = "bench/dfa_%s_%s.coalgebra" % (n, m)
+subprocess.run([converter, n, m, filename])
+
+# Run ma on it
+out = subprocess.run([ma, 'refine', '--stats-json', filename],
+                     stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+stats = json.loads(out.stderr.decode('utf-8'))
+print("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (n, m,
+                                              stats['states'],
+                                              stats['edges'],
+                                              stats['initial-partition-size'],
+                                              stats['final-partition-size'],
+                                              stats['overall-duration'],
+                                              stats['parse-duration'],
+                                              stats['refine-duration']))

dfa/benchmark.org

+#+TITLE: Random DFAs
+
+#+BEGIN_SRC sh :hlines yes :colnames '("N" "M" "Stats" "Edges" "I" "Q" "t (s)" "t_p (s)" "t_r (s)")
+for n in 10 100 1000; do
+    for m in 10 1000; do
+	python bench.py ../../ma/bin/random-dfa ../../ma/bin/ma $n $m;
+    done
+done
+#+END_SRC
+
+#+RESULTS:
+|    N |    M | Stats |   Edges | I |    Q |         t (s) |     t_p (s) |       t_r (s) |
+|------+------+-------+---------+---+------+---------------+-------------+---------------|
+|   10 |   10 |    20 |     110 | 3 |   20 |   0.002219078 | 0.001427121 |   0.000741096 |
+|   10 | 1000 |    20 |   10010 | 3 |   20 |   0.207475028 | 0.032543624 |   0.174878671 |
+|  100 |   10 |   200 |    1100 | 3 |  200 |   0.010397627 | 0.006340301 |   0.003751082 |
+|  100 | 1000 |   200 |  100100 | 3 |  200 |   3.801247482 | 0.325236763 |   3.475673292 |
+| 1000 |   10 |  2000 |   11000 | 3 | 2000 |   0.137027394 | 0.057381985 |   0.078047319 |
+| 1000 | 1000 |  2000 | 1001000 | 3 | 2000 | 108.246297423 | 3.376023696 | 104.862454495 |