From 6d4285d403bef7da71285ef7c6769561ff9f6734 Mon Sep 17 00:00:00 2001
From: Hans-Peter Deifel <hpd@hpdeifel.de>
Date: Tue, 25 Sep 2018 11:08:49 +0200
Subject: [PATCH] Add benchmarks for random DFAs

---
 .gitignore        |  1 +
 dfa/bench.py      | 33 +++++++++++++++++++++++++++++++++
 dfa/benchmark.org | 19 +++++++++++++++++++
 3 files changed, 53 insertions(+)
 create mode 100644 dfa/bench.py
 create mode 100644 dfa/benchmark.org

diff --git a/.gitignore b/.gitignore
index d58b115..ee7737a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,4 @@ valmari_fig4/benchmark.tex
 valmari_fig4/benchmark.tmp
 valmari_fig4/benchmarks.png
 prism/bench/
+dfa/bench/
diff --git a/dfa/bench.py b/dfa/bench.py
new file mode 100644
index 0000000..d712de6
--- /dev/null
+++ b/dfa/bench.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+import sys
+import json
+import subprocess
+import os
+
+if len(sys.argv) != 5:
+    print("Usage: %s GENERATOR MA N M" % sys.argv[0])
+    sys.exit(1)
+
+converter = sys.argv[1]
+ma = sys.argv[2]
+n = sys.argv[3]
+m = sys.argv[4]
+
+# Generate random dfa
+os.makedirs("bench", exist_ok=True)
+filename = "bench/dfa_%s_%s.coalgebra" % (n, m)
+subprocess.run([converter, n, m, filename])
+
+# Run ma on it
+out = subprocess.run([ma, 'refine', '--stats-json', filename],
+                     stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+stats = json.loads(out.stderr.decode('utf-8'))
+print("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (n, m,
+                                              stats['states'],
+                                              stats['edges'],
+                                              stats['initial-partition-size'],
+                                              stats['final-partition-size'],
+                                              stats['overall-duration'],
+                                              stats['parse-duration'],
+                                              stats['refine-duration']))
diff --git a/dfa/benchmark.org b/dfa/benchmark.org
new file mode 100644
index 0000000..b48e657
--- /dev/null
+++ b/dfa/benchmark.org
@@ -0,0 +1,19 @@
+#+TITLE: Random DFAs
+
+#+BEGIN_SRC sh :hlines yes :colnames '("N" "M" "Stats" "Edges" "I" "Q" "t (s)" "t_p (s)" "t_r (s)")
+for n in 10 100 1000; do
+    for m in 10 1000; do
+	python bench.py ../../ma/bin/random-dfa ../../ma/bin/ma $n $m;
+    done
+done
+#+END_SRC
+
+#+RESULTS:
+|    N |    M | Stats |   Edges | I |    Q |         t (s) |     t_p (s) |       t_r (s) |
+|------+------+-------+---------+---+------+---------------+-------------+---------------|
+|   10 |   10 |    20 |     110 | 3 |   20 |   0.002219078 | 0.001427121 |   0.000741096 |
+|   10 | 1000 |    20 |   10010 | 3 |   20 |   0.207475028 | 0.032543624 |   0.174878671 |
+|  100 |   10 |   200 |    1100 | 3 |  200 |   0.010397627 | 0.006340301 |   0.003751082 |
+|  100 | 1000 |   200 |  100100 | 3 |  200 |   3.801247482 | 0.325236763 |   3.475673292 |
+| 1000 |   10 |  2000 |   11000 | 3 | 2000 |   0.137027394 | 0.057381985 |   0.078047319 |
+| 1000 | 1000 |  2000 | 1001000 | 3 | 2000 | 108.246297423 | 3.376023696 | 104.862454495 |
-- 
GitLab