diff --git a/.gitignore b/.gitignore index d58b115f4dbd27fde63344c12623a2d49d02419c..ee7737ae915dab77b41305a79eed086720200125 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ valmari_fig4/benchmark.tex valmari_fig4/benchmark.tmp valmari_fig4/benchmarks.png prism/bench/ +dfa/bench/ diff --git a/dfa/bench.py b/dfa/bench.py new file mode 100644 index 0000000000000000000000000000000000000000..d712de62780f060f26fdec3ab206f6e08e6f58b4 --- /dev/null +++ b/dfa/bench.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +import sys +import json +import subprocess +import os + +if len(sys.argv) != 5: + print("Usage: %s GENERATOR MA N M" % sys.argv[0]) + sys.exit(1) + +converter = sys.argv[1] +ma = sys.argv[2] +n = sys.argv[3] +m = sys.argv[4] + +# Generate random dfa +os.makedirs("bench", exist_ok=True) +filename = "bench/dfa_%s_%s.coalgebra" % (n, m) +subprocess.run([converter, n, m, filename]) + +# Run ma on it +out = subprocess.run([ma, 'refine', '--stats-json', filename], + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) +stats = json.loads(out.stderr.decode('utf-8')) +print("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (n, m, + stats['states'], + stats['edges'], + stats['initial-partition-size'], + stats['final-partition-size'], + stats['overall-duration'], + stats['parse-duration'], + stats['refine-duration'])) diff --git a/dfa/benchmark.org b/dfa/benchmark.org new file mode 100644 index 0000000000000000000000000000000000000000..b48e65715527dcf49036dbf2bce3fe29adb937be --- /dev/null +++ b/dfa/benchmark.org @@ -0,0 +1,19 @@ +#+TITLE: Random DFAs + +#+BEGIN_SRC sh :hlines yes :colnames '("N" "M" "Stats" "Edges" "I" "Q" "t (s)" "t_p (s)" "t_r (s)") +for n in 10 100 1000; do + for m in 10 1000; do + python bench.py ../../ma/bin/random-dfa ../../ma/bin/ma $n $m; + done +done +#+END_SRC + +#+RESULTS: +| N | M | Stats | Edges | I | Q | t (s) | t_p (s) | t_r (s) | +|------+------+-------+---------+---+------+---------------+-------------+---------------| +| 10 | 10 | 20 | 110 | 3 | 20 | 0.002219078 | 0.001427121 | 0.000741096 | +| 10 | 1000 | 20 | 10010 | 3 | 20 | 0.207475028 | 0.032543624 | 0.174878671 | +| 100 | 10 | 200 | 1100 | 3 | 200 | 0.010397627 | 0.006340301 | 0.003751082 | +| 100 | 1000 | 200 | 100100 | 3 | 200 | 3.801247482 | 0.325236763 | 3.475673292 | +| 1000 | 10 | 2000 | 11000 | 3 | 2000 | 0.137027394 | 0.057381985 | 0.078047319 | +| 1000 | 1000 | 2000 | 1001000 | 3 | 2000 | 108.246297423 | 3.376023696 | 104.862454495 |