Commit 21eef418 authored by Hans-Peter Deifel's avatar Hans-Peter Deifel
Browse files

dfa: Repeat benchmarks for final=10%

parent 8b148883
......@@ -53,6 +53,9 @@ def generate(args):
("valmari:%s" % valmari_file(n, m, i, final))]
if dot:
cmd.append("dot:%s" % dot_file(n, m, i, final))
if final is not None:
cmd.append("--final")
cmd.append(str(final))
subprocess.run(cmd)
......
......@@ -253,7 +253,7 @@ done
*** Generating some benchmarks
#+begin_src sh :results output silent
for n in 100000 110000 120000 130000 140000 150000 160000 170000 180000 190000 200000 200000 210000 220000 230000 240000 250000 260000 270000 280000 290000 300000; do
for n in 100000 110000 120000 130000 140000 150000 160000 170000 180000 190000; do
python bench.py generate --final 10 ../../copar/bin/random-dfa $n 2
done
#+end_src
......@@ -261,36 +261,25 @@ done
*** And running them
#+begin_src sh :hlines yes :colnames '("N" "M" "i" "States" "Edges" "I" "Q" "Qs1" "Opts" "t (s)" "sdt" "tp (s)" "sdtp" "ta (s)" "sdta" "ti (s)" "sdti" "tr (s)" "sdtr" "tv (s)" "sdtv" "Qv")
for n in 100000 110000 120000 130000 140000 150000 160000 170000 180000 190000 200000 210000 220000 230000 240000 250000 260000 270000 280000 290000 300000; do
for n in 100000 110000 120000 130000 140000 150000 160000 170000 180000 190000; do
python bench.py run ../../copar/bin/copar ../../valmari_cpp/mdpmin $n 2 --stddev --final 10
done
#+end_src
#+NAME: results2f
#+RESULTS:
| N | M | i | States | Edges | I | Q | Qs1 | Opts | t (s) | sdt | tp (s) | sdtp | ta (s) | sdta | ti (s) | sdti | tr (s) | sdtr | tv (s) | sdtv | Qv |
|--------+---+----+--------+--------+---+--------+--------+--------+--------------------+----------------------+--------------------+----------------------+--------------------+----------------------+--------------------+----------------------+--------------------+----------------------+---------------------+-----------------------+--------|
| 100000 | 2 | 10 | 100000 | 200000 | 2 | 100000 | 100000 | 180429 | 4.6568444347 | 0.058527883396038406 | 1.7348818524999998 | 0.01462464143493326 | 2.6936296465000003 | 0.054734468222638714 | 0.6604005990999999 | 0.00887773341749612 | 1.9302998316 | 0.06813873002661908 | 0.260243685 | 0.00619607132237678 | 100000 |
| 110000 | 2 | 10 | 110000 | 220000 | 2 | 110000 | 110000 | 194181 | 5.487984598399999 | 0.03677489548715484 | 2.0849763404999995 | 0.013033096209312523 | 3.1538232323 | 0.03891841637706378 | 0.7408289973000001 | 0.004290242621115111 | 2.3083699563999995 | 0.03862573600897192 | 0.3247625314 | 0.08778332069569607 | 110000 |
| 120000 | 2 | 10 | 120000 | 240000 | 2 | 120000 | 120000 | 216978 | 5.872806709400001 | 0.04057075606408527 | 2.1735978853 | 0.010461575320678795 | 3.4058449672999997 | 0.03955230946142951 | 1.0130289243 | 0.012553965655794792 | 2.2804973610999997 | 0.04094161827320676 | 0.34707446389999996 | 0.011063358108450386 | 120000 |
| 130000 | 2 | 10 | 130000 | 260000 | 2 | 129998 | 129998 | 234142 | 6.6249887927 | 0.051553196456040506 | 2.2687783852 | 0.017899831914457928 | 3.9798134438999995 | 0.09002661144330304 | 0.8564925456999999 | 0.020559302138173568 | 2.8150207071 | 0.039084047776092486 | 0.3920059233 | 0.008585050142312313 | 129998 |
| 140000 | 2 | 10 | 140000 | 280000 | 2 | 140000 | 140000 | 253054 | 7.188063440799999 | 0.11439343317463105 | 2.4625089368 | 0.010484528850477478 | 4.3730445522 | 0.10680711080329569 | 0.9280604829000001 | 0.013106168580195563 | 3.0580406714999997 | 0.08901168304407762 | 0.4342303363 | 0.0061027343432039785 | 140000 |
| 150000 | 2 | 10 | 150000 | 300000 | 2 | 150000 | 150000 | 269532 | 7.7675284943000005 | 0.057302901719499595 | 2.6507870193 | 0.021638322386182515 | 4.735069655399999 | 0.049109113626300264 | 0.9872577331000001 | 0.033615407395654946 | 3.3284584128000008 | 0.047805994580287836 | 0.4858470692 | 0.008638716696392634 | 150000 |
| 160000 | 2 | 10 | 160000 | 320000 | 2 | 159998 | 159998 | 288317 | 8.162501730399999 | 0.132239973649742 | 2.8353262398 | 0.019333427454770916 | 4.9234886459 | 0.1339251022439333 | 0.8381714475 | 0.14734010612015583 | 3.5649518792 | 0.049808602339306625 | 0.5243544054999999 | 0.007972531054517262 | 159998 |
| 170000 | 2 | 10 | 170000 | 340000 | 2 | 170000 | 170000 | 302734 | 8.7710292719 | 0.1656880332431635 | 3.039674026 | 0.015630910160107554 | 5.2914982907 | 0.16092504514739314 | 0.9055937719999998 | 0.17913360978473875 | 3.8363213732000006 | 0.07044769639545183 | 0.5807324186 | 0.012099886381819783 | 170000 |
| 180000 | 2 | 10 | 180000 | 360000 | 2 | 180000 | 180000 | 325689 | 9.2239578424 | 0.0560281001133905 | 3.0734243586 | 0.03773679098170366 | 5.3476711036 | 0.07135131997481992 | 1.1999275460999999 | 0.02053233042395153 | 3.9828866976 | 0.04561030910330849 | 0.6096904333000001 | 0.012359930956957494 | 180000 |
| 190000 | 2 | 10 | 190000 | 380000 | 2 | 190000 | 190000 | 343720 | 9.798373725800001 | 0.06171425409658723 | 3.2575650536000005 | 0.015220548743439194 | 5.6846106574 | 0.06431141207499487 | 1.2697596982000001 | 0.02242630520965308 | 4.2416012297 | 0.04541661526261202 | 0.651785648 | 0.005198776520454344 | 190000 |
| 200000 | 2 | 10 | 200000 | 400000 | 2 | 200000 | 200000 | 364749 | 10.3989935911 | 0.051947115582823604 | 3.4345249083 | 0.011669143634083457 | 6.0665379446 | 0.05010174311342416 | 1.3352725344 | 0.024650684130856693 | 4.5513793046 | 0.03143908359614278 | 0.6999923338999999 | 0.008710238267230191 | 200000 |
| 210000 | 2 | 10 | 210000 | 420000 | 2 | 210000 | 210000 | 380963 | 10.9311541506 | 0.11059089249288392 | 3.9589549679 | 0.024537898559361306 | 6.461900529999999 | 0.09118072448621002 | 1.4144355546 | 0.029356151481915515 | 4.8529435599 | 0.0699696111206333 | 0.7555080152 | 0.007129740996986406 | 210000 |
| 220000 | 2 | 10 | 220000 | 440000 | 2 | 220000 | 220000 | 389261 | 11.4050210695 | 0.09243069521170397 | 4.0585876911 | 0.020408283591629547 | 6.7290650021000005 | 0.08662883541233364 | 1.4688709124 | 0.014937438973863433 | 5.0498051762 | 0.08174896796681241 | 0.7966641716 | 0.009123862511326363 | 220000 |
| 230000 | 2 | 10 | 230000 | 460000 | 2 | 230000 | 230000 | 417097 | 11.791442098800001 | 0.10025678043713196 | 4.163783029999999 | 0.019994967324833007 | 6.985631379200001 | 0.05702333205654814 | 1.928874545 | 0.029511701477768375 | 4.8496531518 | 0.05600437806476062 | 0.8467098109 | 0.006685651197362439 | 230000 |
| 240000 | 2 | 10 | 240000 | 480000 | 2 | 240000 | 240000 | 433432 | 12.659923065400001 | 0.2675885379453588 | 4.405317828199999 | 0.08197248302375727 | 7.6254524386 | 0.17830942326894064 | 1.7368250154000002 | 0.04205186500025638 | 5.6393416299000005 | 0.17482780861915292 | 0.9323254727 | 0.024270229811434167 | 240000 |
| 250000 | 2 | 10 | 250000 | 500000 | 2 | 250000 | 250000 | 452283 | 13.8911977978 | 0.08006921968638082 | 4.5636413582 | 0.02324751159168931 | 8.543071309899998 | 0.18630101543142064 | 1.7371997350000001 | 0.05140599251079133 | 6.1768473797 | 0.07337087951253254 | 0.9865221917000001 | 0.017418052961143994 | 250000 |
| 260000 | 2 | 10 | 260000 | 520000 | 2 | 260000 | 260000 | 468979 | 14.7337205455 | 0.1158449050093252 | 4.7341499426 | 0.05903349667999552 | 8.7368639348 | 0.11474688974638758 | 1.8829399810999998 | 0.04249956749628621 | 6.598601957 | 0.07142703179554347 | 1.0579299272999998 | 0.012347994607491182 | 260000 |
| 270000 | 2 | 10 | 270000 | 540000 | 2 | 270000 | 270000 | 479175 | 15.2313977862 | 0.11121813515453423 | 4.8893074053 | 0.02071757215362744 | 9.017199511 | 0.0915946997733599 | 1.9277806137 | 0.054222853614453295 | 6.8408429623 | 0.05219021195485631 | 1.1233513739999998 | 0.011059440482615417 | 270000 |
| 280000 | 2 | 10 | 280000 | 560000 | 2 | 280000 | 280000 | 505093 | 15.1466690124 | 0.16259218002854425 | 4.9144513019 | 0.06820816781846911 | 8.941522280900001 | 0.12558439686776457 | 1.9068776829 | 0.046194701662063374 | 6.7797285739 | 0.1456938164513414 | 1.1177862293999998 | 0.008421228210162581 | 280000 |
| 290000 | 2 | 10 | 290000 | 580000 | 2 | 290000 | 290000 | 524556 | 15.9283874981 | 0.4545466290044535 | 5.6799609357 | 0.12279721971070262 | 9.5173838921 | 0.32497052262318527 | 2.0465387604 | 0.07256452884066436 | 7.183017400599999 | 0.24323250979794542 | 1.1964544571000002 | 0.03254000449685574 | 290000 |
| 300000 | 2 | 10 | 300000 | 600000 | 2 | 299998 | 299998 | 542526 | 17.195959921100002 | 0.13704580986960677 | 6.0720172687 | 0.021744378723982227 | 10.330271570499999 | 0.11671365124705374 | 2.2160266842 | 0.03075408800026372 | 7.800040716299999 | 0.12370272908701148 | 1.3249585834000002 | 0.13266759446729978 | 299998 |
| N | M | i | States | Edges | I | Q | Qs1 | Opts | t (s) | sdt | tp (s) | sdtp | ta (s) | sdta | ti (s) | sdti | tr (s) | sdtr | tv (s) | sdtv | Qv |
|--------+---+----+--------+--------+---+--------+--------+--------+--------------------+----------------------+--------------------+----------------------+--------------------+----------------------+---------------------+----------------------+--------------------+----------------------+---------------------+-----------------------+--------|
| 100000 | 2 | 10 | 100000 | 200000 | 2 | 100000 | 100000 | 160352 | 3.0936953513 | 0.015467818313428423 | 1.4124696133 | 0.005943672222068957 | 1.4574320088000001 | 0.017000822749439697 | 0.42203191440000004 | 0.004520660260577265 | 0.9797939001000001 | 0.018105061634433976 | 0.2695312297 | 0.0069361300204224706 | 100000 |
| 110000 | 2 | 10 | 110000 | 220000 | 2 | 109999 | 109999 | 176657 | 3.5784416617 | 0.01508095774483378 | 1.5098241521 | 0.004227983506536663 | 1.6316288086 | 0.012239722621501683 | 0.4385282887 | 0.006495824492836184 | 1.1376198136 | 0.014767401631046848 | 0.31170653470000004 | 0.006848544046025738 | 109999 |
| 120000 | 2 | 10 | 120000 | 240000 | 2 | 119999 | 119999 | 191218 | 3.9010189149000007 | 0.021209169527059025 | 1.7667944406 | 0.008668441762979982 | 1.8386231196 | 0.02984760040605402 | 0.5990321109 | 0.014804404733360246 | 1.1508279068 | 0.021326970463733233 | 0.3524312471 | 0.004316307776005543 | 119999 |
| 130000 | 2 | 10 | 130000 | 260000 | 2 | 130000 | 130000 | 208170 | 4.2108411467 | 0.030429870784823763 | 1.8758927709999997 | 0.014720813295526257 | 2.0216873166 | 0.02944166619615172 | 0.5635494682000001 | 0.006692369051929955 | 1.3816380033999998 | 0.029729760194715515 | 0.3983277031 | 0.0024183986266932186 | 130000 |
| 140000 | 2 | 10 | 140000 | 280000 | 2 | 139999 | 139999 | 225471 | 4.6525796292 | 0.08226665976530252 | 2.0223978788 | 0.02782859024824926 | 2.2457517173 | 0.06419042143691525 | 0.43292923410000006 | 0.07437008468651318 | 1.5179340291999999 | 0.028571780925593183 | 0.4464015892 | 0.008337702174814625 | 139999 |
| 150000 | 2 | 10 | 150000 | 300000 | 2 | 150000 | 150000 | 240331 | 4.9966277785 | 0.05561432863860171 | 2.1685035476999994 | 0.011273739740092388 | 2.4161792503 | 0.05218254165871221 | 0.4491742889 | 0.04435758243758258 | 1.6487869333999998 | 0.03158639366001329 | 0.4903136348 | 0.0035377975274226065 | 150000 |
| 160000 | 2 | 10 | 160000 | 320000 | 2 | 159999 | 159999 | 256952 | 5.355174741499999 | 0.01181544267095224 | 2.3217772239000003 | 0.009423441961012942 | 2.5912845906 | 0.009615174702699738 | 0.4742331759000001 | 0.00567499301007789 | 1.774249567 | 0.00497004110981295 | 0.5373223992 | 0.006688940145895668 | 159999 |
| 170000 | 2 | 10 | 170000 | 340000 | 2 | 169999 | 169999 | 273299 | 5.757032531199999 | 0.02661150144562761 | 2.4794814604999997 | 0.012372017151358293 | 2.8041343125 | 0.015352615509927985 | 0.5093437867 | 0.005037497584885556 | 1.9293367177999996 | 0.009725668899383278 | 0.58678976 | 0.006087850747350132 | 169999 |
| 180000 | 2 | 10 | 180000 | 360000 | 2 | 180000 | 180000 | 289908 | 6.1609400376 | 0.03188833978682093 | 2.6309119938999994 | 0.00791867193670523 | 3.0293672266000002 | 0.035760581136465915 | 0.5431468218 | 0.006141095107119764 | 2.0932277568999997 | 0.0332985663387355 | 0.6429327362999999 | 0.007674756785721323 | 180000 |
| 190000 | 2 | 10 | 190000 | 380000 | 2 | 190000 | 190000 | 307527 | 6.778789601 | 0.06991943979616976 | 2.7624881334999998 | 0.010831227316887074 | 3.3906752104999995 | 0.1397668131349721 | 0.8261375824 | 0.00893049817644368 | 2.2897163365 | 0.043731496282112814 | 0.6881156208999999 | 0.012255461536873485 | 190000 |
*** And plotting them
......@@ -299,6 +288,37 @@ done
#+RESULTS:
[[file:results2f.png]]
** For m = 2 and 0% finality
*** Generating some benchmarks
#+begin_src sh :results output silent
for n in 100000 110000 120000 130000 140000 150000 160000 170000 180000 190000; do
python bench.py generate --final 0 ../../copar/bin/random-dfa $n 2
done
#+end_src
*** And running them
#+begin_src sh :hlines yes :colnames '("N" "M" "i" "States" "Edges" "I" "Q" "Qs1" "Opts" "t (s)" "sdt" "tp (s)" "sdtp" "ta (s)" "sdta" "ti (s)" "sdti" "tr (s)" "sdtr" "tv (s)" "sdtv" "Qv")
for n in 100000 110000 120000 130000 140000 150000 160000 170000 180000 190000; do
python bench.py run ../../copar/bin/copar ../../valmari_cpp/mdpmin $n 2 --stddev --final 0
done
#+end_src
#+RESULTS:
| N | M | i | States | Edges | I | Q | Qs1 | Opts | t (s) | sdt | tp (s) | sdtp | ta (s) | sdta | ti (s) | sdti | tr (s) | sdtr | tv (s) | sdtv | Qv |
|--------+---+----+--------+--------+---+---+-----+------+--------------------+----------------------+--------------------+----------------------+--------------------+-----------------------+---------------------+-----------------------+------------------------+------------------------+---------------------+-----------------------+----|
| 100000 | 2 | 10 | 100000 | 200000 | 1 | 1 | 1 | 0 | 1.9286943634 | 0.00878107688302739 | 1.3837160297 | 0.008180649292894082 | 0.4171904952 | 0.003120493919147368 | 0.3641316149 | 0.0032160904550749985 | 1.0225800000000001e-05 | 2.1150455734517354e-07 | 0.1024724376 | 0.006073081115665208 | 1 |
| 110000 | 2 | 10 | 110000 | 220000 | 1 | 1 | 1 | 0 | 2.2182804934999996 | 0.01412802800557772 | 1.4834582776 | 0.00738948570673236 | 0.4289651487 | 0.0055944857852055704 | 0.3764284657 | 0.005279970052798764 | 1.19171e-05 | 3.2553220851058316e-06 | 0.11168206160000001 | 0.0021092801129365952 | 1 |
| 120000 | 2 | 10 | 120000 | 240000 | 1 | 1 | 1 | 0 | 2.4764642443 | 0.012935219317291618 | 1.7297758319 | 0.006418203929536284 | 0.5863125886 | 0.009762956606817539 | 0.5235213498 | 0.01025042704825142 | 1.0723899999999999e-05 | 2.6905367081267974e-07 | 0.1266497688 | 0.00756630532110257 | 1 |
| 130000 | 2 | 10 | 130000 | 260000 | 1 | 1 | 1 | 0 | 2.5684661536 | 0.006060926053458022 | 1.8325813717 | 0.004199590188170929 | 0.5551686706 | 0.0026852027358281088 | 0.4815377142 | 0.0030480000798772133 | 1.0317600000000001e-05 | 3.9572190010438157e-07 | 0.1354930515 | 0.003756031590903835 | 1 |
| 140000 | 2 | 10 | 140000 | 280000 | 1 | 1 | 1 | 0 | 2.8226806149000003 | 0.04107118493117886 | 1.9613554895 | 0.00623230387960241 | 0.6217279724999999 | 0.03774254529931291 | 0.33219782079999993 | 0.05402562365110335 | 1.21055e-05 | 3.7677216590401157e-06 | 0.1476372001 | 0.0057871447980288974 | 1 |
| 150000 | 2 | 10 | 150000 | 300000 | 1 | 1 | 1 | 0 | 3.0114131203 | 0.02544552047477694 | 2.1203642880999998 | 0.008153746807082824 | 0.6600289866 | 0.006795559380887498 | 0.347185776 | 0.006751335332853107 | 1.1084300000000002e-05 | 2.0330387245587713e-06 | 0.1582478537 | 0.004157373435323506 | 1 |
| 160000 | 2 | 10 | 160000 | 320000 | 1 | 1 | 1 | 0 | 3.2158140727999998 | 0.015839540316263775 | 2.2723906333999997 | 0.006996337136486422 | 0.7061300778999999 | 0.003739872502854762 | 0.373723791 | 0.004235039390227649 | 1.15646e-05 | 3.6649735848676696e-06 | 0.16917011399999998 | 0.0026638888864231733 | 1 |
| 170000 | 2 | 10 | 170000 | 340000 | 1 | 1 | 1 | 0 | 3.4702468513999998 | 0.0509720428289049 | 2.4451624861 | 0.029551826817010306 | 0.7609187292999999 | 0.013753158673689668 | 0.404949446 | 0.004671610869945024 | 1.05429e-05 | 2.703024848819066e-07 | 0.1830823594 | 0.006077950328655399 | 1 |
| 180000 | 2 | 10 | 180000 | 360000 | 1 | 1 | 1 | 0 | 3.650301588 | 0.014122237424641738 | 2.5787219889 | 0.011997897515023564 | 0.8084984614999999 | 0.00382175626616949 | 0.4325368038 | 0.001311000193778083 | 1.02732e-05 | 4.443144031976556e-07 | 0.19291228479999997 | 0.00398230081639267 | 1 |
| 190000 | 2 | 10 | 190000 | 380000 | 1 | 1 | 1 | 0 | 4.0598550992 | 0.03655234344413544 | 2.7205397726999996 | 0.015378199595674087 | 0.9662325937 | 0.10889493389502679 | 0.7044185453 | 0.008811645674713221 | 1.0971300000000001e-05 | 3.923952230992513e-07 | 0.20248723759999998 | 0.0030261990594331093 | 1 |
** For m = 1000:
#+BEGIN_SRC sh :results output silent
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment