wta: Publish more results

d647ff0c · Hans-Peter Deifel · e476eb90 · d647ff0c
Commit d647ff0c authored 6 years ago by Hans-Peter Deifel
--- a/wta/benchmarks.org
+++ b/wta/benchmarks.org
@@ -580,3 +580,353 @@ python bench.py run ../../copar/bin/copar --monoid 'Word,or' --symbols 0,0,0,0,0
 | 2 | 1595591 | 9573480 |                     14 |               473700 |                            11 |       2679546 |     61.459443653 |   24.801616514 |       34.389540062 |        10.893824596 |    21.133849133 |
 | 3 | 1593825 | 9562884 |                     14 |               473580 |                            11 |       2676789 |     61.147557505 |    24.98879381 |       33.942496577 |        10.818043846 |    20.839884809 |
 | 4 | 1596528 | 9579102 |                     14 |               473600 |                            11 |       2681048 |     61.864094058 |   24.884111862 |       34.762990208 |        10.893522867 |    21.485077775 |
+* Same Question, but with limited transition count
+** Powerset
+*** GHC 8.4.4 with symbols 0,8 and transitions 500
+#+BEGIN_SRC sh
+./bench.py bisect ../../copar/bin/{random-wta,copar} --monoid powerset --symbols 0,8 --transitions 5000
+#+END_SRC
+#+RESULTS:
+: Trying 1000...
+: Trying 2000...
+: Trying 1500...
+: Trying 1250...
+: Trying 1375...
+: Trying 1437...
+: Trying 1468...
+: Trying 1484...
+: Trying 1476...
+: Trying 1480...
+: Trying 1478...
+: Trying 1479...
+: First bad state count: 1479
+**** File Size
+#+BEGIN_SRC sh :results output
+ls -sh bench/wta_powerset_0,8_0.7_1478*
+#+END_SRC
+#+RESULTS:
+: 82M bench/wta_powerset_0,8_0.7_1478_0.coalgebra
+: 82M bench/wta_powerset_0,8_0.7_1478_1.coalgebra
+: 82M bench/wta_powerset_0,8_0.7_1478_2.coalgebra
+: 82M bench/wta_powerset_0,8_0.7_1478_3.coalgebra
+: 82M bench/wta_powerset_0,8_0.7_1478_4.coalgebra
+**** Automaton size
+#+begin_src sh :hlines yes
+python bench.py run ../../copar/bin/copar --monoid powerset --symbols 0,8 --zero-frequency 0.7 --states 1478 --indiv --header
+#+end_src
+#+RESULTS:
+| i |  states |    edges | initial-partition-size | final-partition-size | explicit-final-partition-size | size1-skipped | overall-duration | parse-duration | algorithm-duration | initialize-duration | refine-duration |
+| 0 | 5243856 | 10484756 |                     10 |                   18 |                             2 |             0 |    115.339077581 |   34.439478263 |       79.458232767 |        27.712258503 |    37.668249537 |
+| 1 | 5245405 | 10487854 |                     10 |                   18 |                             2 |             0 |    114.001430765 |   34.477874858 |        78.07613373 |        27.764347556 |    37.074617863 |
+| 2 | 5243549 | 10484142 |                     10 |                   18 |                             2 |             0 |    114.862812706 |   34.668409012 |       78.756016051 |        27.741448159 |     37.09480403 |
+| 3 | 5241660 | 10480364 |                     10 |                   18 |                             2 |             0 |    114.228588077 |   34.362576104 |       78.424601434 |        27.648526393 |    37.043127108 |
+| 4 | 5243032 | 10483108 |                     10 |                   18 |                             2 |             0 |    114.692551901 |   34.827430372 |       78.414037199 |        27.723198479 |    38.276163878 |
+*** GHC 8.4.4 with symbols 0,8 and out-degree 1000
+#+BEGIN_SRC sh
+./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,8 --out-degree 1000 --good 1000
+#+END_SRC
+#+RESULTS:
+: Trying 1001...
+: Trying 2002...
+: Trying 4004...
+: Trying 8008...
+: Trying 6006...
+: Trying 7007...
+: Trying 7507...
+: Trying 7257...
+: Trying 7382...
+: Trying 7444...
+: Trying 7413...
+: Trying 7428...
+: Trying 7420...
+: Trying 7416...
+: Trying 7414...
+: Trying 7415...
+: First bad state count: 7416
+**** File Size
+#+BEGIN_SRC sh :results output
+ls -sh bench/wta_powerset_0,8_t7415000__7415*
+#+END_SRC
+#+RESULTS:
+: 120M bench/wta_powerset_0,8_t7415000__7415_0.coalgebra
+: 120M bench/wta_powerset_0,8_t7415000__7415_1.coalgebra
+: 120M bench/wta_powerset_0,8_t7415000__7415_2.coalgebra
+: 120M bench/wta_powerset_0,8_t7415000__7415_3.coalgebra
+: 120M bench/wta_powerset_0,8_t7415000__7415_4.coalgebra
+**** Automaton Size
+#+begin_src sh :hlines yes
+python bench.py run ../../copar/bin/copar --monoid powerset --symbols 0,8 --transitions 7415000 --states 7415 --indiv --header
+#+end_src
+#+RESULTS:
+| i |  states |    edges | initial-partition-size | final-partition-size | explicit-final-partition-size | size1-skipped | overall-duration | parse-duration | algorithm-duration | initialize-duration | refine-duration |
+| 0 | 7422415 | 14830000 |                     10 |                   18 |                             2 |             0 |    157.872966092 |   51.221314999 |      104.615248956 |          33.7012898 |    58.664739269 |
+| 1 | 7422415 | 14830000 |                     10 |                   18 |                             2 |             0 |    157.428380106 |   51.433510926 |      103.926494829 |        33.216537153 |    58.775334589 |
+| 2 | 7422415 | 14830000 |                     10 |                   18 |                             2 |             0 |    156.808413994 |   51.343226544 |       103.40051371 |        32.969125294 |    58.865798525 |
+| 3 | 7422415 | 14830000 |                     10 |                   18 |                             2 |             0 |    157.791363724 |   51.873644617 |      103.855838314 |         33.05713406 |    58.931293578 |
+| 4 | 7422415 | 14830000 |                     10 |                   18 |                             2 |             0 |    157.126256596 |   51.400301031 |      103.666329167 |        32.976280595 |    58.830566981 |
+* Next Try
+We now use the following parameters:
+- out-degree :: 500
+- Signature :: 4×X^k k ∈ { 0, ..., 5 }
+- GHC :: 8.4.4
+** Powerset
+*** 4xX
+#+BEGIN_SRC shell
+./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,4 --out-degree 500 --good 8000 --start-states 12000 --bad 16000
+#+END_SRC
+: Trying 12000...
+: Trying 14000...
+: Trying 15000...
+: Trying 14500...
+: Trying 14250...
+: Trying 14375...
+: Trying 14312...
+: Trying 14281...
+: Trying 14296...
+: Trying 14288...
+: Trying 14284...
+: Trying 14282...
+: First bad state count: 14282
+**** File Size
+#+BEGIN_SRC sh :results output
+ls -sh bench/wta_powerset_0,4_*__14281*
+#+END_SRC
+#+RESULTS:
+: 118M bench/wta_powerset_0,4_t7140500__14281_0.coalgebra
+**** Automaton Size
+#+BEGIN_SRC shell
+./bench.py generate ../../copar/bin/random-wta --monoid powerset --symbols 0,4 --out-degree 500 --states 14281
+./bench.py run ../../copar/bin/copar --monoid powerset --symbols 0,4 --out-degree 500 --states 14281 --indiv --header
+#+END_SRC
+#+RESULTS:
+| i |  states |    edges | initial-partition-size | final-partition-size | explicit-final-partition-size | size1-skipped | overall-duration | parse-duration | algorithm-duration | initialize-duration | refine-duration |
+| 0 | 7154781 | 14281000 |                      6 |                   10 |                             2 |             0 |    155.635043092 |   50.511676534 |      103.239235361 |        30.135940008 |    61.445316625 |
+| 1 | 7154781 | 14281000 |                      6 |                   10 |                             2 |             0 |    155.345664537 |   50.391815858 |      103.046448356 |         29.52881681 |    61.823502784 |
+| 2 | 7154781 | 14281000 |                      6 |                   10 |                             2 |             0 |    154.746389702 |   50.356318464 |      102.487287332 |        29.561887582 |    61.245616512 |
+| 3 | 7154781 | 14281000 |                      6 |                   10 |                             2 |             0 |    156.681042711 |   50.477922803 |      104.306181251 |        30.132009036 |    62.231759233 |
+| 4 | 7154781 | 14281000 |                      6 |                   10 |                             2 |             0 |    156.206931801 |   50.552217812 |      103.775304876 |        30.106012782 |    61.793692847 |
+*** 4xX mit out-degree 20
+#+BEGIN_SRC shell
+./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,4 --out-degree 20 --good 8000 --start-states 14281
+#+END_SRC
+: Trying 14281...
+: Trying 28562...
+: Trying 57124...
+: Trying 114248...
+: Trying 228496...
+: Trying 456992...
+: Trying 342744...
+: Trying 399868...
+: Trying 371306...
+: Trying 357025...
+: Trying 349884...
+: Trying 353454...
+: Trying 355239...
+: Trying 356132...
+: Trying 355685...
+: Trying 355462...
+: Trying 355573...
+: Trying 355629...
+: Trying 355657...
+: Trying 355671...
+: Trying 355678...
+: Trying 355681...
+: Trying 355683...
+: Trying 355684...
+: First bad state count: 355685
+**** File Size
+**** Automaton Size
+#+BEGIN_SRC shell
+./bench.py generate ../../copar/bin/random-wta --monoid powerset --symbols 0,4 --out-degree 20 --states 355684
+./bench.py run ../../copar/bin/copar --monoid powerset --symbols 0,4 --out-degree 20 --states 355684 --indiv --header
+#+END_SRC
+#+RESULTS:
+| i |  states |    edges | initial-partition-size | final-partition-size | explicit-final-partition-size | size1-skipped | overall-duration | parse-duration | algorithm-duration | initialize-duration | refine-duration |
+| 0 | 7469364 | 14227360 |                      6 |              1768728 |                        355684 |      10883373 |    344.399732107 |   61.942569869 |      262.572157652 |        32.047148894 |   209.333188906 |
+| 1 | 7469364 | 14227360 |                      6 |              1768935 |                        355684 |      10883077 |    340.732038227 |   60.912573518 |      260.176273796 |        31.928906963 |   208.837646082 |
+| 2 | 7469364 | 14227360 |                      6 |              1768908 |                        355684 |      10885631 |    340.838739608 |   61.120402905 |      260.008098908 |        31.951679391 |   208.775189688 |
+| 3 | 7469364 | 14227360 |                      6 |              1768880 |                        355684 |      10880372 |    340.237713603 |   61.448840572 |      259.027481579 |        31.936433445 |   207.318948419 |
+| 4 | 7469364 | 14227360 |                      6 |              1768875 |                        355684 |      10886155 |    342.792217839 |   61.487509806 |      261.434824046 |        32.045849362 |   209.995181997 |
+*** 4xX mit out-degree 50
+#+BEGIN_SRC shell
+./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,4 --out-degree 50 --good 130000 --bad 140000
+#+END_SRC
+: Trying 135000...
+: Trying 132500...
+: Trying 131250...
+: Trying 131875...
+: Trying 132187...
+: Trying 132031...
+: Trying 132109...
+: Trying 132148...
+: Trying 132167...
+: Trying 132177...
+: Trying 132182...
+: Trying 132179...
+: Trying 132178...
+: First bad state count: 132179
+**** Automaton Size
+#+BEGIN_SRC shell
+./bench.py run ../../copar/bin/copar --monoid powerset --symbols 0,4 --out-degree 50 --states 132178 --indiv --header
+#+END_SRC
+#+RESULTS:
+| i |  states |    edges | initial-partition-size | final-partition-size | explicit-final-partition-size | size1-skipped | overall-duration | parse-duration | algorithm-duration | initialize-duration | refine-duration |
+| 0 | 6741078 | 13217800 |                      6 |               660888 |                        132178 |       9275215 |    263.557139582 |   53.134795016 |      197.899988998 |        33.273151066 |   153.369899287 |
+*** 4xX^2 mit out-degree 50
+#+BEGIN_SRC shell
+./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,0,4 --out-degree 50 --start-states 80000
+#+END_SRC
+: Trying 80000...
+: Trying 160000...
+: Trying 120000...
+: Trying 100000...
+: Trying 90000...
+: Trying 95000...
+: Trying 97500...
+: Trying 98750...
+: Trying 98125...
+: Trying 98437...
+: Trying 98593...
+: Trying 98671...
+: Trying 98632...
+: Trying 98651...
+: Trying 98661...
+: Trying 98666...
+: Trying 98668...
+: Trying 98669...
+: Trying 98670...
+: First bad state count: 98671
+**** Automaton Size
+#+BEGIN_SRC shell
+./bench.py generate ../../copar/bin/random-wta --monoid powerset --symbols 0,0,4 --out-degree 50 --states 98670
+./bench.py run ../../copar/bin/copar --monoid powerset --symbols 0,0,4 --out-degree 50 --states 98670 --indiv --header
+#+END_SRC
+#+RESULTS:
+| i |  states |    edges | initial-partition-size | final-partition-size | explicit-final-partition-size | size1-skipped | overall-duration | parse-duration | algorithm-duration | initialize-duration | refine-duration |
+| 0 | 5032170 | 14800500 |                      6 |              5031861 |                         98670 |      11563672 |    320.579655827 |   46.524207289 |       261.58724583 |        31.537501676 |   213.989949977 |
+*** 4xX^3 mit out-degree 50
+#+BEGIN_SRC shell
+./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,0,0,4 --out-degree 50 --start-states 60000
+#+END_SRC
+: Trying 60000...
+: Trying 120000...
+: Trying 90000...
+: Trying 75000...
+: Trying 82500...
+: Trying 86250...
+: Trying 84375...
+: Trying 85312...
+: Trying 84843...
+: Trying 85077...
+: Trying 84960...
+: Trying 85018...
+: Trying 84989...
+: Trying 85003...
+: Trying 85010...
+: Trying 85014...
+: Trying 85016...
+: Trying 85017...
+: First bad state count: 85018
+*** 4xX^4 mit out-degree 50
+#+BEGIN_SRC shell
+./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,0,0,0,4 --out-degree 50 --start-states 48000
+#+END_SRC
+: Trying 48000...
+: Trying 96000...
+: Trying 72000...
+: Trying 60000...
+: Trying 54000...
+: Trying 57000...
+: Trying 58500...
+: Trying 59250...
+: Trying 59625...
+: Trying 59437...
+: Trying 59531...
+: Trying 59578...
+: Trying 59601...
+: Trying 59589...
+: Trying 59595...
+: Trying 59598...
+: Trying 59596...
+: First bad state count: 59596
+*** 4xX^5 mit out-degree 50
+#+BEGIN_SRC shell
+./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,0,0,0,0,4 --out-degree 50 --start-states 40000
+#+END_SRC
+: Trying 40000...
+: Trying 80000...
+: Trying 60000...
+: Trying 50000...
+: Trying 45000...
+: Trying 47500...
+: Trying 48750...
+: Trying 49375...
+: Trying 49687...
+: Trying 49531...
+: Trying 49453...
+: Trying 49414...
+: Trying 49394...
+: Trying 49384...
+: Trying 49379...
+: Trying 49377...
+: Trying 49376...
+: First bad state count: 49376