diff --git a/wta/benchmarks.org b/wta/benchmarks.org index 97f87c7efb68f5abe969c1e22cb816897ce44f97..26c0f6e16114acfd5716246f241c2c1db129c598 100644 --- a/wta/benchmarks.org +++ b/wta/benchmarks.org @@ -580,3 +580,353 @@ python bench.py run ../../copar/bin/copar --monoid 'Word,or' --symbols 0,0,0,0,0 | 2 | 1595591 | 9573480 | 14 | 473700 | 11 | 2679546 | 61.459443653 | 24.801616514 | 34.389540062 | 10.893824596 | 21.133849133 | | 3 | 1593825 | 9562884 | 14 | 473580 | 11 | 2676789 | 61.147557505 | 24.98879381 | 33.942496577 | 10.818043846 | 20.839884809 | | 4 | 1596528 | 9579102 | 14 | 473600 | 11 | 2681048 | 61.864094058 | 24.884111862 | 34.762990208 | 10.893522867 | 21.485077775 | +* Same Question, but with limited transition count +** Powerset +*** GHC 8.4.4 with symbols 0,8 and transitions 500 + +#+BEGIN_SRC sh +./bench.py bisect ../../copar/bin/{random-wta,copar} --monoid powerset --symbols 0,8 --transitions 5000 +#+END_SRC + +#+RESULTS: +: Trying 1000... +: Trying 2000... +: Trying 1500... +: Trying 1250... +: Trying 1375... +: Trying 1437... +: Trying 1468... +: Trying 1484... +: Trying 1476... +: Trying 1480... +: Trying 1478... +: Trying 1479... +: First bad state count: 1479 + +**** File Size + +#+BEGIN_SRC sh :results output +ls -sh bench/wta_powerset_0,8_0.7_1478* +#+END_SRC + +#+RESULTS: +: 82M bench/wta_powerset_0,8_0.7_1478_0.coalgebra +: 82M bench/wta_powerset_0,8_0.7_1478_1.coalgebra +: 82M bench/wta_powerset_0,8_0.7_1478_2.coalgebra +: 82M bench/wta_powerset_0,8_0.7_1478_3.coalgebra +: 82M bench/wta_powerset_0,8_0.7_1478_4.coalgebra + +**** Automaton size + +#+begin_src sh :hlines yes +python bench.py run ../../copar/bin/copar --monoid powerset --symbols 0,8 --zero-frequency 0.7 --states 1478 --indiv --header +#+end_src + +#+RESULTS: +| i | states | edges | initial-partition-size | final-partition-size | explicit-final-partition-size | size1-skipped | overall-duration | parse-duration | algorithm-duration | initialize-duration | refine-duration | +| 0 | 5243856 | 10484756 | 10 | 18 | 2 | 0 | 115.339077581 | 34.439478263 | 79.458232767 | 27.712258503 | 37.668249537 | +| 1 | 5245405 | 10487854 | 10 | 18 | 2 | 0 | 114.001430765 | 34.477874858 | 78.07613373 | 27.764347556 | 37.074617863 | +| 2 | 5243549 | 10484142 | 10 | 18 | 2 | 0 | 114.862812706 | 34.668409012 | 78.756016051 | 27.741448159 | 37.09480403 | +| 3 | 5241660 | 10480364 | 10 | 18 | 2 | 0 | 114.228588077 | 34.362576104 | 78.424601434 | 27.648526393 | 37.043127108 | +| 4 | 5243032 | 10483108 | 10 | 18 | 2 | 0 | 114.692551901 | 34.827430372 | 78.414037199 | 27.723198479 | 38.276163878 | + +*** GHC 8.4.4 with symbols 0,8 and out-degree 1000 + +#+BEGIN_SRC sh +./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,8 --out-degree 1000 --good 1000 +#+END_SRC + +#+RESULTS: +: Trying 1001... +: Trying 2002... +: Trying 4004... +: Trying 8008... +: Trying 6006... +: Trying 7007... +: Trying 7507... +: Trying 7257... +: Trying 7382... +: Trying 7444... +: Trying 7413... +: Trying 7428... +: Trying 7420... +: Trying 7416... +: Trying 7414... +: Trying 7415... +: First bad state count: 7416 + +**** File Size + +#+BEGIN_SRC sh :results output +ls -sh bench/wta_powerset_0,8_t7415000__7415* +#+END_SRC + +#+RESULTS: +: 120M bench/wta_powerset_0,8_t7415000__7415_0.coalgebra +: 120M bench/wta_powerset_0,8_t7415000__7415_1.coalgebra +: 120M bench/wta_powerset_0,8_t7415000__7415_2.coalgebra +: 120M bench/wta_powerset_0,8_t7415000__7415_3.coalgebra +: 120M bench/wta_powerset_0,8_t7415000__7415_4.coalgebra + +**** Automaton Size + +#+begin_src sh :hlines yes +python bench.py run ../../copar/bin/copar --monoid powerset --symbols 0,8 --transitions 7415000 --states 7415 --indiv --header +#+end_src + +#+RESULTS: +| i | states | edges | initial-partition-size | final-partition-size | explicit-final-partition-size | size1-skipped | overall-duration | parse-duration | algorithm-duration | initialize-duration | refine-duration | +| 0 | 7422415 | 14830000 | 10 | 18 | 2 | 0 | 157.872966092 | 51.221314999 | 104.615248956 | 33.7012898 | 58.664739269 | +| 1 | 7422415 | 14830000 | 10 | 18 | 2 | 0 | 157.428380106 | 51.433510926 | 103.926494829 | 33.216537153 | 58.775334589 | +| 2 | 7422415 | 14830000 | 10 | 18 | 2 | 0 | 156.808413994 | 51.343226544 | 103.40051371 | 32.969125294 | 58.865798525 | +| 3 | 7422415 | 14830000 | 10 | 18 | 2 | 0 | 157.791363724 | 51.873644617 | 103.855838314 | 33.05713406 | 58.931293578 | +| 4 | 7422415 | 14830000 | 10 | 18 | 2 | 0 | 157.126256596 | 51.400301031 | 103.666329167 | 32.976280595 | 58.830566981 | + +* Next Try + +We now use the following parameters: + +- out-degree :: 500 +- Signature :: 4×X^k k ∈ { 0, ..., 5 } +- GHC :: 8.4.4 + +** Powerset +*** 4xX + +#+BEGIN_SRC shell +./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,4 --out-degree 500 --good 8000 --start-states 12000 --bad 16000 +#+END_SRC + +: Trying 12000... +: Trying 14000... +: Trying 15000... +: Trying 14500... +: Trying 14250... +: Trying 14375... +: Trying 14312... +: Trying 14281... +: Trying 14296... +: Trying 14288... +: Trying 14284... +: Trying 14282... +: First bad state count: 14282 + +**** File Size + +#+BEGIN_SRC sh :results output +ls -sh bench/wta_powerset_0,4_*__14281* +#+END_SRC + +#+RESULTS: +: 118M bench/wta_powerset_0,4_t7140500__14281_0.coalgebra + +**** Automaton Size + +#+BEGIN_SRC shell +./bench.py generate ../../copar/bin/random-wta --monoid powerset --symbols 0,4 --out-degree 500 --states 14281 +./bench.py run ../../copar/bin/copar --monoid powerset --symbols 0,4 --out-degree 500 --states 14281 --indiv --header +#+END_SRC + +#+RESULTS: +| i | states | edges | initial-partition-size | final-partition-size | explicit-final-partition-size | size1-skipped | overall-duration | parse-duration | algorithm-duration | initialize-duration | refine-duration | +| 0 | 7154781 | 14281000 | 6 | 10 | 2 | 0 | 155.635043092 | 50.511676534 | 103.239235361 | 30.135940008 | 61.445316625 | +| 1 | 7154781 | 14281000 | 6 | 10 | 2 | 0 | 155.345664537 | 50.391815858 | 103.046448356 | 29.52881681 | 61.823502784 | +| 2 | 7154781 | 14281000 | 6 | 10 | 2 | 0 | 154.746389702 | 50.356318464 | 102.487287332 | 29.561887582 | 61.245616512 | +| 3 | 7154781 | 14281000 | 6 | 10 | 2 | 0 | 156.681042711 | 50.477922803 | 104.306181251 | 30.132009036 | 62.231759233 | +| 4 | 7154781 | 14281000 | 6 | 10 | 2 | 0 | 156.206931801 | 50.552217812 | 103.775304876 | 30.106012782 | 61.793692847 | + +*** 4xX mit out-degree 20 + +#+BEGIN_SRC shell +./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,4 --out-degree 20 --good 8000 --start-states 14281 +#+END_SRC + +: Trying 14281... +: Trying 28562... +: Trying 57124... +: Trying 114248... +: Trying 228496... +: Trying 456992... +: Trying 342744... +: Trying 399868... +: Trying 371306... +: Trying 357025... +: Trying 349884... +: Trying 353454... +: Trying 355239... +: Trying 356132... +: Trying 355685... +: Trying 355462... +: Trying 355573... +: Trying 355629... +: Trying 355657... +: Trying 355671... +: Trying 355678... +: Trying 355681... +: Trying 355683... +: Trying 355684... +: First bad state count: 355685 + +**** File Size + + +**** Automaton Size + +#+BEGIN_SRC shell +./bench.py generate ../../copar/bin/random-wta --monoid powerset --symbols 0,4 --out-degree 20 --states 355684 +./bench.py run ../../copar/bin/copar --monoid powerset --symbols 0,4 --out-degree 20 --states 355684 --indiv --header +#+END_SRC + +#+RESULTS: +| i | states | edges | initial-partition-size | final-partition-size | explicit-final-partition-size | size1-skipped | overall-duration | parse-duration | algorithm-duration | initialize-duration | refine-duration | +| 0 | 7469364 | 14227360 | 6 | 1768728 | 355684 | 10883373 | 344.399732107 | 61.942569869 | 262.572157652 | 32.047148894 | 209.333188906 | +| 1 | 7469364 | 14227360 | 6 | 1768935 | 355684 | 10883077 | 340.732038227 | 60.912573518 | 260.176273796 | 31.928906963 | 208.837646082 | +| 2 | 7469364 | 14227360 | 6 | 1768908 | 355684 | 10885631 | 340.838739608 | 61.120402905 | 260.008098908 | 31.951679391 | 208.775189688 | +| 3 | 7469364 | 14227360 | 6 | 1768880 | 355684 | 10880372 | 340.237713603 | 61.448840572 | 259.027481579 | 31.936433445 | 207.318948419 | +| 4 | 7469364 | 14227360 | 6 | 1768875 | 355684 | 10886155 | 342.792217839 | 61.487509806 | 261.434824046 | 32.045849362 | 209.995181997 | + +*** 4xX mit out-degree 50 + +#+BEGIN_SRC shell +./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,4 --out-degree 50 --good 130000 --bad 140000 +#+END_SRC + +: Trying 135000... +: Trying 132500... +: Trying 131250... +: Trying 131875... +: Trying 132187... +: Trying 132031... +: Trying 132109... +: Trying 132148... +: Trying 132167... +: Trying 132177... +: Trying 132182... +: Trying 132179... +: Trying 132178... +: First bad state count: 132179 + + +**** Automaton Size + +#+BEGIN_SRC shell +./bench.py run ../../copar/bin/copar --monoid powerset --symbols 0,4 --out-degree 50 --states 132178 --indiv --header +#+END_SRC + +#+RESULTS: +| i | states | edges | initial-partition-size | final-partition-size | explicit-final-partition-size | size1-skipped | overall-duration | parse-duration | algorithm-duration | initialize-duration | refine-duration | +| 0 | 6741078 | 13217800 | 6 | 660888 | 132178 | 9275215 | 263.557139582 | 53.134795016 | 197.899988998 | 33.273151066 | 153.369899287 | + +*** 4xX^2 mit out-degree 50 + +#+BEGIN_SRC shell +./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,0,4 --out-degree 50 --start-states 80000 +#+END_SRC + +: Trying 80000... +: Trying 160000... +: Trying 120000... +: Trying 100000... +: Trying 90000... +: Trying 95000... +: Trying 97500... +: Trying 98750... +: Trying 98125... +: Trying 98437... +: Trying 98593... +: Trying 98671... +: Trying 98632... +: Trying 98651... +: Trying 98661... +: Trying 98666... +: Trying 98668... +: Trying 98669... +: Trying 98670... +: First bad state count: 98671 + +**** Automaton Size + +#+BEGIN_SRC shell +./bench.py generate ../../copar/bin/random-wta --monoid powerset --symbols 0,0,4 --out-degree 50 --states 98670 +./bench.py run ../../copar/bin/copar --monoid powerset --symbols 0,0,4 --out-degree 50 --states 98670 --indiv --header +#+END_SRC + +#+RESULTS: +| i | states | edges | initial-partition-size | final-partition-size | explicit-final-partition-size | size1-skipped | overall-duration | parse-duration | algorithm-duration | initialize-duration | refine-duration | +| 0 | 5032170 | 14800500 | 6 | 5031861 | 98670 | 11563672 | 320.579655827 | 46.524207289 | 261.58724583 | 31.537501676 | 213.989949977 | + +*** 4xX^3 mit out-degree 50 + +#+BEGIN_SRC shell +./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,0,0,4 --out-degree 50 --start-states 60000 +#+END_SRC + +: Trying 60000... +: Trying 120000... +: Trying 90000... +: Trying 75000... +: Trying 82500... +: Trying 86250... +: Trying 84375... +: Trying 85312... +: Trying 84843... +: Trying 85077... +: Trying 84960... +: Trying 85018... +: Trying 84989... +: Trying 85003... +: Trying 85010... +: Trying 85014... +: Trying 85016... +: Trying 85017... +: First bad state count: 85018 + +*** 4xX^4 mit out-degree 50 + +#+BEGIN_SRC shell +./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,0,0,0,4 --out-degree 50 --start-states 48000 +#+END_SRC + +: Trying 48000... +: Trying 96000... +: Trying 72000... +: Trying 60000... +: Trying 54000... +: Trying 57000... +: Trying 58500... +: Trying 59250... +: Trying 59625... +: Trying 59437... +: Trying 59531... +: Trying 59578... +: Trying 59601... +: Trying 59589... +: Trying 59595... +: Trying 59598... +: Trying 59596... +: First bad state count: 59596 + +*** 4xX^5 mit out-degree 50 + +#+BEGIN_SRC shell +./bench.py bisect ../../copar/bin/random-wta ../../copar/bin/copar --monoid powerset --symbols 0,0,0,0,0,4 --out-degree 50 --start-states 40000 +#+END_SRC + +: Trying 40000... +: Trying 80000... +: Trying 60000... +: Trying 50000... +: Trying 45000... +: Trying 47500... +: Trying 48750... +: Trying 49375... +: Trying 49687... +: Trying 49531... +: Trying 49453... +: Trying 49414... +: Trying 49394... +: Trying 49384... +: Trying 49379... +: Trying 49377... +: Trying 49376... +: First bad state count: 49376