Skip to content

Commit 3732b4a

Browse files
pfetschpfetsch
committed
Merge branch 'safeguard_Wilcoxon_test' into 'v90-bugfix'
Safeguard Wilcoxon test See merge request integer/scip!3382
2 parents 2b8be5e + 74a10d7 commit 3732b4a

File tree

1 file changed

+81
-42
lines changed

1 file changed

+81
-42
lines changed

‎check/cmpres.awk‎

Lines changed: 81 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
#!/usr/bin/awk -f
32
#* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
43
#* *
@@ -464,6 +463,7 @@ BEGIN {
464463
printconfs = 0;
465464
printgap = 0; # if timeout, then print absolute gap at termination in time column, if gap is finite
466465
printsoltimes = !short && printsoltimes; # short deactivates the detailed solution times output
466+
setnamelen = 50; # display length of test sets
467467
infinity = 1e+20;
468468
timegeomshift = 1.0;
469469
nodegeomshift = 100.0;
@@ -793,24 +793,24 @@ END {
793793
{
794794
if( printsoltimes && printconfs == 0 )
795795
{
796-
if ( length(sname) <= 58 )
797-
printf(" %58s |", sname);
796+
if ( length(sname) <= 59 )
797+
printf("%59s |", sname);
798798
else
799-
printf(" *%57s |", substr(sname, length(sname)-58));
799+
printf("*%58s |", substr(sname, length(sname) - 57));
800800
}
801801
else if( printsoltimes == 0 && printconfs )
802802
{
803-
if ( length(sname) <= 55 )
804-
printf(" %55s |", sname);
803+
if ( length(sname) <= 56 )
804+
printf("%56s |", sname);
805805
else
806-
printf(" *%54s |", substr(sname, length(sname)-54));
806+
printf("*%55s |", substr(sname, length(sname) - 54));
807807
}
808808
else
809809
{
810-
if ( length(sname) <= 39 )
811-
printf(" %39s |", sname)
810+
if ( length(sname) <= 40 )
811+
printf("%40s |", sname)
812812
else
813-
printf(" *%38s |", substr(sname, length(sname)-39));
813+
printf("*%39s |", substr(sname, length(sname) - 38));
814814
}
815815
}
816816
else
@@ -820,28 +820,28 @@ END {
820820
if( length(sname) <= 19 )
821821
printf("%19s |", sname);
822822
else
823-
printf("*%16s |", substr(sname, length(sname)-17));
823+
printf("*%18s |", substr(sname, length(sname) - 17));
824824
}
825825
else if( printsoltimes && printconfs == 0 )
826826
{
827827
if( length(sname) <= 47 )
828828
printf("%47s |", sname);
829829
else
830-
printf("*%46s |", substr(sname, length(sname)-47));
830+
printf("*%46s |", substr(sname, length(sname) - 45));
831831
}
832832
else if( printsoltimes == 0 && printconfs )
833833
{
834834
if( length(sname) <= 48 )
835835
printf("%48s |", sname);
836836
else
837-
printf("*%47s |", substr(sname, length(sname)-48));
837+
printf("*%47s |", substr(sname, length(sname) - 46));
838838
}
839839
else
840840
{
841841
if( length(sname) <= 33 )
842-
printf("%31s |", sname);
842+
printf("%33s |", sname);
843843
else
844-
printf("*%30s |", substr(sname, length(sname)-31));
844+
printf("*%32s |", substr(sname, length(sname) - 31));
845845
}
846846
}
847847
}
@@ -1697,20 +1697,28 @@ END {
16971697
{
16981698
s = printorder[o];
16991699

1700-
parse_time(ref_array,solver_array,time,o,printorder,probidx,problistlen);
1700+
parse_time(ref_array, solver_array, time, o, printorder, probidx, problistlen);
17011701
n = filter(ref_array, solver_array, problistlen, 0.01, 0.01);
17021702
factorize(ref_array, solver_array, n, timelimit[s])
17031703

1704-
z = wilcoxon(ref_array, solver_array, n, timelimit[s]);
1705-
printf(" z %8.4f",z);
1706-
z_to_p(z);
1704+
# do not print test if number of items is less than 20, because the test is not reliable there
1705+
if( n > 20 )
1706+
{
1707+
z = wilcoxon(ref_array, solver_array, n, timelimit[s]);
1708+
printf(" z %8.4f", z);
1709+
z_to_p(z);
1710+
}
1711+
else
1712+
{
1713+
printf(" %8s", "");
1714+
printf(" -- (N = %2d <= 20)", n);
1715+
}
17071716

17081717
if( printconfs )
17091718
printf("%18s", "");
17101719
}
17111720
printf("\n");
17121721

1713-
17141722
# compute and print result for Wilcoxon signed rank test for time to first solution w.r.t. reference setting
17151723
if( printsoltimes )
17161724
{
@@ -1724,13 +1732,22 @@ END {
17241732
{
17251733
s = printorder[o];
17261734

1727-
parse_time(ref_array,solver_array,timetofirst,o,printorder,probidx,problistlen);
1735+
parse_time(ref_array, solver_array, timetofirst, o, printorder, probidx, problistlen);
17281736
n = filter(ref_array, solver_array, problistlen, 0.01, 0.01);
17291737
factorize(ref_array, solver_array, n, timelimit[s])
17301738

1731-
z = wilcoxon(ref_array, solver_array, n, timelimit[s]);
1732-
printf(" z %8.4f",z);
1733-
z_to_p(z);
1739+
# do not print test if number of items is less than 20, because the test is not reliable there
1740+
if( n > 20 )
1741+
{
1742+
z = wilcoxon(ref_array, solver_array, n, timelimit[s]);
1743+
printf(" z %8.4f", z);
1744+
z_to_p(z);
1745+
}
1746+
else
1747+
{
1748+
printf(" %8s", "");
1749+
printf(" -- (N = %2d <= 20)", n);
1750+
}
17341751

17351752
if( printconfs )
17361753
printf("%18s", "");
@@ -1751,9 +1768,18 @@ END {
17511768
n = filter(ref_array, solver_array, problistlen, 0.01, 0.01);
17521769
factorize(ref_array, solver_array, n, infinity)
17531770

1754-
z = wilcoxon(ref_array, solver_array, n, infinity);
1755-
printf(" z %8.4f",z);
1756-
z_to_p(z);
1771+
# do not print test if number of items is less than 20, because the test is not reliable there
1772+
if( n > 20 )
1773+
{
1774+
z = wilcoxon(ref_array, solver_array, n, infinity);
1775+
printf(" z %8.4f", z);
1776+
z_to_p(z);
1777+
}
1778+
else
1779+
{
1780+
printf(" %8s", "");
1781+
printf(" -- (N = %2d <= 20)", n);
1782+
}
17571783

17581784
if( printconfs )
17591785
printf("%18s", "");
@@ -1768,13 +1794,22 @@ END {
17681794
{
17691795
s = printorder[o];
17701796

1771-
parse_time(ref_array,solver_array,confs,o,printorder,probidx,problistlen);
1797+
parse_time(ref_array, solver_array, confs, o, printorder, probidx, problistlen);
17721798
n = filter(ref_array, solver_array, problistlen, 0.01, 0.01);
17731799
factorize(ref_array, solver_array, n, timelimit[s])
17741800

1775-
z = wilcoxon(ref_array, solver_array, n, timelimit[s]);
1776-
printf(" z %8.4f", z );
1777-
z_to_p(z);
1801+
# do not print test if number of items is less than 20, because the test is not reliable there
1802+
if( n > 20 )
1803+
{
1804+
z = wilcoxon(ref_array, solver_array, n, timelimit[s]);
1805+
printf(" z %8.4f", z);
1806+
z_to_p(z);
1807+
}
1808+
else
1809+
{
1810+
printf(" %8s", "");
1811+
printf(" -- (N = %2d <= 20)", n);
1812+
}
17781813

17791814
if( printconfs )
17801815
printf("%18s", "");
@@ -1881,8 +1916,8 @@ END {
18811916

18821917
header = (cat == -1 ? "optimal" : (cat == 0 ? "all" : (cat == 1 ? "diff" : (cat == 2 ? "equal" : "timeout"))));
18831918
printf("\n");
1884-
printf("%-7s proc eval fail time solv wins bett wors bobj wobj feas gnodes shnodes gnodesQ shnodesQ gtime shtime gtimeQ shtimeQ score\n",
1885-
header);
1919+
printf("%-*s proc eval fail time solv wins bett wors bobj wobj feas gnodes shnodes gnodesQ shnodesQ gtime shtime gtimeQ shtimeQ score\n",
1920+
setnamelen, header);
18861921

18871922
for( o = 0; o < nsolver; ++o )
18881923
{
@@ -1897,11 +1932,13 @@ END {
18971932
}
18981933
if( (o > 0 || cat == 0 || cat == -1) && nevalprobs[s,cat] > 0 )
18991934
{
1900-
if ( length(sname) <= 50 )
1901-
printf("%-50s %4d %4d %4d %4d %4d %4d", sname, nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat],
1935+
if ( length(sname) <= setnamelen )
1936+
printf("%-*s %4d %4d %4d %4d %4d %4d", setnamelen, sname,
1937+
nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat],
19021938
ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]);
19031939
else
1904-
printf("*%-49s %4d %4d %4d %4d %4d %4d", substr(sname, length(sname)-48), nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat],
1940+
printf("*%-*s %4d %4d %4d %4d %4d %4d", setnamelen - 1, substr(sname, length(sname) - setnamelen + 2),
1941+
nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat],
19051942
ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]);
19061943

19071944
printf(" %4d %4d", better[s,cat], worse[s,cat]);
@@ -1915,7 +1952,7 @@ END {
19151952
}
19161953
if( cat == 0 )
19171954
{
1918-
printf("%-50s %4d %4d %4d %4s", "optimal auto settings", bestnfails, bestntimeouts, bestnsolved, "");
1955+
printf("%-*s %4d %4d %4d %4s", setnamelen, "optimal auto settings", bestnfails, bestntimeouts, bestnsolved, "");
19191956
printf(" %4d %4s", bestbetter, "");
19201957
printf(" %4d %4s %4d %9d %9d %9.2f %9.2f %7.1f %7.1f %7.2f %7.2f %7s\n",
19211958
bestbetterobj, "", bestfeasibles,
@@ -1928,7 +1965,8 @@ END {
19281965
# output the all optimal case
19291966
header = "all optimal";
19301967
printf("\n");
1931-
printf("%-11s proc eval fail time solv wins bett wors bobj wobj feas gnodes shnodes gnodesQ shnodesQ gtime shtime gtimeQ shtimeQ score\n", header);
1968+
printf("%-*s proc eval fail time solv wins bett wors bobj wobj feas gnodes shnodes gnodesQ shnodesQ gtime shtime gtimeQ shtimeQ score\n",
1969+
setnamelen, header);
19321970
cat = -1;
19331971
for( o = 0; o < nsolver; ++o )
19341972
{
@@ -1943,11 +1981,12 @@ END {
19431981
}
19441982
if( (o > 0 || cat == 0 || cat == -1) && nevalprobs[s,cat] > 0 )
19451983
{
1946-
if ( length(sname) <= 50 )
1947-
printf("%-50s %4d %4d %4d %4d %4d %4d", sname, nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat],
1984+
if ( length(sname) <= setnamelen )
1985+
printf("%-*s %4d %4d %4d %4d %4d %4d", setnamelen, sname, nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat],
19481986
ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]);
19491987
else
1950-
printf("*%-49s %4d %4d %4d %4d %4d %4d", substr(sname, length(sname)-48), nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat],
1988+
printf("*%-*s %4d %4d %4d %4d %4d %4d", setnamelen - 1, substr(sname, length(sname) - setnamelen + 2),
1989+
nprocessedprobs[s,cat], nevalprobs[s,cat], nfails[s,cat],
19511990
ntimeouts[s,cat], nsolved[s,cat], wins[s,cat]);
19521991
printf(" %4d %4d", better[s,cat], worse[s,cat]);
19531992
printf(" %4d %4d %4d %9d %9d %9.2f %9.2f %7.1f %7.1f %7.2f %7.2f %7.2f\n",
@@ -1960,7 +1999,7 @@ END {
19601999
}
19612000
if( cat == 0 )
19622001
{
1963-
printf("%-50s %4d %4d %4d %4s", "optimal auto settings", bestnfails, bestntimeouts, bestnsolved, "");
2002+
printf("%-*s %4d %4d %4d %4s", setnamelen, "optimal auto settings", bestnfails, bestntimeouts, bestnsolved, "");
19642003
printf(" %4d %4s", bestbetter, "");
19652004
printf(" %4d %4s %4d %9d %9d %9.2f %9.2f %7.1f %7.1f %7.2f %7.2f %7s\n",
19662005
bestbetterobj, "", bestfeasibles,

0 commit comments

Comments
 (0)