Skip to content

Commit 160e3f7

Browse files
cabaumancabauman
committed
Add tests for the newer xml format result files
1 parent 1e25295 commit 160e3f7

File tree

5 files changed

+323
-10
lines changed

5 files changed

+323
-10
lines changed

‎UnityPerformanceBenchmarkReporterTests/PerformanceBenchmarkTests.cs‎

Lines changed: 80 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,26 @@ public void Verify_AddPerformanceTestRunResults()
4848
Assert.IsTrue(performanceTestRunResults.Count > 0);
4949
}
5050

51+
[Test]
52+
public void Verify_AddPerformanceTestRunResultsV2()
53+
{
54+
// Arrange
55+
var resultXmlFilePath = EnsureFullPath("results_v2.xml");
56+
var args = new[] { string.Format("--testresultsxmlsource={0}", resultXmlFilePath) };
57+
optionsParser.ParseOptions(PerformanceBenchmark, args);
58+
59+
// Act
60+
PerformanceBenchmark.AddPerformanceTestRunResults(testResultXmlParser, performanceTestRunResults, testResults, new List<TestResult>());
61+
62+
// Assert
63+
Assert.IsTrue(PerformanceBenchmark.ResultFilesExist);
64+
AssertCorrectResultXmlFilePaths(new[] { resultXmlFilePath });
65+
Assert.NotNull(testResults);
66+
Assert.IsTrue(testResults.Count > 0);
67+
Assert.NotNull(performanceTestRunResults);
68+
Assert.IsTrue(performanceTestRunResults.Count > 0);
69+
}
70+
5171
[Test]
5272
public void Verify_AddPerformanceTestRunResults_TwoResultFiles()
5373
{
@@ -73,6 +93,31 @@ public void Verify_AddPerformanceTestRunResults_TwoResultFiles()
7393
Assert.IsTrue(performanceTestRunResults.Count > 0);
7494
}
7595

96+
[Test]
97+
public void Verify_AddPerformanceTestRunResults_OneV1ResultFiles_OneV2ResultFiles()
98+
{
99+
// Arrange
100+
var resultXmlFilePath = EnsureFullPath("results.xml");
101+
var resultFileName2 = EnsureFullPath("results_v2.xml");
102+
var args = new[]
103+
{
104+
string.Format("--testresultsxmlsource={0}", resultXmlFilePath),
105+
string.Format("--testresultsxmlsource={0}", resultFileName2)
106+
};
107+
optionsParser.ParseOptions(PerformanceBenchmark, args);
108+
109+
// Act
110+
PerformanceBenchmark.AddPerformanceTestRunResults(testResultXmlParser, performanceTestRunResults, testResults, new List<TestResult>());
111+
112+
// Assert
113+
Assert.IsTrue(PerformanceBenchmark.ResultFilesExist);
114+
AssertCorrectResultXmlFilePaths(new[] { resultXmlFilePath, resultFileName2 });
115+
Assert.NotNull(testResults);
116+
Assert.IsTrue(testResults.Count > 0);
117+
Assert.NotNull(performanceTestRunResults);
118+
Assert.IsTrue(performanceTestRunResults.Count > 0);
119+
}
120+
76121
[Test]
77122
public void Verify_AddPerformanceTestRunResults_OneResultFiles_OneResultDirectory()
78123
{
@@ -81,8 +126,8 @@ public void Verify_AddPerformanceTestRunResults_OneResultFiles_OneResultDirector
81126
var resultsXmlDir = EnsureFullPath("Results");
82127
var args = new[]
83128
{
84-
string.Format("--testresultsxmlsource={0}", resultXmlFilePath),
85-
string.Format("--testresultsxmlsource={0}", resultsXmlDir)
129+
string.Format("--testresultsxmlsource={0}", resultXmlFilePath),
130+
string.Format("--testresultsxmlsource={0}", resultsXmlDir)
86131
};
87132
optionsParser.ParseOptions(PerformanceBenchmark, args);
88133

@@ -126,11 +171,11 @@ public void Verify_AddBaselinePerformanceTestRunResults()
126171
}
127172

128173
[Test]
129-
public void Verify_AddBaselinePerformanceTestRunResultsDirectory()
174+
public void Verify_AddBaselinePerformanceTestRunResults_V2BaselineFile()
130175
{
131176
// Arrange
132177
var resultXmlFilePath = EnsureFullPath("results.xml");
133-
var baselineXmlFilePath = EnsureFullPath("baseline.xml");
178+
var baselineXmlFilePath = EnsureFullPath("baseline_v2.xml");
134179
var args = new[]
135180
{
136181
string.Format("--testresultsxmlsource={0}", resultXmlFilePath),
@@ -143,6 +188,34 @@ public void Verify_AddBaselinePerformanceTestRunResultsDirectory()
143188

144189
// Assert
145190
Assert.IsTrue(PerformanceBenchmark.BaselineResultFilesExist);
191+
AssertCorrectBaselineXmlFilePaths(new[] { baselineXmlFilePath });
192+
AssertCorrectResultXmlFilePaths(new[] { resultXmlFilePath });
193+
Assert.NotNull(baselineTestResults);
194+
Assert.IsTrue(baselineTestResults.Count > 0);
195+
Assert.NotNull(baselinePerformanceTestRunResults);
196+
Assert.IsTrue(baselinePerformanceTestRunResults.Count > 0);
197+
}
198+
199+
[Test]
200+
public void Verify_AddBaselinePerformanceTestRunResultsDirectory()
201+
{
202+
// Arrange
203+
var resultsXmlDir = EnsureFullPath("Results");
204+
var baselineXmlFilePath = EnsureFullPath("baseline.xml");
205+
var args = new[]
206+
{
207+
string.Format("--testresultsxmlsource={0}", resultsXmlDir),
208+
string.Format("--baselinexmlsource={0}", baselineXmlFilePath)
209+
};
210+
optionsParser.ParseOptions(PerformanceBenchmark, args);
211+
212+
// Act
213+
PerformanceBenchmark.AddBaselinePerformanceTestRunResults(testResultXmlParser, baselinePerformanceTestRunResults, baselineTestResults);
214+
215+
// Assert
216+
Assert.IsTrue(PerformanceBenchmark.BaselineResultFilesExist);
217+
AssertCorrectBaselineXmlFilePaths(new[] { baselineXmlFilePath });
218+
AssertCorrectResultsXmlDirectoryPaths(new[] { resultsXmlDir });
146219
Assert.NotNull(baselineTestResults);
147220
Assert.IsTrue(baselineTestResults.Count > 0);
148221
Assert.NotNull(baselinePerformanceTestRunResults);
@@ -157,9 +230,9 @@ public void Verify_Verify_AddBaselineAndNonBaselinePerformanceTestRunResults()
157230
var baselineXmlFilePath = EnsureFullPath("baseline.xml");
158231
var args = new[]
159232
{
160-
string.Format("--testresultsxmlsource={0}", resultXmlFilePath),
161-
string.Format("--baselinexmlsource={0}", baselineXmlFilePath)
162-
};
233+
string.Format("--testresultsxmlsource={0}", resultXmlFilePath),
234+
string.Format("--baselinexmlsource={0}", baselineXmlFilePath)
235+
};
163236
optionsParser.ParseOptions(PerformanceBenchmark, args);
164237

165238
// Act

‎UnityPerformanceBenchmarkReporterTests/TestData/Results/results_v2.xml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<test-run id="2" testcasecount="5" result="Passed" total="5" passed="5" failed="0" inconclusive="0" skipped="0" asserts="0" engine-version="3.5.0.0" clr-version="4.0.30319.42000" start-time="2020-05-15 09:37:30Z" end-time="2020-05-15 09:37:37Z" duration="7.0829062">
3+
<test-suite type="TestSuite" id="1000" name="PerformanceTests" fullname="PerformanceTests" runstate="Runnable" testcasecount="18" result="Passed" start-time="2020-05-15 09:37:30Z" end-time="2020-05-15 09:37:37Z" duration="7.082906" total="5" passed="5" failed="0" inconclusive="0" skipped="0" asserts="0">
4+
<properties />
5+
<test-suite type="Assembly" id="1007" name="PerformanceTests.PlayMode.dll" fullname="C:/Unity/PerformanceTests/Library/ScriptAssemblies/PerformanceTests.PlayMode.dll" runstate="Runnable" testcasecount="5" result="Passed" start-time="2020-05-15 09:37:30Z" end-time="2020-05-15 09:37:37Z" duration="6.988342" total="5" passed="5" failed="0" inconclusive="0" skipped="0" asserts="0">
6+
<properties>
7+
<property name="_PID" value="15600" />
8+
<property name="_APPDOMAIN" value="Unity Child Domain" />
9+
<property name="platform" value="PlayMode" />
10+
</properties>
11+
<test-suite type="TestSuite" id="1008" name="PerformanceTests" fullname="PerformanceTests" runstate="Runnable" testcasecount="5" result="Passed" start-time="2020-05-15 09:37:30Z" end-time="2020-05-15 09:37:37Z" duration="6.952032" total="5" passed="5" failed="0" inconclusive="0" skipped="0" asserts="0">
12+
<properties />
13+
<test-suite type="TestSuite" id="1009" name="PlayMode" fullname="PerformanceTests.PlayMode" runstate="Runnable" testcasecount="5" result="Passed" start-time="2020-05-15 09:37:30Z" end-time="2020-05-15 09:37:37Z" duration="6.922271" total="5" passed="5" failed="0" inconclusive="0" skipped="0" asserts="0">
14+
<properties />
15+
<test-suite type="TestFixture" id="1001" name="PerformanceTests" fullname="PerformanceTests.PlayMode.PerformanceTests" classname="PerformanceTests.PlayMode.PerformanceTests" runstate="Runnable" testcasecount="5" result="Passed" start-time="2020-05-15 09:37:30Z" end-time="2020-05-15 09:37:37Z" duration="6.890612" total="5" passed="5" failed="0" inconclusive="0" skipped="0" asserts="0">
16+
<properties />
17+
<test-case id="1005" name="ArrayForEachTest" fullname="PerformanceTests.PlayMode.PerformanceTests.ArrayForEachTest" methodname="ArrayForEachTest" classname="PerformanceTests.PlayMode.PerformanceTests" runstate="Runnable" seed="576498059" result="Passed" start-time="2020-05-15 09:37:30Z" end-time="2020-05-15 09:37:32Z" duration="1.581622" asserts="0">
18+
<properties>
19+
<property name="Category" value="Performance" />
20+
<property name="Category" value="Loops" />
21+
</properties>
22+
<output><![CDATA[Time.GC() Undefined Median:0.00 Min:0.00 Max:0.02 Avg:0.00 Std:0.01 SampleCount: 9 Sum: 0.04
23+
Time Microsecond Median:37.57 Min:37.19 Max:45.71 Avg:38.57 Std:2.58 SampleCount: 9 Sum: 347.11
24+
25+
##performancetestresult2:{"SampleGroups":[{"Name":"Time.GC()","Unit":8,"IncreaseIsBetter":false,"Samples":[0.0,0.0,0.0,0.020448289421942586,0.0,0.0,0.0,0.0,0.015729453401494297],"Min":0.0,"Max":0.020448289421942586,"Median":0.0,"Average":0.0040197492026040988,"StandardDeviation":0.00760206690992495,"Sum":0.036177742823436887},{"Name":"Time","Unit":2,"IncreaseIsBetter":false,"Samples":[0.037669838773102621,0.037187259142744795,0.037367636649626418,0.037707353519465213,0.037311561148250086,0.037567518678725889,0.037426071569012967,0.039161934722768424,0.045708611875737264],"Min":0.037187259142744795,"Max":0.045708611875737264,"Median":0.037567518678725889,"Average":0.038567531786603743,"StandardDeviation":0.0025845571270634482,"Sum":0.34710778607943366}],"Name":"PerformanceTests.PlayMode.PerformanceTests.ArrayForEachTest","Version":"1","Categories":["Performance","Loops"]}
26+
##performancetestruninfo2:{"TestSuite":"Playmode","Date":1589535447192,"Player":{"Development":true,"ScreenWidth":1920,"ScreenHeight":1080,"ScreenRefreshRate":60,"Fullscreen":false,"Vsync":1,"AntiAliasing":2,"Batchmode":false,"RenderThreadingMode":"MultiThreaded","GpuSkinning":true,"Platform":"WindowsEditor","ColorSpace":"Gamma","AnisotropicFiltering":"ForceEnable","BlendWeights":"FourBones","GraphicsApi":"Direct3D11","ScriptingBackend":"IL2CPP","AndroidTargetSdkVersion":"AndroidApiLevelAuto","AndroidBuildSystem":"Gradle","BuildTarget":"Android","StereoRenderingPath":"MultiPass"},"Hardware":{"OperatingSystem":"Windows 10 (10.0.0) 64bit","DeviceModel":"930X5J/910S5J/940X5J (SAMSUNG ELECTRONICS CO., LTD.)","DeviceName":"DESKTOP-FS0JNSD","ProcessorType":"Intel(R) Core(TM) i5-4200U CPU @ 1.60GHz","ProcessorCount":4,"GraphicsDeviceName":"Intel(R) HD Graphics Family","SystemMemorySizeMB":8106},"Editor":{"Version":"2019.3.0f6","Branch":"2019.3/staging","Changeset":"27ab2135bccf","Date":1579669995},"Dependencies":["[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]","[email protected]"],"Results":[]}
27+
]]></output>
28+
</test-case>
29+
<test-case id="1002" name="ArrayTest" fullname="PerformanceTests.PlayMode.PerformanceTests.ArrayTest" methodname="ArrayTest" classname="PerformanceTests.PlayMode.PerformanceTests" runstate="Runnable" seed="810510945" result="Passed" start-time="2020-05-15 09:37:32Z" end-time="2020-05-15 09:37:33Z" duration="1.061096" asserts="0">
30+
<properties>
31+
<property name="Category" value="Performance" />
32+
<property name="Category" value="Loops" />
33+
</properties>
34+
<output><![CDATA[Time Microsecond Median:41.67 Min:40.55 Max:43.14 Avg:41.71 Std:0.89 SampleCount: 9 Sum: 375.43
35+
36+
##performancetestresult2:{"SampleGroups":[{"Name":"Time","Unit":2,"IncreaseIsBetter":false,"Samples":[0.043138577154308619,0.040829559118236457,0.040554108216432877,0.040893336673346707,0.040984368737474949,0.042761873747495,0.042405110220440911,0.042186472945891736,0.041671793587174315],"Min":0.040554108216432877,"Max":0.043138577154308619,"Median":0.041671793587174315,"Average":0.041713911155644622,"StandardDeviation":0.00089215964495631753,"Sum":0.37542520040080157}],"Name":"PerformanceTests.PlayMode.PerformanceTests.ArrayTest","Version":"1","Categories":["Performance","Loops"]}
37+
]]></output>
38+
</test-case>
39+
<test-case id="1006" name="ListForEachTest" fullname="PerformanceTests.PlayMode.PerformanceTests.ListForEachTest" methodname="ListForEachTest" classname="PerformanceTests.PlayMode.PerformanceTests" runstate="Runnable" seed="2127200165" result="Passed" start-time="2020-05-15 09:37:33Z" end-time="2020-05-15 09:37:35Z" duration="1.415228" asserts="0">
40+
<properties>
41+
<property name="Category" value="Performance" />
42+
<property name="Category" value="Loops" />
43+
<property name="Version" value="Unity.PerformanceTesting.VersionAttribute" />
44+
</properties>
45+
<output><![CDATA[Time.GC() Undefined Median:0.00 Min:0.00 Max:0.09 Avg:0.02 Std:0.04 SampleCount: 9 Sum: 0.18
46+
Time Microsecond Median:257.00 Min:248.14 Max:295.52 Avg:260.46 Std:13.18 SampleCount: 9 Sum: 2344.16
47+
48+
##performancetestresult2:{"SampleGroups":[{"Name":"Time.GC()","Unit":8,"IncreaseIsBetter":false,"Samples":[0.087855297157622733,0.0,0.0,0.0,0.0,0.087855297157622733,0.0,0.0,0.0],"Min":0.0,"Max":0.087855297157622733,"Median":0.0,"Average":0.019523399368360606,"StandardDeviation":0.036524935730782089,"Sum":0.17571059431524547},{"Name":"Time","Unit":2,"IncreaseIsBetter":false,"Samples":[0.29551808785529732,0.25604031007751954,0.2570033591731265,0.24813875968992261,0.25387674418604661,0.25015219638242919,0.26157803617571079,0.261019638242894,0.26083074935400491],"Min":0.24813875968992261,"Max":0.29551808785529732,"Median":0.2570033591731265,"Average":0.26046198679299454,"StandardDeviation":0.013181026783500202,"Sum":2.344157881136951}],"Name":"PerformanceTests.PlayMode.PerformanceTests.ListForEachTest","Version":"1.0.0","Categories":["Performance","Loops"]}
49+
]]></output>
50+
</test-case>
51+
<test-case id="1003" name="ListTest" fullname="PerformanceTests.PlayMode.PerformanceTests.ListTest" methodname="ListTest" classname="PerformanceTests.PlayMode.PerformanceTests" runstate="Runnable" seed="1248558080" result="Passed" start-time="2020-05-15 09:37:35Z" end-time="2020-05-15 09:37:36Z" duration="1.288495" asserts="0">
52+
<properties>
53+
<property name="Category" value="Performance" />
54+
<property name="Category" value="Loops" />
55+
</properties>
56+
<output><![CDATA[Time.GC() Undefined Median:0.00 Min:0.00 Max:0.05 Avg:0.01 Std:0.02 SampleCount: 9 Sum: 0.10
57+
Time Microsecond Median:153.60 Min:148.25 Max:162.99 Avg:154.13 Std:4.40 SampleCount: 9 Sum: 1387.16
58+
59+
##performancetestresult2:{"SampleGroups":[{"Name":"Time.GC()","Unit":8,"IncreaseIsBetter":false,"Samples":[0.0,0.0,0.05128205128205128,0.0,0.0,0.0,0.05128205128205128,0.0,0.0],"Min":0.0,"Max":0.05128205128205128,"Median":0.0,"Average":0.011396011396011395,"StandardDeviation":0.021319985109823025,"Sum":0.10256410256410256},{"Name":"Time","Unit":2,"IncreaseIsBetter":false,"Samples":[0.15097677224736053,0.14996561085972854,0.16299245852187033,0.1535962292609353,0.15083951734539977,0.14825158371040723,0.15637978883861237,0.15749758672699843,0.15666485671191549],"Min":0.14825158371040723,"Max":0.16299245852187033,"Median":0.1535962292609353,"Average":0.15412937824702533,"StandardDeviation":0.004401058002247338,"Sum":1.3871644042232278}],"Name":"PerformanceTests.PlayMode.PerformanceTests.ListTest","Version":"1","Categories":["Performance","Loops"]}
60+
]]></output>
61+
</test-case>
62+
<test-case id="1004" name="ToArrayTest" fullname="PerformanceTests.PlayMode.PerformanceTests.ToArrayTest" methodname="ToArrayTest" classname="PerformanceTests.PlayMode.PerformanceTests" runstate="Runnable" seed="296807729" result="Passed" start-time="2020-05-15 09:37:36Z" end-time="2020-05-15 09:37:37Z" duration="1.467734" asserts="0">
63+
<properties>
64+
<property name="Category" value="Performance" />
65+
<property name="Category" value="Loops" />
66+
</properties>
67+
<output><![CDATA[Time.GC() Undefined Median:1.00 Min:1.00 Max:1.02 Avg:1.00 Std:0.01 SampleCount: 9 Sum: 9.04
68+
Time Microsecond Median:58.28 Min:57.13 Max:104.18 Avg:67.68 Std:14.78 SampleCount: 9 Sum: 609.15
69+
70+
##performancetestresult2:{"SampleGroups":[{"Name":"Time.GC()","Unit":8,"IncreaseIsBetter":false,"Samples":[1.0,1.0,1.0218790218790219,1.0,1.0,1.0,1.0,1.0218790218790219,1.0],"Min":1.0,"Max":1.0218790218790219,"Median":1.0,"Average":1.0048620048620047,"StandardDeviation":0.0090959782032256683,"Sum":9.0437580437580429},{"Name":"Time","Unit":2,"IncreaseIsBetter":false,"Samples":[0.10418120978120975,0.067075160875160911,0.058284813384813408,0.057146460746460764,0.057129794079794113,0.057665379665379719,0.057984813384813289,0.078527413127413,0.0711528314028314],"Min":0.057129794079794113,"Max":0.10418120978120975,"Median":0.058284813384813408,"Average":0.067683097383097371,"StandardDeviation":0.014784571833400106,"Sum":0.60914787644787638}],"Name":"PerformanceTests.PlayMode.PerformanceTests.ToArrayTest","Version":"1","Categories":["Performance","Loops"]}
71+
]]></output>
72+
</test-case>
73+
</test-suite>
74+
</test-suite>
75+
</test-suite>
76+
</test-suite>
77+
</test-suite>
78+
</test-run>

0 commit comments

Comments
 (0)