Merge branch '2018.2' into 2018.1

Author: seanstolberg-unity

MismatchedConfigTooltip.png

@@ -13,6 +13,15 @@ public class OptionsParser
         private readonly string learnMore =
             "To learn more about the Unity Performance Benchmark Reporter visit the Unity Performance Benchmark Reporter GitHub wiki at https://github.com/Unity-Technologies/PerformanceBenchmarkReporter/wiki.";
 
+        private readonly string commandLineOptionFormat =
+            "// Command line option format\r\n--results=<Path to a test result XML filename OR directory>... [--baseline=\"Path to a baseline XML filename\"] [--reportdirpath=\"Path to where the report will be written\"]";
+
+        private readonly string example1 = "// Run reporter with one performance test result .xml file\r\n--results=\"G:\\My Drive\\XRPerfRuns\\results\\results.xml\"";
+
+        private readonly string example2 = "// Run reporter against a directory containing one or more performance test result .xml files\r\n--results=\"G:\\My Drive\\XRPerfRuns\\results\"  ";
+
+        private readonly string example3 = "// Run reporter against a directory containing one or more performance test result .xml files, and a baseline .xml result file\r\n--results=\"G:\\My Drive\\XRPerfRuns\\results\" --baseline=\"G:\\My Drive\\XRPerfRuns\\baselines\\baseline.xml\" ";
+
         public enum ResultType
         {
             Test,
@@ -53,17 +62,17 @@ private OptionSet GetOptions(PerformanceBenchmark performanceBenchmark)
         {
             return new OptionSet()
                 .Add("?|help|h", "Prints out the options.", option => help = option != null)
-                .Add("testresultsxmlsource=", "REQUIRED - Path to a test result XML filename or directory. Directories are searched resursively. You can repeat this option with multiple result file or directory paths.",
+                .Add("results|testresultsxmlsource=", "REQUIRED - Path to a test result XML filename OR directory. Directories are searched resursively. You can repeat this option with multiple result file or directory paths.",
                     xmlsource =>
                     {
                         performanceBenchmark.AddXmlSourcePath(xmlsource, "testresultsxmlsource", ResultType.Test);
                     })
-                .Add("baselinexmlsource:", "OPTIONAL - Path to a baseline XML filename or directory. Directories are searched resursively. You can repeat this option with multiple baseline file or directory paths.",
+                .Add("baseline|baselinexmlsource:", "OPTIONAL - Path to a baseline XML filename.",
                         xmlsource =>
                         {
                             performanceBenchmark.AddXmlSourcePath(xmlsource, "baselinexmlsource", ResultType.Baseline);
                         })
-                .Add("reportdirpath:", "OPTIONAL - Path to directory where the UnityPerformanceBenchmark report will be written. Default is current working directory.",
+                .Add("report|reportdirpath:", "OPTIONAL - Path to where the report will be written. Default is current working directory.",
                     performanceBenchmark.AddReportDirPath);
         }
 
@@ -78,7 +87,12 @@ private void ShowHelp(string message, OptionSet optionSet)
 
             Console.WriteLine(about + "\r\n");
             Console.WriteLine(learnMore + "\r\n");
-            Console.WriteLine("Usage is:");
+            Console.WriteLine("Usage is:" + "\r\n");
+            Console.WriteLine(commandLineOptionFormat + "\r\n");
+            Console.WriteLine(example1 + "\r\n");
+            Console.WriteLine(example2 + "\r\n");
+            Console.WriteLine(example3 + "\r\n");
+            Console.WriteLine("Options: \r\n");
             optionSet.WriteOptionDescriptions(Console.Error);
             Environment.Exit(-1);
         }

TestConfigWithMismatchedItems.png

@@ -23,19 +23,25 @@ public class PerformanceBenchmark
         private PerformanceTestRun firstTestRun = new PerformanceTestRun();
         private readonly PerformanceTestRunProcessor performanceTestRunProcessor = new PerformanceTestRunProcessor();
         private readonly string xmlFileExtension = ".xml";
+        private readonly Dictionary<string, string[]> excludedConfigFieldNames = new Dictionary<string, string[]>();
 
 
         public bool BaselineResultFilesExist => BaselineXmlFilePaths.Any() || BaselineXmlDirectoryPaths.Any();
 
         public bool ResultFilesExist => ResultXmlFilePaths.Any() || ResultXmlDirectoryPaths.Any();
 
-        public PerformanceBenchmark()
+        public PerformanceBenchmark(Dictionary<string, string[]> configFieldNames = null)
         {
             // Default significant figures to use for non-integer metrics if user doesn't specify another value.
             // Most values are in milliseconds or a count of something, so using more often creates an artificial baseline
             // failure based on insignificant digits equating to a microsecond, or less, time difference. The Unity Profiler only shows
-            // up to three significant figures for milliseconds as well.
+            // up to 2 significant figures for milliseconds as well, so this is what folks are used to working with.
             SigFig = 2;
+
+            if (configFieldNames != null)
+            {
+                excludedConfigFieldNames = configFieldNames;
+            }
         }
 
         public void AddPerformanceTestRunResults(
@@ -55,7 +61,7 @@ public void AddBaselinePerformanceTestRunResults(
             AddTestResults(testResultXmlParser, baselinePerformanceTestRunResults, baselineTestResults, baselineTestResults, BaselineXmlDirectoryPaths, BaselineXmlFilePaths, true);
         }
 
-        public void AddTestResults(
+        private void AddTestResults(
             TestResultXmlParser testResultXmlParser,  
             List<PerformanceTestRunResult> runResults, 
             List<TestResult> testResults, 
@@ -64,7 +70,7 @@ public void AddTestResults(
             HashSet<string> xmlFileNamePaths,
             bool isBaseline = false)
         {
-            if (xmlDirectoryPaths.Any())
+            if (!isBaseline && xmlDirectoryPaths.Any())
             {
                 foreach (var xmlDirectory in xmlDirectoryPaths)
                 {
@@ -140,15 +146,22 @@ private void ValidateMetadata(PerformanceTestRun performanceTestRun, string xmlF
             }
             else
             {
-                MetadataValidator.ValidatePlayerSystemInfo(firstTestRun, performanceTestRun, firstTestRunResultPath, xmlFileNamePath);
-                MetadataValidator.ValidatePlayerSettings(firstTestRun, performanceTestRun, firstTestRunResultPath, xmlFileNamePath);
-                MetadataValidator.ValidateQualitySettings(firstTestRun, performanceTestRun, firstTestRunResultPath, xmlFileNamePath);
-                MetadataValidator.ValidateScreenSettings(firstTestRun, performanceTestRun, firstTestRunResultPath, xmlFileNamePath);
-                MetadataValidator.ValidateBuildSettings(firstTestRun, performanceTestRun, firstTestRunResultPath, xmlFileNamePath);
-                MetadataValidator.ValidateEditorVersion(firstTestRun, performanceTestRun, firstTestRunResultPath, xmlFileNamePath);
+                MetadataValidator.ValidatePlayerSystemInfo(firstTestRun, performanceTestRun, firstTestRunResultPath, xmlFileNamePath, ExcludedFieldNames<PlayerSystemInfo>());
+                MetadataValidator.ValidatePlayerSettings(firstTestRun, performanceTestRun, firstTestRunResultPath, xmlFileNamePath, ExcludedFieldNames<PlayerSettings>());
+                MetadataValidator.ValidateQualitySettings(firstTestRun, performanceTestRun, firstTestRunResultPath, xmlFileNamePath, ExcludedFieldNames<QualitySettings>());
+                MetadataValidator.ValidateScreenSettings(firstTestRun, performanceTestRun, firstTestRunResultPath, xmlFileNamePath, ExcludedFieldNames<ScreenSettings>());
+                MetadataValidator.ValidateBuildSettings(firstTestRun, performanceTestRun, firstTestRunResultPath, xmlFileNamePath, ExcludedFieldNames<BuildSettings>());
+                MetadataValidator.ValidateEditorVersion(firstTestRun, performanceTestRun, firstTestRunResultPath, xmlFileNamePath, ExcludedFieldNames<EditorVersion>());
             }
         }
 
+        private string[] ExcludedFieldNames<T>()
+        {
+            return excludedConfigFieldNames.ContainsKey(typeof(T).Name)
+                ? excludedConfigFieldNames[typeof(T).Name]
+                : null;
+        }
+
         public void AddXmlSourcePath(string xmlSourcePath, string optionName, OptionsParser.ResultType resultType)
         {
             if (string.IsNullOrEmpty(xmlSourcePath))

UnityPerformanceBenchmarkReport.png

@@ -8,14 +8,19 @@ namespace UnityPerformanceBenchmarkReporter
 {
     internal class Program
     {
+        private static readonly Dictionary<string, string[]> ExcludedConfigFieldNames = new Dictionary<string, string[]>
+        {
+            {typeof(EditorVersion).Name, new []{"DateSeconds", "RevisionValue", "Branch"}}
+        };
+
         private static void Main(string[] args)
         {
             var aggregateTestRunResults = new List<PerformanceTestRunResult>();
             var baselinePerformanceTestRunResults = new List<PerformanceTestRunResult>();
             var baselineTestResults = new List<TestResult>();
             var performanceTestRunResults = new List<PerformanceTestRunResult>();
             var testResults = new List<TestResult>();
-            var performanceBenchmark = new PerformanceBenchmark();
+            var performanceBenchmark = new PerformanceBenchmark(ExcludedConfigFieldNames);
             var optionsParser = new OptionsParser();
 
             optionsParser.ParseOptions(performanceBenchmark, args);
@@ -49,8 +54,13 @@ private static void Main(string[] args)
                 }
             }
 
-            var reportWriter = new ReportWriter();
-            reportWriter.WriteReport(aggregateTestRunResults, performanceBenchmark.MetadataValidator, performanceBenchmark.SigFig, performanceBenchmark.ReportDirPath, performanceBenchmark.BaselineResultFilesExist);
+            var reportWriter = new ReportWriter(ExcludedConfigFieldNames);
+            reportWriter.WriteReport(
+                aggregateTestRunResults, 
+                performanceBenchmark.MetadataValidator, 
+                performanceBenchmark.SigFig, 
+                performanceBenchmark.ReportDirPath, 
+                performanceBenchmark.BaselineResultFilesExist);
         }
     }
 }