Updates to the jit bench harness (#14467)

Updates to the jit bench harness

Introduce a number of new options:
* use-existing-setup to reuse a previously set up environment
* iterations to control number of iterations
* tiering to enable tiered jit
* disable-r2r to disable loading of R2R images
* disable-ngen to disable loading of NGEN images
* minopts to force jit to generate minopts code

Also start tracking average response time. This is not yet "uploaded"
since the underlying data reported by the app needs some updates.

1 files changed, 156 insertions, 47 deletions

diff --git a/tests/src/performance/Scenario/JitBench/JitBenchHarness.cs b/tests/src/performance/Scenario/JitBench/JitBenchHarness.cs
index 36f7c8423f..035b74f56f 100644
--- a/tests/src/performance/Scenario/JitBench/JitBenchHarness.cs
+++ b/tests/src/performance/Scenario/JitBench/JitBenchHarness.cs
@@ -1,5 +1,6 @@
-// Copyright (c) Microsoft. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
 
 using CommandLine;
 using CommandLine.Text;
@@ -22,6 +23,62 @@ namespace JitBench
         {
             var options = JitBenchHarnessOptions.Parse(args);
 
+            SetupStatics(options);
+
+            using (var h = new XunitPerformanceHarness(args))
+            {
+                ProcessStartInfo startInfo = options.UseExistingSetup ? UseExistingSetup() : CreateNewSetup();
+
+                string scenarioName = "MusicStore";
+
+                if (options.EnableTiering)
+                {
+                    startInfo.Environment.Add("COMPlus_EXPERIMENTAL_TieredCompilation", "1");
+                    scenarioName += " Tiering";
+                }
+
+                if (options.Minopts)
+                {
+                    startInfo.Environment.Add("COMPlus_JITMinOpts", "1");
+                    scenarioName += " Minopts";
+                }
+
+                if (options.DisableR2R)
+                {
+                    startInfo.Environment.Add("COMPlus_ReadyToRun", "0");
+                    scenarioName += " NoR2R";
+                }
+
+                if (options.DisableNgen)
+                {
+                    startInfo.Environment.Add("COMPlus_ZapDisable", "1");
+                    scenarioName += " NoNgen";
+                }
+
+                var scenarioConfiguration = CreateScenarioConfiguration();
+
+                h.RunScenario(startInfo, () => { PrintHeader(scenarioName); }, PostIteration, PostProcessing, scenarioConfiguration);
+            }
+        }
+
+        private static ScenarioConfiguration CreateScenarioConfiguration()
+        {
+            var config = new ScenarioConfiguration(TimeSpan.FromMilliseconds(60000))
+            {
+                Iterations = (int)s_iterations
+            };
+            return config;
+        }
+
+        private static void SetupStatics(JitBenchHarnessOptions options)
+        {
+            // Set variables we will need to store results.
+            s_iterations = options.Iterations;
+            s_iteration = 0;
+            s_startupTimes = new double[s_iterations];
+            s_requestTimes = new double[s_iterations];
+            s_steadystateTimes = new double[s_iterations];
+
             s_temporaryDirectory = options.IntermediateOutputDirectory;
             s_targetArchitecture = options.TargetArchitecture;
             if (string.IsNullOrWhiteSpace(s_targetArchitecture))
@@ -36,25 +93,8 @@ namespace JitBench
             // C:\j\w\perf_scenario---5b001a46\bin\sandbox\JitBench\JitBench-dev
             // C:\j\w\perf_scenario---5b001a46\bin\sandbox\J
             s_jitBenchDevDirectory = Path.Combine(s_temporaryDirectory, "J");
-
-            using (var h = new XunitPerformanceHarness(args))
-            {
-                ProcessStartInfo startInfo = Setup();
-                h.RunScenario(startInfo, () => { PrintHeader("Running Benchmark Scenario"); }, PostIteration, PostProcessing, s_ScenarioConfiguration);
-            }
-        }
-
-        static Program()
-        {
-            s_ScenarioConfiguration = new ScenarioConfiguration(TimeSpan.FromMilliseconds(60000)) {
-                Iterations = 11
-            };
-
-            // Set variables we will need to store results.
-            s_iteration = 0;
-            s_startupTimes = new double[s_ScenarioConfiguration.Iterations];
-            s_requestTimes = new double[s_ScenarioConfiguration.Iterations];
-            s_targetArchitecture = "";
+            s_dotnetProcessFileName = Path.Combine(s_jitBenchDevDirectory, ".dotnet", "dotnet.exe");
+            s_musicStoreDirectory = Path.Combine(s_jitBenchDevDirectory, "src", "MusicStore");
         }
 
         private static void DownloadAndExtractJitBenchRepo()
@@ -101,8 +141,7 @@ namespace JitBench
             };
             LaunchProcess(psi, 180000);
 
-            // TODO: This is currently hardcoded, but we could probably pull it from the powershell cmdlet call.
-            return new Dictionary<string, string> { { "PATH", $"{Path.Combine(s_jitBenchDevDirectory, ".dotnet")};{psi.Environment["PATH"]}" } };
+            return GetInitialEnvironment();
         }
 
         private static void ModifySharedFramework()
@@ -153,17 +192,16 @@ namespace JitBench
         private static IDictionary<string, string> GenerateStore(IDictionary<string, string> environment)
         {
             // This step generates some environment variables needed later.
-            var environmentFileName = "JitBenchEnvironment.txt";
             var psi = new ProcessStartInfo() {
                 WorkingDirectory = s_jitBenchDevDirectory,
                 FileName = "powershell.exe",
-                Arguments = $"-Command \".\\AspNet-GenerateStore.ps1 -InstallDir .store -Architecture {s_targetArchitecture} -Runtime win7-{s_targetArchitecture}; gi env:JITBENCH_*, env:DOTNET_SHARED_STORE | %{{ \\\"$($_.Name)=$($_.Value)\\\" }} 1>>{environmentFileName}\""
+                Arguments = $"-Command \".\\AspNet-GenerateStore.ps1 -InstallDir .store -Architecture {s_targetArchitecture} -Runtime win7-{s_targetArchitecture}; gi env:JITBENCH_*, env:DOTNET_SHARED_STORE | %{{ \\\"$($_.Name)=$($_.Value)\\\" }} 1>>{EnvironmentFileName}\""
             };
 
             LaunchProcess(psi, 1800000, environment);
 
             // Return the generated environment variables.
-            return GetEnvironment(environment, Path.Combine(s_jitBenchDevDirectory, environmentFileName));
+            return GetEnvironment(environment, Path.Combine(s_jitBenchDevDirectory, EnvironmentFileName));
         }
 
         private static IDictionary<string, string> GetEnvironment(IDictionary<string, string> environment, string fileName)
@@ -208,7 +246,36 @@ namespace JitBench
             LaunchProcess(psi, 300000, environment);
         }
 
-        private static ProcessStartInfo Setup()
+        // Return an environment with the downloaded dotnet on the path.
+        private static IDictionary<string, string> GetInitialEnvironment()
+        {
+            // TODO: This is currently hardcoded, but we could probably pull it from the powershell cmdlet call.
+            var environment = new Dictionary<string, string> { { "PATH", $"{Path.Combine(s_jitBenchDevDirectory, ".dotnet")};{Environment.GetEnvironmentVariable("PATH")}" } };
+
+            return environment;
+        }
+
+        private static ProcessStartInfo UseExistingSetup()
+        {
+            PrintHeader("Using existing SETUP");
+
+            var environment = GetInitialEnvironment();
+            environment = GetEnvironment(environment, Path.Combine(s_jitBenchDevDirectory, EnvironmentFileName));
+            ValidateEnvironment(environment);
+
+            var psi = new ProcessStartInfo()
+            {
+                FileName = "cmd.exe",
+                Arguments = $"/C \"{s_dotnetProcessFileName} MusicStore.dll 1>{MusicStoreRedirectedStandardOutputFileName}\"",
+                WorkingDirectory = Path.Combine(s_musicStoreDirectory, "bin", "Release", JitBenchTargetFramework, "publish")
+            };
+
+            foreach (KeyValuePair<string, string> pair in environment)
+                psi.Environment.Add(pair.Key, pair.Value);
+
+            return psi;
+        }
+        private static ProcessStartInfo CreateNewSetup()
         {
             PrintHeader("Starting SETUP");
 
@@ -221,28 +288,17 @@ namespace JitBench
 
             environment = GenerateStore(environment);
 
-            var expectedVariables = new string[] {
-                "PATH",
-                "JITBENCH_ASPNET_MANIFEST",
-                "JITBENCH_FRAMEWORK_VERSION",
-                "JITBENCH_ASPNET_VERSION",
-                "DOTNET_SHARED_STORE"
-            };
-            if (expectedVariables.Except(environment.Keys, StringComparer.OrdinalIgnoreCase).Any())
-                throw new Exception("Missing expected environment variables.");
+            ValidateEnvironment(environment);
 
             ModifySharedFramework();
 
-            var dotnetProcessFileName = Path.Combine(s_jitBenchDevDirectory, ".dotnet", "dotnet.exe");
-            var musicStoreDirectory = Path.Combine(s_jitBenchDevDirectory, "src", "MusicStore");
-
-            RestoreMusicStore(musicStoreDirectory, dotnetProcessFileName, environment);
-            PublishMusicStore(musicStoreDirectory, dotnetProcessFileName, environment);
+            RestoreMusicStore(s_musicStoreDirectory, s_dotnetProcessFileName, environment);
+            PublishMusicStore(s_musicStoreDirectory, s_dotnetProcessFileName, environment);
 
             var psi = new ProcessStartInfo() {
                 FileName = "cmd.exe",
-                Arguments = $"/C \"{dotnetProcessFileName} MusicStore.dll 1>{MusicStoreRedirectedStandardOutputFileName}\"",
-                WorkingDirectory = Path.Combine(musicStoreDirectory, "bin", "Release", JitBenchTargetFramework, "publish")
+                Arguments = $"/C \"{s_dotnetProcessFileName} MusicStore.dll 1>{MusicStoreRedirectedStandardOutputFileName}\"",
+                WorkingDirectory = Path.Combine(s_musicStoreDirectory, "bin", "Release", JitBenchTargetFramework, "publish")
             };
 
             foreach (KeyValuePair<string, string> pair in environment)
@@ -251,10 +307,24 @@ namespace JitBench
             return psi;
         }
 
+        private static void ValidateEnvironment(IDictionary<string, string> environment)
+        {
+            var expectedVariables = new string[] {
+                "PATH",
+                "JITBENCH_ASPNET_MANIFEST",
+                "JITBENCH_FRAMEWORK_VERSION",
+                "JITBENCH_ASPNET_VERSION",
+                "DOTNET_SHARED_STORE"
+            };
+            if (expectedVariables.Except(environment.Keys, StringComparer.OrdinalIgnoreCase).Any())
+                throw new Exception("Missing expected environment variables.");
+        }
+
         private const string MusicStoreRedirectedStandardOutputFileName = "measures.txt";
         private const string JitBenchRepoUrl = "https://github.com/aspnet/JitBench";
         private const string JitBenchCommitSha1Id = "b7e7b786c60daa255aacaea85006afe4d4ec8306";
         private const string JitBenchTargetFramework = "netcoreapp2.1";
+        private const string EnvironmentFileName = "JitBenchEnvironment.txt";
 
         private static void PostIteration()
         {
@@ -263,6 +333,7 @@ namespace JitBench
 
             double? startupTime = null;
             double? requestTime = null;
+            double? steadyStateAverageTime = null;
             foreach (string line in File.ReadLines(path))
             {
                 Match match = Regex.Match(line, @"^Server started in (\d+)ms$");
@@ -276,6 +347,13 @@ namespace JitBench
                 if (match.Success && match.Groups.Count == 2)
                 {
                     requestTime = Convert.ToDouble(match.Groups[1].Value);
+                    continue;
+                }
+
+                match = Regex.Match(line, @"^Steadystate average response time: (\d+)ms$");
+                if (match.Success && match.Groups.Count == 2)
+                {
+                    steadyStateAverageTime = Convert.ToDouble(match.Groups[1].Value);
                     break;
                 }
             }
@@ -284,13 +362,17 @@ namespace JitBench
                 throw new Exception("Startup time was not found.");
             if (!requestTime.HasValue)
                 throw new Exception("First Request time was not found.");
+            if (!steadyStateAverageTime.HasValue)
+                throw new Exception("Steady state average response time not found.");
 
             s_startupTimes[s_iteration] = startupTime.Value;
             s_requestTimes[s_iteration] = requestTime.Value;
+            s_steadystateTimes[s_iteration] = steadyStateAverageTime.Value;
 
             PrintRunningStepInformation($"{s_iteration} Server started in {s_startupTimes[s_iteration]}ms");
             PrintRunningStepInformation($"{s_iteration} Request took {s_requestTimes[s_iteration]}ms");
             PrintRunningStepInformation($"{s_iteration} Cold start time (server start + first request time): {s_startupTimes[s_iteration] + s_requestTimes[s_iteration]}ms");
+            PrintRunningStepInformation($"{s_iteration} Average steady state response {s_steadystateTimes[s_iteration]}ms");
 
             ++s_iteration;
         }
@@ -310,6 +392,9 @@ namespace JitBench
             var request = new ScenarioTestModel("First Request");
             scenarioBenchmark.Tests.Add(request);
 
+            // TODO: add response time once jit bench is updated to
+            // report more reasonable numbers.
+
             // Add measured metrics to each test.
             startup.Performance.Metrics.Add(new MetricModel {
                 Name = "Duration",
@@ -322,7 +407,7 @@ namespace JitBench
                 Unit = "ms"
             });
 
-            for (int i = 0; i < s_ScenarioConfiguration.Iterations; ++i)
+            for (int i = 0; i < s_iterations; ++i)
             {
                 var startupIteration = new IterationModel { Iteration = new Dictionary<string, double>() };
                 startupIteration.Iteration.Add("Duration", s_startupTimes[i]);
@@ -381,13 +466,15 @@ namespace JitBench
             Console.WriteLine($"-- {message}");
         }
 
-        private static readonly ScenarioConfiguration s_ScenarioConfiguration;
-
-        private static int s_iteration;
+        private static uint s_iterations;
+        private static uint s_iteration;
         private static double[] s_startupTimes;
         private static double[] s_requestTimes;
+        private static double[] s_steadystateTimes;
         private static string s_temporaryDirectory;
         private static string s_jitBenchDevDirectory;
+        private static string s_dotnetProcessFileName;
+        private static string s_musicStoreDirectory;
         private static string s_targetArchitecture;
 
         /// <summary>
@@ -398,8 +485,27 @@ namespace JitBench
             public JitBenchHarnessOptions()
             {
                 _tempDirectory = Directory.GetCurrentDirectory();
+                _iterations = 11;
             }
 
+            [Option("use-existing-setup", Required = false, HelpText = "Use existing setup.")]
+            public Boolean UseExistingSetup { get; set; }
+
+            [Option("tiering", Required = false, HelpText = "Enable tiered jit.")]
+            public Boolean EnableTiering { get; set; }
+
+            [Option("minopts", Required = false, HelpText = "Force jit to use minopt codegen.")]
+            public Boolean Minopts { get; set; }
+
+            [Option("disable-r2r", Required = false, HelpText = "Disable loading of R2R images.")]
+            public Boolean DisableR2R { get; set; }
+
+            [Option("disable-ngen", Required = false, HelpText = "Disable loading of ngen images.")]
+            public Boolean DisableNgen { get; set; }
+
+            [Option("iterations", Required = false, HelpText = "Number of iterations to run.")]
+            public uint Iterations { get { return _iterations; } set { _iterations = value; } }
+
             [Option('o', Required = false, HelpText = "Specifies the intermediate output directory name.")]
             public string IntermediateOutputDirectory
             {
@@ -452,6 +558,8 @@ namespace JitBench
                                     case ErrorType.BadFormatTokenError:
                                     case ErrorType.UnknownOptionError:
                                     case ErrorType.MissingRequiredOptionError:
+                                        throw new ArgumentException(
+                                                $"Missing required  command line argument '{(error as MissingRequiredOptionError).NameInfo.NameText}'");
                                     case ErrorType.MutuallyExclusiveSetError:
                                     case ErrorType.BadFormatConversionError:
                                     case ErrorType.SequenceOutOfRangeError:
@@ -488,6 +596,7 @@ namespace JitBench
             }
 
             private string _tempDirectory;
+            private uint _iterations;
         }
     }
 }