Add JitMinOpts throughput jobs

This change adds an opt_level throughput option so that we can run
throughput jobs in minopts mode. This also adds jobs to the lab for min
opts runs

2 files changed, 143 insertions, 110 deletions

diff --git a/perf.groovy b/perf.groovy
index e3f42c7f6d..609fccaabf 100644
--- a/perf.groovy
+++ b/perf.groovy
@@ -154,68 +154,75 @@ def static getOSGroup(def os) {
 [true, false].each { isPR ->
     ['Windows_NT'].each { os ->
         ['x64', 'x86'].each { arch ->
-            def architecture = arch
+            ['full_opt', 'min_opt'].each { opt_level ->
+                def architecture = arch
 
-            def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}", isPR)) {
-                // Set the label.
-                label('windows_clr_perf')
-                wrappers {
-                    credentialsBinding {
-                        string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
+                def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}", isPR)) {
+                    // Set the label.
+                    label('windows_clr_perf')
+                    wrappers {
+                        credentialsBinding {
+                            string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
+                        }
                     }
-                }
 
-            if (isPR)
-            {
-                parameters
-                {
-                    stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
-                }
-            }
-            def configuration = 'Release'
-            def runType = isPR ? 'private' : 'rolling'
-            def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
+                    if (isPR)
+                    {
+                        parameters
+                        {
+                            stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
+                        }
+                    }
+                    def configuration = 'Release'
+                    def runType = isPR ? 'private' : 'rolling'
+                    def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
 
-                steps {
-                    // Batch
+                    steps {
+                        // Batch
 
-                    batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
-                    batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
-                    batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
-                    batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
-                    //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
-                    //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
-                    batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
-                    "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
-                    "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
-                    "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
-                    "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
-                    batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
-                    batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
-                    batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
-                    batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
+                        batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
+                        batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
+                        batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
+                        batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
+                        //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
+                        //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
+                        batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
+                        "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
+                        "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
+                        "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
+                        "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
+                        batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
+                        batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
+                        batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
+                        batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
+                    }
                 }
-            }
 
-            // Save machinedata.json to /artifact/bin/ Jenkins dir
-            def archiveSettings = new ArchivalSettings()
-            archiveSettings.addFiles('throughput-*.csv')
-            Utilities.addArchival(newJob, archiveSettings)
+                // Save machinedata.json to /artifact/bin/ Jenkins dir
+                def archiveSettings = new ArchivalSettings()
+                archiveSettings.addFiles('throughput-*.csv')
+                Utilities.addArchival(newJob, archiveSettings)
 
-            Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
+                Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 
-            if (isPR) {
-                TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
-                builder.setGithubContext("${os} ${arch} CoreCLR Throughput Perf Tests")
-                builder.triggerOnlyOnComment()
-                builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+throughput.*")
-                builder.triggerForBranch(branch)
-                builder.emitTrigger(newJob)
-            }
-            else {
-                // Set a push trigger
-                TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
-                builder.emitTrigger(newJob)
+                if (isPR) {
+                    def opts = ""
+                    if (opt_level == 'min_opts')
+                    {
+                        opts = '\\W+min_opts'
+                    }
+                    TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
+                    builder.setGithubContext("${os} ${arch} ${opt_level} CoreCLR Throughput Perf Tests")
+                    builder.triggerOnlyOnComment()
+                    builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}\\W+throughput.*")
+                    builder.triggerForBranch(branch)
+                    builder.emitTrigger(newJob)
+                }
+                else {
+                    // Set a push trigger
+                    TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
+                    builder.emitTrigger(newJob)
+                }
             }
         }
     }
@@ -384,76 +391,89 @@ def static getFullThroughputJobName(def project, def os, def isPR) {
 
     // Actual perf testing on the following OSes
     def throughputOSList = ['Ubuntu14.04']
+    def throughputOptLevelList = ['full_opt', 'min_opt']
+
+    def throughputOSOptLevelList = []
+
     throughputOSList.each { os ->
-        def newJob = job(getFullThroughputJobName(project, os, isPR)) {
+        throughputOptLevelList.each { opt_level ->
+            throughputOSOptLevelList.add("${os}_${opt_level}")
+        }
+    }
 
-            label('linux_clr_perf')
-                wrappers {
-                    credentialsBinding {
-                        string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
+    throughputOSList.each { os ->
+        throughputOptLevelList.each { opt_level ->
+            def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
+
+                label('linux_clr_perf')
+                    wrappers {
+                        credentialsBinding {
+                            string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
+                        }
                     }
-                }
 
-            if (isPR)
-            {
-                parameters
+                if (isPR)
                 {
-                    stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
+                    parameters
+                    {
+                        stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
+                    }
                 }
-            }
 
-            parameters {
-                stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
-            }
+                parameters {
+                    stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
+                }
 
-            def osGroup = getOSGroup(os)
-            def runType = isPR ? 'private' : 'rolling'
-            def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
+                def osGroup = getOSGroup(os)
+                def runType = isPR ? 'private' : 'rolling'
+                def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 
-            steps {
-                shell("bash ./tests/scripts/perf-prep.sh --throughput")
-                shell("./init-tools.sh")
-                copyArtifacts(fullBuildJobName) {
-                    includePatterns("bin/Product/**")
-                    buildSelector {
-                        buildNumber('\${PRODUCT_BUILD}')
+                steps {
+                    shell("bash ./tests/scripts/perf-prep.sh --throughput")
+                    shell("./init-tools.sh")
+                    copyArtifacts(fullBuildJobName) {
+                        includePatterns("bin/Product/**")
+                        buildSelector {
+                            buildNumber('\${PRODUCT_BUILD}')
+                        }
                     }
+                    shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
+                    "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
+                    "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
+                    shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
+                    -arch \"${architecture}\" \\
+                    -os \"${os}\" \\
+                    -configuration \"${configuration}\" \\
+                    -opt_level \"${opt_level}\" \\
+                    -clr_root \"\${WORKSPACE}\" \\
+                    -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
+                    -run_type \"${runType}\" \\
+                    -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
                 }
-                shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
-                "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
-                "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
-                shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
-                -arch \"${architecture}\" \\
-                -os \"${os}\" \\
-                -configuration \"${configuration}\" \\
-                -clr_root \"\${WORKSPACE}\" \\
-                -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
-                -run_type \"${runType}\" \\
-                -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
             }
-        }
 
-        // Save machinedata.json to /artifact/bin/ Jenkins dir
-        def archiveSettings = new ArchivalSettings()
-        archiveSettings.addFiles('throughput-*.csv')
-        archiveSettings.addFiles('machinedata.json')
-        Utilities.addArchival(newJob, archiveSettings)
+            // Save machinedata.json to /artifact/bin/ Jenkins dir
+            def archiveSettings = new ArchivalSettings()
+            archiveSettings.addFiles('throughput-*.csv')
+            archiveSettings.addFiles('machinedata.json')
+            Utilities.addArchival(newJob, archiveSettings)
 
-        Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
+            Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 
-        // For perf, we need to keep the run results longer
-        newJob.with {
-            // Enable the log rotator
-            logRotator {
-                artifactDaysToKeep(7)
-                daysToKeep(300)
-                artifactNumToKeep(25)
-                numToKeep(1000)
+            // For perf, we need to keep the run results longer
+            newJob.with {
+                // Enable the log rotator
+                logRotator {
+                    artifactDaysToKeep(7)
+                    daysToKeep(300)
+                    artifactNumToKeep(25)
+                    numToKeep(1000)
+                }
             }
-        }
+        } // opt_level
     } // os
 
-    def flowJobTPRunList = throughputOSList.collect { os ->
+    def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
         "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
     }
     def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
diff --git a/tests/scripts/run-throughput-perf.py b/tests/scripts/run-throughput-perf.py
index 4fa43e9b7d..c0ed43a459 100644
--- a/tests/scripts/run-throughput-perf.py
+++ b/tests/scripts/run-throughput-perf.py
@@ -116,6 +116,7 @@ parser.add_argument('-clr_root', dest='clr_root', default=None)
 parser.add_argument('-assembly_root', dest='assembly_root', default=None)
 parser.add_argument('-benchview_path', dest='benchview_path', default=None)
 parser.add_argument('-iterations', dest='iterations', default=5, type=int)
+parser.add_argument('-opt_level', dest='opt_level', default='full_opt')
 
 ##########################################################################
 # Helper Functions
@@ -141,6 +142,7 @@ def validate_args(args):
     assembly_root = args.assembly_root
     benchview_path = args.benchview_path
     iterations = args.iterations
+    opt_level = args.opt_level.lower()
 
     def validate_arg(arg, check):
         """ Validate an individual arg
@@ -162,6 +164,7 @@ def validate_args(args):
     valid_build_types = ['Release']
     valid_run_types = ['rolling', 'private']
     valid_os = ['Windows_NT', 'Ubuntu14.04']
+    valid_opt_levels = ['full_opt', 'min_opt']
 
     arch = next((a for a in valid_archs if a.lower() == arch.lower()), arch)
     build_type = next((b for b in valid_build_types if b.lower() == build_type.lower()), build_type)
@@ -174,6 +177,7 @@ def validate_args(args):
     validate_arg(build_type, lambda item: item in valid_build_types)
     validate_arg(run_type, lambda item: item in valid_run_types)
     validate_arg(iterations, lambda item: item > 0)
+    validate_arg(opt_level, lambda item: item in valid_opt_levels)
 
     if clr_root is None:
         raise Exception('--clr_root must be set')
@@ -191,7 +195,7 @@ def validate_args(args):
         benchview_path = os.path.normpath(benchview_path)
         validate_arg(benchview_path, lambda item: os.path.isdir(benchview_path))
 
-    args = (arch, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations)
+    args = (arch, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations, opt_level)
 
     # Log configuration
     log('Configuration:')
@@ -199,6 +203,7 @@ def validate_args(args):
     log(' os: %s' % operating_system)
     log(' os_group: %s' % os_group)
     log(' build_type: %s' % build_type)
+    log(' opt_level: %s' % opt_level)
     log(' run_type: %s' % run_type)
     log(' iterations: %d' % iterations)
     log(' clr_root: %s' % clr_root)
@@ -256,7 +261,7 @@ def generateCSV(dll_name, dll_runtimes):
 
     return csv_file_name
 
-def runIterations(dll_name, dll_path, iterations, crossgen_path, jit_path, assemblies_path):
+def runIterations(dll_name, dll_path, iterations, crossgen_path, jit_path, assemblies_path, opt_level):
     """ Run throughput testing for a given dll
     Args:
         dll_name: the name of the dll
@@ -280,13 +285,18 @@ def runIterations(dll_name, dll_path, iterations, crossgen_path, jit_path, assem
             dll_path
             ]
 
+    my_env = os.environ
+
+    if opt_level == 'min_opt':
+        my_env['COMPlus_JITMinOpts'] = '1'
+
     log(" ".join(run_args))
 
     # Time.clock() returns seconds, with a resolution of 0.4 microseconds, so multiply by the multiplier to get milliseconds
     multiplier = 1000
 
     for iteration in range(0,iterations + 1):
-        proc = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        proc = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=my_env)
 
         start_time = timeit.default_timer()
         (out, err) = proc.communicate()
@@ -312,7 +322,7 @@ def main(args):
     global os_group_list
     global python_exe_list
 
-    architecture, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations = validate_args(args)
+    architecture, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations, opt_level = validate_args(args)
     arch = architecture
 
     current_dir = os.getcwd()
@@ -351,7 +361,7 @@ def main(args):
                 (not dll_file_name in dll_exclude_list["All"])):
             dll_name = dll_file_name.replace(".dll", "")
             dll_path = os.path.join(assembly_root, dll_file_name)
-            dll_elapsed_times = runIterations(dll_file_name, dll_path, iterations, crossgen_path, jit_path, assembly_root)
+            dll_elapsed_times = runIterations(dll_file_name, dll_path, iterations, crossgen_path, jit_path, assembly_root, opt_level)
 
             if len(dll_elapsed_times) != 0:
                 if not benchview_path is None:
@@ -404,6 +414,9 @@ def main(args):
                 "--config",
                 "OS",
                 operating_system,
+                "--config",
+                "OptLevel",
+                opt_level,
                 "--arch",
                 architecture,
                 "--machinepool",