Separate large perf benchmarks into their own legs (#15231)

Separate large perf benchmarks into their own legs

This change splits the windows perf test stages into 6 pipelined legs per flavor to reduce the amount of time we spend running the perf tests and reduce the total time of the job. This change also decreases the size of the stashed bin directory by deleting the obj directory. Finally, we move the benchstones suite into one directory (moving BenchF and BenchI into a shared dir called Benchstones)

1 files changed, 69 insertions, 20 deletions

diff --git a/buildpipeline/perf-pipeline.groovy b/buildpipeline/perf-pipeline.groovy
index 5921cf6f12..918b4ef092 100644
--- a/buildpipeline/perf-pipeline.groovy
+++ b/buildpipeline/perf-pipeline.groovy
@@ -4,6 +4,18 @@
 // Note that the parameters will be set as env variables so we cannot use names that conflict
 // with the engineering system parameter names.
 
+//-------------------------- Globals ---------------------------------//
+
+def validTestFolders = [
+    'Benchstones',
+    'BenchmarksGame',
+    'Bytemark',
+    'Math',
+    'Span',
+    'first half',
+    'last half'
+    ]
+
 //--------------------- Windows Functions ----------------------------//
 
 def windowsBuild(String arch, String config, String pgo, boolean isBaseline) {
@@ -19,13 +31,15 @@ def windowsBuild(String arch, String config, String pgo, boolean isBaseline) {
     }
 
     bat "set __TestIntermediateDir=int&&.\\build.cmd -${config} -${arch} -skipbuildpackages ${pgoBuildFlag}"
+    bat "tests\\runtest.cmd ${config} ${arch} GenerateLayoutOnly"
+    bat "rd /s /q bin\\obj"
 
     // Stash build artifacts. Stash tests in an additional stash to be used by Linux test runs
     stash name: "nt-${arch}-${pgo}${baselineString}-build-artifacts", includes: 'bin/**'
     stash name: "nt-${arch}-${pgo}${baselineString}-test-artifacts", includes: 'bin/tests/**'
 }
 
-def windowsPerf(String arch, String config, String uploadString, String runType, String opt_level, String jit, String pgo, String scenario, boolean isBaseline, boolean isProfileOn) {
+def windowsPerf(String arch, String config, String uploadString, String runType, String opt_level, String jit, String pgo, String scenario, boolean isBaseline, boolean isProfileOn, String testFolder) {
     withCredentials([string(credentialsId: 'CoreCLR Perf BenchView Sas', variable: 'BV_UPLOAD_SAS_TOKEN')]) {
         checkout scm
         String baselineString = ""
@@ -33,11 +47,16 @@ def windowsPerf(String arch, String config, String uploadString, String runType,
             baselineString = "-baseline"
         }
         dir ('.') {
-            unstash "nt-${arch}-${pgo}${baselineString}-build-artifacts"
+            unstash "nt-${arch}-${pgo}${baselineString}-test-artifacts"
             unstash "benchview-tools"
             unstash "metadata"
         }
 
+        String test = ''
+        if (testFolder != 'all') {
+            test = testFolder
+        }
+
         String pgoTestFlag = ((pgo == 'nopgo') ? '-nopgo' : '')
 
         // We want to use the baseline metadata for baseline runs. We expect to find the submission metadata in
@@ -52,8 +71,6 @@ def windowsPerf(String arch, String config, String uploadString, String runType,
 
         bat "py \".\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\""
         bat ".\\init-tools.cmd"
-        bat "run.cmd build -Project=\"tests\\build.proj\" -BuildOS=Windows_NT -BuildType=${config} -BuildArch=${arch} -BatchRestorePackages"
-        bat "tests\\runtest.cmd ${config} ${arch} GenerateLayoutOnly"
 
         // We run run-xunit-perf differently for each of the different job types
 
@@ -62,13 +79,43 @@ def windowsPerf(String arch, String config, String uploadString, String runType,
         String runXUnitCommonArgs = "-arch ${arch} -configuration ${config} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} ${pgoTestFlag} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\""
         if (scenario == 'perf') {
             String runXUnitPerfCommonArgs = "${runXUnitCommonArgs} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
-            String runXUnitPerflabArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\performance\\perflab\\Perflab -library"
+            if (test == 'first half' || testFolder == 'all')
+            {
+                String runXUnitPerflabArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\performance\\perflab\\Perflab -library"
 
-            profileArg = isProfileOn ? "default+${profileArg}+gcapi" : profileArg
-            bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerflabArgs} -collectionFlags ${profileArg}"
+                profileArg = isProfileOn ? "default+${profileArg}+gcapi" : profileArg
+                bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerflabArgs} -collectionFlags ${profileArg}"
+            }
 
-            String runXUnitCodeQualityArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\Jit\\Performance\\CodeQuality"
-            bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitCodeQualityArgs} -collectionFlags ${profileArg}"
+            if (test == 'first half') {
+
+                [
+                    'Burgers',
+                    'Devirtualization',
+                    'FractalPerf',
+                    'Inlining',
+                    'Layout'
+                ].each { benchmark ->
+                    String runXUnitCodeQualityArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\Jit\\Performance\\CodeQuality\\${benchmark}"
+                    bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitCodeQualityArgs} -collectionFlags ${profileArg}"
+                }
+            }
+            else if (test == 'last half') {
+                [
+                    'Linq',
+                    'Roslyn',
+                    'SciMark',
+                    'Serialization',
+                    'V8'
+                ].each { benchmark ->
+                    String runXUnitCodeQualityArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\Jit\\Performance\\CodeQuality\\${benchmark}"
+                    bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitCodeQualityArgs} -collectionFlags ${profileArg}"
+                }
+            }
+            else {
+                String runXUnitCodeQualityArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\Jit\\Performance\\CodeQuality\\${test}"
+                bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitCodeQualityArgs} -collectionFlags ${profileArg}"
+            }
         }
         else if (scenario == 'jitbench') {
             String runXUnitPerfCommonArgs = "${runXUnitCommonArgs} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH\" -scenarioTest"
@@ -307,14 +354,16 @@ def innerLoopTests = [:]
 ['x64', 'x86'].each { arch ->
     ['full_opt'].each { opt_level ->
         [false].each { isBaseline ->
-            String baseline = ""
-            if (isBaseline) {
-                baseline = " baseline"
-            }
-            if (isPR() || !isBaseline) {
-                innerLoopTests["windows ${arch} ryujit ${opt_level} pgo${baseline} perf"] = {
-                    simpleNode('windows_server_2016_clr_perf', 180) {
-                        windowsPerf(arch, config, uploadString, runType, opt_level, 'ryujit', 'pgo', 'perf', isBaseline, true)
+            validTestFolders.each { benchmark ->
+                String baseline = ""
+                if (isBaseline) {
+                    baseline = " baseline"
+                }
+                if (isPR() || !isBaseline) {
+                    innerLoopTests["windows ${arch} ryujit ${opt_level} pgo ${benchmark}${baseline} perf"] = {
+                        simpleNode('windows_server_2016_clr_perf', 180) {
+                            windowsPerf(arch, config, uploadString, runType, opt_level, 'ryujit', 'pgo', 'perf', isBaseline, true, benchmark)
+                        }
                     }
                 }
             }
@@ -328,13 +377,13 @@ def outerLoopTests = [:]
 if (!isPR()) {
     outerLoopTests["windows ${arch} ryujit full_opt pgo${baseline} jitbench"] = {
         simpleNode('windows_server_2016_clr_perf', 180) {
-            windowsPerf(arch, config, uploadString, runType, 'full_opt', 'ryujit', 'pgo', 'jitbench', false)
+            windowsPerf(arch, config, uploadString, runType, 'full_opt', 'ryujit', 'pgo', 'jitbench', false, false, '')
         }
     }
 
     outerLoopTests["windows ${arch} ryujit full_opt pgo${baseline} illink"] = {
         simpleNode('Windows_NT', '20170427-elevated') {
-            windowsPerf(arch, config, uploadString, runType, 'full_opt', 'ryujit', 'pgo', 'illink', false)
+            windowsPerf(arch, config, uploadString, runType, 'full_opt', 'ryujit', 'pgo', 'illink', false, false, '')
         }
     }
 
@@ -345,7 +394,7 @@ if (!isPR()) {
                     [true, false].each { isProfileOn ->
                         outerLoopTests["windows ${arch} ${jit} ${opt_level} ${pgo_enabled} perf"] = {
                             simpleNode('windows_server_2016_clr_perf', 180) {
-                                windowsPerf(arch, config, uploadString, runType, opt_level, jit, pgo_enabled, 'perf', false, isProfileOn)
+                                windowsPerf(arch, config, uploadString, runType, opt_level, jit, pgo_enabled, 'perf', false, isProfileOn, 'all')
                             }
                         }