Add perf pipeline job full groovy (#14397)

This change converts our perf testing to use pipeline jobs. Pipeline
jobs allow us to do the following:

1) Test on the same commit for each of the test legs
2) Parallelize the build and test steps.
3) Separate the build and test steps from one another. This gives us the
ability to use the same build assets for all of the test legs of the
same configuration. It also allows us to build on virtual machines and
test on perf machines, so we only use the perf resources for testing.
4) Have different test scenarios for PRs and rolling. This isn't
strictly a benefit of pipeline jobs, but certainly is made easier by
them.
5) Allows us to have one trigger for PR jobs which will get us all the
perf testing scenarios.

This change also cleans up the groovy scripting for perf testing.

1 files changed, 93 insertions, 115 deletions

diff --git a/perf.groovy b/perf.groovy
index c1628b9c33..22fa822d1c 100644
--- a/perf.groovy
+++ b/perf.groovy
@@ -29,149 +29,127 @@ def static getOSGroup(def os) {
 [true, false].each { isPR ->
     ['Windows_NT'].each { os ->
         ['x64', 'x86'].each { arch ->
-            [true, false].each { isSmoketest ->
-                ['ryujit'].each { jit ->
+            ['ryujit'].each { jit ->
+                if (arch == 'x64' && jit == 'legacy_backend') {
+                    return
+                }
 
-                    if (arch == 'x64' && jit == 'legacy_backend') {
-                        return
-                    }
+                ['full_opt', 'min_opt'].each { opt_level ->
 
-                    ['full_opt', 'min_opt'].each { opt_level ->
-                        if (isSmoketest && opt_level == 'min_opt') {
-                            return
-                        }
+                    def architecture = arch
+                    def jobName = "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
+                    def testEnv = ""
 
-                        def architecture = arch
-                        def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
-                        def testEnv = ""
+                    if (jit == 'legacy_backend') {
+                        testEnv = '-testEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd'
+                    }
 
-                        if (jit == 'legacy_backend') {
-                            testEnv = '-testEnv %WORKSPACE%\\tests\\legacyjit_x86_testenv.cmd'
+                    def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
+                        // Set the label.
+                        label('windows_server_2016_clr_perf')
+                        wrappers {
+                            credentialsBinding {
+                                string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
+                            }
                         }
 
-                        def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
-                            // Set the label.
-                            label('windows_server_2016_clr_perf')
-                            wrappers {
-                                credentialsBinding {
-                                    string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
-                                }
+                        if (isPR) {
+                            parameters {
+                                stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
                             }
+                        }
 
-                            if (isPR) {
-                                parameters {
-                                    stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
-                                }
-                            }
-                            if (isSmoketest) {
-                                parameters {
-                                    stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
-                                    stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two.  We want to do this so that we can run as fast as possible as this is just for smoke testing')
-                                }
-                            }
-                            else {
-                                parameters {
-                                    stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
-                                    stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
-                                }
-                            }
+                        parameters {
+                            stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
+                            stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample')
+                        }
 
-                            def configuration = 'Release'
-                            def runType = isPR ? 'private' : 'rolling'
-                            def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
-                            def uploadString = isSmoketest ? '' : '-uploadToBenchview'
+                        def configuration = 'Release'
+                        def runType = isPR ? 'private' : 'rolling'
+                        def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
+                        def uploadString = '-uploadToBenchview'
 
-                            steps {
-                                // Batch
+                        steps {
+                            // Batch
 
-                                batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
-                                batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
-                                batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
-                                //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
-                                //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
-                                batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
-                                "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
-                                "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
-                                "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
-                                "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
-                                batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
-                                batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
+                            batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
+                            batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
+                            batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
+                            //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
+                            //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
+                            batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
+                            "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
+                            "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
+                            "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
+                            "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
+                            batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
+                            batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
 
-                                batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
+                            batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
 
-                                def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
+                            def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
 
-                                // Run with just stopwatch: Profile=Off
-                                batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
-                                batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\Off\\")
+                            // Run with just stopwatch: Profile=Off
+                            batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
+                            batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\Off\\")
 
-                                batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
-                                batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\Off\\")
+                            batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
+                            batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\Off\\")
 
-                                // Run with the full set of counters enabled: Profile=On
-                                if (opt_level != 'min_opt') {
-                                    batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
-                                    batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\On\\")
+                            // Run with the full set of counters enabled: Profile=On
+                            if (opt_level != 'min_opt') {
+                                batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
+                                batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\On\\")
 
-                                    batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
-                                    batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\On\\")
-                                }
+                                batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
+                                batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\On\\")
                             }
                         }
+                    }
 
-                        if (isSmoketest) {
-                            Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
-                        }
-
-                        def archiveSettings = new ArchivalSettings()
-                        archiveSettings.addFiles('bin/toArchive/**')
-                        archiveSettings.addFiles('machinedata.json')
+                    def archiveSettings = new ArchivalSettings()
+                    archiveSettings.addFiles('bin/toArchive/**')
+                    archiveSettings.addFiles('machinedata.json')
 
-                        Utilities.addArchival(newJob, archiveSettings)
-                        Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
+                    Utilities.addArchival(newJob, archiveSettings)
+                    Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
 
-                        newJob.with {
-                            logRotator {
-                                artifactDaysToKeep(30)
-                                daysToKeep(30)
-                                artifactNumToKeep(200)
-                                numToKeep(200)
-                            }
-                            wrappers {
-                                timeout {
-                                    absolute(240)
-                                }
-                            }
+                    newJob.with {
+                        logRotator {
+                            artifactDaysToKeep(30)
+                            daysToKeep(30)
+                            artifactNumToKeep(200)
+                            numToKeep(200)
                         }
-
-                        if (isPR) {
-                            TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
-                            if (isSmoketest) {
-                                builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests Correctness")
+                        wrappers {
+                            timeout {
+                                absolute(240)
                             }
-                            else {
-                                builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
+                        }
+                    }
 
-                                def opts = ""
-                                if (opt_level == 'min_opt') {
-                                    opts = '\\W+min_opts'
-                                }
-                                def jitt = ""
-                                if (jit != 'ryujit') {
-                                    jitt = "\\W+${jit}"
-                                }
+                    if (isPR) {
+                        TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
+                        builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
 
-                                builder.triggerOnlyOnComment()
-                                builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
-                            }
-                            builder.triggerForBranch(branch)
-                            builder.emitTrigger(newJob)
+                        def opts = ""
+                        if (opt_level == 'min_opt') {
+                            opts = '\\W+min_opts'
                         }
-                        else {
-                            // Set a push trigger
-                            TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
-                            builder.emitTrigger(newJob)
+                        def jitt = ""
+                        if (jit != 'ryujit') {
+                            jitt = "\\W+${jit}"
                         }
+
+                        builder.triggerOnlyOnComment()
+                        builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*")
+                        builder.triggerForBranch(branch)
+                        builder.emitTrigger(newJob)
+                    }
+                    else {
+                        // Set a push trigger
+                        TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
+                        builder.emitTrigger(newJob)
                     }
                 }
             }