Add CoreCLR perf test support to Jenkins

Made changes to the perf.groovy to add support for running the CoreCLR
performance tests in the Perflab directory.  Also made changes to the
runner script for CoreCLR to allow it to run more than one type of test.

1 files changed, 61 insertions, 5 deletions

diff --git a/perf.groovy b/perf.groovy
index ebf63638d4..434e0c1896 100644
--- a/perf.groovy
+++ b/perf.groovy
@@ -24,21 +24,77 @@ def static getOSGroup(def os) {
     assert osGroup != null : "Could not find os group for ${os}"
     return osGroupMap[os]
 }
+// Setup perflab tests runs
+[true, false].each { isPR ->
+    ['Windows_NT'].each { os ->
+		['x64'].each { architecture ->
+			def configuration = 'Release'
+			def runType = isPR ? 'private' : 'rolling'
+			def benchViewName = isPR ? 'coreclr private %ghprbPullTitle%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
+			def newJob = job(Utilities.getFullJobName(project, "perf_perflab_${os}", isPR)) {
+				// Set the label.
+				label('windows_clr_perf')
+				wrappers {
+					credentialsBinding {
+						string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
+					}
+				}
+
+				steps {
+					// Batch
+					
+					batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory C:\\tools -Prerelease -ExcludeVersion")
+					//Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
+					//we have to do it all as one statement because cmd is called each time and we lose the set environment variable
+					batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
+					"py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py --name " + "\"" + benchViewName + "\"" + " --user " + "\"dotnet-bot@microsoft.com\"\n" +
+					"py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\build.py git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type " + runType)
+					batchFile("py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py")
+					batchFile("set __TestIntermediateDir=int&&build.cmd release ${architecture}")
+					batchFile("tests\\runtest.cmd release ${architecture} GenerateLayoutOnly")
+					batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${architecture} -configuration ${configuration} -testBinLoc bin\\tests\\Windows_NT.${architecture}.Release\\performance\\perflab\\Perflab -library -uploadToBenchview C:\\Tools\\Microsoft.Benchview.JSONFormat\\tools -runtype " + runType)
+				}
+			}
+
+			// Save machinedata.json to /artifact/bin/ Jenkins dir
+			def archiveSettings = new ArchivalSettings()
+			archiveSettings.addFiles('sandbox\\perf-*.xml')
+			Utilities.addArchival(newJob, archiveSettings)
+
+			Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
+			
+			if (isPR) {
+				TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
+				builder.setGithubContext("${os} CoreCLR Perf Tests")
+				builder.triggerOnlyOnComment()
+				builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
+				builder.triggerForBranch(branch)
+				builder.emitTrigger(newJob)
+			}
+			else {
+				// Set a push trigger
+				TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
+				builder.emitTrigger(newJob)
+			}
+		}
+    }
+}
 
 [true, false].each { isPR ->
     ['Windows_NT'].each { os ->
         def architecture = 'x64'
         def configuration = 'Release'
         def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
+
             // Set the label.
             label('performance')
             steps {
                     // Batch
-                    batchFile("C:\\tools\\nuget install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory C:\\tools -Prerelease -ExcludeVersion")
-                    batchFile("python C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py")
+                    batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory C:\\tools -Prerelease -ExcludeVersion")
+                    batchFile("py C:\\tools\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py")
                     batchFile("set __TestIntermediateDir=int&&build.cmd release ${architecture}")
                     batchFile("tests\\runtest.cmd release ${architecture}")
-                    batchFile("tests\\scripts\\run-xunit-perf.cmd")
+                    batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${architecture} -configuration ${configuration} -testBinLoc bin\\tests\\Windows_NT.x64.Release\\Jit\\Performance\\CodeQuality")
             }
         }
 
@@ -96,7 +152,7 @@ def static getOSGroup(def os) {
             }
         }
 
-        Utilities.setMachineAffinity(newJob, os, 'latest-or-auto') // Just run against Linux VM’s for now.
+        Utilities.setMachineAffinity(newJob, os, 'latest-or-auto') // Just run against Linux VM's for now.
 
         // Save machinedata.json to /artifact/bin/ Jenkins dir
         def archiveSettings = new ArchivalSettings()
@@ -130,4 +186,4 @@ def static getOSGroup(def os) {
             builder.emitTrigger(newJob)
         }
     } // os
-} // isPR
+} // isPR
\ No newline at end of file