summaryrefslogtreecommitdiff
path: root/perf.groovy
diff options
context:
space:
mode:
authorJosé Rivero <jorive@microsoft.com>2017-08-25 16:26:45 -0700
committerGitHub <noreply@github.com>2017-08-25 16:26:45 -0700
commit35dc8bbd313d82d56d81e3ffac79b16605c90183 (patch)
treece00539f888a9d0967f966982f69ca5cbcfac400 /perf.groovy
parentc539df9eda6e8221c132f003f0266bf18b5a2499 (diff)
downloadcoreclr-35dc8bbd313d82d56d81e3ffac79b16605c90183.tar.gz
coreclr-35dc8bbd313d82d56d81e3ffac79b16605c90183.tar.bz2
coreclr-35dc8bbd313d82d56d81e3ffac79b16605c90183.zip
Minor performance infrastructure changes. (#13557)
- IlLink scenario will run on a VM (currently it is only measuring MSIL size) - Updated the patterns of files to search for on scenario runs - Updated the JitBench commit hash being used () - Disable min_opt from the PR correctness mode. - String was not being interpolated.
Diffstat (limited to 'perf.groovy')
-rw-r--r--perf.groovy142
1 files changed, 118 insertions, 24 deletions
diff --git a/perf.groovy b/perf.groovy
index 92073c7c32..0ef3509219 100644
--- a/perf.groovy
+++ b/perf.groovy
@@ -37,6 +37,10 @@ def static getOSGroup(def os) {
}
['full_opt', 'min_opt'].each { opt_level ->
+ if (isSmoketest && opt_level == 'min_opt') {
+ return
+ }
+
def architecture = arch
def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
def testEnv = ""
@@ -105,11 +109,13 @@ def static getOSGroup(def os) {
batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\Off\\")
// Run with the full set of counters enabled: Profile=On
- batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\On\\")
+ if (opt_level != 'min_opt') {
+ batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
+ batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\On\\")
- batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\On\\")
+ batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
+ batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\On\\")
+ }
}
}
@@ -147,12 +153,12 @@ def static getOSGroup(def os) {
builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests")
def opts = ""
- if (opt_level == 'min_opts') {
+ if (opt_level == 'min_opt') {
opts = '\\W+min_opts'
}
def jitt = ""
if (jit != 'ryujit') {
- jitt = '\\W+${jit}'
+ jitt = "\\W+${jit}"
}
builder.triggerOnlyOnComment()
@@ -234,13 +240,13 @@ def static getOSGroup(def os) {
if (isPR) {
def opts = ""
- if (opt_level == 'min_opts') {
+ if (opt_level == 'min_opt') {
opts = '\\W+min_opts'
}
def jitt = ""
if (jit != 'ryujit') {
- jitt = '\\W+${jit}'
+ jitt = "\\W+${jit}"
}
TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
@@ -608,23 +614,10 @@ parallel(
batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -scenarioTest"
- def failedOutputLogFilename = "run-xunit-perf-scenario.log"
-
- // Using a sentinel file to
- batchFile("if exist \"${failedOutputLogFilename}\" del /q /f \"${failedOutputLogFilename}\"")
- batchFile("if exist \"${failedOutputLogFilename}\" (echo [ERROR] Failed to delete previously created \"${failedOutputLogFilename}\" file.& exit /b 1)")
// Scenario: JitBench
- batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios || (echo [ERROR] JitBench failed. 1>>\"${failedOutputLogFilename}\"& exit /b 0)")
+ batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
batchFile("xcopy.exe /VYQK bin\\sandbox\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\JitBench\\")
-
- // Scenario: ILLink
- if (arch == 'x64' && opt_level == 'full_opt') {
- batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup || (echo [ERROR] IlLink failed. 1>>\"${failedOutputLogFilename}\"& exit /b 0)")
- batchFile("xcopy.exe /VYQK bin\\sandbox\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\LinkBench\\")
- }
-
- batchFile("if exist \"${failedOutputLogFilename}\" (type \"${failedOutputLogFilename}\"& exit /b 1)")
}
}
@@ -651,12 +644,12 @@ parallel(
if (isPR) {
def opts = ""
- if (opt_level == 'min_opts') {
+ if (opt_level == 'min_opt') {
opts = '\\W+min_opts'
}
def jitt = ""
if (jit != 'ryujit') {
- jitt = '\\W+${jit}'
+ jitt = "\\W+${jit}"
}
TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
@@ -677,6 +670,107 @@ parallel(
}
}
+// Setup CoreCLR-Scenarios tests
+[true, false].each { isPR ->
+ ['Windows_NT'].each { os ->
+ ['x64'].each { arch ->
+ ['ryujit'].each { jit ->
+ ['full_opt'].each { opt_level ->
+ def architecture = arch
+ def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {
+
+ def testEnv = ""
+
+ // Set the label (currently we are only measuring size, therefore we are running on VM).
+ label('20170427-elevated')
+ wrappers {
+ credentialsBinding {
+ string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
+ }
+ }
+
+ if (isPR) {
+ parameters {
+ stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
+ }
+ }
+
+ parameters {
+ stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient')
+ stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient')
+ }
+
+ def configuration = 'Release'
+ def runType = isPR ? 'private' : 'rolling'
+ def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
+ def uploadString = '-uploadToBenchview'
+
+ steps {
+ // Batch
+ batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"")
+ batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
+ batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
+
+ //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
+ //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
+ batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
+ "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
+ "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
+ "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
+ "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
+ batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
+ batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
+
+ batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
+
+ def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -scenarioTest"
+
+ // Scenario: ILLink
+ batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
+ batchFile("xcopy.exe /VYQK bin\\sandbox\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\LinkBench\\")
+ }
+ }
+
+ def archiveSettings = new ArchivalSettings()
+ archiveSettings.addFiles('bin/toArchive/**')
+ archiveSettings.addFiles('machinedata.json')
+
+ Utilities.addArchival(newJob, archiveSettings)
+ Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
+
+ newJob.with {
+ logRotator {
+ artifactDaysToKeep(30)
+ daysToKeep(30)
+ artifactNumToKeep(200)
+ numToKeep(200)
+ }
+ wrappers {
+ timeout {
+ absolute(240)
+ }
+ }
+ }
+
+ if (isPR) {
+ TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
+ builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests")
+ builder.triggerOnlyOnComment()
+ builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*")
+ builder.triggerForBranch(branch)
+ builder.emitTrigger(newJob)
+ }
+ else {
+ // Set a push trigger
+ TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
+ builder.emitTrigger(newJob)
+ }
+ }
+ }
+ }
+ }
+}
+
Utilities.createHelperJob(this, project, branch,
"Welcome to the ${project} Perf help",
"Have a nice day!")