diff options
author | José Rivero <jorive@microsoft.com> | 2017-08-25 16:26:45 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-08-25 16:26:45 -0700 |
commit | 35dc8bbd313d82d56d81e3ffac79b16605c90183 (patch) | |
tree | ce00539f888a9d0967f966982f69ca5cbcfac400 /perf.groovy | |
parent | c539df9eda6e8221c132f003f0266bf18b5a2499 (diff) | |
download | coreclr-35dc8bbd313d82d56d81e3ffac79b16605c90183.tar.gz coreclr-35dc8bbd313d82d56d81e3ffac79b16605c90183.tar.bz2 coreclr-35dc8bbd313d82d56d81e3ffac79b16605c90183.zip |
Minor performance infrastructure changes. (#13557)
- IlLink scenario will run on a VM (currently it is only measuring MSIL size)
- Updated the patterns of files to search for on scenario runs
- Updated the JitBench commit hash being used ()
- Disable min_opt from the PR correctness mode.
- String was not being interpolated.
Diffstat (limited to 'perf.groovy')
-rw-r--r-- | perf.groovy | 142 |
1 files changed, 118 insertions, 24 deletions
diff --git a/perf.groovy b/perf.groovy index 92073c7c32..0ef3509219 100644 --- a/perf.groovy +++ b/perf.groovy @@ -37,6 +37,10 @@ def static getOSGroup(def os) { } ['full_opt', 'min_opt'].each { opt_level -> + if (isSmoketest && opt_level == 'min_opt') { + return + } + def architecture = arch def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}" def testEnv = "" @@ -105,11 +109,13 @@ def static getOSGroup(def os) { batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\Off\\") // Run with the full set of counters enabled: Profile=On - batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi") - batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\On\\") + if (opt_level != 'min_opt') { + batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi") + batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Perflab\\On\\") - batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi") - batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\On\\") + batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi") + batchFile("xcopy.exe /VYQK bin\\sandbox\\Logs\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\CodeQuality\\On\\") + } } } @@ -147,12 +153,12 @@ def static getOSGroup(def os) { builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests") def opts = "" - if (opt_level == 'min_opts') { + if (opt_level == 'min_opt') { opts = '\\W+min_opts' } def jitt = "" if (jit != 'ryujit') { - jitt = '\\W+${jit}' + jitt = "\\W+${jit}" } builder.triggerOnlyOnComment() @@ -234,13 +240,13 @@ def static getOSGroup(def os) { if (isPR) { def opts = "" - if (opt_level == 'min_opts') { + if (opt_level == 'min_opt') { opts = '\\W+min_opts' } def jitt = "" if (jit != 'ryujit') { - jitt = '\\W+${jit}' + jitt = "\\W+${jit}" } TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest() @@ -608,23 +614,10 @@ parallel( batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly") def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -scenarioTest" - def failedOutputLogFilename = "run-xunit-perf-scenario.log" - - // Using a sentinel file to - batchFile("if exist \"${failedOutputLogFilename}\" del /q /f \"${failedOutputLogFilename}\"") - batchFile("if exist \"${failedOutputLogFilename}\" (echo [ERROR] Failed to delete previously created \"${failedOutputLogFilename}\" file.& exit /b 1)") // Scenario: JitBench - batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios || (echo [ERROR] JitBench failed. 1>>\"${failedOutputLogFilename}\"& exit /b 0)") + batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios") batchFile("xcopy.exe /VYQK bin\\sandbox\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\JitBench\\") - - // Scenario: ILLink - if (arch == 'x64' && opt_level == 'full_opt') { - batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup || (echo [ERROR] IlLink failed. 1>>\"${failedOutputLogFilename}\"& exit /b 0)") - batchFile("xcopy.exe /VYQK bin\\sandbox\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\LinkBench\\") - } - - batchFile("if exist \"${failedOutputLogFilename}\" (type \"${failedOutputLogFilename}\"& exit /b 1)") } } @@ -651,12 +644,12 @@ parallel( if (isPR) { def opts = "" - if (opt_level == 'min_opts') { + if (opt_level == 'min_opt') { opts = '\\W+min_opts' } def jitt = "" if (jit != 'ryujit') { - jitt = '\\W+${jit}' + jitt = "\\W+${jit}" } TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest() @@ -677,6 +670,107 @@ parallel( } } +// Setup CoreCLR-Scenarios tests +[true, false].each { isPR -> + ['Windows_NT'].each { os -> + ['x64'].each { arch -> + ['ryujit'].each { jit -> + ['full_opt'].each { opt_level -> + def architecture = arch + def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) { + + def testEnv = "" + + // Set the label (currently we are only measuring size, therefore we are running on VM). + label('20170427-elevated') + wrappers { + credentialsBinding { + string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') + } + } + + if (isPR) { + parameters { + stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName') + } + } + + parameters { + stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient') + stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient') + } + + def configuration = 'Release' + def runType = isPR ? 'private' : 'rolling' + def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%' + def uploadString = '-uploadToBenchview' + + steps { + // Batch + batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"") + batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"") + batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion") + + //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView + //we have to do it all as one statement because cmd is called each time and we lose the set environment variable + batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" + + "set \"BENCHVIEWNAME=${benchViewName}\"\n" + + "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" + + "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" + + "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}") + batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"") + batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}") + + batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly") + + def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -scenarioTest" + + // Scenario: ILLink + batchFile("tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup") + batchFile("xcopy.exe /VYQK bin\\sandbox\\Perf-*.* bin\\toArchive\\sandbox\\Logs\\Scenario\\LinkBench\\") + } + } + + def archiveSettings = new ArchivalSettings() + archiveSettings.addFiles('bin/toArchive/**') + archiveSettings.addFiles('machinedata.json') + + Utilities.addArchival(newJob, archiveSettings) + Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") + + newJob.with { + logRotator { + artifactDaysToKeep(30) + daysToKeep(30) + artifactNumToKeep(200) + numToKeep(200) + } + wrappers { + timeout { + absolute(240) + } + } + } + + if (isPR) { + TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest() + builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests") + builder.triggerOnlyOnComment() + builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*") + builder.triggerForBranch(branch) + builder.emitTrigger(newJob) + } + else { + // Set a push trigger + TriggerBuilder builder = TriggerBuilder.triggerOnCommit() + builder.emitTrigger(newJob) + } + } + } + } + } +} + Utilities.createHelperJob(this, project, branch, "Welcome to the ${project} Perf help", "Have a nice day!") |