From ef0fcc8b6f28d380cd193ca393557dc10aa0fb96 Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Thu, 8 Jun 2017 20:30:02 -0700 Subject: Move ILLink Perf to a separate job This changes ILLink performance testing to its own leg. With this change, ILLink perf test will not be run for every PR by default. It can be requested explicitly on a PR, and will run on the rolling builds. JitBench build breaks have broken the PR builds a few times, so this change moves ILLink perf tests out of the PR job, so that MusicStore testing can be enabled again. --- perf.groovy | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) (limited to 'perf.groovy') diff --git a/perf.groovy b/perf.groovy index 3ba7c88807..c61ff1579c 100644 --- a/perf.groovy +++ b/perf.groovy @@ -108,7 +108,6 @@ def static getOSGroup(def os) { // Run with just stopwatch: Profile=Off batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"") batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"") - batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -nowarmup -runtype ${runType} -scenarioTest -group ILLink") // Run with the full set of counters enabled: Profile=On batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"") @@ -398,3 +397,92 @@ def static getOSGroup(def os) { } } // os } // isPR + +// Setup ILLink tests +[true, false].each { isPR -> + ['Windows_NT'].each { os -> + ['x64'].each { arch -> + def architecture = arch + def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}", isPR)) { + // Set the label. + label('windows_Illink_perf') + wrappers { + credentialsBinding { + string('BV_UPLOAD_SAS_TOKEN', 'ILLink BenchView Sas') + } + } + + if (isPR) + { + parameters + { + stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form private BenchviewCommitName') + } + } + + parameters + { + stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient') + stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient') + } + def configuration = 'Release' + def runType = isPR ? 'private' : 'rolling' + def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%' + def uploadString = '-uploadToBenchview' + + steps { + // Batch + batchFile("powershell wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"") + batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"") + batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion") + + //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView + //we have to do it all as one statement because cmd is called each time and we lose the set environment variable + batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" + + "set \"BENCHVIEWNAME=${benchViewName}\"\n" + + "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" + + "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" + + "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}") + batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"") + batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}") + + batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly") + + // Run with just stopwatch: Profile=Off + batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -nowarmup -runtype ${runType} -scenarioTest -group ILLink") + } + } + + // Save machinedata.json to /artifact/bin/ Jenkins dir + def archiveSettings = new ArchivalSettings() + archiveSettings.addFiles('Perf-*.xml') + archiveSettings.addFiles('Perf-*.log') + archiveSettings.addFiles('machinedata.json') + Utilities.addArchival(newJob, archiveSettings) + + Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") + + newJob.with { + wrappers { + timeout { + absolute(240) + } + } + } + + if (isPR) { + TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest() + builder.setGithubContext("${os} ${arch} ILLink Perf Tests") + builder.triggerOnlyOnComment() + builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink\\W+perf.*") + builder.triggerForBranch(branch) + builder.emitTrigger(newJob) + } + else { + // Set a push trigger + TriggerBuilder builder = TriggerBuilder.triggerOnCommit() + builder.emitTrigger(newJob) + } + } + } +} -- cgit v1.2.3