summaryrefslogtreecommitdiff
path: root/perf.groovy
diff options
context:
space:
mode:
Diffstat (limited to 'perf.groovy')
-rw-r--r--perf.groovy190
1 files changed, 148 insertions, 42 deletions
diff --git a/perf.groovy b/perf.groovy
index 9be47c0..d2a4c32 100644
--- a/perf.groovy
+++ b/perf.groovy
@@ -88,9 +88,9 @@ def static getOSGroup(def os) {
batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
//Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
//we have to do it all as one statement because cmd is called each time and we lose the set environment variable
- batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
- "set BENCHVIEWNAME=${benchViewName}\n" +
- "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
+ batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
+ "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
+ "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
@@ -198,9 +198,9 @@ def static getOSGroup(def os) {
batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
//Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
//we have to do it all as one statement because cmd is called each time and we lose the set environment variable
- batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
- "set BENCHVIEWNAME=${benchViewName}\n" +
- "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
+ batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
+ "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
+ "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
@@ -234,10 +234,32 @@ def static getOSGroup(def os) {
}
}
+def static getFullPerfJobName(def project, def os, def isPR) {
+ return Utilities.getFullJobName(project, "perf_${os}", isPR)
+}
+
// Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
[true, false].each { isPR ->
- ['Ubuntu14.04'].each { os ->
- def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
+ def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
+ def architecture = 'x64'
+ def configuration = 'Release'
+
+ // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
+ ['RHEL7.2'].each { os ->
+ def newBuildJob = job(fullBuildJobName) {
+ steps {
+ shell("./build.sh verbose ${architecture} ${configuration}")
+ }
+ }
+ Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
+ Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
+ Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
+ }
+
+ // Actual perf testing on the following OSes
+ def perfOSList = ['Ubuntu14.04']
+ perfOSList.each { os ->
+ def newJob = job(getFullPerfJobName(project, os, isPR)) {
label('linux_clr_perf')
wrappers {
@@ -254,22 +276,26 @@ def static getOSGroup(def os) {
}
}
- // Cap the maximum number of iterations to 21.
parameters {
+ // Cap the maximum number of iterations to 21.
stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
+ stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
}
def osGroup = getOSGroup(os)
- def architecture = 'x64'
- def configuration = 'Release'
def runType = isPR ? 'private' : 'rolling'
def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
steps {
shell("./tests/scripts/perf-prep.sh")
shell("./init-tools.sh")
- shell("./build.sh ${architecture} ${configuration}")
+ copyArtifacts(fullBuildJobName) {
+ includePatterns("bin/**")
+ buildSelector {
+ buildNumber('\${PRODUCT_BUILD}')
+ }
+ }
shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
"python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
"python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
@@ -306,26 +332,73 @@ def static getOSGroup(def os) {
numToKeep(1000)
}
}
+ } // os
+
+ def flowJobPerfRunList = perfOSList.collect { os ->
+ "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
+ }
+ def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
if (isPR) {
- TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
- builder.setGithubContext("${os} Perf Tests")
- builder.triggerOnlyOnComment()
- builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
- builder.triggerForBranch(branch)
- builder.emitTrigger(newJob)
- }
- else {
- // Set a push trigger
- TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
- builder.emitTrigger(newJob)
+ parameters {
+ stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
+ }
}
- } // os
+ buildFlow("""
+// First, build the bits on RHEL7.2
+b = build(params, '${fullBuildJobName}')
+
+// Then, run the perf tests
+parallel(
+ ${flowJobPerfRunList.join(",\n ")}
+)
+""")
+ }
+
+ Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
+ Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
+
+ if (isPR) {
+ TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
+ builder.setGithubContext("Linux Perf Test Flow")
+ builder.triggerOnlyOnComment()
+ builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
+ builder.triggerForBranch(branch)
+ builder.emitTrigger(newFlowJob)
+ }
+ else {
+ // Set a push trigger
+ TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
+ builder.emitTrigger(newFlowJob)
+ }
+
} // isPR
+def static getFullThroughputJobName(def project, def os, def isPR) {
+ return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
+}
+
// Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
[true, false].each { isPR ->
- ['Ubuntu14.04'].each { os ->
- def newJob = job(Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)) {
+ def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
+ def architecture = 'x64'
+ def configuration = 'Release'
+
+ // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
+ ['RHEL7.2'].each { os ->
+ def newBuildJob = job(fullBuildJobName) {
+ steps {
+ shell("./build.sh verbose ${architecture} ${configuration}")
+ }
+ }
+ Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
+ Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
+ Utilities.addArchival(newBuildJob, "bin/Product/**")
+ }
+
+ // Actual perf testing on the following OSes
+ def throughputOSList = ['Ubuntu14.04']
+ throughputOSList.each { os ->
+ def newJob = job(getFullThroughputJobName(project, os, isPR)) {
label('linux_clr_perf')
wrappers {
@@ -341,16 +414,24 @@ def static getOSGroup(def os) {
stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
}
}
+
+ parameters {
+ stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
+ }
+
def osGroup = getOSGroup(os)
- def architecture = 'x64'
- def configuration = 'Release'
def runType = isPR ? 'private' : 'rolling'
def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
steps {
shell("bash ./tests/scripts/perf-prep.sh --throughput")
shell("./init-tools.sh")
- shell("./build.sh ${architecture} ${configuration}")
+ copyArtifacts(fullBuildJobName) {
+ includePatterns("bin/Product/**")
+ buildSelector {
+ buildNumber('\${PRODUCT_BUILD}')
+ }
+ }
shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
"python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
"python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
@@ -359,7 +440,7 @@ def static getOSGroup(def os) {
-os \"${os}\" \\
-configuration \"${configuration}\" \\
-clr_root \"\${WORKSPACE}\" \\
- -assembly_root \"\${WORKSPACE}/_/fx/bin/runtime/netcoreapp-${osGroup}-${configuration}-${architecture}\" \\
+ -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
-run_type \"${runType}\" \\
-benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
}
@@ -383,18 +464,43 @@ def static getOSGroup(def os) {
numToKeep(1000)
}
}
+ } // os
+
+ def flowJobTPRunList = throughputOSList.collect { os ->
+ "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
+ }
+ def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
if (isPR) {
- TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
- builder.setGithubContext("${os} Throughput Perf Tests")
- builder.triggerOnlyOnComment()
- builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+throughput.*")
- builder.triggerForBranch(branch)
- builder.emitTrigger(newJob)
- }
- else {
- // Set a push trigger
- TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
- builder.emitTrigger(newJob)
+ parameters {
+ stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
+ }
}
- } // os
+ buildFlow("""
+// First, build the bits on RHEL7.2
+b = build(params, '${fullBuildJobName}')
+
+// Then, run the perf tests
+parallel(
+ ${flowJobTPRunList.join(",\n ")}
+)
+""")
+ }
+
+ Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
+ Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
+
+ if (isPR) {
+ TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
+ builder.setGithubContext("Linux Throughput Perf Test Flow")
+ builder.triggerOnlyOnComment()
+ builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
+ builder.triggerForBranch(branch)
+ builder.emitTrigger(newFlowJob)
+ }
+ else {
+ // Set a push trigger
+ TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
+ builder.emitTrigger(newFlowJob)
+ }
+
} // isPR