Imported Upstream version 2.0.0.12082upstream/2.0.0.12082

1 files changed, 148 insertions, 42 deletions

diff --git a/perf.groovy b/perf.groovy
index 9be47c0b3c..d2a4c327ee 100644
--- a/perf.groovy
+++ b/perf.groovy
@@ -88,9 +88,9 @@ def static getOSGroup(def os) {
                         batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
                         //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
                         //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
-                        batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
-                        "set BENCHVIEWNAME=${benchViewName}\n" +
-                        "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
+                        batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
+                        "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
+                        "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
                         batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
@@ -198,9 +198,9 @@ def static getOSGroup(def os) {
                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
-                    batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
-                    "set BENCHVIEWNAME=${benchViewName}\n" +
-                    "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
+                    batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
+                    "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
+                    "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
@@ -234,10 +234,32 @@ def static getOSGroup(def os) {
     }
 }
 
+def static getFullPerfJobName(def project, def os, def isPR) {
+    return Utilities.getFullJobName(project, "perf_${os}", isPR)
+}
+
 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 [true, false].each { isPR ->
-    ['Ubuntu14.04'].each { os ->
-        def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
+    def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
+    def architecture = 'x64'
+    def configuration = 'Release'
+
+    // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
+    ['RHEL7.2'].each { os ->
+        def newBuildJob = job(fullBuildJobName) {
+            steps {
+                shell("./build.sh verbose ${architecture} ${configuration}")
+            }
+        }
+        Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
+        Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
+        Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
+    }
+
+    // Actual perf testing on the following OSes
+    def perfOSList = ['Ubuntu14.04']
+    perfOSList.each { os ->
+        def newJob = job(getFullPerfJobName(project, os, isPR)) {
 
             label('linux_clr_perf')
             wrappers {
@@ -254,22 +276,26 @@ def static getOSGroup(def os) {
                 }
             }
 
-            // Cap the maximum number of iterations to 21.
             parameters {
+                // Cap the maximum number of iterations to 21.
                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
+                stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
             }
 
             def osGroup = getOSGroup(os)
-            def architecture = 'x64'
-            def configuration = 'Release'
             def runType = isPR ? 'private' : 'rolling'
             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 
             steps {
                 shell("./tests/scripts/perf-prep.sh")
                 shell("./init-tools.sh")
-                shell("./build.sh ${architecture} ${configuration}")
+                copyArtifacts(fullBuildJobName) {
+                    includePatterns("bin/**")
+                    buildSelector {
+                        buildNumber('\${PRODUCT_BUILD}')
+                    }
+                }
                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
@@ -306,26 +332,73 @@ def static getOSGroup(def os) {
                 numToKeep(1000)
             }
         }
+    } // os
+
+    def flowJobPerfRunList = perfOSList.collect { os ->
+        "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
+    }
+    def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
         if (isPR) {
-            TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
-            builder.setGithubContext("${os} Perf Tests")
-            builder.triggerOnlyOnComment()
-            builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
-            builder.triggerForBranch(branch)
-            builder.emitTrigger(newJob)
-        }
-        else {
-            // Set a push trigger
-            TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
-            builder.emitTrigger(newJob)
+            parameters {
+                stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
+            }
         }
-    } // os
+        buildFlow("""
+// First, build the bits on RHEL7.2
+b = build(params, '${fullBuildJobName}')
+
+// Then, run the perf tests
+parallel(
+    ${flowJobPerfRunList.join(",\n    ")}
+)
+""")
+    }
+
+    Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
+    Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
+
+    if (isPR) {
+        TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
+        builder.setGithubContext("Linux Perf Test Flow")
+        builder.triggerOnlyOnComment()
+        builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
+        builder.triggerForBranch(branch)
+        builder.emitTrigger(newFlowJob)
+    }
+    else {
+        // Set a push trigger
+        TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
+        builder.emitTrigger(newFlowJob)
+    }
+
 } // isPR
 
+def static getFullThroughputJobName(def project, def os, def isPR) {
+    return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
+}
+
 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 [true, false].each { isPR ->
-    ['Ubuntu14.04'].each { os ->
-        def newJob = job(Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)) {
+    def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
+    def architecture = 'x64'
+    def configuration = 'Release'
+
+    // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
+    ['RHEL7.2'].each { os ->
+        def newBuildJob = job(fullBuildJobName) {
+            steps {
+                shell("./build.sh verbose ${architecture} ${configuration}")
+            }
+        }
+        Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
+        Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
+        Utilities.addArchival(newBuildJob, "bin/Product/**")
+    }
+
+    // Actual perf testing on the following OSes
+    def throughputOSList = ['Ubuntu14.04']
+    throughputOSList.each { os ->
+        def newJob = job(getFullThroughputJobName(project, os, isPR)) {
 
             label('linux_clr_perf')
                 wrappers {
@@ -341,16 +414,24 @@ def static getOSGroup(def os) {
                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
                 }
             }
+
+            parameters {
+                stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
+            }
+
             def osGroup = getOSGroup(os)
-            def architecture = 'x64'
-            def configuration = 'Release'
             def runType = isPR ? 'private' : 'rolling'
             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 
             steps {
                 shell("bash ./tests/scripts/perf-prep.sh --throughput")
                 shell("./init-tools.sh")
-                shell("./build.sh ${architecture} ${configuration}")
+                copyArtifacts(fullBuildJobName) {
+                    includePatterns("bin/Product/**")
+                    buildSelector {
+                        buildNumber('\${PRODUCT_BUILD}')
+                    }
+                }
                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
@@ -359,7 +440,7 @@ def static getOSGroup(def os) {
                 -os \"${os}\" \\
                 -configuration \"${configuration}\" \\
                 -clr_root \"\${WORKSPACE}\" \\
-                -assembly_root \"\${WORKSPACE}/_/fx/bin/runtime/netcoreapp-${osGroup}-${configuration}-${architecture}\" \\
+                -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
                 -run_type \"${runType}\" \\
                 -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
             }
@@ -383,18 +464,43 @@ def static getOSGroup(def os) {
                 numToKeep(1000)
             }
         }
+    } // os
+
+    def flowJobTPRunList = throughputOSList.collect { os ->
+        "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
+    }
+    def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
         if (isPR) {
-            TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
-            builder.setGithubContext("${os} Throughput Perf Tests")
-            builder.triggerOnlyOnComment()
-            builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+throughput.*")
-            builder.triggerForBranch(branch)
-            builder.emitTrigger(newJob)
-        }
-        else {
-            // Set a push trigger
-            TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
-            builder.emitTrigger(newJob)
+            parameters {
+                stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
+            }
         }
-    } // os
+        buildFlow("""
+// First, build the bits on RHEL7.2
+b = build(params, '${fullBuildJobName}')
+
+// Then, run the perf tests
+parallel(
+    ${flowJobTPRunList.join(",\n    ")}
+)
+""")
+    }
+
+    Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
+    Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
+
+    if (isPR) {
+        TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
+        builder.setGithubContext("Linux Throughput Perf Test Flow")
+        builder.triggerOnlyOnComment()
+        builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
+        builder.triggerForBranch(branch)
+        builder.emitTrigger(newFlowJob)
+    }
+    else {
+        // Set a push trigger
+        TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
+        builder.emitTrigger(newFlowJob)
+    }
+
 } // isPR