Enable arm64 Linux testing in CI

Testing is enabled on a set of Qualcomm Centriq arm64 servers running Ubuntu 16.04.
The set of jobs enabled almost matches the set run for arm32 Linux testing,
including innerloop, JIT and GC Stress, corefx, and R2R. Temporarily, the
innerloop jobs are commit jobs (invoked when a PR is merged) instead of
"default trigger" jobs (invoked when a PR is submitted), until we get more
experience with the robustness of the machines and jobs. The machines are
fast enough that they are not marked as "limited hardware" (like arm32 Linux
machines). That means that many jobs are run daily, not weekly, as periodic jobs.

Notes about the changes:
1. The Linux arm64 machines are managed by Helix, which allocates them to Jenkins.
2. The arm64 OS used has been renamed from "small_page_size" to "Ubuntu16.04". If we
add large page size machines, we'll need to add a differentiator.
3. The Jenkins "copy artifacts" plug-in runs ridiculously slowly on this hardware,
for unknown reasons, so we copy artifacts directly using "wget".
4. Tests are built using "build-test.sh" on the (cross) build machine; we don't use
Windows-built tests.
5. Added Jenkins archiving of build .log/.wrn/.err files.
6. Various tests were disabled in issues.targets, and with a new
arm64/corefx_linux_test_exclusions.txt file, to get jobs to run clean. (Several
issues have been opened to track these and other known failures.)

1 files changed, 268 insertions, 211 deletions

diff --git a/netci.groovy b/netci.groovy
index f9088dc819..5a854c4ed8 100755
--- a/netci.groovy
+++ b/netci.groovy
@@ -17,16 +17,16 @@ folder('illink')
 Utilities.addStandardFolderView(this, 'illink', project)
 
 def static getOSGroup(def os) {
-    def osGroupMap = ['Ubuntu':'Linux',
-        'RHEL7.2': 'Linux',
-        'Ubuntu16.04': 'Linux',
-        'Ubuntu16.10': 'Linux',
-        'Debian8.4':'Linux',
-        'Fedora24':'Linux',
-        'OSX10.12':'OSX',
-        'Windows_NT':'Windows_NT',
-        'CentOS7.1': 'Linux',
-        'Tizen': 'Linux']
+    def osGroupMap = ['Ubuntu'      : 'Linux',
+                      'Ubuntu16.04' : 'Linux',
+                      'Ubuntu16.10' : 'Linux',
+                      'RHEL7.2'     : 'Linux',
+                      'Debian8.4'   : 'Linux',
+                      'Fedora24'    : 'Linux',
+                      'CentOS7.1'   : 'Linux',
+                      'Tizen'       : 'Linux',
+                      'OSX10.12'    : 'OSX',
+                      'Windows_NT'  : 'Windows_NT']
     def osGroup = osGroupMap.get(os, null)
     assert osGroup != null : "Could not find os group for ${os}"
     return osGroupMap[os]
@@ -212,13 +212,15 @@ class Constants {
             'x64': [
                 'Checked'
             ],
-            'arm64': [
-                'Debug'
-            ],
             'arm': [
                 'Checked'
             ]
         ],
+        'Ubuntu16.04': [
+            'arm64': [
+                'Checked'
+            ]
+        ],
         'CentOS7.1': [
             'x64': [
                 'Debug',
@@ -474,8 +476,11 @@ class Constants {
                'r2r_jitminopts',
                'r2r_jitforcerelocs',
                'r2r_gcstress15',
+               'r2r_no_tiered_compilation',
                'minopts',
                'tieredcompilation',
+               'no_tiered_compilation',
+               'no_tiered_compilation_innerloop',
                'forcerelocs',
                'jitstress1',
                'jitstress2',
@@ -712,15 +717,19 @@ def static setMachineAffinity(def job, def os, def architecture, def options = n
     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true
     // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto
     //       |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true
-    // Arm32 hardware (Test) -> ubuntu.1404.arm32.open
+    // Arm32 hardware (Test) -> Helix ubuntu.1404.arm32.open queue
     //       |-> os == "Ubuntu" && (architecture == "arm")
     //
     // Arm64 (Build) -> arm64-cross-latest
-    //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true
-    // Arm64 Small Page Size (Test) -> arm64-small-page-size
-    //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == false
-    // Arm64 Large Page Size (Test) -> arm64-huge-page-size
-    //       |-> os != "Windows_NT" && architecture == "arm64" && options['large_pages'] == true
+    //       |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_job'] == true
+    // Arm64 (Test) -> Helix Ubuntu.1604.Arm64.Open queue
+    //       |-> os != "Windows_NT" && architecture == "arm64"
+    //
+    // Note: we are no longer using Jenkins tags "arm64-huge-page-size", "arm64-small-page-size".
+    // Support for Linux arm64 large page size has been removed for now, as it wasn't being used.
+    //
+    // Note: we are no longer using Jenkins tag 'latest-arm64' for arm/arm64 Windows build machines. Instead,
+    // we are using public VS2017 arm/arm64 tools in a VM from Helix.
 
     // This has to be a arm arch
     assert architecture in armArches
@@ -729,10 +738,6 @@ def static setMachineAffinity(def job, def os, def architecture, def options = n
         def isBuild = options['use_arm64_build_machine'] == true
 
         if (isBuild == true) {
-            // Current set of machines with private Windows arm64 toolset:
-            // Utilities.setMachineAffinity(job, os, 'latest-arm64')
-            //
-            // New set of machines with public Windows arm64 toolset, coming from Helix:
             job.with {
                 label('Windows.10.Amd64.ClientRS4.DevEx.Open')
             }
@@ -742,48 +747,34 @@ def static setMachineAffinity(def job, def os, def architecture, def options = n
     } else {
         assert os != 'Windows_NT'
 
-        if (architecture == 'arm64') {
-            assert os == 'Ubuntu'
-            def isFlow  = (options != null) && (options['is_flow_job'] == true)
-            def isBuild = (options != null) && (options['is_build_job'] == true)
-            if (isFlow || isBuild) {
-                // Arm64 Ubuntu build machine. Build uses docker, so the actual host OS is not
-                // very important. Therefore, use latest or auto. Flow jobs don't need to use
-                // Arm64 hardware.
-                Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
-            } else {
-                // Arm64 Linux test machines
-                if ((options != null) && (options['large_pages'] == true)) {
-                    Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size')
-                } else {
-                    Utilities.setMachineAffinity(job, os, 'arm64-small-page-size')
-                }
-            }
-        }
-        else if (architecture == 'armem') {
+        if (architecture == 'armem') {
             // arm emulator (Tizen). Build and test on same machine,
             // using Docker.
             assert os == 'Tizen'
             Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest')
         }
         else {
-            // arm Ubuntu on hardware.
-            assert architecture == 'arm'
-            assert os == 'Ubuntu'
+            // arm/arm64 Ubuntu on hardware.
+            assert architecture == 'arm' || architecture == 'arm64'
             def isFlow  = (options != null) && (options['is_flow_job'] == true)
             def isBuild = (options != null) && (options['is_build_job'] == true)
             if (isFlow || isBuild) {
-                // arm Ubuntu build machine. Build uses docker, so the actual host OS is not
-                // very important. Therefore, use latest or auto. Flow jobs don't need to use
-                // arm hardware.
+                // arm/arm64 Ubuntu build machine. Build uses docker, so the actual host OS is not
+                // very important. Therefore, use latest or auto. Flow jobs don't need to use arm hardware.
                 Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto')
             } else {
-                // arm Ubuntu test machine
-                // There is no tag (like, e.g., "arm-latest") for this, so don't call
-                // Utilities.setMachineAffinity. Just add the machine affinity
-                // manually. We specify the Helix queue name here.
-                job.with {
-                    label('ubuntu.1404.arm32.open')
+                // arm/arm64 Ubuntu test machine. Specify the Helix queue name here.
+                if (architecture == 'arm64') {
+                    assert os == 'Ubuntu16.04'
+                    job.with {
+                        label('Ubuntu.1604.Arm64.Open')
+                    }
+                }
+                else {
+                    assert os == 'Ubuntu'
+                    job.with {
+                        label('ubuntu.1404.arm32.open')
+                    }
                 }
             }
         }
@@ -814,16 +805,7 @@ def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def i
         }
     }
     else {
-        if (architecture == 'arm64') {
-            if (isBuildJob) {
-                affinityOptions = ['is_build_job': true]
-            } else if (isFlowJob) {
-                affinityOptions = ['is_flow_job': true]
-            } else if (isTestJob) {
-                affinityOptions = [ "large_pages" : false ]
-            }
-        }
-        else if (architecture == 'arm') {
+        if ((architecture == 'arm64') || (architecture == 'arm')) {
             if (isBuildJob) {
                 affinityOptions = ['is_build_job': true]
             } else if (isFlowJob) {
@@ -1185,7 +1167,7 @@ def static isNeedDocker(def architecture, def os, def isBuild) {
             }
         }
         else if (architecture == 'arm64') {
-            if (os == 'Ubuntu') {
+            if (os == 'Ubuntu16.04') {
                 return true
             }
         }
@@ -1215,7 +1197,7 @@ def static getDockerImageName(def architecture, def os, def isBuild) {
             }
         }
         else if (architecture == 'arm64') {
-            if (os == 'Ubuntu') {
+            if (os == 'Ubuntu16.04') {
                 return "microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921"
             }
         }
@@ -1233,17 +1215,19 @@ def static getDockerImageName(def architecture, def os, def isBuild) {
 // We have a limited amount of some hardware. For these, scale back the periodic testing we do,
 // and only allowing using this hardware in some specific branches.
 def static jobRequiresLimitedHardware(def architecture, def os) {
-    if (((architecture == 'arm64') || (architecture == 'arm')) && (os == 'Windows_NT')) {
-        // These test jobs require ARM64 hardware
-        return true
-    }
-    else if ((architecture == 'arm') && (os == 'Ubuntu')) {
-        // These test jobs require Linux/arm32 hardware
+    if (architecture == 'arm') {
+        // arm Windows and Linux hardware is limited.
         return true
     }
-    else if ((architecture == 'arm64') && (os == 'Ubuntu')) {
-        // These test jobs require Linux/arm64 hardware
-        return true
+    else if (architecture == 'arm64') {
+        if (os == 'Windows_NT') {
+            // arm64 Windows hardware is limited.
+            return true
+        }
+        else {
+            // arm64 Linux hardware is fast enough to allow more frequent jobs
+            return false
+        }
     }
     else {
         return false
@@ -1276,22 +1260,14 @@ def static getJobName(def configuration, def architecture, def os, def scenario,
                 baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase()
             }
             break
-        case 'arm64':
-            if (os.toLowerCase() == "windows_nt") {
-                // These are cross builds
-                baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
-            }
-            else {
-                // Defaults to a small page size set of machines.
-                baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size"
-            }
-            break
         case 'armem':
             // These are cross builds
             assert os == 'Tizen'
             baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
             break
         case 'arm':
+        case 'arm64':
+            // These are cross builds
             baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase()
             break
         case 'x86':
@@ -1316,11 +1292,6 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def
         return
     }
 
-    // No arm64 Ubuntu cron jobs for now: we don't have enough hardware.
-    if ((architecture == 'arm64') && (os != 'Windows_NT')) {
-         return
-    }
-
     // Ubuntu x86 CI jobs are failing. Disable non-PR triggered jobs to avoid these constant failures
     // until this is fixed. Tracked by https://github.com/dotnet/coreclr/issues/19003.
     if (architecture == 'x86' && os == 'Ubuntu') {
@@ -1331,6 +1302,13 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def
     switch (scenario) {
         case 'innerloop':
         case 'no_tiered_compilation_innerloop':
+            // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
+            //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
+            // We should never get here (in the "innerloop cases) for non-PR jobs, except for this TEMPORARY exception.
+            assert (isInnerloopTestScenario(scenario) && (architecture == 'arm64') && (os == 'Ubuntu16.04') && (configuration == 'Checked'))
+            addGithubPushTriggerHelper(job)
+            break
+
         case 'crossgen_comparison':
             break
         case 'normal':
@@ -1340,7 +1318,7 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def
                     if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') {
                         addPeriodicTriggerHelper(job, '@daily')
                     }
-                    else if (isFlowJob || os == 'Windows_NT' || !(os in Constants.crossList)) {
+                    else if (isFlowJob || os == 'Windows_NT' || (architecture == 'x64' && !(os in Constants.crossList))) {
                         addGithubPushTriggerHelper(job)
                     }
                     break
@@ -1397,7 +1375,6 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def
             assert !(os in bidailyCrossList)
             // r2r gets a push trigger for checked/release
             if (configuration == 'Checked' || configuration == 'Release') {
-                assert (os == 'Windows_NT') || (os in Constants.crossList)
                 if (architecture == 'x64' && os != 'OSX10.12') {
                     //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS
                     if (isFlowJob || os == 'Windows_NT') {
@@ -1415,21 +1392,18 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def
                         addGithubPushTriggerHelper(job)
                     }
                 }
-                // arm64 r2r jobs should only run weekly.
-                // arm64 r2r jobs are only run on Windows (Q: should they run on non-Windows?)
-                else if (architecture == 'arm64') {
-                    if (os == 'Windows_NT') {
-                        if (isFlowJob) {
-                            addPeriodicTriggerHelper(job, '@weekly')
-                        }
-                    }
-                }
                 // arm r2r jobs should only run weekly.
                 else if (architecture == 'arm') {
                     if (isFlowJob) {
                         addPeriodicTriggerHelper(job, '@weekly')
                     }
                 }
+                // arm64 r2r jobs should only run weekly.
+                else if (architecture == 'arm64') {
+                    if (isFlowJob) {
+                        addPeriodicTriggerHelper(job, '@weekly')
+                    }
+                }
             }
             break
         case 'r2r_jitstress1':
@@ -1455,13 +1429,10 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def
                 break
             }
 
-            // GC Stress 15 r2r gets a push trigger for checked/release
             if (configuration == 'Checked' || configuration == 'Release') {
-                assert (os == 'Windows_NT') || (os in Constants.crossList)
                 if (architecture == 'x64') {
                     //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS
                     if (isFlowJob || os == 'Windows_NT') {
-                        // Add a weekly periodic trigger
                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
                     }
                 }
@@ -1471,15 +1442,12 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def
                         addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday
                     }
                 }
-                // arm64 r2r jobs are only run on Windows (Q: should they run on non-Windows?)
-                else if (architecture == 'arm64') {
-                    if (os == 'Windows_NT') {
-                        if (isFlowJob) {
-                            addPeriodicTriggerHelper(job, '@weekly')
-                        }
+                else if (architecture == 'arm') {
+                    if (isFlowJob) {
+                        addPeriodicTriggerHelper(job, '@weekly')
                     }
                 }
-                else if (architecture == 'arm') {
+                else if (architecture == 'arm64') {
                     if (isFlowJob) {
                         addPeriodicTriggerHelper(job, '@weekly')
                     }
@@ -1518,8 +1486,6 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def
             assert !(os in bidailyCrossList)
             // ILASM/ILDASM roundtrip one gets a daily build, and only for release
             if (architecture == 'x64' && configuration == 'Release') {
-                // We don't expect to see a job generated except in these scenarios
-                assert (os == 'Windows_NT') || (os in Constants.crossList)
                 if (isFlowJob || os == 'Windows_NT') {
                     addPeriodicTriggerHelper(job, '@daily')
                 }
@@ -1598,7 +1564,6 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def
             if ((architecture == 'arm64') && isCoreFxScenario(scenario) && !isFlowJob) {
                 break
             }
-            assert (os == 'Windows_NT') || (os in Constants.crossList)
             if (jobRequiresLimitedHardware(architecture, os)) {
                 addPeriodicTriggerHelper(job, '@weekly')
             }
@@ -1614,11 +1579,6 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def
             if (os in bidailyCrossList) {
                 break
             }
-            if ((architecture == 'arm64') && (os != 'Windows_NT')) {
-                // TODO: should we have cron jobs for arm64 Linux GCStress?
-                break
-            }
-            assert (os == 'Windows_NT') || (os in Constants.crossList)
             addPeriodicTriggerHelper(job, '@weekly')
             break
         case 'gcstress0xc':
@@ -1628,21 +1588,16 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def
         case 'gcstress0xc_jitstress1':
         case 'gcstress0xc_jitstress2':
         case 'gcstress0xc_minopts_heapverify1':
-            if (os == 'CentOS7.1') {
-                break
-            }
             if (os == 'OSX10.12') {
                 // GCStress=C is currently not supported on OS X
                 break
             }
-            if (os in bidailyCrossList) {
+            if (os == 'CentOS7.1') {
                 break
             }
-            if ((architecture == 'arm64') && (os != 'Windows_NT')) {
-                // TODO: should we have cron jobs for arm64 Linux GCStress?
+            if (os in bidailyCrossList) {
                 break
             }
-            assert (os == 'Windows_NT') || (os in Constants.crossList)
             addPeriodicTriggerHelper(job, '@weekly')
             break
 
@@ -1954,34 +1909,26 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os,
 
             switch (os) {
                 case 'Ubuntu':
-                    // TODO: make arm and arm64 Ubuntu more alike
-
-                    if (architecture == 'arm') {
-                        // Triggers on the non-flow jobs aren't necessary
-                        if (!isFlowJob) {
-                            needsTrigger = false
-                            break
-                        }
+                case 'Ubuntu16.04':
 
-                        switch (scenario) {
-                            case 'innerloop':
-                            case 'no_tiered_compilation_innerloop':
-                                if (configuration == 'Checked') {
-                                    isDefaultTrigger = true
-                                }
-                                break
-                        }
+                    // Triggers on the non-flow jobs aren't necessary
+                    if (!isFlowJob) {
+                        needsTrigger = false
+                        break
                     }
-                    else {
-                        assert architecture == 'arm64'
 
-                        switch (scenario) {
-                            case 'innerloop':
-                                if (configuration == 'Debug' && !isFlowJob) {
-                                    isDefaultTrigger = true
+                    switch (scenario) {
+                        case 'innerloop':
+                        case 'no_tiered_compilation_innerloop':
+                            if (configuration == 'Checked') {
+                                // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
+                                //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
+                                if (architecture == 'arm64') {
+                                    break
                                 }
-                                break
-                        }
+                                isDefaultTrigger = true
+                            }
+                            break
                     }
                     break
 
@@ -2469,7 +2416,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                 case 'arm64':
                 case 'arm':
                     // Non-Windows ARM cross builds on hardware run on Ubuntu only
-                    assert (os == 'Ubuntu')
+                    assert (os == 'Ubuntu') || (os == 'Ubuntu16.04')
 
                     // Add some useful information to the log file. Ignore return codes.
                     buildCommands += "uname -a || true"
@@ -2534,7 +2481,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                         buildCommands += "${dockerCmd}zip -r ${workspaceRelativeArtifactsArchive} ${workspaceRelativeCoreLib} ${workspaceRelativeCoreRootDir} ${workspaceRelativeCrossGenComparisonScript} ${workspaceRelativeResultsDir}"
                         Utilities.addArchival(newJob, "${workspaceRelativeArtifactsArchive}")
                     }
-                    else {
+                    else if (architecture == 'arm') {
                         // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
                         // download the appropriate CoreFX packages.
                         // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it
@@ -2554,6 +2501,31 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
 
                         Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
                     }
+                    else {
+                        assert architecture == 'arm64'
+
+                        // Then, using the same docker image, build the tests and generate the CORE_ROOT layout.
+                        // Linux/arm64 does not use Windows-built tests.
+
+                        def testBuildOpts = ""
+                        if (priority == '1') {
+                            testBuildOpts = "priority1"
+                        }
+
+                        buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross ${testBuildOpts}"
+
+                        // ZIP up the built tests (including CORE_ROOT and native test components copied to the CORE_ROOT) for the test job (created in the flow job code)
+                        buildCommands += "zip -r tests.${lowerConfiguration}.zip ./bin/tests/Linux.${architecture}.${configuration}"
+
+                        // We still use the testnativebin files until they get placed properly in the tests directory (next to their respective tests).
+                        // With https://github.com/dotnet/coreclr/pull/19918 this shouldn't be needed anymore.
+                        buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.${architecture}.${configuration}/tests"
+
+                        Utilities.addArchival(newJob, "tests.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "")
+                    }
+
+                    // Archive the build logs from both product and test builds.
+                    Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err", "")
 
                     // We need to clean up the build machines; the docker build leaves newly built files with root permission, which
                     // the cleanup task in Jenkins can't remove.
@@ -2585,15 +2557,22 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
 // Returns true if the job should be generated.
 def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly)
 {
-    // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
-    if (scenario == 'innerloop' && !isPR) {
-        return false
-    }
+    // The various "innerloop" jobs are only available as PR triggered.
 
-    // Run basic corefx tests only on PR-triggered jobs
-    // Runs under Release and Checked 
-    if (scenario == 'corefx_innerloop' && !isPR) {
-        return false
+    if (!isPR) {
+        if (isInnerloopTestScenario(scenario) && (architecture == 'arm64') && (os == 'Ubuntu16.04') && (configuration == 'Checked')) {
+            // TEMPORARY: make arm64 Linux innerloop jobs push jobs, not default triggered jobs, until we have experience
+            //            with the machines running these jobs (and the jobs themselves), to understand how robust they are.
+            return true
+        }
+
+        if (isInnerloopTestScenario(scenario)) {
+            return false
+        }
+
+        if (scenario == 'corefx_innerloop') {
+            return false
+        }
     }
 
     // Tizen is only supported for armem architecture
@@ -2604,12 +2583,16 @@ def static shouldGenerateJob(def scenario, def isPR, def architecture, def confi
     // Filter based on architecture.
 
     switch (architecture) {
-        case 'arm64':
         case 'arm':
             if ((os != 'Windows_NT') && (os != 'Ubuntu')) {
                 return false
             }
             break
+        case 'arm64':
+            if ((os != 'Windows_NT') && (os != 'Ubuntu16.04')) {
+                return false
+            }
+            break
         case 'armem':
             if (os != 'Tizen') {
                 return false
@@ -2666,7 +2649,10 @@ def static shouldGenerateJob(def scenario, def isPR, def architecture, def confi
             return false
         }
 
-        def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && (isCoreFxScenario(scenario) || architecture == 'arm' || architecture == 'arm64'))
+        def isEnabledOS = (os == 'Windows_NT') ||
+                          (os == 'Ubuntu' && (architecture == 'x64') && isCoreFxScenario(scenario)) ||
+                          (os == 'Ubuntu' && architecture == 'arm') ||
+                          (os == 'Ubuntu16.04' && architecture == 'arm64')
         if (!isEnabledOS) {
             return false
         }
@@ -2685,6 +2671,7 @@ def static shouldGenerateJob(def scenario, def isPR, def architecture, def confi
                 break
 
             case 'arm':
+            case 'arm64':
                 // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that.
                 // No "regular" Windows arm corefx jobs, e.g.
                 // For Ubuntu arm corefx testing, we use regular jobs (not "build only" since only Windows has "build only", and
@@ -2701,21 +2688,7 @@ def static shouldGenerateJob(def scenario, def isPR, def architecture, def confi
                 }
                 break
 
-            case 'arm64':
-                if (os == 'Windows_NT') {
-                    if (! (isBuildOnly && isCoreFxScenario(scenario)) ) {
-                        return false
-                    }
-                }
-                else {
-                    if (!isCoreFxScenario(scenario)) {
-                        return false
-                    }
-                }
-                break
-
             default:
-                // arm64: stress is handled through flow jobs.
                 // armem: no stress jobs for ARM emulator.
                 return false
         }
@@ -3150,7 +3123,12 @@ def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture
 // Returns the newly created job.
 def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName)
 {
-    def isUbuntuArmJob = (os == "Ubuntu") && ((architecture == 'arm') || (architecture == 'arm64')) // ARM Ubuntu running on hardware (not emulator)
+    def lowerConfiguration = configuration.toLowerCase()
+
+    def isUbuntuArm64Job = ((os == "Ubuntu16.04") && (architecture == 'arm64'))
+    def isUbuntuArm32Job = ((os == "Ubuntu") && (architecture == 'arm'))
+    def isUbuntuArmJob = isUbuntuArm32Job || isUbuntuArm64Job
+
     def doCoreFxTesting = isCoreFxScenario(scenario)
 
     def workspaceRelativeFxRootLinux = "_/fx" // only used for CoreFX testing
@@ -3289,11 +3267,16 @@ def static CreateOtherTestJob(def dslFactory, def project, def branch, def archi
             // Coreclr build we are trying to test
             //
             //  ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive
+            //
+            // HACK: the Ubuntu arm64 copyArtifacts Jenkins plug-in is ridiculously slow (45 minutes to
+            // 1.5 hours for this step). Instead, directly use wget, which is fast (1 minute).
 
-            copyArtifacts(inputCoreCLRBuildName) {
-                excludePatterns('**/testResults.xml', '**/*.ni.dll')
-                buildSelector {
-                    buildNumber('${CORECLR_BUILD}')
+            if (!isUbuntuArm64Job) {
+                copyArtifacts(inputCoreCLRBuildName) {
+                    excludePatterns('**/testResults.xml', '**/*.ni.dll')
+                    buildSelector {
+                        buildNumber('${CORECLR_BUILD}')
+                    }
                 }
             }
 
@@ -3302,6 +3285,64 @@ def static CreateOtherTestJob(def dslFactory, def project, def branch, def archi
                 shell("uname -a || true")
             }
 
+            if (isUbuntuArm64Job) {
+                // Copy the required artifacts directly, using wget, e.g.:
+                // 
+                //  https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/testnativebin.checked.zip
+                //  https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/tests.checked.zip
+                // 
+                // parameterized as:
+                //
+                //  https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/testnativebin.checked.zip
+                //  https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/tests.checked.zip
+                //
+                // CoreFX example artifact URLs:
+                //
+                //  https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxruntime.zip
+                //  https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxtests.zip
+                //
+                // Note that the source might be in a "jitstress" folder.
+                //
+                // Use `--progress=dot:giga` to display some progress output, but limit it in the log file.
+                //
+                // Use `--directory-prefix=_/fx` to specify where to put the corefx files (to match what other platforms do). Use this instead of `-O`.
+
+                shell("echo \"Using wget instead of the Jenkins copy artifacts plug-in to copy artifacts from ${inputCoreCLRBuildName}\"")
+
+                def mungedProjectName = Utilities.getFolderName(project)
+                def mungedBranchName = Utilities.getFolderName(branch)
+
+                def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
+                def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
+                if (doCoreFxTesting || doCrossGenComparison) {
+                    // These depend on unique builds for each scenario
+                    inputCoreCLRBuildScenario = scenario
+                }
+                def sourceJobName = getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, false)
+                def inputJobName = Utilities.getFullJobName(sourceJobName, isPR)
+
+                // Need to add the sub-folder if necessary.
+                def inputJobPath = "job/${inputJobName}"
+                def folderName = getJobFolder(inputCoreCLRBuildScenario)
+                if (folderName != '') {
+                    inputJobPath = "job/${folderName}/job/${inputJobName}"
+                }
+
+                def inputUrlRoot = "https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/${inputJobPath}/\${CORECLR_BUILD}/artifact"
+
+                if (doCoreFxTesting) {
+                    shell("mkdir -p ${workspaceRelativeFxRootLinux}")
+                    shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxtests.zip")
+                    shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxruntime.zip")
+                    shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/run-test.sh")
+                    shell("chmod +x ${workspaceRelativeFxRootLinux}/run-test.sh")
+                }
+                else {
+                    shell("wget --progress=dot:giga ${inputUrlRoot}/testnativebin.${lowerConfiguration}.zip")
+                    shell("wget --progress=dot:giga ${inputUrlRoot}/tests.${lowerConfiguration}.zip")
+                }
+            }
+
             if (architecture == 'x86') {
                 shell("mkdir ./bin/CoreFxNative")
 
@@ -3322,10 +3363,12 @@ def static CreateOtherTestJob(def dslFactory, def project, def branch, def archi
 
             // CoreFX testing downloads the CoreFX tests, not the coreclr tests. Also, unzip the built CoreFX layout/runtime directories.
             if (doCoreFxTesting) {
-                shell("unzip -o ${workspaceRelativeFxRootLinux}/fxtests.zip || exit 0")
-                shell("unzip -o ${workspaceRelativeFxRootLinux}/fxruntime.zip || exit 0")
+                shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxtests.zip || exit 0")
+                shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxruntime.zip || exit 0")
             }
-            else {
+            else if (architecture != 'arm64') {
+                // ARM64 copies the tests from the build machine; this is for unzip'ing tests copied from a Windows build.
+                //
                 // Unzip the tests first.  Exit with 0
                 shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0")
                 shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0")
@@ -3338,9 +3381,17 @@ def static CreateOtherTestJob(def dslFactory, def project, def branch, def archi
             // copied correctly.
             if (!doCoreFxTesting) {
                 if (isUbuntuArmJob) {
-                    def lowerConfiguration = configuration.toLowerCase()
-                    shell("unzip -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root
-                    shell("unzip -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
+                    if (architecture == 'arm') {
+                        shell("unzip -q -o ./coreroot.${lowerConfiguration}.zip || exit 0")      // unzips to ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root
+                        shell("unzip -q -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
+                    }
+                    else {
+                        assert architecture == 'arm64'
+                        shell("unzip -q -o ./tests.${lowerConfiguration}.zip || exit 0")         // unzips to ./bin/tests/Linux.${architecture}.${configuration}
+
+                        // We still the testnativebin files until they get placed properly in the tests directory (next to their respective tests).
+                        shell("unzip -q -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.${architecture}.${configuration}/tests
+                    }
                 }
                 else {
                     shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly")
@@ -3385,6 +3436,8 @@ def static CreateOtherTestJob(def dslFactory, def project, def branch, def archi
             else {
                 def runScript = "${dockerCmd}./tests/runtest.sh"
 
+                // TODO: the testNativeBinDir shouldn't be necessary if the native test binaries are placed properly with their corresponding managed test code.
+
                 shell("""\
 ${runScript} \\
     --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\
@@ -3584,24 +3637,31 @@ build(params + [CORECLR_BUILD: coreclrBuildJob.build.number,
 // Returns true if the job should be generated.
 def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os)
 {
-    // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered.
-    if (scenario == 'innerloop' && !isPR) {
-        return false
-    }
-    
-    if (scenario == 'corefx_innerloop') {
-        return false
+    // The various "innerloop" jobs are only available as PR triggered.
+
+    if (!isPR) {
+        if (isInnerloopTestScenario(scenario)) {
+            return false
+        }
+
+        if (scenario == 'corefx_innerloop') {
+            return false
+        }
     }
 
     // Filter based on OS and architecture.
 
     switch (architecture) {
-        case 'arm64':
         case 'arm':
             if (os != "Ubuntu" && os != "Windows_NT") {
                 return false
             }
             break
+        case 'arm64':
+            if (os != "Ubuntu16.04" && os != "Windows_NT") {
+                return false
+            }
+            break
         case 'x86':
             if (os != "Ubuntu") {
                 return false
@@ -3643,19 +3703,13 @@ def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def c
     }
     else {
         // Non-Windows
-        if (architecture == 'arm64') {
-            if (!(scenario in Constants.validLinuxArm64Scenarios)) {
-                return false
-            }
-
-            if (isNormalOrInnerloop && (configuration == 'Debug')) {
-                // The arm32/arm64 Debug configuration for innerloop/normal scenario is a special case: it does a build only, and no test run.
-                // To do that, it doesn't require a flow job.
+        if (architecture == 'arm') {
+            if (!(scenario in Constants.validLinuxArmScenarios)) {
                 return false
             }
         }
-        else if (architecture == 'arm') {
-            if (!(scenario in Constants.validLinuxArmScenarios)) {
+        else if (architecture == 'arm64') {
+            if (!(scenario in Constants.validLinuxArm64Scenarios)) {
                 return false
             }
         }
@@ -3760,6 +3814,10 @@ def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def c
                 }
                 break
 
+            case 'corefx_innerloop':
+                // No flow job needed
+                return false
+
             default:
                 println("Unknown scenario: ${scenario}")
                 assert false
@@ -3814,15 +3872,14 @@ Constants.allScenarios.each { scenario ->
 
                     def inputTestsBuildName = null
 
-                    if (!windowsArmJob && !doCoreFxTesting & !doCrossGenComparison) {
+                    // Ubuntu Arm64 jobs do the test build on the build machine, and thus don't depend on a Windows build.
+                    def isUbuntuArm64Job = ((os == "Ubuntu16.04") && (architecture == 'arm64'))
+
+                    if (!windowsArmJob && !doCoreFxTesting & !doCrossGenComparison && !isUbuntuArm64Job) {
                         def testBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal'
 
                         def inputTestsBuildArch = architecture
-                        if (architecture == "arm64") {
-                            // Use the x64 test build for arm64 unix
-                            inputTestsBuildArch = "x64"
-                        }
-                        else if (architecture == "arm") {
+                        if (architecture == "arm") {
                             // Use the x86 test build for arm unix
                             inputTestsBuildArch = "x86"
                         }