Enable arm64 Linux testing in CI

Testing is enabled on a set of Qualcomm Centriq arm64 servers running Ubuntu 16.04.
The set of jobs enabled almost matches the set run for arm32 Linux testing,
including innerloop, JIT and GC Stress, corefx, and R2R. Temporarily, the
innerloop jobs are commit jobs (invoked when a PR is merged) instead of
"default trigger" jobs (invoked when a PR is submitted), until we get more
experience with the robustness of the machines and jobs. The machines are
fast enough that they are not marked as "limited hardware" (like arm32 Linux
machines). That means that many jobs are run daily, not weekly, as periodic jobs.

Notes about the changes:
1. The Linux arm64 machines are managed by Helix, which allocates them to Jenkins.
2. The arm64 OS used has been renamed from "small_page_size" to "Ubuntu16.04". If we
add large page size machines, we'll need to add a differentiator.
3. The Jenkins "copy artifacts" plug-in runs ridiculously slowly on this hardware,
for unknown reasons, so we copy artifacts directly using "wget".
4. Tests are built using "build-test.sh" on the (cross) build machine; we don't use
Windows-built tests.
5. Added Jenkins archiving of build .log/.wrn/.err files.
6. Various tests were disabled in issues.targets, and with a new
arm64/corefx_linux_test_exclusions.txt file, to get jobs to run clean. (Several
issues have been opened to track these and other known failures.)

5 files changed, 115 insertions, 94 deletions

diff --git a/tests/arm64/corefx_linux_test_exclusions.txt b/tests/arm64/corefx_linux_test_exclusions.txt
new file mode 100644
index 0000000000..1331a07ba3
--- /dev/null
+++ b/tests/arm64/corefx_linux_test_exclusions.txt
@@ -0,0 +1,13 @@
+System.Buffers.Tests                 # timeout. baseline and jit stress.
+System.Drawing.Common.Tests          # bad result data. baseline and jit stress.
+System.Collections.Immutable.Tests   # JitStress=2
+System.Collections.Tests             # JitStress=2
+System.Linq.Expressions.Tests        # JitStress=1
+System.Linq.Tests                    # JitStress=2
+System.Net.NameResolution.Functional.Tests # baseline
+System.Net.NameResolution.Pal.Tests  # baseline
+System.Numerics.Vectors.Tests        # JitStress=2
+System.Runtime.Serialization.Formatters.Tests # baseline + many stress variants
+System.Security.Cryptography.Algorithms.Tests # bad result. baseline and jit stress.
+System.Text.RegularExpressions.Tests # https://github.com/dotnet/coreclr/issues/17754 -- timeout -- JitMinOpts + Tiered only
+System.Threading.Tests               # bad result. baseline
diff --git a/tests/issues.targets b/tests/issues.targets
index a9db89618c..a4b7279f6c 100644
--- a/tests/issues.targets
+++ b/tests/issues.targets
@@ -222,6 +222,12 @@
         <ExcludeList Include="$(XunitTestBinBase)/CoreMangLib/cti/system/string/StringFormat2/*">
             <Issue>needs triage</Issue>
         </ExcludeList>        
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/HardwareIntrinsics/Arm64/Simd/*">
+            <Issue>18895</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/Methodical/MDArray/DataTypes/float_cs_ro/*">
+            <Issue>18989</Issue>
+        </ExcludeList>
         <ExcludeList Include="$(XunitTestBinBase)/JIT/Regression/JitBlue/DevDiv_590772/DevDiv_590772/*">
             <Issue>needs triage</Issue>
         </ExcludeList>
@@ -357,9 +363,6 @@
         <ExcludeList Include="$(XunitTestBinBase)/reflection/DefaultInterfaceMethods/InvokeConsumer/*">
             <Issue>9565</Issue>
         </ExcludeList>
-        <ExcludeList Include="$(XunitTestBinBase)/JIT/HardwareIntrinsics/Arm64/Simd/*">
-            <Issue>18895</Issue>
-        </ExcludeList>
         <ExcludeList Include="$(XunitTestBinBase)/reflection/DefaultInterfaceMethods/Emit/*">
             <Issue>9565</Issue>
         </ExcludeList>
@@ -468,6 +471,12 @@
         <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/eh/FinallyExec/nonlocalexitinroot/*">
             <Issue>Test times out</Issue>
         </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/Regression/VS-ia64-JIT/M00/b80373/b80373/*">
+            <Issue>20029</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/GC/Scenarios/DoublinkList/dlstack/*">
+            <Issue>Release only crash</Issue>
+        </ExcludeList>
     </ItemGroup>
 
     <!-- Unix arm32 specific -->
diff --git a/tests/runtest.proj b/tests/runtest.proj
index 2db2918353..fc81b1528a 100644
--- a/tests/runtest.proj
+++ b/tests/runtest.proj
@@ -57,7 +57,9 @@
   <!-- Target to check the test build, to see if it looks ok. We've had several cases where a change inadvertently and drastically changes
        the set of tests that are built, and that change is unnoticed. The most common case is for a build of the Priority 1 tests
        to only build the Priority 0 tests. This target is run after a test build to verify that the basic number of tests that were
-       built is basically what was expected.
+       built is basically what was expected. When this was written, there were about 2500 Priority 0 tests and about 12270 Priority 1
+       tests (differing slightly based on platform). We currently check that the number of Priority 0 tests is greater than 2000 and
+       less than 3000, and the number of Priority 1 tests is greater than 11000.
   -->
   <Target Name="CheckTestBuild" DependsOnTargets="GetListOfTestCmds">
     <Error Condition="!Exists('$(XunitTestBinBase)')"
@@ -71,7 +73,7 @@
 
     <Error Condition="'$(CLRTestPriorityToBuild)' == '0' and '$(TestCount)' &lt;= 2000" Text="Unexpected test count. Expected &gt; 2000, found $(TestCount).'" />
     <Error Condition="'$(CLRTestPriorityToBuild)' == '0' and '$(TestCount)' &gt;= 3000" Text="Unexpected test count. Expected &lt; 3000, found $(TestCount).'" />
-    <Error Condition="'$(CLRTestPriorityToBuild)' == '1' and '$(TestCount)' &lt;= 11500" Text="Unexpected test count. Expected &gt; 1150, found $(TestCount).'" />
+    <Error Condition="'$(CLRTestPriorityToBuild)' == '1' and '$(TestCount)' &lt;= 11000" Text="Unexpected test count. Expected &gt; 11000, found $(TestCount).'" />
     <Error Condition="'$(CLRTestPriorityToBuild)' != '0' and '$(CLRTestPriorityToBuild)' != '1'" Text="Unknown priority $(CLRTestPriorityToBuild)" />
   </Target>
 
diff --git a/tests/runtest.py b/tests/runtest.py
index bd45d1118b..0845183690 100755
--- a/tests/runtest.py
+++ b/tests/runtest.py
@@ -427,7 +427,7 @@ def create_and_use_test_env(_os, env, func):
 
     if len(list(complus_vars.keys())) > 0:
         print("Found COMPlus variables in the current environment")
-        print()
+        print("")
 
         file_header = None
 
@@ -470,13 +470,13 @@ REM Temporary test env for test run.
                 test_env.write(line)
                 contents += line
 
-            print()
+            print("")
             print("TestEnv: %s" % test_env.name)
-            print() 
+            print("")
             print("Contents:")
-            print()
+            print("")
             print(contents)
-            print()
+            print("")
 
             return func(test_env.name)
 
@@ -601,6 +601,8 @@ def call_msbuild(coreclr_repo_location,
                 "/p:__LogsDir=%s" % logs_dir]
 
     print(" ".join(command))
+
+    sys.stdout.flush() # flush output before creating sub-process
     proc = subprocess.Popen(command)
 
     try:
@@ -743,6 +745,7 @@ def run_tests(host_os,
         os.environ["__TestTimeout"] = str(120*60*1000) # 1,800,000 ms
 
     # Set Core_Root
+    print("Setting CORE_ROOT=%s" % core_root)
     os.environ["CORE_ROOT"] = core_root
 
     # Set test env if exists
@@ -826,7 +829,7 @@ def setup_args(args):
 
             print("Using default test location.")
             print("TestLocation: %s" % default_test_location)
-            print()
+            print("")
 
         else:
             # The tests for the default location have not been built.
@@ -888,7 +891,7 @@ def setup_args(args):
 
             print("Using default test location.")
             print("TestLocation: %s" % default_test_location)
-            print()
+            print("")
 
     if core_root is None:
         default_core_root = os.path.join(test_location, "Tests", "Core_Root")
@@ -898,7 +901,7 @@ def setup_args(args):
 
             print("Using default location for core_root.")
             print("Core_Root: %s" % core_root)
-            print()
+            print("")
 
         elif args.generate_layout is False:
             # CORE_ROOT has not been setup correctly.
@@ -918,7 +921,7 @@ def setup_args(args):
             print("Using default location for test_native_bin_location.")
             test_native_bin_location = os.path.join(os.path.join(coreclr_repo_location, "bin", "obj", "%s.%s.%s" % (host_os, arch, build_type), "tests"))
             print("Native bin location: %s" % test_native_bin_location)
-            print()
+            print("")
             
         if not os.path.isdir(test_native_bin_location):
             print("Error, test_native_bin_location: %s, does not exist." % test_native_bin_location)
@@ -982,43 +985,29 @@ def setup_coredis_tools(coreclr_repo_location, host_os, arch, core_root):
         core_root(str)              : core_root
     """
 
-    test_location = os.path.join(coreclr_repo_location, "tests")
-
-    def is_coredis_tools_supported(host_os, arch):
-        """ Is coredis tools supported on this os/arch
-
-        Args:
-            host_os(str): os
-            arch(str)   : arch
-
-        """
-        unsupported_unix_arches = ["arm", "arm64"]
-
-        if host_os.lower() == "osx":
-            return False
-        
-        return True
+    if host_os.lower() == "osx":
+        print("GCStress C is not supported on your platform.")
+        sys.exit(1)
 
-        if host_os != "Windows_NT" and arch in unsupported_unix_arches:
-            return False
+    unsupported_arches = ["arm", "arm64"]
 
-        return True
+    if arch in unsupported_arches:
+        # Nothing to do; CoreDisTools unneeded.
+        return
 
-    if is_coredis_tools_supported(host_os, arch):
-        command = None
-        if host_os == "Windows_NT":
-            command = [os.path.join(test_location, "setup-stress-dependencies.cmd"), "/arch", arch, "/outputdir", core_root]
-        else:
-            command = [os.path.join(test_location, "setup-stress-dependencies.sh"), "--outputDir=%s" % core_root]
+    command = None
+    test_location = os.path.join(coreclr_repo_location, "tests")
+    if host_os == "Windows_NT":
+        command = [os.path.join(test_location, "setup-stress-dependencies.cmd"), "/arch", arch, "/outputdir", core_root]
+    else:
+        command = [os.path.join(test_location, "setup-stress-dependencies.sh"), "--outputDir=%s" % core_root]
 
-        proc = subprocess.Popen(command)
-        proc.communicate()
+    sys.stdout.flush() # flush output before creating sub-process
+    proc = subprocess.Popen(command)
+    proc.communicate()
 
-        if proc.returncode != 0:
-            print("setup_stress_dependencies.sh failed.")
-            sys.exit(1)
-    else:
-        print("GCStress C is not supported on your platform.")
+    if proc.returncode != 0:
+        print("Failed to set up stress dependencies.")
         sys.exit(1)
 
 def precompile_core_root(test_location,
@@ -1094,22 +1083,19 @@ def precompile_core_root(test_location,
 
         return_code = proc.returncode
 
-        passed = False
         if return_code == -2146230517:
             print("%s is not a managed assembly." % file)
-            return passed
+            return False
 
         if return_code != 0:
-            print("Unable to precompile %s" % file)
-            return passed
+            print("Unable to precompile %s (%d)" % (file, return_code))
+            return False
 
         print("Successfully precompiled %s" % file)
-        passed = True
-
-        return passed
+        return True
 
     print("Precompiling all assemblies in %s" % core_root)
-    print()
+    print("")
 
     env = os.environ.copy()
 
@@ -1135,7 +1121,7 @@ def precompile_core_root(test_location,
     for dll in dlls:
         call_crossgen(dll, env)
 
-    print()
+    print("")
 
 def setup_core_root(host_os, 
                     arch, 
@@ -1218,6 +1204,7 @@ def setup_core_root(host_os,
     print("Restoring packages...")
     print(" ".join(command))
 
+    sys.stdout.flush() # flush output before creating sub-process
     if not g_verbose:
         proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     else:
@@ -1230,7 +1217,7 @@ def setup_core_root(host_os,
         sys.exit(1)
 
     if proc.returncode == 1:
-        "Error test dependency resultion failed."
+        print("Error: package restore failed.")
         return False
 
     os.environ["__BuildLogRootName"] = ""
@@ -1283,6 +1270,7 @@ def setup_core_root(host_os,
     print("Creating Core_Root...")
     print(" ".join(command))
 
+    sys.stdout.flush() # flush output before creating sub-process
     if not g_verbose:
         proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     else:
@@ -1295,7 +1283,7 @@ def setup_core_root(host_os,
         sys.exit(1)
 
     if proc.returncode == 1:
-        "Error test dependency resultion failed."
+        print("Error: creating Core_Root failed.")
         return False
 
     os.environ["__BuildLogRootName"] = ""
@@ -1327,12 +1315,12 @@ def setup_core_root(host_os,
                 shutil.copytree(item, new_dir)
 
     # Copy the product dir to the core_root directory
-    print()
+    print("")
     print("Copying Product Bin to Core_Root:")
     print("cp -r %s%s* %s" % (product_location, os.path.sep, core_root))
     copy_tree(product_location, core_root)
     print("---------------------------------------------------------------------")
-    print()
+    print("")
 
     if is_corefx:
         corefx_utility_setup = os.path.join(coreclr_repo_location,
@@ -1347,11 +1335,13 @@ def setup_core_root(host_os,
                            "msbuild",
                            os.path.join(coreclr_repo_location, "tests", "runtest.proj"),
                            "/p:GenerateRuntimeLayout=true"]
+
+        sys.stdout.flush() # flush output before creating sub-process
         proc = subprocess.Popen(msbuild_command)
         proc.communicate()
 
         if not proc.returncode == 0:
-            "Error test dependency resultion failed."
+            print("Error: generating test host failed.")
             return False
 
         os.environ["__BuildLogRootName"] = ""
@@ -1361,11 +1351,12 @@ def setup_core_root(host_os,
                            "/t:Restore",
                            corefx_utility_setup]
 
+        sys.stdout.flush() # flush output before creating sub-process
         proc = subprocess.Popen(msbuild_command)
         proc.communicate()
 
         if proc.returncode == 1:
-            "Error test dependency resultion failed."
+            print("Error: msbuild failed.")
             return False
 
         corefx_logpath = os.path.join(coreclr_repo_location, 
@@ -1383,11 +1374,12 @@ def setup_core_root(host_os,
                            "/p:OutputPath=%s" % corefx_logpath,
                            corefx_utility_setup]
 
+        sys.stdout.flush() # flush output before creating sub-process
         proc = subprocess.Popen(msbuild_command)
         proc.communicate()
 
         if proc.returncode == 1:
-            "Error test dependency resultion failed."
+            print("Error: msbuild failed.")
             return False
 
     print("Core_Root setup.")
@@ -1476,6 +1468,7 @@ def build_test_wrappers(host_os,
     print("Creating test wrappers...")
     print(" ".join(command))
 
+    sys.stdout.flush() # flush output before creating sub-process
     if not g_verbose:
         proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
@@ -1515,7 +1508,7 @@ def build_test_wrappers(host_os,
         sys.exit(1)
 
     if proc.returncode == 1:
-        "Error test dependency resultion failed."
+        print("Error: creating test wrappers failed.")
         return False
 
 def find_test_from_name(host_os, test_location, test_name):
@@ -1740,14 +1733,6 @@ def print_summary(tests):
         else:
             skipped_tests.append(test)
 
-    print()
-    print("Total tests run: %d" % len(tests))
-    print()
-    print("Total passing tests: %d" % len(passed_tests))
-    print("Total failed tests: %d" % len(failed_tests))
-    print("Total skipped tests: %d" % len(skipped_tests))
-    print()
-
     failed_tests.sort(key=lambda item: item["time"], reverse=True)
     passed_tests.sort(key=lambda item: item["time"], reverse=True)
     skipped_tests.sort(key=lambda item: item["time"], reverse=True)
@@ -1788,41 +1773,51 @@ def print_summary(tests):
                     break
 
     if len(failed_tests) > 0:
-        print("Failed tests:")
-        print()
+        print("%d failed tests:" % len(failed_tests))
+        print("")
         print_tests_helper(failed_tests, None)
         
-
-    if len(passed_tests) > 50:
-        print()
-        print("50 slowest passing tests:")
-        print()
-        print_tests_helper(passed_tests, 50)
+    # The following code is currently disabled, as it produces too much verbosity in a normal
+    # test run. It could be put under a switch, or else just enabled as needed when investigating
+    # test slowness.
+    #
+    # if len(passed_tests) > 50:
+    #     print("")
+    #     print("50 slowest passing tests:")
+    #     print("")
+    #     print_tests_helper(passed_tests, 50)
 
     if len(failed_tests) > 0:
-        print()
+        print("")
         print("#################################################################")
         print("Output of failing tests:")
-        print()
+        print("")
 
         for item in failed_tests:
             print("[%s]: " % item["test_path"])
-            print()
+            print("")
             
             test_output = item["test_output"]
 
-            # XUnit results are captured as escaped, escaped characters.
+            # XUnit results are captured as escaped characters.
             test_output = test_output.replace("\\r", "\r")
             test_output = test_output.replace("\\n", "\n")
 
             print(test_output)
-            print()
+            print("")
 
-        print()
+        print("")
         print("#################################################################")
         print("End of output of failing tests")
         print("#################################################################")
-        print()
+        print("")
+
+    print("")
+    print("Total tests run    : %d" % len(tests))
+    print("Total passing tests: %d" % len(passed_tests))
+    print("Total failed tests : %d" % len(failed_tests))
+    print("Total skipped tests: %d" % len(skipped_tests))
+    print("")
 
 def create_repro(host_os, arch, build_type, env, core_root, coreclr_repo_location, tests):
     """ Go through the failing tests and create repros for them
@@ -1849,13 +1844,11 @@ def create_repro(host_os, arch, build_type, env, core_root, coreclr_repo_locatio
     repro_location = os.path.join(bin_location, "repro", "%s.%s.%s" % (host_os, arch, build_type))
     if os.path.isdir(repro_location):
         shutil.rmtree(repro_location)
-    
-    print("mkdir %s" % repro_location)
-    os.makedirs(repro_location)
 
-    print()
-    print("Creating repo files, they can be found at: %s" % repro_location)
+    print("")
+    print("Creating repro files at: %s" % repro_location)
 
+    os.makedirs(repro_location)
     assert os.path.isdir(repro_location)
 
     # Now that the repro_location exists under <coreclr_location>/bin/repro
@@ -1865,7 +1858,6 @@ def create_repro(host_os, arch, build_type, env, core_root, coreclr_repo_locatio
         debug_env.write_repro()
 
     print("Repro files written.")
-    print("They can be found at %s" % repro_location)
 
 def do_setup(host_os, 
              arch, 
@@ -1893,7 +1885,7 @@ def do_setup(host_os,
                                   core_root)
 
         if not success:
-            print("Error GenerateLayout has failed.")
+            print("Error: GenerateLayout failed.")
             sys.exit(1)
 
         if unprocessed_args.generate_layout_only:
@@ -1932,7 +1924,7 @@ def do_setup(host_os,
         else:
             build_test_wrappers(host_os, arch, build_type, coreclr_repo_location, test_location)
 
-    run_tests(host_os, 
+    return run_tests(host_os, 
               arch,
               build_type,
               core_root, 
@@ -1992,6 +1984,8 @@ def main(args):
         print_summary(tests)
         create_repro(host_os, arch, build_type, env, core_root, coreclr_repo_location, tests)
 
+    return ret_code
+
 ################################################################################
 # __main__
 ################################################################################
diff --git a/tests/tests.targets b/tests/tests.targets
index 6c3afcd7f1..f6b62a79c8 100644
--- a/tests/tests.targets
+++ b/tests/tests.targets
@@ -48,6 +48,9 @@
       <XunitArgs>$(XunitArgs) @(IncludeTraitsItems->'-trait %(Identity)', ' ')</XunitArgs>
       <XunitArgs>$(XunitArgs) @(ExcludeTraitsItems->'-notrait %(Identity)', ' ')</XunitArgs>
 
+      <!-- Color output doesn't work well when capturing the output in the CI system -->
+      <XunitArgs>$(XunitArgs) -nocolor</XunitArgs>
+
       <CorerunExecutable Condition="'$(RunningOnUnix)' == 'true'">$(CORE_ROOT)\corerun</CorerunExecutable>
       <CorerunExecutable Condition="'$(RunningOnUnix)' != 'true'">$(CORE_ROOT)\corerun.exe</CorerunExecutable>
     </PropertyGroup>