Runtest.py on Windows Arm(64) (#20227)

* Initial infrastructure work to get arm(64) runtest.cmd working

* Add excludes and improve runtest.py

1) Adds a build_test_wrappers only to build-test.sh.
2) Adds arm64 windows excludes
3) Adds printlastresults to runtest.cmd
4) corrects runsequential in runtest.sh
5) Corrects SequentialRun in runtest.py
6) Minor improvements to printing test results and copying native test binaries

* Address pr feedback

* Add to issues targets for arm64

* Working Pri 0 testing.

* Remove unecessary common msbuild arguments

* add pri1 excludes

* Remove common msbuild args

* Fix silly python3 issue

4 files changed, 182 insertions, 45 deletions

diff --git a/tests/issues.targets b/tests/issues.targets
index d49472f9e9..1c1ca62504 100644
--- a/tests/issues.targets
+++ b/tests/issues.targets
@@ -387,9 +387,48 @@
         <ExcludeList Include="$(XunitTestBinBase)/Loader/classloader/DefaultInterfaceMethods/diamondshape/diamondshape_r/*">
             <Issue>9565</Issue>
         </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/GC/API/NoGCRegion/NoGC/*">
+            <Issue>needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/baseservices/varargs/varargsupport_r/*">
+            <Issue>Varargs supported on this platform</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/baseservices/varargs/varargsupport/*">
+            <Issue>Varargs supported on this platform</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/Directed/arglist/vararg/*">
+            <Issue>Needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/tracing/runtimeeventsource/runtimeeventsource/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/tracing/tracevalidation/inducedgc/inducedgc/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/opt/rngchk/RngchkStress3/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i53/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i13/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i03/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i33/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i63/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i73/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
     </ItemGroup>
 
-    <!-- Windows arm32 specific excludes -->
+    <!-- arm32 All OS specific excludes -->
     <ItemGroup Condition="'$(XunitTestBinBase)' != '' and '$(BuildArch)' == 'arm'">
         <ExcludeList Include="$(XunitTestBinBase)/JIT/Methodical/tailcall_v4/hijacking/*">
             <Issue>6217</Issue>
@@ -397,6 +436,45 @@
         <ExcludeList Include="$(XunitTestBinBase)/JIT/opt/Tailcall/TailcallVerifyWithPrefix/*">
             <Issue>2420</Issue>
         </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/baseservices/varargs/varargsupport_r/*">
+            <Issue>Varargs supported on this platform</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/baseservices/varargs/varargsupport/*">
+            <Issue>Varargs supported on this platform</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/Directed/arglist/vararg/*">
+            <Issue>Needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/Interop/IJW/ManagedCallingNative/ManagedCallingNative/*">
+            <Issue>Needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/Interop/IJW/NativeCallingManaged/NativeCallingManaged/*">
+            <Issue>Needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/baseservices/exceptions/WindowsEventLog/WindowsEventLog/*">
+            <Issue>Needs triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i53/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i13/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i03/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i33/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i63/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i73/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i83/*">
+            <Issue>Needs Triage</Issue>
+        </ExcludeList>
     </ItemGroup>
 
     <!-- The following are x64 Unix failures. -->
diff --git a/tests/runtest.cmd b/tests/runtest.cmd
index 45e722f1e7..c0991ce1ee 100644
--- a/tests/runtest.cmd
+++ b/tests/runtest.cmd
@@ -7,14 +7,18 @@ set "__MsgPrefix=RUNTEST: "
 
 set __ThisScriptDir="%~dp0"
 
-call "%__ThisScriptDir%"\..\setup_vs_tools.cmd
-if NOT '%ERRORLEVEL%' == '0' exit /b 1
-
-if defined VS150COMNTOOLS (
-    set __VSVersion=vs2017
-) else (
-    set __VSVersion=vs2015
-)
+if /I not "%PROCESSOR_ARCHITECTURE%"=="arm64" (
+    if /I not "%PROCESSOR_ARCHITECTURE%"=="arm" (
+        call "%__ThisScriptDir%"\..\setup_vs_tools.cmd
+        if NOT '%ERRORLEVEL%' == '0' exit /b 1
+
+        if defined VS150COMNTOOLS (
+            set __VSVersion=vs2017
+        ) else (
+            set __VSVersion=vs2015
+        )
+    )
+)   
 
 :: Set the default arguments
 set __BuildArch=x64
@@ -43,6 +47,7 @@ set __CoreFXTests=
 set __CoreFXTestsRunAllAvailable=
 set __SkipGenerateLayout=
 set __BuildXUnitWrappers=
+set __PrintLastResultsOnly=
 
 :Arg_Loop
 if "%1" == "" goto ArgsDone
@@ -82,6 +87,7 @@ if /i "%1" == "ilasmroundtrip"        (set __IlasmRoundTrip=1&shift&goto Arg_Loo
 if /i "%1" == "GenerateLayoutOnly"    (set __GenerateLayoutOnly=1&shift&goto Arg_Loop)
 if /i "%1" == "skipgeneratelayout"    (set __SkipGenerateLayout=1&shift&goto Arg_Loop)
 if /i "%1" == "buildxunitwrappers"    (set __BuildXunitWrappers=1&shift&goto Arg_Loop)
+if /i "%1" == "printlastresultsonly"  (set __PrintLastResultsOnly=1&shift&goto Arg_Loop)
 if /i "%1" == "PerfTests"             (set __PerfTests=true&shift&goto Arg_Loop)
 if /i "%1" == "CoreFXTests"           (set __CoreFXTests=true&shift&goto Arg_Loop)
 if /i "%1" == "CoreFXTestsAll"        (set __CoreFXTests=true&set __CoreFXTestsRunAllAvailable=true&shift&goto Arg_Loop)
@@ -198,6 +204,10 @@ if defined __DoCrossgen (
     set __RuntestPyArgs=%__RuntestPyArgs% --precompile_core_root
 )
 
+if defined __PrintLastResultsOnly (
+    set __RuntestPyArgs=%__RuntestPyArgs% --analyze_results_only
+)
+
 REM __ProjectDir is poorly named, it is actually <projectDir>/tests
 set NEXTCMD=python "%__ProjectDir%\runtest.py" %__RuntestPyArgs%
 echo !NEXTCMD!
@@ -691,6 +701,7 @@ echo VSVersion ^<vs_version^>    - VS2015 or VS2017 ^(default: VS2017^).
 echo TestEnv ^<test_env_script^> - Run a custom script before every test to set custom test environment settings.
 echo AgainstPackages           - This indicates that we are running tests that were built against packages.
 echo GenerateLayoutOnly        - If specified will not run the tests and will only create the Runtime Dependency Layout
+echo skipgeneratelayout        - Do not generate the core root. Used for cross target testing.
 echo sequential                - Run tests sequentially (no parallelism).
 echo crossgen                  - Precompile ^(crossgen^) the managed assemblies in CORE_ROOT before running the tests.
 echo crossgenaltjit ^<altjit^>   - Precompile ^(crossgen^) the managed assemblies in CORE_ROOT before running the tests, using the given altjit.
@@ -716,6 +727,7 @@ echo timeout ^<n^>               - Sets the per-test timeout in milliseconds ^(d
 echo                             Note: some options override this ^(gcstresslevel, longgc, gcsimulator^).
 echo msbuildargs ^<args...^>     - Pass all subsequent args directly to msbuild invocations.
 echo ^<CORE_ROOT^>               - Path to the runtime to test ^(if specified^).
+echo printlastresultsonly        - Print the last test results without running tests.
 echo.
 echo Note that arguments are not case-sensitive.
 echo.
diff --git a/tests/runtest.py b/tests/runtest.py
index 0845183690..a90a0eab99 100755
--- a/tests/runtest.py
+++ b/tests/runtest.py
@@ -560,12 +560,10 @@ def call_msbuild(coreclr_repo_location,
     """
     global g_verbose
 
-    common_msbuild_arguments = ["/nologo", "/nodeReuse:false", "/p:Platform=%s" % arch]
+    common_msbuild_arguments = []
 
     if sequential:
-        common_msbuild_arguments += ["/p:ParallelRun=false"]
-    else:
-        common_msbuild_arguments += ["/maxcpucount"]
+        common_msbuild_arguments += ["/p:ParallelRun=none"]
 
     logs_dir = os.path.join(coreclr_repo_location, "bin", "Logs")
     if not os.path.isdir(logs_dir):
@@ -577,6 +575,8 @@ def call_msbuild(coreclr_repo_location,
                  "/p:Runtests=true",
                  "/clp:showcommandline"]
 
+    command += common_msbuild_arguments
+
     if is_illink:
         command += ["/p:RunTestsViaIllink=true"]
 
@@ -671,16 +671,28 @@ def correct_line_endings(host_os, test_location, root=True):
         for item in os.listdir(test_location):
             correct_line_endings(host_os, os.path.join(test_location, item), False)
     elif test_location.endswith(extension):
-        content = None
-        with open(test_location) as file_handle:
-            content = file_handle.read()
-        
-        assert content != None
-        subbed_content = content.replace(incorrect_line_ending, correct_line_ending)
+        if sys.version_info < (3,0):
+
+            content = None
+            with open(test_location) as file_handle:
+                content = file_handle.read()
+     
+            assert content != None
+            subbed_content = content.replace(incorrect_line_ending, correct_line_ending)
+
+            if content != subbed_content:
+                with open(test_location, 'w') as file_handle:
+                    file_handle.write(subbed_content)
 
-        if content != subbed_content:
+        else:
+            # Python3 will correct line endings automatically.
+ 
+            content = None
+            with open(test_location) as file_handle:
+                content = file_handle.read()
+     
             with open(test_location, 'w') as file_handle:
-                file_handle.write(subbed_content)
+                file_handle.write(content)
 
 def run_tests(host_os,
               arch,
@@ -866,7 +878,7 @@ def setup_args(args):
         if test_location[-1] == os.path.sep:
             test_location = test_location[:-1]
 
-        if test_location != default_test_location and os.path.isdir(default_test_location):
+        if test_location.lower() != default_test_location.lower() and os.path.isdir(default_test_location):
             # Remove the existing directory if there is one.
             shutil.rmtree(default_test_location)
 
@@ -916,7 +928,20 @@ def setup_args(args):
     else:
         print("Core_Root: %s" % core_root)
 
-    if host_os != "Windows_NT":
+    is_same_os = False
+    is_same_arch = False
+    is_same_build_type = False
+
+    # We will write out build information into the test directory. This is used
+    # by runtest.py to determine whether we need to rebuild the test wrappers.
+    if os.path.isfile(os.path.join(test_location, "build_info.json")):
+        with open(os.path.join(test_location, "build_info.json")) as file_handle:
+            build_info = json.load(file_handle)
+        is_same_os = build_info["build_os"] == host_os
+        is_same_arch = build_info["build_arch"] == arch
+        is_same_build_type = build_info["build_type"] == build_type
+
+    if host_os != "Windows_NT" and not (is_same_os and is_same_arch and is_same_build_type):
         if test_native_bin_location is None:
             print("Using default location for test_native_bin_location.")
             test_native_bin_location = os.path.join(os.path.join(coreclr_repo_location, "bin", "obj", "%s.%s.%s" % (host_os, arch, build_type), "tests"))
@@ -1804,6 +1829,16 @@ def print_summary(tests):
             test_output = test_output.replace("\\n", "\n")
 
             print(test_output)
+            test_output = test_output.replace("/r", "\r")
+            test_output = test_output.replace("/n", "\n")
+            unicode_output = None
+            if sys.version_info < (3,0):
+                # Handle unicode characters in output in python2.*
+                unicode_output = unicode(test_output, "utf-8")
+            else:
+                unicode_output = test_output
+
+            print(unicode_output)
             print("")
 
         print("")
@@ -1898,31 +1933,33 @@ def do_setup(host_os,
     if gc_stress_c:
         setup_coredis_tools(coreclr_repo_location, host_os, arch, core_root)
     
+    build_info = None
+    is_same_os = None
+    is_same_arch = None
+    is_same_build_type = None
+
+    # We will write out build information into the test directory. This is used
+    # by runtest.py to determine whether we need to rebuild the test wrappers.
+    if os.path.isfile(os.path.join(test_location, "build_info.json")):
+        with open(os.path.join(test_location, "build_info.json")) as file_handle:
+            build_info = json.load(file_handle)
+        is_same_os = build_info["build_os"] == host_os
+        is_same_arch = build_info["build_arch"] == arch
+        is_same_build_type = build_info["build_type"] == build_type
+
     # Copy all the native libs to core_root
-    if host_os != "Windows_NT":
+    if host_os != "Windows_NT"  and not (is_same_os and is_same_arch and is_same_build_type):
         copy_native_test_bin_to_core_root(host_os, os.path.join(test_native_bin_location, "src"), core_root)
 
-    correct_line_endings(host_os, test_location)
+        # Line ending only need to be corrected if this is a cross build.
+        correct_line_endings(host_os, test_location)
 
     if unprocessed_args.build_test_wrappers:
         build_test_wrappers(host_os, arch, build_type, coreclr_repo_location, test_location)
-    else:
-        # We will write out build information into the test directory. This is used
-        # by runtest.py to determine whether we need to rebuild the test wrappers.
-        if os.path.isfile(os.path.join(test_location, "build_info.json")):
-            build_info = None
-            with open(os.path.join(test_location, "build_info.json")) as file_handle:
-                build_info = json.load(file_handle)
-
-            is_same_os = build_info["build_os"] == host_os
-            is_same_arch = build_info["build_arch"] == arch
-            is_same_build_type = build_info["build_type"] == build_type
-
-            # We will force a build of the test wrappers if they were cross built
-            if not (is_same_os and is_same_arch and is_same_build_type):
-                build_test_wrappers(host_os, arch, build_type, coreclr_repo_location, test_location)
-        else:
-            build_test_wrappers(host_os, arch, build_type, coreclr_repo_location, test_location)
+    elif build_info is None:
+        build_test_wrappers(host_os, arch, build_type, coreclr_repo_location, test_location)
+    elif not (is_same_os and is_same_arch and is_same_build_type):
+        build_test_wrappers(host_os, arch, build_type, coreclr_repo_location, test_location)
 
     return run_tests(host_os, 
               arch,
@@ -1950,6 +1987,8 @@ def main(args):
 
     host_os, arch, build_type, coreclr_repo_location, product_location, core_root, test_location, test_native_bin_location = setup_args(args)
 
+    ret_code = 0
+
     env = get_environment(test_env=args.test_env)
     if not args.analyze_results_only:
         if args.test_env is not None:
diff --git a/tests/runtest.sh b/tests/runtest.sh
index 0917094710..e8bd197251 100755
--- a/tests/runtest.sh
+++ b/tests/runtest.sh
@@ -223,6 +223,7 @@ buildXUnitWrappers=
 printLastResultsOnly=
 generateLayoutOnly=
 generateLayout=
+runSequential=
 
 for i in "$@"
 do
@@ -340,7 +341,7 @@ do
             export testHostDir=${i#*=}
             ;;
         --sequential)
-            ((maxProcesses = 1))
+            runSequential=1
             ;;
         --useServerGC)
             ((serverGC = 1))
@@ -535,7 +536,7 @@ if [ ! -z "$generateLayout" ]; then
     runtestPyArguments+=("--generate_layout")
 fi
 
-if [ ! -z "$sequential" ]; then
+if [ ! "$runSequential" -eq 0 ]; then
     echo "Run tests sequentially."
     runtestPyArguments+=("--sequential")
 fi
@@ -552,7 +553,14 @@ if (($doCrossgen!=0)); then
     runtestPyArguments+=("--precompile_core_root")
 fi
 
+
+# Default to python3 if it is installed
+__Python=python
+ if command -v python3 &>/dev/null; then
+    __Python=python3
+fi
+
 # Run the tests using cross platform runtest.py
 echo "python ${scriptPath}/runtest.py ${runtestPyArguments[@]}"
-python "${scriptPath}/runtest.py" "${runtestPyArguments[@]}"
+$__Python "${scriptPath}/runtest.py" "${runtestPyArguments[@]}"
 exit "$?"