diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/issues.targets | 80 | ||||
| -rw-r--r-- | tests/runtest.cmd | 28 | ||||
| -rwxr-xr-x | tests/runtest.py | 105 | ||||
| -rwxr-xr-x | tests/runtest.sh | 14 |
4 files changed, 182 insertions, 45 deletions
diff --git a/tests/issues.targets b/tests/issues.targets index d49472f9e9..1c1ca62504 100644 --- a/tests/issues.targets +++ b/tests/issues.targets @@ -387,9 +387,48 @@ <ExcludeList Include="$(XunitTestBinBase)/Loader/classloader/DefaultInterfaceMethods/diamondshape/diamondshape_r/*"> <Issue>9565</Issue> </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/GC/API/NoGCRegion/NoGC/*"> + <Issue>needs triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/baseservices/varargs/varargsupport_r/*"> + <Issue>Varargs supported on this platform</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/baseservices/varargs/varargsupport/*"> + <Issue>Varargs supported on this platform</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/Directed/arglist/vararg/*"> + <Issue>Needs triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/tracing/runtimeeventsource/runtimeeventsource/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/tracing/tracevalidation/inducedgc/inducedgc/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/opt/rngchk/RngchkStress3/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i53/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i13/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i03/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i33/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i63/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i73/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> </ItemGroup> - <!-- Windows arm32 specific excludes --> + <!-- arm32 All OS specific excludes --> <ItemGroup Condition="'$(XunitTestBinBase)' != '' and '$(BuildArch)' == 'arm'"> <ExcludeList Include="$(XunitTestBinBase)/JIT/Methodical/tailcall_v4/hijacking/*"> <Issue>6217</Issue> @@ -397,6 +436,45 @@ <ExcludeList Include="$(XunitTestBinBase)/JIT/opt/Tailcall/TailcallVerifyWithPrefix/*"> <Issue>2420</Issue> </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/baseservices/varargs/varargsupport_r/*"> + <Issue>Varargs supported on this platform</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/baseservices/varargs/varargsupport/*"> + <Issue>Varargs supported on this platform</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/Directed/arglist/vararg/*"> + <Issue>Needs triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/Interop/IJW/ManagedCallingNative/ManagedCallingNative/*"> + <Issue>Needs triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/Interop/IJW/NativeCallingManaged/NativeCallingManaged/*"> + <Issue>Needs triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/baseservices/exceptions/WindowsEventLog/WindowsEventLog/*"> + <Issue>Needs triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i53/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i13/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i03/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i33/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i63/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i73/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> + <ExcludeList Include="$(XunitTestBinBase)/JIT/jit64/mcc/interop/mcc_i83/*"> + <Issue>Needs Triage</Issue> + </ExcludeList> </ItemGroup> <!-- The following are x64 Unix failures. --> diff --git a/tests/runtest.cmd b/tests/runtest.cmd index 45e722f1e7..c0991ce1ee 100644 --- a/tests/runtest.cmd +++ b/tests/runtest.cmd @@ -7,14 +7,18 @@ set "__MsgPrefix=RUNTEST: " set __ThisScriptDir="%~dp0" -call "%__ThisScriptDir%"\..\setup_vs_tools.cmd -if NOT '%ERRORLEVEL%' == '0' exit /b 1 - -if defined VS150COMNTOOLS ( - set __VSVersion=vs2017 -) else ( - set __VSVersion=vs2015 -) +if /I not "%PROCESSOR_ARCHITECTURE%"=="arm64" ( + if /I not "%PROCESSOR_ARCHITECTURE%"=="arm" ( + call "%__ThisScriptDir%"\..\setup_vs_tools.cmd + if NOT '%ERRORLEVEL%' == '0' exit /b 1 + + if defined VS150COMNTOOLS ( + set __VSVersion=vs2017 + ) else ( + set __VSVersion=vs2015 + ) + ) +) :: Set the default arguments set __BuildArch=x64 @@ -43,6 +47,7 @@ set __CoreFXTests= set __CoreFXTestsRunAllAvailable= set __SkipGenerateLayout= set __BuildXUnitWrappers= +set __PrintLastResultsOnly= :Arg_Loop if "%1" == "" goto ArgsDone @@ -82,6 +87,7 @@ if /i "%1" == "ilasmroundtrip" (set __IlasmRoundTrip=1&shift&goto Arg_Loo if /i "%1" == "GenerateLayoutOnly" (set __GenerateLayoutOnly=1&shift&goto Arg_Loop) if /i "%1" == "skipgeneratelayout" (set __SkipGenerateLayout=1&shift&goto Arg_Loop) if /i "%1" == "buildxunitwrappers" (set __BuildXunitWrappers=1&shift&goto Arg_Loop) +if /i "%1" == "printlastresultsonly" (set __PrintLastResultsOnly=1&shift&goto Arg_Loop) if /i "%1" == "PerfTests" (set __PerfTests=true&shift&goto Arg_Loop) if /i "%1" == "CoreFXTests" (set __CoreFXTests=true&shift&goto Arg_Loop) if /i "%1" == "CoreFXTestsAll" (set __CoreFXTests=true&set __CoreFXTestsRunAllAvailable=true&shift&goto Arg_Loop) @@ -198,6 +204,10 @@ if defined __DoCrossgen ( set __RuntestPyArgs=%__RuntestPyArgs% --precompile_core_root ) +if defined __PrintLastResultsOnly ( + set __RuntestPyArgs=%__RuntestPyArgs% --analyze_results_only +) + REM __ProjectDir is poorly named, it is actually <projectDir>/tests set NEXTCMD=python "%__ProjectDir%\runtest.py" %__RuntestPyArgs% echo !NEXTCMD! @@ -691,6 +701,7 @@ echo VSVersion ^<vs_version^> - VS2015 or VS2017 ^(default: VS2017^). echo TestEnv ^<test_env_script^> - Run a custom script before every test to set custom test environment settings. echo AgainstPackages - This indicates that we are running tests that were built against packages. echo GenerateLayoutOnly - If specified will not run the tests and will only create the Runtime Dependency Layout +echo skipgeneratelayout - Do not generate the core root. Used for cross target testing. echo sequential - Run tests sequentially (no parallelism). echo crossgen - Precompile ^(crossgen^) the managed assemblies in CORE_ROOT before running the tests. echo crossgenaltjit ^<altjit^> - Precompile ^(crossgen^) the managed assemblies in CORE_ROOT before running the tests, using the given altjit. @@ -716,6 +727,7 @@ echo timeout ^<n^> - Sets the per-test timeout in milliseconds ^(d echo Note: some options override this ^(gcstresslevel, longgc, gcsimulator^). echo msbuildargs ^<args...^> - Pass all subsequent args directly to msbuild invocations. echo ^<CORE_ROOT^> - Path to the runtime to test ^(if specified^). +echo printlastresultsonly - Print the last test results without running tests. echo. echo Note that arguments are not case-sensitive. echo. diff --git a/tests/runtest.py b/tests/runtest.py index 0845183690..a90a0eab99 100755 --- a/tests/runtest.py +++ b/tests/runtest.py @@ -560,12 +560,10 @@ def call_msbuild(coreclr_repo_location, """ global g_verbose - common_msbuild_arguments = ["/nologo", "/nodeReuse:false", "/p:Platform=%s" % arch] + common_msbuild_arguments = [] if sequential: - common_msbuild_arguments += ["/p:ParallelRun=false"] - else: - common_msbuild_arguments += ["/maxcpucount"] + common_msbuild_arguments += ["/p:ParallelRun=none"] logs_dir = os.path.join(coreclr_repo_location, "bin", "Logs") if not os.path.isdir(logs_dir): @@ -577,6 +575,8 @@ def call_msbuild(coreclr_repo_location, "/p:Runtests=true", "/clp:showcommandline"] + command += common_msbuild_arguments + if is_illink: command += ["/p:RunTestsViaIllink=true"] @@ -671,16 +671,28 @@ def correct_line_endings(host_os, test_location, root=True): for item in os.listdir(test_location): correct_line_endings(host_os, os.path.join(test_location, item), False) elif test_location.endswith(extension): - content = None - with open(test_location) as file_handle: - content = file_handle.read() - - assert content != None - subbed_content = content.replace(incorrect_line_ending, correct_line_ending) + if sys.version_info < (3,0): + + content = None + with open(test_location) as file_handle: + content = file_handle.read() + + assert content != None + subbed_content = content.replace(incorrect_line_ending, correct_line_ending) + + if content != subbed_content: + with open(test_location, 'w') as file_handle: + file_handle.write(subbed_content) - if content != subbed_content: + else: + # Python3 will correct line endings automatically. + + content = None + with open(test_location) as file_handle: + content = file_handle.read() + with open(test_location, 'w') as file_handle: - file_handle.write(subbed_content) + file_handle.write(content) def run_tests(host_os, arch, @@ -866,7 +878,7 @@ def setup_args(args): if test_location[-1] == os.path.sep: test_location = test_location[:-1] - if test_location != default_test_location and os.path.isdir(default_test_location): + if test_location.lower() != default_test_location.lower() and os.path.isdir(default_test_location): # Remove the existing directory if there is one. shutil.rmtree(default_test_location) @@ -916,7 +928,20 @@ def setup_args(args): else: print("Core_Root: %s" % core_root) - if host_os != "Windows_NT": + is_same_os = False + is_same_arch = False + is_same_build_type = False + + # We will write out build information into the test directory. This is used + # by runtest.py to determine whether we need to rebuild the test wrappers. + if os.path.isfile(os.path.join(test_location, "build_info.json")): + with open(os.path.join(test_location, "build_info.json")) as file_handle: + build_info = json.load(file_handle) + is_same_os = build_info["build_os"] == host_os + is_same_arch = build_info["build_arch"] == arch + is_same_build_type = build_info["build_type"] == build_type + + if host_os != "Windows_NT" and not (is_same_os and is_same_arch and is_same_build_type): if test_native_bin_location is None: print("Using default location for test_native_bin_location.") test_native_bin_location = os.path.join(os.path.join(coreclr_repo_location, "bin", "obj", "%s.%s.%s" % (host_os, arch, build_type), "tests")) @@ -1804,6 +1829,16 @@ def print_summary(tests): test_output = test_output.replace("\\n", "\n") print(test_output) + test_output = test_output.replace("/r", "\r") + test_output = test_output.replace("/n", "\n") + unicode_output = None + if sys.version_info < (3,0): + # Handle unicode characters in output in python2.* + unicode_output = unicode(test_output, "utf-8") + else: + unicode_output = test_output + + print(unicode_output) print("") print("") @@ -1898,31 +1933,33 @@ def do_setup(host_os, if gc_stress_c: setup_coredis_tools(coreclr_repo_location, host_os, arch, core_root) + build_info = None + is_same_os = None + is_same_arch = None + is_same_build_type = None + + # We will write out build information into the test directory. This is used + # by runtest.py to determine whether we need to rebuild the test wrappers. + if os.path.isfile(os.path.join(test_location, "build_info.json")): + with open(os.path.join(test_location, "build_info.json")) as file_handle: + build_info = json.load(file_handle) + is_same_os = build_info["build_os"] == host_os + is_same_arch = build_info["build_arch"] == arch + is_same_build_type = build_info["build_type"] == build_type + # Copy all the native libs to core_root - if host_os != "Windows_NT": + if host_os != "Windows_NT" and not (is_same_os and is_same_arch and is_same_build_type): copy_native_test_bin_to_core_root(host_os, os.path.join(test_native_bin_location, "src"), core_root) - correct_line_endings(host_os, test_location) + # Line ending only need to be corrected if this is a cross build. + correct_line_endings(host_os, test_location) if unprocessed_args.build_test_wrappers: build_test_wrappers(host_os, arch, build_type, coreclr_repo_location, test_location) - else: - # We will write out build information into the test directory. This is used - # by runtest.py to determine whether we need to rebuild the test wrappers. - if os.path.isfile(os.path.join(test_location, "build_info.json")): - build_info = None - with open(os.path.join(test_location, "build_info.json")) as file_handle: - build_info = json.load(file_handle) - - is_same_os = build_info["build_os"] == host_os - is_same_arch = build_info["build_arch"] == arch - is_same_build_type = build_info["build_type"] == build_type - - # We will force a build of the test wrappers if they were cross built - if not (is_same_os and is_same_arch and is_same_build_type): - build_test_wrappers(host_os, arch, build_type, coreclr_repo_location, test_location) - else: - build_test_wrappers(host_os, arch, build_type, coreclr_repo_location, test_location) + elif build_info is None: + build_test_wrappers(host_os, arch, build_type, coreclr_repo_location, test_location) + elif not (is_same_os and is_same_arch and is_same_build_type): + build_test_wrappers(host_os, arch, build_type, coreclr_repo_location, test_location) return run_tests(host_os, arch, @@ -1950,6 +1987,8 @@ def main(args): host_os, arch, build_type, coreclr_repo_location, product_location, core_root, test_location, test_native_bin_location = setup_args(args) + ret_code = 0 + env = get_environment(test_env=args.test_env) if not args.analyze_results_only: if args.test_env is not None: diff --git a/tests/runtest.sh b/tests/runtest.sh index 0917094710..e8bd197251 100755 --- a/tests/runtest.sh +++ b/tests/runtest.sh @@ -223,6 +223,7 @@ buildXUnitWrappers= printLastResultsOnly= generateLayoutOnly= generateLayout= +runSequential= for i in "$@" do @@ -340,7 +341,7 @@ do export testHostDir=${i#*=} ;; --sequential) - ((maxProcesses = 1)) + runSequential=1 ;; --useServerGC) ((serverGC = 1)) @@ -535,7 +536,7 @@ if [ ! -z "$generateLayout" ]; then runtestPyArguments+=("--generate_layout") fi -if [ ! -z "$sequential" ]; then +if [ ! "$runSequential" -eq 0 ]; then echo "Run tests sequentially." runtestPyArguments+=("--sequential") fi @@ -552,7 +553,14 @@ if (($doCrossgen!=0)); then runtestPyArguments+=("--precompile_core_root") fi + +# Default to python3 if it is installed +__Python=python + if command -v python3 &>/dev/null; then + __Python=python3 +fi + # Run the tests using cross platform runtest.py echo "python ${scriptPath}/runtest.py ${runtestPyArguments[@]}" -python "${scriptPath}/runtest.py" "${runtestPyArguments[@]}" +$__Python "${scriptPath}/runtest.py" "${runtestPyArguments[@]}" exit "$?" |