diff options
| -rwxr-xr-x | netci.groovy | 2251 | ||||
| -rwxr-xr-x | tests/runtest.sh | 61 | ||||
| -rwxr-xr-x | tests/runtesttilstable.sh | 158 | ||||
| -rwxr-xr-x | tests/scripts/arm32_ci_script.sh | 32 | ||||
| -rw-r--r-- | tests/testsFailing.arm.txt | 1 |
5 files changed, 1515 insertions, 988 deletions
diff --git a/netci.groovy b/netci.groovy index 886b4624f7..7d7c686519 100755 --- a/netci.groovy +++ b/netci.groovy @@ -56,7 +56,13 @@ class Constants { 'Fedora24', 'Tizen'] - def static crossList = ['Ubuntu', 'OSX10.12', 'CentOS7.1', 'RHEL7.2', 'Debian8.4', 'Windows_NT'] + def static crossList = [ + 'Ubuntu', + 'Debian8.4', + 'OSX10.12', + 'Windows_NT', + 'CentOS7.1', + 'RHEL7.2'] // This is a set of JIT stress modes combined with the set of variables that // need to be set to actually enable that stress mode. The key of the map is the stress mode and @@ -169,7 +175,7 @@ class Constants { 'Checked' ] ], - 'Windows_NT_buildOnly': [ + 'Windows_NT_BuildOnly': [ 'x64': [ 'Checked', 'Release' @@ -178,6 +184,9 @@ class Constants { 'Checked', 'Release' ], + 'arm': [ + 'Checked' + ], ], 'Ubuntu': [ 'x64': [ @@ -185,6 +194,9 @@ class Constants { ], 'arm64': [ 'Debug' + ], + 'arm': [ + 'Checked' ] ], 'CentOS7.1': [ @@ -199,7 +211,7 @@ class Constants { ] ], 'Tizen': [ - 'arm': [ + 'armem': [ 'Checked' ] ], @@ -317,17 +329,76 @@ class Constants { ] def static validLinuxArm64Scenarios = [ - 'normal', - 'r2r', - 'innerloop', - 'gcstress0x3', - 'gcstress0xc' + 'innerloop', + 'normal', + 'r2r', + 'gcstress0x3', + 'gcstress0xc' + ] + + // Note: no GCStress-related scenario is enabled currently. + def static validLinuxArmScenarios = [ + 'innerloop', + 'normal', + 'r2r', + 'r2r_jitstress1', + 'r2r_jitstress2', + 'r2r_jitstressregs1', + 'r2r_jitstressregs2', + 'r2r_jitstressregs3', + 'r2r_jitstressregs4', + 'r2r_jitstressregs8', + 'r2r_jitstressregs0x10', + 'r2r_jitstressregs0x80', + 'r2r_jitstressregs0x1000', + 'r2r_jitminopts', + 'r2r_jitforcerelocs', + // 'r2r_gcstress15', + 'minopts', + 'forcerelocs', + 'jitstress1', + 'jitstress2', + 'jitstressregs1', + 'jitstressregs2', + 'jitstressregs3', + 'jitstressregs4', + 'jitstressregs8', + 'jitstressregs0x10', + 'jitstressregs0x80', + 'jitstressregs0x1000', + 'jitstress2_jitstressregs1', + 'jitstress2_jitstressregs2', + 'jitstress2_jitstressregs3', + 'jitstress2_jitstressregs4', + 'jitstress2_jitstressregs8', + 'jitstress2_jitstressregs0x10', + 'jitstress2_jitstressregs0x80', + 'jitstress2_jitstressregs0x1000', + 'tailcallstress' + // 'gcstress0x3', + // 'gcstress0xc', + // 'zapdisable', + // 'heapverify1', + // 'gcstress0xc_zapdisable', + // 'gcstress0xc_zapdisable_jitstress2', + // 'gcstress0xc_zapdisable_heapverify1', + // 'gcstress0xc_jitstress1', + // 'gcstress0xc_jitstress2', + // 'gcstress0xc_minopts_heapverify1' ] def static configurationList = ['Debug', 'Checked', 'Release'] // This is the set of architectures - def static architectureList = ['arm', 'armlb', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86'] + // Some of these are pseudo-architectures: + // armlb -- same as arm, but use the LEGACY_BACKEND JIT + // armem -- ARM builds/runs using an emulator. Used for Ubuntu/Ubuntu16.04/Tizen runs. + // x86_arm_altjit -- ARM runs on x86 using the ARM altjit + // x64_arm64_altjit -- ARM64 runs on x64 using the ARM64 altjit + def static architectureList = ['arm', 'armlb', 'armem', 'x86_arm_altjit', 'x64_arm64_altjit', 'arm64', 'x64', 'x86'] + + // This set of architectures that cross build on Windows and run on Windows ARM64 hardware. + def static armWindowsCrossArchitectureList = ['arm', 'armlb', 'arm64'] } // ************************************************************** @@ -465,7 +536,7 @@ def static setMachineAffinity(def job, def os, def architecture, def options = n assert os instanceof String assert architecture instanceof String - def armArches = ['arm', 'armlb', 'arm64'] + def armArches = ['arm', 'armlb', 'armem', 'arm64'] def supportedArmLinuxOs = ['Ubuntu', 'Ubuntu16.04', 'Tizen'] if (!(architecture in armArches)) { @@ -482,9 +553,9 @@ def static setMachineAffinity(def job, def os, def architecture, def options = n // Windows_NT // // Arm32 (Build) -> latest-arm64 - // |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == true + // |-> os == "Windows_NT" && (architecture == "arm" || architecture == "armlb") && options['use_arm64_build_machine'] == true // Arm32 (Test) -> arm64-windows_nt - // |-> os == "Windows_NT" && architecture == "arm" || architecture == "armlb" && options['use_arm64_build_machine'] == false + // |-> os == "Windows_NT" && (architecture == "arm" || architecture == "armlb") && options['use_arm64_build_machine'] == false // // Arm64 (Build) -> latest-arm64 // |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true @@ -493,10 +564,15 @@ def static setMachineAffinity(def job, def os, def architecture, def options = n // // Ubuntu // - // Arm32 (Build) -> arm-cross-latest - // |-> os in supportedArmLinuxOs && architecture == "arm" || architecture == "armlb" - // Arm32 (Test) -> NYI Arch not supported - // |-> + // Arm32 emulator (Build, Test) -> arm-cross-latest + // |-> os in supportedArmLinuxOs && (architecture == "armem") + // + // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware) + // |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true + // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto + // |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true + // Arm32 hardware (Test) -> ubuntu.1404.arm32.open + // |-> os == "Ubuntu" && (architecture == "arm") // // Arm64 (Build) -> arm64-cross-latest // |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_only'] == true @@ -508,7 +584,7 @@ def static setMachineAffinity(def job, def os, def architecture, def options = n // This has to be a arm arch assert architecture in armArches if (os == "Windows_NT") { - // Arm(64) Windows jobs share the same machines for now + // arm32/arm64 Windows jobs share the same machines for now def isBuild = options['use_arm64_build_machine'] == true if (isBuild == true) { @@ -520,24 +596,90 @@ def static setMachineAffinity(def job, def os, def architecture, def options = n assert os != 'Windows_NT' assert os in supportedArmLinuxOs - if (architecture == 'arm' || architecture == 'armlb') { - Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest') - } else { - // Arm64 Linux - if (options['is_build_only'] == true) { + if (architecture == 'arm64') { + if ((options != null) && (options['is_build_only'] == true)) { + // Arm64 Linux build machine Utilities.setMachineAffinity(job, os, 'arm64-cross-latest') } else { - // Arm64 Test Machines - if (options['large_pages'] == false) { - Utilities.setMachineAffinity(job, os, 'arm64-small-page-size') - } else { + // Arm64 Linux test machines + if ((options != null) && (options['large_pages'] == true)) { Utilities.setMachineAffinity(job, os, 'arm64-huge-page-size') + } else { + Utilities.setMachineAffinity(job, os, 'arm64-small-page-size') + } + } + } + else if (architecture == 'armem') { + // arm emulator (Ubuntu/Ubuntu16.04/Tizen). Build and test on same machine, + // using Docker. + Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest') + } + else { + // arm Ubuntu on hardware. + assert (architecture == 'arm') && (os == 'Ubuntu') + def isFlow = (options != null) && (options['is_flow_job'] == true) + def isBuild = (options != null) && (options['is_build_job'] == true) + if (isFlow || isBuild) { + // arm Ubuntu build machine. Build uses docker, so the actual host OS is not + // very important. Therefore, use latest or auto. Flow jobs don't need to use + // arm hardware. + Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto') + } else { + // arm Ubuntu test machine + // There is no tag (like, e.g., "arm-latest") for this, so don't call + // Utilities.setMachineAffinity. Just add the machine affinity + // manually. We specify the Helix queue name here. + job.with { + label('ubuntu.1404.arm32.open') } } } } } +// setJobMachineAffinity: compute the machine affinity options for a job, +// then set the job with those affinity options. +def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job) +{ + assert (isBuildJob && !isTestJob && !isFlowJob) || + (!isBuildJob && isTestJob && !isFlowJob) || + (!isBuildJob && !isTestJob && isFlowJob) + + def affinityOptions = null + def affinityArchitecture = architecture + + if (os == "Windows_NT") { + if (architecture in Constants.armWindowsCrossArchitectureList) { + if (isBuildJob) { + affinityOptions = [ "use_arm64_build_machine" : true ] + } else if (isTestJob) { + affinityOptions = [ "use_arm64_build_machine" : false ] + } else if (isFlowJob) { + // For the flow jobs set the machine affinity as x64 + affinityArchitecture = 'x64' + } + } + } + else { + if (architecture == 'arm64') { + if (isBuildJob) { + affinityOptions = ['is_build_only': true] + } else if (isTestJob) { + affinityOptions = [ "large_pages" : false ] + } + } + else if (architecture == 'arm') { + if (isBuildJob) { + affinityOptions = ['is_build_job': true] + } else if (isFlowJob) { + affinityOptions = ['is_flow_job': true] + } + } + } + + setMachineAffinity(job, os, affinityArchitecture, affinityOptions) +} + def static isGCStressRelatedTesting(def scenario) { // The 'r2r_gcstress15' scenario is a basic scenario. // Detect it and make it a GCStress related. @@ -599,7 +741,7 @@ def static isArmWindowsScenario(def scenario) { def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) { if (isBuildOnly == true) { - os = 'Windows_NT_buildOnly' + os = 'Windows_NT_BuildOnly' } def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os] @@ -655,20 +797,25 @@ def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, is else if (isGcReliabilityFramework(scenario)) { timeout = 1440 } - else if (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') { + else if (architecture == 'armlb' || architecture == 'armem' || architecture == 'arm64') { timeout = 240 } + + if (architecture == 'arm') { + // ARM32 machines are particularly slow. + timeout += 120 + } } if (configuration == 'Debug') { // Debug runs can be very slow. Add an hour. timeout += 60 } - - if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') { - // AltJit runs compile all methods twice. - timeout *= 2 - } + + if (architecture == 'x86_arm_altjit' || architecture == 'x64_arm64_altjit') { + // AltJit runs compile all methods twice. + timeout *= 2 + } // If we've changed the timeout from the default, set it in the job. @@ -802,8 +949,11 @@ def static isNeedDocker(def architecture, def os, def isBuild) { if (architecture == 'x86' && os == 'Ubuntu') { return true } + else if (architecture == 'armem') { + return true + } else if (architecture == 'arm') { - if (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen') { + if (os == 'Ubuntu') { return true } } @@ -822,7 +972,7 @@ def static getDockerImageName(def architecture, def os, def isBuild) { if (architecture == 'x86' && os == 'Ubuntu') { return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548" } - else if (architecture == 'arm') { + else if (architecture == 'armem') { if (os == 'Ubuntu') { return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20172211042239" } @@ -833,6 +983,11 @@ def static getDockerImageName(def architecture, def os, def isBuild) { return "hqueue/dotnetcore:ubuntu1404_cross_prereqs_v4-tizen_rootfs" } } + else if (architecture == 'arm') { + if (os == 'Ubuntu') { + return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-0cd4667-20170319080304" + } + } } else { if (architecture == 'x86' && os == 'Ubuntu') { @@ -843,6 +998,22 @@ def static getDockerImageName(def architecture, def os, def isBuild) { assert false } + +// We have a limited amount of some hardware. For these, scale back the periodic testing we do. +def static jobRequiresLimitedHardware(def architecture, def os) { + if (((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) && (os == 'Windows_NT')) { + // These test jobs require ARM64 hardware + return true + } + else if ((architecture == 'arm') && (os == 'Ubuntu')) { + // These test jobs require Linux/arm32 hardware + return true + } + else { + return false + } +} + // Calculates the name of the build job based on some typical parameters. // def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) { @@ -879,7 +1050,7 @@ def static getJobName(def configuration, def architecture, def os, def scenario, baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + "small_page_size" } break - case 'arm': + case 'armem': // These are cross builds if (os == 'Tizen') { // ABI: softfp @@ -890,6 +1061,7 @@ def static getJobName(def configuration, def architecture, def os, def scenario, } break case 'armlb': + case 'arm': baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase() break case 'x86': @@ -933,6 +1105,16 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def } break case 'arm': + if (os == 'Windows_NT') { + addGithubPushTriggerHelper(job) + } + else { + // Currently no push triggers, with limited arm Linux hardware. + assert os == 'Ubuntu' + addPeriodicTriggerHelper(job, '@daily') + } + break + case 'armem': case 'armlb': case 'x86_arm_altjit': case 'x64_arm64_altjit': @@ -1109,11 +1291,8 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def case 'zapdisable': if (os != 'CentOS7.1' && !(os in bidailyCrossList)) { assert (os == 'Windows_NT') || (os in Constants.crossList) - if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) { - if (os == 'Windows_NT') { - // We don't have enough ARM64 machines to run these more frequently than weekly. - addPeriodicTriggerHelper(job, '@weekly') - } + if (jobRequiresLimitedHardware(architecture, os)) { + addPeriodicTriggerHelper(job, '@weekly') } else { addPeriodicTriggerHelper(job, '@daily') @@ -1124,17 +1303,7 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def case 'gcstress0x3': if (os != 'CentOS7.1' && !(os in bidailyCrossList)) { assert (os == 'Windows_NT') || (os in Constants.crossList) - if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) { - if (os == 'Windows_NT') { - // We don't have enough ARM64 machines to run these more frequently than weekly. - addPeriodicTriggerHelper(job, '@weekly') - } - // TODO: Add once external email sending is available again - // addEmailPublisher(job, 'dotnetonarm64@microsoft.com') - } - else { - addPeriodicTriggerHelper(job, '@weekly') - } + addPeriodicTriggerHelper(job, '@weekly') } break case 'gcstress0xc': @@ -1147,17 +1316,7 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def // GCStress=C is currently not supported on OS X if (os != 'CentOS7.1' && os != 'OSX10.12' && !(os in bidailyCrossList)) { assert (os == 'Windows_NT') || (os in Constants.crossList) - if ((architecture == 'arm64') || (architecture == 'arm') || (architecture == 'armlb')) { - if (os == 'Windows_NT') { - // We don't have enough ARM64 machines to run these more frequently than weekly. - addPeriodicTriggerHelper(job, '@weekly') - } - // TODO: Add once external email sending is available again - // addEmailPublisher(job, 'dotnetonarm64@microsoft.com') - } - else { - addPeriodicTriggerHelper(job, '@weekly') - } + addPeriodicTriggerHelper(job, '@weekly') } break @@ -1486,42 +1645,27 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, } break - // editor brace matching: } - case 'armlb': - case 'arm': // editor brace matching: { + + case 'armem': // editor brace matching: { + job.with { + publishers { + azureVMAgentPostBuildAction { + agentPostBuildAction('Delete agent if the build was not successful (when idle).') + } + } + } + switch (os) { case 'Ubuntu': case 'Ubuntu16.04': - if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu - break - } - assert scenario != 'innerloop' - job.with { - publishers { - azureVMAgentPostBuildAction { - agentPostBuildAction('Delete agent if the build was not successful (when idle).') - } - } - } Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Cross ${configuration} Build", "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*") break case 'Tizen': - if (architecture == 'armlb') { // No arm legacy backend testing for Tizen armel - break - } - architecture = 'armel' - job.with { - publishers { - azureVMAgentPostBuildAction { - agentPostBuildAction('Delete agent if the build was not successful (when idle).') - } - } - } if (scenario == 'innerloop') { if (configuration == 'Checked') { @@ -1533,40 +1677,62 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}\\W+Build.*") } break + } - case 'Windows_NT': - if (architecture == "armlb") { - // Disable armlb windows jobs - break - } + break + // editor brace matching: } - // Triggers on the non-flow jobs aren't necessary here - if (!isFlowJob) { + case 'armlb': + case 'arm': // editor brace matching: { + + // Triggers on the non-flow jobs aren't necessary + if (!isFlowJob) { + break + } + + // Set up a private trigger + def contextString = "${os} ${architecture} Cross ${configuration}" + def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}" + if (scenario == 'innerloop') { + contextString += " Innerloop" + triggerString += "\\W+Innerloop" + } + else { + contextString += " ${scenario}" + triggerString += "\\W+${scenario}" + } + + if (configuration == 'Debug') { + contextString += " Build" + triggerString += "\\W+Build" + } else { + contextString += " Build and Test" + triggerString += "\\W+Build and Test" + } + + triggerString += ".*" + + switch (os) { + case 'Ubuntu': + if (architecture == 'armlb') { // No arm legacy backend testing for Ubuntu break } - // Set up a private trigger - def contextString = "${os} ${architecture} Cross ${configuration}" - def triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}" if (scenario == 'innerloop') { - contextString += " Innerloop" - triggerString += "\\W+Innerloop" + if (configuration == 'Checked') { + Utilities.addGithubPRTriggerForBranch(job, branch, contextString) + } } else { - contextString += " ${scenario}" - triggerString += "\\W+${scenario}" + Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString) } + break - if (configuration == 'Debug') { - contextString += " Build" - triggerString += "\\W+Build" - } else { - contextString += " Build and Test" - triggerString += "\\W+Build and Test" + case 'Windows_NT': + if (architecture == "armlb") { + // Disable armlb windows jobs + break } - - triggerString += ".*" - switch (scenario) { case 'innerloop': // Only Checked is an innerloop trigger. @@ -1781,7 +1947,7 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, } def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) { - def buildCommands = []; + def buildCommands = [] def osGroup = getOSGroup(os) def lowerConfiguration = configuration.toLowerCase() @@ -1790,9 +1956,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR priority = '0' } - setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) - - def enableCorefxTesting = isCoreFxScenario(scenario) + def doCoreFxTesting = isCoreFxScenario(scenario) // Calculate the build steps, archival, and xunit results switch (os) { @@ -1821,7 +1985,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}" } - // If it is a release build for windows, ensure PGO is used, else fail the build + // If it is a release build for Windows, ensure PGO is used, else fail the build. if ((lowerConfiguration == 'release') && (scenario in Constants.basicScenarios) && (architecture != 'x86_arm_altjit') && @@ -1830,7 +1994,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR buildOpts += ' -enforcepgo' } - if (enableCorefxTesting) { + if (doCoreFxTesting) { buildOpts += ' skiptests'; } else { buildOpts += " -priority=${priority}" @@ -1928,7 +2092,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}" - if (enableCorefxTesting) { + if (doCoreFxTesting) { def workspaceRelativeFxRoot = "_/fx" def absoluteFxRoot = "%WORKSPACE%\\_\\fx" @@ -1949,15 +2113,15 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR else { buildCommands += "tests\\runtest.cmd ${runtestArguments}" } - } + } // end if (!isBuildOnly) - if (!enableCorefxTesting) { + if (!doCoreFxTesting) { // Run the rest of the build // Build the mscorlib for the other OS's buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib" buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib" - if (arch == "x64") { + if (arch == 'x64') { buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib" } @@ -1988,9 +2152,6 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR case 'arm': assert isArmWindowsScenario(scenario) - def machineAffinityOptions = ['use_arm64_build_machine' : true] - setMachineAffinity(newJob, os, architecture, machineAffinityOptions) - def buildArchitecture = 'arm' def buildOpts = '' @@ -2001,7 +2162,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR buildOpts += ' -crossgenaltjit legacyjit.dll' } - if (enableCorefxTesting) { + if (doCoreFxTesting) { // We shouldn't need to build the tests. However, run-corefx-tests.py currently depends on having the restored corefx // package available, to determine the correct corefx version git commit hash, and we need to build the tests before // running "tests\\runtest.cmd GenerateLayoutOnly". So build the pri-0 tests to make this happen. @@ -2015,7 +2176,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR // This is now a build only job. Do not run tests. Use the flow job. buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${buildArchitecture} ${buildOpts}" - if (enableCorefxTesting) { + if (doCoreFxTesting) { assert isBuildOnly assert architecture == 'arm' @@ -2056,9 +2217,6 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR case 'arm64': assert isArmWindowsScenario(scenario) - def machineAffinityOptions = ['use_arm64_build_machine' : true] - setMachineAffinity(newJob, os, architecture, machineAffinityOptions) - // This is now a build only job. Do not run tests. Use the flow job. buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} toolset_dir C:\\ats2 -priority=${priority}" @@ -2075,7 +2233,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR break } break - // editor brace matching: } + // end case 'Windows_NT'; editor brace matching: } case 'Ubuntu': case 'Ubuntu16.04': case 'Ubuntu16.10': @@ -2110,7 +2268,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}" } - if (!enableCorefxTesting) { + if (!doCoreFxTesting) { // We run pal tests on all OS but generate mscorlib (and thus, nuget packages) // only on supported OS platforms. def bootstrapRid = Utilities.getBoostrapPublishRid(os) @@ -2153,7 +2311,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR } break case 'arm64': - if (!enableCorefxTesting) { + if (!doCoreFxTesting) { buildCommands += "ROOTFS_DIR=/opt/arm64-xenial-rootfs ./build.sh verbose ${lowerConfiguration} ${architecture} cross clang3.8" // HACK -- Arm64 does not have corefx jobs yet. @@ -2166,19 +2324,19 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**") } break - case 'arm': - // Cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently + case 'armem': + // Emulator cross builds for ARM runs on Ubuntu, Ubuntu16.04 and Tizen currently assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') || (os == 'Tizen') // default values for Ubuntu - def arm_abi="arm" - def linuxCodeName="trusty" + def arm_abi = "arm" + def linuxCodeName = "trusty" if (os == 'Ubuntu16.04') { - linuxCodeName="xenial" + linuxCodeName = "xenial" } else if (os == 'Tizen') { - arm_abi="armel" - linuxCodeName="tizen" + arm_abi = "armel" + linuxCodeName = "tizen" } // Unzip the Windows test binaries first. Exit with 0 @@ -2209,6 +2367,40 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR // Basic archiving of the build, no pal tests Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**") break + case 'arm': + // Non-Windows ARM cross builds on hardware run on Ubuntu only + assert (os == 'Ubuntu') + + // Add some useful information to the log file. Ignore return codes. + buildCommands += "uname -a || true" + + // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct + // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host). + + def dockerImage = getDockerImageName(architecture, os, true) + def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/arm ${dockerImage} " + + buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross" + + // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to + // download the appropriate CoreFX packages. + // Note that docker should not be necessary here, for the "generatelayoutonly" case, but we use it + // just to be consistent with the "build.sh" case -- so both are run with the same environment. + + buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly" + + // ZIP up for the test job (created in the flow job code): + // (1) the built CORE_ROOT, /home/user/coreclr/bin/tests/Linux.arm.Checked/Tests/Core_Root, + // used by runtest.sh as the "--coreOverlayDir" argument. + // (2) the native parts of the test build: /home/user/coreclr/bin/obj/Linux.arm.Checked/tests, + // used by runtest.sh as the "--testNativeBinDir" argument. + + // These commands are assumed to be run from the root of the workspace. + buildCommands += "zip -r coreroot.${lowerConfiguration}.zip ./bin/tests/Linux.arm.${configuration}/Tests/Core_Root" + buildCommands += "zip -r testnativebin.${lowerConfiguration}.zip ./bin/obj/Linux.arm.${configuration}/tests" + + Utilities.addArchival(newJob, "coreroot.${lowerConfiguration}.zip,testnativebin.${lowerConfiguration}.zip", "") + break default: println("Unknown architecture: ${architecture}"); assert false @@ -2225,6 +2417,235 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR return buildCommands } +// Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only. +// Returns true if the job should be generated. +def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly) +{ + // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered. + // All other scenarios do Pri-1 testing. + if (scenario == 'innerloop' && !isPR) { + return false + } + + // Tizen is only supported for armem architecture + if (os == 'Tizen' && architecture != 'armem') { + return false + } + + // Filter based on architecture. + + switch (architecture) { + case 'arm64': + case 'arm': + if ((os != 'Windows_NT') && (os != 'Ubuntu')) { + return false + } + break + case 'armem': + if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen')) { + return false + } + break + case 'armlb': + // Do not create armlb jobs + return false + case 'x86_arm_altjit': + case 'x64_arm64_altjit': + if (os != 'Windows_NT') { + return false + } + break + case 'x86': + if ((os != 'Windows_NT') && (os != 'Ubuntu')) { + return false + } + break + case 'x64': + // Everything implemented + break + default: + println("Unknown architecture: ${architecture}") + assert false + break + } + + // Which (Windows) build only jobs are required? + + def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal') + + if (isBuildOnly) { + switch (architecture) { + case 'arm': + // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that. + if (!isCoreFxScenario(scenario)) { + return false + } + break + case 'x64': + case 'x86': + if (!isNormalOrInnerloop) { + return false + } + break + default: + return false + } + } + + // Filter based on scenario. + + if (isJitStressScenario(scenario)) { + if (configuration != 'Checked') { + return false + } + + def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && architecture == 'arm') || (os == 'Ubuntu' && isCoreFxScenario(scenario)) + if (!isEnabledOS) { + return false + } + + switch (architecture) { + case 'x64': + case 'x86_arm_altjit': + case 'x64_arm64_altjit': + break + + case 'x86': + // x86 ubuntu: no stress modes + if (os == 'Ubuntu') { + return false + } + break + + case 'arm': + // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that. + if (! (isBuildOnly && isCoreFxScenario(scenario)) ) { + return false + } + break + + default: + // arm64, armlb: stress is handled through flow jobs. + // armem: no stress jobs for ARM emulator. + return false + } + } + else if (isR2RScenario(scenario)) { + if (os != 'Windows_NT') { + return false + } + // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow). + if ((configuration != 'Checked') && isR2RStressScenario(scenario)) { + return false + } + } + else { + // Skip scenarios + switch (scenario) { + case 'ilrt': + // The ilrt build isn't necessary except for Windows_NT2003. Non-Windows NT uses + // the default scenario build + if (os != 'Windows_NT') { + return false + } + // Only x64 for now + if (architecture != 'x64') { + return false + } + // Release only + if (configuration != 'Release') { + return false + } + break + case 'jitdiff': + if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') { + return false + } + if (architecture != 'x64') { + return false + } + if (configuration != 'Checked') { + return false + } + break + case 'longgc': + case 'gcsimulator': + if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') { + return false + } + if (architecture != 'x64') { + return false + } + if (configuration != 'Release') { + return false + } + break + case 'gc_reliability_framework': + case 'standalone_gc': + if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') { + return false + } + + if (architecture != 'x64') { + return false + } + + if (configuration != 'Release' && configuration != 'Checked') { + return false + } + break + // We only run Windows and Ubuntu x64 Checked for formatting right now + case 'formatting': + if (os != 'Windows_NT' && os != 'Ubuntu') { + return false + } + if (architecture != 'x64') { + return false + } + if (configuration != 'Checked') { + return false + } + break + case 'illink': + if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) { + return false + } + if (architecture != 'x64' && architecture != 'x86') { + return false + } + break + case 'normal': + // Nothing skipped + break + case 'innerloop': + if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) { + return false + } + break + default: + println("Unknown scenario: ${scenario}") + assert false + break + } + } + + // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change + // runtime behavior, not compile-time behavior, are not interesting. + switch (architecture) { + case 'x86_arm_altjit': + case 'x64_arm64_altjit': + if (isGCStressRelatedTesting(scenario)) { + return false + } + break + default: + break + } + + // The job was not filtered out, so we should generate it! + return true +} + Constants.allScenarios.each { scenario -> [true, false].each { isPR -> Constants.architectureList.each { architecture -> @@ -2238,212 +2659,11 @@ Constants.allScenarios.each { scenario -> os = 'Windows_NT' } - // Tizen is only supported for arm architecture - if (os == 'Tizen' && architecture != 'arm') { + if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) { return } - // Skip totally unimplemented (in CI) configurations. - switch (architecture) { - case 'arm64': - if (os == 'Ubuntu16.04') { - os = 'Ubuntu' - } - - // Windows and Ubuntu only - if ((os != 'Windows_NT' && os != 'Ubuntu') || isBuildOnly) { - return - } - break - case 'arm': - if ((os != 'Ubuntu') && (os != 'Ubuntu16.04') && (os != 'Tizen') && (os != 'Windows_NT')) { - return - } - break - case 'armlb': - // Do not create armlb jobs - return - break - case 'x86': - if ((os != 'Ubuntu') && (os != 'Windows_NT')) { - return - } - break - case 'x86_arm_altjit': - case 'x64_arm64_altjit': - if (os != 'Windows_NT') { - return - } - break - case 'x64': - // Everything implemented - break - default: - println("Unknown architecture: ${architecture}") - assert false - break - } - - // Skip scenarios (blanket skipping for jit stress modes, which are good most everywhere - // with checked builds) - if (isJitStressScenario(scenario)) { - if (configuration != 'Checked') { - return - } - - // Since these are just execution time differences, - // skip platforms that don't execute the tests here (Windows_NT only) - def isEnabledOS = (os == 'Windows_NT') || (os == 'Ubuntu' && isCoreFxScenario(scenario)) - if (!isEnabledOS) { - return - } - - switch (architecture) { - case 'x64': - case 'x86': - case 'x86_arm_altjit': - case 'x64_arm64_altjit': - // x86 ubuntu: default only - if ((os == 'Ubuntu') && (architecture == 'x86')) { - return - } - if (isBuildOnly) { - return - } - break - - case 'arm': - // We use build only jobs for Windows arm cross-compilation corefx testing, so we need to generate builds for that. - if (!isBuildOnly || !isCoreFxScenario(scenario)) { - return - } - break - - default: - // arm64, armlb: stress is handled through flow jobs. - return - } - } - else if (isR2RScenario(scenario)) { - if (os != 'Windows_NT') { - return - } - // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow). - if ((configuration != 'Checked') && isR2RStressScenario(scenario)) { - return - } - } - else { - // Skip scenarios - switch (scenario) { - case 'ilrt': - // The ilrt build isn't necessary except for Windows_NT2003. Non-Windows NT uses - // the default scenario build - if (os != 'Windows_NT') { - return - } - // Only x64 for now - if (architecture != 'x64') { - return - } - // Release only - if (configuration != 'Release') { - return - } - break - case 'jitdiff': - if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') { - return - } - if (architecture != 'x64') { - return - } - if (configuration != 'Checked') { - return - } - break - case 'longgc': - case 'gcsimulator': - if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') { - return - } - if (architecture != 'x64') { - return - } - if (configuration != 'Release') { - return - } - break - case 'gc_reliability_framework': - case 'standalone_gc': - if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') { - return - } - - if (architecture != 'x64') { - return - } - - if (configuration != 'Release' && configuration != 'Checked') { - return - } - break - // We only run Windows and Ubuntu x64 Checked for formatting right now - case 'formatting': - if (os != 'Windows_NT' && os != 'Ubuntu') { - return - } - if (architecture != 'x64') { - return - } - if (configuration != 'Checked') { - return - } - if (isBuildOnly) { - return - } - break - case 'illink': - if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) { - return - } - if (architecture != 'x64' && architecture != 'x86') { - return - } - if (isBuildOnly) { - return - } - break - case 'normal': - // Nothing skipped - break - case 'innerloop': - if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) { - return - } - break - default: - println("Unknown scenario: ${scenario}") - assert false - break - } - } - - // For altjit, don't do any scenarios that don't change compilation. That is, scenarios that only change - // runtime behavior, not compile-time behavior, are not interesting. - switch (architecture) { - case 'x86_arm_altjit': - case 'x64_arm64_altjit': - if (isGCStressRelatedTesting(scenario)) { - return - } - break - default: - break - } - // Calculate names - def lowerConfiguration = configuration.toLowerCase() def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly) def folderName = getJobFolder(scenario) @@ -2451,23 +2671,59 @@ Constants.allScenarios.each { scenario -> def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {} addToViews(newJob, isPR, architecture, os) - def machineAffinityOptions = null - - if (os != 'Windows_NT') { - machineAffinityOptions = architecture == 'arm64' ? ['is_build_only': true] : null - } - else { - machineAffinityOptions = (architecture == 'arm' || architecture == 'armlb' || architecture == 'arm64') ? ['use_arm64_build_machine': false] : null - } - - setMachineAffinity(newJob, os, architecture, machineAffinityOptions) + setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false - // Add all the standard options Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false + setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) + + // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem. + // We don't use a flow job for this, but we do depend on there being existing builds with these + // artifacts produced. + if (architecture == 'armem' && (os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) { + // Define the Windows Tests and Corefx build job names + def lowerConfiguration = configuration.toLowerCase() + def WindowsTestsName = projectFolder + '/' + + Utilities.getFullJobName(project, + getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true), + false) + def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + + Utilities.getFolderName(branch) + + def arm_abi = 'arm' + def corefx_os = 'linux' + if (os == 'Tizen') { + arm_abi = 'armel' + corefx_os = 'tizen' + } + + // Let's use release CoreFX to test checked CoreCLR, + // because we do not generate checked CoreFX in CoreFX CI yet. + def corefx_lowerConfiguration = lowerConfiguration + if (lowerConfiguration == 'checked') { + corefx_lowerConfiguration = 'release' + } + + // Copy the Windows test binaries and the Corefx build binaries + newJob.with { + steps { + copyArtifacts(WindowsTestsName) { + includePatterns('bin/tests/tests.zip') + buildSelector { + latestSuccessful(true) + } + } + copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") { + includePatterns('bin/build.tar.gz') + buildSelector { + latestSuccessful(true) + } + } + } // steps + } // newJob.with + } def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly) - def osGroup = getOSGroup(os) newJob.with { steps { @@ -2477,52 +2733,11 @@ Constants.allScenarios.each { scenario -> } } else { - // Setup corefx and Windows test binaries for Linux cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel - if ( architecture == 'arm' && ( os == 'Ubuntu' || os == 'Ubuntu16.04' || os == 'Tizen')) { - // Cross build for ubuntu-arm, ubuntu16.04-arm and tizen-armel - // Define the Windows Tests and Corefx build job names - def WindowsTestsName = projectFolder + '/' + - Utilities.getFullJobName(project, - getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true), - false) - def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + - Utilities.getFolderName(branch) - - // Copy the Windows test binaries and the Corefx build binaries - copyArtifacts(WindowsTestsName) { - includePatterns('bin/tests/tests.zip') - buildSelector { - latestSuccessful(true) - } - } - - def arm_abi = 'arm' - def corefx_os = 'linux' - if (os == 'Tizen') { - arm_abi = 'armel' - corefx_os = 'tizen' - } - - // Let's use release CoreFX to test checked CoreCLR, - // because we do not generate checked CoreFX in CoreFX CI yet. - def corefx_lowerConfiguration = lowerConfiguration - if ( lowerConfiguration == 'checked' ) { - corefx_lowerConfiguration='release' - } - - copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") { - includePatterns('bin/build.tar.gz') - buildSelector { - latestSuccessful(true) - } - } - } - buildCommands.each { buildCommand -> shell(buildCommand) } } - } + } // steps } // newJob.with } // os @@ -2531,685 +2746,787 @@ Constants.allScenarios.each { scenario -> } // isPR } // scenario +// Create a Windows ARM/ARMLB/ARM64 test job that will be used by a flow job. +// Returns the newly created job. +def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName) +{ + def osGroup = getOSGroup(os) + def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst" -// Create jobs requiring flow jobs. This includes x64 non-Windows, arm64 Ubuntu, and arm/arm64/armlb Windows. -Constants.allScenarios.each { scenario -> - def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal') + def jobFolder = getJobFolder(scenario) + def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) { + parameters { + stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from") + } - [true, false].each { isPR -> - ['arm', 'armlb', 'x64', 'arm64', 'x86'].each { architecture -> - Constants.crossList.each { os -> - if (architecture == 'arm64') { - if (os != "Ubuntu" && os != "Windows_NT") { - return - } - } else if (architecture == 'arm') { - if (os != 'Windows_NT') { - return - } - } else if (architecture == 'armlb') { - // Do not create armlb windows jobs. - return - } else if (architecture == 'x86') { - if (os != "Ubuntu") { - return - } - } + steps { + // Set up the copies - def validWindowsNTCrossArches = ["arm", "armlb", "arm64"] + // Coreclr build we are trying to test + // + // ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive - if (os == "Windows_NT" && !(architecture in validWindowsNTCrossArches)) { - return + copyArtifacts(inputCoreCLRBuildName) { + excludePatterns('**/testResults.xml', '**/*.ni.dll') + buildSelector { + buildNumber('${CORECLR_BUILD}') } + } - Constants.configurationList.each { configuration -> + if (isCoreFxScenario(scenario)) { - // First, filter based on OS. + // Only arm supported for corefx testing now. + assert architecture == 'arm' - if (os == 'Windows_NT') { - if (!isArmWindowsScenario(scenario)) { - return - } + // Unzip CoreFx runtime + batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm')") + + // Unzip CoreFx tests. + batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')") + + // Add the script to run the corefx tests + def corefx_runtime_path = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm" + def corefx_tests_dir = "%WORKSPACE%\\_\\fx\\bin\\tests" + def corefx_exclusion_file = "%WORKSPACE%\\tests\\arm\\corefx_test_exclusions.txt" + batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file}") + + } else { // !isCoreFxScenario(scenario) + + // Unzip tests. + batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')") + + def buildCommands = "" + + def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root" + def addEnvVariable = { variable, value -> buildCommands += "set ${variable}=${value}\r\n"} + def addCommand = { cmd -> buildCommands += "${cmd}\r\n"} + + // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available. + addCommand("SETLOCAL ENABLEEXTENSIONS") + + // For all jobs + addEnvVariable("CORE_ROOT", coreRootLocation) + addEnvVariable("COMPlus_NoGuiOnAssert", "1") + addEnvVariable("COMPlus_ContinueOnAssert", "0") + + // ARM legacy backend; this is an altjit. + if (architecture == 'armlb') { + addEnvVariable("COMPlus_AltJit", "*") + addEnvVariable("COMPlus_AltJitNgen", "*") + addEnvVariable("COMPlus_AltJitName", "legacyjit.dll") + addEnvVariable("COMPlus_AltJitAssertOnNYI", "1") + } + + // If we are running a stress mode, we'll set those variables as well + if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) { + def stressValues = null + if (isJitStressScenario(scenario)) { + stressValues = Constants.jitStressModeScenarios[scenario] } else { - // Non-Windows - if (architecture == 'arm64') { - if (!(scenario in Constants.validLinuxArm64Scenarios)) { - return - } - } - else if (architecture == 'x86') { - // Linux/x86 only want innerloop and default test - if (!isNormalOrInnerloop) { - return - } - } + stressValues = Constants.r2rStressScenarios[scenario] } - // For CentOS, we only want Checked/Release builds. - if (os == 'CentOS7.1') { - if (configuration != 'Checked' && configuration != 'Release') { - return - } - if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) { - return - } + stressValues.each { key, value -> + addEnvVariable(key, value) } + } - // For RedHat and Debian, we only do Release builds. - else if (os == 'RHEL7.2' || os == 'Debian8.4') { - if (configuration != 'Release') { - return - } - if (!isNormalOrInnerloop) { - return - } - } + if (isR2RScenario(scenario)) { + // Crossgen the framework assemblies. + buildCommands += """ +@for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF +@goto skip_PrecompileAssembly - // Next, filter based on scenario. +:PrecompileAssembly +@REM Skip mscorlib since it is already precompiled. +@if /I "%3" == "mscorlib.dll" exit /b 0 +@if /I "%3" == "mscorlib.ni.dll" exit /b 0 - if (isJitStressScenario(scenario)) { - if (configuration != 'Checked') { - return - } +"%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul +@if "%errorlevel%" == "-2146230517" ( + echo %2 is not a managed assembly. +) else if "%errorlevel%" == "-2146234344" ( + echo %2 is not a managed assembly. +) else if %errorlevel% neq 0 ( + echo Unable to precompile %2 +) else ( + echo Precompiled %2 +) +@exit /b 0 - // CoreFx JIT stress tests currently only implemented for ARM. - if (isCoreFxScenario(scenario) && (architecture != 'arm')) { - return - } - } - else if (isR2RBaselineScenario(scenario)) { - if (configuration != 'Checked' && configuration != 'Release') { - return - } - } - else if (isR2RStressScenario(scenario)) { - if (configuration != 'Checked') { - return - } - } - else { - // Skip scenarios - switch (scenario) { - case 'ilrt': - case 'longgc': - case 'gcsimulator': - // Long GC tests take a long time on non-Release builds - // ilrt is also Release only - if (configuration != 'Release') { - return - } - break +:skip_PrecompileAssembly +""" - case 'jitdiff': - if (configuration != 'Checked') { - return; - } - break + // Set RunCrossGen variable to cause test wrappers to invoke their logic to run + // crossgen on tests before running them. + addEnvVariable("RunCrossGen", "true") + } // isR2RScenario(scenario) - case 'gc_reliability_framework': - case 'standalone_gc': - if (configuration != 'Release' && configuration != 'Checked') { - return - } - break + // Create the smarty command + def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS " + def addSmartyFlag = { flag -> smartyCommand += flag + " "} + def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)} + def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } } - case 'formatting': - return - case 'illink': - if (os != 'Windows_NT' && os != 'Ubuntu') { - return - } - break + if (architecture == 'armlb') { + addExclude("LEGACYJIT_FAIL") + } - case 'normal': - // Nothing skipped - break + // Exclude tests based on scenario. + Constants.validArmWindowsScenarios[scenario].each { excludeTag -> + addArchSpecificExclude(architecture, excludeTag) + } - case 'innerloop': - // Nothing skipped - if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) { - return - } - break + // Innerloop jobs run Pri-0 tests; everyone else runs Pri-1. + if (scenario == 'innerloop') { + addExclude("pri1") + } - default: - println("Unknown scenario: ${scenario}") - assert false - break - } - } + // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result. + // TODO: We should create a "long running" job that runs these with a longer timeout. + addExclude("LONG_RUNNING") - // Done filtering. Now, create the jobs. + smartyCommand += "/lstFile Tests.lst" - // ============================================================================================= - // Create the test job - // ============================================================================================= + def testListArch = [ + 'arm64': 'arm64', + 'arm': 'arm', + 'armlb': 'arm' + ] - def windowsArmJob = (os == "Windows_NT" && architecture in validWindowsNTCrossArches) + def archLocation = testListArch[architecture] - def lowerConfiguration = configuration.toLowerCase() - def osGroup = getOSGroup(os) - def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst" + addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}") + addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}") + addCommand("${smartyCommand}") - def inputCoreCLRBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal' - def inputCoreCLRBuildIsBuildOnly = false - if (isCoreFxScenario(scenario)) { - // Every CoreFx test depends on its own unique build. - inputCoreCLRBuildScenario = scenario - inputCoreCLRBuildIsBuildOnly = true - } - def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario) - def inputCoreCLRBuildName = projectFolder + '/' + - Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName) + // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file. + // However, we also need to remove all the variables that were set during this batch file, so we + // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell + // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well + // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel% + // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command. + // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity. + addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%") - def inputWindowsTestsBuildName = "" - if (windowsArmJob != true) { - // If this is a stress scenario, there isn't any difference in the build job, so we didn't create a build only - // job for Windows_NT specific to that stress mode. Just copy from the default scenario. + // ZIP up the smarty output, no matter what the smarty result. + addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"") - def testBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal' + addCommand("echo %errorlevel%") + addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}") - def inputWindowsTestBuildArch = architecture - if (architecture == "arm64" && os != "Windows_NT") { - // Use the x64 test build for arm64 unix - inputWindowsTestBuildArch = "x64" - } + // Use the smarty errorlevel as the script errorlevel. + addCommand("exit /b %__save_smarty_errorlevel%") - if (isJitStressScenario(scenario)) { - inputWindowsTestsBuildName = projectFolder + '/' + - Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, false), isPR) - } else { - inputWindowsTestsBuildName = projectFolder + '/' + - Utilities.getFullJobName(project, getJobName(configuration, inputWindowsTestBuildArch, 'windows_nt', testBuildScenario, true), isPR) - } - } // if (windowsArmJob != true) + batchFile(buildCommands) + } // non-corefx testing + } // steps + } // job - def serverGCString = '' - def testOpts = '' + if (!isCoreFxScenario(scenario)) { + Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false) - if (windowsArmJob != true) { - // Enable Server GC for Ubuntu PR builds - if (os == 'Ubuntu' && isPR) { - serverGCString = '--useServerGC' - } + // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much, + // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get + // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large. + Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false) + } - if (isR2RScenario(scenario)) { + return newJob +} - testOpts += ' --crossgen --runcrossgentests' +// Create a test job not covered by the "Windows ARM" case that will be used by a flow job. +// E.g., non-Windows tests. +// Returns the newly created job. +def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName) +{ + def isUbuntuArmJob = ((os == "Ubuntu") && (architecture == 'arm')) // ARM Ubuntu running on hardware (not emulator) - if (scenario == 'r2r_jitstress1') { - testOpts += ' --jitstress=1' - } - else if (scenario == 'r2r_jitstress2') { - testOpts += ' --jitstress=2' - } - else if (scenario == 'r2r_jitstressregs1') { - testOpts += ' --jitstressregs=1' - } - else if (scenario == 'r2r_jitstressregs2') { - testOpts += ' --jitstressregs=2' - } - else if (scenario == 'r2r_jitstressregs3') { - testOpts += ' --jitstressregs=3' - } - else if (scenario == 'r2r_jitstressregs4') { - testOpts += ' --jitstressregs=4' - } - else if (scenario == 'r2r_jitstressregs8') { - testOpts += ' --jitstressregs=8' - } - else if (scenario == 'r2r_jitstressregs0x10') { - testOpts += ' --jitstressregs=0x10' - } - else if (scenario == 'r2r_jitstressregs0x80') { - testOpts += ' --jitstressregs=0x80' - } - else if (scenario == 'r2r_jitstressregs0x1000') { - testOpts += ' --jitstressregs=0x1000' - } - else if (scenario == 'r2r_jitminopts') { - testOpts += ' --jitminopts' - } - else if (scenario == 'r2r_jitforcerelocs') { - testOpts += ' --jitforcerelocs' - } - else if (scenario == 'r2r_gcstress15') { - testOpts += ' --gcstresslevel=0xF' - } - } - else if (scenario == 'jitdiff') { - testOpts += ' --jitdisasm --crossgen' - } - else if (scenario == 'illink') { - testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink' - } - else if (isLongGc(scenario)) { - // Long GC tests behave very poorly when they are not - // the only test running (many of them allocate until OOM). - testOpts += ' --sequential' - - // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options - // for running long GC and GCSimulator tests, respectively. We don't use them - // here because using a playlist file produces much more readable output on the CI machines - // and reduces running time. - // - // The Long GC playlist contains all of the tests that are - // going to be run. The GCSimulator playlist contains all of - // the GC simulator tests. - if (scenario == 'longgc') { - testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt' - } - else if (scenario == 'gcsimulator') { - testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt' - } - } - else if (isGcReliabilityFramework(scenario)) { - testOpts += ' --build-overlay-only' - } - else if (scenario == 'standalone_gc') { - if (osGroup == 'OSX') { - testOpts += ' --gcname=libclrgc.dylib' - } - else if (osGroup == 'Linux') { - testOpts += ' --gcname=libclrgc.so' - } - else { - println("Unexpected OS group: ${osGroup} for os ${os}") - assert false - } - } - } // if (windowsArmJob != true) + def osGroup = getOSGroup(os) + def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst" - def folder = getJobFolder(scenario) - def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folder)) { - // Add parameters for the inputs + def testOpts = '' + def useServerGC = false - if (windowsArmJob == true) { - parameters { - stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from") - } - } - else { - parameters { - stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from') - stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from") - } - } + // Enable Server GC for Ubuntu PR builds + // REVIEW: why? Does this apply to all architectures? Why only PR? + if (os == 'Ubuntu' && isPR) { + testOpts += ' --useServerGC' + useServerGC = true + } - steps { - // Set up the copies + if (isR2RScenario(scenario)) { - // Coreclr build containing the tests and mscorlib - // pri1 jobs still need to copy windows_nt built tests - if (windowsArmJob != true) { - copyArtifacts(inputWindowsTestsBuildName) { - excludePatterns('**/testResults.xml', '**/*.ni.dll') - buildSelector { - buildNumber('${CORECLR_WINDOWS_BUILD}') - } - } - } + testOpts += ' --crossgen --runcrossgentests' - // Coreclr build we are trying to test - // - // ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive + if (scenario == 'r2r_jitstress1') { + testOpts += ' --jitstress=1' + } + else if (scenario == 'r2r_jitstress2') { + testOpts += ' --jitstress=2' + } + else if (scenario == 'r2r_jitstressregs1') { + testOpts += ' --jitstressregs=1' + } + else if (scenario == 'r2r_jitstressregs2') { + testOpts += ' --jitstressregs=2' + } + else if (scenario == 'r2r_jitstressregs3') { + testOpts += ' --jitstressregs=3' + } + else if (scenario == 'r2r_jitstressregs4') { + testOpts += ' --jitstressregs=4' + } + else if (scenario == 'r2r_jitstressregs8') { + testOpts += ' --jitstressregs=8' + } + else if (scenario == 'r2r_jitstressregs0x10') { + testOpts += ' --jitstressregs=0x10' + } + else if (scenario == 'r2r_jitstressregs0x80') { + testOpts += ' --jitstressregs=0x80' + } + else if (scenario == 'r2r_jitstressregs0x1000') { + testOpts += ' --jitstressregs=0x1000' + } + else if (scenario == 'r2r_jitminopts') { + testOpts += ' --jitminopts' + } + else if (scenario == 'r2r_jitforcerelocs') { + testOpts += ' --jitforcerelocs' + } + else if (scenario == 'r2r_gcstress15') { + testOpts += ' --gcstresslevel=0xF' + } + } + else if (scenario == 'jitdiff') { + testOpts += ' --jitdisasm --crossgen' + } + else if (scenario == 'illink') { + testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink' + } + else if (isLongGc(scenario)) { + // Long GC tests behave very poorly when they are not + // the only test running (many of them allocate until OOM). + testOpts += ' --sequential' + + // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options + // for running long GC and GCSimulator tests, respectively. We don't use them + // here because using a playlist file produces much more readable output on the CI machines + // and reduces running time. + // + // The Long GC playlist contains all of the tests that are + // going to be run. The GCSimulator playlist contains all of + // the GC simulator tests. + if (scenario == 'longgc') { + testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt' + } + else if (scenario == 'gcsimulator') { + testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt' + } + } + else if (isGcReliabilityFramework(scenario)) { + testOpts += ' --build-overlay-only' + } + else if (scenario == 'standalone_gc') { + if (osGroup == 'OSX') { + testOpts += ' --gcname=libclrgc.dylib' + } + else if (osGroup == 'Linux') { + testOpts += ' --gcname=libclrgc.so' + } + else { + println("Unexpected OS group: ${osGroup} for os ${os}") + assert false + } + } - copyArtifacts(inputCoreCLRBuildName) { - excludePatterns('**/testResults.xml', '**/*.ni.dll') - buildSelector { - buildNumber('${CORECLR_BUILD}') - } - } + def jobFolder = getJobFolder(scenario) + def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) { + parameters { + stringParam('CORECLR_WINDOWS_BUILD', '', 'Build number to copy CoreCLR Windows test binaries from') + stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from") + } - // Windows CoreCLR Arm(64) will restore corefx - // packages correctly. - // - // In addition, test steps are entirely different - // because we do not have a unified runner - if (windowsArmJob != true) { - def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch) - - // HACK -- Arm64 does not have corefx jobs yet. - // Clone corefx and build the native packages overwriting the x64 packages. - if (architecture == 'arm64') { - shell("mkdir -p ./bin/CoreFxBinDir") - shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir") - shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun") - } - else if (architecture == 'x86') { - shell("mkdir ./bin/CoreFxNative") - - copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") { - includePatterns('bin/build.tar.gz') - targetDirectory('bin/CoreFxNative') - buildSelector { - latestSuccessful(true) - } - } + steps { + // Set up the copies - shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir") - } + // Coreclr build containing the tests and mscorlib + // pri1 jobs still need to copy windows_nt built tests + assert inputTestsBuildName != null + copyArtifacts(inputTestsBuildName) { + excludePatterns('**/testResults.xml', '**/*.ni.dll') + buildSelector { + buildNumber('${CORECLR_WINDOWS_BUILD}') + } + } - // Unzip the tests first. Exit with 0 - shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0") - shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0") + // Coreclr build we are trying to test + // + // ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive - shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly") + copyArtifacts(inputCoreCLRBuildName) { + excludePatterns('**/testResults.xml', '**/*.ni.dll') + buildSelector { + buildNumber('${CORECLR_BUILD}') + } + } - // Execute the tests - def runDocker = isNeedDocker(architecture, os, false) - def dockerPrefix = "" - def dockerCmd = "" - if (runDocker) { - def dockerImage = getDockerImageName(architecture, os, false) - dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} " - dockerCmd = dockerPrefix + "${dockerImage} " - } + if (isUbuntuArmJob) { + // Add some useful information to the log file. Ignore return codes. + shell("uname -a || true") + } - // If we are running a stress mode, we'll set those variables first - def testEnvOpt = "" - if (isJitStressScenario(scenario)) { - def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh" - def envScriptCmds = envScriptCreate(os, scriptFileName) - envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName) - envScriptCmds += envScriptFinalize(os, scriptFileName) - shell("${envScriptCmds}") - testEnvOpt = "--test-env=" + scriptFileName - } + if (architecture == 'arm64') { + shell("mkdir -p ./bin/CoreFxBinDir") + shell("cp ./bin/Product/Linux.arm64.${configuration}/corefxNative/* ./bin/CoreFxBinDir") + shell("chmod +x ./bin/Product/Linux.arm64.${configuration}/corerun") + } + else if (architecture == 'x86') { + shell("mkdir ./bin/CoreFxNative") - if (isGCStressRelatedTesting(scenario)) { - shell('./init-tools.sh') - } + def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(branch) - shell("""${dockerCmd}./tests/runtest.sh \\ - --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\ - --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\ - --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\ - --copyNativeTestBin --limitedDumpGeneration ${testEnvOpt} ${serverGCString} ${testOpts}""") - - if (isGcReliabilityFramework(scenario)) { - // runtest.sh doesn't actually execute the reliability framework - do it here. - if (serverGCString != '') { - if (runDocker) { - dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} " - } - else { - shell("export COMPlus_gcServer=1") - } - } + copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") { + includePatterns('bin/build.tar.gz') + targetDirectory('bin/CoreFxNative') + buildSelector { + latestSuccessful(true) + } + } - shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}") - } - } - else { // windowsArmJob == true - - if (isCoreFxScenario(scenario)) { + shell("tar -xf ./bin/CoreFxNative/bin/build.tar.gz -C ./bin/CoreFxBinDir") + } - // Only arm supported for corefx testing now. - assert architecture == 'arm' + // Unzip the tests first. Exit with 0 + shell("unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/${osGroup}.${architecture}.${configuration} || exit 0") + shell("rm -r ./bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root || exit 0") + + // For arm Ubuntu (on hardware), we do the "build-test" step on the build machine, not on the test + // machine. The arm Ubuntu test machines do no building -- they have no CLI, for example. + // We should probably do the "generatelayoutonly" step on the build machine for all architectures. + // However, it's believed that perhaps there's an issue with executable permission bits not getting + // copied correctly. + if (isUbuntuArmJob) { + def lowerConfiguration = configuration.toLowerCase() + shell("unzip -o ./coreroot.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/tests/Linux.arm.${configuration}/Tests/Core_Root + shell("unzip -o ./testnativebin.${lowerConfiguration}.zip || exit 0") // unzips to ./bin/obj/Linux.arm.${configuration}/tests + } + else { + shell("./build-test.sh ${architecture} ${configuration} generatelayoutonly") + } - // Unzip CoreFx runtime - batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm')") + // Execute the tests + def runDocker = isNeedDocker(architecture, os, false) + def dockerPrefix = "" + def dockerCmd = "" + if (runDocker) { + def dockerImage = getDockerImageName(architecture, os, false) + dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} " + dockerCmd = dockerPrefix + "${dockerImage} " + } - // Unzip CoreFx tests. - batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\bin\\tests')") + // If we are running a stress mode, we'll set those variables first + if (isJitStressScenario(scenario)) { + def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh" + def envScriptCmds = envScriptCreate(os, scriptFileName) + envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName) + envScriptCmds += envScriptFinalize(os, scriptFileName) + shell("${envScriptCmds}") + testOpts += " --test-env=${scriptFileName}" + } - // Add the script to run the corefx tests - def corefx_runtime_path = "%WORKSPACE%\\_\\fx\\bin\\testhost\\netcoreapp-Windows_NT-Release-arm" - def corefx_tests_dir = "%WORKSPACE%\\_\\fx\\bin\\tests" - def corefx_exclusion_file = "%WORKSPACE%\\tests\\arm\\corefx_test_exclusions.txt" - batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file}") + // TODO: how to handle GCStress-related testing for Ubuntu/arm? + if (isGCStressRelatedTesting(scenario)) { + shell('./init-tools.sh') + } - } else { // !isCoreFxScenario(scenario) + def runScript = "" + if (isUbuntuArmJob) { + // Use 'runtesttilstable.sh' to rerun failing tests (in sequential mode); + // there are many tests that pass on rerun (currently), and we don't want + // that flakiness to affect overall test job robustness. + runScript = "${dockerCmd}./tests/runtesttilstable.sh" + } else { + runScript = "${dockerCmd}./tests/runtest.sh" + } - // Unzip tests. - batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')") + shell("""\ +${runScript} \\ + --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\ + --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\ + --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\ + --copyNativeTestBin --limitedDumpGeneration ${testOpts}""") - def buildCommands = "" + if (isGcReliabilityFramework(scenario)) { + // runtest.sh doesn't actually execute the reliability framework - do it here. + if (useServerGC) { + if (runDocker) { + dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} " + } + else { + shell("export COMPlus_gcServer=1") + } + } - def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root" - def addEnvVariable = { variable, value -> buildCommands += "set ${variable}=${value}\r\n"} - def addCommand = { cmd -> buildCommands += "${cmd}\r\n"} + shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}") + } + } // steps + } // job + + // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links + if (os in ['Ubuntu']) { + SummaryBuilder summaries = new SummaryBuilder() + summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt') + summaries.emit(newJob) + } - // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available. - addCommand("SETLOCAL ENABLEEXTENSIONS") - - // For all jobs - addEnvVariable("CORE_ROOT", coreRootLocation) - - addEnvVariable("COMPlus_NoGuiOnAssert", "1") - addEnvVariable("COMPlus_ContinueOnAssert", "0") - - // ARM legacy backend; this is an altjit. - if (architecture == "armlb") { - addEnvVariable("COMPlus_AltJit", "*") - addEnvVariable("COMPlus_AltJitNgen", "*") - addEnvVariable("COMPlus_AltJitName", "legacyjit.dll") - addEnvVariable("COMPlus_AltJitAssertOnNYI", "1") - } + Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/coreclrtests.*.txt") + Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml') - // If we are running a stress mode, we'll set those variables as well - if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) { - def stressValues = null - if (isJitStressScenario(scenario)) { - stressValues = Constants.jitStressModeScenarios[scenario] - } - else { - stressValues = Constants.r2rStressScenarios[scenario] - } - - stressValues.each { key, value -> - addEnvVariable(key, value) - } - } + return newJob +} - if (isR2RScenario(scenario)) { - // Crossgen the framework assemblies. - buildCommands += """ -@for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF -@goto skip_PrecompileAssembly +// Create a test job that will be used by a flow job. +// Returns the newly created job. +def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName, def inputTestsBuildName) +{ + def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList)) -:PrecompileAssembly -@REM Skip mscorlib since it is already precompiled. -@if /I "%3" == "mscorlib.dll" exit /b 0 -@if /I "%3" == "mscorlib.ni.dll" exit /b 0 + def newJob = null + if (windowsArmJob) { + assert inputTestsBuildName == null + newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName) + } else { + newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName) + } -"%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul -@if "%errorlevel%" == "-2146230517" ( - echo %2 is not a managed assembly. -) else if "%errorlevel%" == "-2146234344" ( - echo %2 is not a managed assembly. -) else if %errorlevel% neq 0 ( - echo Unable to precompile %2 -) else ( - echo Precompiled %2 -) -@exit /b 0 + setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false -:skip_PrecompileAssembly -""" + addToViews(newJob, isPR, architecture, os) - // Set RunCrossGen variable to cause test wrappers to invoke their logic to run - // crossgen on tests before running them. - addEnvVariable("RunCrossGen", "true") - } // isR2RScenario(scenario) + if (scenario == 'jitdiff') { + def osGroup = getOSGroup(os) + Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**") + } - // Create the smarty command - def smartyCommand = "C:\\Tools\\Smarty.exe /noecid /noie /workers 9 /inc EXPECTED_PASS " - def addSmartyFlag = { flag -> smartyCommand += flag + " "} - def addExclude = { exclude -> addSmartyFlag("/exc " + exclude)} + Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") + setJobTimeout(newJob, isPR, architecture, configuration, scenario, false) - def addArchSpecificExclude = { architectureToExclude, exclude -> if (architectureToExclude == "armlb") { addExclude("LEGACYJIT_" + exclude) } else { addExclude(exclude) } } + return newJob +} - if (architecture == "armlb") { - addExclude("LEGACYJIT_FAIL") - } +// Create a flow job to tie together a build job with the given test job. +// Returns the new flow job. +def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName, def inputTestsBuildName) +{ + if (os == 'RHEL7.2' || os == 'Debian8.4') { + // Do not create the flow job for RHEL jobs. + return + } - // Exclude tests based on scenario. - Constants.validArmWindowsScenarios[scenario].each { excludeTag -> - addArchSpecificExclude(architecture, excludeTag) - } + // Windows CoreCLR build and Linux CoreCLR build (in parallel) -> + // Linux CoreCLR test + def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow" + def jobFolder = getJobFolder(scenario) - // Innerloop jobs run Pri-0 tests; everyone else runs Pri-1. - if (scenario == 'innerloop') { - addExclude("pri1") - } + def newFlowJob = null - // Exclude any test marked LONG_RUNNING; these often exceed the standard timeout and fail as a result. - // TODO: We should create a "long running" job that runs these with a longer timeout. - addExclude("LONG_RUNNING") + def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList)) + if (windowsArmJob) { - smartyCommand += "/lstFile Tests.lst" + assert inputTestsBuildName == null - def testListArch = [ - 'arm64': 'arm64', - 'arm': 'arm', - 'armlb': 'arm' - ] + // For Windows arm jobs there is no reason to build a parallel test job. + // The product build supports building and archiving the tests. - def archLocation = testListArch[architecture] + newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) { + buildFlow("""\ +coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') - addCommand("copy %WORKSPACE%\\tests\\${archLocation}\\Tests.lst bin\\tests\\${osGroup}.${architecture}.${configuration}") - addCommand("pushd bin\\tests\\${osGroup}.${architecture}.${configuration}") - addCommand("${smartyCommand}") +// And then build the test build +build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}') +""") + } + JobReport.Report.addReference(inputCoreCLRBuildName) + JobReport.Report.addReference(fullTestJobName) + } + else { + newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) { + buildFlow("""\ +// Build the input jobs in parallel +parallel ( +{ coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') }, +{ windowsBuildJob = build(params, '${inputTestsBuildName}') } +) - // Save the errorlevel from the smarty command to be used as the errorlevel of this batch file. - // However, we also need to remove all the variables that were set during this batch file, so we - // can run the ZIP powershell command (below) in a clean environment. (We can't run the powershell - // command with the COMPlus_AltJit variables set, for example.) To do that, we do ENDLOCAL as well - // as save the current errorlevel on the same line. This works because CMD evaluates the %errorlevel% - // variable expansion (or any variable expansion on the line) BEFORE it executes the ENDLOCAL command. - // Note that the ENDLOCAL also undoes the pushd command, but we add the popd here for clarity. - addCommand("popd & ENDLOCAL & set __save_smarty_errorlevel=%errorlevel%") +// And then build the test build +build(params + [CORECLR_BUILD: coreclrBuildJob.build.number, + CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}') +""") + } + JobReport.Report.addReference(inputCoreCLRBuildName) + JobReport.Report.addReference(inputTestsBuildName) + JobReport.Report.addReference(fullTestJobName) + } - // ZIP up the smarty output, no matter what the smarty result. - addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0', '.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Smarty.run.0.zip')\"") + addToViews(newFlowJob, isPR, architecture, os) - addCommand("echo %errorlevel%") - addCommand("dir .\\bin\\tests\\${osGroup}.${architecture}.${configuration}") + setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true - // Use the smarty errorlevel as the script errorlevel. - addCommand("exit /b %__save_smarty_errorlevel%") + Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}") + addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false - batchFile(buildCommands) - } // non-corefx testing - } // windowsArmJob == true - } // steps - } // job + return newFlowJob +} - addToViews(newJob, isPR, architecture, os) +// Determine if we should generate a flow job for the given parameters. +// Returns true if the job should be generated. +def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os) +{ + // The "innerloop" (Pri-0 testing) scenario is only available as PR triggered. + // All other scenarios do Pri-1 testing. + if (scenario == 'innerloop' && !isPR) { + return false + } - if (scenario == 'jitdiff') { - Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**") - } + // Filter based on OS and architecture. - // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links - if (os in ['Ubuntu']) { - SummaryBuilder summaries = new SummaryBuilder() - summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt') - summaries.emit(newJob) - } + switch (architecture) { + case 'arm64': + if (os != "Ubuntu" && os != "Windows_NT") { + return false + } + break + case 'armlb': + if (os != 'Windows_NT') { + return false + } + // Do not create armlb windows jobs. + return false + case 'arm': + if (os != "Ubuntu" && os != "Windows_NT") { + return false + } + break + case 'x86': + if (os != "Ubuntu") { + return false + } + break + case 'x64': + if (!(os in Constants.crossList)) { + return false + } + if (os == "Windows_NT") { + return false + } + break + case 'armem': + case 'x86_arm_altjit': + case 'x64_arm64_altjit': + // No flow jobs + return false + default: + println("Unknown architecture: ${architecture}") + assert false + break + } - def affinityOptions = null + def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal') - if (windowsArmJob == true) { - affinityOptions = [ - "use_arm64_build_machine" : false - ] - } + // Filter based on scenario in OS. - else if (architecture == 'arm64' && os != 'Windows_NT') { - affinityOptions = [ - "large_pages" : false - ] - } + if (os == 'Windows_NT') { + if (!isArmWindowsScenario(scenario)) { + return false + } + } + else { + // Non-Windows + if (architecture == 'arm64') { + if (!(scenario in Constants.validLinuxArm64Scenarios)) { + return false + } + } + else if (architecture == 'arm') { + if (!(scenario in Constants.validLinuxArmScenarios)) { + return false + } + } + else if (architecture == 'x86') { + // Linux/x86 only want innerloop and default test + if (!isNormalOrInnerloop) { + return false + } + } + } - setMachineAffinity(newJob, os, architecture, affinityOptions) - Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") + // For CentOS, we only want Checked/Release builds. + if (os == 'CentOS7.1') { + if (configuration != 'Checked' && configuration != 'Release') { + return false + } + if (!isNormalOrInnerloop && !isR2RScenario(scenario) && !isJitStressScenario(scenario)) { + return false + } + } - setJobTimeout(newJob, isPR, architecture, configuration, scenario, false) + // For RedHat and Debian, we only do Release builds. + else if (os == 'RHEL7.2' || os == 'Debian8.4') { + if (configuration != 'Release') { + return false + } + if (!isNormalOrInnerloop) { + return false + } + } - if (windowsArmJob != true) { - Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml') - } - else { - if (!isCoreFxScenario(scenario)) { - Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0/*.smrt", '', true, false) + // Next, filter based on scenario. - // Archive a ZIP file of the entire Smarty.run.0 directory. This is possibly a little too much, - // but there is no easy way to only archive the HTML/TXT files of the failing tests, so we get - // all the passing test info as well. Not necessarily a bad thing, but possibly somewhat large. - Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/Smarty.run.0.zip", '', true, false) - } - } + if (isJitStressScenario(scenario)) { + if (configuration != 'Checked') { + return false + } - // ============================================================================================= - // Create a build flow to join together the build and tests required to run this test. - // ============================================================================================= + // CoreFx JIT stress tests currently only implemented for Windows ARM. + if (isCoreFxScenario(scenario) && !( (architecture == 'arm') && (os == 'Windows_NT') )) { + return false + } + } + else if (isR2RBaselineScenario(scenario)) { + if (configuration != 'Checked' && configuration != 'Release') { + return false + } + } + else if (isR2RStressScenario(scenario)) { + if (configuration != 'Checked') { + return false + } + } + else { + // Skip scenarios + switch (scenario) { + case 'ilrt': + case 'longgc': + case 'gcsimulator': + // Long GC tests take a long time on non-Release builds + // ilrt is also Release only + if (configuration != 'Release') { + return false + } + break - // Windows CoreCLR build and Linux CoreCLR build (in parallel) -> - // Linux CoreCLR test - def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow" - def fullTestJobName = projectFolder + '/' + newJob.name - // Add a reference to the input jobs for report purposes - JobReport.Report.addReference(inputCoreCLRBuildName) - if (windowsArmJob != true) { - JobReport.Report.addReference(inputWindowsTestsBuildName) - } - JobReport.Report.addReference(fullTestJobName) - def newFlowJob = null + case 'jitdiff': + if (configuration != 'Checked') { + return false + } + break + + case 'gc_reliability_framework': + case 'standalone_gc': + if (configuration != 'Release' && configuration != 'Checked') { + return false + } + break + + case 'formatting': + return false + case 'illink': + if (os != 'Windows_NT' && os != 'Ubuntu') { + return false + } + break + + case 'normal': + // Nothing skipped + break + + case 'innerloop': + // Nothing skipped + if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) { + return false + } + break + + default: + println("Unknown scenario: ${scenario}") + assert false + break + } + } + + // The job was not filtered out, so we should generate it! + return true +} - if (os == 'RHEL7.2' || os == 'Debian8.4') { - // Do not create the flow job for RHEL jobs. +// Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64/armlb Windows. +// Note: no armlb non-Windows; we expect to deprecate/remove armlb soon, so don't want to add new testing for it. +Constants.allScenarios.each { scenario -> + [true, false].each { isPR -> + Constants.architectureList.each { architecture -> + Constants.configurationList.each { configuration -> + Constants.osList.each { os -> + + if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) { return } - - // For pri0 jobs we can build tests on unix - if (windowsArmJob) { - // For Windows arm jobs there is no reason to build a parallel test job. - // The product build supports building and archiving the tests. - newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) { - buildFlow("""\ -coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') + // Figure out the job name of the CoreCLR build the test will depend on. -// And then build the test build -build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}') -""") - } + def inputCoreCLRBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal' + def inputCoreCLRBuildIsBuildOnly = false + if (isCoreFxScenario(scenario)) { + // Every CoreFx test depends on its own unique build. + inputCoreCLRBuildScenario = scenario + inputCoreCLRBuildIsBuildOnly = true } - else { - newFlowJob = buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, folder)) { - buildFlow("""\ -// Build the input jobs in parallel -parallel ( -{ coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') }, -{ windowsBuildJob = build(params, '${inputWindowsTestsBuildName}') } -) + def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario) + def inputCoreCLRBuildName = projectFolder + '/' + + Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName) -// And then build the test build -build(params + [CORECLR_BUILD: coreclrBuildJob.build.number, - CORECLR_WINDOWS_BUILD: windowsBuildJob.build.number], '${fullTestJobName}') -""") + // Figure out the name of the build job that the test job will depend on. + // For Windows ARM tests, this is not used, as the CoreCLR build creates the tests. For other + // tests (e.g., Linux ARM), we depend on a Windows build to get the tests. + + def inputTestsBuildName = null + + def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList)) + if (!windowsArmJob) { + def testBuildScenario = scenario == 'innerloop' ? 'innerloop' : 'normal' + + def inputTestsBuildArch = architecture + if (architecture == "arm64") { + // Use the x64 test build for arm64 unix + inputTestsBuildArch = "x64" + } + else if (architecture == "arm") { + // Use the x86 test build for arm unix + inputTestsBuildArch = "x86" } + + def inputTestsBuildIsBuildOnly = true + + inputTestsBuildName = projectFolder + '/' + + Utilities.getFullJobName(project, getJobName(configuration, inputTestsBuildArch, 'windows_nt', testBuildScenario, inputTestsBuildIsBuildOnly), isPR) } - addToViews(newFlowJob, isPR, architecture, os) + // ============================================================================================= + // Create the test job + // ============================================================================================= - // For the flow jobs set the machine affinity as x64 if an armarch. - def flowArch = architecture + def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName) - if (flowArch in validWindowsNTCrossArches) { - flowArch = 'x64' - affinityOptions = null - } + // ============================================================================================= + // Create a build flow to join together the build and tests required to run this test. + // ============================================================================================= - setMachineAffinity(newFlowJob, os, flowArch, affinityOptions) - Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}") - addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false - } // configuration - } // os + def fullTestJobName = projectFolder + '/' + testJob.name + def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName, inputTestsBuildName) + + } // os + } // configuration } // architecture } // isPR } // scenario diff --git a/tests/runtest.sh b/tests/runtest.sh index 64d12e7d80..d0eea533d8 100755 --- a/tests/runtest.sh +++ b/tests/runtest.sh @@ -100,6 +100,12 @@ countSkippedTests=0 xunitOutputPath= xunitTestOutputPath= +# Variables for text file output. These can be passed back to runtest.sh using the "--playlist" argument +# to rerun specific tests. +testsPassOutputPath= +testsFailOutputPath= +testsSkipOutputPath= + # libExtension determines extension for dynamic library files # runtimeName determines where CoreFX Runtime files will be located OSName=$(uname -s) @@ -301,6 +307,49 @@ function xunit_output_end { echo '</assemblies>' >>"$xunitOutputPath" } +function text_file_output_begin { + if [ -z "$testsPassOutputPath" ]; then + testsPassOutputPath=$testRootDir/coreclrtests.pass.txt + fi + if ! [ -e $(basename "$testsPassOutputPath") ]; then + testsPassOutputPath=$testRootDir/coreclrtests.pass.txt + fi + if [ -e "$testsPassOutputPath" ]; then + rm -f "$testsPassOutputPath" + fi + if [ -z "$testsFailOutputPath" ]; then + testsFailOutputPath=$testRootDir/coreclrtests.fail.txt + fi + if ! [ -e $(basename "$testsFailOutputPath") ]; then + testsFailOutputPath=$testRootDir/coreclrtests.fail.txt + fi + if [ -e "$testsFailOutputPath" ]; then + rm -f "$testsFailOutputPath" + fi + if [ -z "$testsSkipOutputPath" ]; then + testsSkipOutputPath=$testRootDir/coreclrtests.skip.txt + fi + if ! [ -e $(basename "$testsSkipOutputPath") ]; then + testsSkipOutputPath=$testRootDir/coreclrtests.skip.txt + fi + if [ -e "$testsSkipOutputPath" ]; then + rm -f "$testsSkipOutputPath" + fi +} + +function text_file_output_add_test { + local scriptFilePath=$1 + local testResult=$2 # Pass, Fail, or Skip + + if [ "$testResult" == "Pass" ]; then + echo "$scriptFilePath" >>"$testsPassOutputPath" + elif [ "$testResult" == "Skip" ]; then + echo "$scriptFilePath" >>"$testsSkipOutputPath" + else + echo "$scriptFilePath" >>"$testsFailOutputPath" + fi +} + function exit_with_error { local errorSource=$1 local errorMessage=$2 @@ -820,11 +869,11 @@ function finish_test { header=$header$(printf "[%4ds]" $testRunningTime) fi - local xunitTestResult + local testResult case $testScriptExitCode in 0) let countPassedTests++ - xunitTestResult='Pass' + testResult='Pass' if ((verbose == 1 || runFailingTestsOnly == 1)); then echo "PASSED - ${header}${scriptFilePath}" else @@ -833,12 +882,12 @@ function finish_test { ;; 2) let countSkippedTests++ - xunitTestResult='Skip' + testResult='Skip' echo "SKIPPED - ${header}${scriptFilePath}" ;; *) let countFailedTests++ - xunitTestResult='Fail' + testResult='Fail' echo "FAILED - ${header}${scriptFilePath}" ;; esac @@ -850,7 +899,8 @@ function finish_test { done <"$outputFilePath" fi - xunit_output_add_test "$scriptFilePath" "$outputFilePath" "$xunitTestResult" "$testScriptExitCode" "$testRunningTime" + xunit_output_add_test "$scriptFilePath" "$outputFilePath" "$testResult" "$testScriptExitCode" "$testRunningTime" + text_file_output_add_test "$scriptFilePath" "$testResult" } function finish_remaining_tests { @@ -1258,6 +1308,7 @@ then fi xunit_output_begin +text_file_output_begin create_core_overlay precompile_overlay_assemblies diff --git a/tests/runtesttilstable.sh b/tests/runtesttilstable.sh new file mode 100755 index 0000000000..43f29e67f5 --- /dev/null +++ b/tests/runtesttilstable.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash + +function print_usage { + echo '' + echo 'CoreCLR test runner wrapper script.' + echo '' + echo 'Run tests using runtest.sh, then rerun the failures, if any,' + echo 'until the number of failures stabilizes. Thus, when running' + echo 'flaky tests, or running tests on a flaky platform, only the' + echo 'repeatable, "real", failures are reported.' + echo '' + echo 'Tests are rerun in sequential mode (passing --sequential to runtest.sh).' + echo 'This hopefully avoids resource exhaustion and other parallel run problems.' + echo '' + echo 'A maximum number of iterations can be specified.' + echo '' + echo 'Command line:' + echo '' + echo 'runtesttilstable.sh [options] [arguments for runtest.sh]' + echo '' + echo 'Any unknown argument is passed directly to runtest.sh.' + echo '' + echo 'Optional arguments:' + echo ' -h|--help : Show usage information.' + echo ' --max-iterations=<count> : Specify the maximum number of iterations. Default: 4.' + echo '' +} + +function exit_with_error { + local errorMessage=$1 + local printUsage=$2 + + if [ -z "$printUsage" ]; then + ((printUsage = 0)) + fi + + echo "$errorMessage" + if ((printUsage != 0)); then + print_usage + fi + exit $EXIT_CODE_EXCEPTION +} + +# Handle Ctrl-C. We will stop execution and print the results that +# we gathered so far. +function handle_ctrl_c { + echo "" + echo "*** Stopping... ***" + print_results + exit_with_error "Test run aborted by Ctrl+C." +} + +# Register the Ctrl-C handler +trap handle_ctrl_c INT + +# Where are we? +scriptPath=$(dirname $0) + +# Exit code constants +readonly EXIT_CODE_SUCCESS=0 # Script ran normally. +readonly EXIT_CODE_EXCEPTION=1 # Script exited because something exceptional happened (e.g. bad arguments, Ctrl-C interrupt). +readonly EXIT_CODE_TEST_FAILURE=2 # Script completed successfully, but one or more tests failed. + +# Argument variables +((maxIterations = 4)) + +# Handle arguments +__UnprocessedBuildArgs= + +# We need to capture the --testRootDir argument so we know where the test pass/fail/skip files will be placed. +testRootDir= + +# We need to handle the --playlist argument specially. The first run, we pass it through (if passed). +# After that, we use the --playlist argument ourselves, so we don't pass through the original one. +playlistArgument= + +for i in "$@" +do + case $i in + -h|--help) + print_usage + exit $EXIT_CODE_SUCCESS + ;; + --max-iterations=*) + maxIterations=${i#*=} + ;; + --playlist=*) + playlistArgument=$i + ;; + --testRootDir=*) + testRootDir=${i#*=} + # Also pass it on to runtest.sh + __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i" + ;; + *) + __UnprocessedBuildArgs="$__UnprocessedBuildArgs $i" + ;; + esac +done + +# Check testRootDir; this check is also done by runtest.sh. + +if [ -z "$testRootDir" ]; then + echo "--testRootDir is required." + print_usage + exit $EXIT_CODE_EXCEPTION +fi +if [ ! -d "$testRootDir" ]; then + echo "Directory specified by --testRootDir does not exist: $testRootDir" + exit $EXIT_CODE_EXCEPTION +fi + +# Now start running the tests. + +nextcmd="${scriptPath}/runtest.sh ${playlistArgument} ${__UnprocessedBuildArgs}" +echo "Running: $nextcmd" +$nextcmd +exitCode=$? +if [ $exitCode -eq $EXIT_CODE_TEST_FAILURE ]; then + # Now, we loop, rerunning the failed tests up to maxIterations times minus one + # (the initial run counts as an iteration). + ((totalRerunCount = $maxIterations - 1)) + for (( i=1; i<=$totalRerunCount; i++ )); do + if [ ! -e "$testRootDir/coreclrtests.fail.txt" ]; then + exit_with_error "Error: couldn't find $testRootDir/coreclrtests.fail.txt" + fi + + num_errors=$(grep -c '' "$testRootDir/coreclrtests.fail.txt") + echo "Test run failed with $num_errors errors:" + cat "$testRootDir/coreclrtests.fail.txt" + echo '' + + echo "Rerunning failures ($i of $totalRerunCount reruns)..." + + # Move the fail file to a different location, so it can be used without getting trashed by the + # next run's error file. + retryFile="$testRootDir/coreclrtests.retry.txt" + if [ -e "$retryFile" ]; then + rm -f "$retryFile" + if [ -e "$retryFile" ]; then + exit_with_error "Error: couldn't delete $retryFile" + fi + fi + mv "$testRootDir/coreclrtests.fail.txt" "$retryFile" + + nextcmd="${scriptPath}/runtest.sh --sequential --playlist=${retryFile} ${__UnprocessedBuildArgs}" + echo "Running: $nextcmd" + $nextcmd + exitCode=$? + if [ $exitCode -ne $EXIT_CODE_TEST_FAILURE ]; then + # Either success or exceptional failure; we're done. For test failure, we loop, + # if we haven't hit the maximum number of allowed iterations. + break + fi + done +fi + +exit $exitCode diff --git a/tests/scripts/arm32_ci_script.sh b/tests/scripts/arm32_ci_script.sh index aecb3e890a..3ffff2c9ab 100755 --- a/tests/scripts/arm32_ci_script.sh +++ b/tests/scripts/arm32_ci_script.sh @@ -183,21 +183,21 @@ function mount_emulator { fi if [ ! -d "$__ARMEmulRootfs" ]; then - sudo mkdir "$__ARMEmulRootfs" - fi + sudo mkdir "$__ARMEmulRootfs" + fi - if [ ! -f "$__ARMEmulRootfs/arm-emulator-rootfs.tar" ]; then - if mountpoint -q -- "$__ARMRootfsMountPath"; then - sudo umount -l $__ARMRootfsMountPath + if [ ! -f "$__ARMEmulRootfs/arm-emulator-rootfs.tar" ]; then + if mountpoint -q -- "$__ARMRootfsMountPath"; then + sudo umount -l $__ARMRootfsMountPath fi - mount_with_checking "" "$__ARMEmulPath/platform/rootfs-t30.ext4" "$__ARMRootfsMountPath" - - cd $__ARMRootfsMountPath - sudo tar -cf "$__ARMEmulRootfs/arm-emulator-rootfs.tar" * - cd - - fi + mount_with_checking "" "$__ARMEmulPath/platform/rootfs-t30.ext4" "$__ARMRootfsMountPath" + + cd $__ARMRootfsMountPath + sudo tar -cf "$__ARMEmulRootfs/arm-emulator-rootfs.tar" * + cd - + fi - sudo tar -xf "$__ARMEmulRootfs/arm-emulator-rootfs.tar" -C "$__ARMEmulRootfs" + sudo tar -xf "$__ARMEmulRootfs/arm-emulator-rootfs.tar" -C "$__ARMEmulRootfs" mount_with_checking "-t proc" "/proc" "$__ARMEmulRootfs/proc" mount_with_checking "-o bind" "/dev/" "$__ARMEmulRootfs/dev" @@ -209,9 +209,9 @@ function mount_emulator { fi mount_with_checking "-o bind" "/mnt" "$__ARMEmulRootfs/bindings/tmp" - if [ ! -d "$__ARMEmulRootfs/$__TempFolder" ]; then + if [ ! -d "$__ARMEmulRootfs/$__TempFolder" ]; then sudo mkdir "$__ARMEmulRootfs/$__TempFolder" - fi + fi } #Cross builds coreclr @@ -354,7 +354,7 @@ function run_tests { sudo chroot $__ARMEmulRootfs /bin/bash -x <<EOF cd "$__ARMEmulCoreclr" ./tests/runtest.sh --sequential\ - --testRootDir=$__testRootDirBase \ + --testRootDir=$__testRootDirBase \ --mscorlibDir=$__mscorlibDirBase \ --coreFxNativeBinDir=$__coreFxNativeBinDirBase \ --coreFxBinDir="$__coreFxBinDirBase" \ @@ -509,7 +509,7 @@ if [ "$__ciMode" == "emulator" ]; then __skipTests=1 fi -__coreFxBinDir="./bin/CoreFxBinDir" # TODO-clenup: Just for testing.... +__coreFxBinDir="./bin/CoreFxBinDir" # TODO-cleanup: Just for testing.... #Check if the optional arguments are present in the case that testing is to be done if [ $__skipTests == 0 ]; then exit_if_empty "$__testRootDir" "Testing requested, but --testRootDir not provided" true diff --git a/tests/testsFailing.arm.txt b/tests/testsFailing.arm.txt new file mode 100644 index 0000000000..68ad6a3226 --- /dev/null +++ b/tests/testsFailing.arm.txt @@ -0,0 +1 @@ +GC/API/GC/GetAllocatedBytesForCurrentThread/GetAllocatedBytesForCurrentThread.sh |