diff options
| -rwxr-xr-x | netci.groovy | 3839 | ||||
| -rw-r--r-- | perf.groovy | 957 | ||||
| -rw-r--r-- | tests/CoreFX/CoreFX.issues.json | 1880 | ||||
| -rw-r--r-- | tests/CoreFX/CoreFXTestListURL.txt | 1 | ||||
| -rw-r--r-- | tests/CoreFX/CoreFXTestListURL_Linux.txt | 1 | ||||
| -rw-r--r-- | tests/CoreFX/CoreFXTestListURL_OSX.txt | 1 | ||||
| -rw-r--r-- | tests/src/Common/CoreFX/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj | 36 | ||||
| -rw-r--r-- | tests/src/Common/CoreFX/TestFileSetup/Helpers/TestFileHelper.cs | 326 | ||||
| -rw-r--r-- | tests/src/Common/CoreFX/TestFileSetup/Helpers/TestRunHelper.cs | 158 | ||||
| -rw-r--r-- | tests/src/Common/CoreFX/TestFileSetup/Program.cs | 174 | ||||
| -rw-r--r-- | tests/src/Common/CoreFX/TestFileSetup/RSPGenerator.cs | 83 | ||||
| -rw-r--r-- | tests/src/Common/CoreFX/TestFileSetup/XUnit/XUnitTestAssembly.cs | 63 |
12 files changed, 0 insertions, 7519 deletions
diff --git a/netci.groovy b/netci.groovy deleted file mode 100755 index bb31ba93d3..0000000000 --- a/netci.groovy +++ /dev/null @@ -1,3839 +0,0 @@ -// Import the utility functionality. - -import jobs.generation.* - -// The input project name (e.g. dotnet/coreclr) -def project = GithubProject -// The input branch name (e.g. master) -def branch = GithubBranchName -def projectFolder = Utilities.getFolderName(project) + '/' + Utilities.getFolderName(branch) - -// Create a folder for JIT stress jobs and associated folder views -folder('jitstress') -Utilities.addStandardFolderView(this, 'jitstress', project) - -// Create a folder for testing via illink -folder('illink') -Utilities.addStandardFolderView(this, 'illink', project) - -def static getOSGroup(def os) { - def osGroupMap = ['Ubuntu' : 'Linux', - 'Ubuntu16.04' : 'Linux', - 'Ubuntu16.10' : 'Linux', - 'RHEL7.2' : 'Linux', - 'Debian8.4' : 'Linux', - 'Fedora24' : 'Linux', - 'CentOS7.1' : 'Linux', - 'Tizen' : 'Linux', - 'OSX10.12' : 'OSX', - 'Windows_NT' : 'Windows_NT'] - def osGroup = osGroupMap.get(os, null) - assert osGroup != null : "Could not find os group for ${os}" - return osGroupMap[os] -} - -// We use this class (vs variables) so that the static functions can access data here. -class Constants { - - // We have very limited Windows ARM64 hardware (used for ARM/ARM64 testing) and Linux/arm32 and Linux/arm64 hardware. - // So only allow certain branches to use it. - def static LimitedHardwareBranches = [ - 'master'] - - // Innerloop build OS's - // The Windows_NT_BuildOnly OS is a way to speed up the Non-Windows builds by avoiding - // test execution in the build flow runs. It generates the exact same build - // as Windows_NT but without running the tests. - def static osList = [ - 'Ubuntu', - 'Debian8.4', - 'OSX10.12', - 'Windows_NT', - 'Windows_NT_BuildOnly', - 'CentOS7.1', - 'RHEL7.2', - 'Ubuntu16.04', - 'Ubuntu16.10', - 'Fedora24', - 'Tizen'] - - def static crossList = [ - 'Ubuntu', - 'Debian8.4', - 'OSX10.12', - 'Windows_NT', - 'CentOS7.1', - 'RHEL7.2'] - - // This is a set of JIT stress modes combined with the set of variables that - // need to be set to actually enable that stress mode. The key of the map is the stress mode and - // the values are the environment variables - def static jitStressModeScenarios = [ - 'minopts' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'], - 'tieredcompilation' : ['COMPlus_TieredCompilation' : '1'], // this can be removed once tiered compilation is on by default - 'no_tiered_compilation' : ['COMPlus_TieredCompilation' : '0'], - 'no_tiered_compilation_innerloop': ['COMPlus_TieredCompilation' : '0'], - 'forcerelocs' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ForceRelocs' : '1'], - 'jitstress1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'], - 'jitstress2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'], - 'jitstress1_tiered' : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '1'], - 'jitstress2_tiered' : ['COMPlus_TieredCompilation' : '1', 'COMPlus_JitStress' : '2'], - 'jitstressregs1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'], - 'jitstressregs2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'], - 'jitstressregs3' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'], - 'jitstressregs4' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'], - 'jitstressregs8' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'], - 'jitstressregs0x10' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'], - 'jitstressregs0x80' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'], - 'jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'], - 'jitstress2_jitstressregs1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '1'], - 'jitstress2_jitstressregs2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '2'], - 'jitstress2_jitstressregs3' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '3'], - 'jitstress2_jitstressregs4' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '4'], - 'jitstress2_jitstressregs8' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '8'], - 'jitstress2_jitstressregs0x10' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x10'], - 'jitstress2_jitstressregs0x80' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x80'], - 'jitstress2_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2', 'COMPlus_JitStressRegs' : '0x1000'], - 'tailcallstress' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_TailcallStress' : '1'], - 'jitsse2only' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableAVX' : '0', 'COMPlus_EnableSSE3_4' : '0'], - 'jitnosimd' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_FeatureSIMD' : '0'], - 'jitincompletehwintrinsic' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1'], - 'jitx86hwintrinsicnoavx' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX' : '0'], // testing the legacy SSE encoding - 'jitx86hwintrinsicnoavx2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableAVX2' : '0'], // testing SNB/IVB - 'jitx86hwintrinsicnosimd' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_FeatureSIMD' : '0'], // match "jitnosimd", may need to remove after decoupling HW intrinsic from FeatureSIMD - 'jitnox86hwintrinsic' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_EnableIncompleteISAClass' : '1', 'COMPlus_EnableSSE' : '0' , 'COMPlus_EnableSSE2' : '0' , 'COMPlus_EnableSSE3' : '0' , 'COMPlus_EnableSSSE3' : '0' , 'COMPlus_EnableSSE41' : '0' , 'COMPlus_EnableSSE42' : '0' , 'COMPlus_EnableAVX' : '0' , 'COMPlus_EnableAVX2' : '0' , 'COMPlus_EnableAES' : '0' , 'COMPlus_EnableBMI1' : '0' , 'COMPlus_EnableBMI2' : '0' , 'COMPlus_EnableFMA' : '0' , 'COMPlus_EnableLZCNT' : '0' , 'COMPlus_EnablePCLMULQDQ' : '0' , 'COMPlus_EnablePOPCNT' : '0'], - 'corefx_baseline' : ['COMPlus_TieredCompilation' : '0'], // corefx baseline - 'corefx_minopts' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JITMinOpts' : '1'], - 'corefx_tieredcompilation' : ['COMPlus_TieredCompilation' : '1'], // this can be removed once tiered compilation is on by default - 'corefx_jitstress1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '1'], - 'corefx_jitstress2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStress' : '2'], - 'corefx_jitstressregs1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '1'], - 'corefx_jitstressregs2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '2'], - 'corefx_jitstressregs3' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '3'], - 'corefx_jitstressregs4' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '4'], - 'corefx_jitstressregs8' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '8'], - 'corefx_jitstressregs0x10' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x10'], - 'corefx_jitstressregs0x80' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x80'], - 'corefx_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_JitStressRegs' : '0x1000'], - 'gcstress0x3' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0x3'], - 'gcstress0xc' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC'], - 'zapdisable' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'], - 'heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_HeapVerify' : '1'], - 'gcstress0xc_zapdisable' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0'], - 'gcstress0xc_zapdisable_jitstress2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_JitStress' : '2'], - 'gcstress0xc_zapdisable_heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_ZapDisable' : '1', 'COMPlus_ReadyToRun' : '0', 'COMPlus_HeapVerify' : '1'], - 'gcstress0xc_jitstress1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress' : '1'], - 'gcstress0xc_jitstress2' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JitStress' : '2'], - 'gcstress0xc_minopts_heapverify1' : ['COMPlus_TieredCompilation' : '0', 'COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1'] - ] - - // This is a set of ReadyToRun stress scenarios - def static r2rStressScenarios = [ - 'r2r_jitstress1' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "1"], - 'r2r_jitstress2' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStress": "2"], - 'r2r_jitstress1_tiered' : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "1"], - 'r2r_jitstress2_tiered' : ['COMPlus_TieredCompilation' : '1', "COMPlus_JitStress": "2"], - 'r2r_jitstressregs1' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "1"], - 'r2r_jitstressregs2' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "2"], - 'r2r_jitstressregs3' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "3"], - 'r2r_jitstressregs4' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "4"], - 'r2r_jitstressregs8' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "8"], - 'r2r_jitstressregs0x10' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x10"], - 'r2r_jitstressregs0x80' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x80"], - 'r2r_jitstressregs0x1000' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JitStressRegs": "0x1000"], - 'r2r_jitminopts' : ['COMPlus_TieredCompilation' : '0', "COMPlus_JITMinOpts": "1"], - 'r2r_jitforcerelocs' : ['COMPlus_TieredCompilation' : '0', "COMPlus_ForceRelocs": "1"], - 'r2r_gcstress15' : ['COMPlus_TieredCompilation' : '0', "COMPlus_GCStress": "0xF"], - 'r2r_no_tiered_compilation' : ['COMPlus_TieredCompilation' : '0'], - ] - - // This is the basic set of scenarios - def static basicScenarios = [ - 'innerloop', - 'normal', - 'ilrt', - 'r2r', - 'longgc', - 'gcsimulator', - // 'jitdiff', // jitdiff is currently disabled, until someone spends the effort to make it fully work - 'standalone_gc', - 'gc_reliability_framework', - 'illink', - 'corefx_innerloop', - 'crossgen_comparison', - 'pmi_asm_diffs'] - - def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet() - - // Valid PR trigger combinations. - def static prTriggeredValidInnerLoopCombos = [ - 'Windows_NT': [ - 'arm': [ - 'Checked' - ], - 'arm64': [ - 'Checked' - ] - ], - 'Windows_NT_BuildOnly': [ - 'arm': [ - 'Checked' - ], - ] - ] - - // A set of scenarios that are valid for arm/arm64 tests run on hardware. This is a map from valid scenario name - // to Tests.lst file categories to exclude. - // - // This list should contain a subset of the scenarios from `allScenarios`. Please keep this in the same order as that, - // and with the same values, with some commented out, for easier maintenance. - // - // Note that some scenarios that are commented out should be enabled, but haven't yet been. - // - def static validArmWindowsScenarios = [ - 'innerloop', - 'normal', - // 'ilrt' - 'r2r', - // 'longgc' - // 'formatting' - // 'gcsimulator' - // 'jitdiff' - // 'standalone_gc' - // 'gc_reliability_framework' - // 'illink' - // 'corefx_innerloop' - // 'crossgen_comparison' - // 'pmi_asm_diffs' - 'r2r_jitstress1', - 'r2r_jitstress2', - 'r2r_jitstress1_tiered', - 'r2r_jitstress2_tiered', - 'r2r_jitstressregs1', - 'r2r_jitstressregs2', - 'r2r_jitstressregs3', - 'r2r_jitstressregs4', - 'r2r_jitstressregs8', - 'r2r_jitstressregs0x10', - 'r2r_jitstressregs0x80', - 'r2r_jitstressregs0x1000', - 'r2r_jitminopts', - 'r2r_jitforcerelocs', - 'r2r_gcstress15', - 'r2r_no_tiered_compilation', - 'minopts', - 'tieredcompilation', - 'no_tiered_compilation', - 'no_tiered_compilation_innerloop', - 'forcerelocs', - 'jitstress1', - 'jitstress2', - 'jitstress1_tiered', - 'jitstress2_tiered', - 'jitstressregs1', - 'jitstressregs2', - 'jitstressregs3', - 'jitstressregs4', - 'jitstressregs8', - 'jitstressregs0x10', - 'jitstressregs0x80', - 'jitstressregs0x1000', - 'jitstress2_jitstressregs1', - 'jitstress2_jitstressregs2', - 'jitstress2_jitstressregs3', - 'jitstress2_jitstressregs4', - 'jitstress2_jitstressregs8', - 'jitstress2_jitstressregs0x10', - 'jitstress2_jitstressregs0x80', - 'jitstress2_jitstressregs0x1000', - 'tailcallstress', - // 'jitsse2only' // Only relevant to xarch - 'jitnosimd', // Only interesting on platforms where SIMD support exists. - // 'jitincompletehwintrinsic' - // 'jitx86hwintrinsicnoavx' - // 'jitx86hwintrinsicnoavx2' - // 'jitx86hwintrinsicnosimd' - // 'jitnox86hwintrinsic' - 'corefx_baseline', // corefx tests don't use smarty - 'corefx_minopts', // corefx tests don't use smarty - 'corefx_tieredcompilation', // corefx tests don't use smarty - 'corefx_jitstress1', // corefx tests don't use smarty - 'corefx_jitstress2', // corefx tests don't use smarty - 'corefx_jitstressregs1', // corefx tests don't use smarty - 'corefx_jitstressregs2', // corefx tests don't use smarty - 'corefx_jitstressregs3', // corefx tests don't use smarty - 'corefx_jitstressregs4', // corefx tests don't use smarty - 'corefx_jitstressregs8', // corefx tests don't use smarty - 'corefx_jitstressregs0x10', // corefx tests don't use smarty - 'corefx_jitstressregs0x80', // corefx tests don't use smarty - 'corefx_jitstressregs0x1000', // corefx tests don't use smarty - 'gcstress0x3', - 'gcstress0xc', - 'zapdisable', - 'heapverify1', - 'gcstress0xc_zapdisable', - 'gcstress0xc_zapdisable_jitstress2', - 'gcstress0xc_zapdisable_heapverify1', - 'gcstress0xc_jitstress1', - 'gcstress0xc_jitstress2', - 'gcstress0xc_minopts_heapverify1', - - // - // NOTE: the following scenarios are not defined in the 'allScenarios' list! Is this a bug? - // - - 'minopts_zapdisable', - 'gcstress0x3_jitstress1', - 'gcstress0x3_jitstress2', - 'gcstress0x3_jitstressregs1', - 'gcstress0x3_jitstressregs2', - 'gcstress0x3_jitstressregs3', - 'gcstress0x3_jitstressregs4', - 'gcstress0x3_jitstressregs8', - 'gcstress0x3_jitstressregs0x10', - 'gcstress0x3_jitstressregs0x80', - 'gcstress0x3_jitstressregs0x1000', - 'gcstress0xc_jitstressregs1', - 'gcstress0xc_jitstressregs2', - 'gcstress0xc_jitstressregs3', - 'gcstress0xc_jitstressregs4', - 'gcstress0xc_jitstressregs8', - 'gcstress0xc_jitstressregs0x10', - 'gcstress0xc_jitstressregs0x80', - 'gcstress0xc_jitstressregs0x1000' - ] - - def static validLinuxArmScenarios = [ - 'innerloop', - 'normal', - // 'ilrt' - 'r2r', - // 'longgc' - // 'formatting' - // 'gcsimulator' - // 'jitdiff' - // 'standalone_gc' - // 'gc_reliability_framework' - // 'illink' - // 'corefx_innerloop' - 'crossgen_comparison', - 'pmi_asm_diffs', - 'r2r_jitstress1', - 'r2r_jitstress2', - 'r2r_jitstress1_tiered', - 'r2r_jitstress2_tiered', - 'r2r_jitstressregs1', - 'r2r_jitstressregs2', - 'r2r_jitstressregs3', - 'r2r_jitstressregs4', - 'r2r_jitstressregs8', - 'r2r_jitstressregs0x10', - 'r2r_jitstressregs0x80', - 'r2r_jitstressregs0x1000', - 'r2r_jitminopts', - 'r2r_jitforcerelocs', - 'r2r_gcstress15', - 'r2r_no_tiered_compilation', - 'minopts', - 'tieredcompilation', - 'no_tiered_compilation', - 'no_tiered_compilation_innerloop', - 'forcerelocs', - 'jitstress1', - 'jitstress2', - 'jitstress1_tiered', - 'jitstress2_tiered', - 'jitstressregs1', - 'jitstressregs2', - 'jitstressregs3', - 'jitstressregs4', - 'jitstressregs8', - 'jitstressregs0x10', - 'jitstressregs0x80', - 'jitstressregs0x1000', - 'jitstress2_jitstressregs1', - 'jitstress2_jitstressregs2', - 'jitstress2_jitstressregs3', - 'jitstress2_jitstressregs4', - 'jitstress2_jitstressregs8', - 'jitstress2_jitstressregs0x10', - 'jitstress2_jitstressregs0x80', - 'jitstress2_jitstressregs0x1000', - 'tailcallstress', - // 'jitsse2only' // Only relevant to xarch - // 'jitnosimd' // Only interesting on platforms where SIMD support exists. - // 'jitincompletehwintrinsic' - // 'jitx86hwintrinsicnoavx' - // 'jitx86hwintrinsicnoavx2' - // 'jitx86hwintrinsicnosimd' - // 'jitnox86hwintrinsic' - 'corefx_baseline', - 'corefx_minopts', - 'corefx_tieredcompilation', - 'corefx_jitstress1', - 'corefx_jitstress2', - 'corefx_jitstressregs1', - 'corefx_jitstressregs2', - 'corefx_jitstressregs3', - 'corefx_jitstressregs4', - 'corefx_jitstressregs8', - 'corefx_jitstressregs0x10', - 'corefx_jitstressregs0x80', - 'corefx_jitstressregs0x1000', - 'gcstress0x3', - 'gcstress0xc', - 'zapdisable', - 'heapverify1', - 'gcstress0xc_zapdisable', - 'gcstress0xc_zapdisable_jitstress2', - 'gcstress0xc_zapdisable_heapverify1', - 'gcstress0xc_jitstress1', - 'gcstress0xc_jitstress2', - 'gcstress0xc_minopts_heapverify1' - ] - - def static validLinuxArm64Scenarios = [ - 'innerloop', - 'normal', - // 'ilrt' - 'r2r', - // 'longgc' - // 'formatting' - // 'gcsimulator' - // 'jitdiff' - // 'standalone_gc' - // 'gc_reliability_framework' - // 'illink' - // 'corefx_innerloop' - 'crossgen_comparison', - 'pmi_asm_diffs', - 'r2r_jitstress1', - 'r2r_jitstress2', - 'r2r_jitstress1_tiered', - 'r2r_jitstress2_tiered', - 'r2r_jitstressregs1', - 'r2r_jitstressregs2', - 'r2r_jitstressregs3', - 'r2r_jitstressregs4', - 'r2r_jitstressregs8', - 'r2r_jitstressregs0x10', - 'r2r_jitstressregs0x80', - 'r2r_jitstressregs0x1000', - 'r2r_jitminopts', - 'r2r_jitforcerelocs', - 'r2r_gcstress15', - 'r2r_no_tiered_compilation', - 'minopts', - 'tieredcompilation', - 'no_tiered_compilation', - 'no_tiered_compilation_innerloop', - 'forcerelocs', - 'jitstress1', - 'jitstress2', - 'jitstress1_tiered', - 'jitstress2_tiered', - 'jitstressregs1', - 'jitstressregs2', - 'jitstressregs3', - 'jitstressregs4', - 'jitstressregs8', - 'jitstressregs0x10', - 'jitstressregs0x80', - 'jitstressregs0x1000', - 'jitstress2_jitstressregs1', - 'jitstress2_jitstressregs2', - 'jitstress2_jitstressregs3', - 'jitstress2_jitstressregs4', - 'jitstress2_jitstressregs8', - 'jitstress2_jitstressregs0x10', - 'jitstress2_jitstressregs0x80', - 'jitstress2_jitstressregs0x1000', - 'tailcallstress', - // 'jitsse2only' // Only relevant to xarch - 'jitnosimd', // Only interesting on platforms where SIMD support exists. - // 'jitincompletehwintrinsic' - // 'jitx86hwintrinsicnoavx' - // 'jitx86hwintrinsicnoavx2' - // 'jitx86hwintrinsicnosimd' - // 'jitnox86hwintrinsic' - 'corefx_baseline', - 'corefx_minopts', - 'corefx_tieredcompilation', - 'corefx_jitstress1', - 'corefx_jitstress2', - 'corefx_jitstressregs1', - 'corefx_jitstressregs2', - 'corefx_jitstressregs3', - 'corefx_jitstressregs4', - 'corefx_jitstressregs8', - 'corefx_jitstressregs0x10', - 'corefx_jitstressregs0x80', - 'corefx_jitstressregs0x1000', - 'gcstress0x3', - 'gcstress0xc', - 'zapdisable', - 'heapverify1', - 'gcstress0xc_zapdisable', - 'gcstress0xc_zapdisable_jitstress2', - 'gcstress0xc_zapdisable_heapverify1', - 'gcstress0xc_jitstress1', - 'gcstress0xc_jitstress2', - 'gcstress0xc_minopts_heapverify1' - ] - - def static configurationList = ['Debug', 'Checked', 'Release'] - - // This is the set of architectures - // Some of these are pseudo-architectures: - // armem -- ARM builds/runs using an emulator. Used for Tizen runs. - def static architectureList = ['arm', 'armem', 'arm64', 'x64', 'x86'] - - // This set of architectures that cross build on Windows and run on Windows ARM64 hardware. - def static armWindowsCrossArchitectureList = ['arm', 'arm64'] -} - -// ************************************************************** -// Create some specific views -// -// These aren't using the Utilities.addStandardFolderView() function, because that creates -// views based on a single regular expression. These views will be generated by adding a -// specific set of jobs to them. -// -// Utilities.addStandardFolderView() also creates a lot of additional stuff around the -// view, like "Build Statistics", "Job Statistics", "Unstable Jobs". Until it is determined -// those are required, don't add them (which simplifies the view pages, as well). -// ************************************************************** - -class Views { - def static MergeJobView = null - def static PeriodicJobView = null - def static ArchitectureViews = [:] - def static OSViews = [:] -} - -// MergeJobView: include all jobs that execute when a PR change is merged. -Views.MergeJobView = listView('Merge') { - recurse() - columns { - status() - weather() - name() - lastSuccess() - lastFailure() - lastDuration() - buildButton() - } -} - -// PeriodicJobView: include all jobs that execute on a schedule -Views.PeriodicJobView = listView('Periodic') { - recurse() - columns { - status() - weather() - name() - lastSuccess() - lastFailure() - lastDuration() - buildButton() - } -} - -// Create a view for non-PR jobs for each architecture. -Constants.architectureList.each { architecture -> - Views.ArchitectureViews[architecture] = listView(architecture) { - recurse() - columns { - status() - weather() - name() - lastSuccess() - lastFailure() - lastDuration() - buildButton() - } - } -} - -// Create a view for non-PR jobs for each OS. -Constants.osList.each { os -> - // Don't create one for the special 'Windows_NT_BuildOnly' - if (os == 'Windows_NT_BuildOnly') { - return - } - Views.OSViews[os] = listView(os) { - recurse() - columns { - status() - weather() - name() - lastSuccess() - lastFailure() - lastDuration() - buildButton() - } - } -} - -def static addToMergeView(def job) { - Views.MergeJobView.with { - jobs { - name(job.name) - } - } -} - -def static addToPeriodicView(def job) { - Views.PeriodicJobView.with { - jobs { - name(job.name) - } - } -} - -def static addToViews(def job, def isFlowJob, def isPR, def architecture, def os, def configuration, def scenario) { - if (isPR) { - // No views want PR jobs currently. - return - } - - // We don't want to include in view any job that is only used by a flow job (because we want the views to have only the - // "top-level" jobs. Build only jobs are such jobs. - if (os == 'Windows_NT_BuildOnly') { - return - } - - if (!isFlowJob) { - // For non-flow jobs, which ones are only used by flow jobs? - if ((architecture == 'arm') || (architecture == 'arm64')) { - if (isCoreFxScenario(scenario)) { - // We have corefx-specific scenario builds for each of the runs, but these are driven by flow jobs. - return - } - - // We're left with the basic normal/innerloop builds. We might want to remove these from the views also, if desired. - // However, there are a few, like the Debug Build, that is build only, not "Build and Test", that we should leave. - } - } - - // Add to architecture view. - Views.ArchitectureViews[architecture].with { - jobs { - name(job.name) - } - } - - // Add to OS view. - Views.OSViews[os].with { - jobs { - name(job.name) - } - } -} - -def static addPeriodicTriggerHelper(def job, String cronString, boolean alwaysRuns = false) { - addToPeriodicView(job) - Utilities.addPeriodicTrigger(job, cronString, alwaysRuns) -} - -def static addGithubPushTriggerHelper(def job) { - // Disable all Push trigger jobs. All jobs will need to be requested. - // addToMergeView(job) - // Utilities.addGithubPushTrigger(job) -} - - -def static setMachineAffinity(def job, def os, def architecture, def options = null) { - assert os instanceof String - assert architecture instanceof String - - def armArches = ['arm', 'armem', 'arm64'] - - if (!(architecture in armArches)) { - assert options == null - Utilities.setMachineAffinity(job, os, 'latest-or-auto') - - return - } - - // This is an arm(64) job. - // - // There are several options. - // - // Windows_NT - // - // Arm32 (Build) -> latest-arm64 - // |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == true - // Arm32 (Test) -> arm64-windows_nt - // |-> os == "Windows_NT" && (architecture == "arm") && options['use_arm64_build_machine'] == false - // - // Arm64 (Build) -> latest-arm64 - // |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == true - // Arm64 (Test) -> arm64-windows_nt - // |-> os == "Windows_NT" && architecture == "arm64" && options['use_arm64_build_machine'] == false - // - // Ubuntu - // - // Arm32 emulator (Build, Test) -> arm-cross-latest - // |-> os == "Tizen" && (architecture == "armem") - // - // Arm32 hardware (Flow) -> Ubuntu 16.04 latest-or-auto (don't use limited arm hardware) - // |-> os == "Ubuntu" && (architecture == "arm") && options['is_flow_job'] == true - // Arm32 hardware (Build) -> Ubuntu 16.04 latest-or-auto - // |-> os == "Ubuntu" && (architecture == "arm") && options['is_build_job'] == true - // Arm32 hardware (Test) -> Helix ubuntu.1404.arm32.open queue - // |-> os == "Ubuntu" && (architecture == "arm") - // - // Arm64 (Build) -> arm64-cross-latest - // |-> os != "Windows_NT" && architecture == "arm64" && options['is_build_job'] == true - // Arm64 (Test) -> Helix Ubuntu.1604.Arm64.Iron.Open queue - // |-> os != "Windows_NT" && architecture == "arm64" - // - // Note: we are no longer using Jenkins tags "arm64-huge-page-size", "arm64-small-page-size". - // Support for Linux arm64 large page size has been removed for now, as it wasn't being used. - // - // Note: we are no longer using Jenkins tag 'latest-arm64' for arm/arm64 Windows build machines. Instead, - // we are using public VS2017 arm/arm64 tools in a VM from Helix. - - // This has to be a arm arch - assert architecture in armArches - if (os == "Windows_NT") { - // arm32/arm64 Windows jobs share the same machines for now - def isBuild = options['use_arm64_build_machine'] == true - - if (isBuild == true) { - job.with { - label('Windows.10.Amd64.ClientRS4.DevEx.Open') - } - } else { - Utilities.setMachineAffinity(job, 'windows.10.arm64.open') - } - } else { - assert os != 'Windows_NT' - - if (architecture == 'armem') { - // arm emulator (Tizen). Build and test on same machine, - // using Docker. - assert os == 'Tizen' - Utilities.setMachineAffinity(job, 'Ubuntu', 'arm-cross-latest') - } - else { - // arm/arm64 Ubuntu on hardware. - assert architecture == 'arm' || architecture == 'arm64' - def isFlow = (options != null) && (options['is_flow_job'] == true) - def isBuild = (options != null) && (options['is_build_job'] == true) - if (isFlow || isBuild) { - // arm/arm64 Ubuntu build machine. Build uses docker, so the actual host OS is not - // very important. Therefore, use latest or auto. Flow jobs don't need to use arm hardware. - Utilities.setMachineAffinity(job, 'Ubuntu16.04', 'latest-or-auto') - } else { - // arm/arm64 Ubuntu test machine. Specify the Helix queue name here. - if (architecture == 'arm64') { - assert os == 'Ubuntu16.04' - job.with { - label('Ubuntu.1604.Arm64.Iron.Open') - } - } - else { - assert os == 'Ubuntu' - job.with { - label('ubuntu.1404.arm32.open') - } - } - } - } - } -} - -// setJobMachineAffinity: compute the machine affinity options for a job, -// then set the job with those affinity options. -def static setJobMachineAffinity(def architecture, def os, def isBuildJob, def isTestJob, def isFlowJob, def job) -{ - assert (isBuildJob && !isTestJob && !isFlowJob) || - (!isBuildJob && isTestJob && !isFlowJob) || - (!isBuildJob && !isTestJob && isFlowJob) - - def affinityOptions = null - def affinityArchitecture = architecture - - if (os == "Windows_NT") { - if (architecture in Constants.armWindowsCrossArchitectureList) { - if (isBuildJob) { - affinityOptions = [ "use_arm64_build_machine" : true ] - } else if (isTestJob) { - affinityOptions = [ "use_arm64_build_machine" : false ] - } else if (isFlowJob) { - // For the flow jobs set the machine affinity as x64 - affinityArchitecture = 'x64' - } - } - } - else { - if ((architecture == 'arm64') || (architecture == 'arm')) { - if (isBuildJob) { - affinityOptions = ['is_build_job': true] - } else if (isFlowJob) { - affinityOptions = ['is_flow_job': true] - } - } - } - - setMachineAffinity(job, os, affinityArchitecture, affinityOptions) -} - -def static isGCStressRelatedTesting(def scenario) { - // The 'r2r_gcstress15' scenario is a basic scenario. - // Detect it and make it a GCStress related. - if (scenario == 'r2r_gcstress15') - { - return true; - } - - def gcStressTestEnvVars = [ 'COMPlus_GCStress', 'COMPlus_ZapDisable', 'COMPlus_HeapVerify'] - def scenarioName = scenario.toLowerCase() - def isGCStressTesting = false - Constants.jitStressModeScenarios[scenario].each{ k, v -> - if (k in gcStressTestEnvVars) { - isGCStressTesting = true; - } - } - return isGCStressTesting -} - -def static isCoreFxScenario(def scenario) { - def corefx_prefix = 'corefx_' - if (scenario.length() < corefx_prefix.length()) { - return false - } - return scenario.substring(0,corefx_prefix.length()) == corefx_prefix -} - -def static isR2RBaselineScenario(def scenario) { - return (scenario == 'r2r') -} - -def static isR2RStressScenario(def scenario) { - return Constants.r2rStressScenarios.containsKey(scenario) -} - -def static isR2RScenario(def scenario) { - return isR2RBaselineScenario(scenario) || isR2RStressScenario(scenario) -} - -def static isJitStressScenario(def scenario) { - return Constants.jitStressModeScenarios.containsKey(scenario) -} - -def static isLongGc(def scenario) { - return (scenario == 'longgc' || scenario == 'gcsimulator') -} - -def static isJitDiff(def scenario) { - return (scenario == 'jitdiff') -} - -def static isGcReliabilityFramework(def scenario) { - return (scenario == 'gc_reliability_framework') -} - -def static isArmWindowsScenario(def scenario) { - return Constants.validArmWindowsScenarios.contains(scenario) -} - -def static isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly) { - if (isBuildOnly == true) { - os = 'Windows_NT_BuildOnly' - } - - def validOsPrTriggerArchConfigs = Constants.prTriggeredValidInnerLoopCombos[os] - if (validOsPrTriggerArchConfigs != null) { - def validOsPrTriggerConfigs = validOsPrTriggerArchConfigs[architecture] - if (validOsPrTriggerConfigs != null) { - if (configuration in validOsPrTriggerConfigs) { - return true - } - } - } - - return false -} - -// This means the job builds and runs the 'innerloop' test set. This does not mean the job is -// scheduled with a default PR trigger despite the correlation being true at the moment. -def static isInnerloopTestScenario(def scenario) { - return (scenario == 'innerloop' || scenario == 'no_tiered_compilation_innerloop') -} - -def static isCrossGenComparisonScenario(def scenario) { - return (scenario == 'crossgen_comparison') -} - -def static shouldGenerateCrossGenComparisonJob(def os, def architecture, def configuration, def scenario) { - assert isCrossGenComparisonScenario(scenario) - return ((os == 'Ubuntu' && architecture == 'arm') || (os == 'Ubuntu16.04' && architecture == 'arm64')) && (configuration == 'Checked' || configuration == 'Release') -} - -def static getFxBranch(def branch) { - def fxBranch = branch - // Map 'dev/unix_test_workflow' to 'master' so we can test CoreFX jobs in the CoreCLR dev/unix_test_workflow - // branch even though CoreFX doesn't have such a branch. - if (branch == 'dev/unix_test_workflow') { - fxBranch = 'master' - } - return fxBranch -} - -def static setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) { - // 2 hours (120 minutes) is the default timeout - def timeout = 120 - - if (!isInnerloopTestScenario(scenario)) { - // Pri-1 test builds take a long time (see calculateBuildCommands()). So up the Pri-1 build jobs timeout. - timeout = 240 - } - - if (!isBuildOnly) { - // Note that these can only increase, never decrease, the Pri-1 timeout possibly set above. - if (isGCStressRelatedTesting(scenario)) { - timeout = 4320 - } - else if (isCoreFxScenario(scenario)) { - timeout = 360 - } - else if (isJitStressScenario(scenario)) { - timeout = 300 - } - else if (isR2RBaselineScenario(scenario)) { - timeout = 240 - } - else if (isLongGc(scenario)) { - timeout = 1440 - } - else if (isJitDiff(scenario)) { - timeout = 240 - } - else if (isGcReliabilityFramework(scenario)) { - timeout = 1440 - } - else if (architecture == 'armem' || architecture == 'arm64') { - timeout = 240 - } - - if (architecture == 'arm') { - // ARM32 machines are particularly slow. - timeout += 120 - } - } - - if (configuration == 'Debug') { - // Debug runs can be very slow. Add an hour. - timeout += 60 - } - - // If we've changed the timeout from the default, set it in the job. - - if (timeout != 120) { - Utilities.setJobTimeout(newJob, timeout) - } -} - -def static getJobFolder(def scenario) { - if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) { - return 'jitstress' - } - if (scenario == 'illink') { - return 'illink' - } - return '' -} - -def static getStressModeDisplayName(def scenario) { - def displayStr = '' - Constants.jitStressModeScenarios[scenario].each{ k, v -> - def prefixLength = 'COMPlus_'.length() - if (k.length() >= prefixLength) { - def modeName = k.substring(prefixLength, k.length()) - if (displayStr != '') { - // Separate multiple variables with a space. - displayStr += ' ' - } - displayStr += modeName + '=' + v - } - } - - if (isCoreFxScenario(scenario)) { - displayStr = ('CoreFx ' + displayStr).trim() - } - - return displayStr -} - -def static getR2RDisplayName(def scenario) { - // Assume the scenario name is one from the r2rStressScenarios dict, and remove its "r2r_" prefix. - def displayStr = scenario - def prefixLength = 'r2r_'.length() - if (displayStr.length() >= prefixLength) { - displayStr = "R2R " + displayStr.substring(prefixLength, displayStr.length()) - } else if (scenario == 'r2r') { - displayStr = "R2R" - } - return displayStr -} - -def static getScenarioDisplayString(def scenario) { - switch (scenario) { - case 'innerloop': - return "Innerloop Build and Test" - - case 'no_tiered_compilation_innerloop': - def displayStr = getStressModeDisplayName(scenario) - return "Innerloop Build and Test (Jit - ${displayStr})" - - case 'corefx_innerloop': - return "CoreFX Tests" - - case 'normal': - return "Build and Test" - - case 'jitdiff': - return "Jit Diff Build and Test" - - case 'ilrt': - return "IL RoundTrip Build and Test" - - case 'longgc': - return "Long-Running GC Build & Test" - - case 'gcsimulator': - return "GC Simulator" - - case 'standalone_gc': - return "Standalone GC" - - case 'gc_reliability_framework': - return "GC Reliability Framework" - - case 'illink': - return "via ILLink" - - default: - if (isJitStressScenario(scenario)) { - def displayStr = getStressModeDisplayName(scenario) - return "Build and Test (Jit - ${displayStr})" - } - else if (isR2RScenario(scenario)) { - def displayStr = getR2RDisplayName(scenario) - return "${displayStr} Build and Test" - } - else { - return "${scenario}" - } - break - } - - println("Unknown scenario: ${scenario}"); - assert false -} - -// -// Functions to create an environment script. -// envScriptCreate -- initialize the script (call first) -// envScriptFinalize -- finalize the script (call last) -// envScriptSetStressModeVariables -- set stress mode variables in the env script -// envScriptAppendExistingScript -- append an existing script to the generated script -// -// Each script returns a string of commands. Concatenate all the strings together before -// adding them to the builds commands, to make sure they get executed as one Jenkins script. -// - -// Initialize the environment setting script. -def static envScriptCreate(def os, def stepScriptLocation) { - def stepScript = '' - if (os == 'Windows_NT') { - stepScript += "echo Creating TestEnv script\r\n" - stepScript += "if exist ${stepScriptLocation} del ${stepScriptLocation}\r\n" - - // Create at least an empty script. - stepScript += "echo. > ${stepScriptLocation}\r\n" - } - else { - stepScript += "echo Creating environment setting script\n" - stepScript += "echo \\#\\!/usr/bin/env bash > ${stepScriptLocation}\n" - } - - return stepScript -} - -// Generates the string for setting stress mode variables. -def static envScriptSetStressModeVariables(def os, def stressModeVars, def stepScriptLocation) { - def stepScript = '' - if (os == 'Windows_NT') { - stressModeVars.each{ k, v -> - // Write out what we are writing to the script file - stepScript += "echo Setting ${k}=${v}\r\n" - // Write out the set itself to the script file` - stepScript += "echo set ${k}=${v} >> ${stepScriptLocation}\r\n" - } - } - else { - stressModeVars.each{ k, v -> - // Write out what we are writing to the script file - stepScript += "echo Setting ${k}=${v}\n" - // Write out the set itself to the script file` - stepScript += "echo export ${k}=${v} >> ${stepScriptLocation}\n" - } - } - - return stepScript -} - -// Append an existing script to an environment script. -// Returns string of commands to do this. -def static envScriptAppendExistingScript(def os, def appendScript, def stepScriptLocation) { - assert (os == 'Windows_NT') - def stepScript = '' - - stepScript += "echo Appending ${appendScript} to ${stepScriptLocation}\r\n" - stepScript += "type ${appendScript} >> ${stepScriptLocation}\r\n" - - return stepScript -} - -// Finalize an environment setting script. -// Returns string of commands to do this. -def static envScriptFinalize(def os, def stepScriptLocation) { - def stepScript = '' - - if (os == 'Windows_NT') { - // Display the resulting script. This is useful when looking at the output log file. - stepScript += "echo Display the total script ${stepScriptLocation}\r\n" - stepScript += "type ${stepScriptLocation}\r\n" - } - else { - stepScript += "chmod +x ${stepScriptLocation}\n" - } - - return stepScript -} - -def static isNeedDocker(def architecture, def os, def isBuild) { - if (isBuild) { - if (architecture == 'x86' && os == 'Ubuntu') { - return true - } - else if (architecture == 'armem') { - return true - } - else if (architecture == 'arm') { - if (os == 'Ubuntu') { - return true - } - } - else if (architecture == 'arm64') { - if (os == 'Ubuntu16.04') { - return true - } - } - } - else { - if (architecture == 'x86' && os == 'Ubuntu') { - return true - } - } - return false -} - -def static getDockerImageName(def architecture, def os, def isBuild) { - // We must change some docker private images to official later - if (isBuild) { - if (architecture == 'x86' && os == 'Ubuntu') { - return "hseok82/dotnet-buildtools-prereqs:ubuntu-16.04-crossx86-ef0ac75-20175511035548" - } - else if (architecture == 'armem') { - if (os == 'Tizen') { - return "tizendotnet/dotnet-buildtools-prereqs:ubuntu-16.04-cross-e435274-20180426002255-tizen-rootfs-5.0m1" - } - } - else if (architecture == 'arm') { - if (os == 'Ubuntu') { - return "mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-14.04-cross-e435274-20180426002420" - } - } - else if (architecture == 'arm64') { - if (os == 'Ubuntu16.04') { - return "mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921" - } - } - } - else { - if (architecture == 'x86' && os == 'Ubuntu') { - return "hseok82/dotnet-buildtools-prereqs:ubuntu1604_x86_test" - } - } - println("Unknown architecture to use docker: ${architecture} ${os}"); - assert false -} - -def static getTestArtifactsTgzFileName(def osGroup, def architecture, def configuration) { - return "bin-tests-${osGroup}.${architecture}.${configuration}.tgz" -} - -// We have a limited amount of some hardware. For these, scale back the periodic testing we do, -// and only allowing using this hardware in some specific branches. -def static jobRequiresLimitedHardware(def architecture, def os) { - if (architecture == 'arm') { - // arm Windows and Linux hardware is limited. - return true - } - else if (architecture == 'arm64') { - // arm64 Windows and Linux hardware is limited. - return true - } - else { - return false - } -} - -// Calculates the name of the build job based on some typical parameters. -// -def static getJobName(def configuration, def architecture, def os, def scenario, def isBuildOnly) { - // If the architecture is x64, do not add that info into the build name. - // Need to change around some systems and other builds to pick up the right builds - // to do that. - - def suffix = scenario != 'normal' ? "_${scenario}" : ''; - if (isBuildOnly) { - suffix += '_bld' - } - def baseName = '' - switch (architecture) { - case 'x64': - if (scenario == 'normal') { - // For now we leave x64 off of the name for compatibility with other jobs - baseName = configuration.toLowerCase() + '_' + os.toLowerCase() - } - else if (scenario == 'formatting') { - // we don't care about the configuration for the formatting job. It runs all configs - baseName = architecture.toLowerCase() + '_' + os.toLowerCase() - } - else { - baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase() - } - break - case 'armem': - // These are cross builds - assert os == 'Tizen' - baseName = 'armel_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase() - break - case 'arm': - case 'arm64': - // These are cross builds - baseName = architecture.toLowerCase() + '_cross_' + configuration.toLowerCase() + '_' + os.toLowerCase() - break - case 'x86': - baseName = architecture.toLowerCase() + '_' + configuration.toLowerCase() + '_' + os.toLowerCase() - break - default: - println("Unknown architecture: ${architecture}"); - assert false - break - } - - return baseName + suffix -} - -def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob, def bidailyCrossList) { - - // The dev/unix_test_workflow branch is used for Jenkins CI testing. We generally do not need any non-PR - // triggers in the branch, because that would use machine resources unnecessarily. - if (branch == 'dev/unix_test_workflow') { - return - } - - // Limited hardware is restricted for non-PR triggers to certain branches. - if (jobRequiresLimitedHardware(architecture, os) && (!(branch in Constants.LimitedHardwareBranches))) { - return - } - - // Ubuntu x86 CI jobs are failing. Disable non-PR triggered jobs to avoid these constant failures - // until this is fixed. Tracked by https://github.com/dotnet/coreclr/issues/19003. - if (architecture == 'x86' && os == 'Ubuntu') { - return - } - - def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop") - - // Check scenario. - switch (scenario) { - case 'crossgen_comparison': - if (isFlowJob && (configuration == 'Checked' || configuration == 'Release')) { - if (os == 'Ubuntu' && architecture == 'arm') { - // Not enough Linux/arm32 hardware for this. - // addPeriodicTriggerHelper(job, '@daily') - } - if (os == 'Ubuntu16.04' && architecture == 'arm64') { - addPeriodicTriggerHelper(job, '@daily') - } - } - break - - case 'pmi_asm_diffs': - // No non-PR triggers for now. - break - - case 'normal': - switch (architecture) { - case 'x64': - case 'x86': - if (isFlowJob && architecture == 'x86' && os == 'Ubuntu') { - addPeriodicTriggerHelper(job, '@daily') - } - else if (isFlowJob || os == 'Windows_NT' || (architecture == 'x64' && !(os in Constants.crossList))) { - addGithubPushTriggerHelper(job) - } - break - case 'arm64': - if (os == 'Windows_NT') { - if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) { - // We would normally want a per-push trigger, but with limited hardware we can't keep up. - // Do the builds daily. - addPeriodicTriggerHelper(job, '@daily') - } - } - else { - // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job. - if (isFlowJob) { - addPeriodicTriggerHelper(job, '@daily') - } - } - break - case 'arm': - if (os == 'Windows_NT') { - if (isFlowJob || (isNormalOrInnerloop && (configuration == 'Debug'))) { - // We would normally want a push trigger, but with limited hardware we can't keep up. - // Do the builds daily. - addPeriodicTriggerHelper(job, '@daily') - } - } - else { - assert os == 'Ubuntu' - // Only the flow jobs get push triggers; the build and test jobs are triggered by the flow job. - if (isFlowJob) { - // Currently no push triggers, with limited arm Linux hardware. - // TODO: If we have enough machine capacity, add some arm Linux push triggers. - - // Duplicated by AzDO - // addPeriodicTriggerHelper(job, '@daily') - } - } - break - case 'armem': - addGithubPushTriggerHelper(job) - break - default: - println("Unknown architecture: ${architecture}"); - assert false - break - } - break - case 'r2r': - assert !(os in bidailyCrossList) - // r2r gets a push trigger for checked/release - if (configuration == 'Checked' || configuration == 'Release') { - if (architecture == 'x64' && os != 'OSX10.12') { - //Flow jobs should be Windows, Ubuntu, OSX0.12, or CentOS - if (isFlowJob || os == 'Windows_NT') { - addGithubPushTriggerHelper(job) - } - // OSX10.12 r2r jobs should only run every 12 hours, not daily. - } else if (architecture == 'x64' && os == 'OSX10.12'){ - if (isFlowJob) { - addPeriodicTriggerHelper(job, 'H H/12 * * *') - } - } - // For x86, only add per-commit jobs for Windows - else if (architecture == 'x86') { - if (os == 'Windows_NT') { - addGithubPushTriggerHelper(job) - } - } - // arm r2r jobs should only run weekly. - else if (architecture == 'arm') { - if (isFlowJob) { - // Linux arm32 done in AzDO - if (os == 'Windows_NT') { - addPeriodicTriggerHelper(job, '@weekly') - } - } - } - // arm64 r2r jobs should only run weekly. - else if (architecture == 'arm64') { - if (isFlowJob) { - addPeriodicTriggerHelper(job, '@weekly') - } - } - } - break - case 'r2r_jitstress1': - case 'r2r_jitstress2': - case 'r2r_jitstress1_tiered': - case 'r2r_jitstress2_tiered': - case 'r2r_jitstressregs1': - case 'r2r_jitstressregs2': - case 'r2r_jitstressregs3': - case 'r2r_jitstressregs4': - case 'r2r_jitstressregs8': - case 'r2r_jitstressregs0x10': - case 'r2r_jitstressregs0x80': - case 'r2r_jitstressregs0x1000': - case 'r2r_jitminopts': - case 'r2r_jitforcerelocs': - case 'r2r_gcstress15': - case 'r2r_no_tiered_compilation': - assert !(os in bidailyCrossList) - - // GCStress=C is currently not supported on OS X - if (os == 'OSX10.12' && isGCStressRelatedTesting(scenario)) { - break - } - - if (configuration == 'Checked' || configuration == 'Release') { - if (architecture == 'x64') { - //Flow jobs should be Windows, Ubuntu, OSX10.12, or CentOS - if (isFlowJob || os == 'Windows_NT') { - addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday - } - } - // For x86, only add periodic jobs for Windows - else if (architecture == 'x86') { - if (os == 'Windows_NT') { - addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday - } - } - else if (architecture == 'arm') { - if (isFlowJob) { - // Linux arm32 duplicated by AzDO - if (os == 'Windows_NT') { - addPeriodicTriggerHelper(job, '@weekly') - } - } - } - else if (architecture == 'arm64') { - if (isFlowJob) { - addPeriodicTriggerHelper(job, '@weekly') - } - } - } - break - case 'longgc': - assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12') - assert configuration == 'Release' - assert architecture == 'x64' - addPeriodicTriggerHelper(job, '@daily') - // TODO: Add once external email sending is available again - // addEmailPublisher(job, 'dotnetgctests@microsoft.com') - break - case 'gcsimulator': - assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12') - assert configuration == 'Release' - assert architecture == 'x64' - addPeriodicTriggerHelper(job, 'H H * * 3,6') // some time every Wednesday and Saturday - // TODO: Add once external email sending is available again - // addEmailPublisher(job, 'dotnetgctests@microsoft.com') - break - case 'standalone_gc': - assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12') - assert (configuration == 'Release' || configuration == 'Checked') - // TODO: Add once external email sending is available again - // addEmailPublisher(job, 'dotnetgctests@microsoft.com') - addPeriodicTriggerHelper(job, '@daily') - break - case 'gc_reliability_framework': - assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12') - assert (configuration == 'Release' || configuration == 'Checked') - // Only triggered by phrase. - break - case 'ilrt': - assert !(os in bidailyCrossList) - // ILASM/ILDASM roundtrip one gets a daily build, and only for release - if (architecture == 'x64' && configuration == 'Release') { - if (isFlowJob || os == 'Windows_NT') { - addPeriodicTriggerHelper(job, '@daily') - } - } - break - case 'jitdiff': - assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12') - assert configuration == 'Checked' - assert (architecture == 'x64' || architecture == 'x86') - addGithubPushTriggerHelper(job) - break - case 'formatting': - assert (os == 'Windows_NT' || os == "Ubuntu") - assert architecture == 'x64' - addGithubPushTriggerHelper(job) - break - case 'jitstressregs1': - case 'jitstressregs2': - case 'jitstressregs3': - case 'jitstressregs4': - case 'jitstressregs8': - case 'jitstressregs0x10': - case 'jitstressregs0x80': - case 'jitstressregs0x1000': - case 'minopts': - case 'tieredcompilation': - case 'no_tiered_compilation': - case 'forcerelocs': - case 'jitstress1': - case 'jitstress2': - case 'jitstress1_tiered': - case 'jitstress2_tiered': - case 'jitstress2_jitstressregs1': - case 'jitstress2_jitstressregs2': - case 'jitstress2_jitstressregs3': - case 'jitstress2_jitstressregs4': - case 'jitstress2_jitstressregs8': - case 'jitstress2_jitstressregs0x10': - case 'jitstress2_jitstressregs0x80': - case 'jitstress2_jitstressregs0x1000': - case 'tailcallstress': - case 'jitsse2only': - case 'jitnosimd': - case 'jitnox86hwintrinsic': - case 'jitincompletehwintrinsic': - case 'jitx86hwintrinsicnoavx': - case 'jitx86hwintrinsicnoavx2': - case 'jitx86hwintrinsicnosimd': - case 'corefx_baseline': - case 'corefx_minopts': - case 'corefx_tieredcompilation': - case 'corefx_jitstress1': - case 'corefx_jitstress2': - case 'corefx_jitstressregs1': - case 'corefx_jitstressregs2': - case 'corefx_jitstressregs3': - case 'corefx_jitstressregs4': - case 'corefx_jitstressregs8': - case 'corefx_jitstressregs0x10': - case 'corefx_jitstressregs0x80': - case 'corefx_jitstressregs0x1000': - case 'zapdisable': - if (os == 'CentOS7.1') { - break - } - if (os in bidailyCrossList) { - break - } - if ((os == 'Ubuntu') && (architecture == 'arm') && !isCoreFxScenario(scenario)) { - // Linux arm32 duplicated by AzDO - break - } - // ARM corefx testing uses non-flow jobs to provide the configuration-specific - // build for the flow job. We don't need cron jobs for these. Note that the - // Windows ARM jobs depend on a Windows "build only" job that exits the trigger - // function very early, so only non-Windows gets here. - if ((architecture == 'arm') && isCoreFxScenario(scenario) && !isFlowJob) { - break - } - if ((architecture == 'arm64') && isCoreFxScenario(scenario) && !isFlowJob) { - break - } - if (jobRequiresLimitedHardware(architecture, os)) { - if ((architecture == 'arm64') && (os == 'Ubuntu16.04')) { - // These jobs are very fast on Linux/arm64 hardware, so run them daily. - addPeriodicTriggerHelper(job, '@daily') - } - else if (scenario == 'corefx_baseline') { - addPeriodicTriggerHelper(job, '@daily') - } - else { - addPeriodicTriggerHelper(job, '@weekly') - } - } - else { - addPeriodicTriggerHelper(job, '@daily') - } - break - case 'heapverify1': - case 'gcstress0x3': - if (os == 'CentOS7.1') { - break - } - if (os in bidailyCrossList) { - break - } - if ((os == 'Ubuntu') && (architecture == 'arm')) { - // Linux arm32 duplicated by AzDO - break - } - addPeriodicTriggerHelper(job, '@weekly') - break - case 'gcstress0xc': - case 'gcstress0xc_zapdisable': - case 'gcstress0xc_zapdisable_jitstress2': - case 'gcstress0xc_zapdisable_heapverify1': - case 'gcstress0xc_jitstress1': - case 'gcstress0xc_jitstress2': - case 'gcstress0xc_minopts_heapverify1': - if (os == 'OSX10.12') { - // GCStress=C is currently not supported on OS X - break - } - if (os == 'CentOS7.1') { - break - } - if (os in bidailyCrossList) { - break - } - if ((os == 'Ubuntu') && (architecture == 'arm')) { - // Linux arm32 duplicated by AzDO - break - } - addPeriodicTriggerHelper(job, '@weekly') - break - - case 'illink': - // Testing on other operating systems TBD - assert (os == 'Windows_NT' || os == 'Ubuntu') - if (architecture == 'x64' || architecture == 'x86') { - if (configuration == 'Checked') { - addPeriodicTriggerHelper(job, '@daily') - } - } - break - - default: - println("Unknown scenario: ${scenario}"); - assert false - break - } - return -} - -// ************************** -// Define the basic inner loop builds for PR and commit. This is basically just the set -// of coreclr builds over linux/osx 10.12/windows and debug/release/checked. In addition, the windows -// builds will do a couple extra steps. -// ************************** - -// Adds a trigger for the PR build if one is needed. If isFlowJob is true, then this is the -// flow job that rolls up the build and test for non-windows OS's. // If the job is a windows build only job, -// it's just used for internal builds -// If you add a job with a trigger phrase, please add that phrase to coreclr/Documentation/project-docs/ci-trigger-phrases.md -def static addTriggers(def job, def branch, def isPR, def architecture, def os, def configuration, def scenario, def isFlowJob, def isWindowsBuildOnlyJob) { - def isNormalOrInnerloop = (scenario == "normal" || scenario == "innerloop") - - if (isWindowsBuildOnlyJob) { - return - } - - def bidailyCrossList = ['RHEL7.2', 'Debian8.4'] - // Non pull request builds. - if (!isPR) { - addNonPRTriggers(job, branch, isPR, architecture, os, configuration, scenario, isFlowJob, isWindowsBuildOnlyJob, bidailyCrossList) - return - } - - def arm64Users = [ - 'adityamandaleeka', - 'AndyAyersMS', - 'briansull', - 'BruceForstall', - 'CarolEidt', - 'davidwrighton', - 'echesakovMSFT', - 'erozenfeld', - 'janvorli', - 'jashook', - 'pgodeq', - 'RussKeldorph', - 'sandreenko', - 'swaroop-sridhar', - 'jkotas', - 'markwilkie', - 'weshaggard', - 'tannergooding' - ] - - // Pull request builds. Generally these fall into two categories: default triggers and on-demand triggers - // We generally only have a distinct set of default triggers but a bunch of on-demand ones. - - def contextString = "" - def triggerString = "" - def needsTrigger = true - def isDefaultTrigger = false - def isArm64PrivateJob = false - def scenarioString = "" - - // Set up default context string and trigger phrases. This is overridden in places, sometimes just to keep - // the existing non-standard descriptions and phrases. In some cases, the scenarios are asymmetric, as for - // some jobs where the Debug configuration just does builds, no tests. - // - // Some configurations, like arm32/arm64, always use the exact scenario name as part of the context string. - // This makes it possible to copy/paste the displayed context string as "@dotnet-bot test <context-string>" - // to invoke the trigger. Any "fancy" context string makes that impossible, requiring the user to either - // remember the mapping from context string to trigger string, or use "@dotnet-bot help" to look it up. - - if (architecture == 'armem') { - assert os == 'Tizen' - architecture = 'armel' - } - - switch (architecture) { - case 'armel': - case 'arm': - case 'arm64': - contextString = "${os} ${architecture} Cross ${configuration}" - triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+Cross\\W+${configuration}" - - if (scenario == 'innerloop') { - contextString += " Innerloop" - triggerString += "\\W+Innerloop" - } - else { - contextString += " ${scenario}" - triggerString += "\\W+${scenario}" - } - - if (scenario == 'pmi_asm_diffs') { - // Don't add the "Build and Test" part - } - else if (configuration == 'Debug') { - contextString += " Build" - triggerString += "\\W+Build" - } - else { - contextString += " Build and Test" - triggerString += "\\W+Build and Test" - } - - triggerString += ".*" - break - - default: - scenarioString = getScenarioDisplayString(scenario) - contextString = "${os} ${architecture} ${configuration} ${scenarioString}" - triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}" - - switch (scenario) { - case 'normal': - triggerString += "\\W+Build and Test.*" - break - - case 'corefx_innerloop': // maintain this asymmetry - triggerString += "\\W+CoreFX Tests.*" - break - - default: - triggerString += "\\W+${scenario}.*" - break - } - - triggerString += ".*" - break - } - - // Now determine what kind of trigger this job needs, if any. Any job should be triggerable, except for - // non-flow jobs that are only used as part of flow jobs. - - switch (architecture) { - case 'x64': // editor brace matching: { - if (scenario == 'formatting') { - assert configuration == 'Checked' - if (os == 'Windows_NT' || os == 'Ubuntu') { - isDefaultTrigger = true - contextString = "${os} ${architecture} Formatting" - } - break - } - - if (scenario == 'pmi_asm_diffs') { - // Everything is already set. - // No default triggers. - break - } - - switch (os) { - // OpenSUSE, Debian & RedHat get trigger phrases for pri 0 build, and pri 1 build & test - case 'Debian8.4': - case 'RHEL7.2': - if (scenario == 'innerloop') { - assert !isFlowJob - contextString = "${os} ${architecture} ${configuration} Innerloop Build" - isDefaultTrigger = true - break - } - - // fall through - - case 'Fedora24': - case 'Ubuntu16.04': - case 'Ubuntu16.10': - assert !isFlowJob - assert scenario != 'innerloop' - contextString = "${os} ${architecture} ${configuration} Build" - triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+Build.*" - break - - case 'Ubuntu': - if (scenario == 'illink') { - break - } - else if (scenario == 'corefx_innerloop') { - if (configuration == 'Checked') { - isDefaultTrigger = true - } - break - } - - // fall through - - case 'OSX10.12': - // Triggers on the non-flow jobs aren't necessary here - // Corefx testing uses non-flow jobs. - if (!isFlowJob && !isCoreFxScenario(scenario)) { - needsTrigger = false - break - } - switch (scenario) { - case 'innerloop': - isDefaultTrigger = true - break - - case 'no_tiered_compilation_innerloop': - if (os == 'Ubuntu') { - isDefaultTrigger = true - } - break - - default: - break - } - break - - case 'CentOS7.1': - switch (scenario) { - case 'innerloop': - // CentOS uses checked for default PR tests while debug is build only - if (configuration == 'Debug') { - isDefaultTrigger = true - contextString = "${os} ${architecture} ${configuration} Innerloop Build" - break - } - - // Make sure this is a flow job to get build and test. - if (!isFlowJob) { - needsTrigger = false - break - } - - if (configuration == 'Checked') { - assert job.name.contains("flow") - isDefaultTrigger = true - contextString = "${os} ${architecture} ${configuration} Innerloop Build and Test" - } - break - - case 'normal': - // Make sure this is a flow job to get build and test. - if (!isFlowJob) { - needsTrigger = false - break - } - break - - default: - break - } - break - - case 'Windows_NT': - switch (scenario) { - case 'innerloop': - case 'no_tiered_compilation_innerloop': - isDefaultTrigger = true - break - - case 'corefx_innerloop': - if (configuration == 'Checked' || configuration == 'Release') { - isDefaultTrigger = true - } - break - - default: - break - } - break - - default: - println("Unknown os: ${os}"); - assert false - break - - } // switch (os) - - break - // editor brace matching: } - - case 'armel': // editor brace matching: { - job.with { - publishers { - azureVMAgentPostBuildAction { - agentPostBuildAction('Delete agent if the build was not successful (when idle).') - } - } - } - - switch (os) { - case 'Tizen': - if (scenario == 'innerloop') { - if (configuration == 'Checked') { - isDefaultTrigger = true - } - } - break - } - - break - // editor brace matching: } - - case 'arm': - case 'arm64': // editor brace matching: { - - switch (os) { - case 'Ubuntu': - case 'Ubuntu16.04': - - // Triggers on the non-flow jobs aren't necessary - if (!isFlowJob) { - needsTrigger = false - break - } - if (os == 'Ubuntu' && architecture == 'arm') { - switch (scenario) { - case 'innerloop': - case 'no_tiered_compilation_innerloop': - if (configuration == 'Checked') { - isDefaultTrigger = true - } - break - case 'crossgen_comparison': - if (configuration == 'Checked' || configuration == 'Release') { - isDefaultTrigger = true - } - break - } - } - break - - case 'Windows_NT': - assert isArmWindowsScenario(scenario) - - // For Debug normal/innerloop scenario, we don't do test runs, so we don't use flow jobs. That means we need a trigger for - // the non-flow Build job. All others need a trigger on the flow job. - def needsFlowJobTrigger = !(isNormalOrInnerloop && (configuration == 'Debug')) - if (isFlowJob != needsFlowJobTrigger) { - needsTrigger = false - break - } - - switch (scenario) { - case 'innerloop': - if (configuration == 'Checked') { - isDefaultTrigger = true - isArm64PrivateJob = true - } - break - default: - isArm64PrivateJob = true - break - } - break - default: - println("NYI os: ${os}"); - assert false - break - } - break - - // editor brace matching: } - case 'x86': // editor brace matching: { - assert ((os == 'Windows_NT') || ((os == 'Ubuntu') && isNormalOrInnerloop)) - if (os == 'Ubuntu') { - // Triggers on the non-flow jobs aren't necessary here - if (!isFlowJob) { - needsTrigger = false - break - } - - // on-demand only for ubuntu x86 - contextString = "${os} ${architecture} ${configuration} Build" - triggerString = "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}.*" - break - } - switch (scenario) { - case 'innerloop': - case 'no_tiered_compilation_innerloop': - isDefaultTrigger = true - break - default: - break - } - break - - // editor brace matching: } - - default: - println("Unknown architecture: ${architecture}"); - assert false - break - } - - if (needsTrigger) { - if (isArm64PrivateJob) { - // ignore isDefaultTrigger to disable Jenkins by default - if (false) { - Utilities.addDefaultPrivateGithubPRTriggerForBranch(job, branch, contextString, null, arm64Users) - } - else { - Utilities.addPrivateGithubPRTriggerForBranch(job, branch, contextString, triggerString, null, arm64Users) - } - } - else { - // ignore isDefaultTrigger to disable Jenkins by default - if (false) { - Utilities.addGithubPRTriggerForBranch(job, branch, contextString) - } - else { - Utilities.addGithubPRTriggerForBranch(job, branch, contextString, triggerString) - } - } - } -} - -def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR, def architecture, def configuration, def os, def isBuildOnly) { - def buildCommands = [] - def osGroup = getOSGroup(os) - def lowerConfiguration = configuration.toLowerCase() - - // Which set of tests to build? Innerloop tests build Pri-0. - // Currently, we only generate asm diffs on Pri-0 tests, if we generate asm diffs on tests at all. - // CoreFX testing skipts building tests altogether (done below). - // All other scenarios build Pri-1 tests. - def priority = '1' - if (isInnerloopTestScenario(scenario)) { - priority = '0' - } - - def doCoreFxTesting = isCoreFxScenario(scenario) - - def buildCoreclrTests = true - if (doCoreFxTesting || (scenario == 'pmi_asm_diffs')) { - // These scenarios don't need the coreclr tests build. - buildCoreclrTests = false - } - - // Calculate the build steps, archival, and xunit results - switch (os) { - case 'Windows_NT': // editor brace matching: { - switch (architecture) { - case 'x64': - case 'x86': - def arch = architecture - def buildOpts = '' - - if (scenario == 'formatting') { - buildCommands += "python -u tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}" - Utilities.addArchival(newJob, "format.patch", "", true, false) - break - } - - if (scenario == 'illink') { - buildCommands += "tests\\scripts\\build_illink.cmd clone ${arch}" - } - - // If it is a release build for Windows, ensure PGO is used, else fail the build. - if ((lowerConfiguration == 'release') && - (scenario in Constants.basicScenarios)) { - - buildOpts += ' -enforcepgo' - } - - if (buildCoreclrTests) { - buildOpts += " -priority=${priority}" - } else { - buildOpts += ' skiptests'; - } - - // Set __TestIntermediateDir to something short. If __TestIntermediateDir is already set, build-test.cmd will - // output test binaries to that directory. If it is not set, the binaries are sent to a default directory whose name is about - // 35 characters long. - - buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${arch} ${buildOpts}" - - if (scenario == 'pmi_asm_diffs') { - // Now, generate the layout. We don't have any tests, but we need to restore the packages before calling runtest.cmd. - // Call build-test.cmd to do this. It will do a little more than we need, but that's ok. - buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} skipmanaged skipnative" - buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${arch} GenerateLayoutOnly" - - // TODO: Add -target_branch and -commit_hash arguments based on GitHub variables. - buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-pmi-diffs.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration}" - - // ZIP up the asm - buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('_\\pmi\\asm', '.\\dasm.${os}.${architecture}.${configuration}.zip')\""; - - // Archive the asm - Utilities.addArchival(newJob, "dasm.${os}.${architecture}.${configuration}.zip") - break - } - - if (!isBuildOnly) { - def runtestArguments = '' - def testOpts = 'collectdumps' - - if (isR2RScenario(scenario)) { - - // If this is a ReadyToRun scenario, pass 'crossgen' - // to cause framework assemblies to be crossgen'ed. Pass 'runcrossgentests' - // to cause the tests to be crossgen'ed. - - testOpts += ' crossgen runcrossgentests' - } - else if (scenario == 'jitdiff') { - testOpts += ' jitdisasm crossgen' - } - else if (scenario == 'ilrt') { - testOpts += ' ilasmroundtrip' - } - else if (isLongGc(scenario)) { - testOpts += " ${scenario} sequential" - } - else if (scenario == 'standalone_gc') { - testOpts += ' gcname clrgc.dll' - } - else if (scenario == 'illink') { - testOpts += " link %WORKSPACE%\\linker\\linker\\bin\\netcore_Release\\netcoreapp2.0\\win10-${arch}\\publish\\illink.exe" - } - - // Default per-test timeout is 10 minutes. For stress modes and Debug scenarios, increase this - // to 30 minutes (30 * 60 * 1000 = 180000). The "timeout" argument to runtest.cmd sets this, by - // taking a timeout value in milliseconds. (Note that it sets the __TestTimeout environment variable, - // which is read by the xunit harness.) - if (isJitStressScenario(scenario) || isR2RStressScenario(scenario) || (lowerConfiguration == 'debug')) - { - def timeout = 1800000 - testOpts += " timeout ${timeout}" - } - - // If we are running a stress mode, we should write out the set of key - // value env pairs to a file at this point and then we'll pass that to runtest.cmd - - def envScriptPath = '' - if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) { - def buildCommandsStr = '' - envScriptPath = "%WORKSPACE%\\SetStressModes.bat" - buildCommandsStr += envScriptCreate(os, envScriptPath) - - if (isJitStressScenario(scenario)) { - buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath) - } - else if (isR2RStressScenario(scenario)) { - buildCommandsStr += envScriptSetStressModeVariables(os, Constants.r2rStressScenarios[scenario], envScriptPath) - } - - envScriptFinalize(os, envScriptPath) - - // Note that buildCommands is an array of individually executed commands; we want all the commands used to - // create the SetStressModes.bat script to be executed together, hence we accumulate them as strings - // into a single script. - buildCommands += buildCommandsStr - } - if (envScriptPath != '') { - testOpts += " TestEnv ${envScriptPath}" - } - - runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}" - - if (doCoreFxTesting) { - if (scenario == 'corefx_innerloop') { - // Create CORE_ROOT and testhost - buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly" - buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll" - - // Archive and process (only) the test results - Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) - Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml") - } - else { - def workspaceRelativeFxRoot = "_/fx" - def absoluteFxRoot = "%WORKSPACE%\\_\\fx" - def fxBranch = getFxBranch(branch) - def exclusionRspPath = "%WORKSPACE%\\tests\\scripts\\run-corefx-tests-exclusions.txt" - - buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -exclusion_rsp_file ${exclusionRspPath}" - - // Archive and process (only) the test results - Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) - Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml") - - //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail - Utilities.addArchival(newJob, "SetStressModes.bat", "", true, false) - Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/testhost/**", "", true, false) - } - } - else if (isGcReliabilityFramework(scenario)) { - buildCommands += "tests\\runtest.cmd ${runtestArguments} GenerateLayoutOnly" - buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}" - } - else { - def buildCommandsStr = "call tests\\runtest.cmd ${runtestArguments}\r\n" - if (!isBuildOnly) { - // If we ran the tests, collect the test logs collected by xunit. We want to do this even if the tests fail, so we - // must do it in the same batch file as the test run. - - buildCommandsStr += "echo on\r\n" // Show the following commands in the log. "echo" doesn't alter the errorlevel. - buildCommandsStr += "set saved_errorlevel=%errorlevel%\r\n" - buildCommandsStr += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}\\Reports', '.\\bin\\tests\\testReports.zip')\"\r\n"; - buildCommandsStr += "exit /b %saved_errorlevel%\r\n" - - def doNotFailIfNothingArchived = true - def archiveOnlyIfSuccessful = false - Utilities.addArchival(newJob, "bin/tests/testReports.zip", "", doNotFailIfNothingArchived, archiveOnlyIfSuccessful) - } - buildCommands += buildCommandsStr - } - } // end if (!isBuildOnly) - - if (!doCoreFxTesting) { - // Run the rest of the build - // Build the mscorlib for the other OS's - buildCommands += "build.cmd ${lowerConfiguration} ${arch} linuxmscorlib" - buildCommands += "build.cmd ${lowerConfiguration} ${arch} osxmscorlib" - - if (arch == 'x64') { - buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib" - } - - if (!isJitStressScenario(scenario)) { - // Zip up the tests directory so that we don't use so much space/time copying - // 10s of thousands of files around. - buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\""; - - // For Windows, pull full test results and test drops for x86/x64. - // No need to pull for stress mode scenarios (downstream builds use the default scenario) - Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**") - } - - if (scenario == 'jitdiff') { - // retrieve jit-dasm output for base commit, and run jit-diff - if (!isBuildOnly) { - // if this is a build only job, we want to keep the default (build) artifacts for the flow job - Utilities.addArchival(newJob, "bin/tests/${osGroup}.${arch}.${configuration}/dasm/**") - } - } - - if (!isBuildOnly) { - Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true) - } - } - - // Archive the logs, even if the build failed (which is when they are most interesting). - Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err,bin/Logs/MsbuildDebugLogs/*", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) - break - case 'arm': - case 'arm64': - assert isArmWindowsScenario(scenario) - - def buildOpts = '' - - if (buildCoreclrTests) { - buildOpts += " -priority=${priority}" - } else { - buildOpts += ' skiptests' - } - - // This is now a build only job. Do not run tests. Use the flow job. - buildCommands += "set __TestIntermediateDir=int&&build.cmd ${lowerConfiguration} ${architecture} ${buildOpts}" - - if (doCoreFxTesting) { - assert isBuildOnly - - // Set the stress mode variables; this is incorporated into the generated CoreFx RunTests.cmd files. - def envScriptPath = '' - def buildCommandsStr = '' - envScriptPath = "%WORKSPACE%\\SetStressModes.bat" - buildCommandsStr += envScriptCreate(os, envScriptPath) - buildCommandsStr += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], envScriptPath) - envScriptFinalize(os, envScriptPath) - buildCommands += buildCommandsStr - - def workspaceRelativeFxRootLinux = "_/fx" - def workspaceRelativeFxRootWin = "_\\fx" - def absoluteFxRoot = "%WORKSPACE%\\_\\fx" - def fxBranch = getFxBranch(branch) - - buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath} -no_run_tests" - - // Zip up the CoreFx runtime and tests. We don't need the CoreCLR binaries; they have been copied to the CoreFX tree. - buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\artifacts\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}', '${workspaceRelativeFxRootWin}\\fxruntime.zip')\""; - buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('${workspaceRelativeFxRootWin}\\artifacts\\bin\\tests', '${workspaceRelativeFxRootWin}\\fxtests.zip')\""; - - Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip") - Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip") - } else { - // Zip up the tests directory so that we don't use so much space/time copying - // 10s of thousands of files around. - buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}', '.\\bin\\tests\\tests.zip')\""; - - // Add archival. - Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**") - } - - // Archive the logs, even if the build failed (which is when they are most interesting). - Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err,bin/Logs/MsbuildDebugLogs/*", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) - break - default: - println("Unknown architecture: ${architecture}"); - assert false - break - } - break - // end case 'Windows_NT'; editor brace matching: } - case 'Ubuntu': - case 'Ubuntu16.04': - case 'Ubuntu16.10': - case 'Debian8.4': - case 'OSX10.12': - case 'CentOS7.1': - case 'RHEL7.2': - case 'Tizen': - case 'Fedora24': // editor brace matching: { - switch (architecture) { - case 'x86': - if (os == 'Ubuntu') { - // build and PAL test - def dockerImage = getDockerImageName(architecture, os, true) - buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code -e ROOTFS_DIR=/crossrootfs/x86 ${dockerImage} ./build.sh ${architecture} cross ${lowerConfiguration}" - dockerImage = getDockerImageName(architecture, os, false) - buildCommands += "docker run -i --rm -v \${WORKSPACE}:/opt/code -w /opt/code ${dockerImage} ./src/pal/tests/palsuite/runpaltests.sh /opt/code/bin/obj/${osGroup}.${architecture}.${configuration} /opt/code/bin/paltestout" - Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**") - Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml') - } - break - - case 'x64': - if (scenario == 'formatting') { - buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${architecture}" - Utilities.addArchival(newJob, "format.patch", "", true, false) - break - } - - if (scenario == 'pmi_asm_diffs') { - buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests skipbuildpackages" - buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatelayoutonly" - - // TODO: Add -target_branch and -commit_hash arguments based on GitHub variables. - buildCommands += "python -u \${WORKSPACE}/tests/scripts/run-pmi-diffs.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration}" - - // Archive the asm - buildCommands += "tar -czf dasm.${os}.${architecture}.${configuration}.tgz ./_/pmi/asm" - Utilities.addArchival(newJob, "dasm.${os}.${architecture}.${configuration}.tgz") - break - } - - if (scenario == 'illink') { - assert(os == 'Ubuntu') - buildCommands += "./tests/scripts/build_illink.sh --clone --arch=${architecture}" - } - - if (!doCoreFxTesting) { - // We run pal tests on all OS but generate mscorlib (and thus, nuget packages) - // only on supported OS platforms. - def bootstrapRid = Utilities.getBoostrapPublishRid(os) - def bootstrapRidEnv = bootstrapRid != null ? "__PUBLISH_RID=${bootstrapRid} " : '' - - buildCommands += "${bootstrapRidEnv}./build.sh ${lowerConfiguration} ${architecture}" - - def testBuildOpts = "" - if (priority == '1') { - testBuildOpts = "priority1" - } - - buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} ${testBuildOpts}" - buildCommands += "src/pal/tests/palsuite/runpaltests.sh \${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration} \${WORKSPACE}/bin/paltestout" - - // Archive the bin/tests folder for *_tst jobs - def testArtifactsTgzFileName = getTestArtifactsTgzFileName(osGroup, architecture, configuration) - buildCommands += "tar -czf ${testArtifactsTgzFileName} bin/tests/${osGroup}.${architecture}.${configuration}" - Utilities.addArchival(newJob, "${testArtifactsTgzFileName}", "") - // And pal tests - Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml') - } - else { - if (scenario == 'corefx_innerloop') { - assert os == 'Ubuntu' || 'OSX10.12' - assert architecture == 'x64' - - buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests" - buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly" - buildCommands += "./tests/runtest.sh ${lowerConfiguration} --corefxtestsall --testHostDir=\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=\${WORKSPACE}" - - // Archive and process (only) the test results - Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) - Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml") - } - else { - // Corefx stress testing - assert os == 'Ubuntu' - assert architecture == 'x64' - assert lowerConfiguration == 'checked' - assert isJitStressScenario(scenario) - - // Build coreclr - buildCommands += "./build.sh ${lowerConfiguration} ${architecture}" - - def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh" - - def envScriptCmds = envScriptCreate(os, scriptFileName) - envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName) - envScriptCmds += envScriptFinalize(os, scriptFileName) - buildCommands += envScriptCmds - - // Build and text corefx - def workspaceRelativeFxRoot = "_/fx" - def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}" - def fxBranch = getFxBranch(branch) - def exclusionRspPath = "\$WORKSPACE/tests/scripts/run-corefx-tests-exclusions.txt" - - buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -exclusion_rsp_file ${exclusionRspPath}" - - // Archive and process (only) the test results - Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) - Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml") - } - } - - // Archive the logs, even if the build failed (which is when they are most interesting). - Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err,bin/Logs/MsbuildDebugLogs/*", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) - break - case 'armem': - // Emulator cross builds for ARM runs on Tizen currently - assert os == 'Tizen' - - def arm_abi = "armel" - def linuxCodeName = "tizen" - - // Unzip the Windows test binaries first. Exit with 0 - buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0" - - // Unpack the corefx binaries - buildCommands += "mkdir ./bin/CoreFxBinDir" - buildCommands += "tar -xf ./artifacts/bin/build.tar.gz -C ./bin/CoreFxBinDir" - - // Call the ARM CI script to cross build and test using docker - buildCommands += """./tests/scripts/arm32_ci_script.sh \\ - --mode=docker \\ - --${arm_abi} \\ - --linuxCodeName=${linuxCodeName} \\ - --buildConfig=${lowerConfiguration} \\ - --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\ - --coreFxBinDir=./bin/CoreFxBinDir \\ - --testDirFile=./tests/testsRunningInsideARM.txt""" - - // Basic archiving of the build, no pal tests - Utilities.addArchival(newJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**") - break - case 'arm64': - case 'arm': - // Non-Windows ARM cross builds on hardware run on Ubuntu only - assert (os == 'Ubuntu') || (os == 'Ubuntu16.04') - - // Add some useful information to the log file. Ignore return codes. - buildCommands += "uname -a || true" - - // Cross build the Ubuntu/arm product using docker with a docker image that contains the correct - // Ubuntu cross-compilation toolset (running on a Ubuntu x64 host). - // For CoreFX testing, we only need the product build; we don't need to generate the layouts. The product - // build is then copied into the corefx layout by the run-corefx-test.py script. For CoreFX testing, we - // ZIP up the generated CoreFX runtime and tests. - - def dockerImage = getDockerImageName(architecture, os, true) - def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${dockerImage} " - - buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh ${lowerConfiguration} ${architecture} cross" - - if (doCoreFxTesting) { - def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh" - - def envScriptCmds = envScriptCreate(os, scriptFileName) - envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName) - envScriptCmds += envScriptFinalize(os, scriptFileName) - buildCommands += envScriptCmds - - // Build and text corefx - def workspaceRelativeFxRootLinux = "_/fx" - def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRootLinux}" - def fxBranch = getFxBranch(branch) - - buildCommands += "${dockerCmd}python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName} -no_run_tests" - - // Docker creates files with root permission, so we need to zip in docker also, or else we'll get permission errors. - buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxruntime.zip ${workspaceRelativeFxRootLinux}/artifacts/bin/testhost/netcoreapp-Linux-Release-${architecture}" - buildCommands += "${dockerCmd}zip -r ${workspaceRelativeFxRootLinux}/fxtests.zip ${workspaceRelativeFxRootLinux}/artifacts/bin/tests" - - Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxruntime.zip") - Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip") - } - else if (isCrossGenComparisonScenario(scenario)) { - buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly" - - def workspaceRelativeProductBinDir = "bin/Product/${osGroup}.${architecture}.${configuration}" - def workspaceRelativeCoreLib = "${workspaceRelativeProductBinDir}/IL/System.Private.CoreLib.dll" - def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root" - def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py" - def workspaceRelativeResultsDir = "_" - def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip" - def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} " - def crossArch = "x64" - def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeProductBinDir}/${crossArch}/crossgen" - def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}" - - buildCommands += "${dockerCmd}mkdir -p \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}" - buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}" - buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}" - - buildCommands += "${dockerCmd}zip -r ${workspaceRelativeArtifactsArchive} ${workspaceRelativeCoreLib} ${workspaceRelativeCoreRootDir} ${workspaceRelativeCrossGenComparisonScript} ${workspaceRelativeResultsDir}" - Utilities.addArchival(newJob, "${workspaceRelativeArtifactsArchive}") - } - else if (scenario == 'pmi_asm_diffs') { - buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly" - - // Pass `--skip_diffs` -- the actual diffs will be done on an arm machine in the test job. This is the build job. - // TODO: Add -target_branch and -commit_hash arguments based on GitHub variables. - buildCommands += "python -u \${WORKSPACE}/tests/scripts/run-pmi-diffs.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} --skip_diffs" - - // Archive what we created. - buildCommands += "tar -czf product.${os}.${architecture}.${lowerConfiguration}.tgz ./bin/Product/Linux.${architecture}.${configuration}" - buildCommands += "tar -czf product.baseline.${os}.${architecture}.${lowerConfiguration}.tgz ./_/pmi/base/bin/Product/Linux.${architecture}.${configuration}" - buildCommands += "tar -czf coreroot.${os}.${architecture}.${lowerConfiguration}.tgz ./bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root" - buildCommands += "tar -czf coreroot.baseline.${os}.${architecture}.${lowerConfiguration}.tgz ./_/pmi/base/bin/tests/Linux.${architecture}.${configuration}/Tests/Core_Root" - - // Archive the built artifacts - Utilities.addArchival(newJob, "product.${os}.${architecture}.${lowerConfiguration}.tgz,product.baseline.${os}.${architecture}.${lowerConfiguration}.tgz,coreroot.${os}.${architecture}.${lowerConfiguration}.tgz,coreroot.baseline.${os}.${architecture}.${lowerConfiguration}.tgz") - } - else { - // Then, using the same docker image, build the tests and generate the CORE_ROOT layout. - - def testBuildOpts = "" - if (priority == '1') { - testBuildOpts = "priority1" - } - - buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross ${testBuildOpts}" - - // ZIP up the built tests (including CORE_ROOT and native test components copied to the CORE_ROOT) for the test job (created in the flow job code) - def testArtifactsTgzFileName = getTestArtifactsTgzFileName(osGroup, architecture, configuration) - buildCommands += "tar -czf ${testArtifactsTgzFileName} bin/tests/${osGroup}.${architecture}.${configuration}" - - Utilities.addArchival(newJob, "${testArtifactsTgzFileName}", "") - } - - // Archive the logs, even if the build failed (which is when they are most interesting). - Utilities.addArchival(newJob, "bin/Logs/*.log,bin/Logs/*.wrn,bin/Logs/*.err,bin/Logs/MsbuildDebugLogs/*", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) - - // We need to clean up the build machines; the docker build leaves newly built files with root permission, which - // the cleanup task in Jenkins can't remove. - newJob.with { - publishers { - azureVMAgentPostBuildAction { - agentPostBuildAction('Delete agent after build execution (when idle).') - } - } - } - break - default: - println("Unknown architecture: ${architecture}"); - assert false - break - } - break - // editor brace matching: } - default: - println("Unknown os: ${os}"); - assert false - break - } // os - - return buildCommands -} - -// Determine if we should generate a job for the given parameters. This is for non-flow jobs: either build and test, or build only. -// Returns true if the job should be generated. -def static shouldGenerateJob(def scenario, def isPR, def architecture, def configuration, def os, def isBuildOnly) -{ - def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList)) - - // Innerloop jobs (except corefx_innerloop) are no longer created in Jenkins - // The only exception is windows arm(64) - if (isInnerloopTestScenario(scenario) && isPR && !windowsArmJob) { - assert scenario != 'corefx_innerloop' - return false; - } - - if (!isPR) { - if (isInnerloopTestScenario(scenario)) { - return false - } - - if (scenario == 'corefx_innerloop') { - return false - } - } - - // Tizen is only supported for armem architecture - if (os == 'Tizen' && architecture != 'armem') { - return false - } - - // Filter based on architecture. - - switch (architecture) { - case 'arm': - if ((os != 'Windows_NT') && (os != 'Ubuntu')) { - return false - } - break - case 'arm64': - if ((os != 'Windows_NT') && (os != 'Ubuntu16.04')) { - return false - } - break - case 'armem': - if (os != 'Tizen') { - return false - } - break - case 'x86': - if ((os != 'Windows_NT') && (os != 'Ubuntu')) { - return false - } - break - case 'x64': - // Everything implemented - break - default: - println("Unknown architecture: ${architecture}") - assert false - break - } - - // Which (Windows) build only jobs are required? - - def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal') - - if (isBuildOnly) { - switch (architecture) { - case 'arm': - case 'arm64': - // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that. - if (!isCoreFxScenario(scenario)) { - return false - } - break - case 'x64': - case 'x86': - if (!isNormalOrInnerloop) { - return false - } - break - default: - return false - } - } - - // Filter based on scenario. - - if (isJitStressScenario(scenario)) { - if (configuration != 'Checked') { - return false - } - - def isEnabledOS = (os == 'Windows_NT') || - (os == 'Ubuntu' && (architecture == 'x64') && isCoreFxScenario(scenario)) || - (os == 'Ubuntu' && architecture == 'arm') || - (os == 'Ubuntu16.04' && architecture == 'arm64') - if (!isEnabledOS) { - return false - } - - switch (architecture) { - case 'x64': - break - - case 'x86': - // x86 ubuntu: no stress modes - if (os == 'Ubuntu') { - return false - } - break - - case 'arm': - case 'arm64': - // We use build only jobs for Windows arm/arm64 cross-compilation corefx testing, so we need to generate builds for that. - // No "regular" Windows arm corefx jobs, e.g. - // For Ubuntu arm corefx testing, we use regular jobs (not "build only" since only Windows has "build only", and - // the Ubuntu arm "regular" jobs don't run tests anyway). - if (os == 'Windows_NT') { - if (! (isBuildOnly && isCoreFxScenario(scenario)) ) { - return false - } - } - else { - if (!isCoreFxScenario(scenario)) { - return false - } - } - break - - default: - // armem: no stress jobs for ARM emulator. - return false - } - } - else if (isR2RScenario(scenario)) { - if (os != 'Windows_NT') { - return false - } - - if (isR2RBaselineScenario(scenario)) { - // no need for Debug scenario; Checked is sufficient - if (configuration != 'Checked' && configuration != 'Release') { - return false - } - } - else if (isR2RStressScenario(scenario)) { - // Stress scenarios only run with Checked builds, not Release (they would work with Debug, but be slow). - if (configuration != 'Checked') { - return false - } - } - - switch (architecture) { - case 'arm': - case 'arm64': - // Windows arm/arm64 ready-to-run jobs use flow jobs and test jobs, but depend on "normal" (not R2R specific) build jobs. - return false - - default: - break - } - } - else if (isCrossGenComparisonScenario(scenario)) { - return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario) - } - else { - // Skip scenarios - switch (scenario) { - case 'ilrt': - // The ilrt build isn't necessary except for Windows_NT2003. Non-Windows NT uses - // the default scenario build - if (os != 'Windows_NT') { - return false - } - // Only x64 for now - if (architecture != 'x64') { - return false - } - // Release only - if (configuration != 'Release') { - return false - } - break - case 'jitdiff': - if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') { - return false - } - if (architecture != 'x64') { - return false - } - if (configuration != 'Checked') { - return false - } - break - case 'longgc': - case 'gcsimulator': - if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') { - return false - } - if (architecture != 'x64') { - return false - } - if (configuration != 'Release') { - return false - } - break - case 'gc_reliability_framework': - case 'standalone_gc': - if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') { - return false - } - - if (architecture != 'x64') { - return false - } - - if (configuration != 'Release' && configuration != 'Checked') { - return false - } - break - // We only run Windows and Ubuntu x64 Checked for formatting right now - case 'formatting': - if (os != 'Windows_NT' && os != 'Ubuntu') { - return false - } - if (architecture != 'x64') { - return false - } - if (configuration != 'Checked') { - return false - } - break - case 'illink': - if (os != 'Windows_NT' && (os != 'Ubuntu' || architecture != 'x64')) { - return false - } - if (architecture != 'x64' && architecture != 'x86') { - return false - } - break - case 'normal': - // Nothing skipped - break - case 'innerloop': - if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, isBuildOnly)) { - return false - } - break - case 'corefx_innerloop': - if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') { - return false - } - if (architecture != 'x64') { - return false - } - break - case 'pmi_asm_diffs': - if (configuration != 'Checked') { - return false - } - if (architecture == 'armem') { - return false - } - // Currently, we don't support pmi_asm_diffs for Windows arm/arm64. We don't have a dotnet CLI available to - // build jitutils. The jobs are not in validArmWindowsScenarios. - if ((os == 'Windows_NT') && (architecture == 'arm' || architecture == 'arm64')) { - return false - } - // Currently, no support for Linux x86. - if ((os != 'Windows_NT') && (architecture == 'x86')) { - return false - } - break - default: - println("Unknown scenario: ${scenario}") - assert false - break - } - } - - // The job was not filtered out, so we should generate it! - return true -} - -Constants.allScenarios.each { scenario -> - [true, false].each { isPR -> - Constants.architectureList.each { architecture -> - Constants.configurationList.each { configuration -> - Constants.osList.each { os -> - // If the OS is Windows_NT_BuildOnly, set the isBuildOnly flag to true - // and reset the os to Windows_NT - def isBuildOnly = false - if (os == 'Windows_NT_BuildOnly') { - isBuildOnly = true - os = 'Windows_NT' - } - - if (!shouldGenerateJob(scenario, isPR, architecture, configuration, os, isBuildOnly)) { - return - } - - // Calculate names - def jobName = getJobName(configuration, architecture, os, scenario, isBuildOnly) - def folderName = getJobFolder(scenario) - - // Create the new job - def newJob = job(Utilities.getFullJobName(project, jobName, isPR, folderName)) {} - - addToViews(newJob, false, isPR, architecture, os, configuration, scenario) // isFlowJob == false - - setJobMachineAffinity(architecture, os, true, false, false, newJob) // isBuildJob = true, isTestJob = false, isFlowJob = false - - Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") - addTriggers(newJob, branch, isPR, architecture, os, configuration, scenario, false, isBuildOnly) // isFlowJob==false - setJobTimeout(newJob, isPR, architecture, configuration, scenario, isBuildOnly) - - // Copy Windows build test binaries and corefx build artifacts for Linux cross build for armem. - // We don't use a flow job for this, but we do depend on there being existing builds with these - // artifacts produced. - if ((architecture == 'armem') && (os == 'Tizen')) { - // Define the Windows Tests and Corefx build job names - def lowerConfiguration = configuration.toLowerCase() - def WindowsTestsName = projectFolder + '/' + - Utilities.getFullJobName(project, - getJobName(lowerConfiguration, 'x64' , 'windows_nt', 'normal', true), - false) - def fxBranch = getFxBranch(branch) - def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + - Utilities.getFolderName(fxBranch) - - def arm_abi = 'armel' - def corefx_os = 'tizen' - - // Let's use release CoreFX to test checked CoreCLR, - // because we do not generate checked CoreFX in CoreFX CI yet. - def corefx_lowerConfiguration = lowerConfiguration - if (lowerConfiguration == 'checked') { - corefx_lowerConfiguration = 'release' - } - - // Copy the Windows test binaries and the Corefx build binaries - newJob.with { - steps { - copyArtifacts(WindowsTestsName) { - includePatterns('bin/tests/tests.zip') - buildSelector { - latestSuccessful(true) - } - } - copyArtifacts("${corefxFolder}/${corefx_os}_${arm_abi}_cross_${corefx_lowerConfiguration}") { - includePatterns('artifacts/bin/build.tar.gz') - buildSelector { - latestSuccessful(true) - } - } - } // steps - } // newJob.with - } - - def buildCommands = calculateBuildCommands(newJob, scenario, branch, isPR, architecture, configuration, os, isBuildOnly) - - newJob.with { - steps { - if (os == 'Windows_NT') { - buildCommands.each { buildCommand -> - batchFile(buildCommand) - } - } - else { - buildCommands.each { buildCommand -> - shell(buildCommand) - } - } - } // steps - } // newJob.with - - } // os - } // configuration - } // architecture - } // isPR -} // scenario - -// Create a Windows ARM/ARM64 test job that will be used by a flow job. -// Returns the newly created job. -def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName) -{ - def osGroup = getOSGroup(os) - def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst" - - def jobFolder = getJobFolder(scenario) - def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) { - parameters { - stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from") - } - - steps { - // Set up the copies - - // Coreclr build we are trying to test - // - // ** NOTE ** This will, correctly, overwrite the CORE_ROOT from the Windows test archive - - copyArtifacts(inputCoreCLRBuildName) { - excludePatterns('**/testResults.xml', '**/*.ni.dll') - buildSelector { - buildNumber('${CORECLR_BUILD}') - } - } - - if (isCoreFxScenario(scenario)) { - - // Only arm/arm64 supported for corefx testing now. - assert architecture == 'arm' || architecture == 'arm64' - - // Unzip CoreFx runtime - batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxruntime.zip', '_\\fx\\artifacts\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}')\"") - - // Unzip CoreFx tests. - batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('_\\fx\\fxtests.zip', '_\\fx\\artifacts\\bin\\tests')\"") - - // Add the script to run the corefx tests - def corefx_runtime_path = "%WORKSPACE%\\_\\fx\\artifacts\\bin\\testhost\\netcoreapp-Windows_NT-Release-${architecture}" - def corefx_tests_dir = "%WORKSPACE%\\_\\fx\\artifacts\\bin\\tests" - def corefx_exclusion_file = "%WORKSPACE%\\tests\\${architecture}\\corefx_test_exclusions.txt" - def exclusionRspPath = "%WORKSPACE%\\tests\\scripts\\run-corefx-tests-exclusions.txt" - batchFile("call %WORKSPACE%\\tests\\scripts\\run-corefx-tests.bat ${corefx_runtime_path} ${corefx_tests_dir} ${corefx_exclusion_file} ${architecture} ${exclusionRspPath}") - - } else { // !isCoreFxScenario(scenario) - - // Unzip tests. - batchFile("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('bin\\tests\\tests.zip', 'bin\\tests\\${osGroup}.${architecture}.${configuration}')\"") - - def buildCommands = "" - - def coreRootLocation = "%WORKSPACE%\\bin\\tests\\Windows_NT.${architecture}.${configuration}\\Tests\\Core_Root" - def addEnvVariable = { variable, value -> buildCommands += "set ${variable}=${value}\r\n"} - def addCommand = { cmd -> buildCommands += "${cmd}\r\n"} - - // Make sure Command Extensions are enabled. Used so %ERRORLEVEL% is available. - addCommand("SETLOCAL ENABLEEXTENSIONS") - - // For all jobs - addEnvVariable("CORE_ROOT", coreRootLocation) - addEnvVariable("COMPlus_NoGuiOnAssert", "1") - addEnvVariable("COMPlus_ContinueOnAssert", "0") - - // If we are running a stress mode, we'll set those variables as well - if (isJitStressScenario(scenario) || isR2RStressScenario(scenario)) { - def stressValues = null - if (isJitStressScenario(scenario)) { - stressValues = Constants.jitStressModeScenarios[scenario] - } - else { - stressValues = Constants.r2rStressScenarios[scenario] - } - - stressValues.each { key, value -> - addEnvVariable(key, value) - } - } - - if (isR2RScenario(scenario)) { - // Crossgen the framework assemblies. - buildCommands += """ -@for %%F in (%CORE_ROOT%\\*.dll) do @call :PrecompileAssembly "%CORE_ROOT%" "%%F" %%~nxF -@goto skip_PrecompileAssembly - -:PrecompileAssembly -@REM Skip mscorlib since it is already precompiled. -@if /I "%3" == "mscorlib.dll" exit /b 0 -@if /I "%3" == "mscorlib.ni.dll" exit /b 0 - -"%CORE_ROOT%\\crossgen.exe" /Platform_Assemblies_Paths "%CORE_ROOT%" %2 >nul 2>nul -@if "%errorlevel%" == "-2146230517" ( - echo %2 is not a managed assembly. -) else if "%errorlevel%" == "-2146234344" ( - echo %2 is not a managed assembly. -) else if %errorlevel% neq 0 ( - echo Unable to precompile %2 -) else ( - echo Precompiled %2 -) -@exit /b 0 - -:skip_PrecompileAssembly -""" - - // Set RunCrossGen variable to cause test wrappers to invoke their logic to run - // crossgen on tests before running them. - addEnvVariable("RunCrossGen", "true") - } // isR2RScenario(scenario) - - // Run runtest.cmd - // Do not run generate layout. It will delete the correct CORE_ROOT, and we do not have a correct product - // dir to copy from. - def runtestCommand = "call %WORKSPACE%\\tests\\runtest.cmd ${architecture} ${configuration} skipgeneratelayout" - - addCommand("${runtestCommand}") - addCommand("echo on") // Show the following commands in the log. "echo" doesn't alter the errorlevel. - addCommand("set saved_errorlevel=%errorlevel%") - - // Collect the test logs collected by xunit. Ignore errors here. We want to collect these even if the run - // failed for some reason, so it needs to be in this batch file. - - addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Reports', '.\\bin\\tests\\testReports.zip')\""); - - // Use the runtest.cmd errorlevel as the script errorlevel. - addCommand("exit /b %saved_errorlevel%") - - batchFile(buildCommands) - } // non-corefx testing - } // steps - } // job - - if (!isCoreFxScenario(scenario)) { - def doNotFailIfNothingArchived = true - def archiveOnlyIfSuccessful = false - Utilities.addArchival(newJob, "bin/tests/testReports.zip", "", doNotFailIfNothingArchived, archiveOnlyIfSuccessful) - - Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true) - } - - return newJob -} - -// Create a test job not covered by the "Windows ARM" case that will be used by a flow job. -// E.g., non-Windows tests. -// Returns the newly created job. -def static CreateOtherTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName) -{ - def lowerConfiguration = configuration.toLowerCase() - - def isUbuntuArm64Job = ((os == "Ubuntu16.04") && (architecture == 'arm64')) - def isUbuntuArm32Job = ((os == "Ubuntu") && (architecture == 'arm')) - def isUbuntuArmJob = isUbuntuArm32Job || isUbuntuArm64Job - - def doCoreFxTesting = isCoreFxScenario(scenario) - def isPmiAsmDiffsScenario = (scenario == 'pmi_asm_diffs') - - def workspaceRelativeFxRootLinux = "_/fx" // only used for CoreFX testing - - def osGroup = getOSGroup(os) - def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst" - - def testOpts = '' - def useServerGC = false - - // Enable Server GC for Ubuntu PR builds - // REVIEW: why? Does this apply to all architectures? Why only PR? - if (os == 'Ubuntu' && isPR) { - testOpts += ' --useServerGC' - useServerGC = true - } - - if (isR2RScenario(scenario)) { - - testOpts += ' --crossgen --runcrossgentests' - - if (scenario == 'r2r_jitstress1') { - testOpts += ' --jitstress=1' - } - else if (scenario == 'r2r_jitstress2') { - testOpts += ' --jitstress=2' - } - else if (scenario == 'r2r_jitstress1_tiered') { - testOpts += ' --jitstress=1' - } - else if (scenario == 'r2r_jitstress2_tiered') { - testOpts += ' --jitstress=2' - } - else if (scenario == 'r2r_jitstressregs1') { - testOpts += ' --jitstressregs=1' - } - else if (scenario == 'r2r_jitstressregs2') { - testOpts += ' --jitstressregs=2' - } - else if (scenario == 'r2r_jitstressregs3') { - testOpts += ' --jitstressregs=3' - } - else if (scenario == 'r2r_jitstressregs4') { - testOpts += ' --jitstressregs=4' - } - else if (scenario == 'r2r_jitstressregs8') { - testOpts += ' --jitstressregs=8' - } - else if (scenario == 'r2r_jitstressregs0x10') { - testOpts += ' --jitstressregs=0x10' - } - else if (scenario == 'r2r_jitstressregs0x80') { - testOpts += ' --jitstressregs=0x80' - } - else if (scenario == 'r2r_jitstressregs0x1000') { - testOpts += ' --jitstressregs=0x1000' - } - else if (scenario == 'r2r_jitminopts') { - testOpts += ' --jitminopts' - } - else if (scenario == 'r2r_jitforcerelocs') { - testOpts += ' --jitforcerelocs' - } - else if (scenario == 'r2r_gcstress15') { - testOpts += ' --gcstresslevel=0xF' - } - } - else if (scenario == 'jitdiff') { - testOpts += ' --jitdisasm --crossgen' - } - else if (scenario == 'illink') { - testOpts += ' --link=\$WORKSPACE/linker/linker/bin/netcore_Release/netcoreapp2.0/ubuntu-x64/publish/illink' - } - else if (isLongGc(scenario)) { - // Long GC tests behave very poorly when they are not - // the only test running (many of them allocate until OOM). - testOpts += ' --sequential' - - // A note - runtest.sh does have "--long-gc" and "--gcsimulator" options - // for running long GC and GCSimulator tests, respectively. We don't use them - // here because using a playlist file produces much more readable output on the CI machines - // and reduces running time. - // - // The Long GC playlist contains all of the tests that are - // going to be run. The GCSimulator playlist contains all of - // the GC simulator tests. - if (scenario == 'longgc') { - testOpts += ' --long-gc --playlist=./tests/longRunningGcTests.txt' - } - else if (scenario == 'gcsimulator') { - testOpts += ' --gcsimulator --playlist=./tests/gcSimulatorTests.txt' - } - } - else if (isGcReliabilityFramework(scenario)) { - testOpts += ' --build-overlay-only' - } - else if (scenario == 'standalone_gc') { - if (osGroup == 'OSX') { - testOpts += ' --gcname=libclrgc.dylib' - } - else if (osGroup == 'Linux') { - testOpts += ' --gcname=libclrgc.so' - } - else { - println("Unexpected OS group: ${osGroup} for os ${os}") - assert false - } - } - - def jobFolder = getJobFolder(scenario) - def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) { - parameters { - stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from") - } - - steps { - // Set up the copies - - // Coreclr build we are trying to test - // - // HACK: the Ubuntu arm64 copyArtifacts Jenkins plug-in is ridiculously slow (45 minutes to - // 1.5 hours for this step). Instead, directly use wget, which is fast (1 minute). - - if (!isUbuntuArm64Job) { - copyArtifacts(inputCoreCLRBuildName) { - excludePatterns('**/testResults.xml', '**/*.ni.dll') - buildSelector { - buildNumber('${CORECLR_BUILD}') - } - } - } - - if (isUbuntuArmJob) { - // Add some useful information to the log file. Ignore return codes. - shell("uname -a || true") - } - - if (isUbuntuArm64Job) { - // Copy the required artifacts directly, using wget, e.g.: - // - // https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/testnativebin.checked.zip - // https://ci.dot.net/job/dotnet_coreclr/job/master/job/arm64_cross_checked_ubuntu16.04_innerloop_prtest/16/artifact/tests.checked.zip - // - // parameterized as: - // - // https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/testnativebin.checked.zip - // https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/job/${inputJobName}/${CORECLR_BUILD}/artifact/tests.checked.zip - // - // CoreFX example artifact URLs: - // - // https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxruntime.zip - // https://ci.dot.net/job/dotnet_coreclr/job/dev_unix_test_workflow/job/jitstress/job/arm64_cross_checked_ubuntu16.04_corefx_baseline_prtest/1/artifact/_/fx/fxtests.zip - // - // Note that the source might be in a "jitstress" folder. - // - // Use `--progress=dot:giga` to display some progress output, but limit it in the log file. - // - // Use `--directory-prefix=_/fx` to specify where to put the corefx files (to match what other platforms do). Use this instead of `-O`. - - shell("echo \"Using wget instead of the Jenkins copy artifacts plug-in to copy artifacts from ${inputCoreCLRBuildName}\"") - - def mungedProjectName = Utilities.getFolderName(project) - def mungedBranchName = Utilities.getFolderName(branch) - - def doCrossGenComparison = isCrossGenComparisonScenario(scenario) - def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal' - if (isPmiAsmDiffsScenario || doCoreFxTesting || doCrossGenComparison) { - // These depend on unique builds for each scenario - inputCoreCLRBuildScenario = scenario - } - def sourceJobName = getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, false) - def inputJobName = Utilities.getFullJobName(sourceJobName, isPR) - - // Need to add the sub-folder if necessary. - def inputJobPath = "job/${inputJobName}" - def folderName = getJobFolder(inputCoreCLRBuildScenario) - if (folderName != '') { - inputJobPath = "job/${folderName}/job/${inputJobName}" - } - - def inputUrlRoot = "https://ci.dot.net/job/${mungedProjectName}/job/${mungedBranchName}/${inputJobPath}/\${CORECLR_BUILD}/artifact" - - if (isPmiAsmDiffsScenario) { - def workspaceRelativeRootLinux = "_/pmi" - shell("mkdir -p ${workspaceRelativeRootLinux}") - shell("wget --progress=dot:giga ${inputUrlRoot}/product.${os}.${architecture}.${lowerConfiguration}.tgz") - shell("wget --progress=dot:giga ${inputUrlRoot}/product.baseline.${os}.${architecture}.${lowerConfiguration}.tgz") - shell("wget --progress=dot:giga ${inputUrlRoot}/coreroot.${os}.${architecture}.${lowerConfiguration}.tgz") - shell("wget --progress=dot:giga ${inputUrlRoot}/coreroot.baseline.${os}.${architecture}.${lowerConfiguration}.tgz") - } - else if (doCoreFxTesting) { - shell("mkdir -p ${workspaceRelativeFxRootLinux}") - shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxtests.zip") - shell("wget --progress=dot:giga --directory-prefix=${workspaceRelativeFxRootLinux} ${inputUrlRoot}/${workspaceRelativeFxRootLinux}/fxruntime.zip") - } - else { - def testArtifactsTgzFileName = getTestArtifactsTgzFileName(osGroup, architecture, configuration) - shell("wget --progress=dot:giga ${inputUrlRoot}/${testArtifactsTgzFileName}") - } - } - - if (architecture == 'x86') { - shell("mkdir ./bin/CoreFxNative") - - def fxBranch = getFxBranch(branch) - def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' + Utilities.getFolderName(fxBranch) - - copyArtifacts("${corefxFolder}/ubuntu16.04_x86_release") { - includePatterns('artifacts/bin/build.tar.gz') - targetDirectory('bin/CoreFxNative') - buildSelector { - latestSuccessful(true) - } - } - - shell("mkdir ./bin/CoreFxBinDir") - shell("tar -xf ./bin/CoreFxNative/artifacts/bin/build.tar.gz -C ./bin/CoreFxBinDir") - } - - if (isPmiAsmDiffsScenario) { - shell("tar -xzf ./product.${os}.${architecture}.${lowerConfiguration}.tgz || exit 0") - shell("tar -xzf ./product.baseline.${os}.${architecture}.${lowerConfiguration}.tgz || exit 0") - shell("tar -xzf ./coreroot.${os}.${architecture}.${lowerConfiguration}.tgz || exit 0") - shell("tar -xzf ./coreroot.baseline.${os}.${architecture}.${lowerConfiguration}.tgz || exit 0") - } - // CoreFX testing downloads the CoreFX tests, not the coreclr tests. Also, unzip the built CoreFX layout/runtime directories. - else if (doCoreFxTesting) { - shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxtests.zip || exit 0") - shell("unzip -q -o ${workspaceRelativeFxRootLinux}/fxruntime.zip || exit 0") - } - else { - def testArtifactsTgzFileName = getTestArtifactsTgzFileName(osGroup, architecture, configuration) - shell("tar -xzf ./${testArtifactsTgzFileName} || exit 0") // extracts to ./bin/tests/${osGroup}.${architecture}.${configuration} - } - - // Execute the tests - def runDocker = isNeedDocker(architecture, os, false) - def dockerPrefix = "" - def dockerCmd = "" - if (runDocker) { - def dockerImage = getDockerImageName(architecture, os, false) - dockerPrefix = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} " - dockerCmd = dockerPrefix + "${dockerImage} " - } - - // If we are running a stress mode, we'll set those variables first. - // For CoreFX, the stress variables are already built into the CoreFX test build per-test wrappers. - if (!doCoreFxTesting && isJitStressScenario(scenario)) { - def scriptFileName = "\${WORKSPACE}/set_stress_test_env.sh" - def envScriptCmds = envScriptCreate(os, scriptFileName) - envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName) - envScriptCmds += envScriptFinalize(os, scriptFileName) - shell("${envScriptCmds}") - testOpts += " --test-env=${scriptFileName}" - } - - // setup-stress-dependencies.sh, invoked by runtest.sh to download the coredistools package, depends on the "dotnet" - // tool downloaded by the "init-tools.sh" script. However, it only invokes setup-stress-dependencies.sh for x64. The - // coredistools package is used by GCStress on x86 and x64 to disassemble code to determine instruction boundaries. - // On arm/arm64, it is not required as determining instruction boundaries is trivial. - if (isGCStressRelatedTesting(scenario)) { - if (architecture == 'x64') { - shell('./init-tools.sh') - } - } - - if (isPmiAsmDiffsScenario) { - shell("""\ -python -u \${WORKSPACE}/tests/scripts/run-pmi-diffs.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} --skip_baseline_build""") - - shell("tar -czf dasm.${os}.${architecture}.${configuration}.tgz ./_/pmi/asm") - } - else if (doCoreFxTesting) { - def exclusionRspPath = "\${WORKSPACE}/tests/scripts/run-corefx-tests-exclusions.txt" - shell("""\ -\${WORKSPACE}/tests/scripts/run-corefx-tests.sh --test-exclude-file \${WORKSPACE}/tests/${architecture}/corefx_linux_test_exclusions.txt --runtime \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin/testhost/netcoreapp-Linux-Release-${architecture} --arch ${architecture} --corefx-tests \${WORKSPACE}/${workspaceRelativeFxRootLinux}/artifacts/bin --configurationGroup Release --exclusion-rsp-file ${exclusionRspPath}""") - } - else { - def runScript = "${dockerCmd}./tests/runtest.sh" - - shell("""\ -${runScript} \\ - ${lowerConfiguration} \\ - --testRootDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}\" \\ - --coreOverlayDir=\"\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root\" \\ - --limitedDumpGeneration ${testOpts}""") - } - - if (isGcReliabilityFramework(scenario)) { - // runtest.sh doesn't actually execute the reliability framework - do it here. - if (useServerGC) { - if (runDocker) { - dockerCmd = dockerPrefix + "-e COMPlus_gcServer=1 ${dockerImage} " - } - else { - shell("export COMPlus_gcServer=1") - } - } - - shell("${dockerCmd}./tests/scripts/run-gc-reliability-framework.sh ${architecture} ${configuration}") - } - } // steps - } // job - - // Experimental: If on Ubuntu 14.04, then attempt to pull in crash dump links - if (os in ['Ubuntu']) { - SummaryBuilder summaries = new SummaryBuilder() - summaries.addLinksSummaryFromFile('Crash dumps from this run:', 'dumplings.txt') - summaries.emit(newJob) - } - - if (isPmiAsmDiffsScenario) { - // Archive the asm - Utilities.addArchival(newJob, "dasm.${os}.${architecture}.${configuration}.tgz") - } - else if (doCoreFxTesting) { - Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/artifacts/bin/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) - if ((os == "Ubuntu") && (architecture == 'arm')) { - // We have a problem with the xunit plug-in, where it is consistently failing on Ubuntu arm32 test result uploading with this error: - // - // [xUnit] [ERROR] - The plugin hasn't been performed correctly: remote file operation failed: /ssd/j/workspace/dotnet_coreclr/master/jitstress/arm_cross_checked_ubuntu_corefx_baseline_tst at hudson.remoting.Channel@3697f46d:JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529: java.io.IOException: Remote call on JNLP4-connect connection from 131.107.159.149/131.107.159.149:58529 failed - // - // We haven't been able to identify the reason. So, do not add xunit parsing of the test data in this scenario. - // This is tracked by: https://github.com/dotnet/coreclr/issues/19447. - } - else { - Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRootLinux}/artifacts/bin/**/testResults.xml") - } - } - else { - Utilities.addXUnitDotNETResults(newJob, '**/coreclrtests.xml') - } - - return newJob -} - -def static CreateNonWindowsCrossGenComparisonTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName) -{ - assert isCrossGenComparisonScenario(scenario) - - def osGroup = getOSGroup(os) - def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst" - - def workspaceRelativeResultsDir = "_" - def workspaceRelativeNativeArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${architecture}_${architecture}.${configuration}" - - def crossArch = "x64" - def workspaceRelativeCrossArchResultDir = "${workspaceRelativeResultsDir}/${osGroup}.${crossArch}_${architecture}.${configuration}" - - def jobFolder = getJobFolder(scenario) - def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) { - parameters { - stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from") - } - - def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip" - - steps { - copyArtifacts(inputCoreCLRBuildName) { - includePatterns("${workspaceRelativeArtifactsArchive}") - buildSelector { - buildNumber('${CORECLR_BUILD}') - } - } - - shell("unzip -o ${workspaceRelativeArtifactsArchive} || exit 0") - - def workspaceRelativeCoreLib = "bin/Product/${osGroup}.${architecture}.${configuration}/IL/System.Private.CoreLib.dll" - def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root" - def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py" - def workspaceRelativeCrossGenExecutable = "${workspaceRelativeCoreRootDir}/crossgen" - - def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} " - def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeCrossGenExecutable}" - - shell("mkdir -p ${workspaceRelativeNativeArchResultDir}") - shell("${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir}") - shell("${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir}") - - shell("${crossGenComparisonCmd}compare --base_dir \${WORKSPACE}/${workspaceRelativeNativeArchResultDir} --diff_dir \${WORKSPACE}/${workspaceRelativeCrossArchResultDir}") - } // steps - } // job - - Utilities.addArchival(newJob, "${workspaceRelativeNativeArchResultDir}/**", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) - Utilities.addArchival(newJob, "${workspaceRelativeCrossArchResultDir}/**", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) - - return newJob -} - -// Create a test job that will be used by a flow job. -// Returns the newly created job. -// Note that we don't add tests jobs to the various views, since they are always used by a flow job, which is in the views, -// and we want the views to be the minimal set of "top-level" jobs that represent all work. -def static CreateTestJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName) -{ - def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList)) - - def newJob = null - if (windowsArmJob) { - newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName) - } - else if (isCrossGenComparisonScenario(scenario)) { - newJob = CreateNonWindowsCrossGenComparisonTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName) - } - else { - newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName) - } - - setJobMachineAffinity(architecture, os, false, true, false, newJob) // isBuildJob = false, isTestJob = true, isFlowJob = false - - if (scenario == 'jitdiff') { - def osGroup = getOSGroup(os) - Utilities.addArchival(newJob, "bin/tests/${osGroup}.${architecture}.${configuration}/dasm/**") - } - - Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") - setJobTimeout(newJob, isPR, architecture, configuration, scenario, false) - - return newJob -} - -// Create a flow job to tie together a build job with the given test job. -// Returns the new flow job. -def static CreateFlowJob(def dslFactory, def project, def branch, def architecture, def os, def configuration, def scenario, def isPR, def fullTestJobName, def inputCoreCLRBuildName) -{ - // Windows CoreCLR build and Linux CoreCLR build (in parallel) -> - // Linux CoreCLR test - def flowJobName = getJobName(configuration, architecture, os, scenario, false) + "_flow" - def jobFolder = getJobFolder(scenario) - - def newFlowJob = dslFactory.buildFlowJob(Utilities.getFullJobName(project, flowJobName, isPR, jobFolder)) { - buildFlow("""\ -coreclrBuildJob = build(params, '${inputCoreCLRBuildName}') - -// And then build the test build -build(params + [CORECLR_BUILD: coreclrBuildJob.build.number], '${fullTestJobName}') -""") - } - JobReport.Report.addReference(inputCoreCLRBuildName) - JobReport.Report.addReference(fullTestJobName) - - addToViews(newFlowJob, true, isPR, architecture, os, configuration, scenario) // isFlowJob = true - - setJobMachineAffinity(architecture, os, false, false, true, newFlowJob) // isBuildJob = false, isTestJob = false, isFlowJob = true - - Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}") - addTriggers(newFlowJob, branch, isPR, architecture, os, configuration, scenario, true, false) // isFlowJob==true, isWindowsBuildOnlyJob==false - - return newFlowJob -} - -// Determine if we should generate a flow job for the given parameters. -// Returns true if the job should be generated. -def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def configuration, def os) -{ - // The various "innerloop" jobs are only available as PR triggered. - - if (!isPR) { - if (isInnerloopTestScenario(scenario)) { - return false - } - - if (scenario == 'corefx_innerloop') { - return false - } - } - - // Disable flow jobs for innerloop pr. - // - // The only exception is windows arm(64) - if (isInnerloopTestScenario(scenario) && isPR && os != 'Windows_NT') { - assert scenario != 'corefx_innerloop' - - return false; - } - - // Filter based on OS and architecture. - - switch (architecture) { - case 'arm': - if (os != "Ubuntu" && os != "Windows_NT") { - return false - } - break - case 'arm64': - if (os != "Ubuntu16.04" && os != "Windows_NT") { - return false - } - break - case 'x86': - if (os != "Ubuntu") { - return false - } - break - case 'x64': - if (!(os in Constants.crossList)) { - return false - } - if (os == "Windows_NT") { - return false - } - break - case 'armem': - // No flow jobs - return false - default: - println("Unknown architecture: ${architecture}") - assert false - break - } - - def isNormalOrInnerloop = (scenario == 'innerloop' || scenario == 'normal') - - // Filter based on scenario in OS. - - if (os == 'Windows_NT') { - assert architecture == 'arm' || architecture == 'arm64' - if (!isArmWindowsScenario(scenario)) { - return false - } - if (isNormalOrInnerloop && (configuration == 'Debug')) { - // The arm32/arm64 Debug configuration for innerloop/normal scenario is a special case: it does a build only, and no test run. - // To do that, it doesn't require a flow job. - return false - } - } - else { - // Non-Windows - if (architecture == 'arm') { - if (!(scenario in Constants.validLinuxArmScenarios)) { - return false - } - } - else if (architecture == 'arm64') { - if (!(scenario in Constants.validLinuxArm64Scenarios)) { - return false - } - } - else if (architecture == 'x86') { - // Linux/x86 only want innerloop and default test - if (!isNormalOrInnerloop) { - return false - } - } - else if (architecture == 'x64') { - // Linux/x64 corefx testing doesn't need a flow job; the "build" job runs run-corefx-tests.py which - // builds and runs the corefx tests. Other Linux/x64 flow jobs are required to get the test - // build from a Windows machine. - if (isCoreFxScenario(scenario)) { - return false - } - } - } - - // For CentOS, we only want Checked/Release builds. - if (os == 'CentOS7.1') { - if (configuration != 'Checked' && configuration != 'Release') { - return false - } - if (!isNormalOrInnerloop && !isR2RScenario(scenario)) { - return false - } - } - - // For RedHat and Debian, we only do Release builds. - else if (os == 'RHEL7.2' || os == 'Debian8.4') { - if (configuration != 'Release') { - return false - } - if (!isNormalOrInnerloop) { - return false - } - } - - // Next, filter based on scenario. - - if (isJitStressScenario(scenario)) { - if (configuration != 'Checked') { - return false - } - } - else if (isR2RBaselineScenario(scenario)) { - if (configuration != 'Checked' && configuration != 'Release') { - return false - } - } - else if (isR2RStressScenario(scenario)) { - if (configuration != 'Checked') { - return false - } - } - else if (isCrossGenComparisonScenario(scenario)) { - return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario) - } - else { - // Skip scenarios - switch (scenario) { - case 'ilrt': - case 'longgc': - case 'gcsimulator': - // Long GC tests take a long time on non-Release builds - // ilrt is also Release only - if (configuration != 'Release') { - return false - } - break - - case 'jitdiff': - if (configuration != 'Checked') { - return false - } - break - - case 'gc_reliability_framework': - case 'standalone_gc': - if (configuration != 'Release' && configuration != 'Checked') { - return false - } - break - - case 'formatting': - return false - - case 'illink': - if (os != 'Windows_NT' && os != 'Ubuntu') { - return false - } - break - - case 'normal': - // Nothing skipped - break - - case 'innerloop': - if (!isValidPrTriggeredInnerLoopJob(os, architecture, configuration, false)) { - return false - } - break - - case 'pmi_asm_diffs': - if (configuration != 'Checked') { - return false - } - // No need for flow job except for Linux arm/arm64 - if ((os != 'Windows_NT') && (architecture != 'arm') && (architecture != 'arm64')) { - return false - } - break - - case 'corefx_innerloop': - // No flow job needed - return false - - default: - println("Unknown scenario: ${scenario}") - assert false - break - } - } - - // The job was not filtered out, so we should generate it! - return true -} - -// Create jobs requiring flow jobs. This includes x64 non-Windows, arm/arm64 Ubuntu, and arm/arm64 Windows. -Constants.allScenarios.each { scenario -> - [true, false].each { isPR -> - Constants.architectureList.each { architecture -> - Constants.configurationList.each { configuration -> - Constants.osList.each { os -> - - if (!shouldGenerateFlowJob(scenario, isPR, architecture, configuration, os)) { - return - } - - def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList)) - def doCoreFxTesting = isCoreFxScenario(scenario) - def doCrossGenComparison = isCrossGenComparisonScenario(scenario) - def isPmiAsmDiffsScenario = (scenario == 'pmi_asm_diffs') - - // Figure out the job name of the CoreCLR build the test will depend on. - - def inputCoreCLRBuildScenario = isInnerloopTestScenario(scenario) ? 'innerloop' : 'normal' - def inputCoreCLRBuildIsBuildOnly = false - if (doCoreFxTesting || isPmiAsmDiffsScenario) { - // Every CoreFx test depends on its own unique build. - inputCoreCLRBuildScenario = scenario - if (windowsArmJob) { - // Only Windows ARM corefx jobs use "build only" jobs. Others, such as Ubuntu ARM corefx, use "regular" jobs. - inputCoreCLRBuildIsBuildOnly = true - } - } - else if (doCrossGenComparison) { - inputCoreCLRBuildScenario = scenario - } - - def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario) - def inputCoreCLRBuildName = projectFolder + '/' + - Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName) - - // ============================================================================================= - // Create the test job - // ============================================================================================= - - def testJob = CreateTestJob(this, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName) - - // ============================================================================================= - // Create a build flow to join together the build and tests required to run this test. - // ============================================================================================= - - if (os == 'RHEL7.2' || os == 'Debian8.4') { - // Do not create the flow job for RHEL jobs. - return - } - - def fullTestJobName = projectFolder + '/' + testJob.name - def flowJob = CreateFlowJob(this, project, branch, architecture, os, configuration, scenario, isPR, fullTestJobName, inputCoreCLRBuildName) - - } // os - } // configuration - } // architecture - } // isPR -} // scenario - -JobReport.Report.generateJobReport(out) - -// Make the call to generate the help job -Utilities.createHelperJob(this, project, branch, - "Welcome to the ${project} Repository", // This is prepended to the help message - "Have a nice day!") // This is appended to the help message. You might put known issues here. - -Utilities.addCROSSCheck(this, project, branch) diff --git a/perf.groovy b/perf.groovy deleted file mode 100644 index a8316e4731..0000000000 --- a/perf.groovy +++ /dev/null @@ -1,957 +0,0 @@ -// Import the utility functionality. - -import jobs.generation.*; - -def project = GithubProject -def branch = GithubBranchName -def projectName = Utilities.getFolderName(project) -def projectFolder = projectName + '/' + Utilities.getFolderName(branch) - -def static getOSGroup(def os) { - def osGroupMap = ['Ubuntu14.04':'Linux', - 'RHEL7.2': 'Linux', - 'Ubuntu16.04': 'Linux', - 'Debian8.4':'Linux', - 'Fedora24':'Linux', - 'OSX':'OSX', - 'Windows_NT':'Windows_NT', - 'FreeBSD':'FreeBSD', - 'CentOS7.1': 'Linux', - 'OpenSUSE13.2': 'Linux', - 'OpenSUSE42.1': 'Linux', - 'LinuxARMEmulator': 'Linux'] - def osGroup = osGroupMap.get(os, null) - assert osGroup != null : "Could not find os group for ${os}" - return osGroupMap[os] -} - -// Setup perflab tests runs -[true, false].each { isPR -> - ['Windows_NT'].each { os -> - ['x64', 'x86'].each { arch -> - [true, false].each { isSmoketest -> - ['ryujit'].each { jit -> - ['full_opt', 'min_opt'].each { opt_level -> - - def architecture = arch - def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}" - def testEnv = "" - def python = "C:\\Python35\\python.exe" - - def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) { - // Set the label. - if (isSmoketest) { - label('Windows.10.Amd64.ClientRS4.DevEx.15.8.Open') - python = "C:\\python3.7.0\\python.exe" - } - else { - label('windows_server_2016_clr_perf') - } - - wrappers { - credentialsBinding { - string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') - } - } - - if (isPR) { - parameters { - stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName') - } - } - - if (isSmoketest) { - parameters { - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample') - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample') - } - } - else{ - parameters { - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample') - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample') - } - } - - def configuration = 'Release' - def runType = isPR ? 'private' : 'rolling' - def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%' - def uploadString = isSmoketest ? '' : '-uploadToBenchview' - - steps { - // Batch - - batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"") - batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"") - batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion") - //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView - //we have to do it all as one statement because cmd is called each time and we lose the set environment variable - batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" + - "set \"BENCHVIEWNAME=${benchViewName}\"\n" + - "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" + - "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" + - "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}") - batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"") - batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}") - - batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly") - - def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH /AFFINITY 0x2\"" - - // Run with just stopwatch: Profile=Off - batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library") - batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality") - - // Run with the full set of counters enabled: Profile=On - if (opt_level != 'min_opt') { - batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi") - batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi") - } - } - } - - def archiveSettings = new ArchivalSettings() - archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt') - archiveSettings.addFiles('bin/sandbox_logs/**/*.csv') - archiveSettings.addFiles('bin/sandbox_logs/**/*.xml') - archiveSettings.addFiles('bin/sandbox_logs/**/*.log') - archiveSettings.addFiles('bin/sandbox_logs/**/*.md') - archiveSettings.addFiles('bin/sandbox_logs/**/*.etl') - archiveSettings.addFiles('machinedata.json') - archiveSettings.setAlwaysArchive() - - Utilities.addArchival(newJob, archiveSettings) - Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") - - newJob.with { - logRotator { - artifactDaysToKeep(14) - daysToKeep(30) - artifactNumToKeep(100) - numToKeep(200) - } - wrappers { - timeout { - absolute(240) - } - } - } - - if (isPR) { - TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest() - if (isSmoketest) { - builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests Correctness") - } - else { - builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} CoreCLR Perf Tests") - - def opts = "" - if (opt_level == 'min_opt') { - opts = '\\W+min_opts' - } - def jitt = "" - if (jit != 'ryujit') { - jitt = "\\W+${jit}" - } - - builder.triggerOnlyOnComment() - builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf.*") - } - - builder.triggerForBranch(branch) - builder.emitTrigger(newJob) - } - else if (opt_level == 'full_opt') { - // Set a push trigger - TriggerBuilder builder = TriggerBuilder.triggerOnCommit() - builder.emitTrigger(newJob) - } - else { - // Set periodic trigger - Utilities.addPeriodicTrigger(newJob, '@daily') - } - } - } - } - } - } -} - -// Setup throughput perflab tests runs -[true, false].each { isPR -> - ['Windows_NT'].each { os -> - ['x64', 'x86'].each { arch -> - ['ryujit'].each { jit -> - [true, false].each { pgo_optimized -> - ['full_opt', 'min_opt'].each { opt_level -> - def architecture = arch - - def python = "C:\\Python35\\python.exe" - - pgo_build = "" - pgo_test = "" - pgo_string = "pgo" - if (!pgo_optimized) { - pgo_build = " -nopgooptimize" - pgo_test = " -nopgo" - pgo_string = "nopgo" - } - - def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}_${jit}_${pgo_string}", isPR)) { - // Set the label. - label('windows_server_2016_clr_perf') - wrappers { - credentialsBinding { - string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') - } - } - - if (isPR) { - parameters { - stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.') - } - } - - def configuration = 'Release' - def runType = isPR ? 'private' : 'rolling' - def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%' - - steps { - // Batch - batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"") - batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"") - batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion") - batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion") - //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView - //we have to do it all as one statement because cmd is called each time and we lose the set environment variable - batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" + - "set \"BENCHVIEWNAME=${benchViewName}\"\n" + - "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" + - "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" + - "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}") - batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"") - batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests") - batchFile("${python} -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}") - } - } - - // Save machinedata.json to /artifact/bin/ Jenkins dir - def archiveSettings = new ArchivalSettings() - archiveSettings.addFiles('throughput-*.csv') - archiveSettings.setAlwaysArchive() - Utilities.addArchival(newJob, archiveSettings) - - Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") - - if (isPR) { - def opts = "" - if (opt_level == 'min_opt') { - opts = '\\W+min_opts' - } - - def jitt = "" - if (jit != 'ryujit') { - jitt = "\\W+${jit}" - } - - def pgo_trigger = "" - if (pgo_optimized) { - pgo_trigger = "\\W+nopgo" - } - - - TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest() - builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} ${pgo_string} CoreCLR Throughput Perf Tests") - builder.triggerOnlyOnComment() - builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}${pgo_trigger}\\W+throughput.*") - builder.triggerForBranch(branch) - builder.emitTrigger(newJob) - } - else if (opt_level == 'full_opt' && pgo_optimized) { - // Set a push trigger - TriggerBuilder builder = TriggerBuilder.triggerOnCommit() - builder.emitTrigger(newJob) - } - else { - // Set periodic trigger - Utilities.addPeriodicTrigger(newJob, '@daily') - } - } - } - } - } - } -} - -def static getFullPerfJobName(def project, def os, def arch, def isPR) { - return Utilities.getFullJobName(project, "perf_${os}_${arch}", isPR) -} - -// Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario -[true, false].each { isPR -> - ['x64'].each { architecture -> - def fullBuildJobName = Utilities.getFullJobName(project, "perf_linux_build", isPR) - def configuration = 'Release' - - def crossCompile = "" - def crossLayout = "" - def python = "python3.5" - - // Build has to happen on RHEL7.2 (that's where we produce the bits we ship) - ['RHEL7.2'].each { os -> - def newBuildJob = job(fullBuildJobName) { - steps { - shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}") - shell("./build-test.sh generatelayoutonly ${architecture} ${configuration}${crossLayout}") - } - } - Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto') - Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}") - Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so,bin/tests/**", "bin/Product/**/.nuget/**") - } - - - // Actual perf testing on the following OSes - def perfOSList = ['Ubuntu16.04'] - - perfOSList.each { os -> - def newJob = job(getFullPerfJobName(project, os, architecture, isPR)) { - - def machineLabel = 'ubuntu_1604_clr_perf' - - label(machineLabel) - wrappers { - credentialsBinding { - string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') - } - } - - if (isPR) { - parameters { - stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName') - } - } - - parameters { - // Cap the maximum number of iterations to 21. - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample') - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enough to get a good sample') - stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux') - } - - def osGroup = getOSGroup(os) - def runType = isPR ? 'private' : 'rolling' - def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT' - def uploadString = '-uploadToBenchview' - - def runXUnitCommonArgs = "-arch ${architecture} -os ${os} -configuration ${configuration} -stabilityPrefix \"taskset 0x00000002 nice --adjustment=-10\" -generateBenchviewData \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" ${uploadString} -runtype ${runType} -outputdir \"\${WORKSPACE}/bin/sandbox_logs\"" - - steps { - shell("./tests/scripts/perf-prep.sh --nocorefx") - shell("./init-tools.sh") - copyArtifacts(fullBuildJobName) { - includePatterns("bin/**") - buildSelector { - buildNumber('\${PRODUCT_BUILD}') - } - } - shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" + - "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" + - "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}") - shell("""${python} ./tests/scripts/run-xunit-perf.py -testBinLoc bin/tests/Windows_NT.${architecture}.${configuration}/JIT/Performance/CodeQuality ${runXUnitCommonArgs}""") - } - } - - def archiveSettings = new ArchivalSettings() - archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt') - archiveSettings.addFiles('bin/sandbox_logs/**/*.csv') - archiveSettings.addFiles('bin/sandbox_logs/**/*.xml') - archiveSettings.addFiles('bin/sandbox_logs/**/*.log') - archiveSettings.addFiles('bin/sandbox_logs/**/*.md') - archiveSettings.addFiles('bin/sandbox_logs/**/*.etl') - archiveSettings.addFiles('machinedata.json') - archiveSettings.setAlwaysArchive() - - Utilities.addArchival(newJob, archiveSettings) - Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") - - // For perf, we need to keep the run results longer - newJob.with { - // Enable the log rotator - logRotator { - artifactDaysToKeep(14) - daysToKeep(30) - artifactNumToKeep(100) - numToKeep(200) - } - wrappers { - timeout { - absolute(240) - } - } - } - } // os - - def flowJobPerfRunList = perfOSList.collect { os -> - "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, architecture, isPR)}') }" - } - def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_${architecture}_flow", isPR, '')) { - if (isPR) { - parameters { - stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName') - } - } - buildFlow(""" -// First, build the bits on RHEL7.2 -b = build(params, '${fullBuildJobName}') - -// Then, run the perf tests -parallel( - ${flowJobPerfRunList.join(",\n ")} -) -""") - } - - Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto') - Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}") - - if (isPR) { - TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest() - builder.setGithubContext("Linux Perf Test Flow") - builder.triggerOnlyOnComment() - builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*") - builder.triggerForBranch(branch) - builder.emitTrigger(newFlowJob) - } - else { - // Set a push trigger - TriggerBuilder builder = TriggerBuilder.triggerOnCommit() - builder.emitTrigger(newFlowJob) - } - } // architecture -} // isPR - -def static getDockerImageName(def architecture, def os, def isBuild) { - // We must change some docker private images to official later - if (isBuild) { - if (architecture == 'arm') { - if (os == 'Ubuntu') { - return "microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420" - } - } - } - println("Unknown architecture to use docker: ${architecture} ${os}"); - assert false -} - -def static getFullThroughputJobName(def project, def os, def arch, def isPR) { - return Utilities.getFullJobName(project, "perf_throughput_${os}_${arch}", isPR) -} - -// Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario -[true, false].each { isPR -> - ['x64','arm'].each { architecture -> - def fullBuildJobName = Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_build", isPR) - def configuration = 'Release' - - - def crossCompile = "" - def python = "python3.5" - def tgzFile = "bin-Product-Linux.${architecture}.${configuration}.tgz" - - if (architecture == "arm") { - python = "python3.6" - def buildCommands = [] - def newBuildJob = job(fullBuildJobName) { - def dockerImage = getDockerImageName(architecture, 'Ubuntu', true) - def dockerCmd = "docker run -i --rm -v \${WORKSPACE}:\${WORKSPACE} -w \${WORKSPACE} -e ROOTFS_DIR=/crossrootfs/${architecture} ${dockerImage} " - - buildCommands += "${dockerCmd}\${WORKSPACE}/build.sh release ${architecture} cross" - - steps { - buildCommands.each { buildCommand -> - shell(buildCommand) - shell("tar -czf ${tgzFile} bin/Product/Linux.${architecture}.${configuration}") - } - } - - publishers { - azureVMAgentPostBuildAction { - agentPostBuildAction('Delete agent after build execution (when idle).') - } - } - } - Utilities.setMachineAffinity(newBuildJob, "Ubuntu16.04", 'latest-or-auto') - Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}") - Utilities.addArchival(newBuildJob, "${tgzFile}") - } - else { - // Build has to happen on RHEL7.2 (that's where we produce the bits we ship) - ['RHEL7.2'].each { os -> - def newBuildJob = job(fullBuildJobName) { - steps { - shell("./build.sh verbose ${architecture} ${configuration}${crossCompile}") - } - } - Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto') - Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}") - Utilities.addArchival(newBuildJob, "bin/Product/**") - } - } - - // Actual perf testing on the following OSes - def throughputOSList = ['Ubuntu16.04'] - if (architecture == 'arm') { - throughputOSList = ['Ubuntu14.04'] - } - def throughputOptLevelList = ['full_opt', 'min_opt'] - - def throughputOSOptLevelList = [] - - throughputOSList.each { os -> - throughputOptLevelList.each { opt_level -> - throughputOSOptLevelList.add("${os}_${opt_level}") - } - } - - throughputOSList.each { os -> - throughputOptLevelList.each { opt_level -> - def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", architecture, isPR)) { - - def machineLabel = 'ubuntu_1604_clr_perf' - if (architecture == 'arm') { - machineLabel = 'ubuntu_1404_clr_perf_arm' - } - - label(machineLabel) - wrappers { - credentialsBinding { - string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') - } - } - - if (isPR) { - parameters { - stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.') - } - } - - parameters { - stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux') - } - - def osGroup = getOSGroup(os) - def runType = isPR ? 'private' : 'rolling' - def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT' - def archString = architecture == 'arm' ? ' --arch=arm' : '' - - steps { - shell("bash ./tests/scripts/perf-prep.sh --throughput${archString}") - copyArtifacts(fullBuildJobName) { - if (architecture == 'arm') { - includePatterns("${tgzFile}") - } - else { - includePatterns("bin/Product/**") - } - buildSelector { - buildNumber('\${PRODUCT_BUILD}') - } - } - if (architecture == 'arm') { - shell("tar -xzf ./${tgzFile} || exit 0") - } - shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" + - "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user-email \"dotnet-bot@microsoft.com\"\n" + - "${python} \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}") - shell("""${python} ./tests/scripts/run-throughput-perf.py \\ - -arch \"${architecture}\" \\ - -os \"${os}\" \\ - -configuration \"${configuration}\" \\ - -opt_level \"${opt_level}\" \\ - -clr_root \"\${WORKSPACE}\" \\ - -assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.x64.Windows_NT/lib\" \\ - -run_type \"${runType}\" \\ - -benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""") - } - } - - // Save machinedata.json to /artifact/bin/ Jenkins dir - def archiveSettings = new ArchivalSettings() - archiveSettings.addFiles('throughput-*.csv') - archiveSettings.addFiles('machinedata.json') - archiveSettings.setAlwaysArchive() - Utilities.addArchival(newJob, archiveSettings) - - Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") - - // For perf, we need to keep the run results longer - newJob.with { - // Enable the log rotator - logRotator { - artifactDaysToKeep(7) - daysToKeep(300) - artifactNumToKeep(25) - numToKeep(1000) - } - } - } // opt_level - } // os - - def flowJobTPRunList = throughputOSOptLevelList.collect { os -> - "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, architecture, isPR)}') }" - } - def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_${architecture}_flow", isPR, '')) { - if (isPR) { - parameters { - stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName') - } - } - buildFlow(""" - // First, build the bits on RHEL7.2 - b = build(params, '${fullBuildJobName}') - - // Then, run the perf tests - parallel( - ${flowJobTPRunList.join(",\n ")} - ) - """) - } - - Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto') - Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}") - - if (isPR) { - TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest() - builder.setGithubContext("Linux ${architecture} Throughput Perf Test Flow") - builder.triggerOnlyOnComment() - builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*") - builder.triggerForBranch(branch) - builder.emitTrigger(newFlowJob) - } - else { - // Set a push trigger - TriggerBuilder builder = TriggerBuilder.triggerOnCommit() - builder.emitTrigger(newFlowJob) - } - } // architecture -} // isPR - -// Setup CoreCLR-Scenarios tests -[true, false].each { isPR -> - ['Windows_NT'].each { os -> - ['x64', 'x86'].each { arch -> - ['ryujit'].each { jit -> - ['full_opt', 'min_opt', 'tiered'].each { opt_level -> - def architecture = arch - def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) { - - def testEnv = "" - def python = "C:\\Python35\\python.exe" - - // Set the label. - label('windows_server_2016_clr_perf') - wrappers { - credentialsBinding { - string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') - } - } - - if (isPR) { - parameters { - stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName') - } - } - - parameters { - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient') - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient') - } - - def configuration = 'Release' - def runType = isPR ? 'private' : 'rolling' - def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%' - def uploadString = '-uploadToBenchview' - - steps { - // Batch - batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"") - batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"") - batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion") - - //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView - //we have to do it all as one statement because cmd is called each time and we lose the set environment variable - batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" + - "set \"BENCHVIEWNAME=${benchViewName}\"\n" + - "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" + - "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" + - "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}") - batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"") - batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}") - - batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly") - - def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH\" -scenarioTest" - - // Profile=Off - batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios") - - // Profile=On - if (opt_level != 'min_opt') { - batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired") - } - } - } - - def archiveSettings = new ArchivalSettings() - archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt') - archiveSettings.addFiles('bin/sandbox_logs/**/*.csv') - archiveSettings.addFiles('bin/sandbox_logs/**/*.xml') - archiveSettings.addFiles('bin/sandbox_logs/**/*.log') - archiveSettings.addFiles('bin/sandbox_logs/**/*.md') - archiveSettings.addFiles('bin/sandbox_logs/**/*.etl') - archiveSettings.addFiles('machinedata.json') - archiveSettings.setAlwaysArchive() - - Utilities.addArchival(newJob, archiveSettings) - Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") - - newJob.with { - logRotator { - artifactDaysToKeep(14) - daysToKeep(30) - artifactNumToKeep(100) - numToKeep(200) - } - wrappers { - timeout { - absolute(240) - } - } - } - - if (isPR) { - def opts = "" - if (opt_level == 'min_opt') { - opts = '\\W+min_opts' - } - def jitt = "" - if (jit != 'ryujit') { - jitt = "\\W+${jit}" - } - - TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest() - builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} Performance Scenarios Tests") - builder.triggerOnlyOnComment() - builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}${jitt}\\W+perf\\W+scenarios.*") - builder.triggerForBranch(branch) - builder.emitTrigger(newJob) - } - else if (opt_level == 'full_opt') { - // Set a push trigger - TriggerBuilder builder = TriggerBuilder.triggerOnCommit() - builder.emitTrigger(newJob) - } - else { - // Set periodic trigger - Utilities.addPeriodicTrigger(newJob, '@daily') - } - } - } - } - } -} - -// Setup size-on-disk test -['Windows_NT'].each { os -> - ['x64', 'x86'].each { arch -> - def architecture = arch - def newJob = job(Utilities.getFullJobName(project, "sizeondisk_${arch}", false)) { - label('Windows.10.Amd64.ClientRS4.DevEx.15.8.Open') - - wrappers { - credentialsBinding { - string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') - } - } - - def channel = 'master' - def configuration = 'Release' - def runType = 'rolling' - def benchViewName = 'Dotnet Size on Disk %DATE% %TIME%' - def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}" - def coreRoot = "${testBin}\\Tests\\Core_Root" - def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools" - def python = "C:\\python3.7.0\\python.exe" - - steps { - // Install nuget and get BenchView tools - batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"") - batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"") - batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion") - - // Generate submission metadata for BenchView - // Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView - // we have to do it all as one statement because cmd is called each time and we lose the set environment variable - batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" + - "set \"BENCHVIEWNAME=${benchViewName}\"\n" + - "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" + - "${python} \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" + - "${python} \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}") - - // Generate machine data from BenchView - batchFile("${python} \"${benchViewTools}\\machinedata.py\"") - - // Build CoreCLR and gnerate test layout - batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}") - batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly") - - // Run the size on disk benchmark - batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}") - - // From sodbench.csv, create measurment.json, then submission.json - batchFile("${python} \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"") - batchFile("${python} \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}") - - // If this is a PR, upload submission.json - batchFile("${python} \"${benchViewTools}\\upload.py\" submission.json --container coreclr") - } - } - - def archiveSettings = new ArchivalSettings() - archiveSettings.addFiles('bin/toArchive/**') - archiveSettings.addFiles('machinedata.json') - archiveSettings.setAlwaysArchive() - - Utilities.addArchival(newJob, archiveSettings) - Utilities.standardJobSetup(newJob, project, false, "*/${branch}") - - // Set the cron job here. We run nightly on each flavor, regardless of code changes - Utilities.addPeriodicTrigger(newJob, "@daily", true /*always run*/) - - newJob.with { - logRotator { - artifactDaysToKeep(14) - daysToKeep(30) - artifactNumToKeep(100) - numToKeep(200) - } - wrappers { - timeout { - absolute(240) - } - } - } - } -} - -// Setup IlLink tests -[true, false].each { isPR -> - ['Windows_NT'].each { os -> - ['x64'].each { arch -> - ['ryujit'].each { jit -> - ['full_opt'].each { opt_level -> - def architecture = arch - def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) { - label('Windows.10.Amd64.ClientRS4.DevEx.15.8.Open') - - def testEnv = "" - def python = "C:\\python3.7.0\\python.exe" - wrappers { - credentialsBinding { - string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') - } - } - - if (isPR) { - parameters { - stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName') - } - } - - parameters { - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '1', 'Size test, one iteration is sufficient') - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '1', 'Size test, one iteration is sufficient') - } - - def configuration = 'Release' - def runType = isPR ? 'private' : 'rolling' - def benchViewName = isPR ? 'CoreCLR-Scenarios private %BenchviewCommitName%' : 'CoreCLR-Scenarios rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%' - def uploadString = '-uploadToBenchview' - - steps { - // Batch - batchFile("powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \"%WORKSPACE%\\nuget.exe\"") - batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"") - batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion") - - //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView - //we have to do it all as one statement because cmd is called each time and we lose the set environment variable - batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" + - "set \"BENCHVIEWNAME=${benchViewName}\"\n" + - "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" + - "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" + - "${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}") - batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"") - batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}") - - batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly") - - def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -scenarioTest" - - // Scenario: ILLink - batchFile("\"%VS140COMNTOOLS%\\..\\..\\VC\\vcvarsall.bat\" x86_amd64 && " + - "${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup") - } - } - - def archiveSettings = new ArchivalSettings() - archiveSettings.addFiles('bin/sandbox_logs/**/*_log.txt') - archiveSettings.addFiles('bin/sandbox_logs/**/*.csv') - archiveSettings.addFiles('bin/sandbox_logs/**/*.xml') - archiveSettings.addFiles('bin/sandbox_logs/**/*.log') - archiveSettings.addFiles('bin/sandbox_logs/**/*.md') - archiveSettings.addFiles('bin/sandbox_logs/**/*.etl') - archiveSettings.addFiles('machinedata.json') - archiveSettings.setAlwaysArchive() - - // Set the label (currently we are only measuring size, therefore we are running on VM). - Utilities.addArchival(newJob, archiveSettings) - Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") - - newJob.with { - logRotator { - artifactDaysToKeep(14) - daysToKeep(30) - artifactNumToKeep(100) - numToKeep(200) - } - wrappers { - timeout { - absolute(240) - } - } - } - - if (isPR) { - TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest() - builder.setGithubContext("${os} ${arch} ${opt_level} ${jit} IlLink Tests") - builder.triggerOnlyOnComment() - builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+illink.*") - builder.triggerForBranch(branch) - builder.emitTrigger(newJob) - } - else { - // Set a push trigger - TriggerBuilder builder = TriggerBuilder.triggerOnCommit() - builder.emitTrigger(newJob) - } - } - } - } - } -} - -Utilities.createHelperJob(this, project, branch, - "Welcome to the ${project} Perf help", - "Have a nice day!") diff --git a/tests/CoreFX/CoreFX.issues.json b/tests/CoreFX/CoreFX.issues.json deleted file mode 100644 index cee5f488b9..0000000000 --- a/tests/CoreFX/CoreFX.issues.json +++ /dev/null @@ -1,1880 +0,0 @@ -[ - { - "name": "System.Diagnostics.TraceSource.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Diagnostics.TraceSourceTests.DefaultTraceListenerClassTests.EntryAssemblyName_Null_NotIncludedInTrace", - "reason": "outdated" - } - ] - } - }, - { - "name": "System.Globalization.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Globalization.Tests.CharUnicodeInfoGetUnicodeCategoryTests.GetUnicodeCategory", - "reason": "We updated the Unicode data and didn't pick yet the updated tests from corefx. we should remove this entry after updating the tests in coreclr." - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoNegativeInfinitySymbol.NegativeInfinitySymbol_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoPercentDecimalDigits.PercentDecimalDigits_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoCurrencyGroupSeparator.CurrencyGroupSeparator_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoNumberNegativePattern.NumberNegativePattern_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoPositiveInfinitySymbol.PositiveInfinitySymbol_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.StringInfoString.String_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoNumberDecimalSeparator.NumberDecimalSeparator_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoNumberGroupSeparator.NumberGroupSeparator_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoNaNSymbol.NaNSymbol_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoPositiveSign.PositiveSign_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoCurrencyGroupSizes.CurrencyGroupSizes_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoPercentSymbol.PercentSymbol_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.StringInfoCtorTests.Ctor_String_Null_ThrowsArgumentNullException", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoCurrencySymbol.CurrencySymbol_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoPercentDecimalSeparator.PercentDecimalSeparator_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.DateTimeFormatInfoGetAbbreviatedDayName.GetAbbreviatedDayName_Invalid_ThrowsArgumentOutOfRangeException", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoNumberDecimalDigits.NumberDecimalDigits_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoNegativeSign.NegativeSign_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoPercentGroupSizes.PercentGroupSizes_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoNumberGroupSizes.NumberGroupSizes_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoPercentPositivePattern.PercentPositivePattern_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoPerMilleSymbol.PerMilleSymbol_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoPercentGroupSeparator.PercentGroupSeparator_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoCurrencyNegativePattern.CurrencyNegativePattern_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoCurrencyPositivePattern.CurrencyPositivePattern_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoPercentNegativePattern.PercentNegativePattern_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoCurrencyDecimalDigits.CurrencyDecimalDigits_Set_Invalid", - "reason": "outdated" - }, - { - "name": "System.Globalization.Tests.NumberFormatInfoCurrencyDecimalSeparator.CurrencyDecimalSeparator_Set_Invalid", - "reason": "outdated" - } - ] - } - }, - { - "name": "System.Globalization.Calendars.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Globalization.Tests.GregorianCalendarCtor.CalendarType_Set_InvalidType_ThrowsArgumentOutOfRangeException", - "reason": "Fixed param name in .NET Core." - } - ] - } - }, - { - "name": "System.Data.SqlClient.ManualTesting.Tests", - "enabled": false, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": null - } - }, - { - "name": "System.Data.SqlClient.Stress.Tests", - "enabled": false, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": null - } - }, - { - "name": "System.Data.SqlClient.Tests", - "enabled": false, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": null - } - }, - { - "name": "System.ComponentModel.Composition.Tests", - "enabled": false, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": null - } - }, - { - "name": "System.Runtime.Serialization.Formatters.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Runtime.Serialization.Formatters.Tests.BinaryFormatterTests.ValidateAgainstBlobs", - "reason": "The stored blob for type System.CodeDom.MemberAttributes is outdated and needs to be updated." - }, - { - "name": "System.Runtime.Serialization.Formatters.Tests.BinaryFormatterTests.ValidateEqualityComparersAgainstBlobs", - "reason": "outdated" - }, - { - "name": "System.Runtime.Serialization.Formatters.Tests.BinaryFormatterTests.ValidateBasicObjectsRoundtrip", - "reason": "outdated" - }, - { - "name": "System.Runtime.Serialization.Formatters.Tests.BinaryFormatterTests.RoundtripManyObjectsInOneStream", - "reason": "outdated" - } - ] - } - }, - { - "name": "System.Collections.Immutable.Tests", - "enabled": false, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": null - } - }, - { - "name": "System.Reflection.Metadata.Tests", - "enabled": false, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": null - } - }, - { - "name": "System.Text.Encodings.Web.Tests", - "enabled": false, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": null - } - }, - { - "name": "System.IO.Pipelines.Tests", - "enabled": false, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": null - } - }, - { - "name": "System.Data.Common.Tests", - "enabled": false, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": null - } - }, - { - "name": "System.Diagnostics.StackTrace.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Diagnostics.Tests.StackTraceTests.ToString_NullFrame_ThrowsNullReferenceException", - "reason": "outdated" - } - ] - } - }, - { - "name": "System.Diagnostics.Tracing.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "BasicEventSourceTests.TestsUserErrors.Test_BadEventSource_MismatchedIds", - "reason": "Assert.Equal() Failure Expected: 1 Actual: 2" - }, - { - "name": "BasicEventSourceTests.TestsWrite.Test_Write_T_EventListener", - "reason": "Assert.Equal() Failure Expected: Actual: System.Runtime " - }, - { - "name": "BasicEventSourceTests.TestsWrite.Test_Write_T_EventListener_UseEvents", - "reason": "Assert.Equal() Failure Expected: Actual: System.Runtime " - }, - { - "name": "BasicEventSourceTests.TestsUserErrors.Test_BadEventSource_MismatchedIds_WithEtwListener", - "reason": "Assert.Equal() Failure Expected: Actual: System.Runtime " - }, - { - "name": "BasicEventSourceTests.FuzzyTests.Test_Write_Fuzzy", - "reason": "Assert.Equal() Failure Expected: Actual: System.Runtime " - }, - { - "name": "BasicEventSourceTests.TestsWriteEventToListener.Test_WriteEvent_InvalidCalls", - "reason": "Assert.Equal() Failure Expected: Actual: System.Runtime " - }, - { - "name": "BasicEventSourceTests.TestsWriteEventToListener.Test_WriteEvent_ZeroKwds", - "reason": "Assert.Equal() Failure Expected: Actual: System.Runtime " - }, - { - "name": "BasicEventSourceTests.TestsWriteEventToListener.Test_EventSourceCreatedEvents_BeforeListener", - "reason": "Assert.Equal() Failure Expected: Actual: System.Runtime " - }, - { - "name": "BasicEventSourceTests.TestsWriteEventToListener.Test_WriteEvent_ToChannel_Coverage", - "reason": "Assert.Equal() Failure Expected: Actual: System.Runtime " - }, - { - "name": "BasicEventSourceTests.TestsWriteEventToListener.Test_WriteEvent_ArgsCornerCases", - "reason": "Assert.Equal() Failure Expected: Actual: System.Runtime " - }, - { - "name": "BasicEventSourceTests.TestsWriteEventToListener.Test_EventSourceCreatedEvents_AfterListener", - "reason": "Assert.Equal() Failure Expected: Actual: System.Runtime " - }, - { - "name": "BasicEventSourceTests.TestsWriteEventToListener.Test_WriteEvent_ArgsBasicTypes", - "reason": "Assert.Equal() Failure Expected: Actual: System.Runtime " - }, - { - "name": "BasicEventSourceTests.TestsEventSourceLifetime.Test_EventSource_Lifetime", - "reason": "Assert.Equal() Failure Expected: Actual: System.Runtime " - }, - { - "name": "BasicEventSourceTests.TestsManifestNegative.Test_GenerateManifest_InvalidEventSources", - "reason": "Assert.Equal() Failure Expected: Actual: System.Runtime " - }, - { - "name": "BasicEventSourceTests.TestsTraits.Test_EventSource_Traits_Dynamic", - "reason": "Assert.Equal() Failure Expected: Actual: System.Runtime " - }, - { - "name": "BasicEventSourceTests.TestsTraits.Test_EventSource_Traits_Contract", - "reason": "Assert.Equal() Failure Expected: Actual: System.Runtime " - } - ] - } - }, - { - "name": "System.Diagnostics.DiagnosticSource.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Diagnostics.Tests.DiagnosticSourceEventSourceBridgeTests.TestShortcutKeywords", - "reason": "Assert.Equal() Failure ↓ (pos 0) Expected: Activity1Start Actual: Event ↑ (pos 0)" - }, - { - "name": "System.Diagnostics.Tests.ActivityTests.IdGenerationInternalParent", - "reason": "Assert.Equal() Failure Expected: úúúda-4ecabd4ccb066678.child1-1. Actual: úúúda-4ecabd4ccb066678.1." - } - ] - } - }, - { - "name": "System.Net.HttpListener.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Net.Tests.HttpRequestStreamTests.Read_NullBuffer_ThrowsArgumentNullException", - "reason": "Assert.Throws() Failure Expected: typeof(System.ArgumentNullException) Actual: typeof(System.NullReferenceException): Object reference not set to an instance of an object." - }, - { - "name": "System.Net.Tests.HttpResponseStreamTests.Write_NullBuffer_ThrowsArgumentNullException", - "reason": "Assert.Throws() Failure Expected: typeof(System.ArgumentNullException) Actual: typeof(System.NullReferenceException): Object reference not set to an instance of an object." - } - ] - } - }, - { - "name": "System.Net.Sockets.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Net.Sockets.Tests.KeepAliveTest.Socket_Set_KeepAlive_Interval_Failure", - "reason": "outdated" - } - ] - } - }, - { - "name": "System.Net.WebProxy.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name" : "System.Net.Tests.WebProxyTest.WebProxy_BypassOnLocal_SpecialCases", - "reason" : "outdated" - } - ] - } - }, - { - "name": "System.ComponentModel.TypeConverter.Tests", - "enabled": false, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": null - } - }, - { - "name": "System.Collections.NonGeneric.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Collections.Tests.HashtableBasicTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx update" - }, - { - "name": "System.Collections.Tests.HashtableValuesTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx update" - }, - { - "name": "System.Collections.Tests.Hashtable_SyncRootTests.SyncRoot", - "reason": "waiting on corefx update" - }, - { - "name": "System.Collections.Tests.HashtableSynchronizedTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx update" - }, - { - "name": "System.Collections.Tests.DictionaryBaseTests.SyncRoot", - "reason": "waiting on corefx update" - }, - { - "name": "System.Collections.Tests.HashtableKeysTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx update" - }, - { - "name": "System.Collections.Tests.ArrayList_SyncRootTests.GetSyncRoot ", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.ArrayListAdapterTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.ArrayListBasicTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.ArrayListFixedIListSizeTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.ArrayListFixedSizeTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.ArrayListRangeTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.ArrayListReadOnlyIListTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.ArrayListReadOnlyTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.ArrayListSynchronizedILstTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.ArrayListSynchronizedTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.CollectionBaseTests.SyncRoot ", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.Queue_SyncRootTests.SyncRoot ", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.ReadOnlyCollectionBaseTests.SyncRoot ", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.Stack_SyncRootTests.GetSyncRootBasic ", - "reason": "waiting on corefx test update" - }, - ] - } - }, - { - "name": "System.Collections.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_SimpleInt_int_With_Comparer_WrapStructural_SimpleInt.IEnumerable_Generic_Enumerator_Reset_ModifiedBeforeEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_SimpleInt_int_With_Comparer_WrapStructural_SimpleInt.IEnumerable_Generic_Enumerator_MoveNext_ModifiedDuringEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_SimpleInt_int_With_Comparer_WrapStructural_SimpleInt.IEnumerable_Generic_Enumerator_MoveNext_ModifiedBeforeEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_SimpleInt_int_With_Comparer_WrapStructural_SimpleInt.IEnumerable_Generic_Enumerator_Reset_ModifiedDuringEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_SimpleInt_int_With_Comparer_WrapStructural_SimpleInt.IEnumerable_Generic_Enumerator_Reset_ModifiedAfterEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_SimpleInt_int_With_Comparer_WrapStructural_SimpleInt.IEnumerable_Generic_Enumerator_MoveNext_ModifiedAfterEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_IDictionary_NonGeneric_Tests.IEnumerable_NonGeneric_Enumerator_MoveNext_ModifiedBeforeEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_IDictionary_NonGeneric_Tests.IEnumerable_NonGeneric_Enumerator_Reset_ModifiedBeforeEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_IDictionary_NonGeneric_Tests.IEnumerable_NonGeneric_Enumerator_Reset_ModifiedDuringEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_IDictionary_NonGeneric_Tests.IEnumerable_NonGeneric_Enumerator_MoveNext_ModifiedDuringEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_IDictionary_NonGeneric_Tests.IEnumerable_NonGeneric_Enumerator_Reset_ModifiedAfterEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_IDictionary_NonGeneric_Tests.IEnumerable_NonGeneric_Enumerator_MoveNext_ModifiedAfterEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_string_string.IEnumerable_Generic_Enumerator_Reset_ModifiedBeforeEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_string_string.IEnumerable_Generic_Enumerator_Reset_ModifiedAfterEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_string_string.IEnumerable_Generic_Enumerator_MoveNext_ModifiedAfterEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_string_string.IEnumerable_Generic_Enumerator_MoveNext_ModifiedBeforeEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_string_string.IEnumerable_Generic_Enumerator_Reset_ModifiedDuringEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_string_string.IEnumerable_Generic_Enumerator_MoveNext_ModifiedDuringEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_int_int.IEnumerable_Generic_Enumerator_Reset_ModifiedDuringEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_int_int.IEnumerable_Generic_Enumerator_MoveNext_ModifiedAfterEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_int_int.IEnumerable_Generic_Enumerator_MoveNext_ModifiedBeforeEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_int_int.IEnumerable_Generic_Enumerator_Reset_ModifiedBeforeEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_int_int.IEnumerable_Generic_Enumerator_Reset_ModifiedAfterEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_int_int.IEnumerable_Generic_Enumerator_MoveNext_ModifiedDuringEnumeration_ThrowsInvalidOperationException", - "reason": "outdated" - }, - { - "name": "System.Collections.Tests.Dictionary_IDictionary_NonGeneric_Tests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx update" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_Keys_AsICollection.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx update" - }, - { - "name": "System.Collections.Tests.Dictionary_Generic_Tests_Values_AsICollection.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx update" - }, - { - "name": "System.Collections.Tests.LinkedList_ICollection_NonGeneric_Tests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.Queue_ICollection_NonGeneric_Tests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.SortedDictionary_Generic_Tests_Keys_AsICollection.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.SortedDictionary_Generic_Tests_Values_AsICollection.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.SortedDictionary_IDictionary_NonGeneric_Tests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.SortedList_Generic_Tests_Keys_AsICollection.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.SortedList_Generic_Tests_Values_AsICollection.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.SortedList_IDictionary_NonGeneric_Tests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.SortedSet_ICollection_NonGeneric_Tests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Tests.Stack_ICollection_NonGeneric_Tests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - ] - } - }, - { - "name": "System.Collections.Specialized.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Collections.Specialized.Tests.StringDictionarySyncRootTests.SyncRoot", - "reason": "waiting on corefx update" - }, - { - "name": "System.Collections.Specialized.Tests.HybridDictionaryValuesTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx update" - }, - { - "name": "System.Collections.Specialized.Tests.HybridDictionaryKeysTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx update" - }, - { - "name": "System.Collections.Specialized.Tests.ListDictionary_CustomComparer_Tests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Specialized.Tests.ListDictionary_NoComparer_Tests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Specialized.Tests.ListDictionaryKeysTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Specialized.Tests.ListDictionaryValuesTests.ICollection_NonGeneric_SyncRoot", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Specialized.Tests.NameObjectCollectionBaseSyncRootTests.SyncRoot ", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Specialized.Tests.OrderedDictionaryTests.KeysPropertyContainsAllKeys ", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Specialized.Tests.OrderedDictionaryTests.SyncRootTests ", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Specialized.Tests.OrderedDictionaryTests.ValuesPropertyContainsAllValues ", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Collections.Specialized.Tests.StringCollectionTests.SyncRootTest", - "reason": "waiting on corefx test update" - }, - ] - } - }, - { - "name": "System.Net.Http.Functional.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Net.Http.Functional.Tests.PlatformHandler_DiagnosticsTest.SendAsync_ExpectedDiagnosticSourceActivityLogging", - "reason": "System.Diagnostics.RemoteExecutorTestBase+RemoteInvokeHandle+RemoteExecutionException System.Diagnostics.RemoteExecutorTestBase+RemoteInvokeHandle+RemoteExecutionException : Remote process failed with an unhandled exception." - }, - { - "name": "System.Net.Http.Functional.Tests.SocketsHttpHandler_DiagnosticsTest.SendAsync_ExpectedDiagnosticSourceActivityLogging", - "reason": "System.Diagnostics.RemoteExecutorTestBase+RemoteInvokeHandle+RemoteExecutionException System.Diagnostics.RemoteExecutorTestBase+RemoteInvokeHandle+RemoteExecutionException : Remote process failed with an unhandled exception." - }, - { - "name": "System.Net.Http.Functional.Tests.SocketsHttpHandler_HttpClientHandlerTest.GetAsync_AllowAutoRedirectTrue_RetainsOriginalFragmentIfAppropriate", - "reason": "outdated" - }, - { - "name": "System.Net.Http.Functional.Tests.SocketsHttpHandler_HttpClientHandler_Http2_Test.Http2_InitialSettings_SentAndAcked", - "reason": "outdated" - } - ] - } - }, - { - "name": "System.IO.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.IO.Tests.CloseTests.AfterDisposeThrows", - "reason": "Readers/writers changed to not null out base stream on dispose" - }, - { - "name": "System.IO.Tests.CloseTests.AfterCloseThrows", - "reason": "Readers/writers changed to not null out base stream on dispose" - }, - { - "name": "System.IO.Tests.StreamWriterWithBufferedStream_CloseTests.AfterDisposeThrows", - "reason": "Readers/writers changed to not null out base stream on dispose" - }, - { - "name": "System.IO.Tests.StreamWriterWithBufferedStream_CloseTests.AfterCloseThrows", - "reason": "Readers/writers changed to not null out base stream on dispose" - }, - { - "name": "System.IO.Tests.CtorTests.NullEncodingThrows", - "reason": "StreamReader/StreamWriter changed to accept null encoding" - } - ] - } - }, - { - "name": "System.IO.FileSystem.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.IO.Tests.DirectoryInfo_GetSetTimes.SettingUpdatesProperties", - "reason": "Assert.All() Failure: 6 out of 6 items in the collection did not pass." - }, - { - "name": "System.IO.Tests.Directory_GetSetTimes.SettingUpdatesProperties", - "reason": "Assert.All() Failure: 6 out of 6 items in the collection did not pass." - }, - { - "name": "System.IO.Tests.File_GetSetTimes.SettingUpdatesProperties", - "reason": "Assert.All() Failure: 6 out of 6 items in the collection did not pass." - }, - { - "name": "System.IO.Tests.FileInfo_GetSetTimes.SettingUpdatesProperties", - "reason": "Assert.All() Failure: 6 out of 6 items in the collection did not pass." - }, - { - "name": "System.IO.Tests.EncryptDecrypt.EncryptDecrypt_NotSupported", - "reason": "outdated" - }, - { - "name": "System.IO.Tests.EncryptDecrypt.NullArg_ThrowsException", - "reason": "outdated" - }, - { - "name": "System.IO.Tests.File_Delete.Unix_NonExistentPath_Nop", - "reason": "outdated (deleted from corefx)" - }, - { - "name": "System.IO.Tests.FileInfo_Delete.Unix_NonExistentPath_Nop", - "reason": "outdated (deleted from corefx)" - } - ] - } - }, - { - "name": "System.IO.Pipelines.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.IO.Pipelines.Tests.FlushAsyncCancellationTests.ReadAsyncCompletesIfFlushAsyncCanceledMidFlush", - "reason": "Unstable" - } - ] - } - }, - { - "name": "System.Drawing.Common.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Drawing.Tests.GraphicsTests.InterpolationMode_SetInvalid_ThrowsInvalidEnumArgumentException", - "reason": "Xunit.Sdk.ThrowsException Assert.Throws() Failure\\nExpected: typeof(System.ComponentModel.InvalidEnumArgumentException)\\nActual: typeof(System.ComponentModel.InvalidEnumArgumentException): The value of argument 'value' (-2) is invalid for Enum type 'InterpolationMode'.\\nParameter name: value" - }, - { - "name": "System.Drawing.Tests.GraphicsTests.CompositingQuality_SetInvalid_ThrowsInvalidEnumArgumentException", - "reason": "Xunit.Sdk.ThrowsException Assert.Throws() Failure\\nExpected: typeof(System.ComponentModel.InvalidEnumArgumentException)\\nActual: typeof(System.ComponentModel.InvalidEnumArgumentException): The value of argument 'value' (-2) is invalid for Enum type 'CompositingQuality'.\\nParameter name: value" - }, - { - "name": "System.Drawing.Tests.GraphicsTests.PixelOffsetMode_SetInvalid_ThrowsInvalidEnumArgumentException", - "reason": "Xunit.Sdk.ThrowsException Assert.Throws() Failure\\nExpected: typeof(System.ComponentModel.InvalidEnumArgumentException)\\nActual: typeof(System.ComponentModel.InvalidEnumArgumentException): The value of argument 'value' (-2) is invalid for Enum type 'PixelOffsetMode'.\\nParameter name: value" - }, - { - "name": "System.Drawing.Tests.GraphicsTests.PageUnit_SetInvalid_ThrowsInvalidEnumArgumentException", - "reason": "Xunit.Sdk.ThrowsException Assert.Throws() Failure\\nExpected: typeof(System.ComponentModel.InvalidEnumArgumentException)\\nActual: typeof(System.ComponentModel.InvalidEnumArgumentException): The value of argument 'value' (-1) is invalid for Enum type 'GraphicsUnit'.\\nParameter name: value" - }, - { - "name": "System.Drawing.Tests.GraphicsTests.SmoothingMode_SetInvalid_ThrowsInvalidEnumArgumentException", - "reason": "Xunit.Sdk.ThrowsException Assert.Throws() Failure\\nExpected: typeof(System.ComponentModel.InvalidEnumArgumentException)\\nActual: typeof(System.ComponentModel.InvalidEnumArgumentException): The value of argument 'value' (-2) is invalid for Enum type 'SmoothingMode'.\\nParameter name: value" - }, - { - "name": "System.Drawing.Tests.GraphicsTests.CompositingMode_SetInvalid_ThrowsInvalidEnumArgumentException", - "reason": "Xunit.Sdk.ThrowsException Assert.Throws() Failure\\nExpected: typeof(System.ComponentModel.InvalidEnumArgumentException)\\nActual: typeof(System.ComponentModel.InvalidEnumArgumentException): The value of argument 'value' (-1) is invalid for Enum type 'CompositingMode'.\\nParameter name: value" - }, - { - "name": "System.Drawing.Tests.GraphicsTests.TextRenderingHint_SetInvalid_ThrowsInvalidEnumArgumentException", - "reason": "Xunit.Sdk.ThrowsException Assert.Throws() Failure\\nExpected: typeof(System.ComponentModel.InvalidEnumArgumentException)\\nActual: typeof(System.ComponentModel.InvalidEnumArgumentException): The value of argument 'value' (-1) is invalid for Enum type 'TextRenderingHint'.\\nParameter name: value" - }, - { - "name": "System.Drawing.Tests.Graphics_DrawBezierTests.DrawBezier_PointFs", - "reason": "Xunit.Sdk.EqualException Assert.Equal() Failure\\r\\nExpected: Byte[] [208, 0, 8, 33, 6, ...]\\r\\nActual: Byte[] [153, 27, 131, 228, 197, ...]" - }, - { - "name": "System.Drawing.Tests.Graphics_DrawLineTests.DrawLines_PointFs", - "reason": "Xunit.Sdk.EqualException Assert.Equal() Failure\\r\\nExpected: Byte[] [142, 194, 251, 180, 222, ...]\\r\\nActual: Byte[] [212, 29, 140, 217, 143, ...]" - }, - { - "name": "System.Drawing.Tests.FontTests.ToLogFont_NullLogFont_ThrowsArgumentNullException", - "reason": "Xunit.Sdk.ThrowsException Assert.Throws() Failure\\r\\nExpected: typeof(System.ArgumentNullException)\\r\\nActual: typeof(System.AccessViolationException): Attempted to read or write protected memory. This is often an indication that other memory is corrupt." - }, - { - "name": "System.Drawing.Tests.Graphics_DrawBezierTests.DrawBezier_Point", - "reason": "https://github.com/dotnet/corefx/issues/30180" - }, - { - "name": "System.Drawing.Tests.Graphics_DrawBezierTests.DrawBezier_Point", - "reason": "https://github.com/dotnet/corefx/issues/30180" - }, - { - "name": "System.Drawing.Tests.IconTests.ExtractAssociatedIcon_InvalidFilePath_ThrowsArgumentException", - "reason": "https://github.com/dotnet/corefx/issues/34122" - }, - { - "name": "System.Drawing.Tests.FontTests.ToLogFont_DisposedGraphics_ThrowsArgumentException", - "reason": "outdated" - } - ] - } - }, - { - "name": "System.Text.Encoding.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Text.Tests.EncoderConvert2.EncoderASCIIConvertMixedASCIIUnicodeCharArrayPartial", - "reason": "https://github.com/dotnet/coreclr/issues/23864" - }, - { - "name": "System.Text.Tests.EncoderConvert2.EncoderUTF8ConvertMixedASCIIUnicodeCharArrayPartial", - "reason": "https://github.com/dotnet/coreclr/issues/23864" - }, - { - "name": "System.Text.Tests.EncoderConvert2.EncoderUTF8ConvertUnicodeCharArrayPartial", - "reason": "https://github.com/dotnet/coreclr/issues/23864" - }, - { - "name": "System.Text.Tests.NegativeEncodingTests.GetByteCount_Invalid", - "reason": "https://github.com/dotnet/coreclr/issues/23864" - }, - { - "name": "System.Text.Tests.UTF8EncodingDecode.Decode_InvalidBytes", - "reason": "https://github.com/dotnet/coreclr/issues/23864" - } - ] - } - }, - { - "name": "System.Text.RegularExpressions.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Text.RegularExpressions.Tests.RegexMatchTests.Match_ExcessPrefix", - "reason": "https://github.com/dotnet/coreclr/issues/18912" - } - ] - } - }, - { - "name": "System.Threading.Thread.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Threading.Threads.Tests.ThreadTests.ApartmentState_NoAttributePresent_STA_Windows_Core", - "reason": "https://github.com/dotnet/coreclr/issues/19478" - } - ] - } - }, - { - "name": "System.Threading.ThreadPool.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Threading.ThreadPools.Tests.ThreadPoolTests.SetMinMaxThreadsTest", - "reason": "https://github.com/dotnet/corefx/issues/36885" - } - ] - } - }, - { - "name": "System.Memory.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Buffers.Text.Tests.FormatterTests.TestFormatterDecimal", - "reason": "https://github.com/dotnet/coreclr/pull/19775" - }, - { - "name": "System.SpanTests.MemoryMarshalTests.CreateFromPinnedArrayIntSliceRemainsPinned", - "reason": "https://github.com/dotnet/corefx/pull/32994" - }, - { - "name": "System.SpanTests.MemoryMarshalTests.CreateFromPinnedArrayIntReadOnlyMemorySliceRemainsPinned", - "reason": "https://github.com/dotnet/corefx/pull/32994" - }, - { - "name": "System.Buffers.Text.Tests.ParserTests.TestParserSingle", - "reason": "https://github.com/dotnet/coreclr/pull/20707" - }, - { - "name": "System.Buffers.Text.Tests.ParserTests.TestParserDouble", - "reason": "https://github.com/dotnet/coreclr/pull/20707" - }, - { - "name": "System.Buffers.Text.Tests.FormatterTests.TestGFormatWithPrecisionNotSupported", - "reason": "https://github.com/dotnet/coreclr/pull/22434" - }, - { - "name": "System.Buffers.Text.Tests.FormatterTests.TestBadFormat", - "reason": "https://github.com/dotnet/coreclr/pull/22434" - }, - ] - } - }, - { - "name": "System.Runtime.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Tests.TimeSpanTests.FromMilliseconds", - "reason": "The test will be enabled after https://github.com/dotnet/corefx/pull/37235" - }, - { - "name": "System.Tests.TimeSpanTests.FromMilliseconds_Invalid", - "reason": "The test will be enabled after https://github.com/dotnet/corefx/pull/37235" - }, - { - "name": "System.Tests.TimeSpanTests.NamedMultiplication", - "reason": "The test will be enabled after https://github.com/dotnet/corefx/pull/37235" - }, - { - "name": "System.Tests.TimeSpanTests.Multiplication", - "reason": "The test will be enabled after https://github.com/dotnet/corefx/pull/37235" - }, - { - "name": "System.Tests.TimeSpanTests.Division", - "reason": "The test will be enabled after https://github.com/dotnet/corefx/pull/37235" - }, - { - "name": "System.Tests.TimeSpanTests.NamedDivision", - "reason": "The test will be enabled after https://github.com/dotnet/corefx/pull/37235" - }, - { - "name": "System.Text.Tests.StringBuilderTests.Equals", - "reason": "Because of a change in StringBuilder, this test is outdated." - }, - { - "name": "System.Tests.DecimalTests.Remainder", - "reason": "https://github.com/dotnet/coreclr/issues/12605" - }, - { - "name": "System.Tests.DecimalTests.Remainder_Invalid", - "reason": "https://github.com/dotnet/coreclr/issues/12605" - }, - { - "name": "System.Tests.DecimalTests+BigIntegerMod.Test", - "reason": "https://github.com/dotnet/coreclr/issues/12605" - }, - { - "name": "System.Tests.DecimalTests.Round_InvalidMidpointRounding_ThrowsArgumentException", - "reason": "outdated" - }, - { - "name": "System.Tests.ArrayTests.Copy", - "reason": "Needs updates for XUnit 2.4" - }, - { - "name": "System.Tests.ArraySegment_Tests.CopyTo_Invalid", - "reason": "Needs parameter name updated to 'destination'." - }, - { - "name": "System.Tests.TimeSpanTests.Parse_Invalid", - "reason": "Temporary disabling till merging the PR https://github.com/dotnet/corefx/pull/34561" - }, - { - "name": "System.Tests.TimeSpanTests.Parse_Span", - "reason": "Temporary disabling till merging the PR https://github.com/dotnet/corefx/pull/34561" - }, - { - "name": "System.Tests.TimeSpanTests.Parse_Span_Invalid", - "reason": "Temporary disabling till merging the PR https://github.com/dotnet/corefx/pull/34561" - }, - { - "name": "System.Tests.TimeSpanTests.Parse", - "reason": "Temporary disabling till merging the PR https://github.com/dotnet/corefx/pull/34561" - }, - { - "name": "System.Tests.SingleTests.Test_ToString", - "reason" : "https://github.com/dotnet/coreclr/pull/22040" - }, - { - "name": "System.Tests.SingleTests.TryFormat", - "reason" : "https://github.com/dotnet/coreclr/pull/22040" - }, - { - "name": "System.Tests.DoubleTests.Test_ToString", - "reason" : "https://github.com/dotnet/coreclr/pull/22040" - }, - { - "name": "System.Tests.DoubleTests.TryFormat", - "reason" : "https://github.com/dotnet/coreclr/pull/22040" - }, - { - "name": "System.Tests.TimeSpanTests.Total_Days_Hours_Minutes_Seconds_Milliseconds", - "reason" : "https://github.com/dotnet/coreclr/pull/22040" - }, - { - "name": "System.Tests.BufferTests.BlockCopy_Invalid", - "reason": "https://github.com/dotnet/coreclr/pull/23636" - }, - { - "name": "System.Tests.VersionTests.Comparisons_NullArgument_ThrowsArgumentNullException", - "reason": "Version was improved to no longer throw from comparison operators" - }, - ] - } - }, - { - "name": "System.Runtime.Extensions.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Tests.ConvertToStringTests.FromBoxedObject", - "reason": "https://github.com/dotnet/coreclr/pull/19775" - }, - { - "name": "System.Tests.ConvertToDoubleTests.FromString", - "reason": "https://github.com/dotnet/coreclr/pull/20707" - }, - { - "name": "System.Tests.ConvertToSingleTests.FromString", - "reason": "https://github.com/dotnet/coreclr/pull/20707" - }, - { - "name": "System.Tests.MathTests.Max_Double", - "reason": "https://github.com/dotnet/coreclr/pull/20912" - }, - { - "name": "System.Tests.MathTests.Max_Single", - "reason": "https://github.com/dotnet/coreclr/pull/20912" - }, - { - "name": "System.Tests.MathTests.Min_Double", - "reason": "https://github.com/dotnet/coreclr/pull/20912" - }, - { - "name": "System.Tests.MathTests.Min_Single", - "reason": "https://github.com/dotnet/coreclr/pull/20912" - }, - { - "name": "System.Tests.MathFTests.Max", - "reason": "https://github.com/dotnet/coreclr/pull/20912" - }, - { - "name": "System.Tests.MathFTests.Min", - "reason": "https://github.com/dotnet/coreclr/pull/20912" - }, - { - "name": "System.Tests.EnvironmentTests.Version_MatchesFixedVersion", - "reason": "outdated" - }, - { - "name": "System.Tests.AppDomainTests.ApplyPolicy", - "reason": "outdated" - }, - { - "name": "System.Tests.AppDomainTests.GetAssemblies", - "reason": "outdated" - } - ] - } - }, - { - "name": "System.Runtime.InteropServices.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Runtime.InteropServices.HandleCollectorTests.Ctor_InitialThresholdGreaterThanMaximumThreshold_ThrowsArgumentException ", - "reason": "outdated" - }, - { - "name": "System.Runtime.InteropServices.Tests.GCHandleTests.Alloc_InvalidPinnedObject_ThrowsArgumentException", - "reason": "outdated" - }, - { - "name": "System.Runtime.InteropServices.Tests.GetHINSTANCETests.GetHINSTANCE_NonRuntimeModule_ThrowsArgumentNullException", - "reason": "https://github.com/dotnet/coreclr/pull/30925" - }, - { - "name" : "System.Runtime.InteropServices.Tests.SafeBufferTests.Initialize_NumBytesTimesSizeOfEachElement_ThrowsOverflowException", - "reason" : "https://github.com/dotnet/coreclr/pull/20132" - }, - { - "name" : "System.Runtime.InteropServices.Tests.GetFunctionPointerForDelegateTests.GetFunctionPointerForDelegate_DelegateCollectible_ThrowsNotSupportedException", - "reason" : "outdated" - }, - { - "name" : "System.Runtime.InteropServices.Tests.GetComInterfaceForObjectTests.GetComInterfaceForObject_ObjectNotCollectible_ThrowsNotSupportedException", - "reason" : "outdated" - }, - { - "name" : "System.Runtime.InteropServices.Tests.ChangeWrapperHandleStrengthTests.ChangeWrapperHandleStrength_ObjectNotCollectible_ThrowsNotSupportedException", - "reason" : "outdated" - }, - { - "name" : "System.Runtime.InteropServices.Tests.CreateAggregatedObjectTests.CreateAggregatedObject_ObjectNotCollectible_ReturnsExpected", - "reason" : "outdated" - }, - { - "name" : "System.Runtime.InteropServices.Tests.GetNativeVariantForObjectTests.GetNativeVariantForObject_ObjectNotCollectible_ThrowsNotSupportedException", - "reason" : "outdated" - }, - { - "name" : "System.Runtime.InteropServices.Tests.GetIUnknownForObjectTests.GetIUnknownForObject_ObjectNotCollectible_ThrowsNotSupportedException", - "reason" : "outdated" - }, - { - "name": "System.Runtime.InteropServices.Tests.DispatchWrapperTests.Ctor_NullUnix_ThrowsPlatformNotSupportedException", - "reason": "outdated" - }, - { - "name": "System.Runtime.InteropServices.Tests.ArrayWithOffsetTests.Ctor_Array_Offset", - "reason": "outdated" - }, - { - "name": "System.Runtime.InteropServices.Tests.StringToCoTaskMemAutoTests.StringToCoTaskMemAuto_NonNullString_Roundtrips", - "reason": "outdated" - }, - { - "name": "System.Runtime.InteropServices.Tests.StringToHGlobalAutoTests.StringToHGlobalAuto_NonNullString_Roundtrips", - "reason": "https://github.com/dotnet/coreclr/pull/23664" - }, - { - "name": "System.Runtime.InteropServices.Tests.PtrToStringUTF8.PtrToStringUTF8_NegativeLength_ThrowsArgumentOutOfRangeExeption", - "reason": "https://github.com/dotnet/coreclr/pull/23731" - }, - { - "name": "System.Runtime.InteropServices.Tests.PtrToStringUTF8.PtrToStringUTF8_ZeroPointer_ReturnsNull", - "reason": "https://github.com/dotnet/coreclr/pull/23731" - }, - { - "name": "System.Runtime.InteropServices.Tests.PtrToStringUTF8.PtrToStringUTF8_Win32AtomPointer_ReturnsNull", - "reason": "https://github.com/dotnet/coreclr/pull/23731" - }, - { - "name": "System.Runtime.InteropServices.Tests.PtrToStringAnsiTests.PtrToStringAnsi_NegativeLength_ThrowsArgumentExeption", - "reason": "https://github.com/dotnet/coreclr/pull/23731" - }, - { - "name": "System.Runtime.InteropServices.Tests.PtrToStringUniTests.PtrToStringUni_NegativeLength_ThrowsArgumentExeption", - "reason": "https://github.com/dotnet/coreclr/pull/23731" - }, - { - "name": "System.Runtime.InteropServices.Tests.ThrowExceptionForHRTests.ThrowExceptionForHR_NoErrorInfo_ReturnsValidException", - "reason": "outdated" - }, - { - "name": "System.Runtime.InteropServices.Tests.ThrowExceptionForHRTests.ThrowExceptionForHR_ErrorInfo_ReturnsValidException", - "reason": "outdated" - } - ] - } - }, - { - "name": "System.Runtime.Loader.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name" : "System.Runtime.Loader.Tests.AssemblyLoadContextTest.Unsupported_DelegateMarshalling", - "reason" : "outdated" - }, - { - "name" : "System.Runtime.Loader.Tests.AssemblyLoadContextTest.Unsupported_ThreadStaticAndFixedAddressValueType", - "reason" : "outdated" - } - ] - } - }, - { - "name": "System.Security.Cryptography.Pkcs.Tests", - "enabled": false, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": null - } - }, - { - "name": "Microsoft.VisualBasic.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToChar_InvalidObject_ThrowsInvalidCastException", - "reason" : "outdated" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToSingle_InvalidObject_ThrowsInvalidCastException", - "reason" : "outdated" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToBoolean_InvalidObject_ThrowsInvalidCastException", - "reason" : "outdated" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToSByte_InvalidObject_ThrowsInvalidCastException", - "reason" : "outdated" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToDouble_InvalidObject_ThrowsInvalidCastException", - "reason" : "outdated" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToDecimal_InvalidObject_ThrowsInvalidCastException", - "reason" : "outdated" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToByte_InvalidObject_ThrowsInvalidCastException", - "reason" : "outdated" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToLong_InvalidObject_ThrowsInvalidCastException", - "reason" : "outdated" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToUShort_InvalidObject_ThrowsInvalidCastException", - "reason" : "outdated" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToUInteger_InvalidObject_ThrowsInvalidCastException", - "reason" : "outdated" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToInteger_InvalidObject_ThrowsInvalidCastException", - "reason" : "outdated" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToShort_InvalidObject_ThrowsInvalidCastException", - "reason" : "outdated" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToULong_InvalidObject_ThrowsInvalidCastException", - "reason" : "outdated" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToSingle_String_ReturnsExpected", - "reason" : "https://github.com/dotnet/coreclr/pull/22040" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToDouble_String_ReturnsExpected", - "reason" : "https://github.com/dotnet/coreclr/pull/22040" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToSingle_Object_ReturnsExpected", - "reason" : "https://github.com/dotnet/coreclr/pull/22040" - }, - { - "name" : "Microsoft.VisualBasic.Tests.ConversionsTests.ToDouble_Object_ReturnsExpected", - "reason" : "https://github.com/dotnet/coreclr/pull/22040" - }, - ] - } - }, - { - "name": "System.Numerics.Vectors.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name" : "System.Numerics.Tests.Matrix4x4Tests.Matrix4x4LerpTest", - "reason" : "https://github.com/dotnet/coreclr/pull/20994" - }, - { - "name" : "System.Numerics.Tests.Matrix4x4Tests.Matrix4x4MultiplyTest5", - "reason" : "https://github.com/dotnet/coreclr/pull/20994" - }, - { - "name" : "System.Numerics.Tests.Matrix4x4Tests.Matrix4x4MultiplyTest6", - "reason" : "https://github.com/dotnet/coreclr/pull/20994" - }, - { - "name" : "System.Numerics.Tests.Matrix4x4Tests.Matrix4x4NegateTest", - "reason" : "https://github.com/dotnet/coreclr/pull/20994" - }, - { - "name" : "System.Numerics.Tests.Matrix4x4Tests.Matrix4x4UnaryNegationTest", - "reason" : "https://github.com/dotnet/coreclr/pull/20994" - }, - { - "name" : "System.Numerics.Tests.Vector2Tests.Vector2ClampTest", - "reason" : "outdated" - }, - { - "name" : "System.Numerics.Tests.Vector3Tests.Vector3ClampTest", - "reason" : "outdated" - }, - { - "name" : "System.Numerics.Tests.Vector4Tests.Vector4ClampTest", - "reason" : "outdated" - } - ] - } - }, - { - "name": "System.Diagnostics.Debug.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": [ - { - "name" : "System.Diagnostics.Tests.DebugTests", - "reason" : "refactoring Debug" - } - ], - "methods": null - } - }, - { - "name": "System.Xml.Linq.SDMSample.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "XDocumentTests.SDMSample.SDM_Attribute.AttributeExplicitToDouble", - "reason": "https://github.com/dotnet/coreclr/pull/20707" - }, - { - "name": "XDocumentTests.SDMSample.SDM_Attribute.AttributeExplicitToFloat", - "reason": "https://github.com/dotnet/coreclr/pull/20707" - }, - { - "name": "XDocumentTests.SDMSample.SDM_Element.ElementExplicitToFloat", - "reason": "https://github.com/dotnet/coreclr/pull/20707" - }, - { - "name": "XDocumentTests.SDMSample.SDM_Element.ElementExplicitToDouble", - "reason": "https://github.com/dotnet/coreclr/pull/20707" - }, - ] - } - }, - { - "name": "System.Json.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Json.Tests.JsonValueTests.Parse_DoubleTooLarge_ThrowsOverflowException", - "reason": "https://github.com/dotnet/coreclr/pull/20707" - }, - { - "name": "System.Json.Tests.JsonPrimitiveTests.ToString_DateTimeOffset", - "reason": "waiting on corefx test update" - }, - { - "name": "System.Json.Tests.JsonValueTests.Parse_IntegralBoundaries_LessThanMaxDouble_Works", - "reason": "https://github.com/dotnet/coreclr/pull/22040" - }, - ] - } - }, - { - "name": "System.Xml.RW.XmlWriterApi.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Xml.Tests.TCFullEndElement+TCWriteValue.writeValue_27", - "reason": "https://github.com/dotnet/coreclr/pull/20707" - }, - ] - } - }, - { - "name": "System.ComponentModel.Annotations.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.ComponentModel.DataAnnotations.Tests.RangeAttributeTests.Validate_DoubleConversionOverflows_ThrowsOverflowException", - "reason": "https://github.com/dotnet/coreclr/pull/20707" - }, - { - "name": "System.ComponentModel.DataAnnotations.Tests.AssociationAttributeTests.Constructor", - "reason": "https://github.com/dotnet/coreclr/pull/22595" - }, - ] - } - }, - { - "name": "System.Linq.Expressions.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Linq.Expressions.Tests.BindTests.StaticReadonlyField", - "reason": "https://github.com/dotnet/coreclr/pull/20886 and https://github.com/dotnet/corefx/pull/33413" - }, - { - "name": "System.Linq.Expressions.Tests.StackSpillerTests.Spill_Optimizations_LiteralField", - "reason": "https://github.com/dotnet/coreclr/pull/22040" - }, - ] - } - }, - { - "name": "System.Xml.XmlSerializer.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "XmlSerializerTests.Xml_FloatAsRoot", - "reason": "https://github.com/dotnet/coreclr/pull/22040" - }, - ] - } - }, - { - "name": "System.Xml.XmlSerializer.ReflectionOnly.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "XmlSerializerTests.Xml_FloatAsRoot", - "reason": "https://github.com/dotnet/coreclr/pull/22040" - }, - ] - } - }, - { - "name": "System.Runtime.Serialization.Xml.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "DataContractSerializerTests.DCS_FloatAsRoot", - "reason": "https://github.com/dotnet/coreclr/pull/22040" - }, - { - "name": "DataContractSerializerTests.DCS_BasicPerSerializerRoundTripAndCompare_EnumStruct", - "reason": "https://github.com/dotnet/coreclr/pull/22040" - }, - { - "name": "DataContractSerializerTests.DCS_BasicRoundTripResolvePrimitiveTypes", - "reason": "https://github.com/dotnet/coreclr/pull/22040" - }, - ] - } - }, - { - "name": "System.Runtime.Serialization.Xml.ReflectionOnly.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "DataContractSerializerTests.DCS_FloatAsRoot", - "reason": "https://github.com/dotnet/coreclr/pull/22040" - }, - { - "name": "DataContractSerializerTests.DCS_BasicPerSerializerRoundTripAndCompare_EnumStruct", - "reason": "https://github.com/dotnet/coreclr/pull/22040" - }, - { - "name": "DataContractSerializerTests.DCS_BasicRoundTripResolvePrimitiveTypes", - "reason": "https://github.com/dotnet/coreclr/pull/22040" - }, - ] - } - }, - { - "name": "System.Runtime.Serialization.Json.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "DataContractJsonSerializerTests.DCJS_FloatAsRoot", - "reason": "https://github.com/dotnet/coreclr/pull/22040" - }, - ] - } - }, - { - "name": "System.Runtime.Serialization.Json.ReflectionOnly.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "DataContractJsonSerializerTests.DCJS_FloatAsRoot", - "reason": "https://github.com/dotnet/coreclr/pull/22040" - }, - ] - } - }, - { - "name": "System.ObjectModel.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTest.StringComparer", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntInt.AddExistingKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntInt.ChangeItemDoesNotExistThrows", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntInt.ChangeItemKeyKeyAlreadyChanged", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntInt.ChangeItemKeyKeyAlreadyChangedNewKeyIsDifferent", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntInt.ChangeItemKeyKeyAlreadyChangedNewKeyIsOldKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntInt.ChangeItemKeyThrowsPreexistingKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntInt.Insert", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntInt.NonGenericAddExistingKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntString.AddExistingKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntString.ChangeItemDoesNotExistThrows", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntString.ChangeItemKeyKeyAlreadyChanged", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntString.ChangeItemKeyKeyAlreadyChangedNewKeyIsDifferent", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntString.ChangeItemKeyKeyAlreadyChangedNewKeyIsOldKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntString.ChangeItemKeyThrowsPreexistingKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntString.Insert", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsIntString.NonGenericAddExistingKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringInt.AddExistingKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringInt.ChangeItemDoesNotExistThrows", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringInt.ChangeItemKeyKeyAlreadyChanged", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringInt.ChangeItemKeyKeyAlreadyChangedNewKeyIsDifferent", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringInt.ChangeItemKeyKeyAlreadyChangedNewKeyIsOldKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringInt.ChangeItemKeyNullItemNotPresent", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringInt.ChangeItemKeyNullToNewKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringInt.ChangeItemKeyNullToOldKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringInt.ChangeItemKeyNullToOtherKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringInt.ChangeItemKeyThrowsPreexistingKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringInt.Insert", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringInt.NonGenericAddExistingKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringString.AddExistingKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringString.ChangeItemDoesNotExistThrows", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringString.ChangeItemKeyKeyAlreadyChanged", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringString.ChangeItemKeyKeyAlreadyChangedNewKeyIsDifferent", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringString.ChangeItemKeyKeyAlreadyChangedNewKeyIsOldKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringString.ChangeItemKeyNullItemNotPresent", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringString.ChangeItemKeyNullToNewKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringString.ChangeItemKeyNullToOldKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringString.ChangeItemKeyNullToOtherKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringString.ChangeItemKeyThrowsPreexistingKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringString.Insert", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - { - "name": "System.Collections.ObjectModel.Tests.KeyedCollectionTestsStringString.NonGenericAddExistingKey", - "reason": "https://github.com/dotnet/coreclr/pull/22455" - }, - ] - } - }, - { - "name": "System.Net.Ping.Functional.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Net.NetworkInformation.Tests.PingTest.SendPingAsyncWithIPAddress", - "reason": "https://github.com/dotnet/corefx/issues/35351" - } - ] - } - }, - { - "name": "System.CodeDom.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.CodeDom.Compiler.Tests.CSharpCodeGenerationTests.ProviderSupports", - "reason": "outdated" - }, - { - "name": "System.CodeDom.Compiler.Tests.CSharpCodeGenerationTests.MetadataAttributes", - "reason": "outdated" - }, - { - "name": "System.CodeDom.Compiler.Tests.CSharpCodeGenerationTests.RegionsSnippetsAndLinePragmas", - "reason": "outdated" - }, - { - "name": "System.CodeDom.Compiler.Tests.VBCodeGenerationTests.RegionsSnippetsAndLinePragmas", - "reason": "outdated" - }, - { - "name": "System.CodeDom.Compiler.Tests.VBCodeGenerationTests.MetadataAttributes", - "reason": "outdated" - }, - { - "name": "System.CodeDom.Compiler.Tests.VBCodeGenerationTests.ProviderSupports", - "reason": "outdated" - } - ] - } - }, - { - "name": "System.Runtime.InteropServices.RuntimeInformation.Tests", - "enabled": true, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": [ - { - "name": "System.Runtime.InteropServices.RuntimeInformationTests.DescriptionNameTests.VerifyRuntimeDebugNameOnNetCoreApp", - "reason": "outdated" - } - ] - } - }, - { - "name": "System.IO.Compression.Brotli.Tests", - "enabled": false, - "exclusions": { - "namespaces": null, - "classes": null, - "methods": null - } - } -] diff --git a/tests/CoreFX/CoreFXTestListURL.txt b/tests/CoreFX/CoreFXTestListURL.txt deleted file mode 100644 index 926cf3c8f3..0000000000 --- a/tests/CoreFX/CoreFXTestListURL.txt +++ /dev/null @@ -1 +0,0 @@ -https://cloudcijobs.blob.core.windows.net/coreclrci/CoreFXArchives/TestList.json
\ No newline at end of file diff --git a/tests/CoreFX/CoreFXTestListURL_Linux.txt b/tests/CoreFX/CoreFXTestListURL_Linux.txt deleted file mode 100644 index 29b8c2615b..0000000000 --- a/tests/CoreFX/CoreFXTestListURL_Linux.txt +++ /dev/null @@ -1 +0,0 @@ -https://cloudcijobs.blob.core.windows.net/coreclrci/CoreFXArchives_Linux/TestList.json
\ No newline at end of file diff --git a/tests/CoreFX/CoreFXTestListURL_OSX.txt b/tests/CoreFX/CoreFXTestListURL_OSX.txt deleted file mode 100644 index bb4b850f68..0000000000 --- a/tests/CoreFX/CoreFXTestListURL_OSX.txt +++ /dev/null @@ -1 +0,0 @@ -https://cloudcijobs.blob.core.windows.net/coreclrci/CoreFXArchives_OSX/TestList.json
\ No newline at end of file diff --git a/tests/src/Common/CoreFX/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj b/tests/src/Common/CoreFX/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj deleted file mode 100644 index 751b3ab8d3..0000000000 --- a/tests/src/Common/CoreFX/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj +++ /dev/null @@ -1,36 +0,0 @@ -<Project Sdk="Microsoft.NET.Sdk"> - <Import Project="..\..\..\..\..\dependencies.props" /> - <PropertyGroup> - <GenerateAssemblyInfo>false</GenerateAssemblyInfo> - <SystemCommandLineVersion>0.1.0-e160909-1</SystemCommandLineVersion> - <NewtonsoftJsonVersion>11.0.2</NewtonsoftJsonVersion> - <NewtonsoftJsonSchemaVersion>3.0.10</NewtonsoftJsonSchemaVersion> - <XunitPackageVersion>2.3.0-beta1-build3642</XunitPackageVersion> - <!-- Due to an API surface mismatch, if the xunit.netcore executable attempts to run the wrong version of the - runner utility, running the tests wil break, so separate both into different version definitions --> - <XunitRunnerUtilityVersion>2.2.0-beta2-build3300</XunitRunnerUtilityVersion> - <XunitAbstractionsVersion>2.0.1</XunitAbstractionsVersion> - <XunitNetcoreExtensionsVersion>2.1.0-preview2-02516-02</XunitNetcoreExtensionsVersion> - <CoreFxTestUtilitiesVersion>4.5.0-preview2-26219-0</CoreFxTestUtilitiesVersion> - <RestoreSources>$(RestoreSources);https://dotnet.myget.org/F/dotnet-corefxlab/api/v3/index.json</RestoreSources> - </PropertyGroup> - - <PropertyGroup> - <TargetFramework>netcoreapp2.0</TargetFramework> - <OutputType>Exe</OutputType> - <NETCoreAppMaximumVersion>99.9</NETCoreAppMaximumVersion> - </PropertyGroup> - - <ItemGroup> - <PackageReference Include="Newtonsoft.Json"> - <Version>$(NewtonsoftJsonVersion)</Version> - </PackageReference> - <PackageReference Include="Newtonsoft.Json.Schema"> - <Version>$(NewtonsoftJsonSchemaVersion)</Version> - </PackageReference> - <!-- TODO Remove this reference - System.CommandLine is now archived and not under active development --> - <PackageReference Include="System.CommandLine"> - <Version>$(SystemCommandLineVersion)</Version> - </PackageReference> - </ItemGroup> -</Project> diff --git a/tests/src/Common/CoreFX/TestFileSetup/Helpers/TestFileHelper.cs b/tests/src/Common/CoreFX/TestFileSetup/Helpers/TestFileHelper.cs deleted file mode 100644 index e6e2c4ed6c..0000000000 --- a/tests/src/Common/CoreFX/TestFileSetup/Helpers/TestFileHelper.cs +++ /dev/null @@ -1,326 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.CommandLine; -using System.Diagnostics; -using System.IO; -using System.IO.Compression; -using System.Net.Http; -using System.Text; -using System.Threading.Tasks; -using Newtonsoft.Json; -using Newtonsoft.Json.Schema; -using Newtonsoft.Json.Schema.Generation; - -namespace CoreFX.TestUtils.TestFileSetup.Helpers -{ - /// <summary> - /// Defines the set of flags that represent exit codes - /// </summary> - [Flags] - public enum ExitCode : int - { - Success = 0, - TestFailure = 1, - HttpError = 2, - IOError = 3, - JsonSchemaValidationError = 4, - UnknownError = 10 - - } - - /// <summary> - /// This helper class is used to fetch CoreFX tests from a specified URL, unarchive them and create a flat directory structure - /// through which to iterate. - /// </summary> - public class TestFileHelper - { - private HttpClient httpClient; - public HttpClient HttpClient - { - get - { - if (httpClient == null) - { - httpClient = new HttpClient(); - } - return httpClient; - } - set{ httpClient = value; } - } - - private HashSet<string> disabledTests; - - /// <summary> - /// Default constructor - initialize list of disabled tests - /// </summary> - public TestFileHelper() { - disabledTests = new HashSet<string>(); - } - - /// <summary> - /// Deserialize a list of JSON objects defining test assemblies - /// </summary> - /// <param name="testDefinitionFilePath">The path on disk to the test list. The test list must conform to a schema generated from XUnitTestAssembly</param> - /// <returns></returns> - public Dictionary<string, XUnitTestAssembly> DeserializeTestJson(string testDefinitionFilePath) - { - JSchemaGenerator jsonGenerator = new JSchemaGenerator(); - - // Generate a JSON schema from the XUnitTestAssembly class against which to validate the test list - JSchema testDefinitionSchema = jsonGenerator.Generate(typeof(IList<XUnitTestAssembly>)); - IList<XUnitTestAssembly> testAssemblies = new List<XUnitTestAssembly>(); - - IList<string> validationMessages = new List<string>(); - - using (var sr = new StreamReader(testDefinitionFilePath)) - using (var jsonReader = new JsonTextReader(sr)) - using (var jsonValidationReader = new JSchemaValidatingReader(jsonReader)) - { - // Create schema validator - jsonValidationReader.Schema = testDefinitionSchema; - jsonValidationReader.ValidationEventHandler += (o, a) => validationMessages.Add(a.Message); - - // Deserialize json test assembly definitions - JsonSerializer serializer = new JsonSerializer(); - try - { - testAssemblies = serializer.Deserialize<List<XUnitTestAssembly>>(jsonValidationReader); - } - catch (JsonSerializationException ex) - { - // Invalid definition - throw new AggregateException(ex); - } - } - - if (validationMessages.Count != 0) - { - StringBuilder aggregateExceptionMessage = new StringBuilder(); - foreach (string validationMessage in validationMessages) - { - aggregateExceptionMessage.Append("JSON Validation Error: "); - aggregateExceptionMessage.Append(validationMessage); - aggregateExceptionMessage.AppendLine(); - } - - throw new AggregateException(new JSchemaValidationException(aggregateExceptionMessage.ToString())); - - } - // Generate a map of test assembly names to their object representations - this is used to download and match them to their on-disk representations - var nameToTestAssemblyDef = new Dictionary<string, XUnitTestAssembly>(); - - // Map test names to their definitions - foreach (XUnitTestAssembly assembly in testAssemblies) - { - // Filter disabled tests - if(assembly.IsEnabled) - nameToTestAssemblyDef.Add(assembly.Name, assembly); - else - disabledTests.Add(assembly.Name); - } - - return nameToTestAssemblyDef; - } - - /// <summary> - /// Layout tests on disk. This method sets up every downloaded test as it would appear after running build-test.[cmd/sh] in CoreFX - /// </summary> - /// <param name="jsonUrl">URL to a test list - we expect a test list, which conforms to the Helix layout</param> - /// <param name="destinationDirectory">Directory to which the tests are downloaded</param> - /// <param name="testDefinitions">The mapping of tests parsed from a test definition list to their names</param> - /// <param name="runAllTests">Optional argument, which denotes whether all tests available in the test list downloaded from jsonUrl should be run</param> - /// <returns></returns> - public async Task SetupTests(string jsonUrl, string destinationDirectory, Dictionary<string, XUnitTestAssembly> testDefinitions = null, bool runAllTests = false) - { - Debug.Assert(Directory.Exists(destinationDirectory)); - // testDefinitions should not be empty unless we're running all tests with no exclusions - Debug.Assert(runAllTests || testDefinitions != null); - - // Download archives to a temporary directory - string tempDirPath = Path.Combine(destinationDirectory, "temp"); - if (!Directory.Exists(tempDirPath)) - { - Directory.CreateDirectory(tempDirPath); - } - // Map test names to their URLs, specified by the test list found at jsonUrl - Dictionary<string, XUnitTestAssembly> testPayloads = await GetTestUrls(jsonUrl, testDefinitions, runAllTests); - - // If none were found or the testList did not have the expected format - return - if (testPayloads == null) - { - return; - } - - // Download and unzip tests - await GetTestArchives(testPayloads, tempDirPath); - ExpandArchivesInDirectory(tempDirPath, destinationDirectory); - - // Generate response file for each tests - RSPGenerator rspGenerator = new RSPGenerator(); - foreach (XUnitTestAssembly assembly in testDefinitions.Values) - { - rspGenerator.GenerateRSPFile(assembly, Path.Combine(destinationDirectory, assembly.Name)); - } - - Directory.Delete(tempDirPath); - } - - /// <summary> - /// Maps test names to their respective URLs as found in the test list found at the specified URL - /// </summary> - /// <param name="jsonUrl">URL to a test list - we expect a test list, which conforms to the Helix layout</param> - /// <param name="testDefinitions">The mapping of tests parsed from a test definition list to their names</param> - /// <param name="runAllTests">Optional argument, which denotes whether all tests available in the test list downloaded from jsonUrl should be run</param> - /// <returns></returns> - public async Task<Dictionary<string, XUnitTestAssembly>> GetTestUrls(string jsonUrl, Dictionary<string, XUnitTestAssembly> testDefinitions = null, bool runAllTests = false) - { - // testDefinitions should not be empty unless we're running all tests with no exclusions - Debug.Assert(runAllTests || testDefinitions != null); - // Set up the json stream reader - using (var responseStream = await HttpClient.GetStreamAsync(jsonUrl)) - using (var streamReader = new StreamReader(responseStream)) - using (var jsonReader = new JsonTextReader(streamReader)) - { - // Manual parsing - we only need to key-value pairs from each object and this avoids deserializing all of the work items into objects - string markedTestName = string.Empty; - string currentPropertyName = string.Empty; - - // The expected layout is produced by regular Helix runs - this allows us to parse and run tests from any Helix test list without special considerations - // The expected fields are - // { "WorkItemId": "<Fully Qualified Test Name>" , "PayloadUri":"<Url Of Test>" } - - while (jsonReader.Read()) - { - if (jsonReader.Value != null) - { - switch (jsonReader.TokenType) - { - case JsonToken.PropertyName: - currentPropertyName = jsonReader.Value.ToString(); - break; - case JsonToken.String: - // Test Name Value - if (currentPropertyName.Equals("WorkItemId")) - { - string currentTestName = jsonReader.Value.ToString(); - - // If the test has been marked as disabled in the test list - ignore it - if ((runAllTests || testDefinitions.ContainsKey(currentTestName)) && !disabledTests.Contains(currentTestName)) - { - markedTestName = currentTestName; - } - } - // Test URL value - else if (currentPropertyName.Equals("PayloadUri") && markedTestName != string.Empty) - { - if (!testDefinitions.ContainsKey(markedTestName)) - { - testDefinitions[markedTestName] = new XUnitTestAssembly() { Name = markedTestName }; - } - testDefinitions[markedTestName].Url = jsonReader.Value.ToString(); - markedTestName = string.Empty; - } - break; - } - } - } - - } - return testDefinitions; - } - - /// <summary> - /// Download each test from its specified URL - /// </summary> - /// <param name="testPayloads">The mapping of tests parsed from a test definition list to their names. The test definitions are populated with test URLs</param> - /// <param name="downloadDir">Directory to which to download tests</param> - /// <returns></returns> - public async Task GetTestArchives(Dictionary<string, XUnitTestAssembly> testPayloads, string downloadDir) - { - foreach (string testName in testPayloads.Keys) - { - string payloadUri = testPayloads[testName].Url; - - // Check URL for validity - if (!Uri.IsWellFormedUriString(payloadUri, UriKind.Absolute)) - continue; - Console.WriteLine("Downloading " + testName + " from " + payloadUri); - // Download tests from specified URL - using (var response = await HttpClient.GetStreamAsync(payloadUri)) - { - if (response.CanRead) - { - // Create the test setup directory if it doesn't exist - if (!Directory.Exists(downloadDir)) - { - Directory.CreateDirectory(downloadDir); - } - - // CoreFX test archives are output as .zip regardless of platform - string archivePath = Path.Combine(downloadDir, testName + ".zip"); - - // Copy to a temp folder - using (FileStream file = new FileStream(archivePath, FileMode.Create)) - { - await response.CopyToAsync(file); - } - - } - } - } - } - - /// <summary> - /// Expand Archives - /// </summary> - /// <param name="archiveDirectory">Directory containing archives</param> - /// <param name="destinationDirectory">Directory to which to unpack archives</param> - /// <param name="cleanup">Optional parameter stating, whether archives should be deleted once downloaded</param> - public void ExpandArchivesInDirectory(string archiveDirectory, string destinationDirectory, bool cleanup = true) - { - Debug.Assert(Directory.Exists(archiveDirectory)); - Debug.Assert(Directory.Exists(destinationDirectory)); - - // Get all archives in the directory - string[] archives = Directory.GetFiles(archiveDirectory, "*.zip", SearchOption.TopDirectoryOnly); - - foreach (string archivePath in archives) - { - string destinationDirName = Path.Combine(destinationDirectory, Path.GetFileNameWithoutExtension(archivePath)); - - ZipFile.ExtractToDirectory(archivePath, destinationDirName); - - // Delete archives if cleanup was - if (cleanup) - { - File.Delete(archivePath); - } - } - } - - /// <summary> - /// Cleans build directory - /// </summary> - /// <param name="directoryToClean">Directory the contents of which to delete.</param> - public void CleanBuild(string directoryToClean) - { - Debug.Assert(Directory.Exists(directoryToClean)); - DirectoryInfo dirInfo = new DirectoryInfo(directoryToClean); - - foreach (FileInfo file in dirInfo.EnumerateFiles()) - { - file.Delete(); - } - - foreach (DirectoryInfo dir in dirInfo.EnumerateDirectories()) - { - dir.Delete(true); - } - } - - } -} diff --git a/tests/src/Common/CoreFX/TestFileSetup/Helpers/TestRunHelper.cs b/tests/src/Common/CoreFX/TestFileSetup/Helpers/TestRunHelper.cs deleted file mode 100644 index a8a97916c5..0000000000 --- a/tests/src/Common/CoreFX/TestFileSetup/Helpers/TestRunHelper.cs +++ /dev/null @@ -1,158 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Runtime.InteropServices; -using System.Text; -using System.Threading.Tasks; - -namespace CoreFX.TestUtils.TestFileSetup.Helpers -{ - /// <summary> - /// A class which runs all tests conforming to the current format of CoreFX tests - - /// Each folder: - /// is named for the test it contains - e.g. System.Collections.Tests - /// contains a test assembly named for the library it tests - e.g. System.Collections.Tests.dll - /// contains a test executable with the specified name - e.g. xunit.console.netcore.exe - /// </summary> - public class NetCoreTestRunHelper - { - - public string DotnetExecutablePath { get; set; } - - public string logRootOutputPath { get; set; } - - /// <summary> - /// Default Constructor - /// </summary> - /// <param name="DotnetExecutablePath"> Path to the dotnet executable, which is used to run the test executable In CoreFX tests this will be the built test host</param> - /// <param name="logRootOutputPath">Path to which to output test run logs</param> - public NetCoreTestRunHelper(string DotnetExecutablePath, string logRootOutputPath) - { - this.DotnetExecutablePath = DotnetExecutablePath; - this.logRootOutputPath = logRootOutputPath; - } - - /// <summary> - /// Run a single test executabke - /// </summary> - /// <param name="workingDirectory">Directory from which to start the test executable</param> - /// <param name="executableName">Name of the test executable</param> - /// <param name="xunitTestTraits">Test trait exclusions to pass to the runner.</param> - /// <param name="logOutputPath">Path to which to output the single test run's results</param> - /// <returns>0 if single test run is succesful; 1 if not</returns> - public int RunExecutable(string workingDirectory, string executableName, IReadOnlyList<string> xunitTestTraits, string logOutputPath) - { - // Calculate and create the path to the test log - string logPath = Path.Combine(logOutputPath, Path.GetFileName(workingDirectory)); - if (!Directory.Exists(logPath)) - Directory.CreateDirectory(logPath); - - // Calculate the arguments to pass to the test runner - string arguments = CalculateCommandLineArguments(workingDirectory, executableName, xunitTestTraits, Path.Combine(logPath,"testResults.xml")); - - // Create and initialize the test executable process - ProcessStartInfo startInfo = new ProcessStartInfo(DotnetExecutablePath, arguments) - { - Arguments = arguments, - WorkingDirectory = workingDirectory - }; - - - Process executableProcess = new Process(); - executableProcess.StartInfo = startInfo; - executableProcess.EnableRaisingEvents = true; - executableProcess.Start(); - executableProcess.WaitForExit(); - - return executableProcess.ExitCode; - } - /// <summary> - /// Run all test executables conforming to the specified pattern in a directory - /// </summary> - /// <param name="rootDirectory">Directory containing tests to run</param> - /// <param name="executableName">Name of the test executable contained in folders</param> - /// <param name="xunitTestTraits">Test trait exclusions to pass to the runner.</param> - /// <param name="processLimit">Maximum number of tests to run in parallel</param> - /// <param name="logRootOutputPath">Root path to which to output the all test runs' results</param> - /// <returns>0 if entire test run is succesful; 1 if not</returns> - public int RunAllExecutablesInDirectory(string rootDirectory, string executableName, IReadOnlyList<string> xunitTestTraits, int processLimit, string logRootOutputPath = null) - { - int result = 0; - // Do a Depth-First Search to find and run executables with the same name - Stack<string> directories = new Stack<string>(); - List<string> testDirectories = new List<string>(); - // Push rootdir - directories.Push(rootDirectory); - - while (directories.Count > 0) - { - string currentDirectory = directories.Pop(); - - // If a directory contains an executable with the specified name - add it - if (File.Exists(Path.Combine(currentDirectory, executableName))) - testDirectories.Add(currentDirectory); - - foreach (string subDir in Directory.GetDirectories(currentDirectory)) - directories.Push(subDir); - } - - // Initialize max degree of parallelism - ParallelOptions parallelOptions = new ParallelOptions(); - parallelOptions.MaxDegreeOfParallelism = processLimit; - - Parallel.ForEach(testDirectories, parallelOptions, - (testDirectory) => - { - if (RunExecutable(testDirectory, executableName, xunitTestTraits, logRootOutputPath) != 0) - { - // If any tests fail mark the whole run as failed - Console.WriteLine("Test Run Failed " + testDirectory); - result = 1; - } - } - ); - return result; - } - - /// <summary> - /// Calculate the commandline arguments to pass to the test executable - /// </summary> - /// <param name="testDirectory">Current test directory name - assumed to have the same name as the test</param> - /// <param name="executableName">>Name of the test executable contained in the folder</param> - /// <param name="xunitTestTraits">Test trait exclusions to pass to the runner.</param> - /// <param name="logPath">Path to which to output the single test run's results</param> - /// <returns>A string representing command line arguments to be passed to the console test runner</returns> - private string CalculateCommandLineArguments(string testDirectory, string executableName, IReadOnlyList<string> xunitTestTraits, string logPath) - { - StringBuilder arguments = new StringBuilder(); - - // Append test executable name - arguments.Append($"\"{Path.Combine(testDirectory, Path.GetFileName(executableName))}\" "); - - // Append test name dll\ - arguments.Append($"\"{Path.Combine(testDirectory, Path.GetFileName(testDirectory))}.dll\" "); - - // Append RSP file - arguments.Append($"@\"{Path.Combine(testDirectory, Path.GetFileName(testDirectory))}.rsp\" "); - - if (!String.IsNullOrEmpty(logPath)) - { - // Add logging information - arguments.Append($"-xml {logPath} "); - } - - // Append all additional arguments - foreach (string traitToExclude in xunitTestTraits) - { - arguments.Append($"-notrait {traitToExclude} "); - } - - return arguments.ToString(); - } - } -} diff --git a/tests/src/Common/CoreFX/TestFileSetup/Program.cs b/tests/src/Common/CoreFX/TestFileSetup/Program.cs deleted file mode 100644 index 599dd6d717..0000000000 --- a/tests/src/Common/CoreFX/TestFileSetup/Program.cs +++ /dev/null @@ -1,174 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.CommandLine; -using System.IO; -using System.Net.Http; -using System.Text; -using CoreFX.TestUtils.TestFileSetup.Helpers; -using Newtonsoft.Json; -using Newtonsoft.Json.Schema; - -namespace CoreFX.TestUtils.TestFileSetup -{ - /// <summary> - /// This is a driver class, which downloads archived CoreFX test assemblies from a specified URL, lays out their contents - /// and subsequently runs them in parallel. - /// This is invoked from .tests/runtests.[cmd|sh] and depends on a test host (CoreCLR components with a dotnet executable) being present - /// </summary> - public class Program - { - // Helper class to lay out files on disk - private static TestFileHelper testFileHelper; - // Helper class to run tests in parallel - private static NetCoreTestRunHelper testRunHelper; - - // Test Set-up Options - private static string outputDir; - private static string testUrl; - private static string testListPath; - private static bool cleanTestBuild; - - // Test Run Options - private static string dotnetPath; - private static bool runSpecifiedTests; - private static bool runAllTests; - private static int maximumDegreeOfParalellization; - private static string logRootOutputPath; - - private static ExitCode exitCode; - private static string executableName; - private static IReadOnlyList<string> traitExclusions = Array.Empty<string>(); - - public static void Main(string[] args) - { - // Initialize default options - exitCode = ExitCode.Success; - maximumDegreeOfParalellization = Environment.ProcessorCount; - runSpecifiedTests = false; - runAllTests = false; - cleanTestBuild = false; - - ArgumentSyntax argSyntax = ParseCommandLine(args); - try - { - // Download and lay out files on disk - SetupTests(runAllTests); - - // Only run tests if the relevant commandline switch is passed - if (runSpecifiedTests || runAllTests) - exitCode = RunTests(); - } - catch (AggregateException e) - { - // Handle failure cases and exit gracefully - e.Handle(innerExc => - { - - if (innerExc is HttpRequestException) - { - exitCode = ExitCode.HttpError; - Console.WriteLine("Error downloading tests from: " + testUrl); - Console.WriteLine(innerExc.Message); - return true; - } - else if (innerExc is IOException) - { - exitCode = ExitCode.IOError; - Console.WriteLine(innerExc.Message); - return true; - } - else if (innerExc is JSchemaValidationException || innerExc is JsonSerializationException) - { - exitCode = ExitCode.JsonSchemaValidationError; - Console.WriteLine("Error validating test list: "); - Console.WriteLine(innerExc.Message); - return true; - } - else - { - exitCode = ExitCode.UnknownError; - } - - return false; - }); - } - - Environment.Exit((int)exitCode); - } - - /// <summary> - /// Parse passed Command Line arguments - /// </summary> - /// <param name="args"></param> - /// <returns></returns> - private static ArgumentSyntax ParseCommandLine(string[] args) - { - ArgumentSyntax argSyntax = ArgumentSyntax.Parse(args, syntax => - { - syntax.DefineOption("out|outDir|outputDirectory", ref outputDir, "Directory where tests are downloaded"); - syntax.DefineOption("testUrl", ref testUrl, "URL, pointing to the list of tests"); - syntax.DefineOption("testListJsonPath", ref testListPath, "JSON-formatted list of test assembly names to download"); - syntax.DefineOption("clean|cleanOutputDir", ref cleanTestBuild, "Clean test assembly output directory"); - syntax.DefineOption("runSpecified|runSpecifiedTests", ref runSpecifiedTests, "Run specified Tests after setup"); - syntax.DefineOption("runAll|runAllTests", ref runAllTests, "Run All available Tests in the specified TestList"); - syntax.DefineOption("dotnet|dotnetPath", ref dotnetPath, "Path to dotnet executable used to run tests."); - syntax.DefineOption("executable|executableName", ref executableName, "Name of the test executable to start"); - syntax.DefineOption("log|logPath|logRootOutputPath", ref logRootOutputPath, "Run Tests after setup"); - syntax.DefineOption("maxProcessCount|numberOfParallelTests|maximumDegreeOfParalellization", ref maximumDegreeOfParalellization, "Maximum number of concurrently executing processes"); - syntax.DefineOptionList("notrait", ref traitExclusions, "Traits to be excluded from test runs"); - - }); - - if (runSpecifiedTests || runAllTests) - { - if (String.IsNullOrEmpty(dotnetPath)) - throw new ArgumentException("Please supply a test host location to run tests."); - - if (!File.Exists(dotnetPath)) - throw new ArgumentException("Invalid testhost path. Please supply a test host location to run tests."); - } - - return argSyntax; - } - - /// <summary> - /// Method, which calls into the Test File Setup helper class to download and layout test assemblies on disk - /// </summary> - /// <param name="runAll">Specifies whether all tests available in the test list should be run</param> - private static void SetupTests(bool runAll = false) - { - testFileHelper = new TestFileHelper(); - - if (!Directory.Exists(outputDir)) - Directory.CreateDirectory(outputDir); - - // If the --clean switch has been specified delete existing assemblies - if (cleanTestBuild) - { - testFileHelper.CleanBuild(outputDir); - } - - // Map test names to their definitions - Dictionary<string, XUnitTestAssembly> testAssemblyDefinitions = testFileHelper.DeserializeTestJson(testListPath); - - testFileHelper.SetupTests(testUrl, outputDir, testAssemblyDefinitions, runAll).Wait(); - } - - /// <summary> - /// Runs all tests in a directory - /// Only tests, the executable driver of which has the same name as the passed argument are executed - e.g. xunit.console.netcore.exe - /// </summary> - /// <returns></returns> - private static ExitCode RunTests() - { - testRunHelper = new NetCoreTestRunHelper(dotnetPath, logRootOutputPath); - int result = testRunHelper.RunAllExecutablesInDirectory(outputDir, executableName, traitExclusions, maximumDegreeOfParalellization, logRootOutputPath); - - return result == 0 ? ExitCode.Success : ExitCode.TestFailure; - } - } -} diff --git a/tests/src/Common/CoreFX/TestFileSetup/RSPGenerator.cs b/tests/src/Common/CoreFX/TestFileSetup/RSPGenerator.cs deleted file mode 100644 index ae323026ca..0000000000 --- a/tests/src/Common/CoreFX/TestFileSetup/RSPGenerator.cs +++ /dev/null @@ -1,83 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; - -namespace CoreFX.TestUtils.TestFileSetup -{ - /// <summary> - /// A class which generates .rsp files to be passed to the test executable - /// The file contains a list of methods, classes and namespaces to be excluded from running. - /// </summary> - public class RSPGenerator - { - /// <summary> - /// Generate an rsp file from an XUnitTestAssembly class - /// </summary> - /// <param name="testDefinition">The XUnitTestAssembly object parsed from a specified test list</param> - /// <param name="outputPath">Path to which to output a .rsp file</param> - public void GenerateRSPFile(XUnitTestAssembly testDefinition, string outputPath) - { - if (!Directory.Exists(outputPath)) - { - Directory.CreateDirectory(outputPath); - } - string rspFilePath = Path.Combine(outputPath, testDefinition.Name + ".rsp"); - - if (File.Exists(rspFilePath)) - File.Delete(rspFilePath); - - // Write RSP file to disk - using (StreamWriter sr = File.CreateText(rspFilePath)) - { - // If no exclusions are defined, we don't need to generate an .rsp file - if (testDefinition.Exclusions == null) - return; - - // Method exclusions - if (testDefinition.Exclusions.Methods != null) - { - foreach (Exclusion exclusion in testDefinition.Exclusions.Methods) - { - if (String.IsNullOrWhiteSpace(exclusion.Name)) - continue; - sr.Write("-skipmethod "); - sr.Write(exclusion.Name); - sr.WriteLine(); - } - } - - // Class exclusions - if (testDefinition.Exclusions.Classes != null) - { - foreach (Exclusion exclusion in testDefinition.Exclusions.Classes) - { - if (String.IsNullOrWhiteSpace(exclusion.Name)) - continue; - sr.Write("-skipclass "); - sr.Write(exclusion.Name); - sr.WriteLine(); - } - - } - - // Namespace exclusions - if (testDefinition.Exclusions.Namespaces != null) - { - foreach (Exclusion exclusion in testDefinition.Exclusions.Namespaces) - { - if (String.IsNullOrWhiteSpace(exclusion.Name)) - continue; - sr.Write("-skipnamespace "); - sr.Write(exclusion.Name); - sr.WriteLine(); - } - } - } - } - } -} diff --git a/tests/src/Common/CoreFX/TestFileSetup/XUnit/XUnitTestAssembly.cs b/tests/src/Common/CoreFX/TestFileSetup/XUnit/XUnitTestAssembly.cs deleted file mode 100644 index efe78853c8..0000000000 --- a/tests/src/Common/CoreFX/TestFileSetup/XUnit/XUnitTestAssembly.cs +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Text; -using Newtonsoft.Json; - -namespace CoreFX.TestUtils.TestFileSetup -{ - /// <summary> - /// A class representing a CoreFX test assembly to be run - /// </summary> - public class XUnitTestAssembly - { - [JsonRequired] - [JsonProperty("name")] - public string Name; - - [JsonRequired] - [JsonProperty("enabled")] - public bool IsEnabled; - - [JsonRequired] - [JsonProperty("exclusions")] - public Exclusions Exclusions; - - // Used to assign a test url or to override it via the json file definition - mark it as optional in the test definition - [JsonIgnore] - [JsonProperty(Required = Required.Default)] - public string Url; - - } - /// <summary> - /// Class representing a collection of test exclusions - /// </summary> - public class Exclusions - { - [JsonProperty("namespaces")] - public Exclusion[] Namespaces; - - [JsonProperty("classes")] - public Exclusion[] Classes; - - [JsonProperty("methods")] - public Exclusion[] Methods; - } - - /// <summary> - /// Class representing a single test exclusion - /// </summary> - public class Exclusion - { - [JsonRequired] - [JsonProperty("name", Required = Required.DisallowNull)] - public string Name; - - [JsonRequired] - [JsonProperty("reason", Required = Required.DisallowNull)] - public string Reason; - } -} |