Add CrossGen Comparison scenario to netci.groovy

1 files changed, 120 insertions, 3 deletions

diff --git a/netci.groovy b/netci.groovy
index b86a835b4a..400d17b7db 100755
--- a/netci.groovy
+++ b/netci.groovy
@@ -32,6 +32,18 @@ def static getOSGroup(def os) {
     return osGroupMap[os]
 }
 
+def static getCrossArchitecture(def os, def architecture, def scenario) {
+    switch (architecture) {
+        case 'arm':
+            return 'x86'
+
+        case 'arm64':
+            return 'x64'
+    }
+
+    assert false
+}
+
 // We use this class (vs variables) so that the static functions can access data here.
 class Constants {
 
@@ -159,7 +171,8 @@ class Constants {
                'standalone_gc',
                'gc_reliability_framework',
                'illink',
-               'corefx_innerloop']
+               'corefx_innerloop',
+               'crossgen_comparison']
 
     def static allScenarios = basicScenarios + r2rStressScenarios.keySet() + jitStressModeScenarios.keySet()
 
@@ -244,6 +257,8 @@ class Constants {
                // 'standalone_gc'
                // 'gc_reliability_framework'
                // 'illink'
+               // 'corefx_innerloop'
+               // 'crossgen_comparison'
                'r2r_jitstress1':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
                'r2r_jitstress2':                         ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
                'r2r_jitstress1_tiered':                  ["R2R_FAIL", "R2R_EXCLUDE", "JITSTRESS_FAIL", "JITSTRESS_EXCLUDE"],
@@ -354,6 +369,8 @@ class Constants {
                // 'standalone_gc'
                // 'gc_reliability_framework'
                // 'illink'
+               // 'corefx_innerloop'
+               'crossgen_comparison',
                'r2r_jitstress1',
                'r2r_jitstress2',
                'r2r_jitstress1_tiered',
@@ -440,6 +457,8 @@ class Constants {
                // 'standalone_gc'
                // 'gc_reliability_framework'
                // 'illink'
+               // 'corefx_innerloop'
+               // 'crossgen_comparison'
                'r2r_jitstress1',
                'r2r_jitstress2',
                'r2r_jitstress1_tiered',
@@ -899,6 +918,15 @@ def static isPri0TestScenario(def scenario) {
     return (scenario == 'innerloop' || scenario == 'no_tiered_compilation_innerloop')
 }
 
+def static isCrossGenComparisonScenario(def scenario) {
+    return (scenario == 'crossgen_comparison')
+}
+
+def static shouldGenerateCrossGenComparisonJob(def os, def architecture, def configuration, def scenario) {
+    assert isCrossGenComparisonScenario(scenario)
+    return (os == 'Ubuntu' && architecture == 'arm' && configuration == 'Checked')
+}
+
 def static getFxBranch(def branch) {
     def fxBranch = branch
     // Map 'dev/unix_test_workflow' to 'master' so we can test CoreFX jobs in the CoreCLR dev/unix_test_workflow
@@ -1303,6 +1331,7 @@ def static addNonPRTriggers(def job, def branch, def isPR, def architecture, def
     switch (scenario) {
         case 'innerloop':
         case 'no_tiered_compilation_innerloop':
+        case 'crossgen_comparison':
             break
         case 'normal':
             switch (architecture) {
@@ -2484,6 +2513,28 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/fxtests.zip")
                         Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/run-test.sh")
                     }
+                    else if (isCrossGenComparisonScenario(scenario)) {
+                        buildCommands += "${dockerCmd}\${WORKSPACE}/build-test.sh ${lowerConfiguration} ${architecture} cross generatelayoutonly"
+
+                        def crossArchitecture = getCrossArchitecture(os, architecture, scenario)
+
+                        def workspaceRelativeProductBinDir = "bin/Product/${osGroup}.${architecture}.${configuration}"
+                        def workspaceRelativeCoreLib = "${workspaceRelativeProductBinDir}/IL/System.Private.CoreLib.dll"
+                        def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
+                        def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
+                        def workspaceRelativeCrossResultDir = "_/${osGroup}.${crossArchitecture}_${architecture}.${configuration}"
+                        def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
+
+                        def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
+                        def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeProductBinDir}/${crossArchitecture}/crossgen"
+
+                        buildCommands += "${dockerCmd}mkdir -p \${WORKSPACE}/${workspaceRelativeCrossResultDir}"
+                        buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeCrossResultDir}"
+                        buildCommands += "${dockerCmd}${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeCrossResultDir}"
+                        buildCommands += "${dockerCmd}zip -r ${workspaceRelativeArtifactsArchive} ${workspaceRelativeCoreLib} ${workspaceRelativeCoreRootDir} ${workspaceRelativeCrossGenComparisonScript} ${workspaceRelativeCrossResultDir}"
+
+                        Utilities.addArchival(newJob, "${workspaceRelativeArtifactsArchive}")
+                    }
                     else {
                         // Then, using the same docker image, generate the CORE_ROOT layout using build-test.sh to
                         // download the appropriate CoreFX packages.
@@ -2698,6 +2749,9 @@ def static shouldGenerateJob(def scenario, def isPR, def architecture, def confi
                 break
         }
     }
+    else if (isCrossGenComparisonScenario(scenario)) {
+        return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
+    }
     else {
         // Skip scenarios
         switch (scenario) {
@@ -3374,6 +3428,56 @@ ${runScript} \\
     return newJob
 }
 
+def static CreateNonWindowsCrossGenComparisonTestJob(def dslFactory, def project, def architecture, def os, def configuration, def scenario, def isPR, def inputCoreCLRBuildName)
+{
+    assert isCrossGenComparisonScenario(scenario)
+
+    def osGroup = getOSGroup(os)
+    def jobName = getJobName(configuration, architecture, os, scenario, false) + "_tst"
+
+    def crossArchitecture = getCrossArchitecture(os, architecture, scenario)
+    def workspaceRelativeCrossResultDir = "_/${osGroup}.${crossArchitecture}_${architecture}.${configuration}"
+    def workspaceRelativeNativeResultDir = "_/${osGroup}.${architecture}_${architecture}.${configuration}"
+
+    def jobFolder = getJobFolder(scenario)
+    def newJob = dslFactory.job(Utilities.getFullJobName(project, jobName, isPR, jobFolder)) {
+        parameters {
+            stringParam('CORECLR_BUILD', '', "Build number to copy CoreCLR ${osGroup} binaries from")
+        }
+
+        def workspaceRelativeArtifactsArchive = "${os}.${architecture}.${configuration}.${scenario}.zip"
+
+        steps {
+            copyArtifacts(inputCoreCLRBuildName) {
+                includePatterns("${workspaceRelativeArtifactsArchive}")
+                buildSelector {
+                    buildNumber('${CORECLR_BUILD}')
+                }
+            }
+
+            shell("unzip -o ${workspaceRelativeArtifactsArchive} || exit 0")
+
+            def workspaceRelativeCoreLib = "bin/Product/${osGroup}.${architecture}.${configuration}/IL/System.Private.CoreLib.dll"
+            def workspaceRelativeCoreRootDir = "bin/tests/${osGroup}.${architecture}.${configuration}/Tests/Core_Root"
+            def workspaceRelativeCrossGenComparisonScript = "tests/scripts/crossgen_comparison.py"
+            def workspaceRelativeCrossGenExecutable = "${workspaceRelativeCoreRootDir}/crossgen"
+
+            def crossGenComparisonCmd = "python -u \${WORKSPACE}/${workspaceRelativeCrossGenComparisonScript} "
+            def crossGenExecutable = "\${WORKSPACE}/${workspaceRelativeCrossGenExecutable}"
+
+            shell("mkdir -p ${workspaceRelativeNativeResultDir}")
+            shell("${crossGenComparisonCmd}crossgen_corelib --crossgen ${crossGenExecutable} --il_corelib \${WORKSPACE}/${workspaceRelativeCoreLib} --result_dir \${WORKSPACE}/${workspaceRelativeNativeResultDir}")
+            shell("${crossGenComparisonCmd}crossgen_framework --crossgen ${crossGenExecutable} --core_root \${WORKSPACE}/${workspaceRelativeCoreRootDir} --result_dir \${WORKSPACE}/${workspaceRelativeNativeResultDir}")
+            shell("${crossGenComparisonCmd}compare --base_dir \${WORKSPACE}/${workspaceRelativeNativeResultDir} --diff_dir \${WORKSPACE}/${workspaceRelativeCrossResultDir}")
+        } // steps
+    }  // job
+
+    Utilities.addArchival(newJob, "${workspaceRelativeCrossResultDir}/**")
+    Utilities.addArchival(newJob, "${workspaceRelativeNativeResultDir}/**")
+
+    return newJob
+}
+
 // Create a test job that will be used by a flow job.
 // Returns the newly created job.
 // Note that we don't add tests jobs to the various views, since they are always used by a flow job, which is in the views,
@@ -3386,7 +3490,12 @@ def static CreateTestJob(def dslFactory, def project, def branch, def architectu
     if (windowsArmJob) {
         assert inputTestsBuildName == null
         newJob = CreateWindowsArmTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
-    } else {
+    }
+    else if (isCrossGenComparisonScenario(scenario)) {
+        assert inputTestsBuildName == null
+        newJob = CreateNonWindowsCrossGenComparisonTestJob(dslFactory, project, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName)
+    }
+    else {
         newJob = CreateOtherTestJob(dslFactory, project, branch, architecture, os, configuration, scenario, isPR, inputCoreCLRBuildName, inputTestsBuildName)
     }
 
@@ -3588,6 +3697,9 @@ def static shouldGenerateFlowJob(def scenario, def isPR, def architecture, def c
             return false
         }
     }
+    else if (isCrossGenComparisonScenario(scenario)) {
+        return shouldGenerateCrossGenComparisonJob(os, architecture, configuration, scenario)
+    }
     else {
         // Skip scenarios
         switch (scenario) {
@@ -3657,6 +3769,7 @@ Constants.allScenarios.each { scenario ->
 
                     def windowsArmJob = ((os == "Windows_NT") && (architecture in Constants.armWindowsCrossArchitectureList))
                     def doCoreFxTesting = isCoreFxScenario(scenario)
+                    def doCrossGenComparison = isCrossGenComparisonScenario(scenario)
 
                     // Figure out the job name of the CoreCLR build the test will depend on.
 
@@ -3670,6 +3783,10 @@ Constants.allScenarios.each { scenario ->
                             inputCoreCLRBuildIsBuildOnly = true
                         }
                     }
+                    if (doCrossGenComparison) {
+                        inputCoreCLRBuildScenario = scenario
+                    }
+
                     def inputCoreCLRFolderName = getJobFolder(inputCoreCLRBuildScenario)
                     def inputCoreCLRBuildName = projectFolder + '/' +
                         Utilities.getFullJobName(project, getJobName(configuration, architecture, os, inputCoreCLRBuildScenario, inputCoreCLRBuildIsBuildOnly), isPR, inputCoreCLRFolderName)
@@ -3682,7 +3799,7 @@ Constants.allScenarios.each { scenario ->
 
                     def inputTestsBuildName = null
 
-                    if (!windowsArmJob && !doCoreFxTesting) {
+                    if (!windowsArmJob && !doCoreFxTesting & !doCrossGenComparison) {
                         def testBuildScenario = isPri0TestScenario(scenario) ? 'innerloop' : 'normal'
 
                         def inputTestsBuildArch = architecture