Merge pull request #6050 from prajwal-aithal/devel/arm-ci-runtests

ARM-CI: Add tests to CI script

2 files changed, 429 insertions, 33 deletions

diff --git a/netci.groovy b/netci.groovy
index 310fc6cca1..225ae57fda 100755
--- a/netci.groovy
+++ b/netci.groovy
@@ -1775,11 +1775,24 @@ combinedScenarios.each { scenario ->
                                         def armemul_path = '/opt/linux-arm-emulator'
                                         def armrootfs_mountpath = '/opt/linux-arm-emulator-root'
 
-                                        // Call the ARM emulator build script to cross build using the ARM emulator rootfs
-                                        buildCommands += "./tests/scripts/arm32_ci_script.sh --emulatorPath=${armemul_path} --mountPath=${armrootfs_mountpath} --buildConfig=${lowerConfiguration}"
+                                        // Unzip the Windows test binaries first. Exit with 0
+                                        buildCommands += "unzip -q -o ./bin/tests/tests.zip -d ./bin/tests/Windows_NT.x64.${configuration} || exit 0"
 
+                                        // Unpack the corefx binaries
+                                        buildCommands += "tar -xf ./bin/build.tar.gz"
 
-                                        // Basic archiving of the build, no pal tests
+                                        // Call the ARM emulator build script to cross build and test using the ARM emulator rootfs
+                                        buildCommands += """./tests/scripts/arm32_ci_script.sh \\
+                                        --emulatorPath=${armemul_path} \\
+                                        --mountPath=${armrootfs_mountpath} \\
+                                        --buildConfig=${lowerConfiguration} \\
+                                        --testRootDir=./bin/tests/Windows_NT.x64.${configuration} \\
+                                        --coreFxNativeBinDir=./bin/Linux.arm-softfp.${configuration} \\
+                                        --coreFxBinDir=\"./bin/Linux.AnyCPU.${configuration};./bin/Unix.AnyCPU.${configuration};./bin/AnyOS.AnyCPU.${configuration}\" \\
+                                        --testDirFile=./tests/testsRunningInsideARM.txt"""
+
+
+                                        // Basic archiving of the build
                                         Utilities.addArchival(newJob, "bin/Product/**")
                                         break
                                     }
@@ -1803,6 +1816,35 @@ combinedScenarios.each { scenario ->
                                 }
                             }
                             else {
+                                // Setup corefx and Windows test binaries for Linux ARM Emulator Build
+                                if (isLinuxEmulatorBuild) {
+                                    // Define the Windows Tests and Corefx build job names
+                                    def WindowTestsName = projectFolder + '/' +
+                                                          Utilities.getFullJobName(project,
+                                                                                   getJobName(lowerConfiguration,
+                                                                                              'x64' ,
+                                                                                              'windows_nt',
+                                                                                              'default',
+                                                                                              true),
+                                                                                   false)
+                                    def corefxFolder = Utilities.getFolderName('dotnet/corefx') + '/' +
+                                                       Utilities.getFolderName(branch)
+
+                                    // Copy the Windows test binaries and the Corefx build binaries
+                                    copyArtifacts(WindowTestsName) {
+                                        excludePatterns('**/testResults.xml', '**/*.ni.dll')
+                                        buildSelector {
+                                            latestSuccessful(true)
+                                        }
+                                    }
+                                    copyArtifacts("${corefxFolder}/linuxarmemulator_cross_${lowerConfiguration}") {
+                                        includePatterns('bin/build.tar.gz')
+                                        buildSelector {
+                                            latestSuccessful(true)
+                                        }
+                                    }
+                                }
+
                                 buildCommands.each { buildCommand ->
                                     shell(buildCommand)
                                 }
diff --git a/tests/scripts/arm32_ci_script.sh b/tests/scripts/arm32_ci_script.sh
index 404673ac88..223da2e26c 100755
--- a/tests/scripts/arm32_ci_script.sh
+++ b/tests/scripts/arm32_ci_script.sh
@@ -1,31 +1,285 @@
 #!/bin/bash
 
-usage() {
-    echo 'ARM Emulator Cross Build Script'
+#Usage message
+function usage {
+    echo 'ARM Emulator Cross Build and Test Script'
+    echo 'This script cross builds coreclr source and tests the binaries generated'
     echo ''
     echo 'Typical usage:'
-    echo './tests/scripts/arm32_ci_script.sh'
+    echo '    coreclr source is at ~/clr'
+    echo '    corefx source is at ~/cfx'
+    echo '    --testRootDir and --mscorlibDir have been built on Windows/downloaded from dotnet-ci.cloudapp.net'
+    echo '    --coreFxNativeBinDir has been built using cross build'
+    echo '    --coreFxBinDir has been built on Linux'
+    echo '$ cd ~/clr'
+    echo '$ ./tests/scripts/arm32_ci_script.sh'
     echo '    --emulatorPath=/opt/linux-arm-emulator'
     echo '    --mountPath=/opt/linux-arm-emulator-root'
     echo '    --buildConfig=Release'
+    echo '    --testRootDir=~/Downloads/Windows_NT.x64.Release'
+    echo '    --mscorlibDir=~/clr/bin/Product/Linux.arm-softfp.Release'
+    echo '    --coreFxNativeBinDir=~/cfx/bin/Linux.arm-softfp.Release'
+    echo '    --coreFxBinDir="~/cfx/bin/Linux.AnyCPU.Release;~/cfx/bin/Unix.AnyCPU.Release;~/cfx/bin/AnyOS.AnyCPU.Release"'
+    echo '    --testDirFile=~/clr/tests/testsRunningInsideARM.txt'
     echo ''
     echo 'Required Arguments:'
+    echo '    --emulatorPath=<path>              : Path of the emulator folder (without ending /)'
+    echo '                                         <path>/platform/rootfs-t30.ext4 should exist'
+    echo '    --mountPath=<path>                 : The desired path for mounting the emulator rootfs (without ending /)'
+    echo '                                         This path is created if not already present'
+    echo '    --buildConfig=<config>             : The value of config should be either Debug or Release'
+    echo '                                         Any other value is not accepted'
+    echo 'Optional Arguments:'
+    echo '    --skipTests                        : Presenting this option skips testing the generated binaries'
+    echo '                                         If this option is not presented, then tests are run by default'
+    echo '                                         using the other test related options'
+    echo '    --skipmscorlib                     : Skips generating mscorlib.dll on Linux'
+    echo '                                         If tests are run and this option is not used,'
+    echo '                                         then --mscorlibDir option to this script is mandatory'
+    echo '    -v --verbose                       : Build made verbose'
+    echo '    -h --help                          : Prints this usage message and exits'
     echo ''
-    echo '    --emulatorPath=<path>    Path of the emulator folder (without ending /)'
-    echo '                             <path>/platform/rootfs-t30.ext4 should exist'
-    echo '    --mountPath=<path>       The desired path for mounting the emulator rootfs (without ending /)'
-    echo '                             This path is created if not already present'
-    echo '    --buildConfig=<config>   The value of config should be either Debug or Release'
-    echo '                             Any other value is not accepted'
+    echo 'Test related Arguments (mandatory if --skipTests is not used):'
+    echo '    --testRootDir=<path>               : The root directory of the test build'
+    echo '    --mscorlibDir=<path>               : The directory containing the mscorlib.dll binary'
+    echo '                                         If provided, then the mscorlib.dll in this directory is'
+    echo '                                         used for tests instead of the built mscorlib.dll'
+    echo '    --coreFxNativeBinDir=<path>        : The directory of the CoreFX native build'
+    echo '    --coreFxBinDir="<path>[;<path>]"   : List one or more directories with CoreFX managed build binaries'
+    echo '    --testDirFile=<path>               : Runs tests only in the directories specified by the file at <path>'
+    echo '                                         The directories are listed in lines in the file at <path>'
     echo ''
-    echo 'Any other argument triggers an error and this message is displayed'
+    echo 'Any other argument triggers an error and this usage message is displayed'
     exit 1
 }
 
+#Display error message and exit
+function exit_with_error {
+    set +x
+
+    local errorMessage="$1"
+    local printUsage=$2
+
+    echo "ERROR: $errorMessage"
+    if [ "$printUsage" == "true" ]; then
+        echo ''
+        usage
+    fi
+    exit 1
+}
+
+#Exit if input string is empty
+function exit_if_empty {
+    local inputString="$1"
+    local errorMessage="$2"
+    local printUsage=$3
+
+    if [ -z "$inputString" ]; then
+        exit_with_error "$errorMessage" $printUsage
+    fi
+}
+
+#Exit if the input path does not exist
+function exit_if_path_absent {
+    local path="$1"
+    local errorMessage="$2"
+    local printUsage=$3
+
+    if [ ! -f "$path" -a ! -d "$path" ]; then
+        exit_with_error "$errorMessage" $printUsage
+    fi
+}
+
+#Check if the git changes were reverted completely
+function check_git_head {
+    local currentGitHead=`git rev-parse --verify HEAD`
+
+    if [[ "$__initialGitHead" != "$currentGitHead" ]]; then
+        exit_with_error "Some changes made to the code history were not completely reverted. Intial Git HEAD: $__initialGitHead, current Git HEAD: $currentGitHead" false
+    fi
+}
+
+function unmount_rootfs {
+    local rootfsFolder="$1"
+
+    if grep -qs "$rootfsFolder" /proc/mounts; then
+        sudo umount "$rootfsFolder"
+    fi
+}
+
+#Clean the previous build files inside the emulator
+function clean_emulator {
+    #Remove any previous copies of the coreclr and the corefx directories in the emulator
+    sudo rm -rf "$__ARMRootfsCoreclrPath" "$__ARMRootfsCorefxPath"
+}
+
+#Unmount the emulator file systems
+function unmount_emulator {
+    (set +x; echo 'Unmounting emulator...')
+
+    #Unmount all the mounted emulator file systems
+    unmount_rootfs "$__ARMRootfsMountPath/proc"
+    unmount_rootfs "$__ARMRootfsMountPath/dev/pts"
+    unmount_rootfs "$__ARMRootfsMountPath/dev"
+    unmount_rootfs "$__ARMRootfsMountPath/run/shm"
+    unmount_rootfs "$__ARMRootfsMountPath/sys"
+    unmount_rootfs "$__ARMRootfsMountPath"
+}
+
+#Clean the changes made to the environment by the script
+function clean_env {
+    #Clean the emulator
+    clean_emulator
+
+    #Unmount the emulator
+    unmount_emulator
+
+    #Check for revert of git changes
+    check_git_head
+}
+
+#Trap Ctrl-C and handle it
+function handle_ctrl_c {
+    set +x
+
+    echo 'ERROR: Ctrl-C handled. Script aborted before complete execution.'
+
+    clean_env
+
+    exit 1
+}
+trap handle_ctrl_c INT
+
+#Mount emulator to the target mount path
+function mount_emulator {
+    #Check if the mount path exists and create if neccessary
+    if [ ! -d "$__ARMRootfsMountPath" ]; then
+        sudo mkdir "$__ARMRootfsMountPath"
+    fi
+
+    #Unmount the emulator if already mounted at the mount path and mount again
+    unmount_emulator
+
+    sudo mount "$__ARMEmulPath"/platform/rootfs-t30.ext4 "$__ARMRootfsMountPath"
+    sudo mount -t proc /proc    "$__ARMRootfsMountPath"/proc
+    sudo mount -o bind /dev/    "$__ARMRootfsMountPath"/dev
+    sudo mount -o bind /dev/pts "$__ARMRootfsMountPath"/dev/pts
+    sudo mount -t tmpfs shm     "$__ARMRootfsMountPath"/run/shm
+    sudo mount -o bind /sys     "$__ARMRootfsMountPath"/sys
+}
+
+#Cross builds coreclr
+function cross_build_coreclr {
+#Export the needed environment variables
+    (set +x; echo 'Exporting LINUX_ARM_* environment variable')
+    source "$__ARMRootfsMountPath"/dotnet/setenv/setenv_incpath.sh "$__ARMRootfsMountPath"
+
+    #Apply the changes needed to build for the emulator rootfs
+    (set +x; echo 'Applying cross build patch to suit Linux ARM emulator rootfs')
+    git am < "$__ARMRootfsMountPath"/dotnet/setenv/coreclr_cross.patch
+
+    #Apply release optimization patch if needed
+    if [[ "$__buildConfig" == "Release" ]]; then
+        (set +x; echo 'Applying release optimization patch to build in Release mode')
+        git am < "$__ARMRootfsMountPath"/dotnet/setenv/coreclr_release.patch
+    fi
+
+    #Cross building for emulator rootfs
+    ROOTFS_DIR="$__ARMRootfsMountPath" CPLUS_INCLUDE_PATH=$LINUX_ARM_INCPATH CXXFLAGS=$LINUX_ARM_CXXFLAGS ./build.sh $__buildArch clean cross $__verboseFlag $__skipMscorlib clang3.5 $__buildConfig
+
+    #Reset the code to the upstream version
+    (set +x; echo 'Rewinding HEAD to master code')
+    git reset --hard HEAD^
+    if [[ "$__buildConfig" == "Release" ]]; then
+        git reset --hard HEAD^
+    fi
+}
+
+#Copy the needed files to the emulator to run tests
+function copy_to_emulator {
+
+    #Create the coreclr and corefx directories in the emulator
+    sudo mkdir -p "$__ARMRootfsCoreclrPath/bin/obj/$__buildDirName"
+    sudo mkdir -p "$__ARMRootfsCoreclrPath/bin/Product"
+    sudo mkdir "$__ARMRootfsCorefxPath"
+
+    #Copy all coreclr files to the coreclr root in the emulator and set the paths accordingly
+    local testRootDirBase=`basename "$__testRootDir"`
+    sudo cp -R "$__testRootDir" "$__ARMRootfsCoreclrPath/$testRootDirBase"
+    __testRootDirBase="$__ARMEmulCoreclr/$testRootDirBase"
+
+    sudo cp -R "./$__testNativeBinDirBase" "$__ARMRootfsCoreclrPath/$__testNativeBinDirBase"
+    __testNativeBinDirBase="$__ARMEmulCoreclr/$__testNativeBinDirBase"
+
+    sudo cp -R "./$__coreClrBinDirBase" "$__ARMRootfsCoreclrPath/$__coreClrBinDirBase"
+    if [ ! -z "$__mscorlibDir" ]; then
+        sudo cp "$__mscorlibDir/mscorlib.dll" "$__ARMRootfsCoreclrPath/$__coreClrBinDirBase/"
+    else
+        sudo cp "./$__coreClrBinDirBase/mscorlib.dll" "$__ARMRootfsCoreclrPath/$__coreClrBinDirBase/"
+    fi
+    __coreClrBinDirBase="$__ARMEmulCoreclr/$__coreClrBinDirBase"
+    __mscorlibDirBase="$__coreClrBinDirBase"
+
+    local testDirFileBase=`basename "$__testDirFile"`
+    sudo cp "$__testDirFile" "$__ARMRootfsCoreclrPath/$testDirFileBase"
+    __testDirFileBase="$__ARMEmulCoreclr/$testDirFileBase"
+
+    sudo cp -R ./tests "$__ARMRootfsCoreclrPath/"
+    sudo cp -R ./packages "$__ARMRootfsCoreclrPath/"
+    sudo cp -R ./Tools "$__ARMRootfsCoreclrPath/"
+
+    #Copy corefx binary directories to the corefx root in the emulator (first native and then managed)
+    local coreFxNativeBinDirBase=`basename "$__coreFxNativeBinDir"`
+    sudo cp -R "$__coreFxNativeBinDir" "$__ARMRootfsCorefxPath/$coreFxNativeBinDirBase"
+    __coreFxNativeBinDirBase="$__ARMEmulCorefx/$coreFxNativeBinDirBase"
+
+    __coreFxBinDirBase=
+    while IFS=';' read -ra coreFxBinDirectories; do
+        for currDir in "${coreFxBinDirectories[@]}"; do
+            local currDirBase=`basename "$currDir"`
+            sudo cp -R "$currDir" "$__ARMRootfsCorefxPath/$currDirBase"
+
+            if [ -z "$__coreFxBinDirBase" ]; then
+                __coreFxBinDirBase="$__ARMEmulCorefx/$currDirBase"
+            else
+                __coreFxBinDirBase="$__coreFxBinDirBase;$__ARMEmulCorefx/$currDirBase"
+            fi
+        done
+    done <<< "$__coreFxBinDir" 
+}
+
+#Runs tests in an emulated mode
+function run_tests {
+    sudo chroot $__ARMRootfsMountPath /bin/bash -x <<EOF
+        cd /home/coreclr
+        ./tests/runtest.sh --testRootDir=$__testRootDirBase \
+                           --mscorlibDir=$__mscorlibDirBase \
+                           --coreFxNativeBinDir=$__coreFxNativeBinDirBase \
+                           --coreFxBinDir="$__coreFxBinDirBase" \
+                           --testDirFile=$__testDirFileBase \
+                           --testNativeBinDir=$__testNativeBinDirBase \
+                           --coreClrBinDir=$__coreClrBinDirBase
+EOF
+}
+
+#Define script variables
 __ARMEmulPath=
 __ARMRootfsMountPath=
-__BuildConfig=
+__buildConfig=
+__skipTests=0
+__skipMscorlib=
+__testRootDir=
+__mscorlibDir=
+__coreFxNativeBinDir=
+__coreFxBinDir=
+__testDirFile=
+__verboseFlag=
+__buildOS="Linux"
+__buildArch="arm-softfp"
+__buildDirName=
+__initialGitHead=`git rev-parse --verify HEAD`
 
+#Parse command line arguments
 for arg in "$@"
 do
     case $arg in
@@ -36,41 +290,141 @@ do
         __ARMRootfsMountPath=${arg#*=}
         ;;
     --buildConfig=*)
-        __BuildConfig="$(echo ${arg#*=} | awk '{print tolower($0)}')"
-        if [[ "$__BuildConfig" != "debug" && "$__BuildConfig" != "release" ]]; then
-            usage
+        __buildConfig="$(echo ${arg#*=} | awk '{print tolower($0)}')"
+        if [[ "$__buildConfig" != "debug" && "$__buildConfig" != "release" ]]; then
+            exit_with_error "--buildConfig can be only Debug or Release" true
         fi
         ;;
-    *)
+    --skipTests)
+        __skipTests=1
+        ;;
+    --skipmscorlib)
+        __skipMscorlib="skipmscorlib"
+        ;;
+    -v|--verbose)
+        __verboseFlag="verbose"
+        ;;
+    --testRootDir=*)
+        __testRootDir=${arg#*=}
+        ;;
+    --mscorlibDir=*)
+        __mscorlibDir=${arg#*=}
+        ;;
+    --coreFxNativeBinDir=*)
+        __coreFxNativeBinDir=${arg#*=}
+        ;;
+    --coreFxBinDir=*)
+        __coreFxBinDir=${arg#*=}
+        ;;
+    --testDirFile=*)
+        __testDirFile=${arg#*=}
+        ;;
+    -h|--help)
         usage
         ;;
+    *)
+        exit_with_error "$arg not a recognized argument" true
+        ;;
     esac
 done
 
-if [ -z "$__ARMEmulPath" -o -z "$__ARMRootfsMountPath" -o -z "$__BuildConfig" ]; then
-    usage
+#Check if there are any uncommited changes in the source directory as git adds and removes patches
+if [[ $(git status -s) != "" ]]; then
+   echo 'ERROR: There are some uncommited changes. To avoid losing these changes commit them and try again.'
+   echo ''
+   git status
+   exit 1
+fi
+
+#Check if the compulsory arguments have been presented to the script and if the input paths exist
+exit_if_empty "$__ARMEmulPath" "--emulatorPath is a mandatory argument, not provided" true
+exit_if_empty "$__ARMRootfsMountPath" "--mountPath is a mandatory argument, not provided" true
+exit_if_empty "$__buildConfig" "--buildConfig is a mandatory argument, not provided" true
+exit_if_path_absent "$__ARMEmulPath/platform/rootfs-t30.ext4" "Path specified in --emulatorPath does not have the rootfs" false
+
+#Check if the optional arguments are present in the case that testing is to be done
+if [ $__skipTests == 0 ]; then
+    exit_if_empty "$__testRootDir" "Testing requested, but --testRootDir not provided" true
+    exit_if_path_absent "$__testRootDir" "Path specified in --testRootDir does not exist" false
+
+    exit_if_empty "$__coreFxNativeBinDir" "Testing requested but --coreFxNativeBinDir not provided" true
+    exit_if_path_absent "$__coreFxNativeBinDir" "Path specified in --coreFxNativeBinDir does not exist" false
+
+    exit_if_empty "$__coreFxBinDir" "Testing requested, but --coreFxBinDir not provided" true
+    while IFS=';' read -ra coreFxBinDirectories; do
+        for currDir in "${coreFxBinDirectories[@]}"; do
+            exit_if_path_absent "$currDir" "Path specified in --coreFxBinDir, $currDir does not exist" false
+        done
+    done <<< "$__coreFxBinDir"
+
+    exit_if_empty "$__testDirFile" "Testing requested, but --testDirFile not provided" true
+    exit_if_path_absent "$__testDirFile" "Path specified in --testDirFile does not exist" false
+
+    if [ ! -z "$__skipMscorlib" ]; then
+        exit_if_empty "$__mscorlibDir" "Testing and skipmscorlib requested, but --mscorlibDir not provided" true
+    fi
+    if [ ! -z "$__mscorlibDir" ]; then
+        echo '--mscorlibDir provided; will be using this path for running tests and ignoring the generated mscorlib.dll'
+        exit_if_path_absent "$__mscorlibDir/mscorlib.dll" "Path specified in --mscorlibDir does not contain mscorlib.dll"
+    fi
+fi
+
+#Change build configuration to the capitalized form to create build product paths correctly
+if [[ "$__buildConfig" == "release" ]]; then
+    __buildConfig="Release"
+else
+    __buildConfig="Debug"
 fi
+__buildDirName="$__buildOS.$__buildArch.$__buildConfig"
+
+#Define emulator paths
+__ARMRootfsCoreclrPath="$__ARMRootfsMountPath/home/coreclr"
+__ARMRootfsCorefxPath="$__ARMRootfsMountPath/home/corefx"
+__ARMEmulCoreclr="/home/coreclr"
+__ARMEmulCorefx="/home/corefx"
+__testRootDirBase=
+__mscorlibDirBase=
+__coreFxNativeBinDirBase=
+__coreFxBinDirBase=
+__testDirFileBase=
+__testNativeBinDirBase="bin/obj/$__buildDirName/tests"
+__coreClrBinDirBase="bin/Product/$__buildDirName"
 
 set -x
 set -e
 
-if [ ! -d $__ARMRootfsMountPath ]; then
-    sudo mkdir $__ARMRootfsMountPath
-fi
+## Begin cross build
+(set +x; echo "Git HEAD @ $__initialGitHead")
+
+#Mount the emulator
+(set +x; echo 'Mounting emulator...')
+mount_emulator
+
+#Clean the emulator
+(set +x; echo 'Cleaning emulator...')
+clean_emulator
+
+#Complete the cross build
+(set +x; echo 'Building coreclr...')
+cross_build_coreclr
 
-if grep -qs $__ARMRootfsMountPath /proc/mounts; then
-    sudo umount $__ARMRootfsMountPath
+#If tests are to be skipped end the script here, else continue
+if [ $__skipTests == 1 ]; then
+    exit 0
 fi
 
-sudo mount $__ARMEmulPath/platform/rootfs-t30.ext4 $__ARMRootfsMountPath
+## Tests are going to be performed in an emulated environment
 
-echo "Exporting LINUX_ARM_* environment variable"
-source $__ARMRootfsMountPath/dotnet/setenv/setenv_incpath.sh $__ARMRootfsMountPath
+#Copy the needed files to the emulator before entering the emulated environment
+(set +x; echo 'Setting up emulator to run tests...')
+copy_to_emulator
 
-echo "Applying cross build patch to suit Linux ARM emulator rootfs"
-git am < $__ARMRootfsMountPath/dotnet/setenv/coreclr_cross.patch
+#Enter the emulated mode and run the tests
+(set +x; echo 'Running tests...')
+run_tests
 
-ROOTFS_DIR=$__ARMRootfsMountPath CPLUS_INCLUDE_PATH=$LINUX_ARM_INCPATH CXXFLAGS=$LINUX_ARM_CXXFLAGS ./build.sh arm-softfp clean cross verbose skipmscorlib clang3.5 $__BuildConfig
+#Clean the environment
+(set +x; echo 'Cleaning environment...')
+clean_env
 
-echo "Rewinding HEAD to master code"
-git reset --hard HEAD^
+(set +x; echo 'Build and test complete')