Move groovy logic to python script

3 files changed, 159 insertions, 70 deletions

diff --git a/Documentation/project-docs/ci-trigger-phrases.md b/Documentation/project-docs/ci-trigger-phrases.md
index 4105b5d75a..9751ed0f6a 100644
--- a/Documentation/project-docs/ci-trigger-phrases.md
+++ b/Documentation/project-docs/ci-trigger-phrases.md
@@ -67,6 +67,7 @@ To trigger a job, post a comment on your PR with "@dotnet-bot {trigger-phrase}".
 - **Windows_NT x64 Checked GCStress=0xc JitStress=2 Build & Test:** "test Windows_NT gcstress0xc_jitstress2"
 - **Windows_NT x64 Checked GCStress=0xc MinOpts Heap Verify 1 Build & Test:** "test Windows_NT gcstress0xc_minopts_heapverify1"
 - **Windows_NT x64 Checked Long-Running GC Build & Test:**: "test Windows_NT x64 Checked longgc"
+- **Windows_NT x64 Formatting:**: "test Windows_NT formatting"
 - **Windows_NT x64 Checked CoreFX Baseline Build & Test:** "test Windows_NT corefx_baseline"
 - **Windows_NT x64 Checked CoreFX MinOpts Build & Test:** "test Windows_NT corefx_minopts"
 - **Windows_NT x64 Checked CoreFX JitStress=1 Build & Test:** "test Windows_NT corefx_jitstress1"
@@ -192,6 +193,7 @@ To trigger a job, post a comment on your PR with "@dotnet-bot {trigger-phrase}".
 - **Ubuntu x64 Checked GCStress=0xc JitStress=2 Build & Test:** "test Ubuntu gcstress0xc_jitstress2"
 - **Ubuntu x64 Checked GCStress=0xc MinOpts Heap Verify 1 Build & Test:** "test Ubuntu gcstress0xc_minopts_heapverify1"
 - **Ubuntu x64 Checked Long-Running GC Build & Test:**: "test Ubuntu Checked longgc"
+- **Ubuntu x64 Formatting:**: "test Ubuntu formatting"
 - **Ubuntu x64 Checked CoreFX Baseline Build & Test:** "test Ubuntu corefx_baseline"
 - **Ubuntu x64 Checked CoreFX MinOpts Build & Test:** "test Ubuntu corefx_minopts"
 - **Ubuntu x64 Checked CoreFX JitStress=1 Build & Test:** "test Ubuntu corefx_jitstress1"
diff --git a/netci.groovy b/netci.groovy
index dda99118ed..f083400354 100755
--- a/netci.groovy
+++ b/netci.groovy
@@ -556,7 +556,7 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os,
             if (scenario == 'formatting') {
                 assert configuration == 'Checked'
                 if (os == 'Windows_NT' || os == 'Ubuntu') {
-                    Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting Build & Test", "(?i).*test\\W+formatting.*")
+                    Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} Formatting", "(?i).*test\\W+${os}\\W+formatting.*")
                 }
                 break
             }
@@ -1476,7 +1476,9 @@ combinedScenarios.each { scenario ->
                                 if (configuration != 'Checked') {
                                     return
                                 }
-                                println("Formatting not skipped: ${os} ${architecture} ${configuration}")
+                                if (isBuildOnly) {
+                                    return
+                                }
                                 break
                             case 'default':
                                 // Nothing skipped
@@ -1544,35 +1546,7 @@ combinedScenarios.each { scenario ->
                                         buildCommands += "set __TestIntermediateDir=int&&build-test.cmd ${lowerConfiguration} ${arch}"
                                     }
                                     else if (scenario == 'formatting') {
-                                        // Build with NMake Makefiles for all configurations
-                                        buildCommands += "set __TestIntermediateDir=int&&build.cmd Debug ${arch} usenmakemakefiles"
-                                        buildCommands += "set __TestIntermediateDir=int&&build.cmd Checked ${arch} usenmakemakefiles"
-                                        buildCommands += "set __TestIntermediateDir=int&&build.cmd Release ${arch} usenmakemakefiles"
-
-                                        // Download jitutils
-                                        def url = "https://raw.githubusercontent.com/dotnet/jitutils/master/bootstrap.cmd"
-                                        def outfile = "bootstrap.cmd"
-                                        buildCommands += "powershell Invoke-WebRequest -Uri ${url} -OutFile ${outfile}"
-
-                                        // Install x64 dotnet cli version 1.0.0.0
-                                        def dotnetcliUrl = "https://go.microsoft.com/fwlink/?LinkID=809115"
-                                        buildCommands += "mkdir Tools\\dotnetcli-jitutils&&powershell -Command \"Invoke-WebRequest -Uri ${dotnetcliUrl} -OutFile Tools\\dotnetcli-jitutils\\dotnetcli-jitutils.zip; Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory(Tools\\dotnetcli-jitutils\\dotnetcli-jitutils.zip, Tools\\dotnetcli-jitutils)\""
-                                        
-                                        // Run bootstrap script
-                                        buildCommands += "set PATH=%PATH%;%WORKSPACE%\\Tools\\dotnetcli-jitutils&&bootstrap.cmd"
-                                        
-                                        // run jit-format
-                                        buildCommands += "set PATH=%PATH%;%WORKSPACE%\\Tools\\dotnetcli-jitutils;%WORKSPACE%\\jitutils\\bin&&jit-format.cmd -a ${arch} -b Checked -o Windows_NT -c %WORKSPACE% --verbose --projects dll"
-                                        buildCommands += "set PATH=%PATH%;%WORKSPACE%\\Tools\\dotnetcli-jitutils;%WORKSPACE%\\jitutils\\bin&&jit-format.cmd -a ${arch} -b Checked -o Windows_NT -c %WORKSPACE% --verbose --projects standalone"
-                                        buildCommands += "set PATH=%PATH%;%WORKSPACE%\\Tools\\dotnetcli-jitutils;%WORKSPACE%\\jitutils\\bin&&jit-format.cmd -a ${arch} -b Checked -o Windows_NT -c %WORKSPACE% --verbose --projects crossgen"
-                                        
-                                        buildCommands += "set PATH=%PATH%;%WORKSPACE%\\Tools\\dotnetcli-jitutils;%WORKSPACE%\\jitutils\\bin&&jit-format.cmd -a ${arch} -b Debug -o Windows_NT -c %WORKSPACE% --verbose --projects dll"
-                                        buildCommands += "set PATH=%PATH%;%WORKSPACE%\\Tools\\dotnetcli-jitutils;%WORKSPACE%\\jitutils\\bin&&jit-format.cmd -a ${arch} -b Debug -o Windows_NT -c %WORKSPACE% --verbose --projects standalone"
-                                        buildCommands += "set PATH=%PATH%;%WORKSPACE%\\Tools\\dotnetcli-jitutils;%WORKSPACE%\\jitutils\\bin&&jit-format.cmd -a ${arch} -b Debug -o Windows_NT -c %WORKSPACE% --verbose --projects crossgen"
-                                        
-                                        buildCommands += "set PATH=%PATH%;%WORKSPACE%\\Tools\\dotnetcli-jitutils;%WORKSPACE%\\jitutils\\bin&&jit-format.cmd -a ${arch} -b Release -o Windows_NT -c %WORKSPACE% --verbose --projects dll"
-                                        buildCommands += "set PATH=%PATH%;%WORKSPACE%\\Tools\\dotnetcli-jitutils;%WORKSPACE%\\jitutils\\bin&&jit-format.cmd -a ${arch} -b Release -o Windows_NT -c %WORKSPACE% --verbose --projects standalone"
-                                        buildCommands += "set PATH=%PATH%;%WORKSPACE%\\Tools\\dotnetcli-jitutils;%WORKSPACE%\\jitutils\\bin&&jit-format.cmd -a ${arch} -b Release -o Windows_NT -c %WORKSPACE% --verbose --projects crossgen"
+                                        buildCommands += "python tests\\scripts\\format.py -c %WORKSPACE% -o Windows_NT -a ${arch}"
                                         break
                                     }
                                     else {
@@ -1583,7 +1557,7 @@ combinedScenarios.each { scenario ->
                                     // If we are running a stress mode, we should write out the set of key
                                     // value env pairs to a file at this point and then we'll pass that to runtest.cmd
 
-                                    if (!isBuildOnly && scenario != 'formatting') {
+                                    if (!isBuildOnly) {
                                         //If this is a crossgen build, pass 'crossgen' to runtest.cmd
                                         def crossgenStr = ''
                                         def runcrossgentestsStr = ''
@@ -1765,36 +1739,7 @@ combinedScenarios.each { scenario ->
                                     }
 
                                     if (scenario == 'formatting') {
-                                        // configure all of the configurations
-                                        buildCommands += "./build.sh verbose configureonly release x64"
-                                        buildCommands += "./build.sh verbose configureonly checked x64"
-                                        buildCommands += "./build.sh verbose configureonly debug x64"
-
-                                        // install correct version of dotnet cli
-                                        def dotnetCliUrl = "https://go.microsoft.com/fwlink/?LinkID=809118"
-                                        def dotnetCliTar = "dotnetcli.tar"
-                                        def dotnetCliPath = "\${WORKSPACE}/Tools/dotnetcli-jitutils"
-                                        buildCommands += "wget --trust-server-names ${dotnetCliUrl} -O ${dotnetCliTar}.gz; gunzip ${dotnetCliTar}.gz; mkdir ${dotnetCliPath}; tar -xf ${dotnetCliTar} -C ${dotnetCliPath}"
-                                        
-                                        // get and run bootstrap script
-                                        def url = "https://raw.githubusercontent.com/dotnet/jitutils/master/bootstrap.sh"
-                                        buildCommands += "wget ${url}"
-                                        buildCommands += "chmod 751 bootstrap.sh"
-                                        buildCommands += "export PATH=\${PATH}:${dotnetCliPath}; ./bootstrap.sh"
-                                        def jitutilsPath = "\${WORKSPACE}/jitutils/bin"
-
-                                        // run jit-format
-                                        buildCommands += "export PATH=\${PATH}:${dotnetCliPath}:${jitutilsPath}&&jit-format -a ${arch} -b Checked -o Linux -c \${WORKSPACE} --verbose --projects dll"
-                                        buildCommands += "export PATH=\${PATH}:${dotnetCliPath}:${jitutilsPath}&&jit-format -a ${arch} -b Checked -o Linux -c \${WORKSPACE} --verbose --projects standalone"
-                                        buildCommands += "export PATH=\${PATH}:${dotnetCliPath}:${jitutilsPath}&&jit-format -a ${arch} -b Checked -o Linux -c \${WORKSPACE} --verbose --projects crossgen"
-
-                                        buildCommands += "export PATH=\${PATH}:${dotnetCliPath}:${jitutilsPath}&&jit-format -a ${arch} -b Debug -o Linux -c \${WORKSPACE} --verbose --projects dll"
-                                        buildCommands += "export PATH=\${PATH}:${dotnetCliPath}:${jitutilsPath}&&jit-format -a ${arch} -b Debug -o Linux -c \${WORKSPACE} --verbose --projects standalone"
-                                        buildCommands += "export PATH=\${PATH}:${dotnetCliPath}:${jitutilsPath}&&jit-format -a ${arch} -b Debug -o Linux -c \${WORKSPACE} --verbose --projects crossgen"
-
-                                        buildCommands += "export PATH=\${PATH}:${dotnetCliPath}:${jitutilsPath}&&jit-format -a ${arch} -b Release -o Linux -c \${WORKSPACE} --verbose --projects dll"
-                                        buildCommands += "export PATH=\${PATH}:${dotnetCliPath}:${jitutilsPath}&&jit-format -a ${arch} -b Release -o Linux -c \${WORKSPACE} --verbose --projects standalone"
-                                        buildCommands += "export PATH=\${PATH}:${dotnetCliPath}:${jitutilsPath}&&jit-format -a ${arch} -b Release -o Linux -c \${WORKSPACE} --verbose --projects crossgen"
+                                        buildCommands += "python tests/scripts/format.py -c \${WORKSPACE} -o Linux -a ${arch}"
                                         break
                                     }
                                 
@@ -2065,14 +2010,7 @@ combinedScenarios.each { scenario ->
                                     return
                                 }
                             case 'formatting':
-                                // We only want Ubuntu Checked for formatting
-                                if (os != 'Ubuntu') {
-                                    return
-                                }
-                                if (configuration != 'Checked') {
-                                    return
-                                }
-                                break
+                                return
                             case 'default':
                                 // Nothing skipped
                                 break
diff --git a/tests/scripts/format.py b/tests/scripts/format.py
new file mode 100644
index 0000000000..e89aad7e1b
--- /dev/null
+++ b/tests/scripts/format.py
@@ -0,0 +1,149 @@
+#!/usr/bin/env python
+#
+## Licensed to the .NET Foundation under one or more agreements.
+## The .NET Foundation licenses this file to you under the MIT license.
+## See the LICENSE file in the project root for more information.
+#
+##
+# Title               :format.py
+#
+################################################################################
+# Script to install and run jit-format over jit source for all configurations.
+################################################################################
+
+
+import urllib
+import argparse
+import os
+import sys
+import tarfile
+import zipfile
+import subprocess
+import urllib2
+import shutil
+
+def expandPath(path):
+    return os.path.abspath(os.path.expanduser(path))
+
+def main(argv):
+    parser = argparse.ArgumentParser()
+    required = parser.add_argument_group('required arguments')
+    required.add_argument('-a', '--arch', type=str, 
+            default=None, help='architecture to run jit-format on')
+    required.add_argument('-o', '--os', type=str,
+            default=None, help='operating system')
+    required.add_argument('-c', '--coreclr', type=str,
+            default=None, help='full path to coreclr')
+
+    args, unknown = parser.parse_known_args(argv)
+
+    if unknown:
+        print('Ignorning argument(s): ', ','.join(unknown))
+
+    if args.coreclr is None:
+        print('Specify --coreclr')
+        return -1
+    if args.os is None:
+        print('Specifiy --os')
+        return -1
+    if args.arch is None:
+        print('Specify --arch')
+        return -1
+
+    if not os.path.isdir(expandPath(args.coreclr)):
+        print('Bad path to coreclr')
+        return -1
+
+    coreclr = args.coreclr
+    platform = args.os
+    arch = args.arch
+
+    # Download dotnetcli
+    dotnetcliUrl = ""
+    dotnetcliFilename = ""
+    dotnetcliPath = os.path.join(coreclr, 'Tools', 'dotnetcli-jitutils')
+
+    # Try to make the dotnetcli-jitutils directory if it doesn't exist
+    try: 
+        os.makedirs(dotnetcliPath)
+    except OSError:
+        if not os.path.isdir(dotnetcliPath):
+            raise
+
+    if platform == 'Linux' or platform == 'OSX':
+        dotnetcliUrl = "https://go.microsoft.com/fwlink/?LinkID=809118"
+        dotnetcliFilename = os.path.join(dotnetcliPath, 'dotnetcli-jitutils.tar.gz')
+    elif platform == 'Windows_NT':
+        dotnetcliUrl = "https://go.microsoft.com/fwlink/?LinkID=809115"
+        dotnetcliFilename = os.path.join(dotnetcliPath, 'dotnetcli-jitutils.zip')
+    else:
+        print('Unknown os ', os)
+        return -1
+
+    response = urllib2.urlopen(dotnetcliUrl)
+    request_url = response.geturl()
+    print(request_url)
+    testfile = urllib.URLopener()
+    testfile.retrieve(request_url, dotnetcliFilename)
+
+    # Install dotnetcli
+
+    if platform == 'Linux' or platform == 'OSX':
+        tar = tarfile.open(dotnetcliFilename)
+        tar.extractall(dotnetcliPath)
+        tar.close()
+    elif platform == 'Windows_NT':
+        with zipfile.ZipFile(dotnetcliFilename, "r") as z:
+            z.extractall(dotnetcliPath)
+
+    # Download bootstrap
+    bootstrapFilename = ""
+
+    jitUtilsPath = os.path.join(coreclr, "jitutils")
+
+    if os.path.isdir(jitUtilsPath):
+        shutil.rmtree(dest, ignore_errors=True)
+
+    if platform == 'Linux' or platform == 'OSX':
+        bootstrapFilename = "bootstrap.sh"
+    elif platform == 'Windows_NT':
+        bootstrapFilename = "bootstrap.cmd"
+
+    bootstrapUrl = "https://raw.githubusercontent.com/dotnet/jitutils/master/" + bootstrapFilename
+
+    testfile.retrieve(bootstrapUrl, bootstrapFilename)
+
+    # On Linux platforms, we need to make the bootstrap file executable
+    if platform == 'Linux' or platform == 'OSX':
+        os.chmod(bootstrapFilename, 0751)
+
+    # Run bootstrap
+    os.environ["PATH"] += os.pathsep + dotnetcliPath
+    proc = subprocess.Popen([os.path.join(coreclr, bootstrapFilename)], shell=True)
+
+    output,error = proc.communicate()
+    print(output)
+    print(error)
+
+    # Run jit-format
+    returncode = 0
+    os.environ["PATH"] += os.pathsep + os.path.join(coreclr, "jitutils", "bin")
+
+    for build in ["Checked", "Debug", "Release"]:
+        for project in ["dll", "standalone", "crossgen"]:
+            proc = subprocess.Popen(["jit-format", "-a", arch, "-b", build, "-o", platform,
+                "-c", coreclr, "--verbose", "--projects", project], shell=True)
+            output,error = proc.communicate()
+            errorcode = proc.returncode
+
+            print(output)
+            print(error)
+
+            if errorcode != 0:
+                returncode = errorcode
+
+    return returncode
+
+if __name__ == '__main__':
+    return_code = main(sys.argv[1:])
+    sys.exit(return_code)