Port Windows PGO support to master (#9985)

The bulk of this PR is a cherry-pick of commit fa02660 that shipped in
release/1.1.0, updating the build system support for PGO to support
consuming PGDs properly during release builds on Windows.

Also included are the following new changes:
* Skip restore of opdata if the requisite project.json is missing
* If the optdata package restore fails, fail the build.
* Add new build option: 'skiprestoreoptdata'

Note: This change doesn't by itself enable PGO in master yet, because
training data (optdata packages) for master don't exist on myget yet.
However, with these changes, the only step remaining to enable PGO
optimizations is to add a project.json referencing the correct optdata
package.

6 files changed, 145 insertions, 5 deletions

diff --git a/build.cmd b/build.cmd
index 1e69da4337..685e84cfab 100644
--- a/build.cmd
+++ b/build.cmd
@@ -40,6 +40,7 @@ set "__SourceDir=%__ProjectDir%\src"
 set "__PackagesDir=%__ProjectDir%\packages"
 set "__RootBinDir=%__ProjectDir%\bin"
 set "__LogsDir=%__RootBinDir%\Logs"
+set "__OptDataVersion="
 
 set __BuildAll=
 
@@ -71,6 +72,7 @@ set __BuildNative=1
 set __BuildTests=1
 set __BuildPackages=1
 set __BuildNativeCoreLib=1
+set __RestoreOptData=1
 
 :Arg_Loop
 if "%1" == "" goto ArgsDone
@@ -105,6 +107,7 @@ if /i "%1" == "skipmscorlib"        (set __BuildCoreLib=0&set __BuildNativeCoreL
 if /i "%1" == "skipnative"          (set __BuildNative=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "skiptests"           (set __BuildTests=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "skipbuildpackages"   (set __BuildPackages=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
+if /i "%1" == "skiprestoreoptdata"  (set __RestoreOptData=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "usenmakemakefiles"   (set __NMakeMakefiles=1&set __ConfigureOnly=1&set __BuildNative=1&set __BuildNativeCoreLib=0&set __BuildCoreLib=0&set __BuildTests=0&set __BuildPackages=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "buildjit32"          (set __BuildJit32="-DBUILD_JIT32=1"&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "pgoinstrument"       (set __PgoInstrument=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
@@ -204,6 +207,33 @@ call                                 "%__VSToolsRoot%\VsDevCmd.bat"
 
 REM =========================================================================================
 REM ===
+REM === Restore optimization profile data
+REM ===
+REM =========================================================================================
+
+REM Parse the package version out of project.json so that we can pass it on to CMake
+where /q python || (
+    echo %__MsgPrefix%Error: Python not found on PATH, please make sure that it is installed.
+    exit /b 1
+)
+set OptDataProjectJsonPath=%__ProjectDir%\src\.nuget\optdata\project.json
+if EXIST "%OptDataProjectJsonPath%" (
+    for /f "tokens=*" %%s in ('python "%__ProjectDir%\extract-from-json.py" -rf "%OptDataProjectJsonPath%" dependencies optimization.PGO.CoreCLR') do @(
+        set __OptDataVersion=%%s
+    )
+)
+
+if %__RestoreOptData% EQU 1 (
+    echo %__MsgPrefix%Restoring the OptimizationData Package
+    @call %__ProjectDir%\run.cmd sync -optdata
+    if not !errorlevel! == 0 (
+        echo %__MsgPrefix%Error: Failed to restore the optimization data package.
+        exit /b 1
+    )
+)
+
+REM =========================================================================================
+REM ===
 REM === Build the CLR VM
 REM ===
 REM =========================================================================================
@@ -251,7 +281,7 @@ if %__BuildNative% EQU 1 (
     echo %__MsgPrefix%Regenerating the Visual Studio solution
 
     pushd "%__IntermediatesDir%"
-    set __ExtraCmakeArgs=!___SDKVersion! "-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%"
+    set __ExtraCmakeArgs=!___SDKVersion! "-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%" "-DCLR_CMAKE_OPTDATA_VERSION=%__OptDataVersion%"
     call "%__SourceDir%\pal\tools\gen-buildsys-win.bat" "%__ProjectDir%" %__VSVersion% %__BuildArch% %__BuildJit32% %__BuildStandaloneGC% !__ExtraCmakeArgs!
 	@if defined _echo @echo on
     popd
@@ -305,7 +335,7 @@ if /i "%__DoCrossArchBuild%"=="1" (
     pushd "%__CrossCompIntermediatesDir%"
     set __CMakeBinDir=%__CrossComponentBinDir%
     set "__CMakeBinDir=!__CMakeBinDir:\=/!"
-    set __ExtraCmakeArgs="-DCLR_CROSS_COMPONENTS_BUILD=1" "-DCLR_CMAKE_TARGET_ARCH=%__BuildArch%" "-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%"
+    set __ExtraCmakeArgs="-DCLR_CROSS_COMPONENTS_BUILD=1" "-DCLR_CMAKE_TARGET_ARCH=%__BuildArch%" "-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%" "-DCLR_CMAKE_OPTDATA_VERSION=%__OptDataVersion%"
     call "%__SourceDir%\pal\tools\gen-buildsys-win.bat" "%__ProjectDir%" %__VSVersion% %__CrossArch% !__ExtraCmakeArgs!
     @if defined _echo @echo on
     popd
@@ -593,6 +623,7 @@ echo skipmscorlib: skip building System.Private.CoreLib ^(default: System.Privat
 echo skipnative: skip building native components ^(default: native components are built^).
 echo skiptests: skip building tests ^(default: tests are built^).
 echo skipbuildpackages: skip building nuget packages ^(default: packages are built^).
+echo skiprestoreoptdata: skip restoring optimization data used by profile-based optimizations.
 echo buildstandalonegc: builds the GC in a standalone mode.
 echo -skiprestore: skip restoring packages ^(default: packages are restored during build^).
 echo -disableoss: Disable Open Source Signing for System.Private.CoreLib.
diff --git a/build.proj b/build.proj
index 7df2904e36..9992bdc9c6 100644
--- a/build.proj
+++ b/build.proj
@@ -24,6 +24,14 @@
     <Delete Files="$(BinDir)System.Private.CoreLib.*" />
   </Target>
 
+  <PropertyGroup>
+    <OptDataProjectJson>$(SourceDir).nuget/optdata/project.json</OptDataProjectJson>
+    <OptDataPackageFeed>https://dotnet.myget.org/F/dotnet-core-optimization-data/api/v3/index.json</OptDataPackageFeed>
+  </PropertyGroup>
+  <Target Name="RestoreOptData">
+    <Exec Condition="Exists('$(OptDataProjectJson)')" Command="$(DnuRestoreCommand) &quot;$(OptDataProjectJson)&quot; --source &quot;$(OptDataPackageFeed)&quot;" />
+  </Target>
+
   <Target Name="RestoreNETCorePlatforms" AfterTargets="Build" Condition="'$(RestoreDuringBuild)'=='true'">
     <Exec Command="$(DnuRestoreCommand) &quot;$(SourceDir).nuget/init/project.json&quot; --source https://dotnet.myget.org/F/dotnet-core/api/v3/index.json" />
   </Target>
diff --git a/build.sh b/build.sh
index 778ad63346..b0dcc3c937 100755
--- a/build.sh
+++ b/build.sh
@@ -36,6 +36,7 @@ usage()
     echo "skipmscorlib - do not build mscorlib.dll."
     echo "skiptests - skip the tests in the 'tests' subdirectory."
     echo "skipnuget - skip building nuget packages."
+    echo "skiprestoreoptdata - skip restoring optimization data used by profile-based optimizations."
     echo "portableLinux - build for Portable Linux Distribution"
     echo "verbose - optional argument to enable verbose build output."
     echo "-skiprestore: skip restoring packages ^(default: packages are restored during build^)."
@@ -117,6 +118,19 @@ check_prereqs()
 
 }
 
+restore_optdata()
+{
+    # if msbuild is not supported, then set __SkipRestoreOptData to 1
+    if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipRestoreOptData=1; fi
+    if [ $__SkipRestoreOptData == 0 ]; then
+        echo "Restoring the OptimizationData package"
+        "$__ProjectRoot/run.sh" sync -optdata
+        if [ $? != 0 ]; then
+            echo "Failed to restore the optimization data package."
+            exit 1
+        fi
+    fi
+}
 
 generate_event_logging_sources()
 {
@@ -215,6 +229,7 @@ build_native()
             fi
         fi
 
+
         pushd "$intermediatesForBuild"
         # Regenerate the CMake solution
         echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion $__ClangMinorVersion $platformArch $__BuildType $__CodeCoverage $__IncludeTests $generator $extraCmakeArguments $__cmakeargs"
@@ -289,7 +304,7 @@ build_cross_arch_component()
         fi
     fi
 
-    __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument"
+    __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__OptDataVersion"
     build_native $__SkipCrossArchBuild "$__CrossArch" "$__CrossCompIntermediatesDir" "$__ExtraCmakeArgs" "cross-architecture component"
    
     # restore ROOTFS_DIR, CROSSCOMPONENT, and CROSSCOMPILE 
@@ -548,6 +563,7 @@ __SkipRestore=""
 __SkipNuget=0
 __SkipCoreCLR=0
 __SkipMSCorLib=0
+__SkipRestoreOptData=0
 __CrossBuild=0
 __ClangMajorVersion=0
 __ClangMinorVersion=0
@@ -559,6 +575,7 @@ __SkipGenerateVersion=0
 __DoCrossArchBuild=0
 __PortableLinux=0
 __msbuildonunsupportedplatform=0
+__OptDataVersion=""
 
 while :; do
     if [ $# -le 0 ]; then
@@ -694,6 +711,10 @@ while :; do
             __SkipGenerateVersion=1
             ;;
 
+        skiprestoreoptdata)
+            __SkipRestoreOptData=1
+            ;;
+
         includetests)
             ;;
 
@@ -815,6 +836,12 @@ if [ $__CrossBuild == 1 ]; then
     fi
 fi
 
+# Parse the optdata package version from its project.json file
+optDataProjectJsonPath="$__ProjectRoot/src/.nuget/optdata/project.json"
+if [ -f $optDataProjectJsonPath ]; then
+    __OptDataVersion=$("$__ProjectRoot/extract-from-json.py" -rf $optDataProjectJsonPath dependencies optimization.PGO.CoreCLR)
+fi
+
 # init the target distro name
 initTargetDistroRid
 
@@ -824,11 +851,14 @@ setup_dirs
 # Check prereqs.
 check_prereqs
 
+# Restore the package containing profile counts for profile-guided optimizations
+restore_optdata
+
 # Generate event logging infrastructure sources
 generate_event_logging_sources
 
 # Build the coreclr (native) components.
-__ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument"
+__ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__OptDataVersion"
 build_native $__SkipCoreCLR "$__BuildArch" "$__IntermediatesDir" "$__ExtraCmakeArgs" "CoreCLR component"
 
 # Build cross-architecture components
diff --git a/config.json b/config.json
index 2d25b6a013..140b1eb147 100644
--- a/config.json
+++ b/config.json
@@ -54,6 +54,12 @@
       "values": [],
       "defaultValue": ""
     },
+    "RestoreOptData": {
+      "description": "MsBuild target that restores optimization profile data.",
+      "valueType": "target",
+      "values": [],
+      "defaultValue": ""
+    },
     "RestoreDuringBuild": {
       "description": "Enables/disables package restore.",
       "valueType": "property",
@@ -435,6 +441,14 @@
             "RestoreNETCorePlatforms": "default"
           }
         },
+        "optdata": {
+          "description": "Restores optimization profile data for the repository.",
+          "settings": {
+            "Project": "./build.proj",
+            "RestoreDuringBuild": true,
+            "RestoreOptData": "default"
+          }
+        },
         "ab": {
           "description": "Downloads the latests product packages from Azure. The values for '-AzureAccount' and '-AzureToken' are required",
           "settings": {
diff --git a/extract-from-json.py b/extract-from-json.py
new file mode 100755
index 0000000000..e432b2b067
--- /dev/null
+++ b/extract-from-json.py
@@ -0,0 +1,56 @@
+#!/usr/bin/python
+
+import argparse
+import json
+import sys
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="""Extracts information from a json file by navigating the JSON object using a
+            sequence of property accessors and returning the JSON subtree, or the raw data, found
+            at that location."""
+    )
+
+    parser.add_argument(
+        '-f', '--file',
+        metavar='<project.json>',
+        help="Path to project.json file to parse",
+        required=True,
+    )
+
+    parser.add_argument(
+        'property',
+        metavar='property_name',
+        help="""Name of property to extract using object notation.
+            Pass multiple values to drill down into nested objects (in order).""",
+        nargs='*',
+    )
+
+    parser.add_argument(
+        '-r', '--raw',
+        help="""Dumps the raw object found at the requested location.
+            If omitted, returns a JSON formatted object instead.""",
+        action='store_true',
+        default=False
+    )
+
+    return parser.parse_args()
+
+def main():
+    args = parse_args()
+
+    with open(args.file) as json_file:
+        selected_property = json.load(json_file)
+
+    for prop in args.property:
+        selected_property = selected_property[prop]
+
+    if args.raw:
+        print(selected_property)
+    else:
+        print(json.dumps(selected_property))
+
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/pgosupport.cmake b/pgosupport.cmake
index 015eed6a0f..e4e6bd5091 100644
--- a/pgosupport.cmake
+++ b/pgosupport.cmake
@@ -12,8 +12,9 @@ function(add_pgo TargetName)
         set(ProfileFileName "${TargetName}.pgd")
     endif(WIN32)
 
+    set(CLR_CMAKE_OPTDATA_PACKAGEWITHRID "optimization.${CLR_CMAKE_TARGET_OS}-${CLR_CMAKE_TARGET_ARCH}.PGO.CoreCLR")
     file(TO_NATIVE_PATH
-        "${CLR_CMAKE_PACKAGES_DIR}/Microsoft.DotNet.OptimizationData.Coreclr/${CLR_CMAKE_TARGET_OS}.${CLR_CMAKE_TARGET_ARCH}/${ProfileFileName}"
+        "${CLR_CMAKE_PACKAGES_DIR}/${CLR_CMAKE_OPTDATA_PACKAGEWITHRID}/${CLR_CMAKE_OPTDATA_VERSION}/data/${ProfileFileName}"
         ProfilePath
     )