diff options
| -rw-r--r-- | build.cmd | 35 | ||||
| -rw-r--r-- | build.proj | 8 | ||||
| -rwxr-xr-x | build.sh | 34 | ||||
| -rw-r--r-- | config.json | 14 | ||||
| -rwxr-xr-x | extract-from-json.py | 56 | ||||
| -rw-r--r-- | pgosupport.cmake | 3 |
6 files changed, 145 insertions, 5 deletions
@@ -40,6 +40,7 @@ set "__SourceDir=%__ProjectDir%\src" set "__PackagesDir=%__ProjectDir%\packages" set "__RootBinDir=%__ProjectDir%\bin" set "__LogsDir=%__RootBinDir%\Logs" +set "__OptDataVersion=" set __BuildAll= @@ -71,6 +72,7 @@ set __BuildNative=1 set __BuildTests=1 set __BuildPackages=1 set __BuildNativeCoreLib=1 +set __RestoreOptData=1 :Arg_Loop if "%1" == "" goto ArgsDone @@ -105,6 +107,7 @@ if /i "%1" == "skipmscorlib" (set __BuildCoreLib=0&set __BuildNativeCoreL if /i "%1" == "skipnative" (set __BuildNative=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "skiptests" (set __BuildTests=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "skipbuildpackages" (set __BuildPackages=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "skiprestoreoptdata" (set __RestoreOptData=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "usenmakemakefiles" (set __NMakeMakefiles=1&set __ConfigureOnly=1&set __BuildNative=1&set __BuildNativeCoreLib=0&set __BuildCoreLib=0&set __BuildTests=0&set __BuildPackages=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "buildjit32" (set __BuildJit32="-DBUILD_JIT32=1"&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "pgoinstrument" (set __PgoInstrument=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) @@ -204,6 +207,33 @@ call "%__VSToolsRoot%\VsDevCmd.bat" REM ========================================================================================= REM === +REM === Restore optimization profile data +REM === +REM ========================================================================================= + +REM Parse the package version out of project.json so that we can pass it on to CMake +where /q python || ( + echo %__MsgPrefix%Error: Python not found on PATH, please make sure that it is installed. + exit /b 1 +) +set OptDataProjectJsonPath=%__ProjectDir%\src\.nuget\optdata\project.json +if EXIST "%OptDataProjectJsonPath%" ( + for /f "tokens=*" %%s in ('python "%__ProjectDir%\extract-from-json.py" -rf "%OptDataProjectJsonPath%" dependencies optimization.PGO.CoreCLR') do @( + set __OptDataVersion=%%s + ) +) + +if %__RestoreOptData% EQU 1 ( + echo %__MsgPrefix%Restoring the OptimizationData Package + @call %__ProjectDir%\run.cmd sync -optdata + if not !errorlevel! == 0 ( + echo %__MsgPrefix%Error: Failed to restore the optimization data package. + exit /b 1 + ) +) + +REM ========================================================================================= +REM === REM === Build the CLR VM REM === REM ========================================================================================= @@ -251,7 +281,7 @@ if %__BuildNative% EQU 1 ( echo %__MsgPrefix%Regenerating the Visual Studio solution pushd "%__IntermediatesDir%" - set __ExtraCmakeArgs=!___SDKVersion! "-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%" + set __ExtraCmakeArgs=!___SDKVersion! "-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%" "-DCLR_CMAKE_OPTDATA_VERSION=%__OptDataVersion%" call "%__SourceDir%\pal\tools\gen-buildsys-win.bat" "%__ProjectDir%" %__VSVersion% %__BuildArch% %__BuildJit32% %__BuildStandaloneGC% !__ExtraCmakeArgs! @if defined _echo @echo on popd @@ -305,7 +335,7 @@ if /i "%__DoCrossArchBuild%"=="1" ( pushd "%__CrossCompIntermediatesDir%" set __CMakeBinDir=%__CrossComponentBinDir% set "__CMakeBinDir=!__CMakeBinDir:\=/!" - set __ExtraCmakeArgs="-DCLR_CROSS_COMPONENTS_BUILD=1" "-DCLR_CMAKE_TARGET_ARCH=%__BuildArch%" "-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%" + set __ExtraCmakeArgs="-DCLR_CROSS_COMPONENTS_BUILD=1" "-DCLR_CMAKE_TARGET_ARCH=%__BuildArch%" "-DCLR_CMAKE_TARGET_OS=%__BuildOs%" "-DCLR_CMAKE_PACKAGES_DIR=%__PackagesDir%" "-DCLR_CMAKE_PGO_INSTRUMENT=%__PgoInstrument%" "-DCLR_CMAKE_OPTDATA_VERSION=%__OptDataVersion%" call "%__SourceDir%\pal\tools\gen-buildsys-win.bat" "%__ProjectDir%" %__VSVersion% %__CrossArch% !__ExtraCmakeArgs! @if defined _echo @echo on popd @@ -593,6 +623,7 @@ echo skipmscorlib: skip building System.Private.CoreLib ^(default: System.Privat echo skipnative: skip building native components ^(default: native components are built^). echo skiptests: skip building tests ^(default: tests are built^). echo skipbuildpackages: skip building nuget packages ^(default: packages are built^). +echo skiprestoreoptdata: skip restoring optimization data used by profile-based optimizations. echo buildstandalonegc: builds the GC in a standalone mode. echo -skiprestore: skip restoring packages ^(default: packages are restored during build^). echo -disableoss: Disable Open Source Signing for System.Private.CoreLib. diff --git a/build.proj b/build.proj index 7df2904e36..9992bdc9c6 100644 --- a/build.proj +++ b/build.proj @@ -24,6 +24,14 @@ <Delete Files="$(BinDir)System.Private.CoreLib.*" /> </Target> + <PropertyGroup> + <OptDataProjectJson>$(SourceDir).nuget/optdata/project.json</OptDataProjectJson> + <OptDataPackageFeed>https://dotnet.myget.org/F/dotnet-core-optimization-data/api/v3/index.json</OptDataPackageFeed> + </PropertyGroup> + <Target Name="RestoreOptData"> + <Exec Condition="Exists('$(OptDataProjectJson)')" Command="$(DnuRestoreCommand) "$(OptDataProjectJson)" --source "$(OptDataPackageFeed)"" /> + </Target> + <Target Name="RestoreNETCorePlatforms" AfterTargets="Build" Condition="'$(RestoreDuringBuild)'=='true'"> <Exec Command="$(DnuRestoreCommand) "$(SourceDir).nuget/init/project.json" --source https://dotnet.myget.org/F/dotnet-core/api/v3/index.json" /> </Target> @@ -36,6 +36,7 @@ usage() echo "skipmscorlib - do not build mscorlib.dll." echo "skiptests - skip the tests in the 'tests' subdirectory." echo "skipnuget - skip building nuget packages." + echo "skiprestoreoptdata - skip restoring optimization data used by profile-based optimizations." echo "portableLinux - build for Portable Linux Distribution" echo "verbose - optional argument to enable verbose build output." echo "-skiprestore: skip restoring packages ^(default: packages are restored during build^)." @@ -117,6 +118,19 @@ check_prereqs() } +restore_optdata() +{ + # if msbuild is not supported, then set __SkipRestoreOptData to 1 + if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipRestoreOptData=1; fi + if [ $__SkipRestoreOptData == 0 ]; then + echo "Restoring the OptimizationData package" + "$__ProjectRoot/run.sh" sync -optdata + if [ $? != 0 ]; then + echo "Failed to restore the optimization data package." + exit 1 + fi + fi +} generate_event_logging_sources() { @@ -215,6 +229,7 @@ build_native() fi fi + pushd "$intermediatesForBuild" # Regenerate the CMake solution echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion $__ClangMinorVersion $platformArch $__BuildType $__CodeCoverage $__IncludeTests $generator $extraCmakeArguments $__cmakeargs" @@ -289,7 +304,7 @@ build_cross_arch_component() fi fi - __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument" + __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__OptDataVersion" build_native $__SkipCrossArchBuild "$__CrossArch" "$__CrossCompIntermediatesDir" "$__ExtraCmakeArgs" "cross-architecture component" # restore ROOTFS_DIR, CROSSCOMPONENT, and CROSSCOMPILE @@ -548,6 +563,7 @@ __SkipRestore="" __SkipNuget=0 __SkipCoreCLR=0 __SkipMSCorLib=0 +__SkipRestoreOptData=0 __CrossBuild=0 __ClangMajorVersion=0 __ClangMinorVersion=0 @@ -559,6 +575,7 @@ __SkipGenerateVersion=0 __DoCrossArchBuild=0 __PortableLinux=0 __msbuildonunsupportedplatform=0 +__OptDataVersion="" while :; do if [ $# -le 0 ]; then @@ -694,6 +711,10 @@ while :; do __SkipGenerateVersion=1 ;; + skiprestoreoptdata) + __SkipRestoreOptData=1 + ;; + includetests) ;; @@ -815,6 +836,12 @@ if [ $__CrossBuild == 1 ]; then fi fi +# Parse the optdata package version from its project.json file +optDataProjectJsonPath="$__ProjectRoot/src/.nuget/optdata/project.json" +if [ -f $optDataProjectJsonPath ]; then + __OptDataVersion=$("$__ProjectRoot/extract-from-json.py" -rf $optDataProjectJsonPath dependencies optimization.PGO.CoreCLR) +fi + # init the target distro name initTargetDistroRid @@ -824,11 +851,14 @@ setup_dirs # Check prereqs. check_prereqs +# Restore the package containing profile counts for profile-guided optimizations +restore_optdata + # Generate event logging infrastructure sources generate_event_logging_sources # Build the coreclr (native) components. -__ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument" +__ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__OptDataVersion" build_native $__SkipCoreCLR "$__BuildArch" "$__IntermediatesDir" "$__ExtraCmakeArgs" "CoreCLR component" # Build cross-architecture components diff --git a/config.json b/config.json index 2d25b6a013..140b1eb147 100644 --- a/config.json +++ b/config.json @@ -54,6 +54,12 @@ "values": [], "defaultValue": "" }, + "RestoreOptData": { + "description": "MsBuild target that restores optimization profile data.", + "valueType": "target", + "values": [], + "defaultValue": "" + }, "RestoreDuringBuild": { "description": "Enables/disables package restore.", "valueType": "property", @@ -435,6 +441,14 @@ "RestoreNETCorePlatforms": "default" } }, + "optdata": { + "description": "Restores optimization profile data for the repository.", + "settings": { + "Project": "./build.proj", + "RestoreDuringBuild": true, + "RestoreOptData": "default" + } + }, "ab": { "description": "Downloads the latests product packages from Azure. The values for '-AzureAccount' and '-AzureToken' are required", "settings": { diff --git a/extract-from-json.py b/extract-from-json.py new file mode 100755 index 0000000000..e432b2b067 --- /dev/null +++ b/extract-from-json.py @@ -0,0 +1,56 @@ +#!/usr/bin/python + +import argparse +import json +import sys + +def parse_args(): + parser = argparse.ArgumentParser( + description="""Extracts information from a json file by navigating the JSON object using a + sequence of property accessors and returning the JSON subtree, or the raw data, found + at that location.""" + ) + + parser.add_argument( + '-f', '--file', + metavar='<project.json>', + help="Path to project.json file to parse", + required=True, + ) + + parser.add_argument( + 'property', + metavar='property_name', + help="""Name of property to extract using object notation. + Pass multiple values to drill down into nested objects (in order).""", + nargs='*', + ) + + parser.add_argument( + '-r', '--raw', + help="""Dumps the raw object found at the requested location. + If omitted, returns a JSON formatted object instead.""", + action='store_true', + default=False + ) + + return parser.parse_args() + +def main(): + args = parse_args() + + with open(args.file) as json_file: + selected_property = json.load(json_file) + + for prop in args.property: + selected_property = selected_property[prop] + + if args.raw: + print(selected_property) + else: + print(json.dumps(selected_property)) + + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pgosupport.cmake b/pgosupport.cmake index 015eed6a0f..e4e6bd5091 100644 --- a/pgosupport.cmake +++ b/pgosupport.cmake @@ -12,8 +12,9 @@ function(add_pgo TargetName) set(ProfileFileName "${TargetName}.pgd") endif(WIN32) + set(CLR_CMAKE_OPTDATA_PACKAGEWITHRID "optimization.${CLR_CMAKE_TARGET_OS}-${CLR_CMAKE_TARGET_ARCH}.PGO.CoreCLR") file(TO_NATIVE_PATH - "${CLR_CMAKE_PACKAGES_DIR}/Microsoft.DotNet.OptimizationData.Coreclr/${CLR_CMAKE_TARGET_OS}.${CLR_CMAKE_TARGET_ARCH}/${ProfileFileName}" + "${CLR_CMAKE_PACKAGES_DIR}/${CLR_CMAKE_OPTDATA_PACKAGEWITHRID}/${CLR_CMAKE_OPTDATA_VERSION}/data/${ProfileFileName}" ProfilePath ) |