diff options
author | Michelle McDaniel <adiaaida@gmail.com> | 2018-08-20 15:21:58 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-08-20 15:21:58 -0700 |
commit | e30616f7810838f94170786d9477df2423f4fa09 (patch) | |
tree | 607fe1a181096cda83e861acfeec375cd7d8c701 | |
parent | 895bfa452da4aeeda52447e76d9e9e987d68f331 (diff) | |
download | coreclr-e30616f7810838f94170786d9477df2423f4fa09.tar.gz coreclr-e30616f7810838f94170786d9477df2423f4fa09.tar.bz2 coreclr-e30616f7810838f94170786d9477df2423f4fa09.zip |
Add IBC support (#19046)
This change adds support to consume IBC data packages that are created by the dotnet/optimization repository. With these changes and dotnet/buildtools#2103 to enable IBC optimizations for corefx, we will see a 32% decrease in the size of NetCoreApp (and an overall decrease of 16%), a 30% reduction in ref set, a 5% improvement in time to first request and steady state performance for MusicStore.
Size
Crossgen Partial Partial vs Crossgen (lower is better)
Shared (MB) 185.6 137.3 0.74
Microsoft.NETCore.App (MB) 115.6 67.4 0.58
RefSet
MusicStore
Crossgen Partial Partial vs Crossgen (lower is better)
Total 19.389 13.472 0.69
AllReady
Crossgen Partial Partial vs Crossgen (lower is better)
Total 17.58 12.214
Performance
MusicStore
Crossgen Partial Partial vs Crossgen (lower is better)
Server Start (ms) 870 870.6 1.00
First request (ms) 3532.6 3386.6 0.95
Steady State (ms) 2.926 2.79 0.95
AllReady
Crossgen Partial Partial vs Crossgen (lower is better)
Server Start (ms) 2102 1942.4
First Request (ms) 4263.2 4126
Steady State (ms) 5.69 5.68
TechEmpower Plaintext
Crossgen Partial Partial vs Crossgen (lower is better)
Requests per Second 1928649.8 1893183.8
First Request (ms) 76.14 80.11
Startup (ms) 391 372.8
Working Set (MB) 382.2 373.4
-rw-r--r-- | build.cmd | 88 | ||||
-rwxr-xr-x | build.sh | 2 | ||||
-rw-r--r-- | buildpipeline/DotNet-CoreClr-Trusted-Windows.json | 22 | ||||
-rw-r--r-- | dependencies.props | 12 | ||||
-rw-r--r-- | src/.nuget/optdata/ibcmerge.csproj | 19 |
5 files changed, 136 insertions, 7 deletions
@@ -74,6 +74,7 @@ set __PgoInstrument=0 set __PgoOptimize=1 set __EnforcePgo=0 set __IbcTuning= +set __IbcOptimize=0 REM __PassThroughArgs is a set of things that will be passed through to nested calls to build.cmd REM when using "all". @@ -100,6 +101,7 @@ set __CrossArch= set __SkipNugetPackage=0 set __PgoOptDataVersion= set __IbcOptDataVersion= +set __IbcMergeVersion= @REM CMD has a nasty habit of eating "=" on the argument list, so passing: @REM -priority=1 @@ -178,6 +180,7 @@ if /i "%1" == "-usenmakemakefiles" (set __NMakeMakefiles=1&set __ConfigureOnly if /i "%1" == "-pgoinstrument" (set __PgoInstrument=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "-enforcepgo" (set __EnforcePgo=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "-nopgooptimize" (set __PgoOptimize=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "-ibcoptimize" (set __IbcOptimize=1&set __PartialNgen=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "-ibcinstrument" (set __IbcTuning=/Tuning&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "-crossgenaltjit" (set __CrossgenAltJit=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) @@ -580,10 +583,6 @@ if %__BuildCoreLib% EQU 1 ( set Platform= set __ExtraBuildArgs= - if not defined __IbcTuning ( - set __ExtraBuildArgs=!__ExtraBuildArgs! -OptimizationDataDir="%__PackagesDir%/optimization.%__BuildOS%-%__BuildArch%.IBC.CoreCLR/%__IbcOptDataVersion%/data/" - set __ExtraBuildArgs=!__ExtraBuildArgs! -EnableProfileGuidedOptimization=true - ) if "%__BuildSOS%" == "0" ( set __ExtraBuildArgs=!__ExtraBuildArgs! -SkipSOS=true @@ -676,6 +675,87 @@ if %__BuildNativeCoreLib% EQU 1 ( set COMPlus_ContinueOnAssert=0 ) + if %__IbcOptimize% EQU 1 ( + set IbcMergeProjectFilePath=%__ProjectDir%\src\.nuget\optdata\ibcmerge.csproj + for /f "tokens=*" %%s in ('%DotNetCli% msbuild "!IbcMergeProjectFilePath!" /t:DumpIbcMergePackageVersion /nologo') do @( + set __IbcMergeVersion=%%s + ) + + set IbcMergePath=%__PackagesDir%\microsoft.dotnet.ibcmerge\!__IbcMergeVersion!\lib\net45\ibcmerge.exe + if exist !IbcMergePath! ( + echo %__MsgPrefix%Optimizing using IBC training data + set OptimizationDataDir=%__PackagesDir%\optimization.%__BuildOS%-%__BuildArch%.IBC.CoreCLR\!__IbcOptDataVersion!\data\ + set InputAssemblyFile=!OptimizationDataDir!System.Private.CoreLib.dll + set TargetOptimizationDataFile=!OptimizationDataDir!System.Private.CoreLib.pgo + + if exist "!InputAssemblyFile!" ( + set RawOptimizationDataFile=!OptimizationDataDir!System.Private.CoreLib.ibc + + REM Merge the optimization data into the source DLL + set NEXTCMD="!IbcMergePath!" -q -f -delete -mo "!InputAssemblyFile!" "!RawOptimizationDataFile!" + echo %__MsgPrefix%!NEXTCMD! >> "%__CrossGenCoreLibLog%" + !NEXTCMD! >> "%__CrossGenCoreLibLog%" 2>&1 + if NOT !errorlevel! == 0 ( + echo %__MsgPrefix%Error: CrossGen System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog% + REM Put it in the same log, helpful for Jenkins + type %__CrossGenCoreLibLog% + goto CrossgenFailure + ) + + REM Verify that the optimization data has been merged + set NEXTCMD="!IbcMergePath!" -mi "!InputAssemblyFile!" + echo %__MsgPrefix%!NEXTCMD! >> "%__CrossGenCoreLibLog%" + !NEXTCMD! >> "%__CrossGenCoreLibLog%" 2>&1 + if NOT !errorlevel! == 0 ( + echo %__MsgPrefix%Error: CrossGen System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog% + REM Put it in the same log, helpful for Jenkins + type %__CrossGenCoreLibLog% + goto CrossgenFailure + ) + + REM Save the module as *.pgo to match the convention expected + copy /y !InputAssemblyFile! !TargetOptimizationDataFile! + ) + + if exist "!TargetOptimizationDataFile!" ( + REM Customize IBCMerge's arguments depending on input props + set IBCMergeArguments=-q -f -delete -mo "%__BinDir%\IL\System.Private.CoreLib.dll" -incremental "!TargetOptimizationDataFile!" + + REM Apply optimization data to the compiled assembly + set NEXTCMD="!IbcMergePath!" !IBCMergeArguments! + echo %__MsgPrefix%!NEXTCMD! >> "%__CrossGenCoreLibLog%" + !NEXTCMD! >> "%__CrossGenCoreLibLog%" 2>&1 + if NOT !errorlevel! == 0 ( + echo %__MsgPrefix%Error: CrossGen System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog% + REM Put it in the same log, helpful for Jenkins + type %__CrossGenCoreLibLog% + goto CrossgenFailure + ) + + REM Verify that the optimization data has been applied + set NEXTCMD="!IbcMergePath!" -mi "%__BinDir%\IL\System.Private.CoreLib.dll" + echo %__MsgPrefix%!NEXTCMD! >> "%__CrossGenCoreLibLog%" + !NEXTCMD! >> "%__CrossGenCoreLibLog%" 2>&1 + if NOT !errorlevel! == 0 ( + echo %__MsgPrefix%Error: CrossGen System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog% + REM Put it in the same log, helpful for Jenkins + type %__CrossGenCoreLibLog% + goto CrossgenFailure + ) + ) else ( + echo %__MsgPrefix%!TargetOptimizationDataFile! does not exist >> %__CrossGenCoreLibLog% + echo %__MsgPrefix%Error: CrossGen System.Private.CoreLib build failed. Refer to %__CrossGenCoreLibLog% + REM Put it in the same log, helpful for Jenkins + type %__CrossGenCoreLibLog% + goto CrossgenFailure + ) + ) + ) + + if defined __PartialNgen ( + set COMPlus_PartialNgen=1 + ) + set NEXTCMD="%__CrossgenExe%" %__IbcTuning% /Platform_Assemblies_Paths "%__BinDir%"\IL /out "%__BinDir%\System.Private.CoreLib.dll" "%__BinDir%\IL\System.Private.CoreLib.dll" echo %__MsgPrefix%!NEXTCMD! echo %__MsgPrefix%!NEXTCMD! >> "%__CrossGenCoreLibLog%" @@ -193,7 +193,7 @@ restore_optdata() fi local OptDataProjectFilePath="$__ProjectRoot/src/.nuget/optdata/optdata.csproj" __PgoOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpPgoDataPackageVersion /nologo | sed 's/^\s*//') - __IbcOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpIbcDataPackageVersion /nologo | sed 's/^\s*//') + __IbcOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpIbcDataPackageVersion /nologo | sed 's/^[[:blank:]]*//') fi } diff --git a/buildpipeline/DotNet-CoreClr-Trusted-Windows.json b/buildpipeline/DotNet-CoreClr-Trusted-Windows.json index 10115873a7..f550430fa8 100644 --- a/buildpipeline/DotNet-CoreClr-Trusted-Windows.json +++ b/buildpipeline/DotNet-CoreClr-Trusted-Windows.json @@ -102,7 +102,7 @@ }, "inputs": { "filename": "sync.cmd", - "arguments": "-p -- /p:BuildType=$(PB_BuildType)", + "arguments": "-p -- /p:BuildType=$(PB_BuildType) $(PB_OptionalToolingSyncArguments)", "workingFolder": "", "failOnStandardError": "false" } @@ -122,7 +122,7 @@ }, "inputs": { "filename": "build.cmd", - "arguments": "$(Architecture) $(PB_BuildType) skiptests skipbuildpackages $(PB_EnforcePGO) -OfficialBuildId=$(OfficialBuildId) -Priority=$(Priority) -skiprestore -- /p:SignType=$(PB_SignType) /flp:\"v=diag\"", + "arguments": "$(Architecture) $(PB_BuildType) skiptests skipbuildpackages -ibcoptimize $(PB_EnforcePGO) -OfficialBuildId=$(OfficialBuildId) -Priority=$(Priority) -skiprestore -- /p:SignType=$(PB_SignType) /flp:\"v=diag\"", "workingFolder": "", "failOnStandardError": "false" } @@ -386,6 +386,24 @@ "Label": { "value": "$(Build.BuildNumber)" }, + "PB_OptionalToolingSyncArguments": { + "value": "/p:OptionalToolSource=$(PB_OptionalToolSource) /p:OptionalToolSourceUser=$(PB_OptionalToolSourceUser);OptionalToolSourcePassword=$(PB_OptionalToolSourcePAT)" + }, + "PB_OptionalToolSource": { + "value": null, + "allowOverride": true, + "isSecret": true + }, + "PB_OptionalToolSourcePAT": { + "value": null, + "allowOverride": true, + "isSecret": true + }, + "PB_OptionalToolSourceUser": { + "value": null, + "allowOverride": true, + "isSecret": true + }, "PB_SignType": { "value": "test", "allowOverride": true diff --git a/dependencies.props b/dependencies.props index 0c3724f689..9b23470bc3 100644 --- a/dependencies.props +++ b/dependencies.props @@ -28,6 +28,7 @@ <BuildToolsCurrentRef>e0c1d36d8a19fed21cadfff4890e027bd4c2c214</BuildToolsCurrentRef> <PgoDataCurrentRef>e0c1d36d8a19fed21cadfff4890e027bd4c2c214</PgoDataCurrentRef> <CoreSetupCurrentRef>e0c1d36d8a19fed21cadfff4890e027bd4c2c214</CoreSetupCurrentRef> + <IbcDataCurrentRef>e0c1d36d8a19fed21cadfff4890e027bd4c2c214</IbcDataCurrentRef> </PropertyGroup> <!-- Tests/infrastructure dependency versions. --> @@ -38,6 +39,8 @@ <MicrosoftNETCoreRuntimeCoreCLRPackageVersion>3.0.0-preview1-26817-05</MicrosoftNETCoreRuntimeCoreCLRPackageVersion> <MicrosoftNETCoreAppPackageVersion>3.0.0-preview1-26817-01</MicrosoftNETCoreAppPackageVersion> <XunitPackageVersion>2.4.0-beta.2.build4010</XunitPackageVersion> + <IbcDataPackageVersion>99.99.99-master-20180817-0151</IbcDataPackageVersion> + <IbcMergePackageVersion>4.6.0-alpha-00001</IbcMergePackageVersion> <XunitPerformanceApiPackageVersion>1.0.0-beta-build0015</XunitPerformanceApiPackageVersion> <MicrosoftDiagnosticsTracingTraceEventPackageVersion>2.0.4</MicrosoftDiagnosticsTracingTraceEventPackageVersion> <CommandLineParserVersion>2.2.0</CommandLineParserVersion> @@ -91,6 +94,10 @@ <BuildInfoPath>$(BaseDotNetBuildInfo)optimization/$(DependencyBranch)</BuildInfoPath> <CurrentRef>$(PgoDataCurrentRef)</CurrentRef> </RemoteDependencyBuildInfo> + <RemoteDependencyBuildInfo Include="IbcData"> + <BuildInfoPath>$(BaseDotNetBuildInfo)optimization/$(DependencyBranch)</BuildInfoPath> + <CurrentRef>$(IbcDataCurrentRef)</CurrentRef> + </RemoteDependencyBuildInfo> <RemoteDependencyBuildInfo Include="CoreSetup"> <BuildInfoPath>$(BaseDotNetBuildInfo)core-setup/$(DependencyBranch)</BuildInfoPath> <CurrentRef>$(CoreSetupCurrentRef)</CurrentRef> @@ -135,6 +142,11 @@ <ElementName>PgoDataPackageVersion</ElementName> <PackageId>optimization.PGO.CoreCLR</PackageId> </XmlUpdateStep> + <XmlUpdateStep Include="Data"> + <Path>$(MSBuildThisFileFullPath)</Path> + <ElementName>IbcDataPackageVersion</ElementName> + <PackageId>optimization.IBC.CoreCLR</PackageId> + </XmlUpdateStep> <UpdateStep Include="ILAsm"> <UpdaterType>File</UpdaterType> <Path>$(MSBuildThisFileDirectory)ILAsmVersion.txt</Path> diff --git a/src/.nuget/optdata/ibcmerge.csproj b/src/.nuget/optdata/ibcmerge.csproj new file mode 100644 index 0000000000..e69ac17b4a --- /dev/null +++ b/src/.nuget/optdata/ibcmerge.csproj @@ -0,0 +1,19 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" /> + + <PropertyGroup> + <TargetFramework>netstandard</TargetFramework> + <DisableImplicitFrameworkReferences>true</DisableImplicitFrameworkReferences> + <RuntimeIdentifiers>win7-x64;win7-x86;linux-x64</RuntimeIdentifiers> + </PropertyGroup> + + <ItemGroup> + <PackageReference Include="microsoft.dotnet.ibcmerge" Version="$(IbcMergePackageVersion)" Condition="'$(IbcMergePackageVersion)'!=''" /> + </ItemGroup> + + <Target Name="DumpIbcMergePackageVersion"> + <Message Importance="high" Text="$(IbcMergePackageVersion)" /> + </Target> + +</Project> |