diff options
136 files changed, 3967 insertions, 1715 deletions
diff --git a/Documentation/design-docs/event-counter.md b/Documentation/design-docs/event-counter.md index ffff8505b2..b5ffc056f5 100644 --- a/Documentation/design-docs/event-counter.md +++ b/Documentation/design-docs/event-counter.md @@ -34,7 +34,6 @@ We believe adding some new top-level types will satisfy these requests: class EventCounter { EventCounter(string name, EventSource eventSource); string DisplayName; - TimeSpan DisplayRateTimeScale; void WriteMetric(float metric); void AddMetaData(string key, string value); } @@ -42,7 +41,6 @@ We believe adding some new top-level types will satisfy these requests: class PollingCounter { PollingCounter(string name, EventSource eventSource Func<float> getMetricFunction); string DisplayName; - TimeSpan DisplayRateTimeScale; void AddMetaData(string key, string value); } @@ -61,6 +61,7 @@ usage() echo "-msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported." echo "-numproc - set the number of build processes." echo "-portablebuild - pass -portablebuild=false to force a non-portable build." + echo "-staticanalyzer - build with clang static analyzer enabled." exit 1 } @@ -276,11 +277,15 @@ build_native() # Regenerate the CMake solution if [[ $__GccBuild == 0 ]]; then - echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion \"$__ClangMinorVersion\" $platformArch $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs" - "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion "$__ClangMinorVersion" $platformArch $__BuildType $__CodeCoverage $generator "$extraCmakeArguments" "$__cmakeargs" + scan_build= + if [[ $__StaticAnalyzer == 1 ]]; then + scan_build=scan-build + fi + echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion \"$__ClangMinorVersion\" $platformArch $__BuildType $__CodeCoverage $scan_build $generator $extraCmakeArguments $__cmakeargs" + source "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion "$__ClangMinorVersion" $platformArch $__BuildType $__CodeCoverage $scan_build $generator "$extraCmakeArguments" "$__cmakeargs" else echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-gcc.sh\" \"$__ProjectRoot\" $__GccMajorVersion \"$__GccMinorVersion\" $platformArch $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs" - "$__ProjectRoot/src/pal/tools/gen-buildsys-gcc.sh" "$__ProjectRoot" "$__GccMajorVersion" "$__CGccMinorVersion" $platformArch $__BuildType $__CodeCoverage $generator "$extraCmakeArguments" "$__cmakeargs" + source "$__ProjectRoot/src/pal/tools/gen-buildsys-gcc.sh" "$__ProjectRoot" "$__GccMajorVersion" "$__CGccMinorVersion" $platformArch $__BuildType $__CodeCoverage $generator "$extraCmakeArguments" "$__cmakeargs" fi popd fi @@ -299,6 +304,10 @@ build_native() # Check that the makefiles were created. pushd "$intermediatesForBuild" + if [ $__StaticAnalyzer == 1 ]; then + buildTool="$SCAN_BUILD_COMMAND $buildTool" + fi + echo "Executing $buildTool install -j $__NumProc" $buildTool install -j $__NumProc @@ -649,6 +658,7 @@ __BuildManagedTools=1 __SkipRestoreArg="" __SignTypeArg="" __OfficialBuildIdArg="" +__StaticAnalyzer=0 # Get the number of processors available to the scheduler # Other techniques such as `nproc` only get the number of @@ -937,6 +947,10 @@ while :; do __OfficialBuildIdArg="/p:OfficialBuildId=$__Id" ;; + -staticanalyzer) + __StaticAnalyzer=1 + ;; + --) # Skip -Option=Value style argument passing ;; diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ffce3df54c..90bd899b2b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -2,33 +2,33 @@ <Dependencies> <ProductDependencies></ProductDependencies> <ToolsetDependencies> - <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19128.3"> + <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19160.2"> <Uri>https://github.com/dotnet/arcade</Uri> - <Sha>6c034531b2af9e6b2f76c86d471fd308a400269e</Sha> + <Sha>89ab8b2b806397e5e444809a6ac12e275e0e20a2</Sha> </Dependency> - <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19128.3"> + <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19160.2"> <Uri>https://github.com/dotnet/arcade</Uri> - <Sha>6c034531b2af9e6b2f76c86d471fd308a400269e</Sha> + <Sha>89ab8b2b806397e5e444809a6ac12e275e0e20a2</Sha> </Dependency> - <Dependency Name="Microsoft.Private.CoreFx.NETCoreApp" Version="4.6.0-preview4.19154.9"> + <Dependency Name="Microsoft.Private.CoreFx.NETCoreApp" Version="4.6.0-preview4.19160.8"> <Uri>https://github.com/dotnet/corefx</Uri> - <Sha>8a730a6ecd96fef04d74e1807c8b1d193e0a5f16</Sha> + <Sha>6d7126e0a329db795930ea17e84369eacb7bae5a</Sha> </Dependency> - <Dependency Name="Microsoft.NETCore.Platforms" Version="3.0.0-preview4.19154.9"> + <Dependency Name="Microsoft.NETCore.Platforms" Version="3.0.0-preview4.19160.8"> <Uri>https://github.com/dotnet/corefx</Uri> - <Sha>8a730a6ecd96fef04d74e1807c8b1d193e0a5f16</Sha> + <Sha>6d7126e0a329db795930ea17e84369eacb7bae5a</Sha> </Dependency> - <Dependency Name="Microsoft.NETCore.App" Version="3.0.0-preview4-27504-10"> + <Dependency Name="Microsoft.NETCore.App" Version="3.0.0-preview4-27511-01"> <Uri>https://github.com/dotnet/core-setup</Uri> - <Sha>ed6636c463f0ac5fe4183348b6b18e57c8345cb5</Sha> + <Sha>10b856b0402bc70f2190baa22df989bc0e9dac99</Sha> </Dependency> - <Dependency Name="optimization.IBC.CoreCLR" Version="99.99.99-master-20190306.3"> + <Dependency Name="optimization.IBC.CoreCLR" Version="99.99.99-master-20190308.5"> <Uri>https://dnceng@dev.azure.com/dnceng/internal/_git/dotnet-optimization</Uri> - <Sha>96f5b504695366ad94f8c0e45939f094ca515272</Sha> + <Sha>5fdf7f90d84111d4897438d4e5d8e88e0af4f3fb</Sha> </Dependency> - <Dependency Name="optimization.PGO.CoreCLR" Version="99.99.99-master-20190306.3"> + <Dependency Name="optimization.PGO.CoreCLR" Version="99.99.99-master-20190308.5"> <Uri>https://dnceng@dev.azure.com/dnceng/internal/_git/dotnet-optimization</Uri> - <Sha>96f5b504695366ad94f8c0e45939f094ca515272</Sha> + <Sha>5fdf7f90d84111d4897438d4e5d8e88e0af4f3fb</Sha> </Dependency> </ToolsetDependencies> </Dependencies> diff --git a/eng/Versions.props b/eng/Versions.props index b9d43bf570..b35189d914 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -8,11 +8,11 @@ <UsingToolXliff>false</UsingToolXliff> <UsingToolNetFrameworkReferenceAssemblies>true</UsingToolNetFrameworkReferenceAssemblies> <MicrosoftNetFrameworkReferenceAssembliesVersion>1.0.0-alpha-004</MicrosoftNetFrameworkReferenceAssembliesVersion> - <MicrosoftPrivateCoreFxNETCoreAppVersion>4.6.0-preview4.19154.9</MicrosoftPrivateCoreFxNETCoreAppVersion> - <MicrosoftNETCorePlatformsVersion>3.0.0-preview4.19154.9</MicrosoftNETCorePlatformsVersion> - <MicrosoftNETCoreAppVersion>3.0.0-preview4-27504-10</MicrosoftNETCoreAppVersion> - <optimizationIBCCoreCLRVersion>99.99.99-master-20190306.3</optimizationIBCCoreCLRVersion> - <optimizationPGOCoreCLRVersion>99.99.99-master-20190306.3</optimizationPGOCoreCLRVersion> + <MicrosoftPrivateCoreFxNETCoreAppVersion>4.6.0-preview4.19160.8</MicrosoftPrivateCoreFxNETCoreAppVersion> + <MicrosoftNETCorePlatformsVersion>3.0.0-preview4.19160.8</MicrosoftNETCorePlatformsVersion> + <MicrosoftNETCoreAppVersion>3.0.0-preview4-27511-01</MicrosoftNETCoreAppVersion> + <optimizationIBCCoreCLRVersion>99.99.99-master-20190308.5</optimizationIBCCoreCLRVersion> + <optimizationPGOCoreCLRVersion>99.99.99-master-20190308.5</optimizationPGOCoreCLRVersion> </PropertyGroup> <!--Package names--> <PropertyGroup> diff --git a/eng/common/PublishToPackageFeed.proj b/eng/common/PublishToPackageFeed.proj index 8149e3fb6a..b26d28a90b 100644 --- a/eng/common/PublishToPackageFeed.proj +++ b/eng/common/PublishToPackageFeed.proj @@ -11,7 +11,6 @@ </PropertyGroup> <Import Project="$(MSBuildThisFileDirectory)DefaultVersions.props" Condition="Exists('$(MSBuildThisFileDirectory)DefaultVersions.props')" /> - <Import Project="$(MSBuildThisFileDirectory)Versions.props" Condition="Exists('$(MSBuildThisFileDirectory)Versions.props')" /> <Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" /> diff --git a/eng/common/build.sh b/eng/common/build.sh index 0227c6e1e5..40b1e8ec73 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -137,13 +137,16 @@ while [[ $# > 0 ]]; do node_reuse=$2 shift ;; - /p:*) + -p:*|/p:*) properties="$properties $1" ;; - /m:*) + -m:*|/m:*) properties="$properties $1" ;; - /bl:*) + -bl:*|/bl:*) + properties="$properties $1" + ;; + -dl:*|/dl:*) properties="$properties $1" ;; *) diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 24676b261d..2467ebdd42 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -19,7 +19,7 @@ function InstallDarcCli ($darcVersion) { # Until we can anonymously query the BAR API for the latest arcade-services # build applied to the PROD channel, this is hardcoded. if (-not $darcVersion) { - $darcVersion = '1.1.0-beta.19120.2' + $darcVersion = '1.1.0-beta.19151.3' } $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-arcade/index.json' diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index d4dfdc94fc..8d63dd711b 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash source="${BASH_SOURCE[0]}" -darcVersion="1.1.0-beta.19120.2" +darcVersion="1.1.0-beta.19151.3" while [[ $# > 0 ]]; do opt="$(echo "$1" | awk '{print tolower($0)}')" diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 new file mode 100644 index 0000000000..c04c80e4f6 --- /dev/null +++ b/eng/common/generate-graph-files.ps1 @@ -0,0 +1,61 @@ +Param( + [Parameter(Mandatory=$true)][string] $barToken, # Token generated at https://maestro-prod.westus2.cloudapp.azure.com/Account/Tokens + [Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed) + [Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed) + [Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created + [string] $darcVersion = '1.1.0-beta.19156.4', # darc's version + [switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about + # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies +) + +$ErrorActionPreference = "Stop" +. $PSScriptRoot\tools.ps1 + +function CheckExitCode ([string]$stage) +{ + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + Write-Host "Something failed in stage: '$stage'. Check for errors above. Exiting now..." + ExitWithExitCode $exitCode + } +} + +try { + Push-Location $PSScriptRoot + + Write-Host "Installing darc..." + . .\darc-init.ps1 -darcVersion $darcVersion + CheckExitCode "Running darc-init" + + $darcExe = "$env:USERPROFILE\.dotnet\tools" + $darcExe = Resolve-Path "$darcExe\darc.exe" + + Create-Directory $outputFolder + + $graphVizFilePath = "$outputFolder\graphviz.txt" + $graphFilePath = "$outputFolder\graph.txt" + $options = "get-dependency-graph --graphviz '$graphVizFilePath' --github-pat $gitHubPat --azdev-pat $azdoPat --password $barToken --output-file $graphFilePath" + + if ($includeToolset) { + Write-Host "Toolsets will be included in the graph..." + $options += " --include-toolset" + } + + Write-Host "Generating dependency graph..." + $darc = Invoke-Expression "& `"$darcExe`" $options" + CheckExitCode "Generating dependency graph" + + $graph = Get-Content $graphVizFilePath + Set-Content $graphVizFilePath -Value "Paste the following digraph object in http://www.webgraphviz.com `r`n", $graph + Write-Host "'$graphVizFilePath' and '$graphFilePath' created!" +} +catch { + if (!$includeToolset) { + Write-Host "This might be a toolset repo which includes only toolset dependencies. " -NoNewline -ForegroundColor Yellow + Write-Host "Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again..." -ForegroundColor Yellow + } + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +}
\ No newline at end of file diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index cd4e5731a6..74dd81fdc0 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -179,7 +179,7 @@ jobs: continueOnError: true condition: always() - - ${{ if and(eq(parameters.enablePublishBuildAssets, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(variables['_PublishUsingPipelines'], 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: CopyFiles@2 displayName: Gather Asset Manifests inputs: @@ -194,4 +194,4 @@ jobs: PublishLocation: Container ArtifactName: AssetManifests continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
\ No newline at end of file + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 1fbf8b8897..0925e8ebd1 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -1,32 +1,33 @@ parameters: - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true - XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects - XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner - XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json - EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false steps: - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: Send job to Helix (Windows) + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) env: BuildConfig: $(_BuildConfig) HelixSource: ${{ parameters.HelixSource }} @@ -50,10 +51,11 @@ steps: EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: Send job to Helix (Unix) + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) env: BuildConfig: $(_BuildConfig) HelixSource: ${{ parameters.HelixSource }} @@ -77,5 +79,6 @@ steps: EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.continueOnError }}
\ No newline at end of file diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index ae33b37d33..de7523cae5 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -321,12 +321,10 @@ function LocateVisualStudio([object]$vsRequirements = $null){ function InitializeBuildTool() { if (Test-Path variable:global:_BuildTool) { - Write-Host "variable:global:_BuildTool initialized." -ForegroundColor Red return $global:_BuildTool } if (-not $msbuildEngine) { - Write-Host "-not $msbuildEngine" -ForegroundColor Red $msbuildEngine = GetDefaultMSBuildEngine } diff --git a/eng/install-native-dependencies.sh b/eng/install-native-dependencies.sh index 4b3eda7c00..aca0e6155f 100644 --- a/eng/install-native-dependencies.sh +++ b/eng/install-native-dependencies.sh @@ -6,7 +6,7 @@ if [ "$1" = "Linux" ]; then exit 1; fi sudo apt install cmake llvm-3.9 clang-3.9 lldb-3.9 liblldb-3.9-dev libunwind8 libunwind8-dev gettext libicu-dev liblttng-ust-dev libcurl4-openssl-dev libssl-dev libkrb5-dev libnuma-dev - if [ "$?" != "0"]; then + if [ "$?" != "0" ]; then exit 1; fi elif [ "$1" = "OSX" ]; then diff --git a/eng/xplat-job.yml b/eng/xplat-job.yml index d39a7c1385..d8c88a2449 100644 --- a/eng/xplat-job.yml +++ b/eng/xplat-job.yml @@ -35,21 +35,36 @@ jobs: enableMicrobuild: ${{ parameters.enableMicrobuild }} pool: + + # Public Linux Build Pool ${{ if and(eq(parameters.osGroup, 'Linux'), eq(variables['System.TeamProject'], 'public')) }}: - name: dnceng-linux-external-temp + name: NetCorePublic-Int-Pool + queue: BuildPool.Ubuntu.1604.Amd64.Open + + # Official Build Linux Pool ${{ if and(eq(parameters.osGroup, 'Linux'), ne(variables['System.TeamProject'], 'public')) }}: name: dnceng-linux-internal-temp + # FreeBSD builds only in the internal project ${{ if and(eq(parameters.osGroup, 'FreeBSD'), ne(variables['System.TeamProject'], 'public')) }}: name: dnceng-freebsd-internal + + # Public OSX Build Pool ${{ if and(eq(parameters.osGroup, 'OSX'), ne(variables['System.TeamProject'], 'public')) }}: name: Hosted Mac Internal + + # Official Build OSX Pool ${{ if and(eq(parameters.osGroup, 'OSX'), eq(variables['System.TeamProject'], 'public')) }}: name: Hosted MacOS + + # Public Windows Build Pool ${{ if and(eq(parameters.osGroup, 'Windows_NT'), ne(variables['System.TeamProject'], 'public')) }}: name: dotnet-internal-temp + + # Official Build Windows Pool ${{ if and(eq(parameters.osGroup, 'Windows_NT'), eq(variables['System.TeamProject'], 'public')) }}: - name: dotnet-external-temp + name: NetCorePublic-Int-Pool + queue: BuildPool.Windows.10.Amd64.VS2017.Open workspace: clean: all diff --git a/global.json b/global.json index dac26384fa..7d183ef805 100644 --- a/global.json +++ b/global.json @@ -7,7 +7,7 @@ "python": "2.7.15" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19128.3", - "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19128.3" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19160.2", + "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19160.2" } } diff --git a/netci.groovy b/netci.groovy index 31ce12dbbe..c72a9c2649 100755 --- a/netci.groovy +++ b/netci.groovy @@ -2173,7 +2173,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll" // Archive and process (only) the test results - Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml") + Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml") } else { @@ -2184,7 +2184,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR buildCommands += "python -u %WORKSPACE%\\tests\\scripts\\run-corefx-tests.py -arch ${arch} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${envScriptPath}" // Archive and process (only) the test results - Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml") + Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml") //Archive additional build stuff to diagnose why my attempt at fault injection isn't causing CI to fail @@ -2388,10 +2388,9 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests" buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly" buildCommands += "./tests/runtest.sh ${lowerConfiguration} --corefxtestsall --testHostDir=\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=\${WORKSPACE}" - - break + // Archive and process (only) the test results - Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml") + Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml") } else { @@ -2419,7 +2418,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}" // Archive and process (only) the test results - Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml") + Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/artifacts/bin/**/testResults.xml") } } @@ -3422,7 +3421,7 @@ ${runScript} \\ Utilities.addArchival(newJob, "dasm.${os}.${architecture}.${configuration}.zip") } else if (doCoreFxTesting) { - Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/artifacts/bin/**/testResults.xml") + Utilities.addArchival(newJob, "${workspaceRelativeFxRootLinux}/artifacts/bin/**/testResults.xml", "", /* doNotFailIfNothingArchived */ true, /* archiveOnlyIfSuccessful */ false) if ((os == "Ubuntu") && (architecture == 'arm')) { // We have a problem with the xunit plug-in, where it is consistently failing on Ubuntu arm32 test result uploading with this error: // diff --git a/src/.nuget/Microsoft.NET.Sdk.IL/targets/Microsoft.NET.Sdk.IL.targets b/src/.nuget/Microsoft.NET.Sdk.IL/targets/Microsoft.NET.Sdk.IL.targets index 035db37526..cea740ddab 100644 --- a/src/.nuget/Microsoft.NET.Sdk.IL/targets/Microsoft.NET.Sdk.IL.targets +++ b/src/.nuget/Microsoft.NET.Sdk.IL/targets/Microsoft.NET.Sdk.IL.targets @@ -29,6 +29,7 @@ Copyright (c) .NET Foundation. All rights reserved. <MicrosoftNetCoreIlasmPackageRuntimeId Condition="'$(MicrosoftNetCoreIlasmPackageRuntimeId)' == ''">$(_OSPlatform)-$(_OSArchitecture.ToLower())</MicrosoftNetCoreIlasmPackageRuntimeId> <MicrosoftNetCoreIlasmPackageVersion Condition="'$(MicrosoftNetCoreIlasmPackageVersion)' == ''">3.0.0</MicrosoftNetCoreIlasmPackageVersion> <MicrosoftNetCoreIlasmPackageName>runtime.$(MicrosoftNetCoreIlasmPackageRuntimeId).microsoft.netcore.ilasm</MicrosoftNetCoreIlasmPackageName> + <MicrosoftNetCoreIldasmPackageName>runtime.$(MicrosoftNetCoreIlasmPackageRuntimeId).microsoft.netcore.ildasm</MicrosoftNetCoreIldasmPackageName> <MicrosoftNetCoreRuntimeCoreClrPackageName>runtime.$(MicrosoftNetCoreIlasmPackageRuntimeId).microsoft.netcore.runtime.coreclr</MicrosoftNetCoreRuntimeCoreClrPackageName> <MicrosoftNetCoreJitPackageName>runtime.$(MicrosoftNetCoreIlasmPackageRuntimeId).microsoft.netcore.jit</MicrosoftNetCoreJitPackageName> @@ -43,6 +44,7 @@ Copyright (c) .NET Foundation. All rights reserved. <ItemGroup Condition="'$(ILAsmToolPath)' == ''"> <_IlasmPackageReference Include="$(MicrosoftNetCoreIlasmPackageName)" Version="$(MicrosoftNetCoreIlasmPackageVersion)" /> + <_IlasmPackageReference Include="$(MicrosoftNetCoreIldasmPackageName)" Version="$(MicrosoftNetCoreIlasmPackageVersion)" /> <_IlasmPackageReference Include="$(MicrosoftNetCoreRuntimeCoreClrPackageName)" Version="$(MicrosoftNetCoreIlasmPackageVersion)" /> <_IlasmPackageReference Include="$(MicrosoftNetCoreJitPackageName)" Version="$(MicrosoftNetCoreIlasmPackageVersion)" /> <PackageReference Include="@(_IlasmPackageReference)" PrivateAssets="all" IsImplicitlyDefined="true" /> @@ -70,6 +72,48 @@ Copyright (c) .NET Foundation. All rights reserved. <Copy DestinationFolder="$(_IlasmDir)" SourceFiles="@(_IlasmSourceFiles)" /> </Target> + <!-- projects can define an ILResourceReference and we'll decompile it to get native resources --> + <Target Name="DisassembleIlasmResourceFile" + BeforeTargets="CoreCompile" + Condition="'$(OS)'=='Windows_NT'" + Inputs="@(ILResourceReference)" + Outputs="$(IntermediateOutputPath)$(MSBuildProjectName).ref.res.obj"> + <Error Condition="'@(ILResourceReference->Count())' != '1'" Text="Only one ILResourceReference can be specified" /> + <PropertyGroup> + <_ilResourceReference>%(ILResourceReference.FullPath)</_ilResourceReference> + <_IldasmCommand>$(_IlasmDir)ildasm</_IldasmCommand> + <_IldasmCommand>$(_IldasmCommand) "$(_ilResourceReference)"</_IldasmCommand> + <_IldasmCommand>$(_IldasmCommand) /OUT="$(IntermediateOutputPath)/$(MSBuildProjectName).ref.il"</_IldasmCommand> + + <!-- Try to use cvtres.exe from the framework and fallback to the one that is on the PATH in case we can't find it --> + <_CvtResCommand>$(SystemRoot)\Microsoft.NET\Framework\v4.0.30319\cvtres.exe</_CvtResCommand> + <_CvtResCommand Condition="!Exists('$(_CvtResCommand)')">cvtres.exe</_CvtResCommand> + <_CvtResCommand>$(_CvtResCommand) /MACHINE:x86</_CvtResCommand> + <_CvtResCommand>$(_CvtResCommand) /OUT:"$(IntermediateOutputPath)$(MSBuildProjectName).ref.res.obj"</_CvtResCommand> + <_CvtResCommand>$(_CvtResCommand) "$(IntermediateOutputPath)$(MSBuildProjectName).ref.res"</_CvtResCommand> + </PropertyGroup> + + <ItemGroup> + <FileWrites Include="$(IntermediateOutputPath)$(MSBuildProjectName).ref.*" /> + </ItemGroup> + + <!-- Getting the res file by disassemblying the contract assembly --> + <Exec Command="$(_IldasmCommand)" ConsoleToMSBuild="true" StandardOutputImportance="Low"> + <Output TaskParameter="ExitCode" PropertyName="_IldasmCommandExitCode" /> + </Exec> + <Error Condition="'$(_IldasmCommandExitCode)' != '0'" Text="ILDasm failed while running command: "$(_IldasmCommand)"" /> + + <!-- Calling cvtres.exe which should be on the path in order to transform the resource file to an obj --> + <Exec Command="$(_CvtResCommand)" ConsoleToMSBuild="true" StandardOutputImportance="Low"> + <Output TaskParameter="ExitCode" PropertyName="_CvtResCommandExitCode" /> + </Exec> + <Error Condition="'$(_CvtResCommandExitCode)' != '0'" Text="cvtres failed while running command: "$(_CvtResCommand)"" /> + + <PropertyGroup> + <IlasmResourceFile>$(IntermediateOutputPath)$(MSBuildProjectName).ref.res.obj</IlasmResourceFile> + </PropertyGroup> + </Target> + <Target Name="CoreCompile" Inputs="$(MSBuildAllProjects); @(Compile)" diff --git a/src/System.Private.CoreLib/System.Private.CoreLib.csproj b/src/System.Private.CoreLib/System.Private.CoreLib.csproj index 816d3cef33..384babf674 100644 --- a/src/System.Private.CoreLib/System.Private.CoreLib.csproj +++ b/src/System.Private.CoreLib/System.Private.CoreLib.csproj @@ -122,7 +122,7 @@ <Compile Include="$(BclSourcesRoot)\System\Array.CoreCLR.cs" /> <Compile Include="$(BclSourcesRoot)\System\Attribute.CoreCLR.cs" /> <Compile Include="$(BclSourcesRoot)\System\BadImageFormatException.CoreCLR.cs" /> - <Compile Include="$(BclSourcesRoot)\System\Buffer.cs" /> + <Compile Include="$(BclSourcesRoot)\System\Buffer.CoreCLR.cs" /> <Compile Include="$(BclSourcesRoot)\System\CLRConfig.cs" /> <Compile Include="$(BclSourcesRoot)\System\Collections\EmptyReadOnlyDictionaryInternal.cs" /> <Compile Include="$(BclSourcesRoot)\System\Collections\Generic\ArraySortHelper.CoreCLR.cs" /> @@ -239,7 +239,6 @@ <Compile Include="$(BclSourcesRoot)\System\Runtime\InteropServices\NativeLibrary.cs" /> <Compile Include="$(BclSourcesRoot)\System\Runtime\Loader\AssemblyDependencyResolver.cs" /> <Compile Include="$(BclSourcesRoot)\System\Runtime\Loader\AssemblyLoadContext.CoreCLR.cs" /> - <Compile Include="$(BclSourcesRoot)\System\Runtime\Serialization\FormatterServices.cs" /> <Compile Include="$(BclSourcesRoot)\System\Runtime\Versioning\CompatibilitySwitch.cs" /> <Compile Include="$(BclSourcesRoot)\System\RuntimeArgumentHandle.cs" /> <Compile Include="$(BclSourcesRoot)\System\RuntimeHandles.cs" /> diff --git a/src/System.Private.CoreLib/shared/Interop/Unix/System.Native/Interop.Stat.cs b/src/System.Private.CoreLib/shared/Interop/Unix/System.Native/Interop.Stat.cs index cb0f4284f0..d06fbda718 100644 --- a/src/System.Private.CoreLib/shared/Interop/Unix/System.Native/Interop.Stat.cs +++ b/src/System.Private.CoreLib/shared/Interop/Unix/System.Native/Interop.Stat.cs @@ -54,13 +54,13 @@ internal static partial class Interop HasBirthTime = 1, } - [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_FStat2", SetLastError = true)] + [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_FStat", SetLastError = true)] internal static extern int FStat(SafeFileHandle fd, out FileStatus output); - [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_Stat2", SetLastError = true)] + [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_Stat", SetLastError = true)] internal static extern int Stat(string path, out FileStatus output); - [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_LStat2", SetLastError = true)] + [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_LStat", SetLastError = true)] internal static extern int LStat(string path, out FileStatus output); } } diff --git a/src/System.Private.CoreLib/shared/System.Private.CoreLib.Shared.projitems b/src/System.Private.CoreLib/shared/System.Private.CoreLib.Shared.projitems index a168f09a01..ce0f102587 100644 --- a/src/System.Private.CoreLib/shared/System.Private.CoreLib.Shared.projitems +++ b/src/System.Private.CoreLib/shared/System.Private.CoreLib.Shared.projitems @@ -54,6 +54,7 @@ <Compile Include="$(MSBuildThisFileDirectory)System\BadImageFormatException.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\BitConverter.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Boolean.cs" /> + <Compile Include="$(MSBuildThisFileDirectory)System\Buffer.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Buffers\ArrayPool.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Buffers\ArrayPoolEventSource.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Buffers\ConfigurableArrayPool.cs" /> @@ -575,6 +576,7 @@ <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\CompilerServices\ReferenceAssemblyAttribute.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\CompilerServices\RuntimeCompatibilityAttribute.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\CompilerServices\RuntimeFeature.cs" /> + <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\CompilerServices\RuntimeHelpers.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\CompilerServices\RuntimeWrappedException.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\CompilerServices\SpecialNameAttribute.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\CompilerServices\StateMachineAttribute.cs" /> @@ -759,6 +761,7 @@ <Compile Include="$(MSBuildThisFileDirectory)System\StringSplitOptions.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\SystemException.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Text\ASCIIEncoding.cs" /> + <Compile Include="$(MSBuildThisFileDirectory)System\Text\ASCIIUtility.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Text\StringBuilderCache.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Text\CodePageDataItem.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Text\Decoder.cs" /> @@ -774,6 +777,7 @@ <Compile Include="$(MSBuildThisFileDirectory)System\Text\EncoderFallback.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Text\EncoderReplacementFallback.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Text\Encoding.cs" /> + <Compile Include="$(MSBuildThisFileDirectory)System\Text\Encoding.Internal.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Text\EncodingData.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Text\EncodingInfo.cs" /> <Compile Include="$(MSBuildThisFileDirectory)System\Text\EncodingNLS.cs" /> diff --git a/src/System.Private.CoreLib/shared/System/AppDomain.cs b/src/System.Private.CoreLib/shared/System/AppDomain.cs index fe3706bd1e..707f8278a7 100644 --- a/src/System.Private.CoreLib/shared/System/AppDomain.cs +++ b/src/System.Private.CoreLib/shared/System/AppDomain.cs @@ -4,6 +4,7 @@ #pragma warning disable CS0067 // events are declared but not used +using System.Diagnostics; using System.IO; using System.Reflection; using System.Runtime.ExceptionServices; @@ -398,10 +399,12 @@ namespace System if (s_getUnauthenticatedPrincipal == null) { Type type = Type.GetType("System.Security.Principal.GenericPrincipal, System.Security.Claims", throwOnError: true); + MethodInfo mi = type.GetMethod("GetDefaultInstance", BindingFlags.NonPublic | BindingFlags.Static); + Debug.Assert(mi != null); // Don't throw PNSE if null like for WindowsPrincipal as UnauthenticatedPrincipal should // be available on all platforms. Volatile.Write(ref s_getUnauthenticatedPrincipal, - (Func<IPrincipal>)Delegate.CreateDelegate(typeof(Func<IPrincipal>), type, "GetDefaultInstance")); + (Func<IPrincipal>)mi.CreateDelegate(typeof(Func<IPrincipal>))); } principal = s_getUnauthenticatedPrincipal(); @@ -411,9 +414,13 @@ namespace System if (s_getWindowsPrincipal == null) { Type type = Type.GetType("System.Security.Principal.WindowsPrincipal, System.Security.Principal.Windows", throwOnError: true); + MethodInfo mi = type.GetMethod("GetDefaultInstance", BindingFlags.NonPublic | BindingFlags.Static); + if (mi == null) + { + throw new PlatformNotSupportedException(SR.PlatformNotSupported_Principal); + } Volatile.Write(ref s_getWindowsPrincipal, - (Func<IPrincipal>)Delegate.CreateDelegate(typeof(Func<IPrincipal>), type, "GetDefaultInstance", ignoreCase: false, throwOnBindFailure: false) - ?? throw new PlatformNotSupportedException(SR.PlatformNotSupported_Principal)); + (Func<IPrincipal>)mi.CreateDelegate(typeof(Func<IPrincipal>))); } principal = s_getWindowsPrincipal(); diff --git a/src/System.Private.CoreLib/src/System/Buffer.cs b/src/System.Private.CoreLib/shared/System/Buffer.cs index af94e8e08d..dda25b827f 100644 --- a/src/System.Private.CoreLib/src/System/Buffer.cs +++ b/src/System.Private.CoreLib/shared/System/Buffer.cs @@ -23,30 +23,8 @@ using nuint = System.UInt32; namespace System { - public static class Buffer + public static partial class Buffer { - // Copies from one primitive array to another primitive array without - // respecting types. This calls memmove internally. The count and - // offset parameters here are in bytes. If you want to use traditional - // array element indices and counts, use Array.Copy. - [MethodImplAttribute(MethodImplOptions.InternalCall)] - public static extern void BlockCopy(Array src, int srcOffset, - Array dst, int dstOffset, int count); - - // Returns a bool to indicate if the array is of primitive data types - // or not. - [MethodImplAttribute(MethodImplOptions.InternalCall)] - private static extern bool IsPrimitiveTypeArray(Array array); - - // Gets the length of the array in bytes. The array must be an - // array of primitives. - // - // This essentially does the following: - // return array.length * sizeof(array.UnderlyingElementType). - // - [MethodImplAttribute(MethodImplOptions.InternalCall)] - private static extern int _ByteLength(Array array); - public static int ByteLength(Array array) { // Is the array present? @@ -161,24 +139,6 @@ namespace System } } - // This method has a slightly different behavior on arm and other platforms. - // On arm this method behaves like memcpy and does not handle overlapping buffers. - // While on other platforms it behaves like memmove and handles overlapping buffers. - // This behavioral difference is unfortunate but intentional because - // 1. This method is given access to other internal dlls and this close to release we do not want to change it. - // 2. It is difficult to get this right for arm and again due to release dates we would like to visit it later. -#if ARM - [MethodImplAttribute(MethodImplOptions.InternalCall)] - internal static extern unsafe void Memcpy(byte* dest, byte* src, int len); -#else // ARM - [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] - internal static unsafe void Memcpy(byte* dest, byte* src, int len) - { - Debug.Assert(len >= 0, "Negative length in memcpy!"); - Memmove(dest, src, (nuint)len); - } -#endif // ARM - // This method has different signature for x64 and other platforms and is done for performance reasons. internal static unsafe void Memmove(byte* dest, byte* src, nuint len) { @@ -615,9 +575,6 @@ namespace System __Memmove(pDest, pSrc, len); } - [DllImport(JitHelpers.QCall, CharSet = CharSet.Unicode)] - private static extern unsafe void __Memmove(byte* dest, byte* src, nuint len); - #if HAS_CUSTOM_BLOCKS [StructLayout(LayoutKind.Sequential, Size = 16)] private struct Block16 { } diff --git a/src/System.Private.CoreLib/shared/System/Buffers/Text/Utf8Formatter/Utf8Formatter.Float.cs b/src/System.Private.CoreLib/shared/System/Buffers/Text/Utf8Formatter/Utf8Formatter.Float.cs index 6dde973a89..96e70bada9 100644 --- a/src/System.Private.CoreLib/shared/System/Buffers/Text/Utf8Formatter/Utf8Formatter.Float.cs +++ b/src/System.Private.CoreLib/shared/System/Buffers/Text/Utf8Formatter/Utf8Formatter.Float.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Globalization; +using System.Text; namespace System.Buffers.Text { @@ -107,14 +108,16 @@ namespace System.Buffers.Text return false; } - for (int i = 0; i < utf16Text.Length; i++) + try { - Debug.Assert(utf16Text[i] < 128, "A culture-invariant ToString() of a floating point expected to produce ASCII characters only."); - destination[i] = (byte)utf16Text[i]; + bytesWritten = Encoding.UTF8.GetBytes(utf16Text, destination); + return true; + } + catch + { + bytesWritten = 0; + return false; } - - bytesWritten = utf16Text.Length; - return true; } } } diff --git a/src/System.Private.CoreLib/shared/System/Collections/ObjectModel/Collection.cs b/src/System.Private.CoreLib/shared/System/Collections/ObjectModel/Collection.cs index c96a1576f3..40c8ed8dba 100644 --- a/src/System.Private.CoreLib/shared/System/Collections/ObjectModel/Collection.cs +++ b/src/System.Private.CoreLib/shared/System/Collections/ObjectModel/Collection.cs @@ -69,6 +69,8 @@ namespace System.Collections.ObjectModel InsertItem(index, item); } + public void AddRange(IEnumerable<T> collection) => InsertItemsRange(items.Count, collection); + public void Clear() { if (items.IsReadOnly) @@ -114,6 +116,8 @@ namespace System.Collections.ObjectModel InsertItem(index, item); } + public void InsertRange(int index, IEnumerable<T> collection) => InsertItemsRange(index, collection); + public bool Remove(T item) { if (items.IsReadOnly) @@ -127,6 +131,10 @@ namespace System.Collections.ObjectModel return true; } + public void RemoveRange(int index, int count) => RemoveItemsRange(index, count); + + public void ReplaceRange(int index, int count, IEnumerable<T> collection) => ReplaceItemsRange(index, count, collection); + public void RemoveAt(int index) { if (items.IsReadOnly) @@ -162,6 +170,77 @@ namespace System.Collections.ObjectModel items[index] = item; } + protected virtual void InsertItemsRange(int index, IEnumerable<T> collection) + { + if (items.IsReadOnly) + { + ThrowHelper.ThrowNotSupportedException(ExceptionResource.NotSupported_ReadOnlyCollection); + } + + if (collection == null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.list); + } + + if ((uint)index > (uint)items.Count) + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.index, ExceptionResource.ArgumentOutOfRange_ListInsert); + } + + if (GetType() == typeof(Collection<T>) && items is List<T> list) + { + list.InsertRange(index, collection); + } + else + { + foreach (T item in collection) + { + InsertItem(index++, item); + } + } + } + + protected virtual void RemoveItemsRange(int index, int count) + { + if (items.IsReadOnly) + { + ThrowHelper.ThrowNotSupportedException(ExceptionResource.NotSupported_ReadOnlyCollection); + } + + if ((uint)index > (uint)items.Count) + { + ThrowHelper.ThrowArgumentOutOfRange_IndexException(); + } + + if (count < 0) + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count, ExceptionResource.ArgumentOutOfRange_NeedNonNegNum); + } + + if (index > items.Count - count) + { + ThrowHelper.ThrowArgumentException(ExceptionResource.Argument_InvalidOffLen); + } + + if (GetType() == typeof(Collection<T>) && items is List<T> list) + { + list.RemoveRange(index, count); + } + else + { + for (int i = 0; i < count; i++) + { + RemoveItem(index); + } + } + } + + protected virtual void ReplaceItemsRange(int index, int count, IEnumerable<T> collection) + { + RemoveItemsRange(index, count); + InsertItemsRange(index, collection); + } + bool ICollection<T>.IsReadOnly { get diff --git a/src/System.Private.CoreLib/shared/System/ComponentModel/DefaultValueAttribute.cs b/src/System.Private.CoreLib/shared/System/ComponentModel/DefaultValueAttribute.cs index d506ffae4c..136e4324ed 100644 --- a/src/System.Private.CoreLib/shared/System/ComponentModel/DefaultValueAttribute.cs +++ b/src/System.Private.CoreLib/shared/System/ComponentModel/DefaultValueAttribute.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System.Globalization; +using System.Reflection; using System.Threading; namespace System.ComponentModel @@ -64,13 +65,21 @@ namespace System.ComponentModel if (s_convertFromInvariantString == null) { Type typeDescriptorType = Type.GetType("System.ComponentModel.TypeDescriptor, System.ComponentModel.TypeConverter", throwOnError: false); - Volatile.Write(ref s_convertFromInvariantString, typeDescriptorType == null ? new object() : Delegate.CreateDelegate(typeof(Func<Type, string, object>), typeDescriptorType, "ConvertFromInvariantString", ignoreCase: false)); + MethodInfo mi = typeDescriptorType?.GetMethod("ConvertFromInvariantString", BindingFlags.NonPublic | BindingFlags.Static); + Volatile.Write(ref s_convertFromInvariantString, mi == null ? new object() : mi.CreateDelegate(typeof(Func<Type, string, object>))); } if (!(s_convertFromInvariantString is Func<Type, string, object> convertFromInvariantString)) return false; - conversionResult = convertFromInvariantString(typeToConvert, stringValue); + try + { + conversionResult = convertFromInvariantString(typeToConvert, stringValue); + } + catch + { + return false; + } return true; } diff --git a/src/System.Private.CoreLib/shared/System/Globalization/DateTimeFormatInfoScanner.cs b/src/System.Private.CoreLib/shared/System/Globalization/DateTimeFormatInfoScanner.cs index ae3c1c24fc..d18bd3e4a8 100644 --- a/src/System.Private.CoreLib/shared/System/Globalization/DateTimeFormatInfoScanner.cs +++ b/src/System.Private.CoreLib/shared/System/Globalization/DateTimeFormatInfoScanner.cs @@ -635,7 +635,7 @@ namespace System.Globalization // Check each string for (int i = 0; i < array1.Length; i++) { - if (!array1[i].Equals(array2[i])) + if (array1[i] != array2[i]) { return false; } diff --git a/src/System.Private.CoreLib/shared/System/Number.NumberToFloatingPointBits.cs b/src/System.Private.CoreLib/shared/System/Number.NumberToFloatingPointBits.cs index 37609f1818..ac4c7727f3 100644 --- a/src/System.Private.CoreLib/shared/System/Number.NumberToFloatingPointBits.cs +++ b/src/System.Private.CoreLib/shared/System/Number.NumberToFloatingPointBits.cs @@ -336,6 +336,8 @@ namespace System Debug.Assert(number.Scale <= FloatingPointMaxExponent); Debug.Assert(number.Scale >= FloatingPointMinExponent); + Debug.Assert(number.DigitsCount != 0); + // The input is of the form 0.Mantissa x 10^Exponent, where 'Mantissa' are // the decimal digits of the mantissa and 'Exponent' is the decimal exponent. // We decompose the mantissa into two parts: an integer part and a fractional @@ -363,11 +365,6 @@ namespace System byte* src = number.GetDigitsPointer(); - if (totalDigits == 0) - { - return info.ZeroBits; - } - if ((info.DenormalMantissaBits == 23) && (totalDigits <= 7) && (fastExponent <= 10)) { // It is only valid to do this optimization for single-precision floating-point diff --git a/src/System.Private.CoreLib/shared/System/Number.Parsing.cs b/src/System.Private.CoreLib/shared/System/Number.Parsing.cs index 37facda3bb..fc845a1ea8 100644 --- a/src/System.Private.CoreLib/shared/System/Number.Parsing.cs +++ b/src/System.Private.CoreLib/shared/System/Number.Parsing.cs @@ -1975,13 +1975,13 @@ namespace System number.CheckConsistency(); double result; - if (number.Scale > DoubleMaxExponent) + if ((number.DigitsCount == 0) || (number.Scale < DoubleMinExponent)) { - result = double.PositiveInfinity; + result = 0; } - else if (number.Scale < DoubleMinExponent) + else if (number.Scale > DoubleMaxExponent) { - result = 0; + result = double.PositiveInfinity; } else { @@ -1997,13 +1997,13 @@ namespace System number.CheckConsistency(); float result; - if (number.Scale > SingleMaxExponent) + if ((number.DigitsCount == 0) || (number.Scale < SingleMinExponent)) { - result = float.PositiveInfinity; + result = 0; } - else if (number.Scale < SingleMinExponent) + else if (number.Scale > SingleMaxExponent) { - result = 0; + result = float.PositiveInfinity; } else { diff --git a/src/System.Private.CoreLib/shared/System/Resources/ManifestBasedResourceGroveler.cs b/src/System.Private.CoreLib/shared/System/Resources/ManifestBasedResourceGroveler.cs index f2ef9dda1f..485417d52a 100644 --- a/src/System.Private.CoreLib/shared/System/Resources/ManifestBasedResourceGroveler.cs +++ b/src/System.Private.CoreLib/shared/System/Resources/ManifestBasedResourceGroveler.cs @@ -237,10 +237,23 @@ namespace System.Resources } else { - Type readerType = Type.GetType(readerTypeName, throwOnError: true); - object[] args = new object[1]; - args[0] = store; - IResourceReader reader = (IResourceReader)Activator.CreateInstance(readerType, args); + IResourceReader reader; + + // Permit deserialization as long as the default ResourceReader is used + if (ResourceManager.IsDefaultType(readerTypeName, ResourceManager.ResReaderTypeName)) + { + reader = new ResourceReader( + store, + new Dictionary<string, ResourceLocator>(FastResourceComparer.Default), + permitDeserialization: true); + } + else + { + Type readerType = Type.GetType(readerTypeName, throwOnError: true); + object[] args = new object[1]; + args[0] = store; + reader = (IResourceReader)Activator.CreateInstance(readerType, args); + } object[] resourceSetArgs = new object[1]; resourceSetArgs[0] = reader; diff --git a/src/System.Private.CoreLib/shared/System/Runtime/CompilerServices/RuntimeHelpers.cs b/src/System.Private.CoreLib/shared/System/Runtime/CompilerServices/RuntimeHelpers.cs new file mode 100644 index 0000000000..fcb69eceec --- /dev/null +++ b/src/System.Private.CoreLib/shared/System/Runtime/CompilerServices/RuntimeHelpers.cs @@ -0,0 +1,66 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Runtime.Serialization; + +namespace System.Runtime.CompilerServices +{ + public static partial class RuntimeHelpers + { + public delegate void TryCode(object userData); + + public delegate void CleanupCode(object userData, bool exceptionThrown); + + /// <summary> + /// GetSubArray helper method for the compiler to slice an array using a range. + /// </summary> + public static T[] GetSubArray<T>(T[] array, Range range) + { + Type elementType = array.GetType().GetElementType(); + Span<T> source = array.AsSpan(range); + + if (elementType.IsValueType) + { + return source.ToArray(); + } + else + { + T[] newArray = (T[])Array.CreateInstance(elementType, source.Length); + source.CopyTo(newArray); + return newArray; + } + } + + public static object GetUninitializedObject(Type type) + { + if (type is null) + { + throw new ArgumentNullException(nameof(type), SR.ArgumentNull_Type); + } + + if (!type.IsRuntimeImplemented()) + { + throw new SerializationException(SR.Format(SR.Serialization_InvalidType, type.ToString())); + } + + return GetUninitializedObjectInternal(type); + } + + public static void PrepareContractedDelegate(Delegate d) + { + } + + public static void ProbeForSufficientStack() + { + } + + public static void PrepareConstrainedRegions() + { + } + + public static void PrepareConstrainedRegionsNoOP() + { + } + } +}
\ No newline at end of file diff --git a/src/System.Private.CoreLib/shared/System/String.cs b/src/System.Private.CoreLib/shared/System/String.cs index 22f830a0e4..49afbc8c8c 100644 --- a/src/System.Private.CoreLib/shared/System/String.cs +++ b/src/System.Private.CoreLib/shared/System/String.cs @@ -480,7 +480,7 @@ namespace System Debug.Assert(byteLength >= 0); // Get our string length - int stringLength = encoding.GetCharCount(bytes, byteLength, null); + int stringLength = encoding.GetCharCount(bytes, byteLength); Debug.Assert(stringLength >= 0, "stringLength >= 0"); // They gave us an empty string if they needed one @@ -491,7 +491,7 @@ namespace System string s = FastAllocateString(stringLength); fixed (char* pTempChars = &s._firstChar) { - int doubleCheck = encoding.GetChars(bytes, byteLength, pTempChars, stringLength, null); + int doubleCheck = encoding.GetChars(bytes, byteLength, pTempChars, stringLength); Debug.Assert(stringLength == doubleCheck, "Expected encoding.GetChars to return same length as encoding.GetCharCount"); } diff --git a/src/System.Private.CoreLib/shared/System/Text/ASCIIEncoding.cs b/src/System.Private.CoreLib/shared/System/Text/ASCIIEncoding.cs index 217d934677..8cf1f57ccb 100644 --- a/src/System.Private.CoreLib/shared/System/Text/ASCIIEncoding.cs +++ b/src/System.Private.CoreLib/shared/System/Text/ASCIIEncoding.cs @@ -2,8 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; +using System.Buffers; using System.Diagnostics; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; namespace System.Text @@ -18,10 +19,30 @@ namespace System.Text // Note: IsAlwaysNormalized remains false because 1/2 the code points are unassigned, so they'd // use fallbacks, and we cannot guarantee that fallbacks are normalized. - public class ASCIIEncoding : Encoding + public partial class ASCIIEncoding : Encoding { - // Allow for devirtualization (see https://github.com/dotnet/coreclr/pull/9230) - internal sealed class ASCIIEncodingSealed : ASCIIEncoding { } + // This specialized sealed type has two benefits: + // 1) it allows for devirtualization (see https://github.com/dotnet/coreclr/pull/9230), and + // 2) it allows us to provide highly optimized implementations of certain routines because + // we can make assumptions about the fallback mechanisms in use (in particular, always + // replace with "?"). + // + // (We don't take advantage of #2 yet, but we can do so in the future because the implementation + // of cloning below allows us to make assumptions about the behaviors of the sealed type.) + internal sealed class ASCIIEncodingSealed : ASCIIEncoding + { + public override object Clone() + { + // The base implementation of Encoding.Clone calls object.MemberwiseClone and marks the new object mutable. + // We don't want to do this because it violates the invariants we have set for the sealed type. + // Instead, we'll create a new instance of the base ASCIIEncoding type and mark it mutable. + + return new ASCIIEncoding() + { + IsReadOnly = false + }; + } + } // Used by Encoding.ASCII for lazy initialization // The initialization code will not be run until a static member of the class is referenced @@ -58,22 +79,26 @@ namespace System.Text public override unsafe int GetByteCount(char[] chars, int index, int count) { // Validate input parameters - if (chars == null) - throw new ArgumentNullException(nameof(chars), SR.ArgumentNull_Array); - if (index < 0 || count < 0) - throw new ArgumentOutOfRangeException((index < 0 ? nameof(index) : nameof(count)), SR.ArgumentOutOfRange_NeedNonNegNum); + if (chars is null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.chars, ExceptionResource.ArgumentNull_Array); + } - if (chars.Length - index < count) - throw new ArgumentOutOfRangeException(nameof(chars), SR.ArgumentOutOfRange_IndexCountBuffer); + if ((index | count) < 0) + { + ThrowHelper.ThrowArgumentOutOfRangeException((index < 0) ? ExceptionArgument.index : ExceptionArgument.count, ExceptionResource.ArgumentOutOfRange_NeedNonNegNum); + } - // If no input, return 0, avoid fixed empty array problem - if (count == 0) - return 0; + if (chars.Length - index < count) + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.chars, ExceptionResource.ArgumentOutOfRange_IndexCountBuffer); + } - // Just call the pointer version fixed (char* pChars = chars) - return GetByteCount(pChars + index, count, null); + { + return GetByteCountCommon(pChars + index, count); + } } // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS) @@ -83,12 +108,17 @@ namespace System.Text public override unsafe int GetByteCount(string chars) { - // Validate input - if (chars==null) - throw new ArgumentNullException(nameof(chars)); + // Validate input parameters + + if (chars is null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.chars); + } fixed (char* pChars = chars) - return GetByteCount(pChars, chars.Length, null); + { + return GetByteCountCommon(pChars, chars.Length); + } } // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS) @@ -99,22 +129,81 @@ namespace System.Text public override unsafe int GetByteCount(char* chars, int count) { // Validate Parameters + if (chars == null) - throw new ArgumentNullException(nameof(chars), SR.ArgumentNull_Array); + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.chars); + } if (count < 0) - throw new ArgumentOutOfRangeException(nameof(count), SR.ArgumentOutOfRange_NeedNonNegNum); + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count, ExceptionResource.ArgumentOutOfRange_NeedNonNegNum); + } - // Call it with empty encoder - return GetByteCount(chars, count, null); + return GetByteCountCommon(chars, count); } public override unsafe int GetByteCount(ReadOnlySpan<char> chars) { - fixed (char* charsPtr = &MemoryMarshal.GetNonNullPinnableReference(chars)) + // It's ok for us to pass null pointers down to the workhorse below. + + fixed (char* charsPtr = &MemoryMarshal.GetReference(chars)) + { + return GetByteCountCommon(charsPtr, chars.Length); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe int GetByteCountCommon(char* pChars, int charCount) + { + // Common helper method for all non-EncoderNLS entry points to GetByteCount. + // A modification of this method should be copied in to each of the supported encodings: ASCII, UTF8, UTF16, UTF32. + + Debug.Assert(charCount >= 0, "Caller should't specify negative length buffer."); + Debug.Assert(pChars != null || charCount == 0, "Input pointer shouldn't be null if non-zero length specified."); + + // First call into the fast path. + + int totalByteCount = GetByteCountFast(pChars, charCount, EncoderFallback, out int charsConsumed); + + if (charsConsumed != charCount) + { + // If there's still data remaining in the source buffer, go down the fallback path. + // We need to check for integer overflow since the fallback could change the required + // output count in unexpected ways. + + totalByteCount += GetByteCountWithFallback(pChars, charCount, charsConsumed); + if (totalByteCount < 0) + { + ThrowConversionOverflow(); + } + } + + return totalByteCount; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] // called directly by GetByteCountCommon + private protected sealed override unsafe int GetByteCountFast(char* pChars, int charsLength, EncoderFallback fallback, out int charsConsumed) + { + // First: Can we short-circuit the entire calculation? + // If an EncoderReplacementFallback is in use, all non-ASCII chars + // (including surrogate halves) are replaced with the default string. + // If the default string consists of a single ASCII value, then we + // know there's a 1:1 char->byte transcoding in all cases. + + int byteCount = charsLength; + + if (!(fallback is EncoderReplacementFallback replacementFallback + && replacementFallback.MaxCharCount == 1 + && replacementFallback.DefaultString[0] <= 0x7F)) { - return GetByteCount(charsPtr, chars.Length, encoder: null); + // Unrecognized fallback mechanism - count chars manually. + + byteCount = (int)ASCIIUtility.GetIndexOfFirstNonAsciiChar(pChars, (uint)charsLength); } + + charsConsumed = byteCount; + return byteCount; } // Parent method is safe. @@ -125,22 +214,37 @@ namespace System.Text public override unsafe int GetBytes(string chars, int charIndex, int charCount, byte[] bytes, int byteIndex) { - if (chars == null || bytes == null) - throw new ArgumentNullException((chars == null ? nameof(chars) : nameof(bytes)), SR.ArgumentNull_Array); + // Validate Parameters - if (charIndex < 0 || charCount < 0) - throw new ArgumentOutOfRangeException((charIndex < 0 ? nameof(charIndex) : nameof(charCount)), SR.ArgumentOutOfRange_NeedNonNegNum); + if (chars is null || bytes is null) + { + ThrowHelper.ThrowArgumentNullException( + argument: (chars is null) ? ExceptionArgument.chars : ExceptionArgument.bytes, + resource: ExceptionResource.ArgumentNull_Array); + } - if (chars.Length - charIndex < charCount) - throw new ArgumentOutOfRangeException(nameof(chars), SR.ArgumentOutOfRange_IndexCount); + if ((charIndex | charCount) < 0) + { + ThrowHelper.ThrowArgumentOutOfRangeException( + argument: (charIndex < 0) ? ExceptionArgument.charIndex : ExceptionArgument.charCount, + resource: ExceptionResource.ArgumentOutOfRange_NeedNonNegNum); + } - if (byteIndex < 0 || byteIndex > bytes.Length) - throw new ArgumentOutOfRangeException(nameof(byteIndex), SR.ArgumentOutOfRange_Index); + if (chars.Length - charIndex < charCount) + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.chars, ExceptionResource.ArgumentOutOfRange_IndexCount); + } - int byteCount = bytes.Length - byteIndex; + if ((uint)byteIndex > bytes.Length) + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.byteIndex, ExceptionResource.ArgumentOutOfRange_Index); + } - fixed (char* pChars = chars) fixed (byte* pBytes = &MemoryMarshal.GetReference((Span<byte>)bytes)) - return GetBytes(pChars + charIndex, charCount, pBytes + byteIndex, byteCount, null); + fixed (char* pChars = chars) + fixed (byte* pBytes = bytes) + { + return GetBytesCommon(pChars + charIndex, charCount, pBytes + byteIndex, bytes.Length - byteIndex); + } } // Encodes a range of characters in a character array into a range of bytes @@ -161,28 +265,36 @@ namespace System.Text byte[] bytes, int byteIndex) { // Validate parameters - if (chars == null || bytes == null) - throw new ArgumentNullException((chars == null ? nameof(chars) : nameof(bytes)), SR.ArgumentNull_Array); - - if (charIndex < 0 || charCount < 0) - throw new ArgumentOutOfRangeException((charIndex < 0 ? nameof(charIndex) : nameof(charCount)), SR.ArgumentOutOfRange_NeedNonNegNum); - if (chars.Length - charIndex < charCount) - throw new ArgumentOutOfRangeException(nameof(chars), SR.ArgumentOutOfRange_IndexCountBuffer); + if (chars is null || bytes is null) + { + ThrowHelper.ThrowArgumentNullException( + argument: (chars is null) ? ExceptionArgument.chars : ExceptionArgument.bytes, + resource: ExceptionResource.ArgumentNull_Array); + } - if (byteIndex < 0 || byteIndex > bytes.Length) - throw new ArgumentOutOfRangeException(nameof(byteIndex), SR.ArgumentOutOfRange_Index); + if ((charIndex | charCount) < 0) + { + ThrowHelper.ThrowArgumentOutOfRangeException( + argument: (charIndex < 0) ? ExceptionArgument.charIndex : ExceptionArgument.charCount, + resource: ExceptionResource.ArgumentOutOfRange_NeedNonNegNum); + } - // If nothing to encode return 0 - if (charCount == 0) - return 0; + if (chars.Length - charIndex < charCount) + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.chars, ExceptionResource.ArgumentOutOfRange_IndexCount); + } - // Just call pointer version - int byteCount = bytes.Length - byteIndex; + if ((uint)byteIndex > bytes.Length) + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.byteIndex, ExceptionResource.ArgumentOutOfRange_Index); + } - fixed (char* pChars = chars) fixed (byte* pBytes = &MemoryMarshal.GetReference((Span<byte>)bytes)) - // Remember that byteCount is # to decode, not size of array. - return GetBytes(pChars + charIndex, charCount, pBytes + byteIndex, byteCount, null); + fixed (char* pChars = chars) + fixed (byte* pBytes = bytes) + { + return GetBytesCommon(pChars + charIndex, charCount, pBytes + byteIndex, bytes.Length - byteIndex); + } } // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS) @@ -193,21 +305,123 @@ namespace System.Text public override unsafe int GetBytes(char* chars, int charCount, byte* bytes, int byteCount) { // Validate Parameters - if (bytes == null || chars == null) - throw new ArgumentNullException(bytes == null ? nameof(bytes) : nameof(chars), SR.ArgumentNull_Array); - if (charCount < 0 || byteCount < 0) - throw new ArgumentOutOfRangeException((charCount < 0 ? nameof(charCount) : nameof(byteCount)), SR.ArgumentOutOfRange_NeedNonNegNum); + if (chars == null || bytes == null) + { + ThrowHelper.ThrowArgumentNullException( + argument: (chars is null) ? ExceptionArgument.chars : ExceptionArgument.bytes, + resource: ExceptionResource.ArgumentNull_Array); + } - return GetBytes(chars, charCount, bytes, byteCount, null); + if ((charCount | byteCount) < 0) + { + ThrowHelper.ThrowArgumentOutOfRangeException( + argument: (charCount < 0) ? ExceptionArgument.charCount : ExceptionArgument.byteCount, + resource: ExceptionResource.ArgumentOutOfRange_NeedNonNegNum); + } + + return GetBytesCommon(chars, charCount, bytes, byteCount); } public override unsafe int GetBytes(ReadOnlySpan<char> chars, Span<byte> bytes) { - fixed (char* charsPtr = &MemoryMarshal.GetNonNullPinnableReference(chars)) - fixed (byte* bytesPtr = &MemoryMarshal.GetNonNullPinnableReference(bytes)) + // It's ok for us to operate on null / empty spans. + + fixed (char* charsPtr = &MemoryMarshal.GetReference(chars)) + fixed (byte* bytesPtr = &MemoryMarshal.GetReference(bytes)) { - return GetBytes(charsPtr, chars.Length, bytesPtr, bytes.Length, encoder: null); + return GetBytesCommon(charsPtr, chars.Length, bytesPtr, bytes.Length); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe int GetBytesCommon(char* pChars, int charCount, byte* pBytes, int byteCount) + { + // Common helper method for all non-EncoderNLS entry points to GetBytes. + // A modification of this method should be copied in to each of the supported encodings: ASCII, UTF8, UTF16, UTF32. + + Debug.Assert(charCount >= 0, "Caller should't specify negative length buffer."); + Debug.Assert(pChars != null || charCount == 0, "Input pointer shouldn't be null if non-zero length specified."); + Debug.Assert(byteCount >= 0, "Caller should't specify negative length buffer."); + Debug.Assert(pBytes != null || byteCount == 0, "Input pointer shouldn't be null if non-zero length specified."); + + // First call into the fast path. + + int bytesWritten = GetBytesFast(pChars, charCount, pBytes, byteCount, out int charsConsumed); + + if (charsConsumed == charCount) + { + // All elements converted - return immediately. + + return bytesWritten; + } + else + { + // Simple narrowing conversion couldn't operate on entire buffer - invoke fallback. + + return GetBytesWithFallback(pChars, charCount, pBytes, byteCount, charsConsumed, bytesWritten); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] // called directly by GetBytesCommon + private protected sealed override unsafe int GetBytesFast(char* pChars, int charsLength, byte* pBytes, int bytesLength, out int charsConsumed) + { + int bytesWritten = (int)ASCIIUtility.NarrowUtf16ToAscii(pChars, pBytes, (uint)Math.Min(charsLength, bytesLength)); + + charsConsumed = bytesWritten; + return bytesWritten; + } + + private protected sealed override unsafe int GetBytesWithFallback(ReadOnlySpan<char> chars, int originalCharsLength, Span<byte> bytes, int originalBytesLength, EncoderNLS encoder) + { + // We special-case EncoderReplacementFallback if it's telling us to write a single ASCII char, + // since we believe this to be relatively common and we can handle it more efficiently than + // the base implementation. + + if (((encoder is null) ? this.EncoderFallback : encoder.Fallback) is EncoderReplacementFallback replacementFallback + && replacementFallback.MaxCharCount == 1 + && replacementFallback.DefaultString[0] <= 0x7F) + { + byte replacementByte = (byte)replacementFallback.DefaultString[0]; + + int numElementsToConvert = Math.Min(chars.Length, bytes.Length); + int idx = 0; + + fixed (char* pChars = &MemoryMarshal.GetReference(chars)) + fixed (byte* pBytes = &MemoryMarshal.GetReference(bytes)) + { + // In a loop, replace the non-convertible data, then bulk-convert as much as we can. + + while (idx < numElementsToConvert) + { + pBytes[idx++] = replacementByte; + + if (idx < numElementsToConvert) + { + idx += (int)ASCIIUtility.NarrowUtf16ToAscii(&pChars[idx], &pBytes[idx], (uint)(numElementsToConvert - idx)); + } + + Debug.Assert(idx <= numElementsToConvert, "Somehow went beyond bounds of source or destination buffer?"); + } + } + + // Slice off how much we consumed / wrote. + + chars = chars.Slice(numElementsToConvert); + bytes = bytes.Slice(numElementsToConvert); + } + + // If we couldn't go through our fast fallback mechanism, or if we still have leftover + // data because we couldn't consume everything in the loop above, we need to go down the + // slow fallback path. + + if (chars.IsEmpty) + { + return originalBytesLength - bytes.Length; // total number of bytes written + } + else + { + return base.GetBytesWithFallback(chars, originalCharsLength, bytes, originalBytesLength, encoder); } } @@ -222,22 +436,26 @@ namespace System.Text public override unsafe int GetCharCount(byte[] bytes, int index, int count) { // Validate Parameters - if (bytes == null) - throw new ArgumentNullException(nameof(bytes), SR.ArgumentNull_Array); - if (index < 0 || count < 0) - throw new ArgumentOutOfRangeException((index < 0 ? nameof(index) : nameof(count)), SR.ArgumentOutOfRange_NeedNonNegNum); + if (bytes is null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.bytes, ExceptionResource.ArgumentNull_Array); + } - if (bytes.Length - index < count) - throw new ArgumentOutOfRangeException(nameof(bytes), SR.ArgumentOutOfRange_IndexCountBuffer); + if ((index | count) < 0) + { + ThrowHelper.ThrowArgumentOutOfRangeException((index < 0) ? ExceptionArgument.index : ExceptionArgument.count, ExceptionResource.ArgumentOutOfRange_NeedNonNegNum); + } - // If no input just return 0, fixed doesn't like 0 length arrays - if (count == 0) - return 0; + if (bytes.Length - index < count) + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.bytes, ExceptionResource.ArgumentOutOfRange_IndexCountBuffer); + } - // Just call pointer version fixed (byte* pBytes = bytes) - return GetCharCount(pBytes + index, count, null); + { + return GetCharCountCommon(pBytes + index, count); + } } // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS) @@ -248,673 +466,367 @@ namespace System.Text public override unsafe int GetCharCount(byte* bytes, int count) { // Validate Parameters + if (bytes == null) - throw new ArgumentNullException(nameof(bytes), SR.ArgumentNull_Array); + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.bytes, ExceptionResource.ArgumentNull_Array); + } if (count < 0) - throw new ArgumentOutOfRangeException(nameof(count), SR.ArgumentOutOfRange_NeedNonNegNum); + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count, ExceptionResource.ArgumentOutOfRange_NeedNonNegNum); + } - return GetCharCount(bytes, count, null); + return GetCharCountCommon(bytes, count); } public override unsafe int GetCharCount(ReadOnlySpan<byte> bytes) { - fixed (byte* bytesPtr = &MemoryMarshal.GetNonNullPinnableReference(bytes)) + // It's ok for us to pass null pointers down to the workhorse routine. + + fixed (byte* bytesPtr = &MemoryMarshal.GetReference(bytes)) { - return GetCharCount(bytesPtr, bytes.Length, decoder: null); + return GetCharCountCommon(bytesPtr, bytes.Length); } } - // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS) - // So if you fix this, fix the others. Currently those include: - // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding - // parent method is safe - - public override unsafe int GetChars(byte[] bytes, int byteIndex, int byteCount, - char[] chars, int charIndex) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe int GetCharCountCommon(byte* pBytes, int byteCount) { - // Validate Parameters - if (bytes == null || chars == null) - throw new ArgumentNullException(bytes == null ? nameof(bytes) : nameof(chars), SR.ArgumentNull_Array); + // Common helper method for all non-DecoderNLS entry points to GetCharCount. + // A modification of this method should be copied in to each of the supported encodings: ASCII, UTF8, UTF16, UTF32. - if (byteIndex < 0 || byteCount < 0) - throw new ArgumentOutOfRangeException((byteIndex < 0 ? nameof(byteIndex) : nameof(byteCount)), SR.ArgumentOutOfRange_NeedNonNegNum); + Debug.Assert(byteCount >= 0, "Caller should't specify negative length buffer."); + Debug.Assert(pBytes != null || byteCount == 0, "Input pointer shouldn't be null if non-zero length specified."); - if ( bytes.Length - byteIndex < byteCount) - throw new ArgumentOutOfRangeException(nameof(bytes), SR.ArgumentOutOfRange_IndexCountBuffer); + // First call into the fast path. - if (charIndex < 0 || charIndex > chars.Length) - throw new ArgumentOutOfRangeException(nameof(charIndex), SR.ArgumentOutOfRange_Index); + int totalCharCount = GetCharCountFast(pBytes, byteCount, DecoderFallback, out int bytesConsumed); - // If no input, return 0 & avoid fixed problem - if (byteCount == 0) - return 0; + if (bytesConsumed != byteCount) + { + // If there's still data remaining in the source buffer, go down the fallback path. + // We need to check for integer overflow since the fallback could change the required + // output count in unexpected ways. - // Just call pointer version - int charCount = chars.Length - charIndex; + totalCharCount += GetCharCountWithFallback(pBytes, byteCount, bytesConsumed); + if (totalCharCount < 0) + { + ThrowConversionOverflow(); + } + } - fixed (byte* pBytes = bytes) fixed (char* pChars = &MemoryMarshal.GetReference((Span<char>)chars)) - // Remember that charCount is # to decode, not size of array - return GetChars(pBytes + byteIndex, byteCount, pChars + charIndex, charCount, null); + return totalCharCount; } - // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS) - // So if you fix this, fix the others. Currently those include: - // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding - - [CLSCompliant(false)] - public unsafe override int GetChars(byte* bytes, int byteCount, char* chars, int charCount) + [MethodImpl(MethodImplOptions.AggressiveInlining)] // called directly by GetCharCountCommon + private protected sealed override unsafe int GetCharCountFast(byte* pBytes, int bytesLength, DecoderFallback fallback, out int bytesConsumed) { - // Validate Parameters - if (bytes == null || chars == null) - throw new ArgumentNullException(bytes == null ? nameof(bytes) : nameof(chars), SR.ArgumentNull_Array); + // First: Can we short-circuit the entire calculation? + // If a DecoderReplacementFallback is in use, all non-ASCII bytes are replaced with + // the default string. If the default string consists of a single BMP value, then we + // know there's a 1:1 byte->char transcoding in all cases. - if (charCount < 0 || byteCount < 0) - throw new ArgumentOutOfRangeException((charCount < 0 ? nameof(charCount) : nameof(byteCount)), SR.ArgumentOutOfRange_NeedNonNegNum); + int charCount = bytesLength; - return GetChars(bytes, byteCount, chars, charCount, null); - } - - public override unsafe int GetChars(ReadOnlySpan<byte> bytes, Span<char> chars) - { - fixed (byte* bytesPtr = &MemoryMarshal.GetNonNullPinnableReference(bytes)) - fixed (char* charsPtr = &MemoryMarshal.GetNonNullPinnableReference(chars)) + if (!(fallback is DecoderReplacementFallback replacementFallback) || replacementFallback.MaxCharCount != 1) { - return GetChars(bytesPtr, bytes.Length, charsPtr, chars.Length, decoder: null); + // Unrecognized fallback mechanism - count bytes manually. + + charCount = (int)ASCIIUtility.GetIndexOfFirstNonAsciiByte(pBytes, (uint)bytesLength); } + + bytesConsumed = charCount; + return charCount; } - // Returns a string containing the decoded representation of a range of - // bytes in a byte array. - // // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS) // So if you fix this, fix the others. Currently those include: // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding // parent method is safe - public override unsafe string GetString(byte[] bytes, int byteIndex, int byteCount) + public override unsafe int GetChars(byte[] bytes, int byteIndex, int byteCount, + char[] chars, int charIndex) { // Validate Parameters - if (bytes == null) - throw new ArgumentNullException(nameof(bytes), SR.ArgumentNull_Array); - if (byteIndex < 0 || byteCount < 0) - throw new ArgumentOutOfRangeException((byteIndex < 0 ? nameof(byteIndex) : nameof(byteCount)), SR.ArgumentOutOfRange_NeedNonNegNum); + if (bytes is null || chars is null) + { + ThrowHelper.ThrowArgumentNullException( + argument: (bytes is null) ? ExceptionArgument.bytes : ExceptionArgument.chars, + resource: ExceptionResource.ArgumentNull_Array); + } + if ((byteIndex | byteCount) < 0) + { + ThrowHelper.ThrowArgumentOutOfRangeException( + argument: (byteIndex < 0) ? ExceptionArgument.byteIndex : ExceptionArgument.byteCount, + resource: ExceptionResource.ArgumentOutOfRange_NeedNonNegNum); + } if (bytes.Length - byteIndex < byteCount) - throw new ArgumentOutOfRangeException(nameof(bytes), SR.ArgumentOutOfRange_IndexCountBuffer); + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.bytes, ExceptionResource.ArgumentOutOfRange_IndexCountBuffer); + } - // Avoid problems with empty input buffer - if (byteCount == 0) return string.Empty; + if ((uint)charIndex > (uint)chars.Length) + { + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.charIndex, ExceptionResource.ArgumentOutOfRange_Index); + } fixed (byte* pBytes = bytes) - return string.CreateStringFromEncoding( - pBytes + byteIndex, byteCount, this); + fixed (char* pChars = chars) + { + return GetCharsCommon(pBytes + byteIndex, byteCount, pChars + charIndex, chars.Length - charIndex); + } } - // - // End of standard methods copied from EncodingNLS.cs - // + // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS) + // So if you fix this, fix the others. Currently those include: + // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding - // GetByteCount - // Note: We start by assuming that the output will be the same as count. Having - // an encoder or fallback may change that assumption - internal sealed override unsafe int GetByteCount(char* chars, int charCount, EncoderNLS encoder) + [CLSCompliant(false)] + public unsafe override int GetChars(byte* bytes, int byteCount, char* chars, int charCount) { - // Just need to ASSERT, this is called by something else internal that checked parameters already - Debug.Assert(charCount >= 0, "[ASCIIEncoding.GetByteCount]count is negative"); - Debug.Assert(chars != null, "[ASCIIEncoding.GetByteCount]chars is null"); - - // Assert because we shouldn't be able to have a null encoder. - Debug.Assert(encoderFallback != null, "[ASCIIEncoding.GetByteCount]Attempting to use null fallback encoder"); - - char charLeftOver = (char)0; - EncoderReplacementFallback fallback = null; - - // Start by assuming default count, then +/- for fallback characters - char* charEnd = chars + charCount; - - // For fallback we may need a fallback buffer, we know we aren't default fallback. - EncoderFallbackBuffer fallbackBuffer = null; - char* charsForFallback; - - if (encoder != null) - { - charLeftOver = encoder._charLeftOver; - Debug.Assert(charLeftOver == 0 || char.IsHighSurrogate(charLeftOver), - "[ASCIIEncoding.GetByteCount]leftover character should be high surrogate"); - - fallback = encoder.Fallback as EncoderReplacementFallback; - - // We mustn't have left over fallback data when counting - if (encoder.InternalHasFallbackBuffer) - { - // We always need the fallback buffer in get bytes so we can flush any remaining ones if necessary - fallbackBuffer = encoder.FallbackBuffer; - if (fallbackBuffer.Remaining > 0 && encoder._throwOnOverflow) - throw new ArgumentException(SR.Format(SR.Argument_EncoderFallbackNotEmpty, this.EncodingName, encoder.Fallback.GetType())); - - // Set our internal fallback interesting things. - fallbackBuffer.InternalInitialize(chars, charEnd, encoder, false); - } + // Validate Parameters - // Verify that we have no fallbackbuffer, for ASCII its always empty, so just assert - Debug.Assert(!encoder._throwOnOverflow || !encoder.InternalHasFallbackBuffer || - encoder.FallbackBuffer.Remaining == 0, - "[ASCIICodePageEncoding.GetByteCount]Expected empty fallback buffer"); - } - else + if (bytes is null || chars is null) { - fallback = this.EncoderFallback as EncoderReplacementFallback; + ThrowHelper.ThrowArgumentNullException( + argument: (bytes is null) ? ExceptionArgument.bytes : ExceptionArgument.chars, + resource: ExceptionResource.ArgumentNull_Array); } - // If we have an encoder AND we aren't using default fallback, - // then we may have a complicated count. - if (fallback != null && fallback.MaxCharCount == 1) + if ((byteCount | charCount) < 0) { - // Replacement fallback encodes surrogate pairs as two ?? (or two whatever), so return size is always - // same as input size. - // Note that no existing SBCS code pages map code points to supplimentary characters, so this is easy. - - // We could however have 1 extra byte if the last call had an encoder and a funky fallback and - // if we don't use the funky fallback this time. - - // Do we have an extra char left over from last time? - if (charLeftOver > 0) - charCount++; - - return (charCount); + ThrowHelper.ThrowArgumentOutOfRangeException( + argument: (byteCount < 0) ? ExceptionArgument.byteCount : ExceptionArgument.charCount, + resource: ExceptionResource.ArgumentOutOfRange_NeedNonNegNum); } - // Count is more complicated if you have a funky fallback - // For fallback we may need a fallback buffer, we know we're not default fallback - int byteCount = 0; - - // We may have a left over character from last time, try and process it. - if (charLeftOver > 0) - { - Debug.Assert(char.IsHighSurrogate(charLeftOver), "[ASCIIEncoding.GetByteCount]leftover character should be high surrogate"); - Debug.Assert(encoder != null, "[ASCIIEncoding.GetByteCount]Expected encoder"); - - // Since left over char was a surrogate, it'll have to be fallen back. - // Get Fallback - fallbackBuffer = encoder.FallbackBuffer; - fallbackBuffer.InternalInitialize(chars, charEnd, encoder, false); - - // This will fallback a pair if *chars is a low surrogate - charsForFallback = chars; // Avoid passing chars by reference to allow it to be enregistered - fallbackBuffer.InternalFallback(charLeftOver, ref charsForFallback); - chars = charsForFallback; - } + return GetCharsCommon(bytes, byteCount, chars, charCount); + } - // Now we may have fallback char[] already from the encoder + public override unsafe int GetChars(ReadOnlySpan<byte> bytes, Span<char> chars) + { + // It's ok for us to pass null pointers down to the workhorse below. - // Go ahead and do it, including the fallback. - char ch; - while ((ch = (fallbackBuffer == null) ? '\0' : fallbackBuffer.InternalGetNextChar()) != 0 || - chars < charEnd) + fixed (byte* bytesPtr = &MemoryMarshal.GetReference(bytes)) + fixed (char* charsPtr = &MemoryMarshal.GetReference(chars)) { - // First unwind any fallback - if (ch == 0) - { - // No fallback, just get next char - ch = *chars; - chars++; - } - - // Check for fallback, this'll catch surrogate pairs too. - // no chars >= 0x80 are allowed. - if (ch > 0x7f) - { - if (fallbackBuffer == null) - { - // Initialize the buffer - if (encoder == null) - fallbackBuffer = this.encoderFallback.CreateFallbackBuffer(); - else - fallbackBuffer = encoder.FallbackBuffer; - fallbackBuffer.InternalInitialize(charEnd - charCount, charEnd, encoder, false); - } - - // Get Fallback - charsForFallback = chars; // Avoid passing chars by reference to allow it to be enregistered - fallbackBuffer.InternalFallback(ch, ref charsForFallback); - chars = charsForFallback; - continue; - } - - // We'll use this one - byteCount++; + return GetCharsCommon(bytesPtr, bytes.Length, charsPtr, chars.Length); } - - Debug.Assert(fallbackBuffer == null || fallbackBuffer.Remaining == 0, - "[ASCIIEncoding.GetByteCount]Expected Empty fallback buffer"); - - return byteCount; } - internal sealed override unsafe int GetBytes( - char* chars, int charCount, byte* bytes, int byteCount, EncoderNLS encoder) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe int GetCharsCommon(byte* pBytes, int byteCount, char* pChars, int charCount) { - // Just need to ASSERT, this is called by something else internal that checked parameters already - Debug.Assert(bytes != null, "[ASCIIEncoding.GetBytes]bytes is null"); - Debug.Assert(byteCount >= 0, "[ASCIIEncoding.GetBytes]byteCount is negative"); - Debug.Assert(chars != null, "[ASCIIEncoding.GetBytes]chars is null"); - Debug.Assert(charCount >= 0, "[ASCIIEncoding.GetBytes]charCount is negative"); + // Common helper method for all non-DecoderNLS entry points to GetChars. + // A modification of this method should be copied in to each of the supported encodings: ASCII, UTF8, UTF16, UTF32. - // Assert because we shouldn't be able to have a null encoder. - Debug.Assert(encoderFallback != null, "[ASCIIEncoding.GetBytes]Attempting to use null encoder fallback"); + Debug.Assert(byteCount >= 0, "Caller should't specify negative length buffer."); + Debug.Assert(pBytes != null || byteCount == 0, "Input pointer shouldn't be null if non-zero length specified."); + Debug.Assert(charCount >= 0, "Caller should't specify negative length buffer."); + Debug.Assert(pChars != null || charCount == 0, "Input pointer shouldn't be null if non-zero length specified."); - // Get any left over characters - char charLeftOver = (char)0; - EncoderReplacementFallback fallback = null; + // First call into the fast path. - // For fallback we may need a fallback buffer, we know we aren't default fallback. - EncoderFallbackBuffer fallbackBuffer = null; - char* charsForFallback; + int charsWritten = GetCharsFast(pBytes, byteCount, pChars, charCount, out int bytesConsumed); - // prepare our end - char* charEnd = chars + charCount; - byte* byteStart = bytes; - char* charStart = chars; - - if (encoder != null) + if (bytesConsumed == byteCount) { - charLeftOver = encoder._charLeftOver; - fallback = encoder.Fallback as EncoderReplacementFallback; - - // We mustn't have left over fallback data when counting - if (encoder.InternalHasFallbackBuffer) - { - // We always need the fallback buffer in get bytes so we can flush any remaining ones if necessary - fallbackBuffer = encoder.FallbackBuffer; - if (fallbackBuffer.Remaining > 0 && encoder._throwOnOverflow) - throw new ArgumentException(SR.Format(SR.Argument_EncoderFallbackNotEmpty, this.EncodingName, encoder.Fallback.GetType())); - - // Set our internal fallback interesting things. - fallbackBuffer.InternalInitialize(charStart, charEnd, encoder, true); - } + // All elements converted - return immediately. - Debug.Assert(charLeftOver == 0 || char.IsHighSurrogate(charLeftOver), - "[ASCIIEncoding.GetBytes]leftover character should be high surrogate"); - - // Verify that we have no fallbackbuffer, for ASCII its always empty, so just assert - Debug.Assert(!encoder._throwOnOverflow || !encoder.InternalHasFallbackBuffer || - encoder.FallbackBuffer.Remaining == 0, - "[ASCIICodePageEncoding.GetBytes]Expected empty fallback buffer"); + return charsWritten; } else { - fallback = this.EncoderFallback as EncoderReplacementFallback; + // Simple narrowing conversion couldn't operate on entire buffer - invoke fallback. + + return GetCharsWithFallback(pBytes, byteCount, pChars, charCount, bytesConsumed, charsWritten); } + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] // called directly by GetCharsCommon + private protected sealed override unsafe int GetCharsFast(byte* pBytes, int bytesLength, char* pChars, int charsLength, out int bytesConsumed) + { + int charsWritten = (int)ASCIIUtility.WidenAsciiToUtf16(pBytes, pChars, (uint)Math.Min(bytesLength, charsLength)); + + bytesConsumed = charsWritten; + return charsWritten; + } - // See if we do the fast default or slightly slower fallback - if (fallback != null && fallback.MaxCharCount == 1) + private protected sealed override unsafe int GetCharsWithFallback(ReadOnlySpan<byte> bytes, int originalBytesLength, Span<char> chars, int originalCharsLength, DecoderNLS decoder) + { + // We special-case DecoderReplacementFallback if it's telling us to write a single BMP char, + // since we believe this to be relatively common and we can handle it more efficiently than + // the base implementation. + + if (((decoder is null) ? this.DecoderFallback: decoder.Fallback) is DecoderReplacementFallback replacementFallback + && replacementFallback.MaxCharCount == 1) { - // Fast version - char cReplacement = fallback.DefaultString[0]; + char replacementChar = replacementFallback.DefaultString[0]; + + int numElementsToConvert = Math.Min( bytes.Length, chars.Length); + int idx = 0; - // Check for replacements in range, otherwise fall back to slow version. - if (cReplacement <= (char)0x7f) + fixed (byte* pBytes = &MemoryMarshal.GetReference(bytes)) + fixed (char* pChars = &MemoryMarshal.GetReference(chars)) { - // We should have exactly as many output bytes as input bytes, unless there's a left - // over character, in which case we may need one more. - // If we had a left over character will have to add a ? (This happens if they had a funky - // fallback last time, but not this time.) (We can't spit any out though - // because with fallback encoder each surrogate is treated as a seperate code point) - if (charLeftOver > 0) - { - // Have to have room - // Throw even if doing no throw version because this is just 1 char, - // so buffer will never be big enough - if (byteCount == 0) - ThrowBytesOverflow(encoder, true); - - // This'll make sure we still have more room and also make sure our return value is correct. - *(bytes++) = (byte)cReplacement; - byteCount--; // We used one of the ones we were counting. - } + // In a loop, replace the non-convertible data, then bulk-convert as much as we can. - // This keeps us from overrunning our output buffer - if (byteCount < charCount) + while (idx < numElementsToConvert) { - // Throw or make buffer smaller? - ThrowBytesOverflow(encoder, byteCount < 1); + pChars[idx++] = replacementChar; - // Just use what we can - charEnd = chars + byteCount; - } - - // We just do a quick copy - while (chars < charEnd) - { - char ch2 = *(chars++); - if (ch2 >= 0x0080) *(bytes++) = (byte)cReplacement; - else *(bytes++) = unchecked((byte)(ch2)); - } + if (idx < numElementsToConvert) + { + idx += (int)ASCIIUtility.WidenAsciiToUtf16(&pBytes[idx], &pChars[idx], (uint)(numElementsToConvert - idx)); + } - // Clear encoder - if (encoder != null) - { - encoder._charLeftOver = (char)0; - encoder._charsUsed = (int)(chars - charStart); + Debug.Assert(idx <= numElementsToConvert, "Somehow went beyond bounds of source or destination buffer?"); } - - return (int)(bytes - byteStart); } - } - - // Slower version, have to do real fallback. - // prepare our end - byte* byteEnd = bytes + byteCount; + // Slice off how much we consumed / wrote. - // We may have a left over character from last time, try and process it. - if (charLeftOver > 0) - { - // Initialize the buffer - Debug.Assert(encoder != null, - "[ASCIIEncoding.GetBytes]Expected non null encoder if we have surrogate left over"); - fallbackBuffer = encoder.FallbackBuffer; - fallbackBuffer.InternalInitialize(chars, charEnd, encoder, true); - - // Since left over char was a surrogate, it'll have to be fallen back. - // Get Fallback - // This will fallback a pair if *chars is a low surrogate - charsForFallback = chars; // Avoid passing chars by reference to allow it to be enregistered - fallbackBuffer.InternalFallback(charLeftOver, ref charsForFallback); - chars = charsForFallback; + bytes = bytes.Slice(numElementsToConvert); + chars = chars.Slice(numElementsToConvert); } - // Now we may have fallback char[] already from the encoder + // If we couldn't go through our fast fallback mechanism, or if we still have leftover + // data because we couldn't consume everything in the loop above, we need to go down the + // slow fallback path. - // Go ahead and do it, including the fallback. - char ch; - while ((ch = (fallbackBuffer == null) ? '\0' : fallbackBuffer.InternalGetNextChar()) != 0 || - chars < charEnd) + if (bytes.IsEmpty) { - // First unwind any fallback - if (ch == 0) - { - // No fallback, just get next char - ch = *chars; - chars++; - } - - // Check for fallback, this'll catch surrogate pairs too. - // All characters >= 0x80 must fall back. - if (ch > 0x7f) - { - // Initialize the buffer - if (fallbackBuffer == null) - { - if (encoder == null) - fallbackBuffer = this.encoderFallback.CreateFallbackBuffer(); - else - fallbackBuffer = encoder.FallbackBuffer; - fallbackBuffer.InternalInitialize(charEnd - charCount, charEnd, encoder, true); - } - - // Get Fallback - charsForFallback = chars; // Avoid passing chars by reference to allow it to be enregistered - fallbackBuffer.InternalFallback(ch, ref charsForFallback); - chars = charsForFallback; - - // Go ahead & continue (& do the fallback) - continue; - } - - // We'll use this one - // Bounds check - if (bytes >= byteEnd) - { - // didn't use this char, we'll throw or use buffer - if (fallbackBuffer == null || fallbackBuffer.bFallingBack == false) - { - Debug.Assert(chars > charStart || bytes == byteStart, - "[ASCIIEncoding.GetBytes]Expected chars to have advanced already."); - chars--; // don't use last char - } - else - fallbackBuffer.MovePrevious(); - - // Are we throwing or using buffer? - ThrowBytesOverflow(encoder, bytes == byteStart); // throw? - break; // don't throw, stop - } - - // Go ahead and add it - *bytes = unchecked((byte)ch); - bytes++; + return originalCharsLength - chars.Length; // total number of chars written } - - // Need to do encoder stuff - if (encoder != null) + else { - // Fallback stuck it in encoder if necessary, but we have to clear MustFlush cases - if (fallbackBuffer != null && !fallbackBuffer.bUsedEncoder) - // Clear it in case of MustFlush - encoder._charLeftOver = (char)0; - - // Set our chars used count - encoder._charsUsed = (int)(chars - charStart); + return base.GetCharsWithFallback(bytes, originalBytesLength, chars, originalCharsLength, decoder); } - - Debug.Assert(fallbackBuffer == null || fallbackBuffer.Remaining == 0 || - (encoder != null && !encoder._throwOnOverflow), - "[ASCIIEncoding.GetBytes]Expected Empty fallback buffer at end"); - - return (int)(bytes - byteStart); } - // This is internal and called by something else, - internal sealed override unsafe int GetCharCount(byte* bytes, int count, DecoderNLS decoder) - { - // Just assert, we're called internally so these should be safe, checked already - Debug.Assert(bytes != null, "[ASCIIEncoding.GetCharCount]bytes is null"); - Debug.Assert(count >= 0, "[ASCIIEncoding.GetCharCount]byteCount is negative"); + // Returns a string containing the decoded representation of a range of + // bytes in a byte array. + // + // All of our public Encodings that don't use EncodingNLS must have this (including EncodingNLS) + // So if you fix this, fix the others. Currently those include: + // EncodingNLS, UTF7Encoding, UTF8Encoding, UTF32Encoding, ASCIIEncoding, UnicodeEncoding + // parent method is safe - // ASCII doesn't do best fit, so don't have to check for it, find out which decoder fallback we're using - DecoderReplacementFallback fallback = null; + public override unsafe string GetString(byte[] bytes, int byteIndex, int byteCount) + { + // Validate Parameters - if (decoder == null) - fallback = this.DecoderFallback as DecoderReplacementFallback; - else + if (bytes is null) { - fallback = decoder.Fallback as DecoderReplacementFallback; - Debug.Assert(!decoder._throwOnOverflow || !decoder.InternalHasFallbackBuffer || - decoder.FallbackBuffer.Remaining == 0, - "[ASCIICodePageEncoding.GetCharCount]Expected empty fallback buffer"); + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.bytes, ExceptionResource.ArgumentNull_Array); } - if (fallback != null && fallback.MaxCharCount == 1) + if ((byteIndex | byteCount) < 0) { - // Just return length, SBCS stay the same length because they don't map to surrogate - // pairs and we don't have a decoder fallback. - - return count; + ThrowHelper.ThrowArgumentOutOfRangeException( + argument: (byteIndex < 0) ? ExceptionArgument.byteIndex : ExceptionArgument.byteCount, + resource: ExceptionResource.ArgumentOutOfRange_NeedNonNegNum); } - // Only need decoder fallback buffer if not using default replacement fallback, no best fit for ASCII - DecoderFallbackBuffer fallbackBuffer = null; - - // Have to do it the hard way. - // Assume charCount will be == count - int charCount = count; - byte[] byteBuffer = new byte[1]; - - // Do it our fast way - byte* byteEnd = bytes + count; - - // Quick loop - while (bytes < byteEnd) + if (bytes.Length - byteIndex < byteCount) { - // Faster if don't use *bytes++; - byte b = *bytes; - bytes++; - - // If unknown we have to do fallback count - if (b >= 0x80) - { - if (fallbackBuffer == null) - { - if (decoder == null) - fallbackBuffer = this.DecoderFallback.CreateFallbackBuffer(); - else - fallbackBuffer = decoder.FallbackBuffer; - fallbackBuffer.InternalInitialize(byteEnd - count, null); - } - - // Use fallback buffer - byteBuffer[0] = b; - charCount--; // Have to unreserve the one we already allocated for b - charCount += fallbackBuffer.InternalFallback(byteBuffer, bytes); - } + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.bytes, ExceptionResource.ArgumentOutOfRange_IndexCountBuffer); } - // Fallback buffer must be empty - Debug.Assert(fallbackBuffer == null || fallbackBuffer.Remaining == 0, - "[ASCIIEncoding.GetCharCount]Expected Empty fallback buffer"); + // Avoid problems with empty input buffer + if (byteCount == 0) + return string.Empty; - // Converted sequence is same length as input - return charCount; + fixed (byte* pBytes = bytes) + { + return string.CreateStringFromEncoding(pBytes + byteIndex, byteCount, this); + } } - internal sealed override unsafe int GetChars( - byte* bytes, int byteCount, char* chars, int charCount, DecoderNLS decoder) - { - // Just need to ASSERT, this is called by something else internal that checked parameters already - Debug.Assert(bytes != null, "[ASCIIEncoding.GetChars]bytes is null"); - Debug.Assert(byteCount >= 0, "[ASCIIEncoding.GetChars]byteCount is negative"); - Debug.Assert(chars != null, "[ASCIIEncoding.GetChars]chars is null"); - Debug.Assert(charCount >= 0, "[ASCIIEncoding.GetChars]charCount is negative"); - - // Do it fast way if using ? replacement fallback - byte* byteEnd = bytes + byteCount; - byte* byteStart = bytes; - char* charStart = chars; + // + // End of standard methods copied from EncodingNLS.cs + // - // Note: ASCII doesn't do best fit, but we have to fallback if they use something > 0x7f - // Only need decoder fallback buffer if not using ? fallback. - // ASCII doesn't do best fit, so don't have to check for it, find out which decoder fallback we're using - DecoderReplacementFallback fallback = null; - char* charsForFallback; + // + // Beginning of methods used by shared fallback logic. + // - if (decoder == null) - fallback = this.DecoderFallback as DecoderReplacementFallback; + internal sealed override bool TryGetByteCount(Rune value, out int byteCount) + { + if (value.IsAscii) + { + byteCount = 1; + return true; + } else { - fallback = decoder.Fallback as DecoderReplacementFallback; - Debug.Assert(!decoder._throwOnOverflow || !decoder.InternalHasFallbackBuffer || - decoder.FallbackBuffer.Remaining == 0, - "[ASCIICodePageEncoding.GetChars]Expected empty fallback buffer"); + byteCount = default; + return false; } + } - if (fallback != null && fallback.MaxCharCount == 1) + internal sealed override OperationStatus EncodeRune(Rune value, Span<byte> bytes, out int bytesWritten) + { + if (value.IsAscii) { - // Try it the fast way - char replacementChar = fallback.DefaultString[0]; - - // Need byteCount chars, otherwise too small buffer - if (charCount < byteCount) + if (!bytes.IsEmpty) { - // Need at least 1 output byte, throw if must throw - ThrowCharsOverflow(decoder, charCount < 1); - - // Not throwing, use what we can - byteEnd = bytes + charCount; + bytes[0] = (byte)value.Value; + bytesWritten = 1; + return OperationStatus.Done; } - - // Quick loop, just do '?' replacement because we don't have fallbacks for decodings. - while (bytes < byteEnd) + else { - byte b = *(bytes++); - if (b >= 0x80) - // This is an invalid byte in the ASCII encoding. - *(chars++) = replacementChar; - else - *(chars++) = unchecked((char)b); + bytesWritten = 0; + return OperationStatus.DestinationTooSmall; } - - // bytes & chars used are the same - if (decoder != null) - decoder._bytesUsed = (int)(bytes - byteStart); - return (int)(chars - charStart); } - - // Slower way's going to need a fallback buffer - DecoderFallbackBuffer fallbackBuffer = null; - byte[] byteBuffer = new byte[1]; - char* charEnd = chars + charCount; - - // Not quite so fast loop - while (bytes < byteEnd) + else { - // Faster if don't use *bytes++; - byte b = *(bytes); - bytes++; + bytesWritten = 0; + return OperationStatus.InvalidData; + } + } - if (b >= 0x80) + internal sealed override OperationStatus DecodeFirstRune(ReadOnlySpan<byte> bytes, out Rune value, out int bytesConsumed) + { + if (!bytes.IsEmpty) + { + byte b = bytes[0]; + if (b <= 0x7F) { - // This is an invalid byte in the ASCII encoding. - if (fallbackBuffer == null) - { - if (decoder == null) - fallbackBuffer = this.DecoderFallback.CreateFallbackBuffer(); - else - fallbackBuffer = decoder.FallbackBuffer; - fallbackBuffer.InternalInitialize(byteEnd - byteCount, charEnd); - } - - // Use fallback buffer - byteBuffer[0] = b; - - // Note that chars won't get updated unless this succeeds - charsForFallback = chars; // Avoid passing chars by reference to allow it to be enregistered - bool fallbackResult = fallbackBuffer.InternalFallback(byteBuffer, bytes, ref charsForFallback); - chars = charsForFallback; + // ASCII byte - if (!fallbackResult) - { - // May or may not throw, but we didn't get this byte - Debug.Assert(bytes > byteStart || chars == charStart, - "[ASCIIEncoding.GetChars]Expected bytes to have advanced already (fallback case)"); - bytes--; // unused byte - fallbackBuffer.InternalReset(); // Didn't fall this back - ThrowCharsOverflow(decoder, chars == charStart); // throw? - break; // don't throw, but stop loop - } + value = new Rune(b); + bytesConsumed = 1; + return OperationStatus.Done; } else { - // Make sure we have buffer space - if (chars >= charEnd) - { - Debug.Assert(bytes > byteStart || chars == charStart, - "[ASCIIEncoding.GetChars]Expected bytes to have advanced already (normal case)"); - bytes--; // unused byte - ThrowCharsOverflow(decoder, chars == charStart); // throw? - break; // don't throw, but stop loop - } + // Non-ASCII byte - *(chars) = unchecked((char)b); - chars++; + value = Rune.ReplacementChar; + bytesConsumed = 1; + return OperationStatus.InvalidData; } } + else + { + // No data to decode - // Might have had decoder fallback stuff. - if (decoder != null) - decoder._bytesUsed = (int)(bytes - byteStart); - - // Expect Empty fallback buffer for GetChars - Debug.Assert(fallbackBuffer == null || fallbackBuffer.Remaining == 0, - "[ASCIIEncoding.GetChars]Expected Empty fallback buffer"); - - return (int)(chars - charStart); + value = Rune.ReplacementChar; + bytesConsumed = 0; + return OperationStatus.NeedMoreData; + } } + // + // End of methods used by shared fallback logic. + // public override int GetMaxByteCount(int charCount) { diff --git a/src/System.Private.CoreLib/shared/System/Text/ASCIIUtility.cs b/src/System.Private.CoreLib/shared/System/Text/ASCIIUtility.cs new file mode 100644 index 0000000000..5bc80c35f5 --- /dev/null +++ b/src/System.Private.CoreLib/shared/System/Text/ASCIIUtility.cs @@ -0,0 +1,76 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Runtime.CompilerServices; + +namespace System.Text +{ + /* + * Contains naive unoptimized (non-SIMD) implementations of ASCII transcoding + * operations. Vectorized methods can be substituted here as a drop-in replacement. + */ + + internal unsafe static class ASCIIUtility + { + [MethodImpl(MethodImplOptions.NoInlining)] // the actual implementation won't be inlined, so this shouldn't be either, lest it throw off benchmarks + public static uint GetIndexOfFirstNonAsciiByte(byte* pBytes, uint byteCount) + { + uint idx = 0; + for (; idx < byteCount; idx++) + { + if ((sbyte)pBytes[idx] < 0) + { + break; + } + } + return idx; + } + + [MethodImpl(MethodImplOptions.NoInlining)] // the actual implementation won't be inlined, so this shouldn't be either, lest it throw off benchmarks + public static uint GetIndexOfFirstNonAsciiChar(char* pChars, uint charCount) + { + uint idx = 0; + for (; idx < charCount; idx++) + { + if (pChars[idx] > 0x7Fu) + { + break; + } + } + return idx; + } + + [MethodImpl(MethodImplOptions.NoInlining)] // the actual implementation won't be inlined, so this shouldn't be either, lest it throw off benchmarks + public static uint NarrowUtf16ToAscii(char* pChars, byte* pBytes, uint elementCount) + { + uint idx = 0; + for (; idx < elementCount; idx++) + { + uint ch = pChars[idx]; + if (ch > 0x7Fu) + { + break; + } + pBytes[idx] = (byte)ch; + } + return idx; + } + + [MethodImpl(MethodImplOptions.NoInlining)] // the actual implementation won't be inlined, so this shouldn't be either, lest it throw off benchmarks + public static uint WidenAsciiToUtf16(byte* pBytes, char* pChars, uint elementCount) + { + uint idx = 0; + for (; idx < elementCount; idx++) + { + byte b = pBytes[idx]; + if (b > 0x7F) + { + break; + } + pChars[idx] = (char)b; + } + return idx; + } + } +} diff --git a/src/System.Private.CoreLib/shared/System/Text/DecoderFallback.cs b/src/System.Private.CoreLib/shared/System/Text/DecoderFallback.cs index fff8ad1d7b..2eb03d8089 100644 --- a/src/System.Private.CoreLib/shared/System/Text/DecoderFallback.cs +++ b/src/System.Private.CoreLib/shared/System/Text/DecoderFallback.cs @@ -67,6 +67,10 @@ namespace System.Text internal unsafe byte* byteStart; internal unsafe char* charEnd; + internal Encoding _encoding; + internal DecoderNLS _decoder; + private int _originalByteCount; + // Internal Reset internal unsafe void InternalReset() { @@ -82,6 +86,22 @@ namespace System.Text this.charEnd = charEnd; } + internal static DecoderFallbackBuffer CreateAndInitialize(Encoding encoding, DecoderNLS decoder, int originalByteCount) + { + // The original byte count is only used for keeping track of what 'index' value needs + // to be passed to the abstract Fallback method. The index value is calculated by subtracting + // 'bytes.Length' (where bytes is expected to be the entire remaining input buffer) + // from the 'originalByteCount' value specified here. + + DecoderFallbackBuffer fallbackBuffer = (decoder is null) ? encoding.DecoderFallback.CreateFallbackBuffer() : decoder.FallbackBuffer; + + fallbackBuffer._encoding = encoding; + fallbackBuffer._decoder = decoder; + fallbackBuffer._originalByteCount = originalByteCount; + + return fallbackBuffer; + } + // Fallback the current byte by sticking it into the remaining char buffer. // This can only be called by our encodings (other have to use the public fallback methods), so // we can use our DecoderNLS here too (except we don't). @@ -191,6 +211,90 @@ namespace System.Text return 0; } + internal int InternalFallbackGetCharCount(ReadOnlySpan<byte> remainingBytes, int fallbackLength) + { + return (Fallback(remainingBytes.Slice(0, fallbackLength).ToArray(), index: _originalByteCount - remainingBytes.Length)) + ? DrainRemainingDataForGetCharCount() + : 0; + } + + internal bool TryInternalFallbackGetChars(ReadOnlySpan<byte> remainingBytes, int fallbackLength, Span<char> chars, out int charsWritten) + { + if (Fallback(remainingBytes.Slice(0, fallbackLength).ToArray(), index: _originalByteCount - remainingBytes.Length)) + { + return TryDrainRemainingDataForGetChars(chars, out charsWritten); + } + else + { + // Return true because we weren't asked to write anything, so this is a "success" in the sense that + // the output buffer was large enough to hold the desired 0 chars of output. + + charsWritten = 0; + return true; + } + } + + private Rune GetNextRune() + { + // Call GetNextChar() and try treating it as a non-surrogate character. + // If that fails, call GetNextChar() again and attempt to treat the two chars + // as a surrogate pair. If that still fails, throw an exception since the fallback + // mechanism is giving us a bad replacement character. + + Rune rune; + char ch = GetNextChar(); + if (!Rune.TryCreate(ch, out rune) && !Rune.TryCreate(ch, GetNextChar(), out rune)) + { + throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex); + } + + return rune; + } + + internal int DrainRemainingDataForGetCharCount() + { + int totalCharCount = 0; + + Rune thisRune; + while ((thisRune = GetNextRune()).Value != 0) + { + // We need to check for overflow while tallying the fallback char count. + + totalCharCount += thisRune.Utf16SequenceLength; + if (totalCharCount < 0) + { + InternalReset(); + Encoding.ThrowConversionOverflow(); + } + } + + return totalCharCount; + } + + internal bool TryDrainRemainingDataForGetChars(Span<char> chars, out int charsWritten) + { + int originalCharCount = chars.Length; + + Rune thisRune; + while ((thisRune = GetNextRune()).Value != 0) + { + if (thisRune.TryEncode(chars, out int charsWrittenJustNow)) + { + chars = chars.Slice(charsWrittenJustNow); + continue; + } + else + { + InternalReset(); + charsWritten = default; + return false; + } + } + + charsWritten = originalCharCount - chars.Length; + return true; + } + // private helper methods internal void ThrowLastBytesRecursive(byte[] bytesUnknown) { diff --git a/src/System.Private.CoreLib/shared/System/Text/DecoderNLS.cs b/src/System.Private.CoreLib/shared/System/Text/DecoderNLS.cs index 8af4dc3a55..597d362bf7 100644 --- a/src/System.Private.CoreLib/shared/System/Text/DecoderNLS.cs +++ b/src/System.Private.CoreLib/shared/System/Text/DecoderNLS.cs @@ -2,9 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System.Runtime.Serialization; -using System.Text; -using System; +using System.Buffers; +using System.Diagnostics; using System.Runtime.InteropServices; namespace System.Text @@ -27,6 +26,8 @@ namespace System.Text private bool _mustFlush; internal bool _throwOnOverflow; internal int _bytesUsed; + private int _leftoverBytes; // leftover data from a previous invocation of GetChars (up to 4 bytes) + private int _leftoverByteCount; // number of bytes of actual data in _leftoverBytes internal DecoderNLS(Encoding encoding) { @@ -44,6 +45,7 @@ namespace System.Text public override void Reset() { + ClearLeftoverData(); _fallbackBuffer?.Reset(); } @@ -238,5 +240,195 @@ namespace System.Text { _mustFlush = false; } + + internal ReadOnlySpan<byte> GetLeftoverData() + { + return MemoryMarshal.AsBytes(new ReadOnlySpan<int>(ref _leftoverBytes, 1)).Slice(0, _leftoverByteCount); + } + + internal void SetLeftoverData(ReadOnlySpan<byte> bytes) + { + bytes.CopyTo(MemoryMarshal.AsBytes(new Span<int>(ref _leftoverBytes, 1))); + _leftoverByteCount = bytes.Length; + } + + internal bool HasLeftoverData => _leftoverByteCount != 0; + + internal void ClearLeftoverData() + { + _leftoverByteCount = 0; + } + + internal int DrainLeftoverDataForGetCharCount(ReadOnlySpan<byte> bytes, out int bytesConsumed) + { + // Quick check: we _should not_ have leftover fallback data from a previous invocation, + // as we'd end up consuming any such data and would corrupt whatever Convert call happens + // to be in progress. Unlike EncoderNLS, this is simply a Debug.Assert. No exception is thrown. + + Debug.Assert(_fallbackBuffer is null || _fallbackBuffer.Remaining == 0, "Should have no data remaining in the fallback buffer."); + + // Copy the existing leftover data plus as many bytes as possible of the new incoming data + // into a temporary concated buffer, then get its char count by decoding it. + + Span<byte> combinedBuffer = stackalloc byte[4]; + combinedBuffer = combinedBuffer.Slice(0, ConcatInto(GetLeftoverData(), bytes, combinedBuffer)); + int charCount = 0; + + switch (_encoding.DecodeFirstRune(combinedBuffer, out Rune value, out int combinedBufferBytesConsumed)) + { + case OperationStatus.Done: + charCount = value.Utf16SequenceLength; + goto Finish; // successfully transcoded bytes -> chars + + case OperationStatus.NeedMoreData: + if (MustFlush) + { + goto case OperationStatus.InvalidData; // treat as equivalent to bad data + } + else + { + goto Finish; // consumed some bytes, output 0 chars + } + + case OperationStatus.InvalidData: + break; + + default: + Debug.Fail("Unexpected OperationStatus return value."); + break; + } + + // Couldn't decode the buffer. Fallback the buffer instead. + + if (FallbackBuffer.Fallback(combinedBuffer.Slice(0, combinedBufferBytesConsumed).ToArray(), index: 0)) + { + charCount = _fallbackBuffer.DrainRemainingDataForGetCharCount(); + Debug.Assert(charCount >= 0, "Fallback buffer shouldn't have returned a negative char count."); + } + + Finish: + + bytesConsumed = combinedBufferBytesConsumed - _leftoverByteCount; // amount of 'bytes' buffer consumed just now + return charCount; + } + + internal int DrainLeftoverDataForGetChars(ReadOnlySpan<byte> bytes, Span<char> chars, out int bytesConsumed) + { + // Quick check: we _should not_ have leftover fallback data from a previous invocation, + // as we'd end up consuming any such data and would corrupt whatever Convert call happens + // to be in progress. Unlike EncoderNLS, this is simply a Debug.Assert. No exception is thrown. + + Debug.Assert(_fallbackBuffer is null || _fallbackBuffer.Remaining == 0, "Should have no data remaining in the fallback buffer."); + + // Copy the existing leftover data plus as many bytes as possible of the new incoming data + // into a temporary concated buffer, then transcode it from bytes to chars. + + Span<byte> combinedBuffer = stackalloc byte[4]; + combinedBuffer = combinedBuffer.Slice(0, ConcatInto(GetLeftoverData(), bytes, combinedBuffer)); + int charsWritten = 0; + + bool persistNewCombinedBuffer = false; + + switch (_encoding.DecodeFirstRune(combinedBuffer, out Rune value, out int combinedBufferBytesConsumed)) + { + case OperationStatus.Done: + if (value.TryEncode(chars, out charsWritten)) + { + goto Finish; // successfully transcoded bytes -> chars + } + else + { + goto DestinationTooSmall; + } + + case OperationStatus.NeedMoreData: + if (MustFlush) + { + goto case OperationStatus.InvalidData; // treat as equivalent to bad data + } + else + { + persistNewCombinedBuffer = true; + goto Finish; // successfully consumed some bytes, output no chars + } + + case OperationStatus.InvalidData: + break; + + default: + Debug.Fail("Unexpected OperationStatus return value."); + break; + } + + // Couldn't decode the buffer. Fallback the buffer instead. + + if (FallbackBuffer.Fallback(combinedBuffer.Slice(0, combinedBufferBytesConsumed).ToArray(), index: 0) + && !_fallbackBuffer.TryDrainRemainingDataForGetChars(chars, out charsWritten)) + { + goto DestinationTooSmall; + } + + Finish: + + if (persistNewCombinedBuffer) + { + Debug.Assert(combinedBufferBytesConsumed == combinedBuffer.Length, "We should be asked to persist the entire combined buffer."); + SetLeftoverData(combinedBuffer); // the buffer still only contains partial data; a future call to Convert will need it + } + else + { + ClearLeftoverData(); // the buffer contains no partial data; we'll go down the normal paths + } + + bytesConsumed = combinedBufferBytesConsumed - _leftoverByteCount; // amount of 'bytes' buffer consumed just now + return charsWritten; + + DestinationTooSmall: + + // If we got to this point, we're trying to write chars to the output buffer, but we're unable to do + // so. Unlike EncoderNLS, this type does not allow partial writes to the output buffer. Since we know + // draining leftover data is the first operation performed by any DecoderNLS API, there was no + // opportunity for any code before us to make forward progress, so we must fail immediately. + + _encoding.ThrowCharsOverflow(this, nothingDecoded: true); + throw null; // will never reach this point + } + + /// <summary> + /// Given a byte buffer <paramref name="dest"/>, concatenates as much of <paramref name="srcLeft"/> followed + /// by <paramref name="srcRight"/> into it as will fit, then returns the total number of bytes copied. + /// </summary> + private static int ConcatInto(ReadOnlySpan<byte> srcLeft, ReadOnlySpan<byte> srcRight, Span<byte> dest) + { + int total = 0; + + for (int i = 0; i < srcLeft.Length; i++) + { + if ((uint)total >= (uint)dest.Length) + { + goto Finish; + } + else + { + dest[total++] = srcLeft[i]; + } + } + + for (int i = 0; i < srcRight.Length; i++) + { + if ((uint)total >= (uint)dest.Length) + { + goto Finish; + } + else + { + dest[total++] = srcRight[i]; + } + } + + Finish: + + return total; + } } } diff --git a/src/System.Private.CoreLib/shared/System/Text/EncoderFallback.cs b/src/System.Private.CoreLib/shared/System/Text/EncoderFallback.cs index f98b15e078..ff895d6788 100644 --- a/src/System.Private.CoreLib/shared/System/Text/EncoderFallback.cs +++ b/src/System.Private.CoreLib/shared/System/Text/EncoderFallback.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System.Buffers; using System.Diagnostics; using System.Threading; @@ -86,12 +87,14 @@ namespace System.Text // These help us with our performance and messages internally internal unsafe char* charStart; internal unsafe char* charEnd; - internal EncoderNLS encoder; + internal EncoderNLS encoder; // TODO: MAKE ME PRIVATE internal bool setEncoder; internal bool bUsedEncoder; internal bool bFallingBack = false; internal int iRecursionCount = 0; private const int iMaxRecursion = 250; + private Encoding encoding; + private int originalCharCount; // Internal Reset // For example, what if someone fails a conversion and wants to reset one of our fallback buffers? @@ -116,6 +119,22 @@ namespace System.Text this.iRecursionCount = 0; } + internal static EncoderFallbackBuffer CreateAndInitialize(Encoding encoding, EncoderNLS encoder, int originalCharCount) + { + // The original char count is only used for keeping track of what 'index' value needs + // to be passed to the abstract Fallback method. The index value is calculated by subtracting + // 'chars.Length' (where chars is expected to be the entire remaining input buffer) + // from the 'originalCharCount' value specified here. + + EncoderFallbackBuffer fallbackBuffer = (encoder is null) ? encoding.EncoderFallback.CreateFallbackBuffer() : encoder.FallbackBuffer; + + fallbackBuffer.encoding = encoding; + fallbackBuffer.encoder = encoder; + fallbackBuffer.originalCharCount = originalCharCount; + + return fallbackBuffer; + } + internal char InternalGetNextChar() { char ch = GetNextChar(); @@ -124,6 +143,170 @@ namespace System.Text return ch; } + private bool InternalFallback(ReadOnlySpan<char> chars, out int charsConsumed) + { + Debug.Assert(!chars.IsEmpty, "Caller shouldn't invoke this if there's no data to fall back."); + + // First, try falling back a single BMP character or a standalone low surrogate. + // If the first char is a high surrogate, we'll try to combine it with the next + // char in the input sequence. + + char firstChar = chars[0]; + char secondChar = default; + + if (!chars.IsEmpty) + { + firstChar = chars[0]; + + if (1 < (uint)chars.Length) + { + secondChar = chars[1]; + } + } + + // Ask the subclassed type to initiate fallback logic. + + int index = originalCharCount - chars.Length; + + if (!char.IsSurrogatePair(firstChar, secondChar)) + { + // This code path is also used when 'firstChar' is a standalone surrogate or + // if it's a high surrogate at the end of the input buffer. + + charsConsumed = 1; + return Fallback(firstChar, index); + } + else + { + charsConsumed = 2; + return Fallback(firstChar, secondChar, index); + } + } + + internal int InternalFallbackGetByteCount(ReadOnlySpan<char> chars, out int charsConsumed) + { + int bytesWritten = 0; + + if (InternalFallback(chars, out charsConsumed)) + { + // There's data in the fallback buffer - pull it out now. + + bytesWritten = DrainRemainingDataForGetByteCount(); + } + + return bytesWritten; + } + + internal bool TryInternalFallbackGetBytes(ReadOnlySpan<char> chars, Span<byte> bytes, out int charsConsumed, out int bytesWritten) + { + if (InternalFallback(chars, out charsConsumed)) + { + // There's data in the fallback buffer - pull it out now. + + return TryDrainRemainingDataForGetBytes(bytes, out bytesWritten); + } + else + { + // There's no data in the fallback buffer. + + bytesWritten = 0; + return true; // true = didn't run out of space in destination buffer + } + } + + internal bool TryDrainRemainingDataForGetBytes(Span<byte> bytes, out int bytesWritten) + { + int originalBytesLength = bytes.Length; + + Rune thisRune; + while ((thisRune = GetNextRune()).Value != 0) + { + switch (encoding.EncodeRune(thisRune, bytes, out int bytesWrittenJustNow)) + { + case OperationStatus.Done: + + bytes = bytes.Slice(bytesWrittenJustNow); + continue; + + case OperationStatus.DestinationTooSmall: + + // Since we're not consuming the Rune we just read, back up as many chars as necessary + // to undo the read we just performed, then report to our caller that we ran out of space. + + for (int i = 0; i < thisRune.Utf16SequenceLength; i++) + { + MovePrevious(); + } + + bytesWritten = originalBytesLength - bytes.Length; + return false; // ran out of destination buffer + + case OperationStatus.InvalidData: + + // We can't fallback the fallback. We can't make forward progress, so report to our caller + // that something went terribly wrong. The error message contains the fallback char that + // couldn't be converted. (Ideally we'd provide the first char that originally triggered + // the fallback, but it's complicated to keep this state around, and a fallback producing + // invalid data should be a very rare occurrence.) + + ThrowLastCharRecursive(thisRune.Value); + break; // will never be hit; call above throws + + default: + + Debug.Fail("Unexpected return value."); + break; + } + } + + bytesWritten = originalBytesLength - bytes.Length; + return true; // finished successfully + } + + internal int DrainRemainingDataForGetByteCount() + { + int totalByteCount = 0; + + Rune thisRune; + while ((thisRune = GetNextRune()).Value != 0) + { + if (!encoding.TryGetByteCount(thisRune, out int byteCountThisIteration)) + { + // We can't fallback the fallback. We can't make forward progress, so report to our caller + // that something went terribly wrong. The error message contains the fallback char that + // couldn't be converted. (Ideally we'd provide the first char that originally triggered + // the fallback, but it's complicated to keep this state around, and a fallback producing + // invalid data should be a very rare occurrence.) + + ThrowLastCharRecursive(thisRune.Value); + } + + Debug.Assert(byteCountThisIteration >= 0, "Encoding shouldn't have returned a negative byte count."); + + // We need to check for overflow while tallying the fallback byte count. + + totalByteCount += byteCountThisIteration; + if (totalByteCount < 0) + { + InternalReset(); + Encoding.ThrowConversionOverflow(); + } + } + + return totalByteCount; + } + + private Rune GetNextRune() + { + char firstChar = GetNextChar(); + if (Rune.TryCreate(firstChar, out Rune value) || Rune.TryCreate(firstChar, GetNextChar(), out value)) + { + return value; + } + + throw new ArgumentException(SR.Argument_InvalidCharSequenceNoIndex); + } + // Fallback the current character using the remaining buffer and encoder if necessary // This can only be called by our encodings (other have to use the public fallback methods), so // we can use our EncoderNLS here too. diff --git a/src/System.Private.CoreLib/shared/System/Text/EncoderNLS.cs b/src/System.Private.CoreLib/shared/System/Text/EncoderNLS.cs index e83666f7a3..2901fc37b9 100644 --- a/src/System.Private.CoreLib/shared/System/Text/EncoderNLS.cs +++ b/src/System.Private.CoreLib/shared/System/Text/EncoderNLS.cs @@ -2,8 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System.Text; -using System; +using System.Buffers; +using System.Diagnostics; using System.Runtime.InteropServices; namespace System.Text @@ -197,9 +197,13 @@ namespace System.Text bytesUsed = _encoding.GetBytes(chars, charCount, bytes, byteCount, this); charsUsed = _charsUsed; - // Its completed if they've used what they wanted AND if they didn't want flush or if we are flushed - completed = (charsUsed == charCount) && (!flush || !this.HasState) && - (_fallbackBuffer == null || _fallbackBuffer.Remaining == 0); + // Per MSDN, "The completed output parameter indicates whether all the data in the input + // buffer was converted and stored in the output buffer." That means we've successfully + // consumed all the input _and_ there's no pending state or fallback data remaining to be output. + + completed = (charsUsed == charCount) + && !this.HasState + && (_fallbackBuffer is null || _fallbackBuffer.Remaining == 0); // Our data thingys are now full, we can return } @@ -220,6 +224,10 @@ namespace System.Text } } + /// <summary> + /// States whether a call to <see cref="Encoding.GetBytes(char*, int, byte*, int, EncoderNLS)"/> must first drain data on this <see cref="EncoderNLS"/> instance. + /// </summary> + internal bool HasLeftoverData => _charLeftOver != default || (_fallbackBuffer != null && _fallbackBuffer.Remaining > 0); // Anything left in our encoder? internal virtual bool HasState @@ -235,5 +243,154 @@ namespace System.Text { _mustFlush = false; } + + internal int DrainLeftoverDataForGetByteCount(ReadOnlySpan<char> chars, out int charsConsumed) + { + // Quick check: we _should not_ have leftover fallback data from a previous invocation, + // as we'd end up consuming any such data and would corrupt whatever Convert call happens + // to be in progress. + + if (_fallbackBuffer != null && _fallbackBuffer.Remaining > 0) + { + throw new ArgumentException(SR.Format(SR.Argument_EncoderFallbackNotEmpty, Encoding.EncodingName, _fallbackBuffer.GetType())); + } + + // If we have a leftover high surrogate from a previous operation, consume it now. + // We won't clear the _charLeftOver field since GetByteCount is supposed to be + // a non-mutating operation, and we need the field to retain its value for the + // next call to Convert. + + charsConsumed = 0; // could be incorrect, will fix up later in the method + + if (_charLeftOver == default) + { + return 0; // no leftover high surrogate char - short-circuit and finish + } + else + { + char secondChar = default; + + if (chars.IsEmpty) + { + // If the input buffer is empty and we're not being asked to flush, no-op and return + // success to our caller. If we're being asked to flush, the leftover high surrogate from + // the previous operation will go through the fallback mechanism by itself. + + if (!MustFlush) + { + return 0; // no-op = success + } + } + else + { + secondChar = chars[0]; + } + + // If we have to fallback the chars we're reading immediately below, populate the + // fallback buffer with the invalid data. We'll just fall through to the "consume + // fallback buffer" logic at the end of the method. + + bool didFallback; + + if (Rune.TryCreate(_charLeftOver, secondChar, out Rune rune)) + { + charsConsumed = 1; // consumed the leftover high surrogate + the first char in the input buffer + + if (_encoding.TryGetByteCount(rune, out int byteCount)) + { + Debug.Assert(byteCount >= 0, "Encoding shouldn't have returned a negative byte count."); + return byteCount; + } + else + { + didFallback = FallbackBuffer.Fallback(_charLeftOver, secondChar, index: 0); + } + } + else + { + didFallback = FallbackBuffer.Fallback(_charLeftOver, index: 0); + } + + // Now tally the number of bytes that would've been emitted as part of fallback. + + return _fallbackBuffer.DrainRemainingDataForGetByteCount(); + } + } + + internal bool TryDrainLeftoverDataForGetBytes(ReadOnlySpan<char> chars, Span<byte> bytes, out int charsConsumed, out int bytesWritten) + { + // We may have a leftover high surrogate data from a previous invocation, or we may have leftover + // data in the fallback buffer, or we may have neither, but we will never have both. Check for these + // conditions and handle them now. + + charsConsumed = 0; // could be incorrect, will fix up later in the method + bytesWritten = 0; // could be incorrect, will fix up later in the method + + if (_charLeftOver != default) + { + char secondChar = default; + + if (chars.IsEmpty) + { + // If the input buffer is empty and we're not being asked to flush, no-op and return + // success to our caller. If we're being asked to flush, the leftover high surrogate from + // the previous operation will go through the fallback mechanism by itself. + + if (!MustFlush) + { + charsConsumed = 0; + bytesWritten = 0; + return true; // no-op = success + } + } + else + { + secondChar = chars[0]; + } + + // If we have to fallback the chars we're reading immediately below, populate the + // fallback buffer with the invalid data. We'll just fall through to the "consume + // fallback buffer" logic at the end of the method. + + if (Rune.TryCreate(_charLeftOver, secondChar, out Rune rune)) + { + charsConsumed = 1; // at the very least, we consumed 1 char from the input + switch (_encoding.EncodeRune(rune, bytes, out bytesWritten)) + { + case OperationStatus.Done: + _charLeftOver = default; // we just consumed this char + return true; // that's all - we've handled the leftover data + + case OperationStatus.DestinationTooSmall: + _charLeftOver = default; // we just consumed this char + _encoding.ThrowBytesOverflow(this, nothingEncoded: true); // will throw + break; + + case OperationStatus.InvalidData: + FallbackBuffer.Fallback(_charLeftOver, secondChar, index: 0); + break; + + default: + Debug.Fail("Unknown return value."); + break; + } + } + else + { + FallbackBuffer.Fallback(_charLeftOver, index: 0); + } + } + + // Now check the fallback buffer for any remaining data. + + if (_fallbackBuffer != null && _fallbackBuffer.Remaining > 0) + { + return _fallbackBuffer.TryDrainRemainingDataForGetBytes(bytes, out bytesWritten); + } + + // And we're done! + + return true; // success + } } } diff --git a/src/System.Private.CoreLib/shared/System/Text/Encoding.Internal.cs b/src/System.Private.CoreLib/shared/System/Text/Encoding.Internal.cs new file mode 100644 index 0000000000..09044afefe --- /dev/null +++ b/src/System.Private.CoreLib/shared/System/Text/Encoding.Internal.cs @@ -0,0 +1,1277 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Internal.Runtime.CompilerServices; + +namespace System.Text +{ + public partial class Encoding + { + /* + * This file contains infrastructure code that supports a simplified way of writing + * internally-implemented Encoding types. In this system, the individual Encoding types + * are no longer responsible for handling anything related to the EncoderNLS / DecoderNLS + * infrastructure, nor are they responsible for implementing anything related to fallback + * buffers logic. + * + * Instead, subclassed types are responsible only for transcoding of individual scalar values + * to and from the encoding's byte representation (see the two methods immediately below). + * They can optionally implement fast-path logic to perform bulk transcoding up until the + * first segment of data that cannot be transcoded. They can special-case certain fallback + * mechanisms if desired. + * + * Most of the fast-path code is written using raw pointers as the exchange types, just as + * in the standard Encoding infrastructure. Since the fallback logic is more complex, most + * of it is written using type-safe constructs like Span<T>, with some amount of glue to + * allow it to work correctly with pointer-based fast-path code. + * + * A typical call graph for GetBytes is represented below, using ASCIIEncoding as an example. + * + * ASCIIEncoding.GetBytes(...) [non-EncoderNLS path, public virtual override] + * `- <parameter validation> + * - ASCIIEncoding.GetBytesCommon [private helper method per derived type, inlined] + * `- ASCIIEncoding.GetBytesFast [overridden fast-path implementation, inlined] + * - <if all data transcoded, return immediately> + * - <if all data not transcoded...> + * `- Encoding.GetBytesWithFallback [non-virtual stub method to call main GetBytesWithFallback worker] + * `- Encoding.GetBytesWithFallback [virtual method whose base implementation contains slow fallback logic] + * `- <may be overridden to provide optimized fallback logic> + * - <create EncodeFallbackBuffer instance> + * - <perform the following in a loop:> + * `- <invoke fast-path logic via virtual method dispatch on derived type> + * - <read next "bad" scalar value from source> + * - <run this bad value through the fallback buffer> + * - <drain the fallback buffer to the destination> + * - <loop until source is fully consumed or destination is full> + * - <signal full or partial success to EncoderNLS instance / throw if necessary> + * + * The call graph for GetBytes(..., EncoderNLS) is similar: + * + * Encoding.GetBytes(..., EncoderNLS) [base implementation] + * `- <if no leftover data from previous invocation, invoke fast-path> + * - <if fast-path invocation above completed, return immediately> + * - <if not all data transcoded, or if there was leftover data from previous invocation...> + * `- Encoding.GetBytesWithFallback [non-virtual stub method] + * `- <drain any leftover data from previous invocation> + * - <invoke fast-path again> + * - <if all data transcoded, return immediately> + * - <if all data not transcoded...> + * `- Encoding.GetBytesWithFallback [virtual method as described above] + * + * There are different considerations in each call graph for things like error handling, + * since the error conditions will be different depending on whether or not an EncoderNLS + * instance is available and what values its properties have. + */ + + /* + * THESE TWO METHODS MUST BE OVERRIDDEN BY A SUBCLASSED TYPE + */ + + internal virtual OperationStatus DecodeFirstRune(ReadOnlySpan<byte> bytes, out Rune value, out int bytesConsumed) + { + Debug.Fail("This should be overridden by a subclassed type."); + throw NotImplemented.ByDesign; + } + + internal virtual OperationStatus EncodeRune(Rune value, Span<byte> bytes, out int bytesWritten) + { + Debug.Fail("This should be overridden by a subclassed type."); + throw NotImplemented.ByDesign; + } + + /* + * ALL OTHER LOGIC CAN BE IMPLEMENTED IN TERMS OF THE TWO METHODS ABOVE. + * FOR IMPROVED PERFORMANCE, SUBCLASSED TYPES MAY WANT TO OVERRIDE ONE OR MORE VIRTUAL METHODS BELOW. + */ + + /* + * GETBYTECOUNT FAMILY OF FUNCTIONS + */ + + /// <summary> + /// Given a <see cref="Rune"/>, determines its byte count under the current <see cref="Encoding"/>. + /// Returns <see langword="false"/> if the <see cref="Rune"/> cannot be represented in the + /// current <see cref="Encoding"/>. + /// </summary> + internal virtual bool TryGetByteCount(Rune value, out int byteCount) + { + // Any production-quality type would override this method and provide a real + // implementation, so we won't provide a base implementation. However, a + // non-shipping slow reference implementation is provided below for convenience. + +#if false + Span<byte> bytes = stackalloc byte[4]; // max 4 bytes per input scalar + + OperationStatus opStatus = EncodeRune(value, bytes, out byteCount); + Debug.Assert(opStatus == OperationStatus.Done || opStatus == OperationStatus.InvalidData, "Unexpected return value."); + + return (opStatus == OperationStatus.Done); +#else + Debug.Fail("This should be overridden by a subclassed type."); + throw NotImplemented.ByDesign; +#endif + } + + /// <summary> + /// Entry point from <see cref="EncoderNLS.GetByteCount"/>. + /// </summary> + internal virtual unsafe int GetByteCount(char* pChars, int charCount, EncoderNLS encoder) + { + Debug.Assert(encoder != null, "This code path should only be called from EncoderNLS."); + Debug.Assert(charCount >= 0, "Caller should've checked this condition."); + Debug.Assert(pChars != null || charCount == 0, "Cannot provide a null pointer and a non-zero count."); + + // We're going to try to stay on the fast-path as much as we can. That means that we have + // no leftover data to drain and the entire source buffer can be consumed in a single + // fast-path invocation. If either of these doesn't hold, we'll go down the slow path of + // creating spans, draining the EncoderNLS instance, and falling back. + + int totalByteCount = 0; + int charsConsumed = 0; + + if (!encoder.HasLeftoverData) + { + totalByteCount = GetByteCountFast(pChars, charCount, encoder.Fallback, out charsConsumed); + if (charsConsumed == charCount) + { + return totalByteCount; + } + } + + // We had leftover data, or we couldn't consume the entire input buffer. + // Let's go down the draining + fallback mechanisms. + + totalByteCount += GetByteCountWithFallback(pChars, charCount, charsConsumed, encoder); + if (totalByteCount < 0) + { + ThrowConversionOverflow(); + } + + return totalByteCount; + } + + /// <summary> + /// Counts the number of <see langword="byte"/>s that would result from transcoding the source + /// data, exiting when the source buffer is consumed or when the first unreadable data is encountered. + /// The implementation may inspect <paramref name="fallback"/> to short-circuit any counting + /// operation, but it should not attempt to call <see cref="EncoderFallback.CreateFallbackBuffer"/>. + /// </summary> + /// <returns> + /// Via <paramref name="charsConsumed"/>, the number of elements from <paramref name="pChars"/> which + /// were consumed; and returns the transcoded byte count up to this point. + /// </returns> + /// <exception cref="ArgumentException"> + /// If the byte count would be greater than <see cref="int.MaxValue"/>. + /// (Implementation should call <see cref="ThrowConversionOverflow"/>.) + /// </exception> + /// <remarks> + /// The implementation should not attempt to perform any sort of fallback behavior. + /// If custom fallback behavior is necessary, override <see cref="GetByteCountWithFallback"/>. + /// </remarks> + private protected virtual unsafe int GetByteCountFast(char* pChars, int charsLength, EncoderFallback fallback, out int charsConsumed) + { + // Any production-quality type would override this method and provide a real + // implementation, so we won't provide a base implementation. However, a + // non-shipping slow reference implementation is provided below for convenience. + +#if false + ReadOnlySpan<char> chars = new ReadOnlySpan<char>(pChars, charsLength); + int totalByteCount = 0; + + while (!chars.IsEmpty) + { + if (Rune.DecodeUtf16(chars, out Rune scalarValue, out int charsConsumedThisIteration) != OperationStatus.Done + || !TryGetByteCount(scalarValue, out int byteCountThisIteration)) + { + // Invalid UTF-16 data, or not convertible to target encoding + + break; + } + + chars = chars.Slice(charsConsumedThisIteration); + + totalByteCount += byteCountThisIteration; + if (totalByteCount < 0) + { + ThrowConversionOverflow(); + } + } + + charsConsumed = charsLength - chars.Length; // number of chars consumed across all loop iterations above + return totalByteCount; +#else + Debug.Fail("This should be overridden by a subclassed type."); + throw NotImplemented.ByDesign; +#endif + } + + /// <summary> + /// Counts the number of bytes that would result from transcoding the provided chars, + /// with no associated <see cref="EncoderNLS"/>. The first two arguments are based on the + /// original input before invoking this method; and <paramref name="charsConsumedSoFar"/> + /// signals where in the provided buffer the fallback loop should begin operating. + /// </summary> + /// <returns> + /// The byte count resulting from transcoding the input data. + /// </returns> + /// <exception cref="ArgumentException"> + /// If the resulting byte count is greater than <see cref="int.MaxValue"/>. + /// (Implementation should call <see cref="ThrowConversionOverflow"/>.) + /// </exception> + [MethodImpl(MethodImplOptions.NoInlining)] // don't stack spill spans into our caller + private protected unsafe int GetByteCountWithFallback(char* pCharsOriginal, int originalCharCount, int charsConsumedSoFar) + { + // This is a stub method that's marked "no-inlining" so that it we don't stack-spill spans + // into our immediate caller. Doing so increases the method prolog in what's supposed to + // be a very fast path. + + Debug.Assert(0 <= charsConsumedSoFar && charsConsumedSoFar < originalCharCount, "Invalid arguments provided to method."); + + return GetByteCountWithFallback( + chars: new ReadOnlySpan<char>(pCharsOriginal, originalCharCount).Slice(charsConsumedSoFar), + originalCharsLength: originalCharCount, + encoder: null); + } + + /// <summary> + /// Gets the number of <see langword="byte"/>s that would result from transcoding the provided + /// input data, with an associated <see cref="EncoderNLS"/>. The first two arguments are + /// based on the original input before invoking this method; and <paramref name="charsConsumedSoFar"/> + /// signals where in the provided source buffer the fallback loop should begin operating. + /// The behavior of this method is to consume (non-destructively) any leftover data in the + /// <see cref="EncoderNLS"/> instance, then to invoke the <see cref="GetByteCountFast"/> virtual method + /// after data has been drained, then to call <see cref="GetByteCountWithFallback(ReadOnlySpan{char}, int, EncoderNLS)"/>. + /// </summary> + /// <returns> + /// The total number of bytes that would result from transcoding the remaining portion of the source buffer. + /// </returns> + /// <exception cref="ArgumentException"> + /// If the return value would exceed <see cref="int.MaxValue"/>. + /// (The implementation should call <see cref="ThrowConversionOverflow"/>.) + /// </exception> + private unsafe int GetByteCountWithFallback(char* pOriginalChars, int originalCharCount, int charsConsumedSoFar, EncoderNLS encoder) + { + Debug.Assert(encoder != null, "This code path should only be called from EncoderNLS."); + Debug.Assert(0 <= charsConsumedSoFar && charsConsumedSoFar < originalCharCount, "Caller should've checked this condition."); + + // First, try draining any data that already exists on the encoder instance. If we can't complete + // that operation, there's no point to continuing down to the main workhorse methods. + + ReadOnlySpan<char> chars = new ReadOnlySpan<char>(pOriginalChars, originalCharCount).Slice(charsConsumedSoFar); + + int totalByteCount = encoder.DrainLeftoverDataForGetByteCount(chars, out int charsConsumedJustNow); + chars = chars.Slice(charsConsumedJustNow); + + // Now try invoking the "fast path" (no fallback) implementation. + // We can use Unsafe.AsPointer here since these spans are created from pinned data (raw pointers). + + totalByteCount += GetByteCountFast( + pChars: (char*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(chars)), + charsLength: chars.Length, + fallback: encoder.Fallback, + charsConsumed: out charsConsumedJustNow); + + if (totalByteCount < 0) + { + ThrowConversionOverflow(); + } + + chars = chars.Slice(charsConsumedJustNow); + + // If there's still data remaining in the source buffer, go down the fallback path. + // Otherwise we're finished. + + if (!chars.IsEmpty) + { + totalByteCount += GetByteCountWithFallback(chars, originalCharCount, encoder); + if (totalByteCount < 0) + { + ThrowConversionOverflow(); + } + } + + return totalByteCount; + } + + /// <summary> + /// Counts the number of bytes that would result from transcoding the provided chars, + /// using the provided <see cref="EncoderFallbackBuffer"/> if necessary. + /// </summary> + /// <returns> + /// The byte count resulting from transcoding the input data. + /// </returns> + /// <exception cref="ArgumentException"> + /// If the resulting byte count is greater than <see cref="int.MaxValue"/>. + /// (Implementation should call <see cref="ThrowConversionOverflow"/>.) + /// </exception> + private protected virtual unsafe int GetByteCountWithFallback(ReadOnlySpan<char> chars, int originalCharsLength, EncoderNLS encoder) + { + Debug.Assert(!chars.IsEmpty, "Caller shouldn't invoke this method with an empty input buffer."); + Debug.Assert(originalCharsLength >= 0, "Caller provided invalid parameter."); + + // Since we're using Unsafe.AsPointer in our central loop, we want to ensure everything is pinned. + + fixed (char* _pChars_Unused = &MemoryMarshal.GetReference(chars)) + { + EncoderFallbackBuffer fallbackBuffer = EncoderFallbackBuffer.CreateAndInitialize(this, encoder, originalCharsLength); + int totalByteCount = 0; + + do + { + // There's still data in the source buffer; why wasn't the previous fast-path able to consume it fully? + // There are two scenarios: (a) the source buffer contained invalid / incomplete UTF-16 data; + // or (b) the encoding can't translate this scalar value. + + if (Rune.DecodeUtf16(chars, out Rune firstScalarValue, out int charsConsumedThisIteration) == OperationStatus.NeedMoreData + && encoder != null + && !encoder.MustFlush) + { + // We saw a standalone high surrogate at the end of the buffer, and the + // active EncoderNLS instance isn't asking us to flush. Since a call to + // GetBytes would've consumed this char by storing it in EncoderNLS._charLeftOver, + // we'll "consume" it by ignoring it. The next call to GetBytes will + // pick it up correctly. + + goto Finish; + } + + // We saw invalid UTF-16 data, or we saw a high surrogate that we need to flush (and + // thus treat as invalid), or we saw valid UTF-16 data that this encoder doesn't support. + // In any case we'll run it through the fallback mechanism. + + int byteCountThisIteration = fallbackBuffer.InternalFallbackGetByteCount(chars, out charsConsumedThisIteration); + + Debug.Assert(byteCountThisIteration >= 0, "Fallback shouldn't have returned a negative value."); + Debug.Assert(charsConsumedThisIteration >= 0, "Fallback shouldn't have returned a negative value."); + + totalByteCount += byteCountThisIteration; + if (totalByteCount < 0) + { + ThrowConversionOverflow(); + } + + chars = chars.Slice(charsConsumedThisIteration); + + if (!chars.IsEmpty) + { + // Still data remaining - run it through the fast-path to find the next data to fallback. + // While building up the tally we need to continually check for integer overflow + // since fallbacks can change the total byte count in unexpected ways. + + byteCountThisIteration = GetByteCountFast( + pChars: (char*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(chars)), + charsLength: chars.Length, + fallback: null, // already tried this earlier and we still fell down the common path, so skip from now on + charsConsumed: out charsConsumedThisIteration); + + Debug.Assert(byteCountThisIteration >= 0, "Workhorse shouldn't have returned a negative value."); + Debug.Assert(charsConsumedThisIteration >= 0, "Workhorse shouldn't have returned a negative value."); + + totalByteCount += byteCountThisIteration; + if (totalByteCount < 0) + { + ThrowConversionOverflow(); + } + + chars = chars.Slice(charsConsumedThisIteration); + } + } while (!chars.IsEmpty); + + Finish: + + Debug.Assert(fallbackBuffer.Remaining == 0, "There should be no data in the fallback buffer after GetByteCount."); + + return totalByteCount; + } + } + + /* + * GETBYTES FAMILY OF FUNCTIONS + */ + + /// <summary> + /// Entry point from <see cref="EncoderNLS.GetBytes"/> and <see cref="EncoderNLS.Convert"/>. + /// </summary> + internal virtual unsafe int GetBytes(char* pChars, int charCount, byte* pBytes, int byteCount, EncoderNLS encoder) + { + Debug.Assert(encoder != null, "This code path should only be called from EncoderNLS."); + Debug.Assert(charCount >= 0, "Caller should've checked this condition."); + Debug.Assert(pChars != null || charCount == 0, "Cannot provide a null pointer and a non-zero count."); + Debug.Assert(byteCount >= 0, "Caller should've checked this condition."); + Debug.Assert(pBytes != null || byteCount == 0, "Cannot provide a null pointer and a non-zero count."); + + // We're going to try to stay on the fast-path as much as we can. That means that we have + // no leftover data to drain and the entire source buffer can be transcoded in a single + // fast-path invocation. If either of these doesn't hold, we'll go down the slow path of + // creating spans, draining the EncoderNLS instance, and falling back. + + int bytesWritten = 0; + int charsConsumed = 0; + + if (!encoder.HasLeftoverData) + { + bytesWritten = GetBytesFast(pChars, charCount, pBytes, byteCount, out charsConsumed); + if (charsConsumed == charCount) + { + encoder._charsUsed = charCount; + return bytesWritten; + } + } + + // We had leftover data, or we couldn't consume the entire input buffer. + // Let's go down the draining + fallback mechanisms. + + return GetBytesWithFallback(pChars, charCount, pBytes, byteCount, charsConsumed, bytesWritten, encoder); + } + + /// <summary> + /// Transcodes <see langword="char"/>s to <see langword="byte"/>s, exiting when the source or destination + /// buffer is consumed or when the first unreadable data is encountered. + /// </summary> + /// <returns> + /// Via <paramref name="charsConsumed"/>, the number of elements from <paramref name="pChars"/> which + /// were consumed; and returns the number of elements written to <paramref name="pBytes"/>. + /// </returns> + /// <remarks> + /// The implementation should not attempt to perform any sort of fallback behavior. + /// If custom fallback behavior is necessary, override <see cref="GetBytesWithFallback"/>. + /// </remarks> + private protected virtual unsafe int GetBytesFast(char* pChars, int charsLength, byte* pBytes, int bytesLength, out int charsConsumed) + { + // Any production-quality type would override this method and provide a real + // implementation, so we won't provide a base implementation. However, a + // non-shipping slow reference implementation is provided below for convenience. + +#if false + ReadOnlySpan<char> chars = new ReadOnlySpan<char>(pChars, charsLength); + Span<byte> bytes = new Span<byte>(pBytes, bytesLength); + + while (!chars.IsEmpty) + { + if (Rune.DecodeUtf16(chars, out Rune scalarValue, out int charsConsumedJustNow) != OperationStatus.Done + || EncodeRune(scalarValue, bytes, out int bytesWrittenJustNow) != OperationStatus.Done) + { + // Invalid UTF-16 data, or not convertible to target encoding, or destination buffer too small to contain encoded value + + break; + } + + chars = chars.Slice(charsConsumedJustNow); + bytes = bytes.Slice(bytesWrittenJustNow); + } + + charsConsumed = charsLength - chars.Length; // number of chars consumed across all loop iterations above + return bytesLength - bytes.Length; // number of bytes written across all loop iterations above +#else + Debug.Fail("This should be overridden by a subclassed type."); + throw NotImplemented.ByDesign; +#endif + } + + /// <summary> + /// Transcodes chars to bytes, with no associated <see cref="EncoderNLS"/>. The first four arguments are + /// based on the original input before invoking this method; and <paramref name="charsConsumedSoFar"/> + /// and <paramref name="bytesWrittenSoFar"/> signal where in the provided buffers the fallback loop + /// should begin operating. The behavior of this method is to call the <see cref="GetBytesWithFallback"/> + /// virtual method as overridden by the specific type, and failing that go down the shared fallback path. + /// </summary> + /// <returns> + /// The total number of bytes written to <paramref name="pOriginalBytes"/>, including <paramref name="bytesWrittenSoFar"/>. + /// </returns> + /// <exception cref="ArgumentException"> + /// If the destination buffer is not large enough to hold the entirety of the transcoded data. + /// </exception> + [MethodImpl(MethodImplOptions.NoInlining)] + private protected unsafe int GetBytesWithFallback(char* pOriginalChars, int originalCharCount, byte* pOriginalBytes, int originalByteCount, int charsConsumedSoFar, int bytesWrittenSoFar) + { + // This is a stub method that's marked "no-inlining" so that it we don't stack-spill spans + // into our immediate caller. Doing so increases the method prolog in what's supposed to + // be a very fast path. + + Debug.Assert(0 <= charsConsumedSoFar && charsConsumedSoFar < originalCharCount, "Invalid arguments provided to method."); + Debug.Assert(0 <= bytesWrittenSoFar && bytesWrittenSoFar <= originalByteCount, "Invalid arguments provided to method."); + + return GetBytesWithFallback( + chars: new ReadOnlySpan<char>(pOriginalChars, originalCharCount).Slice(charsConsumedSoFar), + originalCharsLength: originalCharCount, + bytes: new Span<byte>(pOriginalBytes, originalByteCount).Slice(bytesWrittenSoFar), + originalBytesLength: originalByteCount, + encoder: null); + } + + /// <summary> + /// Transcodes chars to bytes, with an associated <see cref="EncoderNLS"/>. The first four arguments are + /// based on the original input before invoking this method; and <paramref name="charsConsumedSoFar"/> + /// and <paramref name="bytesWrittenSoFar"/> signal where in the provided buffers the fallback loop + /// should begin operating. The behavior of this method is to drain any leftover data in the + /// <see cref="EncoderNLS"/> instance, then to invoke the <see cref="GetBytesFast"/> virtual method + /// after data has been drained, then to call <see cref="GetBytesWithFallback(ReadOnlySpan{char}, int, Span{byte}, int, EncoderNLS)"/>. + /// </summary> + /// <returns> + /// The total number of bytes written to <paramref name="pOriginalBytes"/>, including <paramref name="bytesWrittenSoFar"/>. + /// </returns> + /// <exception cref="ArgumentException"> + /// If the destination buffer is too small to make any forward progress at all, or if the destination buffer is + /// too small to contain the entirety of the transcoded data and the <see cref="EncoderNLS"/> instance disallows + /// partial transcoding. + /// </exception> + private unsafe int GetBytesWithFallback(char* pOriginalChars, int originalCharCount, byte* pOriginalBytes, int originalByteCount, int charsConsumedSoFar, int bytesWrittenSoFar, EncoderNLS encoder) + { + Debug.Assert(encoder != null, "This code path should only be called from EncoderNLS."); + Debug.Assert(0 <= charsConsumedSoFar && charsConsumedSoFar < originalCharCount, "Caller should've checked this condition."); + Debug.Assert(0 <= bytesWrittenSoFar && bytesWrittenSoFar <= originalByteCount, "Caller should've checked this condition."); + + // First, try draining any data that already exists on the encoder instance. If we can't complete + // that operation, there's no point to continuing down to the main workhorse methods. + + ReadOnlySpan<char> chars = new ReadOnlySpan<char>(pOriginalChars, originalCharCount).Slice(charsConsumedSoFar); + Span<byte> bytes = new Span<byte>(pOriginalBytes, originalByteCount).Slice(bytesWrittenSoFar); + + bool drainFinishedSuccessfully = encoder.TryDrainLeftoverDataForGetBytes(chars, bytes, out int charsConsumedJustNow, out int bytesWrittenJustNow); + + chars = chars.Slice(charsConsumedJustNow); // whether or not the drain finished, we may have made some progress + bytes = bytes.Slice(bytesWrittenJustNow); + + if (!drainFinishedSuccessfully) + { + ThrowBytesOverflow(encoder, nothingEncoded: bytes.Length == originalByteCount); // might not throw if we wrote at least one byte + } + else + { + // Now try invoking the "fast path" (no fallback) implementation. + // We can use Unsafe.AsPointer here since these spans are created from pinned data (raw pointers). + + bytesWrittenJustNow = GetBytesFast( + pChars: (char*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(chars)), + charsLength: chars.Length, + pBytes: (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(bytes)), + bytesLength: bytes.Length, + charsConsumed: out charsConsumedJustNow); + + chars = chars.Slice(charsConsumedJustNow); + bytes = bytes.Slice(bytesWrittenJustNow); + + // If there's still data remaining in the source buffer, go down the fallback path. + // Otherwise we're finished. + + if (!chars.IsEmpty) + { + // We'll optimistically tell the encoder that we're using everything; the + // GetBytesWithFallback method will overwrite this field if necessary. + + encoder._charsUsed = originalCharCount; + return GetBytesWithFallback(chars, originalCharCount, bytes, originalByteCount, encoder); + } + } + + encoder._charsUsed = originalCharCount - chars.Length; // total number of characters consumed up until now + return originalByteCount - bytes.Length; // total number of bytes written up until now + } + + /// <summary> + /// Transcodes chars to bytes, using <see cref="Encoding.EncoderFallback"/> or <see cref="Encoder.Fallback"/> if needed. + /// </summary> + /// <returns> + /// The total number of bytes written to <paramref name="bytes"/> (based on <paramref name="originalBytesLength"/>). + /// </returns> + /// <remarks> + /// The derived class should override this method if it might be able to provide a more optimized fallback + /// implementation, deferring to the base implementation if needed. This method calls <see cref="ThrowBytesOverflow"/> + /// if necessary. + /// </remarks> + private protected virtual unsafe int GetBytesWithFallback(ReadOnlySpan<char> chars, int originalCharsLength, Span<byte> bytes, int originalBytesLength, EncoderNLS encoder) + { + Debug.Assert(!chars.IsEmpty, "Caller shouldn't invoke this method with an empty input buffer."); + Debug.Assert(originalCharsLength >= 0, "Caller provided invalid parameter."); + Debug.Assert(originalBytesLength >= 0, "Caller provided invalid parameter."); + + // Since we're using Unsafe.AsPointer in our central loop, we want to ensure everything is pinned. + + fixed (char* _pChars_Unused = &MemoryMarshal.GetReference(chars)) + fixed (byte* _pBytes_Unused = &MemoryMarshal.GetReference(bytes)) + { + EncoderFallbackBuffer fallbackBuffer = EncoderFallbackBuffer.CreateAndInitialize(this, encoder, originalCharsLength); + + do + { + // There's still data in the source buffer; why wasn't the previous fast-path able to consume it fully? + // There are two scenarios: (a) the source buffer contained invalid / incomplete UTF-16 data; + // or (b) the encoding can't translate this scalar value. + + switch (Rune.DecodeUtf16(chars, out Rune firstScalarValue, out int charsConsumedThisIteration)) + { + case OperationStatus.NeedMoreData: + Debug.Assert(charsConsumedThisIteration == chars.Length, "If returning NeedMoreData, should out the entire buffer length as chars consumed."); + if (encoder is null || encoder.MustFlush) + { + goto case OperationStatus.InvalidData; // see comment in GetByteCountWithFallback + } + else + { + encoder._charLeftOver = chars[0]; // squirrel away remaining high surrogate char and finish + chars = ReadOnlySpan<char>.Empty; + goto Finish; + } + + case OperationStatus.InvalidData: + break; + + default: + if (EncodeRune(firstScalarValue, bytes, out _) == OperationStatus.DestinationTooSmall) + { + goto Finish; // source buffer contained valid UTF-16 but encoder ran out of space in destination buffer + } + break; // source buffer contained valid UTF-16 but encoder doesn't support this scalar value + } + + // Now we know the reason for failure was that the original input was invalid + // for the encoding in use. Run it through the fallback mechanism. + + bool fallbackFinished = fallbackBuffer.TryInternalFallbackGetBytes(chars, bytes, out charsConsumedThisIteration, out int bytesWrittenThisIteration); + + // Regardless of whether the fallback finished, it did consume some number of + // chars, and it may have written some number of bytes. + + chars = chars.Slice(charsConsumedThisIteration); + bytes = bytes.Slice(bytesWrittenThisIteration); + + if (!fallbackFinished) + { + goto Finish; // fallback has pending state - it'll get written out on the next GetBytes call + } + + if (!chars.IsEmpty) + { + // Still data remaining - run it through the fast-path to find the next data to fallback. + + bytesWrittenThisIteration = GetBytesFast( + pChars: (char*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(chars)), + charsLength: chars.Length, + pBytes: (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(bytes)), + bytesLength: bytes.Length, + charsConsumed: out charsConsumedThisIteration); + + Debug.Assert(bytesWrittenThisIteration >= 0, "Workhorse shouldn't have returned a negative value."); + Debug.Assert(charsConsumedThisIteration >= 0, "Workhorse shouldn't have returned a negative value."); + + chars = chars.Slice(charsConsumedThisIteration); + bytes = bytes.Slice(bytesWrittenThisIteration); + } + } while (!chars.IsEmpty); + + Finish: + + // We reach this point when we deplete the source or destination buffer. There are a few + // cases to consider now. If the source buffer has been fully consumed and there's no + // leftover data in the EncoderNLS or the fallback buffer, we've completed transcoding. + // If the source buffer isn't empty or there's leftover data in the fallback buffer, + // it means we ran out of space in the destintion buffer. This is an unrecoverable error + // if no EncoderNLS is in use (because only EncoderNLS can handle partial success), and + // even if an EncoderNLS is in use this is only recoverable if the EncoderNLS instance + // allows partial completion. Let's check all of these conditions now. + + if (!chars.IsEmpty || fallbackBuffer.Remaining > 0) + { + // The line below will also throw if the encoder couldn't make any progress at all + // because the output buffer wasn't large enough to contain the result of even + // a single scalar conversion or fallback. + + ThrowBytesOverflow(encoder, nothingEncoded: bytes.Length == originalBytesLength); + } + + // If an EncoderNLS instance is active, update its "total consumed character count" value. + + if (encoder != null) + { + Debug.Assert(originalCharsLength >= chars.Length, "About to report a negative number of chars used?"); + encoder._charsUsed = originalCharsLength - chars.Length; // number of chars consumed + } + + Debug.Assert(fallbackBuffer.Remaining == 0 || encoder != null, "Shouldn't have any leftover data in fallback buffer unless an EncoderNLS is in use."); + + return originalBytesLength - bytes.Length; + } + } + + /* + * GETCHARCOUNT FAMILY OF FUNCTIONS + */ + + /// <summary> + /// Entry point from <see cref="DecoderNLS.GetCharCount"/>. + /// </summary> + internal virtual unsafe int GetCharCount(byte* pBytes, int byteCount, DecoderNLS decoder) + { + Debug.Assert(decoder != null, "This code path should only be called from DecoderNLS."); + Debug.Assert(byteCount >= 0, "Caller should've checked this condition."); + Debug.Assert(pBytes != null || byteCount == 0, "Cannot provide a null pointer and a non-zero count."); + + // We're going to try to stay on the fast-path as much as we can. That means that we have + // no leftover data to drain and the entire source buffer can be consumed in a single + // fast-path invocation. If either of these doesn't hold, we'll go down the slow path of + // creating spans, draining the DecoderNLS instance, and falling back. + + Debug.Assert(!decoder.InternalHasFallbackBuffer || decoder.FallbackBuffer.Remaining == 0, "Fallback buffer can't hold data between GetChars invocations."); + + int totalCharCount = 0; + int bytesConsumed = 0; + + if (!decoder.HasLeftoverData) + { + totalCharCount = GetCharCountFast(pBytes, byteCount, decoder.Fallback, out bytesConsumed); + if (bytesConsumed == byteCount) + { + return totalCharCount; + } + } + + // We had leftover data, or we couldn't consume the entire input buffer. + // Let's go down the draining + fallback mechanisms. + + totalCharCount += GetCharCountWithFallback(pBytes, byteCount, bytesConsumed, decoder); + if (totalCharCount < 0) + { + ThrowConversionOverflow(); + } + + return totalCharCount; + } + + /// <summary> + /// Counts the number of <see langword="char"/>s that would result from transcoding the source + /// data, exiting when the source buffer is consumed or when the first unreadable data is encountered. + /// The implementation may inspect <paramref name="fallback"/> to short-circuit any counting + /// operation, but it should not attempt to call <see cref="DecoderFallback.CreateFallbackBuffer"/>. + /// </summary> + /// <returns> + /// Via <paramref name="bytesConsumed"/>, the number of elements from <paramref name="pBytes"/> which + /// were consumed; and returns the transcoded char count up to this point. + /// </returns> + /// <exception cref="ArgumentException"> + /// If the char count would be greater than <see cref="int.MaxValue"/>. + /// (Implementation should call <see cref="ThrowConversionOverflow"/>.) + /// </exception> + /// <remarks> + /// The implementation should not attempt to perform any sort of fallback behavior. + /// If custom fallback behavior is necessary, override <see cref="GetCharCountWithFallback"/>. + /// </remarks> + private protected virtual unsafe int GetCharCountFast(byte* pBytes, int bytesLength, DecoderFallback fallback, out int bytesConsumed) + { + // Any production-quality type would override this method and provide a real + // implementation, so we won't provide a base implementation. However, a + // non-shipping slow reference implementation is provided below for convenience. + +#if false + ReadOnlySpan<byte> bytes = new ReadOnlySpan<byte>(pBytes, bytesLength); + int totalCharCount = 0; + + while (!bytes.IsEmpty) + { + // We don't care about statuses other than Done. The fallback mechanism will handle those. + + if (DecodeFirstRune(bytes, out Rune value, out int bytesConsumedJustNow) != OperationStatus.Done) + { + break; + } + + totalCharCount += value.Utf16SequenceLength; + if (totalCharCount < 0) + { + ThrowConversionOverflow(); + } + + bytes = bytes.Slice(bytesConsumedJustNow); + } + + bytesConsumed = bytesLength - bytes.Length; // number of bytes consumed across all loop iterations above + return totalCharCount; +#else + Debug.Fail("This should be overridden by a subclassed type."); + throw NotImplemented.ByDesign; +#endif + } + + /// <summary> + /// Counts the number of chars that would result from transcoding the provided bytes, + /// with no associated <see cref="DecoderNLS"/>. The first two arguments are based on the + /// original input before invoking this method; and <paramref name="bytesConsumedSoFar"/> + /// signals where in the provided buffer the fallback loop should begin operating. + /// </summary> + /// <returns> + /// The char count resulting from transcoding the input data. + /// </returns> + /// <exception cref="ArgumentException"> + /// If the resulting char count is greater than <see cref="int.MaxValue"/>. + /// (Implementation should call <see cref="ThrowConversionOverflow"/>.) + /// </exception> + [MethodImpl(MethodImplOptions.NoInlining)] // don't stack spill spans into our caller + private protected unsafe int GetCharCountWithFallback(byte* pBytesOriginal, int originalByteCount, int bytesConsumedSoFar) + { + // This is a stub method that's marked "no-inlining" so that it we don't stack-spill spans + // into our immediate caller. Doing so increases the method prolog in what's supposed to + // be a very fast path. + + Debug.Assert(0 <= bytesConsumedSoFar && bytesConsumedSoFar < originalByteCount, "Invalid arguments provided to method."); + + return GetCharCountWithFallback( + bytes: new ReadOnlySpan<byte>(pBytesOriginal, originalByteCount).Slice(bytesConsumedSoFar), + originalBytesLength: originalByteCount, + decoder: null); + } + + /// <summary> + /// Gets the number of <see langword="char"/>s that would result from transcoding the provided + /// input data, with an associated <see cref="DecoderNLS"/>. The first two arguments are + /// based on the original input before invoking this method; and <paramref name="bytesConsumedSoFar"/> + /// signals where in the provided source buffer the fallback loop should begin operating. + /// The behavior of this method is to consume (non-destructively) any leftover data in the + /// <see cref="DecoderNLS"/> instance, then to invoke the <see cref="GetCharCountFast"/> virtual method + /// after data has been drained, then to call <see cref="GetCharCountWithFallback(ReadOnlySpan{byte}, int, DecoderNLS)"/>. + /// </summary> + /// <returns> + /// The total number of chars that would result from transcoding the remaining portion of the source buffer. + /// </returns> + /// <exception cref="ArgumentException"> + /// If the return value would exceed <see cref="int.MaxValue"/>. + /// (The implementation should call <see cref="ThrowConversionOverflow"/>.) + /// </exception> + private unsafe int GetCharCountWithFallback(byte* pOriginalBytes, int originalByteCount, int bytesConsumedSoFar, DecoderNLS decoder) + { + Debug.Assert(decoder != null, "This code path should only be called from DecoderNLS."); + Debug.Assert(0 <= bytesConsumedSoFar && bytesConsumedSoFar < originalByteCount, "Caller should've checked this condition."); + + // First, try draining any data that already exists on the decoder instance. If we can't complete + // that operation, there's no point to continuing down to the main workhorse methods. + + ReadOnlySpan<byte> bytes = new ReadOnlySpan<byte>(pOriginalBytes, originalByteCount).Slice(bytesConsumedSoFar); + + int totalCharCount = decoder.DrainLeftoverDataForGetCharCount(bytes, out int bytesConsumedJustNow); + bytes = bytes.Slice(bytesConsumedJustNow); + + // Now try invoking the "fast path" (no fallback) implementation. + // We can use Unsafe.AsPointer here since these spans are created from pinned data (raw pointers). + + totalCharCount += GetCharCountFast( + pBytes: (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(bytes)), + bytesLength: bytes.Length, + fallback: decoder.Fallback, + bytesConsumed: out bytesConsumedJustNow); + + if (totalCharCount < 0) + { + ThrowConversionOverflow(); + } + + bytes = bytes.Slice(bytesConsumedJustNow); + + // If there's still data remaining in the source buffer, go down the fallback path. + // Otherwise we're finished. + + if (!bytes.IsEmpty) + { + totalCharCount += GetCharCountWithFallback(bytes, originalByteCount, decoder); + if (totalCharCount < 0) + { + ThrowConversionOverflow(); + } + } + + return totalCharCount; + } + + /// <summary> + /// Counts the number of chars that would result from transcoding the provided bytes, + /// using the provided <see cref="DecoderFallbackBuffer"/> if necessary. + /// </summary> + /// <returns> + /// The char count resulting from transcoding the input data. + /// </returns> + /// <exception cref="ArgumentException"> + /// If the resulting char count is greater than <see cref="int.MaxValue"/>. + /// (Implementation should call <see cref="ThrowConversionOverflow"/>.) + /// </exception> + private unsafe int GetCharCountWithFallback(ReadOnlySpan<byte> bytes, int originalBytesLength, DecoderNLS decoder) + { + Debug.Assert(!bytes.IsEmpty, "Caller shouldn't invoke this method with an empty input buffer."); + Debug.Assert(originalBytesLength >= 0, "Caller provided invalid parameter."); + + // Since we're using Unsafe.AsPointer in our central loop, we want to ensure everything is pinned. + + fixed (byte* _pBytes_Unused = &MemoryMarshal.GetReference(bytes)) + { + DecoderFallbackBuffer fallbackBuffer = DecoderFallbackBuffer.CreateAndInitialize(this, decoder, originalBytesLength); + int totalCharCount = 0; + + do + { + // There's still data in the source buffer; why wasn't the previous fast-path able to consume it fully? + // There are two scenarios: (a) the source buffer contained invalid data, or it contained incomplete data. + + if (DecodeFirstRune(bytes, out Rune firstScalarValue, out int bytesConsumedThisIteration) == OperationStatus.NeedMoreData + && decoder != null + && !decoder.MustFlush) + { + // We saw incomplete data at the end of the buffer, and the active DecoderNLS isntance + // isn't asking us to flush. Since a call to GetChars would've consumed this data by + // storing it in the DecoderNLS instance, we'll "consume" it by ignoring it. + // The next call to GetChars will pick it up correctly. + + goto Finish; + } + + // We saw invalid binary data, or we saw incomplete data that we need to flush (and thus + // treat as invalid). In any case we'll run through the fallback mechanism. + + int charCountThisIteration = fallbackBuffer.InternalFallbackGetCharCount(bytes, bytesConsumedThisIteration); + + Debug.Assert(charCountThisIteration >= 0, "Fallback shouldn't have returned a negative value."); + + totalCharCount += charCountThisIteration; + if (totalCharCount < 0) + { + ThrowConversionOverflow(); + } + + bytes = bytes.Slice(bytesConsumedThisIteration); + + if (!bytes.IsEmpty) + { + // Still data remaining - run it through the fast-path to find the next data to fallback. + // While building up the tally we need to continually check for integer overflow + // since fallbacks can change the total byte count in unexpected ways. + + charCountThisIteration = GetCharCountFast( + pBytes: (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(bytes)), + bytesLength: bytes.Length, + fallback: null, // wasn't able to be short-circuited by our caller; don't bother trying again + bytesConsumed: out bytesConsumedThisIteration); + + Debug.Assert(charCountThisIteration >= 0, "Workhorse shouldn't have returned a negative value."); + Debug.Assert(bytesConsumedThisIteration >= 0, "Workhorse shouldn't have returned a negative value."); + + totalCharCount += charCountThisIteration; + if (totalCharCount < 0) + { + ThrowConversionOverflow(); + } + + bytes = bytes.Slice(bytesConsumedThisIteration); + } + } while (!bytes.IsEmpty); + + Finish: + + Debug.Assert(fallbackBuffer.Remaining == 0, "There should be no data in the fallback buffer after GetCharCount."); + + return totalCharCount; + } + } + + /* + * GETCHARS FAMILY OF FUNCTIONS + */ + + /// <summary> + /// Entry point from <see cref="DecoderNLS.GetChars"/> and <see cref="DecoderNLS.Convert"/>. + /// </summary> + internal virtual unsafe int GetChars(byte* pBytes, int byteCount, char* pChars, int charCount, DecoderNLS decoder) + { + Debug.Assert(decoder != null, "This code path should only be called from DecoderNLS."); + Debug.Assert(byteCount >= 0, "Caller should've checked this condition."); + Debug.Assert(pBytes != null || byteCount == 0, "Cannot provide a null pointer and a non-zero count."); + Debug.Assert(charCount >= 0, "Caller should've checked this condition."); + Debug.Assert(pChars != null || charCount == 0, "Cannot provide a null pointer and a non-zero count."); + + // We're going to try to stay on the fast-path as much as we can. That means that we have + // no leftover data to drain and the entire source buffer can be transcoded in a single + // fast-path invocation. If either of these doesn't hold, we'll go down the slow path of + // creating spans, draining the DecoderNLS instance, and falling back. + + int charsWritten = 0; + int bytesConsumed = 0; + + if (!decoder.HasLeftoverData) + { + charsWritten = GetCharsFast(pBytes, byteCount, pChars, charCount, out bytesConsumed); + if (bytesConsumed == byteCount) + { + decoder._bytesUsed = byteCount; + return charsWritten; + } + } + + // We had leftover data, or we couldn't consume the entire input buffer. + // Let's go down the draining + fallback mechanisms. + + return GetCharsWithFallback(pBytes, byteCount, pChars, charCount, bytesConsumed, charsWritten, decoder); + } + + /// <summary> + /// Transcodes <see langword="byte"/>s to <see langword="char"/>s, exiting when the source or destination + /// buffer is consumed or when the first unreadable data is encountered. + /// </summary> + /// <returns> + /// Via <paramref name="bytesConsumed"/>, the number of elements from <paramref name="pBytes"/> which + /// were consumed; and returns the number of elements written to <paramref name="pChars"/>. + /// </returns> + /// <remarks> + /// The implementation should not attempt to perform any sort of fallback behavior. + /// If custom fallback behavior is necessary, override <see cref="GetCharsWithFallback"/>. + /// </remarks> + private protected virtual unsafe int GetCharsFast(byte* pBytes, int bytesLength, char* pChars, int charsLength, out int bytesConsumed) + { + // Any production-quality type would override this method and provide a real + // implementation, so we won't provide a base implementation. However, a + // non-shipping slow reference implementation is provided below for convenience. + +#if false + ReadOnlySpan<byte> bytes = new ReadOnlySpan<byte>(pBytes, bytesLength); + Span<char> chars = new Span<char>(pChars, charsLength); + + while (!bytes.IsEmpty) + { + if ((DecodeFirstRune(bytes, out Rune firstScalarValue, out int bytesConsumedJustNow) != OperationStatus.Done) + || !firstScalarValue.TryEncode(chars, out int charsWrittenJustNow)) + { + // Invalid or incomplete binary data, or destination buffer too small to contain decoded value + + break; + } + + bytes = bytes.Slice(bytesConsumedJustNow); + chars = chars.Slice(charsWrittenJustNow); + } + + bytesConsumed = bytesLength - bytes.Length; // number of bytes consumed across all loop iterations above + return charsLength - chars.Length; // number of chars written across all loop iterations above +#else + Debug.Fail("This should be overridden by a subclassed type."); + throw NotImplemented.ByDesign; +#endif + } + + /// <summary> + /// Transcodes bytes to chars, with no associated <see cref="DecoderNLS"/>. The first four arguments are + /// based on the original input before invoking this method; and <paramref name="bytesConsumedSoFar"/> + /// and <paramref name="charsWrittenSoFar"/> signal where in the provided buffers the fallback loop + /// should begin operating. The behavior of this method is to call the <see cref="GetCharsWithFallback"/> + /// virtual method as overridden by the specific type, and failing that go down the shared fallback path. + /// </summary> + /// <returns> + /// The total number of chars written to <paramref name="pOriginalChars"/>, including <paramref name="charsWrittenSoFar"/>. + /// </returns> + /// <exception cref="ArgumentException"> + /// If the destination buffer is not large enough to hold the entirety of the transcoded data. + /// </exception> + [MethodImpl(MethodImplOptions.NoInlining)] + private protected unsafe int GetCharsWithFallback(byte* pOriginalBytes, int originalByteCount, char* pOriginalChars, int originalCharCount, int bytesConsumedSoFar, int charsWrittenSoFar) + { + // This is a stub method that's marked "no-inlining" so that it we don't stack-spill spans + // into our immediate caller. Doing so increases the method prolog in what's supposed to + // be a very fast path. + + Debug.Assert(0 <= bytesConsumedSoFar && bytesConsumedSoFar < originalByteCount, "Invalid arguments provided to method."); + Debug.Assert(0 <= charsWrittenSoFar && charsWrittenSoFar <= originalCharCount, "Invalid arguments provided to method."); + + return GetCharsWithFallback( + bytes: new ReadOnlySpan<byte>(pOriginalBytes, originalByteCount).Slice(bytesConsumedSoFar), + originalBytesLength: originalByteCount, + chars: new Span<char>(pOriginalChars, originalCharCount).Slice(charsWrittenSoFar), + originalCharsLength: originalCharCount, + decoder: null); + } + + /// <summary> + /// Transcodes bytes to chars, with an associated <see cref="DecoderNLS"/>. The first four arguments are + /// based on the original input before invoking this method; and <paramref name="bytesConsumedSoFar"/> + /// and <paramref name="charsWrittenSoFar"/> signal where in the provided buffers the fallback loop + /// should begin operating. The behavior of this method is to drain any leftover data in the + /// <see cref="DecoderNLS"/> instance, then to invoke the <see cref="GetCharsFast"/> virtual method + /// after data has been drained, then to call <see cref="GetCharsWithFallback(ReadOnlySpan{byte}, int, Span{char}, int, DecoderNLS)"/>. + /// </summary> + /// <returns> + /// The total number of chars written to <paramref name="pOriginalChars"/>, including <paramref name="charsWrittenSoFar"/>. + /// </returns> + /// <exception cref="ArgumentException"> + /// If the destination buffer is too small to make any forward progress at all, or if the destination buffer is + /// too small to contain the entirety of the transcoded data and the <see cref="DecoderNLS"/> instance disallows + /// partial transcoding. + /// </exception> + private protected unsafe int GetCharsWithFallback(byte* pOriginalBytes, int originalByteCount, char* pOriginalChars, int originalCharCount, int bytesConsumedSoFar, int charsWrittenSoFar, DecoderNLS decoder) + { + Debug.Assert(decoder != null, "This code path should only be called from DecoderNLS."); + Debug.Assert(0 <= bytesConsumedSoFar && bytesConsumedSoFar < originalByteCount, "Caller should've checked this condition."); + Debug.Assert(0 <= charsWrittenSoFar && charsWrittenSoFar <= originalCharCount, "Caller should've checked this condition."); + + // First, try draining any data that already exists on the encoder instance. If we can't complete + // that operation, there's no point to continuing down to the main workhorse methods. + // + // Like GetBytes, there may be leftover data in the DecoderNLS instance. But unlike GetBytes, + // the bytes -> chars conversion doesn't allow leftover data in the fallback buffer. This means + // that the drain operation below will either succeed fully or fail; there's no partial success + // condition as with the chars -> bytes conversion. The drain method will throw if there's not + // enough space in the destination buffer. + + ReadOnlySpan<byte> bytes = new ReadOnlySpan<byte>(pOriginalBytes, originalByteCount).Slice(bytesConsumedSoFar); + Span<char> chars = new Span<char>(pOriginalChars, originalCharCount).Slice(charsWrittenSoFar); + + int charsWrittenJustNow = decoder.DrainLeftoverDataForGetChars(bytes, chars, out int bytesConsumedJustNow); + + bytes = bytes.Slice(bytesConsumedJustNow); + chars = chars.Slice(charsWrittenJustNow); + + Debug.Assert(!decoder.InternalHasFallbackBuffer || decoder.FallbackBuffer.Remaining == 0, "Should be no remaining fallback data at this point."); + + // Now try invoking the "fast path" (no fallback buffer) implementation. + // We can use Unsafe.AsPointer here since these spans are created from pinned data (raw pointers). + + charsWrittenJustNow = GetCharsFast( + pBytes: (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(bytes)), + bytesLength: bytes.Length, + pChars: (char*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(chars)), + charsLength: chars.Length, + bytesConsumed: out bytesConsumedJustNow); + + bytes = bytes.Slice(bytesConsumedJustNow); + chars = chars.Slice(charsWrittenJustNow); + + // We'll optimistically tell the decoder that we're using everything; the + // GetCharsWithFallback method will overwrite this field if necessary. + + decoder._bytesUsed = originalByteCount; + + if (bytes.IsEmpty) + { + return originalCharCount - chars.Length; // total number of chars written + } + else + { + return GetCharsWithFallback(bytes, originalByteCount, chars, originalCharCount, decoder); + } + } + + /// <summary> + /// Transcodes bytes to chars, using <see cref="Encoding.DecoderFallback"/> or <see cref="Decoder.Fallback"/> if needed. + /// </summary> + /// <returns> + /// The total number of chars written to <paramref name="chars"/> (based on <paramref name="originalCharsLength"/>). + /// </returns> + /// <remarks> + /// The derived class should override this method if it might be able to provide a more optimized fallback + /// implementation, deferring to the base implementation if needed. This method calls <see cref="ThrowCharsOverflow"/> + /// if necessary. + /// </remarks> + private protected virtual unsafe int GetCharsWithFallback(ReadOnlySpan<byte> bytes, int originalBytesLength, Span<char> chars, int originalCharsLength, DecoderNLS decoder) + { + Debug.Assert(!bytes.IsEmpty, "Caller shouldn't invoke this method with an empty input buffer."); + Debug.Assert(originalBytesLength >= 0, "Caller provided invalid parameter."); + Debug.Assert(originalCharsLength >= 0, "Caller provided invalid parameter."); + + // Since we're using Unsafe.AsPointer in our central loop, we want to ensure everything is pinned. + + fixed (byte* _pBytes_Unused = &MemoryMarshal.GetReference(bytes)) + fixed (char* _pChars_Unused = &MemoryMarshal.GetReference(chars)) + { + DecoderFallbackBuffer fallbackBuffer = DecoderFallbackBuffer.CreateAndInitialize(this, decoder, originalBytesLength); + + do + { + // There's still data in the source buffer; why wasn't the previous fast-path able to consume it fully? + // There are two scenarios: (a) the source buffer contained invalid data, or it contained incomplete data. + + int charsWrittenThisIteration; + + switch (DecodeFirstRune(bytes, out _, out int bytesConsumedThisIteration)) + { + case OperationStatus.NeedMoreData: + Debug.Assert(bytesConsumedThisIteration == bytes.Length, "If returning NeedMoreData, should out the entire buffer length as bytes consumed."); + if (decoder is null || decoder.MustFlush) + { + goto case OperationStatus.InvalidData; // see comment in GetCharCountWithFallback + } + else + { + decoder.SetLeftoverData(bytes); // squirrel away remaining data and finish + bytes = ReadOnlySpan<byte>.Empty; + goto Finish; + } + + case OperationStatus.InvalidData: + if (fallbackBuffer.TryInternalFallbackGetChars(bytes, bytesConsumedThisIteration, chars, out charsWrittenThisIteration)) + { + // We successfully consumed some bytes, sent it through the fallback, and wrote some chars. + + Debug.Assert(charsWrittenThisIteration >= 0, "Fallback shouldn't have returned a negative value."); + break; + } + else + { + // We generated fallback data, but the destination buffer wasn't large enough to hold it. + // Don't mark any of the bytes we ran through the fallback as consumed, and terminate + // the loop now and let our caller handle this condition. + + goto Finish; + } + + default: + goto Finish; // no error on input, so destination must have been too small + } + + bytes = bytes.Slice(bytesConsumedThisIteration); + chars = chars.Slice(charsWrittenThisIteration); + + if (!bytes.IsEmpty) + { + // Still data remaining - run it through the fast-path to find the next data to fallback. + // We need to figure out why we weren't able to make progress. + + charsWrittenThisIteration = GetCharsFast( + pBytes: (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(bytes)), + bytesLength: bytes.Length, + pChars: (char*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(chars)), + charsLength: chars.Length, + bytesConsumed: out bytesConsumedThisIteration); + + Debug.Assert(charsWrittenThisIteration >= 0, "Workhorse shouldn't have returned a negative value."); + Debug.Assert(bytesConsumedThisIteration >= 0, "Workhorse shouldn't have returned a negative value."); + + bytes = bytes.Slice(bytesConsumedThisIteration); + chars = chars.Slice(charsWrittenThisIteration); + } + } while (!bytes.IsEmpty); + + Finish: + + // We reach this point when we deplete the source or destination buffer. See main comment + // at the end of GetBytesWithFallback for how the below logic works; the primary difference + // here is that GetChars disallows leftover data in the fallback buffer between calls. + + Debug.Assert(fallbackBuffer.Remaining == 0); + + if (!bytes.IsEmpty) + { + // The line below will also throw if the decoder couldn't make any progress at all + // because the output buffer wasn't large enough to contain the result of even + // a single scalar conversion or fallback. + + ThrowCharsOverflow(decoder, nothingDecoded: chars.Length == originalCharsLength); + } + + // If a DecoderNLS instance is active, update its "total consumed byte count" value. + + if (decoder != null) + { + Debug.Assert(originalBytesLength >= bytes.Length, "About to report a negative number of bytes used?"); + decoder._bytesUsed = originalBytesLength - bytes.Length; // number of bytes consumed + } + + return originalCharsLength - chars.Length; // total number of chars written + } + } + } +} diff --git a/src/System.Private.CoreLib/shared/System/Text/Encoding.cs b/src/System.Private.CoreLib/shared/System/Text/Encoding.cs index 175e5442fd..8947b7fca0 100644 --- a/src/System.Private.CoreLib/shared/System/Text/Encoding.cs +++ b/src/System.Private.CoreLib/shared/System/Text/Encoding.cs @@ -3,11 +3,8 @@ // See the LICENSE file in the project root for more information. using System.Diagnostics; -using System.Globalization; -using System.Threading; using System.Runtime.InteropServices; using System.Runtime.Serialization; -using System.Diagnostics.CodeAnalysis; namespace System.Text { @@ -74,7 +71,7 @@ namespace System.Text // generally executes faster. // - public abstract class Encoding : ICloneable + public abstract partial class Encoding : ICloneable { // For netcore we use UTF8 as default encoding since ANSI isn't available private static readonly UTF8Encoding.UTF8EncodingSealed s_defaultEncoding = new UTF8Encoding.UTF8EncodingSealed(encoderShouldEmitUTF8Identifier: false); @@ -559,13 +556,16 @@ namespace System.Text return newEncoding; } - public bool IsReadOnly { get { return (_isReadOnly); } + private protected set + { + _isReadOnly = value; + } } // Returns an encoding for the ASCII character set. The returned encoding @@ -666,16 +666,6 @@ namespace System.Text } } - // For NLS Encodings, workhorse takes an encoder (may be null) - // Always validate parameters before calling internal version, which will only assert. - internal virtual unsafe int GetByteCount(char* chars, int count, EncoderNLS encoder) - { - Debug.Assert(chars != null); - Debug.Assert(count >= 0); - - return GetByteCount(chars, count); - } - // Returns a byte array containing the encoded representation of the given // character array. // @@ -772,14 +762,6 @@ namespace System.Text return GetBytes(s.ToCharArray(), charIndex, charCount, bytes, byteIndex); } - // This is our internal workhorse - // Always validate parameters before calling internal version, which will only assert. - internal virtual unsafe int GetBytes(char* chars, int charCount, - byte* bytes, int byteCount, EncoderNLS encoder) - { - return GetBytes(chars, charCount, bytes, byteCount); - } - // We expect this to be the workhorse for NLS Encodings, but for existing // ones we need a working (if slow) default implementation) // @@ -898,13 +880,6 @@ namespace System.Text } } - // This is our internal workhorse - // Always validate parameters before calling internal version, which will only assert. - internal virtual unsafe int GetCharCount(byte* bytes, int count, DecoderNLS decoder) - { - return GetCharCount(bytes, count); - } - // Returns a character array containing the decoded representation of a // given byte array. // @@ -1011,15 +986,6 @@ namespace System.Text } } - // This is our internal workhorse - // Always validate parameters before calling internal version, which will only assert. - internal virtual unsafe int GetChars(byte* bytes, int byteCount, - char* chars, int charCount, DecoderNLS decoder) - { - return GetChars(bytes, byteCount, chars, charCount); - } - - [CLSCompliant(false)] public unsafe string GetString(byte* bytes, int byteCount) { @@ -1238,6 +1204,12 @@ namespace System.Text encoder.ClearMustFlush(); } + [StackTraceHidden] + internal static void ThrowConversionOverflow() + { + throw new ArgumentException(SR.Argument_ConversionOverflow); + } + internal void ThrowCharsOverflow() { // Special message to include fallback type in case fallback's GetMaxCharCount is broken diff --git a/src/System.Private.CoreLib/shared/System/Text/EncodingNLS.cs b/src/System.Private.CoreLib/shared/System/Text/EncodingNLS.cs index e6fa0627d3..51d0e66044 100644 --- a/src/System.Private.CoreLib/shared/System/Text/EncodingNLS.cs +++ b/src/System.Private.CoreLib/shared/System/Text/EncodingNLS.cs @@ -4,6 +4,7 @@ using System; using System.Collections; +using System.Diagnostics; using System.Globalization; using System.Runtime.InteropServices; using System.Threading; @@ -27,6 +28,7 @@ namespace System.Text { protected EncodingNLS(int codePage) : base(codePage) { + Debug.Assert(GetType() == typeof(Latin1Encoding), "Should be no instantiations of this type except via Latin1Encoding."); } // Returns the number of bytes required to encode a range of characters in diff --git a/src/System.Private.CoreLib/shared/System/Threading/Tasks/Task.cs b/src/System.Private.CoreLib/shared/System/Threading/Tasks/Task.cs index 4d790f483d..5d893c5cee 100644 --- a/src/System.Private.CoreLib/shared/System/Threading/Tasks/Task.cs +++ b/src/System.Private.CoreLib/shared/System/Threading/Tasks/Task.cs @@ -131,8 +131,6 @@ namespace System.Threading.Tasks { [ThreadStatic] internal static Task t_currentTask; // The currently executing task. - [ThreadStatic] - private static StackGuard t_stackGuard; // The stack guard object for this thread internal static int s_taskIdCounter; //static counter used to generate unique task IDs @@ -1258,23 +1256,6 @@ namespace System.Threading.Tasks } /// <summary> - /// Gets the StackGuard object assigned to the current thread. - /// </summary> - internal static StackGuard CurrentStackGuard - { - get - { - StackGuard sg = t_stackGuard; - if (sg == null) - { - t_stackGuard = sg = new StackGuard(); - } - return sg; - } - } - - - /// <summary> /// Gets the <see cref="T:System.AggregateException">Exception</see> that caused the <see /// cref="Task">Task</see> to end prematurely. If the <see /// cref="Task">Task</see> completed successfully or has not yet thrown any @@ -3336,7 +3317,9 @@ namespace System.Threading.Tasks if (AsyncCausalityTracer.LoggingOn) AsyncCausalityTracer.TraceSynchronousWorkStart(this, CausalitySynchronousWork.CompletionNotification); - bool canInlineContinuations = (m_stateFlags & (int)TaskCreationOptions.RunContinuationsAsynchronously) == 0; + bool canInlineContinuations = + (m_stateFlags & (int)TaskCreationOptions.RunContinuationsAsynchronously) == 0 && + RuntimeHelpers.TryEnsureSufficientExecutionStack(); switch (continuationObject) { @@ -6485,51 +6468,6 @@ namespace System.Threading.Tasks ExecuteSynchronously = 0x80000 } - /// <summary> - /// Internal helper class to keep track of stack depth and decide whether we should inline or not. - /// </summary> - internal class StackGuard - { - // current thread's depth of nested inline task executions - private int m_inliningDepth = 0; - - // For relatively small inlining depths we don't want to get into the business of stack probing etc. - // This clearly leaves a window of opportunity for the user code to SO. However a piece of code - // that can SO in 20 inlines on a typical 1MB stack size probably needs to be revisited anyway. - private const int MAX_UNCHECKED_INLINING_DEPTH = 20; - - /// <summary> - /// This method needs to be called before attempting inline execution on the current thread. - /// If false is returned, it means we are too close to the end of the stack and should give up inlining. - /// Each call to TryBeginInliningScope() that returns true must be matched with a - /// call to EndInliningScope() regardless of whether inlining actually took place. - /// </summary> - internal bool TryBeginInliningScope() - { - // If we're still under the 'safe' limit we'll just skip the stack probe to save p/invoke calls - if (m_inliningDepth < MAX_UNCHECKED_INLINING_DEPTH || RuntimeHelpers.TryEnsureSufficientExecutionStack()) - { - m_inliningDepth++; - return true; - } - else - return false; - } - - /// <summary> - /// This needs to be called once for each previous successful TryBeginInliningScope() call after - /// inlining related logic runs. - /// </summary> - internal void EndInliningScope() - { - m_inliningDepth--; - Debug.Assert(m_inliningDepth >= 0, "Inlining depth count should never go negative."); - - // do the right thing just in case... - if (m_inliningDepth < 0) m_inliningDepth = 0; - } - } - // Special internal struct that we use to signify that we are not interested in // a Task<VoidTaskResult>'s result. internal struct VoidTaskResult { } @@ -6609,18 +6547,17 @@ namespace System.Threading.Tasks // For ITaskCompletionAction public void Invoke(Task completingTask) { - // Check the current stack guard. If we're ok to inline, - // process the task, and reset the guard when we're done. - var sg = Task.CurrentStackGuard; - if (sg.TryBeginInliningScope()) + // If we're ok to inline, process the task. Otherwise, we're too deep on the stack, and + // we shouldn't run the continuation chain here, so queue a work item to call back here + // to Invoke asynchronously. + if (RuntimeHelpers.TryEnsureSufficientExecutionStack()) + { + InvokeCore(completingTask); + } + else { - try { InvokeCore(completingTask); } - finally { sg.EndInliningScope(); } + InvokeCoreAsync(completingTask); } - // Otherwise, we're too deep on the stack, and - // we shouldn't run the continuation chain here, so queue a work - // item to call back here to Invoke asynchronously. - else InvokeCoreAsync(completingTask); } /// <summary> diff --git a/src/System.Private.CoreLib/shared/System/Threading/Tasks/TaskScheduler.cs b/src/System.Private.CoreLib/shared/System/Threading/Tasks/TaskScheduler.cs index f274f20558..1d1a581b0b 100644 --- a/src/System.Private.CoreLib/shared/System/Threading/Tasks/TaskScheduler.cs +++ b/src/System.Private.CoreLib/shared/System/Threading/Tasks/TaskScheduler.cs @@ -179,12 +179,11 @@ namespace System.Threading.Tasks // Delegate cross-scheduler inlining requests to target scheduler if (ets != this && ets != null) return ets.TryRunInline(task, taskWasPreviouslyQueued); - StackGuard currentStackGuard; if ((ets == null) || (task.m_action == null) || task.IsDelegateInvoked || task.IsCanceled || - (currentStackGuard = Task.CurrentStackGuard).TryBeginInliningScope() == false) + !RuntimeHelpers.TryEnsureSufficientExecutionStack()) { return false; } @@ -192,27 +191,19 @@ namespace System.Threading.Tasks // Task class will still call into TaskScheduler.TryRunInline rather than TryExecuteTaskInline() so that // 1) we can adjust the return code from TryExecuteTaskInline in case a buggy custom scheduler lies to us // 2) we maintain a mechanism for the TLS lookup optimization that we used to have for the ConcRT scheduler (will potentially introduce the same for TP) - bool bInlined = false; - try - { - if (TplEventSource.Log.IsEnabled()) - task.FireTaskScheduledIfNeeded(this); + if (TplEventSource.Log.IsEnabled()) + task.FireTaskScheduledIfNeeded(this); - bInlined = TryExecuteTaskInline(task, taskWasPreviouslyQueued); - } - finally - { - currentStackGuard.EndInliningScope(); - } + bool inlined = TryExecuteTaskInline(task, taskWasPreviouslyQueued); // If the custom scheduler returned true, we should either have the TASK_STATE_DELEGATE_INVOKED or TASK_STATE_CANCELED bit set // Otherwise the scheduler is buggy - if (bInlined && !(task.IsDelegateInvoked || task.IsCanceled)) + if (inlined && !(task.IsDelegateInvoked || task.IsCanceled)) { throw new InvalidOperationException(SR.TaskScheduler_InconsistentStateAfterTryExecuteTaskInline); } - return bInlined; + return inlined; } /// <summary> diff --git a/src/System.Private.CoreLib/shared/System/ThrowHelper.cs b/src/System.Private.CoreLib/shared/System/ThrowHelper.cs index c3c91d8f0b..06b3ce41a6 100644 --- a/src/System.Private.CoreLib/shared/System/ThrowHelper.cs +++ b/src/System.Private.CoreLib/shared/System/ThrowHelper.cs @@ -452,8 +452,20 @@ namespace System return "startIndex"; case ExceptionArgument.task: return "task"; + case ExceptionArgument.bytes: + return "bytes"; + case ExceptionArgument.byteIndex: + return "byteIndex"; + case ExceptionArgument.byteCount: + return "byteCount"; case ExceptionArgument.ch: return "ch"; + case ExceptionArgument.chars: + return "chars"; + case ExceptionArgument.charIndex: + return "charIndex"; + case ExceptionArgument.charCount: + return "charCount"; case ExceptionArgument.s: return "s"; case ExceptionArgument.input: @@ -612,6 +624,10 @@ namespace System { case ExceptionResource.ArgumentOutOfRange_Index: return SR.ArgumentOutOfRange_Index; + case ExceptionResource.ArgumentOutOfRange_IndexCount: + return SR.ArgumentOutOfRange_IndexCount; + case ExceptionResource.ArgumentOutOfRange_IndexCountBuffer: + return SR.ArgumentOutOfRange_IndexCountBuffer; case ExceptionResource.ArgumentOutOfRange_Count: return SR.ArgumentOutOfRange_Count; case ExceptionResource.Arg_ArrayPlusOffTooSmall: @@ -694,6 +710,8 @@ namespace System return SR.Task_WaitMulti_NullTask; case ExceptionResource.ArgumentException_OtherNotArrayOfCorrectLength: return SR.ArgumentException_OtherNotArrayOfCorrectLength; + case ExceptionResource.ArgumentNull_Array: + return SR.ArgumentNull_Array; case ExceptionResource.ArgumentNull_SafeHandle: return SR.ArgumentNull_SafeHandle; case ExceptionResource.ArgumentOutOfRange_EndIndexStartIndex: @@ -752,7 +770,13 @@ namespace System value, startIndex, task, + bytes, + byteIndex, + byteCount, ch, + chars, + charIndex, + charCount, s, input, ownedMemory, @@ -828,6 +852,8 @@ namespace System internal enum ExceptionResource { ArgumentOutOfRange_Index, + ArgumentOutOfRange_IndexCount, + ArgumentOutOfRange_IndexCountBuffer, ArgumentOutOfRange_Count, Arg_ArrayPlusOffTooSmall, NotSupported_ReadOnlyCollection, @@ -869,6 +895,7 @@ namespace System Task_ThrowIfDisposed, Task_WaitMulti_NullTask, ArgumentException_OtherNotArrayOfCorrectLength, + ArgumentNull_Array, ArgumentNull_SafeHandle, ArgumentOutOfRange_EndIndexStartIndex, ArgumentOutOfRange_Enum, diff --git a/src/System.Private.CoreLib/src/System/Buffer.CoreCLR.cs b/src/System.Private.CoreLib/src/System/Buffer.CoreCLR.cs new file mode 100644 index 0000000000..56f36de771 --- /dev/null +++ b/src/System.Private.CoreLib/src/System/Buffer.CoreCLR.cs @@ -0,0 +1,62 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +#if BIT64 +using nuint = System.UInt64; +#else +using nuint = System.UInt32; +#endif + +namespace System +{ + partial class Buffer + { + // Copies from one primitive array to another primitive array without + // respecting types. This calls memmove internally. The count and + // offset parameters here are in bytes. If you want to use traditional + // array element indices and counts, use Array.Copy. + [MethodImplAttribute(MethodImplOptions.InternalCall)] + public static extern void BlockCopy(Array src, int srcOffset, + Array dst, int dstOffset, int count); + + // Returns a bool to indicate if the array is of primitive data types + // or not. + [MethodImplAttribute(MethodImplOptions.InternalCall)] + private static extern bool IsPrimitiveTypeArray(Array array); + + // Gets the length of the array in bytes. The array must be an + // array of primitives. + // + // This essentially does the following: + // return array.length * sizeof(array.UnderlyingElementType). + // + [MethodImplAttribute(MethodImplOptions.InternalCall)] + private static extern int _ByteLength(Array array); + + // This method has a slightly different behavior on arm and other platforms. + // On arm this method behaves like memcpy and does not handle overlapping buffers. + // While on other platforms it behaves like memmove and handles overlapping buffers. + // This behavioral difference is unfortunate but intentional because + // 1. This method is given access to other internal dlls and this close to release we do not want to change it. + // 2. It is difficult to get this right for arm and again due to release dates we would like to visit it later. +#if ARM + [MethodImplAttribute(MethodImplOptions.InternalCall)] + internal static extern unsafe void Memcpy(byte* dest, byte* src, int len); +#else // ARM + [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] + internal static unsafe void Memcpy(byte* dest, byte* src, int len) + { + Debug.Assert(len >= 0, "Negative length in memcpy!"); + Memmove(dest, src, (nuint)len); + } +#endif // ARM + + [DllImport(JitHelpers.QCall, CharSet = CharSet.Unicode)] + private static extern unsafe void __Memmove(byte* dest, byte* src, nuint len); + } +} diff --git a/src/System.Private.CoreLib/src/System/RtType.cs b/src/System.Private.CoreLib/src/System/RtType.cs index 2a71d2e759..3d03771258 100644 --- a/src/System.Private.CoreLib/src/System/RtType.cs +++ b/src/System.Private.CoreLib/src/System/RtType.cs @@ -3225,6 +3225,25 @@ namespace System return false; } +#if FEATURE_TYPEEQUIVALENCE + // Reflexive, symmetric, transitive. + public override bool IsEquivalentTo(Type other) + { + var otherRtType = other as RuntimeType; + if (otherRtType is null) + { + return false; + } + + if (otherRtType == this) + { + return true; + } + + return RuntimeTypeHandle.IsEquivalentTo(this, otherRtType); + } +#endif // FEATURE_TYPEEQUIVALENCE + public override Type BaseType => GetBaseType(); private RuntimeType GetBaseType() diff --git a/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.cs b/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.cs index 94a6379f5f..1851d9fb2b 100644 --- a/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.cs +++ b/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.cs @@ -2,37 +2,14 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -//////////////////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////// -// -// RuntimeHelpers -// This class defines a set of static methods that provide support for compilers. -// -// +using System.Diagnostics; +using System.Runtime.InteropServices; +using Internal.Runtime.CompilerServices; namespace System.Runtime.CompilerServices { - using System; - using System.Diagnostics; - using System.Security; - using System.Runtime; - using System.Runtime.CompilerServices; - using System.Runtime.InteropServices; - using System.Runtime.ConstrainedExecution; - using System.Runtime.Serialization; - using System.Threading; - using System.Runtime.Versioning; - using Internal.Runtime.CompilerServices; - - public static class RuntimeHelpers + public static partial class RuntimeHelpers { - // Exposed here as a more appropriate place than on FormatterServices itself, - // which is a high level reflection heavy type. - public static object GetUninitializedObject(Type type) - { - return FormatterServices.GetUninitializedObject(type); - } - [MethodImplAttribute(MethodImplOptions.InternalCall)] public static extern void InitializeArray(Array array, RuntimeFieldHandle fldHandle); @@ -113,8 +90,6 @@ namespace System.Runtime.CompilerServices } } - public static void PrepareContractedDelegate(Delegate d) { } - [MethodImplAttribute(MethodImplOptions.InternalCall)] public static extern void PrepareDelegate(Delegate d); @@ -160,27 +135,6 @@ namespace System.Runtime.CompilerServices [MethodImplAttribute(MethodImplOptions.InternalCall)] public static extern bool TryEnsureSufficientExecutionStack(); - public static void ProbeForSufficientStack() - { - } - - // This method is a marker placed immediately before a try clause to mark the corresponding catch and finally blocks as - // constrained. There's no code here other than the probe because most of the work is done at JIT time when we spot a call to this routine. - public static void PrepareConstrainedRegions() - { - ProbeForSufficientStack(); - } - - // When we detect a CER with no calls, we can point the JIT to this non-probing version instead - // as we don't need to probe. - public static void PrepareConstrainedRegionsNoOP() - { - } - - public delegate void TryCode(object userData); - - public delegate void CleanupCode(object userData, bool exceptionThrown); - [MethodImplAttribute(MethodImplOptions.InternalCall)] public static extern void ExecuteCodeWithGuaranteedCleanup(TryCode code, CleanupCode backoutCode, object userData); @@ -197,26 +151,6 @@ namespace System.Runtime.CompilerServices throw new InvalidOperationException(); } - /// <summary> - /// GetSubArray helper method for the compiler to slice an array using a range. - /// </summary> - public static T[] GetSubArray<T>(T[] array, Range range) - { - Type elementType = array.GetType().GetElementType(); - Span<T> source = array.AsSpan(range); - - if (elementType.IsValueType) - { - return source.ToArray(); - } - else - { - T[] newArray = (T[])Array.CreateInstance(elementType, source.Length); - source.CopyTo(newArray); - return newArray; - } - } - // Returns true iff the object has a component size; // i.e., is variable length like System.String or Array. [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -256,5 +190,9 @@ namespace System.Runtime.CompilerServices // Ideally this would just be a single dereference: // mov tmp, qword ptr [rax] ; rax = obj ref, tmp = MethodTable* pointer } + + [MethodImplAttribute(MethodImplOptions.InternalCall)] + private static extern object GetUninitializedObjectInternal(Type type); + } } diff --git a/src/System.Private.CoreLib/src/System/Runtime/Serialization/FormatterServices.cs b/src/System.Private.CoreLib/src/System/Runtime/Serialization/FormatterServices.cs deleted file mode 100644 index 9da385bbf4..0000000000 --- a/src/System.Private.CoreLib/src/System/Runtime/Serialization/FormatterServices.cs +++ /dev/null @@ -1,64 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -/*============================================================ -** -** -** -** Purpose: Provides some static methods to aid with the implementation -** of a Formatter for Serialization. -** -** -============================================================*/ - -using System; -using System.Reflection; -using System.Collections; -using System.Collections.Generic; -using System.Security; -using System.Runtime.CompilerServices; -using System.Runtime.Versioning; -using System.Threading; -using System.IO; -using System.Text; -using System.Globalization; -using System.Diagnostics; - -namespace System.Runtime.Serialization -{ - // This class duplicates a class on CoreFX. We are keeping it here -- just this one method -- - // as it was widely invoked by reflection to workaround it being missing in .NET Core 1.0 - internal static class FormatterServices - { - // Gets a new instance of the object. The entire object is initalized to 0 and no - // constructors have been run. **THIS MEANS THAT THE OBJECT MAY NOT BE IN A STATE - // CONSISTENT WITH ITS INTERNAL REQUIREMENTS** This method should only be used for - // deserialization when the user intends to immediately populate all fields. This method - // will not create an unitialized string because it is non-sensical to create an empty - // instance of an immutable type. - // - public static object GetUninitializedObject(Type type) - { - if ((object)type == null) - { - throw new ArgumentNullException(nameof(type)); - } - - if (!(type is RuntimeType)) - { - throw new SerializationException(SR.Format(SR.Serialization_InvalidType, type)); - } - - return nativeGetUninitializedObject((RuntimeType)type); - } - - [MethodImplAttribute(MethodImplOptions.InternalCall)] - private static extern object nativeGetUninitializedObject(RuntimeType type); - } -} - - - - - diff --git a/src/System.Private.CoreLib/src/System/RuntimeHandles.cs b/src/System.Private.CoreLib/src/System/RuntimeHandles.cs index 1325aa831f..56694c63b8 100644 --- a/src/System.Private.CoreLib/src/System/RuntimeHandles.cs +++ b/src/System.Private.CoreLib/src/System/RuntimeHandles.cs @@ -615,6 +615,11 @@ namespace System { throw new PlatformNotSupportedException(); } + +#if FEATURE_TYPEEQUIVALENCE + [MethodImplAttribute(MethodImplOptions.InternalCall)] + internal static extern bool IsEquivalentTo(RuntimeType rtType1, RuntimeType rtType2); +#endif // FEATURE_TYPEEQUIVALENCE } // This type is used to remove the expense of having a managed reference object that is dynamically diff --git a/src/ToolBox/SOS/Strike/disasm.cpp b/src/ToolBox/SOS/Strike/disasm.cpp index f82c38ac4d..ceb0fed30d 100644 --- a/src/ToolBox/SOS/Strike/disasm.cpp +++ b/src/ToolBox/SOS/Strike/disasm.cpp @@ -530,7 +530,7 @@ INT_PTR ParseHexNumber (__in_z char *ptr, ___out char **endptr) endptr1 = endptr2; } // if the hex number was specified as 000006fbf9b70f50, an overflow occurred - else if (ULONG_MAX == value1 && errno == ERANGE) + else if ((INT_PTR)ULONG_MAX == value1 && errno == ERANGE) { if (!strncmp(ptr, "0x", 2)) ptr += 2; diff --git a/src/ToolBox/SOS/Strike/strike.cpp b/src/ToolBox/SOS/Strike/strike.cpp index f85957ee14..b25d340771 100644 --- a/src/ToolBox/SOS/Strike/strike.cpp +++ b/src/ToolBox/SOS/Strike/strike.cpp @@ -2157,7 +2157,7 @@ DECLARE_API(DumpDelegate) DacpObjectData objData; if (objData.Request(g_sos, invocationList) == S_OK && objData.ObjectType == OBJ_ARRAY && - invocationCount <= objData.dwNumComponents) + invocationCount <= (int)objData.dwNumComponents) { for (int i = 0; i < invocationCount; i++) { @@ -2363,7 +2363,7 @@ Done: // Overload that mirrors the code above when the ExceptionObjectData was already retrieved from LS BOOL IsAsyncException(const DacpExceptionObjectData & excData) { - if (excData.XCode != EXCEPTION_COMPLUS) + if ((DWORD)excData.XCode != EXCEPTION_COMPLUS) return TRUE; HRESULT ehr = excData.HResult; @@ -4605,7 +4605,7 @@ DECLARE_API(DumpAsync) DacpObjectData objData; if (objData.Request(g_sos, TO_CDADDR(listItemsPtr)) == S_OK && objData.ObjectType == OBJ_ARRAY) { - for (int i = 0; i < objData.dwNumComponents; i++) + for (SIZE_T i = 0; i < objData.dwNumComponents; i++) { CLRDATA_ADDRESS elementPtr; MOVE(elementPtr, TO_CDADDR(objData.ArrayDataPtr + (i * objData.dwComponentSize))); @@ -15544,7 +15544,7 @@ GetStackFrame(CONTEXT* context, ULONG numNativeFrames) if (FAILED(hr)) { PDEBUG_STACK_FRAME frame = &g_Frames[0]; - for (int i = 0; i < numNativeFrames; i++, frame++) { + for (unsigned int i = 0; i < numNativeFrames; i++, frame++) { if (frame->InstructionOffset == context->Rip) { if ((i + 1) >= numNativeFrames) { diff --git a/src/ToolBox/SOS/Strike/util.cpp b/src/ToolBox/SOS/Strike/util.cpp index 7098cc8019..0a286d2e6f 100644 --- a/src/ToolBox/SOS/Strike/util.cpp +++ b/src/ToolBox/SOS/Strike/util.cpp @@ -1726,7 +1726,7 @@ int GetValueFieldOffset(CLRDATA_ADDRESS cdaMT, __in_z LPCWSTR wszFieldName, Dacp if (dmtd.ParentMethodTable) { DWORD retVal = GetValueFieldOffset(dmtd.ParentMethodTable, wszFieldName, pDacpFieldDescData); - if (retVal != NOT_FOUND) + if (retVal != (DWORD)NOT_FOUND) { // Return in case of error or success. Fall through for field-not-found. return retVal; diff --git a/src/ToolBox/superpmi/mcs/verbdumptoc.cpp b/src/ToolBox/superpmi/mcs/verbdumptoc.cpp index f9cc69effa..a2366225c2 100644 --- a/src/ToolBox/superpmi/mcs/verbdumptoc.cpp +++ b/src/ToolBox/superpmi/mcs/verbdumptoc.cpp @@ -20,7 +20,7 @@ int verbDumpToc::DoWork(const char* nameOfInput) const TOCElement* te = tf.GetElementPtr(i); printf("%4u: %016llX ", te->Number, te->Offset); - for (int j = 0; j < sizeof(te->Hash); j++) + for (size_t j = 0; j < sizeof(te->Hash); j++) { printf("%02x ", te->Hash[j]); } diff --git a/src/ToolBox/superpmi/superpmi-shared/compileresult.cpp b/src/ToolBox/superpmi/superpmi-shared/compileresult.cpp index 70f54d2c4c..251666a823 100644 --- a/src/ToolBox/superpmi/superpmi-shared/compileresult.cpp +++ b/src/ToolBox/superpmi/superpmi-shared/compileresult.cpp @@ -1043,7 +1043,7 @@ bool CompileResult::fndRecordCallSiteSigInfo(ULONG instrOffset, CORINFO_SIG_INFO Agnostic_RecordCallSite value = RecordCallSite->Get(instrOffset); - if (value.callSig.callConv == -1) + if (value.callSig.callConv == (DWORD)-1) return false; pCallSig->callConv = (CorInfoCallConv)value.callSig.callConv; diff --git a/src/ToolBox/superpmi/superpmi-shared/lightweightmap.h b/src/ToolBox/superpmi/superpmi-shared/lightweightmap.h index 3a425124ea..069287c1d3 100644 --- a/src/ToolBox/superpmi/superpmi-shared/lightweightmap.h +++ b/src/ToolBox/superpmi/superpmi-shared/lightweightmap.h @@ -345,7 +345,7 @@ public: return false; // found it. return position ///// } insert = first; - if (insert != first) + if (insert != (unsigned int)first) { LogDebug("index = %u f %u mid = %u l %u***************************", insert, first, mid, last); __debugbreak(); diff --git a/src/ToolBox/superpmi/superpmi-shared/methodcontext.cpp b/src/ToolBox/superpmi/superpmi-shared/methodcontext.cpp index 96ecb8d5f1..a6b284488a 100644 --- a/src/ToolBox/superpmi/superpmi-shared/methodcontext.cpp +++ b/src/ToolBox/superpmi/superpmi-shared/methodcontext.cpp @@ -3547,7 +3547,7 @@ void MethodContext::recGetClassGClayout(CORINFO_CLASS_HANDLE cls, BYTE* gcPtrs, void MethodContext::dmpGetClassGClayout(DWORDLONG key, const Agnostic_GetClassGClayout& value) { printf("GetClassGCLayout key %016llX, value len %u cnt %u {", key, value.len, value.valCount); - if (value.gcPtrs_Index != -1) + if (value.gcPtrs_Index != (DWORD)-1) { BYTE* ptr = (BYTE*)GetClassGClayout->GetBuffer(value.gcPtrs_Index); for (unsigned int i = 0; i < value.len; i++) @@ -3572,7 +3572,7 @@ unsigned MethodContext::repGetClassGClayout(CORINFO_CLASS_HANDLE cls, BYTE* gcPt unsigned int len = (unsigned int)value.len; unsigned int index = (unsigned int)value.gcPtrs_Index; - if (index != -1) + if (index != (unsigned int)-1) { BYTE* ptr = (BYTE*)GetClassGClayout->GetBuffer(index); for (unsigned int i = 0; i < len; i++) diff --git a/src/ToolBox/superpmi/superpmi-shared/tocfile.cpp b/src/ToolBox/superpmi/superpmi-shared/tocfile.cpp index f0979fc5c3..1994a3bcd7 100644 --- a/src/ToolBox/superpmi/superpmi-shared/tocfile.cpp +++ b/src/ToolBox/superpmi/superpmi-shared/tocfile.cpp @@ -47,7 +47,7 @@ void TOCFile::LoadToc(const char* inputFileName, bool validate) // Get the last 4 byte token (more abuse of LARGE_INTEGER) if (!ReadFile(hIndex, &val.u.HighPart, sizeof(DWORD), &read, nullptr) || (read != sizeof(DWORD)) || - (val.u.LowPart != val.u.HighPart)) + (val.u.LowPart != (DWORD)val.u.HighPart)) { CloseHandle(hIndex); this->Clear(); diff --git a/src/ToolBox/superpmi/superpmi/neardiffer.cpp b/src/ToolBox/superpmi/superpmi/neardiffer.cpp index aa1722eb1a..bb0c67fc34 100644 --- a/src/ToolBox/superpmi/superpmi/neardiffer.cpp +++ b/src/ToolBox/superpmi/superpmi/neardiffer.cpp @@ -330,7 +330,7 @@ bool NearDiffer::compareOffsets( // VSD calling case. size_t Offset1 = (ipRelOffset1 - 8); - if (data->cr->CallTargetTypes->GetIndex((DWORDLONG)Offset1) != (DWORD)-1) + if (data->cr->CallTargetTypes->GetIndex((DWORDLONG)Offset1) != -1) { // This logging is too noisy, so disable it. // LogVerbose("Found VSD callsite, did softer compare than ideal"); @@ -340,13 +340,13 @@ bool NearDiffer::compareOffsets( // x86 VSD calling cases. size_t Offset1b = (size_t)offset1 - 4; size_t Offset2b = (size_t)offset2; - if (data->cr->CallTargetTypes->GetIndex((DWORDLONG)Offset1b) != (DWORD)-1) + if (data->cr->CallTargetTypes->GetIndex((DWORDLONG)Offset1b) != -1) { // This logging is too noisy, so disable it. // LogVerbose("Found VSD callsite, did softer compare than ideal"); return true; } - if (data->cr->CallTargetTypes->GetIndex((DWORDLONG)Offset2b) != (DWORD)-1) + if (data->cr->CallTargetTypes->GetIndex((DWORDLONG)Offset2b) != -1) { // This logging is too noisy, so disable it. // LogVerbose("Found VSD callsite, did softer compare than ideal"); @@ -368,7 +368,7 @@ bool NearDiffer::compareOffsets( return true; realTargetAddr = (size_t)data->cr->searchAddressMap((void*)(gOffset2)); - if (realTargetAddr != -1) // we know this was passed out as a bbloc + if (realTargetAddr != (size_t)-1) // we know this was passed out as a bbloc return true; return false; diff --git a/src/classlibnative/bcltype/system.cpp b/src/classlibnative/bcltype/system.cpp index 37c8b11387..8f3b428a58 100644 --- a/src/classlibnative/bcltype/system.cpp +++ b/src/classlibnative/bcltype/system.cpp @@ -346,7 +346,7 @@ INT32 QCALLTYPE SystemNative::GetProcessorCount() #ifdef FEATURE_PAL uint32_t cpuLimit; - if (PAL_GetCpuLimit(&cpuLimit) && cpuLimit < processorCount) + if (PAL_GetCpuLimit(&cpuLimit) && cpuLimit < (uint32_t)processorCount) processorCount = cpuLimit; #endif diff --git a/src/debug/daccess/dacdbiimpl.cpp b/src/debug/daccess/dacdbiimpl.cpp index a008dc1e38..a68a50d51e 100644 --- a/src/debug/daccess/dacdbiimpl.cpp +++ b/src/debug/daccess/dacdbiimpl.cpp @@ -1734,7 +1734,7 @@ void DacDbiInterfaceImpl::CollectFields(TypeHandle thExact, FALSE); // don't fixup EnC (we can't, we're stopped) PTR_FieldDesc pCurrentFD; - int index = 0; + unsigned int index = 0; while (((pCurrentFD = fdIterator.Next()) != NULL) && (index < pFieldList->Count())) { // fill in the pCurrentEntry structure @@ -3097,7 +3097,7 @@ TypeHandle DacDbiInterfaceImpl::GetExactFnPtrTypeHandle(ArgInfoList * pArgInfo) // convert the type information for each parameter to its corresponding type handle // and store it in the list - for (int i = 0; i < pArgInfo->Count(); i++) + for (unsigned int i = 0; i < pArgInfo->Count(); i++) { pInst[i] = BasicTypeInfoToTypeHandle(&((*pArgInfo)[i])); } @@ -3316,7 +3316,7 @@ void DacDbiInterfaceImpl::GetTypeHandleParams(VMPTR_AppDomain vmAppDomain, pParams->Alloc(typeHandle.GetNumGenericArgs()); // collect type information for each type parameter - for (int i = 0; i < pParams->Count(); ++i) + for (unsigned int i = 0; i < pParams->Count(); ++i) { VMPTR_TypeHandle thInst = VMPTR_TypeHandle::NullPtr(); thInst.SetDacTargetPtr(typeHandle.GetInstantiation()[i].AsTAddr()); @@ -3597,7 +3597,7 @@ void DacDbiInterfaceImpl::GetCachedWinRTTypesForIIDs( { pTypes->Alloc(iids.Count()); - for (int i = 0; i < iids.Count(); ++i) + for (unsigned int i = 0; i < iids.Count(); ++i) { // There is the possiblity that we'll get this far with a dump and not fail, but still // not be able to get full info for a particular param. diff --git a/src/debug/daccess/nidump.cpp b/src/debug/daccess/nidump.cpp index 37de104563..d7d90413d7 100644 --- a/src/debug/daccess/nidump.cpp +++ b/src/debug/daccess/nidump.cpp @@ -2462,7 +2462,7 @@ const NativeImageDumper::Dependency *NativeImageDumper::GetDependencyForFixup(RV { unsigned idx = DacSigUncompressData(sig); - _ASSERTE(idx >= 0 && idx < (int)m_numImports); + _ASSERTE(idx >= 0 && idx < m_numImports); return OpenImport(idx)->dependency; } diff --git a/src/debug/daccess/request.cpp b/src/debug/daccess/request.cpp index 995cfcab74..167069ac39 100644 --- a/src/debug/daccess/request.cpp +++ b/src/debug/daccess/request.cpp @@ -2854,7 +2854,7 @@ ClrDataAccess::GetGCHeapList(unsigned int count, CLRDATA_ADDRESS heaps[], unsign #if !defined(FEATURE_SVR_GC) _ASSERTE(0); #else // !defined(FEATURE_SVR_GC) - int heapCount = GCHeapCount(); + unsigned int heapCount = GCHeapCount(); if (pNeeded) *pNeeded = heapCount; diff --git a/src/debug/di/module.cpp b/src/debug/di/module.cpp index 1263e710e4..5fb335e6d5 100644 --- a/src/debug/di/module.cpp +++ b/src/debug/di/module.cpp @@ -4460,7 +4460,7 @@ HRESULT CordbNativeCode::EnumerateVariableHomes(ICorDebugVariableHomeEnum **ppEn const DacDbiArrayList<ICorDebugInfo::NativeVarInfo> *pOffsetInfoList = m_nativeVarData.GetOffsetInfoList(); _ASSERTE(pOffsetInfoList != NULL); DWORD countHomes = 0; - for (int i = 0; i < pOffsetInfoList->Count(); i++) + for (unsigned int i = 0; i < pOffsetInfoList->Count(); i++) { const ICorDebugInfo::NativeVarInfo *pNativeVarInfo = &((*pOffsetInfoList)[i]); _ASSERTE(pNativeVarInfo != NULL); @@ -4477,7 +4477,7 @@ HRESULT CordbNativeCode::EnumerateVariableHomes(ICorDebugVariableHomeEnum **ppEn rsHomes = new RSSmartPtr<CordbVariableHome>[countHomes]; DWORD varHomeInd = 0; - for (int i = 0; i < pOffsetInfoList->Count(); i++) + for (unsigned int i = 0; i < pOffsetInfoList->Count(); i++) { const ICorDebugInfo::NativeVarInfo *pNativeVarInfo = &((*pOffsetInfoList)[i]); diff --git a/src/debug/di/rsclass.cpp b/src/debug/di/rsclass.cpp index bfd02c75ec..662e2c0cd6 100644 --- a/src/debug/di/rsclass.cpp +++ b/src/debug/di/rsclass.cpp @@ -808,7 +808,7 @@ void CordbClass::Init(ClassLoadLevel desiredLoadLevel) BOOL CordbClass::GotUnallocatedStatic(DacDbiArrayList<FieldData> * pFieldList) { BOOL fGotUnallocatedStatic = FALSE; - int count = 0; + unsigned int count = 0; while ((count < pFieldList->Count()) && !fGotUnallocatedStatic ) { if ((*pFieldList)[count].OkToGetOrSetStaticAddress() && @@ -1145,7 +1145,7 @@ HRESULT CordbClass::SearchFieldInfo( FieldData **ppFieldData ) { - int i; + unsigned int i; IMetaDataImport * pImport = pModule->GetMetaDataImporter(); // throws diff --git a/src/debug/di/rstype.cpp b/src/debug/di/rstype.cpp index f180982404..a85ab0dcc4 100644 --- a/src/debug/di/rstype.cpp +++ b/src/debug/di/rstype.cpp @@ -1382,7 +1382,7 @@ HRESULT CordbType::InstantiateFromTypeHandle(CordbAppDomain * pAppDomain, // means it will simply assert IsNeutered. DacDbiArrayList<CordbType *> typeList; typeList.Alloc(params.Count()); - for (int i = 0; i < params.Count(); ++i) + for (unsigned int i = 0; i < params.Count(); ++i) { IfFailThrow(TypeDataToType(pAppDomain, &(params[i]), &(typeList[i]))); } diff --git a/src/debug/inc/dacdbistructures.h b/src/debug/inc/dacdbistructures.h index fc894f3834..285537d71d 100644 --- a/src/debug/inc/dacdbistructures.h +++ b/src/debug/inc/dacdbistructures.h @@ -99,7 +99,7 @@ public: // returns the number of elements in the list - int Count() const; + unsigned int Count() const; // @dbgtodo Mac - cleaner way to expose this for serialization? void PrepareForDeserialize() diff --git a/src/debug/inc/dacdbistructures.inl b/src/debug/inc/dacdbistructures.inl index 58166c1017..fe1ff98058 100644 --- a/src/debug/inc/dacdbistructures.inl +++ b/src/debug/inc/dacdbistructures.inl @@ -154,7 +154,7 @@ T & DacDbiArrayList<T>::operator [](int i) // get the number of elements in the list template<class T> inline -int DacDbiArrayList<T>::Count() const +unsigned int DacDbiArrayList<T>::Count() const { return m_nEntries; } @@ -392,7 +392,7 @@ inline void SequencePoints::CopyAndSortSequencePoints(const ICorDebugInfo::OffsetMapping mapCopy[]) { // copy information to pSeqPoint and set end offsets - int i; + unsigned int i; ULONG32 lastILOffset = 0; @@ -405,7 +405,7 @@ void SequencePoints::CopyAndSortSequencePoints(const ICorDebugInfo::OffsetMappin if (i < m_map.Count() - 1) { // We need to not use CALL_INSTRUCTION's IL start offset. - int j = i + 1; + unsigned int j = i + 1; while ((mapCopy[j].source & call_inst) == call_inst && j < m_map.Count()-1) j++; diff --git a/src/debug/shared/dbgtransportsession.cpp b/src/debug/shared/dbgtransportsession.cpp index f42d259e4c..e4dd78cf18 100644 --- a/src/debug/shared/dbgtransportsession.cpp +++ b/src/debug/shared/dbgtransportsession.cpp @@ -836,7 +836,7 @@ bool DbgTransportSession::SendBlock(PBYTE pbBuffer, DWORD cbBuffer) if (DBG_TRANSPORT_SHOULD_INJECT_FAULT(Send)) fSuccess = false; else - fSuccess = (m_pipe.Write(pbBuffer, cbBuffer) == cbBuffer); + fSuccess = ((DWORD)m_pipe.Write(pbBuffer, cbBuffer) == cbBuffer); if (!fSuccess) { @@ -867,7 +867,7 @@ bool DbgTransportSession::ReceiveBlock(PBYTE pbBuffer, DWORD cbBuffer) if (DBG_TRANSPORT_SHOULD_INJECT_FAULT(Receive)) fSuccess = false; else - fSuccess = (m_pipe.Read(pbBuffer, cbBuffer) == cbBuffer); + fSuccess = ((DWORD)m_pipe.Read(pbBuffer, cbBuffer) == cbBuffer); if (!fSuccess) { diff --git a/src/gc/gc.cpp b/src/gc/gc.cpp index a26ad8d24f..9c82c11536 100644 --- a/src/gc/gc.cpp +++ b/src/gc/gc.cpp @@ -24392,7 +24392,7 @@ void gc_heap::relocate_shortened_survivor_helper (uint8_t* plug, uint8_t* plug_e while (x < plug_end) { - if (check_short_obj_p && ((plug_end - x) < min_pre_pin_obj_size)) + if (check_short_obj_p && ((plug_end - x) < (DWORD)min_pre_pin_obj_size)) { dprintf (3, ("last obj %Ix is short", x)); diff --git a/src/gc/unix/gcenv.unix.cpp b/src/gc/unix/gcenv.unix.cpp index 693cfa8c86..8148f6ff29 100644 --- a/src/gc/unix/gcenv.unix.cpp +++ b/src/gc/unix/gcenv.unix.cpp @@ -485,7 +485,7 @@ bool GCToOSInterface::GetCurrentProcessAffinityMask(uintptr_t* processAffinityMa { uintptr_t processMask = 0; - for (int i = 0; i < g_logicalCpuCount; i++) + for (unsigned int i = 0; i < g_logicalCpuCount; i++) { if (CPU_ISSET(i, &cpuSet)) { @@ -536,7 +536,7 @@ uint32_t GCToOSInterface::GetCurrentProcessCpuCount() pmask &= smask; - int count = 0; + unsigned int count = 0; while (pmask) { pmask &= (pmask - 1); diff --git a/src/gcdump/gcdumpnonx86.cpp b/src/gcdump/gcdumpnonx86.cpp index 83624c76b7..5707926229 100644 --- a/src/gcdump/gcdumpnonx86.cpp +++ b/src/gcdump/gcdumpnonx86.cpp @@ -213,7 +213,7 @@ BOOL StackSlotStateChangeCallback ( if (pState->fAnythingPrinted) pState->pfnPrintf("\n"); - if ((CodeOffset == -2) && !pState->fAnythingPrinted) + if ((CodeOffset == (UINT32)-2) && !pState->fAnythingPrinted) pState->pfnPrintf("Untracked:"); else pState->pfnPrintf("%08x", CodeOffset); diff --git a/src/inc/clrconfigvalues.h b/src/inc/clrconfigvalues.h index 201a123211..a8b4270d4d 100644 --- a/src/inc/clrconfigvalues.h +++ b/src/inc/clrconfigvalues.h @@ -522,17 +522,17 @@ CONFIG_DWORD_INFO_EX(INTERNAL_SymDiffDump, W("SymDiffDump"), 0, "Used to create /// NGEN /// RETAIL_CONFIG_STRING_INFO_EX(EXTERNAL_NGen_JitName, W("NGen_JitName"), "", CLRConfig::REGUTIL_default) -RETAIL_CONFIG_DWORD_INFO_EX(UNSUPPORTED_NGenFramed, W("NGenFramed"), -1, "Same as JitFramed, but for ngen", CLRConfig::REGUTIL_default) +RETAIL_CONFIG_DWORD_INFO_EX(UNSUPPORTED_NGenFramed, W("NGenFramed"), (DWORD)-1, "Same as JitFramed, but for ngen", CLRConfig::REGUTIL_default) CONFIG_DWORD_INFO_EX(INTERNAL_NGenOnlyOneMethod, W("NGenOnlyOneMethod"), 0, "", CLRConfig::REGUTIL_default) CONFIG_DWORD_INFO_EX(INTERNAL_NgenOrder, W("NgenOrder"), 0, "", CLRConfig::REGUTIL_default) CONFIG_DWORD_INFO_EX(INTERNAL_partialNGenStress, W("partialNGenStress"), 0, "", CLRConfig::REGUTIL_default) CONFIG_DWORD_INFO_EX(INTERNAL_ZapDoNothing, W("ZapDoNothing"), 0, "", CLRConfig::REGUTIL_default) -CONFIG_DWORD_INFO_EX(INTERNAL_NgenForceFailureMask, W("NgenForceFailureMask"), -1, "Bitmask used to control which locations will check and raise the failure (defaults to bits: -1)", CLRConfig::REGUTIL_default) +CONFIG_DWORD_INFO_EX(INTERNAL_NgenForceFailureMask, W("NgenForceFailureMask"), (DWORD)-1, "Bitmask used to control which locations will check and raise the failure (defaults to bits: -1)", CLRConfig::REGUTIL_default) CONFIG_DWORD_INFO_EX(INTERNAL_NgenForceFailureCount, W("NgenForceFailureCount"), 0, "If set to >0 and we have IBC data we will force a failure after we reference an IBC data item <value> times", CLRConfig::REGUTIL_default) CONFIG_DWORD_INFO_EX(INTERNAL_NgenForceFailureKind, W("NgenForceFailureKind"), 1, "If set to 1, We will throw a TypeLoad exception; If set to 2, We will cause an A/V", CLRConfig::REGUTIL_default) RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_NGenEnableCreatePdb, W("NGenEnableCreatePdb"), 0, "If set to >0 ngen.exe displays help on, recognizes createpdb in the command line") RETAIL_CONFIG_DWORD_INFO(INTERNAL_NGenSimulateDiskFull, W("NGenSimulateDiskFull"), 0, "If set to 1, ngen will throw a Disk full exception in ZapWriter.cpp:Save()") -RETAIL_CONFIG_DWORD_INFO(INTERNAL_PartialNGen, W("PartialNGen"), -1, "Generate partial NGen images") +RETAIL_CONFIG_DWORD_INFO(INTERNAL_PartialNGen, W("PartialNGen"), (DWORD)-1, "Generate partial NGen images") CONFIG_DWORD_INFO(INTERNAL_NoASLRForNgen, W("NoASLRForNgen"), 0, "Turn off IMAGE_DLLCHARACTERISTICS_DYNAMIC_BASE bit in generated ngen images. Makes nidump output repeatable from run to run.") diff --git a/src/jit/codegenarm64.cpp b/src/jit/codegenarm64.cpp index 7d6afb7c40..9d1959e850 100644 --- a/src/jit/codegenarm64.cpp +++ b/src/jit/codegenarm64.cpp @@ -280,16 +280,32 @@ void CodeGen::genPrologSaveReg(regNumber reg1, int spOffset, int spDelta, regNum assert(spDelta <= 0); assert((spDelta % 16) == 0); // SP changes must be 16-byte aligned + bool needToSaveRegs = true; if (spDelta != 0) { - // generate sub SP,SP,imm - genStackPointerAdjustment(spDelta, tmpReg, pTmpRegIsZero, /* reportUnwindData */ true); + if ((spOffset == 0) && (spDelta >= -256)) + { + // We can use pre-index addressing. + // str REG, [SP, #spDelta]! + getEmitter()->emitIns_R_R_I(INS_str, EA_PTRSIZE, reg1, REG_SPBASE, spDelta, INS_OPTS_PRE_INDEX); + compiler->unwindSaveRegPreindexed(reg1, spDelta); + + needToSaveRegs = false; + } + else // (spOffset != 0) || (spDelta < -256) + { + // generate sub SP,SP,imm + genStackPointerAdjustment(spDelta, tmpReg, pTmpRegIsZero, /* reportUnwindData */ true); + } } - // str REG, [SP, #offset] - // 64-bit STR offset range: 0 to 32760, multiple of 8. - getEmitter()->emitIns_R_R_I(INS_str, EA_PTRSIZE, reg1, REG_SPBASE, spOffset); - compiler->unwindSaveReg(reg1, spOffset); + if (needToSaveRegs) + { + // str REG, [SP, #offset] + // 64-bit STR offset range: 0 to 32760, multiple of 8. + getEmitter()->emitIns_R_R_I(INS_str, EA_PTRSIZE, reg1, REG_SPBASE, spOffset); + compiler->unwindSaveReg(reg1, spOffset); + } } //------------------------------------------------------------------------ @@ -385,14 +401,30 @@ void CodeGen::genEpilogRestoreReg(regNumber reg1, int spOffset, int spDelta, reg assert(spDelta >= 0); assert((spDelta % 16) == 0); // SP changes must be 16-byte aligned - // ldr reg1, [SP, #offset] - getEmitter()->emitIns_R_R_I(INS_ldr, EA_PTRSIZE, reg1, REG_SPBASE, spOffset); - compiler->unwindSaveReg(reg1, spOffset); - if (spDelta != 0) { - // generate add SP,SP,imm - genStackPointerAdjustment(spDelta, tmpReg, pTmpRegIsZero, /* reportUnwindData */ true); + if ((spOffset == 0) && (spDelta <= 255)) + { + // We can use post-index addressing. + // ldr REG, [SP], #spDelta + getEmitter()->emitIns_R_R_I(INS_ldr, EA_PTRSIZE, reg1, REG_SPBASE, spDelta, INS_OPTS_POST_INDEX); + compiler->unwindSaveRegPreindexed(reg1, -spDelta); + } + else // (spOffset != 0) || (spDelta > 255) + { + // ldr reg1, [SP, #offset] + getEmitter()->emitIns_R_R_I(INS_ldr, EA_PTRSIZE, reg1, REG_SPBASE, spOffset); + compiler->unwindSaveReg(reg1, spOffset); + + // generate add SP,SP,imm + genStackPointerAdjustment(spDelta, tmpReg, pTmpRegIsZero, /* reportUnwindData */ true); + } + } + else + { + // ldr reg1, [SP, #offset] + getEmitter()->emitIns_R_R_I(INS_ldr, EA_PTRSIZE, reg1, REG_SPBASE, spOffset); + compiler->unwindSaveReg(reg1, spOffset); } } @@ -617,28 +649,22 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskTP regsToSaveMask, int lowe // We also can save FP and LR, even though they are not in RBM_CALLEE_SAVED. assert(regsToSaveCount <= genCountBits(RBM_CALLEE_SAVED | RBM_FP | RBM_LR)); - if (genSaveFpLrWithAllCalleeSavedRegisters) - { - // TODO: always save int regs higher than float, to be consistent? - regMaskTP maskSaveRegsFloat = regsToSaveMask & RBM_ALLFLOAT; - regMaskTP maskSaveRegsInt = regsToSaveMask & ~maskSaveRegsFloat; + // Save integer registers at higher addresses than floating-point registers. - if (maskSaveRegsFloat != RBM_NONE) - { - genSaveCalleeSavedRegisterGroup(maskSaveRegsFloat, spDelta, lowestCalleeSavedOffset); - spDelta = 0; - lowestCalleeSavedOffset += genCountBits(maskSaveRegsFloat) * FPSAVE_REGSIZE_BYTES; - } + regMaskTP maskSaveRegsFloat = regsToSaveMask & RBM_ALLFLOAT; + regMaskTP maskSaveRegsInt = regsToSaveMask & ~maskSaveRegsFloat; - if (maskSaveRegsInt != RBM_NONE) - { - genSaveCalleeSavedRegisterGroup(maskSaveRegsInt, spDelta, lowestCalleeSavedOffset); - // No need to update spDelta, lowestCalleeSavedOffset since they're not used after this. - } + if (maskSaveRegsFloat != RBM_NONE) + { + genSaveCalleeSavedRegisterGroup(maskSaveRegsFloat, spDelta, lowestCalleeSavedOffset); + spDelta = 0; + lowestCalleeSavedOffset += genCountBits(maskSaveRegsFloat) * FPSAVE_REGSIZE_BYTES; } - else + + if (maskSaveRegsInt != RBM_NONE) { - genSaveCalleeSavedRegisterGroup(regsToSaveMask, spDelta, lowestCalleeSavedOffset); + genSaveCalleeSavedRegisterGroup(maskSaveRegsInt, spDelta, lowestCalleeSavedOffset); + // No need to update spDelta, lowestCalleeSavedOffset since they're not used after this. } } @@ -740,31 +766,25 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, in static_assert_no_msg(REGSIZE_BYTES == FPSAVE_REGSIZE_BYTES); int spOffset = lowestCalleeSavedOffset + regsToRestoreCount * REGSIZE_BYTES; - if (genSaveFpLrWithAllCalleeSavedRegisters) - { - // TODO: always save int regs higher than float, to be consistent? - regMaskTP maskRestoreRegsFloat = regsToRestoreMask & RBM_ALLFLOAT; - regMaskTP maskRestoreRegsInt = regsToRestoreMask & ~maskRestoreRegsFloat; + // Save integer registers at higher addresses than floating-point registers. - // Restore in the opposite order of saving. + regMaskTP maskRestoreRegsFloat = regsToRestoreMask & RBM_ALLFLOAT; + regMaskTP maskRestoreRegsInt = regsToRestoreMask & ~maskRestoreRegsFloat; - if (maskRestoreRegsInt != RBM_NONE) - { - int spIntDelta = (maskRestoreRegsFloat != RBM_NONE) ? 0 : spDelta; // should we delay the SP adjustment? - genRestoreCalleeSavedRegisterGroup(maskRestoreRegsInt, spIntDelta, spOffset); - spOffset -= genCountBits(maskRestoreRegsInt) * REGSIZE_BYTES; - } + // Restore in the opposite order of saving. - if (maskRestoreRegsFloat != RBM_NONE) - { - // If there is any spDelta, it must be used here. - genRestoreCalleeSavedRegisterGroup(maskRestoreRegsFloat, spDelta, spOffset); - // No need to update spOffset since it's not used after this. - } + if (maskRestoreRegsInt != RBM_NONE) + { + int spIntDelta = (maskRestoreRegsFloat != RBM_NONE) ? 0 : spDelta; // should we delay the SP adjustment? + genRestoreCalleeSavedRegisterGroup(maskRestoreRegsInt, spIntDelta, spOffset); + spOffset -= genCountBits(maskRestoreRegsInt) * REGSIZE_BYTES; } - else + + if (maskRestoreRegsFloat != RBM_NONE) { - genRestoreCalleeSavedRegisterGroup(regsToRestoreMask, spDelta, spOffset); + // If there is any spDelta, it must be used here. + genRestoreCalleeSavedRegisterGroup(maskRestoreRegsFloat, spDelta, spOffset); + // No need to update spOffset since it's not used after this. } } diff --git a/src/jit/flowgraph.cpp b/src/jit/flowgraph.cpp index 2207a0750a..4888ce27ec 100644 --- a/src/jit/flowgraph.cpp +++ b/src/jit/flowgraph.cpp @@ -21152,7 +21152,8 @@ void Compiler::fgDebugCheckFlags(GenTree* tree) // If parent is a TYP_VOID, we don't no need to propagate TYP_INT up. We are fine. if (op2 && op2->gtOper == GT_ASG) { - assert(tree->gtType == TYP_VOID); + // We can have ASGs on the RHS of COMMAs in setup arguments to a call. + assert(tree->gtType == TYP_VOID || tree->gtOper == GT_COMMA); } switch (oper) diff --git a/src/jit/gentree.h b/src/jit/gentree.h index 28888396e9..ef3bca2667 100644 --- a/src/jit/gentree.h +++ b/src/jit/gentree.h @@ -4995,10 +4995,10 @@ struct GenTreePhiArg : public GenTreeLclVarCommon { BasicBlock* gtPredBB; - GenTreePhiArg(var_types type, unsigned lclNum, unsigned snum, BasicBlock* block) + GenTreePhiArg(var_types type, unsigned lclNum, unsigned ssaNum, BasicBlock* block) : GenTreeLclVarCommon(GT_PHI_ARG, type, lclNum), gtPredBB(block) { - SetSsaNum(snum); + SetSsaNum(ssaNum); } #if DEBUGGABLE_GENTREE diff --git a/src/jit/importer.cpp b/src/jit/importer.cpp index 4399381856..9c2236d5a7 100644 --- a/src/jit/importer.cpp +++ b/src/jit/importer.cpp @@ -1359,14 +1359,24 @@ GenTree* Compiler::impAssignStructPtr(GenTree* destAddr, { // Insert op1 after '*pAfterStmt' *pAfterStmt = fgInsertStmtAfter(block, *pAfterStmt, gtNewStmt(src->gtOp.gtOp1, ilOffset)); - // Evaluate the second thing using recursion. - return impAssignStructPtr(destAddr, src->gtOp.gtOp2, structHnd, curLevel, pAfterStmt, ilOffset, block); + } + else if (impTreeLast != nullptr) + { + // Do the side-effect as a separate statement. + impAppendTree(src->gtOp.gtOp1, curLevel, ilOffset); } else { - // We don't have an instruction to insert after, so use the entire comma expression as our rhs. - asgType = impNormStructType(structHnd); + // In this case we have neither been given a statement to insert after, nor are we + // in the importer where we can append the side effect. + // Instead, we're going to sink the assignment below the COMMA. + src->gtOp.gtOp2 = + impAssignStructPtr(destAddr, src->gtOp.gtOp2, structHnd, curLevel, pAfterStmt, ilOffset, block); + return src; } + + // Evaluate the second thing using recursion. + return impAssignStructPtr(destAddr, src->gtOp.gtOp2, structHnd, curLevel, pAfterStmt, ilOffset, block); } else if (src->IsLocal()) { diff --git a/src/jit/register_arg_convention.cpp b/src/jit/register_arg_convention.cpp index 4678cdec41..93c3b0bef8 100644 --- a/src/jit/register_arg_convention.cpp +++ b/src/jit/register_arg_convention.cpp @@ -72,10 +72,9 @@ bool InitVarDscInfo::enoughAvailRegs(var_types type, unsigned numRegs /* = 1 */) return regArgNum(type) + numRegs - backFillCount <= maxRegArgNum(type); } +#ifdef _TARGET_ARM_ unsigned InitVarDscInfo::alignReg(var_types type, unsigned requiredRegAlignment) { - NYI_ARM64("alignReg"); - assert(requiredRegAlignment > 0); if (requiredRegAlignment == 1) { @@ -93,12 +92,10 @@ unsigned InitVarDscInfo::alignReg(var_types type, unsigned requiredRegAlignment) unsigned cAlignSkipped = requiredRegAlignment - alignMask; assert(cAlignSkipped == 1); // Alignment is currently only 1 or 2, so misalignment can only be 1. -#ifdef _TARGET_ARM_ if (varTypeIsFloating(type)) { fltArgSkippedRegMask |= genMapFloatRegArgNumToRegMask(floatRegArgNum); } -#endif // _TARGET_ARM_ assert(regArgNum(type) + cAlignSkipped <= maxRegArgNum(type)); // if equal, then we aligned the last slot, and the // arg can't be enregistered @@ -106,6 +103,7 @@ unsigned InitVarDscInfo::alignReg(var_types type, unsigned requiredRegAlignment) return cAlignSkipped; } +#endif // _TARGET_ARM_ bool InitVarDscInfo::canEnreg(var_types type, unsigned numRegs /* = 1 */) { diff --git a/src/jit/register_arg_convention.h b/src/jit/register_arg_convention.h index f948e45eba..28f29b7c13 100644 --- a/src/jit/register_arg_convention.h +++ b/src/jit/register_arg_convention.h @@ -71,12 +71,14 @@ public: // Returns the first argument register of the allocated set. unsigned allocRegArg(var_types type, unsigned numRegs = 1); +#ifdef _TARGET_ARM_ // We are aligning the register to an ABI-required boundary, such as putting // double-precision floats in even-numbered registers, by skipping one register. // "requiredRegAlignment" is the amount to align to: 1 for no alignment (everything // is 1-aligned), 2 for "double" alignment. // Returns the number of registers skipped. unsigned alignReg(var_types type, unsigned requiredRegAlignment); +#endif // _TARGET_ARM_ // Return true if it is an enregisterable type and there is room. // Note that for "type", we only care if it is float or not. In particular, diff --git a/src/jit/ssabuilder.cpp b/src/jit/ssabuilder.cpp index d7b8b7610c..7fdc37ba93 100644 --- a/src/jit/ssabuilder.cpp +++ b/src/jit/ssabuilder.cpp @@ -881,7 +881,6 @@ void SsaBuilder::TreeRenameVariables(GenTree* tree, BasicBlock* block, SsaRename { // GcHeap and ByrefExposed share the same stacks, SsaMap, and phis assert(!hasByrefHavoc); - assert(pRenameState->CountForMemoryUse(GcHeap) == ssaNum); assert(*m_pCompiler->GetMemorySsaMap(GcHeap)->LookupPointer(tree) == ssaNum); assert(block->bbMemorySsaPhiFunc[GcHeap] == block->bbMemorySsaPhiFunc[ByrefExposed]); } @@ -926,7 +925,7 @@ void SsaBuilder::TreeRenameVariables(GenTree* tree, BasicBlock* block, SsaRename // This is a partial definition of a variable. The node records only the SSA number // of the use that is implied by this partial definition. The SSA number of the new // definition will be recorded in the m_opAsgnVarDefSsaNums map. - tree->AsLclVarCommon()->SetSsaNum(pRenameState->CountForUse(lclNum)); + tree->AsLclVarCommon()->SetSsaNum(pRenameState->Top(lclNum)); m_pCompiler->GetOpAsgnVarDefSsaNums()->Set(tree, ssaNum); } @@ -937,7 +936,7 @@ void SsaBuilder::TreeRenameVariables(GenTree* tree, BasicBlock* block, SsaRename pRenameState->Push(block, lclNum, ssaNum); - // If necessary, add "lclNum/count" to the arg list of a phi def in any + // If necessary, add "lclNum/ssaNum" to the arg list of a phi def in any // handlers for try blocks that "block" is within. (But only do this for "real" definitions, // not phi definitions.) if (!isPhiDefn) @@ -965,13 +964,12 @@ void SsaBuilder::TreeRenameVariables(GenTree* tree, BasicBlock* block, SsaRename return; } } - // Give the count as top of stack. - unsigned count = pRenameState->CountForUse(lclNum); - tree->gtLclVarCommon.SetSsaNum(count); + + tree->AsLclVarCommon()->SetSsaNum(pRenameState->Top(lclNum)); } } -void SsaBuilder::AddDefToHandlerPhis(BasicBlock* block, unsigned lclNum, unsigned count) +void SsaBuilder::AddDefToHandlerPhis(BasicBlock* block, unsigned lclNum, unsigned ssaNum) { assert(m_pCompiler->lvaTable[lclNum].lvTracked); // Precondition. unsigned lclIndex = m_pCompiler->lvaTable[lclNum].lvVarIndex; @@ -981,7 +979,7 @@ void SsaBuilder::AddDefToHandlerPhis(BasicBlock* block, unsigned lclNum, unsigne { DBG_SSA_JITDUMP("Definition of local V%02u/d:%d in block " FMT_BB " has exn handler; adding as phi arg to handlers.\n", - lclNum, count, block->bbNum); + lclNum, ssaNum, block->bbNum); while (true) { BasicBlock* handler = tryBlk->ExFlowBlock(); @@ -1007,7 +1005,7 @@ void SsaBuilder::AddDefToHandlerPhis(BasicBlock* block, unsigned lclNum, unsigne if (tree->gtOp.gtOp1->gtLclVar.gtLclNum == lclNum) { - // It's the definition for the right local. Add "count" to the RHS. + // It's the definition for the right local. Add "ssaNum" to the RHS. GenTree* phi = tree->gtOp.gtOp2; GenTreeArgList* args = nullptr; if (phi->gtOp.gtOp1 != nullptr) @@ -1019,12 +1017,12 @@ void SsaBuilder::AddDefToHandlerPhis(BasicBlock* block, unsigned lclNum, unsigne for (GenTreeArgList* curArgs = args; curArgs != nullptr; curArgs = curArgs->Rest()) { GenTreePhiArg* phiArg = curArgs->Current()->AsPhiArg(); - assert(phiArg->gtSsaNum != count); + assert(phiArg->gtSsaNum != ssaNum); } #endif var_types typ = m_pCompiler->lvaTable[lclNum].TypeGet(); GenTreePhiArg* newPhiArg = - new (m_pCompiler, GT_PHI_ARG) GenTreePhiArg(typ, lclNum, count, block); + new (m_pCompiler, GT_PHI_ARG) GenTreePhiArg(typ, lclNum, ssaNum, block); phi->gtOp.gtOp1 = new (m_pCompiler, GT_LIST) GenTreeArgList(newPhiArg, args); m_pCompiler->gtSetStmtInfo(stmt); @@ -1033,7 +1031,7 @@ void SsaBuilder::AddDefToHandlerPhis(BasicBlock* block, unsigned lclNum, unsigne phiFound = true; #endif DBG_SSA_JITDUMP(" Added phi arg u:%d for V%02u to phi defn in handler block " FMT_BB ".\n", - count, lclNum, handler->bbNum); + ssaNum, lclNum, handler->bbNum); break; } } @@ -1051,7 +1049,7 @@ void SsaBuilder::AddDefToHandlerPhis(BasicBlock* block, unsigned lclNum, unsigne } } -void SsaBuilder::AddMemoryDefToHandlerPhis(MemoryKind memoryKind, BasicBlock* block, unsigned count) +void SsaBuilder::AddMemoryDefToHandlerPhis(MemoryKind memoryKind, BasicBlock* block, unsigned ssaNum) { if (m_pCompiler->ehBlockHasExnFlowDsc(block)) { @@ -1063,7 +1061,7 @@ void SsaBuilder::AddMemoryDefToHandlerPhis(MemoryKind memoryKind, BasicBlock* bl // Otherwise... DBG_SSA_JITDUMP("Definition of %s/d:%d in block " FMT_BB " has exn handler; adding as phi arg to handlers.\n", - memoryKindNames[memoryKind], count, block->bbNum); + memoryKindNames[memoryKind], ssaNum, block->bbNum); EHblkDsc* tryBlk = m_pCompiler->ehGetBlockExnFlowDsc(block); while (true) { @@ -1074,7 +1072,7 @@ void SsaBuilder::AddMemoryDefToHandlerPhis(MemoryKind memoryKind, BasicBlock* bl { assert(handler->bbMemorySsaPhiFunc != nullptr); - // Add "count" to the phi args of memoryKind. + // Add "ssaNum" to the phi args of memoryKind. BasicBlock::MemoryPhiArg*& handlerMemoryPhi = handler->bbMemorySsaPhiFunc[memoryKind]; #if DEBUG @@ -1093,7 +1091,7 @@ void SsaBuilder::AddMemoryDefToHandlerPhis(MemoryKind memoryKind, BasicBlock* bl if (handlerMemoryPhi == BasicBlock::EmptyMemoryPhiDef) { - handlerMemoryPhi = new (m_pCompiler) BasicBlock::MemoryPhiArg(count); + handlerMemoryPhi = new (m_pCompiler) BasicBlock::MemoryPhiArg(ssaNum); } else { @@ -1101,14 +1099,14 @@ void SsaBuilder::AddMemoryDefToHandlerPhis(MemoryKind memoryKind, BasicBlock* bl BasicBlock::MemoryPhiArg* curArg = handler->bbMemorySsaPhiFunc[memoryKind]; while (curArg != nullptr) { - assert(curArg->GetSsaNum() != count); + assert(curArg->GetSsaNum() != ssaNum); curArg = curArg->m_nextArg; } #endif // DEBUG - handlerMemoryPhi = new (m_pCompiler) BasicBlock::MemoryPhiArg(count, handlerMemoryPhi); + handlerMemoryPhi = new (m_pCompiler) BasicBlock::MemoryPhiArg(ssaNum, handlerMemoryPhi); } - DBG_SSA_JITDUMP(" Added phi arg u:%d for %s to phi defn in handler block " FMT_BB ".\n", count, + DBG_SSA_JITDUMP(" Added phi arg u:%d for %s to phi defn in handler block " FMT_BB ".\n", ssaNum, memoryKindNames[memoryKind], memoryKind, handler->bbNum); if ((memoryKind == ByrefExposed) && m_pCompiler->byrefStatesMatchGcHeapStates) @@ -1148,7 +1146,8 @@ void SsaBuilder::BlockRenameVariables(BasicBlock* block, SsaRenameState* pRename assert(block->bbMemorySsaPhiFunc[memoryKind] == block->bbMemorySsaPhiFunc[ByrefExposed]); // so we will have already allocated a defnum for it if needed. assert(memoryKind > ByrefExposed); - assert(pRenameState->CountForMemoryUse(memoryKind) == pRenameState->CountForMemoryUse(ByrefExposed)); + + block->bbMemorySsaNumIn[memoryKind] = pRenameState->TopMemory(ByrefExposed); } else { @@ -1160,11 +1159,14 @@ void SsaBuilder::BlockRenameVariables(BasicBlock* block, SsaRenameState* pRename DBG_SSA_JITDUMP("Ssa # for %s phi on entry to " FMT_BB " is %d.\n", memoryKindNames[memoryKind], block->bbNum, ssaNum); + + block->bbMemorySsaNumIn[memoryKind] = ssaNum; + } + else + { + block->bbMemorySsaNumIn[memoryKind] = pRenameState->TopMemory(memoryKind); } } - - // Record the "in" Ssa # for memoryKind. - block->bbMemorySsaNumIn[memoryKind] = pRenameState->CountForMemoryUse(memoryKind); } // We need to iterate over phi definitions, to give them SSA names, but we need @@ -1199,7 +1201,8 @@ void SsaBuilder::BlockRenameVariables(BasicBlock* block, SsaRenameState* pRename assert(memoryKind > ByrefExposed); assert(((block->bbMemoryDef & memorySet) != 0) == ((block->bbMemoryDef & memoryKindSet(ByrefExposed)) != 0)); - assert(pRenameState->CountForMemoryUse(memoryKind) == pRenameState->CountForMemoryUse(ByrefExposed)); + + block->bbMemorySsaNumOut[memoryKind] = pRenameState->TopMemory(ByrefExposed); } else { @@ -1208,12 +1211,15 @@ void SsaBuilder::BlockRenameVariables(BasicBlock* block, SsaRenameState* pRename unsigned ssaNum = m_pCompiler->lvMemoryPerSsaData.AllocSsaNum(m_allocator); pRenameState->PushMemory(memoryKind, block, ssaNum); AddMemoryDefToHandlerPhis(memoryKind, block, ssaNum); + + block->bbMemorySsaNumOut[memoryKind] = ssaNum; + } + else + { + block->bbMemorySsaNumOut[memoryKind] = pRenameState->TopMemory(memoryKind); } } - // Record the "out" Ssa" # for memoryKind. - block->bbMemorySsaNumOut[memoryKind] = pRenameState->CountForMemoryUse(memoryKind); - DBG_SSA_JITDUMP("Ssa # for %s on entry to " FMT_BB " is %d; on exit is %d.\n", memoryKindNames[memoryKind], block->bbNum, block->bbMemorySsaNumIn[memoryKind], block->bbMemorySsaNumOut[memoryKind]); } @@ -1243,7 +1249,7 @@ void SsaBuilder::AssignPhiNodeRhsVariables(BasicBlock* block, SsaRenameState* pR assert(phiNode->gtOp.gtOp1 == nullptr || phiNode->gtOp.gtOp1->OperGet() == GT_LIST); unsigned lclNum = tree->gtOp.gtOp1->gtLclVar.gtLclNum; - unsigned ssaNum = pRenameState->CountForUse(lclNum); + unsigned ssaNum = pRenameState->Top(lclNum); // Search the arglist for an existing definition for ssaNum. // (Can we assert that its the head of the list? This should only happen when we add // during renaming for a definition that occurs within a try, and then that's the last @@ -1399,8 +1405,7 @@ void SsaBuilder::AssignPhiNodeRhsVariables(BasicBlock* block, SsaRenameState* pR assert(phiNode->gtOp.gtOp1 == nullptr || phiNode->gtOp.gtOp1->OperGet() == GT_LIST); GenTreeArgList* argList = reinterpret_cast<GenTreeArgList*>(phiNode->gtOp.gtOp1); - // What is the current SSAName from the predecessor for this local? - unsigned ssaNum = pRenameState->CountForUse(lclNum); + unsigned ssaNum = pRenameState->Top(lclNum); // See if this ssaNum is already an arg to the phi. bool alreadyArg = false; @@ -1474,35 +1479,10 @@ void SsaBuilder::AssignPhiNodeRhsVariables(BasicBlock* block, SsaRenameState* pR } /** - * Walk the block's tree in the evaluation order and reclaim rename stack for var definitions. - * - * @param block Block for which SSA variables have to be renamed. - * @param pRenameState The incremental rename information stored during renaming process. - * - */ -void SsaBuilder::BlockPopStacks(BasicBlock* block, SsaRenameState* pRenameState) -{ - // Pop the names given to the non-phi nodes. - pRenameState->PopBlockStacks(block); - - // And for memory. - for (MemoryKind memoryKind : allMemoryKinds()) - { - if ((memoryKind == GcHeap) && m_pCompiler->byrefStatesMatchGcHeapStates) - { - // GcHeap and ByrefExposed share a rename stack, so don't try - // to pop it a second time. - continue; - } - pRenameState->PopBlockMemoryStack(memoryKind, block); - } -} - -/** * Perform variable renaming. * * Walks the blocks and renames all var defs with ssa numbers and all uses with the - * current count that is in the top of the stack. Assigns phi node rhs variables + * SSA number that is in the top of the stack. Assigns phi node rhs variables * (i.e., the arguments to the phi.) Then, calls the function recursively on child * nodes in the DOM tree to continue the renaming process. * @@ -1540,7 +1520,7 @@ void SsaBuilder::RenameVariables(BlkToBlkVectorMap* domTree, SsaRenameState* pRe // In ValueNum we'd assume un-inited variables get FIRST_SSA_NUM. assert(ssaNum == SsaConfig::FIRST_SSA_NUM); - pRenameState->Push(nullptr, lclNum, ssaNum); + pRenameState->Push(m_pCompiler->fgFirstBB, lclNum, ssaNum); } } @@ -1602,7 +1582,6 @@ void SsaBuilder::RenameVariables(BlkToBlkVectorMap* domTree, SsaRenameState* pRe // been (recursively) processed, we still need to call BlockPopStacks on it. blocksToDo->push_back(BlockWork(block, true)); - // Walk the block give counts to DEFs and give top of stack count for USEs. BlockRenameVariables(block, pRenameState); // Assign arguments to the phi node of successors, corresponding to the block's index. @@ -1621,8 +1600,8 @@ void SsaBuilder::RenameVariables(BlkToBlkVectorMap* domTree, SsaRenameState* pRe } else { - // Done, pop all the stack count, if there is one for this block. - BlockPopStacks(block, pRenameState); + // Done, pop all SSA numbers pushed in this block. + pRenameState->PopBlockStacks(block); DBG_SSA_JITDUMP("[SsaBuilder::RenameVariables] done with " FMT_BB "\n", block->bbNum); } } @@ -1744,9 +1723,8 @@ void SsaBuilder::Build() InsertPhiFunctions(postOrder, count); // Rename local variables and collect UD information for each ssa var. - SsaRenameState* pRenameState = - new (m_allocator) SsaRenameState(m_allocator, m_pCompiler->lvaCount, m_pCompiler->byrefStatesMatchGcHeapStates); - RenameVariables(domTree, pRenameState); + SsaRenameState renameState(m_allocator, m_pCompiler->lvaCount); + RenameVariables(domTree, &renameState); EndPhase(PHASE_BUILD_SSA_RENAME); #ifdef DEBUG diff --git a/src/jit/ssabuilder.h b/src/jit/ssabuilder.h index eab25549df..86c1d12543 100644 --- a/src/jit/ssabuilder.h +++ b/src/jit/ssabuilder.h @@ -9,7 +9,7 @@ #include "compiler.h" -struct SsaRenameState; +class SsaRenameState; typedef int LclVarNum; @@ -105,12 +105,6 @@ private: // Assigns gtSsaNames to all variables. void RenameVariables(BlkToBlkVectorMap* domTree, SsaRenameState* pRenameState); - // Requires "block" to be any basic block participating in variable renaming, and has at least a - // definition that pushed a ssa number into the rename stack for a variable. Requires "pRenameState" - // to have variable stacks that have counts pushed into them for the block while assigning def - // numbers. Pops the stack for any local variable that has an entry for block on top. - void BlockPopStacks(BasicBlock* block, SsaRenameState* pRenameState); - // Requires "block" to be non-NULL; and is searched for defs and uses to assign ssa numbers. // Requires "pRenameState" to be non-NULL and be currently used for variables renaming. void BlockRenameVariables(BasicBlock* block, SsaRenameState* pRenameState); @@ -120,15 +114,15 @@ private: // implies that any definition occurring within "tree" is a phi definition. void TreeRenameVariables(GenTree* tree, BasicBlock* block, SsaRenameState* pRenameState, bool isPhiDefn); - // Assumes that "block" contains a definition for local var "lclNum", with SSA number "count". + // Assumes that "block" contains a definition for local var "lclNum", with SSA number "ssaNum". // IF "block" is within one or more try blocks, // and the local variable is live at the start of the corresponding handlers, - // add this SSA number "count" to the argument list of the phi for the variable in the start + // add this SSA number "ssaNum" to the argument list of the phi for the variable in the start // block of those handlers. - void AddDefToHandlerPhis(BasicBlock* block, unsigned lclNum, unsigned count); + void AddDefToHandlerPhis(BasicBlock* block, unsigned lclNum, unsigned ssaNum); // Same as above, for memory. - void AddMemoryDefToHandlerPhis(MemoryKind memoryKind, BasicBlock* block, unsigned count); + void AddMemoryDefToHandlerPhis(MemoryKind memoryKind, BasicBlock* block, unsigned ssaNum); // Requires "block" to be non-NULL. Requires "pRenameState" to be non-NULL and be currently used // for variables renaming. Assigns the rhs arguments to the phi, i.e., block's phi node arguments. diff --git a/src/jit/ssarenamestate.cpp b/src/jit/ssarenamestate.cpp index 63f120a521..c715aeb643 100644 --- a/src/jit/ssarenamestate.cpp +++ b/src/jit/ssarenamestate.cpp @@ -6,185 +6,156 @@ #include "ssaconfig.h" #include "ssarenamestate.h" -/** - * Constructor - initialize the stacks and counters maps (lclVar -> stack/counter) map. - * - * @params alloc The allocator class used to allocate jitstd data. - */ -SsaRenameState::SsaRenameState(CompAllocator alloc, unsigned lvaCount, bool byrefStatesMatchGcHeapStates) - : stacks(nullptr) - , definedLocs(alloc) - , memoryStack(alloc) - , lvaCount(lvaCount) - , m_alloc(alloc) - , byrefStatesMatchGcHeapStates(byrefStatesMatchGcHeapStates) +//------------------------------------------------------------------------ +// SsaRenameState: Initialize SsaRenameState +// +// Arguments: +// alloc - A memory allocator +// lvaCount - The number of local variables +// +SsaRenameState::SsaRenameState(CompAllocator alloc, unsigned lvaCount) + : m_alloc(alloc), m_lvaCount(lvaCount), m_stacks(nullptr), m_stackListTail(nullptr) { } -/** - * Allocates memory for holding pointers to lcl's stacks, - * if not allocated already. - * - */ +//------------------------------------------------------------------------ +// EnsureStacks: Allocate memory for the stacks array. +// void SsaRenameState::EnsureStacks() { - if (stacks == nullptr) + if (m_stacks == nullptr) { - stacks = m_alloc.allocate<Stack*>(lvaCount); - for (unsigned i = 0; i < lvaCount; ++i) - { - stacks[i] = nullptr; - } + m_stacks = new (m_alloc) Stack[m_lvaCount](); } } -/** - * Returns a SSA count number for a local variable from top of the stack. - * - * @params lclNum The local variable def for which a count has to be returned. - * @return the current variable name for the "use". - * - * @remarks If the stack is empty, then we have an use before a def. To handle this - * special case, we need to initialize the count with 'default+1', so the - * next definition will always use 'default+1' but return 'default' for - * all uses until a definition. - * - */ -unsigned SsaRenameState::CountForUse(unsigned lclNum) +//------------------------------------------------------------------------ +// Top: Get the SSA number at the top of the stack for the specified variable. +// +// Arguments: +// lclNum - The local variable number +// +// Return Value: +// The SSA number. +// +// Notes: +// The stack must not be empty. Method parameters and local variables that are live in at +// the start of the first block must have associated SSA definitions and their SSA numbers +// must have been pushed first. +// +unsigned SsaRenameState::Top(unsigned lclNum) { - EnsureStacks(); - DBG_SSA_JITDUMP("[SsaRenameState::CountForUse] V%02u\n", lclNum); + DBG_SSA_JITDUMP("[SsaRenameState::Top] V%02u\n", lclNum); - Stack* stack = stacks[lclNum]; - noway_assert((stack != nullptr) && !stack->empty()); - return stack->back().m_count; + noway_assert(m_stacks != nullptr); + StackNode* top = m_stacks[lclNum].Top(); + noway_assert(top != nullptr); + return top->m_ssaNum; } -/** - * Pushes a count value on the variable stack. - * - * @params lclNum The local variable def whose stack the count needs to be pushed onto. - * @params count The current count value that needs to be pushed on to the stack. - * - * @remarks Usually called when renaming a "def." - * Create stack lazily when needed for the first time. - */ -void SsaRenameState::Push(BasicBlock* bb, unsigned lclNum, unsigned count) +//------------------------------------------------------------------------ +// Push: Push a SSA number onto the stack for the specified variable. +// +// Arguments: +// block - The block where the SSA definition occurs +// lclNum - The local variable number +// ssaNum - The SSA number +// +void SsaRenameState::Push(BasicBlock* block, unsigned lclNum, unsigned ssaNum) { - EnsureStacks(); - - // We'll use BB00 here to indicate the "block before any real blocks..." - DBG_SSA_JITDUMP("[SsaRenameState::Push] " FMT_BB ", V%02u, count = %d\n", bb != nullptr ? bb->bbNum : 0, lclNum, - count); + DBG_SSA_JITDUMP("[SsaRenameState::Push] " FMT_BB ", V%02u, count = %d\n", block->bbNum, lclNum, ssaNum); - Stack* stack = stacks[lclNum]; + EnsureStacks(); + Push(&m_stacks[lclNum], block, ssaNum); +} - if (stack == nullptr) - { - DBG_SSA_JITDUMP("\tCreating a new stack\n"); - stack = stacks[lclNum] = new (m_alloc) Stack(m_alloc); - } +//------------------------------------------------------------------------ +// Push: Push a SSA number onto a stack +// +// Arguments: +// stack - The stack to push to +// block - The block where the SSA definition occurs +// ssaNum - The SSA number +// +void SsaRenameState::Push(Stack* stack, BasicBlock* block, unsigned ssaNum) +{ + StackNode* top = stack->Top(); - if (stack->empty() || stack->back().m_bb != bb) + if ((top == nullptr) || (top->m_block != block)) { - stack->push_back(SsaRenameStateForBlock(bb, count)); - // Remember that we've pushed a def for this loc (so we don't have - // to traverse *all* the locs to do the necessary pops later). - definedLocs.push_back(SsaRenameStateLocDef(bb, lclNum)); + stack->Push(AllocStackNode(m_stackListTail, block, ssaNum)); + // Append the stack to the stack list. The stack list allows PopBlockStacks + // to easily find stacks that need popping. + m_stackListTail = stack; } else { - stack->back().m_count = count; + // If we already have a stack node for this block then simply update + // update the SSA number, the previous one is no longer needed. + top->m_ssaNum = ssaNum; } -#ifdef DEBUG - if (JitTls::GetCompiler()->verboseSsa) - { - printf("\tContents of the stack: ["); - for (Stack::iterator iter2 = stack->begin(); iter2 != stack->end(); iter2++) - { - printf("<" FMT_BB ", %d>", ((*iter2).m_bb != nullptr ? (*iter2).m_bb->bbNum : 0), (*iter2).m_count); - } - printf("]\n"); - - DumpStacks(); - } -#endif + INDEBUG(DumpStack(stack)); } void SsaRenameState::PopBlockStacks(BasicBlock* block) { DBG_SSA_JITDUMP("[SsaRenameState::PopBlockStacks] " FMT_BB "\n", block->bbNum); - // Iterate over the stacks for all the variables, popping those that have an entry - // for "block" on top. - while (!definedLocs.empty() && definedLocs.back().m_bb == block) + + while ((m_stackListTail != nullptr) && (m_stackListTail->Top()->m_block == block)) { - unsigned lclNum = definedLocs.back().m_lclNum; - assert(stacks != nullptr); // Cannot be empty because definedLocs is not empty. - Stack* stack = stacks[lclNum]; - assert(stack != nullptr); - assert(stack->back().m_bb == block); - stack->pop_back(); - definedLocs.pop_back(); + StackNode* top = m_stackListTail->Pop(); + INDEBUG(DumpStack(m_stackListTail)); + m_stackListTail = top->m_listPrev; + m_freeStack.Push(top); } + #ifdef DEBUG - // It should now be the case that no stack in stacks has an entry for "block" on top -- - // the loop above popped them all. - for (unsigned i = 0; i < lvaCount; ++i) + if (m_stacks != nullptr) { - if (stacks != nullptr && stacks[i] != nullptr && !stacks[i]->empty()) + // It should now be the case that no stack in stacks has an entry for "block" on top -- + // the loop above popped them all. + for (unsigned i = 0; i < m_lvaCount; ++i) { - assert(stacks[i]->back().m_bb != block); + if (m_stacks[i].Top() != nullptr) + { + assert(m_stacks[i].Top()->m_block != block); + } } } - if (JitTls::GetCompiler()->verboseSsa) - { - DumpStacks(); - } #endif // DEBUG } -void SsaRenameState::PopBlockMemoryStack(MemoryKind memoryKind, BasicBlock* block) -{ - auto& stack = memoryStack[memoryKind]; - while (stack.size() > 0 && stack.back().m_bb == block) - { - stack.pop_back(); - } -} - #ifdef DEBUG -/** - * Print the stack data for each variable in a loop. - */ -void SsaRenameState::DumpStacks() +//------------------------------------------------------------------------ +// DumpStack: Print the specified stack. +// +// Arguments: +// stack - The stack to print +// +void SsaRenameState::DumpStack(Stack* stack) { - printf("Dumping stacks:\n-------------------------------\n"); - if (lvaCount == 0) - { - printf("None\n"); - } - else + if (JitTls::GetCompiler()->verboseSsa) { - EnsureStacks(); - for (unsigned i = 0; i < lvaCount; ++i) + if (stack == &m_memoryStack[ByrefExposed]) { - Stack* stack = stacks[i]; - printf("V%02u:\t", i); - if (stack != nullptr) - { - for (Stack::iterator iter2 = stack->begin(); iter2 != stack->end(); ++iter2) - { - if (iter2 != stack->begin()) - { - printf(", "); - } - printf("<" FMT_BB ", %2d>", ((*iter2).m_bb != nullptr ? (*iter2).m_bb->bbNum : 0), - (*iter2).m_count); - } - } - printf("\n"); + printf("ByrefExposed: "); + } + else if (stack == &m_memoryStack[GcHeap]) + { + printf("GcHeap: "); + } + else + { + printf("V%02u: ", stack - m_stacks); } + + for (StackNode* i = stack->Top(); i != nullptr; i = i->m_stackPrev) + { + printf("%s<" FMT_BB ", %u>", (i == stack->Top()) ? "" : ", ", i->m_block->bbNum, i->m_ssaNum); + } + + printf("\n"); } } #endif // DEBUG diff --git a/src/jit/ssarenamestate.h b/src/jit/ssarenamestate.h index 96abf5aa20..bf7dac15d8 100644 --- a/src/jit/ssarenamestate.h +++ b/src/jit/ssarenamestate.h @@ -4,144 +4,114 @@ #pragma once -#include "jitstd.h" - -// Fixed-size array that can hold elements with no default constructor; -// it will construct them all by forwarding whatever arguments are -// supplied to its constructor. -template <typename T, int N> -class ConstructedArray +class SsaRenameState { - union { - // Storage that gets used to hold the T objects. - unsigned char bytes[N * sizeof(T)]; - -#if defined(_MSC_VER) && (_MSC_VER < 1900) - // With MSVC pre-VS2015, the code in the #else branch would hit error C2621, - // so in that case just count on pointer alignment being sufficient - // (currently T is only ever instantiated as jitstd::list<SsaRenameStateForBlock>) - - // Unused (except to impart alignment requirement) - void* pointer; -#else - // Unused (except to impart alignment requirement) - T alignedArray[N]; -#endif // defined(_MSC_VER) && (_MSC_VER < 1900) - }; + struct StackNode; -public: - T& operator[](size_t i) + class Stack { - return *(reinterpret_cast<T*>(bytes + i * sizeof(T))); - } + StackNode* m_top; - template <typename... Args> - ConstructedArray(Args&&... args) - { - for (int i = 0; i < N; ++i) + public: + Stack() : m_top(nullptr) { - new (bytes + i * sizeof(T), jitstd::placement_t()) T(jitstd::forward<Args>(args)...); } - } - ~ConstructedArray() - { - for (int i = 0; i < N; ++i) + StackNode* Top() { - operator[](i).~T(); + return m_top; } - } -}; -struct SsaRenameStateForBlock -{ - BasicBlock* m_bb; - unsigned m_count; - - SsaRenameStateForBlock(BasicBlock* bb, unsigned count) : m_bb(bb), m_count(count) - { - } - SsaRenameStateForBlock() : m_bb(nullptr), m_count(0) - { - } -}; + void Push(StackNode* node) + { + node->m_stackPrev = m_top; + m_top = node; + } -// A record indicating that local "m_loc" was defined in block "m_bb". -struct SsaRenameStateLocDef -{ - BasicBlock* m_bb; - unsigned m_lclNum; + StackNode* Pop() + { + StackNode* top = m_top; + m_top = top->m_stackPrev; + return top; + } + }; - SsaRenameStateLocDef(BasicBlock* bb, unsigned lclNum) : m_bb(bb), m_lclNum(lclNum) + struct StackNode { - } -}; - -struct SsaRenameState -{ - typedef jitstd::list<SsaRenameStateForBlock> Stack; - typedef Stack** Stacks; - typedef jitstd::list<SsaRenameStateLocDef> DefStack; + // Link to the previous stack top node + StackNode* m_stackPrev; + // Link to the previously pushed stack (used only when popping blocks) + Stack* m_listPrev; + // The basic block (used only when popping blocks) + BasicBlock* m_block; + // The actual information StackNode stores - the SSA number + unsigned m_ssaNum; + + StackNode(Stack* listPrev, BasicBlock* block, unsigned ssaNum) + : m_listPrev(listPrev), m_block(block), m_ssaNum(ssaNum) + { + } + }; - SsaRenameState(CompAllocator allocator, unsigned lvaCount, bool byrefStatesMatchGcHeapStates); + // Memory allocator + CompAllocator m_alloc; + // Number of local variables to allocate stacks for + unsigned m_lvaCount; + // An array of stack objects, one for each local variable + Stack* m_stacks; + // The tail of the list of stacks that have been pushed to + Stack* m_stackListTail; + // Same state for the special implicit memory variables + Stack m_memoryStack[MemoryKindCount]; + // A stack of free stack nodes + Stack m_freeStack; - void EnsureStacks(); +public: + SsaRenameState(CompAllocator alloc, unsigned lvaCount); - // Requires "lclNum" to be a variable number for which an ssa number at the top of the - // stack is required i.e., for variable "uses." - unsigned CountForUse(unsigned lclNum); + // Get the SSA number at the top of the stack for the specified variable. + unsigned Top(unsigned lclNum); - // Requires "lclNum" to be a variable number, and requires "count" to represent - // an ssa number, that needs to be pushed on to the stack corresponding to the lclNum. - void Push(BasicBlock* bb, unsigned lclNum, unsigned count); + // Push a SSA number onto the stack for the specified variable. + void Push(BasicBlock* block, unsigned lclNum, unsigned ssaNum); - // Pop all stacks that have an entry for "bb" on top. - void PopBlockStacks(BasicBlock* bb); + // Pop all stacks that have an entry for "block" on top. + void PopBlockStacks(BasicBlock* block); // Similar functions for the special implicit memory variable. - unsigned CountForMemoryUse(MemoryKind memoryKind) + unsigned TopMemory(MemoryKind memoryKind) { - if ((memoryKind == GcHeap) && byrefStatesMatchGcHeapStates) - { - // Share rename stacks in this configuration. - memoryKind = ByrefExposed; - } - return memoryStack[memoryKind].back().m_count; + return m_memoryStack[memoryKind].Top()->m_ssaNum; } - void PushMemory(MemoryKind memoryKind, BasicBlock* bb, unsigned count) + void PushMemory(MemoryKind memoryKind, BasicBlock* block, unsigned ssaNum) { - if ((memoryKind == GcHeap) && byrefStatesMatchGcHeapStates) - { - // Share rename stacks in this configuration. - memoryKind = ByrefExposed; - } - memoryStack[memoryKind].push_back(SsaRenameStateForBlock(bb, count)); + Push(&m_memoryStack[memoryKind], block, ssaNum); } - void PopBlockMemoryStack(MemoryKind memoryKind, BasicBlock* bb); - -#ifdef DEBUG - // Debug interface - void DumpStacks(); -#endif - private: - // Map of lclNum -> SsaRenameStateForBlock. - Stacks stacks; + void EnsureStacks(); - // This list represents the set of locals defined in the current block. - DefStack definedLocs; + // Allocate a new stack entry (possibly by popping it from the free stack) + template <class... Args> + StackNode* AllocStackNode(Args&&... args) + { + StackNode* stack = m_freeStack.Top(); - // Same state for the special implicit memory variables. - ConstructedArray<Stack, MemoryKindCount> memoryStack; + if (stack != nullptr) + { + m_freeStack.Pop(); + } + else + { + stack = m_alloc.allocate<StackNode>(1); + } - // Number of stacks/counts to allocate. - unsigned lvaCount; + return new (stack, jitstd::placement_t()) StackNode(jitstd::forward<Args>(args)...); + } - // Allocator to allocate stacks. - CompAllocator m_alloc; + // Push a SSA number onto a stack + void Push(Stack* stack, BasicBlock* block, unsigned ssaNum); - // Indicates whether GcHeap and ByrefExposed use the same state. - bool byrefStatesMatchGcHeapStates; + INDEBUG(void DumpStack(Stack* stack);) }; diff --git a/src/pal/src/configure.cmake b/src/pal/src/configure.cmake index c8b6f3bd6e..a4d550e03b 100644 --- a/src/pal/src/configure.cmake +++ b/src/pal/src/configure.cmake @@ -42,13 +42,15 @@ if(NOT CMAKE_SYSTEM_NAME STREQUAL Darwin) check_include_files("libintl.h" HAVE_LIBINTL_H) endif() -if(NOT CMAKE_SYSTEM_NAME STREQUAL FreeBSD AND NOT CMAKE_SYSTEM_NAME STREQUAL NetBSD) - set(CMAKE_REQUIRED_FLAGS "-ldl") -endif() -check_include_files(lttng/tracepoint.h HAVE_LTTNG_TRACEPOINT_H) -if(NOT CMAKE_SYSTEM_NAME STREQUAL FreeBSD AND NOT CMAKE_SYSTEM_NAME STREQUAL NetBSD) - unset(CMAKE_REQUIRED_FLAGS) -endif() +set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_DL_LIBS}) + +check_cxx_source_compiles(" +#include <lttng/tracepoint.h> +int main(int argc, char **argv) { + return 0; +}" HAVE_LTTNG_TRACEPOINT_H) + +set(CMAKE_REQUIRED_LIBRARIES) check_include_files(sys/sysctl.h HAVE_SYS_SYSCTL_H) check_include_files(gnu/lib-names.h HAVE_GNU_LIBNAMES_H) diff --git a/src/pal/src/init/pal.cpp b/src/pal/src/init/pal.cpp index f6bbd6a82c..81504061d3 100644 --- a/src/pal/src/init/pal.cpp +++ b/src/pal/src/init/pal.cpp @@ -1042,9 +1042,6 @@ PALCommonCleanup() PROCDumpThreadList(); #endif } - - // Mark that the PAL is uninitialized - init_count = 0; } BOOL PALIsShuttingDown() diff --git a/src/pal/tools/gen-buildsys-clang.sh b/src/pal/tools/gen-buildsys-clang.sh index 2047b1eccf..95b227d89f 100755 --- a/src/pal/tools/gen-buildsys-clang.sh +++ b/src/pal/tools/gen-buildsys-clang.sh @@ -6,12 +6,13 @@ if [ $# -lt 4 ] then echo "Usage..." - echo "gen-buildsys-clang.sh <path to top level CMakeLists.txt> <ClangMajorVersion> <ClangMinorVersion> <Architecture> [build flavor] [coverage] [ninja] [cmakeargs]" + echo "gen-buildsys-clang.sh <path to top level CMakeLists.txt> <ClangMajorVersion> <ClangMinorVersion> <Architecture> [build flavor] [coverage] [ninja] [scan-build] [cmakeargs]" echo "Specify the path to the top level CMake file - <ProjectK>/src/NDP" echo "Specify the clang version to use, split into major and minor version" echo "Specify the target architecture." echo "Optionally specify the build configuration (flavor.) Defaults to DEBUG." echo "Optionally specify 'coverage' to enable code coverage build." + echo "Optionally specify 'scan-build' to enable build with clang static analyzer." echo "Target ninja instead of make. ninja must be on the PATH." echo "Pass additional arguments to CMake call." exit 1 @@ -42,6 +43,7 @@ build_arch="$4" buildtype=DEBUG code_coverage=OFF build_tests=OFF +scan_build=OFF generator="Unix Makefiles" __UnprocessedCMakeArgs="" @@ -59,6 +61,10 @@ for i in "${@:5}"; do NINJA) generator=Ninja ;; + SCAN-BUILD) + echo "Static analysis is turned on for this build." + scan_build=ON + ;; *) __UnprocessedCMakeArgs="${__UnprocessedCMakeArgs}${__UnprocessedCMakeArgs:+ }$i" esac @@ -152,7 +158,16 @@ fi # Determine the current script directory __currentScriptDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cmake \ +cmake_command=cmake + +if [[ "$scan_build" == "ON" ]]; then + export CCC_CC=$CC + export CCC_CXX=$CXX + export SCAN_BUILD_COMMAND=$(command -v scan-build$desired_llvm_version) + cmake_command="$SCAN_BUILD_COMMAND $cmake_command" +fi + +$cmake_command \ -G "$generator" \ "-DCMAKE_USER_MAKE_RULES_OVERRIDE=${__currentScriptDir}/$overridefile" \ "-DCMAKE_AR=$llvm_ar" \ diff --git a/src/pal/tools/gen-buildsys-gcc.sh b/src/pal/tools/gen-buildsys-gcc.sh index cc141baae9..0d66368048 100755 --- a/src/pal/tools/gen-buildsys-gcc.sh +++ b/src/pal/tools/gen-buildsys-gcc.sh @@ -9,8 +9,8 @@ then echo "gen-buildsys-gcc.sh <path to top level CMakeLists.txt> <GccMajorVersion> <GccMinorVersion> <Architecture> [build flavor] [coverage] [ninja] [cmakeargs]" echo "Specify the path to the top level CMake file - <ProjectK>/src/NDP" echo "Specify the Gcc version to use, split into major and minor version" - echo "Specify the target architecture." - echo "Optionally specify the build configuration (flavor.) Defaults to DEBUG." + echo "Specify the target architecture." + echo "Optionally specify the build configuration (flavor.) Defaults to DEBUG." echo "Optionally specify 'coverage' to enable code coverage build." echo "Target ninja instead of make. ninja must be on the PATH." echo "Pass additional arguments to CMake call." @@ -22,7 +22,7 @@ gcc_prefix="" if [ "$CROSSCOMPILE" = "1" ]; then # Locate gcc - if [ ! -z "$TOOLCHAIN" ]; then + if [ -n "$TOOLCHAIN" ]; then gcc_prefix="$TOOLCHAIN-" fi fi @@ -37,21 +37,28 @@ elif command -v "${gcc_prefix}gcc$2$3" > /dev/null elif command -v "${gcc_prefix}gcc-$2$3" > /dev/null then desired_gcc_version="-$2$3" -elif command -v ${gcc_prefix}gcc > /dev/null +elif command -v "${gcc_prefix}gcc" > /dev/null then desired_gcc_version= else - echo "Unable to find $gcc_prefixgcc Compiler" + echo "Unable to find ${gcc_prefix}gcc Compiler" exit 1 fi -if [ -z "$CC" ]; then - export CC="$(command -v ${gcc_prefix}gcc$desired_gcc_version)" +if [ -z "$CLR_CC" ]; then + CC="$(command -v "${gcc_prefix}gcc$desired_gcc_version")" +else + CC="$CLR_CC" fi -if [ -z "$CXX" ]; then - export CXX="$(command -v ${gcc_prefix}g++$desired_gcc_version)" + +if [ -z "$CLR_CXX" ]; then + CXX="$(command -v "${gcc_prefix}g++$desired_gcc_version")" +else + CXX="$CLR_CXX" fi +export CC CXX + build_arch="$4" buildtype=DEBUG code_coverage=OFF @@ -60,9 +67,9 @@ __UnprocessedCMakeArgs="" ITER=-1 for i in "$@"; do - ITER=$(($ITER + 1)) + ITER=$((ITER + 1)) if [ $ITER -lt 5 ]; then continue; fi - upperI="$(echo $i | awk '{print toupper($0)}')" + upperI="$(echo "$i" | awk '{print toupper($0)}')" case $upperI in # Possible build types are DEBUG, CHECKED, RELEASE, RELWITHDEBINFO, MINSIZEREL. DEBUG | CHECKED | RELEASE | RELWITHDEBINFO | MINSIZEREL) @@ -80,55 +87,39 @@ for i in "$@"; do esac done -OS=`uname` +OS=$(uname) locate_gcc_exec() { + ENV_KNOB="CLR_$(echo "$1" | tr '[:lower:]' '[:upper:]')" + if env | grep -q "^$ENV_KNOB="; then + eval "echo \"\$$ENV_KNOB\"" + return + fi + if command -v "$gcc_prefix$1$desired_gcc_version" > /dev/null 2>&1 then - echo "$(command -v $gcc_prefix$1$desired_gcc_version)" + command -v "$gcc_prefix$1$desired_gcc_version" elif command -v "$gcc_prefix$1" > /dev/null 2>&1 then - echo "$(command -v $gcc_prefix$1)" + command -v "$gcc_prefix$1" else exit 1 fi } -gcc_ar="$(locate_gcc_exec ar)" -[ $? -eq 0 ] || { echo "Unable to locate gcc-ar"; exit 1; } +if ! gcc_link="$(locate_gcc_exec link)"; then { echo "Unable to locate link"; exit 1; } fi -if [ -z "$CC" ]; then - gcc_link="$(locate_gcc_exec gcc)" - [ $? -eq 0 ] || { echo "Unable to locate gcc-link"; exit 1; } -else - gcc_link="$CC" -fi +if ! gcc_ar="$(locate_gcc_exec ar)"; then { echo "Unable to locate gcc-ar"; exit 1; } fi -if [ -z "$NM" ]; then - gcc_nm="$(locate_gcc_exec nm)" - [ $? -eq 0 ] || { echo "Unable to locate gcc-nm"; exit 1; } -else - gcc_nm="$NM" -fi +if ! gcc_nm="$(locate_gcc_exec nm)"; then { echo "Unable to locate gcc-nm"; exit 1; } fi if [ "$OS" = "Linux" ] || [ "$OS" = "FreeBSD" ] || [ "$OS" = "OpenBSD" ] || [ "$OS" = "NetBSD" ] || [ "$OS" = "SunOS" ]; then - if [ -z "$OBJDUMP" ]; then - gcc_objdump="$(locate_gcc_exec objdump)" - [ $? -eq 0 ] || { echo "Unable to locate gcc-objdump"; exit 1; } - else - gcc_objdump="$OBJDUMP" - fi + if ! gcc_objdump="$(locate_gcc_exec objdump)"; then { echo "Unable to locate gcc-objdump"; exit 1; } fi fi -if [ -z "$OBJCOPY" ]; then - gcc_objcopy="$(locate_gcc_exec objcopy)" - [ $? -eq 0 ] || { echo "Unable to locate gcc-objcopy"; exit 1; } -else - gcc_objcopy="$OBJCOPY" -fi +if ! gcc_objcopy="$(locate_gcc_exec objcopy)"; then { echo "Unable to locate gcc-objcopy"; exit 1; } fi -gcc_ranlib="$(locate_gcc_exec ranlib)" -[ $? -eq 0 ] || { echo "Unable to locate gcc-ranlib"; exit 1; } +if ! gcc_ranlib="$(locate_gcc_exec ranlib)"; then { echo "Unable to locate gcc-ranlib"; exit 1; } fi cmake_extra_defines= if [ -n "$LLDB_LIB_DIR" ]; then @@ -184,5 +175,5 @@ cmake \ "-DCLR_CMAKE_ENABLE_CODE_COVERAGE=$code_coverage" \ "-DCLR_CMAKE_COMPILER=GNU" \ $cmake_extra_defines \ - $__UnprocessedCMakeArgs \ + "$__UnprocessedCMakeArgs" \ "$1" diff --git a/src/utilcode/util.cpp b/src/utilcode/util.cpp index d0a6035318..5acb57914f 100644 --- a/src/utilcode/util.cpp +++ b/src/utilcode/util.cpp @@ -1208,7 +1208,7 @@ int GetCurrentProcessCpuCount() if (cCPUs != 0) return cCPUs; - int count = 0; + unsigned int count = 0; DWORD_PTR pmask, smask; if (!GetProcessAffinityMask(GetCurrentProcess(), &pmask, &smask)) diff --git a/src/vm/amd64/profiler.cpp b/src/vm/amd64/profiler.cpp index d43df944d6..7dc1780c67 100644 --- a/src/vm/amd64/profiler.cpp +++ b/src/vm/amd64/profiler.cpp @@ -248,7 +248,7 @@ LPVOID ProfileArgIterator::GetNextArgAddr() } // if we're here we have an enregistered argument - int regStructOfs = (argOffset - TransitionBlock::GetOffsetOfArgumentRegisters()); + unsigned int regStructOfs = (argOffset - TransitionBlock::GetOffsetOfArgumentRegisters()); _ASSERTE(regStructOfs < ARGUMENTREGISTERS_SIZE); CorElementType t = m_argIterator.GetArgType(); diff --git a/src/vm/argdestination.h b/src/vm/argdestination.h index 439761bec2..386ba57c82 100644 --- a/src/vm/argdestination.h +++ b/src/vm/argdestination.h @@ -126,7 +126,7 @@ public: // This function is used rarely and so the overhead of reading the zeros from // the stack is negligible. long long zeros[CLR_SYSTEMV_MAX_EIGHTBYTES_COUNT_TO_PASS_IN_REGISTERS] = {}; - _ASSERTE(sizeof(zeros) >= fieldBytes); + _ASSERTE(sizeof(zeros) >= (size_t)fieldBytes); CopyStructToRegisters(zeros, fieldBytes, 0); } diff --git a/src/vm/callhelpers.cpp b/src/vm/callhelpers.cpp index e194161182..5124560128 100644 --- a/src/vm/callhelpers.cpp +++ b/src/vm/callhelpers.cpp @@ -408,7 +408,7 @@ void MethodDescCallSite::CallTargetWorker(const ARG_SLOT *pArguments, ARG_SLOT * TypeHandle thReturnValueType; if (m_methodSig.GetReturnTypeNormalized(&thReturnValueType) == ELEMENT_TYPE_VALUETYPE) { - _ASSERTE(cbReturnValue >= thReturnValueType.GetSize()); + _ASSERTE((DWORD)cbReturnValue >= thReturnValueType.GetSize()); } } #endif // UNIX_AMD64_ABI @@ -611,7 +611,7 @@ void MethodDescCallSite::CallTargetWorker(const ARG_SLOT *pArguments, ARG_SLOT * if (pReturnValue != NULL) { - _ASSERTE(cbReturnValue <= sizeof(callDescrData.returnValue)); + _ASSERTE((DWORD)cbReturnValue <= sizeof(callDescrData.returnValue)); memcpyNoGCRefs(pReturnValue, &callDescrData.returnValue, cbReturnValue); #if !defined(_WIN64) && BIGENDIAN diff --git a/src/vm/ceeload.cpp b/src/vm/ceeload.cpp index cd46cffaaa..c0fad3c595 100644 --- a/src/vm/ceeload.cpp +++ b/src/vm/ceeload.cpp @@ -1987,8 +1987,8 @@ void Module::BuildStaticsOffsets(AllocMemTracker *pamTracker) #endif DWORD dwNonGCBytes[2] = { - DomainLocalModule::OffsetOfDataBlob() + sizeof(BYTE)*dwNumTypes, - ThreadLocalModule::OffsetOfDataBlob() + sizeof(BYTE)*dwNumTypes + DomainLocalModule::OffsetOfDataBlob() + (DWORD)(sizeof(BYTE)*dwNumTypes), + ThreadLocalModule::OffsetOfDataBlob() + (DWORD)(sizeof(BYTE)*dwNumTypes) }; HENUMInternalHolder hTypeEnum(pImport); diff --git a/src/vm/clrex.h b/src/vm/clrex.h index a38f3a3574..c1cd80da98 100644 --- a/src/vm/clrex.h +++ b/src/vm/clrex.h @@ -844,7 +844,7 @@ LONG CLRNoCatchHandler(EXCEPTION_POINTERS* pExceptionInfo, PVOID pv); } \ PAL_ENDTRY \ } \ - PAL_EXCEPT_FILTER(CLRNoCatchHandler) \ + PAL_EXCEPT_FILTER(EntryPointFilter) \ { \ } \ PAL_ENDTRY diff --git a/src/vm/codeman.cpp b/src/vm/codeman.cpp index 97b6a38366..4188658cca 100644 --- a/src/vm/codeman.cpp +++ b/src/vm/codeman.cpp @@ -302,8 +302,8 @@ void UnwindInfoTable::AddToUnwindInfoTable(UnwindInfoTable** unwindInfoPtr, PT_R // We could imagine being much more efficient for 'bulk' updates, but we don't try // because we assume that this is rare and we want to keep the code simple - int usedSpace = unwindInfo->cTableCurCount - unwindInfo->cDeletedEntries; - int desiredSpace = usedSpace * 5 / 4 + 1; // Increase by 20% + ULONG usedSpace = unwindInfo->cTableCurCount - unwindInfo->cDeletedEntries; + ULONG desiredSpace = usedSpace * 5 / 4 + 1; // Increase by 20% // Be more aggresive if we used all of our space; if (usedSpace == unwindInfo->cTableMaxCount) desiredSpace = usedSpace * 3 / 2 + 1; // Increase by 50% diff --git a/src/vm/codeversion.cpp b/src/vm/codeversion.cpp index 5286815845..e5e8a75e80 100644 --- a/src/vm/codeversion.cpp +++ b/src/vm/codeversion.cpp @@ -47,7 +47,7 @@ bool NativeCodeVersion::operator!=(const NativeCodeVersion & rhs) const { return // it is a bug. Corerror.xml has a comment in it reserving this value for our use but it doesn't // appear in the public headers. -#define CORPROF_E_RUNTIME_SUSPEND_REQUIRED 0x80131381 +#define CORPROF_E_RUNTIME_SUSPEND_REQUIRED _HRESULT_TYPEDEF_(0x80131381L) #ifndef DACCESS_COMPILE NativeCodeVersionNode::NativeCodeVersionNode( @@ -1308,7 +1308,7 @@ HRESULT MethodDescVersioningState::JumpStampNativeCode(PCODE pCode /* = NULL */) // revert. if (GetJumpStampState() == JumpStampNone) { - for (int i = 0; i < sizeof(m_rgSavedCode); i++) + for (unsigned int i = 0; i < sizeof(m_rgSavedCode); i++) { m_rgSavedCode[i] = *FirstCodeByteAddr(pbCode + i, DebuggerController::GetPatchTable()->GetPatch((CORDB_ADDRESS_TYPE *)(pbCode + i))); } @@ -1413,7 +1413,7 @@ HRESULT MethodDescVersioningState::UpdateJumpTarget(BOOL fEESuspended, PCODE pRe // revert. if (GetJumpStampState() == JumpStampNone) { - for (int i = 0; i < sizeof(m_rgSavedCode); i++) + for (unsigned int i = 0; i < sizeof(m_rgSavedCode); i++) { m_rgSavedCode[i] = *FirstCodeByteAddr(pbCode + i, DebuggerController::GetPatchTable()->GetPatch((CORDB_ADDRESS_TYPE *)(pbCode + i))); } @@ -1640,7 +1640,7 @@ HRESULT MethodDescVersioningState::UpdateJumpStampHelper(BYTE* pbCode, INT64 i64 // PERF: we might still want a faster path through here if we aren't debugging that doesn't do // all the patch checks - for (int i = 0; i < MethodDescVersioningState::JumpStubSize; i++) + for (unsigned int i = 0; i < MethodDescVersioningState::JumpStubSize; i++) { *FirstCodeByteAddr(pbCode + i, DebuggerController::GetPatchTable()->GetPatch(pbCode + i)) = ((BYTE*)&i64NewValue)[i]; } diff --git a/src/vm/codeversion.h b/src/vm/codeversion.h index f29f67e3b7..dee4ac312a 100644 --- a/src/vm/codeversion.h +++ b/src/vm/codeversion.h @@ -33,7 +33,7 @@ typedef DPTR(class CodeVersionManager) PTR_CodeVersionManager; // This HRESULT is only used as a private implementation detail. Corerror.xml has a comment in it // reserving this value for our use but it doesn't appear in the public headers. -#define CORPROF_E_RUNTIME_SUSPEND_REQUIRED 0x80131381 +#define CORPROF_E_RUNTIME_SUSPEND_REQUIRED _HRESULT_TYPEDEF_(0x80131381L) #endif diff --git a/src/vm/comdelegate.cpp b/src/vm/comdelegate.cpp index e2660a2458..42f457ff3a 100644 --- a/src/vm/comdelegate.cpp +++ b/src/vm/comdelegate.cpp @@ -479,7 +479,7 @@ VOID GenerateShuffleArray(MethodDesc* pInvoke, MethodDesc *pTargetMeth, SArray<S { filledSlots[i] = false; } - for (int i = 0; i < pShuffleEntryArray->GetCount(); i++) + for (unsigned int i = 0; i < pShuffleEntryArray->GetCount(); i++) { entry = (*pShuffleEntryArray)[i]; @@ -487,7 +487,7 @@ VOID GenerateShuffleArray(MethodDesc* pInvoke, MethodDesc *pTargetMeth, SArray<S // of the entry that filled it in. if (filledSlots[GetNormalizedArgumentSlotIndex(entry.srcofs)]) { - int j; + unsigned int j; for (j = i; (*pShuffleEntryArray)[j].dstofs != entry.srcofs; j--) (*pShuffleEntryArray)[j] = (*pShuffleEntryArray)[j - 1]; diff --git a/src/vm/corhost.cpp b/src/vm/corhost.cpp index 1a376f0918..e02b514f5e 100644 --- a/src/vm/corhost.cpp +++ b/src/vm/corhost.cpp @@ -740,6 +740,12 @@ HRESULT CorHost2::_CreateAppDomain( { pwzAppNiPaths = pPropertyValues[i]; } + else + if (wcscmp(pPropertyNames[i], W("USE_ENTRYPOINT_FILTER")) == 0) + { + extern void ParseUseEntryPointFilter(LPCWSTR value); + ParseUseEntryPointFilter(pPropertyValues[i]); + } #ifdef FEATURE_COMINTEROP else if (wcscmp(pPropertyNames[i], W("APP_LOCAL_WINMETADATA")) == 0) diff --git a/src/vm/dllimport.cpp b/src/vm/dllimport.cpp index 7dedce6f9d..6fb16c51f6 100644 --- a/src/vm/dllimport.cpp +++ b/src/vm/dllimport.cpp @@ -867,13 +867,13 @@ public: // </NOTE> #if defined(PROFILING_SUPPORTED) - DWORD dwMethodDescLocalNum = -1; + DWORD dwMethodDescLocalNum = (DWORD)-1; // Notify the profiler of call out of the runtime if (!SF_IsReverseCOMStub(m_dwStubFlags) && (CORProfilerTrackTransitions() || SF_IsNGENedStubForProfiling(m_dwStubFlags))) { dwMethodDescLocalNum = m_slIL.EmitProfilerBeginTransitionCallback(pcsDispatch, m_dwStubFlags); - _ASSERTE(dwMethodDescLocalNum != -1); + _ASSERTE(dwMethodDescLocalNum != (DWORD)-1); } #endif // PROFILING_SUPPORTED @@ -930,7 +930,7 @@ public: #if defined(PROFILING_SUPPORTED) // Notify the profiler of return back into the runtime - if (dwMethodDescLocalNum != -1) + if (dwMethodDescLocalNum != (DWORD)-1) { m_slIL.EmitProfilerEndTransitionCallback(pcsDispatch, m_dwStubFlags, dwMethodDescLocalNum); } diff --git a/src/vm/ecalllist.h b/src/vm/ecalllist.h index 35fbae2160..b4f9895301 100644 --- a/src/vm/ecalllist.h +++ b/src/vm/ecalllist.h @@ -163,10 +163,6 @@ FCFuncStart(gEnvironmentFuncs) FCFuncElementSig("FailFast", &gsig_SM_Str_Exception_Str_RetVoid, SystemNative::FailFastWithExceptionAndSource) FCFuncEnd() -FCFuncStart(gSerializationFuncs) - FCFuncElement("nativeGetUninitializedObject", ReflectionSerialization::GetUninitializedObject) -FCFuncEnd() - FCFuncStart(gExceptionFuncs) FCFuncElement("IsImmutableAgileException", ExceptionNative::IsImmutableAgileException) FCFuncElement("nIsTransient", ExceptionNative::IsTransient) @@ -268,6 +264,7 @@ FCFuncStart(gCOMTypeHandleFuncs) FCFuncElement("Allocate", RuntimeTypeHandle::Allocate) //for A.CI FCFuncElement("CompareCanonicalHandles", RuntimeTypeHandle::CompareCanonicalHandles) FCIntrinsic("GetValueInternal", RuntimeTypeHandle::GetValueInternal, CORINFO_INTRINSIC_RTH_GetValueInternal) + FCFuncElement("IsEquivalentTo", RuntimeTypeHandle::IsEquivalentTo) FCFuncEnd() FCFuncStart(gMetaDataImport) @@ -918,6 +915,7 @@ FCFuncStart(gRuntimeHelpers) FCFuncElement("Equals", ObjectNative::Equals) FCFuncElement("EnsureSufficientExecutionStack", ReflectionInvocation::EnsureSufficientExecutionStack) FCFuncElement("TryEnsureSufficientExecutionStack", ReflectionInvocation::TryEnsureSufficientExecutionStack) + FCFuncElement("GetUninitializedObjectInternal", ReflectionSerialization::GetUninitializedObject) FCFuncEnd() FCFuncStart(gContextSynchronizationFuncs) @@ -1195,7 +1193,6 @@ FCClassElement("EventPipeInternal", "System.Diagnostics.Tracing", gEventPipeInte #endif // FEATURE_PERFTRACING FCClassElement("Exception", "System", gExceptionFuncs) FCClassElement("FileLoadException", "System.IO", gFileLoadExceptionFuncs) -FCClassElement("FormatterServices", "System.Runtime.Serialization", gSerializationFuncs) FCClassElement("GC", "System", gGCInterfaceFuncs) FCClassElement("GCHandle", "System.Runtime.InteropServices", gGCHandleFuncs) FCClassElement("GCSettings", "System.Runtime", gGCSettingsFuncs) diff --git a/src/vm/eventpipe.cpp b/src/vm/eventpipe.cpp index c2db06de8c..4de0b3c943 100644 --- a/src/vm/eventpipe.cpp +++ b/src/vm/eventpipe.cpp @@ -798,6 +798,7 @@ void EventPipe::WriteEventInternal(EventPipeEvent &event, EventPipeEventPayload payload.GetSize(), pActivityId, pRelatedActivityId); + instance.EnsureStack(*s_pSession); if(s_pFile != NULL) { diff --git a/src/vm/eventpipe.h b/src/vm/eventpipe.h index 3f5919f13f..c77b94dbdc 100644 --- a/src/vm/eventpipe.h +++ b/src/vm/eventpipe.h @@ -20,7 +20,6 @@ class EventPipeBufferManager; class EventPipeEventSource; class EventPipeProvider; class MethodDesc; -class SampleProfilerEventInstance; struct EventPipeProviderConfiguration; class EventPipeSession; diff --git a/src/vm/eventpipebuffer.cpp b/src/vm/eventpipebuffer.cpp index 395725efef..6cc901cb7b 100644 --- a/src/vm/eventpipebuffer.cpp +++ b/src/vm/eventpipebuffer.cpp @@ -83,6 +83,7 @@ bool EventPipeBuffer::WriteEvent(Thread *pThread, EventPipeSession &session, Eve payload.GetSize(), (pThread == NULL) ? NULL : pActivityId, pRelatedActivityId); + pInstance->EnsureStack(session); // TODO: Perform the stackwalk before the constructor // Copy the stack if a separate stack trace was provided. if(pStack != NULL) diff --git a/src/vm/eventpipeconfiguration.cpp b/src/vm/eventpipeconfiguration.cpp index d40f2b789e..9cf8280722 100644 --- a/src/vm/eventpipeconfiguration.cpp +++ b/src/vm/eventpipeconfiguration.cpp @@ -505,6 +505,7 @@ EventPipeEventInstance* EventPipeConfiguration::BuildEventMetadataEvent(EventPip instancePayloadSize, NULL /* pActivityId */, NULL /* pRelatedActivityId */); + _ASSERTE(!m_pMetadataEvent->NeedStack()); // Set the timestamp to match the source event, because the metadata event // will be emitted right before the source event. diff --git a/src/vm/eventpipeeventinstance.cpp b/src/vm/eventpipeeventinstance.cpp index 1424dbc94c..f1e5919bd2 100644 --- a/src/vm/eventpipeeventinstance.cpp +++ b/src/vm/eventpipeeventinstance.cpp @@ -34,7 +34,7 @@ EventPipeEventInstance::EventPipeEventInstance( #endif // _DEBUG m_pEvent = &event; m_threadID = threadID; - if(pActivityId != NULL) + if (pActivityId != NULL) { m_activityId = *pActivityId; } @@ -42,7 +42,7 @@ EventPipeEventInstance::EventPipeEventInstance( { m_activityId = {0}; } - if(pRelatedActivityId != NULL) + if (pRelatedActivityId != NULL) { m_relatedActivityId = *pRelatedActivityId; } @@ -55,15 +55,17 @@ EventPipeEventInstance::EventPipeEventInstance( m_dataLength = length; QueryPerformanceCounter(&m_timeStamp); _ASSERTE(m_timeStamp.QuadPart > 0); +#ifdef _DEBUG + EnsureConsistency(); +#endif // _DEBUG +} - if(event.NeedStack() && !session.RundownEnabled()) +void EventPipeEventInstance::EnsureStack(const EventPipeSession &session) +{ + if (m_pEvent->NeedStack() && !session.RundownEnabled()) { EventPipe::WalkManagedStackForCurrentThread(m_stackContents); } - -#ifdef _DEBUG - EnsureConsistency(); -#endif // _DEBUG } unsigned int EventPipeEventInstance::GetAlignedTotalSize() const @@ -155,10 +157,4 @@ bool EventPipeEventInstance::EnsureConsistency() } #endif // _DEBUG -SampleProfilerEventInstance::SampleProfilerEventInstance(EventPipeSession &session, EventPipeEvent &event, Thread *pThread, BYTE *pData, unsigned int length) - :EventPipeEventInstance(session, event, pThread->GetOSThreadId(), pData, length, NULL /* pActivityId */, NULL /* pRelatedActivityId */) -{ - LIMITED_METHOD_CONTRACT; -} - #endif // FEATURE_PERFTRACING diff --git a/src/vm/eventpipeeventinstance.h b/src/vm/eventpipeeventinstance.h index 47f1b5c35b..80fd49d946 100644 --- a/src/vm/eventpipeeventinstance.h +++ b/src/vm/eventpipeeventinstance.h @@ -23,6 +23,8 @@ public: EventPipeEventInstance(EventPipeSession &session, EventPipeEvent &event, DWORD threadID, BYTE *pData, unsigned int length, LPCGUID pActivityId, LPCGUID pRelatedActivityId); + void EnsureStack(const EventPipeSession &session); + StackContents* GetStack() { LIMITED_METHOD_CONTRACT; @@ -138,17 +140,6 @@ private: void SetTimeStamp(LARGE_INTEGER timeStamp); }; -// A specific type of event instance for use by the SampleProfiler. -// This is needed because the SampleProfiler knows how to walk stacks belonging -// to threads other than the current thread. -class SampleProfilerEventInstance : public EventPipeEventInstance -{ - -public: - - SampleProfilerEventInstance(EventPipeSession &session, EventPipeEvent &event, Thread *pThread, BYTE *pData, unsigned int length); -}; - #endif // FEATURE_PERFTRACING #endif // __EVENTPIPE_EVENTINSTANCE_H__ diff --git a/src/vm/excep.cpp b/src/vm/excep.cpp index 8f5cc089c3..26003f0597 100644 --- a/src/vm/excep.cpp +++ b/src/vm/excep.cpp @@ -4822,20 +4822,6 @@ LONG InternalUnhandledExceptionFilter_Worker( // if (pThread && pThread->HasThreadStateNC(Thread::TSNC_ProcessedUnhandledException)) { - // This assert shouldnt be hit in CoreCLR since: - // - // 1) It has no concept of managed entry point that is invoked by the shim. You can - // only run managed code via hosting APIs that will run code in non-default domains. - // - // 2) Managed threads cannot be created in DefaultDomain since no user code executes - // in default domain. - // - // So, if this is hit, something is not right! - if (pThread->HasThreadStateNC(Thread::TSNC_ProcessedUnhandledException)) - { - _ASSERTE(!"How come a thread with TSNC_ProcessedUnhandledException state entered the UEF on CoreCLR?"); - } - LOG((LF_EH, LL_INFO100, "InternalUnhandledExceptionFilter_Worker: have already processed unhandled exception for this thread.\n")); return EXCEPTION_CONTINUE_SEARCH; } @@ -5152,10 +5138,20 @@ LONG InternalUnhandledExceptionFilter( } // LONG InternalUnhandledExceptionFilter() +static bool s_useEntryPointFilter = false; + +void ParseUseEntryPointFilter(LPCWSTR value) +{ +#ifdef PLATFORM_WINDOWS // This feature has only been tested on Windows, keep it disabled on other platforms + // set s_useEntryPointFilter true if value != "0" + if (value && (_wcsicmp(value, W("0")) != 0)) + { + s_useEntryPointFilter = true; + } +#endif +} + // This filter is used to trigger unhandled exception processing for the entrypoint thread -// incase an exception goes unhandled from it. This makes us independent of the OS -// UEF mechanism to invoke our registered UEF to trigger CLR specific unhandled exception -// processing since that can be skipped if another UEF registered over ours and not chain back. LONG EntryPointFilter(PEXCEPTION_POINTERS pExceptionInfo, PVOID _pData) { CONTRACTL @@ -5168,20 +5164,38 @@ LONG EntryPointFilter(PEXCEPTION_POINTERS pExceptionInfo, PVOID _pData) LONG ret = -1; - // Invoke the UEF worker to perform unhandled exception processing - ret = InternalUnhandledExceptionFilter_Worker (pExceptionInfo); + ret = CLRNoCatchHandler(pExceptionInfo, _pData); + + if (ret != EXCEPTION_CONTINUE_SEARCH) + { + return ret; + } + + if (!s_useEntryPointFilter) + { + return EXCEPTION_CONTINUE_SEARCH; + } Thread* pThread = GetThread(); - if (pThread) + if (pThread && !GetThread()->HasThreadStateNC(Thread::TSNC_ProcessedUnhandledException)) { + // Invoke the UEF worker to perform unhandled exception processing + ret = InternalUnhandledExceptionFilter_Worker (pExceptionInfo); + // Set the flag that we have done unhandled exception processing for this thread // so that we dont duplicate the effort in the UEF. // // For details on this flag, refer to threads.h. LOG((LF_EH, LL_INFO100, "EntryPointFilter: setting TSNC_ProcessedUnhandledException\n")); pThread->SetThreadStateNC(Thread::TSNC_ProcessedUnhandledException); + + if (ret == EXCEPTION_EXECUTE_HANDLER) + { + // Do not swallow the exception, we just want to log it + return EXCEPTION_CONTINUE_SEARCH; + } } - + return ret; } diff --git a/src/vm/exceptionhandling.cpp b/src/vm/exceptionhandling.cpp index c5b78c26a4..838450bb0b 100644 --- a/src/vm/exceptionhandling.cpp +++ b/src/vm/exceptionhandling.cpp @@ -901,7 +901,7 @@ ProcessCLRException(IN PEXCEPTION_RECORD pExceptionRecord { DWORD exceptionCode = pExceptionRecord->ExceptionCode; - if (exceptionCode == STATUS_UNWIND) + if ((NTSTATUS)exceptionCode == STATUS_UNWIND) // If exceptionCode is STATUS_UNWIND, RtlUnwind is called with a NULL ExceptionRecord, // therefore OS uses a faked ExceptionRecord with STATUS_UNWIND code. Then we need to // look at our saved exception code. diff --git a/src/vm/genericdict.cpp b/src/vm/genericdict.cpp index 22453c71ef..687ae2fbb0 100644 --- a/src/vm/genericdict.cpp +++ b/src/vm/genericdict.cpp @@ -1233,7 +1233,7 @@ Dictionary::PopulateEntry( ULONG slotIndex; if (isReadyToRunModule) { - _ASSERT(dictionaryIndexAndSlot != -1); + _ASSERT(dictionaryIndexAndSlot != (DWORD)-1); slotIndex = (ULONG)(dictionaryIndexAndSlot & 0xFFFF); } else diff --git a/src/vm/i386/stublinkerx86.cpp b/src/vm/i386/stublinkerx86.cpp index 9902419c97..c1fbd4db87 100644 --- a/src/vm/i386/stublinkerx86.cpp +++ b/src/vm/i386/stublinkerx86.cpp @@ -1414,7 +1414,7 @@ VOID StubLinkerCPU::X86EmitSPIndexPush(__int32 ofs) // The offset can be expressed in a byte (can use the byte // form of the push esp instruction) - BYTE code[] = {0xff, 0x74, 0x24, ofs8}; + BYTE code[] = {0xff, 0x74, 0x24, (BYTE)ofs8}; EmitBytes(code, sizeof(code)); } else diff --git a/src/vm/jithelpers.cpp b/src/vm/jithelpers.cpp index 04de717cce..303f06130f 100644 --- a/src/vm/jithelpers.cpp +++ b/src/vm/jithelpers.cpp @@ -3834,7 +3834,7 @@ CORINFO_GENERIC_HANDLE JIT_GenericHandleWorker(MethodDesc * pMD, MethodTable * p { #ifdef _DEBUG // Only in R2R mode are the module, dictionary index and dictionary slot provided as an input - _ASSERTE(dictionaryIndexAndSlot != -1); + _ASSERTE(dictionaryIndexAndSlot != (DWORD)-1); _ASSERT(ExecutionManager::FindReadyToRunModule(dac_cast<TADDR>(signature)) == pModule); #endif dictionaryIndex = (dictionaryIndexAndSlot >> 16); @@ -3853,7 +3853,7 @@ CORINFO_GENERIC_HANDLE JIT_GenericHandleWorker(MethodDesc * pMD, MethodTable * p // prepare for every possible derived type of the type containing the method). So instead we have to locate the exactly // instantiated (non-shared) super-type of the class passed in. - _ASSERTE(dictionaryIndexAndSlot == -1); + _ASSERTE(dictionaryIndexAndSlot == (DWORD)-1); IfFailThrow(ptr.GetData(&dictionaryIndex)); } diff --git a/src/vm/jithost.cpp b/src/vm/jithost.cpp index 4a7783ef64..a037af4857 100644 --- a/src/vm/jithost.cpp +++ b/src/vm/jithost.cpp @@ -27,7 +27,7 @@ int JitHost::getIntConfigValue(const wchar_t* name, int defaultValue) WRAPPER_NO_CONTRACT; // Translate JIT call into runtime configuration query - CLRConfig::ConfigDWORDInfo info{ name, defaultValue, CLRConfig::EEConfig_default }; + CLRConfig::ConfigDWORDInfo info{ name, (DWORD)defaultValue, CLRConfig::EEConfig_default }; // Perform a CLRConfig look up on behalf of the JIT. return CLRConfig::GetConfigValue(info); diff --git a/src/vm/jitinterface.cpp b/src/vm/jitinterface.cpp index 42975b6537..6e4b288d66 100644 --- a/src/vm/jitinterface.cpp +++ b/src/vm/jitinterface.cpp @@ -7040,7 +7040,7 @@ bool getILIntrinsicImplementation(MethodDesc * ftn, // Call CompareTo method on the primitive type int tokCompareTo = pCompareToMD->GetMemberDef(); - int index = (et - ELEMENT_TYPE_I1); + unsigned int index = (et - ELEMENT_TYPE_I1); _ASSERTE(index < _countof(ilcode)); ilcode[index][0] = CEE_LDARGA_S; diff --git a/src/vm/managedmdimport.cpp b/src/vm/managedmdimport.cpp index e709575257..5b8a3d5ae8 100644 --- a/src/vm/managedmdimport.cpp +++ b/src/vm/managedmdimport.cpp @@ -20,7 +20,7 @@ void ThrowMetaDataImportException(HRESULT hr) MethodDescCallSite throwError(METHOD__METADATA_IMPORT__THROW_ERROR); - ARG_SLOT args[] = { hr }; + ARG_SLOT args[] = { (ARG_SLOT)hr }; throwError.Call(args); } diff --git a/src/vm/methodtablebuilder.cpp b/src/vm/methodtablebuilder.cpp index d0045411db..568a23136e 100644 --- a/src/vm/methodtablebuilder.cpp +++ b/src/vm/methodtablebuilder.cpp @@ -6326,7 +6326,7 @@ MethodTableBuilder::WriteMethodImplData( // binary search later for (DWORD i = 0; i < cSlots; i++) { - int min = i; + unsigned int min = i; for (DWORD j = i + 1; j < cSlots; j++) { if (rgSlots[j] < rgSlots[min]) diff --git a/src/vm/readytoruninfo.cpp b/src/vm/readytoruninfo.cpp index 378ce2a3f2..c855fd1fa9 100644 --- a/src/vm/readytoruninfo.cpp +++ b/src/vm/readytoruninfo.cpp @@ -685,7 +685,7 @@ PCODE ReadyToRunInfo::GetEntryPoint(MethodDesc * pMD, PrepareCodeConfig* pConfig NativeHashtable::Enumerator lookup = m_instMethodEntryPoints.Lookup(GetVersionResilientMethodHashCode(pMD)); NativeParser entryParser; - offset = -1; + offset = (uint)-1; while (lookup.GetNext(entryParser)) { PCCOR_SIGNATURE pBlob = (PCCOR_SIGNATURE)entryParser.GetBlob(); @@ -702,7 +702,7 @@ PCODE ReadyToRunInfo::GetEntryPoint(MethodDesc * pMD, PrepareCodeConfig* pConfig } } - if (offset == -1) + if (offset == (uint)-1) return NULL; } else diff --git a/src/vm/reflectioninvocation.cpp b/src/vm/reflectioninvocation.cpp index a69f5439c2..1f8aa04593 100644 --- a/src/vm/reflectioninvocation.cpp +++ b/src/vm/reflectioninvocation.cpp @@ -2581,10 +2581,6 @@ FCIMPL1(Object*, ReflectionSerialization::GetUninitializedObject, ReflectClassBa HELPER_METHOD_FRAME_BEGIN_RET_NOPOLL(); - if (objType == NULL) { - COMPlusThrowArgumentNull(W("type"), W("ArgumentNull_Type")); - } - TypeHandle type = objType->GetType(); // Don't allow arrays, pointers, byrefs or function pointers. diff --git a/src/vm/rejit.cpp b/src/vm/rejit.cpp index d0742840d4..9735df011f 100644 --- a/src/vm/rejit.cpp +++ b/src/vm/rejit.cpp @@ -165,7 +165,7 @@ // This HRESULT is only used as a private implementation detail. Corerror.xml has a comment in it // reserving this value for our use but it doesn't appear in the public headers. -#define CORPROF_E_RUNTIME_SUSPEND_REQUIRED 0x80131381 +#define CORPROF_E_RUNTIME_SUSPEND_REQUIRED _HRESULT_TYPEDEF_(0x80131381L) // This is just used as a unique id. Overflow is OK. If we happen to have more than 4+Billion rejits // and somehow manage to not run out of memory, we'll just have to redefine ReJITID as size_t. diff --git a/src/vm/rexcep.h b/src/vm/rexcep.h index 054d6ed4e6..c50608f6e7 100644 --- a/src/vm/rexcep.h +++ b/src/vm/rexcep.h @@ -201,7 +201,7 @@ DEFINE_EXCEPTION(g_IONS, FileNotFoundException, true, DEFINE_EXCEPTION(g_SystemNS, FormatException, false, COR_E_FORMAT) -DEFINE_EXCEPTION(g_SystemNS, IndexOutOfRangeException, false, COR_E_INDEXOUTOFRANGE, 0x800a0009 /*Subscript out of range*/) +DEFINE_EXCEPTION(g_SystemNS, IndexOutOfRangeException, false, COR_E_INDEXOUTOFRANGE, (int)0x800a0009 /*Subscript out of range*/) DEFINE_EXCEPTION(g_SystemNS, InsufficientExecutionStackException, false, COR_E_INSUFFICIENTEXECUTIONSTACK) DEFINE_EXCEPTION(g_SystemNS, InvalidCastException, false, COR_E_INVALIDCAST) #ifdef FEATURE_COMINTEROP diff --git a/src/vm/runtimehandles.cpp b/src/vm/runtimehandles.cpp index dda5d1d139..c8782db7ef 100644 --- a/src/vm/runtimehandles.cpp +++ b/src/vm/runtimehandles.cpp @@ -250,6 +250,25 @@ FCIMPL1_V(EnregisteredTypeHandle, RuntimeTypeHandle::GetValueInternal, FCALLRunt } FCIMPLEND +FCIMPL2(FC_BOOL_RET, RuntimeTypeHandle::IsEquivalentTo, ReflectClassBaseObject *rtType1UNSAFE, ReflectClassBaseObject *rtType2UNSAFE) +{ + FCALL_CONTRACT; + + REFLECTCLASSBASEREF rtType1 = (REFLECTCLASSBASEREF)ObjectToOBJECTREF(rtType1UNSAFE); + REFLECTCLASSBASEREF rtType2 = (REFLECTCLASSBASEREF)ObjectToOBJECTREF(rtType2UNSAFE); + + BOOL areEquivalent = FALSE; + HELPER_METHOD_FRAME_BEGIN_RET_2(rtType1, rtType2); + + if (rtType1 != NULL && rtType2 != NULL) + areEquivalent = rtType1->GetType().IsEquivalentTo(rtType2->GetType()); + + HELPER_METHOD_FRAME_END(); + + FC_RETURN_BOOL(areEquivalent); +} +FCIMPLEND + // TypeEqualsHelper and TypeNotEqualsHelper are almost identical. // Unfortunately we cannot combime them because they need to hardcode the caller's name NOINLINE static BOOL TypeEqualSlow(OBJECTREF refL, OBJECTREF refR, LPVOID __me) diff --git a/src/vm/runtimehandles.h b/src/vm/runtimehandles.h index 6edf61ec99..66a27f4626 100644 --- a/src/vm/runtimehandles.h +++ b/src/vm/runtimehandles.h @@ -152,6 +152,8 @@ public: static FCDECL1_V(ReflectClassBaseObject*, GetTypeFromHandle, FCALLRuntimeTypeHandle th); static FCDECL1_V(EnregisteredTypeHandle, GetValueInternal, FCALLRuntimeTypeHandle RTH); + static FCDECL2(FC_BOOL_RET, IsEquivalentTo, ReflectClassBaseObject *rtType1UNSAFE, ReflectClassBaseObject *rtType2UNSAFE); + static FCDECL2(FC_BOOL_RET, TypeEQ, Object* left, Object* right); static FCDECL2(FC_BOOL_RET, TypeNEQ, Object* left, Object* right); diff --git a/src/vm/sampleprofiler.h b/src/vm/sampleprofiler.h index 871b175604..00c20f3b5a 100644 --- a/src/vm/sampleprofiler.h +++ b/src/vm/sampleprofiler.h @@ -22,7 +22,6 @@ class SampleProfiler // Declare friends. friend class EventPipe; - friend class SampleProfilerEventInstance; public: diff --git a/src/vm/util.cpp b/src/vm/util.cpp index f03c5e3605..8765896c4a 100644 --- a/src/vm/util.cpp +++ b/src/vm/util.cpp @@ -2929,7 +2929,7 @@ void DACNotify::DoGCNotification(const GcEvtArgs& args) if (args.typ == GC_MARK_END) { - TADDR Args[3] = { GC_NOTIFICATION, (TADDR) args.typ, args.condemnedGeneration }; + TADDR Args[3] = { GC_NOTIFICATION, (TADDR) args.typ, (TADDR) args.condemnedGeneration }; DACNotifyExceptionHelper(Args, 3); } } diff --git a/src/vm/yieldprocessornormalized.cpp b/src/vm/yieldprocessornormalized.cpp index 79d983e6dd..33fe113414 100644 --- a/src/vm/yieldprocessornormalized.cpp +++ b/src/vm/yieldprocessornormalized.cpp @@ -72,7 +72,7 @@ static void InitializeYieldProcessorNormalized() { yieldsPerNormalizedYield = 1; } - _ASSERTE(yieldsPerNormalizedYield <= MinNsPerNormalizedYield); + _ASSERTE(yieldsPerNormalizedYield <= (int)MinNsPerNormalizedYield); // Calculate the maximum number of yields that would be optimal for a late spin iteration. Typically, we would not want to // spend excessive amounts of time (thousands of cycles) doing only YieldProcessor, as SwitchToThread/Sleep would do a diff --git a/src/zap/zapcode.cpp b/src/zap/zapcode.cpp index 1c3cd83294..91cf7163e6 100644 --- a/src/zap/zapcode.cpp +++ b/src/zap/zapcode.cpp @@ -1788,7 +1788,7 @@ DWORD ZapLazyHelperThunk::SaveWorker(ZapWriter * pZapWriter) PORTABILITY_ASSERT("ZapLazyHelperThunk::Save"); #endif - _ASSERTE(p - buffer <= sizeof(buffer)); + _ASSERTE((DWORD)(p - buffer) <= sizeof(buffer)); if (pZapWriter != NULL) pZapWriter->Write(&buffer, (int)(p - buffer)); diff --git a/src/zap/zapimage.cpp b/src/zap/zapimage.cpp index 7a849fbfcc..59a9e12a8b 100644 --- a/src/zap/zapimage.cpp +++ b/src/zap/zapimage.cpp @@ -2177,7 +2177,7 @@ ZapImage::CompileStatus ZapImage::TryCompileMethodWorker(CORINFO_METHOD_HANDLE h // We skip the compilation of such methods and we don't want to // issue a warning or error // - if ((hrException == E_NOTIMPL) || (hrException == IDS_CLASSLOAD_GENERAL)) + if ((hrException == E_NOTIMPL) || (hrException == (HRESULT)IDS_CLASSLOAD_GENERAL)) { result = NOT_COMPILED; level = CORZAP_LOGLEVEL_INFO; diff --git a/src/zap/zapimport.cpp b/src/zap/zapimport.cpp index 887996475c..d84ea79e8a 100644 --- a/src/zap/zapimport.cpp +++ b/src/zap/zapimport.cpp @@ -2205,7 +2205,7 @@ DWORD ZapIndirectHelperThunk::SaveWorker(ZapWriter * pZapWriter) PORTABILITY_ASSERT("ZapIndirectHelperThunk::SaveWorker"); #endif - _ASSERTE(p - buffer <= sizeof(buffer)); + _ASSERTE((DWORD)(p - buffer) <= sizeof(buffer)); if (pZapWriter != NULL) pZapWriter->Write(&buffer, (int)(p - buffer)); diff --git a/src/zap/zapinfo.cpp b/src/zap/zapinfo.cpp index ccb796da9c..560400bd44 100644 --- a/src/zap/zapinfo.cpp +++ b/src/zap/zapinfo.cpp @@ -549,7 +549,7 @@ class MethodCodeComparer if (k1 == k2) return TRUE; - for (int i = 0; i < _countof(equivalentNodes); i++) + for (unsigned int i = 0; i < _countof(equivalentNodes); i++) { if (k1 == equivalentNodes[i][0] && k2 == equivalentNodes[i][1]) return TRUE; diff --git a/tests/CoreFX/CoreFX.issues.json b/tests/CoreFX/CoreFX.issues.json index 4489b51ecc..3db01ef4e1 100644 --- a/tests/CoreFX/CoreFX.issues.json +++ b/tests/CoreFX/CoreFX.issues.json @@ -862,6 +862,28 @@ } }, { + "name": "System.Text.Encoding.Tests", + "enabled": true, + "exclusions": { + "namespaces": null, + "classes": null, + "methods": [ + { + "name": "System.Text.Tests.EncoderConvert2.EncoderASCIIConvertMixedASCIIUnicodeCharArrayPartial", + "reason": "https://github.com/dotnet/coreclr/issues/23020" + }, + { + "name": "System.Text.Tests.EncoderConvert2.EncoderUTF8ConvertMixedASCIIUnicodeCharArrayPartial", + "reason": "https://github.com/dotnet/coreclr/issues/23020" + }, + { + "name": "System.Text.Tests.EncoderConvert2.EncoderUTF8ConvertUnicodeCharArrayPartial", + "reason": "https://github.com/dotnet/coreclr/issues/23020" + } + ] + } + }, + { "name": "System.Text.RegularExpressions.Tests", "enabled": true, "exclusions": { diff --git a/tests/issues.targets b/tests/issues.targets index e4bde3b22d..eb34df3ca6 100644 --- a/tests/issues.targets +++ b/tests/issues.targets @@ -71,9 +71,6 @@ <ExcludeList Include="$(XunitTestBinBase)/baseservices/threading/waithandle/waitany/waitanyex2a/*"> <Issue>19406</Issue> </ExcludeList> - <ExcludeList Include="$(XunitTestBinBase)/Loader/classloader/regressions/429802/CMain/*"> - <Issue>23096</Issue> - </ExcludeList> </ItemGroup> <!-- All Unix targets --> diff --git a/tests/scripts/run-corefx-tests.py b/tests/scripts/run-corefx-tests.py index a4e691df8a..594d6c4c19 100644 --- a/tests/scripts/run-corefx-tests.py +++ b/tests/scripts/run-corefx-tests.py @@ -288,23 +288,26 @@ def main(args): # Gather up some arguments to pass to the different build scripts. - config_args = '-restore -build -buildtests -configuration Release -os %s -arch %s' % (clr_os, arch) + common_config_args = '-configuration Release -framework netcoreapp -os %s -arch %s' % (clr_os, arch) + build_args = '-build -restore' + build_test_args = '-buildtests /p:ArchiveTests=Tests' if not no_run_tests: - config_args += ' -test' + build_test_args += ' -test' - build_args = config_args if not Is_windows and arch == 'arm' : # We need to force clang5.0; we are building in a docker container that doesn't have # clang3.9, which is currently the default used by the native build. - build_args += ' /p:BuildNativeClang=--clang5.0' + common_config_args += ' /p:BuildNativeClang=--clang5.0' if not Is_windows and (arch == 'arm' or arch == 'arm64'): # It is needed under docker where LC_ALL is not configured. - build_args += ' --warnAsError false' + common_config_args += ' --warnAsError false' - command = ' '.join(('build.cmd' if Is_windows else './build.sh', build_args)) + build_command = 'build.cmd' if Is_windows else './build.sh' + + command = ' '.join((build_command, common_config_args, build_args)) log(command) returncode = 0 if testing else os.system(command) if returncode != 0: @@ -332,11 +335,6 @@ def main(args): # Build the test command line. - if Is_windows: - command = 'build.cmd' - else: - command = './build.sh' - # If we're doing altjit testing, then don't run any tests that don't work with altjit. if ci_arch is not None and (ci_arch == 'x86_arm_altjit' or ci_arch == 'x64_arm64_altjit'): # The property value we need to specify for the WithoutCategories property is a semicolon @@ -366,8 +364,9 @@ def main(args): without_categories = ' /p:WithoutCategories=IgnoreForCI' command = ' '.join(( - command, - config_args, + build_command, + common_config_args, + build_test_args, without_categories )) diff --git a/tests/src/Loader/classloader/regressions/429802/CMain.il b/tests/src/Loader/classloader/regressions/429802/CMain.il index da86838fae..52b9f8b2a3 100644 --- a/tests/src/Loader/classloader/regressions/429802/CMain.il +++ b/tests/src/Loader/classloader/regressions/429802/CMain.il @@ -82,16 +82,16 @@ IL_002d: call void [mscorlib_1]System.Console::WriteLine(string) IL_0032: ldc.i4.0 IL_0033: stloc.0 - IL_0034: ldc.i4.1 - IL_0035: ldloc.2 - IL_0036: call instance int32 [MyBar]MyBar::DoBar() - IL_003b: beq.s IL_0049 - - IL_003d: ldstr "FAIL: expected MyBar.DoSelfBar to execute, but ano" - + "ther method was executed instead." - IL_0042: call void [mscorlib_1]System.Console::WriteLine(string) - IL_0047: ldc.i4.0 - IL_0048: stloc.0 + IL_0034: + + + + + + + + + IL_0049: ldloc.0 IL_004a: brtrue.s IL_005b diff --git a/tests/src/baseservices/typeequivalence/simple/Simple.cs b/tests/src/baseservices/typeequivalence/simple/Simple.cs index 0b6a54d14c..56c3ecd3fa 100644 --- a/tests/src/baseservices/typeequivalence/simple/Simple.cs +++ b/tests/src/baseservices/typeequivalence/simple/Simple.cs @@ -38,6 +38,34 @@ public class Simple } } + private static void ValidateTypeInstanceEquality() + { + Console.WriteLine($"{nameof(ValidateTypeInstanceEquality)}"); + var inAsm = EmptyType.Create(); + var otherAsm = EmptyType2.Create(); + + Type inAsmInterfaceType = inAsm.GetType().GetInterface(nameof(IEmptyType)); + Type otherAsmInterfaceType = otherAsm.GetType().GetInterface(nameof(IEmptyType)); + + // Sanity checks + Assert.IsTrue(inAsmInterfaceType == inAsmInterfaceType); + Assert.IsTrue(inAsmInterfaceType.IsEquivalentTo(inAsmInterfaceType)); + Assert.IsFalse(inAsmInterfaceType.IsEquivalentTo(inAsm.GetType())); + Assert.IsTrue(otherAsmInterfaceType == otherAsmInterfaceType); + Assert.IsTrue(otherAsmInterfaceType.IsEquivalentTo(otherAsmInterfaceType)); + Assert.IsFalse(otherAsmInterfaceType.IsEquivalentTo(otherAsm.GetType())); + + // The intrinsic equality operations should fail + Assert.IsFalse(inAsmInterfaceType == otherAsmInterfaceType); + Assert.IsFalse(inAsmInterfaceType.Equals(otherAsmInterfaceType)); + Assert.IsFalse(otherAsmInterfaceType == inAsmInterfaceType); + Assert.IsFalse(otherAsmInterfaceType.Equals(inAsmInterfaceType)); + + // Determination of equal types requires API call + Assert.IsTrue(inAsmInterfaceType.IsEquivalentTo(otherAsmInterfaceType)); + Assert.IsTrue(otherAsmInterfaceType.IsEquivalentTo(inAsmInterfaceType)); + } + private class MethodTestDerived : MethodTestBase { private readonly int scaleValue; @@ -127,6 +155,7 @@ public class Simple try { InterfaceTypesFromDifferentAssembliesAreEquivalent(); + ValidateTypeInstanceEquality(); InterfaceTypesMethodOperations(); CallSparseInterface(); } @@ -138,4 +167,4 @@ public class Simple return 100; } -}
\ No newline at end of file +} diff --git a/tests/src/tracing/common/AbstractTraceTest.cs b/tests/src/tracing/common/AbstractTraceTest.cs new file mode 100644 index 0000000000..26b7299ff1 --- /dev/null +++ b/tests/src/tracing/common/AbstractTraceTest.cs @@ -0,0 +1,148 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Diagnostics.Tracing; +using System.IO; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Tracing.Tests.Common; + +using Microsoft.Diagnostics.Tracing; +using Microsoft.Diagnostics.Tracing.Parsers.Clr; + +namespace Tracing.Tests +{ + [EventSource(Name = "My-Simple-Event-Source")] + public sealed class MySimpleEventSource : EventSource + { + private AbstractTraceTest abstractTraceTest; + + [NonEvent] + private void OnEventCommand(object sender, EventCommandEventArgs command) + { + this.abstractTraceTest.OnEventCommand(sender, command); + } + + public MySimpleEventSource(AbstractTraceTest abstractTraceTest) + { + this.abstractTraceTest = abstractTraceTest; + this.EventCommandExecuted += this.OnEventCommand; + } + + public void Request(string message) + { + WriteEvent(1, message); + } + } + + public abstract class AbstractTraceTest + { + protected abstract string GetConfigFileContents(); + + public virtual void OnEventCommand(object sender, EventCommandEventArgs command) + { + } + + protected virtual void InstallValidationCallbacks(TraceEventDispatcher trace) + { + + } + + protected virtual bool Pass() + { + return true; + } + + private static readonly TimeSpan TimeIntervalToReadConfigFile = new TimeSpan(0, 0, 25); + + private const int BytesInOneMB = 1024 * 1024; + + /// <summary> + /// This test collects a trace of itself and then performs some basic validation on the trace. + /// </summary> + public int Execute() + { + MySimpleEventSource MySimpleEventSource = new MySimpleEventSource(this); + + // Logging before tracing is enable - this should be ignored + MySimpleEventSource.Request("Test 1"); + + // Calculate the path to the config file. + string configFileName = Assembly.GetEntryAssembly().GetName().Name + ".eventpipeconfig"; + string configFilePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, configFileName); + Console.WriteLine("Calculated config file path: " + configFilePath); + + // Write the config file to disk. + File.WriteAllText(configFilePath, GetConfigFileContents()); + Console.WriteLine("Wrote contents of config file."); + + // Wait few seconds to ensure that tracing has started. + Console.WriteLine($"Waiting {TimeIntervalToReadConfigFile.TotalSeconds} seconds for the config file to be picked up by the next poll operation."); + Thread.Sleep(TimeIntervalToReadConfigFile); + + // Do some work that we can look for in the trace. + Console.WriteLine("Do some work that will be captured by the trace."); + GC.Collect(2, GCCollectionMode.Forced); + + // Logging while tracing is enabled - this should NOT be ignored + MySimpleEventSource.Request("Test 2"); + + Console.WriteLine("Done with the work."); + + // Delete the config file to start tracing. + File.Delete(configFilePath); + Console.WriteLine("Deleted the config file."); + + // Build the full path to the trace file. + string[] traceFiles = Directory.GetFiles(".", "*.netperf", SearchOption.TopDirectoryOnly); + Assert.Equal("traceFiles.Length == 1", traceFiles.Length, 1); + string traceFilePath = traceFiles[0]; + + // Poll the file system and wait for the trace file to be written. + Console.WriteLine("Wait for the config file deletion to be picked up and for the trace file to be written."); + + // Wait for 1 second, which is the poll time when tracing is enabled. + Thread.Sleep(TimeSpan.FromSeconds(1)); + + // Logging after tracing is disabled - this should be ignored + MySimpleEventSource.Request("Test 3"); + + // Poll for file size changes to the trace file itself. + // When the size of the trace file hasn't changed for few seconds, consider it fully written out. + Console.WriteLine($"Waiting for the trace file to be written. Poll every second to watch for {TimeIntervalToReadConfigFile.TotalSeconds} seconds of no file size changes."); + long lastSizeInBytes = 0; + DateTime timeOfLastChangeUTC = DateTime.UtcNow; + do + { + FileInfo traceFileInfo = new FileInfo(traceFilePath); + long currentSizeInBytes = traceFileInfo.Length; + Console.WriteLine("Trace file size: " + ((double)currentSizeInBytes / BytesInOneMB)); + + if (currentSizeInBytes > lastSizeInBytes) + { + lastSizeInBytes = currentSizeInBytes; + timeOfLastChangeUTC = DateTime.UtcNow; + } + + Thread.Sleep(TimeSpan.FromSeconds(1)); + + } while (DateTime.UtcNow.Subtract(timeOfLastChangeUTC) < TimeIntervalToReadConfigFile); + + // Use TraceEvent to consume the trace file and look for the work that we did. + Console.WriteLine("Using TraceEvent to parse the file to find the work that was done during trace capture."); + using (TraceEventDispatcher trace = TraceEventDispatcher.GetDispatcherFromFileName(traceFilePath)) + { + InstallValidationCallbacks(trace); + trace.Process(); + } + + // Clean-up the resulting trace file. + File.Delete(traceFilePath); + + return this.Pass() ? 100 : 10086; + } + } +}
\ No newline at end of file diff --git a/tests/src/tracing/common/common.csproj b/tests/src/tracing/common/common.csproj index 32a210882c..0de82bc6e2 100644 --- a/tests/src/tracing/common/common.csproj +++ b/tests/src/tracing/common/common.csproj @@ -30,6 +30,7 @@ <Compile Include="NetPerfFile.cs" /> <Compile Include="TraceControl.cs" /> <Compile Include="TraceConfiguration.cs" /> + <Compile Include="AbstractTraceTest.cs" /> </ItemGroup> <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" /> </Project> diff --git a/tests/src/tracing/keyword/TwoKeywords/TwoKeywords.cs b/tests/src/tracing/keyword/TwoKeywords/TwoKeywords.cs new file mode 100644 index 0000000000..89b38cec8a --- /dev/null +++ b/tests/src/tracing/keyword/TwoKeywords/TwoKeywords.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Diagnostics.Tracing; +using System.IO; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Tracing.Tests.Common; + +using Microsoft.Diagnostics.Tracing; +using Microsoft.Diagnostics.Tracing.Parsers.Clr; + +namespace Tracing.Tests +{ + public static class TwoKeywordsTest + { + public static int Main(string[] args) + { + return new TwoKeywordsTraceTest().Execute(); + } + } + + public class TwoKeywordsTraceTest : AbstractTraceTest + { + private bool pass; + + protected override string GetConfigFileContents() + { + return @" +OutputPath=. +CircularMB=2048 +Providers=My-Simple-Event-Source:0xFFFFFFFFFFFFFFFF:5:Key1=Value1;Key2=Value2 +";; + } + + public override void OnEventCommand(object sender, EventCommandEventArgs command) + { + if (command.Command == EventCommand.Enable) + { + this.pass = (command.Arguments.Count == 2); + } + } + + protected override bool Pass() + { + return this.pass; + } + } +} diff --git a/tests/src/tracing/keyword/TwoKeywords/TwoKeywords.csproj b/tests/src/tracing/keyword/TwoKeywords/TwoKeywords.csproj new file mode 100644 index 0000000000..e5882f01c3 --- /dev/null +++ b/tests/src/tracing/keyword/TwoKeywords/TwoKeywords.csproj @@ -0,0 +1,33 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" /> + <PropertyGroup> + <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> + <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> + <SchemaVersion>2.0</SchemaVersion> + <ProjectGuid>{8E3244CB-407F-4142-BAAB-E7A55901A5FA}</ProjectGuid> + <OutputType>Exe</OutputType> + <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids> + <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir> + <CLRTestKind>BuildAndRun</CLRTestKind> + <DefineConstants>$(DefineConstants);STATIC</DefineConstants> + <CLRTestPriority>0</CLRTestPriority> + <GCStressIncompatible>true</GCStressIncompatible> + <!-- Due to https://github.com/dotnet/coreclr/issues/22247 --> + <UnloadabilityIncompatible>true</UnloadabilityIncompatible> + <DisableProjectBuild Condition="'$(Platform)' == 'arm'">true</DisableProjectBuild> + </PropertyGroup> + <!-- Default configurations to help VS understand the configurations --> + <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'"></PropertyGroup> + <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'"></PropertyGroup> + <ItemGroup> + <CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies"> + <Visible>False</Visible> + </CodeAnalysisDependentAssemblyPaths> + </ItemGroup> + <ItemGroup> + <Compile Include="TwoKeywords.cs" /> + <ProjectReference Include="../../common/common.csproj" /> + </ItemGroup> + <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" /> +</Project>
\ No newline at end of file diff --git a/tests/src/tracing/tracecontrol/TraceControl.cs b/tests/src/tracing/tracecontrol/TraceControl.cs index 68870d9fc0..99e552ac4e 100644 --- a/tests/src/tracing/tracecontrol/TraceControl.cs +++ b/tests/src/tracing/tracecontrol/TraceControl.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Diagnostics.Tracing; using System.IO; using System.Reflection; using System.Threading; @@ -16,101 +17,42 @@ namespace Tracing.Tests { public static class TraceControlTest { + public static int Main(string[] args) + { + return new TraceControlTraceTest().Execute(); + } + } + + public class TraceControlTraceTest : AbstractTraceTest + { + private bool pass; + private static string ConfigFileContents = @" OutputPath=. CircularMB=2048 -Providers=*:0xFFFFFFFFFFFFFFFF:5 +Providers=*:0xFFFFFFFFFFFFFFFF:5: "; - - private static readonly TimeSpan TimeIntervalToReadConfigFile = new TimeSpan(0, 0, 25); - - private const int BytesInOneMB = 1024 * 1024; - - /// <summary> - /// This test collects a trace of itself and then performs some basic validation on the trace. - /// </summary> - public static int Main(string[] args) + protected override string GetConfigFileContents() { - // Calculate the path to the config file. - string configFileName = Assembly.GetEntryAssembly().GetName().Name + ".eventpipeconfig"; - string configFilePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, configFileName); - Console.WriteLine("Calculated config file path: " + configFilePath); - - // Write the config file to disk. - File.WriteAllText(configFilePath, ConfigFileContents); - Console.WriteLine("Wrote contents of config file."); - - // Wait few seconds to ensure that tracing has started. - Console.WriteLine($"Waiting {TimeIntervalToReadConfigFile.TotalSeconds} seconds for the config file to be picked up by the next poll operation."); - Thread.Sleep(TimeIntervalToReadConfigFile); - - // Do some work that we can look for in the trace. - Console.WriteLine("Do some work that will be captured by the trace."); - GC.Collect(2, GCCollectionMode.Forced); - Console.WriteLine("Done with the work."); - - // Delete the config file to start tracing. - File.Delete(configFilePath); - Console.WriteLine("Deleted the config file."); - - // Build the full path to the trace file. - string[] traceFiles = Directory.GetFiles(".", "*.netperf", SearchOption.TopDirectoryOnly); - Assert.Equal("traceFiles.Length == 1", traceFiles.Length, 1); - string traceFilePath = traceFiles[0]; - - // Poll the file system and wait for the trace file to be written. - Console.WriteLine("Wait for the config file deletion to be picked up and for the trace file to be written."); - - // Wait for 1 second, which is the poll time when tracing is enabled. - Thread.Sleep(TimeSpan.FromSeconds(1)); + return ConfigFileContents; + } - // Poll for file size changes to the trace file itself. - // When the size of the trace file hasn't changed for few seconds, consider it fully written out. - Console.WriteLine($"Waiting for the trace file to be written. Poll every second to watch for {TimeIntervalToReadConfigFile.TotalSeconds} seconds of no file size changes."); - long lastSizeInBytes = 0; - DateTime timeOfLastChangeUTC = DateTime.UtcNow; - do + protected override void InstallValidationCallbacks(TraceEventDispatcher trace) + { + string gcReasonInduced = GCReason.Induced.ToString(); + trace.Clr.GCTriggered += delegate (GCTriggeredTraceData data) { - FileInfo traceFileInfo = new FileInfo(traceFilePath); - long currentSizeInBytes = traceFileInfo.Length; - Console.WriteLine("Trace file size: " + ((double)currentSizeInBytes / BytesInOneMB)); - - if (currentSizeInBytes > lastSizeInBytes) + if (gcReasonInduced.Equals(data.Reason.ToString())) { - lastSizeInBytes = currentSizeInBytes; - timeOfLastChangeUTC = DateTime.UtcNow; + Console.WriteLine("Detected an induced GC"); + pass = true; } + }; + } - Thread.Sleep(TimeSpan.FromSeconds(1)); - - } while (DateTime.UtcNow.Subtract(timeOfLastChangeUTC) < TimeIntervalToReadConfigFile); - - int retVal = 0; - - // Use TraceEvent to consume the trace file and look for the work that we did. - Console.WriteLine("Using TraceEvent to parse the file to find the work that was done during trace capture."); - using (var trace = TraceEventDispatcher.GetDispatcherFromFileName(traceFilePath)) - { - string gcReasonInduced = GCReason.Induced.ToString(); - string providerName = "Microsoft-Windows-DotNETRuntime"; - string gcTriggeredEventName = "GC/Triggered"; - - trace.Clr.GCTriggered += delegate (GCTriggeredTraceData data) - { - if (gcReasonInduced.Equals(data.Reason.ToString())) - { - Console.WriteLine("Detected an induced GC"); - retVal = 100; - } - }; - - trace.Process(); - } - - // Clean-up the resulting trace file. - File.Delete(traceFilePath); - - return retVal; + protected override bool Pass() + { + return this.pass; } } } |