JIT: enable inline pinvoke in more cases

An inline pinvoke is a pinvoke where the managed/native transition
overhead is reduced by inlining parts of the transition bookkeeping
around the call site. A normal pinvoke does this bookkeeping in
a stub method that interposes between the managed caller and the
native callee.

Previously the jit would not allow pinvoke calls that came from inlines
to be optimized via inline pinvoke. This sometimes caused performance
surprises for users who wrap DLL imports with managed methods. See for
instance #2373.

This change lifts this limitation. Pinvokes from inlined method bodies
are now given the same treatment as pinvokes in the root method. The
legality check for inline pinvokes has been streamlined slightly to
remove a redundant check. Inline pinvokes introduced by inlining are
handled by accumulating the unmanaged method count with the value from
inlinees, and deferring insertion of the special basic blocks until after
inlining, so that if the only inline pinvokes come from inline instances
they are still properly processed.

Inline pinvokes are still disallowed in try and handler regions
(catches, filters, and finallies).

X87 liveness tracking was updated to handle the implicit inline frame
var references. This was a pre-existing issue that now can show up more
frequently. Added a test case that fails with the stock legacy jit
(and also with the new enhancements to pinvoke). Now both the original
failing case and this case pass.

Inline pinvokes are also now suppressed in rarely executed blocks,
for instance blocks leading up to throws or similar.

The inliner is now also changed to preferentially report inline
reasons as forced instead of always when both are applicable.

This change adds a new test case that shows the variety of
situations that can occur with pinvoke, inlining, and EH.

5 files changed, 372 insertions, 0 deletions

diff --git a/tests/src/JIT/Directed/pinvoke/pinvoke-examples.cs b/tests/src/JIT/Directed/pinvoke/pinvoke-examples.cs
new file mode 100644
index 0000000000..26080d8d71
--- /dev/null
+++ b/tests/src/JIT/Directed/pinvoke/pinvoke-examples.cs
@@ -0,0 +1,223 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+// Test cases showing interaction of inlining and inline pinvoke,
+// along with the impact of EH.
+
+using System;
+using System.Runtime.CompilerServices;
+
+
+namespace PInvokeTest
+{
+    internal class Test
+    {
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        static int AsForceInline()
+        {
+            return Environment.ProcessorCount;
+        }
+
+        static int AsNormalInline()
+        {
+            return Environment.ProcessorCount;
+        }
+
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        static int AsNoInline()
+        {
+            return Environment.ProcessorCount;
+        }
+
+        static bool FromTryCatch()
+        {
+            bool result = false;
+            try 
+            {
+                // All pinvokes should be inline, except on x64
+                result = (Environment.ProcessorCount == AsNormalInline());
+            }
+            catch (Exception)
+            {
+                result = false;
+            }
+            return result;
+        }        
+
+        static bool FromTryFinally()
+        {
+            bool result = false;
+            bool result1 = false;
+            bool result2 = false;
+            try 
+            {
+                // All pinvokes should be inline, except on x64
+                result1 = (Environment.ProcessorCount == AsNormalInline());
+                result2 = (Environment.ProcessorCount == AsNormalInline());
+            }
+            finally
+            {
+                result = result1 && result2;
+            }
+            return result;
+        }        
+
+        static bool FromTryFinally2()
+        {
+            bool result = false;
+            bool result1 = false;
+            bool result2 = false;
+
+            try 
+            {
+                // These two pinvokes should be inline, except on x64
+                result1 = (Environment.ProcessorCount == AsNormalInline());
+            }
+            finally
+            {
+                // These two pinvokes should *not* be inline (finally)
+                result2 = (Environment.ProcessorCount == AsNormalInline());
+                result = result1 && result2;
+            }
+
+            return result;
+        }        
+
+        static bool FromTryFinally3()
+        {
+            bool result = false;
+            bool result1 = false;
+            bool result2 = false;
+
+            try 
+            {
+                // These two pinvokes should be inline, except on x64
+                result1 = (Environment.ProcessorCount == AsNormalInline());
+            }
+            finally
+            {
+                try 
+                {
+                    // These two pinvokes should *not* be inline (finally)
+                    result2 = (Environment.ProcessorCount == AsNormalInline());
+                }
+                catch (Exception)
+                {
+                    result2 = false;
+                }
+
+                result = result1 && result2;
+            }
+
+            return result;
+        }        
+
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        static bool FromInline()
+        {
+            // These two pinvokes should be inline
+            bool result = (Environment.ProcessorCount == AsForceInline());
+            return result;
+        }
+
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        static bool FromInline2()
+        {
+            // These four pinvokes should be inline
+            bool result1 = (Environment.ProcessorCount == AsNormalInline());
+            bool result2 = (Environment.ProcessorCount == AsForceInline());
+            return result1 && result2;
+        }
+
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        static bool FromNoInline()
+        {
+            // The only pinvoke should be inline
+            bool result = (Environment.ProcessorCount == AsNoInline());
+            return result;
+        }
+
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        static bool FromNoInline2()
+        {
+            // Three pinvokes should be inline
+            bool result1 = (Environment.ProcessorCount == AsNormalInline());
+            bool result2 = (Environment.ProcessorCount == AsNoInline());
+            return result1 && result2;
+        }
+
+        static bool FromFilter()
+        {
+            bool result = false;
+
+            try
+            {
+                throw new Exception("expected");
+            }
+            // These two pinvokes should *not* be inline (filter)
+            //
+            // For the first call the jit won't inline the wrapper, so
+            // it just calls get_ProcessorCount.
+            //
+            // For the second call, the force inline works, and the
+            // subsequent inline of get_ProcessorCount exposes a call
+            // to the pinvoke GetProcessorCount.  This pinvoke will
+            // not be inline.
+            catch (Exception) when (Environment.ProcessorCount == AsForceInline())
+            {
+                result = true;
+            }
+
+            return result;
+        }
+
+        static bool FromColdCode()
+        {
+            int pc = 0;
+            bool result1 = false;
+            bool result2 = false;
+
+            try
+            {
+                // This pinvoke should not be inline (cold)
+                pc = Environment.ProcessorCount;
+                throw new Exception("expected");
+            }
+            catch (Exception)
+            {
+                // These two pinvokes should not be inline (catch)
+                //
+                // For the first call the jit won't inline the
+                // wrapper, so it just calls get_ProcessorCount.
+                //
+                // For the second call, the force inline works, and
+                // the subsequent inline of get_ProcessorCount exposes
+                // a call to the pinvoke GetProcessorCount.  This
+                // pinvoke will not be inline.
+                result1 = (pc == Environment.ProcessorCount);
+                result2 = (pc == AsForceInline());
+            }
+
+            return result1 && result2;
+        }
+
+        private static int Main()
+        {
+            bool result = true;
+
+            result &= FromTryCatch();
+            result &= FromTryFinally();
+            result &= FromTryFinally2();
+            result &= FromTryFinally3();
+            result &= FromInline();           
+            result &= FromInline2();
+            result &= FromNoInline();
+            result &= FromNoInline2();
+            result &= FromFilter();
+            result &= FromColdCode();
+
+            return (result ? 100 : -1);
+        }
+    }
+}
diff --git a/tests/src/JIT/Directed/pinvoke/pinvoke-examples.csproj b/tests/src/JIT/Directed/pinvoke/pinvoke-examples.csproj
new file mode 100644
index 0000000000..78cf4471fe
--- /dev/null
+++ b/tests/src/JIT/Directed/pinvoke/pinvoke-examples.csproj
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <AssemblyName>$(MSBuildProjectName)</AssemblyName>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{95DFC527-4DC1-495E-97D7-E94EE1F7140D}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <FileAlignment>512</FileAlignment>
+    <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages</ReferencePath>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
+    <NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
+  </PropertyGroup>
+  <!-- Default configurations to help VS understand the configurations -->
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+  </PropertyGroup>
+  <ItemGroup>
+    <CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
+      <Visible>False</Visible>
+    </CodeAnalysisDependentAssemblyPaths>
+  </ItemGroup>
+  <PropertyGroup>
+    <DebugType>PdbOnly</DebugType>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="pinvoke-examples.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
+  </ItemGroup>
+  <PropertyGroup>
+    <ProjectJson>$(JitPackagesConfigFileDirectory)minimal\project.json</ProjectJson>
+    <ProjectLockJson>$(JitPackagesConfigFileDirectory)minimal\project.lock.json</ProjectLockJson>
+  </PropertyGroup>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
+  <PropertyGroup Condition=" '$(MsBuildProjectDirOverride)' != '' ">
+  </PropertyGroup> 
+</Project>
\ No newline at end of file
diff --git a/tests/src/jit/Directed/pinvoke/pinvoke-bug.cs b/tests/src/jit/Directed/pinvoke/pinvoke-bug.cs
new file mode 100644
index 0000000000..2d4b5f6aea
--- /dev/null
+++ b/tests/src/jit/Directed/pinvoke/pinvoke-bug.cs
@@ -0,0 +1,60 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Runtime.InteropServices;
+
+// Test includes an intentional unreachable return 
+#pragma warning disable 162
+
+namespace PInvokeTest
+{
+    internal class Test
+    {
+        [DllImport("msvcrt", EntryPoint = "sin")]
+        private static extern double sin(double x);
+
+        private static double g;
+        private static bool b;
+
+        public static int Main(string[] args)
+        {
+            bool result = false;
+            g = 0.0;
+            double val = 1.0;
+            b = false;
+            try
+            {
+                Func(val);
+            }
+            catch(Exception)
+            {
+                result = (Math.Abs(g - sin(val)) < 0.0001);
+            }
+
+            return (result ? 100 : -1);
+        }
+
+        // An inline pinvoke in a method with float math followed by a
+        // throw may causes trouble for liveness models for the inline
+        // frame var.
+        static double Func(double x)
+        {
+            g = sin(x);
+
+            // A bit of control flow to throw off rareness detection
+            // Also we need float in here 
+            if (b)
+            {
+                g = 0.0;
+            }
+
+            throw new Exception();
+
+            // Deliberately unreachable return
+            return g;
+        }
+    }
+}
+
diff --git a/tests/src/jit/Directed/pinvoke/pinvoke-bug.csproj b/tests/src/jit/Directed/pinvoke/pinvoke-bug.csproj
new file mode 100644
index 0000000000..2f8a24638b
--- /dev/null
+++ b/tests/src/jit/Directed/pinvoke/pinvoke-bug.csproj
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <AssemblyName>$(MSBuildProjectName)</AssemblyName>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{95DFC527-4DC1-495E-97D7-E94EE1F7140D}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <FileAlignment>512</FileAlignment>
+    <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages</ReferencePath>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\</SolutionDir>
+    <NuGetPackageImportStamp>7a9bfb7d</NuGetPackageImportStamp>
+  </PropertyGroup>
+  <!-- Default configurations to help VS understand the configurations -->
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+  </PropertyGroup>
+  <ItemGroup>
+    <CodeAnalysisDependentAssemblyPaths Condition=" '$(VS100COMNTOOLS)' != '' " Include="$(VS100COMNTOOLS)..\IDE\PrivateAssemblies">
+      <Visible>False</Visible>
+    </CodeAnalysisDependentAssemblyPaths>
+  </ItemGroup>
+  <PropertyGroup>
+    <DebugType>PdbOnly</DebugType>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="pinvoke-bug.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
+  </ItemGroup>
+  <PropertyGroup>
+    <ProjectJson>$(JitPackagesConfigFileDirectory)minimal\project.json</ProjectJson>
+    <ProjectLockJson>$(JitPackagesConfigFileDirectory)minimal\project.lock.json</ProjectLockJson>
+  </PropertyGroup>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
+  <PropertyGroup Condition=" '$(MsBuildProjectDirOverride)' != '' ">
+  </PropertyGroup> 
+</Project>
\ No newline at end of file
diff --git a/tests/testsUnsupportedOutsideWindows.txt b/tests/testsUnsupportedOutsideWindows.txt
index 4f68d7136f..0f8f194b41 100644
--- a/tests/testsUnsupportedOutsideWindows.txt
+++ b/tests/testsUnsupportedOutsideWindows.txt
@@ -147,6 +147,7 @@ JIT/Directed/intrinsic/interlocked/rva_rvastatic3/rva_rvastatic3.sh
 JIT/Directed/intrinsic/interlocked/rva_rvastatic4/rva_rvastatic4.sh
 JIT/Directed/pinvoke/calli_excep/calli_excep.sh
 JIT/Directed/pinvoke/jump/jump.sh
+JIT/Directed/pinvoke/pinvoke-bug/pinvoke-bug.sh
 JIT/Directed/pinvoke/sin/sin.sh
 JIT/Directed/pinvoke/sysinfo_cs/sysinfo_cs.sh
 JIT/Directed/pinvoke/sysinfo_il/sysinfo_il.sh