Avoid failures when allocate memory in native (#8028)

* Avoid failures when allocate memory in native

using new in the native side can throw and the app will shutdown. instead we detect the failure and throw a managed exception

* Use same mechanism we return the error code from GetSortHandle

* Add assert

* Dispose the safeHandle

2 files changed, 22 insertions, 8 deletions

diff --git a/src/corefx/System.Globalization.Native/collation.cpp b/src/corefx/System.Globalization.Native/collation.cpp
index 06d41f9c23..f37211208e 100644
--- a/src/corefx/System.Globalization.Native/collation.cpp
+++ b/src/corefx/System.Globalization.Native/collation.cpp
@@ -10,6 +10,8 @@
 #include <map>
 
 #include "icushim.h"
+#include "locale.hpp"
+#include "errors.h"
 
 const int32_t CompareOptionsIgnoreCase = 0x1;
 const int32_t CompareOptionsIgnoreNonSpace = 0x2;
@@ -331,24 +333,30 @@ extern "C" int32_t GlobalizationNative_GetSortVersion()
     return UCOL_RUNTIME_VERSION << 16 | UCOL_BUILDER_VERSION;
 }
 
-extern "C" SortHandle* GlobalizationNative_GetSortHandle(const char* lpLocaleName)
+extern "C" ResultCode GlobalizationNative_GetSortHandle(const char* lpLocaleName, SortHandle** ppSortHandle)
 {
-    SortHandle* pSortHandle = new SortHandle();
+    assert(ppSortHandle != nullptr);
+    
+    *ppSortHandle = new (std::nothrow) SortHandle();
+    if ((*ppSortHandle) == nullptr)
+    {
+        return GetResultCode(U_MEMORY_ALLOCATION_ERROR);
+    }
 
     UErrorCode err = U_ZERO_ERROR;
 
-    pSortHandle->regular = ucol_open(lpLocaleName, &err);
+    (*ppSortHandle)->regular = ucol_open(lpLocaleName, &err);
 
     if (U_FAILURE(err))
     {
-        if (pSortHandle->regular != nullptr)
-              ucol_close(pSortHandle->regular);
+        if ((*ppSortHandle)->regular != nullptr)
+            ucol_close((*ppSortHandle)->regular);
 
-        delete pSortHandle;
-        pSortHandle = nullptr;
+        delete (*ppSortHandle);
+        (*ppSortHandle) = nullptr;
     }
 
-    return pSortHandle;
+    return GetResultCode(err);
 }
 
 extern "C" void GlobalizationNative_CloseSortHandle(SortHandle* pSortHandle)
diff --git a/src/corefx/System.Globalization.Native/errors.h b/src/corefx/System.Globalization.Native/errors.h
index 031ea7b120..b23a0dacd5 100644
--- a/src/corefx/System.Globalization.Native/errors.h
+++ b/src/corefx/System.Globalization.Native/errors.h
@@ -13,6 +13,7 @@ enum ResultCode : int32_t
     Success = 0,
     UnknownError = 1,
     InsufficentBuffer = 2,
+    OutOfMemory = 3
 };
 
 /*
@@ -24,6 +25,11 @@ static ResultCode GetResultCode(UErrorCode err)
     {
         return InsufficentBuffer;
     }
+    
+    if (err == U_MEMORY_ALLOCATION_ERROR)
+    {
+        return OutOfMemory;
+    }
 
     if (U_SUCCESS(err))
     {