UTF8 Marshaling support(UnmanagedType.LPUTF8Str)

Usage: [MarshalAs(UnmanagedType.LPUTF8Str)] applied to string
and stringbuilder.

Implementation mostly use Encoding.UTF8 API to do the byte buffer
to string roundtripping. Introducing two new marshalers,
UTF8StringMarshaler and UTF8BufferMarshaler which handle string
and StringBuilder respectively. [Out] StringBuilder marshaling use
builder capacity as the buffer size ie (builder. Capacity + 1) *3
which is enough for any UTF8 char in BMP plane, infact Encoding.UTF8
mscorlib APIs use the same length.All marshaling flags(ThrowOnUnmapable,
defaultchar) are ignored since they do not make sense in UTF16 to UTD8
context.

The public contracts are not yet updated, the public contracts and
public marshaling API (Marshal.PtrToStringUtf8 and StringToHGlobalUtf8)
will be added once the implementation is in. The marshal api are anyway
going to be a wrapper around Encoding.GetBytes and GetChars.

1 files changed, 242 insertions, 0 deletions

diff --git a/src/vm/ilmarshalers.cpp b/src/vm/ilmarshalers.cpp
index 8062cfd608..ebd8250459 100644
--- a/src/vm/ilmarshalers.cpp
+++ b/src/vm/ilmarshalers.cpp
@@ -507,6 +507,155 @@ void ILOptimizedAllocMarshaler::EmitClearNative(ILCodeStream* pslILEmit)
     }
 }
 
+LocalDesc ILUTF8BufferMarshaler::GetManagedType()
+{
+    STANDARD_VM_CONTRACT;
+    return LocalDesc(MscorlibBinder::GetClass(CLASS__STRING_BUILDER));
+}
+
+void ILUTF8BufferMarshaler::EmitConvertSpaceCLRToNative(ILCodeStream* pslILEmit)
+{
+    STANDARD_VM_CONTRACT;
+
+    ILCodeLabel* pNullRefLabel = pslILEmit->NewCodeLabel();
+
+    pslILEmit->EmitLoadNullPtr();
+    EmitStoreNativeValue(pslILEmit);
+
+    EmitLoadManagedValue(pslILEmit);
+    pslILEmit->EmitBRFALSE(pNullRefLabel);
+
+    EmitLoadManagedValue(pslILEmit);
+    // int System.Text.StringBuilder.get_Capacity()
+    pslILEmit->EmitCALL(METHOD__STRING_BUILDER__GET_CAPACITY, 1, 1);
+    pslILEmit->EmitDUP();
+
+    // static void StubHelpers.CheckStringLength(int length)
+    pslILEmit->EmitCALL(METHOD__STUBHELPERS__CHECK_STRING_LENGTH, 1, 0);
+
+    // Max number of bytes for UTF8 string in BMP plane is ( StringBuilder.Capacity + 1 ) * 3 + 1
+    // first +1 if the high surrogate is '?' and second +1 for null byte.
+
+    // stack: capacity_in_bytes
+    pslILEmit->EmitLDC(1);
+    pslILEmit->EmitADD();
+
+    // stack: capacity
+    pslILEmit->EmitLDC(3);
+    pslILEmit->EmitMUL();
+
+    // stack: offset_of_null
+    DWORD dwTmpOffsetOfSecretNull = pslILEmit->NewLocal(ELEMENT_TYPE_I4);
+    pslILEmit->EmitDUP();
+    pslILEmit->EmitSTLOC(dwTmpOffsetOfSecretNull); // make sure the stack is empty for localloc
+
+    // make space for '\0'
+    pslILEmit->EmitLDC(1);
+    pslILEmit->EmitADD();
+
+    // stack: alloc_size_in_bytes
+    ILCodeLabel *pAllocRejoin = pslILEmit->NewCodeLabel();
+    if (IsCLRToNative(m_dwMarshalFlags) && !IsByref(m_dwMarshalFlags))
+    {
+        ILCodeLabel *pNoOptimize = pslILEmit->NewCodeLabel();
+        m_dwLocalBuffer = pslILEmit->NewLocal(ELEMENT_TYPE_I);
+
+        // LocalBuffer = 0
+        pslILEmit->EmitLoadNullPtr();
+        pslILEmit->EmitSTLOC(m_dwLocalBuffer);
+
+        // if (alloc_size_in_bytes > MAX_LOCAL_BUFFER_LENGTH) goto NoOptimize
+        pslILEmit->EmitDUP();
+        pslILEmit->EmitLDC(MAX_LOCAL_BUFFER_LENGTH);
+        pslILEmit->EmitCGT_UN();
+        pslILEmit->EmitBRTRUE(pNoOptimize);
+
+        pslILEmit->EmitLOCALLOC();
+        pslILEmit->EmitDUP();
+        pslILEmit->EmitSTLOC(m_dwLocalBuffer);
+        pslILEmit->EmitBR(pAllocRejoin);
+
+        pslILEmit->EmitLabel(pNoOptimize);
+    }
+
+    // static IntPtr AllocCoTaskMem(int cb)
+    pslILEmit->EmitCALL(METHOD__MARSHAL__ALLOC_CO_TASK_MEM, 1, 1);
+
+    pslILEmit->EmitLabel(pAllocRejoin);
+
+    // stack: native_addr
+
+    pslILEmit->EmitDUP();
+    EmitStoreNativeValue(pslILEmit);
+
+    pslILEmit->EmitLDLOC(dwTmpOffsetOfSecretNull);
+
+    // stack: native_addr offset_of_null
+    pslILEmit->EmitADD();
+
+    // stack: addr_of_null0    
+    pslILEmit->EmitLDC(0);
+    pslILEmit->EmitSTIND_I1();
+
+    pslILEmit->EmitLabel(pNullRefLabel);
+}
+
+void ILUTF8BufferMarshaler::EmitConvertContentsCLRToNative(ILCodeStream* pslILEmit)
+{
+    STANDARD_VM_CONTRACT;
+    DWORD dwUtf8MarshalFlags =
+        (m_pargs->m_pMarshalInfo->GetBestFitMapping() & 0xFF) |
+        (m_pargs->m_pMarshalInfo->GetThrowOnUnmappableChar() << 8);
+
+    // setup to call UTF8BufferMarshaler.ConvertToNative
+    EmitLoadManagedValue(pslILEmit);
+    EmitLoadNativeValue(pslILEmit);
+    pslILEmit->EmitLDC(dwUtf8MarshalFlags);
+
+    //ConvertToNative(StringBuilder sb,IntPtr pNativeBuffer, int flags)	
+    pslILEmit->EmitCALL(METHOD__UTF8BUFFERMARSHALER__CONVERT_TO_NATIVE, 3, 1);
+    EmitStoreNativeValue(pslILEmit);
+}
+
+void ILUTF8BufferMarshaler::EmitConvertSpaceNativeToCLR(ILCodeStream* pslILEmit)
+{
+    STANDARD_VM_CONTRACT;
+
+    ILCodeLabel* pNullRefLabel = pslILEmit->NewCodeLabel();
+
+    EmitLoadNativeValue(pslILEmit);
+    pslILEmit->EmitBRFALSE(pNullRefLabel);
+
+    if (IsIn(m_dwMarshalFlags) || IsCLRToNative(m_dwMarshalFlags))
+    {
+        EmitLoadNativeValue(pslILEmit);
+        // static int System.StubHelpers.StubHelpers.strlen(sbyte* ptr)
+        pslILEmit->EmitCALL(METHOD__STUBHELPERS__STRLEN, 1, 1);
+    }
+    else
+    {
+        // don't touch the native buffer in the native->CLR out-only case
+        pslILEmit->EmitLDC(0);
+    }
+    // Convert to UTF8 and then call 
+    // System.Text.StringBuilder..ctor(int capacity)
+    pslILEmit->EmitNEWOBJ(METHOD__STRING_BUILDER__CTOR_INT, 1);
+    EmitStoreManagedValue(pslILEmit);
+    pslILEmit->EmitLabel(pNullRefLabel);
+}
+
+void ILUTF8BufferMarshaler::EmitConvertContentsNativeToCLR(ILCodeStream* pslILEmit)
+{
+    STANDARD_VM_CONTRACT;
+
+    EmitLoadManagedValue(pslILEmit);
+    EmitLoadNativeValue(pslILEmit);
+
+    //void UTF8BufferMarshaler.ConvertToManaged(StringBuilder sb, IntPtr pNative)
+    pslILEmit->EmitCALL(METHOD__UTF8BUFFERMARSHALER__CONVERT_TO_MANAGED, 2, 0);
+}
+
+
 LocalDesc ILWSTRBufferMarshaler::GetManagedType()
 {
     STANDARD_VM_CONTRACT;
@@ -1924,6 +2073,99 @@ void ILHSTRINGMarshaler::EmitClearNative(ILCodeStream* pslILEmit)
 
 #endif // FEATURE_COMINTEROP
 
+LocalDesc ILCUTF8Marshaler::GetManagedType()
+{
+	LIMITED_METHOD_CONTRACT;
+
+	return LocalDesc(ELEMENT_TYPE_STRING);
+}
+
+void ILCUTF8Marshaler::EmitConvertContentsCLRToNative(ILCodeStream* pslILEmit)
+{
+	STANDARD_VM_CONTRACT;
+
+	DWORD dwUtf8MarshalFlags =
+		(m_pargs->m_pMarshalInfo->GetBestFitMapping() & 0xFF) |
+		(m_pargs->m_pMarshalInfo->GetThrowOnUnmappableChar() << 8);
+
+	bool bPassByValueInOnly = IsIn(m_dwMarshalFlags) && !IsOut(m_dwMarshalFlags) && !IsByref(m_dwMarshalFlags);
+	if (bPassByValueInOnly)
+	{
+		DWORD dwBufSize = pslILEmit->NewLocal(ELEMENT_TYPE_I4);
+		m_dwLocalBuffer = pslILEmit->NewLocal(ELEMENT_TYPE_I);
+
+		// LocalBuffer = 0
+		pslILEmit->EmitLoadNullPtr();
+		pslILEmit->EmitSTLOC(m_dwLocalBuffer);
+
+		ILCodeLabel* pNoOptimize = pslILEmit->NewCodeLabel();
+
+		// if == NULL, goto NoOptimize
+		EmitLoadManagedValue(pslILEmit);
+		pslILEmit->EmitBRFALSE(pNoOptimize);
+				 		
+		// (String.Length + 1)
+		// Characters would be # of characters + 1 in case left over high surrogate is ?
+		EmitLoadManagedValue(pslILEmit);
+		pslILEmit->EmitCALL(METHOD__STRING__GET_LENGTH, 1, 1);
+		pslILEmit->EmitLDC(1);
+		pslILEmit->EmitADD();
+
+		// Max 3 bytes per char.
+		// (String.Length + 1) * 3		
+		pslILEmit->EmitLDC(3);
+		pslILEmit->EmitMUL();
+
+		// +1 for the 0x0 that we put in.
+		// ((String.Length + 1) * 3) + 1
+		pslILEmit->EmitLDC(1);
+		pslILEmit->EmitADD();
+				
+		// BufSize = ( (String.Length+1) * 3) + 1
+		pslILEmit->EmitSTLOC(dwBufSize);
+
+		// if (MAX_LOCAL_BUFFER_LENGTH < BufSize ) goto NoOptimize
+		pslILEmit->EmitLDC(MAX_LOCAL_BUFFER_LENGTH);
+		pslILEmit->EmitLDLOC(dwBufSize);
+		pslILEmit->EmitCLT();
+		pslILEmit->EmitBRTRUE(pNoOptimize);
+
+		// LocalBuffer = localloc(BufSize);
+		pslILEmit->EmitLDLOC(dwBufSize);
+		pslILEmit->EmitLOCALLOC();
+		pslILEmit->EmitSTLOC(m_dwLocalBuffer);
+
+		// NoOptimize:
+		pslILEmit->EmitLabel(pNoOptimize);
+	}
+
+	// UTF8Marshaler.ConvertToNative(dwUtf8MarshalFlags,pManaged, pLocalBuffer)
+	pslILEmit->EmitLDC(dwUtf8MarshalFlags);
+	EmitLoadManagedValue(pslILEmit);
+
+	if (m_dwLocalBuffer != LOCAL_NUM_UNUSED)
+	{
+		pslILEmit->EmitLDLOC(m_dwLocalBuffer);
+	}
+	else
+	{
+		pslILEmit->EmitLoadNullPtr();
+	}
+
+	pslILEmit->EmitCALL(METHOD__CUTF8MARSHALER__CONVERT_TO_NATIVE, 3, 1);
+
+	EmitStoreNativeValue(pslILEmit);
+}
+
+void ILCUTF8Marshaler::EmitConvertContentsNativeToCLR(ILCodeStream* pslILEmit)
+{
+	STANDARD_VM_CONTRACT;
+
+	EmitLoadNativeValue(pslILEmit);
+	pslILEmit->EmitCALL(METHOD__CUTF8MARSHALER__CONVERT_TO_MANAGED, 1, 1);
+	EmitStoreManagedValue(pslILEmit);
+}
+
 
 LocalDesc ILCSTRMarshaler::GetManagedType()
 {