ASan: add AMD64 supportsandbox/ches01/asan_amd64_v2

Change-Id: I48446ce7c8771a4c75149512bb7d8a8cb3fae8e5
Signed-off-by: Vyacheslav Cherkashin <v.cherkashin@samsung.com>

5 files changed, 261 insertions, 0 deletions

diff --git a/src/vm/CMakeLists.txt b/src/vm/CMakeLists.txt
index e391e1d692..c062139530 100644
--- a/src/vm/CMakeLists.txt
+++ b/src/vm/CMakeLists.txt
@@ -719,6 +719,17 @@ else(WIN32)
             ${ARCH_SOURCES_DIR}/umthunkstub.S
             ${ARCH_SOURCES_DIR}/virtualcallstubamd64.S
         )
+        if (TIZEN_ASAN_ENVIRONMENT)
+            list(APPEND VM_SOURCES_WKS
+                tizenasanenv.cpp
+            )
+            list(APPEND VM_HEADERS_WKS
+                tizenasanenv.h
+            )
+            list(APPEND VM_SOURCES_WKS_ARCH_ASM
+                ${ARCH_SOURCES_DIR}/tizenasanenv.S
+            )
+        endif()
     elseif(CLR_CMAKE_TARGET_ARCH_I386)
         set(VM_SOURCES_WKS_ARCH_ASM
             ${ARCH_SOURCES_DIR}/ehhelpers.S
diff --git a/src/vm/amd64/cgenamd64.cpp b/src/vm/amd64/cgenamd64.cpp
index 814f04dd9f..7d09871baa 100644
--- a/src/vm/amd64/cgenamd64.cpp
+++ b/src/vm/amd64/cgenamd64.cpp
@@ -26,6 +26,10 @@
 #include "clrtocomcall.h"
 #endif // FEATURE_COMINTEROP
 
+#ifdef TIZEN_ASAN_ENVIRONMENT
+#include <tizenasanenv.h>
+#endif // TIZEN_ASAN_ENVIRONMENT
+
 void UpdateRegDisplayFromCalleeSavedRegisters(REGDISPLAY * pRD, CalleeSavedRegisters * pRegs)
 {
     LIMITED_METHOD_CONTRACT;
@@ -563,6 +567,12 @@ void UMEntryThunkCode::Encode(BYTE* pTargetCode, void* pvSecretParam)
     }
     CONTRACTL_END;
 
+#ifdef TIZEN_ASAN_ENVIRONMENT
+    pTargetCode = (BYTE *)TizenASanEnv::CreateWrapperJmp((LPVOID)pTargetCode,
+                                                         TizenASanEnv::PushAndDisableASan,
+                                                         TizenASanEnv::PopAndEnableASan);
+#endif // TIZEN_ASAN_ENVIRONMENT
+
     // padding                  // CC CC CC CC
     // mov r10, pUMEntryThunk   // 49 ba xx xx xx xx xx xx xx xx    // METHODDESC_REGISTER
     // mov rax, pJmpDest        // 48 b8 xx xx xx xx xx xx xx xx    // need to ensure this imm64 is qword aligned
diff --git a/src/vm/amd64/tizenasanenv.S b/src/vm/amd64/tizenasanenv.S
new file mode 100644
index 0000000000..f0c22c4509
--- /dev/null
+++ b/src/vm/amd64/tizenasanenv.S
@@ -0,0 +1,201 @@
+//
+// Scheme of saving registers to the stack
+// +--------+-----------------+
+// |        |      value      |
+// | offset | size |     name |
+// +--------+------+----------+
+// |  0x00  |    8 | ret addr |
+// | -0x08  |    8 |      rax |
+// | -0x10  |    8 |      r11 |
+// | -0x18  |    8 |      r10 |
+// | -0x20  |    8 |       r9 |
+// | -0x28  |    8 |       r8 |
+// | -0x30  |    8 |      rcx |
+// | -0x38  |    8 |      rdx |
+// | -0x40  |    8 |      rsi |
+// | -0x48  |    8 |      rdi |
+// | -0x50  |   16 |     xmm7 |
+// | -0x60  |   16 |     xmm6 |
+// | -0x70  |   16 |     xmm5 |
+// | -0x80  |   16 |     xmm4 |
+// | -0x90  |   16 |     xmm3 |
+// | -0xa0  |   16 |     xmm2 |
+// | -0xb0  |   16 |     xmm1 |
+// | -0xc0  |   16 |     xmm0 |
+
+
+#define GENERAL_SAVED_REGS_COUNT 9
+#define XMM_SAVED_REGS_COUNT 8
+#define GENERAL_SAVED_REGS_SIZE (8 * GENERAL_SAVED_REGS_COUNT)
+#define XMM_SAVED_REGS_SIZE (16 * XMM_SAVED_REGS_COUNT)
+#define RETADDR_OFFSET (GENERAL_SAVED_REGS_SIZE + XMM_SAVED_REGS_SIZE)
+
+
+.macro PUSH_GENERAL_REGS
+        push %rdi       # 0, arg0
+        push %rsi       # 1, arg1
+        push %rdx       # 2, arg2
+        push %rcx       # 3, arg3
+        push %r8        # 4, arg4
+        push %r9        # 5, arg5
+        push %r10       # 6, ...
+        push %r11       # 7, temprory reg
+        push %rax       # 8
+.endm
+
+.macro POP_GENERAL_REGS
+        pop %rax
+        pop %r11
+        pop %r10
+        pop %r9
+        pop %r8
+        pop %rcx
+        pop %rdx
+        pop %rsi
+        pop %rdi
+.endm
+
+.macro PUSH_XMM_REGS
+        sub $XMM_SAVED_REGS_SIZE, %rsp
+        movaps %xmm0, 0x00(%rsp)
+        movaps %xmm1, 0x10(%rsp)
+        movaps %xmm2, 0x20(%rsp)
+        movaps %xmm3, 0x30(%rsp)
+        movaps %xmm4, 0x40(%rsp)
+        movaps %xmm5, 0x50(%rsp)
+        movaps %xmm6, 0x60(%rsp)
+        movaps %xmm7, 0x70(%rsp)
+.endm
+
+.macro POP_XMM_REGS
+        movaps 0x70(%rsp), %xmm7
+        movaps 0x60(%rsp), %xmm6
+        movaps 0x50(%rsp), %xmm5
+        movaps 0x40(%rsp), %xmm4
+        movaps 0x30(%rsp), %xmm3
+        movaps 0x20(%rsp), %xmm2
+        movaps 0x10(%rsp), %xmm1
+        movaps 0x00(%rsp), %xmm0
+        add $XMM_SAVED_REGS_SIZE, %rsp
+.endm
+
+.macro PUSH_REGS offset=0
+        sub $\offset, %rsp
+        PUSH_GENERAL_REGS
+        PUSH_XMM_REGS
+.endm
+
+.macro POP_REGS offset=0
+        POP_XMM_REGS
+        POP_GENERAL_REGS
+        add $\offset, %rsp
+.endm
+
+
+// Export symbols
+.global tizenASanWrapper
+.global tizenASanWrapperSize
+.global tizenASanWrapperEntryOffset
+
+.text
+.code64
+
+tizenASanWrapper:
+// !!! ATTENTION !!!
+// Don't move this labels (target, pushAddr, popAddr)
+// because they mapped to AuxiliaryCalls struct from src/vm/tizenasanenv.cpp
+target:                 .quad 0xdeadbeef0badc0de
+pushAddr:               .quad 0xdeadbeef0badc0de        // void pushAddr(LPVOID addr)
+popAddr:                .quad 0xdeadbeef0badc0de        // LPVOID popAddr()
+
+
+entryPointer:
+        // Save context
+        PUSH_REGS
+
+        // Save the return address and call 'pre handler'
+        mov RETADDR_OFFSET(%rsp), %rdi	// rdi: get return address
+        call *pushAddr(%rip)            // save the return address
+
+        // Change the return address
+        call next
+next:
+        pop %rax                        // rax: get current rip
+        add $(postLabel - next), %rax   // rax: add offset to 'postLabel'
+        mov %rax, RETADDR_OFFSET(%rsp)  // change the return address
+
+        // Restore context
+        POP_REGS
+
+        // Call original function
+        jmp *target(%rip)
+postLabel:
+        // Save context
+        PUSH_REGS 8
+
+        // Get the return address and call 'post handler'
+        call *popAddr(%rip)                     // rax: get the return address
+        mov %rax, (RETADDR_OFFSET + 8)(%rsp)    // restore the return address
+
+        // Restore context
+        POP_REGS 8
+
+        // Return
+        ret
+
+tizenASanWrapperSize:           .long . - tizenASanWrapper
+tizenASanWrapperEntryOffset:    .long entryPointer - tizenASanWrapper
+
+
+// Export symbols
+.global tizenASanWrapperJmp
+.global tizenASanWrapperJmpSize
+.global tizenASanWrapperJmpEntryOffset
+
+tizenASanWrapperJmp:
+// !!! ATTENTION !!!
+// Don't move this labels (target, pushAddr, popAddr)
+// because they mapped to AuxiliaryCalls struct from src/vm/tizenasanenv.cpp
+targetJmp:              .quad 0xdeadbeef0badc0de
+pushAddrJmp:            .quad 0xdeadbeef0badc0de        // void pushAddr(LPVOID addr)
+popAddrJmp:             .quad 0xdeadbeef0badc0de        // LPVOID popAddr()
+
+
+entryPointerJmp:
+        // Save context
+        PUSH_REGS
+
+        // Save the return address and call 'pre handler'
+        mov RETADDR_OFFSET(%rsp), %rdi  // rdi: get return address
+        call *pushAddrJmp(%rip)         // save the return address
+
+        // Change the return address
+        call nextJmp
+nextJmp:
+        pop %rax                            // rax: get current rip
+        add $(postLabelJmp - nextJmp), %rax // rax: add offset to 'postLabelJmp'
+        mov %rax, RETADDR_OFFSET(%rsp)      // change the return address
+
+        // Restore context
+        POP_REGS
+
+        // Call original function
+        jmp *targetJmp(%rip)
+postLabelJmp:
+        sub $8, %rsp                      // add space for the return addr
+
+        // Save context
+        PUSH_REGS
+
+        // Get the return address and call 'post handler'
+        call *popAddrJmp(%rip)             // rax: get the return address
+        mov %rax, RETADDR_OFFSET(%rsp)     // restore the return address
+
+        // Restore context
+        POP_REGS
+
+        // Return
+        ret
+
+tizenASanWrapperJmpSize:        .long . - tizenASanWrapperJmp
+tizenASanWrapperJmpEntryOffset: .long entryPointerJmp - tizenASanWrapperJmp
diff --git a/src/vm/tizenasanenv.cpp b/src/vm/tizenasanenv.cpp
index afb78aeaec..9c34c67174 100644
--- a/src/vm/tizenasanenv.cpp
+++ b/src/vm/tizenasanenv.cpp
@@ -54,6 +54,12 @@ extern LPVOID tizenASanWrapper;
 extern UINT32 tizenASanWrapperSize;
 extern UINT32 tizenASanWrapperEntryOffset;
 
+#ifdef _AMD64_
+extern LPVOID tizenASanWrapperJmp;
+extern UINT32 tizenASanWrapperJmpSize;
+extern UINT32 tizenASanWrapperJmpEntryOffset;
+#endif // _AMD64_
+
 static __thread StaticStack<LPVOID, 128> s_retaddrStack;
 static __thread int s_enableCounter;
 
@@ -170,4 +176,33 @@ LPVOID CreateWrapper(LPVOID target, void (*pushAddr)(LPVOID addr), LPVOID (*popA
     dprintf(1, "#### CreateWrapper E\n");
     return entryPointer;
 }
+
+#ifdef _AMD64_
+LPVOID CreateWrapperJmp(LPVOID target, void (*pushAddr)(LPVOID addr), LPVOID (*popAddr)())
+{
+    _ASSERTE(tizenASanWrapperJmpEntryOffset == sizeof(AuxiliaryCalls));
+
+    LPVOID wrapperSpace = (LPVOID)SystemDomain::GetGlobalLoaderAllocator()->GetExecutableHeap()->AllocMem(S_SIZE_T(tizenASanWrapperJmpSize));
+
+    AuxiliaryCalls calls = {
+        .target = target,
+	.pushAddr = pushAddr,
+	.popAddr = popAddr,
+    };
+
+    // copy auxiliary calls
+    memcpy(wrapperSpace, &calls, sizeof(calls));
+
+    LPVOID entryPointer = (LPVOID)((UINT_PTR)wrapperSpace + tizenASanWrapperJmpEntryOffset);
+    LPVOID wrapperEntryPointer = (LPVOID)((UINT_PTR)&tizenASanWrapperJmp + tizenASanWrapperJmpEntryOffset);
+    UINT32 wrapperCodeSize = tizenASanWrapperJmpSize - tizenASanWrapperJmpEntryOffset;
+
+    // copy executable code wrapper
+    memcpy(entryPointer, wrapperEntryPointer, wrapperCodeSize);
+
+    FlushInstructionCache(GetCurrentProcess(), wrapperSpace, tizenASanWrapperJmpSize);
+
+    return entryPointer;
+}
+#endif // _AMD64_
 } // namespace TizenASanEnv
diff --git a/src/vm/tizenasanenv.h b/src/vm/tizenasanenv.h
index 7200936d16..1a4412c991 100644
--- a/src/vm/tizenasanenv.h
+++ b/src/vm/tizenasanenv.h
@@ -10,6 +10,10 @@ LPVOID PopAndDisableASan();
 
 LPVOID CreateWrapper(LPVOID target, void (*pushAddr)(LPVOID addr), LPVOID (*popAddr)());
 
+#ifdef _AMD64_
+LPVOID CreateWrapperJmp(LPVOID target, void (*pushAddr)(LPVOID addr), LPVOID (*popAddr)());
+#endif // _AMD64_
+
 } // namespace TizenASanEnv
 
 #endif // TIZENASANENV_H_