[Tizen] Implement ASan wrapper for Linux AMD64

Change-Id: I48446ce7c8771a4c75149512bb7d8a8cb3fae8e5
Signed-off-by: Vyacheslav Cherkashin <v.cherkashin@samsung.com>

3 files changed, 157 insertions, 0 deletions

diff --git a/src/vm/CMakeLists.txt b/src/vm/CMakeLists.txt
index 3f37720742..c90c955003 100644
--- a/src/vm/CMakeLists.txt
+++ b/src/vm/CMakeLists.txt
@@ -740,6 +740,11 @@ else(WIN32)
             ${ARCH_SOURCES_DIR}/umthunkstub.S
             ${ARCH_SOURCES_DIR}/virtualcallstubamd64.S
         )
+        if (TIZEN_ASAN_ENVIRONMENT)
+            list(APPEND VM_SOURCES_WKS_ARCH_ASM
+                ${ARCH_SOURCES_DIR}/tizenasanenv.S
+            )
+        endif()
     elseif(CLR_CMAKE_TARGET_ARCH_I386)
         set(VM_SOURCES_WKS_ARCH_ASM
             ${ARCH_SOURCES_DIR}/ehhelpers.S
diff --git a/src/vm/amd64/cgenamd64.cpp b/src/vm/amd64/cgenamd64.cpp
index 2a93fa4413..d6fcf16d9c 100644
--- a/src/vm/amd64/cgenamd64.cpp
+++ b/src/vm/amd64/cgenamd64.cpp
@@ -26,6 +26,10 @@
 #include "clrtocomcall.h"
 #endif // FEATURE_COMINTEROP
 
+#ifdef TIZEN_ASAN_ENVIRONMENT
+#include <tizenasanenv.h>
+#endif // TIZEN_ASAN_ENVIRONMENT
+
 void UpdateRegDisplayFromCalleeSavedRegisters(REGDISPLAY * pRD, CalleeSavedRegisters * pRegs)
 {
     LIMITED_METHOD_CONTRACT;
@@ -559,6 +563,10 @@ void UMEntryThunkCode::Encode(BYTE* pTargetCode, void* pvSecretParam)
     }
     CONTRACTL_END;
 
+#ifdef TIZEN_ASAN_ENVIRONMENT
+    pTargetCode = (BYTE *)TizenASanEnv::CreateWrapperILCode((LPVOID)pTargetCode);
+#endif // TIZEN_ASAN_ENVIRONMENT
+
     // padding                  // CC CC CC CC
     // mov r10, pUMEntryThunk   // 49 ba xx xx xx xx xx xx xx xx    // METHODDESC_REGISTER
     // mov rax, pJmpDest        // 48 b8 xx xx xx xx xx xx xx xx    // need to ensure this imm64 is qword aligned
diff --git a/src/vm/amd64/tizenasanenv.S b/src/vm/amd64/tizenasanenv.S
new file mode 100644
index 0000000000..ee4728561e
--- /dev/null
+++ b/src/vm/amd64/tizenasanenv.S
@@ -0,0 +1,144 @@
+// Scheme of saving registers to the stack
+// +--------+----------------------+
+// |        |         value        |
+// | offset | size | register name |
+// +--------+------+---------------+
+// |  0x00  |    8 |           --- | (returnt addr)
+// | -0x08  |    8 |           rax |
+// | -0x10  |    8 |           r11 |
+// | -0x18  |    8 |           r10 |
+// | -0x20  |    8 |            r9 |
+// | -0x28  |    8 |            r8 |
+// | -0x30  |    8 |           rcx |
+// | -0x38  |    8 |           rdx |
+// | -0x40  |    8 |           rsi |
+// | -0x48  |    8 |           rdi |
+// | -0x50  |   16 |          xmm7 |
+// | -0x60  |   16 |          xmm6 |
+// | -0x70  |   16 |          xmm5 |
+// | -0x80  |   16 |          xmm4 |
+// | -0x90  |   16 |          xmm3 |
+// | -0xa0  |   16 |          xmm2 |
+// | -0xb0  |   16 |          xmm1 |
+// | -0xc0  |   16 |          xmm0 |
+
+
+#define GENERAL_SAVED_REGS_COUNT 9
+#define GENERAL_SAVED_REGS_SIZE (8 * GENERAL_SAVED_REGS_COUNT)
+.macro PUSH_GENERAL_REGS
+        push %rdi       // 1st argument
+        push %rsi       // 2nd argument
+        push %rdx       // 3rd argument, 2nd return register
+        push %rcx       // 4th argument
+        push %r8        // 5th argument
+        push %r9        // 6th argument
+        push %r10       // 1st return register
+        push %r11       // temporary register
+        push %rax       // temporary register
+.endm
+
+.macro POP_GENERAL_REGS
+        pop %rax
+        pop %r11
+        pop %r10
+        pop %r9
+        pop %r8
+        pop %rcx
+        pop %rdx
+        pop %rsi
+        pop %rdi
+.endm
+
+#define XMM_SAVED_REGS_COUNT 8
+#define XMM_SAVED_REGS_SIZE (16 * XMM_SAVED_REGS_COUNT)
+.macro PUSH_XMM_REGS
+        sub $XMM_SAVED_REGS_SIZE, %rsp
+        movaps %xmm0, 0x00(%rsp)
+        movaps %xmm1, 0x10(%rsp)
+        movaps %xmm2, 0x20(%rsp)
+        movaps %xmm3, 0x30(%rsp)
+        movaps %xmm4, 0x40(%rsp)
+        movaps %xmm5, 0x50(%rsp)
+        movaps %xmm6, 0x60(%rsp)
+        movaps %xmm7, 0x70(%rsp)
+.endm
+
+.macro POP_XMM_REGS
+        movaps 0x70(%rsp), %xmm7
+        movaps 0x60(%rsp), %xmm6
+        movaps 0x50(%rsp), %xmm5
+        movaps 0x40(%rsp), %xmm4
+        movaps 0x30(%rsp), %xmm3
+        movaps 0x20(%rsp), %xmm2
+        movaps 0x10(%rsp), %xmm1
+        movaps 0x00(%rsp), %xmm0
+        add $XMM_SAVED_REGS_SIZE, %rsp
+.endm
+
+#define RETADDR_OFFSET (GENERAL_SAVED_REGS_SIZE + XMM_SAVED_REGS_SIZE)
+.macro PUSH_REGS
+        PUSH_GENERAL_REGS
+        PUSH_XMM_REGS
+.endm
+
+.macro POP_REGS
+        POP_XMM_REGS
+        POP_GENERAL_REGS
+.endm
+
+
+// Export symbols
+.global tizenASanWrapper
+.global tizenASanWrapperSize
+.global tizenASanWrapperEntryOffset
+
+.text
+.code64
+
+tizenASanWrapper:
+// !!! ATTENTION !!!
+// Don't move this labels (target, pushAddr, popAddr)
+// because they mapped to AuxiliaryCalls struct from src/vm/tizenasanenv.cpp
+target:                 .quad 0xdeadbeef0badc0de
+pushAddr:               .quad 0xdeadbeef0badc0de  // void pushAddr(LPVOID addr)
+popAddr:                .quad 0xdeadbeef0badc0de  // LPVOID popAddr()
+
+
+entryPointer:
+        // Save context
+        PUSH_REGS
+
+        // Save the return address and call 'pre handler'
+        mov RETADDR_OFFSET(%rsp), %rdi  // rdi: get return address
+        call *pushAddr(%rip)            // save the return address
+
+        // Change the return address
+        call next
+next:
+        pop %rax                        // rax: get current rip
+        add $(postLabel - next), %rax   // rax: add offset to 'postLabel'
+        mov %rax, RETADDR_OFFSET(%rsp)  // change the return address
+
+        // Restore context
+        POP_REGS
+
+        // Call original function
+        jmp *target(%rip)
+postLabel:
+        sub $8, %rsp                    // add space for the return addr
+
+        // Save context
+        PUSH_REGS
+
+        // Get the return address and call 'post handler'
+        call *popAddr(%rip)                 // rax: get the return address
+        mov %rax, (RETADDR_OFFSET)(%rsp)    // restore the return address
+
+        // Restore context
+        POP_REGS
+
+        // Return
+        ret
+
+tizenASanWrapperSize:           .long . - tizenASanWrapper
+tizenASanWrapperEntryOffset:    .long entryPointer - tizenASanWrapper