[TTC-8][Sanitizers] Secondary allocator respects allocator_may_return_null=1.

Summary:
Context: https://github.com/google/sanitizers/issues/740.

Making secondary allocator to respect allocator_may_return_null=1 flag
and return nullptr when "out of memory" happens.

More changes in primary allocator and operator new will follow.

Reviewers: eugenis

Subscribers: kubamracek, llvm-commits

Differential Revision: https://reviews.llvm.org/D34243

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@305569 91177308-0d34-0410-b5e6-96231b3b80d8

Change-Id: I534117600099ec8953a6d83c559da0cf3bdfb5ad

4 files changed, 34 insertions, 2 deletions

diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h b/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h
index 91c2ecc5b26..60a3e03a16b 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h
@@ -34,9 +34,12 @@ class LargeMmapAllocator {
     if (alignment > page_size_)
       map_size += alignment;
     // Overflow.
-    if (map_size < size) return ReturnNullOrDieOnBadRequest();
+    if (map_size < size)
+      return ReturnNullOrDieOnBadRequest();
     uptr map_beg = reinterpret_cast<uptr>(
-        MmapOrDie(map_size, "LargeMmapAllocator"));
+        MmapOrDieOnFatalError(map_size, "LargeMmapAllocator"));
+    if (!map_beg)
+      return ReturnNullOrDieOnOOM();
     CHECK(IsAligned(map_beg, page_size_));
     MapUnmapCallback().OnMap(map_beg, map_size);
     uptr map_end = map_beg + map_size;
diff --git a/libsanitizer/sanitizer_common/sanitizer_common.h b/libsanitizer/sanitizer_common/sanitizer_common.h
index 3767a1cb368..ca91ddb7bcd 100644
--- a/libsanitizer/sanitizer_common/sanitizer_common.h
+++ b/libsanitizer/sanitizer_common/sanitizer_common.h
@@ -83,6 +83,9 @@ INLINE void *MmapOrDieQuietly(uptr size, const char *mem_type) {
   return MmapOrDie(size, mem_type, /*raw_report*/ true);
 }
 void UnmapOrDie(void *addr, uptr size);
+// Behaves just like MmapOrDie, but tolerates out of memory condition, in that
+// case returns nullptr.
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type);
 void *MmapFixedNoReserve(uptr fixed_addr, uptr size,
                          const char *name = nullptr);
 void *MmapNoReserveOrDie(uptr size, const char *mem_type);
diff --git a/libsanitizer/sanitizer_common/sanitizer_posix.cc b/libsanitizer/sanitizer_common/sanitizer_posix.cc
index eccbdf6b1fd..2a03ce2f4a0 100644
--- a/libsanitizer/sanitizer_common/sanitizer_posix.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_posix.cc
@@ -20,6 +20,7 @@
 #include "sanitizer_procmaps.h"
 #include "sanitizer_stacktrace.h"
 
+#include <errno.h>
 #include <fcntl.h>
 #include <signal.h>
 #include <sys/mman.h>
@@ -144,6 +145,21 @@ void UnmapOrDie(void *addr, uptr size) {
   DecreaseTotalMmap(size);
 }
 
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
+  size = RoundUpTo(size, GetPageSizeCached());
+  uptr res = internal_mmap(nullptr, size,
+                           PROT_READ | PROT_WRITE,
+                           MAP_PRIVATE | MAP_ANON, -1, 0);
+  int reserrno;
+  if (internal_iserror(res, &reserrno)) {
+    if (reserrno == ENOMEM)
+      return nullptr;
+    ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno);
+  }
+  IncreaseTotalMmap(size);
+  return (void *)res;
+}
+
 // We want to map a chunk of address space aligned to 'alignment'.
 // We do it by maping a bit more and then unmaping redundant pieces.
 // We probably can do it with fewer syscalls in some OS-dependent way.
diff --git a/libsanitizer/sanitizer_common/sanitizer_win.cc b/libsanitizer/sanitizer_common/sanitizer_win.cc
index 785883ce767..d16edbb70d5 100644
--- a/libsanitizer/sanitizer_common/sanitizer_win.cc
+++ b/libsanitizer/sanitizer_common/sanitizer_win.cc
@@ -113,6 +113,16 @@ void UnmapOrDie(void *addr, uptr size) {
   }
 }
 
+void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
+  void *rv = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
+  if (rv == 0) {
+    error_t last_error = GetLastError();
+    if (last_error != ERROR_NOT_ENOUGH_MEMORY)
+      ReportMmapFailureAndDie(size, mem_type, "allocate", last_error);
+  }
+  return rv;
+}
+
 // We want to map a chunk of address space aligned to 'alignment'.
 void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type) {
   CHECK(IsPowerOfTwo(size));