Fix for running in a container without mem limit set on Windows; (#23297)

also fixing the LocalGC standalone case on Linux

4 files changed, 45 insertions, 28 deletions

diff --git a/src/gc/unix/cgroup.cpp b/src/gc/unix/cgroup.cpp
index b66323a3b9..88eb415474 100644
--- a/src/gc/unix/cgroup.cpp
+++ b/src/gc/unix/cgroup.cpp
@@ -434,10 +434,19 @@ void CleanupCGroup()
 
 size_t GetRestrictedPhysicalMemoryLimit()
 {
-    size_t physical_memory_limit;
+    size_t physical_memory_limit = 0;
  
     if (!CGroup::GetPhysicalMemoryLimit(&physical_memory_limit))
-         physical_memory_limit = SIZE_T_MAX;
+         return 0;
+
+    // If there's no memory limit specified on the container this 
+    // actually returns 0x7FFFFFFFFFFFF000 (2^63-1 rounded down to 
+    // 4k which is a common page size). So we know we are not
+    // running in a memory restricted environment.
+    if (physical_memory_limit > 0x7FFFFFFF00000000)
+    {
+        return 0;
+    }
 
     struct rlimit curr_rlimit;
     size_t rlimit_soft_limit = (size_t)RLIM_INFINITY;
diff --git a/src/gc/windows/gcenv.windows.cpp b/src/gc/windows/gcenv.windows.cpp
index f032405c66..f7d069c874 100644
--- a/src/gc/windows/gcenv.windows.cpp
+++ b/src/gc/windows/gcenv.windows.cpp
@@ -286,16 +286,21 @@ static size_t GetRestrictedPhysicalMemoryLimit()
             if ((limit_info.BasicLimitInformation.LimitFlags & JOB_OBJECT_LIMIT_WORKINGSET) != 0)
                 job_workingset_limit = limit_info.BasicLimitInformation.MaximumWorkingSetSize;
 
-            job_physical_memory_limit = min (job_memory_limit, job_process_memory_limit);
-            job_physical_memory_limit = min (job_physical_memory_limit, job_workingset_limit);
+            if ((job_memory_limit != (size_t)UINTPTR_MAX) ||
+                (job_process_memory_limit != (size_t)UINTPTR_MAX) ||
+                (job_workingset_limit != (size_t)UINTPTR_MAX))
+            {
+                job_physical_memory_limit = min (job_memory_limit, job_process_memory_limit);
+                job_physical_memory_limit = min (job_physical_memory_limit, job_workingset_limit);
 
-            MEMORYSTATUSEX ms;
-            ::GetProcessMemoryLoad(&ms);
-            total_virtual = ms.ullTotalVirtual;
-            total_physical = ms.ullAvailPhys;
+                MEMORYSTATUSEX ms;
+                ::GetProcessMemoryLoad(&ms);
+                total_virtual = ms.ullTotalVirtual;
+                total_physical = ms.ullAvailPhys;
 
-            // A sanity check in case someone set a larger limit than there is actual physical memory.
-            job_physical_memory_limit = (size_t) min (job_physical_memory_limit, ms.ullTotalPhys);
+                // A sanity check in case someone set a larger limit than there is actual physical memory.
+                job_physical_memory_limit = (size_t) min (job_physical_memory_limit, ms.ullTotalPhys);
+            }
         }
     }
 
diff --git a/src/pal/src/misc/cgroup.cpp b/src/pal/src/misc/cgroup.cpp
index 304b813f28..144ac669df 100644
--- a/src/pal/src/misc/cgroup.cpp
+++ b/src/pal/src/misc/cgroup.cpp
@@ -386,20 +386,18 @@ size_t
 PALAPI
 PAL_GetRestrictedPhysicalMemoryLimit()
 {
-    size_t physical_memory_limit;
+    size_t physical_memory_limit = 0;
 
     if (!CGroup::GetPhysicalMemoryLimit(&physical_memory_limit))
          return 0;
-    else
+
+    // If there's no memory limit specified on the container this 
+    // actually returns 0x7FFFFFFFFFFFF000 (2^63-1 rounded down to 
+    // 4k which is a common page size). So we know we are not
+    // running in a memory restricted environment.
+    if (physical_memory_limit > 0x7FFFFFFF00000000)
     {
-        // If there's no memory limit specified on the container this 
-        // actually returns 0x7FFFFFFFFFFFF000 (2^63-1 rounded down to 
-        // 4k which is a common page size). So we know we are not
-        // running in a memory restricted environment.
-        if (physical_memory_limit > 0x7FFFFFFF00000000)
-        {
-            return 0;
-        }
+        return 0;
     }
 
     struct rlimit curr_rlimit;
diff --git a/src/vm/gcenv.os.cpp b/src/vm/gcenv.os.cpp
index 0e37931fe9..4d6a19c98d 100644
--- a/src/vm/gcenv.os.cpp
+++ b/src/vm/gcenv.os.cpp
@@ -494,16 +494,21 @@ static size_t GetRestrictedPhysicalMemoryLimit()
             if ((limit_info.BasicLimitInformation.LimitFlags & JOB_OBJECT_LIMIT_WORKINGSET) != 0)
                 job_workingset_limit = limit_info.BasicLimitInformation.MaximumWorkingSetSize;
 
-            job_physical_memory_limit = min (job_memory_limit, job_process_memory_limit);
-            job_physical_memory_limit = min (job_physical_memory_limit, job_workingset_limit);
+            if ((job_memory_limit != (size_t)MAX_PTR) ||
+                (job_process_memory_limit != (size_t)MAX_PTR) ||
+                (job_workingset_limit != (size_t)MAX_PTR))
+            {
+                job_physical_memory_limit = min (job_memory_limit, job_process_memory_limit);
+                job_physical_memory_limit = min (job_physical_memory_limit, job_workingset_limit);
 
-            MEMORYSTATUSEX ms;
-            ::GetProcessMemoryLoad(&ms);
-            total_virtual = ms.ullTotalVirtual;
-            total_physical = ms.ullAvailPhys;
+                MEMORYSTATUSEX ms;
+                ::GetProcessMemoryLoad(&ms);
+                total_virtual = ms.ullTotalVirtual;
+                total_physical = ms.ullAvailPhys;
 
-            // A sanity check in case someone set a larger limit than there is actual physical memory.
-            job_physical_memory_limit = (size_t) min (job_physical_memory_limit, ms.ullTotalPhys);
+                // A sanity check in case someone set a larger limit than there is actual physical memory.
+                job_physical_memory_limit = (size_t) min (job_physical_memory_limit, ms.ullTotalPhys);
+            }
         }
     }