prevent exceeding 32 bit memory limitations with multithreaded xz compression

As 32 bit build suffers under the limitation of 32 bit address space,
regardless of it's environment would be ie. 64 bit and not have this
constration, rpm must make sure not to exceed this memory limitation.
When using multithreaded xz compression, the number of threads used will
increase the memory usage, making it necessary to check the memory
required with the number of threads to be used.

Number of compression threads will therefore be kept reduced untill
amount of memory required won't exceed this limitation.

For 32 bit binaries running under 64 bit host environment, where less
available memory will be reserved for kernel, easing memory constraints,
determination of this will be done by a combination of checking host
arch as well as whether 32 bit personality flag is set, thereby still
allow a sligthly greater memory usage for such cases to avoid
imposing unnecessatry limitations under such environments.

Change-Id: Ie72827d3067a353a8a48faff9b65595c99f60a4c

1 files changed, 43 insertions, 0 deletions

diff --git a/rpmio/rpmio.c b/rpmio/rpmio.c
index 7add8536b..e8c55ff39 100644
--- a/rpmio/rpmio.c
+++ b/rpmio/rpmio.c
@@ -6,6 +6,10 @@
 #include <stdarg.h>
 #include <errno.h>
 #include <ctype.h>
+#if defined(__linux__)
+#include <sys/personality.h>
+#endif
+#include <sys/utsname.h>
 
 #include <rpm/rpmlog.h>
 #include <rpm/rpmmacro.h>
@@ -952,6 +956,45 @@ static LZFILE *lzopen_internal(const char *path, const char *mode, int fd, int x
 		    .filters = NULL,
 		    .check = LZMA_CHECK_SHA256 };
 
+#if __WORDSIZE == 32
+		/* In 32 bit environment, required memory easily exceeds memory address
+		 * space limit if compressing using multiple threads.
+		 * By setting a memory limit, liblzma will automatically adjust number
+		 * of threads to avoid exceeding memory.
+		 */
+		if (threads > 1) {
+		    struct utsname u;
+		    uint32_t memlimit = (SIZE_MAX>>1) + (SIZE_MAX>>3);
+		    uint64_t memory_usage;
+		    /* While a 32 bit linux kernel will have an address limit of 3GiB
+		     * for processes (which is why set the memory limit to 2.5GiB as a safety
+		     * margin), 64 bit kernels will have a limit of 4GiB for 32 bit binaries.
+		     * Therefore the memory limit should be higher if running on a 64 bit
+		     * kernel, so we increase it to 3,5GiB.
+		     */
+		    uname(&u);
+		    if (strstr(u.machine, "64") || strstr(u.machine, "s390x")
+#if defined(__linux__)
+				    || ((personality(0xffffffff) & PER_MASK) == PER_LINUX32)
+#endif
+			    )
+			memlimit += (SIZE_MAX>>2);
+
+		    /* keep reducing the number of threads untill memory usage gets below limit */
+		    while ((memory_usage = lzma_stream_encoder_mt_memusage(&mt_options)) > memlimit) {
+			/* number of threads shouldn't be able to hit zero with compression
+			 * settings aailable to set through rpm... */
+			assert(--mt_options.threads != 0);
+		    }
+		    lzma_memlimit_set(&lzfile->strm, memlimit);
+
+		    if (threads != (int)mt_options.threads)
+			rpmlog(RPMLOG_NOTICE,
+				"XZ: Adjusted the number of threads from %d to %d to not exceed the memory usage limit of %lu bytes",
+				threads, mt_options.threads, memlimit);
+		}
+#endif
+
 		ret = lzma_stream_encoder_mt(&lzfile->strm, &mt_options);
 	    }
 	} else {