Port of multithreaded XZ memory limit on 32 bit

rpm-5.4.10-multithreaded-xz-memlimit.patch

@@ -0,0 +1,93 @@
+From 0ae3d55d40be97727e64b20a6bec6758f9ef023e Mon Sep 17 00:00:00 2001
+From: Mikhail Novosyolov <m.novosyolov@rosalinux.ru>
+Date: Tue, 25 Dec 2018 03:32:04 +0300
+Subject: [PATCH] prevent exceeding 32 bit memory limitations with
+ multithreaded xz compression
+
+Port of https://github.com/rpm-software-management/rpm/commit/a60f36a55cff3331e8bef3a1ab95c87d313911bb
+
+As 32 bit build suffers under the limitation of 32 bit address space,
+regardless of it's environment would be ie. 64 bit and not have this
+constration, rpm must make sure not to exceed this memory limitation.
+When using multithreaded xz compression, the number of threads used will
+increase the memory usage, making it necessary to check the memory
+required with the number of threads to be used.
+
+Number of compression threads will therefore be kept reduced untill
+amount of memory required won't exceed this limitation.
+
+For 32 bit binaries running under 64 bit host environment, where less
+available memory will be reserved for kernel, easing memory constraints,
+determination of this will be done by a combination of checking host
+arch as well as whether 32 bit personality flag is set, thereby still
+allow a sligthly greater memory usage for such cases to avoid
+imposing unnecessatry limitations under such environments.
+---
+ rpmio/xzdio.c | 44 +++++++++++++++++++++++++++++++++++++++++++-
+ 1 file changed, 43 insertions(+), 1 deletion(-)
+
+diff --git a/rpmio/xzdio.c b/rpmio/xzdio.c
+index 0e5e612..a1492f1 100644
+--- a/rpmio/xzdio.c
++++ b/rpmio/xzdio.c
+@@ -8,6 +8,10 @@
+ #include <rpmmacro.h>
+ #include <rpmcb.h>
+ #include <ctype.h>
++#if defined(__linux__)
++#include <sys/personality.h>
++#endif
++#include <sys/utsname.h>
+ 
+ #if defined(WITH_XZ)
+ 
+@@ -132,7 +136,45 @@ static XZFILE *xzopen_internal(const char *path, const char *mode, int fdno, int
+ 	    	    .timeout = 0,
+ 	    	    .preset = level,
+ 	    	    .filters = NULL,
+-	    	    .check = LZMA_CHECK_SHA256 };
++	    	    .check = LZMA_CHECK_CRC32 };
++#if __WORDSIZE == 32
++		/* In 32 bit environment, required memory easily exceeds memory address
++		 * space limit if compressing using multiple threads.
++		 * By setting a memory limit, liblzma will automatically adjust number
++		 * of threads to avoid exceeding memory.
++		 */
++		if (threads > 1) {
++		    struct utsname u;
++		    uint32_t memlimit = (SIZE_MAX>>1) + (SIZE_MAX>>3);
++		    uint64_t memory_usage;
++		    /* While a 32 bit linux kernel will have an address limit of 3GiB
++		     * for processes (which is why set the memory limit to 2.5GiB as a safety
++		     * margin), 64 bit kernels will have a limit of 4GiB for 32 bit binaries.
++		     * Therefore the memory limit should be higher if running on a 64 bit
++		     * kernel, so we increase it to 3,5GiB.
++		     */
++		    uname(&u);
++		    if (strstr(u.machine, "64") || strstr(u.machine, "s390x")
++#if defined(__linux__)
++				    || ((personality(0xffffffff) & PER_MASK) == PER_LINUX32)
++#endif
++			    )
++			memlimit += (SIZE_MAX>>2);
++
++		    /* keep reducing the number of threads untill memory usage gets below limit */
++		    while ((memory_usage = lzma_stream_encoder_mt_memusage(&mt_options)) > memlimit) {
++			/* number of threads shouldn't be able to hit zero with compression
++			 * settings aailable to set through rpm... */
++			assert(--mt_options.threads != 0);
++		    }
++		    lzma_memlimit_set(&xzfile->strm, memlimit);
++
++		    if (threads != (int)mt_options.threads)
++			rpmlog(RPMLOG_NOTICE,
++				"XZ: Adjusted the number of threads from %d to %d to not exceed the memory usage limit of %lu bytes",
++				threads, mt_options.threads, memlimit);
++		}
++#endif
+ 
+ 	    	ret = lzma_stream_encoder_mt(&xzfile->strm, &mt_options);
+ 	    }
+-- 
+2.17.1
+

rpm.spec

@@ -531,6 +531,7 @@ Patch519:	rpm-5.4.10-debuginfo-too-many-notes.patch
 
 Patch520:	rpm-5.4.10-Multithreaded-XZ.patch
 Patch521:	rpm-5.4.10-Use-multithreaded-XZ-by-default-for-both-binary-and-.patch
+Patch522:	rpm-5.4.10-multithreaded-xz-memlimit.patch
 
 BuildRequires:	autoconf >= 2.57
 BuildRequires:	bzip2-devel
@@ -1198,6 +1199,7 @@ This package contains the RPM API documentation generated in HTML format.
 
 %patch520 -p1 -b .MultithreadedXZ
 %patch521 -p1 -b .MultithreadedXZbyDefault
+%patch522 -p1 -b .MultithreadedXZ32bitMemLimit
 
 #required by P55, P80, P81, P94..
 ./autogen.sh