diff --git a/rpm-5.4.10-multithreaded-xz-memlimit.patch b/rpm-5.4.10-multithreaded-xz-memlimit.patch new file mode 100644 index 0000000..f2bd5d2 --- /dev/null +++ b/rpm-5.4.10-multithreaded-xz-memlimit.patch @@ -0,0 +1,93 @@ +From 0ae3d55d40be97727e64b20a6bec6758f9ef023e Mon Sep 17 00:00:00 2001 +From: Mikhail Novosyolov +Date: Tue, 25 Dec 2018 03:32:04 +0300 +Subject: [PATCH] prevent exceeding 32 bit memory limitations with + multithreaded xz compression + +Port of https://github.com/rpm-software-management/rpm/commit/a60f36a55cff3331e8bef3a1ab95c87d313911bb + +As 32 bit build suffers under the limitation of 32 bit address space, +regardless of it's environment would be ie. 64 bit and not have this +constration, rpm must make sure not to exceed this memory limitation. +When using multithreaded xz compression, the number of threads used will +increase the memory usage, making it necessary to check the memory +required with the number of threads to be used. + +Number of compression threads will therefore be kept reduced untill +amount of memory required won't exceed this limitation. + +For 32 bit binaries running under 64 bit host environment, where less +available memory will be reserved for kernel, easing memory constraints, +determination of this will be done by a combination of checking host +arch as well as whether 32 bit personality flag is set, thereby still +allow a sligthly greater memory usage for such cases to avoid +imposing unnecessatry limitations under such environments. +--- + rpmio/xzdio.c | 44 +++++++++++++++++++++++++++++++++++++++++++- + 1 file changed, 43 insertions(+), 1 deletion(-) + +diff --git a/rpmio/xzdio.c b/rpmio/xzdio.c +index 0e5e612..a1492f1 100644 +--- a/rpmio/xzdio.c ++++ b/rpmio/xzdio.c +@@ -8,6 +8,10 @@ + #include + #include + #include ++#if defined(__linux__) ++#include ++#endif ++#include + + #if defined(WITH_XZ) + +@@ -132,7 +136,45 @@ static XZFILE *xzopen_internal(const char *path, const char *mode, int fdno, int + .timeout = 0, + .preset = level, + .filters = NULL, +- .check = LZMA_CHECK_SHA256 }; ++ .check = LZMA_CHECK_CRC32 }; ++#if __WORDSIZE == 32 ++ /* In 32 bit environment, required memory easily exceeds memory address ++ * space limit if compressing using multiple threads. ++ * By setting a memory limit, liblzma will automatically adjust number ++ * of threads to avoid exceeding memory. ++ */ ++ if (threads > 1) { ++ struct utsname u; ++ uint32_t memlimit = (SIZE_MAX>>1) + (SIZE_MAX>>3); ++ uint64_t memory_usage; ++ /* While a 32 bit linux kernel will have an address limit of 3GiB ++ * for processes (which is why set the memory limit to 2.5GiB as a safety ++ * margin), 64 bit kernels will have a limit of 4GiB for 32 bit binaries. ++ * Therefore the memory limit should be higher if running on a 64 bit ++ * kernel, so we increase it to 3,5GiB. ++ */ ++ uname(&u); ++ if (strstr(u.machine, "64") || strstr(u.machine, "s390x") ++#if defined(__linux__) ++ || ((personality(0xffffffff) & PER_MASK) == PER_LINUX32) ++#endif ++ ) ++ memlimit += (SIZE_MAX>>2); ++ ++ /* keep reducing the number of threads untill memory usage gets below limit */ ++ while ((memory_usage = lzma_stream_encoder_mt_memusage(&mt_options)) > memlimit) { ++ /* number of threads shouldn't be able to hit zero with compression ++ * settings aailable to set through rpm... */ ++ assert(--mt_options.threads != 0); ++ } ++ lzma_memlimit_set(&xzfile->strm, memlimit); ++ ++ if (threads != (int)mt_options.threads) ++ rpmlog(RPMLOG_NOTICE, ++ "XZ: Adjusted the number of threads from %d to %d to not exceed the memory usage limit of %lu bytes", ++ threads, mt_options.threads, memlimit); ++ } ++#endif + + ret = lzma_stream_encoder_mt(&xzfile->strm, &mt_options); + } +-- +2.17.1 + diff --git a/rpm.spec b/rpm.spec index 00f996e..767edab 100644 --- a/rpm.spec +++ b/rpm.spec @@ -531,6 +531,7 @@ Patch519: rpm-5.4.10-debuginfo-too-many-notes.patch Patch520: rpm-5.4.10-Multithreaded-XZ.patch Patch521: rpm-5.4.10-Use-multithreaded-XZ-by-default-for-both-binary-and-.patch +Patch522: rpm-5.4.10-multithreaded-xz-memlimit.patch BuildRequires: autoconf >= 2.57 BuildRequires: bzip2-devel @@ -1198,6 +1199,7 @@ This package contains the RPM API documentation generated in HTML format. %patch520 -p1 -b .MultithreadedXZ %patch521 -p1 -b .MultithreadedXZbyDefault +%patch522 -p1 -b .MultithreadedXZ32bitMemLimit #required by P55, P80, P81, P94.. ./autogen.sh