From 0ae3d55d40be97727e64b20a6bec6758f9ef023e Mon Sep 17 00:00:00 2001 From: Mikhail Novosyolov Date: Tue, 25 Dec 2018 03:32:04 +0300 Subject: [PATCH] prevent exceeding 32 bit memory limitations with multithreaded xz compression Port of https://github.com/rpm-software-management/rpm/commit/a60f36a55cff3331e8bef3a1ab95c87d313911bb As 32 bit build suffers under the limitation of 32 bit address space, regardless of it's environment would be ie. 64 bit and not have this constration, rpm must make sure not to exceed this memory limitation. When using multithreaded xz compression, the number of threads used will increase the memory usage, making it necessary to check the memory required with the number of threads to be used. Number of compression threads will therefore be kept reduced untill amount of memory required won't exceed this limitation. For 32 bit binaries running under 64 bit host environment, where less available memory will be reserved for kernel, easing memory constraints, determination of this will be done by a combination of checking host arch as well as whether 32 bit personality flag is set, thereby still allow a sligthly greater memory usage for such cases to avoid imposing unnecessatry limitations under such environments. --- rpmio/xzdio.c | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/rpmio/xzdio.c b/rpmio/xzdio.c index 0e5e612..a1492f1 100644 --- a/rpmio/xzdio.c +++ b/rpmio/xzdio.c @@ -8,6 +8,10 @@ #include #include #include +#if defined(__linux__) +#include +#endif +#include #if defined(WITH_XZ) @@ -132,7 +136,45 @@ static XZFILE *xzopen_internal(const char *path, const char *mode, int fdno, int .timeout = 0, .preset = level, .filters = NULL, - .check = LZMA_CHECK_SHA256 }; + .check = LZMA_CHECK_CRC32 }; +#if __WORDSIZE == 32 + /* In 32 bit environment, required memory easily exceeds memory address + * space limit if compressing using multiple threads. + * By setting a memory limit, liblzma will automatically adjust number + * of threads to avoid exceeding memory. + */ + if (threads > 1) { + struct utsname u; + uint32_t memlimit = (SIZE_MAX>>1) + (SIZE_MAX>>3); + uint64_t memory_usage; + /* While a 32 bit linux kernel will have an address limit of 3GiB + * for processes (which is why set the memory limit to 2.5GiB as a safety + * margin), 64 bit kernels will have a limit of 4GiB for 32 bit binaries. + * Therefore the memory limit should be higher if running on a 64 bit + * kernel, so we increase it to 3,5GiB. + */ + uname(&u); + if (strstr(u.machine, "64") || strstr(u.machine, "s390x") +#if defined(__linux__) + || ((personality(0xffffffff) & PER_MASK) == PER_LINUX32) +#endif + ) + memlimit += (SIZE_MAX>>2); + + /* keep reducing the number of threads untill memory usage gets below limit */ + while ((memory_usage = lzma_stream_encoder_mt_memusage(&mt_options)) > memlimit) { + /* number of threads shouldn't be able to hit zero with compression + * settings aailable to set through rpm... */ + assert(--mt_options.threads != 0); + } + lzma_memlimit_set(&xzfile->strm, memlimit); + + if (threads != (int)mt_options.threads) + rpmlog(RPMLOG_NOTICE, + "XZ: Adjusted the number of threads from %d to %d to not exceed the memory usage limit of %lu bytes", + threads, mt_options.threads, memlimit); + } +#endif ret = lzma_stream_encoder_mt(&xzfile->strm, &mt_options); } -- 2.17.1