mirror of
https://abf.rosa.ru/djam/rpm.git
synced 2025-02-23 10:23:04 +00:00
93 lines
3.6 KiB
Diff
93 lines
3.6 KiB
Diff
From 0ae3d55d40be97727e64b20a6bec6758f9ef023e Mon Sep 17 00:00:00 2001
|
|
From: Mikhail Novosyolov <m.novosyolov@rosalinux.ru>
|
|
Date: Tue, 25 Dec 2018 03:32:04 +0300
|
|
Subject: [PATCH] prevent exceeding 32 bit memory limitations with
|
|
multithreaded xz compression
|
|
|
|
Port of https://github.com/rpm-software-management/rpm/commit/a60f36a55cff3331e8bef3a1ab95c87d313911bb
|
|
|
|
As 32 bit build suffers under the limitation of 32 bit address space,
|
|
regardless of it's environment would be ie. 64 bit and not have this
|
|
constration, rpm must make sure not to exceed this memory limitation.
|
|
When using multithreaded xz compression, the number of threads used will
|
|
increase the memory usage, making it necessary to check the memory
|
|
required with the number of threads to be used.
|
|
|
|
Number of compression threads will therefore be kept reduced untill
|
|
amount of memory required won't exceed this limitation.
|
|
|
|
For 32 bit binaries running under 64 bit host environment, where less
|
|
available memory will be reserved for kernel, easing memory constraints,
|
|
determination of this will be done by a combination of checking host
|
|
arch as well as whether 32 bit personality flag is set, thereby still
|
|
allow a sligthly greater memory usage for such cases to avoid
|
|
imposing unnecessatry limitations under such environments.
|
|
---
|
|
rpmio/xzdio.c | 44 +++++++++++++++++++++++++++++++++++++++++++-
|
|
1 file changed, 43 insertions(+), 1 deletion(-)
|
|
|
|
diff --git a/rpmio/xzdio.c b/rpmio/xzdio.c
|
|
index 0e5e612..a1492f1 100644
|
|
--- a/rpmio/xzdio.c
|
|
+++ b/rpmio/xzdio.c
|
|
@@ -8,6 +8,10 @@
|
|
#include <rpmmacro.h>
|
|
#include <rpmcb.h>
|
|
#include <ctype.h>
|
|
+#if defined(__linux__)
|
|
+#include <sys/personality.h>
|
|
+#endif
|
|
+#include <sys/utsname.h>
|
|
|
|
#if defined(WITH_XZ)
|
|
|
|
@@ -132,7 +136,45 @@ static XZFILE *xzopen_internal(const char *path, const char *mode, int fdno, int
|
|
.timeout = 0,
|
|
.preset = level,
|
|
.filters = NULL,
|
|
- .check = LZMA_CHECK_SHA256 };
|
|
+ .check = LZMA_CHECK_CRC32 };
|
|
+#if __WORDSIZE == 32
|
|
+ /* In 32 bit environment, required memory easily exceeds memory address
|
|
+ * space limit if compressing using multiple threads.
|
|
+ * By setting a memory limit, liblzma will automatically adjust number
|
|
+ * of threads to avoid exceeding memory.
|
|
+ */
|
|
+ if (threads > 1) {
|
|
+ struct utsname u;
|
|
+ uint32_t memlimit = (SIZE_MAX>>1) + (SIZE_MAX>>3);
|
|
+ uint64_t memory_usage;
|
|
+ /* While a 32 bit linux kernel will have an address limit of 3GiB
|
|
+ * for processes (which is why set the memory limit to 2.5GiB as a safety
|
|
+ * margin), 64 bit kernels will have a limit of 4GiB for 32 bit binaries.
|
|
+ * Therefore the memory limit should be higher if running on a 64 bit
|
|
+ * kernel, so we increase it to 3,5GiB.
|
|
+ */
|
|
+ uname(&u);
|
|
+ if (strstr(u.machine, "64") || strstr(u.machine, "s390x")
|
|
+#if defined(__linux__)
|
|
+ || ((personality(0xffffffff) & PER_MASK) == PER_LINUX32)
|
|
+#endif
|
|
+ )
|
|
+ memlimit += (SIZE_MAX>>2);
|
|
+
|
|
+ /* keep reducing the number of threads untill memory usage gets below limit */
|
|
+ while ((memory_usage = lzma_stream_encoder_mt_memusage(&mt_options)) > memlimit) {
|
|
+ /* number of threads shouldn't be able to hit zero with compression
|
|
+ * settings aailable to set through rpm... */
|
|
+ assert(--mt_options.threads != 0);
|
|
+ }
|
|
+ lzma_memlimit_set(&xzfile->strm, memlimit);
|
|
+
|
|
+ if (threads != (int)mt_options.threads)
|
|
+ rpmlog(RPMLOG_NOTICE,
|
|
+ "XZ: Adjusted the number of threads from %d to %d to not exceed the memory usage limit of %lu bytes",
|
|
+ threads, mt_options.threads, memlimit);
|
|
+ }
|
|
+#endif
|
|
|
|
ret = lzma_stream_encoder_mt(&xzfile->strm, &mt_options);
|
|
}
|
|
--
|
|
2.17.1
|
|
|