From 8ab3a545c6fb72263fd0b3c1aae508e59ad95d29 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Mon, 22 Jan 2024 07:13:49 -0800 Subject: [PATCH] Increase build memory quota If you install qemu-user from apt then glibc links a lot of address space bloat that causes pthread_create() to ENOMEM (a.k.a. EAGAIN). Boosting the virtual memory quota from 512m to 2048m will hopefully future proof the build for the future, as Linux distros get fatter. Please note this only applies to MODE=aarch64 on x86_64 builds when you're using QEMU from Debian/Ubuntu rather than installing the one cosmo provides in third_party/qemu/qemu-aarch64.gz. This change may also be useful to people who are using the host compiler toolchain. --- Makefile | 2 +- examples/BUILD.mk | 1 - libc/runtime/zipos-get.c | 2 +- libc/str/smoothsort.c | 1 - third_party/chibicc/test/BUILD.mk | 2 -- third_party/mbedtls/test/BUILD.mk | 2 -- third_party/python/BUILD.mk | 15 +++++---------- third_party/sqlite3/BUILD.mk | 8 +++----- tool/build/compile.c | 18 +++++++++--------- tool/cosmocc/package.sh | 6 +++--- 10 files changed, 22 insertions(+), 35 deletions(-) diff --git a/Makefile b/Makefile index ad265cd28..6e6453a4d 100644 --- a/Makefile +++ b/Makefile @@ -116,7 +116,7 @@ ZIPCOPY = build/bootstrap/zipcopy.com PECHECK = build/bootstrap/pecheck.com FIXUPOBJ = build/bootstrap/fixupobj.com MKDIR = build/bootstrap/mkdir.com -p -COMPILE = build/bootstrap/compile.com -V9 -P4096 $(QUOTA) +COMPILE = build/bootstrap/compile.com -V9 -M2048m -P8192 $(QUOTA) IGNORE := $(shell $(MKDIR) $(TMPDIR)) diff --git a/examples/BUILD.mk b/examples/BUILD.mk index ffe8625d2..e69b297b0 100644 --- a/examples/BUILD.mk +++ b/examples/BUILD.mk @@ -152,7 +152,6 @@ o/$(MODE)/examples/picol.com.dbg: \ $(APE_NO_MODIFY_SELF) @$(APELINK) -o/$(MODE)/examples/nesemu1.o: private QUOTA += -M512m o/$(MODE)/usr/share/dict/words.zip.o: private ZIPOBJ_FLAGS += -C2 $(EXAMPLES_OBJS): examples/BUILD.mk diff --git a/libc/runtime/zipos-get.c b/libc/runtime/zipos-get.c index c6d7184fa..b06e5f0e4 100644 --- a/libc/runtime/zipos-get.c +++ b/libc/runtime/zipos-get.c @@ -96,7 +96,7 @@ static void __zipos_generate_index(struct Zipos *zipos) { zipos->index[i] = c; } // smoothsort() isn't the fastest algorithm, but it guarantees - // o(logn), won't smash the stack and doesn't depend on malloc + // o(nlogn) won't smash the stack and doesn't depend on malloc smoothsort_r(zipos->index, zipos->records, sizeof(size_t), __zipos_compare_names, zipos); } diff --git a/libc/str/smoothsort.c b/libc/str/smoothsort.c index fc984f7b7..e9cf02b26 100644 --- a/libc/str/smoothsort.c +++ b/libc/str/smoothsort.c @@ -21,7 +21,6 @@ │ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS │ │ IN THE SOFTWARE. │ └─────────────────────────────────────────────────────────────────────────────*/ -#include "libc/assert.h" #include "libc/intrin/bsf.h" #include "libc/limits.h" #include "libc/mem/alg.h" diff --git a/third_party/chibicc/test/BUILD.mk b/third_party/chibicc/test/BUILD.mk index 3434e33ac..08f704c31 100644 --- a/third_party/chibicc/test/BUILD.mk +++ b/third_party/chibicc/test/BUILD.mk @@ -74,8 +74,6 @@ o/$(MODE)/third_party/chibicc/test/%.o: \ $(CHIBICC) @$(COMPILE) -wAOBJECTIFY.c $(CHIBICC) $(CHIBICC_FLAGS) $(OUTPUT_OPTION) -c $< -o/$(MODE)/third_party/chibicc/test/int128_test.o: private QUOTA = -M1024m - endif .PHONY: o/$(MODE)/third_party/chibicc/test diff --git a/third_party/mbedtls/test/BUILD.mk b/third_party/mbedtls/test/BUILD.mk index fff68ee1e..a892433bc 100644 --- a/third_party/mbedtls/test/BUILD.mk +++ b/third_party/mbedtls/test/BUILD.mk @@ -1358,8 +1358,6 @@ o/$(MODE)/third_party/mbedtls/test/secp384r1_test.com.dbg: \ $(APE_NO_MODIFY_SELF) @$(APELINK) -o/$(MODE)/third_party/mbedtls/test/test_suite_asn1parse.com.runs: private QUOTA = -M512m - # these need to be explictly defined because landlock make won't sandbox # prerequisites with a trailing slash. o/$(MODE)/third_party/mbedtls/test/data/.zip.o: \ diff --git a/third_party/python/BUILD.mk b/third_party/python/BUILD.mk index 479368dc7..33ef97a47 100644 --- a/third_party/python/BUILD.mk +++ b/third_party/python/BUILD.mk @@ -3900,8 +3900,8 @@ $(THIRD_PARTY_PYTHON_STAGE2_A_DATA_OBJS): private ZIPOBJ_FLAGS += -P.python -C3 $(THIRD_PARTY_PYTHON_PYTEST_A_PYS_OBJS): private PYFLAGS += -P.python -C3 $(THIRD_PARTY_PYTHON_PYTEST_A_DATA_OBJS): private ZIPOBJ_FLAGS += -P.python -C3 -o/$(MODE)/third_party/python/Python/ceval.o: private QUOTA = -C64 -M1024m -L300 -o/$(MODE)/third_party/python/Objects/unicodeobject.o: private QUOTA += -C64 -M1024m -L300 +o/$(MODE)/third_party/python/Python/ceval.o: private QUOTA = -C64 -L300 +o/$(MODE)/third_party/python/Objects/unicodeobject.o: private QUOTA += -C64 -L300 o/$(MODE)/third_party/python/Objects/unicodeobject.o: \ third_party/python/Objects/unicodeobject.c \ @@ -3944,18 +3944,13 @@ o/$(MODE)/third_party/python/Lib/test/pystone.o: private PYFLAGS += -m -O2 -P.py o/$(MODE)/third_party/python/Lib/test/test_long.py.runs: private QUOTA = -C64 -L180 o/$(MODE)/third_party/python/Lib/test/test_hash.py.runs: private QUOTA = -C64 o/$(MODE)/third_party/python/Lib/test/test_exceptions.py.runs: private QUOTA = -C64 -o/$(MODE)/third_party/python/Lib/test/test_tuple.py.runs: private QUOTA = -M512m -o/$(MODE)/third_party/python/Lib/test/test_decimal.py.runs: private QUOTA = -M512m -C64 -L300 -o/$(MODE)/third_party/python/Lib/test/test_longexp.py.runs: private QUOTA = -M1024m -o/$(MODE)/third_party/python/Lib/test/test_unicode.py.runs: private QUOTA = -M1400m -L300 +o/$(MODE)/third_party/python/Lib/test/test_decimal.py.runs: private QUOTA = -C64 -L300 +o/$(MODE)/third_party/python/Lib/test/test_unicode.py.runs: private QUOTA = -L300 o/$(MODE)/third_party/python/Lib/test/test_unicodedata.py.runs: private QUOTA = -C64 -L300 -o/$(MODE)/third_party/python/Lib/test/test_logging.py.runs: private QUOTA = -M512m -o/$(MODE)/third_party/python/Lib/test/test_itertools.py.runs: private QUOTA = -M1024m o/$(MODE)/third_party/python/Lib/test/test_tarfile.py.runs: private QUOTA = -L300 -C64 o/$(MODE)/third_party/python/Lib/test/test_sqlite.py.runs: private QUOTA = -L120 o/$(MODE)/third_party/python/Lib/test/test_gzip.py.runs: private QUOTA = -L120 -o/$(MODE)/third_party/python/Lib/test/test_logging.py.runs: private QUOTA = -M512m -o/$(MODE)/third_party/python/Lib/test/test_email/test_email.py.runs: private QUOTA = -C32 -M1024m +o/$(MODE)/third_party/python/Lib/test/test_email/test_email.py.runs: private QUOTA = -C32 o/$(MODE)/third_party/python/Lib/test/test_selectors.py.runs: private QUOTA = -L180 o/$(MODE)/third_party/python/Lib/test/test_trace.py.runs: private QUOTA = -L300 o/$(MODE)/third_party/python/Lib/test/test_multibytecodec.py.runs: private QUOTA = -C128 -L600 -L300 diff --git a/third_party/sqlite3/BUILD.mk b/third_party/sqlite3/BUILD.mk index 234ee0599..4177cdb19 100644 --- a/third_party/sqlite3/BUILD.mk +++ b/third_party/sqlite3/BUILD.mk @@ -187,11 +187,9 @@ o/$(MODE)/third_party/sqlite3/parse.o: private \ CFLAGS += \ -fpie -o/$(MODE)/third_party/sqlite3/shell.o: private QUOTA = -M512m -C32 -L180 -o/$(MODE)/third_party/sqlite3/vdbe.o: private QUOTA = -M1024m -o/$(MODE)/third_party/sqlite3/vdbe.shell.o: private QUOTA = -M1024m -o/$(MODE)/third_party/sqlite3/fts5.o: private QUOTA = -M512m -C32 -o/$(MODE)/third_party/sqlite3/fts5.shell.o: private QUOTA = -M512m -C32 -L180 +o/$(MODE)/third_party/sqlite3/shell.o: private QUOTA = -C32 -L180 +o/$(MODE)/third_party/sqlite3/fts5.o: private QUOTA = -C32 +o/$(MODE)/third_party/sqlite3/fts5.shell.o: private QUOTA = -C32 -L180 o/$(MODE)/third_party/sqlite3/rtree.o: \ third_party/sqlite3/rtree.c \ diff --git a/tool/build/compile.c b/tool/build/compile.c index a73baf964..aee8f9e30 100644 --- a/tool/build/compile.c +++ b/tool/build/compile.c @@ -32,7 +32,6 @@ #include "libc/fmt/itoa.h" #include "libc/fmt/libgen.h" #include "libc/fmt/magnumstrs.internal.h" -#include "libc/serialize.h" #include "libc/intrin/safemacros.internal.h" #include "libc/limits.h" #include "libc/log/appendresourcereport.internal.h" @@ -48,6 +47,7 @@ #include "libc/nexgen32e/x86info.h" #include "libc/proc/posix_spawn.h" #include "libc/runtime/runtime.h" +#include "libc/serialize.h" #include "libc/stdio/append.h" #include "libc/str/str.h" #include "libc/sysv/consts/auxv.h" @@ -112,9 +112,9 @@ FLAGS\n\ -V NUMBER specifies compiler version\n\ -C SECS set cpu limit [default 16]\n\ -L SECS set lat limit [default 90]\n\ - -P PROCS set pro limit [default 2048]\n\ + -P PROCS set pro limit [default 4096]\n\ -S BYTES set stk limit [default 8m]\n\ - -M BYTES set mem limit [default 512m]\n\ + -M BYTES set mem limit [default 2048m]\n\ -F BYTES set fsz limit [default 256m]\n\ -O BYTES set out limit [default 1m]\n\ -s decrement verbosity [default 4]\n\ @@ -845,12 +845,12 @@ int main(int argc, char *argv[]) { // parse prefix arguments verbose = 4; - timeout = 90; /* secs */ - cpuquota = 32; /* secs */ - proquota = 2048; /* procs */ - stkquota = 8 * 1024 * 1024; /* bytes */ - fszquota = 256 * 1000 * 1000; /* bytes */ - memquota = 512 * 1024 * 1024; /* bytes */ + timeout = 90; // secs + cpuquota = 32; // secs + proquota = 4096; // procs + stkquota = 8 * 1024 * 1024; // bytes + fszquota = 256 * 1000 * 1000; // bytes + memquota = 2048L * 1024 * 1024; // bytes if ((s = getenv("V"))) verbose = atoi(s); while ((opt = getopt(argc, argv, "hnstvwA:C:F:L:M:O:P:T:V:S:")) != -1) { switch (opt) { diff --git a/tool/cosmocc/package.sh b/tool/cosmocc/package.sh index 5b9ef866e..2864d0214 100755 --- a/tool/cosmocc/package.sh +++ b/tool/cosmocc/package.sh @@ -20,10 +20,10 @@ APELINK=o/$(mode)/tool/build/apelink.com AMD64=${2:-x86_64} ARM64=${3:-aarch64} -make -j8 m= \ +make -j32 m= \ $APELINK -make -j8 m=$AMD64 \ +make -j32 m=$AMD64 \ o/cosmocc.h.txt \ o/$AMD64/ape/ape.lds \ o/$AMD64/libc/crt/crt.o \ @@ -46,7 +46,7 @@ make -j8 m=$AMD64 \ o/$AMD64/third_party/make/make.com.dbg \ o/$AMD64/third_party/ctags/ctags.com.dbg -make -j8 m=$ARM64 \ +make -j32 m=$ARM64 \ o/$ARM64/ape/ape.elf \ o/$ARM64/ape/aarch64.lds \ o/$ARM64/libc/crt/crt.o \