From c5659b93f802661aca7ea6ccf1aa63878e0799e2 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Sat, 3 Sep 2022 18:01:38 -0700 Subject: [PATCH] Create unit test framework for subprocesses --- build/rules.mk | 48 ++++++++++++++++++++++--- libc/testlib/subprocess.h | 66 ++++++++++++++++++++++++++++++++++ libc/testlib/testlib.mk | 31 ++++++++-------- libc/testlib/testrunner.c | 2 +- libc/testlib/waitforexit.c | 65 +++++++++++++++++++++++++++++++++ libc/testlib/waitforterm.c | 65 +++++++++++++++++++++++++++++++++ test/libc/calls/openbsd_test.c | 38 +++++++++----------- test/libc/calls/pledge2_test.c | 23 +----------- test/libc/calls/test.mk | 3 ++ test/libc/calls/unveil_test.c | 15 +------- 10 files changed, 279 insertions(+), 77 deletions(-) create mode 100644 libc/testlib/subprocess.h create mode 100644 libc/testlib/waitforexit.c create mode 100644 libc/testlib/waitforterm.c diff --git a/build/rules.mk b/build/rules.mk index 62718a2fe..1fc646402 100644 --- a/build/rules.mk +++ b/build/rules.mk @@ -43,7 +43,6 @@ o/$(MODE)/%.initabi.o: %.initabi.c ; @$(COMPILE) -AOBJECTIFY.init $(OBJECTIFY.in o/$(MODE)/%.ncabi.o: %.ncabi.c ; @$(COMPILE) -AOBJECTIFY.nc $(OBJECTIFY.ncabi.c) $(OUTPUT_OPTION) $< o/$(MODE)/%.real.o: %.c ; @$(COMPILE) -AOBJECTIFY.real $(OBJECTIFY.real.c) $(OUTPUT_OPTION) $< -o/$(MODE)/%.runs: o/$(MODE)/% ; @$(COMPILE) -ACHECK -wtT$@ $< $(TESTARGS) o/$(MODE)/%-gcc.asm: %.c ; @$(COMPILE) -AOBJECTIFY.c $(OBJECTIFY.c) -S -g0 $(OUTPUT_OPTION) $< o/$(MODE)/%-gcc.asm: %.cc ; @$(COMPILE) -AOBJECTIFY.c $(OBJECTIFY.cxx) -S -g0 $(OUTPUT_OPTION) $< o/$(MODE)/%-clang.asm: %.c ; @$(COMPILE) -AOBJECTIFY.c $(OBJECTIFY.c) -S -g0 $(OUTPUT_OPTION) $< @@ -99,7 +98,48 @@ o/$(MODE)/%: o/$(MODE)/%.com o/$(MODE)/tool/build/cp.com o/$(MODE)/tool/build/as @$(COMPILE) -wAASSIMILATE -T$@ o/$(MODE)/tool/build/assimilate.com $@ ################################################################################ -# elf zip files +# LOCAL UNIT TESTS +# +# We always run unit tests as part of the normal `make` invocation. you +# may override the $(TESTARGS) variable to do things such as enable the +# benchmarking feature. For example: +# +# TESTARGS = -b +# +# May be specified in your ~/.cosmo.mk file. You can also use this to +# enable things like function tracing. For example: +# +# TESTARGS = --ftrace +# .PLEDGE += prot_exec +# +# You could then run a command like: +# +# make -j8 o//test/libc/calls/openbsd_test.com.runs +# +# You need PROT_EXEC permission since ftrace morphs the binary. It's +# also worth mentioning that the pledge.com command can simulate what +# Landlock Make does: +# +# o//tool/build/pledge.com \ +# -v. -p 'stdio rpath wpath cpath tty prot_exec' \ +# o//test/libc/calls/openbsd_test.com \ +# ----ftrace +# +# This is useful in the event a test binary should run by itself, but +# fails to run beneath Landlock Make. It's also useful sometimes to +# override the verbosity when running tests: +# +# make V=5 TESTARGS=-b o//test/libc/calls/openbsd_test.com.runs +# +# This way, if for some reason a test should fail but calls exit(0), +# then the stdout/stderr output, which would normally be suppressed, +# will actually be displayed. + +o/$(MODE)/%.runs: o/$(MODE)/% + @$(COMPILE) -ACHECK -wtT$@ $< $(TESTARGS) + +################################################################################ +# ELF ZIP FILES # # zipobj.com lets us do fast incremental linking of compressed data. # it's nice because if we link a hundred binaries that use the time zone @@ -123,7 +163,7 @@ o/$(MODE)%/.zip.o: % @$(COMPILE) -wAZIPOBJ $(ZIPOBJ) $(ZIPOBJ_FLAGS) $(OUTPUT_OPTION) $< ################################################################################ -# strict header checking +# STRICT HEADER CHECKING # # these rules are unsandboxed since they're already a sandboxing test, # and it would be too costly in terms of make latency to have every @@ -150,7 +190,7 @@ o/$(MODE)/%.okk: % @$(COMPILE) -ACHECK.h $(COMPILE.cxx) -xc++ -g0 -o $@ $< ################################################################################ -# executable helpers +# EXECUTABLE HELPERS MAKE_OBJCOPY = \ $(COMPILE) -AOBJCOPY -T$@ \ diff --git a/libc/testlib/subprocess.h b/libc/testlib/subprocess.h new file mode 100644 index 000000000..5e1611448 --- /dev/null +++ b/libc/testlib/subprocess.h @@ -0,0 +1,66 @@ +#ifndef COSMOPOLITAN_LIBC_TESTLIB_SUBPROCESS_H_ +#define COSMOPOLITAN_LIBC_TESTLIB_SUBPROCESS_H_ +#include "libc/calls/calls.h" +#include "libc/macros.internal.h" +#include "libc/runtime/runtime.h" +#include "libc/testlib/testlib.h" +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +/** + * @fileoverview Test Subprocess Helper Macros + * + * These macros may be used to run unit tests in a subprocess. It's + * particularly useful when testing code that causes the process to die: + * + * TEST(term, test) { + * SPAWN(fork); + * raise(SIGUSR1); + * TERMS(SIGUSR1); + * } + * + * The default action for both EXITS() and TERMS() is _Exit(0), unless + * an EXPECT_FOO() macro failed, in which case the status is non-zero. + * + * TEST(my, test) { + * SPAWN(fork); + * // do stuff + * EXITS(0); + * } + * + * You can also nest subprocesses: + * + * TEST(exit, test) { + * SPAWN(fork); + * SPAWN(vfork); + * _Exit(2); + * EXITS(2); + * _Exit(1); + * EXITS(1); + * } + */ + +#define SPAWN(METHOD) \ + { \ + int _pid, _failed = g_testlib_failed; \ + ASSERT_NE(-1, (_pid = METHOD())); \ + if (!_pid) { + +#define EXITS(rc) \ + _Exit(MAX(0, MIN(255, g_testlib_failed - _failed))); \ + } \ + testlib_waitforexit(__FILE__, __LINE__, #rc, rc, _pid); \ + } + +#define TERMS(sig) \ + _Exit(MAX(0, MIN(255, g_testlib_failed - _failed))); \ + } \ + testlib_waitforterm(__FILE__, __LINE__, #sig, sig, _pid); \ + } + +void testlib_waitforexit(const char *, int, const char *, int, int); +void testlib_waitforterm(const char *, int, const char *, int, int); + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_LIBC_TESTLIB_SUBPROCESS_H_ */ diff --git a/libc/testlib/testlib.mk b/libc/testlib/testlib.mk index e80b5c5e0..a4024ebc2 100644 --- a/libc/testlib/testlib.mk +++ b/libc/testlib/testlib.mk @@ -21,24 +21,24 @@ LIBC_TESTLIB_A_ASSETS = \ LIBC_TESTLIB_A_HDRS = \ libc/testlib/bench.h \ libc/testlib/blocktronics.h \ - libc/testlib/viewables.h \ libc/testlib/ezbench.h \ libc/testlib/fastrandomstring.h \ libc/testlib/hyperion.h \ libc/testlib/moby.h \ - libc/testlib/testlib.h + libc/testlib/subprocess.h \ + libc/testlib/testlib.h \ + libc/testlib/viewables.h LIBC_TESTLIB_A_SRCS_S = \ libc/testlib/bench.S \ libc/testlib/blocktronics.S \ - libc/testlib/viewables.S \ libc/testlib/combo.S \ libc/testlib/fixture.S \ libc/testlib/hyperion.S \ libc/testlib/moby.S \ + libc/testlib/polluteregisters.S \ libc/testlib/testcase.S \ libc/testlib/thrashcodecache.S \ - libc/testlib/polluteregisters.S \ libc/testlib/thunks/assert_eq.S \ libc/testlib/thunks/assert_false.S \ libc/testlib/thunks/assert_ne.S \ @@ -48,26 +48,20 @@ LIBC_TESTLIB_A_SRCS_S = \ libc/testlib/thunks/expect_ne.S \ libc/testlib/thunks/expect_true.S \ libc/testlib/thunks/free.S \ - libc/testlib/thunks/jump.S + libc/testlib/thunks/jump.S \ + libc/testlib/viewables.S LIBC_TESTLIB_A_SRCS_C = \ libc/testlib/almostequallongdouble.c \ libc/testlib/benchrunner.c \ - libc/testlib/getcore.c \ - libc/testlib/geterrno.c \ - libc/testlib/seterrno.c \ - libc/testlib/strerror.c \ - libc/testlib/getinterrupts.c \ - libc/testlib/ezbenchwarn.c \ libc/testlib/binequals.c \ - libc/testlib/quota.c \ libc/testlib/clearxmmregisters.c \ libc/testlib/comborunner.c \ libc/testlib/contains.c \ libc/testlib/endswith.c \ - libc/testlib/yield.c \ libc/testlib/ezbenchcontrol.c \ libc/testlib/ezbenchreport.c \ + libc/testlib/ezbenchwarn.c \ libc/testlib/fixturerunner.c \ libc/testlib/formatbinaryasglyphs.c \ libc/testlib/formatbinaryashex.c \ @@ -76,15 +70,24 @@ LIBC_TESTLIB_A_SRCS_C = \ libc/testlib/formatint.c \ libc/testlib/formatrange.c \ libc/testlib/formatstr.c \ + libc/testlib/getcore.c \ + libc/testlib/geterrno.c \ + libc/testlib/getinterrupts.c \ libc/testlib/globals.c \ libc/testlib/hexequals.c \ libc/testlib/incrementfailed.c \ + libc/testlib/quota.c \ + libc/testlib/seterrno.c \ libc/testlib/shoulddebugbreak.c \ libc/testlib/showerror.c \ libc/testlib/startswith.c \ libc/testlib/strcaseequals.c \ libc/testlib/strequals.c \ - libc/testlib/testrunner.c + libc/testlib/strerror.c \ + libc/testlib/testrunner.c \ + libc/testlib/waitforexit.c \ + libc/testlib/waitforterm.c \ + libc/testlib/yield.c LIBC_TESTLIB_A_SRCS = \ $(LIBC_TESTLIB_A_SRCS_S) \ diff --git a/libc/testlib/testrunner.c b/libc/testlib/testrunner.c index c75117db5..8a144a491 100644 --- a/libc/testlib/testrunner.c +++ b/libc/testlib/testrunner.c @@ -83,7 +83,7 @@ void testlib_error_leave(void) { wontreturn void testlib_abort(void) { testlib_finish(); __restorewintty(); - _Exit(MIN(255, g_testlib_failed)); + _Exit(MAX(1, MIN(255, g_testlib_failed))); } static void SetupTmpDir(void) { diff --git a/libc/testlib/waitforexit.c b/libc/testlib/waitforexit.c new file mode 100644 index 000000000..79b8beba8 --- /dev/null +++ b/libc/testlib/waitforexit.c @@ -0,0 +1,65 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" +#include "libc/intrin/kprintf.h" +#include "libc/runtime/runtime.h" +#include "libc/str/str.h" +#include "libc/testlib/testlib.h" + +STATIC_YOINK("strsignal"); + +void testlib_waitforexit(const char *file, int line, const char *code, int rc, + int pid) { + int ws; + char host[64]; + ASSERT_NE(-1, wait(&ws)); + if (WIFEXITED(ws)) { + if (WEXITSTATUS(ws) == rc) { + return; + } + kprintf("%s:%d: test failed\n" + "\tEXITS(%s)\n" + "\t want WEXITSTATUS(%d)\n" + "\t got WEXITSTATUS(%d)\n", + file, line, code, rc, WEXITSTATUS(ws)); + } else if (WIFSIGNALED(ws)) { + kprintf("%s:%d: test failed\n" + "\tEXITS(%s)\n" + "\t want _Exit(%d)\n" + "\t got WTERMSIG(%G)\n", + file, line, code, rc, WTERMSIG(ws)); + } else if (WIFSTOPPED(ws)) { + kprintf("%s:%d: test failed\n" + "\tEXITS(%s)\n" + "\t want _Exit(%d)\n" + "\t got WSTOPSIG(%G)\n", + file, line, code, rc, WSTOPSIG(ws)); + } else { + kprintf("%s:%d: test failed\n" + "\tEXITS(%s)\n" + "\t want _Exit(%d)\n" + "\t got ws=%#x\n", + file, line, code, rc, ws); + } + if (gethostname(host, sizeof(host))) { + strcpy(host, "unknown"); + } + kprintf("\t%s\n", host); + exit(1); +} diff --git a/libc/testlib/waitforterm.c b/libc/testlib/waitforterm.c new file mode 100644 index 000000000..cf3b0512c --- /dev/null +++ b/libc/testlib/waitforterm.c @@ -0,0 +1,65 @@ +/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│ +│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2022 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ +#include "libc/calls/calls.h" +#include "libc/intrin/kprintf.h" +#include "libc/runtime/runtime.h" +#include "libc/str/str.h" +#include "libc/testlib/testlib.h" + +STATIC_YOINK("strsignal"); + +void testlib_waitforterm(const char *file, int line, const char *code, int sig, + int pid) { + int ws; + char host[64]; + ASSERT_NE(-1, waitpid(pid, &ws, 0)); + if (WIFSIGNALED(ws)) { + if (WTERMSIG(ws) == sig) { + return; + } + kprintf("%s:%d: test failed\n" + "\tTERMS(%s)\n" + "\t want WTERMSIG(%G)\n" + "\t got WTERMSIG(%G)\n", + file, line, code, sig, WTERMSIG(ws)); + } else if (WIFEXITED(ws)) { + kprintf("%s:%d: test failed\n" + "\tTERMS(%s)\n" + "\t want WTERMSIG(%G)\n" + "\t got _Exit(%d)\n", + file, line, code, sig, WEXITSTATUS(ws)); + } else if (WIFSTOPPED(ws)) { + kprintf("%s:%d: test failed\n" + "\tTERMS(%s)\n" + "\t want WTERMSIG(%G)\n" + "\t got WSTOPSIG(%G)\n", + file, line, code, sig, WSTOPSIG(ws)); + } else { + kprintf("%s:%d: test failed\n" + "\tTERMS(%s)\n" + "\t want WTERMSIG(%G)\n" + "\t got ws=%#x\n", + file, line, code, sig, ws); + } + if (gethostname(host, sizeof(host))) { + strcpy(host, "unknown"); + } + kprintf("\t%s\n", host); + exit(1); +} diff --git a/test/libc/calls/openbsd_test.c b/test/libc/calls/openbsd_test.c index 981f82a6d..ba0e777eb 100644 --- a/test/libc/calls/openbsd_test.c +++ b/test/libc/calls/openbsd_test.c @@ -17,33 +17,27 @@ │ PERFORMANCE OF THIS SOFTWARE. │ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/calls/calls.h" +#include "libc/calls/pledge.h" +#include "libc/calls/syscall_support-sysv.internal.h" +#include "libc/dce.h" +#include "libc/intrin/kprintf.h" #include "libc/sysv/consts/o.h" +#include "libc/testlib/subprocess.h" #include "libc/testlib/testlib.h" -#define SPAWN(METHOD) \ - { \ - int ws, pid; \ - ASSERT_NE(-1, (pid = METHOD())); \ - if (!pid) { - -#define EXITS(rc) \ - _Exit(0); \ - } \ - ASSERT_NE(-1, wait(&ws)); \ - ASSERT_TRUE(WIFEXITED(ws)); \ - ASSERT_EQ(rc, WEXITSTATUS(ws)); \ - } - -#define TERMS(sig) \ - _Exit(0); \ - } \ - ASSERT_NE(-1, wait(&ws)); \ - ASSERT_TRUE(WIFSIGNALED(ws)); \ - ASSERT_EQ(sig, WTERMSIG(ws)); \ - } - char testlib_enable_tmp_setup_teardown; +void CheckPlatform(void) { + if (IsOpenbsd()) return; + if (__is_linux_2_6_23()) return; + kprintf("skipping openbsd_test\n"); + exit(0); +} + +void SetUp(void) { + CheckPlatform(); +} + TEST(pledge, promisedSyscallsCanBeCalled_ieEnosysIsIgnored) { SPAWN(fork); EXPECT_SYS(0, 0, pledge("stdio", 0)); diff --git a/test/libc/calls/pledge2_test.c b/test/libc/calls/pledge2_test.c index a5a1bfe79..b09bd1838 100644 --- a/test/libc/calls/pledge2_test.c +++ b/test/libc/calls/pledge2_test.c @@ -29,30 +29,9 @@ #include "libc/sysv/consts/ipproto.h" #include "libc/sysv/consts/sig.h" #include "libc/sysv/consts/sock.h" +#include "libc/testlib/subprocess.h" #include "libc/testlib/testlib.h" -#define SPAWN(METHOD) \ - { \ - int ws, pid; \ - ASSERT_NE(-1, (pid = METHOD())); \ - if (!pid) { - -#define EXITS(rc) \ - _Exit(0); \ - } \ - ASSERT_NE(-1, wait(&ws)); \ - ASSERT_TRUE(WIFEXITED(ws)); \ - ASSERT_EQ(rc, WEXITSTATUS(ws)); \ - } - -#define TERMS(sig) \ - _Exit(0); \ - } \ - ASSERT_NE(-1, wait(&ws)); \ - ASSERT_TRUE(WIFSIGNALED(ws)); \ - ASSERT_EQ(sig, WTERMSIG(ws)); \ - } - void SetUp(void) { if (!__is_linux_2_6_23() && !IsOpenbsd()) exit(0); } diff --git a/test/libc/calls/test.mk b/test/libc/calls/test.mk index 25b3e1846..16c467504 100644 --- a/test/libc/calls/test.mk +++ b/test/libc/calls/test.mk @@ -103,6 +103,9 @@ o/$(MODE)/test/libc/calls/poll_test.com.runs: \ o/$(MODE)/test/libc/calls/fcntl_test.com.runs: \ private .PLEDGE = stdio rpath wpath cpath fattr proc flock +o/$(MODE)/test/libc/calls/openbsd_test.com.runs: \ + private .PLEDGE = stdio rpath wpath cpath fattr proc unveil + # TODO(jart): Update nointernet() to allow AF_INET6 o/$(MODE)/test/libc/calls/pledge_test.com.runs: \ private .INTERNET = 1 diff --git a/test/libc/calls/unveil_test.c b/test/libc/calls/unveil_test.c index 3f5dc9ca4..44853111b 100644 --- a/test/libc/calls/unveil_test.c +++ b/test/libc/calls/unveil_test.c @@ -39,6 +39,7 @@ #include "libc/sysv/consts/s.h" #include "libc/sysv/consts/sig.h" #include "libc/sysv/consts/sock.h" +#include "libc/testlib/subprocess.h" #include "libc/testlib/testlib.h" #include "libc/thread/spawn.h" #include "libc/x/x.h" @@ -47,20 +48,6 @@ STATIC_YOINK("zip_uri_support"); #define EACCES_OR_ENOENT (IsOpenbsd() ? ENOENT : EACCES) -#define SPAWN(METHOD) \ - { \ - int ws, pid; \ - ASSERT_NE(-1, (pid = METHOD())); \ - if (!pid) { - -#define EXITS(rc) \ - _Exit(0); \ - } \ - ASSERT_NE(-1, wait(&ws)); \ - ASSERT_TRUE(WIFEXITED(ws)); \ - ASSERT_EQ(rc, WEXITSTATUS(ws)); \ - } - char testlib_enable_tmp_setup_teardown; struct stat st;