Greatly expand system() shell code features

The cosmopolitan command interpreter now has 13 builtin commands,
variable support, support for ; / && / || syntax, asynchronous support,
and plenty of unit tests with bug fixes.

This change fixes a bug in posix_spawn() with null envp arg. strace
logging now uses atomic writes for scatter functions. Breaking change
renaming GetCpuCount() to _getcpucount(). TurfWar is now updated to use
the new token bucket algorithm. WIN32 affinity masks now inherit across
fork() and execve().

libc/calls/execve-nt.greg.c

@@ -99,7 +99,7 @@ textwindows int sys_execve_nt(const char *program, char *const argv[],
   startinfo.hStdError = __getfdhandleactual(2);
 
   // spawn the process
-  rc = ntspawn(program, argv, envp, 0, 0, 0, 1, 0, 0, &startinfo, &procinfo);
+  rc = ntspawn(program, argv, envp, 0, 0, 0, true, 0, 0, &startinfo, &procinfo);
   if (rc == -1) {
     STRACE("panic: unrecoverable ntspawn(%#s) error: %m", program);
     __imp_ExitProcess(6543);

libc/calls/getcpucount.c

@@ -34,7 +34,7 @@
 #define HW_NCPUONLINE_NETBSD  16
 #define ALL_PROCESSOR_GROUPS  0xffff
 
-static unsigned GetCpuCountLinux(void) {
+static unsigned _getcpucount_linux(void) {
   cpu_set_t s = {0};
   if (sys_sched_getaffinity(0, sizeof(s), &s) != -1) {
     return CPU_COUNT(&s);
@@ -43,7 +43,7 @@ static unsigned GetCpuCountLinux(void) {
   }
 }
 
-static unsigned GetCpuCountBsd(void) {
+static unsigned _getcpucount_bsd(void) {
   size_t n;
   int c, cmd[2];
   n = sizeof(c);
@@ -62,12 +62,12 @@ static unsigned GetCpuCountBsd(void) {
   }
 }
 
-static unsigned GetCpuCountImpl(void) {
+static unsigned _getcpucount_impl(void) {
   if (!IsWindows()) {
     if (!IsBsd()) {
-      return GetCpuCountLinux();
+      return _getcpucount_linux();
     } else {
-      return GetCpuCountBsd();
+      return _getcpucount_bsd();
     }
   } else {
     return GetMaximumProcessorCount(ALL_PROCESSOR_GROUPS);
@@ -77,18 +77,24 @@ static unsigned GetCpuCountImpl(void) {
 static int g_cpucount;
 
 // precompute because process affinity on linux may change later
-__attribute__((__constructor__)) static void init(void) {
-  g_cpucount = GetCpuCountImpl();
+__attribute__((__constructor__)) static void _getcpucount_init(void) {
+  g_cpucount = _getcpucount_impl();
 }
 
 /**
  * Returns number of CPUs in system.
  *
+ * This is the same as the standard interface:
+ *
+ *     sysconf(_SC_NPROCESSORS_ONLN);
+ *
+ * Except this function isn't a bloated diamond dependency.
+ *
  * On Intel systems with HyperThreading this will return the number of
  * cores multiplied by two.
  *
  * @return cpu count or 0 if it couldn't be determined
  */
-unsigned GetCpuCount(void) {
+unsigned _getcpucount(void) {
   return g_cpucount;
 }

libc/calls/getloadavg-nt.c

@@ -20,10 +20,10 @@
 #include "libc/calls/syscall_support-nt.internal.h"
 #include "libc/dce.h"
 #include "libc/fmt/conv.h"
-#include "libc/thread/thread.h"
 #include "libc/macros.internal.h"
 #include "libc/nt/accounting.h"
-#include "libc/runtime/sysconf.h"
+#include "libc/runtime/runtime.h"
+#include "libc/thread/thread.h"
 
 #define FT(x) (x.dwLowDateTime | (uint64_t)x.dwHighDateTime << 32)
 
@@ -60,7 +60,7 @@ static textstartup void sys_getloadavg_nt_init(void) {
   double a[3];
   if (IsWindows()) {
     load = 1;
-    cpus = GetCpuCount() / 2;
+    cpus = _getcpucount() / 2;
     cpus = MAX(1, cpus);
     GetSystemTimes(&idle1, &kern1, &user1);
   }

libc/calls/ntspawn.c

@@ -16,10 +16,11 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/intrin/pushpop.h"
 #include "libc/calls/ntspawn.h"
-#include "libc/intrin/strace.internal.h"
 #include "libc/calls/syscall_support-nt.internal.h"
+#include "libc/intrin/kprintf.h"
+#include "libc/intrin/pushpop.h"
+#include "libc/intrin/strace.internal.h"
 #include "libc/macros.internal.h"
 #include "libc/nt/enum/filemapflags.h"
 #include "libc/nt/enum/pageflags.h"
@@ -87,7 +88,8 @@ textwindows int ntspawn(
       mkntenvblock(block->envvars, envp, extravar, block->buf) != -1 &&
       CreateProcess(prog16, block->cmdline, opt_lpProcessAttributes,
                     opt_lpThreadAttributes, bInheritHandles,
-                    dwCreationFlags | kNtCreateUnicodeEnvironment,
+                    dwCreationFlags | kNtCreateUnicodeEnvironment |
+                        kNtInheritParentAffinity,
                     block->envvars, opt_lpCurrentDirectory, lpStartupInfo,
                     opt_out_lpProcessInformation)) {
     rc = 0;

libc/calls/preadv.c

@@ -129,12 +129,7 @@ static ssize_t Preadv(int fd, struct iovec *iov, int iovlen, int64_t off) {
 ssize_t preadv(int fd, struct iovec *iov, int iovlen, int64_t off) {
   ssize_t rc;
   rc = Preadv(fd, iov, iovlen, off);
-#if defined(SYSDEBUG) && _DATATRACE
-  if (UNLIKELY(__strace > 0)) {
-    kprintf(STRACE_PROLOGUE "preadv(%d, [", fd);
-    DescribeIov(iov, iovlen, rc != -1 ? rc : 0);
-    kprintf("], %d, %'ld) → %'ld% m\n", iovlen, off, rc);
-  }
-#endif
+  STRACE("preadv(%d, [%s], %d, %'ld) → %'ld% m", fd,
+         DescribeIovec(rc, iov, iovlen), iovlen, off, rc);
   return rc;
 }

libc/calls/pwritev.c

@@ -135,12 +135,7 @@ static ssize_t Pwritev(int fd, const struct iovec *iov, int iovlen,
 ssize_t pwritev(int fd, const struct iovec *iov, int iovlen, int64_t off) {
   ssize_t rc;
   rc = Pwritev(fd, iov, iovlen, off);
-#if defined(SYSDEBUG) && _DATATRACE
-  if (UNLIKELY(__strace > 0)) {
-    kprintf(STRACE_PROLOGUE "pwritev(%d, ", fd);
-    DescribeIov(iov, iovlen, rc != -1 ? rc : 0);
-    kprintf(", %d, %'ld) → %'ld% m\n", iovlen, off, rc);
-  }
-#endif
+  STRACE("pwritev(%d, %s, %d, %'ld) → %'ld% m", fd,
+         DescribeIovec(rc != -1 ? rc : -2, iov, iovlen), iovlen, off, rc);
   return rc;
 }

libc/calls/readv.c

@@ -73,17 +73,7 @@ ssize_t readv(int fd, const struct iovec *iov, int iovlen) {
     rc = einval();
   }
 
-#if defined(SYSDEBUG) && _DATATRACE
-  if (UNLIKELY(__strace > 0)) {
-    if (rc == -1 && errno == EFAULT) {
-      STRACE("readv(%d, %p, %d) → %'zd% m", fd, iov, iovlen, rc);
-    } else {
-      kprintf(STRACE_PROLOGUE "readv(%d, [", fd);
-      DescribeIov(iov, iovlen, rc != -1 ? rc : 0);
-      kprintf("], %d) → %'ld% m\n", iovlen, rc);
-    }
-  }
-#endif
-
+  STRACE("readv(%d, [%s], %d) → %'ld% m", fd, DescribeIovec(rc, iov, iovlen),
+         iovlen, rc);
   return rc;
 }

libc/calls/sched_setaffinity.c

@@ -60,6 +60,8 @@ static dontinline textwindows int sys_sched_setaffinity_nt(
 /**
  * Asks kernel to only schedule process on particular CPUs.
  *
+ * Affinity masks are inherited across fork() and execve() boundaries.
+ *
  * @param pid is the process or process id (or 0 for caller)
  * @param size is bytes in bitset, which should be `sizeof(cpuset_t)`
  * @return 0 on success, or -1 w/ errno

libc/calls/sched_setparam.c

@@ -29,6 +29,7 @@
  *
  * @return 0 on success, or -1 w/ errno
  * @raise ENOSYS on XNU, Windows, OpenBSD
+ * @vforksafe
  */
 int sched_setparam(int pid, const struct sched_param *param) {
   int rc, policy;

libc/calls/sched_setscheduler.c

@@ -91,6 +91,7 @@
  * @raise EINVAL if `param` is NULL
  * @raise EINVAL if `policy` is invalid
  * @raise EINVAL if `param` has value out of ranges defined by `policy`
+ * @vforksafe
  */
 int sched_setscheduler(int pid, int policy, const struct sched_param *param) {
   int rc, old;

libc/calls/struct/iovec.h

@@ -13,7 +13,6 @@ ssize_t pwritev(int, const struct iovec *, int, int64_t);
 ssize_t readv(int, const struct iovec *, int);
 ssize_t vmsplice(int, const struct iovec *, int64_t, uint32_t);
 ssize_t writev(int, const struct iovec *, int);
-void DescribeIov(const struct iovec *, int, ssize_t);
 
 COSMOPOLITAN_C_END_
 #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

libc/calls/struct/iovec.internal.h

@@ -2,6 +2,7 @@
 #define COSMOPOLITAN_LIBC_CALLS_STRUCT_IOVEC_INTERNAL_H_
 #include "libc/calls/struct/fd.internal.h"
 #include "libc/calls/struct/iovec.h"
+#include "libc/mem/alloca.h"
 #if !(__ASSEMBLER__ + __LINKER__ + 0)
 COSMOPOLITAN_C_START_
 
@@ -24,6 +25,9 @@ ssize_t sys_send_nt(int, const struct iovec *, size_t, uint32_t) hidden;
 ssize_t sys_sendto_nt(int, const struct iovec *, size_t, uint32_t, void *,
                       uint32_t) hidden;
 
+const char *DescribeIovec(char[300], ssize_t, const struct iovec *, int);
+#define DescribeIovec(x, y, z) DescribeIovec(alloca(300), x, y, z)
+
 COSMOPOLITAN_C_END_
 #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
 #endif /* COSMOPOLITAN_LIBC_CALLS_STRUCT_IOVEC_INTERNAL_H_ */

libc/calls/syscall-nt.internal.h

@@ -16,7 +16,7 @@ int sys_fchmodat_nt(int, const char *, uint32_t, int) hidden;
 int sys_fcntl_nt(int, int, uintptr_t) hidden;
 int sys_fdatasync_nt(int) hidden;
 int sys_flock_nt(int, int) hidden;
-int sys_fork_nt(void) hidden;
+int sys_fork_nt(uint32_t) hidden;
 int sys_ftruncate_nt(int64_t, uint64_t) hidden;
 int sys_getloadavg_nt(double *, int) hidden;
 int sys_getppid_nt(void) hidden;

libc/calls/syscall_support-sysv.internal.h

@@ -24,6 +24,7 @@ void cosmo2flock(uintptr_t) hidden;
 void flock2cosmo(uintptr_t) hidden;
 int _ptsname(int, char *, size_t) hidden;
 int _isptmaster(int) hidden;
+int _fork(uint32_t) hidden;
 
 COSMOPOLITAN_C_END_
 #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */

libc/calls/writev.c

@@ -77,13 +77,7 @@ ssize_t writev(int fd, const struct iovec *iov, int iovlen) {
     rc = einval();
   }
 
-#if defined(SYSDEBUG) && _DATATRACE
-  if (UNLIKELY(__strace > 0)) {
-    kprintf(STRACE_PROLOGUE "writev(%d, ", fd);
-    DescribeIov(iov, iovlen, rc != -1 ? rc : 0);
-    kprintf(", %d) → %'ld% m\n", iovlen, rc);
-  }
-#endif
-
+  STRACE("writev(%d, %s, %d) → %'ld% m", fd,
+         DescribeIovec(rc != -1 ? rc : -2, iov, iovlen), iovlen, rc);
   return rc;
 }