vista: backport execve escaping and using cocmd as shell for system, etc. (#660)

* Introduce testlib_extract() helper

* Have execve() escape double quotes in cmd.exe's preferred style

This makes it possible for us to use system() and popen() with paths
that redirect to filenames that contain spaces, e.g.

    system("echo.com hello >\"hello there.txt\"")

It's difficult to solve this problem, because WIN32 only allows passing
one single argument when launching programs and each program is allowed
to tokenize that however it wants. Most software follows the convention
of cmd.exe which is poorly documented and positively byzantine.

In the future we're going to solve this by not using cmd.exe at all and
instead embedding the cocmd.com interpreter into the system() function.
In the meantime, our documentation has been updated to help recalibrate
any expectation the user might hold regarding the security of using the
Windows command interpreter.

Fixes #644

* Introduce double quote support in cocmd.com shell

* Add some tests for execve()

* Embed cocmd.com interpreter for system() / open()

This change lets you use system() in an easier and portable way. The
problem with the call in the past has always been that bourne and
cmd.com on Windows have less than nothing in common, so pretty much the
only command system() could be used for across platforms was maybe echo.
cmd.exe is also a security liability due to its escaping rules.

Since cocmd.com implements 85% of what we need from bourne, in a really
tiny way, it makes perfect sense to be embedded in these functionss. We
get a huge performance boost too.

Fixes #644

* Support whitespace after cocmd output redirection

Co-authored-by: Justine Tunney <jtunney@gmail.com>

test/libc/calls/execve_test.c

@@ -0,0 +1,60 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/calls.h"
+#include "libc/fmt/conv.h"
+#include "libc/fmt/itoa.h"
+#include "libc/runtime/runtime.h"
+#include "libc/str/str.h"
+#include "libc/testlib/subprocess.h"
+#include "libc/testlib/testlib.h"
+
+#define N 127
+
+char *GenBuf(char buf[8], int x) {
+  int i;
+  bzero(buf, 8);
+  for (i = 0; i < 7; ++i) {
+    buf[i] = x & 127;  // nt doesn't respect invalid unicode?
+    x >>= 1;
+  }
+  return buf;
+}
+
+__attribute__((__constructor__)) static void init(void) {
+  char buf[8];
+  if (__argc == 4 && !strcmp(__argv[1], "-")) {
+    ASSERT_STREQ(GenBuf(buf, atoi(__argv[2])), __argv[3]);
+    exit(0);
+  }
+}
+
+TEST(execve, testArgPassing) {
+  int i;
+  char ibuf[12], buf[8];
+  for (i = 0; i < N; ++i) {
+    FormatInt32(ibuf, i);
+    GenBuf(buf, i);
+    SPAWN(vfork);
+    execve(GetProgramExecutableName(),
+           (char *const[]){GetProgramExecutableName(), "-", ibuf, buf, 0},
+           (char *const[]){0});
+    notpossible;
+    EXITS(0);
+  }
+}

test/libc/calls/mkntcmdline_test.c

@@ -61,12 +61,6 @@ TEST(mkntcmdline, justSlash) {
   EXPECT_STREQ(u"\\", cmdline);
 }
 
-TEST(mkntcmdline, basicQuoting) {
-  char *argv[] = {"a\"b c", "d", NULL};
-  EXPECT_NE(-1, mkntcmdline(cmdline, argv[0], argv));
-  EXPECT_STREQ(u"\"a\\\"b c\" d" /* "a\"b c" d */, cmdline);
-}
-
 TEST(mkntcmdline, testUnicode) {
   char *argv1[] = {
       gc(strdup("(╯°□°)╯")),
@@ -82,12 +76,12 @@ TEST(mkntcmdline, fixAsBestAsWeCanForNow1) {
   char *argv1[] = {
       "/C/WINDOWS/system32/cmd.exe",
       "/C",
-      "more < \"/C/Users/jart/AppData/Local/Temp/tmplquaa_d6\"",
+      "more <\"/C/Users/jart/AppData/Local/Temp/tmplquaa_d6\"",
       NULL,
   };
   EXPECT_NE(-1, mkntcmdline(cmdline, argv1[0], argv1));
-  EXPECT_STREQ(u"C:\\WINDOWS\\system32\\cmd.exe /C \"more < "
-               u"\\\"C:/Users/jart/AppData/Local/Temp/tmplquaa_d6\\\"\"",
+  EXPECT_STREQ(u"C:\\WINDOWS\\system32\\cmd.exe /C \"more <"
+               u"\"\"\"C:/Users/jart/AppData/Local/Temp/tmplquaa_d6\"\"\"\"",
                cmdline);
 }
 
@@ -106,6 +100,6 @@ TEST(mkntcmdline, fixAsBestAsWeCanForNow2) {
 
 TEST(mkntcmdline, testWut) {
   char *argv[] = {"redbean.com", "--strace", NULL};
-  EXPECT_NE(-1, mkntcmdline(cmdline, "C:\\Users\\jart\\redbean.com", argv));
-  EXPECT_STREQ(u"C:\\Users\\jart\\redbean.com --strace", cmdline);
+  EXPECT_NE(-1, mkntcmdline(cmdline, "C:\\Users\\jart\\𝑟𝑒𝑑𝑏𝑒𝑎𝑛.com", argv));
+  EXPECT_STREQ(u"C:\\Users\\jart\\𝑟𝑒𝑑𝑏𝑒𝑎𝑛.com --strace", cmdline);
 }

test/libc/calls/mkntpath_test.c

@@ -47,11 +47,4 @@ TEST(mkntpath, testUnicode) {
 TEST(mkntpath, testRemoveDoubleSlash) {
   EXPECT_EQ(21, __mkntpath("C:\\Users\\jart\\\\.config", p));
   EXPECT_STREQ(u"C:\\Users\\jart\\.config", p);
-  EXPECT_EQ(8, __mkntpath("\\\\?\\doge", p));
-  EXPECT_STREQ(u"\\\\?\\doge", p);
-}
-
-TEST(mkntpath, testJustC) {
-  EXPECT_EQ(7, __mkntpath("/C", p));
-  EXPECT_STREQ(u"\\\\?\\C:\\", p);
 }

test/libc/calls/pledge_test.c

@@ -56,8 +56,6 @@
 #include "libc/time/time.h"
 #include "libc/x/x.h"
 
-STATIC_YOINK("zip_uri_support");
-
 char testlib_enable_tmp_setup_teardown;
 
 void OnSig(int sig) {
@@ -66,26 +64,11 @@ void OnSig(int sig) {
 
 int sys_memfd_secret(unsigned int);  // our ENOSYS threshold
 
-int extract(const char *from, const char *to, int mode) {
-  int fdin, fdout;
-  if ((fdin = open(from, O_RDONLY)) == -1) return -1;
-  if ((fdout = creat(to, mode)) == -1) {
-    close(fdin);
-    return -1;
-  }
-  if (_copyfd(fdin, fdout, -1) == -1) {
-    close(fdout);
-    close(fdin);
-    return -1;
-  }
-  return close(fdout) | close(fdin);
-}
-
 void SetUp(void) {
   __enable_threads();
   if (!__is_linux_2_6_23() && !IsOpenbsd()) exit(0);
-  ASSERT_SYS(0, 0, extract("/zip/life.elf", "life.elf", 0755));
-  ASSERT_SYS(0, 0, extract("/zip/sock.elf", "sock.elf", 0755));
+  testlib_extract("/zip/life.elf", "life.elf", 0755);
+  testlib_extract("/zip/sock.elf", "sock.elf", 0755);
   __pledge_mode = PLEDGE_PENALTY_RETURN_EPERM;
 }
 

test/libc/calls/unveil_test.c

@@ -44,8 +44,6 @@
 #include "libc/thread/spawn.h"
 #include "libc/x/x.h"
 
-STATIC_YOINK("zip_uri_support");
-
 #define EACCES_OR_ENOENT (IsOpenbsd() ? ENOENT : EACCES)
 
 char testlib_enable_tmp_setup_teardown;
@@ -69,21 +67,6 @@ void SetUp(void) {
   ASSERT_SYS(0, 0, stat("/zip/life.elf", &st));
 }
 
-int extract(const char *from, const char *to, int mode) {
-  int fdin, fdout;
-  if ((fdin = open(from, O_RDONLY)) == -1) return -1;
-  if ((fdout = creat(to, mode)) == -1) {
-    close(fdin);
-    return -1;
-  }
-  if (_copyfd(fdin, fdout, -1) == -1) {
-    close(fdout);
-    close(fdin);
-    return -1;
-  }
-  return close(fdout) | close(fdin);
-}
-
 TEST(unveil, api_differences) {
   SPAWN(fork);
   ASSERT_SYS(0, 0, mkdir("foo", 0755));
@@ -110,7 +93,7 @@ TEST(unveil, api_differences) {
 TEST(unveil, rx_readOnlyPreexistingExecutable_worksFine) {
   SPAWN(fork);
   ASSERT_SYS(0, 0, mkdir("folder", 0755));
-  ASSERT_SYS(0, 0, extract("/zip/life.elf", "folder/life.elf", 0755));
+  testlib_extract("/zip/life.elf", "folder/life.elf", 0755);
   ASSERT_SYS(0, 0, unveil("folder", "rx"));
   ASSERT_SYS(0, 0, unveil(0, 0));
   SPAWN(fork);
@@ -124,7 +107,7 @@ TEST(unveil, rx_readOnlyPreexistingExecutable_worksFine) {
 TEST(unveil, r_noExecutePreexistingExecutable_raisesEacces) {
   SPAWN(fork);
   ASSERT_SYS(0, 0, mkdir("folder", 0755));
-  ASSERT_SYS(0, 0, extract("/zip/life.elf", "folder/life.elf", 0755));
+  testlib_extract("/zip/life.elf", "folder/life.elf", 0755);
   ASSERT_SYS(0, 0, unveil("folder", "r"));
   ASSERT_SYS(0, 0, unveil(0, 0));
   SPAWN(fork);
@@ -155,7 +138,7 @@ TEST(unveil, rwc_createExecutableFile_isAllowedButCantBeRun) {
   ASSERT_SYS(0, 0, mkdir("folder", 0755));
   ASSERT_SYS(0, 0, unveil("folder", "rwc"));
   ASSERT_SYS(0, 0, unveil(0, 0));
-  ASSERT_SYS(0, 0, extract("/zip/life.elf", "folder/life.elf", 0755));
+  testlib_extract("/zip/life.elf", "folder/life.elf", 0755);
   SPAWN(fork);
   ASSERT_SYS(0, 0, stat("folder/life.elf", &st));
   ASSERT_SYS(EACCES, -1, execl("folder/life.elf", "folder/life.elf", 0));
@@ -168,7 +151,7 @@ TEST(unveil, rwcx_createExecutableFile_canAlsoBeRun) {
   ASSERT_SYS(0, 0, mkdir("folder", 0755));
   ASSERT_SYS(0, 0, unveil("folder", "rwcx"));
   ASSERT_SYS(0, 0, unveil(0, 0));
-  ASSERT_SYS(0, 0, extract("/zip/life.elf", "folder/life.elf", 0755));
+  testlib_extract("/zip/life.elf", "folder/life.elf", 0755);
   SPAWN(fork);
   ASSERT_SYS(0, 0, stat("folder/life.elf", &st));
   execl("folder/life.elf", "folder/life.elf", 0);
@@ -205,7 +188,7 @@ TEST(unveil, mostRestrictivePolicy) {
 TEST(unveil, overlappingDirectories_inconsistentBehavior) {
   SPAWN(fork);
   ASSERT_SYS(0, 0, makedirs("f1/f2", 0755));
-  ASSERT_SYS(0, 0, extract("/zip/life.elf", "f1/f2/life.elf", 0755));
+  testlib_extract("/zip/life.elf", "f1/f2/life.elf", 0755);
   ASSERT_SYS(0, 0, unveil("f1", "x"));
   ASSERT_SYS(0, 0, unveil("f1/f2", "r"));
   ASSERT_SYS(0, 0, unveil(0, 0));

test/libc/log/backtrace_test.c

@@ -40,24 +40,15 @@
 #include "libc/x/x.h"
 #include "net/http/escape.h"
 
-STATIC_YOINK("zip_uri_support");
 STATIC_YOINK("backtrace.com");
 STATIC_YOINK("backtrace.com.dbg");
 
 char testlib_enable_tmp_setup_teardown_once;
 
-void Extract(const char *from, const char *to, int mode) {
-  ASSERT_SYS(0, 3, open(from, O_RDONLY));
-  ASSERT_SYS(0, 4, creat(to, mode));
-  ASSERT_NE(-1, _copyfd(3, 4, -1));
-  EXPECT_SYS(0, 0, close(4));
-  EXPECT_SYS(0, 0, close(3));
-}
-
 void SetUpOnce(void) {
   ASSERT_NE(-1, mkdir("bin", 0755));
-  Extract("/zip/backtrace.com", "bin/backtrace.com", 0755);
-  Extract("/zip/backtrace.com.dbg", "bin/backtrace.com.dbg", 0755);
+  testlib_extract("/zip/backtrace.com", "bin/backtrace.com", 0755);
+  testlib_extract("/zip/backtrace.com.dbg", "bin/backtrace.com.dbg", 0755);
 }
 
 static bool OutputHasSymbol(const char *output, const char *s) {

test/libc/runtime/getdosargv_test.c

@@ -169,3 +169,18 @@ TEST(GetDosArgv, waqQuoting2) {
   free(argv);
   free(buf);
 }
+
+TEST(GetDosArgv, cmdToil) {
+  size_t max = 4;
+  size_t size = ARG_MAX / 2;
+  char *buf = malloc(size * sizeof(char));
+  char **argv = malloc(max * sizeof(char *));
+  EXPECT_EQ(3, GetDosArgv(u"cmd.exe /C \"echo hi >\"\"\"𝑓𝑜𝑜 bar.txt\"\"\"\"",
+                          buf, size, argv, max));
+  EXPECT_STREQ("cmd.exe", argv[0]);
+  EXPECT_STREQ("/C", argv[1]);
+  EXPECT_STREQ("echo hi >\"𝑓𝑜𝑜 bar.txt\"", argv[2]);
+  EXPECT_EQ(NULL, argv[3]);
+  free(argv);
+  free(buf);
+}

test/libc/stdio/getrandom_test.c

@@ -16,13 +16,14 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/errno.h"
 #include "libc/intrin/bits.h"
 #include "libc/log/check.h"
 #include "libc/math.h"
 #include "libc/nexgen32e/x86feature.h"
+#include "libc/runtime/runtime.h"
 #include "libc/stdio/lcg.internal.h"
 #include "libc/stdio/rand.h"
-#include "libc/runtime/runtime.h"
 #include "libc/stdio/stdio.h"
 #include "libc/sysv/consts/grnd.h"
 #include "libc/testlib/ezbench.h"
@@ -241,3 +242,7 @@ TEST(getrandom, sanityTest) {
     }
   }
 }
+
+TEST(getrandom, badflags_einval) {
+  ASSERT_SYS(EINVAL, -1, getrandom(0, 0, -1));
+}

test/libc/stdio/system_test.c

@@ -0,0 +1,55 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/calls.h"
+#include "libc/mem/io.h"
+#include "libc/paths.h"
+#include "libc/runtime/gc.h"
+#include "libc/runtime/runtime.h"
+#include "libc/stdio/stdio.h"
+#include "libc/sysv/consts/o.h"
+#include "libc/testlib/ezbench.h"
+#include "libc/testlib/testlib.h"
+#include "libc/x/x.h"
+
+char testlib_enable_tmp_setup_teardown;
+
+TEST(system, testStdoutRedirect) {
+  int ws;
+  testlib_extract("/zip/echo.com", "echo.com", 0755);
+  ASSERT_TRUE(system(0));
+  ws = system("./echo.com hello >hello.txt");
+  ASSERT_TRUE(WIFEXITED(ws));
+  ASSERT_EQ(0, WEXITSTATUS(ws));
+  EXPECT_STREQ("hello\n", _gc(xslurp("hello.txt", 0)));
+}
+
+TEST(system, testStdoutRedirect_withSpacesInFilename) {
+  int ws;
+  testlib_extract("/zip/echo.com", "echo.com", 0755);
+  ASSERT_TRUE(system(0));
+  ws = system("./echo.com hello >\"hello there.txt\"");
+  ASSERT_TRUE(WIFEXITED(ws));
+  ASSERT_EQ(0, WEXITSTATUS(ws));
+  EXPECT_STREQ("hello\n", _gc(xslurp("hello there.txt", 0)));
+}
+
+BENCH(system, bench) {
+  testlib_extract("/zip/echo.com", "echo.com", 0755);
+  EZBENCH2("system", donothing, system("./echo.com hi >/dev/null"));
+}

test/libc/stdio/test.mk

@@ -63,9 +63,20 @@ o/$(MODE)/test/libc/stdio/%.com.dbg:			\
 		$(APE_NO_MODIFY_SELF)
 	@$(APELINK)
 
+o/$(MODE)/test/libc/stdio/system_test.com.dbg:		\
+		$(TEST_LIBC_STDIO_DEPS)			\
+		o/$(MODE)/test/libc/stdio/system_test.o	\
+		o/$(MODE)/test/libc/stdio/stdio.pkg	\
+		o/$(MODE)/tool/build/echo.com.zip.o	\
+		o/$(MODE)/tool/build/cocmd.com.zip.o	\
+		$(LIBC_TESTMAIN)			\
+		$(CRT)					\
+		$(APE_NO_MODIFY_SELF)
+	@$(APELINK)
+
 $(TEST_LIBC_STDIO_OBJS): private			\
-	DEFAULT_CCFLAGS +=				\
-		-fno-builtin
+		DEFAULT_CCFLAGS +=			\
+			-fno-builtin
 
 .PHONY: o/$(MODE)/test/libc/stdio
 o/$(MODE)/test/libc/stdio:				\