fexecve: renable and fix, assimilate via running shell script instead of running it

fexecve_test.c: fix zipread.com. renable tinyelf test (x86_64) only

fexecve_test.c: make memfd_create test work on all archs cosmopolitan is compiled to

(f)execve_test.c add checking for /proc/self/fd/ for fexecve

fexecve_test: get /zip/echo working again

move execve test out of fexecve_test.c

assimilate via running embedded shell script instead of parsing it

satisfy -Werror=discarded-qualifiers

fexecve: fix zipos when executable is not an APE

comment out broken tests

get most fexecve tests working again

switch memfd_create test back to vfork

add life.elf back into zipread.c, move zipread into proc

get zipos execve working again

libc/calls/isapemagic.c

@@ -28,3 +28,11 @@ bool IsApeLoadable(char buf[8]) {
          READ64LE(buf) == READ64LE("jartsr='") ||
          READ64LE(buf) == READ64LE("APEDBG='");
 }
+
+/**
+ * Returns true if executable image is an APE
+ */
+bool IsApeMagic(char buf[8]) {
+  return READ64LE(buf) == READ64LE("MZqFpD='") ||
+         READ64LE(buf) == READ64LE("JTqFpD='");
+}

libc/proc/execve.c

@@ -76,7 +76,7 @@ int execve(const char *prog, char *const argv[], char *const envp[]) {
       rc = _weaken(sys_pledge_linux)(__execpromises, __pledge_mode);
     }
     if (!rc) {
-      if (0 && _weaken(__zipos_parseuri) &&
+      if (_weaken(__zipos_parseuri) &&
           (_weaken(__zipos_parseuri)(prog, &uri) != -1)) {
         rc = _weaken(__zipos_open)(&uri, O_RDONLY | O_CLOEXEC);
         if (rc != -1) {

libc/proc/execve.internal.h

@@ -2,7 +2,8 @@
 #define COSMOPOLITAN_LIBC_CALLS_EXECVE_SYSV_H_
 COSMOPOLITAN_C_START_
 
-bool IsApeLoadable(char[8]) libcesque;
+bool IsApeLoadable(char[8]);
+bool IsApeMagic(char[8]);
 
 COSMOPOLITAN_C_END_
 #endif /* COSMOPOLITAN_LIBC_CALLS_EXECVE_SYSV_H_ */

libc/proc/fexecve.c

@@ -34,6 +34,7 @@
 #include "libc/intrin/strace.internal.h"
 #include "libc/intrin/weaken.h"
 #include "libc/limits.h"
+#include "libc/paths.h"
 #include "libc/proc/execve.internal.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/f.h"
@@ -42,12 +43,14 @@
 #include "libc/sysv/consts/mfd.h"
 #include "libc/sysv/consts/o.h"
 #include "libc/sysv/consts/prot.h"
+#include "libc/sysv/consts/s.h"
 #include "libc/sysv/consts/shm.h"
 #include "libc/sysv/errfuns.h"
+#include "libc/zip.internal.h"
 
 static bool IsAPEFd(const int fd) {
   char buf[8];
-  return (sys_pread(fd, buf, 8, 0, 0) == 8) && IsApeLoadable(buf);
+  return (sys_pread(fd, buf, 8, 0, 0) == 8) && IsApeMagic(buf);
 }
 
 static int fexecve_impl(const int fd, char *const argv[], char *const envp[]) {
@@ -64,6 +67,57 @@ static int fexecve_impl(const int fd, char *const argv[], char *const envp[]) {
   return rc;
 }
 
+#define defer(fn) __attribute__((cleanup(fn)))
+
+void cleanup_close(int *pFD) {
+  STRACE("time to close");
+  if (*pFD != -1) {
+    close(*pFD);
+  }
+}
+#define defer_close defer(cleanup_close)
+
+void cleanup_unlink(const char **path) {
+  STRACE("time to unlink");
+  if (*path != NULL) {
+    sys_unlink(*path);
+  }
+}
+#define defer_unlink defer(cleanup_unlink)
+
+static bool ape_to_elf_execve(void *ape, const size_t apesize) {
+  if (!_weaken(fork) || !_weaken(exit) || !IsLinux()) {
+    return false;
+  }
+  defer_unlink const char *tempfile = "/dev/shm/ape_to_elf_execve_XXXXXX";
+  defer_close int fd = open(tempfile, O_RDWR | O_CREAT | O_EXCL, S_IRWXU);
+  if (fd == -1) {
+    tempfile = NULL;
+    return false;
+  }
+  if ((sys_ftruncate(fd, apesize, apesize) == -1) || (write(fd, ape, apesize) != apesize)) {
+    return false;
+  }
+  close(fd);
+  fd = -1;
+  int child = _weaken(fork)();
+  if (child == -1) {
+    return false;
+  } else if (child == 0) {
+    __sys_execve(_PATH_BSHELL, (char *const[]){_PATH_BSHELL, (char*)tempfile, "--assimilate", NULL}, (char *const[]){NULL});
+    _weaken(exit)(1);
+  }
+  int wstatus;
+  if ((child != waitpid(child, &wstatus, 0)) || !WIFEXITED(wstatus) || (WEXITSTATUS(wstatus) != 0)) {
+    return false;
+  }
+  return ((fd = open(tempfile, O_RDWR, S_IRWXU)) != -1) && (pread(fd, ape, apesize, 0) == apesize);
+}
+
+#undef defer_unlink
+#undef defer_close
+#undef defer
+
 typedef enum {
   PTF_NUM = 1 << 0,
   PTF_NUM2 = 1 << 1,
@@ -72,6 +126,7 @@ typedef enum {
 } PTF_PARSE;
 
 static bool ape_to_elf(void *ape, const size_t apesize) {
+  return ape_to_elf_execve(ape, apesize);
   static const char printftok[] = "printf '";
   static const size_t printftoklen = sizeof(printftok) - 1;
   const char *tok = memmem(ape, apesize, printftok, printftoklen);
@@ -142,15 +197,21 @@ static int fd_to_mem_fd(const int infd, char *path) {
     ssize_t readRc;
     readRc = pread(infd, space, st.st_size, 0);
     bool success = readRc != -1;
-    if (success && (st.st_size > 8) && IsApeLoadable(space)) {
+    if (success) {
-      int flags = fcntl(fd, F_GETFD);
+      int ziperror;
-      if ((success = (flags != -1) &&
+      if ((st.st_size > 8) && IsApeMagic(space)) {
-                     (fcntl(fd, F_SETFD, flags & (~FD_CLOEXEC)) != -1) &&
+        success = ape_to_elf(space, st.st_size);
-                     ape_to_elf(space, st.st_size))) {
+      }
-        const int newfd = fcntl(fd, F_DUPFD, 9001);
+      // we need to preserve the fd over exec if there's zipos
-        if (newfd != -1) {
+      if (success && _weaken(GetZipEocd) && _weaken(GetZipEocd)(space, st.st_size, &ziperror)) {
-          close(fd);
+        int flags = fcntl(fd, F_GETFD);
-          fd = newfd;
+        if ((success = (flags != -1) &&
+                       (fcntl(fd, F_SETFD, flags & (~FD_CLOEXEC)) != -1))) {
+          const int newfd = fcntl(fd, F_DUPFD, 9001);
+          if (newfd != -1) {
+            close(fd);
+            fd = newfd;
+          }
         }
       }
     }
@@ -233,7 +294,6 @@ int fexecve(int fd, char *const argv[], char *const envp[]) {
       }
       size_t numenvs;
       for (numenvs = 0; envp[numenvs];) ++numenvs;
-      // const size_t desenvs = min(500, max(numenvs + 1, 2));
       static _Thread_local char *envs[500];
       memcpy(envs, envp, numenvs * sizeof(char *));
       envs[numenvs] = path;

test/libc/calls/BUILD.mk

@@ -19,9 +19,7 @@ TEST_LIBC_CALLS_BINS =							\
 	$(TEST_LIBC_CALLS_COMS)						\
 	$(TEST_LIBC_CALLS_COMS:%=%.dbg)					\
 	o/$(MODE)/test/libc/calls/life-nomod.com			\
-	o/$(MODE)/test/libc/calls/life-classic.com			\
+	o/$(MODE)/test/libc/calls/life-classic.com
-	o/$(MODE)/test/libc/calls/zipread.com.dbg			\
-	o/$(MODE)/test/libc/calls/zipread.com
 
 TEST_LIBC_CALLS_TESTS =							\
 	$(TEST_LIBC_CALLS_SRCS_TEST:%.c=o/$(MODE)/%.com.ok)
@@ -118,8 +116,7 @@ o/$(MODE)/test/libc/calls/life-nomod.com.dbg:				\
 
 o/$(MODE)/test/libc/calls/tiny64.elf.zip.o				\
 o/$(MODE)/test/libc/calls/life-nomod.com.zip.o				\
-o/$(MODE)/test/libc/calls/life-classic.com.zip.o			\
+o/$(MODE)/test/libc/calls/life-classic.com.zip.o: private	\
-o/$(MODE)/test/libc/calls/zipread.com.zip.o: private			\
 		ZIPOBJ_FLAGS +=						\
 			-B
 

test/libc/proc/BUILD.mk

@@ -14,7 +14,9 @@ TEST_LIBC_PROC_COMS =						\
 
 TEST_LIBC_PROC_BINS =						\
 	$(TEST_LIBC_PROC_COMS)					\
-	$(TEST_LIBC_PROC_COMS:%=%.dbg)
+	$(TEST_LIBC_PROC_COMS:%=%.dbg)			\
+	o/$(MODE)/test/libc/proc/zipread.com.dbg	\
+	o/$(MODE)/test/libc/proc/zipread.com
 
 TEST_LIBC_PROC_TESTS =						\
 	$(TEST_LIBC_PROC_SRCS_TEST:%.c=o/$(MODE)/%.com.ok)
@@ -90,6 +92,7 @@ o/$(MODE)/test/libc/proc/execve_test.com.dbg:				\
 		o/$(MODE)/test/libc/proc/execve_test.o			\
 		o/$(MODE)/test/libc/calls/life-nomod.com.zip.o		\
 		o/$(MODE)/test/libc/proc/execve_test_prog1.com.zip.o	\
+		o/$(MODE)/test/libc/proc/echo.elf.zip.o			\
 		o/$(MODE)/test/libc/mem/prog/life.elf.zip.o		\
 		o/$(MODE)/test/libc/mem/prog/sock.elf.zip.o		\
 		o/$(MODE)/test/libc/proc/proc.pkg			\
@@ -102,16 +105,38 @@ o/$(MODE)/test/libc/proc/fexecve_test.com.dbg:				\
 		$(TEST_LIBC_PROC_DEPS)					\
 		o/$(MODE)/test/libc/proc/fexecve_test.o		\
 		o/$(MODE)/test/libc/proc/proc.pkg			\
+		o/$(MODE)/test/libc/proc/echo.elf.zip.o		\
 		o/$(MODE)/test/libc/mem/prog/life.elf.zip.o		\
 		o/$(MODE)/test/libc/calls/life-nomod.com.zip.o		\
-		o/$(MODE)/test/libc/calls/zipread.com.zip.o		\
+		o/$(MODE)/test/libc/proc/zipread.com.zip.o		\
 		$(LIBC_TESTMAIN)					\
 		$(CRT)							\
 		$(APE_NO_MODIFY_SELF)
 	@$(APELINK)
 
+o/$(MODE)/test/libc/proc/zipread.com.dbg:			\
+		$(LIBC_RUNTIME)					\
+		o/$(MODE)/test/libc/proc/zipread.o			\
+		o/$(MODE)/test/libc/mem/prog/life.elf.zip.o	\
+		$(CRT)						\
+		$(APE)
+	@$(APELINK)
+
+o/$(MODE)/test/libc/proc/echo.elf:				\
+		o/$(MODE)/tool/build/assimilate.com		\
+		o/$(MODE)/tool/build/echo.com
+	@$(COMPILE) -wACP -T$@					\
+		build/bootstrap/cp.com				\
+		o/$(MODE)/tool/build/echo.com		\
+		o/$(MODE)/test/libc/proc/echo.elf
+	@$(COMPILE) -wAASSIMILATE -T$@				\
+		o/$(MODE)/tool/build/assimilate.com -bcef	\
+		o/$(MODE)/test/libc/proc/echo.elf
+
+o/$(MODE)/test/libc/proc/echo.elf.zip.o					\
 o/$(MODE)/test/libc/proc/execve_test_prog1.com.zip.o		\
-o/$(MODE)/test/libc/proc/life-pe.com.zip.o: private		\
+o/$(MODE)/test/libc/proc/life-pe.com.zip.o					\
+o/$(MODE)/test/libc/proc/zipread.com.zip.o: private		\
 		ZIPOBJ_FLAGS +=					\
 			-B
 

test/libc/proc/execve_test.c

@@ -18,6 +18,7 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/calls/calls.h"
 #include "libc/calls/struct/rusage.h"
+#include "libc/calls/struct/stat.h"
 #include "libc/calls/syscall_support-sysv.internal.h"
 #include "libc/dce.h"
 #include "libc/errno.h"
@@ -26,6 +27,8 @@
 #include "libc/intrin/kprintf.h"
 #include "libc/runtime/runtime.h"
 #include "libc/str/str.h"
+#include "libc/sysv/consts/o.h"
+#include "libc/sysv/consts/s.h"
 #include "libc/temp.h"
 #include "libc/testlib/ezbench.h"
 #include "libc/testlib/subprocess.h"
@@ -35,6 +38,9 @@ __static_yoink("zipos");
 
 #define N 16
 
+int fds[2];
+char buf[8];
+bool HasProcFSAndMemfd = false;
 void SetUpOnce(void) {
   testlib_enable_tmp_setup_teardown();
 }
@@ -48,6 +54,21 @@ void GenBuf(char buf[8], int x) {
   }
 }
 
+__attribute__((__constructor__)) static void init(void) {
+  char buf[8];
+  if (__argc == 4 && !strcmp(__argv[1], "-")) {
+    GenBuf(buf, atoi(__argv[2]));
+    ASSERT_STREQ(buf, __argv[3]);
+    exit(0);
+  }
+  // zipos execve requires /proc and memfd_create
+  // TODO check for memfd
+  if (IsLinux()) {
+    struct stat st;
+    HasProcFSAndMemfd = stat("/proc/self/fd", &st) == 0 && S_ISDIR(st.st_mode);
+  }
+}
+
 TEST(execve, testArgPassing) {
   int i;
   char ibuf[12], buf[8];
@@ -64,10 +85,28 @@ TEST(execve, testArgPassing) {
   }
 }
 
+TEST(execve, elfIsUnreadable_mayBeExecuted) {
+  if (IsWindows() || IsXnu()) return;
+  testlib_extract("/zip/echo.elf", "echo", 0111);
+  ASSERT_SYS(0, 0, pipe2(fds, O_CLOEXEC));
+  SPAWN(vfork);
+  ASSERT_SYS(0, 1, dup2(4, 1));
+  ASSERT_SYS(
+      0, 0,
+      execve("echo", (char *const[]){"echo", "hi", 0}, (char *const[]){0}));
+  exit(1);
+  EXITS(0);
+  bzero(buf, 8);
+  ASSERT_SYS(0, 0, close(4));
+  ASSERT_SYS(0, 3, read(3, buf, 7));
+  ASSERT_SYS(0, 0, close(3));
+  ASSERT_STREQ("hi\n", buf);
+}
+
 TEST(execve, ziposELF) {
-  if (1) return;                                  // TODO: rewrite
   if (IsFreebsd()) return;                        // TODO: fixme on freebsd
   if (IsLinux() && !__is_linux_2_6_23()) return;  // TODO: fixme on old linux
+  if (IsLinux() && !HasProcFSAndMemfd) return;
   if (!IsLinux() && !IsFreebsd()) {
     EXPECT_SYS(ENOSYS, -1,
                execve("/zip/life.elf", (char *const[]){0}, (char *const[]){0}));
@@ -80,9 +119,9 @@ TEST(execve, ziposELF) {
 }
 
 TEST(execve, ziposAPE) {
-  if (1) return;                                  // TODO: rewrite
   if (IsFreebsd()) return;                        // TODO: fixme on freebsd
   if (IsLinux() && !__is_linux_2_6_23()) return;  // TODO: fixme on old linux
+  if (IsLinux() && !HasProcFSAndMemfd) return;
   if (!IsLinux() && !IsFreebsd()) {
     EXPECT_EQ(-1, execve("/zip/life-nomod.com", (char *const[]){0},
                          (char *const[]){0}));

test/libc/proc/fexecve_test.c

@@ -16,8 +16,8 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#if 0  // TODO(G4Vi): improve reliability of fexecve() implementation
 #include "libc/calls/calls.h"
+#include "libc/calls/struct/stat.h"
 #include "libc/calls/syscall_support-sysv.internal.h"
 #include "libc/dce.h"
 #include "libc/errno.h"
@@ -25,6 +25,7 @@
 #include "libc/str/str.h"
 #include "libc/sysv/consts/mfd.h"
 #include "libc/sysv/consts/o.h"
+#include "libc/sysv/consts/s.h"
 #include "libc/testlib/subprocess.h"
 #include "libc/testlib/testlib.h"
 // clang-format off
@@ -33,6 +34,7 @@ __static_yoink("zipos");
 
 int fds[2];
 char buf[8];
+uint8_t elf_buf[4096];
 
 void SetUpOnce(void) {
   testlib_enable_tmp_setup_teardown();
@@ -41,29 +43,18 @@ void SetUpOnce(void) {
 void SetUp(void) {
   if (IsFreebsd()) exit(0);           // TODO: fixme on freebsd
   if (IsLinux() && !__is_linux_2_6_23()) exit(0);  // TODO: fixme on old linux
-}
+  // linux fexecve relies on execve from /proc
-
+  if (IsLinux()) {
-TEST(execve, elfIsUnreadable_mayBeExecuted) {
+    struct stat st;
-  if (IsWindows() || IsXnu()) return;
+    if (stat("/proc/self/fd", &st) != 0 || !S_ISDIR(st.st_mode)) {
-  testlib_extract("/zip/echo", "echo", 0111);
+      exit(0);
-  ASSERT_SYS(0, 0, pipe2(fds, O_CLOEXEC));
+    }
-  SPAWN(vfork);
+  }
-  ASSERT_SYS(0, 1, dup2(4, 1));
-  ASSERT_SYS(
-      0, 0,
-      execve("echo", (char *const[]){"echo", "hi", 0}, (char *const[]){0}));
-  notpossible;
-  EXITS(0);
-  bzero(buf, 8);
-  ASSERT_SYS(0, 0, close(4));
-  ASSERT_SYS(0, 3, read(3, buf, 7));
-  ASSERT_SYS(0, 0, close(3));
-  ASSERT_STREQ("hi\n", buf);
 }
 
 TEST(fexecve, elfIsUnreadable_mayBeExecuted) {
   if (!IsLinux() && !IsFreebsd()) return;
-  testlib_extract("/zip/echo", "echo", 0111);
+  testlib_extract("/zip/echo.elf", "echo", 0111);
   ASSERT_SYS(0, 0, pipe2(fds, O_CLOEXEC));
   SPAWN(vfork);
   ASSERT_SYS(0, 1, dup2(4, 1));
@@ -71,7 +62,7 @@ TEST(fexecve, elfIsUnreadable_mayBeExecuted) {
   if (IsFreebsd()) ASSERT_SYS(0, 1, lseek(5, 1, SEEK_SET));
   ASSERT_SYS(0, 0,
              fexecve(5, (char *const[]){"echo", "hi", 0}, (char *const[]){0}));
-  notpossible;
+  exit(1);
   EXITS(0);
   bzero(buf, 8);
   ASSERT_SYS(0, 0, close(4));
@@ -81,23 +72,27 @@ TEST(fexecve, elfIsUnreadable_mayBeExecuted) {
 }
 
 TEST(fexecve, memfd_create) {
-  if (1) return; // TODO: fixme
   if (!IsLinux()) return;
-  SPAWN(vfork);
+  int life_fd = open("/zip/life.elf", O_RDONLY);
-#define TINY_ELF_PROGRAM "\
+  ASSERT_NE(-1, life_fd);
-\177\105\114\106\002\001\001\000\000\000\000\000\000\000\000\000\
-\002\000\076\000\001\000\000\000\170\000\100\000\000\000\000\000\
-\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\
-\000\000\000\000\100\000\070\000\001\000\000\000\000\000\000\000\
-\001\000\000\000\005\000\000\000\000\000\000\000\000\000\000\000\
-\000\000\100\000\000\000\000\000\000\000\100\000\000\000\000\000\
-\200\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\
-\000\020\000\000\000\000\000\000\152\052\137\152\074\130\017\005"
   int fd = memfd_create("foo", MFD_CLOEXEC);
-  if (fd == -1 && errno == ENOSYS) _Exit(42);
+  if(fd == -1) {
-  write(fd, TINY_ELF_PROGRAM, sizeof(TINY_ELF_PROGRAM) - 1);
+    ASSERT_EQ(ENOSYS, errno);
+    return;
+  }
+  while(1) {
+    const ssize_t bytes_read = read(life_fd, elf_buf, sizeof(elf_buf));
+    if (bytes_read <= 0) {
+      ASSERT_LE(0, bytes_read);
+      break;
+    }
+    ASSERT_EQ(bytes_read, write(fd, elf_buf, bytes_read));
+  }
+  ASSERT_SYS(0, 0, close(life_fd));
+  SPAWN(vfork);
   fexecve(fd, (char *const[]){0}, (char *const[]){0});
   EXITS(42);
+  ASSERT_SYS(0, 0, close(fd));
 }
 
 TEST(fexecve, APE) {
@@ -141,16 +136,11 @@ TEST(fexecve, ziposAPE) {
 }
 
 TEST(fexecve, ziposAPEHasZipos) {
-  if (1) return; // TODO: fixme
   if (!IsLinux() && !IsFreebsd()) return;
   int fd = open("/zip/zipread.com", O_RDONLY);
   ASSERT_NE(-1, fd);
   SPAWN(fork);
-  ASSERT_NE(-1, fd);
-  if (fd == -1 && errno == ENOSYS) _Exit(42);
   fexecve(fd, (char *const[]){0}, (char *const[]){0});
   EXITS(42);
   close(fd);
 }
-
-#endif