Undiamond Python headers

This change gets the Python codebase into a state where it conforms to
the conventions of this codebase. It's now possible to include headers
from Python, without worrying about ordering. Python has traditionally
solved that problem by "diamonding" everything in Python.h, but that's
problematic since it means any change to any Python header invalidates
all the build artifacts. Lastly it makes tooling not work. Since it is
hard to explain to Emacs when I press C-c C-h to add an import line it
shouldn't add the header that actually defines the symbol, and instead
do follow the nonstandard Python convention.

Progress has been made on letting Python load source code from the zip
executable structure via the standard C library APIs. System calss now
recognizes zip!FILENAME alternative URIs as equivalent to zip:FILENAME
since Python uses colon as its delimiter.

Some progress has been made on embedding the notice license terms into
the Python object code. This is easier said than done since Python has
an extremely complicated ownership story.

- Some termios APIs have been added
- Implement rewinddir() dirstream API
- GetCpuCount() API added to Cosmopolitan Libc
- More bugs in Cosmopolitan Libc have been fixed
- zipobj.com now has flags for mangling the path
- Fixed bug a priori with sendfile() on certain BSDs
- Polyfill F_DUPFD and F_DUPFD_CLOEXEC across platforms
- FIOCLEX / FIONCLEX now polyfilled for fast O_CLOEXEC changes
- APE now supports a hybrid solution to no-self-modify for builds
- Many BSD-only magnums added, e.g. O_SEARCH, O_SHLOCK, SF_NODISKIO

libc/zipos/close.c

@@ -20,6 +20,7 @@
 #include "libc/calls/internal.h"
 #include "libc/mem/mem.h"
 #include "libc/nt/runtime.h"
+#include "libc/zip.h"
 #include "libc/zipos/zipos.internal.h"
 
 /**
@@ -30,6 +31,9 @@
 int __zipos_close(int fd) {
   struct ZiposHandle *h;
   h = (struct ZiposHandle *)(intptr_t)g_fds.p[fd].handle;
+  ZTRACE("__zipos_close(%`'.*s)",
+         ZIP_CFILE_NAMESIZE(__zipos_get()->map + h->cfile),
+         ZIP_CFILE_NAME(__zipos_get()->map + h->cfile));
   if (!IsWindows()) {
     sys_close(fd);
   } else {

libc/zipos/find.c

@@ -24,13 +24,17 @@
 #include "libc/zipos/zipos.internal.h"
 
 ssize_t __zipos_find(struct Zipos *zipos, const struct ZiposUri *name) {
-  size_t i, n, c;
+  const char *zname;
+  size_t i, n, c, znamesize;
   c = GetZipCdirOffset(zipos->cdir);
   n = GetZipCdirRecords(zipos->cdir);
   for (i = 0; i < n; ++i, c += ZIP_CFILE_HDRSIZE(zipos->map + c)) {
     assert(ZIP_CFILE_MAGIC(zipos->map + c) == kZipCfileHdrMagic);
-    if (name->len == ZIP_CFILE_NAMESIZE(zipos->map + c) &&
-        memcmp(name->path, ZIP_CFILE_NAME(zipos->map + c), name->len) == 0) {
+    zname = ZIP_CFILE_NAME(zipos->map + c);
+    znamesize = ZIP_CFILE_NAMESIZE(zipos->map + c);
+    if ((name->len == znamesize && !memcmp(name->path, zname, name->len)) ||
+        (name->len + 1 == znamesize && !memcmp(name->path, zname, name->len) &&
+         zname[name->len] == '/')) {
       return c;
     }
   }

libc/zipos/fstat.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/zip.h"
 #include "libc/zipos/zipos.internal.h"
 
 /**
@@ -25,5 +26,8 @@
  * @asyncsignalsafe
  */
 int __zipos_fstat(const struct ZiposHandle *h, struct stat *st) {
+  ZTRACE("__zipos_fstat(%`'.*s)",
+         ZIP_CFILE_NAMESIZE(__zipos_get()->map + h->cfile),
+         ZIP_CFILE_NAME(__zipos_get()->map + h->cfile));
   return __zipos_stat_impl(__zipos_get(), h->cfile, st);
 }

libc/zipos/lseek.c

@@ -18,6 +18,7 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/calls/calls.h"
 #include "libc/sysv/errfuns.h"
+#include "libc/zip.h"
 #include "libc/zipos/zipos.internal.h"
 
 /**
@@ -47,5 +48,8 @@ int64_t __zipos_lseek(struct ZiposHandle *h, int64_t offset, unsigned whence) {
     return einval();
   }
   h->pos = i;
+  ZTRACE("__zipos_lseek(%`'.*s, %ld)",
+         ZIP_CFILE_NAMESIZE(__zipos_get()->map + h->cfile),
+         ZIP_CFILE_NAME(__zipos_get()->map + h->cfile), i);
   return i;
 }

libc/zipos/open.c

@@ -18,6 +18,7 @@
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "ape/relocations.h"
 #include "libc/assert.h"
+#include "libc/bits/weaken.h"
 #include "libc/calls/calls.h"
 #include "libc/calls/internal.h"
 #include "libc/calls/struct/sigset.h"
@@ -28,6 +29,7 @@
 #include "libc/mem/mem.h"
 #include "libc/nexgen32e/crc32.h"
 #include "libc/runtime/runtime.h"
+#include "libc/stdio/stdio.h"
 #include "libc/str/str.h"
 #include "libc/str/undeflate.h"
 #include "libc/sysv/consts/map.h"
@@ -140,11 +142,14 @@ int __zipos_open(const struct ZiposUri *name, unsigned flags, int mode) {
   if ((zipos = __zipos_get())) {
     if ((cf = __zipos_find(zipos, name)) != -1) {
       fd = __zipos_load(zipos, cf, flags, mode);
+      ZTRACE("__zipos_open(%`'.*s)", name->len, name->path);
     } else {
+      ZTRACE("__zipos_open(%`'.*s) enoent", name->len, name->path);
       fd = enoent();
     }
   } else {
     fd = enoexec();
+    ZTRACE("__zipos_open(%`'.*s) enoexec", name->len, name->path);
   }
   return fd;
 }

libc/zipos/parseuri.c

@@ -26,10 +26,11 @@ const char kZiposSchemePrefix[4] hidden = "zip:";
  */
 ssize_t __zipos_parseuri(const char *uri, struct ZiposUri *out) {
   size_t len;
-  if ((len = strlen(uri)) >= sizeof(kZiposSchemePrefix) && len < PATH_MAX &&
-      memcmp(uri, kZiposSchemePrefix, sizeof(kZiposSchemePrefix)) == 0) {
-    out->path = uri + sizeof(kZiposSchemePrefix);
-    return (out->len = len - sizeof(kZiposSchemePrefix));
+  if ((uri[0] == 'z' && uri[1] == 'i' && uri[2] == 'p' &&
+       (uri[3] == ':' || uri[3] == '!')) &&
+      (len = strlen(uri)) < PATH_MAX) {
+    out->path = uri + 4;
+    return (out->len = len - 4);
   } else {
     return -1;
   }

libc/zipos/read.c

@@ -21,8 +21,15 @@
 #include "libc/calls/internal.h"
 #include "libc/calls/struct/iovec.h"
 #include "libc/str/str.h"
+#include "libc/zip.h"
 #include "libc/zipos/zipos.internal.h"
 
+static size_t GetIovSize(const struct iovec *iov, size_t iovlen) {
+  size_t i, r;
+  for (r = i = 0; i < iovlen; ++i) r += iov[i].iov_len;
+  return r;
+}
+
 /**
  * Reads data from zip store object.
  *
@@ -39,5 +46,9 @@ ssize_t __zipos_read(struct ZiposHandle *h, const struct iovec *iov,
     memcpy(iov[i].iov_base, h->mem + y, b);
   }
   if (opt_offset == -1) h->pos = y;
+  ZTRACE("__zipos_read(%`'.*s, cap=%ld, off=%ld) → got=%ld",
+         ZIP_CFILE_NAMESIZE(__zipos_get()->map + h->cfile),
+         ZIP_CFILE_NAME(__zipos_get()->map + h->cfile), GetIovSize(iov, iovlen),
+         x, y - x);
   return y - x;
 }

libc/zipos/stat-impl.c

@@ -19,11 +19,74 @@
 #include "libc/bits/safemacros.internal.h"
 #include "libc/calls/calls.h"
 #include "libc/calls/struct/stat.h"
+#include "libc/fmt/conv.h"
 #include "libc/str/str.h"
 #include "libc/sysv/errfuns.h"
 #include "libc/zip.h"
 #include "libc/zipos/zipos.internal.h"
 
+static int popcnt3(int x) {
+  return !!(x & 1) + !!(x & 2) + !!(x & 4);
+}
+
+static struct timespec WindowsTimeToTime(uint64_t x) {
+  return (struct timespec){x / HECTONANOSECONDS - MODERNITYSECONDS,
+                           x % HECTONANOSECONDS * 100};
+}
+
+static void GetZipCfileTimestamps(const uint8_t *zcf, struct stat *st) {
+  const uint8_t *p, *pe;
+  for (p = ZIP_CFILE_EXTRA(zcf), pe = p + ZIP_CFILE_EXTRASIZE(zcf); p + 4 <= pe;
+       p += ZIP_EXTRA_SIZE(p)) {
+    if (ZIP_EXTRA_HEADERID(p) == kZipExtraNtfs &&
+        ZIP_EXTRA_CONTENTSIZE(p) >= 4 + 4 + 8 &&
+        READ16LE(ZIP_EXTRA_CONTENT(p) + 4) == 1 &&
+        READ16LE(ZIP_EXTRA_CONTENT(p) + 6) >= 8) {
+      st->st_mtim = WindowsTimeToTime(READ64LE(ZIP_EXTRA_CONTENT(p) + 8));
+      if (ZIP_EXTRA_CONTENTSIZE(p) >= 4 + 4 + 8 * 2 &&
+          READ16LE(ZIP_EXTRA_CONTENT(p) + 6) >= 16) {
+        st->st_atim = WindowsTimeToTime(READ64LE(ZIP_EXTRA_CONTENT(p) + 8 * 2));
+      }
+      if (ZIP_EXTRA_CONTENTSIZE(p) >= 4 + 4 + 8 * 3 &&
+          READ16LE(ZIP_EXTRA_CONTENT(p) + 6) >= 24) {
+        st->st_ctim = WindowsTimeToTime(READ64LE(ZIP_EXTRA_CONTENT(p) + 8 * 3));
+      }
+      return;
+    }
+  }
+  for (p = ZIP_CFILE_EXTRA(zcf), pe = p + ZIP_CFILE_EXTRASIZE(zcf); p + 4 <= pe;
+       p += ZIP_EXTRA_SIZE(p)) {
+    if (ZIP_EXTRA_HEADERID(p) == kZipExtraExtendedTimestamp &&
+        ZIP_EXTRA_CONTENTSIZE(p) > 1 &&
+        ZIP_EXTRA_CONTENTSIZE(p) ==
+            1 + 4 * popcnt3(*ZIP_EXTRA_CONTENT(p) & 7)) {
+      if (*ZIP_EXTRA_CONTENT(p) & 1) {
+        st->st_mtim.tv_sec = READ32LE(ZIP_EXTRA_CONTENT(p) + 1);
+      }
+      if (*ZIP_EXTRA_CONTENT(p) & 2) {
+        st->st_atim.tv_sec = READ32LE(ZIP_EXTRA_CONTENT(p) + 1 +
+                                      4 * (*ZIP_EXTRA_CONTENT(p) & 1));
+      }
+      if (*ZIP_EXTRA_CONTENT(p) & 4) {
+        st->st_ctim.tv_sec = READ32LE(ZIP_EXTRA_CONTENT(p) + 1 +
+                                      4 * popcnt3(*ZIP_EXTRA_CONTENT(p) & 3));
+      }
+      return;
+    }
+  }
+  for (p = ZIP_CFILE_EXTRA(zcf), pe = p + ZIP_CFILE_EXTRASIZE(zcf); p + 4 <= pe;
+       p += ZIP_EXTRA_SIZE(p)) {
+    if (ZIP_EXTRA_HEADERID(p) == kZipExtraUnix &&
+        ZIP_EXTRA_CONTENTSIZE(p) >= 4 + 4) {
+      st->st_atim.tv_sec = (int32_t)READ32LE(ZIP_EXTRA_CONTENT(p) + 0);
+      st->st_mtim.tv_sec = (int32_t)READ32LE(ZIP_EXTRA_CONTENT(p) + 4);
+      return;
+    }
+  }
+  st->st_mtim.tv_sec = DosDateTimeToUnix(ZIP_CFILE_LASTMODIFIEDDATE(zcf),
+                                         ZIP_CFILE_LASTMODIFIEDTIME(zcf));
+}
+
 int __zipos_stat_impl(struct Zipos *zipos, size_t cf, struct stat *st) {
   size_t lf;
   if (zipos && st) {
@@ -37,6 +100,7 @@ int __zipos_stat_impl(struct Zipos *zipos, size_t cf, struct stat *st) {
     st->st_size = GetZipLfileUncompressedSize(zipos->map + lf);
     st->st_blocks =
         roundup(GetZipLfileCompressedSize(zipos->map + lf), 512) / 512;
+    GetZipCfileTimestamps(zipos->map + cf, st);
     return 0;
   } else {
     return einval();

libc/zipos/stat.c

@@ -16,6 +16,8 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/bits/weaken.h"
+#include "libc/stdio/stdio.h"
 #include "libc/sysv/errfuns.h"
 #include "libc/zipos/zipos.internal.h"
 
@@ -32,9 +34,11 @@ int __zipos_stat(const struct ZiposUri *name, struct stat *st) {
     if ((cf = __zipos_find(zipos, name)) != -1) {
       return __zipos_stat_impl(zipos, cf, st);
     } else {
+      ZTRACE("__zipos_stat(%`'.*s) → enoent", name->len, name->path);
       return enoent();
     }
   } else {
+    ZTRACE("__zipos_stat(%`'.*s) → enoexec", name->len, name->path);
     return enoexec();
   }
 }

libc/zipos/zipos.internal.h

@@ -1,8 +1,15 @@
 #ifndef COSMOPOLITAN_LIBC_ZIPOS_ZIPOS_H_
 #define COSMOPOLITAN_LIBC_ZIPOS_ZIPOS_H_
+#include "libc/calls/calls.h"
 #if !(__ASSEMBLER__ + __LINKER__ + 0)
 COSMOPOLITAN_C_START_
 
+#if 0
+#define ZTRACE(FMT, ...) (dprintf)(2, FMT "\n", ##__VA_ARGS__)
+#else
+#define ZTRACE(FMT, ...) (void)0
+#endif
+
 struct stat;
 struct iovec;
 
@@ -26,10 +33,8 @@ struct ZiposHandle {
   uint8_t *freeme;
 };
 
-extern const char kZiposSchemePrefix[4];
-
 int __zipos_close(int) hidden;
-struct Zipos *__zipos_get(void) hidden;
+struct Zipos *__zipos_get(void) pureconst hidden;
 ssize_t __zipos_parseuri(const char *, struct ZiposUri *) hidden;
 ssize_t __zipos_find(struct Zipos *, const struct ZiposUri *);
 int __zipos_open(const struct ZiposUri *, unsigned, int) hidden;