Undiamond Python headers

This change gets the Python codebase into a state where it conforms to
the conventions of this codebase. It's now possible to include headers
from Python, without worrying about ordering. Python has traditionally
solved that problem by "diamonding" everything in Python.h, but that's
problematic since it means any change to any Python header invalidates
all the build artifacts. Lastly it makes tooling not work. Since it is
hard to explain to Emacs when I press C-c C-h to add an import line it
shouldn't add the header that actually defines the symbol, and instead
do follow the nonstandard Python convention.

Progress has been made on letting Python load source code from the zip
executable structure via the standard C library APIs. System calss now
recognizes zip!FILENAME alternative URIs as equivalent to zip:FILENAME
since Python uses colon as its delimiter.

Some progress has been made on embedding the notice license terms into
the Python object code. This is easier said than done since Python has
an extremely complicated ownership story.

- Some termios APIs have been added
- Implement rewinddir() dirstream API
- GetCpuCount() API added to Cosmopolitan Libc
- More bugs in Cosmopolitan Libc have been fixed
- zipobj.com now has flags for mangling the path
- Fixed bug a priori with sendfile() on certain BSDs
- Polyfill F_DUPFD and F_DUPFD_CLOEXEC across platforms
- FIOCLEX / FIONCLEX now polyfilled for fast O_CLOEXEC changes
- APE now supports a hybrid solution to no-self-modify for builds
- Many BSD-only magnums added, e.g. O_SEARCH, O_SHLOCK, SF_NODISKIO
This commit is contained in:
Justine Tunney 2021-08-12 00:42:14 -07:00
parent 20bb8db9f8
commit b420ed8248
762 changed files with 18410 additions and 53772 deletions

View file

@ -20,6 +20,7 @@
#include "libc/calls/internal.h"
#include "libc/mem/mem.h"
#include "libc/nt/runtime.h"
#include "libc/zip.h"
#include "libc/zipos/zipos.internal.h"
/**
@ -30,6 +31,9 @@
int __zipos_close(int fd) {
struct ZiposHandle *h;
h = (struct ZiposHandle *)(intptr_t)g_fds.p[fd].handle;
ZTRACE("__zipos_close(%`'.*s)",
ZIP_CFILE_NAMESIZE(__zipos_get()->map + h->cfile),
ZIP_CFILE_NAME(__zipos_get()->map + h->cfile));
if (!IsWindows()) {
sys_close(fd);
} else {

View file

@ -24,13 +24,17 @@
#include "libc/zipos/zipos.internal.h"
ssize_t __zipos_find(struct Zipos *zipos, const struct ZiposUri *name) {
size_t i, n, c;
const char *zname;
size_t i, n, c, znamesize;
c = GetZipCdirOffset(zipos->cdir);
n = GetZipCdirRecords(zipos->cdir);
for (i = 0; i < n; ++i, c += ZIP_CFILE_HDRSIZE(zipos->map + c)) {
assert(ZIP_CFILE_MAGIC(zipos->map + c) == kZipCfileHdrMagic);
if (name->len == ZIP_CFILE_NAMESIZE(zipos->map + c) &&
memcmp(name->path, ZIP_CFILE_NAME(zipos->map + c), name->len) == 0) {
zname = ZIP_CFILE_NAME(zipos->map + c);
znamesize = ZIP_CFILE_NAMESIZE(zipos->map + c);
if ((name->len == znamesize && !memcmp(name->path, zname, name->len)) ||
(name->len + 1 == znamesize && !memcmp(name->path, zname, name->len) &&
zname[name->len] == '/')) {
return c;
}
}

View file

@ -16,6 +16,7 @@
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/zip.h"
#include "libc/zipos/zipos.internal.h"
/**
@ -25,5 +26,8 @@
* @asyncsignalsafe
*/
int __zipos_fstat(const struct ZiposHandle *h, struct stat *st) {
ZTRACE("__zipos_fstat(%`'.*s)",
ZIP_CFILE_NAMESIZE(__zipos_get()->map + h->cfile),
ZIP_CFILE_NAME(__zipos_get()->map + h->cfile));
return __zipos_stat_impl(__zipos_get(), h->cfile, st);
}

View file

@ -18,6 +18,7 @@
*/
#include "libc/calls/calls.h"
#include "libc/sysv/errfuns.h"
#include "libc/zip.h"
#include "libc/zipos/zipos.internal.h"
/**
@ -47,5 +48,8 @@ int64_t __zipos_lseek(struct ZiposHandle *h, int64_t offset, unsigned whence) {
return einval();
}
h->pos = i;
ZTRACE("__zipos_lseek(%`'.*s, %ld)",
ZIP_CFILE_NAMESIZE(__zipos_get()->map + h->cfile),
ZIP_CFILE_NAME(__zipos_get()->map + h->cfile), i);
return i;
}

View file

@ -18,6 +18,7 @@
*/
#include "ape/relocations.h"
#include "libc/assert.h"
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/calls/internal.h"
#include "libc/calls/struct/sigset.h"
@ -28,6 +29,7 @@
#include "libc/mem/mem.h"
#include "libc/nexgen32e/crc32.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
#include "libc/str/undeflate.h"
#include "libc/sysv/consts/map.h"
@ -140,11 +142,14 @@ int __zipos_open(const struct ZiposUri *name, unsigned flags, int mode) {
if ((zipos = __zipos_get())) {
if ((cf = __zipos_find(zipos, name)) != -1) {
fd = __zipos_load(zipos, cf, flags, mode);
ZTRACE("__zipos_open(%`'.*s)", name->len, name->path);
} else {
ZTRACE("__zipos_open(%`'.*s) enoent", name->len, name->path);
fd = enoent();
}
} else {
fd = enoexec();
ZTRACE("__zipos_open(%`'.*s) enoexec", name->len, name->path);
}
return fd;
}

View file

@ -26,10 +26,11 @@ const char kZiposSchemePrefix[4] hidden = "zip:";
*/
ssize_t __zipos_parseuri(const char *uri, struct ZiposUri *out) {
size_t len;
if ((len = strlen(uri)) >= sizeof(kZiposSchemePrefix) && len < PATH_MAX &&
memcmp(uri, kZiposSchemePrefix, sizeof(kZiposSchemePrefix)) == 0) {
out->path = uri + sizeof(kZiposSchemePrefix);
return (out->len = len - sizeof(kZiposSchemePrefix));
if ((uri[0] == 'z' && uri[1] == 'i' && uri[2] == 'p' &&
(uri[3] == ':' || uri[3] == '!')) &&
(len = strlen(uri)) < PATH_MAX) {
out->path = uri + 4;
return (out->len = len - 4);
} else {
return -1;
}

View file

@ -21,8 +21,15 @@
#include "libc/calls/internal.h"
#include "libc/calls/struct/iovec.h"
#include "libc/str/str.h"
#include "libc/zip.h"
#include "libc/zipos/zipos.internal.h"
static size_t GetIovSize(const struct iovec *iov, size_t iovlen) {
size_t i, r;
for (r = i = 0; i < iovlen; ++i) r += iov[i].iov_len;
return r;
}
/**
* Reads data from zip store object.
*
@ -39,5 +46,9 @@ ssize_t __zipos_read(struct ZiposHandle *h, const struct iovec *iov,
memcpy(iov[i].iov_base, h->mem + y, b);
}
if (opt_offset == -1) h->pos = y;
ZTRACE("__zipos_read(%`'.*s, cap=%ld, off=%ld) → got=%ld",
ZIP_CFILE_NAMESIZE(__zipos_get()->map + h->cfile),
ZIP_CFILE_NAME(__zipos_get()->map + h->cfile), GetIovSize(iov, iovlen),
x, y - x);
return y - x;
}

View file

@ -19,11 +19,74 @@
#include "libc/bits/safemacros.internal.h"
#include "libc/calls/calls.h"
#include "libc/calls/struct/stat.h"
#include "libc/fmt/conv.h"
#include "libc/str/str.h"
#include "libc/sysv/errfuns.h"
#include "libc/zip.h"
#include "libc/zipos/zipos.internal.h"
static int popcnt3(int x) {
return !!(x & 1) + !!(x & 2) + !!(x & 4);
}
static struct timespec WindowsTimeToTime(uint64_t x) {
return (struct timespec){x / HECTONANOSECONDS - MODERNITYSECONDS,
x % HECTONANOSECONDS * 100};
}
static void GetZipCfileTimestamps(const uint8_t *zcf, struct stat *st) {
const uint8_t *p, *pe;
for (p = ZIP_CFILE_EXTRA(zcf), pe = p + ZIP_CFILE_EXTRASIZE(zcf); p + 4 <= pe;
p += ZIP_EXTRA_SIZE(p)) {
if (ZIP_EXTRA_HEADERID(p) == kZipExtraNtfs &&
ZIP_EXTRA_CONTENTSIZE(p) >= 4 + 4 + 8 &&
READ16LE(ZIP_EXTRA_CONTENT(p) + 4) == 1 &&
READ16LE(ZIP_EXTRA_CONTENT(p) + 6) >= 8) {
st->st_mtim = WindowsTimeToTime(READ64LE(ZIP_EXTRA_CONTENT(p) + 8));
if (ZIP_EXTRA_CONTENTSIZE(p) >= 4 + 4 + 8 * 2 &&
READ16LE(ZIP_EXTRA_CONTENT(p) + 6) >= 16) {
st->st_atim = WindowsTimeToTime(READ64LE(ZIP_EXTRA_CONTENT(p) + 8 * 2));
}
if (ZIP_EXTRA_CONTENTSIZE(p) >= 4 + 4 + 8 * 3 &&
READ16LE(ZIP_EXTRA_CONTENT(p) + 6) >= 24) {
st->st_ctim = WindowsTimeToTime(READ64LE(ZIP_EXTRA_CONTENT(p) + 8 * 3));
}
return;
}
}
for (p = ZIP_CFILE_EXTRA(zcf), pe = p + ZIP_CFILE_EXTRASIZE(zcf); p + 4 <= pe;
p += ZIP_EXTRA_SIZE(p)) {
if (ZIP_EXTRA_HEADERID(p) == kZipExtraExtendedTimestamp &&
ZIP_EXTRA_CONTENTSIZE(p) > 1 &&
ZIP_EXTRA_CONTENTSIZE(p) ==
1 + 4 * popcnt3(*ZIP_EXTRA_CONTENT(p) & 7)) {
if (*ZIP_EXTRA_CONTENT(p) & 1) {
st->st_mtim.tv_sec = READ32LE(ZIP_EXTRA_CONTENT(p) + 1);
}
if (*ZIP_EXTRA_CONTENT(p) & 2) {
st->st_atim.tv_sec = READ32LE(ZIP_EXTRA_CONTENT(p) + 1 +
4 * (*ZIP_EXTRA_CONTENT(p) & 1));
}
if (*ZIP_EXTRA_CONTENT(p) & 4) {
st->st_ctim.tv_sec = READ32LE(ZIP_EXTRA_CONTENT(p) + 1 +
4 * popcnt3(*ZIP_EXTRA_CONTENT(p) & 3));
}
return;
}
}
for (p = ZIP_CFILE_EXTRA(zcf), pe = p + ZIP_CFILE_EXTRASIZE(zcf); p + 4 <= pe;
p += ZIP_EXTRA_SIZE(p)) {
if (ZIP_EXTRA_HEADERID(p) == kZipExtraUnix &&
ZIP_EXTRA_CONTENTSIZE(p) >= 4 + 4) {
st->st_atim.tv_sec = (int32_t)READ32LE(ZIP_EXTRA_CONTENT(p) + 0);
st->st_mtim.tv_sec = (int32_t)READ32LE(ZIP_EXTRA_CONTENT(p) + 4);
return;
}
}
st->st_mtim.tv_sec = DosDateTimeToUnix(ZIP_CFILE_LASTMODIFIEDDATE(zcf),
ZIP_CFILE_LASTMODIFIEDTIME(zcf));
}
int __zipos_stat_impl(struct Zipos *zipos, size_t cf, struct stat *st) {
size_t lf;
if (zipos && st) {
@ -37,6 +100,7 @@ int __zipos_stat_impl(struct Zipos *zipos, size_t cf, struct stat *st) {
st->st_size = GetZipLfileUncompressedSize(zipos->map + lf);
st->st_blocks =
roundup(GetZipLfileCompressedSize(zipos->map + lf), 512) / 512;
GetZipCfileTimestamps(zipos->map + cf, st);
return 0;
} else {
return einval();

View file

@ -16,6 +16,8 @@
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/bits/weaken.h"
#include "libc/stdio/stdio.h"
#include "libc/sysv/errfuns.h"
#include "libc/zipos/zipos.internal.h"
@ -32,9 +34,11 @@ int __zipos_stat(const struct ZiposUri *name, struct stat *st) {
if ((cf = __zipos_find(zipos, name)) != -1) {
return __zipos_stat_impl(zipos, cf, st);
} else {
ZTRACE("__zipos_stat(%`'.*s) → enoent", name->len, name->path);
return enoent();
}
} else {
ZTRACE("__zipos_stat(%`'.*s) → enoexec", name->len, name->path);
return enoexec();
}
}

View file

@ -1,8 +1,15 @@
#ifndef COSMOPOLITAN_LIBC_ZIPOS_ZIPOS_H_
#define COSMOPOLITAN_LIBC_ZIPOS_ZIPOS_H_
#include "libc/calls/calls.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
#if 0
#define ZTRACE(FMT, ...) (dprintf)(2, FMT "\n", ##__VA_ARGS__)
#else
#define ZTRACE(FMT, ...) (void)0
#endif
struct stat;
struct iovec;
@ -26,10 +33,8 @@ struct ZiposHandle {
uint8_t *freeme;
};
extern const char kZiposSchemePrefix[4];
int __zipos_close(int) hidden;
struct Zipos *__zipos_get(void) hidden;
struct Zipos *__zipos_get(void) pureconst hidden;
ssize_t __zipos_parseuri(const char *, struct ZiposUri *) hidden;
ssize_t __zipos_find(struct Zipos *, const struct ZiposUri *);
int __zipos_open(const struct ZiposUri *, unsigned, int) hidden;