mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-07-06 11:18:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve ZIP filesystem and change its prefix

Browse source 
The ZIP filesystem has a breaking change. You now need to use /zip/ to
open() / opendir() / etc. assets within the ZIP structure of your APE
binary, instead of the previous convention of using zip: or zip! URIs.
This is needed because Python likes to use absolute paths, and having
ZIP paths encoded like URIs simply broke too many things.

Many more system calls have been updated to be able to operate on ZIP
files and file descriptors. In particular fcntl() and ioctl() since
Python would do things like ask if a ZIP file is a terminal and get
confused when the old implementation mistakenly said yes, because the
fastest way to guarantee native file descriptors is to dup(2). This
change also improves the async signal safety of zipos and ensures it
doesn't maintain any open file descriptors beyond that which the user
has opened.

This change makes a lot of progress towards adding magic numbers that
are specific to platforms other than Linux. The philosophy here is that,
if you use an operating system like FreeBSD, then you should be able to
take advantage of FreeBSD exclusive features, even if we don't polyfill
them on other platforms. For example, you can now open() a file with the
O_VERIFY flag. If your program runs on other platforms, then Cosmo will
automatically set O_VERIFY to zero. This lets you safely use it without
the need for #ifdef or ifstatements which detract from readability.

One of the blindspots of the ASAN memory hardening we use to offer Rust
like assurances has always been that memory passed to the kernel via
system calls (e.g. writev) can't be checked automatically since the
kernel wasn't built with MODE=asan. This change makes more progress
ensuring that each system call will verify the soundness of memory
before it's passed to the kernel. The code for doing these checks is
fast, particularly for buffers, where it can verify 64 bytes a cycle.

- Correct O_LOOP definition on NT
- Introduce program_executable_name
- Add ASAN guards to more system calls
- Improve termios compatibility with BSDs
- Fix bug in Windows auxiliary value encoding
- Add BSD and XNU specific errnos and open flags
- Add check to ensure build doesn't talk to internet
This commit is contained in:
Justine Tunney 2021-08-22 01:04:18 -07:00
parent 2730c66f4a
commit 00611e9b06
319 changed files with 4418 additions and 2599 deletions
Download patch file Download diff file Expand all files Collapse all files

57
libc/zipos/fcntl.c Normal file
View file

@ -0,0 +1,57 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/internal.h"
#include "libc/sysv/consts/f.h"
#include "libc/sysv/consts/fd.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/errfuns.h"
#include "libc/zip.h"
#include "libc/zipos/zipos.internal.h"
#define ZIPOS __zipos_get()
#define HANDLE ((struct ZiposHandle *)(intptr_t)g_fds.p[fd].handle)
int __zipos_fcntl(int fd, int cmd, uintptr_t arg) {
if (cmd == F_GETFD) {
ZTRACE("__zipos_fcntl(%`'.*s, %s)",
ZIP_CFILE_NAMESIZE(ZIPOS->map + HANDLE->cfile),
ZIP_CFILE_NAME(ZIPOS->map + HANDLE->cfile), "F_GETFD");
if (g_fds.p[fd].flags & O_CLOEXEC) {
return FD_CLOEXEC;
} else {
return 0;
}
} else if (cmd == F_SETFD) {
ZTRACE("__zipos_fcntl(%`'.*s, %s, 0x%016lx)",
ZIP_CFILE_NAMESIZE(ZIPOS->map + HANDLE->cfile),
ZIP_CFILE_NAME(ZIPOS->map + HANDLE->cfile), "F_SETFD", arg);
if (arg & FD_CLOEXEC) {
g_fds.p[fd].flags |= O_CLOEXEC;
return FD_CLOEXEC;
} else {
g_fds.p[fd].flags &= ~O_CLOEXEC;
return 0;
}
} else {
ZTRACE("__zipos_fcntl(%`'.*s, %d, 0x%016lx) → EINVAL",
ZIP_CFILE_NAMESIZE(ZIPOS->map + HANDLE->cfile),
ZIP_CFILE_NAME(ZIPOS->map + HANDLE->cfile), cmd, arg);
return einval();
}
}

2
libc/zipos/fstat.c
View file

@ -16,6 +16,7 @@
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/sysv/errfuns.h"
#include "libc/zip.h"
#include "libc/zipos/zipos.internal.h"
@ -27,6 +28,7 @@
*/
int __zipos_fstat(const struct ZiposHandle *h, struct stat *st) {
int rc;
if (!st) return efault();
if (!(rc = __zipos_stat_impl(__zipos_get(), h->cfile, st))) {
ZTRACE("__zipos_fstat(%`'.*s) → %ld",
ZIP_CFILE_NAMESIZE(__zipos_get()->map + h->cfile),

53
libc/zipos/get.c
View file

@ -18,14 +18,18 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/safemacros.internal.h"
#include "libc/calls/calls.h"
#include "libc/calls/sigbits.h"
#include "libc/calls/struct/stat.h"
#include "libc/errno.h"
#include "libc/limits.h"
#include "libc/mem/alloca.h"
#include "libc/runtime/runtime.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/auxv.h"
#include "libc/sysv/consts/map.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/prot.h"
#include "libc/sysv/consts/sig.h"
#include "libc/zip.h"
#include "libc/zipos/zipos.internal.h"
@ -33,41 +37,44 @@
* Returns pointer to zip central directory of current executable.
*/
struct Zipos *__zipos_get(void) {
static struct Zipos zipos;
static bool once;
const char *exe, *dir;
char path[PATH_MAX];
size_t size;
uint8_t *map, *base, *cdir;
static struct Zipos zipos;
int fd;
size_t n;
char *path;
sigset_t neu, old;
uint8_t *p, *base, *cdir;
if (!once) {
dir = nulltoempty(getenv("PWD")); /* suboptimal */
exe = (const char *)getauxval(AT_EXECFN);
if (!fileexists(exe) && strlen(dir) + 1 + strlen(exe) + 1 <= PATH_MAX) {
stpcpy(stpcpy(stpcpy(path, dir), "/"), exe);
exe = path;
}
if ((zipos.fd = open(exe, O_RDONLY)) != -1) {
if ((size = getfiledescriptorsize(zipos.fd)) != SIZE_MAX &&
(map = mmap(NULL, size, PROT_READ, MAP_SHARED, zipos.fd, 0)) !=
MAP_FAILED) {
if (endswith(exe, ".com.dbg")) {
if ((base = memmem(map, size, "MZqFpD", 6))) {
size -= base - map;
sigfillset(&neu);
sigprocmask(SIG_BLOCK, &neu, &old);
if ((fd = open(program_executable_name, O_RDONLY)) != -1) {
if ((n = getfiledescriptorsize(fd)) != SIZE_MAX &&
(p = mmap(0, n, PROT_READ, MAP_SHARED, fd, 0)) != MAP_FAILED) {
if (endswith(program_executable_name, ".com.dbg")) {
if ((base = memmem(p, n, "MZqFpD", 6))) {
n -= base - p;
} else {
base = map;
base = p;
}
} else {
base = map;
base = p;
}
if ((cdir = GetZipCdir(base, size))) {
if ((cdir = GetZipCdir(base, n))) {
zipos.map = base;
zipos.cdir = cdir;
} else {
munmap(map, size);
munmap(p, n);
ZTRACE("__zipos_get(%`'s) → eocd not found", program_executable_name);
}
} else {
ZTRACE("__zipos_get(%`'s) → stat/mmap %m", program_executable_name);
}
close(fd);
} else {
ZTRACE("__zipos_get(%`'s) → open %m", program_executable_name);
}
once = true;
sigprocmask(SIG_SETMASK, &old, 0);
}
return zipos.cdir ? &zipos : NULL;
return zipos.cdir ? &zipos : 0;
}

32
libc/zipos/notat.c Normal file
View file

@ -0,0 +1,32 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/bits/weaken.h"
#include "libc/calls/internal.h"
#include "libc/sysv/errfuns.h"
#include "libc/zipos/zipos.internal.h"
int __zipos_notat(int dirfd, const char *path) {
struct ZiposUri zipname;
if (!path) return efault();
if (__isfdkind(dirfd, kFdZip) || __zipos_parseuri(path, &zipname) != -1) {
ZTRACE("__zipos_notat(%d, %`'s) → EINVAL", dirfd, path);
return einval();
}
return 0;
}

47
libc/zipos/open.c
View file

@ -86,23 +86,34 @@ static int __zipos_inflate_tiny(struct ZiposHandle *h, uint8_t *data,
static int __zipos_load(struct Zipos *zipos, size_t cf, unsigned flags,
int mode) {
int fd;
size_t lf;
int rc, fd;
size_t size;
uint8_t *data;
struct ZiposHandle *h;
lf = GetZipCfileOffset(zipos->map + cf);
assert(ZIP_LFILE_MAGIC(zipos->map + lf) == kZipLfileHdrMagic);
assert(ZIP_LFILE_COMPRESSIONMETHOD(zipos->map + lf) == kZipCompressionNone ||
ZIP_LFILE_COMPRESSIONMETHOD(zipos->map + lf) ==
kZipCompressionDeflate);
if (!IsTiny() &&
(ZIP_LFILE_MAGIC(zipos->map + lf) != kZipLfileHdrMagic ||
(ZIP_LFILE_COMPRESSIONMETHOD(zipos->map + lf) != kZipCompressionNone &&
ZIP_LFILE_COMPRESSIONMETHOD(zipos->map + lf) !=
kZipCompressionDeflate))) {
return eio();
}
if (!(h = calloc(1, sizeof(*h)))) return -1;
h->cfile = cf;
h->size = GetZipLfileUncompressedSize(zipos->map + lf);
if (ZIP_LFILE_COMPRESSIONMETHOD(zipos->map + lf)) {
if ((h->freeme = malloc(h->size)) &&
(IsTiny() ? __zipos_inflate_tiny : __zipos_inflate_fast)(
h, ZIP_LFILE_CONTENT(zipos->map + lf),
GetZipLfileCompressedSize(zipos->map + lf)) != -1) {
h->mem = h->freeme;
if ((h->freeme = malloc(h->size))) {
data = ZIP_LFILE_CONTENT(zipos->map + lf);
size = GetZipLfileCompressedSize(zipos->map + lf);
if (IsTiny()) {
rc = __zipos_inflate_tiny(h, data, size);
} else {
rc = __zipos_inflate_fast(h, data, size);
}
if (rc != -1) {
h->mem = h->freeme;
}
}
} else {
h->mem = ZIP_LFILE_CONTENT(zipos->map + lf);
@ -113,14 +124,16 @@ static int __zipos_load(struct Zipos *zipos, size_t cf, unsigned flags,
h->mem = NULL;
}
if (h->mem) {
if (__ensurefds((fd = dup(zipos->fd))) != -1) {
h->handle = g_fds.p[fd].handle;
g_fds.p[fd].kind = kFdZip;
g_fds.p[fd].handle = (intptr_t)h;
g_fds.p[fd].flags = flags;
return fd;
if ((fd = dup(2)) != -1) {
if (__ensurefds(fd) != -1) {
h->handle = g_fds.p[fd].handle;
g_fds.p[fd].kind = kFdZip;
g_fds.p[fd].handle = (intptr_t)h;
g_fds.p[fd].flags = flags | O_CLOEXEC;
return fd;
}
close(fd);
}
close(fd);
}
free(h->freeme);
free(h);

10
libc/zipos/parseuri.c
View file

@ -19,18 +19,16 @@
#include "libc/str/str.h"
#include "libc/zipos/zipos.internal.h"
const char kZiposSchemePrefix[4] hidden = "zip:";
/**
* Extracts information about ZIP URI if it is one.
*/
ssize_t __zipos_parseuri(const char *uri, struct ZiposUri *out) {
size_t len;
if ((uri[0] == 'z' && uri[1] == 'i' && uri[2] == 'p' &&
(uri[3] == ':' || uri[3] == '!')) &&
if ((uri[0] == '/' && uri[1] == 'z' && uri[2] == 'i' && uri[3] == 'p' &&
(!uri[4] || uri[4] == '/')) &&
(len = strlen(uri)) < PATH_MAX) {
out->path = uri + 4;
return (out->len = len - 4);
out->path = uri + 4 + !!uri[4];
return (out->len = len - 4 - !!uri[4]);
} else {
return -1;
}

1
libc/zipos/stat.c
View file

@ -30,6 +30,7 @@
int __zipos_stat(const struct ZiposUri *name, struct stat *st) {
ssize_t cf;
struct Zipos *zipos;
if (!st) return efault();
if ((zipos = __zipos_get())) {
if ((cf = __zipos_find(zipos, name)) != -1) {
return __zipos_stat_impl(zipos, cf, st);

8
libc/zipos/zipos.S
View file

@ -18,16 +18,18 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
// static_yoink this symbol for open(zip:...) support.
// static_yoink this symbol for open(/zip/...) support.
zip_uri_support = 0
.globl zip_uri_support
.yoink __zip_start
.yoink __zip_end
.yoink __zip_start
.yoink __zipos_close
.yoink __zipos_fcntl
.yoink __zipos_fstat
.yoink __zipos_open
.yoink __zipos_lseek
.yoink __zipos_open
.yoink __zipos_parseuri
.yoink __zipos_read
.yoink __zipos_stat
.yoink __zipos_notat

3
libc/zipos/zipos.internal.h
View file

@ -14,7 +14,6 @@ struct stat;
struct iovec;
struct Zipos {
int fd;
uint8_t *map;
uint8_t *cdir;
};
@ -46,6 +45,8 @@ ssize_t __zipos_read(struct ZiposHandle *, const struct iovec *, size_t,
ssize_t __zipos_write(struct ZiposHandle *, const struct iovec *, size_t,
ssize_t) hidden;
int64_t __zipos_lseek(struct ZiposHandle *, int64_t, unsigned) hidden;
int __zipos_fcntl(int, int, uintptr_t) hidden;
int __zipos_notat(int, const char *) hidden;
COSMOPOLITAN_C_END_
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */