Make numerous improvements

- Python static hello world now 1.8mb
- Python static fully loaded now 10mb
- Python HTTPS client now uses MbedTLS
- Python REPL now completes import stmts
- Increase stack size for Python for now
- Begin synthesizing posixpath and ntpath
- Restore Python \N{UNICODE NAME} support
- Restore Python NFKD symbol normalization
- Add optimized code path for Intel SHA-NI
- Get more Python unit tests passing faster
- Get Python help() pagination working on NT
- Python hashlib now supports MbedTLS PBKDF2
- Make memcpy/memmove/memcmp/bcmp/etc. faster
- Add Mersenne Twister and Vigna to LIBC_RAND
- Provide privileged __printf() for error code
- Fix zipos opendir() so that it reports ENOTDIR
- Add basic chmod() implementation for Windows NT
- Add Cosmo's best functions to Python cosmo module
- Pin function trace indent depth to that of caller
- Show memory diagram on invalid access in MODE=dbg
- Differentiate stack overflow on crash in MODE=dbg
- Add stb_truetype and tools for analyzing font files
- Upgrade to UNICODE 13 and reduce its binary footprint
- COMPILE.COM now logs resource usage of build commands
- Start implementing basic poll() support on bare metal
- Set getauxval(AT_EXECFN) to GetModuleFileName() on NT
- Add descriptions to strerror() in non-TINY build modes
- Add COUNTBRANCH() macro to help with micro-optimizations
- Make error / backtrace / asan / memory code more unbreakable
- Add fast perfect C implementation of μ-Law and a-Law audio codecs
- Make strtol() functions consistent with other libc implementations
- Improve Linenoise implementation (see also github.com/jart/bestline)
- COMPILE.COM now suppresses stdout/stderr of successful build commands
This commit is contained in:
Justine Tunney 2021-09-27 22:58:51 -07:00
parent fa7b4f5bd1
commit 39bf41f4eb
806 changed files with 77494 additions and 63859 deletions

View file

@ -306,7 +306,7 @@ static textwindows int err_check_handle(int64_t handle) {
}
static textwindows void tree_init(struct Tree *tree) {
memset(tree, 0, sizeof *tree);
bzero(tree, sizeof *tree);
}
static textwindows void ts_tree_init(struct TsTree *ts_tree) {
@ -492,7 +492,7 @@ static textwindows int port__close_iocp(struct PortState *port_state) {
}
static textwindows void tree_node_init(struct TreeNode *node) {
memset(node, 0, sizeof *node);
bzero(node, sizeof *node);
}
static textwindows void reflock_init(struct RefLock *reflock) {
@ -567,7 +567,7 @@ static textwindows struct PortState *port_new(int64_t *iocp_handle_out) {
if (!port_state) goto err1;
iocp_handle = port__create_iocp();
if (!iocp_handle) goto err2;
memset(port_state, 0, sizeof *port_state);
bzero(port_state, sizeof *port_state);
port_state->iocp_handle = iocp_handle;
tree_init(&port_state->sock_tree);
queue_init(&port_state->sock_update_queue);
@ -867,7 +867,7 @@ static textwindows struct PollGroup *poll_group__new(
struct Queue *poll_group_queue = port_get_poll_group_queue(port_state);
struct PollGroup *poll_group = malloc(sizeof *poll_group);
if (!poll_group) RETURN_SET_ERROR(NULL, kNtErrorNotEnoughMemory);
memset(poll_group, 0, sizeof *poll_group);
bzero(poll_group, sizeof *poll_group);
queue_node_init(&poll_group->queue_node);
poll_group->port_state = port_state;
if (afd_create_device_handle(iocp_handle, &poll_group->afd_device_handle) <
@ -1225,7 +1225,7 @@ static textwindows struct SockState *sock_new(struct PortState *port_state,
if (!poll_group) return NULL;
sock_state = sock__alloc();
if (!sock_state) goto err1;
memset(sock_state, 0, sizeof *sock_state);
bzero(sock_state, sizeof *sock_state);
sock_state->base_socket = base_socket;
sock_state->poll_group = poll_group;
tree_node_init(&sock_state->tree_node);

View file

@ -108,6 +108,7 @@ int sys_setsockopt(int, int, int, const void *, uint32_t) hidden;
int32_t sys_epoll_create(int32_t) hidden;
int32_t sys_epoll_ctl(int32_t, int32_t, int32_t, void *) hidden;
int32_t sys_epoll_wait(int32_t, void *, int32_t, int32_t) hidden;
int sys_poll_metal(struct pollfd *, size_t, unsigned);
int sys_poll_nt(struct pollfd *, uint64_t, uint64_t) hidden;
int sys_getsockopt_nt(struct Fd *, int, int, void *, uint32_t *) hidden;

80
libc/sock/poll-metal.c Normal file
View file

@ -0,0 +1,80 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi
Copyright 2021 Justine Alexandra Roberts Tunney
Permission to use, copy, modify, and/or distribute this software for
any purpose with or without fee is hereby granted, provided that the
above copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/bits/safemacros.internal.h"
#include "libc/nexgen32e/rdtsc.h"
#include "libc/nexgen32e/uart.internal.h"
#include "libc/runtime/pc.internal.h"
#include "libc/sock/internal.h"
#include "libc/sysv/consts/poll.h"
int sys_poll_metal(struct pollfd *fds, size_t nfds, unsigned timeout_ms) {
int rc;
size_t i;
bool blocking;
uint64_t start, timeout;
if (!timeout_ms) {
start = 0;
timeout = 0;
blocking = false;
} else {
start = rdtsc();
timeout = timeout_ms;
timeout *= 3; /* approx. cycles to nanoseconds */
timeout *= 1000000;
blocking = true;
}
for (rc = 0;;) {
for (i = 0; i < nfds; ++i) {
fds[i].revents = 0;
if (fds[i].fd >= 0) {
if (__isfdopen(fds[i].fd)) {
switch (g_fds.p[fds[i].fd].kind) {
case kFdSerial:
if ((fds[i].events & POLLIN) &&
(inb(g_fds.p[fds[i].fd].handle + UART_LSR) & UART_TTYDA)) {
fds[i].revents |= POLLIN;
}
if ((fds[i].events & POLLOUT) &&
(inb(g_fds.p[fds[i].fd].handle + UART_LSR) & UART_TTYTXR)) {
fds[i].revents |= POLLOUT;
}
break;
case kFdFile:
if (fds[i].events & (POLLIN | POLLOUT)) {
fds[i].revents |= fds[i].events & (POLLIN | POLLOUT);
}
break;
default:
fds[i].revents = POLLNVAL;
break;
}
} else {
fds[i].revents = POLLNVAL;
}
}
if (fds[i].revents) ++rc;
}
if (rc || !blocking || unsignedsubtract(rdtsc(), start) >= timeout) {
break;
} else {
asm("pause");
}
}
return rc;
}

View file

@ -42,7 +42,11 @@ int poll(struct pollfd *fds, uint64_t nfds, int32_t timeout_ms) {
return efault();
}
if (!IsWindows()) {
return sys_poll(fds, nfds, timeout_ms);
if (!IsMetal()) {
return sys_poll(fds, nfds, timeout_ms);
} else {
return sys_poll_metal(fds, nfds, timeout_ms);
}
} else {
return sys_poll_nt(fds, nfds, timeout_ms);
}

View file

@ -15,7 +15,7 @@ typedef struct fd_set {
#define FD_ISSET(FD, SET) (((SET)->fds_bits[(FD) >> 6] >> ((FD)&63)) & 1)
#define FD_SET(FD, SET) ((SET)->fds_bits[(FD) >> 6] |= 1ull << ((FD)&63))
#define FD_CLR(FD, SET) ((SET)->fds_bits[(FD) >> 6] &= ~(1ull << ((FD)&63)))
#define FD_ZERO(SET) memset((SET)->fds_bits, 0, sizeof((SET)->fds_bits))
#define FD_ZERO(SET) bzero((SET)->fds_bits, sizeof((SET)->fds_bits))
int select(int, fd_set *, fd_set *, fd_set *, struct timeval *);

View file

@ -33,7 +33,7 @@ static textwindows ssize_t sendfile_linux2nt(int outfd, int infd,
if (!__isfdkind(outfd, kFdSocket) || !__isfdkind(outfd, kFdFile))
return ebadf();
if (inout_opt_inoffset) {
memset(&Overlapped, 0, sizeof(Overlapped));
bzero(&Overlapped, sizeof(Overlapped));
Overlapped.Pointer = (void *)(intptr_t)(*inout_opt_inoffset);
lpOverlapped = &Overlapped;
} else {

View file

@ -54,7 +54,6 @@ textwindows int sys_socket_nt(int family, int type, int protocol) {
return __winsockerr();
}
}
sockfd = calloc(1, sizeof(struct SockFd));
sockfd->family = family;
sockfd->type = truetype;

View file

@ -16,25 +16,26 @@
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
*/
#include "libc/calls/calls.h"
#include "libc/calls/weirdtypes.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/fmt/fmt.h"
#include "libc/log/internal.h"
#include "libc/sysv/consts/log.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/af.h"
#include "libc/sysv/consts/sock.h"
#include "libc/nt/events.h"
#include "libc/nt/runtime.h"
#include "libc/runtime/valist.h"
#include "libc/stdio/stdio.h"
#include "libc/calls/calls.h"
#include "libc/str/str.h"
#include "libc/time/struct/tm.h"
#include "libc/time/time.h"
#include "libc/calls/weirdtypes.h"
#include "libc/sock/sock.h"
#include "libc/sock/syslog.h"
#include "libc/nt/runtime.h"
#include "libc/nt/events.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/af.h"
#include "libc/sysv/consts/clock.h"
#include "libc/sysv/consts/log.h"
#include "libc/sysv/consts/o.h"
#include "libc/sysv/consts/sock.h"
#include "libc/time/struct/tm.h"
#include "libc/time/time.h"
/* Note: log_facility should be initialized with LOG_USER by default,
* but since LOG_USER is not a constant value, we cannot initialize it
@ -45,46 +46,46 @@ static int log_facility = -1;
static char log_ident[32];
static int log_opt;
static int log_mask;
static uint16_t log_id; /* Used for Windows EvtID */
static uint16_t log_id; /* Used for Windows EvtID */
static int64_t log_fd = -1;
static const struct sockaddr_un log_addr = {AF_UNIX, "/dev/log"};
static const struct sockaddr_un log_addr = {
AF_UNIX,
"/dev/log"
};
static int64_t Time(int64_t *tp) {
struct timespec ts;
clock_gettime(CLOCK_REALTIME, &ts);
if (tp) *tp = ts.tv_sec;
return ts.tv_sec;
}
static void __initlog() {
log_ident[0] = '\0';
log_opt = LOG_ODELAY;
log_facility = LOG_USER;
log_mask = LOG_PRI(0xff); // Initially use max verbosity
log_mask = LOG_PRI(0xff); // Initially use max verbosity
log_fd = -1;
}
forceinline int is_lost_conn(int e) {
return (e==ECONNREFUSED) ||
(e==ECONNRESET) ||
(e==ENOTCONN) ||
(e==EPIPE);
return (e == ECONNREFUSED) || (e == ECONNRESET) || (e == ENOTCONN) ||
(e == EPIPE);
}
static void __openlog() {
if (IsWindows()) {
log_fd = RegisterEventSourceA(NULL, log_ident);
} else {
log_fd = socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0);
log_fd = socket(AF_UNIX, SOCK_DGRAM | SOCK_CLOEXEC, 0);
if (log_fd >= 0) {
int rc = connect(log_fd, (void *)&log_addr, sizeof(log_addr));
if (rc < 0) {
printf("ERR: connect(openlog) failed: %s (errno=%d)\n", strerror(errno), errno);
}
int rc = connect(log_fd, (void *)&log_addr, sizeof(log_addr));
if (rc < 0) {
printf("ERR: connect(openlog) failed: %s (errno=%d)\n", strerror(errno),
errno);
}
}
}
}
/**
* Generates a log message which will be distributed by syslogd
*
@ -95,14 +96,14 @@ static void __openlog() {
* the level value. If no facility value is ORed into priority,
* then the default value set by openlog() is used.
* it set to NULL, the program name is used.
* Level is one of LOG_EMERG, LOG_ALERT, LOG_CRIT, LOG_ERR,
* Level is one of LOG_EMERG, LOG_ALERT, LOG_CRIT, LOG_ERR,
* LOG_WARNING, LOG_NOTICE, LOG_INFO, LOG_DEBUG
* @param message the format of the message to be processed by vprintf()
* @param ap the va_list of arguments to be applied to the message
* @asyncsignalsafe
*/
void vsyslog(int priority, const char *message, va_list ap) {
char timebuf[16]; /* Store formatted time */
char timebuf[16]; /* Store formatted time */
time_t now;
struct tm tm;
char buf[1024];
@ -117,7 +118,7 @@ void vsyslog(int priority, const char *message, va_list ap) {
if (!(priority & LOG_FACMASK)) priority |= log_facility;
/* Build the time string */
now = time(NULL);
now = Time(NULL);
gmtime_r(&now, &tm);
strftime(timebuf, sizeof(timebuf), "%b %e %T", &tm);
@ -126,33 +127,34 @@ void vsyslog(int priority, const char *message, va_list ap) {
* only if pid != 0. When pid==0, the while "[%.0d]" is skipped:
* %s%.0d%s -> String, pidValue, String
* Each of those %s:
* - if pid == 0 -> !pid is true (1), so "[" + 1 points to the
* - if pid == 0 -> !pid is true (1), so "[" + 1 points to the
* NULL terminator after the "[".
* - if pid != 0 -> !pid is false (0), so the string printed is
* the "[".
*/
l = snprintf(buf, sizeof(buf), "<%d>%s ", priority, timebuf);
hlen = l;
l += snprintf(buf+l, sizeof(buf)-l, "%s%s%.0d%s: ", log_ident, "["+!pid, pid, "]"+!pid);
l += snprintf(buf + l, sizeof(buf) - l, "%s%s%.0d%s: ", log_ident, "[" + !pid,
pid, "]" + !pid);
errno = errno_save;
/* Append user message */
l2 = vsnprintf(buf+l, sizeof(buf)-l, message, ap);
l2 = vsnprintf(buf + l, sizeof(buf) - l, message, ap);
if (l2 >= 0) {
if (l2 >= sizeof(buf) - l) {
l = sizeof(buf) - 1;
} else {
l += l2;
}
if (buf[l-1] != '\n') {
if (buf[l - 1] != '\n') {
buf[l++] = '\n';
}
if (!IsWindows()) {
/* Unix:
/* Unix:
* - First try to send it to syslogd
* - If fails and LOG_CONS is provided, writes to /dev/console
*/
#if 0
if (send(log_fd, buf, l, 0) < 0 && (!is_lost_conn(errno)
|| connect(log_fd, (void *)&log_addr, sizeof(log_addr)) < 0
@ -171,11 +173,13 @@ void vsyslog(int priority, const char *message, va_list ap) {
if (!is_lost_conn(errno)) {
rc = connect(log_fd, (void *)&log_addr, sizeof(log_addr));
if (rc < 0) {
printf("ERR: connect(syslog) failed: %s (errno=%d)\n", strerror(errno), errno);
printf("ERR: connect(syslog) failed: %s (errno=%d)\n",
strerror(errno), errno);
} else {
rc = send(log_fd, buf, l, 0);
if (rc < 0) {
printf("ERR: send(2) failed: %s (errno=%d)\n", strerror(errno), errno);
printf("ERR: send(2) failed: %s (errno=%d)\n", strerror(errno),
errno);
}
}
}
@ -184,15 +188,15 @@ void vsyslog(int priority, const char *message, va_list ap) {
} else {
uint16_t evtType;
uint32_t evtID;
const char *bufArr[] = { &buf[hlen] }; /* Only print the message without time*/
/* Windows only have 3 usable event types
const char *bufArr[] = {
&buf[hlen]}; /* Only print the message without time*/
/* Windows only have 3 usable event types
* Event ID are not supported
* For more information on message types and event IDs, see:
* For more information on message types and event IDs, see:
* https://docs.microsoft.com/en-us/windows/win32/eventlog/event-identifiers
*/
priority &= LOG_PRIMASK; // Remove facility from the priority field
if (priority == LOG_EMERG ||
priority == LOG_ALERT ||
if (priority == LOG_EMERG || priority == LOG_ALERT ||
priority == LOG_CRIT) {
evtType = EVENTLOG_ERROR_TYPE;
evtID = 0xe << 28 | (LOG_USER) << 16;
@ -204,30 +208,27 @@ void vsyslog(int priority, const char *message, va_list ap) {
evtType = EVENTLOG_INFORMATION_TYPE;
evtID = 0x6 << 28 | (LOG_USER) << 16;
}
ReportEventA(log_fd,
evtType, /* Derived from priority */
0 /* Category unsupported */,
evtID, /* Unsupported */
NULL, /* User SID */
1, /* Number of strings */
0, /* Raw data size */
bufArr, /* String(s) */
NULL /* Arguments */ );
ReportEventA(log_fd, evtType, /* Derived from priority */
0 /* Category unsupported */, evtID, /* Unsupported */
NULL, /* User SID */
1, /* Number of strings */
0, /* Raw data size */
bufArr, /* String(s) */
NULL /* Arguments */);
++log_id;
}
if (log_opt & LOG_PERROR) dprintf(2, "%.*s", l-hlen, buf+hlen);
if (log_opt & LOG_PERROR) dprintf(2, "%.*s", l - hlen, buf + hlen);
}
}
/**
* Sets log priority mask
*
* Modifies the log priority mask that determines which calls to
* Modifies the log priority mask that determines which calls to
* syslog() may be logged.
* Log priority values are LOG_EMERG, LOG_ALERT, LOG_CRIT, LOG_ERR,
* LOG_WARNING, LOG_NOTICE, LOG_INFO, and LOG_DEBUG.
* LOG_WARNING, LOG_NOTICE, LOG_INFO, and LOG_DEBUG.
*
* @param mask the new priority mask to use by syslog()
* @return the previous log priority mask
@ -243,12 +244,11 @@ int setlogmask(int maskpri) {
return ret;
}
/**
* Opens a connection to the system logger
*
* Calling this function before calling syslog() is optional and
* only allow customizing the identity, options and facility of
* Calling this function before calling syslog() is optional and
* only allow customizing the identity, options and facility of
* the messages logged.
*
* @param ident a string that prepends every logged message. If
@ -287,10 +287,9 @@ void openlog(const char *ident, int opt, int facility) {
log_facility = facility;
log_id = 0;
if ((opt & LOG_NDELAY) && log_fd<0) __openlog();
if ((opt & LOG_NDELAY) && log_fd < 0) __openlog();
}
/**
* Generates a log message which will be distributed by syslogd
*
@ -301,7 +300,7 @@ void openlog(const char *ident, int opt, int facility) {
* the level value. If no facility value is ORed into priority,
* then the default value set by openlog() is used.
* it set to NULL, the program name is used.
* Level is one of LOG_EMERG, LOG_ALERT, LOG_CRIT, LOG_ERR,
* Level is one of LOG_EMERG, LOG_ALERT, LOG_CRIT, LOG_ERR,
* LOG_WARNING, LOG_NOTICE, LOG_INFO, LOG_DEBUG
* @param message the message formatted using the same rules as printf()
* @asyncsignalsafe
@ -318,7 +317,6 @@ void syslog(int priority, const char *message, ...) {
}
}
/**
* Closes the file descriptor being used to write to the system logger
*
@ -339,5 +337,3 @@ void closelog(void) {
}
log_fd = -1;
}

View file

@ -29,7 +29,7 @@ textwindows int64_t __winsockblock(int64_t fh, unsigned eventbit, int64_t rc) {
if (rc != -1) return rc;
if (WSAGetLastError() != EWOULDBLOCK) return __winsockerr();
eh = WSACreateEvent();
memset(&ev, 0, sizeof(ev));
bzero(&ev, sizeof(ev));
if (WSAEventSelect(fh, eh, 1u << eventbit) != -1 &&
WSAEnumNetworkEvents(fh, eh, &ev) != -1) {
if (!ev.iErrorCode[eventbit]) {