Add HTTP/HTTPS Fetch() API to redbean

You can now say the following in your redbean Lua code:

    status,headers,payload = Fetch("https://foo.example")

The following Lua APIs have been introduced:

  - Fetch(str) → str,{str:str},str
  - GetHttpReason(int) → str
  - GetHttpReason(int) → str
  - ProgramSslFetchVerify(bool)
  - ProgramSslClientVerify(bool)

The following flags have been introduced:

  - `-j` enables client SSL verification
  - `-k` disables Fetch() SSL verification
  - `-t INT` may now be passed a negative value for keepalive

Lua exceptions now invoke Cosmopolitan's garbage collector when
unwinding the stack. So it's now safe to use _gc() w/ Lua 𝔱𝔥𝔯𝔬𝔴

See #97

Makefile

@@ -130,13 +130,14 @@ include libc/sock/sock.mk			#─┐
 include dsp/tty/tty.mk				# ├──ONLINE RUNTIME
 include libc/dns/dns.mk				# │  You can communicate with the network
 include net/http/http.mk			# │
+include third_party/mbedtls/mbedtls.mk		# │
+include net/https/https.mk			# │
 include third_party/regex/regex.mk		#─┘
 include third_party/third_party.mk
 include libc/testlib/testlib.mk
 include tool/viz/lib/vizlib.mk
 include third_party/lua/lua.mk
 include third_party/sqlite3/sqlite3.mk
-include third_party/mbedtls/mbedtls.mk
 include third_party/mbedtls/test/test.mk
 include third_party/quickjs/quickjs.mk
 include third_party/lz4cli/lz4cli.mk

examples/curl.c

@@ -9,6 +9,7 @@
 #endif
 #include "libc/bits/safemacros.internal.h"
 #include "libc/calls/calls.h"
+#include "libc/dce.h"
 #include "libc/dns/dns.h"
 #include "libc/errno.h"
 #include "libc/fmt/conv.h"
@@ -24,33 +25,26 @@
 #include "libc/str/str.h"
 #include "libc/sysv/consts/af.h"
 #include "libc/sysv/consts/dt.h"
+#include "libc/sysv/consts/ex.h"
+#include "libc/sysv/consts/exit.h"
 #include "libc/sysv/consts/ipproto.h"
 #include "libc/sysv/consts/shut.h"
+#include "libc/sysv/consts/so.h"
 #include "libc/sysv/consts/sock.h"
+#include "libc/sysv/consts/sol.h"
+#include "libc/sysv/consts/tcp.h"
+#include "libc/time/struct/tm.h"
 #include "libc/x/x.h"
 #include "net/http/http.h"
 #include "net/http/url.h"
+#include "net/https/https.h"
+#include "third_party/getopt/getopt.h"
 #include "third_party/mbedtls/ctr_drbg.h"
 #include "third_party/mbedtls/debug.h"
 #include "third_party/mbedtls/error.h"
+#include "third_party/mbedtls/pk.h"
 #include "third_party/mbedtls/ssl.h"
 
-STATIC_YOINK("zip_uri_support");
-STATIC_YOINK("usr/share/ssl/root/amazon.pem");
-STATIC_YOINK("usr/share/ssl/root/certum.pem");
-STATIC_YOINK("usr/share/ssl/root/comodo.pem");
-STATIC_YOINK("usr/share/ssl/root/digicert.pem");
-STATIC_YOINK("usr/share/ssl/root/dst.pem");
-STATIC_YOINK("usr/share/ssl/root/geotrust.pem");
-STATIC_YOINK("usr/share/ssl/root/globalsign.pem");
-STATIC_YOINK("usr/share/ssl/root/godaddy.pem");
-STATIC_YOINK("usr/share/ssl/root/google.pem");
-STATIC_YOINK("usr/share/ssl/root/isrg.pem");
-STATIC_YOINK("usr/share/ssl/root/quovadis.pem");
-STATIC_YOINK("usr/share/ssl/root/redbean.pem");
-STATIC_YOINK("usr/share/ssl/root/starfield.pem");
-STATIC_YOINK("usr/share/ssl/root/verisign.pem");
-
 /**
  * @fileoverview Downloads HTTP URL to stdout.
  *
@@ -58,45 +52,161 @@ STATIC_YOINK("usr/share/ssl/root/verisign.pem");
  *     o//examples/curl.com http://justine.lol/ape.html
  */
 
+#define HasHeader(H)    (!!msg.headers[H].a)
+#define HeaderData(H)   (p + msg.headers[H].a)
+#define HeaderLength(H) (msg.headers[H].b - msg.headers[H].a)
+#define HeaderEqualCase(H, S) \
+  SlicesEqualCase(S, strlen(S), HeaderData(H), HeaderLength(H))
+
+struct Buffer {
+  size_t i, n;
+  char *p;
+};
+
+static inline bool SlicesEqualCase(const char *a, size_t n, const char *b,
+                                   size_t m) {
+  return n == m && !memcasecmp(a, b, n);
+}
+
+static bool TuneSocket(int fd, int a, int b, int x) {
+  if (!b) return false;
+  return setsockopt(fd, a, b, &x, sizeof(x)) != -1;
+}
+
+static int Socket(int family, int type, int protocol) {
+  int fd;
+  if ((fd = socket(family, type, protocol)) != -1) {
+    TuneSocket(fd, SOL_SOCKET, SO_KEEPALIVE, 1);
+    if (protocol == SOL_TCP) {
+      TuneSocket(fd, SOL_TCP, TCP_KEEPIDLE, 60);
+      TuneSocket(fd, SOL_TCP, TCP_KEEPINTVL, 60);
+      TuneSocket(fd, SOL_TCP, TCP_FASTOPEN_CONNECT, 1);
+      if (!TuneSocket(fd, SOL_TCP, TCP_QUICKACK, 1)) {
+        TuneSocket(fd, SOL_TCP, TCP_NODELAY, 1);
+      }
+    }
+  }
+  return fd;
+}
+
+static int TlsSend(void *c, const unsigned char *p, size_t n) {
+  int rc;
+  CHECK_NE(-1, (rc = write(*(int *)c, p, n)));
+  return rc;
+}
+
+static int TlsRecv(void *c, unsigned char *p, size_t n, uint32_t t) {
+  int rc;
+  CHECK_NE(-1, (rc = read(*(int *)c, p, n)));
+  return rc;
+}
+
+static void TlsDebug(void *c, int v, const char *f, int l, const char *s) {
+  flogf(v, f, l, 0, "TLS %s", s);
+}
+
+static char *TlsError(int r) {
+  static char b[128];
+  mbedtls_strerror(r, b, sizeof(b));
+  return b;
+}
+
+static wontreturn void PrintUsage(FILE *f, int rc) {
+  fprintf(f, "usage: %s [-ksvV] URL\n", program_invocation_name);
+  exit(rc);
+}
+
+static wontreturn void TlsDie(const char *s, int r) {
+  if (IsTiny()) {
+    fprintf(stderr, "error: %s (-0x%04x %s)\n", s, -r, TlsError(r));
+  } else {
+    fprintf(stderr, "error: %s (grep -0x%04x)\n", s, -r);
+  }
+  exit(1);
+}
+
 static int GetEntropy(void *c, unsigned char *p, size_t n) {
   CHECK_EQ(n, getrandom(p, n, 0));
   return 0;
 }
 
-static int TlsSend(void *ctx, const unsigned char *buf, size_t len) {
+static int AppendFmt(struct Buffer *b, const char *fmt, ...) {
-  int rc;
+  int n;
-  CHECK_NE(-1, (rc = write(*(int *)ctx, buf, len)));
+  char *p;
-  return rc;
+  va_list va, vb;
-}
+  va_start(va, fmt);
-
+  va_copy(vb, va);
-static int TlsRecv(void *ctx, unsigned char *buf, size_t len, uint32_t tmo) {
+  n = vsnprintf(b->p + b->i, b->n - b->i, fmt, va);
-  int rc;
+  if (n >= b->n - b->i) {
-  CHECK_NE(-1, (rc = read(*(int *)ctx, buf, len)));
+    do {
-  return rc;
+      if (b->n) {
-}
+        b->n += b->n >> 1;
-
+      } else {
-static void TlsDebug(void *ctx, int level, const char *file, int line,
+        b->n = 16;
-                     const char *message) {
+      }
-  flogf(level, file, line, 0, "TLS %s", message);
+    } while (b->i + n > b->n);
-}
+    b->p = realloc(b->p, b->n);
-
+    vsnprintf(b->p + b->i, b->n - b->i, fmt, vb);
-static char *TlsError(int rc) {
+  }
-  static char ebuf[128];
+  va_end(vb);
-  mbedtls_strerror(rc, ebuf, sizeof(ebuf));
+  va_end(va);
-  return ebuf;
+  b->i += n;
+  return n;
 }
 
 int main(int argc, char *argv[]) {
+  if (!NoDebug()) showcrashreports();
+
+  /*
+   * Read flags.
+   */
+  int opt;
+  struct Headers {
+    size_t n;
+    char **p;
+  } headers = {0};
+  int method = kHttpGet;
+  bool authmode = MBEDTLS_SSL_VERIFY_REQUIRED;
+  const char *agent = "hurl/1.o (https://github.com/jart/cosmopolitan)";
+  while ((opt = getopt(argc, argv, "qksvVIX:H:A:")) != -1) {
+    switch (opt) {
+      case 's':
+      case 'q':
+        break;
+      case 'v':
+        break;
+      case 'I':
+        method = kHttpHead;
+        break;
+      case 'A':
+        agent = optarg;
+        break;
+      case 'H':
+        headers.p = realloc(headers.p, ++headers.n * sizeof(*headers.p));
+        headers.p[headers.n - 1] = optarg;
+        break;
+      case 'X':
+        CHECK((method = GetHttpMethod(optarg, strlen(optarg))));
+        break;
+      case 'V':
+        ++mbedtls_debug_threshold;
+        break;
+      case 'k':
+        authmode = MBEDTLS_SSL_VERIFY_NONE;
+        break;
+      case 'h':
+        PrintUsage(stdout, EXIT_SUCCESS);
+      default:
+        PrintUsage(stderr, EX_USAGE);
+    }
+  }
 
   /*
    * Get argument.
    */
   const char *urlarg;
-  if (argc != 2) {
+  if (optind == argc) PrintUsage(stderr, EX_USAGE);
-    fprintf(stderr, "usage: %s URL\n", argv[0]);
+  urlarg = argv[optind];
-    exit(1);
-  }
-  urlarg = argv[1];
 
   /*
    * Parse URL.
@@ -114,10 +224,17 @@ int main(int argc, char *argv[]) {
       exit(1);
     }
   }
-  host = firstnonnull(_gc(strndup(url.host.p, url.host.n)), "127.0.0.1");
+  if (url.host.n) {
-  port = url.port.n ? _gc(strndup(url.port.p, url.port.n))
+    host = _gc(strndup(url.host.p, url.host.n));
-                    : (usessl ? "443" : "80");
+    if (url.port.n) {
-  port = _gc(xasprintf("%hu", atoi(port)));
+      port = _gc(strndup(url.port.p, url.port.n));
+    } else {
+      port = usessl ? "443" : "80";
+    }
+  } else {
+    host = "127.0.0.1";
+    port = usessl ? "443" : "80";
+  }
   if (!IsAcceptableHost(host, -1)) {
     fprintf(stderr, "error: invalid host: %s\n", urlarg);
     exit(1);
@@ -138,40 +255,17 @@ int main(int argc, char *argv[]) {
   /*
    * Create HTTP message.
    */
-  const char *msg;
+  struct Buffer request = {0};
-  msg = _gc(xasprintf("GET %s HTTP/1.1\r\n"
+  AppendFmt(&request,
-                      "Host: %s:%s\r\n"
+            "%s %s HTTP/1.1\r\n"
-                      "Connection: close\r\n"
+            "Host: %s:%s\r\n"
-                      "Content-Length: 0\r\n"
+            "Connection: close\r\n"
-                      "Accept: text/plain; */*\r\n"
+            "User-Agent: %s\r\n",
-                      "User-Agent: github.com/jart/cosmopolitan\r\n"
+            kHttpMethod[method], _gc(EncodeUrl(&url, 0)), host, port, agent);
-                      "\r\n",
+  for (int i = 0; i < headers.n; ++i) {
-                      _gc(EncodeUrl(&url, 0)), host, port));
+    AppendFmt(&request, "%s\r\n", headers.p[i]);
-
-  /*
-   * Load root certificates.
-   */
-  mbedtls_x509_crt cacert;
-  if (usessl) {
-    DIR *zd;
-    size_t calen;
-    const char *dir;
-    char capath[300];
-    uint8_t *cabytes;
-    struct dirent *ze;
-    mbedtls_x509_crt_init(&cacert);
-    dir = "zip:usr/share/ssl/root";
-    CHECK((zd = opendir(dir)), "%s", dir);
-    while ((ze = readdir(zd))) {
-      if (ze->d_type != DT_REG) continue;
-      snprintf(capath, sizeof(capath), "%s/%s", dir, ze->d_name);
-      CHECK((cabytes = xslurp(capath, &calen)));
-      CHECK_EQ(0, mbedtls_x509_crt_parse(&cacert, cabytes, calen + 1), "%s",
-               capath);
-      free(cabytes);
-    }
-    closedir(zd);
   }
+  AppendFmt(&request, "\r\n");
 
   /*
    * Setup crypto.
@@ -183,16 +277,14 @@ int main(int argc, char *argv[]) {
     mbedtls_ssl_init(&ssl);
     mbedtls_ctr_drbg_init(&drbg);
     mbedtls_ssl_config_init(&conf);
-    CHECK_EQ(0, mbedtls_ctr_drbg_seed(&drbg, GetEntropy, 0,
+    CHECK_EQ(0, mbedtls_ctr_drbg_seed(&drbg, GetEntropy, 0, "justine", 7));
-                                      (const uint8_t *)"justine", 7));
     CHECK_EQ(0, mbedtls_ssl_config_defaults(&conf, MBEDTLS_SSL_IS_CLIENT,
                                             MBEDTLS_SSL_TRANSPORT_STREAM,
                                             MBEDTLS_SSL_PRESET_DEFAULT));
-    mbedtls_ssl_conf_ca_chain(&conf, &cacert, 0);
+    mbedtls_ssl_conf_ca_chain(&conf, GetSslRoots(), 0);
-    mbedtls_ssl_conf_authmode(&conf, MBEDTLS_SSL_VERIFY_REQUIRED);
+    mbedtls_ssl_conf_authmode(&conf, authmode);
     mbedtls_ssl_conf_rng(&conf, mbedtls_ctr_drbg_random, &drbg);
-    mbedtls_ssl_conf_dbg(&conf, TlsDebug, 0);
+    if (!IsTiny()) mbedtls_ssl_conf_dbg(&conf, TlsDebug, 0);
-    /* mbedtls_debug_set_threshold(5); */
     CHECK_EQ(0, mbedtls_ssl_setup(&ssl, &conf));
     CHECK_EQ(0, mbedtls_ssl_set_hostname(&ssl, host));
   }
@@ -211,13 +303,14 @@ int main(int argc, char *argv[]) {
    * Connect to server.
    */
   int ret, sock;
-  CHECK_NE(-1, (sock = socket(addr->ai_family, addr->ai_socktype,
+  CHECK_NE(-1, (sock = Socket(addr->ai_family, addr->ai_socktype,
                               addr->ai_protocol)));
   CHECK_NE(-1, connect(sock, addr->ai_addr, addr->ai_addrlen));
+  freeaddrinfo(addr);
   if (usessl) {
     mbedtls_ssl_set_bio(&ssl, &sock, TlsSend, 0, TlsRecv);
     if ((ret = mbedtls_ssl_handshake(&ssl))) {
-      FATALF("ssl handshake failed (-0x%04x %s)", -ret, TlsError(ret));
+      TlsDie("ssl handshake", ret);
     }
   }
 
@@ -225,81 +318,126 @@ int main(int argc, char *argv[]) {
    * Send HTTP Message.
    */
   if (usessl) {
-    CHECK_EQ(strlen(msg), mbedtls_ssl_write(&ssl, (void *)msg, strlen(msg)));
+    ret = mbedtls_ssl_write(&ssl, request.p, request.i);
+    if (ret != request.i) TlsDie("ssl write", ret);
   } else {
-    CHECK_EQ(strlen(msg), write(sock, msg, strlen(msg)));
+    CHECK_EQ(request.i, write(sock, request.p, request.i));
-    shutdown(sock, SHUT_WR);
   }
 
   /*
    * Handle response.
-   * TODO(jart): use response/chunk parsers
    */
-  char buf[1500];
+  int t;
-  size_t got, toto;
+  char *p;
-  ssize_t rc, need;
+  ssize_t rc;
-  const char *crlfcrlf;
+  struct HttpMessage msg;
-  buf[0] = '\0';
+  struct HttpUnchunker u;
-  if (usessl) {
+  size_t g, i, n, hdrlen, paylen;
-    if ((rc = mbedtls_ssl_read(&ssl, (void *)buf, sizeof(buf))) < 0) {
+  InitHttpMessage(&msg, kHttpResponse);
-      FATALF("ssl read failed (-0x%04x %s)", -ret, TlsError(rc));
+  for (p = 0, hdrlen = paylen = t = i = n = 0;;) {
+    if (i == n) {
+      n += 1000;
+      n += n >> 1;
+      p = realloc(p, n);
     }
-  } else {
-    CHECK_NE(-1, (rc = read(sock, buf, sizeof(buf))));
-  }
-  got = rc;
-  CHECK(startswith(buf, "HTTP/1.1 200") || startswith(buf, "HTTP/1.0 200"),
-        "%`'.*s", got, buf);
-  CHECK_NOTNULL((crlfcrlf = memmem(buf, got, "\r\n\r\n", 4)));
-  need = strtol(
-      (char *)firstnonnull(
-          memmem(buf, crlfcrlf - buf, "\r\nContent-Length: ", 18),
-          firstnonnull(memmem(buf, crlfcrlf - buf, "\r\ncontent-length: ", 18),
-                       "\r\nContent-Length: -1")) +
-          18,
-      NULL, 10);
-  got = MIN(got - (crlfcrlf + 4 - buf), need);
-  CHECK_EQ(got, write(1, crlfcrlf + 4, got));
-  for (toto = got; need == -1 || toto < need; toto += got) {
     if (usessl) {
-      if ((rc = mbedtls_ssl_read(&ssl, (void *)buf, sizeof(buf))) < 0) {
+      if ((rc = mbedtls_ssl_read(&ssl, p + i, n - i)) < 0) {
         if (rc == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) {
-          CHECK_EQ(0, mbedtls_ssl_close_notify(&ssl));
+          rc = 0;
-          got = 0;
-          break;
         } else {
-          FATALF("ssl read failed (-0x%04x %s)", -rc, TlsError(rc));
+          TlsDie("ssl read", rc);
         }
       }
     } else {
-      CHECK_NE(-1, (rc = read(sock, buf, sizeof(buf))));
+      CHECK_NE(-1, (rc = read(sock, p + i, n - i)));
+    }
+    g = rc;
+    i += g;
+    switch (t) {
+      case kHttpClientStateHeaders:
+        CHECK(g);
+        CHECK_NE(-1, (rc = ParseHttpMessage(&msg, p, i)));
+        if (rc) {
+          hdrlen = rc;
+          if (100 <= msg.status && msg.status <= 199) {
+            CHECK(!HasHeader(kHttpContentLength) ||
+                  HeaderEqualCase(kHttpContentLength, "0"));
+            CHECK(!HasHeader(kHttpTransferEncoding) ||
+                  HeaderEqualCase(kHttpTransferEncoding, "identity"));
+            DestroyHttpMessage(&msg);
+            InitHttpMessage(&msg, kHttpResponse);
+            memmove(p, p + hdrlen, i - hdrlen);
+            i -= hdrlen;
+            break;
+          }
+          if (method == kHttpHead) {
+            CHECK_EQ(hdrlen, write(1, p, hdrlen));
+            goto Finished;
+          } else if (msg.status == 204 || msg.status == 304) {
+            goto Finished;
+          }
+          if (HasHeader(kHttpTransferEncoding) &&
+              !HeaderEqualCase(kHttpTransferEncoding, "identity")) {
+            CHECK(HeaderEqualCase(kHttpTransferEncoding, "chunked"));
+            t = kHttpClientStateBodyChunked;
+            memset(&u, 0, sizeof(u));
+            goto Chunked;
+          } else if (HasHeader(kHttpContentLength)) {
+            CHECK_NE(-1, (rc = ParseContentLength(
+                              HeaderData(kHttpContentLength),
+                              HeaderLength(kHttpContentLength))));
+            t = kHttpClientStateBodyLengthed;
+            paylen = rc;
+            if (paylen > i - hdrlen) {
+              CHECK_EQ(i - hdrlen, write(1, p + hdrlen, i - hdrlen));
+            } else {
+              CHECK_EQ(paylen, write(1, p + hdrlen, paylen));
+              goto Finished;
+            }
+          } else {
+            t = kHttpClientStateBody;
+            CHECK_EQ(i - hdrlen, write(1, p + hdrlen, i - hdrlen));
+          }
+        }
+        break;
+      case kHttpClientStateBody:
+        if (!g) goto Finished;
+        CHECK_EQ(g, write(1, p + i - g, g));
+        break;
+      case kHttpClientStateBodyLengthed:
+        CHECK(g);
+        if (i - hdrlen > paylen) g = hdrlen + paylen - (i - g);
+        CHECK_EQ(g, write(1, p + i - g, g));
+        if (i - hdrlen >= paylen) goto Finished;
+        break;
+      case kHttpClientStateBodyChunked:
+      Chunked:
+        CHECK_NE(-1, (rc = Unchunk(&u, p + hdrlen, i - hdrlen, &paylen)));
+        if (rc) {
+          CHECK_EQ(paylen, write(1, p + hdrlen, paylen));
+          goto Finished;
+        }
+        break;
+      default:
+        abort();
     }
-    if (!(got = rc)) break;
-    CHECK_EQ(got, write(1, buf, got));
-  }
-  if (need != -1) {
-    CHECK_EQ(need, toto);
   }
 
-  /*
+/*
-   * Close connection.
+ * Close connection.
-   */
+ */
-  if (got) {
+Finished:
-    if (usessl) {
-      CHECK_EQ(0, mbedtls_ssl_close_notify(&ssl));
-      CHECK_EQ(MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY,
-               mbedtls_ssl_read(&ssl, (void *)buf, 1));
-    } else {
-      shutdown(sock, SHUT_RDWR);
-      CHECK_EQ(0, read(sock, buf, 1));
-    }
-  }
   CHECK_NE(-1, close(sock));
+
+  /*
+   * Free memory.
+   */
+  free(p);
+  free(headers.p);
   if (usessl) {
     mbedtls_ssl_free(&ssl);
-    mbedtls_x509_crt_free(&cacert);
-    mbedtls_ssl_config_free(&conf);
     mbedtls_ctr_drbg_free(&drbg);
+    mbedtls_ssl_config_free(&conf);
   }
 
   return 0;

examples/examples.mk

@@ -66,6 +66,7 @@ EXAMPLES_DIRECTDEPS =						\
 	LIBC_X							\
 	LIBC_ZIPOS						\
 	NET_HTTP						\
+	NET_HTTPS						\
 	THIRD_PARTY_COMPILER_RT					\
 	THIRD_PARTY_DLMALLOC					\
 	THIRD_PARTY_GDTOA					\

libc/sysv/consts.sh

@@ -720,11 +720,13 @@ syscon	sol	SOL_X25					262			0			0			0			0			0
 syscon	tcp	TCP_NODELAY				1			1			1			1			1			1			# strong consensus for disabling nagle's algorithm; so be sure to disable it by turning this on
 syscon	tcp	TCP_CORK				3			4			4			16			0			0			# nagle's algorithm strikes again; TCP_NOPUSH on BSD; be sure to turn it off; protip: mmap+writev vs. write+sendfile; see also /proc/sys/net/ipv4/tcp_autocorking; netbsd is 4 but not implemented
 syscon	tcp	TCP_MAXSEG				2			2			2			2			2			0			# reduces tcp segment size; see also tcp offloading
-syscon	tcp	TCP_FASTOPEN				23			0			0x0401			0			0			15                      # reduces roundtrips; for listener; Linux 3.7+ (c. 2012) / or is windows it 0x22? /proc/sys/net/ipv4/tcp_fastopen TODO(jart): MSG_FASTOPEN; XNU sources say 261 but not sure if that's true
+syscon	tcp	TCP_FASTOPEN				23			0			0x0401			0			0			15			# reduces roundtrips; for listener; Linux 3.7+ (c. 2012) / or is windows it 0x22? /proc/sys/net/ipv4/tcp_fastopen TODO(jart): MSG_FASTOPEN; XNU sources say 261 but not sure if that's true
+syscon	tcp	TCP_FASTOPEN_CONNECT			30			0			0			0			0			0			# reduces roundtrips; for listener; Linux 3.7+ (c. 2012) / or is windows it 0x22? /proc/sys/net/ipv4/tcp_fastopen TODO(jart): MSG_FASTOPEN; XNU sources say 261 but not sure if that's true
 syscon	tcp	TCP_KEEPIDLE				4			0			0x100			0			3			0			# keepalives
 syscon	tcp	TCP_KEEPINTVL				5			0x101			0x200			0			5			0			# keepalives
 syscon	tcp	TCP_KEEPCNT				6			0x102			0x400			0			6			0			# keepalives
 syscon	tcp	TCP_SYNCNT				7			0			0			0			0			0			# how hard to syn packet the enemy
+syscon	tcp	TCP_ULP					31			0			0			0			0			0			# setsockopt(sock, IPPROTO_TCP, TCP_ULP, "tls", 4)
 syscon	tcp	TCP_COOKIE_TRANSACTIONS			15			0			0			0			0			0			# defense against the syn packets
 syscon	tcp	TCP_LINGER2				8			0			0			0			0			0			# orphaned fin-wait-2 lifetime cf. net.ipv4.tcp_fin_timeout see cloudflare blog
 syscon	tcp	TCP_NOTSENT_LOWAT			25			513			0			0			0			0			# limit unset byte queue

libc/sysv/consts/TCP_FASTOPEN_CONNECT.S

@@ -0,0 +1,2 @@
+#include "libc/sysv/consts/syscon.internal.h"
+.syscon tcp,TCP_FASTOPEN_CONNECT,30,0,0,0,0,0

libc/sysv/consts/TCP_ULP.S

@@ -0,0 +1,2 @@
+#include "libc/sysv/consts/syscon.internal.h"
+.syscon tcp,TCP_ULP,31,0,0,0,0,0

libc/sysv/consts/tcp.h

@@ -10,6 +10,7 @@ extern const long TCP_COOKIE_TRANSACTIONS;
 extern const long TCP_CORK;
 extern const long TCP_DEFER_ACCEPT;
 extern const long TCP_FASTOPEN;
+extern const long TCP_FASTOPEN_CONNECT;
 extern const long TCP_INFO;
 extern const long TCP_KEEPCNT;
 extern const long TCP_KEEPIDLE;
@@ -31,40 +32,43 @@ extern const long TCP_SYNCNT;
 extern const long TCP_THIN_DUPACK;
 extern const long TCP_THIN_LINEAR_TIMEOUTS;
 extern const long TCP_TIMESTAMP;
+extern const long TCP_ULP;
 extern const long TCP_USER_TIMEOUT;
 extern const long TCP_WINDOW_CLAMP;
 
 COSMOPOLITAN_C_END_
 #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
 
-#define TCP_CC_INFO SYMBOLIC(TCP_CC_INFO)
+#define TCP_CC_INFO              SYMBOLIC(TCP_CC_INFO)
-#define TCP_CONGESTION SYMBOLIC(TCP_CONGESTION)
+#define TCP_CONGESTION           SYMBOLIC(TCP_CONGESTION)
-#define TCP_COOKIE_TRANSACTIONS SYMBOLIC(TCP_COOKIE_TRANSACTIONS)
+#define TCP_COOKIE_TRANSACTIONS  SYMBOLIC(TCP_COOKIE_TRANSACTIONS)
-#define TCP_CORK SYMBOLIC(TCP_CORK)
+#define TCP_CORK                 SYMBOLIC(TCP_CORK)
-#define TCP_DEFER_ACCEPT SYMBOLIC(TCP_DEFER_ACCEPT)
+#define TCP_DEFER_ACCEPT         SYMBOLIC(TCP_DEFER_ACCEPT)
-#define TCP_FASTOPEN SYMBOLIC(TCP_FASTOPEN)
+#define TCP_FASTOPEN             SYMBOLIC(TCP_FASTOPEN)
-#define TCP_INFO SYMBOLIC(TCP_INFO)
+#define TCP_FASTOPEN_CONNECT     SYMBOLIC(TCP_FASTOPEN_CONNECT)
-#define TCP_KEEPCNT SYMBOLIC(TCP_KEEPCNT)
+#define TCP_INFO                 SYMBOLIC(TCP_INFO)
-#define TCP_KEEPIDLE SYMBOLIC(TCP_KEEPIDLE)
+#define TCP_KEEPCNT              SYMBOLIC(TCP_KEEPCNT)
-#define TCP_KEEPINTVL SYMBOLIC(TCP_KEEPINTVL)
+#define TCP_KEEPIDLE             SYMBOLIC(TCP_KEEPIDLE)
-#define TCP_LINGER2 SYMBOLIC(TCP_LINGER2)
+#define TCP_KEEPINTVL            SYMBOLIC(TCP_KEEPINTVL)
-#define TCP_MAXSEG SYMBOLIC(TCP_MAXSEG)
+#define TCP_LINGER2              SYMBOLIC(TCP_LINGER2)
-#define TCP_MD5SIG SYMBOLIC(TCP_MD5SIG)
+#define TCP_MAXSEG               SYMBOLIC(TCP_MAXSEG)
-#define TCP_MD5SIG_MAXKEYLEN SYMBOLIC(TCP_MD5SIG_MAXKEYLEN)
+#define TCP_MD5SIG               SYMBOLIC(TCP_MD5SIG)
-#define TCP_NODELAY LITERALLY(1)
+#define TCP_MD5SIG_MAXKEYLEN     SYMBOLIC(TCP_MD5SIG_MAXKEYLEN)
-#define TCP_NOTSENT_LOWAT SYMBOLIC(TCP_NOTSENT_LOWAT)
+#define TCP_NODELAY              LITERALLY(1)
-#define TCP_QUEUE_SEQ SYMBOLIC(TCP_QUEUE_SEQ)
+#define TCP_NOTSENT_LOWAT        SYMBOLIC(TCP_NOTSENT_LOWAT)
-#define TCP_QUICKACK SYMBOLIC(TCP_QUICKACK)
+#define TCP_QUEUE_SEQ            SYMBOLIC(TCP_QUEUE_SEQ)
-#define TCP_REPAIR SYMBOLIC(TCP_REPAIR)
+#define TCP_QUICKACK             SYMBOLIC(TCP_QUICKACK)
-#define TCP_REPAIR_OPTIONS SYMBOLIC(TCP_REPAIR_OPTIONS)
+#define TCP_REPAIR               SYMBOLIC(TCP_REPAIR)
-#define TCP_REPAIR_QUEUE SYMBOLIC(TCP_REPAIR_QUEUE)
+#define TCP_REPAIR_OPTIONS       SYMBOLIC(TCP_REPAIR_OPTIONS)
-#define TCP_SAVED_SYN SYMBOLIC(TCP_SAVED_SYN)
+#define TCP_REPAIR_QUEUE         SYMBOLIC(TCP_REPAIR_QUEUE)
-#define TCP_SAVE_SYN SYMBOLIC(TCP_SAVE_SYN)
+#define TCP_SAVED_SYN            SYMBOLIC(TCP_SAVED_SYN)
-#define TCP_SYNCNT SYMBOLIC(TCP_SYNCNT)
+#define TCP_SAVE_SYN             SYMBOLIC(TCP_SAVE_SYN)
-#define TCP_THIN_DUPACK SYMBOLIC(TCP_THIN_DUPACK)
+#define TCP_SYNCNT               SYMBOLIC(TCP_SYNCNT)
+#define TCP_THIN_DUPACK          SYMBOLIC(TCP_THIN_DUPACK)
 #define TCP_THIN_LINEAR_TIMEOUTS SYMBOLIC(TCP_THIN_LINEAR_TIMEOUTS)
-#define TCP_TIMESTAMP SYMBOLIC(TCP_TIMESTAMP)
+#define TCP_TIMESTAMP            SYMBOLIC(TCP_TIMESTAMP)
-#define TCP_USER_TIMEOUT SYMBOLIC(TCP_USER_TIMEOUT)
+#define TCP_ULP                  SYMBOLIC(TCP_ULP)
-#define TCP_WINDOW_CLAMP SYMBOLIC(TCP_WINDOW_CLAMP)
+#define TCP_USER_TIMEOUT         SYMBOLIC(TCP_USER_TIMEOUT)
+#define TCP_WINDOW_CLAMP         SYMBOLIC(TCP_WINDOW_CLAMP)
 
 #endif /* COSMOPOLITAN_LIBC_SYSV_CONSTS_TCP_H_ */

net/http/http.h

@@ -36,6 +36,11 @@
 #define kHttpStateLf1     10
 #define kHttpStateLf2     11
 
+#define kHttpClientStateHeaders      0
+#define kHttpClientStateBody         1
+#define kHttpClientStateBodyChunked  2
+#define kHttpClientStateBodyLengthed 3
+
 #define kHttpStateChunkStart   0
 #define kHttpStateChunkSize    1
 #define kHttpStateChunkExt     2

net/http/http.mk

@@ -8,39 +8,22 @@ NET_HTTP = $(NET_HTTP_A_DEPS) $(NET_HTTP_A)
 NET_HTTP_A = o/$(MODE)/net/http/http.a
 NET_HTTP_A_FILES := $(wildcard net/http/*)
 NET_HTTP_A_HDRS = $(filter %.h,$(NET_HTTP_A_FILES))
-NET_HTTP_A_SRCS_S = $(filter %.S,$(NET_HTTP_A_FILES))
+NET_HTTP_A_SRCS = $(filter %.c,$(NET_HTTP_A_FILES))
-NET_HTTP_A_SRCS_C = $(filter %.c,$(NET_HTTP_A_FILES))
+NET_HTTP_A_OBJS = $(NET_HTTP_A_SRCS:%.c=o/$(MODE)/%.o)
-NET_HTTP_A_SRCS_R = $(filter %.rl,$(NET_HTTP_A_FILES))
-
-NET_HTTP_A_SRCS =				\
-	$(NET_HTTP_A_SRCS_S)			\
-	$(NET_HTTP_A_SRCS_C)			\
-	$(NET_HTTP_A_SRCS_R)
-
-NET_HTTP_A_OBJS =				\
-	$(NET_HTTP_A_SRCS_S:%.S=o/$(MODE)/%.o)	\
-	$(NET_HTTP_A_SRCS_C:%.c=o/$(MODE)/%.o)	\
-	$(NET_HTTP_A_SRCS_R:%.rl=o/$(MODE)/%.o)
 
 NET_HTTP_A_CHECKS =				\
 	$(NET_HTTP_A).pkg			\
 	$(NET_HTTP_A_HDRS:%=o/$(MODE)/%.ok)
 
 NET_HTTP_A_DIRECTDEPS =				\
-	LIBC_CALLS				\
 	LIBC_FMT				\
 	LIBC_INTRIN				\
-	LIBC_LOG				\
 	LIBC_MEM				\
 	LIBC_NEXGEN32E				\
-	LIBC_RUNTIME				\
-	LIBC_SOCK				\
-	LIBC_STDIO				\
 	LIBC_STR				\
 	LIBC_STUBS				\
 	LIBC_SYSV				\
-	LIBC_TIME				\
+	LIBC_TIME
-	LIBC_X
 
 NET_HTTP_A_DEPS :=				\
 	$(call uniq,$(foreach x,$(NET_HTTP_A_DIRECTDEPS),$($(x))))
@@ -76,10 +59,8 @@ o/$(MODE)/net/http/formathttpdatetime.o:	\
 NET_HTTP_LIBS = $(foreach x,$(NET_HTTP_ARTIFACTS),$($(x)))
 NET_HTTP_SRCS = $(foreach x,$(NET_HTTP_ARTIFACTS),$($(x)_SRCS))
 NET_HTTP_HDRS = $(foreach x,$(NET_HTTP_ARTIFACTS),$($(x)_HDRS))
-NET_HTTP_CHECKS = $(foreach x,$(NET_HTTP_ARTIFACTS),$($(x)_CHECKS))
 NET_HTTP_OBJS = $(foreach x,$(NET_HTTP_ARTIFACTS),$($(x)_OBJS))
+NET_HTTP_CHECKS = $(foreach x,$(NET_HTTP_ARTIFACTS),$($(x)_CHECKS))
 
 .PHONY: o/$(MODE)/net/http
-o/$(MODE)/net/http:				\
+o/$(MODE)/net/http: $(NET_HTTP_CHECKS)
-		$(NET_HTTP_CHECKS)		\
-		$(NET_HTTP_A_SRCS_R:%.rl=%.svgz)

net/http/isacceptablehost.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/dns/dns.h"
 #include "libc/str/str.h"
 #include "net/http/http.h"
 
@@ -72,6 +73,7 @@ bool IsAcceptableHost(const char *s, size_t n) {
   int c, b, j;
   if (n == -1) n = s ? strlen(s) : 0;
   if (!n) return true;
+  if (n > DNS_NAME_MAX) return false;
   for (b = j = i = 0; i < n; ++i) {
     c = s[i] & 255;
     if (isdigit(c)) {

net/http/unchunk.c

@@ -43,7 +43,7 @@ ssize_t Unchunk(struct HttpUnchunker *u, char *p, size_t n, size_t *l) {
         if ((h = kHexToInt[c]) != -1) {
           u->m *= 16;
           u->m += h;
-          if (u->i + u->m >= n) return ebadmsg();
+          if (u->m >= 0x0000010000000000) return ebadmsg();
           break;
         }
         u->t = kHttpStateChunkExt;

net/https/describesslverifyfailure.c

@@ -0,0 +1,60 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2021 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/macros.internal.h"
+#include "libc/mem/mem.h"
+#include "libc/str/str.h"
+#include "net/https/https.h"
+
+static const struct thatispacked SslVerifyString {
+  int code;
+  const char *str;
+} kSslVerifyStrings[] = {
+    {MBEDTLS_X509_BADCERT_BAD_KEY, "badcert_bad_key"},
+    {MBEDTLS_X509_BADCERT_BAD_MD, "badcert_bad_md"},
+    {MBEDTLS_X509_BADCERT_BAD_PK, "badcert_bad_pk"},
+    {MBEDTLS_X509_BADCERT_CN_MISMATCH, "badcert_cn_mismatch"},
+    {MBEDTLS_X509_BADCERT_EXPIRED, "badcert_expired"},
+    {MBEDTLS_X509_BADCERT_EXT_KEY_USAGE, "badcert_ext_key_usage"},
+    {MBEDTLS_X509_BADCERT_FUTURE, "badcert_future"},
+    {MBEDTLS_X509_BADCERT_KEY_USAGE, "badcert_key_usage"},
+    {MBEDTLS_X509_BADCERT_MISSING, "badcert_missing"},
+    {MBEDTLS_X509_BADCERT_NOT_TRUSTED, "badcert_not_trusted"},
+    {MBEDTLS_X509_BADCERT_NS_CERT_TYPE, "badcert_ns_cert_type"},
+    {MBEDTLS_X509_BADCERT_OTHER, "badcert_other"},
+    {MBEDTLS_X509_BADCERT_REVOKED, "badcert_revoked"},
+    {MBEDTLS_X509_BADCERT_SKIP_VERIFY, "badcert_skip_verify"},
+    {MBEDTLS_X509_BADCRL_BAD_KEY, "badcrl_bad_key"},
+    {MBEDTLS_X509_BADCRL_BAD_MD, "badcrl_bad_md"},
+    {MBEDTLS_X509_BADCRL_BAD_PK, "badcrl_bad_pk"},
+    {MBEDTLS_X509_BADCRL_EXPIRED, "badcrl_expired"},
+    {MBEDTLS_X509_BADCRL_FUTURE, "badcrl_future"},
+    {MBEDTLS_X509_BADCRL_NOT_TRUSTED, "badcrl_not_trusted"},
+};
+
+char *DescribeSslVerifyFailure(int flags) {
+  int i;
+  char *p, *q;
+  p = malloc(1024);
+  q = stpcpy(p, "verify failed");
+  for (i = 0; i < ARRAYLEN(kSslVerifyStrings); ++i) {
+    if (!(flags & kSslVerifyStrings[i].code)) continue;
+    q = stpcpy(stpcpy(q, " "), kSslVerifyStrings[i].str);
+  }
+  return p;
+}

net/https/formatssltime.c

@@ -0,0 +1,37 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2021 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "net/https/https.h"
+
+void FormatSslTime(char p[restrict hasatleast 16], struct tm *tm) {
+  p[0x0] = '0' + (tm->tm_year + 1900) / 1000;
+  p[0x1] = '0' + (tm->tm_year + 1900) / 100 % 10;
+  p[0x2] = '0' + (tm->tm_year + 1900) / 10 % 10;
+  p[0x3] = '0' + (tm->tm_year + 1900) % 10;
+  p[0x4] = '0' + (tm->tm_mon + 1) / 10;
+  p[0x5] = '0' + (tm->tm_mon + 1) % 10;
+  p[0x6] = '0' + tm->tm_mday / 10;
+  p[0x7] = '0' + tm->tm_mday % 10;
+  p[0x8] = '0' + tm->tm_hour / 10;
+  p[0x9] = '0' + tm->tm_hour % 10;
+  p[0xa] = '0' + tm->tm_min / 10;
+  p[0xb] = '0' + tm->tm_min % 10;
+  p[0xc] = '0' + tm->tm_sec / 10;
+  p[0xd] = '0' + tm->tm_sec % 10;
+  p[0xe] = 0;
+}

net/https/getsslroots.c

@@ -0,0 +1,69 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2021 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/calls.h"
+#include "libc/calls/struct/dirent.h"
+#include "libc/errno.h"
+#include "libc/log/check.h"
+#include "libc/runtime/runtime.h"
+#include "libc/str/str.h"
+#include "libc/sysv/consts/dt.h"
+#include "libc/sysv/consts/o.h"
+#include "libc/x/x.h"
+#include "net/https/https.h"
+#include "third_party/mbedtls/x509_crt.h"
+
+STATIC_YOINK("zip_uri_support");
+STATIC_YOINK("usr/share/ssl/root/amazon.pem");
+STATIC_YOINK("usr/share/ssl/root/certum.pem");
+STATIC_YOINK("usr/share/ssl/root/comodo.pem");
+STATIC_YOINK("usr/share/ssl/root/digicert.pem");
+STATIC_YOINK("usr/share/ssl/root/dst.pem");
+STATIC_YOINK("usr/share/ssl/root/geotrust.pem");
+STATIC_YOINK("usr/share/ssl/root/globalsign.pem");
+STATIC_YOINK("usr/share/ssl/root/godaddy.pem");
+STATIC_YOINK("usr/share/ssl/root/google.pem");
+STATIC_YOINK("usr/share/ssl/root/isrg.pem");
+STATIC_YOINK("usr/share/ssl/root/quovadis.pem");
+STATIC_YOINK("usr/share/ssl/root/redbean.pem");
+STATIC_YOINK("usr/share/ssl/root/starfield.pem");
+STATIC_YOINK("usr/share/ssl/root/verisign.pem");
+
+mbedtls_x509_crt *GetSslRoots(void) {
+  int fd;
+  DIR *d;
+  uint8_t *p;
+  size_t n, m;
+  struct dirent *e;
+  mbedtls_x509_crt *c;
+  char path[PATH_MAX + 1];
+  c = calloc(1, sizeof(*c));
+  m = stpcpy(path, "zip:usr/share/ssl/root/") - path;
+  if ((d = opendir(path))) {
+    while ((e = readdir(d))) {
+      if (e->d_type != DT_REG) continue;
+      if (m + (n = strlen(e->d_name)) > PATH_MAX) continue;
+      memcpy(path + m, e->d_name, n + 1);
+      CHECK((p = xslurp(path, &n)));
+      CHECK_GE(mbedtls_x509_crt_parse(c, p, n + 1), 0, "%s", path);
+      free(p);
+    }
+    closedir(d);
+  }
+  return c;
+}

net/https/https.h

@@ -0,0 +1,14 @@
+#ifndef COSMOPOLITAN_NET_HTTPS_HTTPS_H_
+#define COSMOPOLITAN_NET_HTTPS_HTTPS_H_
+#include "libc/time/struct/tm.h"
+#include "third_party/mbedtls/x509_crt.h"
+#if !(__ASSEMBLER__ + __LINKER__ + 0)
+COSMOPOLITAN_C_START_
+
+char *DescribeSslVerifyFailure(int);
+mbedtls_x509_crt *GetSslRoots(void);
+void FormatSslTime(char[restrict hasatleast 16], struct tm *);
+
+COSMOPOLITAN_C_END_
+#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
+#endif /* COSMOPOLITAN_NET_HTTPS_HTTPS_H_ */

net/https/https.mk

@@ -0,0 +1,55 @@
+#-*-mode:makefile-gmake;indent-tabs-mode:t;tab-width:8;coding:utf-8-*-┐
+#───vi: set et ft=make ts=8 tw=8 fenc=utf-8 :vi───────────────────────┘
+
+PKGS += NET_HTTPS
+
+NET_HTTPS_ARTIFACTS += NET_HTTPS_A
+NET_HTTPS = $(NET_HTTPS_A_DEPS) $(NET_HTTPS_A)
+NET_HTTPS_A = o/$(MODE)/net/https/https.a
+NET_HTTPS_A_FILES := $(wildcard net/https/*)
+NET_HTTPS_A_CERTS := $(wildcard usr/share/ssl/root/*)
+NET_HTTPS_A_HDRS = $(filter %.h,$(NET_HTTPS_A_FILES))
+NET_HTTPS_A_SRCS = $(filter %.c,$(NET_HTTPS_A_FILES))
+
+NET_HTTPS_A_OBJS =				\
+	$(NET_HTTPS_A_SRCS:%.c=o/$(MODE)/%.o)	\
+	$(NET_HTTPS_A_CERTS:%=o/$(MODE)/%.zip.o)
+
+NET_HTTPS_A_CHECKS =				\
+	$(NET_HTTPS_A).pkg			\
+	$(NET_HTTPS_A_HDRS:%=o/$(MODE)/%.ok)
+
+NET_HTTPS_A_DIRECTDEPS =			\
+	LIBC_FMT				\
+	LIBC_INTRIN				\
+	LIBC_LOG				\
+	LIBC_MEM				\
+	LIBC_NEXGEN32E				\
+	LIBC_RUNTIME				\
+	LIBC_STDIO				\
+	LIBC_STR				\
+	LIBC_STUBS				\
+	LIBC_SYSV				\
+	LIBC_X					\
+	LIBC_ZIPOS				\
+	THIRD_PARTY_MBEDTLS
+
+NET_HTTPS_A_DEPS :=				\
+	$(call uniq,$(foreach x,$(NET_HTTPS_A_DIRECTDEPS),$($(x))))
+
+$(NET_HTTPS_A):	net/https/			\
+		$(NET_HTTPS_A).pkg		\
+		$(NET_HTTPS_A_OBJS)
+
+$(NET_HTTPS_A).pkg:				\
+		$(NET_HTTPS_A_OBJS)		\
+		$(foreach x,$(NET_HTTPS_A_DIRECTDEPS),$($(x)_A).pkg)
+
+NET_HTTPS_LIBS = $(foreach x,$(NET_HTTPS_ARTIFACTS),$($(x)))
+NET_HTTPS_SRCS = $(foreach x,$(NET_HTTPS_ARTIFACTS),$($(x)_SRCS))
+NET_HTTPS_HDRS = $(foreach x,$(NET_HTTPS_ARTIFACTS),$($(x)_HDRS))
+NET_HTTPS_OBJS = $(foreach x,$(NET_HTTPS_ARTIFACTS),$($(x)_OBJS))
+NET_HTTPS_CHECKS = $(foreach x,$(NET_HTTPS_ARTIFACTS),$($(x)_CHECKS))
+
+.PHONY: o/$(MODE)/net/https
+o/$(MODE)/net/https: $(NET_HTTPS_CHECKS)

net/net.mk

@@ -2,4 +2,5 @@
 #───vi: set et ft=make ts=8 tw=8 fenc=utf-8 :vi───────────────────────┘
 
 .PHONY: 	o/$(MODE)/net
-o/$(MODE)/net:	o/$(MODE)/net/http
+o/$(MODE)/net:	o/$(MODE)/net/http	\
+		o/$(MODE)/net/https

test/libc/dns/parsehoststxt_test.c

@@ -24,7 +24,7 @@
 #include "libc/sysv/consts/af.h"
 #include "libc/testlib/testlib.h"
 
-static const char *ParseIp(unsigned char ip[4]) {
+static const char *parseip(unsigned char ip[4]) {
   static char g_ipbuf[16];
   return inet_ntop(AF_INET, ip, g_ipbuf, sizeof(g_ipbuf));
 }
@@ -51,16 +51,16 @@ TEST(ParseHostsTxt, testCorrectlyTokenizesAndSorts) {
   ASSERT_EQ(4, ht->entries.i);
   EXPECT_STREQ("lol.example.", &ht->strings.p[ht->entries.p[0].name]);
   EXPECT_STREQ("lol.example.", &ht->strings.p[ht->entries.p[0].canon]);
-  EXPECT_STREQ("203.0.113.1", ParseIp(ht->entries.p[0].ip));
+  EXPECT_STREQ("203.0.113.1", parseip(ht->entries.p[0].ip));
   EXPECT_STREQ("lol", &ht->strings.p[ht->entries.p[1].name]);
   EXPECT_STREQ("lol.example.", &ht->strings.p[ht->entries.p[1].canon]);
-  EXPECT_STREQ("203.0.113.1", ParseIp(ht->entries.p[1].ip));
+  EXPECT_STREQ("203.0.113.1", parseip(ht->entries.p[1].ip));
   EXPECT_STREQ("cat.example.", &ht->strings.p[ht->entries.p[2].name]);
   EXPECT_STREQ("cat.example.", &ht->strings.p[ht->entries.p[2].canon]);
-  EXPECT_STREQ("203.0.113.2", ParseIp(ht->entries.p[2].ip));
+  EXPECT_STREQ("203.0.113.2", parseip(ht->entries.p[2].ip));
   EXPECT_STREQ("cat", &ht->strings.p[ht->entries.p[3].name]);
   EXPECT_STREQ("cat.example.", &ht->strings.p[ht->entries.p[3].canon]);
-  EXPECT_STREQ("203.0.113.2", ParseIp(ht->entries.p[3].ip));
+  EXPECT_STREQ("203.0.113.2", parseip(ht->entries.p[3].ip));
   FreeHostsTxt(&ht);
   fclose(f);
 }
@@ -74,7 +74,7 @@ TEST(ParseHostsTxt, testIpv6_isIgnored) {
   ASSERT_EQ(1, ht->entries.i);
   EXPECT_STREQ("cat", &ht->strings.p[ht->entries.p[0].name]);
   EXPECT_STREQ("cat", &ht->strings.p[ht->entries.p[0].canon]);
-  EXPECT_STREQ("203.0.113.2", ParseIp(ht->entries.p[0].ip));
+  EXPECT_STREQ("203.0.113.2", parseip(ht->entries.p[0].ip));
   FreeHostsTxt(&ht);
   fclose(f);
 }

test/net/http/unchunk_test.c

@@ -17,6 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/testlib/testlib.h"
+#include "libc/x/x.h"
 #include "net/http/http.h"
 
 ssize_t EzUnchunk(char *p, size_t n, size_t *l) {

third_party/lua/ldo.c

@@ -7,6 +7,7 @@
 #define ldo_c
 #define LUA_CORE
 
+#include "libc/runtime/gc.h"
 #include "third_party/lua/lapi.h"
 #include "third_party/lua/ldebug.h"
 #include "third_party/lua/ldo.h"
@@ -57,14 +58,14 @@
 #elif defined(LUA_USE_POSIX)				/* }{ */
 
 /* in POSIX, try _longjmp/_setjmp (more efficient) */
-#define LUAI_THROW(L,c)		_longjmp((c)->b, 1)
+#define LUAI_THROW(L,c)		_gclongjmp((c)->b, 1)
 #define LUAI_TRY(L,c,a)		if (_setjmp((c)->b) == 0) { a }
 #define luai_jmpbuf		jmp_buf
 
 #else							/* }{ */
 
 /* ISO C handling with long jumps */
-#define LUAI_THROW(L,c)		longjmp((c)->b, 1)
+#define LUAI_THROW(L,c)		_gclongjmp((c)->b, 1)
 #define LUAI_TRY(L,c,a)		if (setjmp((c)->b) == 0) { a }
 #define luai_jmpbuf		jmp_buf
 

third_party/mbedtls/ctr_drbg.c

@@ -431,7 +431,7 @@ static size_t good_nonce_len( size_t entropy_len )
 int mbedtls_ctr_drbg_seed( mbedtls_ctr_drbg_context *ctx,
                            int (*f_entropy)(void *, unsigned char *, size_t),
                            void *p_entropy,
-                           const unsigned char *custom,
+                           const void *custom,
                            size_t len )
 {
     int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED;

third_party/mbedtls/ctr_drbg.h

@@ -244,7 +244,7 @@ void mbedtls_ctr_drbg_init( mbedtls_ctr_drbg_context * );
 int mbedtls_ctr_drbg_seed( mbedtls_ctr_drbg_context *ctx,
                            int (*f_entropy)(void *, unsigned char *, size_t),
                            void *p_entropy,
-                           const unsigned char *custom,
+                           const void *custom,
                            size_t len );
 
 /**

third_party/mbedtls/mbedtls.mk

@@ -7,13 +7,9 @@ THIRD_PARTY_MBEDTLS_ARTIFACTS += THIRD_PARTY_MBEDTLS_A
 THIRD_PARTY_MBEDTLS = $(THIRD_PARTY_MBEDTLS_A_DEPS) $(THIRD_PARTY_MBEDTLS_A)
 THIRD_PARTY_MBEDTLS_A = o/$(MODE)/third_party/mbedtls/mbedtls.a
 THIRD_PARTY_MBEDTLS_A_FILES := $(wildcard third_party/mbedtls/*)
-THIRD_PARTY_MBEDTLS_A_CERTS := $(wildcard usr/share/ssl/root/*)
 THIRD_PARTY_MBEDTLS_A_HDRS = $(filter %.h,$(THIRD_PARTY_MBEDTLS_A_FILES))
 THIRD_PARTY_MBEDTLS_A_SRCS = $(filter %.c,$(THIRD_PARTY_MBEDTLS_A_FILES))
-
+THIRD_PARTY_MBEDTLS_A_OBJS = $(THIRD_PARTY_MBEDTLS_A_SRCS:%.c=o/$(MODE)/%.o)
-THIRD_PARTY_MBEDTLS_A_OBJS =						\
-	$(THIRD_PARTY_MBEDTLS_A_SRCS:%.c=o/$(MODE)/%.o)			\
-	$(THIRD_PARTY_MBEDTLS_A_CERTS:%=o/$(MODE)/%.zip.o)
 
 THIRD_PARTY_MBEDTLS_A_CHECKS =						\
 	$(THIRD_PARTY_MBEDTLS_A).pkg					\
@@ -33,6 +29,7 @@ THIRD_PARTY_MBEDTLS_A_DIRECTDEPS =					\
 	LIBC_SYSV							\
 	LIBC_TIME							\
 	LIBC_UNICODE							\
+	NET_HTTP							\
 	THIRD_PARTY_COMPILER_RT						\
 	THIRD_PARTY_GDTOA
 

third_party/mbedtls/ssl.h

@@ -1429,7 +1429,7 @@ int mbedtls_ssl_handshake( mbedtls_ssl_context * );
 int mbedtls_ssl_handshake_client_step( mbedtls_ssl_context * );
 int mbedtls_ssl_handshake_server_step( mbedtls_ssl_context * );
 int mbedtls_ssl_handshake_step( mbedtls_ssl_context * );
-int mbedtls_ssl_read( mbedtls_ssl_context *, unsigned char *, size_t );
+int mbedtls_ssl_read( mbedtls_ssl_context *, void *, size_t );
 int mbedtls_ssl_renegotiate( mbedtls_ssl_context * );
 int mbedtls_ssl_send_alert_message( mbedtls_ssl_context *, unsigned char, unsigned char );
 int mbedtls_ssl_session_load( mbedtls_ssl_session *, const unsigned char *, size_t );
@@ -1443,7 +1443,7 @@ int mbedtls_ssl_set_hs_own_cert( mbedtls_ssl_context *, mbedtls_x509_crt *, mbed
 int mbedtls_ssl_set_hs_psk( mbedtls_ssl_context *, const unsigned char *, size_t );
 int mbedtls_ssl_set_session( mbedtls_ssl_context *, const mbedtls_ssl_session * );
 int mbedtls_ssl_setup( mbedtls_ssl_context *, const mbedtls_ssl_config * );
-int mbedtls_ssl_write( mbedtls_ssl_context *, const unsigned char *, size_t );
+int mbedtls_ssl_write( mbedtls_ssl_context *, const void *, size_t );
 size_t mbedtls_ssl_get_bytes_avail( const mbedtls_ssl_context * );
 size_t mbedtls_ssl_get_input_max_frag_len( const mbedtls_ssl_context * );
 size_t mbedtls_ssl_get_output_max_frag_len( const mbedtls_ssl_context * );

third_party/mbedtls/ssl_msg.c

@@ -5614,7 +5614,7 @@ static int ssl_check_ctr_renegotiate( mbedtls_ssl_context *ssl )
  *                   \c mbedtls_ssl_check_pending to check for remaining records.
  *
  */
-int mbedtls_ssl_read( mbedtls_ssl_context *ssl, unsigned char *buf, size_t len )
+int mbedtls_ssl_read( mbedtls_ssl_context *ssl, void *buf, size_t len )
 {
     int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED;
     size_t n;
@@ -6051,7 +6051,7 @@ static int ssl_write_split( mbedtls_ssl_context *ssl,
  * \note           Attempting to write 0 bytes will result in an empty TLS
  *                 application record being sent.
  */
-int mbedtls_ssl_write( mbedtls_ssl_context *ssl, const unsigned char *buf, size_t len )
+int mbedtls_ssl_write( mbedtls_ssl_context *ssl, const void *buf, size_t len )
 {
     int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED;
     MBEDTLS_SSL_DEBUG_MSG( 2, ( "=> write" ) );

third_party/mbedtls/x509_crt.c

@@ -6,6 +6,7 @@
 #include "libc/limits.h"
 #include "libc/mem/mem.h"
 #include "libc/stdio/stdio.h"
+#include "net/http/http.h"
 #include "third_party/mbedtls/common.h"
 #include "third_party/mbedtls/error.h"
 #include "third_party/mbedtls/oid.h"
@@ -2090,7 +2091,7 @@ static const struct x509_crt_verify_string x509_crt_verify_strings[] = {
  *                 terminated nul byte), or a negative error code.
  */
 int mbedtls_x509_crt_verify_info( char *buf, size_t size, const char *prefix,
-                          uint32_t flags )
+                                  uint32_t flags )
 {
     int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED;
     const struct x509_crt_verify_string *cur;
@@ -2794,11 +2795,17 @@ static int x509_crt_check_cn( const mbedtls_x509_buf *name,
 static int x509_crt_check_san( const mbedtls_x509_buf *name,
                                const char *cn, size_t cn_len )
 {
+    int64_t ip;
     const unsigned char san_type = (unsigned char) name->tag &
                                    MBEDTLS_ASN1_TAG_VALUE_MASK;
     /* dNSName */
     if( san_type == MBEDTLS_X509_SAN_DNS_NAME )
         return( x509_crt_check_cn( name, cn, cn_len ) );
+    if( san_type == MBEDTLS_X509_SAN_IP_ADDRESS &&
+        name->len == 4 && ( ip = ParseIp( cn, cn_len ) ) != -1 &&
+        ip == READ32BE( name->p ) ) {
+        return( 0 );
+    }
     /* (We may handle other types here later.) */
     /* Unrecognized type */
     return -1;

tool/build/deltaify.c

@@ -0,0 +1,96 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2021 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/calls.h"
+#include "libc/calls/sigbits.h"
+#include "libc/calls/struct/sigaction.h"
+#include "libc/calls/struct/siginfo.h"
+#include "libc/calls/ucontext.h"
+#include "libc/errno.h"
+#include "libc/log/check.h"
+#include "libc/mem/mem.h"
+#include "libc/runtime/runtime.h"
+#include "libc/stdio/stdio.h"
+#include "libc/sysv/consts/sa.h"
+#include "libc/sysv/consts/sig.h"
+#include "libc/time/time.h"
+#include "libc/x/x.h"
+
+static int pid;
+
+static void RelaySig(int sig, struct siginfo *si, struct ucontext *uc) {
+  kill(pid, sig);
+}
+
+int main(int argc, char *argv[]) {
+  FILE *f;
+  char *s;
+  bool ok;
+  int64_t micros;
+  long double t1, t2;
+  int ws, pipefds[2];
+  t1 = nowl();
+  if (argc < 2) {
+    f = stdin;
+  } else {
+    sigset_t block, mask;
+    struct sigaction saignore = {.sa_handler = SIG_IGN};
+    struct sigaction sadefault = {.sa_handler = SIG_DFL};
+    struct sigaction sarelay = {.sa_sigaction = RelaySig,
+                                .sa_flags = SA_RESTART};
+    sigemptyset(&block);
+    sigaddset(&mask, SIGINT);
+    sigaddset(&mask, SIGQUIT);
+    sigaddset(&mask, SIGCHLD);
+    sigprocmask(SIG_BLOCK, &block, &mask);
+    sigaction(SIGHUP, &sarelay, 0);
+    sigaction(SIGTERM, &sarelay, 0);
+    sigaction(SIGINT, &saignore, 0);
+    sigaction(SIGQUIT, &saignore, 0);
+    CHECK_NE(-1, pipe(pipefds));
+    CHECK_NE(-1, (pid = vfork()));
+    if (!pid) {
+      close(pipefds[0]);
+      dup2(pipefds[1], 1);
+      sigaction(SIGHUP, &sadefault, NULL);
+      sigaction(SIGTERM, &sadefault, NULL);
+      sigaction(SIGINT, &sadefault, NULL);
+      sigaction(SIGQUIT, &sadefault, NULL);
+      sigprocmask(SIG_SETMASK, &mask, NULL);
+      execvp(argv[1], argv + 1);
+      _exit(127);
+    }
+    close(pipefds[1]);
+    sigprocmask(SIG_SETMASK, &mask, NULL);
+    CHECK((f = fdopen(pipefds[0], "r")));
+    CHECK_NE(-1, setvbuf(f, malloc(4096), _IOLBF, 4096));
+  }
+  for (;;) {
+    if (!(s = xgetline(f))) break;
+    t2 = nowl();
+    micros = (t2 - t1) * 1e6;
+    t1 = t2;
+    printf("%,16ld %s", micros, s);
+    free(s);
+  }
+  ok = !ferror(f);
+  if (argc < 2) return ok ? 0 : 1;
+  fclose(f);
+  while (waitpid(pid, &ws, 0) == -1) CHECK_EQ(EINTR, errno);
+  return !WIFEXITED(ws) ? 128 + WTERMSIG(ws) : ok ? WEXITSTATUS(ws) : 1;
+}

tool/net/counters.inc

@@ -84,12 +84,14 @@ C(sslcantciphers)
 C(sslhandshakefails)
 C(sslhandshakes)
 C(sslnociphers)
+C(sslnoclientcert)
 C(sslnoversion)
 C(sslshakemacs)
 C(ssltimeouts)
 C(sslunknownca)
 C(sslunknowncert)
 C(sslupgrades)
+C(sslverifyfailed)
 C(statfails)
 C(staticrequests)
 C(stats)

tool/net/demo/fetch.lua

@@ -0,0 +1,74 @@
+-- Fetch() API Demo
+
+local function WriteForm(url)
+   Write('<!doctype html>\r\n')
+   Write(string.format([[
+     <title>redbean fetch demo</title>
+     <style>
+       body {
+         padding: 1em;
+       }
+       h1 a {
+         color: inherit;
+         text-decoration: none;
+       }
+       h1 img {
+         vertical-align: middle;
+       }
+       input {
+         margin: 1em;
+         padding: .5em;
+       }
+       p {
+         word-break: break-word;
+       }
+       p span {
+         display: block;
+         text-indent: -1em;
+         padding-left: 1em;
+       }
+     </style>
+     <h1>
+       <a href="/"><img src="/redbean.png"></a>
+       <a href="fetch.lua">redbean fetch demo</a>
+     </h1>
+     <form action="fetch.lua" method="post">
+       <input type="text" id="url" name="url" size="70"
+              value="%s" placeholder="uri" autofocus>
+       <input type="submit" value="fetch">
+     </form>
+   ]], EscapeHtml(url)))
+end
+
+local function main()
+   if IsPublicIp(GetClientAddr()) then
+      ServeError(403)
+   elseif GetMethod() == 'GET' or GetMethod() == 'HEAD' then
+      WriteForm("https://www.cloudflare.com/robots.txt")
+   elseif GetMethod() == 'POST' then
+      status, headers, payload = Fetch(GetParam('url'))
+      WriteForm(GetParam('url'))
+      Write('<dl>\r\n')
+      Write('<dt>Status\r\n')
+      Write(string.format('<dd><p>%d %s\r\n', status, GetHttpReason(status)))
+      Write('<dt>Headers\r\n')
+      Write('<dd><p>\r\n')
+      for k,v in pairs(headers) do
+         Write('<span><strong>')
+         Write(EscapeHtml(k))
+         Write('</strong>: ')
+         Write(EscapeHtml(v))
+         Write('</span>\r\n')
+      end
+      Write('<dt>Payload\r\n')
+      Write('<dd><pre>')
+      Write(EscapeHtml(VisualizeControlCodes(payload)))
+      Write('</pre>\r\n')
+      Write('</dl>\r\n')
+   else
+      ServeError(405)
+      SetHeader('Allow', 'GET, HEAD, POST')
+   end
+end
+
+main()

tool/net/demo/hello.lua

@@ -1,2 +1,4 @@
-StoreAsset('/hi', 'sup')
+if not IsPublicIp(GetClientAddr()) then
+   StoreAsset('/hi', 'sup')
+end
 mymodule.hello()

tool/net/help.txt

@@ -34,8 +34,10 @@ FLAGS
   -b        log message bodies
   -a        log resource usage
   -g        log handler latency
-  -f        log worker function calls
+  -j        enable ssl client verify
+  -k        disable ssl fetch verify
   -B        use stronger cryptography
+  -f        log worker function calls
   -s        increase silence                  [repeatable]
   -v        increase verbosity                [repeatable]
   -V        increase ssl verbosity            [repeatable]
@@ -45,8 +47,8 @@ FLAGS
   -R /X=/Y  rewrites X to Y                   [repeatable]
   -K PATH   tls private key path              [repeatable]
   -C PATH   tls certificate(s) path           [repeatable]
-  -t MS     tunes read and write timeouts     [def. 60000]
   -M INT    tunes max message payload size    [def. 65536]
+  -t INT    timeout ms or keepalive sec if <0 [def. 60000]
   -p PORT   listen port                       [def. 8080; repeatable]
   -l ADDR   listen addr                       [def. 0.0.0.0; repeatable]
   -c SEC    configures static cache-control

tool/net/net.mk

@@ -50,6 +50,7 @@ TOOL_NET_DIRECTDEPS =						\
 	LIBC_X							\
 	LIBC_ZIPOS						\
 	NET_HTTP						\
+	NET_HTTPS						\
 	THIRD_PARTY_GDTOA					\
 	THIRD_PARTY_GETOPT					\
 	THIRD_PARTY_LUA						\
@@ -130,6 +131,7 @@ o/$(MODE)/tool/net/redbean-demo.com:				\
 		tool/net/demo/seekable.txt			\
 		tool/net/demo/virtualbean.html			\
 		tool/net/demo/printpayload.lua			\
+		tool/net/demo/fetch.lua				\
 		tool/net/redbean.c				\
 		net/http/parsehttpmessage.c			\
 		net/http/parseurl.c				\
@@ -145,10 +147,10 @@ o/$(MODE)/tool/net/redbean-demo.com:				\
 	@(cd o/$(MODE)/tool/net && ../../host/third_party/infozip/zip.com -qr redbean-demo.com .lua)
 	@$(COMPILE) -AZIP -T$@ o/$(MODE)/host/third_party/infozip/zip.com -qj $@ tool/net/demo/hello.lua tool/net/demo/sql.lua
 	@echo "<-- check out this lua server page" | $(COMPILE) -AZIP -T$@ o/$(MODE)/host/third_party/infozip/zip.com -cqj $@ tool/net/demo/redbean.lua
-	@$(COMPILE) -AZIP -T$@ o/$(MODE)/host/third_party/infozip/zip.com -qj $@ tool/net/demo/404.html tool/net/favicon.ico tool/net/redbean.png tool/net/demo/redbean-form.lua tool/net/demo/redbean-xhr.lua tool/net/demo/printpayload.lua
+	@$(COMPILE) -AZIP -T$@ o/$(MODE)/host/third_party/infozip/zip.com -qj $@ tool/net/demo/404.html tool/net/favicon.ico tool/net/redbean.png tool/net/demo/redbean-form.lua tool/net/demo/redbean-xhr.lua tool/net/demo/printpayload.lua tool/net/demo/fetch.lua
 	@echo Uncompressed for HTTP Range requests | $(COMPILE) -AZIP -T$@ o/$(MODE)/host/third_party/infozip/zip.com -cqj0 $@ tool/net/demo/seekable.txt
 	@$(COMPILE) -AZIP -T$@ o/$(MODE)/host/third_party/infozip/zip.com -q $@ tool/net/ tool/net/demo/ tool/net/demo/index.html tool/net/demo/redbean.css tool/net/redbean.c net/http/parsehttpmessage.c net/http/parseurl.c net/http/encodeurl.c test/net/http/parsehttpmessage_test.c test/net/http/parseurl_test.c
-	@printf "<p>This is a live instance of <a href=https://justine.lol/redbean/>redbean</a>: a tiny multiplatform webserver that <a href=https://news.ycombinator.com/item?id=26271117>went viral</a> on hacker news a few months ago.\r\nSince then, we've added Lua dynamic serving, which also goes as fast as 1,000,000 requests per second on a core i9 (rather than a cheap virtual machine like this)\nin addition to SQLite and SSL. The text you're reading now is a PKZIP End Of Central Directory comment.\r\n<p>redbean aims to be production worthy across six operating systems, using a single executable file (this demo is hosted on FreeBSD 13). redbean has been enhanced to restore the APE header after startup.\r\nIt automatically generates this listing page based on your zip contents. If you use redbean as an application server / web development environment,\r\nthen you'll find other new and useful features like function call logging so you can get that sweet sweet microsecond scale latency." | $(COMPILE) -AZIP -T$@ o/$(MODE)/host/third_party/infozip/zip.com -z $@
+	@printf "<p>Thank you for using <a href=https://justine.lol/redbean/>redbean</a> the tiniest most vertically integrated actually portable web server with superior performance." | $(COMPILE) -AZIP -T$@ o/$(MODE)/host/third_party/infozip/zip.com -z $@
 	@$(COMPILE) -AMKDIR -T$@ mkdir -p o/$(MODE)/tool/net/virtualbean.justine.lol/
 	@$(COMPILE) -ACP -T$@ cp tool/net/redbean.png o/$(MODE)/tool/net/virtualbean.justine.lol/redbean.png
 	@$(COMPILE) -ACP -T$@ cp tool/net/demo/virtualbean.html o/$(MODE)/tool/net/virtualbean.justine.lol/index.html

usr/share/ssl/root/digicert.pem

@@ -1,3 +1,5 @@
+https://www.digicert.com/kb/digicert-root-certificates.htm
+
 -----BEGIN CERTIFICATE-----
 MIIDtzCCAp+gAwIBAgIQDOfg5RfYRv6P5WD8G/AwOTANBgkqhkiG9w0BAQUFADBl
 MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
@@ -20,6 +22,7 @@ NW1fiQG2SVufAQWbqz0lwcy2f8Lxb4bG+mRo64EtlOtCt/qMHt1i8b5QZ7dsvfPx
 H2sMNgcWfzd8qVttevESRmCD1ycEvkvOl77DZypoEd+A5wwzZr8TDRRu838fYxAe
 +o0bJW1sj6W3YQGx0qMmoRBxna3iw/nDmVG3KwcIzi7mULKn+gpFL6Lw8g==
 -----END CERTIFICATE-----
+
 -----BEGIN CERTIFICATE-----
 MIIDljCCAn6gAwIBAgIQC5McOtY5Z+pnI7/Dr5r0SzANBgkqhkiG9w0BAQsFADBl
 MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
@@ -42,6 +45,7 @@ B5yiutkBclzzTcHdDrEcDcRjvq30FPuJ7KJBDkzMyFdA0G4Dqs0MjomZmWzwPDCv
 ON9vvKO+KSAnq3T/EyJ43pdSVR6DtVQgA+6uwE9W3jfMw3+qBCe703e4YtsXfJwo
 IhNzbM8m9Yop5w==
 -----END CERTIFICATE-----
+
 -----BEGIN CERTIFICATE-----
 MIICRjCCAc2gAwIBAgIQC6Fa+h3foLVJRK/NJKBs7DAKBggqhkjOPQQDAzBlMQsw
 CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu
@@ -57,6 +61,7 @@ AwNnADBkAjAlpIFFAmsSS3V0T8gj43DydXLefInwz5FyYZ5eEJJZVrmDxxDnOOlY
 JjZ91eQ0hjkCMHw2U/Aw5WJjOpnitqM7mzT6HtoQknFekROn3aRukswy1vUhZscv
 6pZjamVFkpUBtA==
 -----END CERTIFICATE-----
+
 -----BEGIN CERTIFICATE-----
 MIIDrzCCApegAwIBAgIQCDvgVpBCRrGhdWrJWZHHSjANBgkqhkiG9w0BAQUFADBh
 MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
@@ -79,6 +84,7 @@ PnlUkiaY4IBIqDfv8NZ5YBberOgOzW6sRBc4L0na4UU+Krk2U886UAb3LujEV0ls
 YSEY1QSteDwsOoBrp+uvFRTp2InBuThs4pFsiv9kuXclVzDAGySj4dzp30d8tbQk
 CAUw7C29C79Fv1C5qfPrmAESrciIxpg0X40KPMbp1ZWVbd4=
 -----END CERTIFICATE-----
+
 -----BEGIN CERTIFICATE-----
 MIIDjjCCAnagAwIBAgIQAzrx5qcRqaC7KGSxHQn65TANBgkqhkiG9w0BAQsFADBh
 MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
@@ -101,6 +107,7 @@ Fdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBHQRFXGU7Aj64GxJUTFy8bJZ91
 pLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl
 MrY=
 -----END CERTIFICATE-----
+
 -----BEGIN CERTIFICATE-----
 MIICPzCCAcWgAwIBAgIQBVVWvPJepDU1w6QP1atFcjAKBggqhkjOPQQDAzBhMQsw
 CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu
@@ -116,6 +123,7 @@ AK288mw/EkrRLTnDCgmXc/SINoyIJ7vmiI1Qhadj+Z4y3maTD/HMsQmP3Wyr+mt/
 oAIwOWZbwmSNuJ5Q3KjVSaLtx9zRSX8XAbjIho9OjIgrqJqpisXRAL34VOKa5Vt8
 sycX
 -----END CERTIFICATE-----
+
 -----BEGIN CERTIFICATE-----
 MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBs
 MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
@@ -139,6 +147,7 @@ Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe
 vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep
 +OkuE6N36B9K
 -----END CERTIFICATE-----
+
 -----BEGIN CERTIFICATE-----
 MIIFkDCCA3igAwIBAgIQBZsbV56OITLiOQe9p3d1XDANBgkqhkiG9w0BAQwFADBi
 MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
@@ -171,3 +180,72 @@ r/OSmbaz5mEP0oUA51Aa5BuVnRmhuZyxm7EAHu/QD09CbMkKvO5D+jpxpchNJqU1
 /YldvIViHTLSoCtU7ZpXwdv6EM8Zt4tKG48BtieVU+i2iW1bvGjUI+iLUaJW+fCm
 gKDWHrO8Dw9TdSmq6hN35N6MgSGtBxBHEa2HPQfRdbzP82Z+
 -----END CERTIFICATE-----
+
+-----BEGIN CERTIFICATE-----
+MIIDdzCCAl+gAwIBAgIEAgAAuTANBgkqhkiG9w0BAQUFADBaMQswCQYDVQQGEwJJ
+RTESMBAGA1UEChMJQmFsdGltb3JlMRMwEQYDVQQLEwpDeWJlclRydXN0MSIwIAYD
+VQQDExlCYWx0aW1vcmUgQ3liZXJUcnVzdCBSb290MB4XDTAwMDUxMjE4NDYwMFoX
+DTI1MDUxMjIzNTkwMFowWjELMAkGA1UEBhMCSUUxEjAQBgNVBAoTCUJhbHRpbW9y
+ZTETMBEGA1UECxMKQ3liZXJUcnVzdDEiMCAGA1UEAxMZQmFsdGltb3JlIEN5YmVy
+VHJ1c3QgUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKMEuyKr
+mD1X6CZymrV51Cni4eiVgLGw41uOKymaZN+hXe2wCQVt2yguzmKiYv60iNoS6zjr
+IZ3AQSsBUnuId9Mcj8e6uYi1agnnc+gRQKfRzMpijS3ljwumUNKoUMMo6vWrJYeK
+mpYcqWe4PwzV9/lSEy/CG9VwcPCPwBLKBsua4dnKM3p31vjsufFoREJIE9LAwqSu
+XmD+tqYF/LTdB1kC1FkYmGP1pWPgkAx9XbIGevOF6uvUA65ehD5f/xXtabz5OTZy
+dc93Uk3zyZAsuT3lySNTPx8kmCFcB5kpvcY67Oduhjprl3RjM71oGDHweI12v/ye
+jl0qhqdNkNwnGjkCAwEAAaNFMEMwHQYDVR0OBBYEFOWdWTCCR1jMrPoIVDaGezq1
+BE3wMBIGA1UdEwEB/wQIMAYBAf8CAQMwDgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3
+DQEBBQUAA4IBAQCFDF2O5G9RaEIFoN27TyclhAO992T9Ldcw46QQF+vaKSm2eT92
+9hkTI7gQCvlYpNRhcL0EYWoSihfVCr3FvDB81ukMJY2GQE/szKN+OMY3EU/t3Wgx
+jkzSswF07r51XgdIGn9w/xZchMB5hbgF/X++ZRGjD8ACtPhSNzkE1akxehi/oCr0
+Epn3o0WC4zxe9Z2etciefC7IpJ5OCBRLbf1wbWsaY71k5h+3zvDyny67G7fyUIhz
+ksLi4xaNmjICq44Y3ekQEe5+NauQrz4wlHrQMz2nZQ/1/I6eYs9HRCwBXbsdtTLS
+R9I4LtD+gdwyah617jzV/OeBHRnDJELqYzmp
+-----END CERTIFICATE-----
+
+-----BEGIN CERTIFICATE-----
+MIICGTCCAZ+gAwIBAgIQCeCTZaz32ci5PhwLBCou8zAKBggqhkjOPQQDAzBOMQsw
+CQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xJjAkBgNVBAMTHURp
+Z2lDZXJ0IFRMUyBFQ0MgUDM4NCBSb290IEc1MB4XDTIxMDExNTAwMDAwMFoXDTQ2
+MDExNDIzNTk1OVowTjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDkRpZ2lDZXJ0LCBJ
+bmMuMSYwJAYDVQQDEx1EaWdpQ2VydCBUTFMgRUNDIFAzODQgUm9vdCBHNTB2MBAG
+ByqGSM49AgEGBSuBBAAiA2IABMFEoc8Rl1Ca3iOCNQfN0MsYndLxf3c1TzvdlHJS
+7cI7+Oz6e2tYIOyZrsn8aLN1udsJ7MgT9U7GCh1mMEy7H0cKPGEQQil8pQgO4CLp
+0zVozptjn4S1mU1YoI71VOeVyaNCMEAwHQYDVR0OBBYEFMFRRVBZqz7nLFr6ICIS
+B4CIfBFqMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49
+BAMDA2gAMGUCMQCJao1H5+z8blUD2WdsJk6Dxv3J+ysTvLd6jLRl0mlpYxNjOyZQ
+LgGheQaRnUi/wr4CMEfDFXuxoJGZSZOoPHzoRgaLLPIxAJSdYsiJvRmEFOml+wG4
+DXZDjC5Ty3zfDBeWUA==
+-----END CERTIFICATE-----
+
+-----BEGIN CERTIFICATE-----
+MIIFZjCCA06gAwIBAgIQCPm0eKj6ftpqMzeJ3nzPijANBgkqhkiG9w0BAQwFADBN
+MQswCQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xJTAjBgNVBAMT
+HERpZ2lDZXJ0IFRMUyBSU0E0MDk2IFJvb3QgRzUwHhcNMjEwMTE1MDAwMDAwWhcN
+NDYwMTE0MjM1OTU5WjBNMQswCQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQs
+IEluYy4xJTAjBgNVBAMTHERpZ2lDZXJ0IFRMUyBSU0E0MDk2IFJvb3QgRzUwggIi
+MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCz0PTJeRGd/fxmgefM1eS87IE+
+ajWOLrfn3q/5B03PMJ3qCQuZvWxX2hhKuHisOjmopkisLnLlvevxGs3npAOpPxG0
+2C+JFvuUAT27L/gTBaF4HI4o4EXgg/RZG5Wzrn4DReW+wkL+7vI8toUTmDKdFqgp
+wgscONyfMXdcvyej/Cestyu9dJsXLfKB2l2w4SMXPohKEiPQ6s+d3gMXsUJKoBZM
+pG2T6T867jp8nVid9E6P/DsjyG244gXazOvswzH016cpVIDPRFtMbzCe88zdH5RD
+nU1/cHAN1DrRN/BsnZvAFJNY781BOHW8EwOVfH/jXOnVDdXifBBiqmvwPXbzP6Po
+sMH976pXTayGpxi0KcEsDr9kvimM2AItzVwv8n/vFfQMFawKsPHTDU9qTXeXAaDx
+Zre3zu/O7Oyldcqs4+Fj97ihBMi8ez9dLRYiVu1ISf6nL3kwJZu6ay0/nTvEF+cd
+Lvvyz6b84xQslpghjLSR6Rlgg/IwKwZzUNWYOwbpx4oMYIwo+FKbbuH2TbsGJJvX
+KyY//SovcfXWJL5/MZ4PbeiPT02jP/816t9JXkGPhvnxd3lLG7SjXi/7RgLQZhNe
+XoVPzthwiHvOAbWWl9fNff2C+MIkwcoBOU+NosEUQB+cZtUMCUbW8tDRSHZWOkPL
+tgoRObqME2wGtZ7P6wIDAQABo0IwQDAdBgNVHQ4EFgQUUTMc7TZArxfTJc1paPKv
+TiM+s0EwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcN
+AQEMBQADggIBAGCmr1tfV9qJ20tQqcQjNSH/0GEwhJG3PxDPJY7Jv0Y02cEhJhxw
+GXIeo8mH/qlDZJY6yFMECrZBu8RHANmfGBg7sg7zNOok992vIGCukihfNudd5N7H
+PNtQOa27PShNlnx2xlv0wdsUpasZYgcYQF+Xkdycx6u1UQ3maVNVzDl92sURVXLF
+O4uJ+DQtpBflF+aZfTCIITfNMBc9uPK8qHWgQ9w+iUuQrm0D4ByjoJYJu32jtyoQ
+REtGBzRj7TG5BO6jm5qu5jF49OokYTurWGT/u4cnYiWB39yhL/btp/96j1EuMPik
+AdKFOV8BmZZvWltwGUb+hmA+rYAQCd05JS9Yf7vSdPD3Rh9GOUrYU9DzLjtxpdRv
+/PNn5AeP3SYZ4Y1b+qOTEZvpyDrDVWiakuFSdjjo4bq9+0/V77PnSIMx8IIh47a+
+p6tv75/fTM8BuGJqIz3nCU2AG3swpMPdB380vqQmsvZB6Akd4yCYqjdP//fx4ilw
+MUc/dNAUFvohigLVigmUdy7yWSiLfFCSCmZ4OIN1xLVaqBHG5cGdZlXPU8Sv13WF
+qUITVuwhd4GTWgzqltlJyqEI8pc7bZsEGCREjnwB8twl2F6GmrE52/WRMmrRpnCK
+ovfepEWFJqgejF0pW8hL2JpqA15w8oVPbEtoL8pU9ozaMv7Da4M/OMZ+
+-----END CERTIFICATE-----

usr/share/ssl/root/isrg.pem

@@ -1,3 +1,5 @@
+https://letsencrypt.org/certificates/
+
 -----BEGIN CERTIFICATE-----
 MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
 TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
@@ -29,3 +31,17 @@ oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
 mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
 emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
 -----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIICGzCCAaGgAwIBAgIQQdKd0XLq7qeAwSxs6S+HUjAKBggqhkjOPQQDAzBPMQsw
+CQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2gg
+R3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBYMjAeFw0yMDA5MDQwMDAwMDBaFw00
+MDA5MTcxNjAwMDBaME8xCzAJBgNVBAYTAlVTMSkwJwYDVQQKEyBJbnRlcm5ldCBT
+ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEVMBMGA1UEAxMMSVNSRyBSb290IFgyMHYw
+EAYHKoZIzj0CAQYFK4EEACIDYgAEzZvVn4CDCuwJSvMWSj5cz3es3mcFDR0HttwW
++1qLFNvicWDEukWVEYmO6gbf9yoWHKS5xcUy4APgHoIYOIvXRdgKam7mAHf7AlF9
+ItgKbppbd9/w+kHsOdx1ymgHDB/qo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T
+AQH/BAUwAwEB/zAdBgNVHQ4EFgQUfEKWrt5LSDv6kviejM9ti6lyN5UwCgYIKoZI
+zj0EAwMDaAAwZQIwe3lORlCEwkSHRhtFcP9Ymd70/aTSVaYgLXTWNLxBo1BfASdW
+tL4ndQavEi51mI38AjEAi/V3bNTIZargCyzuFJ0nN6T5U6VR5CmD1/iQMVtCnwr1
+/q4AaOeMSQ+2b1tbFfLn
+-----END CERTIFICATE-----