cosmopolitan/tool/net/fetch.inc

#define FetchHasHeader(H)    (!!msg.headers[H].a)
#define FetchHeaderData(H)   (inbuf.p + msg.headers[H].a)
#define FetchHeaderLength(H) (msg.headers[H].b - msg.headers[H].a)
#define FetchHeaderEqualCase(H, S) \
  SlicesEqualCase(S, strlen(S), FetchHeaderData(H), FetchHeaderLength(H))

#define kaNONE  0
#define kaOPEN  1
#define kaKEEP  2
#define kaCLOSE 3

static int LuaFetch(lua_State *L) {
#define ssl nope  // TODO(jart): make this file less huge
  ssize_t rc;
  bool usingssl;
  uint32_t ip;
  struct Url url;
  int t, ret, sock = -1, hdridx;
  const char *host, *port;
  char *request;
  struct TlsBio *bio;
  struct addrinfo *addr;
  struct Buffer inbuf;     // shadowing intentional
  struct HttpMessage msg;  // shadowing intentional
  struct HttpUnchunker u;
  const char *urlarg, *body, *method;
  char *conlenhdr = "";
  char *headers = 0;
  const char *hosthdr = 0;
  const char *connhdr = 0;
  const char *agenthdr = brand;
  const char *key, *val, *hdr;
  size_t keylen, vallen;
  size_t urlarglen, requestlen, paylen, bodylen;
  size_t i, g, hdrsize;
  int keepalive = kaNONE;
  char canmethod[9] = {0};
  uint64_t imethod;
  int numredirects = 0, maxredirects = 5;
  bool followredirect = true;
  struct addrinfo hints = {.ai_family = AF_INET,
                           .ai_socktype = SOCK_STREAM,
                           .ai_protocol = IPPROTO_TCP,
                           .ai_flags = AI_NUMERICSERV};

  (void)ret;
  (void)usingssl;

  /*
   * Get args: url [, body | {method = "PUT", body = "..."}]
   */
  urlarg = luaL_checklstring(L, 1, &urlarglen);
  if (lua_istable(L, 2)) {
    lua_settop(L, 2);  // discard any extra arguments
    lua_getfield(L, 2, "body");
    body = luaL_optlstring(L, -1, "", &bodylen);
    lua_getfield(L, 2, "method");
    // use GET by default if no method is provided
    method = luaL_optstring(L, -1, "GET");
    if ((imethod = ParseHttpMethod(method, -1))) {
      WRITE64LE(canmethod, imethod);
      method = canmethod;
    } else {
      return LuaNilError(L, "bad method");
    }
    lua_getfield(L, 2, "followredirect");
    if (lua_isboolean(L, -1))
      followredirect = lua_toboolean(L, -1);
    lua_getfield(L, 2, "maxredirects");
    maxredirects = luaL_optinteger(L, -1, maxredirects);
    lua_getfield(L, 2, "numredirects");
    numredirects = luaL_optinteger(L, -1, numredirects);
    lua_getfield(L, 2, "keepalive");
    if (!lua_isnil(L, -1)) {
      if (lua_istable(L, -1)) {
        keepalive = kaOPEN;  // will be updated based on host later
      } else if (lua_isboolean(L, -1)) {
        keepalive = lua_toboolean(L, -1) ? kaOPEN : kaNONE;
        if (keepalive) {
          lua_createtable(L, 0, 1);
          lua_setfield(L, 2, "keepalive");
        }
      } else {
        return luaL_argerror(L, 2,
                             "invalid keepalive value;"
                             " boolean or table expected");
      }
    }
    lua_getfield(L, 2, "headers");
    if (!lua_isnil(L, -1)) {
      if (!lua_istable(L, -1))
        return luaL_argerror(L, 2, "invalid headers value; table expected");

      lua_pushnil(L);
      while (lua_next(L, -2)) {
        if (lua_type(L, -2) == LUA_TSTRING) {  // skip any non-string keys
          key = lua_tolstring(L, -2, &keylen);
          if (!IsValidHttpToken(key, keylen))
            return LuaNilError(L, "invalid header name: %s", key);

          val = lua_tolstring(L, -1, &vallen);
          if (!(hdr = gc(EncodeHttpHeaderValue(val, vallen, 0))))
            return LuaNilError(L, "invalid header %s value encoding", key);

          // Content-Length will be overwritten; skip it to avoid duplicates;
          // also allow unknown headers
          if ((hdridx = GetHttpHeader(key, keylen)) == -1 ||
              hdridx != kHttpContentLength) {
            if (hdridx == kHttpUserAgent) {
              agenthdr = hdr;
            } else if (hdridx == kHttpHost) {
              hosthdr = hdr;
            } else if (hdridx == kHttpConnection) {
              connhdr = hdr;
            } else {
              appendd(&headers, key, keylen);
              appendw(&headers, READ16LE(": "));
              appends(&headers, hdr);
              appendw(&headers, READ16LE("\r\n"));
            }
          }
        }
        lua_pop(L, 1);  // pop the value, keep the key for the next iteration
      }
    }
    lua_settop(L, 2);  // drop all added elements to keep the stack balanced
  } else if (lua_isnoneornil(L, 2)) {
    body = "";
    bodylen = 0;
    method = "GET";
  } else {
    body = luaL_checklstring(L, 2, &bodylen);
    method = "POST";
  }
  // provide Content-Length header unless it's zero and not expected
  imethod = ParseHttpMethod(method, -1);
  if (bodylen > 0 ||
      !(imethod == kHttpGet || imethod == kHttpHead || imethod == kHttpTrace ||
        imethod == kHttpDelete || imethod == kHttpConnect)) {
    conlenhdr = gc(xasprintf("Content-Length: %zu\r\n", bodylen));
  }

  /*
   * Parse URL.
   */
  gc(ParseUrl(urlarg, urlarglen, &url, true));
  gc(url.params.p);
  DEBUGF("(ftch) client fetching %`'s (host=%`'.*s, port=%.*s, path=%`'.*s)",
         urlarg, url.host.n, url.host.p, url.port.n, url.port.p, url.path.n,
         url.path.p);

  usingssl = false;
  if (url.scheme.n) {
#ifndef UNSECURE
    if (!unsecure && url.scheme.n == 5 &&
        !memcasecmp(url.scheme.p, "https", 5)) {
      usingssl = true;
    } else
#endif
        if (!(url.scheme.n == 4 && !memcasecmp(url.scheme.p, "http", 4))) {
      return LuaNilError(L, "bad scheme");
    }
  }

#ifndef UNSECURE
  if (usingssl)
    keepalive = kaNONE;
  if (usingssl && !sslinitialized)
    TlsInit();
#endif

  if (url.host.n) {
    host = gc(strndup(url.host.p, url.host.n));
    if (url.port.n) {
      port = gc(strndup(url.port.p, url.port.n));
#ifndef UNSECURE
    } else if (usingssl) {
      port = "443";
#endif
    } else {
      port = "80";
    }
  } else if ((ip = ParseIp(urlarg, -1)) != -1) {
    host = urlarg;
    port = "80";
  } else {
    return LuaNilError(L, "invalid host");
  }
  if (!IsAcceptableHost(host, -1)) {
    return LuaNilError(L, "invalid host");
  }
  if (!IsAcceptablePort(port, -1)) {
    return LuaNilError(L, "invalid port");
  }
  if (!hosthdr)
    hosthdr = gc(xasprintf("%s:%s", host, port));

  // check if hosthdr is in keepalive table
  if (keepalive && lua_istable(L, 2)) {
    lua_getfield(L, 2, "keepalive");
    lua_getfield(L, -1, "close");  // aft: -2=tbl, -1=close
    lua_getfield(L, -2, hosthdr);  // aft: -3=tbl, -2=close, -1=hosthdr
    if (lua_isinteger(L, -1)) {
      sock = lua_tointeger(L, -1);
      keepalive = lua_toboolean(L, -2) ? kaCLOSE : kaKEEP;
      // remove host mapping, as the socket is ether being closed
      // (so needs to be removed) or will be added after the request is done;
      // this also helps to keep the mapping clean in case of an error
      lua_pushnil(L);  // aft: -4=tbl, -3=close, -2=hosthdr, -1=nil
      lua_setfield(L, -4, hosthdr);
      VERBOSEF("(ftch) reuse socket %d for host %s (and %s)", sock, hosthdr,
               keepalive == kaCLOSE ? "close" : "keep");
    }
    lua_settop(L, 2);  // drop all added elements to keep the stack balanced
  }

  url.fragment.p = 0, url.fragment.n = 0;
  url.scheme.p = 0, url.scheme.n = 0;
  url.user.p = 0, url.user.n = 0;
  url.pass.p = 0, url.pass.n = 0;
  url.host.p = 0, url.host.n = 0;
  url.port.p = 0, url.port.n = 0;
  if (!url.path.n || url.path.p[0] != '/') {
    void *p = gc(xmalloc(1 + url.path.n));
    mempcpy(mempcpy(p, "/", 1), url.path.p, url.path.n);
    url.path.p = p;
    ++url.path.n;
  }

  /*
   * Create HTTP message.
   */
  request = 0;
  appendf(&request,
          "%s %s HTTP/1.1\r\n"
          "Host: %s\r\n"
          "Connection: %s\r\n"
          "User-Agent: %s\r\n"
          "%s%s"
          "\r\n",
          method, gc(EncodeUrl(&url, 0)), hosthdr,
          (keepalive == kaNONE || keepalive == kaCLOSE)
              ? "close"
              : (connhdr ? connhdr : "keep-alive"),
          agenthdr, conlenhdr, headers ? headers : "");
  appendd(&request, body, bodylen);
  requestlen = appendz(request).i;
  gc(request);

  if (keepalive == kaNONE || keepalive == kaOPEN) {
    /*
     * Perform DNS lookup.
     */
    DEBUGF("(ftch) client resolving %s", host);
    if ((rc = getaddrinfo(host, port, &hints, &addr)) != 0) {
      return LuaNilError(L, "getaddrinfo(%s:%s) error: EAI_%s %s", host, port,
                         gai_strerror(rc), strerror(errno));
    }

    /*
     * Connect to server.
     */
    ip = ntohl(((struct sockaddr_in *)addr->ai_addr)->sin_addr.s_addr);
    DEBUGF("(ftch) client connecting %hhu.%hhu.%hhu.%hhu:%d", ip >> 24,
           ip >> 16, ip >> 8, ip,
           ntohs(((struct sockaddr_in *)addr->ai_addr)->sin_port));
    CHECK_NE(-1, (sock = GoodSocket(addr->ai_family, addr->ai_socktype,
                                    addr->ai_protocol, false, &timeout)));
    rc = connect(sock, addr->ai_addr, addr->ai_addrlen);
    freeaddrinfo(addr), addr = 0;
    if (rc == -1) {
      close(sock);
      return LuaNilError(L, "connect(%s:%s) error: %s", host, port,
                         strerror(errno));
    }
  }

  (void)bio;
#ifndef UNSECURE
  if (usingssl) {
    if (sslcliused) {
      mbedtls_ssl_session_reset(&sslcli);
    } else {
      ReseedRng(&rngcli, "child");
    }
    sslcliused = true;
    DEBUGF("(ftch) client handshaking %`'s", host);
    if (!evadedragnetsurveillance) {
      mbedtls_ssl_set_hostname(&sslcli, host);
    }
    bio = gc(malloc(sizeof(struct TlsBio)));
    bio->fd = sock;
    bio->a = 0;
    bio->b = 0;
    bio->c = -1;
    mbedtls_ssl_set_bio(&sslcli, bio, TlsSend, 0, TlsRecvImpl);
    while ((ret = mbedtls_ssl_handshake(&sslcli))) {
      switch (ret) {
        case MBEDTLS_ERR_SSL_WANT_READ:
          break;
        case MBEDTLS_ERR_X509_CERT_VERIFY_FAILED:
          goto VerifyFailed;
        default:
          close(sock);
          return LuaNilTlsError(L, "handshake", ret);
      }
    }
    LockInc(&shared->c.sslhandshakes);
    VERBOSEF("(ftch) shaken %s:%s %s %s", host, port,
             mbedtls_ssl_get_ciphersuite(&sslcli),
             mbedtls_ssl_get_version(&sslcli));
  }
#endif /* UNSECURE */

  /*
   * Send HTTP Message.
   */
  DEBUGF("(ftch) client sending %s request", method);
  for (i = 0; i < requestlen; i += rc) {
#ifndef UNSECURE
    if (usingssl) {
      rc = mbedtls_ssl_write(&sslcli, request + i, requestlen - i);
      if (rc <= 0) {
        if (rc == MBEDTLS_ERR_X509_CERT_VERIFY_FAILED)
          goto VerifyFailed;
        close(sock);
        return LuaNilTlsError(L, "write", rc);
      }
    } else
#endif
        if ((rc = WRITE(sock, request + i, requestlen - i)) <= 0) {
      close(sock);
      return LuaNilError(L, "write error: %s", strerror(errno));
    }
  }
  if (logmessages) {
    LogMessage("sent", request, requestlen);
  }

  /*
   * Handle response.
   */
  bzero(&inbuf, sizeof(inbuf));
  InitHttpMessage(&msg, kHttpResponse);
  for (hdrsize = paylen = t = 0;;) {
    if (inbuf.n == inbuf.c) {
      inbuf.c += 1000;
      inbuf.c += inbuf.c >> 1;
      inbuf.p = realloc(inbuf.p, inbuf.c);
    }
    NOISEF("(ftch) client reading");
#ifndef UNSECURE
    if (usingssl) {
      if ((rc = mbedtls_ssl_read(&sslcli, inbuf.p + inbuf.n,
                                 inbuf.c - inbuf.n)) < 0) {
        if (rc == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) {
          rc = 0;
        } else {
          close(sock);
          free(inbuf.p);
          DestroyHttpMessage(&msg);
          return LuaNilTlsError(L, "read", rc);
        }
      }
    } else
#endif
        if ((rc = READ(sock, inbuf.p + inbuf.n, inbuf.c - inbuf.n)) == -1) {
      close(sock);
      free(inbuf.p);
      DestroyHttpMessage(&msg);
      return LuaNilError(L, "read error: %s", strerror(errno));
    }
    g = rc;
    inbuf.n += g;
    switch (t) {
      case kHttpClientStateHeaders:
        if (!g) {
          WARNF("(ftch) HTTP client %s error", "EOF headers");
          goto TransportError;
        }
        rc = ParseHttpMessage(&msg, inbuf.p, inbuf.n, inbuf.c);
        if (rc == -1) {
          WARNF("(ftch) HTTP client %s error", "ParseHttpMessage");
          goto TransportError;
        }
        if (rc) {
          DEBUGF("(ftch) content-length is %`'.*s",
                 FetchHeaderLength(kHttpContentLength),
                 FetchHeaderData(kHttpContentLength));
          hdrsize = rc;
          if (logmessages) {
            LogMessage("received", inbuf.p, hdrsize);
          }
          if (100 <= msg.status && msg.status <= 199) {
            if ((FetchHasHeader(kHttpContentLength) &&
                 !FetchHeaderEqualCase(kHttpContentLength, "0")) ||
                (FetchHasHeader(kHttpTransferEncoding) &&
                 !FetchHeaderEqualCase(kHttpTransferEncoding, "identity"))) {
              WARNF("(ftch) HTTP client %s error", "Content-Length #1");
              goto TransportError;
            }
            DestroyHttpMessage(&msg);
            InitHttpMessage(&msg, kHttpResponse);
            memmove(inbuf.p, inbuf.p + hdrsize, inbuf.n - hdrsize);
            inbuf.n -= hdrsize;
            break;
          }
          if (msg.status == 204 || msg.status == 304) {
            goto Finished;
          }
          if (FetchHasHeader(kHttpTransferEncoding) &&
              !FetchHeaderEqualCase(kHttpTransferEncoding, "identity")) {
            if (FetchHeaderEqualCase(kHttpTransferEncoding, "chunked")) {
              t = kHttpClientStateBodyChunked;
              bzero(&u, sizeof(u));
              goto Chunked;
            } else {
              WARNF("(ftch) HTTP client %s error", "Transfer-Encoding");
              goto TransportError;
            }
          } else if (FetchHasHeader(kHttpContentLength)) {
            rc = ParseContentLength(FetchHeaderData(kHttpContentLength),
                                    FetchHeaderLength(kHttpContentLength));
            if (rc == -1) {
              WARNF("(ftch) ParseContentLength(%`'.*s) failed",
                    FetchHeaderLength(kHttpContentLength),
                    FetchHeaderData(kHttpContentLength));
              goto TransportError;
            }
            if ((paylen = rc) <= inbuf.n - hdrsize) {
              goto Finished;
            } else {
              t = kHttpClientStateBodyLengthed;
            }
          } else {
            t = kHttpClientStateBody;
          }
        }
        break;
      case kHttpClientStateBody:
        if (!g) {
          paylen = inbuf.n - hdrsize;
          goto Finished;
        }
        break;
      case kHttpClientStateBodyLengthed:
        if (!g) {
          WARNF("(ftch) HTTP client %s error", "EOF body");
          goto TransportError;
        }
        if (inbuf.n - hdrsize >= paylen) {
          goto Finished;
        }
        break;
      case kHttpClientStateBodyChunked:
      Chunked:
        rc = Unchunk(&u, inbuf.p + hdrsize, inbuf.n - hdrsize, &paylen);
        if (rc == -1) {
          WARNF("(ftch) HTTP client %s error", "Unchunk");
          goto TransportError;
        }
        if (rc)
          goto Finished;
        break;
      default:
        __builtin_unreachable();
    }
  }

Finished:
  if (paylen && logbodies)
    LogBody("received", inbuf.p + hdrsize, paylen);
  VERBOSEF("(ftch) completed %s HTTP%02d %d %s %`'.*s", method, msg.version,
           msg.status, urlarg, FetchHeaderLength(kHttpServer),
           FetchHeaderData(kHttpServer));

  // check if the server has requested to close the connection
  // https://www.rfc-editor.org/rfc/rfc2616#section-14.10
  if (keepalive && keepalive != kaCLOSE && FetchHasHeader(kHttpConnection) &&
      FetchHeaderEqualCase(kHttpConnection, "close")) {
    VERBOSEF("(ftch) close keepalive on server request");
    keepalive = kaCLOSE;
  }

  // need to save updated sock for keepalive
  if (keepalive && keepalive != kaCLOSE && lua_istable(L, 2)) {
    lua_getfield(L, 2, "keepalive");
    lua_pushinteger(L, sock);
    lua_setfield(L, -2, hosthdr);
    lua_pop(L, 1);
  }
  if (followredirect && FetchHasHeader(kHttpLocation) &&
      (msg.status == 301 || msg.status == 308 ||  // permanent redirects
       msg.status == 302 || msg.status == 307 ||  // temporary redirects
       msg.status == 303 /* see other; non-GET changes to GET, body lost */) &&
      numredirects < maxredirects) {
    // if 303, then remove body and set method to GET
    if (msg.status == 303) {
      body = "";
      bodylen = 0;
      method = "GET";
    }
    // create table if needed
    if (!lua_istable(L, 2)) {
      lua_settop(L, 1);          // pop body if present
      lua_createtable(L, 0, 3);  // body, method, numredirects
    }
    lua_pushlstring(L, body, bodylen);
    lua_setfield(L, -2, "body");

    lua_pushstring(L, method);
    lua_setfield(L, -2, "method");

    lua_pushinteger(L, numredirects + 1);
    lua_setfield(L, -2, "numredirects");
    // replace URL with Location header, which
    // can be a relative or absolute URL:
    // https://www.rfc-editor.org/rfc/rfc3986#section-4.2
    gc(ParseUrl(FetchHeaderData(kHttpLocation),
                FetchHeaderLength(kHttpLocation), &url, true));
    free(url.params.p);
    VERBOSEF("(ftch) client redirecting %`'.*s "
             "(scheme=%`'.*s, host=%`'.*s, port=%.*s, path=%`'.*s)",
             FetchHeaderLength(kHttpLocation), FetchHeaderData(kHttpLocation),
             url.scheme.n, url.scheme.p, url.host.n, url.host.p, url.port.n,
             url.port.p, url.path.n, url.path.p);
    // while it's possible to check for IsAcceptableHost/IsAcceptablePort
    // it's not clear what to do if they are not;
    // if they are invalid, redirect returns "invalid host" message
    if (url.host.n && url.scheme.n) {
      lua_pushlstring(L, FetchHeaderData(kHttpLocation),
                      FetchHeaderLength(kHttpLocation));
    } else {
      gc(ParseUrl(urlarg, urlarglen, &url, true));
      free(url.params.p);
      // remove user/pass/fragment for the redirect
      url.fragment.p = 0, url.fragment.n = 0;
      url.user.p = 0, url.user.n = 0;
      url.pass.p = 0, url.pass.n = 0;
      if (FetchHeaderData(kHttpLocation)[0] == '/') {
        // if the path is absolute, then use it
        // so `/redir/more` -> `/less` becomes `/less`
        url.path.n = 0;  // replace the path
      } else {
        // if the path is relative, then merge it,
        // so `/redir/more` -> `less` becomes `/redir/less`
        while (url.path.n > 0 && url.path.p[url.path.n - 1] != '/') {
          --url.path.n;
        }
      }
      url.path.p = gc(xasprintf("%.*s%.*s", url.path.n, url.path.p,
                                FetchHeaderLength(kHttpLocation),
                                FetchHeaderData(kHttpLocation)));
      url.path.n = strlen(url.path.p);
      lua_pushstring(L, gc(EncodeUrl(&url, 0)));
    }
    lua_replace(L, -3);

    DestroyHttpMessage(&msg);
    free(inbuf.p);
    if (!keepalive || keepalive == kaCLOSE)
      close(sock);
    return LuaFetch(L);
  } else {
    lua_pushinteger(L, msg.status);
    LuaPushHeaders(L, &msg, inbuf.p);
    lua_pushlstring(L, inbuf.p + hdrsize, paylen);
    DestroyHttpMessage(&msg);
    free(inbuf.p);
    if (!keepalive || keepalive == kaCLOSE)
      close(sock);
    return 3;
  }
TransportError:
  DestroyHttpMessage(&msg);
  free(inbuf.p);
  close(sock);
  return LuaNilError(L, "transport error");
#ifndef UNSECURE
VerifyFailed:
  LockInc(&shared->c.sslverifyfailed);
  close(sock);
  return LuaNilTlsError(
      L, gc(DescribeSslVerifyFailure(sslcli.session_negotiate->verify_result)),
      ret);
#endif
#undef ssl
}