diff --git a/libc/str/isutf8.c b/libc/str/isutf8.c index 30f9600cd..8c385a55e 100644 --- a/libc/str/isutf8.c +++ b/libc/str/isutf8.c @@ -26,8 +26,8 @@ static const char kUtf8Dispatch[] = { 1, 1, 1, 1, 1, 1, 1, 1, // 0320 1, 1, 1, 1, 1, 1, 1, 1, // 0330 2, 3, 3, 3, 3, 3, 3, 3, // 0340 utf8-3 - 3, 3, 3, 3, 3, 3, 3, 3, // 0350 - 4, 5, 5, 5, 5, 0, 0, 0, // 0360 utf8-4 + 3, 3, 3, 3, 3, 4, 3, 3, // 0350 + 5, 6, 6, 6, 7, 0, 0, 0, // 0360 utf8-4 0, 0, 0, 0, 0, 0, 0, 0, // 0370 }; @@ -45,6 +45,8 @@ static const char kUtf8Dispatch[] = { * - Incorrect sequencing of 0300 (FIRST) and 0200 (CONT) chars * - Thompson-Pike varint sequence not encodable as UTF-16 * - Overlong UTF-8 encoding + * - any UTF-16 surrogate code points + * - last character in a multi-byte UTF-8 sequence exceeds the valid limit * * @param size if -1 implies strlen */ @@ -95,6 +97,7 @@ bool32 isutf8(const void *data, size_t size) { } // fallthrough case 3: + case3: if (p + 2 <= e && // (p[0] & 0300) == 0200 && // (p[1] & 0300) == 0200) { // @@ -104,11 +107,17 @@ bool32 isutf8(const void *data, size_t size) { return false; // missing cont } case 4: + if (p < e && (*p & 040)) { + return false; // utf-16 surrogate + } + goto case3; + case 5: if (p < e && (*p & 0377) < 0220) { return false; // overlong } // fallthrough - case 5: + case 6: + case6: if (p + 3 <= e && // (((uint32_t)(p[+2] & 0377) << 030 | // (uint32_t)(p[+1] & 0377) << 020 | // @@ -120,6 +129,11 @@ bool32 isutf8(const void *data, size_t size) { } else { return false; // missing cont } + case 7: + if (p < e && (*p & 0x3F) > 0xF) { + return false; // over limit + } + goto case6; default: __builtin_unreachable(); } diff --git a/net/http/gethttpheader.gperf b/net/http/gethttpheader.gperf index 26c3c2dd0..850a3dab6 100644 --- a/net/http/gethttpheader.gperf +++ b/net/http/gethttpheader.gperf @@ -104,3 +104,4 @@ CF-Visitor, kHttpCfVisitor CF-Connecting-IP, kHttpCfConnectingIp CF-IPCountry, kHttpCfIpcountry CDN-Loop, kHttpCdnLoop +Sec-WebSocket-Key, kHttpWebSocketKey diff --git a/net/http/gethttpheader.inc b/net/http/gethttpheader.inc index 8e58c9279..b90d279cc 100644 --- a/net/http/gethttpheader.inc +++ b/net/http/gethttpheader.inc @@ -39,7 +39,7 @@ #line 12 "gethttpheader.gperf" struct thatispacked HttpHeaderSlot { char *name; char code; }; -#define TOTAL_KEYWORDS 93 +#define TOTAL_KEYWORDS 94 #define MIN_WORD_LENGTH 2 #define MAX_WORD_LENGTH 32 #define MIN_HASH_VALUE 3 @@ -387,7 +387,10 @@ LookupHttpHeader (register const char *str, register size_t len) #line 87 "gethttpheader.gperf" {"Strict-Transport-Security", kHttpStrictTransportSecurity}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, {""}, - {""}, {""}, {""}, {""}, {""}, + {""}, {""}, +#line 107 "gethttpheader.gperf" + {"Sec-WebSocket-Key", kHttpWebSocketKey}, + {""}, {""}, #line 22 "gethttpheader.gperf" {"X-Forwarded-For", kHttpXForwardedFor}, {""}, diff --git a/net/http/gethttpheadername.c b/net/http/gethttpheadername.c index ced63607c..ee661b9fb 100644 --- a/net/http/gethttpheadername.c +++ b/net/http/gethttpheadername.c @@ -206,6 +206,8 @@ const char *GetHttpHeaderName(int h) { return "CDN-Loop"; case kHttpSecChUaPlatform: return "Sec-CH-UA-Platform"; + case kHttpWebSocketKey: + return "Sec-WebSocket-Key"; default: return NULL; } diff --git a/net/http/http.h b/net/http/http.h index c1dbbb13b..77c0c8bf0 100644 --- a/net/http/http.h +++ b/net/http/http.h @@ -138,7 +138,8 @@ #define kHttpCfIpcountry 90 #define kHttpSecChUaPlatform 91 #define kHttpCdnLoop 92 -#define kHttpHeadersMax 93 +#define kHttpWebSocketKey 93 +#define kHttpHeadersMax 94 COSMOPOLITAN_C_START_ diff --git a/test/libc/str/isutf8_test.c b/test/libc/str/isutf8_test.c index e9b0690c1..e24fd5873 100644 --- a/test/libc/str/isutf8_test.c +++ b/test/libc/str/isutf8_test.c @@ -42,10 +42,16 @@ TEST(isutf8, good) { } TEST(isutf8, bad) { - ASSERT_FALSE(isutf8("\300\200", -1)); // overlong nul - ASSERT_FALSE(isutf8("\200\300", -1)); // latin1 c1 control code - ASSERT_FALSE(isutf8("\300\300", -1)); // missing continuation - ASSERT_FALSE(isutf8("\377\200\200\200\200", -1)); // thompson-pike varint + ASSERT_FALSE(isutf8("\300\200", -1)); // overlong nul + ASSERT_FALSE(isutf8("\200\300", -1)); // latin1 c1 control code + ASSERT_FALSE(isutf8("\300\300", -1)); // missing continuation + ASSERT_FALSE(isutf8("\377\200\200\200\200", -1)); // thompson-pike varint + ASSERT_FALSE(isutf8("\355\240\200", -1)); // single utf-16 surrogate (high) + ASSERT_FALSE(isutf8("\355\277\277", -1)); // single utf-16 surrogate (low) + ASSERT_FALSE(isutf8("\355\240\200\355\260\200", -1)); // paired utf-16 surrogates (range begin) + ASSERT_FALSE(isutf8("\355\257\277\355\277\277", -1)); // paired utf-16 surrogates (range end) + ASSERT_FALSE(isutf8("\364\220\200\200", -1)); // boundary condition + ASSERT_FALSE(isutf8("\220", -1)); // boundary condition } TEST(isutf8, oob) { diff --git a/test/tool/net/BUILD.mk b/test/tool/net/BUILD.mk index c1a7ebd4f..a1ac357b1 100644 --- a/test/tool/net/BUILD.mk +++ b/test/tool/net/BUILD.mk @@ -71,11 +71,17 @@ o/$(MODE)/test/tool/net/%.dbg: \ $(APE_NO_MODIFY_SELF) @$(APELINK) + +o/$(MODE)/tool/net/tester/.init.lua.zip.o: private \ + ZIPOBJ_FLAGS += \ + -B + .PRECIOUS: o/$(MODE)/test/tool/net/redbean-tester o/$(MODE)/test/tool/net/redbean-tester.dbg: \ $(TOOL_NET_DEPS) \ o/$(MODE)/tool/net/redbean.o \ $(TOOL_NET_REDBEAN_LUA_MODULES) \ + o/$(MODE)/tool/net/tester/.init.lua.zip.o \ o/$(MODE)/tool/net/demo/seekable.txt.zip.o \ o/$(MODE)/tool/net/net.pkg \ $(CRT) \ diff --git a/test/tool/net/redbean_test.c b/test/tool/net/redbean_test.c index e13c3cfdd..06996f0d0 100644 --- a/test/tool/net/redbean_test.c +++ b/test/tool/net/redbean_test.c @@ -291,3 +291,45 @@ Z\n", EXPECT_NE(-1, wait(0)); EXPECT_NE(-1, sigprocmask(SIG_SETMASK, &savemask, 0)); } + +TEST(redbean, testWebSockets) { + if (IsWindows()) + return; + char portbuf[16]; + int pid, pipefds[2]; + sigset_t chldmask, savemask; + sigaddset(&chldmask, SIGCHLD); + EXPECT_NE(-1, sigprocmask(SIG_BLOCK, &chldmask, &savemask)); + ASSERT_NE(-1, pipe(pipefds)); + ASSERT_NE(-1, (pid = fork())); + if (!pid) { + setpgrp(); + close(0); + open("/dev/null", O_RDWR); + close(pipefds[0]); + dup2(pipefds[1], 1); + sigprocmask(SIG_SETMASK, &savemask, NULL); + execv("bin/redbean-tester", + (char *const[]){"bin/redbean-tester", "-vvszXp0", "-l127.0.0.1", // "-L/tmp/redbean-tester.log", + __strace > 0 ? "--strace" : 0, 0}); + _exit(127); + } + EXPECT_NE(-1, close(pipefds[1])); + EXPECT_NE(-1, read(pipefds[0], portbuf, sizeof(portbuf))); + port = atoi(portbuf); + EXPECT_TRUE(Matches("HTTP/1.1 101 Switching Protocols\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + "Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=\r\n" + "Date: .*\r\n" + "Server: redbean/.*\r\n" + "\r\n", + gc(SendHttpRequest("GET /ws HTTP/1.1\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n")))); + EXPECT_EQ(0, close(pipefds[0])); + EXPECT_NE(-1, kill(pid, SIGTERM)); + EXPECT_NE(-1, wait(0)); + EXPECT_NE(-1, sigprocmask(SIG_SETMASK, &savemask, 0)); +} diff --git a/tool/net/redbean.c b/tool/net/redbean.c index 93816d1aa..a755fec96 100644 --- a/tool/net/redbean.c +++ b/tool/net/redbean.c @@ -42,6 +42,7 @@ #include "libc/intrin/atomic.h" #include "libc/intrin/bsr.h" #include "libc/intrin/likely.h" +#include "libc/intrin/newbie.h" #include "libc/intrin/nomultics.h" #include "libc/intrin/safemacros.h" #include "libc/log/appendresourcereport.internal.h" @@ -125,6 +126,7 @@ #include "third_party/mbedtls/net_sockets.h" #include "third_party/mbedtls/oid.h" #include "third_party/mbedtls/san.h" +#include "third_party/mbedtls/sha1.h" #include "third_party/mbedtls/ssl.h" #include "third_party/mbedtls/ssl_ticket.h" #include "third_party/mbedtls/x509.h" @@ -405,6 +407,7 @@ struct ClearedPerMessage { bool hascontenttype; bool gotcachecontrol; bool gotxcontenttypeoptions; + char wstype; int frags; int statuscode; int isyielding; @@ -485,6 +488,8 @@ static uint8_t *zmap; static uint8_t *zcdir; static size_t hdrsize; static size_t amtread; +static size_t wsfragread; +static char wsfragtype; static reader_f reader; static writer_f writer; static char *extrahdrs; @@ -5159,6 +5164,228 @@ static bool LuaRunAsset(const char *path, bool mandatory) { return !!a; } +static int LuaWSUpgrade(lua_State *L) { + mbedtls_sha1_context ctx; + unsigned char hash[20]; + char *accept, *p, *q; + if (cpm.generator) { + return luaL_error(L, "Cannot upgrade to websocket after yielding normally"); + } + + if (!HasHeader(kHttpWebSocketKey)) { + return luaL_error(L, "No Sec-WebSocket-Key header"); + } + // Prepare Sec-WebSocket-Accept response header (See RFC6455 1.3) + mbedtls_sha1_init(&ctx); + mbedtls_sha1_starts_ret(&ctx); + mbedtls_sha1_update_ret(&ctx, + (unsigned char*)HeaderData(kHttpWebSocketKey), + HeaderLength(kHttpWebSocketKey)); + mbedtls_sha1_update_ret(&ctx, + (unsigned char*)"258EAFA5-E914-47DA-95CA-C5AB0DC85B11", + 36); + mbedtls_sha1_finish_ret(&ctx, hash); + accept = EncodeBase64((char *)hash, 20, NULL); + + // prepare response + p = SetStatus(101, "Switching Protocols"); + // make enough space for the handshake message: + // "Upgrade: websocket\r\n" (20 bytes) + // "Connection: Upgrade\r\n" (21 bytes) + // "Sec-WebSocket-Accept: \r\n" (54 bytes) + // will always be 28 bytes, as len(b64(hash)) = 4*ceil(20/3) = 28 + while (p - hdrbuf.p + 95 + 512 > hdrbuf.n) { + hdrbuf.n += hdrbuf.n >> 1; + q = xrealloc(hdrbuf.p, hdrbuf.n); + cpm.luaheaderp = p = q + (p - hdrbuf.p); + hdrbuf.p = q; + } + p = stpcpy(p, "Upgrade: websocket\r\n"); + p = stpcpy(p, "Connection: Upgrade\r\n"); + p = AppendHeader(p, "Sec-WebSocket-Accept", accept); + cpm.luaheaderp = p; + cpm.wstype = 1; + + return 0; +} + +// see RFC6455 5.2 for details on the websocket data frame structure +static int LuaWSRead(lua_State *L) { + ssize_t rc; + size_t i, got, amt, bufsize; + unsigned char header[10], headerlen, opcode, *extlen, *mask; + char *bufstart; + uint64_t len; + struct iovec iov[2]; + OnlyCallDuringRequest(L, "ws.Read"); + + got = 0; + // read 2 bytes of the frame header + do { + if ((rc = reader(client, header + got, 2 - got)) == -1) { + return luaL_error(L, "Could not read WS header"); + } + } while ((got += rc) < 2); + + // reserved bit set + if (header[0] & 0x70) goto close; + + opcode = header[0] & 0xF; + // reserved opcode + if ((opcode & 0x7) >= 0x3 || opcode > 0xA) goto close; + // payload data is unmasked + if (!(header[1] | (1 << 7))) goto close; + // not in continuation + if (!wsfragtype && !opcode) goto close; + + len = header[1] & ~(1 << 7); + if (header[0] & 0x8) { + // control frame is fragmented or too long + if (!(header[0] & 0x80) || len >= 126) goto close; + } else { + // control frame during fragmented sequence + if (opcode && wsfragtype) goto close; + } + + headerlen = 6; + if (len == 126) { + headerlen = 8; + } else if (len == 127) { + headerlen = 14; + } + + // read rest of header, if necessary + while (got < headerlen) { + if ((rc = reader(client, header + got, headerlen - got)) == -1) { + return luaL_error(L, "Could not read WS extended length"); + } + got += rc; + } + + extlen = &header[2]; + mask = &header[headerlen - 4]; + // multibyte length quantities are expressed in network byte order + if (len == 126) { + len = be16toh(*(uint16_t *)extlen); + } else if (len == 127) { + len = be64toh(*(uint64_t *)extlen); + } + + if (len >= inbuf.n - wsfragread) { + return luaL_error(L, + "Required %d bytes to read WS frame, %d bytes available", + len, + inbuf.n - wsfragread); + } + + // read in frame data + for (got = 0, amt = wsfragread; got < len; got += rc, amt += rc) { + if ((rc = reader(client, inbuf.p + amt, len - got)) == -1) { + return luaL_error(L, "Could not read WS data"); + } + } + + // unmask data + for (i = 0, amt = wsfragread; i < got; ++i, ++amt) { + inbuf.p[amt] ^= mask[i & 0x3]; + } + + // ping received, respond with pong + if (opcode == 0x9) { + header[0] = (header[0] & ~0xF) | 0xA; + header[1] = header[1] & ~0x80; + // pong data must be identical to ping + iov[0].iov_base = header; + iov[0].iov_len = headerlen - 4; + iov[1].iov_base = inbuf.p + wsfragread; + iov[1].iov_len = got; + Send(iov, 2); + } + + // final fragment + if (header[0] & 0x80) { + // non-continuation frame + if (opcode) { + bufstart = inbuf.p + wsfragread; + bufsize = got; + + // text frame with invalid text + if (opcode == 0x1 && !isutf8(bufstart, bufsize)) goto close; + lua_pushlstring(L, bufstart, bufsize); + lua_pushinteger(L, opcode); + } else { + bufstart = inbuf.p + amtread; + bufsize = (wsfragread - amtread) + got; + + // text frame with invalid text + if (wsfragtype == 0x1 && !isutf8(bufstart, bufsize)) goto close; + lua_pushlstring(L, bufstart, bufsize); + lua_pushinteger(L, wsfragtype); + + wsfragread = amtread; + wsfragtype = 0; + } + } else { + lua_pushnil(L); + lua_pushinteger(L, 0); + + if (!wsfragtype) wsfragtype = opcode; + wsfragread += got; + } + + return 2; + +close: + lua_pushnil(L); + lua_pushinteger(L, 0x08); + return 2; +} + +static int LuaWSWrite(lua_State *L) { + int type, retval; + size_t size; + const char *data; + + OnlyCallDuringRequest(L, "ws.Write"); + if (!cpm.wstype) { + retval = LuaWSUpgrade(L); + if (retval != 0) { + return retval; + } + } + + type = luaL_optinteger(L, 2, -1); + if (type == 1 || type == 2) { + cpm.wstype = type; + } else if (type != -1) { + return luaL_error(L, "Invalid WS type"); + } + + if (!lua_isnil(L, 1)) { + data = luaL_checklstring(L, 1, &size); + appendd(&cpm.outbuf, data, size); + } + + return 0; +} + +static const luaL_Reg kLuaWS[] = { + {"Read", LuaWSRead}, // + {"Write", LuaWSWrite}, // + {0} // +}; + +int LuaWS(lua_State *L) { + luaL_newlib(L, kLuaWS); + lua_pushinteger(L, 0); lua_setfield(L, -2, "CONT"); + lua_pushinteger(L, 1); lua_setfield(L, -2, "TEXT"); + lua_pushinteger(L, 2); lua_setfield(L, -2, "BIN"); + lua_pushinteger(L, 8); lua_setfield(L, -2, "CLOSE"); + lua_pushinteger(L, 9); lua_setfield(L, -2, "PING"); + lua_pushinteger(L, 10); lua_setfield(L, -2, "PONG"); + return 1; +} + // // list of functions that can't be run from the repl static const char *const kDontAutoComplete[] = { @@ -5426,6 +5653,7 @@ static const luaL_Reg kLuaLibs[] = { {"path", LuaPath}, // {"re", LuaRe}, // {"unix", LuaUnix}, // + {"ws", LuaWS}, // }; static void LuaSetArgv(lua_State *L) { @@ -6479,6 +6707,80 @@ static bool StreamResponse(char *p) { return true; } +static bool StreamWS(char *p) { + ssize_t rc; + struct iovec iov[2]; + char header[10], *s, *extlen; + int nresults, status; + + p = AppendCrlf(p); + CHECK_LE(p - hdrbuf.p, hdrbuf.n); + if (logmessages) { + LogMessage("sending", hdrbuf.p, p - hdrbuf.p); + } + iov[0].iov_base = hdrbuf.p; + iov[0].iov_len = p - hdrbuf.p; + Send(iov, 1); + + bzero(iov, sizeof(iov)); + iov[0].iov_base = header; + + extlen = &header[2]; + wsfragread = amtread; + wsfragtype = 0; + + for (;;) { + // done yielding + if (!YL || lua_status(YL) != LUA_YIELD) { + break; + } + cpm.contentlength = 0; + status = lua_resume(YL, NULL, 0, &nresults); + if (status == LUA_OK) { + lua_pop(YL, nresults); + break; + } else if (status != LUA_YIELD) { + LogLuaError("resume", lua_tostring(YL, -1)); + lua_pop(YL, 1); + break; + } + lua_pop(YL, nresults); + if (!cpm.contentlength) { + UseOutput(); + } + + DEBUGF("(lua) ws yielded with %ld bytes generated", cpm.contentlength); + + iov[1].iov_base = cpm.content; + iov[1].iov_len = rc = cpm.contentlength; + + if (rc < 126) { + header[1] = rc; + iov[0].iov_len = 2; + } else if (rc <= 0xFFFF) { + header[1] = 126; + *(uint16_t *)extlen = htobe16(rc); + iov[0].iov_len = 4; + } else { + header[1] = 127; + *(uint64_t *)extlen = htobe64(rc); + iov[0].iov_len = 10; + } + header[0] = cpm.wstype | (1 << 7); + if (Send(iov, 2) == -1) { + break; + } + } + + header[0] = 0x8 | (1 << 7); + header[1] = 0; + iov[0].iov_len = 2; + Send(iov, 1); + connectionclose = true; + + return true; +} + static bool HandleMessageActual(void) { int rc; long reqtime, contime; @@ -6543,6 +6845,8 @@ static bool HandleMessageActual(void) { } if (!cpm.generator) { return TransmitResponse(p); + } else if (cpm.wstype) { + return StreamWS(p); } else { return StreamResponse(p); } diff --git a/tool/net/tester/.init.lua b/tool/net/tester/.init.lua new file mode 100644 index 000000000..077843882 --- /dev/null +++ b/tool/net/tester/.init.lua @@ -0,0 +1,38 @@ +-- special script called by main redbean process at startup +HidePath('/usr/share/zoneinfo/') +HidePath('/usr/share/ssl/') + +-- 20Ki, for certain autobahn tests +ProgramMaxPayloadSize(20 * 1024 * 1024) + +function OnHttpRequest() + if GetPath() == "/ws" then + ws.Write(nil) -- upgrade without sending a response + coroutine.yield() + + local fd = GetClientFd() + local client_exit = unix.POLLHUP | unix.POLLRDHUP | unix.POLLERR + local fds = {[fd] = unix.POLLIN | client_exit} + -- simple echo server + while true do + res = unix.poll(fds) + if res[fd] & client_exit > 0 then + return + end + local s, t = ws.Read() + if t == ws.CLOSE then + return + elseif t == ws.TEXT then + ws.Write(s, ws.TEXT) + coroutine.yield() + elseif t == ws.BIN then + ws.Write(s, ws.BIN) + coroutine.yield() + end + end + else + Route() + end +end + +