From 6dcdf91458360638ad377603f1f580bd84c851cc Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Tue, 3 Jan 2023 18:28:35 -0800 Subject: [PATCH] Add a turfwar hilbert decoration --- libc/ar.h | 21 +++++ libc/isystem/ar.h | 4 + libc/isystem/string.h | 1 + libc/sysv/consts.sh | 2 +- net/turfwar/turfwar.c | 128 ++++++++++++++++++++++++++++-- net/turfwar/turfwar.mk | 1 + third_party/stb/stb_image_write.h | 2 + third_party/xed/avx.h | 43 ---------- third_party/xed/x86ild.greg.c | 65 ++++++++------- 9 files changed, 186 insertions(+), 81 deletions(-) create mode 100644 libc/ar.h create mode 100644 libc/isystem/ar.h delete mode 100644 third_party/xed/avx.h diff --git a/libc/ar.h b/libc/ar.h new file mode 100644 index 000000000..01b62e4c5 --- /dev/null +++ b/libc/ar.h @@ -0,0 +1,21 @@ +#ifndef COSMOPOLITAN_LIBC_AR_H_ +#define COSMOPOLITAN_LIBC_AR_H_ +#if !(__ASSEMBLER__ + __LINKER__ + 0) +COSMOPOLITAN_C_START_ + +#define ARMAG "!\n" +#define SARMAG 8 +#define ARFMAG "`\n" + +struct ar_hdr { + char ar_name[16]; + char ar_date[12]; + char ar_uid[6], ar_gid[6]; + char ar_mode[8]; + char ar_size[10]; + char ar_fmag[2]; +}; + +COSMOPOLITAN_C_END_ +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* COSMOPOLITAN_LIBC_AR_H_ */ diff --git a/libc/isystem/ar.h b/libc/isystem/ar.h new file mode 100644 index 000000000..e9ba7b694 --- /dev/null +++ b/libc/isystem/ar.h @@ -0,0 +1,4 @@ +#ifndef COSMOPOLITAN_LIBC_ISYSTEM_AR_H_ +#define COSMOPOLITAN_LIBC_ISYSTEM_AR_H_ +#include "libc/ar.h" +#endif /* COSMOPOLITAN_LIBC_ISYSTEM_AR_H_ */ diff --git a/libc/isystem/string.h b/libc/isystem/string.h index 39dc7c33a..e8a0fc362 100644 --- a/libc/isystem/string.h +++ b/libc/isystem/string.h @@ -1,5 +1,6 @@ #ifndef LIBC_ISYSTEM_STRING_H_ #define LIBC_ISYSTEM_STRING_H_ #include "libc/mem/alg.h" +#include "libc/mem/mem.h" #include "libc/str/str.h" #endif diff --git a/libc/sysv/consts.sh b/libc/sysv/consts.sh index 64b63be77..e69c43010 100755 --- a/libc/sysv/consts.sh +++ b/libc/sysv/consts.sh @@ -785,7 +785,7 @@ syscon tcp TCP_SAVED_SYN 28 0 0 0 0 0 # get recorded syn packets syscon tcp TCP_THIN_DUPACK 17 0 0 0 0 0 # what is it syscon tcp TCP_QUEUE_SEQ 21 0 0 0 0 0 # what is it syscon tcp TCP_WINDOW_CLAMP 10 0 0 0 0 0 # what is it -syscon tcp TCP_DEFER_ACCEPT 9 0 0 0 0 0 # what is it +syscon tcp TCP_DEFER_ACCEPT 9 0 0 0 0 0 # defer accept() until readable data has arrived syscon tcp TCP_REPAIR 19 0 0 0 0 0 # what is it syscon tcp TCP_REPAIR_OPTIONS 22 0 0 0 0 0 # what is it syscon tcp TCP_REPAIR_QUEUE 20 0 0 0 0 0 # what is it diff --git a/net/turfwar/turfwar.c b/net/turfwar/turfwar.c index db7467e80..fe3b5dea0 100644 --- a/net/turfwar/turfwar.c +++ b/net/turfwar/turfwar.c @@ -33,6 +33,8 @@ #include "libc/fmt/itoa.h" #include "libc/intrin/atomic.h" #include "libc/intrin/bits.h" +#include "libc/intrin/bsr.h" +#include "libc/intrin/hilbert.h" #include "libc/intrin/kprintf.h" #include "libc/intrin/strace.internal.h" #include "libc/log/check.h" @@ -51,6 +53,7 @@ #include "libc/sock/struct/pollfd.h" #include "libc/sock/struct/sockaddr.h" #include "libc/stdio/append.h" +#include "libc/stdio/rand.h" #include "libc/stdio/stdio.h" #include "libc/str/slice.h" #include "libc/str/str.h" @@ -83,6 +86,7 @@ #include "third_party/nsync/note.h" #include "third_party/nsync/time.h" #include "third_party/sqlite3/sqlite3.h" +#include "third_party/stb/stb_image_write.h" #include "third_party/zlib/zconf.h" #include "third_party/zlib/zlib.h" #include "tool/net/lfuncs.h" @@ -93,6 +97,8 @@ #define PORT 8080 // default server listening port #define CPUS 64 // number of cpus to actually use +#define XN 64 // plot width in pixels +#define YN 64 // plot height in pixels #define WORKERS 500 // size of http client thread pool #define SUPERVISE_MS 1000 // how often to stat() asset files #define KEEPALIVE_MS 60000 // max time to keep idle conn open @@ -101,7 +107,8 @@ #define SCORE_D_UPDATE_MS 30000 // how often to regenerate /score/day #define SCORE_W_UPDATE_MS 70000 // how often to regenerate /score/week #define SCORE_M_UPDATE_MS 100000 // how often to regenerate /score/month -#define SCORE_UPDATE_MS 200000 // how often to regenerate /score +#define SCORE_UPDATE_MS 210000 // how often to regenerate /score +#define PLOTS_UPDATE_MS 999000 // how often to regenerate /plot/xxx #define ACCEPT_DEADLINE_MS 100 // how long accept() can take to find worker #define CLAIM_DEADLINE_MS 100 // how long /claim may block if queue is full #define CONCERN_LOAD .75 // avoid keepalive, upon this connection load @@ -259,6 +266,7 @@ nsync_time g_started; nsync_counter g_ready; atomic_int g_connections; nsync_note g_shutdown[3]; +int g_hilbert[YN * XN][2]; // whitebox metrics atomic_long g_banned; @@ -330,6 +338,7 @@ struct Assets { struct Asset score_month; struct Asset recent; struct Asset favicon; + struct Asset plot[256]; } g_asset; // queues ListenWorker() to HttpWorker() @@ -894,6 +903,9 @@ void *HttpWorker(void *arg) { ksnprintf(ipbuf, sizeof(ipbuf), "%hhu.%hhu.%hhu.%hhu", ip >> 24, ip >> 16, ip >> 8, ip); + if (UrlStartsWith("/plot/") && (_rand64() % 256)) { + goto SkipSecurity; + } if (!ipv6 && !ContainsInt(&g_whitelisted, ip) && (tok = AcquireToken(g_tok.b, ip, TB_CIDR)) < 32) { if (tok > 4) { @@ -910,6 +922,7 @@ void *HttpWorker(void *arg) { ++g_ratelimits; break; } + SkipSecurity: // we don't support http/1.0 and http/0.9 right now if (msg->version != 11) { @@ -959,6 +972,14 @@ void *HttpWorker(void *arg) { a = &g_asset.score; } else if (UrlStartsWith("/recent")) { a = &g_asset.recent; + } else if (UrlStartsWith("/plot/")) { + int i, block = 0; + for (i = msg->uri.a + 6; i < msg->uri.b && isdigit(inbuf[i]); ++i) { + block *= 10; + block += inbuf[i] - '0'; + block &= 255; + } + a = g_asset.plot + block; } else { a = 0; } @@ -1483,6 +1504,68 @@ OnError: return false; } +// generator function for the big board +bool GeneratePlot(struct Asset *out, long block, long cash) { + _Static_assert(IS2POW(XN * YN), "area must be 2-power"); + _Static_assert(XN == YN, "hilbert algorithm needs square"); + int rc, out_len; + sqlite3 *db = 0; + struct Asset a = {0}; + unsigned char *rgba; + sqlite3_stmt *stmt = 0; + unsigned x, y, i, ip, area, mask, clump; + DEBUG("GeneratePlot %ld\n", block); + a.type = "image/png"; + a.cash = cash; + a.mtim = timespec_real(); + FormatUnixHttpDateTime(a.lastmodified, a.mtim.tv_sec); + CHECK_MEM((rgba = calloc(4, YN * XN))); + for (y = 0; y < YN; ++y) { + for (x = 0; x < XN; ++x) { + rgba[y * XN * 4 + x * 4 + 0] = 255; + rgba[y * XN * 4 + x * 4 + 1] = 255; + rgba[y * XN * 4 + x * 4 + 2] = 255; + } + } + CHECK_SQL(DbOpen("db.sqlite3", &db)); + CHECK_DB(DbPrepare(db, &stmt, + "SELECT ip\n" + " FROM land\n" + "WHERE ip >= ?1\n" + " AND ip <= ?2")); + CHECK_DB(sqlite3_bind_int64(stmt, 1, block << 24 | 0x000000)); + CHECK_DB(sqlite3_bind_int64(stmt, 2, block << 24 | 0xffffff)); + CHECK_SQL(sqlite3_exec(db, "BEGIN TRANSACTION", 0, 0, 0)); + area = XN * YN; + mask = area - 1; + clump = 32 - _bsr(area) - 8; + while ((rc = DbStep(stmt)) != SQLITE_DONE) { + if (rc != SQLITE_ROW) CHECK_DB(rc); + ip = sqlite3_column_int64(stmt, 0); + i = (ip >> clump) & mask; + y = g_hilbert[i][0]; + x = g_hilbert[i][1]; + if (rgba[y * XN * 4 + x * 4 + 3] < 255) { + ++rgba[y * XN * 4 + x * 4 + 3]; + } + } + CHECK_SQL(sqlite3_exec(db, "END TRANSACTION", 0, 0, 0)); + CHECK_DB(sqlite3_finalize(stmt)); + CHECK_SQL(sqlite3_close(db)); + a.data.p = (char *)stbi_write_png_to_mem(rgba, XN * 4, XN, YN, 4, &out_len); + a.data.n = out_len; + a.gzip = Gzip(a.data); + free(rgba); + *out = a; + return true; +OnError: + sqlite3_finalize(stmt); + sqlite3_close(db); + free(a.data.p); + free(rgba); + return false; +} + // single thread for regenerating the user scores json void *ScoreWorker(void *arg) { BlockSignals(); @@ -1562,6 +1645,26 @@ void *ScoreMonthWorker(void *arg) { return 0; } +// single thread for regenerating /8 cell background image charts +void *PlotWorker(void *arg) { + long i, wait; + BlockSignals(); + pthread_setname_np(pthread_self(), "Plotter"); + LOG("%P Plotter started\n"); + wait = PLOTS_UPDATE_MS; + for (i = 0; i < 256; ++i) { + Update(g_asset.plot + i, GeneratePlot, i, MS2CASH(wait)); + } + nsync_counter_add(g_ready, -1); // #6 + do { + for (i = 0; i < 256; ++i) { + Update(g_asset.plot + i, GeneratePlot, i, MS2CASH(wait)); + } + } while (!nsync_note_wait(g_shutdown[1], WaitFor(wait))); + LOG("Plotter exiting\n"); + return 0; +} + // thread for realtime json generation of recent successful claims void *RecentWorker(void *arg) { bool once; @@ -1629,7 +1732,7 @@ StartOver: free(f[1]); // handle startup condition if (!warmedup) { - nsync_counter_add(g_ready, -1); // #6 + nsync_counter_add(g_ready, -1); // #7 warmedup = true; } // wait for wakeup or cancel @@ -1676,7 +1779,7 @@ StartOver: " OR created IS NULL\n" " OR ?3 - created > 3600")); if (!warmedup) { - nsync_counter_add(g_ready, -1); // #7 + nsync_counter_add(g_ready, -1); // #8 warmedup = true; } while ((n = GetClaims(&g_claims, v, BATCH_MAX))) { @@ -1715,7 +1818,7 @@ void *NowWorker(void *arg) { pthread_setname_np(pthread_self(), "NowWorker"); LOG("%P NowWorker started\n"); UpdateNow(); - nsync_counter_add(g_ready, -1); // #8 + nsync_counter_add(g_ready, -1); // #9 for (struct timespec ts = {timespec_real().tv_sec};; ++ts.tv_sec) { if (!nsync_note_wait(g_shutdown[1], ts)) { UpdateNow(); @@ -1849,6 +1952,13 @@ int main(int argc, char *argv[]) { _npassert(2 == open("turfwar.log", O_CREAT | O_WRONLY | O_APPEND, 0644)); } + LOG("Generating Hilbert Curve...\n"); + for (int i = 0; i < YN * XN; ++i) { + axdx_t h = unhilbert(XN, i); + g_hilbert[i][0] = h.ax; + g_hilbert[i][1] = h.dx; + } + // library init __enable_threads(); sqlite3_initialize(); @@ -1890,9 +2000,9 @@ int main(int argc, char *argv[]) { sa.sa_handler = IgnoreSignal; sigaction(SIGUSR1, &sa, 0); - // make 8 helper threads - g_ready = nsync_counter_new(9); - pthread_t scorer, recenter, claimer, nower, replenisher; + // make 9 helper threads + g_ready = nsync_counter_new(10); + pthread_t scorer, recenter, claimer, nower, replenisher, plotter; pthread_t scorer_hour, scorer_day, scorer_week, scorer_month; CHECK_EQ(0, pthread_create(&scorer, 0, ScoreWorker, 0)); CHECK_EQ(0, pthread_create(&scorer_hour, 0, ScoreHourWorker, 0)); @@ -1902,10 +2012,11 @@ int main(int argc, char *argv[]) { CHECK_EQ(0, pthread_create(&replenisher, 0, ReplenishWorker, 0)); CHECK_EQ(0, pthread_create(&recenter, 0, RecentWorker, 0)); CHECK_EQ(0, pthread_create(&claimer, 0, ClaimWorker, 0)); + CHECK_EQ(0, pthread_create(&plotter, 0, PlotWorker, 0)); CHECK_EQ(0, pthread_create(&nower, 0, NowWorker, 0)); // wait for helper threads to warm up creating assets - if (nsync_counter_add(g_ready, -1)) { // #9 + if (nsync_counter_add(g_ready, -1)) { // #10 nsync_counter_wait(g_ready, nsync_time_no_deadline); } @@ -1942,6 +2053,7 @@ int main(int argc, char *argv[]) { LOG("Waiting for helpers to finish...\n"); CHECK_EQ(0, pthread_join(nower, 0)); CHECK_EQ(0, pthread_join(scorer, 0)); + CHECK_EQ(0, pthread_join(plotter, 0)); CHECK_EQ(0, pthread_join(recenter, 0)); CHECK_EQ(0, pthread_join(scorer_day, 0)); CHECK_EQ(0, pthread_join(scorer_hour, 0)); diff --git a/net/turfwar/turfwar.mk b/net/turfwar/turfwar.mk index b6e6a6e71..c200b74a2 100644 --- a/net/turfwar/turfwar.mk +++ b/net/turfwar/turfwar.mk @@ -38,6 +38,7 @@ NET_TURFWAR_DIRECTDEPS = \ THIRD_PARTY_NSYNC \ THIRD_PARTY_NSYNC_MEM \ THIRD_PARTY_SQLITE3 \ + THIRD_PARTY_STB \ THIRD_PARTY_ZLIB NET_TURFWAR_DEPS := \ diff --git a/third_party/stb/stb_image_write.h b/third_party/stb/stb_image_write.h index d72c8b965..69ab97401 100644 --- a/third_party/stb/stb_image_write.h +++ b/third_party/stb/stb_image_write.h @@ -26,6 +26,8 @@ int stbi_write_hdr_to_func(stbi_write_func *, void *, int, int, int, const float *); int stbi_write_jpg_to_func(stbi_write_func *, void *, int, int, int, const void *, int); +unsigned char *stbi_write_png_to_mem(const unsigned char *, int, int, int, int, + int *); void stbi_flip_vertically_on_write(int); diff --git a/third_party/xed/avx.h b/third_party/xed/avx.h deleted file mode 100644 index f23e1fb31..000000000 --- a/third_party/xed/avx.h +++ /dev/null @@ -1,43 +0,0 @@ -#ifndef COSMOPOLITAN_THIRD_PARTY_XED_AVX_H_ -#define COSMOPOLITAN_THIRD_PARTY_XED_AVX_H_ -#if !(__ASSEMBLER__ + __LINKER__ + 0) -COSMOPOLITAN_C_START_ - -union XedAvxC4Payload1 { - struct { - unsigned map : 5; - unsigned b_inv : 1; - unsigned x_inv : 1; - unsigned r_inv : 1; - unsigned pad : 24; - } s; - unsigned u32; -}; - -union XedAvxC4Payload2 { - struct { - unsigned pp : 2; - unsigned l : 1; - unsigned vvv210 : 3; - unsigned v3 : 1; - unsigned w : 1; - unsigned pad : 24; - } s; - unsigned u32; -}; - -union XedAvxC5Payload { - struct { - unsigned pp : 2; - unsigned l : 1; - unsigned vvv210 : 3; - unsigned v3 : 1; - unsigned r_inv : 1; - unsigned pad : 24; - } s; - unsigned u32; -}; - -COSMOPOLITAN_C_END_ -#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ -#endif /* COSMOPOLITAN_THIRD_PARTY_XED_AVX_H_ */ diff --git a/third_party/xed/x86ild.greg.c b/third_party/xed/x86ild.greg.c index 684807cbd..b2eef63e0 100644 --- a/third_party/xed/x86ild.greg.c +++ b/third_party/xed/x86ild.greg.c @@ -23,7 +23,6 @@ #include "libc/macros.internal.h" #include "libc/runtime/runtime.h" #include "libc/str/str.h" -#include "third_party/xed/avx.h" #include "third_party/xed/avx512.h" #include "third_party/xed/private.h" #include "third_party/xed/x86.h" @@ -846,27 +845,32 @@ privileged static void xed_evex_imm_scanner(struct XedDecodedInst *d) { } privileged static void xed_vex_c4_scanner(struct XedDecodedInst *d) { - uint8_t n; - xed_bits_t length, max_bytes; - union XedAvxC4Payload1 c4byte1; - union XedAvxC4Payload2 c4byte2; + unsigned length, b1, b2; if (xed_is_bound_instruction(d)) return; length = d->length; - max_bytes = d->op.max_bytes; length++; - if (length + 2 < max_bytes) { - c4byte1.u32 = d->bytes[length]; - c4byte2.u32 = d->bytes[length + 1]; - d->op.rexr = ~c4byte1.s.r_inv & 1; - d->op.rexx = ~c4byte1.s.x_inv & 1; - d->op.rexb = (xed3_mode_64b(d) & ~c4byte1.s.b_inv) & 1; - d->op.rexw = c4byte2.s.w & 1; - d->op.vexdest3 = c4byte2.s.v3; - d->op.vexdest210 = c4byte2.s.vvv210; - d->op.vl = c4byte2.s.l; - d->op.vex_prefix = kXed.vex_prefix_recoding[c4byte2.s.pp]; - d->op.map = c4byte1.s.map; - if ((c4byte1.s.map & 0x3) == XED_ILD_MAP3) { + if (length + 2 < d->op.max_bytes) { + // map: 5-bit + // rex.b: 1-bit + // rex.x: 1-bit + // rex.r: 1-bit + b1 = d->bytes[length]; + d->op.rexr = !(b1 & 128); + d->op.rexx = !(b1 & 64); + d->op.rexb = xed3_mode_64b(d) & !(b1 & 32); + // prefix: 2-bit → {none, osz, rep3, rep2} + // vector_length: 1-bit → {xmm, ymm} + // vexdest210: 3-bit + // vexdest3: 1-bit + // rex.w: 1-bit + b2 = d->bytes[length + 1]; + d->op.rexw = !!(b2 & 128); + d->op.vexdest3 = !!(b2 & 64); + d->op.vexdest210 = (b2 >> 3) & 7; + d->op.vl = !!(b2 & 4); + d->op.vex_prefix = kXed.vex_prefix_recoding[b2 & 3]; + d->op.map = b1 & 31; + if ((b1 & 3) == XED_ILD_MAP3) { d->op.imm_width = xed_bytes2bits(1); } d->op.vexvalid = 1; @@ -880,19 +884,22 @@ privileged static void xed_vex_c4_scanner(struct XedDecodedInst *d) { } privileged static void xed_vex_c5_scanner(struct XedDecodedInst *d) { - xed_bits_t max_bytes, length; - union XedAvxC5Payload c5byte1; + unsigned length, b; length = d->length; - max_bytes = d->op.max_bytes; if (xed_is_bound_instruction(d)) return; length++; - if (length + 1 < max_bytes) { - c5byte1.u32 = d->bytes[length]; - d->op.rexr = ~c5byte1.s.r_inv & 1; - d->op.vexdest3 = c5byte1.s.v3; - d->op.vexdest210 = c5byte1.s.vvv210; - d->op.vl = c5byte1.s.l; - d->op.vex_prefix = kXed.vex_prefix_recoding[c5byte1.s.pp]; + if (length + 1 < d->op.max_bytes) { + // prefix: 2-bit → {none, osz, rep3, rep2} + // vector_length: 1-bit → {xmm, ymm} + // vexdest210: 3-bit + // vexdest3: 1-bit + // rex.r: 1-bit + b = d->bytes[length]; + d->op.rexr = !(b & 128); + d->op.vexdest3 = !!(b & 64); + d->op.vexdest210 = (b >> 3) & 7; + d->op.vl = (b >> 2) & 1; + d->op.vex_prefix = kXed.vex_prefix_recoding[b & 3]; d->op.map = XED_ILD_MAP1; d->op.vexvalid = 1; length++;