Add shared memory apis to redbean

You can now do things like implement mutexes using futexes in your
redbean lua code. This provides the fastest possible inter-process
communication for your production systems when SQLite alone as ipc
or things like pipes aren't sufficient.

test/libc/intrin/lockipc_test.c

@@ -0,0 +1,92 @@
+/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
+│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8                                :vi│
+╞══════════════════════════════════════════════════════════════════════════════╡
+│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
+│                                                                              │
+│ Permission to use, copy, modify, and/or distribute this software for         │
+│ any purpose with or without fee is hereby granted, provided that the         │
+│ above copyright notice and this permission notice appear in all copies.      │
+│                                                                              │
+│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
+│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
+│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
+│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
+│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
+│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
+│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
+│ PERFORMANCE OF THIS SOFTWARE.                                                │
+╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/calls/calls.h"
+#include "libc/errno.h"
+#include "libc/intrin/bits.h"
+#include "libc/intrin/intrin.h"
+#include "libc/intrin/kprintf.h"
+#include "libc/macros.internal.h"
+#include "libc/runtime/runtime.h"
+#include "libc/testlib/testlib.h"
+#include "libc/thread/thread.h"
+
+#define PROCESSES  8
+#define ITERATIONS 100000
+
+struct SharedMemory {
+  pthread_mutex_t mutex;
+  volatile long x;
+} * shm;
+
+void Worker(void) {
+  long t;
+  for (int i = 0; i < ITERATIONS; ++i) {
+    pthread_mutex_lock(&shm->mutex);
+    t = shm->x;
+    t += 1;
+    shm->x = t;
+    pthread_mutex_unlock(&shm->mutex);
+  }
+}
+
+TEST(lockipc, mutex) {
+  int e, rc, ws, pid;
+
+  // create shared memory
+  shm = _mapshared(FRAMESIZE);
+
+  // create shared mutex
+  pthread_mutexattr_t mattr;
+  pthread_mutexattr_init(&mattr);
+  pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_NORMAL);
+  pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED);
+  pthread_mutex_init(&shm->mutex, &mattr);
+  pthread_mutexattr_destroy(&mattr);
+
+  // create processes
+  for (int i = 0; i < PROCESSES; ++i) {
+    ASSERT_NE(-1, (rc = fork()));
+    if (!rc) {
+      Worker();
+      _Exit(0);
+    }
+  }
+
+  // wait for processes to finish
+  for (;;) {
+    e = errno;
+    if ((pid = waitpid(0, &ws, 0)) != -1) {
+      if (WIFSIGNALED(ws)) {
+        kprintf("process %d terminated with %G\n", pid, WTERMSIG(ws));
+        testlib_incrementfailed();
+      } else if (WEXITSTATUS(ws)) {
+        kprintf("process %d exited with %d\n", pid, WEXITSTATUS(ws));
+        testlib_incrementfailed();
+      }
+    } else {
+      ASSERT_EQ(ECHILD, errno);
+      errno = e;
+      break;
+    }
+  }
+
+  EXPECT_EQ(PROCESSES * ITERATIONS, shm->x);
+  ASSERT_EQ(0, pthread_mutex_destroy(&shm->mutex));
+  ASSERT_SYS(0, 0, munmap(shm, FRAMESIZE));
+}

test/tool/net/futex_test.lua

@@ -0,0 +1,113 @@
+-- Copyright 2022 Justine Alexandra Roberts Tunney
+--
+-- Permission to use, copy, modify, and/or distribute this software for
+-- any purpose with or without fee is hereby granted, provided that the
+-- above copyright notice and this permission notice appear in all copies.
+--
+-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
+-- WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+-- WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
+-- AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
+-- DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
+-- PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+-- TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+-- PERFORMANCE OF THIS SOFTWARE.
+
+assert(unix.pledge("stdio proc"))
+
+-- let's use futexes to implement a scalable mutex in lua
+--
+-- this example is designed to be the kind of thing that would be
+-- extremely expensive if we were using spin locks, because we'll
+-- have a lot of processes that wait a long time for something to
+-- happen. futexes solve that by asking the kernel to help us out
+-- and let the waiting processes sleep until the actual event.
+--
+-- here we have fifty processes, waiting one just one process, to
+-- to finish a long-running task. if we used spin locks, then the
+-- waiter procesess would eat up all the cpu. if you benchmark it
+-- then be sure to note that WALL TIME will be the same, it's the
+-- CPU USER TIME that gets pwnd.
+--
+-- uses    67 ms cpu time with futexes
+-- uses 5,000 ms cpu time with spinlocks
+millis = 300
+waiters = 100
+
+words = 2
+mem = unix.mapshared(words * 8)
+LOCK = 0     -- word index of our lock
+RESULT = 1   -- word index of our result
+
+-- From Futexes Are Tricky Version 1.1 § Mutex, Take 3;
+-- Ulrich Drepper, Red Hat Incorporated, June 27, 2004.
+function Lock()
+    local ok, old = mem:cmpxchg(LOCK, 0, 1)
+    if not ok then
+        if old == 1 then
+            old = mem:xchg(LOCK, 2)
+        end
+        while old > 0 do
+            mem:wait(LOCK, 2)
+            old = mem:xchg(LOCK, 2)
+        end
+    end
+end
+function Unlock()
+    local old = mem:add(LOCK, -1)
+    if old == 2 then
+        mem:store(LOCK, 0)
+        mem:wake(LOCK, 1)
+    end
+end
+
+-- -- try it out with spin locks instead
+-- function Lock()
+--     while mem:xchg(LOCK, 1) == 1 do
+--     end
+-- end
+-- function Unlock()
+--     mem:store(LOCK, 0)
+-- end
+
+function Worker()
+    assert(unix.nanosleep(math.floor(millis / 1000),
+                          millis % 1000 * 1000 * 1000))
+    mem:store(RESULT, 123)
+    Unlock()
+end
+
+function Waiter()
+    Lock()
+    assert(mem:load(RESULT) == 123)
+    Unlock()
+end
+
+Lock()
+
+for i = 1,waiters do
+    pid = assert(unix.fork())
+    if pid == 0 then
+        Waiter()
+        unix.exit(0)
+    end
+end
+
+Worker()
+
+while true do
+    rc, ws = unix.wait(0)
+    if not rc then
+        assert(ws:errno() == unix.ECHILD)
+        break
+    end
+    if unix.WIFEXITED(ws) then
+        if unix.WEXITSTATUS(ws) ~= 0 then
+            print('process %d exited with %s' % {rc, unix.WEXITSTATUS(ws)})
+            unix.exit(1)
+        end
+    else
+        print('process %d terminated with %s' % {rc, unix.WTERMSIG(ws)})
+        unix.exit(1)
+    end
+end

test/tool/net/mapshared_test.lua

@@ -0,0 +1,85 @@
+-- Copyright 2022 Justine Alexandra Roberts Tunney
+--
+-- Permission to use, copy, modify, and/or distribute this software for
+-- any purpose with or without fee is hereby granted, provided that the
+-- above copyright notice and this permission notice appear in all copies.
+--
+-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
+-- WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+-- WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
+-- AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
+-- DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
+-- PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+-- TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+-- PERFORMANCE OF THIS SOFTWARE.
+
+assert(unix.pledge("stdio proc"))
+
+words = 2
+processes = 8
+iterations = 10000
+
+mem = unix.mapshared(words * 8)
+
+--------------------------------------------------------------------------------
+-- test shared memory string reading and writing
+
+mem:write('hello')
+assert(mem:read() == 'hello')
+mem:write('hi')
+assert(mem:read() == 'hi')
+assert(mem:read(0, 5) == 'hi\0lo')
+mem:write('H', 0, 1)
+assert(mem:read(0, 5) == 'Hi\0lo')
+assert(mem:read(1, 1) == 'i')
+
+--------------------------------------------------------------------------------
+-- test shared memory locking primitives
+
+mem:store(0, 0)
+assert(mem:xchg(0, 1) == 0)
+assert(mem:xchg(0, 2) == 1)
+
+mem:store(0, 0)
+ok, old = mem:cmpxchg(0, 0, 1)
+assert(ok and old == 0)
+ok, old = mem:cmpxchg(0, 666, 777)
+assert(not ok and old == 1)
+assert(mem:add(0, 3) == 1)
+assert(mem:load(0) == 4)
+
+--------------------------------------------------------------------------------
+-- test atomic addition across concurrent processes
+
+function Worker()
+    for i = 1,iterations do
+        mem:add(0, 1)
+    end
+end
+
+mem:store(0, 0)
+for i = 1,processes do
+    pid = assert(unix.fork())
+    if pid == 0 then
+        Worker()
+        unix.exit(0)
+    end
+end
+while true do
+    rc, ws = unix.wait(0)
+    if not rc then
+        assert(ws:errno() == unix.ECHILD)
+        break
+    end
+    if unix.WIFEXITED(ws) then
+        if unix.WEXITSTATUS(ws) ~= 0 then
+            print('process %d exited with %s' % {rc, unix.WEXITSTATUS(ws)})
+            unix.exit(1)
+        end
+    else
+        print('process %d terminated with %s' % {rc, unix.WTERMSIG(ws)})
+        unix.exit(1)
+    end
+end
+
+assert(mem:load(0) == processes * iterations)

test/tool/net/spinlock_test.lua

@@ -0,0 +1,76 @@
+-- Copyright 2022 Justine Alexandra Roberts Tunney
+--
+-- Permission to use, copy, modify, and/or distribute this software for
+-- any purpose with or without fee is hereby granted, provided that the
+-- above copyright notice and this permission notice appear in all copies.
+--
+-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
+-- WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+-- WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
+-- AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
+-- DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
+-- PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+-- TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+-- PERFORMANCE OF THIS SOFTWARE.
+
+assert(unix.pledge("stdio proc"))
+
+words = 2
+processes = 8
+iterations = 10000
+
+mem = unix.mapshared(words * 8)
+
+--------------------------------------------------------------------------------
+-- let's use atomics to implement a spin lock in lua
+
+LOCK = 0     -- word index of our lock
+COUNTER = 1  -- word index of our number
+
+function Lock()
+    while mem:xchg(LOCK, 1) == 1 do
+    end
+end
+
+function Unlock()
+    mem:store(LOCK, 0)
+end
+
+function Worker()
+    local x
+    for i = 1,iterations do
+        Lock()
+        x = mem:load(COUNTER)
+        x = x + 1
+        mem:store(COUNTER, x)
+        Unlock()
+    end
+end
+
+mem:store(LOCK, 0)
+mem:store(COUNTER, 0)
+for i = 1,processes do
+    pid = assert(unix.fork())
+    if pid == 0 then
+        Worker()
+        unix.exit(0)
+    end
+end
+while true do
+    rc, ws = unix.wait(0)
+    if not rc then
+        assert(ws:errno() == unix.ECHILD)
+        break
+    end
+    if unix.WIFEXITED(ws) then
+        if unix.WEXITSTATUS(ws) ~= 0 then
+            print('process %d exited with %s' % {rc, unix.WEXITSTATUS(ws)})
+            unix.exit(1)
+        end
+    else
+        print('process %d terminated with %s' % {rc, unix.WTERMSIG(ws)})
+        unix.exit(1)
+    end
+end
+
+assert(mem:load(COUNTER) == processes * iterations)