cosmopolitan/third_party/nsync/common.internal.h

#ifndef NSYNC_COMMON_H_
#define NSYNC_COMMON_H_
#include "libc/assert.h"
#include "libc/intrin/dll.h"
#include "third_party/nsync/atomic.h"
#include "third_party/nsync/atomic.internal.h"
#include "third_party/nsync/cv.h"
#include "third_party/nsync/mu.h"
#include "third_party/nsync/mu_semaphore.h"
#include "third_party/nsync/note.h"
#include "third_party/nsync/time.h"
#include "third_party/nsync/wait_s.internal.h"
COSMOPOLITAN_C_START_

#ifdef MODE_DBG
#define NSYNC_DEBUG 1
#else
#define NSYNC_DEBUG 0
#endif

/* Yield the CPU. Platform specific. */
void nsync_yield_(void);

/* Retrieve the per-thread cache of the waiter object.  Platform specific. */
void *nsync_per_thread_waiter_(void (*dest)(void *));

/* Spin until (*w & test) == 0, then atomically perform *w = ((*w | set) &
   ~clear), perform an acquire barrier, and return the previous value of *w.
   */
uint32_t nsync_spin_test_and_set_(nsync_atomic_uint32_ *w, uint32_t test,
                                  uint32_t set, uint32_t clear, void *symbol);

/* Abort after printing the nul-temrinated string s[]. */
void nsync_panic_(const char *s) wontreturn;

/* ---------- */

#define MIN_(a_, b_) ((a_) < (b_) ? (a_) : (b_))
#define MAX_(a_, b_) ((a_) > (b_) ? (a_) : (b_))

/* ---------- */

/* Fields in nsync_mu.word.

   - At least one of the MU_WLOCK or MU_RLOCK_FIELD fields must be zero.
   - MU_WLOCK indicates that a write lock is held.
   - MU_RLOCK_FIELD is a count of readers with read locks.

   - MU_SPINLOCK represents a spinlock that must be held when manipulating the
     waiter queue.

   - MU_DESIG_WAKER indicates that a former waiter has been woken, but has
     neither acquired the lock nor gone back to sleep.  Legal to fail to set it;
     illegal to set it when no such waiter exists.

   - MU_WAITING indicates whether the waiter queue is non-empty.
     The following bits should be zero if MU_WAITING is zero.
   - MU_CONDITION indicates that some waiter may have an associated condition
     (from nsync_mu_wait, etc.).  Legal to set it with no such waiter exists,
     but illegal to fail to set it with such a waiter.
   - MU_WRITER_WAITING indicates that a reader that has not yet blocked
     at least once should not acquire in order not to starve waiting writers.
     It set when a writer blocks or a reader is woken with a writer waiting.
     It is reset when a writer acquires, but set again when that writer
     releases if it wakes readers and there is a waiting writer.
   - MU_LONG_WAIT indicates that a waiter has been woken many times but
     repeatedly failed to acquire when competing for the lock.  This is used
     only to prevent long-term starvation by writers.  The thread that sets it
     clears it when if acquires.
   - MU_ALL_FALSE indicates that a complete scan of the waiter list found no
     waiters with true conditions, and the lock has not been acquired by a
     writer since then.  This allows a reader lock to be released without
     testing conditions again.  It is legal to fail to set this, but illegal
     to set it inappropriately.
 */
#define MU_WLOCK ((uint32_t)(1 << 0)) /* writer lock is held. */
#define MU_SPINLOCK \
  ((uint32_t)(1 << 1)) /* spinlock is held (protects waiters). */
#define MU_WAITING ((uint32_t)(1 << 2)) /* waiter list is non-empty. */
#define MU_DESIG_WAKER                                                      \
  ((uint32_t)(1 << 3)) /* a former waiter awoke, and hasn't yet acquired or \
                          slept anew */
#define MU_CONDITION \
  ((uint32_t)(1 << 4)) /* the wait list contains some conditional waiters. */
#define MU_WRITER_WAITING ((uint32_t)(1 << 5)) /* there is a writer waiting */
#define MU_LONG_WAIT                                                           \
  ((uint32_t)(1 << 6)) /* the waiter at the head of the queue has been waiting \
                          a long time */
#define MU_ALL_FALSE                                      \
  ((uint32_t)(1 << 7)) /* all waiter conditions are false \
                        */
#define MU_RLOCK                                                            \
  ((uint32_t)(1 << 8)) /* low-order bit of reader count, which uses rest of \
                          word */

/* The constants below are derived from those above. */
#define MU_RLOCK_FIELD \
  (~(uint32_t)(MU_RLOCK - 1)) /* mask of reader count field */

#define MU_ANY_LOCK (MU_WLOCK | MU_RLOCK_FIELD) /* mask for any lock held */

#define MU_WZERO_TO_ACQUIRE \
  (MU_ANY_LOCK | MU_LONG_WAIT) /* bits to be zero to acquire write lock */
#define MU_WADD_TO_ACQUIRE (MU_WLOCK) /* add to acquire a write lock */
#define MU_WHELD_IF_NON_ZERO \
  (MU_WLOCK) /* if any of these bits are set, write lock is held */
#define MU_WSET_WHEN_WAITING \
  (MU_WAITING | MU_WRITER_WAITING) /* a writer is waiting */
#define MU_WCLEAR_ON_ACQUIRE \
  (MU_WRITER_WAITING) /* clear MU_WRITER_WAITING when a writer acquires */
#define MU_WCLEAR_ON_UNCONTENDED_RELEASE \
  (MU_ALL_FALSE) /* clear if a writer releases w/o waking */

/* bits to be zero to acquire read lock */
#define MU_RZERO_TO_ACQUIRE (MU_WLOCK | MU_WRITER_WAITING | MU_LONG_WAIT)
#define MU_RADD_TO_ACQUIRE  (MU_RLOCK) /* add to acquire a read lock */
#define MU_RHELD_IF_NON_ZERO \
  (MU_RLOCK_FIELD) /* if any of these bits are set, read lock is held */
#define MU_RSET_WHEN_WAITING \
  (MU_WAITING) /* indicate that some thread is waiting */
#define MU_RCLEAR_ON_ACQUIRE \
  ((uint32_t)0) /* nothing to clear when a read acquires */
#define MU_RCLEAR_ON_UNCONTENDED_RELEASE \
  ((uint32_t)0) /* nothing to clear when a read releases */

/* A lock_type holds the values needed to manipulate a mu in some mode (read or
   write).  This allows some of the code to be generic, and parameterized by
   the lock type. */
typedef struct lock_type_s {
  uint32_t zero_to_acquire; /* bits that must be zero to acquire */
  uint32_t add_to_acquire;  /* constant to add to acquire */
  uint32_t
      held_if_non_zero; /* if any of these bits are set, the lock is held */
  uint32_t set_when_waiting;             /* set when thread waits */
  uint32_t clear_on_acquire;             /* clear when thread acquires */
  uint32_t clear_on_uncontended_release; /* clear when thread releases without
                                            waking */
} lock_type;

/* writer_type points to a lock_type that describes how to manipulate a mu for a
 * writer. */
extern lock_type *nsync_writer_type_;

/* reader_type points to a lock_type that describes how to manipulate a mu for a
 * reader. */
extern lock_type *nsync_reader_type_;

/* ---------- */

/* Bits in nsync_cv.word */

#define CV_SPINLOCK  ((uint32_t)(1 << 0)) /* protects waiters */
#define CV_NON_EMPTY ((uint32_t)(1 << 1)) /* waiters list is non-empty */

/* ---------- */

/* Hold a pair of  condition function and its argument. */
struct wait_condition_s {
  int (*f)(const void *v);
  const void *v;
  int (*eq)(const void *a, const void *b);
};

/* Return whether wait conditions *a_ and *b_ are equal and non-null. */
#define WAIT_CONDITION_EQ(a_, b_)           \
  ((a_)->f != NULL && (a_)->f == (b_)->f && \
   ((a_)->v == (b_)->v ||                   \
    ((a_)->eq != NULL && (*(a_)->eq)((a_)->v, (b_)->v))))

/* If a waiter has waited this many times, it may set the MU_LONG_WAIT bit. */
#define LONG_WAIT_THRESHOLD 30

/* ---------- */

#define NOTIFIED_TIME(n_)                                 \
  (ATM_LOAD_ACQ(&(n_)->notified) != 0 ? nsync_time_zero   \
   : (n_)->expiry_time_valid          ? (n_)->expiry_time \
                                      : nsync_time_no_deadline)

/* A waiter represents a single waiter on a cv or a mu.

   To wait:
   Allocate a waiter struct *w with new_waiter(), set w.waiting=1, and
   w.cv_mu=nil or to the associated mu if waiting on a condition variable, then
   queue w.nsync_dll on some queue, and then wait using:
      while (ATM_LOAD_ACQ (&w.waiting) != 0) { nsync_mu_semaphore_p (&w.sem); }
   Return *w to the freepool by calling free_waiter (w).

   To wakeup:
   Remove *w from the relevant queue then:
    ATM_STORE_REL (&w.waiting, 0);
    nsync_mu_semaphore_v (&w.sem); */
typedef struct waiter_s {
  uint32_t tag; /* debug DLL_NSYNC_WAITER, DLL_WAITER, DLL_WAITER_SAMECOND */
  int flags;    /* see WAITER_* bits below */
  nsync_semaphore sem;       /* Thread waits on this semaphore. */
  struct nsync_waiter_s nw;  /* An embedded nsync_waiter_s. */
  struct nsync_mu_s_ *cv_mu; /* pointer to nsync_mu associated with a cv wait */
  lock_type
      *l_type; /* Lock type of the mu, or nil if not associated with a mu. */
  nsync_atomic_uint32_ remove_count; /* count of removals from queue */
  struct wait_condition_s cond;      /* A condition on which to acquire a mu. */
  struct Dll same_condition;         /* Links neighbours in nw.q with same
                                        non-nil condition. */
  struct waiter_s * next_free;
} waiter;
static const uint32_t WAITER_TAG = 0x0590239f;
static const uint32_t NSYNC_WAITER_TAG = 0x726d2ba9;

#define WAITER_RESERVED \
  0x1 /* waiter reserved by a thread, even when not in use */
#define WAITER_IN_USE 0x2 /* waiter in use by a thread */

#define ASSERT(x) unassert(x)

/* Return a pointer to the nsync_waiter_s containing struct Dll *e. */
#define DLL_NSYNC_WAITER(e)                 \
  (NSYNC_DEBUG ? nsync_dll_nsync_waiter_(e) \
               : DLL_CONTAINER(struct nsync_waiter_s, q, e))
struct nsync_waiter_s *nsync_dll_nsync_waiter_(struct Dll *e);

/* Return a pointer to the waiter struct that *e is embedded in, where *e is an
 * nw.q field. */
#define DLL_WAITER(e)                 \
  (NSYNC_DEBUG ? nsync_dll_waiter_(e) \
               : DLL_CONTAINER(waiter, nw, DLL_NSYNC_WAITER(e)))
waiter *nsync_dll_waiter_(struct Dll *e);

/* Return a pointer to the waiter struct that *e is embedded in, where *e is a
   same_condition field.  */
#define DLL_WAITER_SAMECOND(e)                 \
  (NSYNC_DEBUG ? nsync_dll_waiter_samecond_(e) \
               : DLL_CONTAINER(struct waiter_s, same_condition, e))
waiter *nsync_dll_waiter_samecond_(struct Dll *e);

/* Return a pointer to an unused waiter struct.
   Ensures that the enclosed timer is stopped and its channel drained. */
waiter *nsync_waiter_new_(void);

/* Return an unused waiter struct *w to the free pool. */
void nsync_waiter_free_(waiter *w);

/* ---------- */

/* The internals of an nync_note.  See internal/note.c for details of locking
   discipline.  */
struct nsync_note_s_ {
  struct Dll parent_child_link; /* parent's children, under parent->note_mu  */
  int expiry_time_valid; /* whether expiry_time is valid; r/o after init */
  nsync_time
      expiry_time;  /* expiry time, if expiry_time_valid != 0; r/o after init */
  nsync_mu note_mu; /* protects fields below except "notified" */
  nsync_cv no_children_cv;       /* signalled when children becomes empty */
  uint32_t disconnecting;        /* non-zero => node is being disconnected */
  nsync_atomic_uint32_ notified; /* non-zero if the note has been notified */
  struct nsync_note_s_ *parent;  /* points to parent, if any */
  struct Dll *children;          /* list of children */
  struct Dll *waiters;           /* list of waiters */
};

/* ---------- */

void nsync_mu_lock_slow_(nsync_mu *mu, waiter *w, uint32_t clear,
                         lock_type *l_type);
void nsync_mu_unlock_slow_(nsync_mu *mu, lock_type *l_type);
struct Dll *nsync_remove_from_mu_queue_(struct Dll *mu_queue, struct Dll *e);
void nsync_maybe_merge_conditions_(struct Dll *p, struct Dll *n);
nsync_time nsync_note_notified_deadline_(nsync_note n);
int nsync_sem_wait_with_cancel_(waiter *w, nsync_time abs_deadline,
                                nsync_note cancel_note);

COSMOPOLITAN_C_END_
#endif /* NSYNC_COMMON_H_ */