#ifndef NSYNC_COMMON_H_ #define NSYNC_COMMON_H_ #include "libc/assert.h" #include "libc/intrin/dll.h" #include "third_party/nsync/atomic.h" #include "third_party/nsync/atomic.internal.h" #include "third_party/nsync/cv.h" #include "third_party/nsync/mu.h" #include "third_party/nsync/mu_semaphore.h" #include "third_party/nsync/note.h" #include "third_party/nsync/time.h" #include "third_party/nsync/wait_s.internal.h" #if !(__ASSEMBLER__ + __LINKER__ + 0) COSMOPOLITAN_C_START_ #ifdef MODE_DBG #define NSYNC_DEBUG 1 #else #define NSYNC_DEBUG 0 #endif /* Yield the CPU. Platform specific. */ void nsync_yield_(void); /* Retrieve the per-thread cache of the waiter object. Platform specific. */ void *nsync_per_thread_waiter_(void (*dest)(void *)); /* Set the per-thread cache of the waiter object. Platform specific. */ void nsync_set_per_thread_waiter_(void *v, void (*dest)(void *)); /* Used in spinloops to delay resumption of the loop. Usage: unsigned attempts = 0; while (try_something) { attempts = nsync_spin_delay_ (attempts); } */ unsigned nsync_spin_delay_(unsigned attempts); /* Spin until (*w & test) == 0, then atomically perform *w = ((*w | set) & ~clear), perform an acquire barrier, and return the previous value of *w. */ uint32_t nsync_spin_test_and_set_(nsync_atomic_uint32_ *w, uint32_t test, uint32_t set, uint32_t clear); /* Abort after printing the nul-temrinated string s[]. */ void nsync_panic_(const char *s); /* ---------- */ #define MIN_(a_, b_) ((a_) < (b_) ? (a_) : (b_)) #define MAX_(a_, b_) ((a_) > (b_) ? (a_) : (b_)) /* ---------- */ /* Fields in nsync_mu.word. - At least one of the MU_WLOCK or MU_RLOCK_FIELD fields must be zero. - MU_WLOCK indicates that a write lock is held. - MU_RLOCK_FIELD is a count of readers with read locks. - MU_SPINLOCK represents a spinlock that must be held when manipulating the waiter queue. - MU_DESIG_WAKER indicates that a former waiter has been woken, but has neither acquired the lock nor gone back to sleep. Legal to fail to set it; illegal to set it when no such waiter exists. - MU_WAITING indicates whether the waiter queue is non-empty. The following bits should be zero if MU_WAITING is zero. - MU_CONDITION indicates that some waiter may have an associated condition (from nsync_mu_wait, etc.). Legal to set it with no such waiter exists, but illegal to fail to set it with such a waiter. - MU_WRITER_WAITING indicates that a reader that has not yet blocked at least once should not acquire in order not to starve waiting writers. It set when a writer blocks or a reader is woken with a writer waiting. It is reset when a writer acquires, but set again when that writer releases if it wakes readers and there is a waiting writer. - MU_LONG_WAIT indicates that a waiter has been woken many times but repeatedly failed to acquire when competing for the lock. This is used only to prevent long-term starvation by writers. The thread that sets it clears it when if acquires. - MU_ALL_FALSE indicates that a complete scan of the waiter list found no waiters with true conditions, and the lock has not been acquired by a writer since then. This allows a reader lock to be released without testing conditions again. It is legal to fail to set this, but illegal to set it inappropriately. */ #define MU_WLOCK ((uint32_t)(1 << 0)) /* writer lock is held. */ #define MU_SPINLOCK \ ((uint32_t)(1 << 1)) /* spinlock is held (protects waiters). */ #define MU_WAITING ((uint32_t)(1 << 2)) /* waiter list is non-empty. */ #define MU_DESIG_WAKER \ ((uint32_t)(1 << 3)) /* a former waiter awoke, and hasn't yet acquired or \ slept anew */ #define MU_CONDITION \ ((uint32_t)(1 << 4)) /* the wait list contains some conditional waiters. */ #define MU_WRITER_WAITING ((uint32_t)(1 << 5)) /* there is a writer waiting */ #define MU_LONG_WAIT \ ((uint32_t)(1 << 6)) /* the waiter at the head of the queue has been waiting \ a long time */ #define MU_ALL_FALSE \ ((uint32_t)(1 << 7)) /* all waiter conditions are false \ */ #define MU_RLOCK \ ((uint32_t)(1 << 8)) /* low-order bit of reader count, which uses rest of \ word */ /* The constants below are derived from those above. */ #define MU_RLOCK_FIELD \ (~(uint32_t)(MU_RLOCK - 1)) /* mask of reader count field */ #define MU_ANY_LOCK (MU_WLOCK | MU_RLOCK_FIELD) /* mask for any lock held */ #define MU_WZERO_TO_ACQUIRE \ (MU_ANY_LOCK | MU_LONG_WAIT) /* bits to be zero to acquire write lock */ #define MU_WADD_TO_ACQUIRE (MU_WLOCK) /* add to acquire a write lock */ #define MU_WHELD_IF_NON_ZERO \ (MU_WLOCK) /* if any of these bits are set, write lock is held */ #define MU_WSET_WHEN_WAITING \ (MU_WAITING | MU_WRITER_WAITING) /* a writer is waiting */ #define MU_WCLEAR_ON_ACQUIRE \ (MU_WRITER_WAITING) /* clear MU_WRITER_WAITING when a writer acquires */ #define MU_WCLEAR_ON_UNCONTENDED_RELEASE \ (MU_ALL_FALSE) /* clear if a writer releases w/o waking */ /* bits to be zero to acquire read lock */ #define MU_RZERO_TO_ACQUIRE (MU_WLOCK | MU_WRITER_WAITING | MU_LONG_WAIT) #define MU_RADD_TO_ACQUIRE (MU_RLOCK) /* add to acquire a read lock */ #define MU_RHELD_IF_NON_ZERO \ (MU_RLOCK_FIELD) /* if any of these bits are set, read lock is held */ #define MU_RSET_WHEN_WAITING \ (MU_WAITING) /* indicate that some thread is waiting */ #define MU_RCLEAR_ON_ACQUIRE \ ((uint32_t)0) /* nothing to clear when a read acquires */ #define MU_RCLEAR_ON_UNCONTENDED_RELEASE \ ((uint32_t)0) /* nothing to clear when a read releases */ /* A lock_type holds the values needed to manipulate a mu in some mode (read or write). This allows some of the code to be generic, and parameterized by the lock type. */ typedef struct lock_type_s { uint32_t zero_to_acquire; /* bits that must be zero to acquire */ uint32_t add_to_acquire; /* constant to add to acquire */ uint32_t held_if_non_zero; /* if any of these bits are set, the lock is held */ uint32_t set_when_waiting; /* set when thread waits */ uint32_t clear_on_acquire; /* clear when thread acquires */ uint32_t clear_on_uncontended_release; /* clear when thread releases without waking */ } lock_type; /* writer_type points to a lock_type that describes how to manipulate a mu for a * writer. */ extern lock_type *nsync_writer_type_; /* reader_type points to a lock_type that describes how to manipulate a mu for a * reader. */ extern lock_type *nsync_reader_type_; /* ---------- */ /* Bits in nsync_cv.word */ #define CV_SPINLOCK ((uint32_t)(1 << 0)) /* protects waiters */ #define CV_NON_EMPTY ((uint32_t)(1 << 1)) /* waiters list is non-empty */ /* ---------- */ /* Hold a pair of condition function and its argument. */ struct wait_condition_s { int (*f)(const void *v); const void *v; int (*eq)(const void *a, const void *b); }; /* Return whether wait conditions *a_ and *b_ are equal and non-null. */ #define WAIT_CONDITION_EQ(a_, b_) \ ((a_)->f != NULL && (a_)->f == (b_)->f && \ ((a_)->v == (b_)->v || \ ((a_)->eq != NULL && (*(a_)->eq)((a_)->v, (b_)->v)))) /* If a waiter has waited this many times, it may set the MU_LONG_WAIT bit. */ #define LONG_WAIT_THRESHOLD 30 /* ---------- */ #define NOTIFIED_TIME(n_) \ (ATM_LOAD_ACQ(&(n_)->notified) != 0 ? nsync_time_zero \ : (n_)->expiry_time_valid ? (n_)->expiry_time \ : nsync_time_no_deadline) /* A waiter represents a single waiter on a cv or a mu. To wait: Allocate a waiter struct *w with new_waiter(), set w.waiting=1, and w.cv_mu=nil or to the associated mu if waiting on a condition variable, then queue w.nsync_dll on some queue, and then wait using: while (ATM_LOAD_ACQ (&w.waiting) != 0) { nsync_mu_semaphore_p (&w.sem); } Return *w to the freepool by calling free_waiter (w). To wakeup: Remove *w from the relevant queue then: ATM_STORE_REL (&w.waiting, 0); nsync_mu_semaphore_v (&w.sem); */ typedef struct waiter_s { uint32_t tag; /* debug DLL_NSYNC_WAITER, DLL_WAITER, DLL_WAITER_SAMECOND */ int flags; /* see WAITER_* bits below */ nsync_semaphore sem; /* Thread waits on this semaphore. */ struct nsync_waiter_s nw; /* An embedded nsync_waiter_s. */ struct nsync_mu_s_ *cv_mu; /* pointer to nsync_mu associated with a cv wait */ lock_type *l_type; /* Lock type of the mu, or nil if not associated with a mu. */ nsync_atomic_uint32_ remove_count; /* count of removals from queue */ struct wait_condition_s cond; /* A condition on which to acquire a mu. */ struct Dll same_condition; /* Links neighbours in nw.q with same non-nil condition. */ } waiter; static const uint32_t WAITER_TAG = 0x0590239f; static const uint32_t NSYNC_WAITER_TAG = 0x726d2ba9; #define WAITER_RESERVED \ 0x1 /* waiter reserved by a thread, even when not in use */ #define WAITER_IN_USE 0x2 /* waiter in use by a thread */ #define ASSERT(x) _npassert(x) /* Return a pointer to the nsync_waiter_s containing struct Dll *e. */ #define DLL_NSYNC_WAITER(e) \ (NSYNC_DEBUG ? nsync_dll_nsync_waiter_(e) \ : DLL_CONTAINER(struct nsync_waiter_s, q, e)) struct nsync_waiter_s *nsync_dll_nsync_waiter_(struct Dll *e); /* Return a pointer to the waiter struct that *e is embedded in, where *e is an * nw.q field. */ #define DLL_WAITER(e) \ (NSYNC_DEBUG ? nsync_dll_waiter_(e) \ : DLL_CONTAINER(waiter, nw, DLL_NSYNC_WAITER(e))) waiter *nsync_dll_waiter_(struct Dll *e); /* Return a pointer to the waiter struct that *e is embedded in, where *e is a same_condition field. */ #define DLL_WAITER_SAMECOND(e) \ (NSYNC_DEBUG ? nsync_dll_waiter_samecond_(e) \ : DLL_CONTAINER(struct waiter_s, same_condition, e)) waiter *nsync_dll_waiter_samecond_(struct Dll *e); /* Return a pointer to an unused waiter struct. Ensures that the enclosed timer is stopped and its channel drained. */ waiter *nsync_waiter_new_(void); /* Return an unused waiter struct *w to the free pool. */ void nsync_waiter_free_(waiter *w); /* ---------- */ /* The internals of an nync_note. See internal/note.c for details of locking discipline. */ struct nsync_note_s_ { struct Dll parent_child_link; /* parent's children, under parent->note_mu */ int expiry_time_valid; /* whether expiry_time is valid; r/o after init */ nsync_time expiry_time; /* expiry time, if expiry_time_valid != 0; r/o after init */ nsync_mu note_mu; /* protects fields below except "notified" */ nsync_cv no_children_cv; /* signalled when children becomes empty */ uint32_t disconnecting; /* non-zero => node is being disconnected */ nsync_atomic_uint32_ notified; /* non-zero if the note has been notified */ struct nsync_note_s_ *parent; /* points to parent, if any */ struct Dll *children; /* list of children */ struct Dll *waiters; /* list of waiters */ }; /* ---------- */ void nsync_mu_lock_slow_(nsync_mu *mu, waiter *w, uint32_t clear, lock_type *l_type); void nsync_mu_unlock_slow_(nsync_mu *mu, lock_type *l_type); struct Dll *nsync_remove_from_mu_queue_(struct Dll *mu_queue, struct Dll *e); void nsync_maybe_merge_conditions_(struct Dll *p, struct Dll *n); nsync_time nsync_note_notified_deadline_(nsync_note n); int nsync_sem_wait_with_cancel_(waiter *w, nsync_time abs_deadline, nsync_note cancel_note); COSMOPOLITAN_C_END_ #endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ #endif /* NSYNC_COMMON_H_ */