cosmopolitan/libc/intrin/pthread_mutex_lock.c

/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2022 Justine Alexandra Roberts Tunney                              │
│                                                                              │
│ Permission to use, copy, modify, and/or distribute this software for         │
│ any purpose with or without fee is hereby granted, provided that the         │
│ above copyright notice and this permission notice appear in all copies.      │
│                                                                              │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
│ PERFORMANCE OF THIS SOFTWARE.                                                │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/calls/blockcancel.internal.h"
#include "libc/calls/calls.h"
#include "libc/calls/state.internal.h"
#include "libc/cosmo.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/intrin/atomic.h"
#include "libc/intrin/describeflags.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/strace.h"
#include "libc/intrin/weaken.h"
#include "libc/macros.h"
#include "libc/runtime/internal.h"
#include "libc/thread/lock.h"
#include "libc/thread/posixthread.internal.h"
#include "libc/thread/thread.h"
#include "libc/thread/tls.h"
#include "third_party/nsync/mu.h"

static errno_t pthread_mutex_lock_normal_success(pthread_mutex_t *mutex,
                                                 uint64_t word) {
  if (IsModeDbg() || MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK) {
    __deadlock_track(mutex, MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK);
    __deadlock_record(mutex, MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK);
  }
  return 0;
}

// see "take 3" algorithm in "futexes are tricky" by ulrich drepper
// slightly improved to attempt acquiring multiple times b4 syscall
static int pthread_mutex_lock_drepper(pthread_mutex_t *mutex, uint64_t word,
                                      bool is_trylock) {
  int val = 0;
  if (atomic_compare_exchange_strong_explicit(
          &mutex->_futex, &val, 1, memory_order_acquire, memory_order_acquire))
    return pthread_mutex_lock_normal_success(mutex, word);
  if (is_trylock)
    return EBUSY;
  LOCKTRACE("acquiring pthread_mutex_lock_drepper(%t)...", mutex);
  if (val == 1)
    val = atomic_exchange_explicit(&mutex->_futex, 2, memory_order_acquire);
  BLOCK_CANCELATION;
  while (val > 0) {
    cosmo_futex_wait(&mutex->_futex, 2, MUTEX_PSHARED(word), 0, 0);
    val = atomic_exchange_explicit(&mutex->_futex, 2, memory_order_acquire);
  }
  ALLOW_CANCELATION;
  return pthread_mutex_lock_normal_success(mutex, word);
}

static errno_t pthread_mutex_lock_recursive(pthread_mutex_t *mutex,
                                            uint64_t word, bool is_trylock) {
  uint64_t lock;
  int backoff = 0;
  int me = gettid();
  bool once = false;
  for (;;) {
    if (MUTEX_OWNER(word) == me) {
      if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) {
        if (atomic_compare_exchange_weak_explicit(
                &mutex->_word, &word, MUTEX_INC_DEPTH(word),
                memory_order_relaxed, memory_order_relaxed))
          return 0;
        continue;
      } else {
        return EAGAIN;
      }
    }
    if (IsModeDbg())
      __deadlock_check(mutex, 0);
    word = MUTEX_UNLOCK(word);
    lock = MUTEX_LOCK(word);
    lock = MUTEX_SET_OWNER(lock, me);
    if (atomic_compare_exchange_weak_explicit(&mutex->_word, &word, lock,
                                              memory_order_acquire,
                                              memory_order_relaxed)) {
      if (IsModeDbg()) {
        __deadlock_track(mutex, 0);
        __deadlock_record(mutex, 0);
      }
      mutex->_pid = __pid;
      return 0;
    }
    if (is_trylock)
      return EBUSY;
    if (!once) {
      LOCKTRACE("acquiring pthread_mutex_lock_recursive(%t)...", mutex);
      once = true;
    }
    for (;;) {
      word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);
      if (MUTEX_OWNER(word) == me)
        break;
      if (word == MUTEX_UNLOCK(word))
        break;
      backoff = pthread_delay_np(mutex, backoff);
    }
  }
}

#if PTHREAD_USE_NSYNC
static errno_t pthread_mutex_lock_recursive_nsync(pthread_mutex_t *mutex,
                                                  uint64_t word,
                                                  bool is_trylock) {
  int me = gettid();
  for (;;) {
    if (MUTEX_OWNER(word) == me) {
      if (MUTEX_DEPTH(word) < MUTEX_DEPTH_MAX) {
        if (atomic_compare_exchange_weak_explicit(
                &mutex->_word, &word, MUTEX_INC_DEPTH(word),
                memory_order_relaxed, memory_order_relaxed))
          return 0;
        continue;
      } else {
        return EAGAIN;
      }
    }
    if (IsModeDbg())
      __deadlock_check(mutex, 0);
    if (!is_trylock) {
      _weaken(nsync_mu_lock)((nsync_mu *)mutex->_nsync);
    } else {
      if (!_weaken(nsync_mu_trylock)((nsync_mu *)mutex->_nsync))
        return EBUSY;
    }
    if (IsModeDbg()) {
      __deadlock_track(mutex, 0);
      __deadlock_record(mutex, 0);
    }
    word = MUTEX_UNLOCK(word);
    word = MUTEX_LOCK(word);
    word = MUTEX_SET_OWNER(word, me);
    mutex->_word = word;
    mutex->_pid = __pid;
    return 0;
  }
}
#endif

static errno_t pthread_mutex_lock_impl(pthread_mutex_t *mutex,
                                       bool is_trylock) {
  uint64_t word = atomic_load_explicit(&mutex->_word, memory_order_relaxed);

  // handle recursive mutexes
  if (MUTEX_TYPE(word) == PTHREAD_MUTEX_RECURSIVE) {
#if PTHREAD_USE_NSYNC
    if (_weaken(nsync_mu_lock) &&
        MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE) {
      return pthread_mutex_lock_recursive_nsync(mutex, word, is_trylock);
    } else {
      return pthread_mutex_lock_recursive(mutex, word, is_trylock);
    }
#else
    return pthread_mutex_lock_recursive(mutex, word, is_trylock);
#endif
  }

  // check if normal mutex is already owned by calling thread
  if (!is_trylock &&
      (MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK ||
       (IsModeDbg() && MUTEX_TYPE(word) == PTHREAD_MUTEX_DEFAULT))) {
    if (__deadlock_tracked(mutex) == 1) {
      if (IsModeDbg() && MUTEX_TYPE(word) != PTHREAD_MUTEX_ERRORCHECK) {
        kprintf("error: attempted to lock non-recursive mutex that's already "
                "held by the calling thread: %t\n",
                mutex);
        DebugBreak();
      }
      return EDEADLK;
    }
  }

  // check if locking will create cycle in lock graph
  if (IsModeDbg() || MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK)
    if (__deadlock_check(mutex, MUTEX_TYPE(word) == PTHREAD_MUTEX_ERRORCHECK))
      return EDEADLK;

#if PTHREAD_USE_NSYNC
  // use superior mutexes if possible
  if (MUTEX_PSHARED(word) == PTHREAD_PROCESS_PRIVATE &&
      _weaken(nsync_mu_lock)) {
    // on apple silicon we should just put our faith in ulock
    // otherwise *nsync gets struck down by the eye of sauron
    if (!IsXnuSilicon()) {
      if (!is_trylock) {
        _weaken(nsync_mu_lock)((nsync_mu *)mutex->_nsync);
        return pthread_mutex_lock_normal_success(mutex, word);
      } else {
        if (_weaken(nsync_mu_trylock)((nsync_mu *)mutex->_nsync))
          return pthread_mutex_lock_normal_success(mutex, word);
        return EBUSY;
      }
    }
  }
#endif

  // isc licensed non-recursive mutex implementation
  return pthread_mutex_lock_drepper(mutex, word, is_trylock);
}

/**
 * Locks mutex, e.g.
 *
 *     pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
 *     pthread_mutex_lock(&lock);
 *     // do work...
 *     pthread_mutex_unlock(&lock);
 *
 * The long way to do that is:
 *
 *     pthread_mutex_t lock;
 *     pthread_mutexattr_t attr;
 *     pthread_mutexattr_init(&attr);
 *     pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT);
 *     pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_PRIVATE);
 *     pthread_mutex_init(&lock, &attr);
 *     pthread_mutexattr_destroy(&attr);
 *     pthread_mutex_lock(&lock);
 *     // do work...
 *     pthread_mutex_unlock(&lock);
 *     pthread_mutex_destroy(&lock);
 *
 * The following non-POSIX initializers are also provided by cosmo libc:
 *
 * - `PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP`
 * - `PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP`
 * - `PTHREAD_NORMAL_MUTEX_INITIALIZER_NP`
 *
 * Locking a mutex that's already locked by the calling thread will make
 * the thread hang indefinitely, i.e. it's a deadlock condition. You can
 * use `PTHREAD_MUTEX_RECURSIVE` to allow recursive locking, which could
 * result in somewhat less performance. An alternative solution is using
 * the `PTHREAD_MUTEX_ERRORCHECK` mode, which raises `EDEADLK` for that.
 *
 * If a thread locks a mutex while other mutexes are already locked then
 * you need to observe a consistent global ordering, otherwise deadlocks
 * might occur. The Cosmopolitan runtime can detect these cycles quickly
 * so you can fix your code before it becomes an issue. With error check
 * mode, an EPERM will be returned. If your app is using `cosmocc -mdbg`
 * then an error message will be printed including the demangled symbols
 * of the mutexes in the strongly connected component that was detected.
 * Please note that, even for debug builds mutexes set to explicitly use
 * the `PTHREAD_MUTEX_ERRORCHECK` mode will return an error code instead
 * which means the cosmo debug mode only influences undefined behaviors.
 *
 * Cosmopolitan only supports error checking on mutexes stored in static
 * memory, i.e. your `mutex` pointer must point inside the .data or .bss
 * sections of your executable. When compiling your programs using -mdbg
 * all your locks will gain error checking automatically. When deadlocks
 * are detected an error message will be printed and a SIGTRAP signal is
 * raised, which may be ignored to force EDEADLK and EPERM to be raised.
 *
 * Using `cosmocc -mdbg` also enhances `--strace` with information about
 * mutexes. First, locks and unlocks will be logged. Since the lock line
 * only appears after the lock is acquired, that might mean you'll never
 * get an indication about a lock that takes a very long time to acquire
 * so, whenever a lock can't immediately be acquired, a second line gets
 * printed *before* the lock is acquired to let you know that the thread
 * is waiting for a particular lock. If your mutex object resides within
 * static memory, then its demangled symbol name will be printed. If you
 * call ShowCrashReports() at the beginning of your main() function then
 * you'll also see a backtrace when a locking violation occurs. When the
 * symbols in the violation error messages show up as numbers, and it is
 * desirable to see demangled symbols without enabling full crash report
 * functionality the GetSymbolTable() function may be called for effect.
 *
 * If you use `PTHREAD_MUTEX_NORMAL`, instead of `PTHREAD_MUTEX_DEFAULT`
 * then deadlocking is actually defined behavior according to POSIX.1 so
 * the helpfulness of `cosmocc -mdbg` will be somewhat weakened.
 *
 * If your `mutex` object resides in `MAP_SHARED` memory, then undefined
 * behavior will happen unless you use `PTHREAD_PROCESS_SHARED` mode, if
 * the lock is used by multiple processes.
 *
 * This function does nothing when the process is in vfork() mode.
 *
 * @return 0 on success, or error number on failure
 * @raise EDEADLK if mutex is recursive and locked by another thread
 * @raise EDEADLK if mutex is non-recursive and locked by current thread
 * @raise EDEADLK if cycle is detected in global nested lock graph
 * @raise EAGAIN if maximum recursive locks is exceeded
 * @see pthread_spin_lock()
 * @vforksafe
 */
errno_t _pthread_mutex_lock(pthread_mutex_t *mutex) {
  if (__tls_enabled && !__vforked) {
    errno_t err = pthread_mutex_lock_impl(mutex, false);
    LOCKTRACE("pthread_mutex_lock(%t) → %s", mutex, DescribeErrno(err));
    return err;
  } else {
    LOCKTRACE("skipping pthread_mutex_lock(%t) due to runtime state", mutex);
    return 0;
  }
}

/**
 * Attempts acquiring lock.
 *
 * Unlike pthread_mutex_lock() this function won't block and instead
 * returns an error immediately if the lock couldn't be acquired.
 *
 * @return 0 if lock was acquired, otherwise an errno
 * @raise EBUSY if lock is currently held by another thread
 * @raise EAGAIN if maximum number of recursive locks is held
 * @raise EDEADLK if `mutex` is `PTHREAD_MUTEX_ERRORCHECK` and the
 *     current thread already holds this mutex
 */
errno_t _pthread_mutex_trylock(pthread_mutex_t *mutex) {
  if (__tls_enabled && !__vforked) {
    errno_t err = pthread_mutex_lock_impl(mutex, true);
    LOCKTRACE("pthread_mutex_trylock(%t) → %s", mutex, DescribeErrno(err));
    return err;
  } else {
    LOCKTRACE("skipping pthread_mutex_trylock(%t) due to runtime state", mutex);
    return 0;
  }
}

__weak_reference(_pthread_mutex_lock, pthread_mutex_lock);
__weak_reference(_pthread_mutex_trylock, pthread_mutex_trylock);