Add LLVM libcxxabi (#1063)

* third_party: Add libcxxabi Added libcxxabi from LLVM 17.0.6 The library implements the Itanium C++ exception handling ABI. * third_party/libcxxabi: Enable __cxa_thread_atexit Enable `__cxa_thread_atexit` from libcxxabi. `__cxa_thread_atexit_impl` is still implemented by the cosmo libc. The original `__cxa_thread_atexit` has been removed. * third_party/libcxx: Build with exceptions Build libcxx with exceptions enabled. - Removed `_LIBCPP_NO_EXCEPTIONS` from `__config`. - Switched the exception implementation to `libcxxabi`. These two files are taken from the same `libcxx` version as mentioned in `README.cosmo`. - Removed `new_handler_fallback` in favor of `libcxxabi` implementation. - Enable `-fexceptions` and `-frtti` for `libcxx`. - Removed `THIRD_PARTY_LIBCXX` dependency from `libcxxabi` and `libunwind`. These libraries do not use any runtime `libcxx` functions, just headers. * libc: Remove remaining redundant cxa functions - `__cxa_pure_virtual` in `libcxxabi` is also a stub similar to the existing one. - `__cxa_guard_*` from `libcxxabi` is used instead of the ones from Android. Now there should be no more duplicate implementations. `__cxa_thread_atexit_impl`, `__cxa_atexit`, and related supporting functions, are still left to other libraries as in `libcxxabi`. `libcxxabi` is also now added to `cosmopolitan.a` to make up for the removed functions. Affected in-tree libraries (`third_party/double-conversion`) have been updated.
2025-07-01 16:58:30 +00:00 · 2024-01-08 23:45:10 +07:00 · 2024-01-08 23:45:10 +07:00 · 8b33204f37
commit 8b33204f37
parent 94bab1618d
53 changed files with 14606 additions and 372 deletions
--- a/third_party/libcxxabi/cxa_guard_impl.h
+++ b/third_party/libcxxabi/cxa_guard_impl.h
@ -0,0 +1,674 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H
+#define LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H
+
+/* cxa_guard_impl.h - Implements the C++ runtime support for function local
+ * static guards.
+ * The layout of the guard object is the same across ARM and Itanium.
+ *
+ * The first "guard byte" (which is checked by the compiler) is set only upon
+ * the completion of cxa release.
+ *
+ * The second "init byte" does the rest of the bookkeeping. It tracks if
+ * initialization is complete or pending, and if there are waiting threads.
+ *
+ * If the guard variable is 64-bits and the platforms supplies a 32-bit thread
+ * identifier, it is used to detect recursive initialization. The thread ID of
+ * the thread currently performing initialization is stored in the second word.
+ *
+ *  Guard Object Layout:
+ * ---------------------------------------------------------------------------
+ * | a+0: guard byte | a+1: init byte | a+2: unused ... | a+4: thread-id ... |
+ * ---------------------------------------------------------------------------
+ *
+ * Note that we don't do what the ABI docs suggest (put a mutex in the guard
+ * object which we acquire in cxa_guard_acquire and release in
+ * cxa_guard_release). Instead we use the init byte to imitate that behaviour,
+ * but without actually holding anything mutex related between aquire and
+ * release/abort.
+ *
+ *  Access Protocol:
+ *    For each implementation the guard byte is checked and set before accessing
+ *    the init byte.
+ *
+ *  Overall Design:
+ *    The implementation was designed to allow each implementation to be tested
+ *    independent of the C++ runtime or platform support.
+ *
+ */
+
+#include "third_party/libcxxabi/include/__cxxabi_config.h"
+#include "third_party/libcxx/include/atomic_support.hh" // from libc++
+#if defined(__has_include)
+#  if __has_include(<sys/syscall.h>)
+#    include <sys/syscall.h>
+#  endif
+#  if __has_include(<unistd.h>)
+#    include <unistd.h>
+#  endif
+#endif
+
+#include "third_party/libcxx/__threading_support"
+#include "third_party/libcxx/cstdint"
+#include "third_party/libcxx/cstring"
+#include "libc/isystem/limits.h"
+#include "libc/isystem/stdlib.h"
+
+#ifndef _LIBCXXABI_HAS_NO_THREADS
+#  if defined(__ELF__) && defined(_LIBCXXABI_LINK_PTHREAD_LIB)
+#    pragma comment(lib, "pthread")
+#  endif
+#endif
+
+#if defined(__clang__)
+#  pragma clang diagnostic push
+#  pragma clang diagnostic ignored "-Wtautological-pointer-compare"
+#elif defined(__GNUC__)
+#  pragma GCC diagnostic push
+#  pragma GCC diagnostic ignored "-Waddress"
+#endif
+
+// To make testing possible, this header is included from both cxa_guard.cpp
+// and a number of tests.
+//
+// For this reason we place everything in an anonymous namespace -- even though
+// we're in a header. We want the actual implementation and the tests to have
+// unique definitions of the types in this header (since the tests may depend
+// on function local statics).
+//
+// To enforce this either `BUILDING_CXA_GUARD` or `TESTING_CXA_GUARD` must be
+// defined when including this file. Only `src/cxa_guard.cpp` should define
+// the former.
+#ifdef BUILDING_CXA_GUARD
+#  include "third_party/libcxxabi/abort_message.h"
+#  define ABORT_WITH_MESSAGE(...) ::abort_message(__VA_ARGS__)
+#elif defined(TESTING_CXA_GUARD)
+#  define ABORT_WITH_MESSAGE(...) ::abort()
+#else
+#  error "Either BUILDING_CXA_GUARD or TESTING_CXA_GUARD must be defined"
+#endif
+
+#if __has_feature(thread_sanitizer)
+extern "C" void __tsan_acquire(void*);
+extern "C" void __tsan_release(void*);
+#else
+#  define __tsan_acquire(addr) ((void)0)
+#  define __tsan_release(addr) ((void)0)
+#endif
+
+namespace __cxxabiv1 {
+// Use an anonymous namespace to ensure that the tests and actual implementation
+// have unique definitions of these symbols.
+namespace {
+
+//===----------------------------------------------------------------------===//
+//                          Misc Utilities
+//===----------------------------------------------------------------------===//
+
+template <class T, T (*Init)()>
+struct LazyValue {
+  LazyValue() : is_init(false) {}
+
+  T& get() {
+    if (!is_init) {
+      value = Init();
+      is_init = true;
+    }
+    return value;
+  }
+
+private:
+  T value;
+  bool is_init = false;
+};
+
+template <class IntType>
+class AtomicInt {
+public:
+  using MemoryOrder = std::__libcpp_atomic_order;
+
+  explicit AtomicInt(IntType* b) : b_(b) {}
+  AtomicInt(AtomicInt const&) = delete;
+  AtomicInt& operator=(AtomicInt const&) = delete;
+
+  IntType load(MemoryOrder ord) { return std::__libcpp_atomic_load(b_, ord); }
+  void store(IntType val, MemoryOrder ord) { std::__libcpp_atomic_store(b_, val, ord); }
+  IntType exchange(IntType new_val, MemoryOrder ord) { return std::__libcpp_atomic_exchange(b_, new_val, ord); }
+  bool compare_exchange(IntType* expected, IntType desired, MemoryOrder ord_success, MemoryOrder ord_failure) {
+    return std::__libcpp_atomic_compare_exchange(b_, expected, desired, ord_success, ord_failure);
+  }
+
+private:
+  IntType* b_;
+};
+
+//===----------------------------------------------------------------------===//
+//                       PlatformGetThreadID
+//===----------------------------------------------------------------------===//
+
+#if defined(__APPLE__) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
+uint32_t PlatformThreadID() {
+  static_assert(sizeof(mach_port_t) == sizeof(uint32_t), "");
+  return static_cast<uint32_t>(pthread_mach_thread_np(std::__libcpp_thread_get_current_id()));
+}
+#elif defined(SYS_gettid) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
+uint32_t PlatformThreadID() {
+  static_assert(sizeof(pid_t) == sizeof(uint32_t), "");
+  return static_cast<uint32_t>(syscall(SYS_gettid));
+}
+#else
+constexpr uint32_t (*PlatformThreadID)() = nullptr;
+#endif
+
+//===----------------------------------------------------------------------===//
+//                          GuardByte
+//===----------------------------------------------------------------------===//
+
+static constexpr uint8_t UNSET = 0;
+static constexpr uint8_t COMPLETE_BIT = (1 << 0);
+static constexpr uint8_t PENDING_BIT = (1 << 1);
+static constexpr uint8_t WAITING_BIT = (1 << 2);
+
+/// Manages reads and writes to the guard byte.
+struct GuardByte {
+  GuardByte() = delete;
+  GuardByte(GuardByte const&) = delete;
+  GuardByte& operator=(GuardByte const&) = delete;
+
+  explicit GuardByte(uint8_t* const guard_byte_address) : guard_byte(guard_byte_address) {}
+
+public:
+  /// The guard byte portion of cxa_guard_acquire. Returns true if
+  /// initialization has already been completed.
+  bool acquire() {
+    // if guard_byte is non-zero, we have already completed initialization
+    // (i.e. release has been called)
+    return guard_byte.load(std::_AO_Acquire) != UNSET;
+  }
+
+  /// The guard byte portion of cxa_guard_release.
+  void release() { guard_byte.store(COMPLETE_BIT, std::_AO_Release); }
+
+  /// The guard byte portion of cxa_guard_abort.
+  void abort() {} // Nothing to do
+
+private:
+  AtomicInt<uint8_t> guard_byte;
+};
+
+//===----------------------------------------------------------------------===//
+//                       InitByte Implementations
+//===----------------------------------------------------------------------===//
+//
+// Each initialization byte implementation supports the following methods:
+//
+//  InitByte(uint8_t* _init_byte_address, uint32_t* _thread_id_address)
+//    Construct the InitByte object, initializing our member variables
+//
+//  bool acquire()
+//    Called before we start the initialization. Check if someone else has already started, and if
+//    not to signal our intent to start it ourselves. We determine the current status from the init
+//    byte, which is one of 4 possible values:
+//      COMPLETE:           Initialization was finished by somebody else. Return true.
+//      PENDING:            Somebody has started the initialization already, set the WAITING bit,
+//                          then wait for the init byte to get updated with a new value.
+//      (PENDING|WAITING):  Somebody has started the initialization already, and we're not the
+//                          first one waiting. Wait for the init byte to get updated.
+//      UNSET:              Initialization hasn't successfully completed, and nobody is currently
+//                          performing the initialization. Set the PENDING bit to indicate our
+//                          intention to start the initialization, and return false.
+//    The return value indicates whether initialization has already been completed.
+//
+//  void release()
+//    Called after successfully completing the initialization. Update the init byte to reflect
+//    that, then if anybody else is waiting, wake them up.
+//
+//  void abort()
+//    Called after an error is thrown during the initialization. Reset the init byte to UNSET to
+//    indicate that we're no longer performing the initialization, then if anybody is waiting, wake
+//    them up so they can try performing the initialization.
+//
+
+//===----------------------------------------------------------------------===//
+//                    Single Threaded Implementation
+//===----------------------------------------------------------------------===//
+
+/// InitByteNoThreads - Doesn't use any inter-thread synchronization when
+/// managing reads and writes to the init byte.
+struct InitByteNoThreads {
+  InitByteNoThreads() = delete;
+  InitByteNoThreads(InitByteNoThreads const&) = delete;
+  InitByteNoThreads& operator=(InitByteNoThreads const&) = delete;
+
+  explicit InitByteNoThreads(uint8_t* _init_byte_address, uint32_t*) : init_byte_address(_init_byte_address) {}
+
+  /// The init byte portion of cxa_guard_acquire. Returns true if
+  /// initialization has already been completed.
+  bool acquire() {
+    if (*init_byte_address == COMPLETE_BIT)
+      return true;
+    if (*init_byte_address & PENDING_BIT)
+      ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization: do you have a function-local static variable whose initialization depends on that function?");
+    *init_byte_address = PENDING_BIT;
+    return false;
+  }
+
+  /// The init byte portion of cxa_guard_release.
+  void release() { *init_byte_address = COMPLETE_BIT; }
+  /// The init byte portion of cxa_guard_abort.
+  void abort() { *init_byte_address = UNSET; }
+
+private:
+  /// The address of the byte used during initialization.
+  uint8_t* const init_byte_address;
+};
+
+//===----------------------------------------------------------------------===//
+//                     Global Mutex Implementation
+//===----------------------------------------------------------------------===//
+
+struct LibcppMutex;
+struct LibcppCondVar;
+
+#ifndef _LIBCXXABI_HAS_NO_THREADS
+struct LibcppMutex {
+  LibcppMutex() = default;
+  LibcppMutex(LibcppMutex const&) = delete;
+  LibcppMutex& operator=(LibcppMutex const&) = delete;
+
+  bool lock() { return std::__libcpp_mutex_lock(&mutex); }
+  bool unlock() { return std::__libcpp_mutex_unlock(&mutex); }
+
+private:
+  friend struct LibcppCondVar;
+  std::__libcpp_mutex_t mutex = _LIBCPP_MUTEX_INITIALIZER;
+};
+
+struct LibcppCondVar {
+  LibcppCondVar() = default;
+  LibcppCondVar(LibcppCondVar const&) = delete;
+  LibcppCondVar& operator=(LibcppCondVar const&) = delete;
+
+  bool wait(LibcppMutex& mut) { return std::__libcpp_condvar_wait(&cond, &mut.mutex); }
+  bool broadcast() { return std::__libcpp_condvar_broadcast(&cond); }
+
+private:
+  std::__libcpp_condvar_t cond = _LIBCPP_CONDVAR_INITIALIZER;
+};
+#else
+struct LibcppMutex {};
+struct LibcppCondVar {};
+#endif // !defined(_LIBCXXABI_HAS_NO_THREADS)
+
+/// InitByteGlobalMutex - Uses a global mutex and condition variable (common to
+/// all static local variables) to manage reads and writes to the init byte.
+template <class Mutex, class CondVar, Mutex& global_mutex, CondVar& global_cond,
+          uint32_t (*GetThreadID)() = PlatformThreadID>
+struct InitByteGlobalMutex {
+
+  explicit InitByteGlobalMutex(uint8_t* _init_byte_address, uint32_t* _thread_id_address)
+      : init_byte_address(_init_byte_address), thread_id_address(_thread_id_address),
+        has_thread_id_support(_thread_id_address != nullptr && GetThreadID != nullptr) {}
+
+public:
+  /// The init byte portion of cxa_guard_acquire. Returns true if
+  /// initialization has already been completed.
+  bool acquire() {
+    LockGuard g("__cxa_guard_acquire");
+    // Check for possible recursive initialization.
+    if (has_thread_id_support && (*init_byte_address & PENDING_BIT)) {
+      if (*thread_id_address == current_thread_id.get())
+        ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization: do you have a function-local static variable whose initialization depends on that function?");
+    }
+
+    // Wait until the pending bit is not set.
+    while (*init_byte_address & PENDING_BIT) {
+      *init_byte_address |= WAITING_BIT;
+      global_cond.wait(global_mutex);
+    }
+
+    if (*init_byte_address == COMPLETE_BIT)
+      return true;
+
+    if (has_thread_id_support)
+      *thread_id_address = current_thread_id.get();
+
+    *init_byte_address = PENDING_BIT;
+    return false;
+  }
+
+  /// The init byte portion of cxa_guard_release.
+  void release() {
+    bool has_waiting;
+    {
+      LockGuard g("__cxa_guard_release");
+      has_waiting = *init_byte_address & WAITING_BIT;
+      *init_byte_address = COMPLETE_BIT;
+    }
+    if (has_waiting) {
+      if (global_cond.broadcast()) {
+        ABORT_WITH_MESSAGE("%s failed to broadcast", "__cxa_guard_release");
+      }
+    }
+  }
+
+  /// The init byte portion of cxa_guard_abort.
+  void abort() {
+    bool has_waiting;
+    {
+      LockGuard g("__cxa_guard_abort");
+      if (has_thread_id_support)
+        *thread_id_address = 0;
+      has_waiting = *init_byte_address & WAITING_BIT;
+      *init_byte_address = UNSET;
+    }
+    if (has_waiting) {
+      if (global_cond.broadcast()) {
+        ABORT_WITH_MESSAGE("%s failed to broadcast", "__cxa_guard_abort");
+      }
+    }
+  }
+
+private:
+  /// The address of the byte used during initialization.
+  uint8_t* const init_byte_address;
+  /// An optional address storing an identifier for the thread performing initialization.
+  /// It's used to detect recursive initialization.
+  uint32_t* const thread_id_address;
+
+  const bool has_thread_id_support;
+  LazyValue<uint32_t, GetThreadID> current_thread_id;
+
+private:
+  struct LockGuard {
+    LockGuard() = delete;
+    LockGuard(LockGuard const&) = delete;
+    LockGuard& operator=(LockGuard const&) = delete;
+
+    explicit LockGuard(const char* calling_func) : calling_func_(calling_func) {
+      if (global_mutex.lock())
+        ABORT_WITH_MESSAGE("%s failed to acquire mutex", calling_func_);
+    }
+
+    ~LockGuard() {
+      if (global_mutex.unlock())
+        ABORT_WITH_MESSAGE("%s failed to release mutex", calling_func_);
+    }
+
+  private:
+    const char* const calling_func_;
+  };
+};
+
+//===----------------------------------------------------------------------===//
+//                         Futex Implementation
+//===----------------------------------------------------------------------===//
+
+#if defined(SYS_futex)
+void PlatformFutexWait(int* addr, int expect) {
+  constexpr int WAIT = 0;
+  syscall(SYS_futex, addr, WAIT, expect, 0);
+  __tsan_acquire(addr);
+}
+void PlatformFutexWake(int* addr) {
+  constexpr int WAKE = 1;
+  __tsan_release(addr);
+  syscall(SYS_futex, addr, WAKE, INT_MAX);
+}
+#else
+constexpr void (*PlatformFutexWait)(int*, int) = nullptr;
+constexpr void (*PlatformFutexWake)(int*) = nullptr;
+#endif
+
+constexpr bool PlatformSupportsFutex() { return +PlatformFutexWait != nullptr; }
+
+/// InitByteFutex - Uses a futex to manage reads and writes to the init byte.
+template <void (*Wait)(int*, int) = PlatformFutexWait, void (*Wake)(int*) = PlatformFutexWake,
+          uint32_t (*GetThreadIDArg)() = PlatformThreadID>
+struct InitByteFutex {
+
+  explicit InitByteFutex(uint8_t* _init_byte_address, uint32_t* _thread_id_address)
+      : init_byte(_init_byte_address),
+        has_thread_id_support(_thread_id_address != nullptr && GetThreadIDArg != nullptr),
+        thread_id(_thread_id_address),
+        base_address(reinterpret_cast<int*>(/*_init_byte_address & ~0x3*/ _init_byte_address - 1)) {}
+
+public:
+  /// The init byte portion of cxa_guard_acquire. Returns true if
+  /// initialization has already been completed.
+  bool acquire() {
+    while (true) {
+      uint8_t last_val = UNSET;
+      if (init_byte.compare_exchange(&last_val, PENDING_BIT, std::_AO_Acq_Rel, std::_AO_Acquire)) {
+        if (has_thread_id_support) {
+          thread_id.store(current_thread_id.get(), std::_AO_Relaxed);
+        }
+        return false;
+      }
+
+      if (last_val == COMPLETE_BIT)
+        return true;
+
+      if (last_val & PENDING_BIT) {
+
+        // Check for recursive initialization
+        if (has_thread_id_support && thread_id.load(std::_AO_Relaxed) == current_thread_id.get()) {
+          ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization: do you have a function-local static variable whose initialization depends on that function?");
+        }
+
+        if ((last_val & WAITING_BIT) == 0) {
+          // This compare exchange can fail for several reasons
+          // (1) another thread finished the whole thing before we got here
+          // (2) another thread set the waiting bit we were trying to thread
+          // (3) another thread had an exception and failed to finish
+          if (!init_byte.compare_exchange(&last_val, PENDING_BIT | WAITING_BIT, std::_AO_Acq_Rel, std::_AO_Release)) {
+            // (1) success, via someone else's work!
+            if (last_val == COMPLETE_BIT)
+              return true;
+
+            // (3) someone else, bailed on doing the work, retry from the start!
+            if (last_val == UNSET)
+              continue;
+
+            // (2) the waiting bit got set, so we are happy to keep waiting
+          }
+        }
+        wait_on_initialization();
+      }
+    }
+  }
+
+  /// The init byte portion of cxa_guard_release.
+  void release() {
+    uint8_t old = init_byte.exchange(COMPLETE_BIT, std::_AO_Acq_Rel);
+    if (old & WAITING_BIT)
+      wake_all();
+  }
+
+  /// The init byte portion of cxa_guard_abort.
+  void abort() {
+    if (has_thread_id_support)
+      thread_id.store(0, std::_AO_Relaxed);
+
+    uint8_t old = init_byte.exchange(UNSET, std::_AO_Acq_Rel);
+    if (old & WAITING_BIT)
+      wake_all();
+  }
+
+private:
+  /// Use the futex to wait on the current guard variable. Futex expects a
+  /// 32-bit 4-byte aligned address as the first argument, so we use the 4-byte
+  /// aligned address that encompasses the init byte (i.e. the address of the
+  /// raw guard object that was passed to __cxa_guard_acquire/release/abort).
+  void wait_on_initialization() { Wait(base_address, expected_value_for_futex(PENDING_BIT | WAITING_BIT)); }
+  void wake_all() { Wake(base_address); }
+
+private:
+  AtomicInt<uint8_t> init_byte;
+
+  const bool has_thread_id_support;
+  // Unsafe to use unless has_thread_id_support
+  AtomicInt<uint32_t> thread_id;
+  LazyValue<uint32_t, GetThreadIDArg> current_thread_id;
+
+  /// the 4-byte-aligned address that encompasses the init byte (i.e. the
+  /// address of the raw guard object).
+  int* const base_address;
+
+  /// Create the expected integer value for futex `wait(int* addr, int expected)`.
+  /// We pass the base address as the first argument, So this function creates
+  /// an zero-initialized integer  with `b` copied at the correct offset.
+  static int expected_value_for_futex(uint8_t b) {
+    int dest_val = 0;
+    std::memcpy(reinterpret_cast<char*>(&dest_val) + 1, &b, 1);
+    return dest_val;
+  }
+
+  static_assert(Wait != nullptr && Wake != nullptr, "");
+};
+
+//===----------------------------------------------------------------------===//
+//                          GuardObject
+//===----------------------------------------------------------------------===//
+
+enum class AcquireResult {
+  INIT_IS_DONE,
+  INIT_IS_PENDING,
+};
+constexpr AcquireResult INIT_IS_DONE = AcquireResult::INIT_IS_DONE;
+constexpr AcquireResult INIT_IS_PENDING = AcquireResult::INIT_IS_PENDING;
+
+/// Co-ordinates between GuardByte and InitByte.
+template <class InitByteT>
+struct GuardObject {
+  GuardObject() = delete;
+  GuardObject(GuardObject const&) = delete;
+  GuardObject& operator=(GuardObject const&) = delete;
+
+private:
+  GuardByte guard_byte;
+  InitByteT init_byte;
+
+public:
+  /// ARM Constructor
+  explicit GuardObject(uint32_t* raw_guard_object)
+      : guard_byte(reinterpret_cast<uint8_t*>(raw_guard_object)),
+        init_byte(reinterpret_cast<uint8_t*>(raw_guard_object) + 1, nullptr) {}
+
+  /// Itanium Constructor
+  explicit GuardObject(uint64_t* raw_guard_object)
+      : guard_byte(reinterpret_cast<uint8_t*>(raw_guard_object)),
+        init_byte(reinterpret_cast<uint8_t*>(raw_guard_object) + 1, reinterpret_cast<uint32_t*>(raw_guard_object) + 1) {
+  }
+
+  /// Implements __cxa_guard_acquire.
+  AcquireResult cxa_guard_acquire() {
+    // Use short-circuit evaluation to avoid calling init_byte.acquire when
+    // guard_byte.acquire returns true. (i.e. don't call it when we know from
+    // the guard byte that initialization has already been completed)
+    if (guard_byte.acquire() || init_byte.acquire())
+      return INIT_IS_DONE;
+    return INIT_IS_PENDING;
+  }
+
+  /// Implements __cxa_guard_release.
+  void cxa_guard_release() {
+    // Update guard byte first, so if somebody is woken up by init_byte.release
+    // and comes all the way back around to __cxa_guard_acquire again, they see
+    // it as having completed initialization.
+    guard_byte.release();
+    init_byte.release();
+  }
+
+  /// Implements __cxa_guard_abort.
+  void cxa_guard_abort() {
+    guard_byte.abort();
+    init_byte.abort();
+  }
+};
+
+//===----------------------------------------------------------------------===//
+//                          Convenience Classes
+//===----------------------------------------------------------------------===//
+
+/// NoThreadsGuard - Manages initialization without performing any inter-thread
+/// synchronization.
+using NoThreadsGuard = GuardObject<InitByteNoThreads>;
+
+/// GlobalMutexGuard - Manages initialization using a global mutex and
+/// condition variable.
+template <class Mutex, class CondVar, Mutex& global_mutex, CondVar& global_cond,
+          uint32_t (*GetThreadID)() = PlatformThreadID>
+using GlobalMutexGuard = GuardObject<InitByteGlobalMutex<Mutex, CondVar, global_mutex, global_cond, GetThreadID>>;
+
+/// FutexGuard - Manages initialization using atomics and the futex syscall for
+/// waiting and waking.
+template <void (*Wait)(int*, int) = PlatformFutexWait, void (*Wake)(int*) = PlatformFutexWake,
+          uint32_t (*GetThreadIDArg)() = PlatformThreadID>
+using FutexGuard = GuardObject<InitByteFutex<Wait, Wake, GetThreadIDArg>>;
+
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+template <class T>
+struct GlobalStatic {
+  static T instance;
+};
+template <class T>
+_LIBCPP_CONSTINIT T GlobalStatic<T>::instance = {};
+
+enum class Implementation { NoThreads, GlobalMutex, Futex };
+
+template <Implementation Impl>
+struct SelectImplementation;
+
+template <>
+struct SelectImplementation<Implementation::NoThreads> {
+  using type = NoThreadsGuard;
+};
+
+template <>
+struct SelectImplementation<Implementation::GlobalMutex> {
+  using type = GlobalMutexGuard<LibcppMutex, LibcppCondVar, GlobalStatic<LibcppMutex>::instance,
+                                GlobalStatic<LibcppCondVar>::instance, PlatformThreadID>;
+};
+
+template <>
+struct SelectImplementation<Implementation::Futex> {
+  using type = FutexGuard<PlatformFutexWait, PlatformFutexWake, PlatformThreadID>;
+};
+
+// TODO(EricWF): We should prefer the futex implementation when available. But
+// it should be done in a separate step from adding the implementation.
+constexpr Implementation CurrentImplementation =
+#if defined(_LIBCXXABI_HAS_NO_THREADS)
+    Implementation::NoThreads;
+#elif defined(_LIBCXXABI_USE_FUTEX)
+    Implementation::Futex;
+#else
+    Implementation::GlobalMutex;
+#endif
+
+static_assert(CurrentImplementation != Implementation::Futex || PlatformSupportsFutex(),
+              "Futex selected but not supported");
+
+using SelectedImplementation = SelectImplementation<CurrentImplementation>::type;
+
+} // end namespace
+} // end namespace __cxxabiv1
+
+#if defined(__clang__)
+#  pragma clang diagnostic pop
+#elif defined(__GNUC__)
+#  pragma GCC diagnostic pop
+#endif
+
+#endif // LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H