cosmopolitan/libc/intrin/memcmp.c

/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney                              │
│                                                                              │
│ Permission to use, copy, modify, and/or distribute this software for         │
│ any purpose with or without fee is hereby granted, provided that the         │
│ above copyright notice and this permission notice appear in all copies.      │
│                                                                              │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
│ PERFORMANCE OF THIS SOFTWARE.                                                │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/dce.h"
#include "libc/intrin/likely.h"
#include "libc/nexgen32e/x86feature.h"
#include "libc/str/str.h"

#define PMOVMSKB(x) __builtin_ia32_pmovmskb128(x)

typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(1)));

#if defined(__x86_64__) && !defined(__chibicc__)

static dontinline antiquity int memcmp_sse(const unsigned char *p,
                                           const unsigned char *q, size_t n) {
  unsigned u;
  if (n > 32) {
    while (n > 16 + 16) {
      if (!(u = PMOVMSKB(*(xmm_t *)p == *(xmm_t *)q) ^ 0xffff)) {
        n -= 16;
        p += 16;
        q += 16;
      } else {
        u = __builtin_ctzl(u);
        return p[u] - q[u];
      }
    }
  }
  if (!(u = PMOVMSKB(*(xmm_t *)p == *(xmm_t *)q) ^ 0xffff)) {
    if (!(u = PMOVMSKB(*(xmm_t *)(p + n - 16) == *(xmm_t *)(q + n - 16)) ^
              0xffff)) {
      return 0;
    } else {
      u = __builtin_ctzl(u);
      return p[n - 16 + u] - q[n - 16 + u];
    }
  } else {
    u = __builtin_ctzl(u);
    return p[u] - q[u];
  }
}

_Microarchitecture("avx") static int memcmp_avx(const unsigned char *p,
                                                const unsigned char *q,
                                                size_t n) {
  uint64_t w;
  unsigned u;
  if (n > 32) {
    while (n >= 16 + 64) {
      w = (uint64_t)PMOVMSKB(((xmm_t *)p)[0] == ((xmm_t *)q)[0]) << 000 |
          (uint64_t)PMOVMSKB(((xmm_t *)p)[1] == ((xmm_t *)q)[1]) << 020 |
          (uint64_t)PMOVMSKB(((xmm_t *)p)[2] == ((xmm_t *)q)[2]) << 040 |
          (uint64_t)PMOVMSKB(((xmm_t *)p)[3] == ((xmm_t *)q)[3]) << 060;
      if (w == -1) {
        n -= 64;
        p += 64;
        q += 64;
      } else {
        w = __builtin_ctzll(w ^ -1);
        return p[w] - q[w];
      }
    }
    while (n > 16 + 16) {
      if (!(u = PMOVMSKB(*(xmm_t *)p == *(xmm_t *)q) ^ 0xffff)) {
        n -= 16;
        p += 16;
        q += 16;
      } else {
        u = __builtin_ctzl(u);
        return p[u] - q[u];
      }
    }
  }
  if (!(u = PMOVMSKB(*(xmm_t *)p == *(xmm_t *)q) ^ 0xffff)) {
    if (!(u = PMOVMSKB(*(xmm_t *)(p + n - 16) == *(xmm_t *)(q + n - 16)) ^
              0xffff)) {
      return 0;
    } else {
      u = __builtin_ctzl(u);
      return p[n - 16 + u] - q[n - 16 + u];
    }
  } else {
    u = __builtin_ctzl(u);
    return p[u] - q[u];
  }
}

#endif /* __x86_64__ */

/**
 * Compares memory byte by byte.
 *
 *     memcmp n=0                           2 nanoseconds
 *     memcmp n=1                           2 ns/byte            357 mb/s
 *     memcmp n=2                           1 ns/byte            530 mb/s
 *     memcmp n=3                           1 ns/byte            631 mb/s
 *     𝗺𝗲𝗺𝗰𝗺𝗽 n=4                           1 ns/byte            849 mb/s
 *     memcmp n=5                         816 ps/byte          1,195 mb/s
 *     memcmp n=6                         888 ps/byte          1,098 mb/s
 *     memcmp n=7                         829 ps/byte          1,176 mb/s
 *     𝗺𝗲𝗺𝗰𝗺𝗽 n=8                         773 ps/byte          1,261 mb/s
 *     memcmp n=9                         629 ps/byte          1,551 mb/s
 *     memcmp n=15                        540 ps/byte          1,805 mb/s
 *     𝗺𝗲𝗺𝗰𝗺𝗽 n=16                        211 ps/byte          4,623 mb/s
 *     memcmp n=17                        268 ps/byte          3,633 mb/s
 *     memcmp n=31                        277 ps/byte          3,524 mb/s
 *     memcmp n=32                        153 ps/byte          6,351 mb/s
 *     memcmp n=33                        179 ps/byte          5,431 mb/s
 *     memcmp n=79                        148 ps/byte          6,576 mb/s
 *     𝗺𝗲𝗺𝗰𝗺𝗽 n=80                         81 ps/byte             11 GB/s
 *     memcmp n=128                        76 ps/byte             12 GB/s
 *     memcmp n=256                        60 ps/byte             15 GB/s
 *     memcmp n=16384                      51 ps/byte             18 GB/s
 *     memcmp n=32768                      51 ps/byte             18 GB/s
 *     memcmp n=131072                     52 ps/byte             18 GB/s
 *
 * @return an integer that's (1) equal to zero if `a` is equal to `b`,
 *     (2) less than zero if `a` is less than `b`, or (3) greater than
 *     zero if `a` is greater than `b`
 * @asyncsignalsafe
 */
int memcmp(const void *a, const void *b, size_t n) {
  int c;
#if defined(__x86_64__) && !defined(__chibicc__)
  unsigned u;
  uint32_t k, i, j;
  uint64_t w, x, y;
#endif
  const unsigned char *p, *q;
  if ((p = a) == (q = b) || !n)
    return 0;
  if ((c = *p - *q))
    return c;
#if defined(__x86_64__) && !defined(__chibicc__)
  if (!IsTiny()) {
    if (n <= 16) {
      if (n >= 8) {
        if (!(w = (x = ((uint64_t)p[0] << 000 | (uint64_t)p[1] << 010 |
                        (uint64_t)p[2] << 020 | (uint64_t)p[3] << 030 |
                        (uint64_t)p[4] << 040 | (uint64_t)p[5] << 050 |
                        (uint64_t)p[6] << 060 | (uint64_t)p[7] << 070)) ^
                  (y = ((uint64_t)q[0] << 000 | (uint64_t)q[1] << 010 |
                        (uint64_t)q[2] << 020 | (uint64_t)q[3] << 030 |
                        (uint64_t)q[4] << 040 | (uint64_t)q[5] << 050 |
                        (uint64_t)q[6] << 060 | (uint64_t)q[7] << 070)))) {
          p += n - 8;
          q += n - 8;
          if (!(w = (x = ((uint64_t)p[0] << 000 | (uint64_t)p[1] << 010 |
                          (uint64_t)p[2] << 020 | (uint64_t)p[3] << 030 |
                          (uint64_t)p[4] << 040 | (uint64_t)p[5] << 050 |
                          (uint64_t)p[6] << 060 | (uint64_t)p[7] << 070)) ^
                    (y = ((uint64_t)q[0] << 000 | (uint64_t)q[1] << 010 |
                          (uint64_t)q[2] << 020 | (uint64_t)q[3] << 030 |
                          (uint64_t)q[4] << 040 | (uint64_t)q[5] << 050 |
                          (uint64_t)q[6] << 060 | (uint64_t)q[7] << 070)))) {
            return 0;
          }
        }
        u = __builtin_ctzll(w);
        u = u & -8;
        return ((x >> u) & 255) - ((y >> u) & 255);
      } else if (n >= 4) {
        if (!(k = (i = ((uint32_t)p[0] << 000 | (uint32_t)p[1] << 010 |
                        (uint32_t)p[2] << 020 | (uint32_t)p[3] << 030)) ^
                  (j = ((uint32_t)q[0] << 000 | (uint32_t)q[1] << 010 |
                        (uint32_t)q[2] << 020 | (uint32_t)q[3] << 030)))) {
          p += n - 4;
          q += n - 4;
          if (!(k = (i = ((uint32_t)p[0] << 000 | (uint32_t)p[1] << 010 |
                          (uint32_t)p[2] << 020 | (uint32_t)p[3] << 030)) ^
                    (j = ((uint32_t)q[0] << 000 | (uint32_t)q[1] << 010 |
                          (uint32_t)q[2] << 020 | (uint32_t)q[3] << 030)))) {
            return 0;
          }
        }
        u = __builtin_ctzl(k);
        u = u & -8;
        return ((i >> u) & 255) - ((j >> u) & 255);
      }
    } else if (LIKELY(X86_HAVE(AVX))) {
      return memcmp_avx(p, q, n);
    } else {
      return memcmp_sse(p, q, n);
    }
  }
#endif /* __x86_64__ */
  for (; n; ++p, ++q, --n) {
    if ((c = *p - *q)) {
      return c;
    }
  }
  return 0;
}