Make mkdeps.com go faster

This program usually runs once at the begininng of each GNU Make
invocation. It generates an o//depend file with 170,000 lines of
Makefile code to define source -> headers relationships.

This change makes that take 650 milliseconds rather than 1,100ms
by improving the performance of strstr(), using longsort(), plus
migrating to the new append library.

libc/str/strstr.c

@@ -16,34 +16,55 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/dce.h"
+#include "libc/intrin/asan.internal.h"
 #include "libc/str/str.h"
 
+typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(16)));
+
 /**
  * Searches for substring.
  *
  * @param haystack is the search area, as a NUL-terminated string
  * @param needle is the desired substring, also NUL-terminated
  * @return pointer to first substring within haystack, or NULL
+ * @note this implementation goes fast in practice but isn't hardened
+ *     against pathological cases, and therefore shouldn't be used on
+ *     untrustworthy data
  * @asyncsignalsafe
  * @see memmem()
  */
-char *strstr(const char *haystack, const char *needle) {
+noasan char *strstr(const char *haystack, const char *needle) {
+  xmm_t *p;
   size_t i;
-  const char *p;
+  unsigned k, m;
-  if (!needle[0]) return haystack;
+  xmm_t v, n, z = {0};
-  if (haystack == needle) return haystack;
+  if (IsAsan()) __asan_verify(needle, 1);
-  p = strchr(haystack, needle[0]);
+  if (IsAsan()) __asan_verify(haystack, 1);
-  if (!needle[1]) return p;
+  if (haystack == needle || !*needle) return haystack;
-  if (p) haystack = p;
+  n = (xmm_t){*needle, *needle, *needle, *needle, *needle, *needle,
-  /* TODO: make not quadratic */
+              *needle, *needle, *needle, *needle, *needle, *needle,
+              *needle, *needle, *needle, *needle};
   for (;;) {
-    for (i = 0;;) {
+    k = (uintptr_t)haystack & 15;
+    p = (const xmm_t *)((uintptr_t)haystack & -16);
+    v = *p;
+    m = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(v, z) |
+                                   __builtin_ia32_pcmpeqb128(v, n));
+    m >>= k;
+    m <<= k;
+    while (!m) {
+      v = *++p;
+      m = __builtin_ia32_pmovmskb128(__builtin_ia32_pcmpeqb128(v, z) |
+                                     __builtin_ia32_pcmpeqb128(v, n));
+    }
+    haystack = (const char *)p + __builtin_ctzl(m);
+    for (i = 0;; ++i) {
       if (!needle[i]) return (/*unconst*/ char *)haystack;
       if (!haystack[i]) break;
       if (needle[i] != haystack[i]) break;
-      ++i;
     }
     if (!*haystack++) break;
   }
-  return NULL;
+  return 0;
 }

third_party/python/Lib/test/test_compile.py

@@ -546,6 +546,7 @@ if 1:
         self.assertIn(b"Non-UTF-8", res.err)
 
     @support.cpython_only
+    @unittest.skipUnless(cosmo.MODE == "dbg", "disabled recursion checking")
     def test_compiler_recursion_limit(self):
         # Expected limit is sys.getrecursionlimit() * the scaling factor
         # in symtable.c (currently 3)

tool/build/build.mk

@@ -110,7 +110,7 @@ o/$(MODE)/tool/build/mkdeps.com.dbg:			\
 		o/$(MODE)/tool/build/mkdeps.o		\
 		$(CRT)					\
 		$(APE)
-	-@$(APELINK)
+	@$(APELINK)
 
 o/$(MODE)/tool/build/compile.com.dbg:			\
 		$(TOOL_BUILD_DEPS)			\

tool/build/mkdeps.c

@@ -25,15 +25,18 @@
 #include "libc/bits/safemacros.internal.h"
 #include "libc/calls/calls.h"
 #include "libc/calls/struct/stat.h"
+#include "libc/dce.h"
 #include "libc/errno.h"
 #include "libc/fmt/fmt.h"
 #include "libc/log/check.h"
 #include "libc/log/log.h"
 #include "libc/macros.internal.h"
+#include "libc/mem/mem.h"
 #include "libc/nexgen32e/crc32.h"
 #include "libc/runtime/ezmap.internal.h"
 #include "libc/runtime/gc.internal.h"
 #include "libc/runtime/runtime.h"
+#include "libc/stdio/append.internal.h"
 #include "libc/stdio/stdio.h"
 #include "libc/str/str.h"
 #include "libc/sysv/consts/madv.h"
@@ -82,19 +85,24 @@ struct Strings {
 };
 
 struct Source {
-  unsigned hash; /* 0 means empty w/ triangle probe */
+  unsigned hash;
-  unsigned name; /* strings.p[name] w/ interning */
+  unsigned name;
-  unsigned id;   /* rehashing changes indexes */
+  unsigned id;
 };
 
 struct Edge {
-  unsigned from; /* sources.p[from.id] */
+  unsigned to;
-  unsigned to;   /* sources.p[to.id] */
+  unsigned from;
 };
 
 struct Sources {
-  size_t i, n;      /* phase 1: hashmap: popcnt(n)==1 if n */
+  size_t i, n;
-  struct Source *p; /* phase 2: arraylist sorted by id */
+  struct Source *p;
+};
+
+struct Sauce {
+  unsigned name;
+  unsigned id;
 };
 
 struct Edges {
@@ -103,22 +111,15 @@ struct Edges {
 };
 
 char *out;
-FILE *fout;
+char *bout;
-int *visited;
 unsigned counter;
+uint32_t *visited;
 struct Edges edges;
+struct Sauce *sauces;
 struct Strings strings;
 struct Sources sources;
 const char *buildroot;
 
-int CompareSourcesById(struct Source *a, struct Source *b) {
-  return a->id > b->id ? 1 : a->id < b->id ? -1 : 0;
-}
-
-int CompareEdgesByFrom(struct Edge *a, struct Edge *b) {
-  return a->from > b->from ? 1 : a->from < b->from ? -1 : 0;
-}
-
 unsigned Hash(const void *s, size_t l) {
   return max(1, crc32c(0, s, l));
 }
@@ -140,14 +141,19 @@ unsigned FindFirstFromEdge(unsigned id) {
 
 void Crunch(void) {
   size_t i, j;
-  for (i = 0, j = 0; j < sources.n; ++j) {
+  sauces = malloc(sizeof(*sauces) * sources.n);
-    if (!sources.p[j].hash) continue;
+  for (j = i = 0; i < sources.n; ++i) {
-    if (i != j) memcpy(&sources.p[i], &sources.p[j], sizeof(sources.p[j]));
+    if (sources.p[i].hash) {
-    i++;
+      sauces[j].name = sources.p[i].name;
+      sauces[j].id = sources.p[i].id;
+      j++;
     }
-  sources.i = i;
+  }
-  qsort(sources.p, sources.i, sizeof(*sources.p), (void *)CompareSourcesById);
+  free(sources.p);
-  qsort(edges.p, edges.i, sizeof(*edges.p), (void *)CompareEdgesByFrom);
+  sources.p = 0;
+  sources.i = j;
+  longsort((const long *)sauces, sources.i);
+  longsort((const long *)edges.p, edges.i);
 }
 
 void Rehash(void) {
@@ -163,7 +169,7 @@ void Rehash(void) {
       j = (old.p[i].hash + step * (step + 1) / 2) & (sources.n - 1);
       step++;
     } while (sources.p[j].hash);
-    memcpy(&sources.p[j], &old.p[i], sizeof(old.p[i]));
+    sources.p[j] = old.p[i];
   }
   free(old.p);
 }
@@ -315,12 +321,21 @@ bool IsObjectSource(const char *name) {
   return false;
 }
 
+forceinline bool Bts(uint32_t *p, size_t i) {
+  bool r;
+  uint32_t k;
+  k = 1u << (i & 31);
+  if (p[i >> 5] & k) return true;
+  p[i >> 5] |= k;
+  return false;
+}
+
 void Dive(unsigned id) {
   int i;
   for (i = FindFirstFromEdge(id); i < edges.i && edges.p[i].from == id; ++i) {
-    if (bts(visited, edges.p[i].to)) continue;
+    if (Bts(visited, edges.p[i].to)) continue;
-    fputs(" \\\n\t", fout);
+    appendw(&bout, READ32LE(" \\\n\t"));
-    fputs(&strings.p[sources.p[edges.p[i].to].name], fout);
+    appends(&bout, strings.p + sauces[edges.p[i].to].name);
     Dive(edges.p[i].to);
   }
 }
@@ -332,31 +347,22 @@ size_t GetFileSizeOrZero(const char *path) {
   return st.st_size;
 }
 
-bool FilesHaveSameContent(const char *path1, const char *path2) {
+/* prevents gnu make from restarting unless necessary */
+bool HasSameContent(void) {
   bool r;
-  int c1, c2;
+  int fd;
-  size_t s1, s2;
+  char *m;
-  FILE *f1, *f2;
+  size_t s;
-  s1 = GetFileSizeOrZero(path1);
+  s = GetFileSizeOrZero(out);
-  s2 = GetFileSizeOrZero(path2);
+  if (s == appendz(bout).i) {
-  if (s1 == s2) {
+    if (s) {
+      CHECK_NE(-1, (fd = open(out, O_RDONLY)));
+      CHECK_NE(MAP_FAILED, (m = mmap(0, s, PROT_READ, MAP_SHARED, fd, 0)));
+      r = !bcmp(bout, m, s);
+      munmap(m, s);
+      close(fd);
+    } else {
       r = true;
-    if (s1) {
-      CHECK_NOTNULL((f1 = fopen(path1, "r")));
-      CHECK_NOTNULL((f2 = fopen(path2, "r")));
-      for (;;) {
-        c1 = getc(f1);
-        c2 = getc(f2);
-        if (c1 != c2) {
-          r = false;
-          break;
-        }
-        if (c1 == -1) {
-          break;
-        }
-      }
-      CHECK_NE(-1, fclose(f2));
-      CHECK_NE(-1, fclose(f1));
     }
   } else {
     r = false;
@@ -365,44 +371,42 @@ bool FilesHaveSameContent(const char *path1, const char *path2) {
 }
 
 int main(int argc, char *argv[]) {
-  char *tp;
+  int fd;
-  bool needprefix;
+  const char *path;
-  size_t i, bitmaplen;
+  size_t i, visilen;
-  const char *path, *prefix;
   if (argc == 2 && !strcmp(argv[1], "-n")) exit(0);
-  showcrashreports();
   out = "/dev/stdout";
   GetOpts(argc, argv);
-  tp = !fileexists(out) || isregularfile(out) ? xasprintf("%s.tmp", out) : NULL;
-  CHECK_NOTNULL((fout = fopen(tp ? tp : out, "w")));
   LoadRelationships(argc, argv);
   Crunch();
-  bitmaplen = roundup((sources.i + 8) / 8, 4);
+  visilen = (sources.i + sizeof(*visited) * CHAR_BIT - 1) /
-  visited = malloc(bitmaplen);
+            (sizeof(*visited) * CHAR_BIT);
+  visited = malloc(visilen * sizeof(*visited));
   for (i = 0; i < sources.i; ++i) {
-    path = &strings.p[sources.p[i].name];
+    path = strings.p + sauces[i].name;
     if (!IsObjectSource(path)) continue;
-    needprefix = !startswith(path, "o/");
+    appendw(&bout, '\n');
-    prefix = !needprefix ? "" : buildroot;
+    if (!startswith(path, "o/")) {
-    fprintf(fout, "\n%s%s.o: \\\n\t%s", prefix, StripExt(path), path);
+      appends(&bout, buildroot);
-    bzero(visited, bitmaplen);
+    }
-    bts(visited, i);
+    appends(&bout, StripExt(path));
+    appendw(&bout, READ64LE(".o: \\\n\t"));
+    appends(&bout, path);
+    bzero(visited, visilen * sizeof(*visited));
+    Bts(visited, i);
     Dive(i);
-    fprintf(fout, "\n");
+    appendw(&bout, '\n');
   }
-  CHECK_NE(-1, fclose(fout));
+  /* if (!fileexists(out) || !HasSameContent()) { */
-  if (tp) {
+  CHECK_NE(-1, (fd = open(out, O_CREAT | O_WRONLY, 0644)));
-    /* prevent gnu make from restarting unless necessary */
+  CHECK_NE(-1, ftruncate(fd, appendz(bout).i));
-    if (!FilesHaveSameContent(tp, out)) {
+  CHECK_NE(-1, xwrite(fd, bout, appendz(bout).i));
-      CHECK_NE(-1, rename(tp, out));
+  CHECK_NE(-1, close(fd));
-    } else {
+  /* } */
-      CHECK_NE(-1, unlink(tp));
+  free(strings.p);
-    }
+  free(edges.p);
-  }
+  free(visited);
-  free_s(&strings.p);
+  free(sauces);
-  free_s(&sources.p);
+  free(bout);
-  free_s(&edges.p);
-  free_s(&visited);
-  free_s(&tp);
   return 0;
 }