More ctl::string optimization (#1232)

Moves some isbig checks into string.h, enabling smarter optimizations to
be made on small strings. Also we no longer zero out our string prior to
calling the various constructors, buying back the performance we lost on
big strings when we made the small-string optimization. We further add a
little optimization to the big_string copy constructor: if the string is
using half or more of its capacity, then we don’t recompute capacity and
just take the old string’s. As well, the copy constructor always makes a
small string when it will fit, even if copied from a big string that got
truncated.

This also reworks the test to follow the idiom adopted elsewhere re stl,
and adds a helper function to tell if a string is small based on data().

ctl/string.cc

@@ -23,45 +23,66 @@
 
 namespace ctl {
 
-string::~string() noexcept
+void
+string::destroy_big() noexcept
 {
-    if (isbig()) {
-        auto* b = big();
-        if (b->n) {
-            if (b->n >= b->c)
-                __builtin_trap();
-            if (b->p[b->n])
-                __builtin_trap();
-        }
-        if (b->c && !b->p)
+    auto* b = big();
+    if (b->n) {
+        if (b->n >= b->c)
             __builtin_trap();
-        free(b->p);
+        if (b->p[b->n])
+            __builtin_trap();
+    }
+    if (b->c && !b->p)
+        __builtin_trap();
+    free(b->p);
+}
+
+void
+string::init_big(const string& s) noexcept
+{
+    char* p2;
+#ifndef NDEBUG
+    if (!s.isbig())
+        __builtin_trap();
+#endif
+    if (s.size() >= s.capacity() >> 1) {
+        if (!(p2 = (char*)malloc(s.capacity())))
+            __builtin_trap();
+        set_big_string(p2, s.size(), s.capacity());
+    } else {
+        init_big(string_view(s));
     }
 }
 
-string::string(const char* s) noexcept : string()
+void
+string::init_big(const string_view s) noexcept
 {
-    append(s, strlen(s));
+    size_t need;
+    char* p2;
+    if (ckd_add(&need, s.n, 1 /* nul */ + 15))
+        __builtin_trap();
+    need &= -16;
+    if (!(p2 = (char*)malloc(need)))
+        __builtin_trap();
+    memcpy(p2, s.p, s.n);
+    p2[s.n] = 0;
+    set_big_string(p2, s.n, need);
 }
 
-string::string(const string& s) noexcept : string()
+void
+string::init_big(const size_t n, const char ch) noexcept
 {
-    append(s.data(), s.size());
-}
-
-string::string(const string_view s) noexcept : string()
-{
-    append(s.p, s.n);
-}
-
-string::string(const size_t size, const char ch) noexcept : string()
-{
-    resize(size, ch);
-}
-
-string::string(const char* s, const size_t size) noexcept : string()
-{
-    append(s, size);
+    size_t need;
+    char* p2;
+    if (ckd_add(&need, n, 1 /* nul */ + 15))
+        __builtin_trap();
+    need &= -16;
+    if (!(p2 = (char*)malloc(need)))
+        __builtin_trap();
+    memset(p2, ch, n);
+    p2[n] = 0;
+    set_big_string(p2, n, need);
 }
 
 const char*

ctl/string.h

@@ -48,12 +48,60 @@ class string
     using const_iterator = const char*;
     static constexpr size_t npos = -1;
 
-    ~string() /* noexcept */;
-    string(string_view) noexcept;
-    string(const char*) noexcept;
-    string(const string&) noexcept;
-    string(const char*, size_t) noexcept;
-    explicit string(size_t, char = 0) noexcept;
+    string() noexcept
+    {
+        __builtin_memset(blob, 0, sizeof(size_t) * 2);
+        // equivalent to set_small_size(0) but also zeroes memory
+        *(((size_t*)blob) + 2) = __::sso_max << (sizeof(size_t) - 1) * 8;
+    }
+
+    string(const string_view s) noexcept
+    {
+        if (s.n <= __::sso_max) {
+            __builtin_memcpy(blob, s.p, s.n);
+            __builtin_memset(blob + s.n, 0, __::sso_max - s.n);
+            set_small_size(s.n);
+        } else {
+            init_big(s);
+        }
+    }
+
+    explicit string(const size_t n, const char ch = 0) noexcept
+    {
+        if (n <= __::sso_max) {
+            __builtin_memset(blob, ch, n);
+            __builtin_memset(blob + n, 0, __::sso_max - n);
+            set_small_size(n);
+        } else {
+            init_big(n, ch);
+        }
+    }
+
+    string(const char* const p) noexcept
+      : string(string_view(p, __builtin_strlen(p)))
+    {
+    }
+
+    string(const string& r) noexcept
+    {
+        if (r.size() <= __::sso_max) {
+            __builtin_memcpy(blob, r.data(), __::string_size);
+            set_small_size(r.size());
+        } else {
+            init_big(r);
+        }
+    }
+
+    string(const char* const p, const size_t n) noexcept
+      : string(string_view(p, n))
+    {
+    }
+
+    ~string() /* noexcept */
+    {
+        if (isbig())
+            destroy_big();
+    }
 
     string& operator=(string) noexcept;
     const char* c_str() const noexcept;
@@ -78,13 +126,6 @@ class string
     size_t find(char, size_t = 0) const noexcept;
     size_t find(string_view, size_t = 0) const noexcept;
 
-    string() noexcept
-    {
-        __builtin_memset(blob, 0, sizeof(size_t) * 2);
-        // equivalent to set_small_size(0) but also zeroes memory
-        *(((size_t*)blob) + 2) = __::sso_max << (sizeof(size_t) - 1) * 8;
-    }
-
     void swap(string& s) noexcept
     {
         ctl::swap(blob, s.blob);
@@ -110,17 +151,17 @@ class string
         return isbig() ? !big()->n : small()->rem >= __::sso_max;
     }
 
-    inline char* data() noexcept
+    char* data() noexcept
     {
         return isbig() ? big()->p : small()->buf;
     }
 
-    inline const char* data() const noexcept
+    const char* data() const noexcept
     {
         return isbig() ? big()->p : small()->buf;
     }
 
-    inline size_t size() const noexcept
+    size_t size() const noexcept
     {
 #if 0
         if (!isbig() && small()->rem > __::sso_max)
@@ -215,7 +256,7 @@ class string
         append(s.p, s.n);
     }
 
-    inline operator string_view() const noexcept
+    operator string_view() const noexcept
     {
         return string_view(data(), size());
     }
@@ -277,63 +318,68 @@ class string
     }
 
   private:
-    inline bool isbig() const noexcept
+    void destroy_big() noexcept;
+    void init_big(const string&) noexcept;
+    void init_big(string_view) noexcept;
+    void init_big(size_t, char) noexcept;
+
+    bool isbig() const noexcept
     {
         return *(blob + __::sso_max) & 0x80;
     }
 
-    inline void set_small_size(const size_t size) noexcept
+    void set_small_size(const size_t size) noexcept
     {
         if (size > __::sso_max)
             __builtin_trap();
-        *(blob + __::sso_max) = (__::sso_max - size);
+        __s.rem = __::sso_max - size;
     }
 
-    inline void set_big_string(char* const p,
-                               const size_t n,
-                               const size_t c2) noexcept
+    void set_big_string(char* const p, const size_t n, const size_t c2) noexcept
     {
         if (c2 > __::big_mask)
             __builtin_trap();
-        *(char**)blob = p;
-        *(((size_t*)blob) + 1) = n;
-        *(((size_t*)blob) + 2) = c2 | ~__::big_mask;
+        __b.p = p;
+        __b.n = n;
+        __b.c = c2 | ~__::big_mask;
     }
 
-    inline __::small_string* small() noexcept
+    __::small_string* small() noexcept
     {
         if (isbig())
             __builtin_trap();
-        return reinterpret_cast<__::small_string*>(blob);
+        return &__s;
     }
 
-    inline const __::small_string* small() const noexcept
+    const __::small_string* small() const noexcept
     {
         if (isbig())
             __builtin_trap();
-        return reinterpret_cast<const __::small_string*>(blob);
+        return &__s;
     }
 
-    inline __::big_string* big() noexcept
+    __::big_string* big() noexcept
     {
         if (!isbig())
             __builtin_trap();
-        return reinterpret_cast<__::big_string*>(blob);
+        return &__b;
     }
 
-    inline const __::big_string* big() const noexcept
+    const __::big_string* big() const noexcept
     {
         if (!isbig())
             __builtin_trap();
-        return reinterpret_cast<const __::big_string*>(blob);
+        return &__b;
     }
 
-    friend string strcat(string_view, string_view);
+    friend string strcat(string_view, string_view) noexcept;
 
-    alignas(union {
-        __::big_string a;
-        __::small_string b;
-    }) char blob[__::string_size];
+    union
+    {
+        __::big_string __b;
+        __::small_string __s;
+        char blob[__::string_size];
+    };
 };
 
 static_assert(sizeof(string) == __::string_size);

test/ctl/string_test.cc

@@ -23,9 +23,19 @@
 #include "libc/runtime/runtime.h"
 #include "libc/str/str.h"
 
-using String = ctl::string;
 // #include <string>
-// using String = std::string;
+// #define ctl std
+
+using String = ctl::string;
+
+#undef ctl
+
+inline bool
+issmall(const String& s)
+{
+    return s.capacity() == sizeof(s) &&
+           s.data() == reinterpret_cast<const char*>(&s);
+}
 
 int
 main()
@@ -358,15 +368,14 @@ main()
         String s;
         if constexpr (std::is_same_v<ctl::string, decltype(s)>) {
             // tests the small-string optimization on ctl::string
-            char* d = s.data();
             for (int i = 0; i < 23; ++i) {
                 s.append("a");
-                if (s.data() != d) {
+                if (!issmall(s)) {
                     return 79 + i;
                 }
             }
             s.append("a");
-            if (s.data() == d) {
+            if (issmall(s)) {
                 return 103;
             }
         } else {
@@ -380,6 +389,21 @@ main()
         }
     }
 
+    {
+        String s("arst", 4);
+        for (int i = 0; i < 30; ++i) {
+            s.append("a");
+        }
+        s.resize(4);
+        if (s != "arst")
+            return 105;
+        if constexpr (std::is_same_v<ctl::string, decltype(s)>) {
+            String r(s);
+            if (issmall(s) || !issmall(r))
+                return 106;
+        }
+    }
+
     CheckForMemoryLeaks();
     return 0;
 }