From 7e780e57d4cd3c2950162841e1071cf68d2af6e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven=20Dee=20=28J=C5=8Dshin=29?= Date: Thu, 20 Jun 2024 11:52:12 -0700 Subject: [PATCH] More ctl::string optimization (#1232) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moves some isbig checks into string.h, enabling smarter optimizations to be made on small strings. Also we no longer zero out our string prior to calling the various constructors, buying back the performance we lost on big strings when we made the small-string optimization. We further add a little optimization to the big_string copy constructor: if the string is using half or more of its capacity, then we don’t recompute capacity and just take the old string’s. As well, the copy constructor always makes a small string when it will fit, even if copied from a big string that got truncated. This also reworks the test to follow the idiom adopted elsewhere re stl, and adds a helper function to tell if a string is small based on data(). --- ctl/string.cc | 81 ++++++++++++++++---------- ctl/string.h | 124 +++++++++++++++++++++++++++------------- test/ctl/string_test.cc | 34 +++++++++-- 3 files changed, 165 insertions(+), 74 deletions(-) diff --git a/ctl/string.cc b/ctl/string.cc index a7552e86e..4e292a188 100644 --- a/ctl/string.cc +++ b/ctl/string.cc @@ -23,45 +23,66 @@ namespace ctl { -string::~string() noexcept +void +string::destroy_big() noexcept { - if (isbig()) { - auto* b = big(); - if (b->n) { - if (b->n >= b->c) - __builtin_trap(); - if (b->p[b->n]) - __builtin_trap(); - } - if (b->c && !b->p) + auto* b = big(); + if (b->n) { + if (b->n >= b->c) __builtin_trap(); - free(b->p); + if (b->p[b->n]) + __builtin_trap(); + } + if (b->c && !b->p) + __builtin_trap(); + free(b->p); +} + +void +string::init_big(const string& s) noexcept +{ + char* p2; +#ifndef NDEBUG + if (!s.isbig()) + __builtin_trap(); +#endif + if (s.size() >= s.capacity() >> 1) { + if (!(p2 = (char*)malloc(s.capacity()))) + __builtin_trap(); + set_big_string(p2, s.size(), s.capacity()); + } else { + init_big(string_view(s)); } } -string::string(const char* s) noexcept : string() +void +string::init_big(const string_view s) noexcept { - append(s, strlen(s)); + size_t need; + char* p2; + if (ckd_add(&need, s.n, 1 /* nul */ + 15)) + __builtin_trap(); + need &= -16; + if (!(p2 = (char*)malloc(need))) + __builtin_trap(); + memcpy(p2, s.p, s.n); + p2[s.n] = 0; + set_big_string(p2, s.n, need); } -string::string(const string& s) noexcept : string() +void +string::init_big(const size_t n, const char ch) noexcept { - append(s.data(), s.size()); -} - -string::string(const string_view s) noexcept : string() -{ - append(s.p, s.n); -} - -string::string(const size_t size, const char ch) noexcept : string() -{ - resize(size, ch); -} - -string::string(const char* s, const size_t size) noexcept : string() -{ - append(s, size); + size_t need; + char* p2; + if (ckd_add(&need, n, 1 /* nul */ + 15)) + __builtin_trap(); + need &= -16; + if (!(p2 = (char*)malloc(need))) + __builtin_trap(); + memset(p2, ch, n); + p2[n] = 0; + set_big_string(p2, n, need); } const char* diff --git a/ctl/string.h b/ctl/string.h index 42fc14dc3..84aac204b 100644 --- a/ctl/string.h +++ b/ctl/string.h @@ -48,12 +48,60 @@ class string using const_iterator = const char*; static constexpr size_t npos = -1; - ~string() /* noexcept */; - string(string_view) noexcept; - string(const char*) noexcept; - string(const string&) noexcept; - string(const char*, size_t) noexcept; - explicit string(size_t, char = 0) noexcept; + string() noexcept + { + __builtin_memset(blob, 0, sizeof(size_t) * 2); + // equivalent to set_small_size(0) but also zeroes memory + *(((size_t*)blob) + 2) = __::sso_max << (sizeof(size_t) - 1) * 8; + } + + string(const string_view s) noexcept + { + if (s.n <= __::sso_max) { + __builtin_memcpy(blob, s.p, s.n); + __builtin_memset(blob + s.n, 0, __::sso_max - s.n); + set_small_size(s.n); + } else { + init_big(s); + } + } + + explicit string(const size_t n, const char ch = 0) noexcept + { + if (n <= __::sso_max) { + __builtin_memset(blob, ch, n); + __builtin_memset(blob + n, 0, __::sso_max - n); + set_small_size(n); + } else { + init_big(n, ch); + } + } + + string(const char* const p) noexcept + : string(string_view(p, __builtin_strlen(p))) + { + } + + string(const string& r) noexcept + { + if (r.size() <= __::sso_max) { + __builtin_memcpy(blob, r.data(), __::string_size); + set_small_size(r.size()); + } else { + init_big(r); + } + } + + string(const char* const p, const size_t n) noexcept + : string(string_view(p, n)) + { + } + + ~string() /* noexcept */ + { + if (isbig()) + destroy_big(); + } string& operator=(string) noexcept; const char* c_str() const noexcept; @@ -78,13 +126,6 @@ class string size_t find(char, size_t = 0) const noexcept; size_t find(string_view, size_t = 0) const noexcept; - string() noexcept - { - __builtin_memset(blob, 0, sizeof(size_t) * 2); - // equivalent to set_small_size(0) but also zeroes memory - *(((size_t*)blob) + 2) = __::sso_max << (sizeof(size_t) - 1) * 8; - } - void swap(string& s) noexcept { ctl::swap(blob, s.blob); @@ -110,17 +151,17 @@ class string return isbig() ? !big()->n : small()->rem >= __::sso_max; } - inline char* data() noexcept + char* data() noexcept { return isbig() ? big()->p : small()->buf; } - inline const char* data() const noexcept + const char* data() const noexcept { return isbig() ? big()->p : small()->buf; } - inline size_t size() const noexcept + size_t size() const noexcept { #if 0 if (!isbig() && small()->rem > __::sso_max) @@ -215,7 +256,7 @@ class string append(s.p, s.n); } - inline operator string_view() const noexcept + operator string_view() const noexcept { return string_view(data(), size()); } @@ -277,63 +318,68 @@ class string } private: - inline bool isbig() const noexcept + void destroy_big() noexcept; + void init_big(const string&) noexcept; + void init_big(string_view) noexcept; + void init_big(size_t, char) noexcept; + + bool isbig() const noexcept { return *(blob + __::sso_max) & 0x80; } - inline void set_small_size(const size_t size) noexcept + void set_small_size(const size_t size) noexcept { if (size > __::sso_max) __builtin_trap(); - *(blob + __::sso_max) = (__::sso_max - size); + __s.rem = __::sso_max - size; } - inline void set_big_string(char* const p, - const size_t n, - const size_t c2) noexcept + void set_big_string(char* const p, const size_t n, const size_t c2) noexcept { if (c2 > __::big_mask) __builtin_trap(); - *(char**)blob = p; - *(((size_t*)blob) + 1) = n; - *(((size_t*)blob) + 2) = c2 | ~__::big_mask; + __b.p = p; + __b.n = n; + __b.c = c2 | ~__::big_mask; } - inline __::small_string* small() noexcept + __::small_string* small() noexcept { if (isbig()) __builtin_trap(); - return reinterpret_cast<__::small_string*>(blob); + return &__s; } - inline const __::small_string* small() const noexcept + const __::small_string* small() const noexcept { if (isbig()) __builtin_trap(); - return reinterpret_cast(blob); + return &__s; } - inline __::big_string* big() noexcept + __::big_string* big() noexcept { if (!isbig()) __builtin_trap(); - return reinterpret_cast<__::big_string*>(blob); + return &__b; } - inline const __::big_string* big() const noexcept + const __::big_string* big() const noexcept { if (!isbig()) __builtin_trap(); - return reinterpret_cast(blob); + return &__b; } - friend string strcat(string_view, string_view); + friend string strcat(string_view, string_view) noexcept; - alignas(union { - __::big_string a; - __::small_string b; - }) char blob[__::string_size]; + union + { + __::big_string __b; + __::small_string __s; + char blob[__::string_size]; + }; }; static_assert(sizeof(string) == __::string_size); diff --git a/test/ctl/string_test.cc b/test/ctl/string_test.cc index 22cc0d6e9..67fcf625e 100644 --- a/test/ctl/string_test.cc +++ b/test/ctl/string_test.cc @@ -23,9 +23,19 @@ #include "libc/runtime/runtime.h" #include "libc/str/str.h" -using String = ctl::string; // #include -// using String = std::string; +// #define ctl std + +using String = ctl::string; + +#undef ctl + +inline bool +issmall(const String& s) +{ + return s.capacity() == sizeof(s) && + s.data() == reinterpret_cast(&s); +} int main() @@ -358,15 +368,14 @@ main() String s; if constexpr (std::is_same_v) { // tests the small-string optimization on ctl::string - char* d = s.data(); for (int i = 0; i < 23; ++i) { s.append("a"); - if (s.data() != d) { + if (!issmall(s)) { return 79 + i; } } s.append("a"); - if (s.data() == d) { + if (issmall(s)) { return 103; } } else { @@ -380,6 +389,21 @@ main() } } + { + String s("arst", 4); + for (int i = 0; i < 30; ++i) { + s.append("a"); + } + s.resize(4); + if (s != "arst") + return 105; + if constexpr (std::is_same_v) { + String r(s); + if (issmall(s) || !issmall(r)) + return 106; + } + } + CheckForMemoryLeaks(); return 0; }