mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-08-05 09:20:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip ctl::string small-string optimization

Browse source 
A small-string optimization is a way of reusing inline storage space for
sufficiently small strings, rather than allocating them on the heap. The
current approach takes after an old Facebook string class: it reuses the
highest-order byte for flags and small-string size, in such a way that a
maximally-sized small string will have its last byte zeroed, making it a
null terminator for the C string.

The only flag we have is in the highest-order bit, that says whether the
string is big (set) or small (cleared.) Most of the logic switches based
on the value of this bit; e.g. data() returns big()->p if it's set, else
small()->buf if it's cleared.

Morally speaking, our class's storage is a union over two POD C structs.
It may be that this winds up being the best way to actually write it but
for now I gravitated towards a slightly more obtuse approach: the string
class itself contains a blob of the right size, and we alias that blob's
pointer for the two structs, taking some care not to run afoul of object
lifetime rules in C++. Only in writing this now do I realize that we may
be able to relatively easily sidestep those rules.

TODO:

- [ ] tests are currently segfaulting
- [ ] think about operator string_view
- [ ] maybe migrate to POD anonymous union
- [ ] benchmark and see if this is even worth it
- [ ] __ namespace needs documented, at least here
- [ ] we are probably incorrectly setting size in a few places
- [ ] explain why assign-by-value and "swapperator", at least here
This commit is contained in:
Jōshin 2024-06-05 23:57:48 -07:00
parent 2c5e7ec547
commit a535cdc417
No known key found for this signature in database
3 changed files with 313 additions and 132 deletions
Download patch file Download diff file Expand all files Collapse all files

11
ctl/strcat.cc
View file

@ -34,10 +34,15 @@ strcat(const string_view lhs, const string_view rhs) noexcept
__builtin_trap(); __builtin_trap();
res.reserve(need); res.reserve(need);
if (lhs.n) if (lhs.n)
memcpy(res.p, lhs.p, lhs.n); memcpy(res.data(), lhs.p, lhs.n);
if (rhs.n) if (rhs.n)
memcpy(res.p + lhs.n, rhs.p, rhs.n); memcpy(res.data() + lhs.n, rhs.p, rhs.n);
res.p[res.n = lhs.n + rhs.n] = 0; if (res.isbig()) {
res.big()->n = lhs.n + rhs.n;
} else {
res.small()->rem = __::sso_max - lhs.n - rhs.n;
}
res.data()[res.size()] = 0;
return res; return res;
} }

231
ctl/string.cc
View file

@ -23,7 +23,9 @@
namespace ctl { namespace ctl {
string::~string() noexcept namespace __ {
big_string::~big_string() /* noexcept */
{ {
if (n) { if (n) {
if (n >= c) if (n >= c)
@ -36,6 +38,15 @@ string::~string() noexcept
free(p); free(p);
} }
} // namespace __
string::~string() /* noexcept */
{
if (isbig()) {
big()->~big_string();
}
}
string::string(const char* s) noexcept string::string(const char* s) noexcept
{ {
append(s, strlen(s)); append(s, strlen(s));
@ -43,7 +54,7 @@ string::string(const char* s) noexcept
string::string(const string& s) noexcept string::string(const string& s) noexcept
{ {
append(s.p, s.n); append(s.data(), s.size());
} }
string::string(const string_view s) noexcept string::string(const string_view s) noexcept
@ -64,29 +75,37 @@ string::string(const char* s, size_t size) noexcept
const char* const char*
string::c_str() const noexcept string::c_str() const noexcept
{ {
if (!n) if (size() >= capacity())
return "";
if (n >= c)
__builtin_trap(); __builtin_trap();
if (p[n]) if (data()[size()])
__builtin_trap(); __builtin_trap();
return p; return data();
} }
void void
string::reserve(size_t c2) noexcept string::reserve(size_t c2) noexcept
{ {
char* p2; char* p2;
if (c2 < n) if (c2 < size())
c2 = n; c2 = size();
if (ckd_add(&c2, c2, 15)) if (ckd_add(&c2, c2, 15))
__builtin_trap(); __builtin_trap();
c2 &= -16; c2 &= -16;
if (!(p2 = (char*)realloc(p, c2))) if (c2 <= __::sso_max)
__builtin_trap(); return;
if (!isbig()) {
if (!(p2 = (char *)malloc(c2)))
__builtin_trap();
__builtin_memcpy(p2, data(), size() + 1);
} else {
if (!(p2 = (char *)realloc(big()->p, c2)))
__builtin_trap();
}
size_t n = size();
std::atomic_signal_fence(std::memory_order_seq_cst); std::atomic_signal_fence(std::memory_order_seq_cst);
c = c2; set_big_capacity(c2);
p = p2; big()->n = n;
big()->p = p2;
} }
void void
@ -96,41 +115,54 @@ string::resize(size_t n2, char ch) noexcept
if (ckd_add(&c2, n2, 1)) if (ckd_add(&c2, n2, 1))
__builtin_trap(); __builtin_trap();
reserve(c2); reserve(c2);
if (n2 > n) if (n2 > size())
memset(p + n, ch, n2 - n); memset(data() + size(), ch, n2 - size());
p[n = n2] = 0; if (isbig()) {
big()->p[big()->n = n2] = 0;
} else {
set_small_size(n2);
data()[size()] = 0;
}
} }
void void
string::append(char ch) noexcept string::append(char ch) noexcept
{ {
if (n + 2 > c) { size_t n2;
size_t c2 = c + 2; if (ckd_add(&n2, size(), 2))
c2 += c2 >> 1; __builtin_trap();
if (n2 > capacity()) {
size_t c2 = capacity() + 2;
if (ckd_add(&c2, c2, c2 >> 1))
__builtin_trap();
reserve(c2); reserve(c2);
} }
p[n++] = ch; // XXX do we care to fence this?
p[n] = 0; data()[size()] = ch;
data()[size() + 1] = 0;
if (isbig()) {
++big()->n;
} else {
--small()->rem;
}
} }
void void
string::grow(size_t size) noexcept string::grow(size_t size) noexcept
{ {
size_t need; size_t need;
if (ckd_add(&need, n, size)) if (ckd_add(&need, this->size(), size))
__builtin_trap(); __builtin_trap();
if (ckd_add(&need, need, 1)) if (ckd_add(&need, need, 1))
__builtin_trap(); __builtin_trap();
if (need <= c) if (need <= capacity())
return; return;
size_t c2 = c; size_t c2 = capacity();
if (!c2) { if (!c2)
c2 = need; __builtin_trap();
} else { while (c2 < need)
while (c2 < need) if (ckd_add(&c2, c2, c2 >> 1))
if (ckd_add(&c2, c2, c2 >> 1)) __builtin_trap();
__builtin_trap();
}
reserve(c2); reserve(c2);
} }
@ -139,8 +171,14 @@ string::append(char ch, size_t size) noexcept
{ {
grow(size); grow(size);
if (size) if (size)
memset(p + n, ch, size); memset(data() + this->size(), ch, size);
p[n += size] = 0; // XXX fence?
if (isbig()) {
big()->n += size;
} else {
small()->rem -= size;
}
data()[this->size()] = 0;
} }
void void
@ -148,55 +186,53 @@ string::append(const void* data, size_t size) noexcept
{ {
grow(size); grow(size);
if (size) if (size)
memcpy(p + n, data, size); memcpy(this->data() + this->size(), data, size);
p[n += size] = 0; if (isbig()) {
big()->n += size;
} else {
small()->rem -= size;
}
this->data()[this->size()] = 0;
} }
void void
string::pop_back() noexcept string::pop_back() noexcept
{ {
if (!n) if (!size())
__builtin_trap(); __builtin_trap();
p[--n] = 0; if (isbig()) {
--big()->n;
} else {
++small()->rem;
}
data()[size()] = 0;
} }
string& string&
string::operator=(string&& s) noexcept string::operator=(string s) noexcept
{ {
if (p != s.p) { swap(s);
if (p) {
clear();
append(s.p, s.n);
} else {
p = s.p;
n = s.n;
c = s.c;
s.p = nullptr;
s.n = 0;
s.c = 0;
}
}
return *this; return *this;
} }
bool bool
string::operator==(const string_view s) const noexcept string::operator==(const string_view s) const noexcept
{ {
if (n != s.n) if (size() != s.n)
return false; return false;
if (!n) if (!s.n)
return true; return true;
return !memcmp(p, s.p, n); return !memcmp(data(), s.p, s.n);
} }
bool bool
string::operator!=(const string_view s) const noexcept string::operator!=(const string_view s) const noexcept
{ {
if (n != s.n) if (size() != s.n)
return true; return true;
if (!n) if (!s.n)
return false; return false;
return !!memcmp(p, s.p, n); return !!memcmp(data(), s.p, s.n);
} }
bool bool
@ -204,35 +240,35 @@ string::contains(const string_view s) const noexcept
{ {
if (!s.n) if (!s.n)
return true; return true;
return !!memmem(p, n, s.p, s.n); return !!memmem(data(), size(), s.p, s.n);
} }
bool bool
string::ends_with(const string_view s) const noexcept string::ends_with(const string_view s) const noexcept
{ {
if (n < s.n) if (size() < s.n)
return false; return false;
if (!s.n) if (!s.n)
return true; return true;
return !memcmp(p + n - s.n, s.p, s.n); return !memcmp(data() + size() - s.n, s.p, s.n);
} }
bool bool
string::starts_with(const string_view s) const noexcept string::starts_with(const string_view s) const noexcept
{ {
if (n < s.n) if (size() < s.n)
return false; return false;
if (!s.n) if (!s.n)
return true; return true;
return !memcmp(p, s.p, s.n); return !memcmp(data(), s.p, s.n);
} }
size_t size_t
string::find(char ch, size_t pos) const noexcept string::find(char ch, size_t pos) const noexcept
{ {
char* q; char* q;
if ((q = (char*)memchr(p, ch, n))) if ((q = (char*)memchr(data(), ch, size())))
return q - p; return q - data();
return npos; return npos;
} }
@ -240,10 +276,10 @@ size_t
string::find(const string_view s, size_t pos) const noexcept string::find(const string_view s, size_t pos) const noexcept
{ {
char* q; char* q;
if (pos > n) if (pos > size())
__builtin_trap(); __builtin_trap();
if ((q = (char*)memmem(p + pos, n - pos, s.p, s.n))) if ((q = (char*)memmem(data() + pos, size() - pos, s.p, s.n)))
return q - p; return q - data();
return npos; return npos;
} }
@ -251,15 +287,15 @@ string
string::substr(size_t pos, size_t count) const noexcept string::substr(size_t pos, size_t count) const noexcept
{ {
size_t last; size_t last;
if (pos > n) if (pos > size())
__builtin_trap(); __builtin_trap();
if (count > n - pos) if (count > size() - pos)
count = n - pos; count = size() - pos;
if (ckd_add(&last, pos, count)) if (ckd_add(&last, pos, count))
last = n; last = size();
if (last > n) if (last > size())
__builtin_trap(); __builtin_trap();
return string(p + pos, count); return string(data() + pos, count);
} }
string& string&
@ -268,12 +304,12 @@ string::replace(size_t pos, size_t count, const string_view& s) noexcept
size_t last; size_t last;
if (ckd_add(&last, pos, count)) if (ckd_add(&last, pos, count))
__builtin_trap(); __builtin_trap();
if (last > n) if (last > size())
__builtin_trap(); __builtin_trap();
size_t need; size_t need;
if (ckd_add(&need, pos, s.n)) if (ckd_add(&need, pos, s.n))
__builtin_trap(); __builtin_trap();
size_t extra = n - last; size_t extra = size() - last;
if (ckd_add(&need, need, extra)) if (ckd_add(&need, need, extra))
__builtin_trap(); __builtin_trap();
size_t c2; size_t c2;
@ -281,42 +317,57 @@ string::replace(size_t pos, size_t count, const string_view& s) noexcept
__builtin_trap(); __builtin_trap();
reserve(c2); reserve(c2);
if (extra) if (extra)
memmove(p + pos + s.n, p + last, extra); memmove(data() + pos + s.n, data() + last, extra);
memcpy(p + pos, s.p, s.n); memcpy(data() + pos, s.p, s.n);
p[n = need] = 0; if (isbig()) {
big()->p[big()->n = need] = 0;
} else {
set_small_size(need);
data()[size()] = 0;
}
return *this; return *this;
} }
string& string&
string::insert(size_t i, const string_view s) noexcept string::insert(size_t i, const string_view s) noexcept
{ {
if (i > n) if (i > size())
__builtin_trap(); __builtin_trap();
size_t extra = n - i; size_t extra = size() - i;
size_t need; size_t need;
if (ckd_add(&need, n, s.n)) if (ckd_add(&need, size(), s.n))
__builtin_trap(); __builtin_trap();
if (ckd_add(&need, need, 1)) if (ckd_add(&need, need, 1))
__builtin_trap(); __builtin_trap();
reserve(need); reserve(need);
if (extra) if (extra)
memmove(p + i + s.n, p + i, extra); memmove(data() + i + s.n, data() + i, extra);
memcpy(p + i, s.p, s.n); memcpy(data() + i, s.p, s.n);
p[n += s.n] = 0; if (isbig()) {
big()->n += s.n;
} else {
small()->rem -= s.n;
}
data()[size()] = 0;
return *this; return *this;
} }
string& string&
string::erase(size_t pos, size_t count) noexcept string::erase(size_t pos, size_t count) noexcept
{ {
if (pos > n) if (pos > size())
__builtin_trap(); __builtin_trap();
if (count > n - pos) if (count > size() - pos)
count = n - pos; count = size() - pos;
size_t extra = n - (pos + count); size_t extra = size() - (pos + count);
if (extra) if (extra)
memmove(p + pos, p + pos + count, extra); memmove(data() + pos, data() + pos + count, extra);
p[n = pos + extra] = 0; if (isbig()) {
big()->n = pos + extra;
} else {
set_small_size(pos + extra);
}
data()[size()] = 0;
return *this; return *this;
} }

203
ctl/string.h
View file

@ -6,29 +6,59 @@
namespace ctl { namespace ctl {
struct string; class string;
string string
strcat(const string_view, const string_view) noexcept __wur; strcat(const string_view, const string_view) noexcept __wur;
struct string namespace __ {
{
char* p = nullptr;
size_t n = 0;
size_t c = 0;
constexpr size_t string_size = 3 * sizeof(size_t);
constexpr size_t sso_max = string_size - 1;
constexpr size_t big_mask = ~(1ull << (8ull * sizeof(size_t) - 1ull));
struct small_string
{
char buf[sso_max];
// interpretation is: size == sso_max - rem
unsigned char rem;
#if 0
size_t rem : 7;
size_t big : 1 /* = 0 */;
#endif
};
struct big_string
{
char* p;
size_t n;
// interpretation is: capacity == c & big_mask
size_t c;
#if 0
size_t c : sizeof(size_t) * 8 - 1;
size_t big : 1 /* = 1 */;
#endif
~big_string() /* noexcept */;
};
} // namespace __
class string
{
public:
using iterator = char*; using iterator = char*;
using const_iterator = const char*; using const_iterator = const char*;
static constexpr size_t npos = -1; static constexpr size_t npos = -1;
~string() noexcept; ~string() /* noexcept */;
string() = default;
string(const string_view) noexcept; string(const string_view) noexcept;
string(const char*) noexcept; string(const char*) noexcept;
string(const string&) noexcept; string(const string&) noexcept;
string(const char*, size_t) noexcept; string(const char*, size_t) noexcept;
explicit string(size_t, char = 0) noexcept; explicit string(size_t, char = 0) noexcept;
string& operator=(string&&) noexcept;
string& operator=(string) noexcept;
const char* c_str() const noexcept; const char* c_str() const noexcept;
void pop_back() noexcept; void pop_back() noexcept;
@ -51,103 +81,137 @@ struct string
size_t find(char, size_t = 0) const noexcept; size_t find(char, size_t = 0) const noexcept;
size_t find(const string_view, size_t = 0) const noexcept; size_t find(const string_view, size_t = 0) const noexcept;
string(string&& s) noexcept : p(s.p), n(s.n), c(s.c) string() noexcept
{ {
s.p = nullptr; set_small_size(0);
s.n = 0; small()->buf[0] = 0;
s.c = 0; }
void swap(string& s) noexcept
{
char tmp[__::string_size];
__builtin_memcpy(tmp, __builtin_launder(blob), sizeof(tmp));
__builtin_memcpy(
__builtin_launder(blob), __builtin_launder(s.blob), sizeof(tmp));
__builtin_memcpy(__builtin_launder(s.blob), tmp, sizeof(tmp));
}
string(string&& s) noexcept
{
__builtin_memcpy(blob, __builtin_launder(s.blob), sizeof(blob));
s.set_small_size(0);
/* shouldn't be necessary, but the spec says s should be left in a valid
state and our c_str() depends on this */
s.small()->buf[0] = 0;
} }
void clear() noexcept void clear() noexcept
{ {
n = 0; if (isbig()) {
big()->n = 0;
} else {
set_small_size(0);
}
} }
bool empty() const noexcept bool empty() const noexcept
{ {
return !n; return isbig() ? !big()->n : small()->rem >= __::sso_max;
} }
char* data() const noexcept inline char* data() noexcept
{ {
return p; return isbig() ? big()->p : small()->buf;
} }
size_t size() const noexcept inline const char* data() const noexcept
{ {
return n; return isbig() ? big()->p : small()->buf;
}
inline size_t size() const noexcept
{
#if 0
if (!isbig() && small()->rem > __::sso_max)
__builtin_trap();
#endif
return isbig() ? big()->n : __::sso_max - small()->rem;
} }
size_t length() const noexcept size_t length() const noexcept
{ {
return n; return size();
} }
size_t capacity() const noexcept size_t capacity() const noexcept
{ {
return c; #if 0
if (isbig() && big()->c <= __::sso_max)
__builtin_trap();
#endif
return isbig() ? big()->c : __::sso_max;
} }
iterator begin() noexcept iterator begin() noexcept
{ {
return p; return data();
} }
iterator end() noexcept iterator end() noexcept
{ {
return p + n; return data() + size();
} }
const_iterator cbegin() const noexcept const_iterator cbegin() const noexcept
{ {
return p; return data();
} }
const_iterator cend() const noexcept const_iterator cend() const noexcept
{ {
return p + n; return data() + size();
} }
char& front() char& front()
{ {
if (!n) if (!size())
__builtin_trap(); __builtin_trap();
return p[0]; return data()[0];
} }
const char& front() const const char& front() const
{ {
if (!n) if (!size())
__builtin_trap(); __builtin_trap();
return p[0]; return data()[0];
} }
char& back() char& back()
{ {
if (!n) if (!size())
__builtin_trap(); __builtin_trap();
return p[n - 1]; return data()[size() - 1];
} }
const char& back() const const char& back() const
{ {
if (!n) if (!size())
__builtin_trap(); __builtin_trap();
return p[n - 1]; return data()[size() - 1];
} }
char& operator[](size_t i) noexcept char& operator[](size_t i) noexcept
{ {
if (i >= n) if (i >= size())
__builtin_trap(); __builtin_trap();
return p[i]; return data()[i];
} }
const char& operator[](size_t i) const noexcept const char& operator[](size_t i) const noexcept
{ {
if (i >= n) if (i >= size())
__builtin_trap(); __builtin_trap();
return p[i]; return data()[i];
} }
void push_back(char ch) noexcept void push_back(char ch) noexcept
@ -160,9 +224,10 @@ struct string
append(s.p, s.n); append(s.p, s.n);
} }
inline constexpr operator string_view() const noexcept // TODO(mrdomino): explicit?
inline operator string_view() const noexcept
{ {
return string_view(p, n); return string_view(data(), size());
} }
string& operator=(const char* s) noexcept string& operator=(const char* s) noexcept
@ -220,8 +285,68 @@ struct string
{ {
return compare(s) >= 0; return compare(s) >= 0;
} }
private:
inline bool isbig() const noexcept
{
return *(__builtin_launder(blob) + __::sso_max) & 0x80;
}
inline void set_small_size(size_t size) noexcept
{
if (size > __::sso_max)
__builtin_trap();
*(__builtin_launder(blob) + __::sso_max) = (__::sso_max - size);
}
inline void set_big_capacity(size_t c2) noexcept
{
if (c2 > __::big_mask)
__builtin_trap();
*((size_t *)__builtin_launder(blob) + 2) = ~__::big_mask | c2;
}
inline __::small_string* small() noexcept
{
if (isbig())
__builtin_trap();
return __builtin_launder(reinterpret_cast<__::small_string*>(blob));
}
inline const __::small_string* small() const noexcept
{
if (isbig())
__builtin_trap();
return __builtin_launder(
reinterpret_cast<const __::small_string*>(blob));
}
inline __::big_string* big() noexcept
{
if (!isbig())
__builtin_trap();
return __builtin_launder(reinterpret_cast<__::big_string*>(blob));
}
inline const __::big_string* big() const noexcept
{
if (!isbig())
__builtin_trap();
return __builtin_launder(reinterpret_cast<const __::big_string*>(blob));
}
friend string strcat(const string_view, const string_view);
alignas(union {
__::big_string a;
__::small_string b;
}) char blob[__::string_size];
}; };
static_assert(sizeof(string) == __::string_size);
static_assert(sizeof(__::small_string) == __::string_size);
static_assert(sizeof(__::big_string) == __::string_size);
} // namespace ctl } // namespace ctl
#pragma GCC diagnostic push #pragma GCC diagnostic push