vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Adding a comment

Browse source
This commit is contained in:
goerch 2023-09-19 19:37:25 +02:00
parent 17ca832717
commit 4abbfb51f9
1 changed files with 1 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
tests/test-tokenizer-1-bpe.cpp
View file

@ -77,6 +77,7 @@ int main(int argc, char **argv) {
}
for (uint32_t cp = 0x0000; cp < 0xffff; ++cp) {
// NOTE: these exceptions seem to be necessary, because the GPT2 tokenizer doesn't want to interfere with some ASCII control characters
if ((cp < 0x03 || cp > 0x05) && cp != 0x0b && cp != 0x11 && (cp < 0x13 || cp > 0x17) && cp != 0x19 && (cp < 0x1c || cp > 0x1e) && (cp < 0xd800 || cp > 0xdfff)) {
std::string str = " " + codepoint_to_utf8(cp);
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);