vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make console usage platform specific

Browse source 
Work on compiler warnings.
This commit is contained in:
goerch 2023-09-09 13:35:57 +02:00
parent e903d5f163
commit 96533e0c22
2 changed files with 9 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

3
tests/test-tokenizer-0-llama.cpp
View file

@ -90,8 +90,11 @@ int main(int argc, char **argv) {
return 2; return 2;
} }
#ifdef _WIN32
// We need this for unicode console support
console::init(false, false); console::init(false, false);
atexit([]() { console::cleanup(); }); atexit([]() { console::cleanup(); });
#endif
bool success = true; bool success = true;

9
tests/test-tokenizer-1-llama.cpp
View file

@ -74,8 +74,11 @@ int main(int argc, char **argv) {
GGML_ASSERT(llama_vocab_type(ctx) == LLAMA_VOCAB_TYPE_SPM); GGML_ASSERT(llama_vocab_type(ctx) == LLAMA_VOCAB_TYPE_SPM);
#ifdef _WIN32
// We need this for unicode console support
console::init(false, false); console::init(false, false);
atexit([]() { console::cleanup(); }); atexit([]() { console::cleanup(); });
#endif
const int n_vocab = llama_n_vocab(ctx); const int n_vocab = llama_n_vocab(ctx);
@ -84,7 +87,7 @@ int main(int argc, char **argv) {
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false); std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize_spm(ctx, tokens); std::string check = llama_detokenize_spm(ctx, tokens);
if (check != str) { if (check != str) {
fprintf(stderr, "%s : error: token %d detokenizes to >%s<(%d) but tokenization of this detokenizes to >%s<(%d)\n", fprintf(stderr, "%s : error: token %d detokenizes to >%s<(%llu) but tokenization of this detokenizes to >%s<(%llu)\n",
__func__, i, str.c_str(), str.length(), check.c_str(), check.length()); __func__, i, str.c_str(), str.length(), check.c_str(), check.length());
if(i != 3) if(i != 3)
return 2; return 2;
@ -97,7 +100,7 @@ int main(int argc, char **argv) {
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false); std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize_spm(ctx, tokens); std::string check = llama_detokenize_spm(ctx, tokens);
if (str != check) { if (str != check) {
fprintf(stderr, "%s : error: codepoint %d detokenizes to >%s<(%d) instead of >%s<(%d)\n", fprintf(stderr, "%s : error: codepoint %d detokenizes to >%s<(%llu) instead of >%s<(%llu)\n",
__func__, cp, check.c_str(), check.length(), str.c_str(), str.length()); __func__, cp, check.c_str(), check.length(), str.c_str(), str.length());
if(cp != 0 && cp != 9601) if(cp != 0 && cp != 9601)
return 3; return 3;
@ -109,7 +112,7 @@ int main(int argc, char **argv) {
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false); std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize_spm(ctx, tokens); std::string check = llama_detokenize_spm(ctx, tokens);
if (str != check) { if (str != check) {
fprintf(stderr, "%s : error: codepoint %d detokenizes to >%s<(%d) instead of >%s<(%d)\n", fprintf(stderr, "%s : error: codepoint %d detokenizes to >%s<(%llu) instead of >%s<(%llu)\n",
__func__, cp, check.c_str(), check.length(), str.c_str(), str.length()); __func__, cp, check.c_str(), check.length(), str.c_str(), str.length());
return 4; return 4;
} }