vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

optimize tokenize method

Browse source
This commit is contained in:
Concedo 2023-08-24 21:16:16 +08:00
parent b95a4ccb22
commit 661bede62f
1 changed files with 10 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

20
gpttype_adapter.cpp
View file

@ -338,34 +338,32 @@ static std::string FileFormatTokenizeID(int id, FileFormat file_format)
} }
} }
static std::vector<int> TokenizeString(const std::string & str_to_tokenize, FileFormat file_format) static void TokenizeString(const std::string & str_to_tokenize, std::vector<int> & output_tokens, FileFormat file_format)
{ {
std::vector<int> tokvec;
if (file_format == FileFormat::GGML || file_format == FileFormat::GGHF || file_format == FileFormat::GGJT || file_format == FileFormat::GGJT_2 || file_format == FileFormat::GGJT_3 || file_format == FileFormat::GGUF_LLAMA) if (file_format == FileFormat::GGML || file_format == FileFormat::GGHF || file_format == FileFormat::GGJT || file_format == FileFormat::GGJT_2 || file_format == FileFormat::GGJT_3 || file_format == FileFormat::GGUF_LLAMA)
{ {
if(file_format == FileFormat::GGHF || file_format == FileFormat::GGJT || file_format == FileFormat::GGJT_2 ) if(file_format == FileFormat::GGHF || file_format == FileFormat::GGJT || file_format == FileFormat::GGJT_2 )
{ {
tokvec = ::llama_v2_tokenize(llama_ctx_v2, str_to_tokenize, true); output_tokens = ::llama_v2_tokenize(llama_ctx_v2, str_to_tokenize, true);
} }
else if (file_format == FileFormat::GGML) else if (file_format == FileFormat::GGML)
{ {
tokvec = ::legacy_llama_v2_tokenize(llama_ctx_v2, str_to_tokenize, true); output_tokens = ::legacy_llama_v2_tokenize(llama_ctx_v2, str_to_tokenize, true);
} }
else if (file_format == FileFormat::GGJT_3) else if (file_format == FileFormat::GGJT_3)
{ {
tokvec = ::llama_v3_tokenize(llama_ctx_v3, str_to_tokenize, true); output_tokens = ::llama_v3_tokenize(llama_ctx_v3, str_to_tokenize, true);
} }
else else
{ {
tokvec = ::llama_tokenize(llama_ctx_v4, str_to_tokenize, true); output_tokens = ::llama_tokenize(llama_ctx_v4, str_to_tokenize, true);
} }
} }
else else
{ {
// tokenize the prompt // tokenize the prompt
tokvec = ::gpt_tokenize(vocab, str_to_tokenize); output_tokens = ::gpt_tokenize(vocab, str_to_tokenize);
} }
return tokvec;
} }
static std::string RemoveBell(const std::string & input) //removes the bell character static std::string RemoveBell(const std::string & input) //removes the bell character
@ -1001,7 +999,8 @@ int gpttype_token_count(const std::string & input)
{ {
printf("\nFileFormat: %d, Tokenizing: %s",file_format ,input.c_str()); printf("\nFileFormat: %d, Tokenizing: %s",file_format ,input.c_str());
} }
auto toks = TokenizeString(input, file_format); std::vector<int> toks;
TokenizeString(input, toks, file_format);
int tokcount = toks.size(); int tokcount = toks.size();
if(debugmode==1) if(debugmode==1)
{ {
@ -1063,7 +1062,8 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
} }
// tokenize the prompt // tokenize the prompt
std::vector<int> embd_inp = TokenizeString(params.prompt, file_format); std::vector<int> embd_inp;
TokenizeString(params.prompt, embd_inp, file_format);
//truncate to front of the prompt if its too long //truncate to front of the prompt if its too long
int32_t nctx = params.n_ctx; int32_t nctx = params.n_ctx;