vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Speedup tokenization

Browse source 
On current master it takes ~3.2 seconds to tokenize
Wikitext. With this change it becomes ~525 ms.
This commit is contained in:
Iwan Kawrakow 2023-08-27 16:28:40 +03:00
parent a6d1189fdd
commit 5021d7bc3f
2 changed files with 11 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

4
examples/perplexity/perplexity.cpp
View file

@ -190,10 +190,14 @@ void perplexity(llama_context * ctx, const gpt_params & params) {
const bool is_spm = llama_vocab_type(ctx) == LLAMA_VOCAB_TYPE_SPM;
const bool add_bos = is_spm;
auto tim1 = std::chrono::high_resolution_clock::now();
fprintf(stderr, "%s: tokenizing the input ..\n", __func__);
auto tokens = ::llama_tokenize(ctx, params.prompt, add_bos);
auto tim2 = std::chrono::high_resolution_clock::now();
fprintf(stderr, "%s: tokenization took %g ms\n",__func__,1e-3*std::chrono::duration_cast<std::chrono::microseconds>(tim2-tim1).count());
const int n_chunk_max = tokens.size() / params.n_ctx;
const int n_chunk = params.n_chunks < 0 ? n_chunk_max : std::min(params.n_chunks, n_chunk_max);

12
llama.cpp
View file

@ -114,12 +114,14 @@ static size_t utf8_len(char src) {
}
void replace_all(std::string & s, const std::string & search, const std::string & replace) {
for (size_t pos = 0; ; pos += replace.length()) {
pos = s.find(search, pos);
if (pos == std::string::npos) break;
s.erase(pos, search.length());
s.insert(pos, replace);
std::string result;
for (size_t pos = 0; ; pos += search.length()) {
auto new_pos = s.find(search, pos);
if (new_pos == std::string::npos) break;
result += s.substr(pos, new_pos - pos) + replace;
pos = new_pos;
}
s = std::move(result);
}
static void zeros(std::ofstream & file, size_t n) {