vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

renamed token to tok

Browse source
This commit is contained in:
Fabio Rossini Sluzala 2023-03-21 11:54:04 -03:00
parent 34596530b2
commit a19aa63ba2
No known key found for this signature in database
GPG key ID: F9D569BBF49F437B
4 changed files with 7 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

8
main.cpp
View file

@ -175,7 +175,7 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
vocab.token_to_id[word] = i;
auto &tok_score = vocab.id_to_token[i];
tok_score.token = word;
tok_score.tok = word;
tok_score.score = score;
//if (i < 30000) {
@ -896,7 +896,7 @@ int main(int argc, char ** argv) {
fprintf(stderr, "%s: prompt: '%s'\n", __func__, params.prompt.c_str());
fprintf(stderr, "%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size());
for (int i = 0; i < (int) embd_inp.size(); i++) {
fprintf(stderr, "%6d -> '%s'\n", embd_inp[i], vocab.id_to_token.at(embd_inp[i]).token.c_str());
fprintf(stderr, "%6d -> '%s'\n", embd_inp[i], vocab.id_to_token.at(embd_inp[i]).tok.c_str());
}
fprintf(stderr, "\n");
if (params.interactive) {
@ -918,7 +918,7 @@ int main(int argc, char ** argv) {
fprintf(stderr, "%s: reverse prompt: '%s'\n", __func__, params.antiprompt.at(apindex).c_str());
fprintf(stderr, "%s: number of tokens in reverse prompt = %zu\n", __func__, antiprompt_inp.size());
for (int i = 0; i < (int) antiprompt_inp.size(); i++) {
fprintf(stderr, "%6d -> '%s'\n", antiprompt_inp[i], vocab.id_to_token.at(antiprompt_inp[i]).token.c_str());
fprintf(stderr, "%6d -> '%s'\n", antiprompt_inp[i], vocab.id_to_token.at(antiprompt_inp[i]).tok.c_str());
}
fprintf(stderr, "\n");
}
@ -1024,7 +1024,7 @@ int main(int argc, char ** argv) {
// display text
if (!input_noecho) {
for (auto id : embd) {
printf("%s", vocab.id_to_token[id].token.c_str());
printf("%s", vocab.id_to_token[id].tok.c_str());
}
fflush(stdout);
}

2
quantize.cpp
View file

@ -146,7 +146,7 @@ bool llama_model_quantize(const std::string & fname_inp, const std::string & fna
vocab.token_to_id[word] = i;
auto &tok_score = vocab.id_to_token[i];
tok_score.token = word;
tok_score.tok = word;
tok_score.score = score;
}
}

2
utils.cpp
View file

@ -444,7 +444,7 @@ bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) {
vocab.id_to_token.resize(vocab.token_to_id.size());
for (const auto & kv : vocab.token_to_id) {
vocab.id_to_token[kv.second].token = kv.first;
vocab.id_to_token[kv.second].tok = kv.first;
}
printf("%s: vocab size = %d\n", __func__, (int) vocab.token_to_id.size());

2
utils.h
View file

@ -57,7 +57,7 @@ struct gpt_vocab {
using token = std::string;
struct token_score {
token token;
token tok;
float score;
};