vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

renamed token to tok

Browse source
This commit is contained in:
Fabio Rossini Sluzala 2023-03-21 11:54:04 -03:00
parent 34596530b2
commit a19aa63ba2
No known key found for this signature in database
GPG key ID: F9D569BBF49F437B
4 changed files with 7 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

8
main.cpp
View file

@ -175,7 +175,7 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab
vocab.token_to_id[word] = i; vocab.token_to_id[word] = i;
auto &tok_score = vocab.id_to_token[i]; auto &tok_score = vocab.id_to_token[i];
tok_score.token = word; tok_score.tok = word;
tok_score.score = score; tok_score.score = score;
//if (i < 30000) { //if (i < 30000) {
@ -896,7 +896,7 @@ int main(int argc, char ** argv) {
fprintf(stderr, "%s: prompt: '%s'\n", __func__, params.prompt.c_str()); fprintf(stderr, "%s: prompt: '%s'\n", __func__, params.prompt.c_str());
fprintf(stderr, "%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size()); fprintf(stderr, "%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size());
for (int i = 0; i < (int) embd_inp.size(); i++) { for (int i = 0; i < (int) embd_inp.size(); i++) {
fprintf(stderr, "%6d -> '%s'\n", embd_inp[i], vocab.id_to_token.at(embd_inp[i]).token.c_str()); fprintf(stderr, "%6d -> '%s'\n", embd_inp[i], vocab.id_to_token.at(embd_inp[i]).tok.c_str());
} }
fprintf(stderr, "\n"); fprintf(stderr, "\n");
if (params.interactive) { if (params.interactive) {
@ -918,7 +918,7 @@ int main(int argc, char ** argv) {
fprintf(stderr, "%s: reverse prompt: '%s'\n", __func__, params.antiprompt.at(apindex).c_str()); fprintf(stderr, "%s: reverse prompt: '%s'\n", __func__, params.antiprompt.at(apindex).c_str());
fprintf(stderr, "%s: number of tokens in reverse prompt = %zu\n", __func__, antiprompt_inp.size()); fprintf(stderr, "%s: number of tokens in reverse prompt = %zu\n", __func__, antiprompt_inp.size());
for (int i = 0; i < (int) antiprompt_inp.size(); i++) { for (int i = 0; i < (int) antiprompt_inp.size(); i++) {
fprintf(stderr, "%6d -> '%s'\n", antiprompt_inp[i], vocab.id_to_token.at(antiprompt_inp[i]).token.c_str()); fprintf(stderr, "%6d -> '%s'\n", antiprompt_inp[i], vocab.id_to_token.at(antiprompt_inp[i]).tok.c_str());
} }
fprintf(stderr, "\n"); fprintf(stderr, "\n");
} }
@ -1024,7 +1024,7 @@ int main(int argc, char ** argv) {
// display text // display text
if (!input_noecho) { if (!input_noecho) {
for (auto id : embd) { for (auto id : embd) {
printf("%s", vocab.id_to_token[id].token.c_str()); printf("%s", vocab.id_to_token[id].tok.c_str());
} }
fflush(stdout); fflush(stdout);
} }

2
quantize.cpp
View file

@ -146,7 +146,7 @@ bool llama_model_quantize(const std::string & fname_inp, const std::string & fna
vocab.token_to_id[word] = i; vocab.token_to_id[word] = i;
auto &tok_score = vocab.id_to_token[i]; auto &tok_score = vocab.id_to_token[i];
tok_score.token = word; tok_score.tok = word;
tok_score.score = score; tok_score.score = score;
} }
} }

2
utils.cpp
View file

@ -444,7 +444,7 @@ bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) {
vocab.id_to_token.resize(vocab.token_to_id.size()); vocab.id_to_token.resize(vocab.token_to_id.size());
for (const auto & kv : vocab.token_to_id) { for (const auto & kv : vocab.token_to_id) {
vocab.id_to_token[kv.second].token = kv.first; vocab.id_to_token[kv.second].tok = kv.first;
} }
printf("%s: vocab size = %d\n", __func__, (int) vocab.token_to_id.size()); printf("%s: vocab size = %d\n", __func__, (int) vocab.token_to_id.size());

2
utils.h
View file

@ -57,7 +57,7 @@ struct gpt_vocab {
using token = std::string; using token = std::string;
struct token_score { struct token_score {
token token; token tok;
float score; float score;
}; };