From 1d7e32bba718860ac5c2b5f925fefcc4721bacea Mon Sep 17 00:00:00 2001 From: Lou Xiao Date: Sun, 19 Mar 2023 17:03:31 +0800 Subject: [PATCH] bugfix: centos 7, gcc (GCC) 11.2.1 20220127 (Red Hat 11.2.1-9) std::string mesh up vocab. --- main.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/main.cpp b/main.cpp index 105dd91ee..dcf6bc90d 100644 --- a/main.cpp +++ b/main.cpp @@ -144,15 +144,15 @@ bool llama_model_load(const std::string & fname, llama_model & model, gpt_vocab // load vocab { std::string word; + uint32_t len; for (int i = 0; i < model.hparams.n_vocab; i++) { - uint32_t len; fin.read((char *) &len, sizeof(len)); - word.resize(len); fin.read((char *) word.data(), len); - vocab.token_to_id[word] = i; - vocab.id_to_token[i] = word; + // force make a new std::string, some compiler may share inner data. + vocab.token_to_id[std::string(word.data(), len)] = i; + vocab.id_to_token[i] = std::string(word.data(), len); //if (i < 30000) { // fprintf(stderr, "%s: vocab[%d] = '%s'\n", __func__, i, word.c_str());