diff --git a/utils.h b/utils.h index a3c2955c9..a6600dae1 100644 --- a/utils.h +++ b/utils.h @@ -52,16 +52,15 @@ std::string gpt_random_prompt(std::mt19937 & rng); // Vocab utils // -struct token_score { - using token_t = std::string; - token_t token; - float score; -}; - struct gpt_vocab { using id = int32_t; using token = std::string; + struct token_score { + token token; + float score; + }; + std::unordered_map token_to_id; std::vector id_to_token; };