From 34596530b2f922a387bd787903ee290c5f048a4d Mon Sep 17 00:00:00 2001 From: Fabio Rossini Sluzala Date: Mon, 20 Mar 2023 21:28:14 -0300 Subject: [PATCH] Nest struct token score inside gpt_vocab --- utils.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/utils.h b/utils.h index a3c2955c9..a6600dae1 100644 --- a/utils.h +++ b/utils.h @@ -52,16 +52,15 @@ std::string gpt_random_prompt(std::mt19937 & rng); // Vocab utils // -struct token_score { - using token_t = std::string; - token_t token; - float score; -}; - struct gpt_vocab { using id = int32_t; using token = std::string; + struct token_score { + token token; + float score; + }; + std::unordered_map token_to_id; std::vector id_to_token; };