From bdd0286bd0903d4578420461edd5793b2046ad42 Mon Sep 17 00:00:00 2001 From: teleprint-me <77757836+teleprint-me@users.noreply.github.com> Date: Mon, 20 May 2024 01:39:09 -0400 Subject: [PATCH] refactor: Use proper names for referenced member variables --- gguf-py/gguf/constants.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index bef04744a..c7565ab8e 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -70,7 +70,8 @@ class Keys: class Tokenizer: MODEL = "tokenizer.ggml.model" - PRE = "tokenizer.ggml.pre" + HASH = "tokenizer.ggml.hash" + TYPE = "tokenizer.ggml.type" LIST = "tokenizer.ggml.tokens" TOKEN_TYPE = "tokenizer.ggml.token_type" TOKEN_TYPE_COUNT = "tokenizer.ggml.token_type_count" # for BERT-style token types @@ -1000,7 +1001,8 @@ KEY_SSM_TIME_STEP_RANK = Keys.SSM.TIME_STEP_RANK # tokenization KEY_TOKENIZER_MODEL = Keys.Tokenizer.MODEL -KEY_TOKENIZER_PRE = Keys.Tokenizer.PRE +KEY_TOKENIZER_HASH = Keys.Tokenizer.HASH +KEY_TOKENIZER_TYPE = Keys.Tokenizer.TYPE KEY_TOKENIZER_LIST = Keys.Tokenizer.LIST KEY_TOKENIZER_TOKEN_TYPE = Keys.Tokenizer.TOKEN_TYPE KEY_TOKENIZER_SCORES = Keys.Tokenizer.SCORES