From 02fbbf90990fb8d5903caee9740b0a74b83eff3e Mon Sep 17 00:00:00 2001 From: Cebtenzzre Date: Sun, 1 Oct 2023 17:04:01 -0400 Subject: [PATCH] fix typo --- convert-falcon-hf-to-gguf.py | 2 +- convert-gptneox-hf-to-gguf.py | 2 +- convert-starcoder-hf-to-gguf.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/convert-falcon-hf-to-gguf.py b/convert-falcon-hf-to-gguf.py index 339c8e4c5..95631e7e2 100755 --- a/convert-falcon-hf-to-gguf.py +++ b/convert-falcon-hf-to-gguf.py @@ -144,7 +144,7 @@ for i in range(vocab_size): print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.") pad_token = f"[PAD{i}]".encode("utf8") text = bytearray(pad_token) - elif i in added_tokens: + elif i in added_token_ids: # these tokens are not encoded, see https://github.com/huggingface/transformers/issues/1133 text = bytearray(reverse_vocab[i].encode('utf-8')) else: diff --git a/convert-gptneox-hf-to-gguf.py b/convert-gptneox-hf-to-gguf.py index 80b09110f..1908e6440 100755 --- a/convert-gptneox-hf-to-gguf.py +++ b/convert-gptneox-hf-to-gguf.py @@ -140,7 +140,7 @@ for i in range(vocab_size): print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.") pad_token = f"[PAD{i}]".encode("utf8") text = bytearray(pad_token) - elif i in added_tokens: + elif i in added_token_ids: # these tokens are not encoded, see https://github.com/huggingface/transformers/issues/1133 text = bytearray(reverse_vocab[i].encode('utf-8')) else: diff --git a/convert-starcoder-hf-to-gguf.py b/convert-starcoder-hf-to-gguf.py index 0e0a1ccc7..e3c06b227 100755 --- a/convert-starcoder-hf-to-gguf.py +++ b/convert-starcoder-hf-to-gguf.py @@ -128,7 +128,7 @@ for i in range(vocab_size): print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.") pad_token = f"[PAD{i}]".encode("utf8") text = bytearray(pad_token) - elif i in added_tokens: + elif i in added_token_ids: # these tokens are not encoded, see https://github.com/huggingface/transformers/issues/1133 text = bytearray(reverse_vocab[i].encode('utf-8')) else: