From e39905050f65c78c0486bdc4ef96ad3127a6e88a Mon Sep 17 00:00:00 2001 From: Galunid Date: Sun, 22 Oct 2023 21:21:31 +0200 Subject: [PATCH] Fix added_tokens crashes --- convert-stablelm-hf-to-gguf.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/convert-stablelm-hf-to-gguf.py b/convert-stablelm-hf-to-gguf.py index 3dceb32be..bc3c84f2d 100755 --- a/convert-stablelm-hf-to-gguf.py +++ b/convert-stablelm-hf-to-gguf.py @@ -112,14 +112,20 @@ vocab_size = hparams.get("vocab_size", len(tokenizer.vocab)) assert max(tokenizer.vocab.values()) < vocab_size reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()} +added_vocab = tokenizer.get_added_vocab() for i in range(vocab_size): - tokens.append(reverse_vocab[i] if i in reverse_vocab else f"[PAD{i}]") - scores.append(0.0) # dummy - toktypes.append(gguf.TokenType.NORMAL) + if i not in reverse_vocab: + tokens.append(f"[PAD{i}]") + toktypes.append(gguf.TokenType.USER_DEFINED) + elif reverse_vocab[i] in added_vocab: + tokens.append(reverse_vocab[i]) + toktypes.append(gguf.TokenType.USER_DEFINED) + else: + tokens.append(reverse_vocab[i]) + toktypes.append(gguf.TokenType.NORMAL) gguf_writer.add_token_list(tokens) -gguf_writer.add_token_scores(scores) gguf_writer.add_token_types(toktypes) special_vocab = gguf.SpecialVocab(dir_model, load_merges = True)