vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

convert : restore Falcon vocab padding

Browse source
This commit is contained in:
cebtenzzre 2023-10-30 23:05:05 -04:00
parent 6e08281e58
commit 08f183c229
6 changed files with 13 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

1
convert-bloom-hf-to-gguf.py
View file

@ -102,7 +102,6 @@ gguf_writer.add_file_type(ftype)
print("gguf: get tokenizer metadata") print("gguf: get tokenizer metadata")
tokens: list[bytearray] = [] tokens: list[bytearray] = []
scores: list[float] = []
toktypes: list[int] = [] toktypes: list[int] = []
# gpt2 tokenizer # gpt2 tokenizer

14
convert-falcon-hf-to-gguf.py
View file

@ -125,7 +125,6 @@ gguf_writer.add_file_type(ftype)
print("gguf: get tokenizer metadata") print("gguf: get tokenizer metadata")
tokens: list[bytearray] = [] tokens: list[bytearray] = []
scores: list[float] = []
toktypes: list[int] = [] toktypes: list[int] = []
# gpt2 tokenizer # gpt2 tokenizer
@ -141,15 +140,24 @@ tokenizer = AutoTokenizer.from_pretrained(dir_model)
vocab_size = hparams.get("vocab_size", len(tokenizer.vocab)) vocab_size = hparams.get("vocab_size", len(tokenizer.vocab))
assert max(tokenizer.vocab.values()) < vocab_size assert max(tokenizer.vocab.values()) < vocab_size
added_vocab = tokenizer.get_added_vocab()
reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()} reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
for i in range(vocab_size): for i in range(vocab_size):
if i not in reverse_vocab:
tokens.append(f"[PAD{i}]")
toktypes.append(gguf.TokenType.USER_DEFINED)
elif reverse_vocab[i] in added_vocab:
tokens.append(reverse_vocab[i])
if tokenizer.added_tokens_decoder[i].special:
toktypes.append(gguf.TokenType.CONTROL)
else:
toktypes.append(gguf.TokenType.USER_DEFINED)
else:
tokens.append(reverse_vocab[i]) tokens.append(reverse_vocab[i])
scores.append(0.0) # dummy
toktypes.append(gguf.TokenType.NORMAL) toktypes.append(gguf.TokenType.NORMAL)
gguf_writer.add_token_list(tokens) gguf_writer.add_token_list(tokens)
gguf_writer.add_token_scores(scores)
gguf_writer.add_token_types(toktypes) gguf_writer.add_token_types(toktypes)
special_vocab = gguf.SpecialVocab(dir_model, load_merges = True, n_vocab = len(tokens)) special_vocab = gguf.SpecialVocab(dir_model, load_merges = True, n_vocab = len(tokens))

1
convert-gptneox-hf-to-gguf.py
View file

@ -107,7 +107,6 @@ gguf_writer.add_layer_norm_eps(hparams["layer_norm_eps"])
print("gguf: get tokenizer metadata") print("gguf: get tokenizer metadata")
tokens: list[bytearray] = [] tokens: list[bytearray] = []
scores: list[float] = []
toktypes: list[int] = [] toktypes: list[int] = []
# gpt2 tokenizer # gpt2 tokenizer

1
convert-mpt-hf-to-gguf.py
View file

@ -110,7 +110,6 @@ gguf_writer.add_max_alibi_bias(hparams["attn_config"]["alibi_bias_max"])
print("gguf: get tokenizer metadata") print("gguf: get tokenizer metadata")
tokens: list[bytearray] = [] tokens: list[bytearray] = []
scores: list[float] = []
toktypes: list[int] = [] toktypes: list[int] = []
# gpt2 tokenizer # gpt2 tokenizer

1
convert-refact-hf-to-gguf.py
View file

@ -123,7 +123,6 @@ gguf_writer.add_file_type(ftype)
print("gguf: get tokenizer metadata") print("gguf: get tokenizer metadata")
tokens: list[bytearray] = [] tokens: list[bytearray] = []
scores: list[float] = []
toktypes: list[int] = [] toktypes: list[int] = []
# gpt2 tokenizer # gpt2 tokenizer

1
convert-starcoder-hf-to-gguf.py
View file

@ -95,7 +95,6 @@ gguf_writer.add_file_type(ftype)
print("gguf: get tokenizer metadata") print("gguf: get tokenizer metadata")
tokens: list[bytearray] = [] tokens: list[bytearray] = []
scores: list[float] = []
toktypes: list[int] = [] toktypes: list[int] = []
# gpt2 tokenizer # gpt2 tokenizer