vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

s/else if/elif/

Browse source
This commit is contained in:
Cebtenzzre 2023-10-01 17:02:39 -04:00
parent 8c8b5b0666
commit f18cfeab62
3 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
convert-falcon-hf-to-gguf.py
View file

@ -144,7 +144,7 @@ for i in range(vocab_size):
print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.") print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
pad_token = f"[PAD{i}]".encode("utf8") pad_token = f"[PAD{i}]".encode("utf8")
text = bytearray(pad_token) text = bytearray(pad_token)
else if i in added_tokens: elif i in added_tokens:
# these tokens are not encoded, see https://github.com/huggingface/transformers/issues/1133 # these tokens are not encoded, see https://github.com/huggingface/transformers/issues/1133
text = bytearray(reverse_vocab[i].encode('utf-8')) text = bytearray(reverse_vocab[i].encode('utf-8'))
else: else:

2
convert-gptneox-hf-to-gguf.py
View file

@ -140,7 +140,7 @@ for i in range(vocab_size):
print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.") print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
pad_token = f"[PAD{i}]".encode("utf8") pad_token = f"[PAD{i}]".encode("utf8")
text = bytearray(pad_token) text = bytearray(pad_token)
else if i in added_tokens: elif i in added_tokens:
# these tokens are not encoded, see https://github.com/huggingface/transformers/issues/1133 # these tokens are not encoded, see https://github.com/huggingface/transformers/issues/1133
text = bytearray(reverse_vocab[i].encode('utf-8')) text = bytearray(reverse_vocab[i].encode('utf-8'))
else: else:

2
convert-starcoder-hf-to-gguf.py
View file

@ -128,7 +128,7 @@ for i in range(vocab_size):
print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.") print(f"Key {i} not in tokenizer vocabulary. Padding with an arbitrary token.")
pad_token = f"[PAD{i}]".encode("utf8") pad_token = f"[PAD{i}]".encode("utf8")
text = bytearray(pad_token) text = bytearray(pad_token)
else if i in added_tokens: elif i in added_tokens:
# these tokens are not encoded, see https://github.com/huggingface/transformers/issues/1133 # these tokens are not encoded, see https://github.com/huggingface/transformers/issues/1133
text = bytearray(reverse_vocab[i].encode('utf-8')) text = bytearray(reverse_vocab[i].encode('utf-8'))
else: else: