vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

byte_decoder -> byte_encoder

Browse source
This commit is contained in:
simonJJJ 2023-09-05 12:08:25 +08:00
parent 2ba85c8609
commit cbdc56491d
2 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
convert-falcon-hf-to-gguf.py
View file

@ -154,7 +154,7 @@ for i in range(vocab_size):
text = bytearray() text = bytearray()
for c in reverse_vocab[i]: for c in reverse_vocab[i]:
if ord(c) < 256: # single byte character if ord(c) < 256: # single byte character
text.append(byte_decoder[ord(c)]) text.append(byte_encoder[ord(c)])
else: # multibyte special token character else: # multibyte special token character
text.extend(c.encode('utf-8')) text.extend(c.encode('utf-8'))
else: else:

2
convert-gptneox-hf-to-gguf.py
View file

@ -149,7 +149,7 @@ for i in range(vocab_size):
text = bytearray() text = bytearray()
for c in reverse_vocab[i]: for c in reverse_vocab[i]:
if ord(c) < 256: # single byte character if ord(c) < 256: # single byte character
text.append(byte_decoder[ord(c)]) text.append(byte_encoder[ord(c)])
else: # multibyte special token character else: # multibyte special token character
text.extend(c.encode('utf-8')) text.extend(c.encode('utf-8'))
else: else: