vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remove debug code

Browse source
This commit is contained in:
wonjun Jang 2023-10-15 09:44:07 +00:00 • committed by GitHub
parent c7b636e950
commit 6ec856b3ef
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 0 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

1
convert.py
View file

@ -465,7 +465,6 @@ class HFVocab:
reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()} reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
byte_encoder = bytes_to_unicode() byte_encoder = bytes_to_unicode()
byte_decoder = {v: k for k, v in byte_encoder.items()} byte_decoder = {v: k for k, v in byte_encoder.items()}
print(len(byte_decoder), byte_decoder)
for i in range(tokenizer.vocab_size): for i in range(tokenizer.vocab_size):
text = reverse_vocab[i].encode("utf-8") text = reverse_vocab[i].encode("utf-8")