vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove inactive code.

Browse source
This commit is contained in:
goerch 2023-08-06 13:51:26 +02:00
parent bb6a58d0c3
commit 5d52192f73
1 changed files with 0 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
convert.py
View file

@ -267,8 +267,6 @@ class SentencePieceVocab:
byte_encoder = tokenization_gpt2.bytes_to_unicode() byte_encoder = tokenization_gpt2.bytes_to_unicode()
byte_decoder = {v: k for k, v in byte_encoder.items()} byte_decoder = {v: k for k, v in byte_encoder.items()}
for i, item in enumerate(tokenizer): for i, item in enumerate(tokenizer):
# text: bytes
# text = b''.join([x.to_bytes(1, byteorder='big') for x in [byte_decoder[y] for y in item]])
text: bytes = item.encode("utf-8") text: bytes = item.encode("utf-8")
score: float = -i score: float = -i
yield text, score yield text, score