vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update convert-hf-to-gguf.py

Browse source 
Co-authored-by: Jared Van Bortel <cebtenzzre@gmail.com>
This commit is contained in:
Douglas Hanley 2024-02-08 17:33:14 -05:00 • committed by GitHub
parent 7218c7b613
commit e3efcf13c8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 3 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

7
convert-hf-to-gguf.py
View file

@ -1666,12 +1666,11 @@ class BertModel(Model):
# convert to phantom space vocab # convert to phantom space vocab
def phantom(tok, typ): def phantom(tok, typ):
if tok.startswith(b'[') and tok.endswith(b']'): if tok.startswith(b"[") and tok.endswith(b"]"):
return tok return tok
elif tok.startswith(b"##"): if tok.startswith(b"##"):
return tok[2:] return tok[2:]
else: return b"\xe2\x96\x81" + tok
return b"\xe2\x96\x81" + tok
tokens = [phantom(t, y) for t, y in zip(tokens, toktypes)] tokens = [phantom(t, y) for t, y in zip(tokens, toktypes)]
# set up bos and eos tokens (cls and sep) # set up bos and eos tokens (cls and sep)