vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

hack in empty tokens for unknown vocab

Browse source
This commit is contained in:
Aman Karmani 2023-07-06 14:08:32 -07:00
parent dedd2067e8
commit 58d663d327
1 changed files with 5 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

5
convert.py
View file

@ -219,6 +219,8 @@ class XgenVocab:
for index in range(0, self.vocab_size_base): for index in range(0, self.vocab_size_base):
token = self.xt._convert_id_to_token(index) token = self.xt._convert_id_to_token(index)
yield (token, float(index)) yield (token, float(index))
for index in range(self.vocab_size_base, self.vocab_size):
yield (b'', float(index))
def __repr__(self) -> str: def __repr__(self) -> str:
return f"<XgenVocab with {self.vocab_size_base} base tokens and {len(self.added_tokens_list)} added tokens>" return f"<XgenVocab with {self.vocab_size_base} base tokens and {len(self.added_tokens_list)} added tokens>"
@ -977,6 +979,9 @@ def check_vocab_size(params: Params, vocab: Vocab) -> None:
vocab.added_tokens_list = [] vocab.added_tokens_list = []
vocab.vocab_size = vocab.vocab_size_base vocab.vocab_size = vocab.vocab_size_base
return return
if isinstance(vocab, XgenVocab):
vocab.vocab_size = params.n_vocab
return
msg = f"Vocab size mismatch (model has {params.n_vocab}, but {vocab.fname_tokenizer}" msg = f"Vocab size mismatch (model has {params.n_vocab}, but {vocab.fname_tokenizer}"
if vocab.fname_added_tokens is not None: if vocab.fname_added_tokens is not None:
msg += f" combined with {vocab.fname_added_tokens}" msg += f" combined with {vocab.fname_added_tokens}"