vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

gguf-py : check chat_template type

Browse source
This commit is contained in:
slaren 2023-11-18 16:59:22 +01:00
parent 72c3a5d28a
commit edd98313ca
1 changed files with 8 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

9
gguf-py/gguf/vocab.py
View file

@ -137,6 +137,14 @@ class SpecialVocab:
return True return True
with open(tokenizer_config_file, encoding = 'utf-8') as f: with open(tokenizer_config_file, encoding = 'utf-8') as f:
tokenizer_config = json.load(f) tokenizer_config = json.load(f)
chat_template = tokenizer_config.get('chat_template')
if chat_template is None or isinstance(chat_template, str):
self.chat_template = chat_template
else:
print(
f'gguf: WARNING: Bad type for chat_template field in {tokenizer_config_file!r} - ignoring',
file = sys.stderr
)
for typ in self.special_token_types: for typ in self.special_token_types:
add_entry = tokenizer_config.get(f'add_{typ}_token') add_entry = tokenizer_config.get(f'add_{typ}_token')
if isinstance(add_entry, bool): if isinstance(add_entry, bool):
@ -161,7 +169,6 @@ class SpecialVocab:
None, None,
) )
self._set_special_token(typ, maybe_token_id) self._set_special_token(typ, maybe_token_id)
self.chat_template = tokenizer_config.get('chat_template')
return True return True
def _try_load_from_config_json(self, path: Path) -> bool: def _try_load_from_config_json(self, path: Path) -> bool: