vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

gguf: Fix special vocab handling when id < 0

Browse source
This commit is contained in:
KerfuffleV2 2023-09-03 01:41:26 -06:00
parent 6a31a3bd98
commit c3d3ea9b3a
2 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
gguf-py/gguf/gguf.py
View file

@ -801,7 +801,7 @@ class SpecialVocab:
else: else:
continue continue
for maybe_token_id in (atok.get('id') for atok in added_tokens if atok.get('content') == tc_content): for maybe_token_id in (atok.get('id') for atok in added_tokens if atok.get('content') == tc_content):
if isinstance(maybe_token_id, int): if isinstance(maybe_token_id, int) and maybe_token_id >= 0:
self.special_token_ids[typ] = maybe_token_id self.special_token_ids[typ] = maybe_token_id
break break
return True return True
@ -814,7 +814,7 @@ class SpecialVocab:
config = json.load(f) config = json.load(f)
for typ in self.special_token_types: for typ in self.special_token_types:
maybe_token_id = config.get(f'{typ}_token_id') maybe_token_id = config.get(f'{typ}_token_id')
if isinstance(maybe_token_id, int): if isinstance(maybe_token_id, int) and maybe_token_id >= 0:
self.special_token_ids[typ] = maybe_token_id self.special_token_ids[typ] = maybe_token_id
return True return True

2
gguf-py/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "gguf" name = "gguf"
version = "0.3.1" version = "0.3.2"
description = "Write ML models in GGUF for GGML" description = "Write ML models in GGUF for GGML"
authors = ["GGML <ggml@ggml.ai>"] authors = ["GGML <ggml@ggml.ai>"]
packages = [ packages = [