vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fail on invalid token id

Browse source
This commit is contained in:
Sigbjørn Skjæret 2024-05-09 12:21:11 +02:00 • committed by GitHub
parent 144d99a00a
commit efd70f3ce9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
gguf-py/scripts/gguf-new-metadata.py
View file

@ -209,14 +209,14 @@ def main() -> None:
if name not in token_names: if name not in token_names:
logger.warning(f'Unknown special token "{name}", ignoring...') logger.warning(f'Unknown special token "{name}", ignoring...')
elif not id_string.isdecimal(): elif not id_string.isdecimal():
logger.warning(f'Token ID "{id_string}" is not a valid ID, ignoring...') raise LookupError(f'Token ID "{id_string}" is not a valid ID!')
else: else:
id_int = int(id_string) id_int = int(id_string)
if id_int >= 0 and id_int < len(token_list): if id_int >= 0 and id_int < len(token_list):
new_metadata[token_names[name]] = MetadataDetails(gguf.GGUFValueType.UINT32, id_int, f'= {token_list[id_int]}') new_metadata[token_names[name]] = MetadataDetails(gguf.GGUFValueType.UINT32, id_int, f'= {token_list[id_int]}')
else: else:
logger.warning(f'Token ID {id_int} is not within token list, ignoring...') raise LookupError(f'Token ID {id_int} is not within token list!')
if os.path.isfile(args.output) and not args.force: if os.path.isfile(args.output) and not args.force:
logger.warning('*** Warning *** Warning *** Warning **') logger.warning('*** Warning *** Warning *** Warning **')