vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Replaced prints with logger calls.

Browse source
This commit is contained in:
Stanisław Szymczyk 2024-05-09 17:13:12 +02:00
parent a89257151f
commit 4ebb52cfc2
1 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
convert-hf-to-gguf.py
View file

@ -2290,7 +2290,7 @@ class ArcticModel(Model):
tokenizer_path = self.dir_model / 'tokenizer.model' tokenizer_path = self.dir_model / 'tokenizer.model'
if not tokenizer_path.is_file(): if not tokenizer_path.is_file():
print(f'Error: Missing {tokenizer_path}', file=sys.stderr) logger.error(f'Error: Missing {tokenizer_path}')
sys.exit(1) sys.exit(1)
# Read the whole vocabulary from the tokenizer.model file # Read the whole vocabulary from the tokenizer.model file
@ -2334,7 +2334,7 @@ class ArcticModel(Model):
for token_id, token_json in added_tokens_decoder.items(): for token_id, token_json in added_tokens_decoder.items():
token_id = int(token_id) token_id = int(token_id)
if (token_id >= vocab_size): if (token_id >= vocab_size):
print(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}') logger.debug(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}')
continue continue
token_content = token_json["content"] token_content = token_json["content"]
@ -2350,7 +2350,7 @@ class ArcticModel(Model):
token_type = SentencePieceTokenTypes.CONTROL token_type = SentencePieceTokenTypes.CONTROL
token_score = 0.0 token_score = 0.0
print(f"Setting token {token_id} to '{token_content}' (type: {token_type}, score: {token_score:.2f})") logger.info(f"Setting added token {token_id} to '{token_content}' (type: {token_type}, score: {token_score:.2f})")
tokens[token_id] = token_content.encode("utf-8") tokens[token_id] = token_content.encode("utf-8")
toktypes[token_id] = token_type toktypes[token_id] = token_type
scores[token_id] = token_score scores[token_id] = token_score