From 6ec856b3efab650c74e659db7393efee7c184c5e Mon Sep 17 00:00:00 2001 From: wonjun Jang Date: Sun, 15 Oct 2023 09:44:07 +0000 Subject: [PATCH] remove debug code --- convert.py | 1 - 1 file changed, 1 deletion(-) diff --git a/convert.py b/convert.py index e84db460f..4868b66f9 100755 --- a/convert.py +++ b/convert.py @@ -465,7 +465,6 @@ class HFVocab: reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()} byte_encoder = bytes_to_unicode() byte_decoder = {v: k for k, v in byte_encoder.items()} - print(len(byte_decoder), byte_decoder) for i in range(tokenizer.vocab_size): text = reverse_vocab[i].encode("utf-8")