convert.py : add python logging instead of print() (#6511)

* convert.py: add python logging instead of print()

* convert.py: verbose flag takes priority over dump flag log suppression

* convert.py: named instance logging

* convert.py: use explicit logger id string

* convert.py: convert extra print() to named logger

* convert.py: sys.stderr.write --> logger.error

* *.py: Convert all python scripts to use logging module

* requirements.txt: remove extra line

* flake8: update flake8 ignore and exclude to match ci settings

* gh-actions: add flake8-no-print to flake8 lint step

* pre-commit: add flake8-no-print to flake8 and also update pre-commit version

* convert-hf-to-gguf.py: print() to logger conversion

* *.py: logging basiconfig refactor to use conditional expression

* *.py: removed commented out logging

* fixup! *.py: logging basiconfig refactor to use conditional expression

* constant.py: logger.error then exit should be a raise exception instead

* *.py: Convert logger error and sys.exit() into a raise exception (for atypical error)

* gguf-convert-endian.py: refactor convert_byteorder() to use tqdm progressbar

* verify-checksum-model.py: This is the result of the program, it should be printed to stdout.

* compare-llama-bench.py: add blank line for readability during missing repo response

* reader.py: read_gguf_file() use print() over logging

* convert.py: warning goes to stderr and won't hurt the dump output

* gguf-dump.py: dump_metadata() should print to stdout

* convert-hf-to-gguf.py: print --> logger.debug or ValueError()

* verify-checksum-models.py: use print() for printing table

* *.py: refactor logging.basicConfig()

* gguf-py/gguf/*.py: use __name__ as logger name

Since they will be imported and not run directly.

* python-lint.yml: use .flake8 file instead

* constants.py: logger no longer required

* convert-hf-to-gguf.py: add additional logging

* convert-hf-to-gguf.py: print() --> logger

* *.py: fix flake8 warnings

* revert changes to convert-hf-to-gguf.py for get_name()

* convert-hf-to-gguf-update.py: use triple quoted f-string instead

* *.py: accidentally corrected the wrong line

* *.py: add compilade warning suggestions and style fixes

tests/test-tokenizer-0-bpe.py

@@ -7,15 +7,20 @@
 #   python3 tests/test-tokenizer-0-bpe.py ~/Data/huggingface/deepseek-coder-6.7b-instruct/
 #
 
+import logging
 import argparse
 
 from transformers import AutoTokenizer
 
+logger = logging.getLogger("test-tokenizer-0-bpe")
+
 parser = argparse.ArgumentParser()
 parser.add_argument("dir_tokenizer", help="directory containing 'tokenizer.model' file")
 parser.add_argument("--fname-tok",   help="path to a text file to tokenize")
-args = parser.parse_args()
+parser.add_argument("--verbose", action="store_true", help="increase output verbosity")
 
+args = parser.parse_args()
+logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
 dir_tokenizer = args.dir_tokenizer
 
 tokenizer = AutoTokenizer.from_pretrained(dir_tokenizer)
@@ -64,30 +69,34 @@ tests = [
 ]
 
 for text in tests:
-    print('text: ', text)
-    print(tokenizer.encode(text))
-    print(tokenizer.decode(tokenizer.encode(text)))
+    logger.info(f"text: {text}")
+    logger.info(tokenizer.encode(text))
+    logger.info(tokenizer.decode(tokenizer.encode(text)))
 
-print("\n\ntests for C++:\n")
+logger.info("tests for C++:")
 for text in tests:
     res = tokenizer.encode(text)
 
+    # Modify text representation for logging
     k = text.replace('\n', '\\n')
     k = k.replace('\t', '\\t')
     k = '"' + k + '"'
-    print("{ %-24s, { " % k, end='')
-    for x in res:
-        print("%7d," % x, end='')
-    print(" }, },")
 
-print(tokenizer.encode('hello'))
-print(tokenizer.encode('world'))
-print(tokenizer.encode(' world'))
-print(tokenizer.encode('hello world'))
+    # Log the modified text and its encoding
+    log_message = "{ %-24s, { " % k
+    for x in res:
+        log_message += "%7d," % x
+    log_message += " }, },"
+    logger.info(log_message)
+
+logger.info(tokenizer.encode('hello'))
+logger.info(tokenizer.encode('world'))
+logger.info(tokenizer.encode(' world'))
+logger.info(tokenizer.encode('hello world'))
 
 fname_tok = args.fname_tok
 if fname_tok:
-    print('tokenizing file: ', fname_tok)
+    logger.info(f"tokenizing file: {fname_tok}")
     fname_out = fname_tok + '.tok'
     with open(fname_tok, 'r', encoding='utf-8') as f:
         lines = f.readlines()
@@ -112,6 +121,6 @@ if fname_tok:
                 # else:
                 #     f.write(str(x) + ' \'' + tokenizer.decode(x) + '\'\n')
                 f.write(str(x) + ' \'' + tokenizer.decode(x).strip() + '\'\n')
-        print('len(res): ', len(res))
-        print('len(lines): ', len(lines))
-    print('results written to: ', fname_out)
+        logger.info(f"len(res): {len(res)}")
+        logger.info(f"len(lines): {len(lines)}")
+    logger.info(f"results written to: {fname_out}")