From 52bdc7e946c5cf05143f5c261aa801b4a9638bb0 Mon Sep 17 00:00:00 2001 From: KerfuffleV2 Date: Thu, 9 Nov 2023 14:52:44 -0700 Subject: [PATCH] Reorganize scripts --- gguf-py/README.md | 8 +- gguf-py/examples/modify_gguf.py | 75 ---------------- gguf-py/gguf/gguf_reader.py | 2 +- .../scripts/gguf-convert-endian.py | 52 +++++++---- .../dump_gguf.py => scripts/gguf-dump.py} | 28 +++--- gguf-py/scripts/gguf-set-metadata.py | 88 +++++++++++++++++++ 6 files changed, 149 insertions(+), 104 deletions(-) delete mode 100755 gguf-py/examples/modify_gguf.py rename convert-gguf-endian.py => gguf-py/scripts/gguf-convert-endian.py (70%) rename gguf-py/{examples/dump_gguf.py => scripts/gguf-dump.py} (66%) create mode 100755 gguf-py/scripts/gguf-set-metadata.py diff --git a/gguf-py/README.md b/gguf-py/README.md index d0f5450a0..502b6a510 100644 --- a/gguf-py/README.md +++ b/gguf-py/README.md @@ -11,13 +11,15 @@ as an example for its usage. pip install gguf ``` -## API Examples +## API Examples/Simple Tools [examples/writer.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/writer.py) — Generates `example.gguf` in the current directory to demonstrate generating a GGUF file. Note that this file cannot be used as a model. -[examples/dump_gguf.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/dump_gguf.py) — Dumps a GGUF file's metadata to the console. +[scripts/gguf-dump.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/scripts/gguf-dump.py) — Dumps a GGUF file's metadata to the console. -[examples/modify_gguf.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/dump_gguf.py) — Allows changing simple metadata values in a GGUF file by key. +[scripts/gguf-set-metadata.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/scripts/gguf-set-metadata.py) — Allows changing simple metadata values in a GGUF file by key. + +[scripts/gguf-convert-endian.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/scripts/gguf-convert-endian.py) — Allows converting the endianness of GGUF files. ## Development Maintainers who participate in development of this package are advised to install it in editable mode: diff --git a/gguf-py/examples/modify_gguf.py b/gguf-py/examples/modify_gguf.py deleted file mode 100755 index c53856d2b..000000000 --- a/gguf-py/examples/modify_gguf.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python3 -import sys -from pathlib import Path - -# Necessary to load the local gguf package -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from gguf import GGUFReader # noqa: E402 - - -def minimal_example(filename: str) -> None: - reader = GGUFReader(filename, 'r+') - field = reader.fields['tokenizer.ggml.bos_token_id'] - if field is None: - return - part_index = field.data[0] - field.parts[part_index][0] = 2 # Set tokenizer.ggml.bos_token_id to 2 - # - # So what's this field.data thing? It's helpful because field.parts contains - # _every_ part of the GGUF field. For example, tokenizer.ggml.bos_token_id consists - # of: - # - # Part index 0: Key length (27) - # Part index 1: Key data ("tokenizer.ggml.bos_token_id") - # Part index 2: Field type (4, the id for GGUFValueType.UINT32) - # Part index 3: Field value - # - # Note also that each part is an NDArray slice, so even a part that - # is only a single value like the key length will be a NDArray of - # the key length type (numpy.uint32). - # - # The .data attribute in the Field is a list of relevant part indexes - # and doesn't contain internal GGUF details like the key length part. - # In this case, .data will be [3] - just the part index of the - # field value itself. - - -def change_gguf(reader: GGUFReader, key: str, value: str) -> None: - field = reader.get_field(key) - if field is None: - print(f'! Field {repr(key)} not found', file = sys.stderr) - sys.exit(1) - # Note that field.types is a list of types. This is because the GGUF - # format supports arrays. For example, an array of UINT32 would - # look like [GGUFValueType.ARRAY, GGUFValueType.UINT32] - handler = reader.gguf_scalar_to_np.get(field.types[0]) if field.types else None - if handler is None: - print(f'! Field {repr(key)} has unsupported type: {field.types}') - sys.exit(1) - current_value = field.parts[field.data[0]][0] - new_value = handler(value) - print(f'* Preparing to change field {repr(key)} from {current_value} to {new_value}') - if current_value == new_value: - print(f'- Key {repr(key)} already set to requested value {current_value}') - sys.exit(0) - print('*** Warning *** Warning *** Warning **') - print('* Changing fields in a GGUF file can damage it. If you are positive then type YES:') - response = input('YES, I am sure> ') - if response != 'YES': - print("You didn't enter YES. Okay then, see ya!") - sys.exit(0) - field.parts[field.data[0]][0] = new_value - print('* Field changed. Successful completion.') - - -if __name__ == '__main__': - if len(sys.argv) < 4: - print( - 'modify_gguf: Error: Missing arguments. Syntax: modify_gguf.py ', - file = sys.stderr, - ) - sys.exit(1) - print(f'* Loading: {sys.argv[1]}') - reader = GGUFReader(sys.argv[1], 'r+') - change_gguf(reader, sys.argv[2], sys.argv[3]) diff --git a/gguf-py/gguf/gguf_reader.py b/gguf-py/gguf/gguf_reader.py index fb4b8a167..a3b331b0f 100644 --- a/gguf-py/gguf/gguf_reader.py +++ b/gguf-py/gguf/gguf_reader.py @@ -1,6 +1,6 @@ # # GGUF file reading/modification support. For API usage information, -# please see examples/modify_gguf.py and examples/dump_gguf.py +# please see the files scripts/ for some fairly simple examples. # from __future__ import annotations diff --git a/convert-gguf-endian.py b/gguf-py/scripts/gguf-convert-endian.py similarity index 70% rename from convert-gguf-endian.py rename to gguf-py/scripts/gguf-convert-endian.py index 96341b6d8..bf8c194a1 100755 --- a/convert-gguf-endian.py +++ b/gguf-py/scripts/gguf-convert-endian.py @@ -1,21 +1,21 @@ #!/usr/bin/env python3 from __future__ import annotations +import argparse import os import sys from pathlib import Path import numpy as np -if "NO_LOCAL_GGUF" not in os.environ: - sys.path.insert(1, str(Path(__file__).parent / "gguf-py")) +# Necessary to load the local gguf package +if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists(): + sys.path.insert(0, str(Path(__file__).parent.parent)) + import gguf -def convert_byteorder(filename: str, order: str) -> None: - if order not in ("big", "little", "native"): - raise ValueError(f"Bad order parameter {order}") - reader = gguf.GGUFReader(filename, "r+") +def convert_byteorder(reader: gguf.GGUFReader, args: argparse.Namespace) -> None: if np.uint32(1) == np.uint32(1).newbyteorder("<"): # Host is little endian host_endian = "little" @@ -28,7 +28,7 @@ def convert_byteorder(filename: str, order: str) -> None: file_endian = swapped_endian else: file_endian = host_endian - if order == "native": + if args.order == "native": order = host_endian print(f"* Host is {host_endian.upper()} endian, GGUF file seems to be {file_endian.upper()} endian") if file_endian == order: @@ -43,10 +43,14 @@ def convert_byteorder(filename: str, order: str) -> None: ): raise ValueError(f"Cannot handle type {tensor.tensor_type.name} for tensor {repr(tensor.name)}") print(f"* Preparing to convert from {file_endian.upper()} to {order.upper()}") + if args.dry_run: + return print("\n*** Warning *** Warning *** Warning **") print("* This conversion process may damage the file. Ensure you have a backup.") + if order != host_endian: + print("* Requested endian differs from host, you will not be able to load the model on this machine.") print("* The file will be modified immediately, so if conversion fails or is interrupted") - print("* the file will be corrupted. If you are positive then type YES:") + print("* the file will be corrupted. Enter exactly YES if you are positive you want to proceed:") response = input("YES, I am sure> ") if response != "YES": print("You didn't enter YES. Okay then, see ya!") @@ -85,11 +89,29 @@ def convert_byteorder(filename: str, order: str) -> None: print("* Completion") +def main() -> None: + parser = argparse.ArgumentParser(description="Convert GGUF file byte order") + parser.add_argument( + "model", + type=str, + help="GGUF format model filename", + ) + parser.add_argument( + "order", + type=str, + choices=['big','little','native'], + help="Requested byte order", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Don't actually change anything" + ) + args = parser.parse_args(None if len(sys.argv) > 1 else ["--help"]) + print(f'* Loading: {args.model}') + reader = gguf.GGUFReader(args.model, 'r' if args.dry_run else 'r+') + convert_byteorder(reader, args) + + if __name__ == "__main__": - if len(sys.argv) < 3: - print( - "convert_endian: Error: Missing arguments. Syntax: modify_gguf.py ", - file=sys.stderr, - ) - sys.exit(1) - convert_byteorder(sys.argv[1], sys.argv[2]) + main() diff --git a/gguf-py/examples/dump_gguf.py b/gguf-py/scripts/gguf-dump.py similarity index 66% rename from gguf-py/examples/dump_gguf.py rename to gguf-py/scripts/gguf-dump.py index a2dab0887..923898d3b 100755 --- a/gguf-py/examples/dump_gguf.py +++ b/gguf-py/scripts/gguf-dump.py @@ -1,18 +1,19 @@ #!/usr/bin/env python3 +import argparse +import os import sys from pathlib import Path # Necessary to load the local gguf package -sys.path.insert(0, str(Path(__file__).parent.parent)) +if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists(): + sys.path.insert(0, str(Path(__file__).parent.parent)) from gguf import GGUFReader, GGUFValueType # noqa: E402 # For more information about what field.parts and field.data represent, # please see the comments in the modify_gguf.py example. -def dump_gguf(filename: str) -> None: - print(f'* Loading: {filename}') - reader = GGUFReader(filename, 'r') +def dump_metadata(reader: GGUFReader, dump_tensors: bool = True) -> None: print(f'\n* Dumping {len(reader.fields)} key/value pair(s)') for n, field in enumerate(reader.fields.values(), 1): if not field.types: @@ -30,16 +31,23 @@ def dump_gguf(filename: str) -> None: elif field.types[0] in reader.gguf_scalar_to_np: print(' = {0}'.format(field.parts[-1][0]), end = '') print() - + if not dump_tensors: + return print(f'\n* Dumping {len(reader.tensors)} tensor(s)') for n, tensor in enumerate(reader.tensors, 1): - prettydims = ', '.join('{0:5}'.format(d) for d in list(tensor.shape) + [1] * (4 - len(tensor.shape))) print(f' {n:5}: {tensor.n_elements:10} | {prettydims} | {tensor.tensor_type.name:7} | {tensor.name}') +def main() -> None: + parser = argparse.ArgumentParser(description="Dump GGUF file metadata") + parser.add_argument("model", type=str, help="GGUF format model filename") + parser.add_argument("--no-tensors", action="store_true", help="Don't dump tensor metadata") + args = parser.parse_args(None if len(sys.argv) > 1 else ["--help"]) + print(f'* Loading: {args.model}') + reader = GGUFReader(args.model, 'r') + dump_metadata(reader, not args.no_tensors) + + if __name__ == '__main__': - if len(sys.argv) < 2: - print('dump_gguf: Error: Specify an input file', file = sys.stderr) - sys.exit(1) - dump_gguf(sys.argv[1]) + main() diff --git a/gguf-py/scripts/gguf-set-metadata.py b/gguf-py/scripts/gguf-set-metadata.py new file mode 100755 index 000000000..fcb0e9cd1 --- /dev/null +++ b/gguf-py/scripts/gguf-set-metadata.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +import argparse +import os +import sys +from pathlib import Path + +# Necessary to load the local gguf package +if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists(): + sys.path.insert(0, str(Path(__file__).parent.parent)) + +from gguf import GGUFReader # noqa: E402 + + +def minimal_example(filename: str) -> None: + reader = GGUFReader(filename, 'r+') + field = reader.fields['tokenizer.ggml.bos_token_id'] + if field is None: + return + part_index = field.data[0] + field.parts[part_index][0] = 2 # Set tokenizer.ggml.bos_token_id to 2 + # + # So what's this field.data thing? It's helpful because field.parts contains + # _every_ part of the GGUF field. For example, tokenizer.ggml.bos_token_id consists + # of: + # + # Part index 0: Key length (27) + # Part index 1: Key data ("tokenizer.ggml.bos_token_id") + # Part index 2: Field type (4, the id for GGUFValueType.UINT32) + # Part index 3: Field value + # + # Note also that each part is an NDArray slice, so even a part that + # is only a single value like the key length will be a NDArray of + # the key length type (numpy.uint32). + # + # The .data attribute in the Field is a list of relevant part indexes + # and doesn't contain internal GGUF details like the key length part. + # In this case, .data will be [3] - just the part index of the + # field value itself. + + +def set_metadata(reader: GGUFReader, args: argparse.Namespace) -> None: + field = reader.get_field(args.key) + if field is None: + print(f'! Field {repr(args.key)} not found', file = sys.stderr) + sys.exit(1) + # Note that field.types is a list of types. This is because the GGUF + # format supports arrays. For example, an array of UINT32 would + # look like [GGUFValueType.ARRAY, GGUFValueType.UINT32] + handler = reader.gguf_scalar_to_np.get(field.types[0]) if field.types else None + if handler is None: + print(f'! This tool only supports changing simple values, {repr(args.key)} has unsupported type {field.types}', + file = sys.stderr) + sys.exit(1) + current_value = field.parts[field.data[0]][0] + new_value = handler(args.value) + print(f'* Preparing to change field {repr(args.key)} from {current_value} to {new_value}') + if current_value == new_value: + print(f'- Key {repr(args.key)} already set to requested value {current_value}') + sys.exit(0) + if args.dry_run: + sys.exit(0) + if not args.force: + print('*** Warning *** Warning *** Warning **') + print('* Changing fields in a GGUF file can make it unusable. Proceed at your own risk.') + print('* Enter exactly YES if you are positive you want to proceed:') + response = input('YES, I am sure> ') + if response != 'YES': + print("You didn't enter YES. Okay then, see ya!") + sys.exit(0) + field.parts[field.data[0]][0] = new_value + print('* Field changed. Successful completion.') + + +def main() -> None: + parser = argparse.ArgumentParser(description="Set a simple value in GGUF file metadata") + parser.add_argument("model", type=str, help="GGUF format model filename") + parser.add_argument("key", type=str, help="Metadata key to set") + parser.add_argument("value", type=str, help="Metadata value to set") + parser.add_argument("--dry-run", action="store_true", help="Don't actually change anything") + parser.add_argument("--force", action="store_true", help="Change the field without confirmation") + args = parser.parse_args(None if len(sys.argv) > 1 else ["--help"]) + print(f'* Loading: {args.model}') + reader = GGUFReader(args.model, 'r' if args.dry_run else 'r+') + set_metadata(reader, args) + + +if __name__ == '__main__': + main()