From fffdac32b548d7b24f36a5a05543c213ddbdd178 Mon Sep 17 00:00:00 2001 From: KerfuffleV2 Date: Wed, 8 Nov 2023 09:01:13 -0700 Subject: [PATCH] Fix an issue with state init in GGUFReader Move examples to an examples/ directory Clean up examples Add an example of modifying keys in a GGUF file Update documentation with info on examples Try to support people importing gguf/gguf.py directly --- gguf-py/README.md | 8 ++++++ gguf-py/examples/dump_gguf.py | 41 ++++++++++++++++++++++++++ gguf-py/examples/modify_gguf.py | 41 ++++++++++++++++++++++++++ gguf-py/examples/writer.py | 37 ++++++++++++++++++++++++ gguf-py/gguf/gguf.py | 43 +++++++-------------------- gguf-py/gguf/gguf_reader.py | 51 +++++++++------------------------ gguf-py/tests/test_gguf.py | 2 +- 7 files changed, 153 insertions(+), 70 deletions(-) create mode 100644 gguf-py/examples/dump_gguf.py create mode 100644 gguf-py/examples/modify_gguf.py create mode 100644 gguf-py/examples/writer.py diff --git a/gguf-py/README.md b/gguf-py/README.md index a28d8c57a..d0f5450a0 100644 --- a/gguf-py/README.md +++ b/gguf-py/README.md @@ -11,6 +11,14 @@ as an example for its usage. pip install gguf ``` +## API Examples + +[examples/writer.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/writer.py) — Generates `example.gguf` in the current directory to demonstrate generating a GGUF file. Note that this file cannot be used as a model. + +[examples/dump_gguf.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/dump_gguf.py) — Dumps a GGUF file's metadata to the console. + +[examples/modify_gguf.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/dump_gguf.py) — Allows changing simple metadata values in a GGUF file by key. + ## Development Maintainers who participate in development of this package are advised to install it in editable mode: diff --git a/gguf-py/examples/dump_gguf.py b/gguf-py/examples/dump_gguf.py new file mode 100644 index 000000000..b4b97fd8f --- /dev/null +++ b/gguf-py/examples/dump_gguf.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +# Necessary to load the local gguf package +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from gguf import GGUFReader, GGUFValueType # noqa: E402 + +def dump_gguf(filename: str) -> None: + print(f'* Loading: {filename}') + reader = GGUFReader(filename, 'r') + print(f'\n* Dumping {len(reader.fields)} key/value pair(s)') + for n, field in enumerate(reader.fields.values(), 1): + if not field.types: + pretty_type = 'N/A' + elif field.types[0] == GGUFValueType.ARRAY: + nest_count = len(field.types) - 1 + pretty_type = '[' * nest_count + str(field.types[-1].name) + ']' * nest_count + else: + pretty_type = str(field.types[-1].name) + print(f' {n:5}: {pretty_type:10} | {len(field.data):8} | {field.name}', end = '') + if len(field.types) == 1: + curr_type = field.types[0] + if curr_type == GGUFValueType.STRING: + print(' = {0}'.format(repr(str(bytes(field.parts[-1]), encoding='utf8')[:60])), end = '') + elif field.types[0] in reader.gguf_scalar_to_np: + print(' = {0}'.format(field.parts[-1][0]), end = '') + print() + + print(f'\n* Dumping {len(reader.tensors)} tensor(s)') + for n, tensor in enumerate(reader.tensors, 1): + + prettydims = ', '.join('{0:5}'.format(d) for d in list(tensor.shape) + [1] * (4 - len(tensor.shape))) + print(f' {n:5}: {tensor.n_elements:10} | {prettydims} | {tensor.tensor_type.name:7} | {tensor.name}') + +if __name__ == '__main__': + if len(sys.argv) < 2: + print('dump_gguf: Error: Specify an input file', file = sys.stderr) + sys.exit(1) + dump_gguf(sys.argv[1]) diff --git a/gguf-py/examples/modify_gguf.py b/gguf-py/examples/modify_gguf.py new file mode 100644 index 000000000..8c7c670d0 --- /dev/null +++ b/gguf-py/examples/modify_gguf.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +# Necessary to load the local gguf package +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from gguf import GGUFReader # noqa: E402 + +def change_gguf(reader: GGUFReader, key: str, value: str) -> None: + field = reader.get_field(key) + if field is None: + print(f'! Field {repr(key)} not found', file = sys.stderr) + sys.exit(1) + + handler = reader.gguf_scalar_to_np.get(field.types[0]) if field.types else None + if handler is None: + print(f'! Field {repr(key)} has unsupported type: {field.types}') + sys.exit(1) + current_value = field.parts[field.data[0]][0] + new_value = handler(value) + print(f'* Preparing to change field {repr(key)} from {current_value} to {new_value}') + if current_value == new_value: + print(f'- Key {repr(key)} already set to requested value {current_value}') + sys.exit(0) + print('*** Warning *** Warning *** Warning **') + print('* Changing fields in a GGUF file can damagage it. If you are positive then type YES:') + response = input('YES, I am sure> ') + if response != 'YES': + print("You didn't enter YES. Okay then, see ya!") + sys.exit(0) + field.parts[field.data[0]][0] = new_value + print('* Field changed. Successful completion.') + +if __name__ == '__main__': + if len(sys.argv) < 4: + print('modify_gguf: Error: Missing arguments. Syntax: modify_gguf.py ', file = sys.stderr) + sys.exit(1) + print(f'* Loading: {sys.argv[1]}') + reader = GGUFReader(sys.argv[1], 'r+') + change_gguf(reader, sys.argv[2], sys.argv[3]) diff --git a/gguf-py/examples/writer.py b/gguf-py/examples/writer.py new file mode 100644 index 000000000..c47c693cc --- /dev/null +++ b/gguf-py/examples/writer.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +import numpy as np + +# Necessary to load the local gguf package +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from gguf import GGUFWriter # noqa: E402 + +# Example usage: +def writer_example() -> None: + # Example usage with a file + gguf_writer = GGUFWriter("example.gguf", "llama") + + gguf_writer.add_architecture() + gguf_writer.add_block_count(12) + gguf_writer.add_uint32("answer", 42) # Write a 32-bit integer + gguf_writer.add_float32("answer_in_float", 42.0) # Write a 32-bit float + gguf_writer.add_custom_alignment(64) + + tensor1 = np.ones((32,), dtype=np.float32) * 100.0 + tensor2 = np.ones((64,), dtype=np.float32) * 101.0 + tensor3 = np.ones((96,), dtype=np.float32) * 102.0 + + gguf_writer.add_tensor("tensor1", tensor1) + gguf_writer.add_tensor("tensor2", tensor2) + gguf_writer.add_tensor("tensor3", tensor3) + + gguf_writer.write_header_to_file() + gguf_writer.write_kv_data_to_file() + gguf_writer.write_tensors_to_file() + + gguf_writer.close() + +writer_example() diff --git a/gguf-py/gguf/gguf.py b/gguf-py/gguf/gguf.py index c484d67c1..651a81eb8 100644 --- a/gguf-py/gguf/gguf.py +++ b/gguf-py/gguf/gguf.py @@ -1,36 +1,15 @@ -#!/usr/bin/env python3 +# This file left for compatibility. If you want to use the GGUF API from Python +# then don't import gguf/gguf.py directly. If you're looking for examples, see the +# examples/ directory for gguf-py -# Example usage: -if __name__ == "__main__": - import sys - from pathlib import Path +import importlib +import sys +from pathlib import Path - # Allow running file in package as a script. - sys.path.insert(0, str(Path(__file__).parent.parent)) +sys.path.insert(0, str(Path(__file__).parent.parent)) - import numpy as np +# Compatibility for people trying to import gguf/gguf.py directly instead of as a package. +importlib.invalidate_caches() +import gguf # noqa: E402 - from gguf.gguf_writer import GGUFWriter - - # Example usage with a file - gguf_writer = GGUFWriter("example.gguf", "llama") - - gguf_writer.add_architecture() - gguf_writer.add_block_count(12) - gguf_writer.add_uint32("answer", 42) # Write a 32-bit integer - gguf_writer.add_float32("answer_in_float", 42.0) # Write a 32-bit float - gguf_writer.add_custom_alignment(64) - - tensor1 = np.ones((32,), dtype=np.float32) * 100.0 - tensor2 = np.ones((64,), dtype=np.float32) * 101.0 - tensor3 = np.ones((96,), dtype=np.float32) * 102.0 - - gguf_writer.add_tensor("tensor1", tensor1) - gguf_writer.add_tensor("tensor2", tensor2) - gguf_writer.add_tensor("tensor3", tensor3) - - gguf_writer.write_header_to_file() - gguf_writer.write_kv_data_to_file() - gguf_writer.write_tensors_to_file() - - gguf_writer.close() +importlib.reload(gguf) diff --git a/gguf-py/gguf/gguf_reader.py b/gguf-py/gguf/gguf_reader.py index 3326e9517..744e18f90 100644 --- a/gguf-py/gguf/gguf_reader.py +++ b/gguf-py/gguf/gguf_reader.py @@ -2,7 +2,7 @@ from __future__ import annotations import os from collections import OrderedDict -from typing import Any, Dict, Literal, NamedTuple, Type, TypeVar +from typing import Any, Dict, Literal, NamedTuple, TypeVar, Union import numpy as np import numpy.typing as npt @@ -58,11 +58,10 @@ class ReaderTensor(NamedTuple): class GGUFReader: byte_order: Literal['I' | 'S' | '<'] = 'I' - fields: 'OrderedDict[str, ReaderField]' = OrderedDict() - tensors: list[ReaderTensor] = [] alignment: int = GGUF_DEFAULT_ALIGNMENT - _simple_value_map: Dict[GGUFValueType, Type[Any]] = { + # Note: Internal helper, API may change. + gguf_scalar_to_np: Dict[GGUFValueType, npt.DTypeLike] = { GGUFValueType.UINT8: np.uint8, GGUFValueType.INT8: np.int8, GGUFValueType.UINT16: np.uint16, @@ -89,6 +88,8 @@ class GGUFReader: version = temp_version[0] if version not in READER_SUPPORTED_VERSIONS: raise ValueError(f'Sorry, file appears to be version {version} which we cannot handle') + self.fields: OrderedDict[str, ReaderField] = OrderedDict() + self.tensors: list[ReaderTensor] = [] offs += self._push_field(ReaderField(offs, 'GGUF.version', [temp_version], [0], [GGUFValueType.UINT32])) temp_counts = self._get(offs, np.uint64, 2) offs += self._push_field(ReaderField(offs, 'GGUF.tensor_count', [temp_counts[:1]], [0], [GGUFValueType.UINT64])) @@ -108,6 +109,14 @@ class GGUFReader: _DT = TypeVar('_DT', bound = npt.DTypeLike) + # Fetch a key/value metadata field by key. + def get_field(self, key: str) -> Union[ReaderField, None]: + return self.fields.get(key, None) + + # Fetch a tensor from the list by index. + def get_tensor(self, idx: int) -> ReaderTensor: + return self.tensors[idx] + def _get( self, offset: int, dtype: npt.DTypeLike, count: int = 1, override_order: None | Literal['I' | 'S' | '<'] = None, ) -> npt.NDArray[Any]: @@ -143,7 +152,7 @@ class GGUFReader: size = sum(int(part.nbytes) for part in sparts) return size, sparts, [1], types # Check if it's a simple scalar type. - nptype = self._simple_value_map.get(gtype) + nptype = self.gguf_scalar_to_np.get(gtype) if nptype is not None: val = self._get(offs, nptype) return int(val.nbytes), [val], [0], types @@ -245,35 +254,3 @@ class GGUFReader: field = field, )) self.tensors = tensors - - -# Example usage: -if __name__ == "__main__": - if len(sys.argv) < 2: - print('gguf_reader: Error: Specify an input file', file = sys.stderr) - sys.exit(1) - print(f'* Loading: {sys.argv[1]}') - reader = GGUFReader(sys.argv[1], 'r') - print(f'\n* Dumping {len(reader.fields)} key/value pair(s)') - for n, field in enumerate(reader.fields.values(), 1): - if not field.types: - pretty_type = 'N/A' - elif field.types[0] == GGUFValueType.ARRAY: - nest_count = len(field.types) - 1 - pretty_type = '[' * nest_count + str(field.types[-1].name) + ']' * nest_count - else: - pretty_type = str(field.types[-1].name) - print(f' {n:5}: {pretty_type:10} | {len(field.data):8} | {field.name}', end = '') - if len(field.types) == 1: - curr_type = field.types[0] - if curr_type == GGUFValueType.STRING: - print(' = {0}'.format(repr(str(bytes(field.parts[-1]), encoding='utf8')[:60])), end = '') - elif field.types[0] in reader._simple_value_map: - print(' = {0}'.format(field.parts[-1][0]), end = '') - print() - - print(f'\n* Dumping {len(reader.tensors)} tensor(s)') - for n, tensor in enumerate(reader.tensors, 1): - - prettydims = ', '.join('{0:5}'.format(d) for d in list(tensor.shape) + [1] * (4 - len(tensor.shape))) - print(f' {n:5}: {tensor.n_elements:10} | {prettydims} | {tensor.tensor_type.name:7} | {tensor.name}') diff --git a/gguf-py/tests/test_gguf.py b/gguf-py/tests/test_gguf.py index 512531dd2..fe680d983 100644 --- a/gguf-py/tests/test_gguf.py +++ b/gguf-py/tests/test_gguf.py @@ -3,5 +3,5 @@ import gguf # TODO: add tests -def test_write_gguf(): +def test_write_gguf() -> None: pass