Fix an issue with state init in GGUFReader

Move examples to an examples/ directory

Clean up examples

Add an example of modifying keys in a GGUF file

Update documentation with info on examples

Try to support people importing gguf/gguf.py directly

gguf-py/README.md

@@ -11,6 +11,14 @@ as an example for its usage.
 pip install gguf
 ```
 
+## API Examples
+
+[examples/writer.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/writer.py) — Generates `example.gguf` in the current directory to demonstrate generating a GGUF file. Note that this file cannot be used as a model.
+
+[examples/dump_gguf.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/dump_gguf.py) — Dumps a GGUF file's metadata to the console.
+
+[examples/modify_gguf.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/dump_gguf.py) — Allows changing simple metadata values in a GGUF file by key.
+
 ## Development
 Maintainers who participate in development of this package are advised to install it in editable mode:
 

gguf-py/examples/dump_gguf.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+import sys
+from pathlib import Path
+
+# Necessary to load the local gguf package
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from gguf import GGUFReader, GGUFValueType  # noqa: E402
+
+def dump_gguf(filename: str) -> None:
+    print(f'* Loading: {filename}')
+    reader = GGUFReader(filename, 'r')
+    print(f'\n* Dumping {len(reader.fields)} key/value pair(s)')
+    for n, field in enumerate(reader.fields.values(), 1):
+        if not field.types:
+            pretty_type = 'N/A'
+        elif field.types[0] == GGUFValueType.ARRAY:
+            nest_count = len(field.types) - 1
+            pretty_type = '[' * nest_count + str(field.types[-1].name) + ']' * nest_count
+        else:
+            pretty_type = str(field.types[-1].name)
+        print(f'  {n:5}: {pretty_type:10} | {len(field.data):8} | {field.name}', end = '')
+        if len(field.types) == 1:
+            curr_type = field.types[0]
+            if curr_type == GGUFValueType.STRING:
+                print(' = {0}'.format(repr(str(bytes(field.parts[-1]), encoding='utf8')[:60])), end = '')
+            elif field.types[0] in reader.gguf_scalar_to_np:
+                print(' = {0}'.format(field.parts[-1][0]), end = '')
+        print()
+
+    print(f'\n* Dumping {len(reader.tensors)} tensor(s)')
+    for n, tensor in enumerate(reader.tensors, 1):
+
+        prettydims = ', '.join('{0:5}'.format(d) for d in list(tensor.shape) + [1] * (4 - len(tensor.shape)))
+        print(f'  {n:5}: {tensor.n_elements:10} | {prettydims} | {tensor.tensor_type.name:7} | {tensor.name}')
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print('dump_gguf: Error: Specify an input file', file = sys.stderr)
+        sys.exit(1)
+    dump_gguf(sys.argv[1])

gguf-py/examples/modify_gguf.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+import sys
+from pathlib import Path
+
+# Necessary to load the local gguf package
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from gguf import GGUFReader # noqa: E402
+
+def change_gguf(reader: GGUFReader, key: str, value: str) -> None:
+    field = reader.get_field(key)
+    if field is None:
+        print(f'! Field {repr(key)} not found', file = sys.stderr)
+        sys.exit(1)
+
+    handler = reader.gguf_scalar_to_np.get(field.types[0]) if field.types else None
+    if handler is None:
+        print(f'! Field {repr(key)} has unsupported type: {field.types}')
+        sys.exit(1)
+    current_value = field.parts[field.data[0]][0]
+    new_value = handler(value)
+    print(f'* Preparing to change field {repr(key)} from {current_value} to {new_value}')
+    if current_value == new_value:
+        print(f'- Key {repr(key)} already set to requested value {current_value}')
+        sys.exit(0)
+    print('*** Warning *** Warning *** Warning **')
+    print('* Changing fields in a GGUF file can damagage it. If you are positive then type YES:')
+    response = input('YES, I am sure> ')
+    if response != 'YES':
+        print("You didn't enter YES. Okay then, see ya!")
+        sys.exit(0)
+    field.parts[field.data[0]][0] = new_value
+    print('* Field changed. Successful completion.')
+
+if __name__ == '__main__':
+    if len(sys.argv) < 4:
+        print('modify_gguf: Error: Missing arguments. Syntax: modify_gguf.py <filename> <key> <value>', file = sys.stderr)
+        sys.exit(1)
+    print(f'* Loading: {sys.argv[1]}')
+    reader = GGUFReader(sys.argv[1], 'r+')
+    change_gguf(reader, sys.argv[2], sys.argv[3])

gguf-py/examples/writer.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+import sys
+from pathlib import Path
+
+import numpy as np
+
+# Necessary to load the local gguf package
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from gguf import GGUFWriter  # noqa: E402
+
+# Example usage:
+def writer_example() -> None:
+    # Example usage with a file
+    gguf_writer = GGUFWriter("example.gguf", "llama")
+
+    gguf_writer.add_architecture()
+    gguf_writer.add_block_count(12)
+    gguf_writer.add_uint32("answer", 42)  # Write a 32-bit integer
+    gguf_writer.add_float32("answer_in_float", 42.0)  # Write a 32-bit float
+    gguf_writer.add_custom_alignment(64)
+
+    tensor1 = np.ones((32,), dtype=np.float32) * 100.0
+    tensor2 = np.ones((64,), dtype=np.float32) * 101.0
+    tensor3 = np.ones((96,), dtype=np.float32) * 102.0
+
+    gguf_writer.add_tensor("tensor1", tensor1)
+    gguf_writer.add_tensor("tensor2", tensor2)
+    gguf_writer.add_tensor("tensor3", tensor3)
+
+    gguf_writer.write_header_to_file()
+    gguf_writer.write_kv_data_to_file()
+    gguf_writer.write_tensors_to_file()
+
+    gguf_writer.close()
+
+writer_example()

gguf-py/gguf/gguf.py

@@ -1,36 +1,15 @@
-#!/usr/bin/env python3
+# This file left for compatibility. If you want to use the GGUF API from Python
+# then don't import gguf/gguf.py directly. If you're looking for examples, see the
+# examples/ directory for gguf-py
 
-# Example usage:
+import importlib
-if __name__ == "__main__":
+import sys
-    import sys
+from pathlib import Path
-    from pathlib import Path
 
-    # Allow running file in package as a script.
+sys.path.insert(0, str(Path(__file__).parent.parent))
-    sys.path.insert(0, str(Path(__file__).parent.parent))
 
-    import numpy as np
+# Compatibility for people trying to import gguf/gguf.py directly instead of as a package.
+importlib.invalidate_caches()
+import gguf  # noqa: E402
 
-    from gguf.gguf_writer import GGUFWriter
+importlib.reload(gguf)
-
-    # Example usage with a file
-    gguf_writer = GGUFWriter("example.gguf", "llama")
-
-    gguf_writer.add_architecture()
-    gguf_writer.add_block_count(12)
-    gguf_writer.add_uint32("answer", 42)  # Write a 32-bit integer
-    gguf_writer.add_float32("answer_in_float", 42.0)  # Write a 32-bit float
-    gguf_writer.add_custom_alignment(64)
-
-    tensor1 = np.ones((32,), dtype=np.float32) * 100.0
-    tensor2 = np.ones((64,), dtype=np.float32) * 101.0
-    tensor3 = np.ones((96,), dtype=np.float32) * 102.0
-
-    gguf_writer.add_tensor("tensor1", tensor1)
-    gguf_writer.add_tensor("tensor2", tensor2)
-    gguf_writer.add_tensor("tensor3", tensor3)
-
-    gguf_writer.write_header_to_file()
-    gguf_writer.write_kv_data_to_file()
-    gguf_writer.write_tensors_to_file()
-
-    gguf_writer.close()

gguf-py/gguf/gguf_reader.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import os
 from collections import OrderedDict
-from typing import Any, Dict, Literal, NamedTuple, Type, TypeVar
+from typing import Any, Dict, Literal, NamedTuple, TypeVar, Union
 
 import numpy as np
 import numpy.typing as npt
@@ -58,11 +58,10 @@ class ReaderTensor(NamedTuple):
 
 class GGUFReader:
     byte_order: Literal['I' | 'S' | '<'] = 'I'
-    fields: 'OrderedDict[str, ReaderField]' = OrderedDict()
-    tensors: list[ReaderTensor] = []
     alignment: int = GGUF_DEFAULT_ALIGNMENT
 
-    _simple_value_map: Dict[GGUFValueType, Type[Any]] = {
+    # Note: Internal helper, API may change.
+    gguf_scalar_to_np: Dict[GGUFValueType, npt.DTypeLike] = {
         GGUFValueType.UINT8:   np.uint8,
         GGUFValueType.INT8:    np.int8,
         GGUFValueType.UINT16:  np.uint16,
@@ -89,6 +88,8 @@ class GGUFReader:
         version = temp_version[0]
         if version not in READER_SUPPORTED_VERSIONS:
             raise ValueError(f'Sorry, file appears to be version {version} which we cannot handle')
+        self.fields: OrderedDict[str, ReaderField] = OrderedDict()
+        self.tensors: list[ReaderTensor] = []
         offs += self._push_field(ReaderField(offs, 'GGUF.version', [temp_version], [0], [GGUFValueType.UINT32]))
         temp_counts = self._get(offs, np.uint64, 2)
         offs += self._push_field(ReaderField(offs, 'GGUF.tensor_count', [temp_counts[:1]], [0], [GGUFValueType.UINT64]))
@@ -108,6 +109,14 @@ class GGUFReader:
 
     _DT = TypeVar('_DT', bound = npt.DTypeLike)
 
+    # Fetch a key/value metadata field by key.
+    def get_field(self, key: str) -> Union[ReaderField, None]:
+        return self.fields.get(key, None)
+
+    # Fetch a tensor from the list by index.
+    def get_tensor(self, idx: int) -> ReaderTensor:
+        return self.tensors[idx]
+
     def _get(
         self, offset: int, dtype: npt.DTypeLike, count: int = 1, override_order: None | Literal['I' | 'S' | '<'] = None,
     ) -> npt.NDArray[Any]:
@@ -143,7 +152,7 @@ class GGUFReader:
             size = sum(int(part.nbytes) for part in sparts)
             return size, sparts, [1], types
         # Check if it's a simple scalar type.
-        nptype = self._simple_value_map.get(gtype)
+        nptype = self.gguf_scalar_to_np.get(gtype)
         if nptype is not None:
             val = self._get(offs, nptype)
             return int(val.nbytes), [val], [0], types
@@ -245,35 +254,3 @@ class GGUFReader:
                 field = field,
             ))
         self.tensors = tensors
-
-
-# Example usage:
-if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print('gguf_reader: Error: Specify an input file', file = sys.stderr)
-        sys.exit(1)
-    print(f'* Loading: {sys.argv[1]}')
-    reader = GGUFReader(sys.argv[1], 'r')
-    print(f'\n* Dumping {len(reader.fields)} key/value pair(s)')
-    for n, field in enumerate(reader.fields.values(), 1):
-        if not field.types:
-            pretty_type = 'N/A'
-        elif field.types[0] == GGUFValueType.ARRAY:
-            nest_count = len(field.types) - 1
-            pretty_type = '[' * nest_count + str(field.types[-1].name) + ']' * nest_count
-        else:
-            pretty_type = str(field.types[-1].name)
-        print(f'  {n:5}: {pretty_type:10} | {len(field.data):8} | {field.name}', end = '')
-        if len(field.types) == 1:
-            curr_type = field.types[0]
-            if curr_type == GGUFValueType.STRING:
-                print(' = {0}'.format(repr(str(bytes(field.parts[-1]), encoding='utf8')[:60])), end = '')
-            elif field.types[0] in reader._simple_value_map:
-                print(' = {0}'.format(field.parts[-1][0]), end = '')
-        print()
-
-    print(f'\n* Dumping {len(reader.tensors)} tensor(s)')
-    for n, tensor in enumerate(reader.tensors, 1):
-
-        prettydims = ', '.join('{0:5}'.format(d) for d in list(tensor.shape) + [1] * (4 - len(tensor.shape)))
-        print(f'  {n:5}: {tensor.n_elements:10} | {prettydims} | {tensor.tensor_type.name:7} | {tensor.name}')

gguf-py/tests/test_gguf.py

@@ -3,5 +3,5 @@ import gguf
 # TODO: add tests
 
 
-def test_write_gguf():
+def test_write_gguf() -> None:
     pass