vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Reorganize scripts

Browse source
This commit is contained in:
KerfuffleV2 2023-11-09 14:52:44 -07:00
parent 5738b2f3b6
commit 52bdc7e946
6 changed files with 149 additions and 104 deletions
Download patch file Download diff file Expand all files Collapse all files

8
gguf-py/README.md
View file

@ -11,13 +11,15 @@ as an example for its usage.
pip install gguf
```
## API Examples
## API Examples/Simple Tools
[examples/writer.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/writer.py) — Generates `example.gguf` in the current directory to demonstrate generating a GGUF file. Note that this file cannot be used as a model.
[examples/dump_gguf.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/dump_gguf.py) — Dumps a GGUF file's metadata to the console.
[scripts/gguf-dump.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/scripts/gguf-dump.py) — Dumps a GGUF file's metadata to the console.
[examples/modify_gguf.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/dump_gguf.py) — Allows changing simple metadata values in a GGUF file by key.
[scripts/gguf-set-metadata.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/scripts/gguf-set-metadata.py) — Allows changing simple metadata values in a GGUF file by key.
[scripts/gguf-convert-endian.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/scripts/gguf-convert-endian.py) — Allows converting the endianness of GGUF files.
## Development
Maintainers who participate in development of this package are advised to install it in editable mode:

75
gguf-py/examples/modify_gguf.py
View file

@ -1,75 +0,0 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
# Necessary to load the local gguf package
sys.path.insert(0, str(Path(__file__).parent.parent))
from gguf import GGUFReader # noqa: E402
def minimal_example(filename: str) -> None:
reader = GGUFReader(filename, 'r+')
field = reader.fields['tokenizer.ggml.bos_token_id']
if field is None:
return
part_index = field.data[0]
field.parts[part_index][0] = 2 # Set tokenizer.ggml.bos_token_id to 2
#
# So what's this field.data thing? It's helpful because field.parts contains
# _every_ part of the GGUF field. For example, tokenizer.ggml.bos_token_id consists
# of:
#
# Part index 0: Key length (27)
# Part index 1: Key data ("tokenizer.ggml.bos_token_id")
# Part index 2: Field type (4, the id for GGUFValueType.UINT32)
# Part index 3: Field value
#
# Note also that each part is an NDArray slice, so even a part that
# is only a single value like the key length will be a NDArray of
# the key length type (numpy.uint32).
#
# The .data attribute in the Field is a list of relevant part indexes
# and doesn't contain internal GGUF details like the key length part.
# In this case, .data will be [3] - just the part index of the
# field value itself.
def change_gguf(reader: GGUFReader, key: str, value: str) -> None:
field = reader.get_field(key)
if field is None:
print(f'! Field {repr(key)} not found', file = sys.stderr)
sys.exit(1)
# Note that field.types is a list of types. This is because the GGUF
# format supports arrays. For example, an array of UINT32 would
# look like [GGUFValueType.ARRAY, GGUFValueType.UINT32]
handler = reader.gguf_scalar_to_np.get(field.types[0]) if field.types else None
if handler is None:
print(f'! Field {repr(key)} has unsupported type: {field.types}')
sys.exit(1)
current_value = field.parts[field.data[0]][0]
new_value = handler(value)
print(f'* Preparing to change field {repr(key)} from {current_value} to {new_value}')
if current_value == new_value:
print(f'- Key {repr(key)} already set to requested value {current_value}')
sys.exit(0)
print('*** Warning *** Warning *** Warning **')
print('* Changing fields in a GGUF file can damage it. If you are positive then type YES:')
response = input('YES, I am sure> ')
if response != 'YES':
print("You didn't enter YES. Okay then, see ya!")
sys.exit(0)
field.parts[field.data[0]][0] = new_value
print('* Field changed. Successful completion.')
if __name__ == '__main__':
if len(sys.argv) < 4:
print(
'modify_gguf: Error: Missing arguments. Syntax: modify_gguf.py <filename> <key> <value>',
file = sys.stderr,
)
sys.exit(1)
print(f'* Loading: {sys.argv[1]}')
reader = GGUFReader(sys.argv[1], 'r+')
change_gguf(reader, sys.argv[2], sys.argv[3])

2
gguf-py/gguf/gguf_reader.py
View file

@ -1,6 +1,6 @@
#
# GGUF file reading/modification support. For API usage information,
# please see examples/modify_gguf.py and examples/dump_gguf.py
# please see the files scripts/ for some fairly simple examples.
#
from __future__ import annotations

52
convert-gguf-endian.py → gguf-py/scripts/gguf-convert-endian.py
View file

@ -1,21 +1,21 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import os
import sys
from pathlib import Path
import numpy as np
if "NO_LOCAL_GGUF" not in os.environ:
sys.path.insert(1, str(Path(__file__).parent / "gguf-py"))
# Necessary to load the local gguf package
if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists():
sys.path.insert(0, str(Path(__file__).parent.parent))
import gguf
def convert_byteorder(filename: str, order: str) -> None:
if order not in ("big", "little", "native"):
raise ValueError(f"Bad order parameter {order}")
reader = gguf.GGUFReader(filename, "r+")
def convert_byteorder(reader: gguf.GGUFReader, args: argparse.Namespace) -> None:
if np.uint32(1) == np.uint32(1).newbyteorder("<"):
# Host is little endian
host_endian = "little"
@ -28,7 +28,7 @@ def convert_byteorder(filename: str, order: str) -> None:
file_endian = swapped_endian
else:
file_endian = host_endian
if order == "native":
if args.order == "native":
order = host_endian
print(f"* Host is {host_endian.upper()} endian, GGUF file seems to be {file_endian.upper()} endian")
if file_endian == order:
@ -43,10 +43,14 @@ def convert_byteorder(filename: str, order: str) -> None:
):
raise ValueError(f"Cannot handle type {tensor.tensor_type.name} for tensor {repr(tensor.name)}")
print(f"* Preparing to convert from {file_endian.upper()} to {order.upper()}")
if args.dry_run:
return
print("\n*** Warning *** Warning *** Warning **")
print("* This conversion process may damage the file. Ensure you have a backup.")
if order != host_endian:
print("* Requested endian differs from host, you will not be able to load the model on this machine.")
print("* The file will be modified immediately, so if conversion fails or is interrupted")
print("* the file will be corrupted. If you are positive then type YES:")
print("* the file will be corrupted. Enter exactly YES if you are positive you want to proceed:")
response = input("YES, I am sure> ")
if response != "YES":
print("You didn't enter YES. Okay then, see ya!")
@ -85,11 +89,29 @@ def convert_byteorder(filename: str, order: str) -> None:
print("* Completion")
if __name__ == "__main__":
if len(sys.argv) < 3:
print(
"convert_endian: Error: Missing arguments. Syntax: modify_gguf.py <filename> <little | big | native>",
file=sys.stderr,
def main() -> None:
parser = argparse.ArgumentParser(description="Convert GGUF file byte order")
parser.add_argument(
"model",
type=str,
help="GGUF format model filename",
)
sys.exit(1)
convert_byteorder(sys.argv[1], sys.argv[2])
parser.add_argument(
"order",
type=str,
choices=['big','little','native'],
help="Requested byte order",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Don't actually change anything"
)
args = parser.parse_args(None if len(sys.argv) > 1 else ["--help"])
print(f'* Loading: {args.model}')
reader = gguf.GGUFReader(args.model, 'r' if args.dry_run else 'r+')
convert_byteorder(reader, args)
if __name__ == "__main__":
main()

26
gguf-py/examples/dump_gguf.py → gguf-py/scripts/gguf-dump.py
View file

@ -1,8 +1,11 @@
#!/usr/bin/env python3
import argparse
import os
import sys
from pathlib import Path
# Necessary to load the local gguf package
if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists():
sys.path.insert(0, str(Path(__file__).parent.parent))
from gguf import GGUFReader, GGUFValueType # noqa: E402
@ -10,9 +13,7 @@ from gguf import GGUFReader, GGUFValueType # noqa: E402
# For more information about what field.parts and field.data represent,
# please see the comments in the modify_gguf.py example.
def dump_gguf(filename: str) -> None:
print(f'* Loading: {filename}')
reader = GGUFReader(filename, 'r')
def dump_metadata(reader: GGUFReader, dump_tensors: bool = True) -> None:
print(f'\n* Dumping {len(reader.fields)} key/value pair(s)')
for n, field in enumerate(reader.fields.values(), 1):
if not field.types:
@ -30,16 +31,23 @@ def dump_gguf(filename: str) -> None:
elif field.types[0] in reader.gguf_scalar_to_np:
print(' = {0}'.format(field.parts[-1][0]), end = '')
print()
if not dump_tensors:
return
print(f'\n* Dumping {len(reader.tensors)} tensor(s)')
for n, tensor in enumerate(reader.tensors, 1):
prettydims = ', '.join('{0:5}'.format(d) for d in list(tensor.shape) + [1] * (4 - len(tensor.shape)))
print(f' {n:5}: {tensor.n_elements:10} | {prettydims} | {tensor.tensor_type.name:7} | {tensor.name}')
def main() -> None:
parser = argparse.ArgumentParser(description="Dump GGUF file metadata")
parser.add_argument("model", type=str, help="GGUF format model filename")
parser.add_argument("--no-tensors", action="store_true", help="Don't dump tensor metadata")
args = parser.parse_args(None if len(sys.argv) > 1 else ["--help"])
print(f'* Loading: {args.model}')
reader = GGUFReader(args.model, 'r')
dump_metadata(reader, not args.no_tensors)
if __name__ == '__main__':
if len(sys.argv) < 2:
print('dump_gguf: Error: Specify an input file', file = sys.stderr)
sys.exit(1)
dump_gguf(sys.argv[1])
main()

88
gguf-py/scripts/gguf-set-metadata.py Executable file
View file

@ -0,0 +1,88 @@
#!/usr/bin/env python3
import argparse
import os
import sys
from pathlib import Path
# Necessary to load the local gguf package
if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists():
sys.path.insert(0, str(Path(__file__).parent.parent))
from gguf import GGUFReader # noqa: E402
def minimal_example(filename: str) -> None:
reader = GGUFReader(filename, 'r+')
field = reader.fields['tokenizer.ggml.bos_token_id']
if field is None:
return
part_index = field.data[0]
field.parts[part_index][0] = 2 # Set tokenizer.ggml.bos_token_id to 2
#
# So what's this field.data thing? It's helpful because field.parts contains
# _every_ part of the GGUF field. For example, tokenizer.ggml.bos_token_id consists
# of:
#
# Part index 0: Key length (27)
# Part index 1: Key data ("tokenizer.ggml.bos_token_id")
# Part index 2: Field type (4, the id for GGUFValueType.UINT32)
# Part index 3: Field value
#
# Note also that each part is an NDArray slice, so even a part that
# is only a single value like the key length will be a NDArray of
# the key length type (numpy.uint32).
#
# The .data attribute in the Field is a list of relevant part indexes
# and doesn't contain internal GGUF details like the key length part.
# In this case, .data will be [3] - just the part index of the
# field value itself.
def set_metadata(reader: GGUFReader, args: argparse.Namespace) -> None:
field = reader.get_field(args.key)
if field is None:
print(f'! Field {repr(args.key)} not found', file = sys.stderr)
sys.exit(1)
# Note that field.types is a list of types. This is because the GGUF
# format supports arrays. For example, an array of UINT32 would
# look like [GGUFValueType.ARRAY, GGUFValueType.UINT32]
handler = reader.gguf_scalar_to_np.get(field.types[0]) if field.types else None
if handler is None:
print(f'! This tool only supports changing simple values, {repr(args.key)} has unsupported type {field.types}',
file = sys.stderr)
sys.exit(1)
current_value = field.parts[field.data[0]][0]
new_value = handler(args.value)
print(f'* Preparing to change field {repr(args.key)} from {current_value} to {new_value}')
if current_value == new_value:
print(f'- Key {repr(args.key)} already set to requested value {current_value}')
sys.exit(0)
if args.dry_run:
sys.exit(0)
if not args.force:
print('*** Warning *** Warning *** Warning **')
print('* Changing fields in a GGUF file can make it unusable. Proceed at your own risk.')
print('* Enter exactly YES if you are positive you want to proceed:')
response = input('YES, I am sure> ')
if response != 'YES':
print("You didn't enter YES. Okay then, see ya!")
sys.exit(0)
field.parts[field.data[0]][0] = new_value
print('* Field changed. Successful completion.')
def main() -> None:
parser = argparse.ArgumentParser(description="Set a simple value in GGUF file metadata")
parser.add_argument("model", type=str, help="GGUF format model filename")
parser.add_argument("key", type=str, help="Metadata key to set")
parser.add_argument("value", type=str, help="Metadata value to set")
parser.add_argument("--dry-run", action="store_true", help="Don't actually change anything")
parser.add_argument("--force", action="store_true", help="Change the field without confirmation")
args = parser.parse_args(None if len(sys.argv) > 1 else ["--help"])
print(f'* Loading: {args.model}')
reader = GGUFReader(args.model, 'r' if args.dry_run else 'r+')
set_metadata(reader, args)
if __name__ == '__main__':
main()