vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Reorganize scripts

Browse source
This commit is contained in:
KerfuffleV2 2023-11-09 14:52:44 -07:00
parent 5738b2f3b6
commit 52bdc7e946
6 changed files with 149 additions and 104 deletions
Download patch file Download diff file Expand all files Collapse all files

8
gguf-py/README.md
View file

@ -11,13 +11,15 @@ as an example for its usage.
pip install gguf pip install gguf
``` ```
## API Examples ## API Examples/Simple Tools
[examples/writer.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/writer.py) — Generates `example.gguf` in the current directory to demonstrate generating a GGUF file. Note that this file cannot be used as a model. [examples/writer.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/writer.py) — Generates `example.gguf` in the current directory to demonstrate generating a GGUF file. Note that this file cannot be used as a model.
[examples/dump_gguf.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/dump_gguf.py) — Dumps a GGUF file's metadata to the console. [scripts/gguf-dump.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/scripts/gguf-dump.py) — Dumps a GGUF file's metadata to the console.
[examples/modify_gguf.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/examples/dump_gguf.py) — Allows changing simple metadata values in a GGUF file by key. [scripts/gguf-set-metadata.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/scripts/gguf-set-metadata.py) — Allows changing simple metadata values in a GGUF file by key.
[scripts/gguf-convert-endian.py](https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/scripts/gguf-convert-endian.py) — Allows converting the endianness of GGUF files.
## Development ## Development
Maintainers who participate in development of this package are advised to install it in editable mode: Maintainers who participate in development of this package are advised to install it in editable mode:

75
gguf-py/examples/modify_gguf.py
View file

@ -1,75 +0,0 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
# Necessary to load the local gguf package
sys.path.insert(0, str(Path(__file__).parent.parent))
from gguf import GGUFReader # noqa: E402
def minimal_example(filename: str) -> None:
reader = GGUFReader(filename, 'r+')
field = reader.fields['tokenizer.ggml.bos_token_id']
if field is None:
return
part_index = field.data[0]
field.parts[part_index][0] = 2 # Set tokenizer.ggml.bos_token_id to 2
#
# So what's this field.data thing? It's helpful because field.parts contains
# _every_ part of the GGUF field. For example, tokenizer.ggml.bos_token_id consists
# of:
#
# Part index 0: Key length (27)
# Part index 1: Key data ("tokenizer.ggml.bos_token_id")
# Part index 2: Field type (4, the id for GGUFValueType.UINT32)
# Part index 3: Field value
#
# Note also that each part is an NDArray slice, so even a part that
# is only a single value like the key length will be a NDArray of
# the key length type (numpy.uint32).
#
# The .data attribute in the Field is a list of relevant part indexes
# and doesn't contain internal GGUF details like the key length part.
# In this case, .data will be [3] - just the part index of the
# field value itself.
def change_gguf(reader: GGUFReader, key: str, value: str) -> None:
field = reader.get_field(key)
if field is None:
print(f'! Field {repr(key)} not found', file = sys.stderr)
sys.exit(1)
# Note that field.types is a list of types. This is because the GGUF
# format supports arrays. For example, an array of UINT32 would
# look like [GGUFValueType.ARRAY, GGUFValueType.UINT32]
handler = reader.gguf_scalar_to_np.get(field.types[0]) if field.types else None
if handler is None:
print(f'! Field {repr(key)} has unsupported type: {field.types}')
sys.exit(1)
current_value = field.parts[field.data[0]][0]
new_value = handler(value)
print(f'* Preparing to change field {repr(key)} from {current_value} to {new_value}')
if current_value == new_value:
print(f'- Key {repr(key)} already set to requested value {current_value}')
sys.exit(0)
print('*** Warning *** Warning *** Warning **')
print('* Changing fields in a GGUF file can damage it. If you are positive then type YES:')
response = input('YES, I am sure> ')
if response != 'YES':
print("You didn't enter YES. Okay then, see ya!")
sys.exit(0)
field.parts[field.data[0]][0] = new_value
print('* Field changed. Successful completion.')
if __name__ == '__main__':
if len(sys.argv) < 4:
print(
'modify_gguf: Error: Missing arguments. Syntax: modify_gguf.py <filename> <key> <value>',
file = sys.stderr,
)
sys.exit(1)
print(f'* Loading: {sys.argv[1]}')
reader = GGUFReader(sys.argv[1], 'r+')
change_gguf(reader, sys.argv[2], sys.argv[3])

2
gguf-py/gguf/gguf_reader.py
View file

@ -1,6 +1,6 @@
# #
# GGUF file reading/modification support. For API usage information, # GGUF file reading/modification support. For API usage information,
# please see examples/modify_gguf.py and examples/dump_gguf.py # please see the files scripts/ for some fairly simple examples.
# #
from __future__ import annotations from __future__ import annotations

52
convert-gguf-endian.py → gguf-py/scripts/gguf-convert-endian.py
View file

@ -1,21 +1,21 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from __future__ import annotations from __future__ import annotations
import argparse
import os import os
import sys import sys
from pathlib import Path from pathlib import Path
import numpy as np import numpy as np
if "NO_LOCAL_GGUF" not in os.environ: # Necessary to load the local gguf package
sys.path.insert(1, str(Path(__file__).parent / "gguf-py")) if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists():
sys.path.insert(0, str(Path(__file__).parent.parent))
import gguf import gguf
def convert_byteorder(filename: str, order: str) -> None: def convert_byteorder(reader: gguf.GGUFReader, args: argparse.Namespace) -> None:
if order not in ("big", "little", "native"):
raise ValueError(f"Bad order parameter {order}")
reader = gguf.GGUFReader(filename, "r+")
if np.uint32(1) == np.uint32(1).newbyteorder("<"): if np.uint32(1) == np.uint32(1).newbyteorder("<"):
# Host is little endian # Host is little endian
host_endian = "little" host_endian = "little"
@ -28,7 +28,7 @@ def convert_byteorder(filename: str, order: str) -> None:
file_endian = swapped_endian file_endian = swapped_endian
else: else:
file_endian = host_endian file_endian = host_endian
if order == "native": if args.order == "native":
order = host_endian order = host_endian
print(f"* Host is {host_endian.upper()} endian, GGUF file seems to be {file_endian.upper()} endian") print(f"* Host is {host_endian.upper()} endian, GGUF file seems to be {file_endian.upper()} endian")
if file_endian == order: if file_endian == order:
@ -43,10 +43,14 @@ def convert_byteorder(filename: str, order: str) -> None:
): ):
raise ValueError(f"Cannot handle type {tensor.tensor_type.name} for tensor {repr(tensor.name)}") raise ValueError(f"Cannot handle type {tensor.tensor_type.name} for tensor {repr(tensor.name)}")
print(f"* Preparing to convert from {file_endian.upper()} to {order.upper()}") print(f"* Preparing to convert from {file_endian.upper()} to {order.upper()}")
if args.dry_run:
return
print("\n*** Warning *** Warning *** Warning **") print("\n*** Warning *** Warning *** Warning **")
print("* This conversion process may damage the file. Ensure you have a backup.") print("* This conversion process may damage the file. Ensure you have a backup.")
if order != host_endian:
print("* Requested endian differs from host, you will not be able to load the model on this machine.")
print("* The file will be modified immediately, so if conversion fails or is interrupted") print("* The file will be modified immediately, so if conversion fails or is interrupted")
print("* the file will be corrupted. If you are positive then type YES:") print("* the file will be corrupted. Enter exactly YES if you are positive you want to proceed:")
response = input("YES, I am sure> ") response = input("YES, I am sure> ")
if response != "YES": if response != "YES":
print("You didn't enter YES. Okay then, see ya!") print("You didn't enter YES. Okay then, see ya!")
@ -85,11 +89,29 @@ def convert_byteorder(filename: str, order: str) -> None:
print("* Completion") print("* Completion")
def main() -> None:
parser = argparse.ArgumentParser(description="Convert GGUF file byte order")
parser.add_argument(
"model",
type=str,
help="GGUF format model filename",
)
parser.add_argument(
"order",
type=str,
choices=['big','little','native'],
help="Requested byte order",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Don't actually change anything"
)
args = parser.parse_args(None if len(sys.argv) > 1 else ["--help"])
print(f'* Loading: {args.model}')
reader = gguf.GGUFReader(args.model, 'r' if args.dry_run else 'r+')
convert_byteorder(reader, args)
if __name__ == "__main__": if __name__ == "__main__":
if len(sys.argv) < 3: main()
print(
"convert_endian: Error: Missing arguments. Syntax: modify_gguf.py <filename> <little | big | native>",
file=sys.stderr,
)
sys.exit(1)
convert_byteorder(sys.argv[1], sys.argv[2])

28
gguf-py/examples/dump_gguf.py → gguf-py/scripts/gguf-dump.py
View file

@ -1,18 +1,19 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import argparse
import os
import sys import sys
from pathlib import Path from pathlib import Path
# Necessary to load the local gguf package # Necessary to load the local gguf package
sys.path.insert(0, str(Path(__file__).parent.parent)) if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists():
sys.path.insert(0, str(Path(__file__).parent.parent))
from gguf import GGUFReader, GGUFValueType # noqa: E402 from gguf import GGUFReader, GGUFValueType # noqa: E402
# For more information about what field.parts and field.data represent, # For more information about what field.parts and field.data represent,
# please see the comments in the modify_gguf.py example. # please see the comments in the modify_gguf.py example.
def dump_gguf(filename: str) -> None: def dump_metadata(reader: GGUFReader, dump_tensors: bool = True) -> None:
print(f'* Loading: {filename}')
reader = GGUFReader(filename, 'r')
print(f'\n* Dumping {len(reader.fields)} key/value pair(s)') print(f'\n* Dumping {len(reader.fields)} key/value pair(s)')
for n, field in enumerate(reader.fields.values(), 1): for n, field in enumerate(reader.fields.values(), 1):
if not field.types: if not field.types:
@ -30,16 +31,23 @@ def dump_gguf(filename: str) -> None:
elif field.types[0] in reader.gguf_scalar_to_np: elif field.types[0] in reader.gguf_scalar_to_np:
print(' = {0}'.format(field.parts[-1][0]), end = '') print(' = {0}'.format(field.parts[-1][0]), end = '')
print() print()
if not dump_tensors:
return
print(f'\n* Dumping {len(reader.tensors)} tensor(s)') print(f'\n* Dumping {len(reader.tensors)} tensor(s)')
for n, tensor in enumerate(reader.tensors, 1): for n, tensor in enumerate(reader.tensors, 1):
prettydims = ', '.join('{0:5}'.format(d) for d in list(tensor.shape) + [1] * (4 - len(tensor.shape))) prettydims = ', '.join('{0:5}'.format(d) for d in list(tensor.shape) + [1] * (4 - len(tensor.shape)))
print(f' {n:5}: {tensor.n_elements:10} | {prettydims} | {tensor.tensor_type.name:7} | {tensor.name}') print(f' {n:5}: {tensor.n_elements:10} | {prettydims} | {tensor.tensor_type.name:7} | {tensor.name}')
def main() -> None:
parser = argparse.ArgumentParser(description="Dump GGUF file metadata")
parser.add_argument("model", type=str, help="GGUF format model filename")
parser.add_argument("--no-tensors", action="store_true", help="Don't dump tensor metadata")
args = parser.parse_args(None if len(sys.argv) > 1 else ["--help"])
print(f'* Loading: {args.model}')
reader = GGUFReader(args.model, 'r')
dump_metadata(reader, not args.no_tensors)
if __name__ == '__main__': if __name__ == '__main__':
if len(sys.argv) < 2: main()
print('dump_gguf: Error: Specify an input file', file = sys.stderr)
sys.exit(1)
dump_gguf(sys.argv[1])

88
gguf-py/scripts/gguf-set-metadata.py Executable file
View file

@ -0,0 +1,88 @@
#!/usr/bin/env python3
import argparse
import os
import sys
from pathlib import Path
# Necessary to load the local gguf package
if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists():
sys.path.insert(0, str(Path(__file__).parent.parent))
from gguf import GGUFReader # noqa: E402
def minimal_example(filename: str) -> None:
reader = GGUFReader(filename, 'r+')
field = reader.fields['tokenizer.ggml.bos_token_id']
if field is None:
return
part_index = field.data[0]
field.parts[part_index][0] = 2 # Set tokenizer.ggml.bos_token_id to 2
#
# So what's this field.data thing? It's helpful because field.parts contains
# _every_ part of the GGUF field. For example, tokenizer.ggml.bos_token_id consists
# of:
#
# Part index 0: Key length (27)
# Part index 1: Key data ("tokenizer.ggml.bos_token_id")
# Part index 2: Field type (4, the id for GGUFValueType.UINT32)
# Part index 3: Field value
#
# Note also that each part is an NDArray slice, so even a part that
# is only a single value like the key length will be a NDArray of
# the key length type (numpy.uint32).
#
# The .data attribute in the Field is a list of relevant part indexes
# and doesn't contain internal GGUF details like the key length part.
# In this case, .data will be [3] - just the part index of the
# field value itself.
def set_metadata(reader: GGUFReader, args: argparse.Namespace) -> None:
field = reader.get_field(args.key)
if field is None:
print(f'! Field {repr(args.key)} not found', file = sys.stderr)
sys.exit(1)
# Note that field.types is a list of types. This is because the GGUF
# format supports arrays. For example, an array of UINT32 would
# look like [GGUFValueType.ARRAY, GGUFValueType.UINT32]
handler = reader.gguf_scalar_to_np.get(field.types[0]) if field.types else None
if handler is None:
print(f'! This tool only supports changing simple values, {repr(args.key)} has unsupported type {field.types}',
file = sys.stderr)
sys.exit(1)
current_value = field.parts[field.data[0]][0]
new_value = handler(args.value)
print(f'* Preparing to change field {repr(args.key)} from {current_value} to {new_value}')
if current_value == new_value:
print(f'- Key {repr(args.key)} already set to requested value {current_value}')
sys.exit(0)
if args.dry_run:
sys.exit(0)
if not args.force:
print('*** Warning *** Warning *** Warning **')
print('* Changing fields in a GGUF file can make it unusable. Proceed at your own risk.')
print('* Enter exactly YES if you are positive you want to proceed:')
response = input('YES, I am sure> ')
if response != 'YES':
print("You didn't enter YES. Okay then, see ya!")
sys.exit(0)
field.parts[field.data[0]][0] = new_value
print('* Field changed. Successful completion.')
def main() -> None:
parser = argparse.ArgumentParser(description="Set a simple value in GGUF file metadata")
parser.add_argument("model", type=str, help="GGUF format model filename")
parser.add_argument("key", type=str, help="Metadata key to set")
parser.add_argument("value", type=str, help="Metadata value to set")
parser.add_argument("--dry-run", action="store_true", help="Don't actually change anything")
parser.add_argument("--force", action="store_true", help="Change the field without confirmation")
args = parser.parse_args(None if len(sys.argv) > 1 else ["--help"])
print(f'* Loading: {args.model}')
reader = GGUFReader(args.model, 'r' if args.dry_run else 'r+')
set_metadata(reader, args)
if __name__ == '__main__':
main()