vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

gguf_dump.py: refactor kv array string handling

Browse source
This commit is contained in:
brian khuu 2024-07-20 14:31:24 +10:00
parent 130d396ec7
commit 923886f7ba
1 changed files with 9 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

20
gguf-py/scripts/gguf_dump.py
View file

@ -249,31 +249,29 @@ def dump_markdown_metadata(reader: GGUFReader, args: argparse.Namespace) -> None
if len(field.types) == 1:
curr_type = field.types[0]
if curr_type == GGUFValueType.STRING:
value = repr(str(bytes(field.parts[-1]), encoding='utf-8')[:60])
value = "\"`{strval}`\"".format(strval=str(bytes(field.parts[-1]), encoding='utf-8')[:60])
elif curr_type in reader.gguf_scalar_to_np:
value = str(field.parts[-1][0])
else:
if field.types[0] == GGUFValueType.ARRAY:
displayed_values = 0
curr_type = field.types[1]
array_elements = []
if curr_type == GGUFValueType.STRING:
render_element = min(5, total_elements)
for element_pos in range(render_element):
truncate_length = 30
value_string = repr(str(bytes(field.parts[-1 - (total_elements - element_pos - 1) * 2]), encoding='utf-8'))
value_string = str(bytes(field.parts[-1 - (total_elements - element_pos - 1) * 2]), encoding='utf-8')
if len(value_string) > truncate_length:
value += value_string[:truncate_length // 2] + "..." + value_string[-truncate_length // 2:]
array_elements.append(value_string[:truncate_length // 2] + "`...`" + value_string[-truncate_length // 2:])
else:
value += value_string
displayed_values += 1
if (total_elements - 1) > element_pos:
value += ", "
array_elements.append(value_string)
value_array_inner = ["\"`{strval}`\"".format(strval=strval) for strval in array_elements]
value = f'[ {", ".join(value_array_inner).strip()}{", ..." if total_elements > len(array_elements) else ""} ]'
elif curr_type in reader.gguf_scalar_to_np:
render_element = min(7, total_elements)
for element_pos in range(render_element):
value += str(field.parts[-1 - (total_elements - element_pos - 1)][0]) + (", " if total_elements > 1 else "")
displayed_values += 1
value = f'[ {value}{"..." if total_elements > displayed_values else ""} ]'
array_elements.append(str(field.parts[-1 - (total_elements - element_pos - 1)][0]))
value = f'[ {", ".join(array_elements).strip()}{", ..." if total_elements > len(array_elements) else ""} ]'
kv_dump_table.append({"n":n, "pretty_type":pretty_type, "total_elements":total_elements, "field_name":field.name, "value":value})
kv_dump_table_header_map = [