From 609df3c703dac923b3b00df212f862d0d050a73e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sat, 4 May 2024 20:29:40 +0200 Subject: [PATCH] add progress bar --- gguf-py/scripts/gguf-new-metadata.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index 316f9bf1c..f2a9e8b14 100755 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -7,6 +7,7 @@ import json from pathlib import Path import numpy as np +from tqdm import tqdm from typing import Any, Sequence, NamedTuple # Necessary to load the local gguf package @@ -113,17 +114,23 @@ def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new writer.add_key(key) writer.add_val(val.value, val.type) + total_bytes = 0 + for tensor in reader.tensors: + total_bytes += tensor.n_bytes # Dimensions are written in reverse order, so flip them first shape = np.flipud(tensor.shape).tolist() writer.add_tensor_info(tensor.name, shape, tensor.data.dtype, tensor.data.nbytes, tensor.tensor_type) + bar = tqdm(desc="Writing", total=total_bytes, unit="byte", unit_scale=True) + writer.write_header_to_file() writer.write_kv_data_to_file() writer.write_ti_data_to_file() for tensor in reader.tensors: writer.write_tensor_data(tensor.data) + bar.update(tensor.n_bytes) writer.close()