vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add layer_norm_rms_eps to checkpoint convert script

Browse source
This commit is contained in:
xaedes 2023-08-27 23:25:41 +02:00
parent ef899fbe89
commit d71069c4fb
No known key found for this signature in database
GPG key ID: 30030EDD817EA2B1
1 changed files with 1 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
examples/train-text-from-scratch/convert-train-checkpoint-to-gguf.py
View file

@ -453,6 +453,7 @@ class Checkpoint:
def save_gguf(self, gguf_writer): def save_gguf(self, gguf_writer):
gguf_writer.add_architecture() gguf_writer.add_architecture()
gguf_writer.add_file_type(gguf.GGMLQuantizationType.F32) gguf_writer.add_file_type(gguf.GGMLQuantizationType.F32)
gguf_writer.add_layer_norm_rms_eps(1e-5)
gguf_writer.add_uint32(LLM_KV_TRAINING_FILE_VERSION, 0) gguf_writer.add_uint32(LLM_KV_TRAINING_FILE_VERSION, 0)
gguf_writer.add_uint32(LLM_KV_TRAINING_ITERATION_COUNT, self.train_its) gguf_writer.add_uint32(LLM_KV_TRAINING_ITERATION_COUNT, self.train_its)
gguf_writer.add_uint32(LLM_KV_TRAINING_SAMPLE_COUNT, self.train_samples) gguf_writer.add_uint32(LLM_KV_TRAINING_SAMPLE_COUNT, self.train_samples)