From d71069c4fba085ea52b6362ac3c9634429b1a66f Mon Sep 17 00:00:00 2001 From: xaedes Date: Sun, 27 Aug 2023 23:25:41 +0200 Subject: [PATCH] add layer_norm_rms_eps to checkpoint convert script --- .../train-text-from-scratch/convert-train-checkpoint-to-gguf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/train-text-from-scratch/convert-train-checkpoint-to-gguf.py b/examples/train-text-from-scratch/convert-train-checkpoint-to-gguf.py index 37d0244fb..d7ea4e6fe 100644 --- a/examples/train-text-from-scratch/convert-train-checkpoint-to-gguf.py +++ b/examples/train-text-from-scratch/convert-train-checkpoint-to-gguf.py @@ -453,6 +453,7 @@ class Checkpoint: def save_gguf(self, gguf_writer): gguf_writer.add_architecture() gguf_writer.add_file_type(gguf.GGMLQuantizationType.F32) + gguf_writer.add_layer_norm_rms_eps(1e-5) gguf_writer.add_uint32(LLM_KV_TRAINING_FILE_VERSION, 0) gguf_writer.add_uint32(LLM_KV_TRAINING_ITERATION_COUNT, self.train_its) gguf_writer.add_uint32(LLM_KV_TRAINING_SAMPLE_COUNT, self.train_samples)