diff --git a/src/llama.cpp b/src/llama.cpp
index 9002ccd24..bc94d7ff0 100644
--- a/src/llama.cpp
+++ b/src/llama.cpp
@@ -4284,7 +4284,21 @@ struct llama_model_loader {
         }
     };
 
-    std::unordered_map<std::string, struct llama_tensor_weight> weights_map;
+    // custom comparator to sort weights more nicely by layer
+    struct weight_name_comparer {
+        bool operator()(const std::string & a, const std::string & b) const {
+            int a_layer = -1;
+            int b_layer = -1;
+            sscanf(a.c_str(), "blk.%d.", &a_layer);
+            sscanf(b.c_str(), "blk.%d.", &b_layer);
+            if (a_layer != b_layer) {
+                return a_layer < b_layer;
+            }
+            return a < b;
+        }
+    };
+
+    std::map<std::string, struct llama_tensor_weight, weight_name_comparer> weights_map;
     std::unordered_map<std::string, struct llama_model_kv_override> kv_overrides;
 
     struct gguf_context * meta = NULL;