From 3b5515bbe0e2224425986ba24f1f5d84aa38dce9 Mon Sep 17 00:00:00 2001 From: xaedes Date: Mon, 14 Aug 2023 22:09:36 +0200 Subject: [PATCH] reverse order of for loop in ggml_build_backward_expand to save memory when using gradient checkpointing and allocator with this loop order gradient checkpointing with allocator on 16 layer model saves 13% memory; 2 layer memory it saves 2% memory. the computation results are the same --- ggml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml.c b/ggml.c index 142d5965a..79098a2fc 100644 --- a/ggml.c +++ b/ggml.c @@ -15809,7 +15809,7 @@ void ggml_build_backward_expand(struct ggml_context * ctx, struct ggml_cgraph * } } - for (int i = gf->n_nodes - 1; i >= 0; i--) { + for (int i = 0; i < gf->n_nodes; i++) { struct ggml_tensor * node = gf->nodes[i]; if (node->is_param) {