From cd160f85f1776fec40010758895685f56c76f723 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 6 Jan 2024 14:05:13 +0200 Subject: [PATCH] metal : fix kernel init + fix compile options --- ggml-metal.m | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/ggml-metal.m b/ggml-metal.m index 1c27dab20..020acf855 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -26,7 +26,7 @@ #define GGML_MAX_CONCUR (2*GGML_DEFAULT_GRAPH_SIZE) -#define GGML_METAL_MAX_KERNELS 1024 +#define GGML_METAL_MAX_KERNELS 256 struct ggml_metal_buffer { const char * name; @@ -294,16 +294,16 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) { return NULL; } - MTLCompileOptions* options = nil; + // dictionary of preprocessor macros + NSMutableDictionary * prep = [NSMutableDictionary dictionary]; + #ifdef GGML_QKK_64 - options = [MTLCompileOptions new]; - options.preprocessorMacros = @{ @"QK_K" : @(64) }; + prep[@"QK_K"] = @(64); #endif - // try to disable fast-math - // NOTE: this seems to have no effect whatsoever - // instead, in order to disable fast-math, we have to build default.metallib from the command line - // using xcrun -sdk macosx metal -fno-fast-math -c ggml-metal.metal -o ggml-metal.air - // and go through the "pre-compiled library found" path above + + MTLCompileOptions* options = [MTLCompileOptions new]; + options.preprocessorMacros = prep; + //[options setFastMathEnabled:false]; ctx->library = [ctx->device newLibraryWithSource:src options:options error:&error]; @@ -355,6 +355,11 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) { { NSError * error = nil; + for (int i = 0; i < GGML_METAL_MAX_KERNELS; ++i) { + ctx->kernels[i].function = nil; + ctx->kernels[i].pipeline = nil; + } + #define GGML_METAL_ADD_KERNEL(e, name, family) \ if ([ctx->device supportsFamily:(family)]) { \ struct ggml_metal_kernel * kernel = &ctx->kernels[e]; \