diff --git a/README.md b/README.md index 4ceeb22f2..d8bd0facf 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ What does it mean? You get llama.cpp with a fancy UI, persistent stories, editin - CLBlast - tested with https://github.com/CNugteren/CLBlast . If you wish to compile it you will need to reference the OpenCL files. It will only generate the ".lib" file if you compile using MSVC. - OpenBLAS - tested with https://github.com/xianyi/OpenBLAS . - Move the respectives .lib files to the /lib folder of your project, overwriting the older files. + - Also, replace the existing versions of the corresponding .dll files located in the project directory root (e.g. libopenblas.dll). - Make the KoboldCPP project using the instructions above. ## OSX and Linux diff --git a/otherarch/gpt2_v2.cpp b/otherarch/gpt2_v2.cpp index 2f4c0b0be..0e68195e0 100644 --- a/otherarch/gpt2_v2.cpp +++ b/otherarch/gpt2_v2.cpp @@ -371,7 +371,7 @@ bool gpt2_eval( const int n_vocab = hparams.n_vocab; //todo: there is a bug that causes the buffer to oom and I cannot figure it out, hack to increase size for now - static size_t buf_size = 512u*1024*1024; + static size_t buf_size = 256u*1024*1024; static void * buf = malloc(buf_size); if (mem_per_token > 0 && mem_per_token*N*1.6 > buf_size) { diff --git a/otherarch/gptj_v2.cpp b/otherarch/gptj_v2.cpp index 040be8478..5ad7d36c0 100644 --- a/otherarch/gptj_v2.cpp +++ b/otherarch/gptj_v2.cpp @@ -382,11 +382,11 @@ bool gptj_eval( const int d_key = n_embd/n_head; //todo: there is a bug that causes the buffer to oom and I cannot figure it out, hack to increase size for now - static size_t buf_size = 512u*1024*1024; + static size_t buf_size = 256u*1024*1024; static void * buf = malloc(buf_size); - if (mem_per_token > 0 && mem_per_token*N*1.4 > buf_size) { - const size_t buf_size_new = 1.6*(mem_per_token*N); // add 10% to account for ggml object overhead + if (mem_per_token > 0 && mem_per_token*N*1.6 > buf_size) { + const size_t buf_size_new = 1.8*(mem_per_token*N); // add 10% to account for ggml object overhead //printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new); // reallocate diff --git a/otherarch/neox.cpp b/otherarch/neox.cpp index 1bcfad61b..caf34200e 100644 --- a/otherarch/neox.cpp +++ b/otherarch/neox.cpp @@ -364,11 +364,11 @@ bool stablelm_eval( const int n_vocab = hparams.n_vocab; const int n_rot = hparams.n_rot; - static size_t buf_size = 512u*1024*1024; + static size_t buf_size = 256u*1024*1024; static void * buf = malloc(buf_size); - if (mem_per_token > 0 && mem_per_token*N*1.4 > buf_size) { - const size_t buf_size_new = 1.6*(mem_per_token*N); // add 10% to account for ggml object overhead + if (mem_per_token > 0 && mem_per_token*N*1.6 > buf_size) { + const size_t buf_size_new = 1.8*(mem_per_token*N); // add 10% to account for ggml object overhead //printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new); // reallocate