From 432cc9164933a0687e7b04f828a8a2bc5a831170 Mon Sep 17 00:00:00 2001
From: Concedo <39025047+LostRuins@users.noreply.github.com>
Date: Sun, 23 Apr 2023 15:01:38 +0800
Subject: [PATCH] still needs to be a bit higher for very small contexts

---
 otherarch/gpt2_v2.cpp | 4 ++--
 otherarch/gptj_v2.cpp | 4 ++--
 otherarch/neox.cpp    | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/otherarch/gpt2_v2.cpp b/otherarch/gpt2_v2.cpp
index 0e68195e0..ee180f86c 100644
--- a/otherarch/gpt2_v2.cpp
+++ b/otherarch/gpt2_v2.cpp
@@ -374,8 +374,8 @@ bool gpt2_eval(
     static size_t buf_size = 256u*1024*1024;    
     static void * buf = malloc(buf_size);
 
-    if (mem_per_token > 0 && mem_per_token*N*1.6 > buf_size) {
-        const size_t buf_size_new = 2*(mem_per_token*N); // add 10% to account for ggml object overhead
+    if (mem_per_token > 0 && mem_per_token*N*1.9 > buf_size) {
+        const size_t buf_size_new = 320u*1024*1024 + 2*(mem_per_token*N); // add 10% to account for ggml object overhead
         //printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
 
         // reallocate
diff --git a/otherarch/gptj_v2.cpp b/otherarch/gptj_v2.cpp
index 5ad7d36c0..212c79a47 100644
--- a/otherarch/gptj_v2.cpp
+++ b/otherarch/gptj_v2.cpp
@@ -385,8 +385,8 @@ bool gptj_eval(
     static size_t buf_size = 256u*1024*1024;    
     static void * buf = malloc(buf_size);
 
-    if (mem_per_token > 0 && mem_per_token*N*1.6 > buf_size) {
-        const size_t buf_size_new = 1.8*(mem_per_token*N); // add 10% to account for ggml object overhead
+    if (mem_per_token > 0 && mem_per_token*N*1.9 > buf_size) {
+        const size_t buf_size_new = 320u*1024*1024 + 2*(mem_per_token*N); // add 10% to account for ggml object overhead
         //printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
 
         // reallocate
diff --git a/otherarch/neox.cpp b/otherarch/neox.cpp
index caf34200e..6866736f8 100644
--- a/otherarch/neox.cpp
+++ b/otherarch/neox.cpp
@@ -367,8 +367,8 @@ bool stablelm_eval(
     static size_t buf_size = 256u*1024*1024;    
     static void * buf = malloc(buf_size);
 
-    if (mem_per_token > 0 && mem_per_token*N*1.6 > buf_size) {
-        const size_t buf_size_new = 1.8*(mem_per_token*N); // add 10% to account for ggml object overhead
+    if (mem_per_token > 0 && mem_per_token*N*1.9 > buf_size) {
+        const size_t buf_size_new = 320u*1024*1024 + 2*(mem_per_token*N); // add 10% to account for ggml object overhead
         //printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
 
         // reallocate