still needs to be a bit higher for very small contexts
This commit is contained in:
parent
4e1ea2ac61
commit
432cc91649
3 changed files with 6 additions and 6 deletions
|
@ -374,8 +374,8 @@ bool gpt2_eval(
|
|||
static size_t buf_size = 256u*1024*1024;
|
||||
static void * buf = malloc(buf_size);
|
||||
|
||||
if (mem_per_token > 0 && mem_per_token*N*1.6 > buf_size) {
|
||||
const size_t buf_size_new = 2*(mem_per_token*N); // add 10% to account for ggml object overhead
|
||||
if (mem_per_token > 0 && mem_per_token*N*1.9 > buf_size) {
|
||||
const size_t buf_size_new = 320u*1024*1024 + 2*(mem_per_token*N); // add 10% to account for ggml object overhead
|
||||
//printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
|
||||
|
||||
// reallocate
|
||||
|
|
|
@ -385,8 +385,8 @@ bool gptj_eval(
|
|||
static size_t buf_size = 256u*1024*1024;
|
||||
static void * buf = malloc(buf_size);
|
||||
|
||||
if (mem_per_token > 0 && mem_per_token*N*1.6 > buf_size) {
|
||||
const size_t buf_size_new = 1.8*(mem_per_token*N); // add 10% to account for ggml object overhead
|
||||
if (mem_per_token > 0 && mem_per_token*N*1.9 > buf_size) {
|
||||
const size_t buf_size_new = 320u*1024*1024 + 2*(mem_per_token*N); // add 10% to account for ggml object overhead
|
||||
//printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
|
||||
|
||||
// reallocate
|
||||
|
|
|
@ -367,8 +367,8 @@ bool stablelm_eval(
|
|||
static size_t buf_size = 256u*1024*1024;
|
||||
static void * buf = malloc(buf_size);
|
||||
|
||||
if (mem_per_token > 0 && mem_per_token*N*1.6 > buf_size) {
|
||||
const size_t buf_size_new = 1.8*(mem_per_token*N); // add 10% to account for ggml object overhead
|
||||
if (mem_per_token > 0 && mem_per_token*N*1.9 > buf_size) {
|
||||
const size_t buf_size_new = 320u*1024*1024 + 2*(mem_per_token*N); // add 10% to account for ggml object overhead
|
||||
//printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
|
||||
|
||||
// reallocate
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue