vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Adjusted look ahead in ggml_cuda_pool_malloc to 5%

Browse source 
This is sufficient it seems.
We end up using about 200 MB less VRAM that way when running
the 13B model with context 8192.
This commit is contained in:
Iwan Kawrakow 2023-07-21 11:58:52 +03:00
parent 1cdbbbb37c
commit b068f2f4b5
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
ggml-cuda.cu
View file

@ -2466,7 +2466,7 @@ static void * ggml_cuda_pool_malloc(size_t size, size_t * actual_size) {
(uint32_t)(max_size/1024/1024), (uint32_t)(tot_size/1024/1024), (uint32_t)(size/1024/1024)); (uint32_t)(max_size/1024/1024), (uint32_t)(tot_size/1024/1024), (uint32_t)(size/1024/1024));
#endif #endif
void * ptr; void * ptr;
size_t look_ahead_size = (size_t) (1.25 * size); size_t look_ahead_size = (size_t) (1.05 * size);
look_ahead_size = 256 * ((look_ahead_size + 255)/256); look_ahead_size = 256 * ((look_ahead_size + 255)/256);
CUDA_CHECK(cudaMalloc((void **) &ptr, look_ahead_size)); CUDA_CHECK(cudaMalloc((void **) &ptr, look_ahead_size));
*actual_size = look_ahead_size; *actual_size = look_ahead_size;