vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

bump scratch buffers

Browse source
This commit is contained in:
Concedo 2023-07-22 18:12:18 +08:00
parent 52c5856a08
commit 3aec3038d4
2 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

2
koboldcpp.py
View file

@ -283,7 +283,7 @@ maxhordectx = 1024
maxhordelen = 256 maxhordelen = 256
modelbusy = threading.Lock() modelbusy = threading.Lock()
defaultport = 5001 defaultport = 5001
KcppVersion = "1.36" KcppVersion = "1.37"
showdebug = True showdebug = True
showsamplerwarning = True showsamplerwarning = True
exitcounter = 0 exitcounter = 0

10
llama.cpp
View file

@ -106,11 +106,11 @@ static const std::map<e_model, size_t> & MEM_REQ_SCRATCH0(int n_ctx)
{ {
static std::map<e_model, size_t> k_sizes = { static std::map<e_model, size_t> k_sizes = {
/* empirical scaling, still a guess */ /* empirical scaling, still a guess */
{ MODEL_3B, ((size_t) n_ctx / 16ull + 180ull) * MB }, { MODEL_3B, ((size_t) n_ctx / 11ull + 320ull) * MB },
{ MODEL_7B, ((size_t) n_ctx / 16ull + 320ull) * MB }, { MODEL_7B, ((size_t) n_ctx / 11ull + 440ull) * MB },
{ MODEL_13B, ((size_t) n_ctx / 12ull + 460ull) * MB }, { MODEL_13B, ((size_t) n_ctx / 10ull + 560ull) * MB },
{ MODEL_30B, ((size_t) n_ctx / 10ull + 620ull) * MB }, { MODEL_30B, ((size_t) n_ctx / 9ull + 680ull) * MB },
{ MODEL_65B, ((size_t) n_ctx / 8ull + 860ull) * MB }, { MODEL_65B, ((size_t) n_ctx / 8ull + 1000ull) * MB },
}; };
return k_sizes; return k_sizes;
} }