vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

common : by default, move the penalties at the end of the sampling chain

Browse source 
ggml-ci
This commit is contained in:
Georgi Gerganov 2024-12-12 22:29:09 +02:00
parent 9847a375f3
commit 97261aa216
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
2 changed files with 12 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

3
common/common.h
View file

@ -146,6 +146,7 @@ struct common_params_sampling {
COMMON_SAMPLER_TYPE_MIN_P,
COMMON_SAMPLER_TYPE_XTC,
COMMON_SAMPLER_TYPE_TEMPERATURE,
COMMON_SAMPLER_TYPE_PENALTIES,
};
std::string grammar; // optional BNF-like grammar to constrain sampling
@ -194,9 +195,11 @@ struct common_params {
// offload params
std::vector<ggml_backend_dev_t> devices; // devices to use for offloading
int32_t n_gpu_layers = -1; // number of layers to store in VRAM (-1 - use default)
int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors
float tensor_split[128] = {0}; // how split tensors should be distributed across GPUs
enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
struct cpu_params cpuparams;