vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama: add llama_model_kv_override_free

Browse source
This commit is contained in:
Pierrick HYMBERT 2024-04-19 13:16:42 +02:00
parent ea0ad80a4f
commit 82e4187f95
3 changed files with 21 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

1
common/common.cpp
View file

@ -263,7 +263,6 @@ bool parse_kv_override(const char * data, std::vector<llama_model_kv_override> &
fprintf(stderr, "%s: invalid boolean value for KV override '%s'\n", __func__, data); fprintf(stderr, "%s: invalid boolean value for KV override '%s'\n", __func__, data);
return false; return false;
} }
} else if (strncmp(sep, "str:", 4) == 0) { } else if (strncmp(sep, "str:", 4) == 0) {
sep += 4; sep += 4;
kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR; kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;

18
llama.cpp
View file

@ -2929,6 +2929,16 @@ namespace GGUFMeta {
return false; return false;
} }
template<typename OT>
static typename std::enable_if<std::is_same<OT, char *>::value, char *>::type
try_override(T & target, const struct llama_model_kv_override * ovrd) {
if (validate_override(LLAMA_KV_OVERRIDE_TYPE_STR, ovrd)) {
target = ovrd->str_value;
return true;
}
return false;
}
template<typename OT> template<typename OT>
static typename std::enable_if<std::is_same<OT, std::string>::value, bool>::type static typename std::enable_if<std::is_same<OT, std::string>::value, bool>::type
try_override(T & target, const struct llama_model_kv_override * ovrd) { try_override(T & target, const struct llama_model_kv_override * ovrd) {
@ -14977,6 +14987,14 @@ void llama_free_model(struct llama_model * model) {
delete model; delete model;
} }
void llama_model_kv_override_free(struct llama_model_kv_override * kv_overrides) {
for (const struct llama_model_kv_override *p = kv_overrides; p->key[0] != 0; p++) {
if (p->tag == LLAMA_KV_OVERRIDE_TYPE_STR) {
delete p->str_value;
}
}
}
struct llama_context * llama_new_context_with_model( struct llama_context * llama_new_context_with_model(
struct llama_model * model, struct llama_model * model,
struct llama_context_params params) { struct llama_context_params params) {

3
llama.h
View file

@ -209,6 +209,9 @@ extern "C" {
}; };
}; };
// Frees all allocated memory
LLAMA_API void llama_model_kv_override_free(struct llama_model_kv_override * ctx);
struct llama_model_params { struct llama_model_params {
int32_t n_gpu_layers; // number of layers to store in VRAM int32_t n_gpu_layers; // number of layers to store in VRAM
enum llama_split_mode split_mode; // how to split the model across multiple GPUs enum llama_split_mode split_mode; // how to split the model across multiple GPUs