vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

common: add llama_model_kv_override_free

Browse source 
common: free kv override if used after model loading
This commit is contained in:
Pierrick HYMBERT 2024-04-20 10:17:03 +02:00
parent 82e4187f95
commit aa0e28f8fc
4 changed files with 15 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

12
common/common.cpp
View file

@ -2232,6 +2232,10 @@ std::tuple<struct llama_model *, struct llama_context *> llama_init_from_gpt_par
return std::make_tuple(nullptr, nullptr); return std::make_tuple(nullptr, nullptr);
} }
if (!params.kv_overrides.empty()) {
llama_model_kv_override_free(params.kv_overrides.data());
}
auto cparams = llama_context_params_from_gpt_params(params); auto cparams = llama_context_params_from_gpt_params(params);
llama_context * lctx = llama_new_context_with_model(model, cparams); llama_context * lctx = llama_new_context_with_model(model, cparams);
@ -2952,3 +2956,11 @@ llama_control_vector_data llama_control_vector_load(const std::vector<llama_cont
return result; return result;
} }
void llama_model_kv_override_free(struct llama_model_kv_override * kv_overrides) {
for (const struct llama_model_kv_override *p = kv_overrides; p->key[0] != 0; p++) {
if (p->tag == LLAMA_KV_OVERRIDE_TYPE_STR) {
delete p->str_value;
}
}
}

3
common/common.h
View file

@ -172,6 +172,9 @@ struct gpt_params {
bool parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides); bool parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides);
// Frees all allocated memory
void llama_model_kv_override_free(struct llama_model_kv_override * ctx);
bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params); bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params);
bool gpt_params_parse(int argc, char ** argv, gpt_params & params); bool gpt_params_parse(int argc, char ** argv, gpt_params & params);

8
llama.cpp
View file

@ -14987,14 +14987,6 @@ void llama_free_model(struct llama_model * model) {
delete model; delete model;
} }
void llama_model_kv_override_free(struct llama_model_kv_override * kv_overrides) {
for (const struct llama_model_kv_override *p = kv_overrides; p->key[0] != 0; p++) {
if (p->tag == LLAMA_KV_OVERRIDE_TYPE_STR) {
delete p->str_value;
}
}
}
struct llama_context * llama_new_context_with_model( struct llama_context * llama_new_context_with_model(
struct llama_model * model, struct llama_model * model,
struct llama_context_params params) { struct llama_context_params params) {

3
llama.h
View file

@ -209,9 +209,6 @@ extern "C" {
}; };
}; };
// Frees all allocated memory
LLAMA_API void llama_model_kv_override_free(struct llama_model_kv_override * ctx);
struct llama_model_params { struct llama_model_params {
int32_t n_gpu_layers; // number of layers to store in VRAM int32_t n_gpu_layers; // number of layers to store in VRAM
enum llama_split_mode split_mode; // how to split the model across multiple GPUs enum llama_split_mode split_mode; // how to split the model across multiple GPUs