vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixed auto rope scaling (+1 squashed commits)

Browse source 
Squashed commits:

[b1767874] wip
This commit is contained in:
Concedo 2023-09-07 14:19:42 +08:00
parent 0bf75b05dc
commit 7fb809b94b
5 changed files with 42 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

33
expose.cpp
View file

@ -27,6 +27,7 @@ extern "C"
//return val: 0=fail, 1=(original ggml, alpaca), 2=(ggmf), 3=(ggjt)
static FileFormat file_format = FileFormat::BADFORMAT;
static FileFormatExtraMeta file_format_meta;
bool load_model(const load_model_inputs inputs)
{
@ -36,11 +37,9 @@ extern "C"
int forceversion = inputs.forceversion;
if(forceversion==0)
{
file_format = check_file_format(model.c_str());
}
else
file_format = check_file_format(model.c_str(),&file_format_meta);
if(forceversion!=0)
{
printf("\nWARNING: FILE FORMAT FORCED TO VER %d\nIf incorrect, loading may fail or crash.\n",forceversion);
file_format = (FileFormat)forceversion;
@ -64,7 +63,7 @@ extern "C"
if(file_format==FileFormat::GPTJ_1 || file_format==FileFormat::GPTJ_2 || file_format==FileFormat::GPTJ_3 || file_format==FileFormat::GPTJ_4 || file_format==FileFormat::GPTJ_5)
{
printf("\n---\nIdentified as GPT-J model: (ver %d)\nAttempting to Load...\n---\n", file_format);
ModelLoadResult lr = gpttype_load_model(inputs, file_format);
ModelLoadResult lr = gpttype_load_model(inputs, file_format, file_format_meta);
if (lr == ModelLoadResult::RETRY_LOAD)
{
if(file_format==FileFormat::GPTJ_1)
@ -73,14 +72,14 @@ extern "C"
//otherwise if we tried 3 first, then try 2
file_format = FileFormat::GPTJ_4;
printf("\n---\nRetrying as GPT-J model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}
if (lr == ModelLoadResult::RETRY_LOAD)
{
file_format = FileFormat::GPTJ_3;
printf("\n---\nRetrying as GPT-J model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}
//lastly try format 2
@ -88,7 +87,7 @@ extern "C"
{
file_format = FileFormat::GPTJ_2;
printf("\n---\nRetrying as GPT-J model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}
}
@ -104,18 +103,18 @@ extern "C"
else if(file_format==FileFormat::GPT2_1||file_format==FileFormat::GPT2_2||file_format==FileFormat::GPT2_3||file_format==FileFormat::GPT2_4)
{
printf("\n---\nIdentified as GPT-2 model: (ver %d)\nAttempting to Load...\n---\n", file_format);
ModelLoadResult lr = gpttype_load_model(inputs, file_format);
ModelLoadResult lr = gpttype_load_model(inputs, file_format, file_format_meta);
if (lr == ModelLoadResult::RETRY_LOAD)
{
file_format = FileFormat::GPT2_3;
printf("\n---\nRetrying as GPT-2 model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}
if (lr == ModelLoadResult::RETRY_LOAD)
{
file_format = FileFormat::GPT2_2;
printf("\n---\nRetrying as GPT-2 model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}
if (lr == ModelLoadResult::FAIL || lr == ModelLoadResult::RETRY_LOAD)
{
@ -129,27 +128,27 @@ extern "C"
else if(file_format==FileFormat::NEOX_1 || file_format==FileFormat::NEOX_2 || file_format==FileFormat::NEOX_3 || file_format==FileFormat::NEOX_4 || file_format==FileFormat::NEOX_5 || file_format==FileFormat::NEOX_6 || file_format==FileFormat::NEOX_7)
{
printf("\n---\nIdentified as GPT-NEO-X model: (ver %d)\nAttempting to Load...\n---\n", file_format);
ModelLoadResult lr = gpttype_load_model(inputs, file_format);
ModelLoadResult lr = gpttype_load_model(inputs, file_format, file_format_meta);
if (lr == ModelLoadResult::RETRY_LOAD)
{
if(file_format==FileFormat::NEOX_2)
{
file_format = FileFormat::NEOX_3;
printf("\n---\nRetrying as GPT-NEO-X model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}
else
{
file_format = FileFormat::NEOX_5;
printf("\n---\nRetrying as GPT-NEO-X model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}
}
if (lr == ModelLoadResult::RETRY_LOAD)
{
file_format = FileFormat::NEOX_1;
printf("\n---\nRetrying as GPT-NEO-X model: (ver %d)\nAttempting to Load...\n---\n", file_format);
lr = gpttype_load_model(inputs, file_format);
lr = gpttype_load_model(inputs, file_format, file_format_meta);
}
if (lr == ModelLoadResult::FAIL || lr == ModelLoadResult::RETRY_LOAD)
{
@ -178,7 +177,7 @@ extern "C"
{
printf("\n---\nIdentified as LLAMA model: (ver %d)\nAttempting to Load...\n---\n", file_format);
}
ModelLoadResult lr = gpttype_load_model(inputs, file_format);
ModelLoadResult lr = gpttype_load_model(inputs, file_format, file_format_meta);
if (lr == ModelLoadResult::FAIL || lr == ModelLoadResult::RETRY_LOAD)
{
return false;

12
gpttype_adapter.cpp
View file

@ -393,7 +393,7 @@ static std::string RemoveBell(const std::string & input) //removes the bell char
return word2;
}
ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in_file_format)
ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in_file_format, FileFormatExtraMeta file_format_meta)
{
ggml_time_init();
@ -438,11 +438,11 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
{
//approximate NTK aware ctx
auto effectivenctx = params.n_ctx;
// if((file_format == FileFormat::GGUF_LLAMA || file_format==FileFormat::GGUF_FALCON) && llama_ctx_v4->model.hparams.n_ctx_train>2048)
// {
// float factor = llama_ctx_v4->model.hparams.n_ctx_train/2048;
// effectivenctx = effectivenctx/factor;
// }
if((file_format == FileFormat::GGUF_LLAMA || file_format==FileFormat::GGUF_FALCON) && file_format_meta.n_ctx_train > 2048)
{
float factor = file_format_meta.n_ctx_train/2048;
effectivenctx = effectivenctx/factor;
}
rope_freq_base = (effectivenctx <= 3072 ? 26000.0f : (effectivenctx <= 4096 ? 32000.0f : (effectivenctx <= 6144 ? 54000.0f : (effectivenctx <= 8192 ? 82684.0f : (effectivenctx <= 12288 ? 140000.0f : 200000.0f)))));
}

2
koboldcpp.py
View file

@ -330,7 +330,7 @@ maxhordectx = 1024
maxhordelen = 256
modelbusy = threading.Lock()
defaultport = 5001
KcppVersion = "1.42.1"
KcppVersion = "1.43"
showdebug = True
showsamplerwarning = True
showmaxctxwarning = True

14
model_adapter.cpp
View file

@ -80,7 +80,7 @@ void print_tok_vec(std::vector<float> &embd)
}
//return val: 0=fail, 1=(original ggml, alpaca), 2=(ggmf), 3=(ggjt)
FileFormat check_file_format(const std::string & fname)
FileFormat check_file_format(const std::string & fname, FileFormatExtraMeta * fileformatmeta)
{
std::vector<char> f_buf(1024*1024);
@ -266,7 +266,7 @@ void print_tok_vec(std::vector<float> &embd)
auto keyidx = gguf_find_key(ctx, "general.architecture");
std::string modelarch = "";
if (keyidx != -1) { modelarch = gguf_get_val_str(ctx, keyidx); }
gguf_free(ctx);
if(modelarch=="llama")
{
fileformat = FileFormat::GGUF_LLAMA;
@ -280,6 +280,16 @@ void print_tok_vec(std::vector<float> &embd)
{
printf("\nERROR: Detected unimplemented GGUF Arch: %s\n",modelarch.c_str());
}
if(modelarch!="" && fileformatmeta!=nullptr)
{
std::string fkey = modelarch+".context_length";
auto keyidx = gguf_find_key(ctx, fkey.c_str());
if (keyidx != -1) {
fileformatmeta->n_ctx_train = gguf_get_val_u32(ctx, keyidx);
}
}
gguf_free(ctx);
}
if(fin.is_open())

9
model_adapter.h
View file

@ -51,6 +51,11 @@ enum FileFormat
};
struct FileFormatExtraMeta
{
int n_ctx_train = 2048;
};
enum ModelLoadResult
{
FAIL = 0,
@ -58,7 +63,7 @@ enum ModelLoadResult
RETRY_LOAD = 2, //used if it's suspected that the model is an older format
};
ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in_file_format);
ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in_file_format, FileFormatExtraMeta file_format_meta);
generation_outputs gpttype_generate(const generation_inputs inputs, generation_outputs &output);
bool gpttype_generate_abort();
const std::string & gpttype_get_pending_output();
@ -73,7 +78,7 @@ std::vector<int> LongestCommonSubseq(const std::vector<int> x, const std::vector
bool ArrStartWith(const std::vector<int> targetArray, const std::vector<int> searchSeq);
int ArrFindIndexOf(const std::vector<int> targetArray, const std::vector<int> searchSeq);
FileFormat check_file_format(const std::string & fname);
FileFormat check_file_format(const std::string & fname, FileFormatExtraMeta * fileformatmeta);
void ContextFastForward(std::vector<int> &current_context_tokens, std::vector<int> &embd_inp,
int &n_past, std::vector<int> &last_n_tokens, const int nctx, std::vector<int> &smartcontext,
const bool useSmartContext, const bool requireFullSubset);