vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Optimizations

Browse source
This commit is contained in:
Concedo 2023-03-24 21:33:53 +08:00
parent 3879d84400
commit c6c60332a4
3 changed files with 10 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

15
expose.cpp
View file

@ -161,25 +161,28 @@ extern "C" {
std::fill(last_n_tokens.begin(), last_n_tokens.end(), 0); std::fill(last_n_tokens.begin(), last_n_tokens.end(), 0);
n_past = 0; n_past = 0;
//fast forward the past based on identical tokens, stop once a divergence is noted //fast forward the past based on identical tokens, stop once a divergence is noted
int embd_inp_len = embd_inp.size();
for(int i=0;i<current_context_tokens.size();++i) for(int i=0;i<current_context_tokens.size();++i)
{ {
if(current_context_tokens[i]==embd_inp[0]) if(current_context_tokens[i]==embd_inp[i])
{ {
n_past += 1; n_past += 1;
embd_inp.erase(embd_inp.begin());
last_n_tokens.erase(last_n_tokens.begin());
last_n_tokens.push_back(current_context_tokens[i]); last_n_tokens.push_back(current_context_tokens[i]);
} }
else else
{ {
break; break;
} }
if(embd_inp.size()<=1) if((i+2)>=embd_inp_len)
{ {
break; break;
} }
} }
last_n_tokens.erase(last_n_tokens.begin(),last_n_tokens.begin()+n_past);
embd_inp.erase(embd_inp.begin(),embd_inp.begin()+n_past);
current_context_tokens.resize(n_past); current_context_tokens.resize(n_past);
int remaining_tokens = params.n_predict; int remaining_tokens = params.n_predict;

2
llama_for_kobold.py
View file

@ -289,7 +289,7 @@ if __name__ == '__main__':
mdl_nparts += 1 mdl_nparts += 1
modelname = os.path.abspath(sys.argv[1]) modelname = os.path.abspath(sys.argv[1])
print("Loading model: " + modelname) print("Loading model: " + modelname)
loadok = load_model(modelname,16,maxctx,mdl_nparts) loadok = load_model(modelname,8,maxctx,mdl_nparts)
print("Load Model OK: " + str(loadok)) print("Load Model OK: " + str(loadok))
#friendlymodelname = Path(modelname).stem ### this wont work on local kobold api, so we must hardcode a known HF model name #friendlymodelname = Path(modelname).stem ### this wont work on local kobold api, so we must hardcode a known HF model name

BIN
llamacpp.dll
View file

Binary file not shown.