vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

still refactoring

Browse source
This commit is contained in:
Concedo 2023-04-01 11:56:34 +08:00
parent 6e6125ebdb
commit 085a9f90a7
11 changed files with 72 additions and 261 deletions
Download patch file Download diff file Expand all files Collapse all files

19
Makefile
View file

@ -206,7 +206,7 @@ endif
BLAS_BUILD = BLAS_BUILD =
ifeq ($(OS),Windows_NT) ifeq ($(OS),Windows_NT)
BLAS_BUILD = $(CXX) $(CXXFLAGS) ggml_blas.o common.o extra.o expose.o model_adapter.o libopenblas.lib -shared -o llamacpp_blas.dll $(LDFLAGS) BLAS_BUILD = $(CXX) $(CXXFLAGS) ggml_blas.o expose.o llama_adapter.o llamaextra.o common.o libopenblas.lib -shared -o llamacpp_blas.dll $(LDFLAGS)
else else
BLAS_BUILD = @echo 'Your OS is $(OS) and does not appear to be Windows. If you want to use openblas, please link it manually with LLAMA_OPENBLAS=1' BLAS_BUILD = @echo 'Your OS is $(OS) and does not appear to be Windows. If you want to use openblas, please link it manually with LLAMA_OPENBLAS=1'
endif endif
@ -247,14 +247,17 @@ llama.o: llama.cpp llama.h
common.o: examples/common.cpp examples/common.h common.o: examples/common.cpp examples/common.h
$(CXX) $(CXXFLAGS) -c examples/common.cpp -o common.o $(CXX) $(CXXFLAGS) -c examples/common.cpp -o common.o
extra.o: extra.cpp extra.h llamaextra.o: llamaextra.cpp llamaextra.h
$(CXX) $(CXXFLAGS) -c extra.cpp -o extra.o $(CXX) $(CXXFLAGS) -c llamaextra.cpp -o llamaextra.o
expose.o: expose.cpp expose.h expose.o: expose.cpp expose.h
$(CXX) $(CXXFLAGS) -c expose.cpp -o expose.o $(CXX) $(CXXFLAGS) -c expose.cpp -o expose.o
model_adapter.o: llama_adapter.o:
$(CXX) $(CXXFLAGS) -c llama_adapter.cpp -o model_adapter.o $(CXX) $(CXXFLAGS) -c llama_adapter.cpp -o llama_adapter.o
gptj_adapter.o: ggml_old_v1.o
$(CXX) $(CXXFLAGS) otherarch/gptj_old.cpp otherarch/utils.cpp ggml_old_v1.o gptj_adapter.cpp -o gptj_adapter.o
clean: clean:
rm -vf *.o main quantize perplexity embedding main.exe quantize.exe llamacpp.dll llamacpp_blas.dll gpt2.exe gptj.exe rm -vf *.o main quantize perplexity embedding main.exe quantize.exe llamacpp.dll llamacpp_blas.dll gpt2.exe gptj.exe
@ -269,10 +272,10 @@ gptj: ggml_old_v1.o
$(CXX) $(CXXFLAGS) otherarch/gptj_old.cpp otherarch/utils.cpp ggml_old_v1.o -o gptj $(LDFLAGS) $(CXX) $(CXXFLAGS) otherarch/gptj_old.cpp otherarch/utils.cpp ggml_old_v1.o -o gptj $(LDFLAGS)
llamalib: ggml.o common.o extra.o expose.o model_adapter.o llamalib: ggml.o expose.o llama_adapter.o llamaextra.o common.o
$(CXX) $(CXXFLAGS) ggml.o common.o extra.o expose.o model_adapter.o -shared -o llamacpp.dll $(LDFLAGS) $(CXX) $(CXXFLAGS) expose.o ggml.o llama_adapter.o llamaextra.o common.o -shared -o llamacpp.dll $(LDFLAGS)
llamalib_blas: ggml_blas.o common.o extra.o expose.o model_adapter.o llamalib_blas: ggml_blas.o expose.o llama_adapter.o llamaextra.o common.o
$(BLAS_BUILD) $(BLAS_BUILD)
quantize: examples/quantize/quantize.cpp ggml.o llama.o quantize: examples/quantize/quantize.cpp ggml.o llama.o

4
expose.cpp
View file

@ -9,7 +9,9 @@
#include "model_adapter.h" #include "model_adapter.h"
#include "expose.h" #include "expose.h"
#include "extra.h" #include "llamaextra.h"
extern "C" extern "C"
{ {

3
expose.h
View file

@ -25,4 +25,5 @@ struct generation_outputs
{ {
int status = -1; int status = -1;
char text[16384]; //16kb should be enough for any response char text[16384]; //16kb should be enough for any response
}; };

259
gptj_adapter.cpp
View file

@ -10,196 +10,50 @@
#include <time.h> #include <time.h>
#include "model_adapter.h" #include "model_adapter.h"
#include "otherarch/otherarch.h" #include "otherarch/otherarch.h"
#include "llamaextra.h"
//return val: 0=fail, 1=(original ggml, alpaca), 2=(ggmf), 3=(ggjt) //return val: 0=fail, 1=(original ggml, alpaca), 2=(ggmf), 3=(ggjt)
static FileFormat file_format = FileFormat::FAIL; static FileFormat file_format = FileFormat::FAIL;
static llama_context_params ctx_params; static gpt_vocab vocab;
static gptj_model model;
static gpt_params params; static gpt_params params;
static int n_past = 0; static int n_past = 0;
static int n_threads = 4; static int n_threads = 4;
static int n_batch = 8; static int n_batch = 8;
static std::string model; static std::string modelname;
static llama_context *ctx; static std::vector<gpt_vocab::id> current_context_tokens;
static std::vector<llama_token> last_n_tokens; static size_t mem_per_token = 0;
static std::vector<llama_token> current_context_tokens; static std::vector<float> logits;
void nnn() bool gptj_load_model(const load_model_inputs inputs, FileFormat in_file_format)
{ {
ggml_time_init(); ggml_time_init();
const int64_t t_main_start_us = ggml_time_us();
gpt_params params;
params.model = "models/gpt-j-6B/ggml-model.bin";
if (params.seed < 0) {
params.seed = time(NULL);
}
printf("%s: seed = %d\n", __func__, params.seed);
std::mt19937 rng(params.seed);
int64_t t_load_us = 0;
gpt_vocab vocab;
gptj_model model;
// load the model
{
const int64_t t_start_us = ggml_time_us();
if (!legacy_gptj_model_load(params.model, model, vocab)) {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
return 1;
}
t_load_us = ggml_time_us() - t_start_us;
}
int n_past = 0;
int64_t t_sample_us = 0;
int64_t t_predict_us = 0;
std::vector<float> logits;
// tokenize the prompt
std::vector<gpt_vocab::id> embd_inp = ::gpt_tokenize(vocab, params.prompt);
params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
printf("%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size());
printf("\n");
std::vector<gpt_vocab::id> embd;
// determine the required inference memory per token:
size_t mem_per_token = 0;
legacy_gptj_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
for (int i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {
// predict
if (embd.size() > 0) {
const int64_t t_start_us = ggml_time_us();
if (!legacy_gptj_eval(model, params.n_threads, n_past, embd, logits, mem_per_token)) {
printf("Failed to predict\n");
return 1;
}
t_predict_us += ggml_time_us() - t_start_us;
}
n_past += embd.size();
embd.clear();
if (i >= embd_inp.size()) {
// sample next token
const int top_k = params.top_k;
const float top_p = params.top_p;
const float temp = params.temp;
const int n_vocab = model.hparams.n_vocab;
gpt_vocab::id id = 0;
{
const int64_t t_start_sample_us = ggml_time_us();
id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);
t_sample_us += ggml_time_us() - t_start_sample_us;
}
// add it to the context
embd.push_back(id);
} else {
// if here, it means we are still processing the input prompt
for (int k = i; k < embd_inp.size(); k++) {
embd.push_back(embd_inp[k]);
if (embd.size() > params.n_batch) {
break;
}
}
i += embd.size() - 1;
}
// display text
for (auto id : embd) {
printf("%s", vocab.id_to_token[id].c_str());
}
fflush(stdout);
// end of text token
if (embd.back() == 50256) {
break;
}
}
// report timing
{
const int64_t t_main_end_us = ggml_time_us();
printf("\n\n");
printf("%s: mem per token = %8zu bytes\n", __func__, mem_per_token);
printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f);
printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f);
printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
}
ggml_free(model.ctx);
return 0;
}
bool llama_load_model(const load_model_inputs inputs, FileFormat in_file_format)
{
printf("System Info: %s\n", llama_print_system_info());
ctx_params = llama_context_default_params();
n_threads = inputs.threads;
n_batch = inputs.batch_size;
model = inputs.model_filename;
ctx_params.n_ctx = inputs.max_context_length;
ctx_params.n_parts = inputs.n_parts_overwrite;
ctx_params.seed = -1;
ctx_params.f16_kv = inputs.f16_kv;
ctx_params.logits_all = false;
file_format = in_file_format; file_format = in_file_format;
n_threads = params.n_threads = inputs.threads;
n_batch = params.n_batch = inputs.batch_size;
modelname = params.model = inputs.model_filename;
if (file_format == FileFormat::GGML || file_format == FileFormat::GGHF) if (!legacy_gptj_model_load(params.model, model, vocab)) {
{ fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
ctx = legacy_llama_init_from_file(model.c_str(), ctx_params);
}
else
{
ctx = llama_init_from_file(model.c_str(), ctx_params);
}
if (ctx == NULL)
{
fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, model.c_str());
return false; return false;
} }
if (file_format < FileFormat::GGJT) if (file_format != FileFormat::GPTJ2)
{ {
printf("\n---\nWarning: Your model has an INVALID or OUTDATED format (ver %d). Please reconvert it for better results!\n---\n", file_format); printf("\n---\nWarning: Your model has an INVALID or OUTDATED format (ver %d). Please reconvert it for better results!\n---\n", file_format);
} }
//determine mem per token // determine the required inference memory per token:
const std::vector<llama_token> tmp = {0, 1, 2, 3}; legacy_gptj_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
llama_eval(ctx, tmp.data(), tmp.size(), 0, params.n_threads);
return true; return true;
} }
generation_outputs llama_generate(const generation_inputs inputs, generation_outputs &output)
generation_outputs gptj_generate(const generation_inputs inputs, generation_outputs &output)
{ {
params.prompt = inputs.prompt; params.prompt = inputs.prompt;
params.seed = inputs.seed; params.seed = inputs.seed;
@ -207,16 +61,9 @@ generation_outputs llama_generate(const generation_inputs inputs, generation_out
params.top_k = inputs.top_k; params.top_k = inputs.top_k;
params.top_p = inputs.top_p; params.top_p = inputs.top_p;
params.temp = inputs.temperature; params.temp = inputs.temperature;
params.repeat_last_n = inputs.rep_pen_range;
params.repeat_penalty = inputs.rep_pen;
params.n_ctx = inputs.max_context_length;
params.n_batch = n_batch; params.n_batch = n_batch;
params.n_threads = n_threads; params.n_threads = n_threads;
if (params.repeat_last_n < 1)
{
params.repeat_last_n = 1;
}
if (params.top_k < 1) if (params.top_k < 1)
{ {
params.top_k = 300; //to disable top_k we actually need to increase this value to a very high number params.top_k = 300; //to disable top_k we actually need to increase this value to a very high number
@ -225,41 +72,20 @@ generation_outputs llama_generate(const generation_inputs inputs, generation_out
{ {
params.seed = time(NULL); params.seed = time(NULL);
} }
params.prompt.insert(0, 1, ' ');
// tokenize the prompt // tokenize the prompt
std::vector<llama_token> embd_inp; std::vector<gpt_vocab::id> embd_inp = ::gpt_tokenize(vocab, params.prompt);
if (file_format == 1)
{
embd_inp = ::legacy_llama_tokenize(ctx, params.prompt, true);
}
else
{
embd_inp = ::llama_tokenize(ctx, params.prompt, true);
}
//params.n_predict = std::min(params.n_predict, params.n_ctx - (int) embd_inp.size());
//truncate to front of the prompt if its too long //truncate to front of the prompt if its too long
if (embd_inp.size() + params.n_predict > params.n_ctx) if (embd_inp.size() + params.n_predict > model.hparams.n_ctx)
{ {
int offset = embd_inp.size() - params.n_ctx + params.n_predict; int offset = embd_inp.size() - model.hparams.n_ctx + params.n_predict;
embd_inp = std::vector<llama_token>(embd_inp.begin() + offset, embd_inp.end()); embd_inp = std::vector<llama_token>(embd_inp.begin() + offset, embd_inp.end());
} }
//determine how much npast we have to rewind from the current state //determine how much npast we have to rewind from the current state
std::vector<gpt_vocab::id> embd;
std::vector<llama_token> embd;
int last_n_size = params.repeat_last_n;
last_n_tokens.resize(last_n_size);
//display usage
// std::string tst = " ";
// char * tst2 = (char*)tst.c_str();
// gpt_print_usage(1,&tst2,params);
std::fill(last_n_tokens.begin(), last_n_tokens.end(), 0);
n_past = 0; n_past = 0;
//fast forward the past based on identical tokens, stop once a divergence is noted //fast forward the past based on identical tokens, stop once a divergence is noted
@ -269,7 +95,6 @@ generation_outputs llama_generate(const generation_inputs inputs, generation_out
if (current_context_tokens[i] == embd_inp[i]) if (current_context_tokens[i] == embd_inp[i])
{ {
n_past += 1; n_past += 1;
last_n_tokens.push_back(current_context_tokens[i]);
} }
else else
{ {
@ -281,11 +106,10 @@ generation_outputs llama_generate(const generation_inputs inputs, generation_out
} }
} }
last_n_tokens.erase(last_n_tokens.begin(), last_n_tokens.begin() + n_past);
embd_inp.erase(embd_inp.begin(), embd_inp.begin() + n_past); embd_inp.erase(embd_inp.begin(), embd_inp.begin() + n_past);
//if using BLAS and prompt is big enough, switch to single thread and use a huge batch //if using BLAS and prompt is big enough, switch to single thread and use a huge batch
bool blasmode = (embd_inp.size() >= 32 && ggml_cpu_has_blas()); bool blasmode = false;// (embd_inp.size() >= 32 && ggml_cpu_has_blas());
int original_batch = params.n_batch; int original_batch = params.n_batch;
int original_threads = params.n_threads; int original_threads = params.n_threads;
if (blasmode) if (blasmode)
@ -306,11 +130,13 @@ generation_outputs llama_generate(const generation_inputs inputs, generation_out
timer_start(); timer_start();
double time1 = 0, time2 = 0; double time1 = 0, time2 = 0;
unsigned int embd_inp_size = embd_inp.size(); unsigned int embd_inp_size = embd_inp.size();
const int n_vocab = model.hparams.n_vocab;
printf("\n"); printf("\n");
while (remaining_tokens > 0) while (remaining_tokens > 0)
{ {
llama_token id = 0; gpt_vocab::id id = 0;
// predict // predict
unsigned int embdsize = embd.size(); unsigned int embdsize = embd.size();
if (embdsize > 0) if (embdsize > 0)
@ -324,8 +150,8 @@ generation_outputs llama_generate(const generation_inputs inputs, generation_out
{ {
printf("\rGenerating (%d / %d tokens)", (1 + params.n_predict - remaining_tokens), params.n_predict); printf("\rGenerating (%d / %d tokens)", (1 + params.n_predict - remaining_tokens), params.n_predict);
} }
//printf("\nnp:%d embd:%d txt:%s",n_past,embd.size(),llama_token_to_str(ctx, embd[0]));
if (llama_eval(ctx, embd.data(), embdsize, n_past, params.n_threads)) if (!legacy_gptj_eval(model, params.n_threads, n_past, embd, logits, mem_per_token))
{ {
fprintf(stderr, "Failed to predict\n"); fprintf(stderr, "Failed to predict\n");
snprintf(output.text, sizeof(output.text), "%s", ""); snprintf(output.text, sizeof(output.text), "%s", "");
@ -342,7 +168,6 @@ generation_outputs llama_generate(const generation_inputs inputs, generation_out
const float top_k = params.top_k; const float top_k = params.top_k;
const float top_p = params.top_p; const float top_p = params.top_p;
const float temp = params.temp; const float temp = params.temp;
const float repeat_penalty = params.repeat_penalty;
if (!startedsampling) if (!startedsampling)
{ {
@ -355,17 +180,13 @@ generation_outputs llama_generate(const generation_inputs inputs, generation_out
} }
{ {
auto logits = llama_get_logits(ctx);
// set the logit of the eos token (2) to zero to avoid sampling it // set the logit of the eos token (2) to zero to avoid sampling it
logits[llama_token_eos()] = 0; logits[50256] = 0;
//set logits of opening square bracket to zero. //set logits of opening square bracket to zero.
logits[518] = 0;
logits[29961] = 0;
id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);
id = llama_sample_top_p_top_k(ctx, last_n_tokens.data(), last_n_tokens.size(), top_k, top_p, temp, repeat_penalty);
last_n_tokens.erase(last_n_tokens.begin());
last_n_tokens.push_back(id);
current_context_tokens.push_back(id); current_context_tokens.push_back(id);
} }
@ -374,17 +195,17 @@ generation_outputs llama_generate(const generation_inputs inputs, generation_out
// decrement remaining sampling budget // decrement remaining sampling budget
--remaining_tokens; --remaining_tokens;
//printf("\nid:%d word:%s\n",id,llama_token_to_str(ctx, id));
concat_output += llama_token_to_str(ctx, id); for (auto id : embd) {
concat_output += vocab.id_to_token[id].c_str();
}
} }
else else
{ {
// some user input remains from prompt or interaction, forward it to processing // some user input remains from prompt or interaction, forward it to processing
while ((int)embd_inp.size() > input_consumed) while ((int)embd_inp.size() > input_consumed)
{ {
embd.push_back(embd_inp[input_consumed]); embd.push_back(embd_inp[input_consumed]);
last_n_tokens.erase(last_n_tokens.begin());
last_n_tokens.push_back(embd_inp[input_consumed]);
current_context_tokens.push_back(embd_inp[input_consumed]); current_context_tokens.push_back(embd_inp[input_consumed]);
++input_consumed; ++input_consumed;
if ((int)embd.size() >= params.n_batch) if ((int)embd.size() >= params.n_batch)

10
llama_adapter.cpp
View file

@ -19,7 +19,7 @@ static gpt_params params;
static int n_past = 0; static int n_past = 0;
static int n_threads = 4; static int n_threads = 4;
static int n_batch = 8; static int n_batch = 8;
static std::string model; static std::string modelname;
static llama_context *ctx; static llama_context *ctx;
static std::vector<llama_token> last_n_tokens; static std::vector<llama_token> last_n_tokens;
static std::vector<llama_token> current_context_tokens; static std::vector<llama_token> current_context_tokens;
@ -32,7 +32,7 @@ bool llama_load_model(const load_model_inputs inputs, FileFormat in_file_format)
n_threads = inputs.threads; n_threads = inputs.threads;
n_batch = inputs.batch_size; n_batch = inputs.batch_size;
model = inputs.model_filename; modelname = inputs.model_filename;
ctx_params.n_ctx = inputs.max_context_length; ctx_params.n_ctx = inputs.max_context_length;
ctx_params.n_parts = inputs.n_parts_overwrite; ctx_params.n_parts = inputs.n_parts_overwrite;
@ -44,16 +44,16 @@ bool llama_load_model(const load_model_inputs inputs, FileFormat in_file_format)
if (file_format == FileFormat::GGML || file_format == FileFormat::GGHF) if (file_format == FileFormat::GGML || file_format == FileFormat::GGHF)
{ {
ctx = legacy_llama_init_from_file(model.c_str(), ctx_params); ctx = legacy_llama_init_from_file(modelname.c_str(), ctx_params);
} }
else else
{ {
ctx = llama_init_from_file(model.c_str(), ctx_params); ctx = llama_init_from_file(modelname.c_str(), ctx_params);
} }
if (ctx == NULL) if (ctx == NULL)
{ {
fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, model.c_str()); fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, modelname.c_str());
return false; return false;
} }

BIN
llamacpp.dll
View file

Binary file not shown.

BIN
llamacpp_blas.dll
View file

Binary file not shown.

2
extra.cpp → llamaextra.cpp
View file

@ -1,5 +1,5 @@
#include "ggml.h" #include "ggml.h"
#include "extra.h" #include "llamaextra.h"
#include "llama.cpp" #include "llama.cpp"
#include <cassert> #include <cassert>

8
extra.h → llamaextra.h
View file

@ -21,13 +21,15 @@ enum FileFormat
FAIL=0, FAIL=0,
GGML=1, GGML=1,
GGHF=2, GGHF=2,
GGJT=3 GGJT=3,
GPTJ1=100,
GPTJ2=101
}; };
void print_tok_vec(std::vector<int> &embd);
void timer_start(); void timer_start();
double timer_check(); double timer_check();
void print_tok_vec(std::vector<int> &embd);
FileFormat check_file_format(const std::string & fname); FileFormat check_file_format(const std::string & fname);
std::vector<llama_token> legacy_llama_tokenize(struct llama_context * ctx, const std::string & text, bool add_bos); std::vector<llama_token> legacy_llama_tokenize(struct llama_context * ctx, const std::string & text, bool add_bos);

6
model_adapter.h
View file

@ -1,6 +1,8 @@
#pragma once #pragma once
#include "expose.h" #include "expose.h"
#include "extra.h" #include "llamaextra.h"
bool llama_load_model(const load_model_inputs inputs, FileFormat file_format); bool llama_load_model(const load_model_inputs inputs, FileFormat file_format);
generation_outputs llama_generate(const generation_inputs inputs, generation_outputs &output); generation_outputs llama_generate(const generation_inputs inputs, generation_outputs &output);
bool gptj_load_model(const load_model_inputs inputs, FileFormat in_file_format);
generation_outputs gptj_generate(const generation_inputs inputs, generation_outputs &output);

22
otherarch/utils.h
View file

@ -7,32 +7,12 @@
#include <vector> #include <vector>
#include <random> #include <random>
#include <thread> #include <thread>
#include "common.h"
// //
// CLI argument parsing // CLI argument parsing
// //
struct gpt_params {
int32_t seed = -1; // RNG seed
int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
int32_t n_predict = 200; // new tokens to predict
// sampling parameters
int32_t top_k = 40;
float top_p = 0.9f;
float temp = 0.9f;
int32_t n_batch = 8; // batch size for prompt processing
std::string model = "models/gpt-2-117M/ggml-model.bin"; // model path
std::string prompt;
};
bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
void gpt_print_usage(int argc, char ** argv, const gpt_params & params);
std::string gpt_random_prompt(std::mt19937 & rng);
// //
// Vocab utils // Vocab utils