vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Are you happy editorconfig?

Browse source
This commit is contained in:
M. Yusuf Sarıgöz 2023-10-09 13:42:04 +03:00
parent 54495c9474
commit 9b0ec4d2cc
2 changed files with 142 additions and 142 deletions
Download patch file Download diff file Expand all files Collapse all files

282
examples/llava/llava-utils.h
View file

@ -1,141 +1,141 @@
// this one and clip lib will be eventually merged to a single lib, let's keep it this way for now // this one and clip lib will be eventually merged to a single lib, let's keep it this way for now
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <vector> #include <vector>
#include "common.h" #include "common.h"
#include "llama.h" #include "llama.h"
bool eval_image_embd(llama_context * ctx_llama, float * embd, int N, int n_batch, int * n_past) { bool eval_image_embd(llama_context * ctx_llama, float * embd, int N, int n_batch, int * n_past) {
int n_embd = llama_n_embd(llama_get_model(ctx_llama)); int n_embd = llama_n_embd(llama_get_model(ctx_llama));
for (int i = 0; i < N; i += n_batch) { for (int i = 0; i < N; i += n_batch) {
int n_eval = N - i; int n_eval = N - i;
if (n_eval > n_batch) { if (n_eval > n_batch) {
n_eval = n_batch; n_eval = n_batch;
} }
llama_batch batch = {int32_t(n_eval), nullptr, (embd+i*n_embd), nullptr, nullptr, nullptr, *n_past, 1, 0, }; llama_batch batch = {int32_t(n_eval), nullptr, (embd+i*n_embd), nullptr, nullptr, nullptr, *n_past, 1, 0, };
if (llama_decode(ctx_llama, batch)) { if (llama_decode(ctx_llama, batch)) {
fprintf(stderr, "%s : failed to eval\n", __func__); fprintf(stderr, "%s : failed to eval\n", __func__);
return false; return false;
} }
*n_past += n_eval; *n_past += n_eval;
} }
return true; return true;
} }
bool eval_tokens(struct llama_context * ctx_llama, std::vector<llama_token> tokens, int n_batch, int * n_past) { bool eval_tokens(struct llama_context * ctx_llama, std::vector<llama_token> tokens, int n_batch, int * n_past) {
int N = (int) tokens.size(); int N = (int) tokens.size();
for (int i = 0; i < N; i += n_batch) { for (int i = 0; i < N; i += n_batch) {
int n_eval = (int) tokens.size() - i; int n_eval = (int) tokens.size() - i;
if (n_eval > n_batch) { if (n_eval > n_batch) {
n_eval = n_batch; n_eval = n_batch;
} }
if (llama_decode(ctx_llama, llama_batch_get_one(&tokens[i], n_eval, *n_past, 0))) { if (llama_decode(ctx_llama, llama_batch_get_one(&tokens[i], n_eval, *n_past, 0))) {
fprintf(stderr, "%s : failed to eval\n", __func__); fprintf(stderr, "%s : failed to eval\n", __func__);
return false; return false;
} }
*n_past += n_eval; *n_past += n_eval;
} }
return true; return true;
} }
bool eval_id(struct llama_context * ctx_llama, int id, int * n_past) { bool eval_id(struct llama_context * ctx_llama, int id, int * n_past) {
std::vector<llama_token> tokens; std::vector<llama_token> tokens;
tokens.push_back(id); tokens.push_back(id);
return eval_tokens(ctx_llama, tokens, 1, n_past); return eval_tokens(ctx_llama, tokens, 1, n_past);
} }
bool eval_string(struct llama_context * ctx_llama, const char* str, int n_batch, int * n_past){ bool eval_string(struct llama_context * ctx_llama, const char* str, int n_batch, int * n_past){
std::string str2 = str; std::string str2 = str;
std::vector<llama_token> embd_inp = ::llama_tokenize(ctx_llama, str2, true); std::vector<llama_token> embd_inp = ::llama_tokenize(ctx_llama, str2, true);
eval_tokens(ctx_llama, embd_inp, n_batch, n_past); eval_tokens(ctx_llama, embd_inp, n_batch, n_past);
return true; return true;
} }
llama_token sample_id(llama_context * ctx_llama, gpt_params & params) { llama_token sample_id(llama_context * ctx_llama, gpt_params & params) {
// out of user input, sample next token // out of user input, sample next token
const float temp = params.temp; const float temp = params.temp;
const int32_t top_k = params.top_k <= 0 ? llama_n_vocab(llama_get_model(ctx_llama)) : params.top_k; const int32_t top_k = params.top_k <= 0 ? llama_n_vocab(llama_get_model(ctx_llama)) : params.top_k;
const float top_p = params.top_p; const float top_p = params.top_p;
const float tfs_z = params.tfs_z; const float tfs_z = params.tfs_z;
const float typical_p = params.typical_p; const float typical_p = params.typical_p;
// const int32_t repeat_last_n = params.repeat_last_n < 0 ? n_ctx : params.repeat_last_n; // const int32_t repeat_last_n = params.repeat_last_n < 0 ? n_ctx : params.repeat_last_n;
// const float repeat_penalty = params.repeat_penalty; // const float repeat_penalty = params.repeat_penalty;
// const float alpha_presence = params.presence_penalty; // const float alpha_presence = params.presence_penalty;
// const float alpha_frequency = params.frequency_penalty; // const float alpha_frequency = params.frequency_penalty;
const int mirostat = params.mirostat; const int mirostat = params.mirostat;
const float mirostat_tau = params.mirostat_tau; const float mirostat_tau = params.mirostat_tau;
const float mirostat_eta = params.mirostat_eta; const float mirostat_eta = params.mirostat_eta;
// const bool penalize_nl = params.penalize_nl; // const bool penalize_nl = params.penalize_nl;
llama_token id = 0; llama_token id = 0;
{ {
auto logits = llama_get_logits(ctx_llama); auto logits = llama_get_logits(ctx_llama);
auto n_vocab = llama_n_vocab(llama_get_model(ctx_llama)); auto n_vocab = llama_n_vocab(llama_get_model(ctx_llama));
// Apply params.logit_bias map // Apply params.logit_bias map
for (auto it = params.logit_bias.begin(); it != params.logit_bias.end(); it++) { for (auto it = params.logit_bias.begin(); it != params.logit_bias.end(); it++) {
logits[it->first] += it->second; logits[it->first] += it->second;
} }
std::vector<llama_token_data> candidates; std::vector<llama_token_data> candidates;
candidates.reserve(n_vocab); candidates.reserve(n_vocab);
for (llama_token token_id = 0; token_id < n_vocab; token_id++) { for (llama_token token_id = 0; token_id < n_vocab; token_id++) {
candidates.emplace_back(llama_token_data{token_id, logits[token_id], 0.0f}); candidates.emplace_back(llama_token_data{token_id, logits[token_id], 0.0f});
} }
llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false }; llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
// TODO: Apply penalties // TODO: Apply penalties
// float nl_logit = logits[llama_token_nl(ctx)]; // float nl_logit = logits[llama_token_nl(ctx)];
// auto last_n_repeat = std::min(std::min((int)last_n_tokens.size(), repeat_last_n), n_ctx); // auto last_n_repeat = std::min(std::min((int)last_n_tokens.size(), repeat_last_n), n_ctx);
// llama_sample_repetition_penalty(ctx, &candidates_p, // llama_sample_repetition_penalty(ctx, &candidates_p,
// last_n_tokens.data() + last_n_tokens.size() - last_n_repeat, // last_n_tokens.data() + last_n_tokens.size() - last_n_repeat,
// last_n_repeat, repeat_penalty); // last_n_repeat, repeat_penalty);
// llama_sample_frequency_and_presence_penalties(ctx, &candidates_p, // llama_sample_frequency_and_presence_penalties(ctx, &candidates_p,
// last_n_tokens.data() + last_n_tokens.size() - last_n_repeat, // last_n_tokens.data() + last_n_tokens.size() - last_n_repeat,
// last_n_repeat, alpha_frequency, alpha_presence); // last_n_repeat, alpha_frequency, alpha_presence);
// if (!penalize_nl) { // if (!penalize_nl) {
// logits[llama_token_nl(ctx)] = nl_logit; // logits[llama_token_nl(ctx)] = nl_logit;
// } // }
if (temp <= 0) { if (temp <= 0) {
// Greedy sampling // Greedy sampling
id = llama_sample_token_greedy(ctx_llama, &candidates_p); id = llama_sample_token_greedy(ctx_llama, &candidates_p);
} else { } else {
if (mirostat == 1) { if (mirostat == 1) {
static float mirostat_mu = 2.0f * mirostat_tau; static float mirostat_mu = 2.0f * mirostat_tau;
const int mirostat_m = 100; const int mirostat_m = 100;
llama_sample_temp(ctx_llama, &candidates_p, temp); llama_sample_temp(ctx_llama, &candidates_p, temp);
id = llama_sample_token_mirostat(ctx_llama, &candidates_p, mirostat_tau, mirostat_eta, mirostat_m, &mirostat_mu); id = llama_sample_token_mirostat(ctx_llama, &candidates_p, mirostat_tau, mirostat_eta, mirostat_m, &mirostat_mu);
} else if (mirostat == 2) { } else if (mirostat == 2) {
static float mirostat_mu = 2.0f * mirostat_tau; static float mirostat_mu = 2.0f * mirostat_tau;
llama_sample_temp(ctx_llama, &candidates_p, temp); llama_sample_temp(ctx_llama, &candidates_p, temp);
id = llama_sample_token_mirostat_v2(ctx_llama, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu); id = llama_sample_token_mirostat_v2(ctx_llama, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu);
} else { } else {
// Temperature sampling // Temperature sampling
llama_sample_top_k(ctx_llama, &candidates_p, top_k, 1); llama_sample_top_k(ctx_llama, &candidates_p, top_k, 1);
llama_sample_tail_free(ctx_llama, &candidates_p, tfs_z, 1); llama_sample_tail_free(ctx_llama, &candidates_p, tfs_z, 1);
llama_sample_typical(ctx_llama, &candidates_p, typical_p, 1); llama_sample_typical(ctx_llama, &candidates_p, typical_p, 1);
llama_sample_top_p(ctx_llama, &candidates_p, top_p, 1); llama_sample_top_p(ctx_llama, &candidates_p, top_p, 1);
llama_sample_temp(ctx_llama, &candidates_p, temp); llama_sample_temp(ctx_llama, &candidates_p, temp);
id = llama_sample_token(ctx_llama, &candidates_p); id = llama_sample_token(ctx_llama, &candidates_p);
} }
} }
} }
return id; return id;
} }
const char * sample(struct llama_context * ctx_llama, gpt_params & params, int * n_past) { const char * sample(struct llama_context * ctx_llama, gpt_params & params, int * n_past) {
int id = sample_id(ctx_llama, params); int id = sample_id(ctx_llama, params);
static std::string ret; static std::string ret;
if (id == llama_token_eos(ctx_llama)) { if (id == llama_token_eos(ctx_llama)) {
ret = "</s>"; ret = "</s>";
} else { } else {
ret = llama_token_to_piece(ctx_llama, id); ret = llama_token_to_piece(ctx_llama, id);
} }
eval_id(ctx_llama, id, n_past); eval_id(ctx_llama, id, n_past);
return ret.c_str(); return ret.c_str();
} }

2
examples/llava/llava.cpp
View file

@ -35,7 +35,7 @@ int main(int argc, char ** argv) {
clip_image_f32 img_res; clip_image_f32 img_res;
clip_image_load_from_file(img_path, &img); clip_image_load_from_file(img_path, &img);
clip_image_preprocess(ctx_clip, &img, &img_res); clip_image_preprocess(ctx_clip, &img, &img_res);
float * image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip)); float * image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip));
if (!image_embd) { if (!image_embd) {
fprintf(stderr, "Unable to allocate memory for CLIP embeddings\n"); fprintf(stderr, "Unable to allocate memory for CLIP embeddings\n");