vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

change batch.logits to batch.output

Browse source
This commit is contained in:
Jia Liu 2024-08-16 16:42:44 +08:00
parent 23fd453544
commit cbb5dd7b12
18 changed files with 60 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

2
common/common.cpp
View file

@ -2666,7 +2666,7 @@ void llama_batch_add(
for (size_t i = 0; i < seq_ids.size(); ++i) {
batch.seq_id[batch.n_tokens][i] = seq_ids[i];
}
batch.logits [batch.n_tokens] = logits;
batch.output [batch.n_tokens] = logits;
batch.n_tokens++;
}

2
common/log.h
View file

@ -686,7 +686,7 @@ inline std::string LOG_BATCH_TOSTR_PRETTY(const C & ctx, const B & batch)
<< ":pos " << std::to_string(batch.pos[i])
<< ":n_seq_id " << std::to_string(batch.n_seq_id[i])
<< ":seq_id " << std::to_string(batch.seq_id[i][0])
<< ":logits " << std::to_string(batch.logits[i]);
<< ":logits " << std::to_string(batch.output[i]);
}
buf << " ]";

4
examples/batched-bench/batched-bench.cpp
View file

@ -94,7 +94,7 @@ int main(int argc, char ** argv) {
batch.pos + i,
batch.n_seq_id + i,
batch.seq_id + i,
batch.logits + i,
batch.output + i,
0, 0, 0, // unused
};
@ -149,7 +149,7 @@ int main(int argc, char ** argv) {
llama_batch_add(batch, 0, i, { j }, false);
}
}
batch.logits[batch.n_tokens - 1] = true;
batch.output[batch.n_tokens - 1] = true;
const auto t_pp_start = ggml_time_us();

6
examples/batched.swift/Sources/main.swift
View file

@ -86,11 +86,11 @@ for (i, token) in tokens.enumerated() {
if let seq_id = batch.seq_id[i] {
seq_id[0] = 0
}
batch.logits[i] = 0
batch.output[i] = 0
}
// llama_decode will output logits only for the last token of the prompt
batch.logits[Int(batch.n_tokens) - 1] = 1
batch.output[Int(batch.n_tokens) - 1] = 1
if llama_decode(context, batch) != 0 {
print("llama_decode() failed")
@ -178,7 +178,7 @@ while n_cur <= n_len {
if let seq_id = batch.seq_id[Int(batch.n_tokens)] {
seq_id[0] = Int32(i)
}
batch.logits[Int(batch.n_tokens)] = 1
batch.output[Int(batch.n_tokens)] = 1
i_batch[i] = batch.n_tokens

2
examples/batched/batched.cpp
View file

@ -122,7 +122,7 @@ int main(int argc, char ** argv) {
}
// llama_decode will output logits only for the last token of the prompt
batch.logits[batch.n_tokens - 1] = true;
batch.output[batch.n_tokens - 1] = true;
if (llama_decode(ctx, batch) != 0) {
LOG_TEE("%s: llama_decode() failed\n", __func__);

2
examples/embedding/embedding.cpp
View file

@ -52,7 +52,7 @@ static void batch_decode(llama_context * ctx, llama_batch & batch, float * outpu
}
for (int i = 0; i < batch.n_tokens; i++) {
if (!batch.logits[i]) {
if (!batch.output[i]) {
continue;
}

12
examples/gritlm/gritlm.cpp
View file

@ -102,21 +102,21 @@ static std::string generate(llama_context * ctx, const std::string & prompt, boo
llama_set_embeddings(ctx, false);
llama_set_causal_attn(ctx, true);
llama_batch bat = llama_batch_init(llama_n_batch(ctx), 0, 1);
llama_batch batch = llama_batch_init(llama_n_batch(ctx), 0, 1);
std::vector<llama_token> inputs = llama_tokenize(mdl, prompt, false, true);
int32_t i_current_token = 0;
while (true) {
llama_batch_clear(bat);
llama_batch_clear(batch);
auto n_inputs = (int32_t)inputs.size();
for (int32_t i = 0; i < n_inputs; i++) {
llama_batch_add(bat, inputs[i], i_current_token++, { 0 }, i == n_inputs - 1);
llama_batch_add(batch, inputs[i], i_current_token++, { 0 }, i == n_inputs - 1);
}
inputs.clear();
llama_decode(ctx, bat);
auto logits = llama_get_logits_ith(ctx, bat.n_tokens - 1);
llama_decode(ctx, batch);
auto logits = llama_get_logits_ith(ctx, batch.n_tokens - 1);
auto candidates = std::vector<llama_token_data>(llama_n_vocab(mdl));
auto n_candidates = (int32_t)candidates.size();
@ -145,7 +145,7 @@ static std::string generate(llama_context * ctx, const std::string & prompt, boo
std::printf("\n");
}
llama_batch_free(bat);
llama_batch_free(batch);
return result;
}

2
examples/imatrix/imatrix.cpp
View file

@ -513,7 +513,7 @@ static bool compute_imatrix(llama_context * ctx, const gpt_params & params) {
tokens[batch_start] = llama_token_bos(llama_get_model(ctx));
}
// TODO: use batch.logits to save computations instead of relying on logits_all == true
// TODO: use batch.output to save computations instead of relying on logits_all == true
if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) {
fprintf(stderr, "%s : failed to eval\n", __func__);
return false;

6
examples/llama.android/llama/src/main/cpp/llama-android.cpp
View file

@ -193,7 +193,7 @@ Java_android_llama_cpp_LLamaAndroid_bench_1model(
llama_batch_add(*batch, 0, i, { 0 }, false);
}
batch->logits[batch->n_tokens - 1] = true;
batch->output[batch->n_tokens - 1] = true;
llama_kv_cache_clear(context);
const auto t_pp_start = ggml_time_us();
@ -306,7 +306,7 @@ Java_android_llama_cpp_LLamaAndroid_new_1batch(JNIEnv *, jobject, jint n_tokens,
for (int i = 0; i < n_tokens; ++i) {
batch->seq_id[i] = (llama_seq_id *) malloc(sizeof(llama_seq_id) * n_seq_max);
}
batch->logits = (int8_t *) malloc(sizeof(int8_t) * n_tokens);
batch->output = (int8_t *) malloc(sizeof(int8_t) * n_tokens);
return reinterpret_cast<jlong>(batch);
}
@ -363,7 +363,7 @@ Java_android_llama_cpp_LLamaAndroid_completion_1init(
}
// llama_decode will output logits only for the last token of the prompt
batch->logits[batch->n_tokens - 1] = true;
batch->output[batch->n_tokens - 1] = true;
if (llama_decode(context, *batch) != 0) {
LOGe("llama_decode() failed");

6
examples/llama.swiftui/llama.cpp.swift/LibLlama.swift
View file

@ -16,7 +16,7 @@ func llama_batch_add(_ batch: inout llama_batch, _ id: llama_token, _ pos: llama
for i in 0..<seq_ids.count {
batch.seq_id[Int(batch.n_tokens)]![Int(i)] = seq_ids[i]
}
batch.logits [Int(batch.n_tokens)] = logits ? 1 : 0
batch.output [Int(batch.n_tokens)] = logits ? 1 : 0
batch.n_tokens += 1
}
@ -132,7 +132,7 @@ actor LlamaContext {
let i = Int(i1)
llama_batch_add(&batch, tokens_list[i], Int32(i), [0], false)
}
batch.logits[Int(batch.n_tokens) - 1] = 1 // true
batch.output[Int(batch.n_tokens) - 1] = 1 // true
if llama_decode(context, batch) != 0 {
print("llama_decode() failed")
@ -214,7 +214,7 @@ actor LlamaContext {
for i in 0..<n_tokens {
llama_batch_add(&batch, 0, Int32(i), [0], false)
}
batch.logits[Int(batch.n_tokens) - 1] = 1 // true
batch.output[Int(batch.n_tokens) - 1] = 1 // true
llama_kv_cache_clear(context)

4
examples/parallel/parallel.cpp
View file

@ -265,7 +265,7 @@ int main(int argc, char ** argv) {
// extract the logits only for the last token
if (batch.n_tokens > 0) {
batch.logits[batch.n_tokens - 1] = true;
batch.output[batch.n_tokens - 1] = true;
}
client.n_prompt = tokens_prompt.size();
@ -308,7 +308,7 @@ int main(int argc, char ** argv) {
batch.pos + i,
batch.n_seq_id + i,
batch.seq_id + i,
batch.logits + i,
batch.output + i,
0, 0, 0, // unused
};

4
examples/passkey/passkey.cpp
View file

@ -140,7 +140,7 @@ int main(int argc, char ** argv) {
}
if (i + n_batch >= n_tokens_all) {
batch.logits[batch.n_tokens - 1] = true;
batch.output[batch.n_tokens - 1] = true;
}
if (llama_decode(ctx, batch) != 0) {
@ -174,7 +174,7 @@ int main(int argc, char ** argv) {
}
if (i + n_batch >= n_tokens_all) {
batch.logits[batch.n_tokens - 1] = true;
batch.output[batch.n_tokens - 1] = true;
}
if (llama_decode(ctx, batch) != 0) {

18
examples/perplexity/perplexity.cpp
View file

@ -407,7 +407,7 @@ static results_perplexity perplexity_v2(llama_context * ctx, const gpt_params &
const int batch_size = std::min(end - batch_start, n_batch);
//fprintf(stderr, " Batch %d: starts at %d, size is %d, n_past is %d\n",j,batch_start,batch_size,j * n_batch);
// TODO: use llama_batch.logits instead of relying on logits_all == true
// TODO: use llama_batch.output instead of relying on logits_all == true
if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) {
//fprintf(stderr, "%s : failed to eval\n", __func__);
return {tokens, -1, logit_history, prob_history};
@ -601,9 +601,9 @@ static results_perplexity perplexity(llama_context * ctx, const gpt_params & par
batch.pos [idx] = j*n_batch + k;
batch.n_seq_id[idx] = 1;
batch.seq_id [idx][0] = seq;
batch.logits [idx] = batch.pos[idx] >= first ? 1 : 0;
batch.output [idx] = batch.pos[idx] >= first ? 1 : 0;
n_outputs += batch.logits[idx] != 0;
n_outputs += batch.output[idx] != 0;
}
batch.n_tokens += batch_size;
@ -697,7 +697,7 @@ static bool decode_helper(llama_context * ctx, llama_batch & batch, std::vector<
batch.pos + i,
batch.n_seq_id + i,
batch.seq_id + i,
batch.logits + i,
batch.output + i,
0, 0, 0, // unused
};
@ -709,7 +709,7 @@ static bool decode_helper(llama_context * ctx, llama_batch & batch, std::vector<
int n_outputs = 0;
for (int i = 0; i < n_tokens; ++i) {
n_outputs += batch_view.logits[i] != 0;
n_outputs += batch_view.output[i] != 0;
}
memcpy(batch_logits.data() + prev_outputs*n_vocab, llama_get_logits(ctx), n_outputs*n_vocab*sizeof(float));
@ -917,7 +917,7 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) {
for (size_t i = 0; i < hs_cur.common_prefix; ++i) {
llama_batch_add(batch, hs_cur.seq_tokens[0][i], i, { s0 + 0, s0 + 1, s0 + 2, s0 + 3 }, false);
}
batch.logits[batch.n_tokens - 1] = true; // we need logits for the last token of the common prefix
batch.output[batch.n_tokens - 1] = true; // we need logits for the last token of the common prefix
n_logits += 1;
for (int s = 0; s < 4; ++s) {
@ -1196,7 +1196,7 @@ static void winogrande_score(llama_context * ctx, const gpt_params & params) {
for (size_t i = 0; i < data[i1].common_prefix; ++i) {
llama_batch_add(batch, data[i1].seq_tokens[0][i], i, { s0 + 0, s0 + 1 }, false);
}
batch.logits[batch.n_tokens - 1] = true;
batch.output[batch.n_tokens - 1] = true;
n_logits += 1;
for (int s = 0; s < 2; ++s) {
@ -1565,7 +1565,7 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & params
//llama_batch_add(batch, cur_task.seq_tokens[0][i], i, { s0 + 0, s0 + 1, s0 + 2, s0 + 3}, false);
llama_batch_add(batch, cur_task.seq_tokens[0][i], i, batch_indeces, false);
}
batch.logits[batch.n_tokens - 1] = true; // we need logits for the last token of the common prefix
batch.output[batch.n_tokens - 1] = true; // we need logits for the last token of the common prefix
n_logits += 1;
for (int s = 0; s < int(cur_task.seq_tokens.size()); ++s) {
@ -1794,7 +1794,7 @@ static void kl_divergence(llama_context * ctx, const gpt_params & params) {
tokens[batch_start] = llama_token_bos(llama_get_model(ctx));
}
// TODO: use llama_batch.logits instead of relying on logits_all == true
// TODO: use llama_batch.output instead of relying on logits_all == true
if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) {
fprintf(stderr, "%s : failed to eval\n", __func__);
return;

2
examples/retrieval/retrieval.cpp
View file

@ -91,7 +91,7 @@ static void batch_decode(llama_context * ctx, llama_batch & batch, float * outpu
}
for (int i = 0; i < batch.n_tokens; i++) {
if (!batch.logits[i]) {
if (!batch.output[i]) {
continue;
}

6
examples/server/server.cpp
View file

@ -1480,7 +1480,7 @@ struct server_context {
std::vector<float> embd_res(n_embd, 0.0f);
for (int i = 0; i < batch.n_tokens; ++i) {
if (!batch.logits[i] || batch.seq_id[i][0] != slot.id + 1) {
if (!batch.output[i] || batch.seq_id[i][0] != slot.id + 1) {
continue;
}
@ -2269,7 +2269,7 @@ struct server_context {
GGML_ASSERT(batch.n_tokens > 0);
// extract the logits only for the last token
batch.logits[batch.n_tokens - 1] = true;
batch.output[batch.n_tokens - 1] = true;
slot.n_decoded = 0;
slot.i_batch = batch.n_tokens - 1;
@ -2341,7 +2341,7 @@ struct server_context {
batch.pos + i,
batch.n_seq_id + i,
batch.seq_id + i,
batch.logits + i,
batch.output + i,
0, 0, 0, // unused
};

2
examples/simple/simple.cpp
View file

@ -93,7 +93,7 @@ int main(int argc, char ** argv) {
}
// llama_decode will output logits only for the last token of the prompt
batch.logits[batch.n_tokens - 1] = true;
batch.output[batch.n_tokens - 1] = true;
if (llama_decode(ctx, batch) != 0) {
LOG_TEE("%s: llama_decode() failed\n", __func__);

12
include/llama.h
View file

@ -220,7 +220,7 @@ extern "C" {
// - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)
// - pos : the positions of the respective token in the sequence
// - seq_id : the sequence to which the respective token belongs
// - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output
// - output : if zero, the logits (and/or the embeddings) for the respective token will not be output
//
typedef struct llama_batch {
int32_t n_tokens;
@ -230,7 +230,7 @@ extern "C" {
llama_pos * pos;
int32_t * n_seq_id;
llama_seq_id ** seq_id;
int8_t * logits; // TODO: rename this to "output"
int8_t * output; // Previously named 'logits', renamed to 'output' now.
// NOTE: helpers for smooth API transition - can be deprecated in the future
// for future-proof code, use the above fields instead and ignore everything below
@ -328,7 +328,7 @@ extern "C" {
enum ggml_type type_v; // data type for V cache [EXPERIMENTAL]
// Keep the booleans together to avoid misalignment during copy-by-value.
bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead)
bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.output instead)
bool embeddings; // if true, extract embeddings (together with logits)
bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU
bool flash_attn; // whether to use flash attention [EXPERIMENTAL]
@ -859,9 +859,9 @@ extern "C" {
LLAMA_API void llama_synchronize(struct llama_context * ctx);
// Token logits obtained from the last call to llama_decode()
// The logits for which llama_batch.logits[i] != 0 are stored contiguously
// The logits for which llama_batch.output[i] != 0 are stored contiguously
// in the order they have appeared in the batch.
// Rows: number of tokens for which llama_batch.logits[i] != 0
// Rows: number of tokens for which llama_batch.output[i] != 0
// Cols: n_vocab
LLAMA_API float * llama_get_logits(struct llama_context * ctx);
@ -873,7 +873,7 @@ extern "C" {
// Get all output token embeddings.
// when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model,
// the embeddings for which llama_batch.logits[i] != 0 are stored contiguously
// the embeddings for which llama_batch.output[i] != 0 are stored contiguously
// in the order they have appeared in the batch.
// shape: [n_outputs*n_embd]
// Otherwise, returns NULL.

28
src/llama.cpp
View file

@ -14500,10 +14500,10 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {
for (int i = 0; i < n_tokens; ++i) {
data[i] = i;
}
} else if (batch.logits) {
} else if (batch.output) {
int32_t n_outputs = 0;
for (int i = 0; i < n_tokens; ++i) {
if (batch.logits[i]) {
if (batch.output[i]) {
data[n_outputs++] = i;
}
}
@ -14972,13 +14972,13 @@ static int llama_decode_internal(
std::vector<llama_seq_id *> seq_id_arr;
std::vector<std::vector<llama_seq_id>> seq_id;
// this indicates we are doing pooled embedding, so we ignore batch.logits and output all tokens
// this indicates we are doing pooled embedding, so we ignore batch.output and output all tokens
const bool embd_pooled = cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE;
// count outputs
if (batch_all.logits && !embd_pooled) {
if (batch_all.output && !embd_pooled) {
for (uint32_t i = 0; i < n_tokens_all; ++i) {
n_outputs += batch_all.logits[i] != 0;
n_outputs += batch_all.output[i] != 0;
}
} else if (lctx.logits_all || embd_pooled) {
n_outputs = n_tokens_all;
@ -14994,10 +14994,10 @@ static int llama_decode_internal(
};
// set output mappings
if (batch_all.logits) {
if (batch_all.output) {
int32_t i_logits = 0;
for (uint32_t i = 0; i < n_tokens_all; ++i) {
if (batch_all.logits[i]) {
if (batch_all.output[i]) {
lctx.output_ids[i] = i_logits++;
}
}
@ -15016,7 +15016,7 @@ static int llama_decode_internal(
/* .pos = */ batch_all.pos ? batch_all.pos + cur_token : nullptr,
/* .n_seq_id = */ batch_all.n_seq_id ? batch_all.n_seq_id + cur_token : nullptr,
/* .seq_id = */ batch_all.seq_id ? batch_all.seq_id + cur_token : nullptr,
/* .logits = */ batch_all.logits ? batch_all.logits + cur_token : nullptr,
/* .logits = */ batch_all.output ? batch_all.output + cur_token : nullptr,
/* .all_pos_0 = */ batch_all.all_pos_0 + (llama_pos) cur_token*batch_all.all_pos_1,
/* .all_pos_1 = */ batch_all.all_pos_1,
/* .all_seq_id = */ batch_all.all_seq_id,
@ -15026,9 +15026,9 @@ static int llama_decode_internal(
{
int32_t n_outputs_new = 0;
if (u_batch.logits && !embd_pooled) {
if (u_batch.output && !embd_pooled) {
for (uint32_t i = 0; i < n_tokens; i++) {
n_outputs_new += u_batch.logits[i] != 0;
n_outputs_new += u_batch.output[i] != 0;
}
} else if (n_outputs == n_tokens_all) {
n_outputs_new = n_tokens;
@ -18881,7 +18881,7 @@ struct llama_batch llama_batch_init(int32_t n_tokens_alloc, int32_t embd, int32_
}
batch.seq_id[n_tokens_alloc] = nullptr;
batch.logits = (int8_t *) malloc(sizeof(int8_t) * n_tokens_alloc);
batch.output = (int8_t *) malloc(sizeof(int8_t) * n_tokens_alloc);
return batch;
}
@ -18897,7 +18897,7 @@ void llama_batch_free(struct llama_batch batch) {
}
free(batch.seq_id);
}
if (batch.logits) free(batch.logits);
if (batch.output) free(batch.output);
}
int32_t llama_encode(
@ -18975,7 +18975,7 @@ float * llama_get_logits_ith(struct llama_context * ctx, int32_t i) {
}
if (j < 0) {
throw std::runtime_error(format("batch.logits[%d] != true", i));
throw std::runtime_error(format("batch.output[%d] != true", i));
}
if (j >= ctx->n_outputs) {
// This should not happen
@ -19020,7 +19020,7 @@ float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i) {
}
if (j < 0) {
throw std::runtime_error(format("batch.logits[%d] != true", i));
throw std::runtime_error(format("batch.output[%d] != true", i));
}
if (j >= ctx->n_outputs) {
// This should not happen