vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

examplse : de-shadow

Browse source 
ggml-ci
This commit is contained in:
Georgi Gerganov 2025-01-12 14:25:32 +02:00
parent 82caffa74e
commit 9a735ae6d8
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
16 changed files with 152 additions and 159 deletions
Download patch file Download diff file Expand all files Collapse all files

20
examples/speculative/speculative.cpp
View file

@ -544,26 +544,26 @@ int main(int argc, char ** argv) {
for (int is = 0; is < (int) sa.size(); ++is) {
const llama_token id = cur_p->data[is].id;
const int s = sa[is];
const int sd = sa[is];
common_sampler_accept(drafts[s].smpl, id, true);
common_sampler_accept(drafts[sd].smpl, id, true);
drafts[s].tokens.push_back(id);
// save cur_p.data into drafts[s].dists
drafts[s].dists.push_back({cur_p->data, cur_p->data + cur_p->size});
drafts[sd].tokens.push_back(id);
// save cur_p.data into drafts[sd].dists
drafts[sd].dists.push_back({cur_p->data, cur_p->data + cur_p->size});
// add unique drafted tokens to the target batch
drafts[s].i_batch_tgt.push_back(batch_tgt.n_tokens);
drafts[sd].i_batch_tgt.push_back(batch_tgt.n_tokens);
common_batch_add(batch_tgt, id, n_past_tgt + i + 1, { s }, true);
common_batch_add(batch_tgt, id, n_past_tgt + i + 1, { sd }, true);
// add the token to the batch for batched decoding with the draft model
drafts[s].i_batch_dft = batch_dft.n_tokens;
drafts[sd].i_batch_dft = batch_dft.n_tokens;
common_batch_add(batch_dft, id, n_past_cur, { s }, true);
common_batch_add(batch_dft, id, n_past_cur, { sd }, true);
if (batch_tgt.n_tokens > n_draft) {
drafts[s].drafting = false;
drafts[sd].drafting = false;
}
}
}