vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lookup : final touches

Browse source
This commit is contained in:
Georgi Gerganov 2023-12-22 18:04:30 +02:00
parent d8ed670c6c
commit 50ea1ef7c8
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
2 changed files with 6 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

2
examples/lookup/README.md
View file

@ -4,7 +4,7 @@ Demonstration of Prompt Lookup Decoding
https://github.com/apoorvumang/prompt-lookup-decoding https://github.com/apoorvumang/prompt-lookup-decoding
The two key parameters for lookup decoding are `max_ngram_size` and `n_draft`. The first, determines how many ngrams to use when searching through the prompt for a match and the second specifies how many subsequent tokens to draft if a match is found. The key parameters for lookup decoding are `ngram_min`, `ngram_max` and `n_draft`. The first two determine the size of the ngrams to search for in the prompt for a match. The latter specifies how many subsequent tokens to draft if a match is found.
More info: More info:

9
examples/lookup/lookup.cpp
View file

@ -9,12 +9,13 @@
int main(int argc, char ** argv){ int main(int argc, char ** argv){
gpt_params params; gpt_params params;
if(gpt_params_parse(argc, argv, params) == false){ if (!gpt_params_parse(argc, argv, params)) {
return 1; return 1;
} }
// maximum n-grams to search for in prompt // max/min n-grams size to search for in prompt
const int max_ngram_size = 3; const int ngram_max = 4;
const int ngram_min = 1;
// length of the candidate / draft sequence, if match is found // length of the candidate / draft sequence, if match is found
const int n_draft = params.n_draft; const int n_draft = params.n_draft;
@ -160,7 +161,7 @@ int main(int argc, char ** argv){
// generate n_pred tokens through prompt lookup // generate n_pred tokens through prompt lookup
auto prompt_lookup = [&]() -> void { auto prompt_lookup = [&]() -> void {
int inp_size = inp.size(); int inp_size = inp.size();
for (int ngram_size = max_ngram_size ; ngram_size > 0; --ngram_size){ for (int ngram_size = ngram_max ; ngram_size > ngram_min; --ngram_size){
const llama_token * ngram = &inp[inp_size - ngram_size]; const llama_token * ngram = &inp[inp_size - ngram_size];
for (int i = 0; i <= (int) inp_size - (ngram_size * 2); ++i) { for (int i = 0; i <= (int) inp_size - (ngram_size * 2); ++i) {