vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge 593627a8b1 into 00ba2ff781

Browse source
This commit is contained in:
pomoke 2024-09-10 16:24:49 +08:00 • committed by GitHub
commit 406d475837
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 18 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
examples/lookahead/README.md
View file

@ -5,3 +5,5 @@ Demonstration of lookahead decoding technique:
https://lmsys.org/blog/2023-11-21-lookahead-decoding/ https://lmsys.org/blog/2023-11-21-lookahead-decoding/
More info: https://github.com/ggerganov/llama.cpp/pull/4207 More info: https://github.com/ggerganov/llama.cpp/pull/4207
To set W, N and G manually, set them as environment variables.

19
examples/lookahead/lookahead.cpp
View file

@ -4,6 +4,7 @@
#include "llama.h" #include "llama.h"
#include <cstdio> #include <cstdio>
#include <cstdlib>
#include <string> #include <string>
#include <vector> #include <vector>
@ -35,6 +36,17 @@ struct ngram_container {
std::vector<llama_token> tokens; std::vector<llama_token> tokens;
}; };
int int_from_env(const char *name, int default_value) {
char *value = std::getenv(name);
if (value != nullptr) {
int v = atoi(value);
v = v ? v : default_value;
return v;
} else {
return default_value;
}
}
int main(int argc, char ** argv) { int main(int argc, char ** argv) {
gpt_params params; gpt_params params;
@ -42,9 +54,10 @@ int main(int argc, char ** argv) {
return 1; return 1;
} }
const int W = 15; // lookahead window int W = int_from_env("W",15); // lookahead window
const int N = 5; // n-gram size int N = int_from_env("N",5); // n-gram size
const int G = 15; // max verification n-grams int G = int_from_env("G",15); // max verification n-grams
const bool dump_kv_cache = params.dump_kv_cache; const bool dump_kv_cache = params.dump_kv_cache;