vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add gqa parameter support to the server

Browse source
This commit is contained in:
Ignacio DM 2023-07-23 13:49:26 -03:00
parent 70d26ac388
commit 161c2c69f2
1 changed files with 14 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

16
examples/server/server.cpp
View file

@ -608,6 +608,7 @@ static void server_print_usage(const char *argv0, const gpt_params &params,
fprintf(stderr, " -v, --verbose verbose output (default: %s)\n", server_verbose ? "enabled" : "disabled");
fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);
fprintf(stderr, " -c N, --ctx-size N size of the prompt context (default: %d)\n", params.n_ctx);
fprintf(stderr, " -gqa N, --gqa N grouped-query attention factor (TEMP!!! use 8 for LLaMAv2 70B) (default: %d)\n", params.n_gqa);
fprintf(stderr, " --rope-freq-base N RoPE base frequency (default: %.1f)\n", params.rope_freq_base);
fprintf(stderr, " --rope-freq-scale N RoPE frequency scaling factor (default: %g)\n", params.rope_freq_scale);
fprintf(stderr, " -b N, --batch-size N batch size for prompt processing (default: %d)\n", params.n_batch);
@ -724,9 +725,19 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
}
params.n_ctx = std::stoi(argv[i]);
}
else if (arg == "-gqa" || arg == "--gqa")
{
if (++i >= argc)
{
invalid_param = true;
break;
}
params.n_gqa = std::stoi(argv[i]);
}
else if (arg == "--rope-freq-base")
{
if (++i >= argc) {
if (++i >= argc)
{
invalid_param = true;
break;
}
@ -734,7 +745,8 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
}
else if (arg == "--rope-freq-scale")
{
if (++i >= argc) {
if (++i >= argc)
{
invalid_param = true;
break;
}