From 64d173bc9c187e01e34d8a718a251631aef4f5d1 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 18 Jan 2024 14:43:33 +0200 Subject: [PATCH] perplexity : option to specify max batched tasks via `n_parallel` --- examples/perplexity/perplexity.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index 218ab6746..343eeaa45 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -567,7 +567,7 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) { GGML_ASSERT(params.n_batch >= n_ctx && "HellaSwag currently requires n_batch >= n_ctx"); - const int max_tasks_per_batch = 32; + const int max_tasks_per_batch = params.n_parallel; const int max_seq = 4*max_tasks_per_batch; llama_batch batch = llama_batch_init(n_ctx, 0, max_seq); @@ -591,7 +591,6 @@ static void hellaswag_score(llama_context * ctx, const gpt_params & params) { const int s0 = 4*(i1 - i0); if (s0 + 4 > max_seq) { - fprintf(stderr, "%s : too many tasks in the batch - increase batch sequence capacity\n", __func__); break; }