From fa3582f509a2715e80a473e79f88dcd1ebff44c2 Mon Sep 17 00:00:00 2001 From: Kawrakow <48489457+ikawrakow@users.noreply.github.com> Date: Tue, 29 Aug 2023 23:55:45 +0300 Subject: [PATCH] Tell users attmepting to run perplexity with too few tokens to use more (#2882) Closes #2858 Co-authored-by: Iwan Kawrakow --- examples/perplexity/perplexity.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index aeb774c5f..7c02b6d40 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -142,6 +142,14 @@ results_perplexity perplexity_v2(llama_context * ctx, const gpt_params & params) fprintf(stderr, "%s: tokenizing the input ..\n", __func__); std::vector tokens = ::llama_tokenize(ctx, params.prompt, add_bos); + + if (int(tokens.size()) < 2*params.n_ctx) { + fprintf(stderr, "%s: you need at least %d tokens to evaluate perplexity with a context of %d\n",__func__,2*params.n_ctx, + params.n_ctx); + fprintf(stderr, "%s: the data file you provided tokenizes to only %zu tokens\n",__func__,tokens.size()); + return {std::move(tokens), 0., {}, {}}; + } + std::vector logit_history; std::vector prob_history; @@ -274,6 +282,13 @@ results_perplexity perplexity(llama_context * ctx, const gpt_params & params) { auto tim2 = std::chrono::high_resolution_clock::now(); fprintf(stderr, "%s: tokenization took %g ms\n",__func__,1e-3*std::chrono::duration_cast(tim2-tim1).count()); + if (int(tokens.size()) < 2*params.n_ctx) { + fprintf(stderr, "%s: you need at least %d tokens to evaluate perplexity with a context of %d\n",__func__,2*params.n_ctx, + params.n_ctx); + fprintf(stderr, "%s: the data file you provided tokenizes to only %zu tokens\n",__func__,tokens.size()); + return {std::move(tokens), 0., {}, {}}; + } + std::vector logit_history; logit_history.resize(tokens.size());