vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update perplexity.cpp

Browse source
This commit is contained in:
klosax 2023-07-22 10:31:43 +02:00 • committed by GitHub
parent 545862ae48
commit 9a36dff0fd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

13
examples/perplexity/perplexity.cpp
View file

@ -4,6 +4,7 @@
#include <cmath> #include <cmath>
#include <ctime> #include <ctime>
#include <sstream>
#if defined(_MSC_VER) #if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data #pragma warning(disable: 4244 4267) // possible loss of data
@ -124,16 +125,10 @@ void perplexity_lines(llama_context * ctx, const gpt_params & params) {
// Calculates perplexity over each line of the prompt // Calculates perplexity over each line of the prompt
std::vector<std::string> prompt_lines; std::vector<std::string> prompt_lines;
std::istringstream strstream(params.prompt);
size_t pos=0;
while( pos < params.prompt.size() ) {
std::string line; std::string line;
while( true ) {
if( params.prompt[pos] == '\n' || pos == params.prompt.size() ) while (std::getline(strstream,line,'\n')) {
break;
line += params.prompt[pos++];
}
pos++;
prompt_lines.push_back(line); prompt_lines.push_back(line);
} }