vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'elbios/concat_output_mutex' into concedo_experimental

Browse source
This commit is contained in:
Concedo 2023-09-01 15:24:13 +08:00
commit 81abd3cb1f
1 changed files with 11 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

12
gpttype_adapter.cpp
View file

@ -8,6 +8,7 @@
//Python will ALWAYS provide the memory, we just write to it. //Python will ALWAYS provide the memory, we just write to it.
#include <time.h> #include <time.h>
#include <mutex>
#include "model_adapter.h" #include "model_adapter.h"
#include "otherarch.h" #include "otherarch.h"
@ -85,7 +86,9 @@ static std::vector<int> banned_token_ids;
static std::vector<llama_token_data> top_picks; static std::vector<llama_token_data> top_picks;
static int remaining_tokens = 0; static int remaining_tokens = 0;
static int stopper_unused_tokens = 0; static int stopper_unused_tokens = 0;
static std::mutex concat_output_mtx;
static std::string concat_output = ""; static std::string concat_output = "";
static std::string concat_output_reader_copy = "";
inline bool IsNanCheck(float f) inline bool IsNanCheck(float f)
{ {
@ -1039,12 +1042,17 @@ int gpttype_token_count(const std::string & input)
const std::string & gpttype_get_pending_output() const std::string & gpttype_get_pending_output()
{ {
return concat_output; concat_output_mtx.lock();
concat_output_reader_copy = concat_output;
concat_output_mtx.unlock();
return concat_output_reader_copy;
} }
generation_outputs gpttype_generate(const generation_inputs inputs, generation_outputs &output) generation_outputs gpttype_generate(const generation_inputs inputs, generation_outputs &output)
{ {
concat_output_mtx.lock();
concat_output = ""; concat_output = "";
concat_output_mtx.unlock();
last_stop_reason = stop_reason::OUT_OF_TOKENS; last_stop_reason = stop_reason::OUT_OF_TOKENS;
stop_sequence.clear(); stop_sequence.clear();
for(int x=0;x<stop_token_max;++x) for(int x=0;x<stop_token_max;++x)
@ -1570,7 +1578,9 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
{ {
generated_tokens.push_back(tokenizedstr); generated_tokens.push_back(tokenizedstr);
} }
concat_output_mtx.lock();
concat_output += tokenizedstr; concat_output += tokenizedstr;
concat_output_mtx.unlock();
} }
if (startedsampling && debugmode!=-1) if (startedsampling && debugmode!=-1)