vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

server : fix initialization thread issues

Browse source
This commit is contained in:
Georgi Gerganov 2024-02-21 13:08:57 +02:00
parent 2a37bd6b86
commit f1d4138c13
No known key found for this signature in database
GPG key ID: BF970631944C16B7
2 changed files with 14 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

26
examples/server/server.cpp
View file

@ -2719,19 +2719,6 @@ int main(int argc, char **argv)
log_data["api_key"] = "api_key: " + std::to_string(sparams.api_keys.size()) + " keys loaded"; log_data["api_key"] = "api_key: " + std::to_string(sparams.api_keys.size()) + " keys loaded";
} }
LOG_INFO("HTTP server listening", log_data);
// run the HTTP server in a thread - see comment below
std::thread t([&]()
{
if (!svr.listen_after_bind())
{
state.store(SERVER_STATE_ERROR);
return 1;
}
return 0;
});
// load the model // load the model
if (!llama.load_model(params)) if (!llama.load_model(params))
{ {
@ -3194,6 +3181,19 @@ int main(int argc, char **argv)
}*/ }*/
//); //);
LOG_INFO("HTTP server listening", log_data);
// run the HTTP server in a thread - see comment below
std::thread t([&]()
{
if (!svr.listen_after_bind())
{
state.store(SERVER_STATE_ERROR);
return 1;
}
return 0;
});
llama.queue_tasks.on_new_task(std::bind( llama.queue_tasks.on_new_task(std::bind(
&llama_server_context::process_single_task, &llama, std::placeholders::_1)); &llama_server_context::process_single_task, &llama, std::placeholders::_1));
llama.queue_tasks.on_finish_multitask(std::bind( llama.queue_tasks.on_finish_multitask(std::bind(

2
examples/server/tests/features/server.feature
View file

@ -84,7 +84,7 @@ Feature: llama.cpp server
Then all prompts are predicted Then all prompts are predicted
# FIXME: #3969 infinite loop on the CI, not locally, if n_prompt * n_predict > kv_size # FIXME: #3969 infinite loop on the CI, not locally, if n_prompt * n_predict > kv_size
@bug @llama.cpp
Scenario: Multi users with total number of tokens to predict exceeds the KV Cache size Scenario: Multi users with total number of tokens to predict exceeds the KV Cache size
Given a prompt: Given a prompt:
""" """