vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

server: only add back deferred tasks when one slot is available

Browse source
This commit is contained in:
ngxson 2024-01-22 23:06:12 +01:00
parent 1bd867894d
commit d083c81761
2 changed files with 12 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

1
examples/server/server.cpp
View file

@ -1383,6 +1383,7 @@ struct llama_server_context
slot.t_last_used = ggml_time_us();
LOG_TEE("slot %d released (%d tokens in cache)\n", slot.id, (int) slot.cache_tokens.size());
queue_tasks.notify_slot_changed();
continue;
}

21
examples/server/utils.hpp
View file

@ -209,7 +209,7 @@ struct llama_server_queue {
return task.id;
}
// Add a new task, but defer until the next loop
// Add a new task, but defer until one slot is available
void defer(task_server task) {
std::unique_lock<std::mutex> lock(mutex_tasks);
queue_tasks_deferred.push_back(std::move(task));
@ -236,6 +236,16 @@ struct llama_server_queue {
callback_all_task_finished = callback;
}
// Call when the state of one slot is changed
void notify_slot_changed() {
// move deferred tasks back to main loop
std::unique_lock<std::mutex> lock(mutex_tasks);
for (auto & task : queue_tasks_deferred) {
queue_tasks.push_back(std::move(task));
}
queue_tasks_deferred.clear();
}
// Start the main loop. This call is blocking
void start_loop() {
while (true) {
@ -255,15 +265,6 @@ struct llama_server_queue {
LOG_VERBOSE("callback_new_task", {});
callback_new_task(task);
}
// move deferred tasks back to main loop
{
std::unique_lock<std::mutex> lock(mutex_tasks);
for (auto & task : queue_tasks_deferred) {
queue_tasks.push_back(std::move(task));
}
queue_tasks_deferred.clear();
lock.unlock();
}
LOG_VERBOSE("callback_all_task_finished", {});
// process and update all the multitasks
auto queue_iterator = queue_multitasks.begin();