diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 9c86407c2..f04d5b037 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1313,6 +1313,7 @@ struct server_context { {"id_slot", slot.id}, {"multimodal", false}, {"index", slot.index}, + {"timings", slot.get_formated_timings()}, }; if (slot.params.sampling.n_probs > 0) { @@ -2274,12 +2275,17 @@ struct server_context { common_sampler_accept(slot.smpl, id, true); slot.n_decoded += 1; + + const int64_t t_current = ggml_time_us(); + if (slot.n_decoded == 1) { - slot.t_start_generation = ggml_time_us(); + slot.t_start_generation = t_current; slot.t_prompt_processing = (slot.t_start_generation - slot.t_start_process_prompt) / 1e3; metrics.on_prompt_eval(slot); } + slot.t_token_generation = (t_current - slot.t_start_generation) / 1e3; + completion_token_output result; result.tok = id; @@ -2995,13 +3001,15 @@ int main(int argc, char ** argv) { ctx_server.queue_tasks.post(tasks); bool stream = json_value(data, "stream", false); + bool timings = json_value(data, "timing_per_token", false); + const auto task_ids = server_task::get_list_id(tasks); const auto completion_id = gen_chatcmplid(); if (!stream) { ctx_server.receive_cmpl_results(task_ids, [&](const std::vector & results) { // multitask is never support in chat completion, there is only one result - json result_oai = format_final_response_oaicompat(data, results[0].data, completion_id, /*.streaming =*/ false, verbose); + json result_oai = format_final_response_oaicompat(data, results[0].data, completion_id, /*.streaming =*/ false, verbose, timings); res_ok(res, result_oai); }, [&](const json & error_data) { res_error(res, error_data); @@ -3009,9 +3017,9 @@ int main(int argc, char ** argv) { ctx_server.queue_results.remove_waiting_task_ids(task_ids); } else { - const auto chunked_content_provider = [task_ids, &ctx_server, completion_id](size_t, httplib::DataSink & sink) { + const auto chunked_content_provider = [task_ids, &ctx_server, completion_id, timings](size_t, httplib::DataSink & sink) { ctx_server.receive_cmpl_results_stream(task_ids, [&](const server_task_result & result) -> bool { - std::vector result_array = format_partial_response_oaicompat(result.data, completion_id); + std::vector result_array = format_partial_response_oaicompat(result.data, completion_id, timings); for (auto & event_data : result_array) { if (event_data.empty()) { continue; // skip the stop token diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 1665e9dc3..072811367 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -604,7 +604,7 @@ static json oaicompat_completion_params_parse( return llama_params; } -static json format_final_response_oaicompat(const json & request, const json & result, const std::string & completion_id, bool streaming = false, bool verbose = false) { +static json format_final_response_oaicompat(const json & request, const json & result, const std::string & completion_id, bool streaming = false, bool verbose = false, bool timings = false) { bool stopped_word = result.count("stopped_word") != 0; bool stopped_eos = json_value(result, "stopped_eos", false); int num_tokens_predicted = json_value(result, "tokens_predicted", 0); @@ -650,11 +650,15 @@ static json format_final_response_oaicompat(const json & request, const json & r res["completion_probabilities"] = json_value(result, "completion_probabilities", json::array()); } + if (timings) { + res.push_back({"timings", json_value(result, "timings", json::object())}); + } + return res; } // return value is vector as there is one case where we might need to generate two responses -static std::vector format_partial_response_oaicompat(const json & result, const std::string & completion_id) { +static std::vector format_partial_response_oaicompat(const json & result, const std::string & completion_id, bool timings = false) { if (!result.contains("model") || !result.contains("oaicompat_token_ctr")) { return std::vector({result}); } @@ -740,6 +744,11 @@ static std::vector format_partial_response_oaicompat(const json & result, {"model", modelname}, {"object", "chat.completion.chunk"} }; + + if (timings) { + ret.push_back({"timings", json_value(result, "timings", json::object())}); + } + if (!finish_reason.empty()) { int num_tokens_predicted = json_value(result, "tokens_predicted", 0); int num_prompt_tokens = json_value(result, "tokens_evaluated", 0);