vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

server : fill usage info in embeddings and rerank responses (#10852)

Browse source 
* server : fill usage info in embeddings response

* server : fill usage info in reranking response
This commit is contained in:
krystiancha 2024-12-17 16:00:24 +00:00 • committed by GitHub
parent 382bc7f2e8
commit 05c3a444b8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 77 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

18
examples/server/utils.hpp
View file

@ -560,6 +560,7 @@ static json oaicompat_completion_params_parse(
static json format_embeddings_response_oaicompat(const json & request, const json & embeddings) {
json data = json::array();
int32_t n_tokens = 0;
int i = 0;
for (const auto & elem : embeddings) {
data.push_back(json{
@ -567,14 +568,16 @@ static json format_embeddings_response_oaicompat(const json & request, const jso
{"index", i++},
{"object", "embedding"}
});
n_tokens += json_value(elem, "tokens_evaluated", 0);
}
json res = json {
{"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},
{"object", "list"},
{"usage", json { // TODO: fill
{"prompt_tokens", 0},
{"total_tokens", 0}
{"usage", json {
{"prompt_tokens", n_tokens},
{"total_tokens", n_tokens}
}},
{"data", data}
};
@ -584,20 +587,23 @@ static json format_embeddings_response_oaicompat(const json & request, const jso
static json format_response_rerank(const json & request, const json & ranks) {
json data = json::array();
int32_t n_tokens = 0;
int i = 0;
for (const auto & rank : ranks) {
data.push_back(json{
{"index", i++},
{"relevance_score", json_value(rank, "score", 0.0)},
});
n_tokens += json_value(rank, "tokens_evaluated", 0);
}
json res = json {
{"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},
{"object", "list"},
{"usage", json { // TODO: fill
{"prompt_tokens", 0},
{"total_tokens", 0}
{"usage", json {
{"prompt_tokens", n_tokens},
{"total_tokens", n_tokens}
}},
{"results", data}
};