From 1574279273740ef9f4153ec497e351cd4de09206 Mon Sep 17 00:00:00 2001 From: JohannesGaessler Date: Wed, 7 Feb 2024 20:11:12 +0100 Subject: [PATCH] partial implementation --- examples/lookup-static/lookup-static.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/examples/lookup-static/lookup-static.cpp b/examples/lookup-static/lookup-static.cpp index 364ce4e18..d468be067 100644 --- a/examples/lookup-static/lookup-static.cpp +++ b/examples/lookup-static/lookup-static.cpp @@ -85,6 +85,23 @@ int main(int argc, char ** argv){ } } printf("\n\n%ld\n\n", hashmap.size()); + std::unordered_map hashmap_max; + for (auto item : hashmap) { + const int64_t key = item.first; + const std::unordered_map frequency = item.second; + + llama_token max_token = -1; + int max_frequency = 0; + for (auto item2 : frequency) { + if (item2.second > max_frequency) { + max_token = item2.first; + max_frequency = item2.second; + } + } + + hashmap_max.emplace(std::make_pair(key, max_token)); + } + printf("\n\n%ld\n\n", hashmap_max.size()); const int max_context_size = llama_n_ctx(ctx); const int max_tokens_list_size = max_context_size - 4;