From 5144a18e6705c87241bdb169538cc4183984ef81 Mon Sep 17 00:00:00 2001
From: lexasub <empty@empty.ru>
Date: Mon, 27 Jan 2025 15:09:34 +0400
Subject: [PATCH] impl::load change map bpe_ranks to onordered map for reduce
 time of impl::load on 30%

---
 src/llama-vocab.cpp | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index 0782d3a41..6a3a9d48e 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -1245,8 +1245,13 @@ struct llama_vocab::impl {
 
     std::vector<llama_token> cache_special_tokens;
     std::vector<std::string> cache_token_to_piece; // llama_token_to_piece(special = true);
-
-    std::map<std::pair<std::string, std::string>, int> bpe_ranks;
+    struct PairHash {
+        size_t operator()(const std::pair<std::string, std::string>& p) const {
+            return std::hash<std::string>{}(p.first) ^  //create some hash for pair
+                   (std::hash<std::string>{}(p.second) << 1);
+        }
+    };
+    std::unordered_map<std::pair<std::string, std::string>, int, PairHash> bpe_ranks;
 
     // set of all tokens that cause "end of generation"
     std::set<llama_token> special_eog_ids;