From a1aa65e069d05ca48f7b80bbbfce941431f20d5e Mon Sep 17 00:00:00 2001 From: Joan Martinez Date: Thu, 2 May 2024 12:02:26 +0200 Subject: [PATCH] feat: change convert hf to gguf --- convert-hf-to-gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 2f146d730..28bf20e99 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -271,7 +271,7 @@ class Model(ABC): chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български \'\'\'\'\'\'```````""""......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL' - chktok = tokenizer.encode(chktxt) + chktok = tokenizer.decode(tokenizer.encode(chktxt)) chkhsh = sha256(str(chktok).encode()).hexdigest() print(f"chktok: {chktok}")