From d8033d9c8cdef056d8a963e92fee25f276180cf5 Mon Sep 17 00:00:00 2001 From: ezosa Date: Mon, 27 May 2024 14:31:28 +0300 Subject: [PATCH] add support for Poro pre-tokenizer --- convert-hf-to-gguf-update.py | 1 + convert-hf-to-gguf.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py index 84b72348d..3bfb2dc28 100755 --- a/convert-hf-to-gguf-update.py +++ b/convert-hf-to-gguf-update.py @@ -82,6 +82,7 @@ models = [ {"name": "jina-v2-es", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-es", }, {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-de", }, {"name": "smaug-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/abacusai/Smaug-Llama-3-70B-Instruct", }, + {"name": "Poro-34B-chat", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LumiOpen/Poro-34B-chat", }, ] diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index a342f6b1c..0ae9e27a3 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -476,6 +476,9 @@ class Model: if chkhsh == "c136ed14d01c2745d4f60a9596ae66800e2b61fa45643e72436041855ad4089d": # ref: https://huggingface.co/abacusai/Smaug-Llama-3-70B-Instruct res = "smaug-bpe" + if chkhsh == "c7ea5862a53e4272c035c8238367063e2b270d51faa48c0f09e9d5b54746c360": + # ref: https://huggingface.co/LumiOpen/Poro-34B-chat + res = "Poro-34B-chat" if res is None: logger.warning("\n")