Added support for Viking pre-tokenizer (#8135)

Co-authored-by: kustaaya <kustaaya@protonmail.com>
2024-06-27 11:58:54 +03:00 · 2024-06-27 11:58:54 +03:00 · f675b20a3b
commit f675b20a3b
parent 911e35bb8b
4 changed files with 14 additions and 0 deletions
--- a/include/llama.h
+++ b/include/llama.h
@ -88,6 +88,7 @@ extern "C" {
        LLAMA_VOCAB_PRE_TYPE_DBRX           = 13,
        LLAMA_VOCAB_PRE_TYPE_SMAUG          = 14,
        LLAMA_VOCAB_PRE_TYPE_PORO           = 15,
+        LLAMA_VOCAB_PRE_TYPE_VIKING         = 16,
    };

    // note: these values should be synchronized with ggml_rope