Merge branch 'master' into compilade/refactor-kv-cache

2024-09-01 20:47:59 -04:00 · 2024-09-01 20:47:59 -04:00 · a03e32a3c9
commit a03e32a3c9
parent fcb889cf7f 8f1d81a0b6
9 changed files with 1273 additions and 107 deletions
--- a/include/llama.h
+++ b/include/llama.h
@ -66,6 +66,7 @@ extern "C" {
        LLAMA_VOCAB_TYPE_BPE  = 2, // GPT-2 tokenizer based on byte-level BPE
        LLAMA_VOCAB_TYPE_WPM  = 3, // BERT tokenizer based on WordPiece
        LLAMA_VOCAB_TYPE_UGM  = 4, // T5 tokenizer based on Unigram
+        LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization
    };

    // pre-tokenization types