vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

convert : refactor vocab selection logic (#6355)

Browse source
This commit is contained in:
Jared Van Bortel 2024-03-28 11:44:36 -04:00 • committed by GitHub
parent 66ba560256
commit be55134a53
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 204 additions and 176 deletions
Download patch file Download diff file Expand all files Collapse all files

6
llama.h
View file

@ -60,9 +60,9 @@ extern "C" {
enum llama_vocab_type {
LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab
LLAMA_VOCAB_TYPE_SPM = 1, // SentencePiece
LLAMA_VOCAB_TYPE_BPE = 2, // Byte Pair Encoding
LLAMA_VOCAB_TYPE_WPM = 3, // WordPiece
LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback
LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE
LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece
};
// note: these values should be synchronized with ggml_rope