vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

examples : rely on new behavior of add_special

Browse source
This commit is contained in:
Jared Van Bortel 2024-04-04 18:12:33 -04:00
parent d1a1b614cd
commit 92591c125f
14 changed files with 53 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

8
common/common.h
View file

@ -221,14 +221,14 @@ void llama_batch_add(
std::vector<llama_token> llama_tokenize(
const struct llama_context * ctx,
const std::string & text,
bool add_bos,
bool special = false);
bool add_special,
bool parse_special = false);
std::vector<llama_token> llama_tokenize(
const struct llama_model * model,
const std::string & text,
bool add_bos,
bool special = false);
bool add_special,
bool parse_special = false);
// tokenizes a token into a piece
// should work similar to Python's `tokenizer.id_to_piece`