vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into speculative-tree

Browse source
This commit is contained in:
Georgi Gerganov 2023-10-17 19:31:40 +03:00 • committed by GitHub
commit bd9451ca2a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 385 additions and 90 deletions
Download patch file Download diff file Expand all files Collapse all files

6
common/common.h
View file

@ -151,12 +151,14 @@ void llama_batch_add(
std::vector<llama_token> llama_tokenize(
const struct llama_context * ctx,
const std::string & text,
bool add_bos);
bool add_bos,
bool special = false);
std::vector<llama_token> llama_tokenize(
const struct llama_model * model,
const std::string & text,
bool add_bos);
bool add_bos,
bool special = false);
// tokenizes a token into a piece
// should work similar to Python's `tokenizer.id_to_piece`