From b693000c2e388cb02b1ab5a8e0e7df8942a76f17 Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Wed, 23 Aug 2023 13:22:41 +0200 Subject: [PATCH] llama.cpp : fix linefeed token --- llama.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/llama.cpp b/llama.cpp index a2ee80ab2..61446cdf7 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1596,6 +1596,9 @@ static void llm_load_hparams( hparams.rope_freq_scale = rope_freq_scale; } +// TODO: This should probably be in llama.h +static std::vector llama_tokenize_internal(const llama_vocab & vocab, const std::string & raw_text, bool bos, bool escape); + static void llm_load_vocab( llama_model_loader & ml, llama_model & model) { @@ -1655,12 +1658,11 @@ static void llm_load_vocab( token_data.score = scores[i]; token_data.type = (llama_token_type) toktypes[i]; - // determine the newline token: 0x0A == 10 == '\n' - if (token_data.text == "<0x0A>") { - vocab.linefeed_id = i; - } } + // determine the newline token: LLaMA "<0x0A>" == 10 == '\n', Falcon 193 == '\n' + vocab.linefeed_id = llama_tokenize_internal(vocab, "\n", false, false)[0]; + // special tokens GGUF_GET_KEY(ctx, vocab.special_bos_id, gguf_get_val_u32, GGUF_TYPE_UINT32, false, kv(LLM_KV_TOKENIZER_BOS_ID)); GGUF_GET_KEY(ctx, vocab.special_eos_id, gguf_get_val_u32, GGUF_TYPE_UINT32, false, kv(LLM_KV_TOKENIZER_EOS_ID));