vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #2 from jploski/master

Browse source 
Fixed bos/eos token (which is both 11 according to config.json of Fal…
This commit is contained in:
John 2023-06-16 19:56:00 +02:00 • committed by GitHub
commit 5005d07f45
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 4 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

9
examples/falcon/falcon_main.cpp
View file

@ -175,10 +175,8 @@ int main(int argc, char ** argv) {
std::vector<llama_token> embd_inp; std::vector<llama_token> embd_inp;
if (params.interactive_first || params.instruct || !params.prompt.empty() || session_tokens.empty()) { if (params.interactive_first || params.instruct || !params.prompt.empty() || session_tokens.empty()) {
// Add a space in front of the first character to match OG llama tokenizer behavior // Falcon does not have a dedicated bos token (bos==eos), so don't inject it here
params.prompt.insert(0, 1, ' '); embd_inp = ::falcon_tokenize(ctx, params.prompt, false);
embd_inp = ::falcon_tokenize(ctx, params.prompt, true);
} else { } else {
embd_inp = session_tokens; embd_inp = session_tokens;
} }
@ -361,9 +359,6 @@ int main(int argc, char ** argv) {
if (n_past + (int) embd.size() > n_ctx) { if (n_past + (int) embd.size() > n_ctx) {
const int n_left = n_past - params.n_keep; const int n_left = n_past - params.n_keep;
// always keep the first token - BOS
n_past = std::max(1, params.n_keep);
// insert n_left/2 tokens at the start of embd from last_n_tokens // insert n_left/2 tokens at the start of embd from last_n_tokens
embd.insert(embd.begin(), last_n_tokens.begin() + n_ctx - n_left/2 - embd.size(), last_n_tokens.end() - embd.size()); embd.insert(embd.begin(), last_n_tokens.begin() + n_ctx - n_left/2 - embd.size(), last_n_tokens.end() - embd.size());

10
libfalcon.cpp
View file

@ -1348,12 +1348,6 @@ static bool falcon_eval_internal(
const int n_threads, const int n_threads,
const char * cgraph_fname) { const char * cgraph_fname) {
// enforce that the first token is BOS
if (n_past == 0 && tokens[0] != falcon_token_bos()) {
fprintf(stderr, "%s: first token must be BOS\n", __func__);
return false;
}
const int64_t t_start_us = ggml_time_us(); const int64_t t_start_us = ggml_time_us();
const int N = n_tokens; const int N = n_tokens;
@ -3389,11 +3383,11 @@ const char * falcon_token_to_str(const struct falcon_context * ctx, llama_token
} }
llama_token falcon_token_bos() { llama_token falcon_token_bos() {
return 1; return 11;
} }
llama_token falcon_token_eos() { llama_token falcon_token_eos() {
return 2; return 11;
} }
llama_token falcon_token_nl() { llama_token falcon_token_nl() {