From 25d60dcf50a9154908ca4ab4a537e9704c076216 Mon Sep 17 00:00:00 2001 From: Bartosz Podkanowicz Date: Wed, 8 Nov 2023 02:08:45 +0100 Subject: [PATCH 1/7] contrastive: initial example --- Makefile | 7 +- examples/CMakeLists.txt | 1 + examples/contrastive/CMakeLists.txt | 8 + examples/contrastive/contrastive.cpp | 213 +++++++++++++++++++++++++++ 4 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 examples/contrastive/CMakeLists.txt create mode 100644 examples/contrastive/contrastive.cpp diff --git a/Makefile b/Makefile index d6be254a0..a3bb4f4f7 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ # Define the default target now so that it is always the first target BUILD_TARGETS = \ main quantize quantize-stats perplexity embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml \ - simple batched batched-bench save-load-state server gguf llama-bench libllava.a llava-cli baby-llama beam-search \ - speculative infill benchmark-matmult parallel finetune export-lora tests/test-c.o + contrastive simple batched batched-bench save-load-state server gguf llama-bench libllava.a llava-cli baby-llama \ + beam-search speculative infill benchmark-matmult parallel finetune export-lora tests/test-c.o # Binaries only useful for tests TEST_TARGETS = \ @@ -614,6 +614,9 @@ train-text-from-scratch: examples/train-text-from-scratch/train-text-from-scratc convert-llama2c-to-ggml: examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp ggml.o llama.o $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) +contrastive: examples/contrastive/contrastive.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) + $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + llama-bench: examples/llama-bench/llama-bench.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 75b8df676..3bac19be7 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -18,6 +18,7 @@ else() add_subdirectory(beam-search) add_subdirectory(benchmark) add_subdirectory(convert-llama2c-to-ggml) + add_subdirectory(contrastive) add_subdirectory(embedding) add_subdirectory(finetune) add_subdirectory(infill) diff --git a/examples/contrastive/CMakeLists.txt b/examples/contrastive/CMakeLists.txt new file mode 100644 index 000000000..36bf76b9b --- /dev/null +++ b/examples/contrastive/CMakeLists.txt @@ -0,0 +1,8 @@ +set(TARGET contrastive) +add_executable(${TARGET} contrastive.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_11) +if(TARGET BUILD_INFO) + add_dependencies(${TARGET} BUILD_INFO) +endif() diff --git a/examples/contrastive/contrastive.cpp b/examples/contrastive/contrastive.cpp new file mode 100644 index 000000000..35cf90582 --- /dev/null +++ b/examples/contrastive/contrastive.cpp @@ -0,0 +1,213 @@ +#include "common.h" +#include "llama.h" + +#include +#include +#include +#include +#include + +int main(int argc, char ** argv) { + gpt_params params_expert; + gpt_params params_amateur; + if (argc == 1 || argv[1][0] == '-') { + printf("usage: %s EXPERT_MODEL_PATH AMATEUR_MODEL_PATH [PROMPT]\n" , argv[0]); + return 1; + } + + if (argc >= 2) { + params_expert.model = argv[1]; + } + + if (argc >= 3) { + params_amateur.model = argv[2]; + } + + if (argc >= 4) { + params_expert.prompt = argv[3]; + params_amateur.prompt = argv[3]; + } + + if (params_expert.prompt.empty()) { + params_expert.prompt = "Hello my name is"; + params_amateur.prompt = "Hello my name is"; + } + + // total length of the sequence including the prompt + const int n_len = 32; + + // init LLM + + llama_backend_init(params_expert.numa); + + // initialize the model + + llama_model_params model_params = llama_model_default_params(); + + // model_params.n_gpu_layers = 99; // offload all layers to the GPU + + llama_model * model_expert = llama_load_model_from_file(params_expert.model.c_str(), model_params); + llama_model * model_amateur = llama_load_model_from_file(params_amateur.model.c_str(), model_params); + + + if (model_expert == NULL or model_amateur == NULL) { + fprintf(stderr , "%s: error: unable to load model\n" , __func__); + return 1; + } + + // initialize the context + + llama_context_params ctx_params = llama_context_default_params(); + + ctx_params.seed = 1234; + ctx_params.n_ctx = 2048; + ctx_params.n_threads = params_expert.n_threads; + ctx_params.n_threads_batch = params_expert.n_threads_batch == -1 ? params_expert.n_threads : params_expert.n_threads_batch; + + llama_context * ctx_expert = llama_new_context_with_model(model_expert, ctx_params); + llama_context * ctx_amateur = llama_new_context_with_model(model_amateur, ctx_params); + + if (ctx_expert == NULL or ctx_amateur == NULL) { + fprintf(stderr , "%s: error: failed to create the llama_context\n" , __func__); + return 1; + } + + // tokenize the prompt + + std::vector tokens_list; + tokens_list = ::llama_tokenize(ctx_expert, params_expert.prompt, true); + + const int n_ctx = llama_n_ctx(ctx_expert); + const int n_kv_req = tokens_list.size() + (n_len - tokens_list.size()); + + LOG_TEE("\n%s: n_len = %d, n_ctx = %d, n_kv_req = %d\n", __func__, n_len, n_ctx, n_kv_req); + + // make sure the KV cache is big enough to hold all the prompt and generated tokens + if (n_kv_req > n_ctx) { + LOG_TEE("%s: error: n_kv_req > n_ctx, the required KV cache size is not big enough\n", __func__); + LOG_TEE("%s: either reduce n_parallel or increase n_ctx\n", __func__); + return 1; + } + + // print the prompt token-by-token + + fprintf(stderr, "\n"); + + for (auto id : tokens_list) { + fprintf(stderr, "%s", llama_token_to_piece(ctx_expert, id).c_str()); + } + + fflush(stderr); + + // create a llama_batch with size 512 + // we use this object to submit token data for decoding + + llama_batch batch = llama_batch_init(512, 0, 1); + + // evaluate the initial prompt + for (size_t i = 0; i < tokens_list.size(); i++) { + llama_batch_add(batch, tokens_list[i], i, { 0 }, false); + } + + // llama_decode will output logits only for the last token of the prompt + batch.logits[batch.n_tokens - 1] = true; + + if (llama_decode(ctx_expert, batch) != 0) { + LOG_TEE("%s: llama_decode() failed\n", __func__); + return 1; + } + + if (llama_decode(ctx_amateur, batch) != 0) { + LOG_TEE("%s: llama_decode() failed\n", __func__); + return 1; + } + + // main loop + + int n_cur = batch.n_tokens; + int n_decode = 0; + + const auto t_main_start = ggml_time_us(); + + float alpha = 0.1; + float beta = 0.5; + + while (n_cur <= n_len) { + // sample the next token + { + auto n_vocab = llama_n_vocab(model_expert); + auto * logits_expert = llama_get_logits_ith(ctx_expert, batch.n_tokens - 1); + auto * logits_amateur = llama_get_logits_ith(ctx_amateur, batch.n_tokens - 1); + + std::vector candidates; + candidates.reserve(n_vocab); + + for (llama_token token_id = 0; token_id < n_vocab; token_id++) { + float cd_logit = std::numeric_limits::lowest(); + if(logits_expert[token_id] > alpha){ + cd_logit = (1+beta)*logits_expert[token_id] - beta*logits_amateur[token_id]; + } + candidates.emplace_back(llama_token_data{ token_id, cd_logit, 0.0f }); + } + + llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false }; + + // sample the most likely token + const llama_token new_token_id_expert = llama_sample_token_greedy(ctx_expert, &candidates_p); + + // is it an end of stream? + if (new_token_id_expert == llama_token_eos(model_expert) || n_cur == n_len) { + LOG_TEE("\n"); + + break; + } + + LOG_TEE("%s", llama_token_to_piece(ctx_expert, new_token_id_expert).c_str()); + fflush(stdout); + + // prepare the next batch + llama_batch_clear(batch); + + // push this new token for next evaluation + llama_batch_add(batch, new_token_id_expert, n_cur, { 0 }, true); + + n_decode += 1; + } + + n_cur += 1; + + // evaluate the current batch with the transformer model + if (llama_decode(ctx_expert, batch)) { + fprintf(stderr, "%s : failed to eval, return code %d\n", __func__, 1); + return 1; + } + if (llama_decode(ctx_amateur, batch)) { + fprintf(stderr, "%s : failed to eval, return code %d\n", __func__, 1); + return 1; + } + } + + LOG_TEE("\n"); + + const auto t_main_end = ggml_time_us(); + + LOG_TEE("%s: decoded %d tokens in %.2f s, speed: %.2f t/s\n", + __func__, n_decode, (t_main_end - t_main_start) / 1000000.0f, n_decode / ((t_main_end - t_main_start) / 1000000.0f)); + + llama_print_timings(ctx_expert); + llama_print_timings(ctx_amateur); + + fprintf(stderr, "\n"); + + llama_batch_free(batch); + + llama_free(ctx_expert); + llama_free(ctx_amateur); + llama_free_model(model_expert); + llama_free_model(model_amateur); + + llama_backend_free(); + + return 0; +} + From 38c5b7ee5f38860e19715310e5fed2bf9334711e Mon Sep 17 00:00:00 2001 From: trt Date: Wed, 8 Nov 2023 03:52:30 +0100 Subject: [PATCH 2/7] fix compilation error for msbuild --- examples/contrastive/contrastive.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/examples/contrastive/contrastive.cpp b/examples/contrastive/contrastive.cpp index 35cf90582..7c6446268 100644 --- a/examples/contrastive/contrastive.cpp +++ b/examples/contrastive/contrastive.cpp @@ -50,8 +50,13 @@ int main(int argc, char ** argv) { llama_model * model_amateur = llama_load_model_from_file(params_amateur.model.c_str(), model_params); - if (model_expert == NULL or model_amateur == NULL) { - fprintf(stderr , "%s: error: unable to load model\n" , __func__); + if (model_expert == NULL) { + fprintf(stderr , "%s: error: unable to load expert model\n" , __func__); + return 1; + } + + if (model_amateur == NULL) { + fprintf(stderr , "%s: error: unable to load amateur model\n" , __func__); return 1; } @@ -67,8 +72,13 @@ int main(int argc, char ** argv) { llama_context * ctx_expert = llama_new_context_with_model(model_expert, ctx_params); llama_context * ctx_amateur = llama_new_context_with_model(model_amateur, ctx_params); - if (ctx_expert == NULL or ctx_amateur == NULL) { - fprintf(stderr , "%s: error: failed to create the llama_context\n" , __func__); + if (ctx_expert == NULL) { + fprintf(stderr , "%s: error: failed to create the llama_context for expert\n" , __func__); + return 1; + } + + if (ctx_amateur == NULL) { + fprintf(stderr , "%s: error: failed to create the llama_context for amateur\n" , __func__); return 1; } From db2a5beef1150e78990e568b0b22d85d0c91813c Mon Sep 17 00:00:00 2001 From: Bartosz Podkanowicz Date: Thu, 9 Nov 2023 15:37:53 +0100 Subject: [PATCH 3/7] erase constant seed, add changing alpha and beta parameters from command line --- examples/contrastive/contrastive.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/examples/contrastive/contrastive.cpp b/examples/contrastive/contrastive.cpp index 7c6446268..f756bbc38 100644 --- a/examples/contrastive/contrastive.cpp +++ b/examples/contrastive/contrastive.cpp @@ -11,7 +11,7 @@ int main(int argc, char ** argv) { gpt_params params_expert; gpt_params params_amateur; if (argc == 1 || argv[1][0] == '-') { - printf("usage: %s EXPERT_MODEL_PATH AMATEUR_MODEL_PATH [PROMPT]\n" , argv[0]); + printf("usage: %s EXPERT_MODEL_PATH AMATEUR_MODEL_PATH [PROMPT] [alpha] [beta]\n" , argv[0]); return 1; } @@ -28,6 +28,17 @@ int main(int argc, char ** argv) { params_amateur.prompt = argv[3]; } + float alpha = 0.1; + float beta = 0.5; + + if(argc >= 5){ + alpha = std::stof(argv[4]); + } + + if(argc >= 6){ + beta = std::stof(argv[5]); + } + if (params_expert.prompt.empty()) { params_expert.prompt = "Hello my name is"; params_amateur.prompt = "Hello my name is"; @@ -64,7 +75,6 @@ int main(int argc, char ** argv) { llama_context_params ctx_params = llama_context_default_params(); - ctx_params.seed = 1234; ctx_params.n_ctx = 2048; ctx_params.n_threads = params_expert.n_threads; ctx_params.n_threads_batch = params_expert.n_threads_batch == -1 ? params_expert.n_threads : params_expert.n_threads_batch; @@ -138,10 +148,6 @@ int main(int argc, char ** argv) { int n_decode = 0; const auto t_main_start = ggml_time_us(); - - float alpha = 0.1; - float beta = 0.5; - while (n_cur <= n_len) { // sample the next token { From 75418dc2c2e1ec875d7517067442730a805c9f79 Mon Sep 17 00:00:00 2001 From: Bartosz Podkanowicz Date: Thu, 9 Nov 2023 15:45:51 +0100 Subject: [PATCH 4/7] change in spaces, change in fprintf formating --- examples/contrastive/contrastive.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/examples/contrastive/contrastive.cpp b/examples/contrastive/contrastive.cpp index f756bbc38..3e52e779b 100644 --- a/examples/contrastive/contrastive.cpp +++ b/examples/contrastive/contrastive.cpp @@ -11,7 +11,7 @@ int main(int argc, char ** argv) { gpt_params params_expert; gpt_params params_amateur; if (argc == 1 || argv[1][0] == '-') { - printf("usage: %s EXPERT_MODEL_PATH AMATEUR_MODEL_PATH [PROMPT] [alpha] [beta]\n" , argv[0]); + printf("usage: %s EXPERT_MODEL_PATH AMATEUR_MODEL_PATH [PROMPT] [alpha] [beta]\n", argv[0]); return 1; } @@ -62,12 +62,12 @@ int main(int argc, char ** argv) { if (model_expert == NULL) { - fprintf(stderr , "%s: error: unable to load expert model\n" , __func__); + fprintf(stderr, "%s: error: unable to load expert model\n", __func__); return 1; } if (model_amateur == NULL) { - fprintf(stderr , "%s: error: unable to load amateur model\n" , __func__); + fprintf(stderr, "%s: error: unable to load amateur model\n", __func__); return 1; } @@ -83,12 +83,12 @@ int main(int argc, char ** argv) { llama_context * ctx_amateur = llama_new_context_with_model(model_amateur, ctx_params); if (ctx_expert == NULL) { - fprintf(stderr , "%s: error: failed to create the llama_context for expert\n" , __func__); + fprintf(stderr, "%s: error: failed to create the llama_context for expert\n", __func__); return 1; } if (ctx_amateur == NULL) { - fprintf(stderr , "%s: error: failed to create the llama_context for amateur\n" , __func__); + fprintf(stderr, "%s: error: failed to create the llama_context for amateur\n", __func__); return 1; } @@ -174,7 +174,6 @@ int main(int argc, char ** argv) { // is it an end of stream? if (new_token_id_expert == llama_token_eos(model_expert) || n_cur == n_len) { LOG_TEE("\n"); - break; } @@ -194,11 +193,11 @@ int main(int argc, char ** argv) { // evaluate the current batch with the transformer model if (llama_decode(ctx_expert, batch)) { - fprintf(stderr, "%s : failed to eval, return code %d\n", __func__, 1); + fprintf(stderr, "%s : failed to eval, return code 1\n", __func__); return 1; } if (llama_decode(ctx_amateur, batch)) { - fprintf(stderr, "%s : failed to eval, return code %d\n", __func__, 1); + fprintf(stderr, "%s : failed to eval, return code 1\n", __func__); return 1; } } From 1cf0b0927396dd33fd68550ee2eb644d77fb6e00 Mon Sep 17 00:00:00 2001 From: Bartosz Podkanowicz Date: Thu, 9 Nov 2023 21:02:59 +0100 Subject: [PATCH 5/7] erase flushing stderr, changes in spaces --- examples/contrastive/contrastive.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/examples/contrastive/contrastive.cpp b/examples/contrastive/contrastive.cpp index 3e52e779b..83610dcfe 100644 --- a/examples/contrastive/contrastive.cpp +++ b/examples/contrastive/contrastive.cpp @@ -31,11 +31,11 @@ int main(int argc, char ** argv) { float alpha = 0.1; float beta = 0.5; - if(argc >= 5){ + if (argc >= 5) { alpha = std::stof(argv[4]); } - if(argc >= 6){ + if (argc >= 6) { beta = std::stof(argv[5]); } @@ -117,8 +117,6 @@ int main(int argc, char ** argv) { fprintf(stderr, "%s", llama_token_to_piece(ctx_expert, id).c_str()); } - fflush(stderr); - // create a llama_batch with size 512 // we use this object to submit token data for decoding @@ -160,7 +158,7 @@ int main(int argc, char ** argv) { for (llama_token token_id = 0; token_id < n_vocab; token_id++) { float cd_logit = std::numeric_limits::lowest(); - if(logits_expert[token_id] > alpha){ + if (logits_expert[token_id] > alpha) { cd_logit = (1+beta)*logits_expert[token_id] - beta*logits_amateur[token_id]; } candidates.emplace_back(llama_token_data{ token_id, cd_logit, 0.0f }); @@ -178,7 +176,6 @@ int main(int argc, char ** argv) { } LOG_TEE("%s", llama_token_to_piece(ctx_expert, new_token_id_expert).c_str()); - fflush(stdout); // prepare the next batch llama_batch_clear(batch); From 3f8e444d0d7fd06835d790732ce2b7c722be6d97 Mon Sep 17 00:00:00 2001 From: Bartosz Podkanowicz Date: Thu, 9 Nov 2023 21:21:35 +0100 Subject: [PATCH 6/7] fix error in the formula - formula now is similar to formula in the paper. --- examples/contrastive/contrastive.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/contrastive/contrastive.cpp b/examples/contrastive/contrastive.cpp index 83610dcfe..24616b347 100644 --- a/examples/contrastive/contrastive.cpp +++ b/examples/contrastive/contrastive.cpp @@ -1,6 +1,7 @@ #include "common.h" #include "llama.h" +#include #include #include #include @@ -144,6 +145,7 @@ int main(int argc, char ** argv) { int n_cur = batch.n_tokens; int n_decode = 0; + float log_alpha = std::log(alpha); const auto t_main_start = ggml_time_us(); while (n_cur <= n_len) { @@ -156,9 +158,10 @@ int main(int argc, char ** argv) { std::vector candidates; candidates.reserve(n_vocab); + auto largest_expert_logit = *std::max_element(logits_expert, logits_expert + n_vocab); for (llama_token token_id = 0; token_id < n_vocab; token_id++) { float cd_logit = std::numeric_limits::lowest(); - if (logits_expert[token_id] > alpha) { + if (logits_expert[token_id] > log_alpha + largest_expert_logit) { cd_logit = (1+beta)*logits_expert[token_id] - beta*logits_amateur[token_id]; } candidates.emplace_back(llama_token_data{ token_id, cd_logit, 0.0f }); From 147ab5eb0ceda47006d8111bae60e9434959007b Mon Sep 17 00:00:00 2001 From: Bartosz Podkanowicz Date: Fri, 10 Nov 2023 15:39:26 +0100 Subject: [PATCH 7/7] restore flushing stdout --- examples/contrastive/contrastive.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/contrastive/contrastive.cpp b/examples/contrastive/contrastive.cpp index 24616b347..0e5c01a59 100644 --- a/examples/contrastive/contrastive.cpp +++ b/examples/contrastive/contrastive.cpp @@ -179,6 +179,7 @@ int main(int argc, char ** argv) { } LOG_TEE("%s", llama_token_to_piece(ctx_expert, new_token_id_expert).c_str()); + fflush(stdout); // prepare the next batch llama_batch_clear(batch);