Implemented basic interface for llamacheck and link to weights, adapting from simple.cpp

Makefile

@@ -2,7 +2,7 @@
 BUILD_TARGETS = \
 	main quantize quantize-stats perplexity imatrix embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml \
 	simple batched batched-bench save-load-state server gguf gguf-split eval-callback llama-bench libllava.a llava-cli baby-llama beam-search  \
-	retrieval speculative infill tokenize benchmark-matmult parallel finetune export-lora lookahead lookup passkey gritlm tests/test-c.o
+	retrieval speculative infill tokenize benchmark-matmult parallel finetune export-lora lookahead lookup passkey gritlm tests/test-c.o llamacheck
 
 # Binaries only useful for tests
 TEST_TARGETS = \
@@ -892,6 +892,10 @@ gbnf-validator: examples/gbnf-validator/gbnf-validator.cpp ggml.o llama.o $(COMM
 	$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
 
+llamacheck: examples/llamacheck/llamacheck.cpp                          ggml.o llama.o $(COMMON_DEPS) $(OBJS)
+	$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
+	$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
+
 ifeq ($(UNAME_S),Darwin)
 swift: examples/batched.swift
 	(cd examples/batched.swift; make build)

examples/CMakeLists.txt

@@ -12,6 +12,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
 if (EMSCRIPTEN)
 else()
+    add_subdirectory(llamacheck)
     add_subdirectory(baby-llama)
     add_subdirectory(batched)
     add_subdirectory(batched-bench)
@@ -45,6 +46,7 @@ else()
     add_subdirectory(gguf)
     add_subdirectory(train-text-from-scratch)
     add_subdirectory(imatrix)
+
     if (LLAMA_BUILD_SERVER)
         add_subdirectory(server)
     endif()

examples/llamacheck/CMakeLists.txt

@@ -0,0 +1,5 @@
+set(TARGET llamacheck)
+add_executable(${TARGET} llamacheck.cpp)
+install(TARGETS ${TARGET} RUNTIME)
+target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
+target_compile_features(${TARGET} PRIVATE cxx_std_11)

examples/llamacheck/README.md

@@ -0,0 +1,15 @@
+# Llamacheck: Basic Spellcheck and Grammarcheck using Llama
+
+
+The attached file provides a basic implementation of LLama to 
+be used for Spell and Grammar checking.
+We use it as follows:
+```console
+make llamacheck
+./llamacheck <./models/llamacheck.gguf> 
+```
+The weights are quantized. On my machine, it runs with as speed of 7.21 t/s
+
+
+Weights are available at:
+https://huggingface.co/azferruolo/llamacheck

examples/llamacheck/llamacheck.cpp

@@ -0,0 +1,194 @@
+#include "common.h"
+#include "llama.h"
+
+#include <cmath>
+#include <cstdio>
+#include <string>
+#include <vector>
+
+int main(int argc, char ** argv) {
+    gpt_params params;
+
+    if (argc == 1 || argv[1][0] == '-') {
+        printf("usage: %s MODEL_PATH [PROMPT]\n" , argv[0]);
+        return 1 ;
+    }
+
+    if (argc >= 2) {
+        params.model = argv[1];
+    }
+
+
+
+    params.prompt = "";
+
+    // total length of the sequence including the prompt
+    const int n_len = 150;
+
+    // init LLM
+
+    llama_backend_init();
+    llama_numa_init(params.numa);
+
+    // initialize the model
+
+    llama_model_params model_params = llama_model_default_params();
+
+    // model_params.n_gpu_layers = 99; // offload all layers to the GPU
+
+    llama_model * model = llama_load_model_from_file(params.model.c_str(), model_params);
+
+    if (model == NULL) {
+        fprintf(stderr , "%s: error: unable to load model\n" , __func__);
+        return 1;
+    }
+
+    // initialize the context
+
+    llama_context_params ctx_params = llama_context_default_params();
+
+    ctx_params.seed  = 1234;
+    ctx_params.n_ctx = 2048;
+    ctx_params.n_threads = params.n_threads;
+    ctx_params.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch;
+
+    llama_context * ctx = llama_new_context_with_model(model, ctx_params);
+
+    if (ctx == NULL) {
+        fprintf(stderr , "%s: error: failed to create the llama_context\n" , __func__);
+        return 1;
+    }
+
+
+
+    // main loop
+
+    std::string prompt_template = "You will see two sentences. The first is marked INCORRECT and has a plethora of spelling and grammatical issues, the second is marked CORRECT and shows the fixed version of the prior sentence. INCORRECT:";
+    std::string prompt_suffix = " CORRECT: ";
+    std::string input_string = "";
+    while (std::getline(std::cin, input_string, '\n')) {
+        if (input_string == "q") {
+            break;
+        }
+
+        // tokenize the prompt
+
+        params.prompt = prompt_template;
+        params.prompt += input_string;
+        params.prompt += prompt_suffix;
+
+        std::vector<llama_token> tokens_list;
+        tokens_list = ::llama_tokenize(ctx, params.prompt, true);
+
+        const int n_ctx    = llama_n_ctx(ctx);
+        const int n_kv_req = tokens_list.size() + (n_len - tokens_list.size());
+
+        LOG_TEE("\n%s: n_len = %d, n_ctx = %d, n_kv_req = %d\n", __func__, n_len, n_ctx, n_kv_req);
+
+        // make sure the KV cache is big enough to hold all the prompt and generated tokens
+        if (n_kv_req > n_ctx) {
+            LOG_TEE("%s: error: n_kv_req > n_ctx, the required KV cache size is not big enough\n", __func__);
+            LOG_TEE("%s:        either reduce n_len or increase n_ctx\n", __func__);
+            return 1;
+        }
+
+        // print the prompt token-by-token
+
+        fprintf(stderr, "\n");
+
+        for (auto id : tokens_list) {
+            fprintf(stderr, "%s", llama_token_to_piece(ctx, id).c_str());
+        }
+
+        fflush(stderr);
+
+        // create a llama_batch with size 512
+        // we use this object to submit token data for decoding
+
+        llama_batch batch = llama_batch_init(512, 0, 1);
+
+        // evaluate the initial prompt
+        for (size_t i = 0; i < tokens_list.size(); i++) {
+            llama_batch_add(batch, tokens_list[i], i, { 0 }, false);
+        }
+
+        // llama_decode will output logits only for the last token of the prompt
+        batch.logits[batch.n_tokens - 1] = true;
+
+        if (llama_decode(ctx, batch) != 0) {
+            LOG_TEE("%s: llama_decode() failed\n", __func__);
+            return 1;
+        }
+
+        int n_cur    = batch.n_tokens;
+        int n_decode = 0;
+
+        const auto t_main_start = ggml_time_us();
+
+        while (n_cur <= n_len) {
+            // sample the next token
+            {
+                auto   n_vocab = llama_n_vocab(model);
+                auto * logits  = llama_get_logits_ith(ctx, batch.n_tokens - 1);
+
+                std::vector<llama_token_data> candidates;
+                candidates.reserve(n_vocab);
+
+                for (llama_token token_id = 0; token_id < n_vocab; token_id++) {
+                    candidates.emplace_back(llama_token_data{ token_id, logits[token_id], 0.0f });
+                }
+
+                llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
+
+                // sample the most likely token
+                const llama_token new_token_id = llama_sample_token_greedy(ctx, &candidates_p);
+
+                // is it an end of generation?
+                if (llama_token_is_eog(model, new_token_id) || n_cur == n_len) {
+                    LOG_TEE("\n");
+
+                    break;
+                }
+
+                LOG_TEE("%s", llama_token_to_piece(ctx, new_token_id).c_str());
+                fflush(stdout);
+
+                // prepare the next batch
+                llama_batch_clear(batch);
+
+                // push this new token for next evaluation
+                llama_batch_add(batch, new_token_id, n_cur, { 0 }, true);
+
+                n_decode += 1;
+            }
+
+            n_cur += 1;
+
+            // evaluate the current batch with the transformer model
+            if (llama_decode(ctx, batch)) {
+                fprintf(stderr, "%s : failed to eval, return code %d\n", __func__, 1);
+                return 1;
+            }
+        }
+        LOG_TEE("\n");
+
+        const auto t_main_end = ggml_time_us();
+
+        LOG_TEE("%s: decoded %d tokens in %.2f s, speed: %.2f t/s\n",
+                __func__, n_decode, (t_main_end - t_main_start) / 1000000.0f, n_decode / ((t_main_end - t_main_start) / 1000000.0f));
+
+        llama_print_timings(ctx);
+
+        fprintf(stderr, "\n");
+
+        llama_batch_free(batch);
+    }
+
+
+    llama_free(ctx);
+    llama_free_model(model);
+
+    llama_backend_free();
+
+    return 0;
+}