From ac69c93ca9aa42b6195a646cb75f569910881cac Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 7 Dec 2023 18:40:52 -0500 Subject: [PATCH] linking --- CMakeLists.txt | 3 +- Makefile | 4 + embedding.py | 35 ++++++- examples/main/main.cpp | 20 ++-- plugin_nodejs.cpp | 209 +++++++++++++++++++++++++++++++++++++++++ plugin_nodejs.hpp | 3 + plugin_python.cpp | 8 +- plugin_python.hpp | 4 +- 8 files changed, 271 insertions(+), 15 deletions(-) create mode 100644 plugin_nodejs.cpp create mode 100644 plugin_nodejs.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 50a31ce18..331897432 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -733,6 +733,7 @@ add_library(ggml OBJECT ggml.h print.hpp plugin_python.cpp + plugin_nodejs.cpp ggml-internal.hpp llama-internal.hpp ggml-alloc.cpp @@ -748,7 +749,7 @@ add_library(ggml OBJECT ${GGML_SOURCES_EXTRA} ${GGML_HEADERS_EXTRA} ) -target_include_directories(ggml PUBLIC . ${LLAMA_EXTRA_INCLUDES}) +target_include_directories(ggml PUBLIC . ${LLAMA_EXTRA_INCLUDES} "/usr/local/include/node/") target_compile_features(ggml PUBLIC c_std_11) # don't bump target_link_libraries(ggml PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS}) if (GGML_USE_CPU_HBM) diff --git a/Makefile b/Makefile index 01c46cd09..507d05851 100644 --- a/Makefile +++ b/Makefile @@ -750,3 +750,7 @@ tests/test-rope: tests/test-rope.cpp ggml.o $(OBJS) tests/test-c.o: tests/test-c.cpp llama.h $(CC) $(CXXFLAGS) -c $(filter-out %.h,$^) -o $@ + + +test123: + ./build2/bin/main -m ~/.ollama/models/mistral --interactive -r STOP -p 'write simple python expression to be evaluated ending WITH TOKEN ' -n -1 diff --git a/embedding.py b/embedding.py index 5d95f663f..e8bb3e688 100644 --- a/embedding.py +++ b/embedding.py @@ -1,6 +1,31 @@ -print("hello llama.cpp, got input:\n" + llm_input + "\n") +#for x in list(globals()): +# print("GLOBAL",x,globals()[x],"\n") +# any global variables set here will be available later as well! -if len(llm_input) > 20: - llm_output = "Reinterpret with emojis " + llm_input + "?\nSTOP"; -else: - llm_output = llm_input +#print("debug input:\n" + llm_input + "\n") +#foobar ="test" +#if llm_state in ("params", statement, antiprompt,) + +def entrypoint(): + global llm_output + global llm_input + global llm_state + llm_output = llm_input + if llm_state == "antiprompt": + #used to check each token if you want to stop early + return + elif llm_state == "params": + # first time it is called it returns the state via llm_output that will be used + return + elif llm_state == "statement": + if "" in llm_input: + llm_input = llm_input.replace("","") + try: + v= eval(llm_input) + llm_output = "Check that the evaluation of```" + llm_input + "``` Produced:"+ str(v) + " STOP"; + except Exception as e: + #print(e) + llm_output = "generate a simple python expression to be evaluated. to evaluate your work emit the word and the python code will be evaluated. Please correct the python error in Evaluation of ```" + llm_input + "``` Produced Output:"+ str(e) + "now consider the original task"+ llm_start + " STOP" + +if __name__ == "__main__": + entrypoint() diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 4cd63650c..354b8f9a0 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -178,9 +178,11 @@ int main(int argc, char ** argv) { std::mt19937 rng(params.seed); if (params.random_prompt) { - params.prompt = gpt_random_prompt(rng); + params.prompt = gpt_random_prompt(rng); } + auto start_prompt = process_output_plugin(params.prompt,"params",params.prompt); + LOG("%s: llama backend init\n", __func__); llama_backend_init(params.numa); @@ -606,7 +608,7 @@ int main(int argc, char ** argv) { int n_eval = std::min(input_size - i, params.n_batch); if (llama_decode(ctx_guidance, llama_batch_get_one(input_buf + i, n_eval, n_past_guidance, 0))) { LOG_TEE("%s : failed to eval\n", __func__); - return 1; + //return 1; } n_past_guidance += n_eval; @@ -685,7 +687,7 @@ int main(int argc, char ** argv) { if (input_echo) { for (auto id : embd) { const std::string token_str = llama_token_to_piece(ctx, id); - printf("TOKEN:%s\n", token_str.c_str()); + printf("\nTOKEN:%s\n", token_str.c_str()); //print_fields(id); @@ -706,8 +708,8 @@ int main(int argc, char ** argv) { // just print the whole thing const std::string last_output1 = output_ss.str(); printf("%s",last_output1.c_str()); - last_output = process_output_plugin(last_output1); - printf("%s",last_output.c_str()); + last_output = process_output_plugin(start_prompt,"statement",last_output1); + printf("\nLASTOUTPUT: '%s'\n",last_output.c_str()); // if not currently processing queued inputs; if ((int) embd_inp.size() <= n_consumed) { @@ -716,7 +718,7 @@ int main(int argc, char ** argv) { const int n_prev = 32; const std::string last_output1 = llama_sampling_prev_str(ctx_sampling, ctx, n_prev); // now plugin the python : - const std::string partial_output = process_output_plugin(last_output1); + const std::string partial_output = process_output_plugin(start_prompt,"antiprompt",last_output1); is_antiprompt = false; // Check if each of the reverse prompts appears at the end of the output. @@ -786,6 +788,12 @@ int main(int argc, char ** argv) { //bool another_line = true; //do { // another_line = console::readline(line, params.multiline_input); + + for (const auto & antiprompt : params.antiprompt) { + size_t found_pos = last_output.find(antiprompt); + if (found_pos != string::npos) { + last_output.erase(found_pos,found_pos+ antiprompt.length()); } + } buffer += last_output; //} while (another_line); diff --git a/plugin_nodejs.cpp b/plugin_nodejs.cpp new file mode 100644 index 000000000..c1a16231d --- /dev/null +++ b/plugin_nodejs.cpp @@ -0,0 +1,209 @@ +#ifdef NDEBUG +#undef NDEBUG +#endif +#include +#include "uv.h" +#include + +#include + +// Note: This file is being referred to from doc/api/embedding.md, and excerpts +// from it are included in the documentation. Try to keep these in sync. +// Snapshot support is not part of the embedder API docs yet due to its +// experimental nature, although it is of course documented in node.h. + +using node::CommonEnvironmentSetup; +using node::Environment; +using node::MultiIsolatePlatform; +using v8::Context; +using v8::HandleScope; +using v8::Isolate; +using v8::Locker; +using v8::MaybeLocal; +using v8::V8; +using v8::Value; + +static int RunNodeInstance(MultiIsolatePlatform* platform, + const std::vector& args, + const std::vector& exec_args); + +std::string process_output_plugin(const std::string start, + const std::string state, + const std::string input) { + + //int main(int argc, char** argv) { + //argv = uv_setup_args(argc, argv); + std::vector args; + std::unique_ptr result = + node::InitializeOncePerProcess( + args, + {node::ProcessInitializationFlags::kNoInitializeV8, + node::ProcessInitializationFlags::kNoInitializeNodeV8Platform}); + + for (const std::string& error : result->errors()) + fprintf(stderr, "%s: %s\n", args[0].c_str(), error.c_str()); + if (result->early_return() != 0) { + return "ERROR"; + } + + std::unique_ptr platform = + MultiIsolatePlatform::Create(4); + V8::InitializePlatform(platform.get()); + V8::Initialize(); + + int ret = + RunNodeInstance(platform.get(), result->args(), result->exec_args()); + + V8::Dispose(); + V8::DisposePlatform(); + + node::TearDownOncePerProcess(); + return "TODO"; +} + +int RunNodeInstance(MultiIsolatePlatform* platform, + const std::vector& args, + const std::vector& exec_args) { + int exit_code = 0; + + // Format of the arguments of this binary: + // Building snapshot: + // embedtest js_code_to_eval arg1 arg2... + // --embedder-snapshot-blob blob-path + // --embedder-snapshot-create + // [--embedder-snapshot-as-file] + // Running snapshot: + // embedtest --embedder-snapshot-blob blob-path + // [--embedder-snapshot-as-file] + // arg1 arg2... + // No snapshot: + // embedtest arg1 arg2... + node::EmbedderSnapshotData::Pointer snapshot; + + std::string binary_path = args[0]; + std::vector filtered_args; + bool is_building_snapshot = false; + bool snapshot_as_file = false; + std::string snapshot_blob_path; + for (size_t i = 0; i < args.size(); ++i) { + const std::string& arg = args[i]; + if (arg == "--embedder-snapshot-create") { + is_building_snapshot = true; + } else if (arg == "--embedder-snapshot-as-file") { + snapshot_as_file = true; + } else if (arg == "--embedder-snapshot-blob") { + assert(i + 1 < args.size()); + snapshot_blob_path = args[i + 1]; + i++; + } else { + filtered_args.push_back(arg); + } + } + + if (!snapshot_blob_path.empty() && !is_building_snapshot) { + FILE* fp = fopen(snapshot_blob_path.c_str(), "r"); + assert(fp != nullptr); + if (snapshot_as_file) { + snapshot = node::EmbedderSnapshotData::FromFile(fp); + } else { + uv_fs_t req = uv_fs_t(); + int statret = + uv_fs_stat(nullptr, &req, snapshot_blob_path.c_str(), nullptr); + assert(statret == 0); + size_t filesize = req.statbuf.st_size; + uv_fs_req_cleanup(&req); + + std::vector vec(filesize); + size_t read = fread(vec.data(), filesize, 1, fp); + assert(read == 1); + snapshot = node::EmbedderSnapshotData::FromBlob(vec); + } + assert(snapshot); + int ret = fclose(fp); + assert(ret == 0); + } + + if (is_building_snapshot) { + // It contains at least the binary path, the code to snapshot, + // and --embedder-snapshot-create (which is filtered, so at least + // 2 arguments should remain after filtering). + assert(filtered_args.size() >= 2); + // Insert an anonymous filename as process.argv[1]. + filtered_args.insert(filtered_args.begin() + 1, + node::GetAnonymousMainPath()); + } + + std::vector errors; + std::unique_ptr setup = + snapshot + ? CommonEnvironmentSetup::CreateFromSnapshot( + platform, &errors, snapshot.get(), filtered_args, exec_args) + : is_building_snapshot ? CommonEnvironmentSetup::CreateForSnapshotting( + platform, &errors, filtered_args, exec_args) + : CommonEnvironmentSetup::Create( + platform, &errors, filtered_args, exec_args); + if (!setup) { + for (const std::string& err : errors) + fprintf(stderr, "%s: %s\n", binary_path.c_str(), err.c_str()); + return 1; + } + + Isolate* isolate = setup->isolate(); + Environment* env = setup->env(); + + { + Locker locker(isolate); + Isolate::Scope isolate_scope(isolate); + HandleScope handle_scope(isolate); + Context::Scope context_scope(setup->context()); + + MaybeLocal loadenv_ret; + if (snapshot) { // Deserializing snapshot + loadenv_ret = node::LoadEnvironment(env, node::StartExecutionCallback{}); + } else if (is_building_snapshot) { + // Environment created for snapshotting must set process.argv[1] to + // the name of the main script, which was inserted above. + loadenv_ret = node::LoadEnvironment( + env, + "const assert = require('assert');" + "assert(require('v8').startupSnapshot.isBuildingSnapshot());" + "globalThis.embedVars = { nön_ascıı: '🏳️‍🌈' };" + "globalThis.require = require;" + "require('vm').runInThisContext(process.argv[2]);"); + } else { + loadenv_ret = node::LoadEnvironment( + env, + "const publicRequire = require('module').createRequire(process.cwd() " + "+ '/');" + "globalThis.require = publicRequire;" + "globalThis.embedVars = { nön_ascıı: '🏳️‍🌈' };" + "require('vm').runInThisContext(process.argv[1]);"); + } + + if (loadenv_ret.IsEmpty()) // There has been a JS exception. + return 1; + + exit_code = node::SpinEventLoop(env).FromMaybe(1); + } + + if (!snapshot_blob_path.empty() && is_building_snapshot) { + snapshot = setup->CreateSnapshot(); + assert(snapshot); + + FILE* fp = fopen(snapshot_blob_path.c_str(), "w"); + assert(fp != nullptr); + if (snapshot_as_file) { + snapshot->ToFile(fp); + } else { + const std::vector vec = snapshot->ToBlob(); + size_t written = fwrite(vec.data(), vec.size(), 1, fp); + assert(written == 1); + } + int ret = fclose(fp); + assert(ret == 0); + } + + node::Stop(env); + + return exit_code; +} diff --git a/plugin_nodejs.hpp b/plugin_nodejs.hpp new file mode 100644 index 000000000..57628d561 --- /dev/null +++ b/plugin_nodejs.hpp @@ -0,0 +1,3 @@ +std::string process_output_plugin(const std::string start, + const std::string state, + const std::string input); diff --git a/plugin_python.cpp b/plugin_python.cpp index 6900a95b0..39f9d52d9 100644 --- a/plugin_python.cpp +++ b/plugin_python.cpp @@ -26,7 +26,9 @@ using namespace boost::python; #endif -std::string process_output_plugin(const std::string input) +std::string process_output_plugin(const std::string start, + const std::string state, + const std::string input) { try { PyImport_AppendInittab((char*)"mymodule", INIT_MODULE); @@ -36,7 +38,9 @@ std::string process_output_plugin(const std::string input) object mymodule = import("mymodule"); main_namespace["precreated_object"] = Base("created on C++ side"); - main_namespace["llm_input"] = input; + main_namespace["llm_input"] = input; + main_namespace["llm_state"] = state; + main_namespace["llm_start"] = start; exec_file("embedding.py", main_namespace, main_namespace); boost::python::object llm_output = main_namespace["llm_output"]; diff --git a/plugin_python.hpp b/plugin_python.hpp index fca613112..8b3e3ae6f 100644 --- a/plugin_python.hpp +++ b/plugin_python.hpp @@ -1 +1,3 @@ -std::string process_output_plugin(const std::string input); +std::string process_output_plugin(const std::string start, + const std::string state, + const std::string input);