diff --git a/CMakeLists.txt b/CMakeLists.txt index 331897432..c8622933e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,7 +36,7 @@ message(STATUS "Boost_LIBRARIES = ${Boost_LIBRARIES}") ENABLE_TESTING() INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS}) -LINK_LIBRARIES(${Boost_LIBRARIES} ${Python3_LIBRARIES}) # Deprecated but so convenient! +LINK_LIBRARIES(${Boost_LIBRARIES} ${Python3_LIBRARIES} ) # Deprecated but so convenient! #PYTHON_ADD_MODULE(plugin_python plugin_python.cpp) Python3_add_library(plugin_python MODULE plugin_python.cpp) @@ -749,8 +749,8 @@ add_library(ggml OBJECT ${GGML_SOURCES_EXTRA} ${GGML_HEADERS_EXTRA} ) -target_include_directories(ggml PUBLIC . ${LLAMA_EXTRA_INCLUDES} "/usr/local/include/node/") -target_compile_features(ggml PUBLIC c_std_11) # don't bump +target_include_directories(ggml PUBLIC "/usr/include/node/" . ${LLAMA_EXTRA_INCLUDES} ) +target_compile_features(ggml PUBLIC c_std_23) # always bump target_link_libraries(ggml PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS}) if (GGML_USE_CPU_HBM) target_link_libraries(ggml PUBLIC memkind) @@ -772,10 +772,15 @@ add_library(llama ) target_include_directories(llama PUBLIC .) -target_compile_features(llama PUBLIC cxx_std_20) # don't bump +target_compile_features(llama PUBLIC cxx_std_20) # always bump target_link_libraries(llama PRIVATE ggml ${LLAMA_EXTRA_LIBS} + libnode.so +# libv8.so +# libv8_libbase.so +# libv8_libplatform.so +# libv8_libsampler.so ) if (BUILD_SHARED_LIBS) diff --git a/Makefile b/Makefile index 507d05851..0132a4fb4 100644 --- a/Makefile +++ b/Makefile @@ -587,8 +587,8 @@ clean: # Examples # -main: examples/main/main.cpp ggml.o llama.o $(COMMON_DEPS) console.o grammar-parser.o $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) +main: examples/main/main.cpp plugin_nodejs.o ggml.o llama.o $(COMMON_DEPS) console.o grammar-parser.o $(OBJS) + $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) /usr/lib/libnode.so @echo @echo '==== Run ./main -h for help. ====' @echo diff --git a/examples/main/CMakeLists.txt b/examples/main/CMakeLists.txt index 366dcf6ae..ed0eebb12 100644 --- a/examples/main/CMakeLists.txt +++ b/examples/main/CMakeLists.txt @@ -1,6 +1,6 @@ set(TARGET main) add_executable(${TARGET} main.cpp) install(TARGETS ${TARGET} RUNTIME) -target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT} ${Boost_LIBRARIES} ${PYTHON_LIBRARIES}) +target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT} ${Boost_LIBRARIES} ${PYTHON_LIBRARIES} libnode.so plugin_nodejs.o) target_compile_features(${TARGET} PRIVATE cxx_std_20) diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 354b8f9a0..44af3fe6d 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -32,7 +32,9 @@ #endif #include "print.hpp" -#include "plugin_python.hpp" +//#include "plugin_python.hpp" +#include "plugin_nodejs.hpp" +#define process_output_plugin process_output_plugin_node static llama_context ** g_ctx; static llama_model ** g_model; diff --git a/plugin_nodejs.cpp b/plugin_nodejs.cpp index c1a16231d..3a21c905a 100644 --- a/plugin_nodejs.cpp +++ b/plugin_nodejs.cpp @@ -1,209 +1,90 @@ -#ifdef NDEBUG -#undef NDEBUG -#endif -#include -#include "uv.h" -#include - -#include - -// Note: This file is being referred to from doc/api/embedding.md, and excerpts -// from it are included in the documentation. Try to keep these in sync. -// Snapshot support is not part of the embedder API docs yet due to its -// experimental nature, although it is of course documented in node.h. - -using node::CommonEnvironmentSetup; -using node::Environment; -using node::MultiIsolatePlatform; -using v8::Context; -using v8::HandleScope; -using v8::Isolate; -using v8::Locker; -using v8::MaybeLocal; -using v8::V8; -using v8::Value; - -static int RunNodeInstance(MultiIsolatePlatform* platform, - const std::vector& args, - const std::vector& exec_args); - -std::string process_output_plugin(const std::string start, +#include +#include +#include +#define NAPI_EXPERIMENTAL +#define NAPI_EMBEDDING +//#include +#include +#include +#include +std::string process_output_plugin_node(const std::string start, const std::string state, const std::string input) { - - //int main(int argc, char** argv) { - //argv = uv_setup_args(argc, argv); - std::vector args; - std::unique_ptr result = - node::InitializeOncePerProcess( - args, - {node::ProcessInitializationFlags::kNoInitializeV8, - node::ProcessInitializationFlags::kNoInitializeNodeV8Platform}); - for (const std::string& error : result->errors()) - fprintf(stderr, "%s: %s\n", args[0].c_str(), error.c_str()); - if (result->early_return() != 0) { - return "ERROR"; - } + // !!! All napi calls for one given environment must + // !!! be made from the same thread that created it + // (except everything napi_threadsafe_function related) - std::unique_ptr platform = - MultiIsolatePlatform::Create(4); - V8::InitializePlatform(platform.get()); - V8::Initialize(); + // This the V8 engine, there must be only one + napi_platform platform; + // This is a V8 isolate, there may be multiple + napi_env env; + // This holds local references, when it is closed + // they become available to the GC + napi_handle_scope scope; + // These are JS values + napi_value global; + napi_value key; + napi_value cb; + napi_value result; - int ret = - RunNodeInstance(platform.get(), result->args(), result->exec_args()); + const char *main_script = "console.log('hello world'); " + "function callMe() { console.log('called you'); }" + // or you can use vm.runInThisContext + "global.callMe = callMe;"; - V8::Dispose(); - V8::DisposePlatform(); + // Do only once + if (napi_create_platform(0, NULL, 0, NULL, NULL, 0, &platform) != napi_ok) { + fprintf(stderr, "Failed creating the platform\n"); + return "error"; + } - node::TearDownOncePerProcess(); - return "TODO"; -} - -int RunNodeInstance(MultiIsolatePlatform* platform, - const std::vector& args, - const std::vector& exec_args) { - int exit_code = 0; - - // Format of the arguments of this binary: - // Building snapshot: - // embedtest js_code_to_eval arg1 arg2... - // --embedder-snapshot-blob blob-path - // --embedder-snapshot-create - // [--embedder-snapshot-as-file] - // Running snapshot: - // embedtest --embedder-snapshot-blob blob-path - // [--embedder-snapshot-as-file] - // arg1 arg2... - // No snapshot: - // embedtest arg1 arg2... - node::EmbedderSnapshotData::Pointer snapshot; - - std::string binary_path = args[0]; - std::vector filtered_args; - bool is_building_snapshot = false; - bool snapshot_as_file = false; - std::string snapshot_blob_path; - for (size_t i = 0; i < args.size(); ++i) { - const std::string& arg = args[i]; - if (arg == "--embedder-snapshot-create") { - is_building_snapshot = true; - } else if (arg == "--embedder-snapshot-as-file") { - snapshot_as_file = true; - } else if (arg == "--embedder-snapshot-blob") { - assert(i + 1 < args.size()); - snapshot_blob_path = args[i + 1]; - i++; - } else { - filtered_args.push_back(arg); - } - } - - if (!snapshot_blob_path.empty() && !is_building_snapshot) { - FILE* fp = fopen(snapshot_blob_path.c_str(), "r"); - assert(fp != nullptr); - if (snapshot_as_file) { - snapshot = node::EmbedderSnapshotData::FromFile(fp); - } else { - uv_fs_t req = uv_fs_t(); - int statret = - uv_fs_stat(nullptr, &req, snapshot_blob_path.c_str(), nullptr); - assert(statret == 0); - size_t filesize = req.statbuf.st_size; - uv_fs_req_cleanup(&req); - - std::vector vec(filesize); - size_t read = fread(vec.data(), filesize, 1, fp); - assert(read == 1); - snapshot = node::EmbedderSnapshotData::FromBlob(vec); - } - assert(snapshot); - int ret = fclose(fp); - assert(ret == 0); - } - - if (is_building_snapshot) { - // It contains at least the binary path, the code to snapshot, - // and --embedder-snapshot-create (which is filtered, so at least - // 2 arguments should remain after filtering). - assert(filtered_args.size() >= 2); - // Insert an anonymous filename as process.argv[1]. - filtered_args.insert(filtered_args.begin() + 1, - node::GetAnonymousMainPath()); - } - - std::vector errors; - std::unique_ptr setup = - snapshot - ? CommonEnvironmentSetup::CreateFromSnapshot( - platform, &errors, snapshot.get(), filtered_args, exec_args) - : is_building_snapshot ? CommonEnvironmentSetup::CreateForSnapshotting( - platform, &errors, filtered_args, exec_args) - : CommonEnvironmentSetup::Create( - platform, &errors, filtered_args, exec_args); - if (!setup) { - for (const std::string& err : errors) - fprintf(stderr, "%s: %s\n", binary_path.c_str(), err.c_str()); - return 1; - } - - Isolate* isolate = setup->isolate(); - Environment* env = setup->env(); - - { - Locker locker(isolate); - Isolate::Scope isolate_scope(isolate); - HandleScope handle_scope(isolate); - Context::Scope context_scope(setup->context()); - - MaybeLocal loadenv_ret; - if (snapshot) { // Deserializing snapshot - loadenv_ret = node::LoadEnvironment(env, node::StartExecutionCallback{}); - } else if (is_building_snapshot) { - // Environment created for snapshotting must set process.argv[1] to - // the name of the main script, which was inserted above. - loadenv_ret = node::LoadEnvironment( - env, - "const assert = require('assert');" - "assert(require('v8').startupSnapshot.isBuildingSnapshot());" - "globalThis.embedVars = { nön_ascıı: '🏳️‍🌈' };" - "globalThis.require = require;" - "require('vm').runInThisContext(process.argv[2]);"); - } else { - loadenv_ret = node::LoadEnvironment( - env, - "const publicRequire = require('module').createRequire(process.cwd() " - "+ '/');" - "globalThis.require = publicRequire;" - "globalThis.embedVars = { nön_ascıı: '🏳️‍🌈' };" - "require('vm').runInThisContext(process.argv[1]);"); - } - - if (loadenv_ret.IsEmpty()) // There has been a JS exception. - return 1; - - exit_code = node::SpinEventLoop(env).FromMaybe(1); - } - - if (!snapshot_blob_path.empty() && is_building_snapshot) { - snapshot = setup->CreateSnapshot(); - assert(snapshot); - - FILE* fp = fopen(snapshot_blob_path.c_str(), "w"); - assert(fp != nullptr); - if (snapshot_as_file) { - snapshot->ToFile(fp); - } else { - const std::vector vec = snapshot->ToBlob(); - size_t written = fwrite(vec.data(), vec.size(), 1, fp); - assert(written == 1); - } - int ret = fclose(fp); - assert(ret == 0); - } - - node::Stop(env); - - return exit_code; + // Do for each environment (V8 isolate) + // 'hello world' will be printed here + if (napi_create_environment(platform, NULL, main_script, &env) != napi_ok) { + fprintf(stderr, "Failed running JS\n"); + return "error1"; + } + + // Here you can interact with the environment through Node-API env + // (refer to the Node-API doc) + if (napi_get_global(env, &global) != napi_ok) { + fprintf(stderr, "Failed accessing the global object\n"); + return "Failed accessing the global object"; + } + napi_create_string_utf8(env, "callMe", strlen("callMe"), &key); + if (napi_get_property(env, global, key, &cb) != napi_ok) { + fprintf(stderr, "Failed accessing the global object\n"); + return "Failed accessing the global object"; + } + + // This cycle can be repeated + { + // Call a JS function + // V8 will run in this thread + if (napi_call_function(env, global, cb, 0, NULL, &result) != napi_ok) { + fprintf(stderr, "Failed calling JS callback\n"); + return "Failed calling JS callback"; + } + // (optional) Call this to flush all pending async callbacks + // V8 will run in this thread + if (napi_run_environment(env) != napi_ok) { + fprintf(stderr, "Failed flushing pending JS callbacks\n"); + return "Failed flushing pending JS callbacks"; + } + } + + // Shutdown everyhing + napi_close_handle_scope(env, scope); + + if (napi_destroy_environment(env, NULL) != napi_ok) { + return "destroy"; + } + + if (napi_destroy_platform(platform) != napi_ok) { + fprintf(stderr, "Failed destroying the platform\n"); + return "Failed destroying the platform"; + } + + return "OK"; } diff --git a/plugin_nodejs.hpp b/plugin_nodejs.hpp index 57628d561..b8ee4b262 100644 --- a/plugin_nodejs.hpp +++ b/plugin_nodejs.hpp @@ -1,3 +1,3 @@ -std::string process_output_plugin(const std::string start, +std::string process_output_plugin_node(const std::string start, const std::string state, const std::string input);