vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

linking, loading, segfaulting

Browse source
This commit is contained in:
mike dupont 2023-12-08 13:01:59 -05:00
parent ac69c93ca9
commit 09a48ec2ae
6 changed files with 98 additions and 210 deletions
Download patch file Download diff file Expand all files Collapse all files

13
CMakeLists.txt
View file

@ -36,7 +36,7 @@ message(STATUS "Boost_LIBRARIES = ${Boost_LIBRARIES}")
ENABLE_TESTING() ENABLE_TESTING()
INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS}) INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS})
LINK_LIBRARIES(${Boost_LIBRARIES} ${Python3_LIBRARIES}) # Deprecated but so convenient! LINK_LIBRARIES(${Boost_LIBRARIES} ${Python3_LIBRARIES} ) # Deprecated but so convenient!
#PYTHON_ADD_MODULE(plugin_python plugin_python.cpp) #PYTHON_ADD_MODULE(plugin_python plugin_python.cpp)
Python3_add_library(plugin_python MODULE plugin_python.cpp) Python3_add_library(plugin_python MODULE plugin_python.cpp)
@ -749,8 +749,8 @@ add_library(ggml OBJECT
${GGML_SOURCES_EXTRA} ${GGML_HEADERS_EXTRA} ${GGML_SOURCES_EXTRA} ${GGML_HEADERS_EXTRA}
) )
target_include_directories(ggml PUBLIC . ${LLAMA_EXTRA_INCLUDES} "/usr/local/include/node/") target_include_directories(ggml PUBLIC "/usr/include/node/" . ${LLAMA_EXTRA_INCLUDES} )
target_compile_features(ggml PUBLIC c_std_11) # don't bump target_compile_features(ggml PUBLIC c_std_23) # always bump
target_link_libraries(ggml PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS}) target_link_libraries(ggml PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS})
if (GGML_USE_CPU_HBM) if (GGML_USE_CPU_HBM)
target_link_libraries(ggml PUBLIC memkind) target_link_libraries(ggml PUBLIC memkind)
@ -772,10 +772,15 @@ add_library(llama
) )
target_include_directories(llama PUBLIC .) target_include_directories(llama PUBLIC .)
target_compile_features(llama PUBLIC cxx_std_20) # don't bump target_compile_features(llama PUBLIC cxx_std_20) # always bump
target_link_libraries(llama PRIVATE target_link_libraries(llama PRIVATE
ggml ggml
${LLAMA_EXTRA_LIBS} ${LLAMA_EXTRA_LIBS}
libnode.so
# libv8.so
# libv8_libbase.so
# libv8_libplatform.so
# libv8_libsampler.so
) )
if (BUILD_SHARED_LIBS) if (BUILD_SHARED_LIBS)

4
Makefile
View file

@ -587,8 +587,8 @@ clean:
# Examples # Examples
# #
main: examples/main/main.cpp ggml.o llama.o $(COMMON_DEPS) console.o grammar-parser.o $(OBJS) main: examples/main/main.cpp plugin_nodejs.o ggml.o llama.o $(COMMON_DEPS) console.o grammar-parser.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) /usr/lib/libnode.so
@echo @echo
@echo '==== Run ./main -h for help. ====' @echo '==== Run ./main -h for help. ===='
@echo @echo

2
examples/main/CMakeLists.txt
View file

@ -1,6 +1,6 @@
set(TARGET main) set(TARGET main)
add_executable(${TARGET} main.cpp) add_executable(${TARGET} main.cpp)
install(TARGETS ${TARGET} RUNTIME) install(TARGETS ${TARGET} RUNTIME)
target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT} ${Boost_LIBRARIES} ${PYTHON_LIBRARIES}) target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT} ${Boost_LIBRARIES} ${PYTHON_LIBRARIES} libnode.so plugin_nodejs.o)
target_compile_features(${TARGET} PRIVATE cxx_std_20) target_compile_features(${TARGET} PRIVATE cxx_std_20)

4
examples/main/main.cpp
View file

@ -32,7 +32,9 @@
#endif #endif
#include "print.hpp" #include "print.hpp"
#include "plugin_python.hpp" //#include "plugin_python.hpp"
#include "plugin_nodejs.hpp"
#define process_output_plugin process_output_plugin_node
static llama_context ** g_ctx; static llama_context ** g_ctx;
static llama_model ** g_model; static llama_model ** g_model;

283
plugin_nodejs.cpp
View file

@ -1,209 +1,90 @@
#ifdef NDEBUG #include <stdio.h>
#undef NDEBUG #include <string>
#endif #include <string.h>
#include <node/node.h> #define NAPI_EXPERIMENTAL
#include "uv.h" #define NAPI_EMBEDDING
#include <assert.h> //#include <node/node_api.h>
#include <libnode/node_api.h>
#include <algorithm> #include <libnode/js_native_api.h>
#include <libnode/js_native_api_types.h>
// Note: This file is being referred to from doc/api/embedding.md, and excerpts std::string process_output_plugin_node(const std::string start,
// from it are included in the documentation. Try to keep these in sync.
// Snapshot support is not part of the embedder API docs yet due to its
// experimental nature, although it is of course documented in node.h.
using node::CommonEnvironmentSetup;
using node::Environment;
using node::MultiIsolatePlatform;
using v8::Context;
using v8::HandleScope;
using v8::Isolate;
using v8::Locker;
using v8::MaybeLocal;
using v8::V8;
using v8::Value;
static int RunNodeInstance(MultiIsolatePlatform* platform,
const std::vector<std::string>& args,
const std::vector<std::string>& exec_args);
std::string process_output_plugin(const std::string start,
const std::string state, const std::string state,
const std::string input) { const std::string input) {
//int main(int argc, char** argv) {
//argv = uv_setup_args(argc, argv);
std::vector<std::string> args;
std::unique_ptr<node::InitializationResult> result =
node::InitializeOncePerProcess(
args,
{node::ProcessInitializationFlags::kNoInitializeV8,
node::ProcessInitializationFlags::kNoInitializeNodeV8Platform});
for (const std::string& error : result->errors()) // !!! All napi calls for one given environment must
fprintf(stderr, "%s: %s\n", args[0].c_str(), error.c_str()); // !!! be made from the same thread that created it
if (result->early_return() != 0) { // (except everything napi_threadsafe_function related)
return "ERROR";
}
std::unique_ptr<MultiIsolatePlatform> platform = // This the V8 engine, there must be only one
MultiIsolatePlatform::Create(4); napi_platform platform;
V8::InitializePlatform(platform.get()); // This is a V8 isolate, there may be multiple
V8::Initialize(); napi_env env;
// This holds local references, when it is closed
// they become available to the GC
napi_handle_scope scope;
// These are JS values
napi_value global;
napi_value key;
napi_value cb;
napi_value result;
int ret = const char *main_script = "console.log('hello world'); "
RunNodeInstance(platform.get(), result->args(), result->exec_args()); "function callMe() { console.log('called you'); }"
// or you can use vm.runInThisContext
"global.callMe = callMe;";
V8::Dispose(); // Do only once
V8::DisposePlatform(); if (napi_create_platform(0, NULL, 0, NULL, NULL, 0, &platform) != napi_ok) {
fprintf(stderr, "Failed creating the platform\n");
return "error";
}
node::TearDownOncePerProcess(); // Do for each environment (V8 isolate)
return "TODO"; // 'hello world' will be printed here
} if (napi_create_environment(platform, NULL, main_script, &env) != napi_ok) {
fprintf(stderr, "Failed running JS\n");
int RunNodeInstance(MultiIsolatePlatform* platform, return "error1";
const std::vector<std::string>& args, }
const std::vector<std::string>& exec_args) {
int exit_code = 0; // Here you can interact with the environment through Node-API env
// (refer to the Node-API doc)
// Format of the arguments of this binary: if (napi_get_global(env, &global) != napi_ok) {
// Building snapshot: fprintf(stderr, "Failed accessing the global object\n");
// embedtest js_code_to_eval arg1 arg2... return "Failed accessing the global object";
// --embedder-snapshot-blob blob-path }
// --embedder-snapshot-create napi_create_string_utf8(env, "callMe", strlen("callMe"), &key);
// [--embedder-snapshot-as-file] if (napi_get_property(env, global, key, &cb) != napi_ok) {
// Running snapshot: fprintf(stderr, "Failed accessing the global object\n");
// embedtest --embedder-snapshot-blob blob-path return "Failed accessing the global object";
// [--embedder-snapshot-as-file] }
// arg1 arg2...
// No snapshot: // This cycle can be repeated
// embedtest arg1 arg2... {
node::EmbedderSnapshotData::Pointer snapshot; // Call a JS function
// V8 will run in this thread
std::string binary_path = args[0]; if (napi_call_function(env, global, cb, 0, NULL, &result) != napi_ok) {
std::vector<std::string> filtered_args; fprintf(stderr, "Failed calling JS callback\n");
bool is_building_snapshot = false; return "Failed calling JS callback";
bool snapshot_as_file = false; }
std::string snapshot_blob_path; // (optional) Call this to flush all pending async callbacks
for (size_t i = 0; i < args.size(); ++i) { // V8 will run in this thread
const std::string& arg = args[i]; if (napi_run_environment(env) != napi_ok) {
if (arg == "--embedder-snapshot-create") { fprintf(stderr, "Failed flushing pending JS callbacks\n");
is_building_snapshot = true; return "Failed flushing pending JS callbacks";
} else if (arg == "--embedder-snapshot-as-file") { }
snapshot_as_file = true; }
} else if (arg == "--embedder-snapshot-blob") {
assert(i + 1 < args.size()); // Shutdown everyhing
snapshot_blob_path = args[i + 1]; napi_close_handle_scope(env, scope);
i++;
} else { if (napi_destroy_environment(env, NULL) != napi_ok) {
filtered_args.push_back(arg); return "destroy";
} }
}
if (napi_destroy_platform(platform) != napi_ok) {
if (!snapshot_blob_path.empty() && !is_building_snapshot) { fprintf(stderr, "Failed destroying the platform\n");
FILE* fp = fopen(snapshot_blob_path.c_str(), "r"); return "Failed destroying the platform";
assert(fp != nullptr); }
if (snapshot_as_file) {
snapshot = node::EmbedderSnapshotData::FromFile(fp); return "OK";
} else {
uv_fs_t req = uv_fs_t();
int statret =
uv_fs_stat(nullptr, &req, snapshot_blob_path.c_str(), nullptr);
assert(statret == 0);
size_t filesize = req.statbuf.st_size;
uv_fs_req_cleanup(&req);
std::vector<char> vec(filesize);
size_t read = fread(vec.data(), filesize, 1, fp);
assert(read == 1);
snapshot = node::EmbedderSnapshotData::FromBlob(vec);
}
assert(snapshot);
int ret = fclose(fp);
assert(ret == 0);
}
if (is_building_snapshot) {
// It contains at least the binary path, the code to snapshot,
// and --embedder-snapshot-create (which is filtered, so at least
// 2 arguments should remain after filtering).
assert(filtered_args.size() >= 2);
// Insert an anonymous filename as process.argv[1].
filtered_args.insert(filtered_args.begin() + 1,
node::GetAnonymousMainPath());
}
std::vector<std::string> errors;
std::unique_ptr<CommonEnvironmentSetup> setup =
snapshot
? CommonEnvironmentSetup::CreateFromSnapshot(
platform, &errors, snapshot.get(), filtered_args, exec_args)
: is_building_snapshot ? CommonEnvironmentSetup::CreateForSnapshotting(
platform, &errors, filtered_args, exec_args)
: CommonEnvironmentSetup::Create(
platform, &errors, filtered_args, exec_args);
if (!setup) {
for (const std::string& err : errors)
fprintf(stderr, "%s: %s\n", binary_path.c_str(), err.c_str());
return 1;
}
Isolate* isolate = setup->isolate();
Environment* env = setup->env();
{
Locker locker(isolate);
Isolate::Scope isolate_scope(isolate);
HandleScope handle_scope(isolate);
Context::Scope context_scope(setup->context());
MaybeLocal<Value> loadenv_ret;
if (snapshot) { // Deserializing snapshot
loadenv_ret = node::LoadEnvironment(env, node::StartExecutionCallback{});
} else if (is_building_snapshot) {
// Environment created for snapshotting must set process.argv[1] to
// the name of the main script, which was inserted above.
loadenv_ret = node::LoadEnvironment(
env,
"const assert = require('assert');"
"assert(require('v8').startupSnapshot.isBuildingSnapshot());"
"globalThis.embedVars = { nön_ascıı: '🏳️‍🌈' };"
"globalThis.require = require;"
"require('vm').runInThisContext(process.argv[2]);");
} else {
loadenv_ret = node::LoadEnvironment(
env,
"const publicRequire = require('module').createRequire(process.cwd() "
"+ '/');"
"globalThis.require = publicRequire;"
"globalThis.embedVars = { nön_ascıı: '🏳️‍🌈' };"
"require('vm').runInThisContext(process.argv[1]);");
}
if (loadenv_ret.IsEmpty()) // There has been a JS exception.
return 1;
exit_code = node::SpinEventLoop(env).FromMaybe(1);
}
if (!snapshot_blob_path.empty() && is_building_snapshot) {
snapshot = setup->CreateSnapshot();
assert(snapshot);
FILE* fp = fopen(snapshot_blob_path.c_str(), "w");
assert(fp != nullptr);
if (snapshot_as_file) {
snapshot->ToFile(fp);
} else {
const std::vector<char> vec = snapshot->ToBlob();
size_t written = fwrite(vec.data(), vec.size(), 1, fp);
assert(written == 1);
}
int ret = fclose(fp);
assert(ret == 0);
}
node::Stop(env);
return exit_code;
} }

2
plugin_nodejs.hpp
View file

@ -1,3 +1,3 @@
std::string process_output_plugin(const std::string start, std::string process_output_plugin_node(const std::string start,
const std::string state, const std::string state,
const std::string input); const std::string input);