linking

CMakeLists.txt

@@ -733,6 +733,7 @@ add_library(ggml OBJECT
             ggml.h
 	    print.hpp
 	    plugin_python.cpp
+	    plugin_nodejs.cpp
 	    ggml-internal.hpp
 	    llama-internal.hpp
             ggml-alloc.cpp
@@ -748,7 +749,7 @@ add_library(ggml OBJECT
             ${GGML_SOURCES_EXTRA} ${GGML_HEADERS_EXTRA}
             )
 
-target_include_directories(ggml PUBLIC . ${LLAMA_EXTRA_INCLUDES})
+target_include_directories(ggml PUBLIC . ${LLAMA_EXTRA_INCLUDES} "/usr/local/include/node/")
 target_compile_features(ggml PUBLIC c_std_11) # don't bump
 target_link_libraries(ggml PUBLIC Threads::Threads ${LLAMA_EXTRA_LIBS})
 if (GGML_USE_CPU_HBM)

Makefile

@@ -750,3 +750,7 @@ tests/test-rope: tests/test-rope.cpp ggml.o $(OBJS)
 
 tests/test-c.o: tests/test-c.cpp llama.h
 	$(CC) $(CXXFLAGS) -c $(filter-out %.h,$^) -o $@
+
+
+test123:
+	./build2/bin/main -m ~/.ollama/models/mistral --interactive -r STOP -p 'write simple python expression to be evaluated ending WITH TOKEN <GO>' -n -1

embedding.py

@@ -1,6 +1,31 @@
-print("hello llama.cpp, got input:\n" + llm_input  + "\n")
+#for x in list(globals()):
+#    print("GLOBAL",x,globals()[x],"\n")
+# any global variables set here will be available later as well!
 
-if len(llm_input) > 20:
-    llm_output = "Reinterpret with emojis " + llm_input + "?\nSTOP";
-else:
-    llm_output =  llm_input
+#print("debug input:\n" + llm_input  + "\n")
+#foobar ="test"
+#if llm_state in ("params", statement, antiprompt,)
+
+def entrypoint():
+    global llm_output
+    global llm_input
+    global llm_state
+    llm_output = llm_input
+    if llm_state == "antiprompt":
+        #used to check each token if you want to stop early
+        return
+    elif llm_state == "params":
+        # first time it is called it returns the state via llm_output that will be used 
+        return 
+    elif llm_state == "statement":
+        if "<GO>" in llm_input:
+            llm_input = llm_input.replace("<GO>","")
+            try:
+                v= eval(llm_input)
+                llm_output = "Check that the evaluation of```" + llm_input + "``` Produced:"+ str(v) + " STOP";
+            except Exception as e:
+                #print(e)
+                llm_output = "generate a simple python expression to be evaluated. to evaluate your work emit the word <GO> and the python code will be evaluated.  Please correct the python error in Evaluation of ```" + llm_input + "``` Produced Output:"+ str(e) + "now consider the original task"+ llm_start + " STOP"
+                
+if __name__ == "__main__":
+    entrypoint()

examples/main/main.cpp

@@ -178,9 +178,11 @@ int main(int argc, char ** argv) {
 
     std::mt19937 rng(params.seed);
     if (params.random_prompt) {
-        params.prompt = gpt_random_prompt(rng);
+      params.prompt = gpt_random_prompt(rng);
     }
 
+    auto start_prompt = process_output_plugin(params.prompt,"params",params.prompt);
+
     LOG("%s: llama backend init\n", __func__);
     llama_backend_init(params.numa);
 
@@ -606,7 +608,7 @@ int main(int argc, char ** argv) {
                     int n_eval = std::min(input_size - i, params.n_batch);
                     if (llama_decode(ctx_guidance, llama_batch_get_one(input_buf + i, n_eval, n_past_guidance, 0))) {
                         LOG_TEE("%s : failed to eval\n", __func__);
-                        return 1;
+                        //return 1;
                     }
 
                     n_past_guidance += n_eval;
@@ -685,7 +687,7 @@ int main(int argc, char ** argv) {
         if (input_echo) {
             for (auto id : embd) {
                 const std::string token_str = llama_token_to_piece(ctx, id);
-                printf("TOKEN:%s\n", token_str.c_str());
+                printf("\nTOKEN:%s\n", token_str.c_str());
 
 		//print_fields(id);
 		
@@ -706,8 +708,8 @@ int main(int argc, char ** argv) {
 	// just print the whole thing       	
 	const std::string last_output1 = output_ss.str();
 	printf("%s",last_output1.c_str());
-	last_output = process_output_plugin(last_output1);
-	printf("%s",last_output.c_str());
+	last_output = process_output_plugin(start_prompt,"statement",last_output1);
+	printf("\nLASTOUTPUT: '%s'\n",last_output.c_str());
 		    
         // if not currently processing queued inputs;
         if ((int) embd_inp.size() <= n_consumed) {
@@ -716,7 +718,7 @@ int main(int argc, char ** argv) {
                 const int n_prev = 32;
                 const std::string last_output1 = llama_sampling_prev_str(ctx_sampling, ctx, n_prev);
 		// now plugin the python :
-		const std::string partial_output = process_output_plugin(last_output1);
+		const std::string partial_output = process_output_plugin(start_prompt,"antiprompt",last_output1);
 
                 is_antiprompt = false;
                 // Check if each of the reverse prompts appears at the end of the output.
@@ -786,6 +788,12 @@ int main(int argc, char ** argv) {
                 //bool another_line = true;
                 //do {
 		//  another_line = console::readline(line, params.multiline_input);
+
+		for (const auto & antiprompt : params.antiprompt) {
+		  size_t found_pos = last_output.find(antiprompt);
+		  if (found_pos != string::npos) {
+		  last_output.erase(found_pos,found_pos+ antiprompt.length());		  }
+		}
 		buffer += last_output;
 		//} while (another_line);
 

plugin_nodejs.cpp

@@ -0,0 +1,209 @@
+#ifdef NDEBUG
+#undef NDEBUG
+#endif
+#include <node/node.h>
+#include "uv.h"
+#include <assert.h>
+
+#include <algorithm>
+
+// Note: This file is being referred to from doc/api/embedding.md, and excerpts
+// from it are included in the documentation. Try to keep these in sync.
+// Snapshot support is not part of the embedder API docs yet due to its
+// experimental nature, although it is of course documented in node.h.
+
+using node::CommonEnvironmentSetup;
+using node::Environment;
+using node::MultiIsolatePlatform;
+using v8::Context;
+using v8::HandleScope;
+using v8::Isolate;
+using v8::Locker;
+using v8::MaybeLocal;
+using v8::V8;
+using v8::Value;
+
+static int RunNodeInstance(MultiIsolatePlatform* platform,
+                           const std::vector<std::string>& args,
+                           const std::vector<std::string>& exec_args);
+
+std::string process_output_plugin(const std::string start,
+				  const std::string state,
+				  const std::string input) {
+  
+  //int main(int argc, char** argv) {
+  //argv = uv_setup_args(argc, argv);
+  std::vector<std::string> args;
+  std::unique_ptr<node::InitializationResult> result =
+      node::InitializeOncePerProcess(
+          args,
+          {node::ProcessInitializationFlags::kNoInitializeV8,
+           node::ProcessInitializationFlags::kNoInitializeNodeV8Platform});
+
+  for (const std::string& error : result->errors())
+    fprintf(stderr, "%s: %s\n", args[0].c_str(), error.c_str());
+  if (result->early_return() != 0) {
+    return "ERROR";
+  }
+
+  std::unique_ptr<MultiIsolatePlatform> platform =
+      MultiIsolatePlatform::Create(4);
+  V8::InitializePlatform(platform.get());
+  V8::Initialize();
+
+  int ret =
+      RunNodeInstance(platform.get(), result->args(), result->exec_args());
+
+  V8::Dispose();
+  V8::DisposePlatform();
+
+  node::TearDownOncePerProcess();
+  return "TODO";
+}
+
+int RunNodeInstance(MultiIsolatePlatform* platform,
+                    const std::vector<std::string>& args,
+                    const std::vector<std::string>& exec_args) {
+  int exit_code = 0;
+
+  // Format of the arguments of this binary:
+  // Building snapshot:
+  // embedtest js_code_to_eval arg1 arg2...
+  //           --embedder-snapshot-blob blob-path
+  //           --embedder-snapshot-create
+  //           [--embedder-snapshot-as-file]
+  // Running snapshot:
+  // embedtest --embedder-snapshot-blob blob-path
+  //           [--embedder-snapshot-as-file]
+  //           arg1 arg2...
+  // No snapshot:
+  // embedtest arg1 arg2...
+  node::EmbedderSnapshotData::Pointer snapshot;
+
+  std::string binary_path = args[0];
+  std::vector<std::string> filtered_args;
+  bool is_building_snapshot = false;
+  bool snapshot_as_file = false;
+  std::string snapshot_blob_path;
+  for (size_t i = 0; i < args.size(); ++i) {
+    const std::string& arg = args[i];
+    if (arg == "--embedder-snapshot-create") {
+      is_building_snapshot = true;
+    } else if (arg == "--embedder-snapshot-as-file") {
+      snapshot_as_file = true;
+    } else if (arg == "--embedder-snapshot-blob") {
+      assert(i + 1 < args.size());
+      snapshot_blob_path = args[i + 1];
+      i++;
+    } else {
+      filtered_args.push_back(arg);
+    }
+  }
+
+  if (!snapshot_blob_path.empty() && !is_building_snapshot) {
+    FILE* fp = fopen(snapshot_blob_path.c_str(), "r");
+    assert(fp != nullptr);
+    if (snapshot_as_file) {
+      snapshot = node::EmbedderSnapshotData::FromFile(fp);
+    } else {
+      uv_fs_t req = uv_fs_t();
+      int statret =
+          uv_fs_stat(nullptr, &req, snapshot_blob_path.c_str(), nullptr);
+      assert(statret == 0);
+      size_t filesize = req.statbuf.st_size;
+      uv_fs_req_cleanup(&req);
+
+      std::vector<char> vec(filesize);
+      size_t read = fread(vec.data(), filesize, 1, fp);
+      assert(read == 1);
+      snapshot = node::EmbedderSnapshotData::FromBlob(vec);
+    }
+    assert(snapshot);
+    int ret = fclose(fp);
+    assert(ret == 0);
+  }
+
+  if (is_building_snapshot) {
+    // It contains at least the binary path, the code to snapshot,
+    // and --embedder-snapshot-create (which is filtered, so at least
+    // 2 arguments should remain after filtering).
+    assert(filtered_args.size() >= 2);
+    // Insert an anonymous filename as process.argv[1].
+    filtered_args.insert(filtered_args.begin() + 1,
+                         node::GetAnonymousMainPath());
+  }
+
+  std::vector<std::string> errors;
+  std::unique_ptr<CommonEnvironmentSetup> setup =
+      snapshot
+          ? CommonEnvironmentSetup::CreateFromSnapshot(
+                platform, &errors, snapshot.get(), filtered_args, exec_args)
+      : is_building_snapshot ? CommonEnvironmentSetup::CreateForSnapshotting(
+                                   platform, &errors, filtered_args, exec_args)
+                             : CommonEnvironmentSetup::Create(
+                                   platform, &errors, filtered_args, exec_args);
+  if (!setup) {
+    for (const std::string& err : errors)
+      fprintf(stderr, "%s: %s\n", binary_path.c_str(), err.c_str());
+    return 1;
+  }
+
+  Isolate* isolate = setup->isolate();
+  Environment* env = setup->env();
+
+  {
+    Locker locker(isolate);
+    Isolate::Scope isolate_scope(isolate);
+    HandleScope handle_scope(isolate);
+    Context::Scope context_scope(setup->context());
+
+    MaybeLocal<Value> loadenv_ret;
+    if (snapshot) {  // Deserializing snapshot
+      loadenv_ret = node::LoadEnvironment(env, node::StartExecutionCallback{});
+    } else if (is_building_snapshot) {
+      // Environment created for snapshotting must set process.argv[1] to
+      // the name of the main script, which was inserted above.
+      loadenv_ret = node::LoadEnvironment(
+          env,
+          "const assert = require('assert');"
+          "assert(require('v8').startupSnapshot.isBuildingSnapshot());"
+          "globalThis.embedVars = { nön_ascıı: '🏳️‍🌈' };"
+          "globalThis.require = require;"
+          "require('vm').runInThisContext(process.argv[2]);");
+    } else {
+      loadenv_ret = node::LoadEnvironment(
+          env,
+          "const publicRequire = require('module').createRequire(process.cwd() "
+          "+ '/');"
+          "globalThis.require = publicRequire;"
+          "globalThis.embedVars = { nön_ascıı: '🏳️‍🌈' };"
+          "require('vm').runInThisContext(process.argv[1]);");
+    }
+
+    if (loadenv_ret.IsEmpty())  // There has been a JS exception.
+      return 1;
+
+    exit_code = node::SpinEventLoop(env).FromMaybe(1);
+  }
+
+  if (!snapshot_blob_path.empty() && is_building_snapshot) {
+    snapshot = setup->CreateSnapshot();
+    assert(snapshot);
+
+    FILE* fp = fopen(snapshot_blob_path.c_str(), "w");
+    assert(fp != nullptr);
+    if (snapshot_as_file) {
+      snapshot->ToFile(fp);
+    } else {
+      const std::vector<char> vec = snapshot->ToBlob();
+      size_t written = fwrite(vec.data(), vec.size(), 1, fp);
+      assert(written == 1);
+    }
+    int ret = fclose(fp);
+    assert(ret == 0);
+  }
+
+  node::Stop(env);
+
+  return exit_code;
+}

plugin_nodejs.hpp

@@ -0,0 +1,3 @@
+std::string process_output_plugin(const std::string start,
+					 const std::string state,
+					 const std::string input);

plugin_python.cpp

@@ -26,7 +26,9 @@ using namespace boost::python;
 #endif
 
 
-std::string process_output_plugin(const std::string input)
+std::string process_output_plugin(const std::string start,
+				  const std::string state,
+				  const std::string input)
 {
     try {
         PyImport_AppendInittab((char*)"mymodule", INIT_MODULE);
@@ -36,7 +38,9 @@ std::string process_output_plugin(const std::string input)
         object mymodule = import("mymodule");
 
         main_namespace["precreated_object"] = Base("created on C++ side");
-	main_namespace["llm_input"] = input;       
+	main_namespace["llm_input"] = input;
+	main_namespace["llm_state"] = state;
+	main_namespace["llm_start"] = start;       
         exec_file("embedding.py", main_namespace, main_namespace);
 
 	boost::python::object llm_output = main_namespace["llm_output"];

plugin_python.hpp

@@ -1 +1,3 @@
-std::string process_output_plugin(const std::string input);
+std::string process_output_plugin(const std::string start,
+				  const std::string state,
+				  const std::string input);