From c2b55cc9171cbefdd0f5cb7c8cfc3d728245e61b Mon Sep 17 00:00:00 2001 From: Randall Fitzgerald Date: Thu, 25 May 2023 12:53:05 -0700 Subject: [PATCH] Added LoRA Loading Someone please test this. I have no LoRAs available to test. The code is direct from the base repo so it should be fine. --- examples/server/server.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 6cc07a955..d77d6b303 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -421,6 +421,8 @@ void server_print_usage(int /*argc*/, char **argv, const gpt_params ¶ms) fprintf(stderr, " number of layers to store in VRAM\n"); fprintf(stderr, " -m FNAME, --model FNAME\n"); fprintf(stderr, " model path (default: %s)\n", params.model.c_str()); + fprintf(stderr, " --lora FNAME apply LoRA adapter (implies --no-mmap)\n"); + fprintf(stderr, " --lora-base FNAME optional model to use as a base for the layers modified by the LoRA adapter\n"); fprintf(stderr, " -host ip address to listen (default 127.0.0.1)\n"); fprintf(stderr, " -port PORT port to listen (default 8080)\n"); fprintf(stderr, "\n"); @@ -505,6 +507,24 @@ bool server_params_parse(int argc, char **argv, server_params &sparams, gpt_para } params.n_gpu_layers = std::stoi(argv[i]); } + else if (arg == "--lora") + { + if (++i >= argc) + { + invalid_param = true; + break; + } + params.lora_adapter = argv[i]; + params.use_mmap = false; + } + else if (arg == "--lora-base") + { + if (++i >= argc) { + invalid_param = true; + break; + } + params.lora_base = argv[i]; + } else { fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());