vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

hmm...

Browse source
This commit is contained in:
trollkotze 2024-03-25 02:07:54 +01:00
parent 0274e6b364
commit 7dbed974dc
1 changed files with 28 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

51
examples/server/server.cpp
View file

@ -3176,10 +3176,10 @@ int main(int argc, char ** argv) {
res.status = 200; // HTTP OK res.status = 200; // HTTP OK
}; };
const auto handle_get_control_vectors = [&ctx_server, &params](const httplib::Request & req, httplib::Response & res) { const auto handle_get_control_vectors = [&ctx_server](const httplib::Request & req, httplib::Response & res) {
json vectors = json::array(); json vectors = json::array();
for (const auto & vec : params.control_vectors) { for (const auto & vec : ctx_server.params.control_vectors) {
vectors.push_back(json { vectors.push_back(json {
{ "fname", vec.fname }, { "fname", vec.fname },
{ "strength", vec.strength } { "strength", vec.strength }
@ -3187,13 +3187,13 @@ int main(int argc, char ** argv) {
} }
json data = { json data = {
{ "vectors", vectors }, { "vectors", vectors },
{ "layer_start", params.control_vector_layer_start }, { "layer_start", ctx_server.params.control_vector_layer_start },
{ "layer_end", params.control_vector_layer_end } { "layer_end", ctx_server.params.control_vector_layer_end }
}; };
res.set_content(data.dump(), "application/json; charset=utf-8"); res.set_content(data.dump(), "application/json; charset=utf-8");
}; };
const auto handle_set_control_vectors = [&ctx_server, &res_error, &params, &handle_get_control_vectors](const httplib::Request & req, httplib::Response & res) { const auto handle_set_control_vectors = [&ctx_server, &res_error, &handle_get_control_vectors](const httplib::Request & req, httplib::Response & res) {
res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin")); res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
json data = json::parse(req.body); json data = json::parse(req.body);
@ -3202,7 +3202,7 @@ int main(int argc, char ** argv) {
if (data.contains("vectors") && data["vectors"].is_array()) { if (data.contains("vectors") && data["vectors"].is_array()) {
for (const auto &item : data["vectors"]) { for (const auto &item : data["vectors"]) {
auto v = item.get<llama_control_vector_load_info>(); auto v = item.get<llama_control_vector_load_info>();
// std::cout << "Add vector: " << v.fname << " " << v.strength << "\n"; std::cout << "Add vector: " << v.fname << " " << v.strength << "\n";
vec_params.push_back(v); vec_params.push_back(v);
} }
} else { } else {
@ -3210,44 +3210,47 @@ int main(int argc, char ** argv) {
res_error(res, format_error_response("No vectors passed", ERROR_TYPE_SERVER)); res_error(res, format_error_response("No vectors passed", ERROR_TYPE_SERVER));
return; return;
} }
for (auto v : params.control_vectors) {
// std::cout << "Subtract vector:" << v.fname << " " << v.strength << "\n";
vec_params.push_back({ -v.strength, v.fname });
}
const auto cvec = llama_control_vector_load(vec_params); const auto cvec = llama_control_vector_load(vec_params);
if (cvec.n_embd == -1) { if (cvec.n_embd == -1) {
// std::cerr << "Could not load control vector\n"; std::cerr << "Could not load control vector\n";
res_error(res, format_error_response("Could not load control vector", ERROR_TYPE_SERVER)); res_error(res, format_error_response("Could not load control vector", ERROR_TYPE_SERVER));
return; return;
} }
if (params.control_vector_layer_start <= 0) { if (ctx_server.params.control_vector_layer_start <= 0) {
params.control_vector_layer_start = 1; ctx_server.params.control_vector_layer_start = 1;
} }
if (params.control_vector_layer_end <= 0){ if (ctx_server.params.control_vector_layer_end <= 0){
params.control_vector_layer_end = llama_n_layer(ctx_server.model); ctx_server.params.control_vector_layer_end = llama_n_layer(ctx_server.model);
} }
int err = llama_control_vector_apply(ctx_server.ctx, int err = llama_control_vector_apply(ctx_server.ctx,
cvec.data.data(), cvec.data.data(),
cvec.data.size(), cvec.data.size(),
cvec.n_embd, cvec.n_embd,
params.control_vector_layer_start, ctx_server.params.control_vector_layer_start,
params.control_vector_layer_end); ctx_server.params.control_vector_layer_end);
if (err) { if (err) {
std::cerr << "Could not apply control vector\n"; std::cerr << "Could not apply control vector\n";
res_error(res, format_error_response("Could not apply control vector", ERROR_TYPE_SERVER)); res_error(res, format_error_response("Could not apply control vector", ERROR_TYPE_SERVER));
return; return;
} }
auto s = params.control_vectors.size(); ctx_server.params.control_vectors.clear();
auto s2 = vec_params.size();
params.control_vectors.clear();
unsigned i = 0;
for (auto v : vec_params) { for (auto v : vec_params) {
if (i++ < s2 - s) {
//std::cout << "set vector param: " << v.fname << " " << v.strength << "\n"; //std::cout << "set vector param: " << v.fname << " " << v.strength << "\n";
params.control_vectors.push_back(v); ctx_server.params.control_vectors.push_back(v);
}
} }
/*std::cerr << "Maybe we need to do this initiation ritual before it werks?\n"; // No, it's still all garbled bullshit.
std::vector<llama_token> tmp = { llama_token_bos(ctx_server.model), llama_token_eos(ctx_server.model), };
std::cerr << "decode, bro\n";
llama_decode(ctx_server.ctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) ctx_server.params.n_batch), 0, 0));
std::cerr << "clear that fucking cache\n";
llama_kv_cache_clear(ctx_server.ctx);
std::cerr << "symcr0nice or what\n";
llama_synchronize(ctx_server.ctx);
std::cerr << "time will tell\n";
llama_reset_timings(ctx_server.ctx);*/
handle_get_control_vectors(req, res); handle_get_control_vectors(req, res);
}; };