vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

n_ctx change

Browse source
This commit is contained in:
pudepiedj 2024-02-29 12:40:53 +00:00
parent ee7f05b52b
commit a451708e90
3 changed files with 51 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

16
Llamaserver.py
View file

@ -96,7 +96,7 @@ if __name__ == "__main__":
global bar global bar
lockbar = threading.Lock() lockbar = threading.Lock()
url = "http://192.168.1.31:8080/completion" url = "http://192.168.1.28:8080/completion"
num_requests = 76 num_requests = 76
q = Queue(maxsize = 80) q = Queue(maxsize = 80)
@ -113,15 +113,23 @@ if __name__ == "__main__":
'Authorization': f'Bearer {api_key}' 'Authorization': f'Bearer {api_key}'
} }
writer_list = ["Plato", "Aristotle", "Thales of Miletus", "Heraclitus", "Socrates",
"The prophet Isaiah", "Jesus of Nazareth", "Plotinus", "Porphyry",
"Irenaeus", "Athanasius", "Augustine of Hippo", "Thomas Aquinas", "Anselm of Canterbury",
"Roget Bacon", "Fibonacci", "Duns Scotus", "William of Ockham", "Nicholas of Cusa",
"Erasmus", "Thomas More", "Luther", "Calvin", "Thomas Cranmer", "Shakespeare",
"Francis Bacon", "Thomas Cromwell", "Thomas Hobbes", "John Locke", "David Hume", "Berkeley", "Immanuel Kant",
"Jeremy Bentham", "William Blake", "John Stuart Mill", "Peirce", "Ralph Waldo Emerson", "Emily Dickinson", "Walt Whitman", "William James", "Henry James", "Henry Sidgwick", "John Dewey"]
country_list = ["France", "Germany", "China", "USA", "Italy", "India", country_list = ["France", "Germany", "China", "USA", "Italy", "India",
"Ukraine", "Japan", "Australia", "New Zealand", "Indonesia", "Nigeria", "Saudi Arabia", "Ukraine", "Japan", "Australia", "New Zealand", "Indonesia", "Nigeria", "Saudi Arabia",
"Israel", "Egypt", "Kenya", "Chile", "Mexico", "Canada", "Israel", "Egypt", "Kenya", "Chile", "Mexico", "Canada", "Ecuador", "Brazil", "Argentina", "Colombia",
"Bulgaria", "Romania", "Finland", "Sweden", "Norway", "Denmark", "Tanzania", "Israel", "Bulgaria", "Romania", "Finland", "Sweden", "Norway", "Denmark", "Tanzania", "Israel",
"Latvia", "Lithuania", "Estonia", "Pakistan", "Sri Lanka", "Malawi", "Mozambique"] "Latvia", "Lithuania", "Estonia", "Pakistan", "Sri Lanka", "Malawi", "Mozambique"]
for i in range(num_requests): for i in range(num_requests):
country = country_list[i % len(country_list)] writer = writer_list[i % len(writer_list)]
question = f"Tell me the political history of {country} up to 2018." question = f"Tell me about the writings of {writer}."
# NOTE: don't pass the parameter as a function call; pass in args # NOTE: don't pass the parameter as a function call; pass in args
print(f"Processing request {i} / {num_requests}: {question}\n") print(f"Processing request {i} / {num_requests}: {question}\n")
event = threading.Event() event = threading.Event()

12
examples/server/server.cpp
View file

@ -2245,6 +2245,7 @@ static void server_print_usage(const char *argv0, const gpt_params &params,
printf(" KV cache data type for V (default: f16)\n"); printf(" KV cache data type for V (default: f16)\n");
printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n"); printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n");
printf(" --log-format log output format: json or text (default: json)\n"); printf(" --log-format log output format: json or text (default: json)\n");
printf(" --stdout-log-file FNAME redirect stdout to a log file.\n");
printf(" --log-disable disables logging to a file.\n"); printf(" --log-disable disables logging to a file.\n");
printf(" --slots-endpoint-disable disables slots monitoring endpoint.\n"); printf(" --slots-endpoint-disable disables slots monitoring endpoint.\n");
printf(" --metrics enable prometheus compatible metrics endpoint (default: %s).\n", sparams.metrics_endpoint ? "enabled" : "disabled"); printf(" --metrics enable prometheus compatible metrics endpoint (default: %s).\n", sparams.metrics_endpoint ? "enabled" : "disabled");
@ -2515,11 +2516,13 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
{ {
llama.skvgraphics = true; // -skvg takes no parameter so we don't test ++i >= argc llama.skvgraphics = true; // -skvg takes no parameter so we don't test ++i >= argc
llama.skvinteract = false; llama.skvinteract = false;
log_settings.stdout_target = "/dev/null";
} }
else if (arg == "-skvi" || arg == "--show-interactive-graphics") else if (arg == "-skvi" || arg == "--show-interactive-graphics")
{ {
llama.skvgraphics = true; // -skvi takes no parameter so we don't test ++i >= argc llama.skvgraphics = true; // -skvi takes no parameter so we don't test ++i >= argc
llama.skvinteract = true; llama.skvinteract = true;
log_settings.stdout_target = "/dev/null";
} }
else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers")
{ {
@ -2759,6 +2762,15 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
break; break;
} }
} }
else if (arg == "--stdout-log-file")
{
if (++i >= argc)
{
invalid_param = true;
break;
}
log_settings.stdout_target = argv[i]; // have just noticed that there is a 'log_set_target()' below but says 'INTERNAL USE: DO NOT USE'
}
else if (arg == "--log-disable") else if (arg == "--log-disable")
{ {
log_set_target(stdout); log_set_target(stdout);

40
examples/server/utils.hpp
View file

@ -29,8 +29,8 @@ extern bool server_log_json;
if (server_verbose) \ if (server_verbose) \
{ \ { \
server_log("VERB", __func__, __LINE__, MSG, __VA_ARGS__, \ server_log("VERB", __func__, __LINE__, MSG, __VA_ARGS__, \
log_settings.stdout_target, log_settings.stdout_reset, \ log_settings.stdout_target, log_settings.stderr_target, \
log_settings.stderr_target, log_settings.stderr_reset); \ log_settings.stdout_reset, log_settings.stderr_reset); \
} \ } \
} while (0) // this is always false so the loop only compiles once but is treated as a single statement } while (0) // this is always false so the loop only compiles once but is treated as a single statement
#endif #endif
@ -39,28 +39,28 @@ extern bool server_log_json;
struct LogRedirection { struct LogRedirection {
// Set default values for redirection targets and reset strings // Set default values for redirection targets and reset strings
std::string stdout_target = "stdout"; std::string stdout_target = "stdout_log.log";
std::string stdout_reset = "stdout"; std::string stdout_reset = "/dev/stdout";
std::string stderr_target = "stderr_log.log"; std::string stderr_target = "stderr_log.log";
std::string stderr_reset = "stderr"; std::string stderr_reset = "/dev/stderr";
}; };
LogRedirection log_settings; LogRedirection log_settings;
#define LOG_ERROR(MSG, ...) \ #define LOG_ERROR(MSG, ...) \
server_log("ERR", __func__, __LINE__, MSG, __VA_ARGS__, \ server_log("ERR", __func__, __LINE__, MSG, __VA_ARGS__, \
log_settings.stdout_target, log_settings.stdout_reset, \ log_settings.stdout_target, log_settings.stderr_target, \
log_settings.stderr_target, log_settings.stderr_reset) log_settings.stdout_reset, log_settings.stderr_reset)
#define LOG_WARNING(MSG, ...) \ #define LOG_WARNING(MSG, ...) \
server_log("WARN", __func__, __LINE__, MSG, __VA_ARGS__, \ server_log("WARN", __func__, __LINE__, MSG, __VA_ARGS__, \
log_settings.stdout_target, log_settings.stdout_reset, \ log_settings.stdout_target, log_settings.stderr_target, \
log_settings.stderr_target, log_settings.stderr_reset) log_settings.stdout_reset, log_settings.stderr_reset)
#define LOG_INFO(MSG, ...) \ #define LOG_INFO(MSG, ...) \
server_log("INFO", __func__, __LINE__, MSG, __VA_ARGS__, \ server_log("INFO", __func__, __LINE__, MSG, __VA_ARGS__, \
log_settings.stdout_target, log_settings.stdout_reset, \ log_settings.stdout_target, log_settings.stderr_target, \
log_settings.stderr_target, log_settings.stderr_reset) log_settings.stdout_reset, log_settings.stderr_reset)
/* /*
// Example usage (WIP): // Example usage (WIP):
@ -214,8 +214,20 @@ static inline void server_log(
{"timestamp", time(nullptr)}, {"timestamp", time(nullptr)},
}; };
freopen(stdout_target.c_str(), "a", stdout); /*
freopen(stderr_target.c_str(), "a", stderr); // we assign stderr to dev/null effectively 'blackholing' the output because log.dump below is redirected too std::cerr << stdout_target.c_str() << std::endl;
FILE* new_stdout = freopen(stdout_target.c_str(), "a", stdout);
if (new_stdout == nullptr) {
std::cerr << "Error on redirecting stdout to " << stdout_target.c_str() << std::endl;
}
*/
std::cerr << stderr_target.c_str() << std::endl;
FILE* new_stderr = freopen(stderr_target.c_str(), "a", stderr);
if (new_stderr == nullptr) {
std::cerr << "Error on redirecting stderr to " << stderr_target.c_str() << std::endl;
}
//freopen(stderr_target.c_str(), "a", stderr); // we assign stderr to dev/null effectively 'blackholing' the output because log.dump below is redirected too
if (server_log_json) { if (server_log_json) {
log.merge_patch( log.merge_patch(
@ -250,8 +262,10 @@ static inline void server_log(
printf("\033[85;0H%.*s\n", (int)str.size(), str.data()); printf("\033[85;0H%.*s\n", (int)str.size(), str.data());
fflush(stderr); // was originally fflush(stdout) fflush(stderr); // was originally fflush(stdout)
/*
freopen(stdout_reset.c_str(), "a", stdout); // decide whether to restore stdout freopen(stdout_reset.c_str(), "a", stdout); // decide whether to restore stdout
freopen(stderr_reset.c_str(), "a", stderr); // decide whether to restore stderr (both need automating) freopen(stderr_reset.c_str(), "a", stderr); // decide whether to restore stderr (both need automating)
*/
} }
} }