diff --git a/Llamaserver.py b/Llamaserver.py index e74330a27..0314336d2 100644 --- a/Llamaserver.py +++ b/Llamaserver.py @@ -96,7 +96,7 @@ if __name__ == "__main__": global bar lockbar = threading.Lock() - url = "http://192.168.1.31:8080/completion" + url = "http://192.168.1.28:8080/completion" num_requests = 76 q = Queue(maxsize = 80) @@ -113,15 +113,23 @@ if __name__ == "__main__": 'Authorization': f'Bearer {api_key}' } + writer_list = ["Plato", "Aristotle", "Thales of Miletus", "Heraclitus", "Socrates", + "The prophet Isaiah", "Jesus of Nazareth", "Plotinus", "Porphyry", + "Irenaeus", "Athanasius", "Augustine of Hippo", "Thomas Aquinas", "Anselm of Canterbury", + "Roget Bacon", "Fibonacci", "Duns Scotus", "William of Ockham", "Nicholas of Cusa", + "Erasmus", "Thomas More", "Luther", "Calvin", "Thomas Cranmer", "Shakespeare", + "Francis Bacon", "Thomas Cromwell", "Thomas Hobbes", "John Locke", "David Hume", "Berkeley", "Immanuel Kant", + "Jeremy Bentham", "William Blake", "John Stuart Mill", "Peirce", "Ralph Waldo Emerson", "Emily Dickinson", "Walt Whitman", "William James", "Henry James", "Henry Sidgwick", "John Dewey"] + country_list = ["France", "Germany", "China", "USA", "Italy", "India", "Ukraine", "Japan", "Australia", "New Zealand", "Indonesia", "Nigeria", "Saudi Arabia", - "Israel", "Egypt", "Kenya", "Chile", "Mexico", "Canada", + "Israel", "Egypt", "Kenya", "Chile", "Mexico", "Canada", "Ecuador", "Brazil", "Argentina", "Colombia", "Bulgaria", "Romania", "Finland", "Sweden", "Norway", "Denmark", "Tanzania", "Israel", "Latvia", "Lithuania", "Estonia", "Pakistan", "Sri Lanka", "Malawi", "Mozambique"] for i in range(num_requests): - country = country_list[i % len(country_list)] - question = f"Tell me the political history of {country} up to 2018." + writer = writer_list[i % len(writer_list)] + question = f"Tell me about the writings of {writer}." # NOTE: don't pass the parameter as a function call; pass in args print(f"Processing request {i} / {num_requests}: {question}\n") event = threading.Event() diff --git a/examples/server/server.cpp b/examples/server/server.cpp index be736efb6..055654e02 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2245,6 +2245,7 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms, printf(" KV cache data type for V (default: f16)\n"); printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n"); printf(" --log-format log output format: json or text (default: json)\n"); + printf(" --stdout-log-file FNAME redirect stdout to a log file.\n"); printf(" --log-disable disables logging to a file.\n"); printf(" --slots-endpoint-disable disables slots monitoring endpoint.\n"); printf(" --metrics enable prometheus compatible metrics endpoint (default: %s).\n", sparams.metrics_endpoint ? "enabled" : "disabled"); @@ -2515,11 +2516,13 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, { llama.skvgraphics = true; // -skvg takes no parameter so we don't test ++i >= argc llama.skvinteract = false; + log_settings.stdout_target = "/dev/null"; } else if (arg == "-skvi" || arg == "--show-interactive-graphics") { llama.skvgraphics = true; // -skvi takes no parameter so we don't test ++i >= argc llama.skvinteract = true; + log_settings.stdout_target = "/dev/null"; } else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") { @@ -2759,6 +2762,15 @@ static void server_params_parse(int argc, char **argv, server_params &sparams, break; } } + else if (arg == "--stdout-log-file") + { + if (++i >= argc) + { + invalid_param = true; + break; + } + log_settings.stdout_target = argv[i]; // have just noticed that there is a 'log_set_target()' below but says 'INTERNAL USE: DO NOT USE' + } else if (arg == "--log-disable") { log_set_target(stdout); diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 7020cd046..53901eb4a 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -29,8 +29,8 @@ extern bool server_log_json; if (server_verbose) \ { \ server_log("VERB", __func__, __LINE__, MSG, __VA_ARGS__, \ - log_settings.stdout_target, log_settings.stdout_reset, \ - log_settings.stderr_target, log_settings.stderr_reset); \ + log_settings.stdout_target, log_settings.stderr_target, \ + log_settings.stdout_reset, log_settings.stderr_reset); \ } \ } while (0) // this is always false so the loop only compiles once but is treated as a single statement #endif @@ -39,28 +39,28 @@ extern bool server_log_json; struct LogRedirection { // Set default values for redirection targets and reset strings - std::string stdout_target = "stdout"; - std::string stdout_reset = "stdout"; + std::string stdout_target = "stdout_log.log"; + std::string stdout_reset = "/dev/stdout"; std::string stderr_target = "stderr_log.log"; - std::string stderr_reset = "stderr"; + std::string stderr_reset = "/dev/stderr"; }; LogRedirection log_settings; #define LOG_ERROR(MSG, ...) \ server_log("ERR", __func__, __LINE__, MSG, __VA_ARGS__, \ - log_settings.stdout_target, log_settings.stdout_reset, \ - log_settings.stderr_target, log_settings.stderr_reset) + log_settings.stdout_target, log_settings.stderr_target, \ + log_settings.stdout_reset, log_settings.stderr_reset) #define LOG_WARNING(MSG, ...) \ server_log("WARN", __func__, __LINE__, MSG, __VA_ARGS__, \ - log_settings.stdout_target, log_settings.stdout_reset, \ - log_settings.stderr_target, log_settings.stderr_reset) + log_settings.stdout_target, log_settings.stderr_target, \ + log_settings.stdout_reset, log_settings.stderr_reset) #define LOG_INFO(MSG, ...) \ server_log("INFO", __func__, __LINE__, MSG, __VA_ARGS__, \ - log_settings.stdout_target, log_settings.stdout_reset, \ - log_settings.stderr_target, log_settings.stderr_reset) + log_settings.stdout_target, log_settings.stderr_target, \ + log_settings.stdout_reset, log_settings.stderr_reset) /* // Example usage (WIP): @@ -214,8 +214,20 @@ static inline void server_log( {"timestamp", time(nullptr)}, }; - freopen(stdout_target.c_str(), "a", stdout); - freopen(stderr_target.c_str(), "a", stderr); // we assign stderr to dev/null effectively 'blackholing' the output because log.dump below is redirected too + /* + std::cerr << stdout_target.c_str() << std::endl; + FILE* new_stdout = freopen(stdout_target.c_str(), "a", stdout); + if (new_stdout == nullptr) { + std::cerr << "Error on redirecting stdout to " << stdout_target.c_str() << std::endl; + } + */ + + std::cerr << stderr_target.c_str() << std::endl; + FILE* new_stderr = freopen(stderr_target.c_str(), "a", stderr); + if (new_stderr == nullptr) { + std::cerr << "Error on redirecting stderr to " << stderr_target.c_str() << std::endl; + } + //freopen(stderr_target.c_str(), "a", stderr); // we assign stderr to dev/null effectively 'blackholing' the output because log.dump below is redirected too if (server_log_json) { log.merge_patch( @@ -250,8 +262,10 @@ static inline void server_log( printf("\033[85;0H%.*s\n", (int)str.size(), str.data()); fflush(stderr); // was originally fflush(stdout) + /* freopen(stdout_reset.c_str(), "a", stdout); // decide whether to restore stdout freopen(stderr_reset.c_str(), "a", stderr); // decide whether to restore stderr (both need automating) + */ } }