From 71d05b9ae438e7d2c3f05212c3dcaf3a53606fa9 Mon Sep 17 00:00:00 2001 From: staviq Date: Wed, 23 Aug 2023 20:27:20 +0200 Subject: [PATCH] remove atomics and add dynamic log target --- common/log.h | 140 ++++++++++++++++++++++++++++++----------- examples/main/main.cpp | 11 +++- 2 files changed, 111 insertions(+), 40 deletions(-) diff --git a/common/log.h b/common/log.h index 0e2e9cd47..cd7b43b4d 100644 --- a/common/log.h +++ b/common/log.h @@ -5,8 +5,9 @@ #endif #include -#include #include +#include +#include // Specifies a log target. // default uses log_handler() with "llama.log" log file @@ -84,66 +85,129 @@ #define LOG_FLF_VAL #endif -#define _LOG(str, ...) \ +#define _LOG(str, ...) \ { \ + /*fprintf(stderr, "DBG:" str, ##__VA_ARGS__);*/ \ fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%c" LOG_TIMESTAMP_VAL LOG_FLF_VAL, ##__VA_ARGS__); \ - fflush(LOG_TARGET); \ + fflush(LOG_TARGET); /*fprintf(stderr, "DBGEND\n");*/ \ } -// This us a trick to bypass the silly +// This is a trick to bypass the silly // "warning: ISO C++11 requires at least one argument for the "..." in a variadic macro" // so we can gave a single macro which can be called just like printf. #define LOG(...) _LOG(__VA_ARGS__, '\0') -inline FILE *log_handler(std::string s = "llama.log") +inline std::string log_get_pid() { - static std::atomic_bool _initialized{false}; - static std::atomic_bool _logfileopened{false}; - - static FILE* logfile = nullptr; - - if (_initialized)[[likely]] + static std::string pid; + if (pid.empty()) [[unlikely]] { - // with fallback in case something went wrong + // std::this_thread::get_id() is the most portable way of obtaining a "process id" + // it's not the same as "pid" but is unique enough to solve multiple instances + // trying to write to the same log. + std::stringstream ss; + ss << std::this_thread::get_id(); + pid = ss.str(); + } + + return pid; +} + +#define LOG_DEFAULT_FILE_NAME std::string().append("llama.").append(log_get_pid()).append(".log") + +inline FILE *_log_handler1(bool change = false, std::string filename = LOG_DEFAULT_FILE_NAME, FILE *target = nullptr) +{ + // std::cerr << "\tFNM:" << filename << "TGT:" << (uint64_t)target << std::endl; + static bool _initialized{false}; + static std::string log_current_filename{filename}; + static FILE *log_current_target{target}; + static FILE *logfile = nullptr; + + if (change && log_current_filename.compare(filename) != 0) [[unlikely]] + { + // std::cerr << "\t\tFNM changed, deinit" << std::endl; + _initialized = false; + } + + if (change && log_current_target != target) [[unlikely]] + { + // std::cerr << "\t\tTGT changed, deinit" << std::endl; + _initialized = false; + } + + // std::cerr << "\tINIT:" << (_initialized ? "true" : "false") << std::endl; + + if (_initialized) [[likely]] + { + // std::cerr << "\t\tIS Inited" << std::endl; + // with fallback in case something went wrong + // std::cerr << "\t\tEarly Done" << std::endl; return logfile ? logfile : stderr; } else { - // Mutex-less threadsafe synchronisation. - // we need to make sure not more than one invocation of this function - // attempts to open a file at once. - // compare_exchange_strong checks and updates a flag - // in a single atomic operation. - bool expected{false}; - if( _logfileopened.compare_exchange_strong(expected,true) ) + // std::cerr << "\t\tIS NOT Inited" << std::endl; + if (target != nullptr) { - // If the flag was previously false, and we managed to turn it true - // ew are now responsible for opening the log file - logfile = fopen( s.c_str(), "wa" ); - - if( !logfile ) + // std::cerr << "\t\t\tTGT not nullptr" << std::endl; + if (logfile != nullptr && logfile != stdout && logfile != stderr) { - // Verify whether the file was opened, otherwise fallback to stderr - logfile = stderr; - - fprintf(stderr, "Failed to open logfile '%s' with error '%s'\n", s.c_str(), std::strerror(errno)); - fflush(stderr); + // std::cerr << "\t\t\t\tF close" << std::endl; + fclose(logfile); } - _initialized.store(true); + log_current_filename = LOG_DEFAULT_FILE_NAME; + log_current_target = target; + + // std::cerr << "\t\t\tTGT set to new target" << std::endl; + logfile = target; } else { - // We are not first to open the log file - // - // TODO: Better thread-safe option, possibly std::condition_variable + // std::cerr << "\t\t\tTGT IS nullptr" << std::endl; + if (log_current_filename.compare(filename) != 0) [[likely]] + { + // std::cerr << "\t\t\t\tFNM changed" << std::endl; - return stderr; + if (logfile != nullptr && logfile != stdout && logfile != stderr) + { + // std::cerr << "\t\t\t\t\tF close 2" << std::endl; + fclose(logfile); + } + + // std::cerr << "\t\t\t\tF reopen" << std::endl; + logfile = nullptr; + logfile = fopen(filename.c_str(), "a"); + } + else + { + // std::cerr << "\t\t\t\tF open" << std::endl; + // logfile = fopen(filename.c_str(), "wa"); + logfile = fopen(filename.c_str(), "a"); + } } - // with fallback in case something went wrong - return logfile ? logfile : stderr; + if (!logfile) + { + // std::cerr << "\t\t\tF invalid" << std::endl; + // Verify whether the file was opened, otherwise fallback to stderr + logfile = stderr; + + fprintf(stderr, "Failed to open logfile '%s' with error '%s'\n", filename.c_str(), std::strerror(errno)); + fflush(stderr); + } + + _initialized = true; } - return stderr; -} \ No newline at end of file + // std::cerr << "\tDone" << std::endl; + return logfile ? logfile : stderr; +} +inline FILE *_log_handler2(bool change = false, FILE *target = nullptr, std::string filename = LOG_DEFAULT_FILE_NAME) +{ + return _log_handler1(change, filename, target); +} + +inline FILE *log_set_target(std::string filename) { return _log_handler1(true, filename); } +inline FILE *log_set_target(FILE *target) { return _log_handler2(true, target); } +inline FILE *log_handler() { return _log_handler1(); } \ No newline at end of file diff --git a/examples/main/main.cpp b/examples/main/main.cpp index aee4b2809..eaaa63f81 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -61,8 +61,15 @@ void sigint_handler(int signo) { int main(int argc, char ** argv) { gpt_params params; - LOG("Hello World!\n") - LOG("Hello Again!\n") + LOG("Hello World to default output!\n") + log_set_target(stderr); + LOG("Hello World to stderr!\n") + log_set_target(LOG_DEFAULT_FILE_NAME); + LOG("Hello World to default log file!\n") + log_set_target(stdout); + LOG("Hello World to stdout!\n") + log_set_target(LOG_DEFAULT_FILE_NAME); + LOG("Hello World to default log file again!\n") if (gpt_params_parse(argc, argv, params) == false) { return 1;