vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

mv defs to log.cpp

Browse source
This commit is contained in:
staviq 2023-09-17 16:27:53 +02:00
parent 06e5c1902a
commit 5b4db36130
3 changed files with 404 additions and 354 deletions
Download patch file Download diff file Expand all files Collapse all files

51
Makefile
View file

@ -484,6 +484,9 @@ OBJS += ggml-alloc.o
llama.o: llama.cpp ggml.h ggml-alloc.h ggml-cuda.h ggml-metal.h llama.h llama.o: llama.cpp ggml.h ggml-alloc.h ggml-cuda.h ggml-metal.h llama.h
$(CXX) $(CXXFLAGS) -c $< -o $@ $(CXX) $(CXXFLAGS) -c $< -o $@
log.o: common/log.cpp common/log.h
$(CXX) $(CXXFLAGS) -c $< -o $@
common.o: common/common.cpp common/common.h build-info.h common/log.h common.o: common/common.cpp common/common.h build-info.h common/log.h
$(CXX) $(CXXFLAGS) -c $< -o $@ $(CXX) $(CXXFLAGS) -c $< -o $@
@ -503,13 +506,13 @@ clean:
# Examples # Examples
# #
main: examples/main/main.cpp build-info.h ggml.o llama.o common.o console.o grammar-parser.o $(OBJS) main: examples/main/main.cpp build-info.h ggml.o llama.o common.o log.o console.o grammar-parser.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
@echo @echo
@echo '==== Run ./main -h for help. ====' @echo '==== Run ./main -h for help. ===='
@echo @echo
simple: examples/simple/simple.cpp ggml.o llama.o common.o $(OBJS) simple: examples/simple/simple.cpp ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
quantize: examples/quantize/quantize.cpp ggml.o llama.o $(OBJS) quantize: examples/quantize/quantize.cpp ggml.o llama.o $(OBJS)
@ -518,44 +521,44 @@ quantize: examples/quantize/quantize.cpp ggml.o llama.o $(O
quantize-stats: examples/quantize-stats/quantize-stats.cpp ggml.o llama.o $(OBJS) quantize-stats: examples/quantize-stats/quantize-stats.cpp ggml.o llama.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
perplexity: examples/perplexity/perplexity.cpp ggml.o llama.o common.o $(OBJS) perplexity: examples/perplexity/perplexity.cpp ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
embedding: examples/embedding/embedding.cpp ggml.o llama.o common.o $(OBJS) embedding: examples/embedding/embedding.cpp ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
save-load-state: examples/save-load-state/save-load-state.cpp ggml.o llama.o common.o $(OBJS) save-load-state: examples/save-load-state/save-load-state.cpp ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
server: examples/server/server.cpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp build-info.h ggml.o llama.o common.o grammar-parser.o $(OBJS) server: examples/server/server.cpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp build-info.h ggml.o llama.o common.o log.o grammar-parser.o $(OBJS)
$(CXX) $(CXXFLAGS) -Iexamples/server $(filter-out %.h,$(filter-out %.hpp,$^)) -o $@ $(LDFLAGS) $(LWINSOCK2) $(CXX) $(CXXFLAGS) -Iexamples/server $(filter-out %.h,$(filter-out %.hpp,$^)) -o $@ $(LDFLAGS) $(LWINSOCK2)
$(LIB_PRE)embdinput$(DSO_EXT): examples/embd-input/embd-input.h examples/embd-input/embd-input-lib.cpp build-info.h ggml.o llama.o common.o $(OBJS) $(LIB_PRE)embdinput$(DSO_EXT): examples/embd-input/embd-input.h examples/embd-input/embd-input-lib.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) --shared $(CXXFLAGS) $(filter-out %.h,$(filter-out %.hpp,$^)) -o $@ $(LDFLAGS) $(CXX) --shared $(CXXFLAGS) $(filter-out %.h,$(filter-out %.hpp,$^)) -o $@ $(LDFLAGS)
embd-input-test: $(LIB_PRE)embdinput$(DSO_EXT) examples/embd-input/embd-input-test.cpp build-info.h ggml.o llama.o common.o $(OBJS) embd-input-test: $(LIB_PRE)embdinput$(DSO_EXT) examples/embd-input/embd-input-test.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %$(DSO_EXT),$(filter-out %.h,$(filter-out %.hpp,$^))) -o $@ $(LDFLAGS) -L. -lembdinput $(CXX) $(CXXFLAGS) $(filter-out %$(DSO_EXT),$(filter-out %.h,$(filter-out %.hpp,$^))) -o $@ $(LDFLAGS) -L. -lembdinput
gguf: examples/gguf/gguf.cpp ggml.o llama.o $(OBJS) gguf: examples/gguf/gguf.cpp ggml.o llama.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
train-text-from-scratch: examples/train-text-from-scratch/train-text-from-scratch.cpp ggml.o llama.o common.o $(OBJS) train-text-from-scratch: examples/train-text-from-scratch/train-text-from-scratch.cpp ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(TTFS_CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(TTFS_CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
convert-llama2c-to-ggml: examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp ggml.o llama.o $(OBJS) convert-llama2c-to-ggml: examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp ggml.o llama.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
llama-bench: examples/llama-bench/llama-bench.cpp build-info.h ggml.o llama.o common.o $(OBJS) llama-bench: examples/llama-bench/llama-bench.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
baby-llama: examples/baby-llama/baby-llama.cpp ggml.o llama.o common.o $(OBJS) baby-llama: examples/baby-llama/baby-llama.cpp ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
beam-search: examples/beam-search/beam-search.cpp build-info.h ggml.o llama.o common.o $(OBJS) beam-search: examples/beam-search/beam-search.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
speculative: examples/speculative/speculative.cpp build-info.h ggml.o llama.o common.o grammar-parser.o $(OBJS) speculative: examples/speculative/speculative.cpp build-info.h ggml.o llama.o common.o log.o grammar-parser.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
ifdef LLAMA_METAL ifdef LLAMA_METAL
@ -584,37 +587,37 @@ benchmark-matmult: examples/benchmark/benchmark-matmult.cpp ggml.o $(OBJS)
vdot: pocs/vdot/vdot.cpp ggml.o $(OBJS) vdot: pocs/vdot/vdot.cpp ggml.o $(OBJS)
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
tests/test-llama-grammar: tests/test-llama-grammar.cpp build-info.h ggml.o common.o grammar-parser.o $(OBJS) tests/test-llama-grammar: tests/test-llama-grammar.cpp build-info.h ggml.o common.o log.o grammar-parser.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-grammar-parser: tests/test-grammar-parser.cpp build-info.h ggml.o llama.o common.o grammar-parser.o $(OBJS) tests/test-grammar-parser: tests/test-grammar-parser.cpp build-info.h ggml.o llama.o common.o log.o grammar-parser.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-double-float: tests/test-double-float.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-double-float: tests/test-double-float.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-grad0: tests/test-grad0.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-grad0: tests/test-grad0.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-opt: tests/test-opt.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-opt: tests/test-opt.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-quantize-fns: tests/test-quantize-fns.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-quantize-fns: tests/test-quantize-fns.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-quantize-perf: tests/test-quantize-perf.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-quantize-perf: tests/test-quantize-perf.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-sampling: tests/test-sampling.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-sampling: tests/test-sampling.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-tokenizer-0-falcon: tests/test-tokenizer-0-falcon.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-tokenizer-0-falcon: tests/test-tokenizer-0-falcon.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-tokenizer-0-llama: tests/test-tokenizer-0-llama.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-tokenizer-0-llama: tests/test-tokenizer-0-llama.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-tokenizer-1-llama: tests/test-tokenizer-1-llama.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-tokenizer-1-llama: tests/test-tokenizer-1-llama.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-c.o: tests/test-c.c llama.h tests/test-c.o: tests/test-c.c llama.h

331
common/log.cpp Normal file
View file

@ -0,0 +1,331 @@
#include "log.h"
LogStateWrapper::~LogStateWrapper()
{
log_flush_all_targets_internal();
for(auto t : _targets){ delete t; }
}
LogStateWrapper & LogStateWrapper::instance()
{
static LogStateWrapper inst;
return inst;
}
LogStateWrapper::LogTargetWrapper::LogTargetWrapper(FILE * handle)
: _type(Type::Stream),
_opened(true),
_handle(handle)
{}
LogStateWrapper::LogTargetWrapper::LogTargetWrapper(const std::string && filename)
: LogTargetWrapper(filename)
{}
LogStateWrapper::LogTargetWrapper::LogTargetWrapper(const std::string & filename)
: _filename(filename)
{
if(_filename.empty())
{
_type = Type::Stream;
_handle = stderr;
_opened = true;
}
else
{
_type = Type::File;
}
}
LogStateWrapper::LogTargetWrapper::~LogTargetWrapper()
{
if(_type == Type::File && _handle != nullptr) { std::fclose(_handle); }
}
LogStateWrapper::LogTargetWrapper::operator FILE * ()
{
if(!_opened)
{
while(_lock.test_and_set(std::memory_order_acquire)){ std::this_thread::yield(); }
if(!_opened && _type == Type::File) // check for opened again after acquiring a lock
{
auto result = std::fopen(_filename.c_str(), "w");
if(result)
{
_handle = result;
}
else
{
std::fprintf(
stderr,
"Failed to open logfile '%s' with error '%s'\n",
_filename.c_str(), std::strerror(errno)
);
std::fflush(stderr);
_handle = stderr;
}
_opened = true;
}
_lock.clear(std::memory_order_release);
}
return _handle;
}
void LogStateWrapper::LogTargetWrapper::flush()
{
while(_lock.test_and_set(std::memory_order_acquire)){ std::this_thread::yield(); }
if(_opened && _type != Type::Invalid && _handle)
{
std::fflush(_handle);
}
_lock.clear(std::memory_order_release);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_internal(const std::string && filename)
{
return log_set_target_internal(filename);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_internal(const std::string & filename)
{
return log_add_select_target_internal(new LogTargetWrapper(filename), true);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_internal(FILE * handle)
{
return log_add_select_target_internal(new LogTargetWrapper(handle), true);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_internal(LogTargetWrapper * target)
{
return log_add_select_target_internal(target);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_add_select_target_internal(LogTargetWrapper * t, bool insert)
{
log_flush_all_targets_internal();
std::lock_guard<std::mutex> lock(_mutex);
if(_global_override_target == LogTriState::True)
{
if(_enabled || _global_override_enabled == LogTriState::True) return _current_target;
return _stored_target;
}
if(_enabled || _global_override_enabled == LogTriState::True) _current_target.store(t);
else _stored_target.store(t);
if(insert) _targets.insert(t);
return t;
}
void LogStateWrapper::log_flush_all_targets_internal()
{
std::lock_guard<std::mutex> lock(_mutex);
for(auto t : _targets){ t->flush(); }
}
FILE * LogStateWrapper::log_handler_internal()
{
return *_current_target;
}
FILE * LogStateWrapper::log_tee_handler_internal()
{
return _stderr_target;
}
void LogStateWrapper::log_disable_internal(bool threadsafe)
{
if(threadsafe)
{
std::lock_guard<std::mutex> lock(_mutex);
log_disable_internal_unsafe();
}
else
{
log_disable_internal_unsafe();
}
}
void LogStateWrapper::log_disable_internal_unsafe()
{
if(_enabled && _global_override_enabled != LogTriState::True)
{
_stored_target.store (_current_target);
_current_target.store (&_null_target);
_enabled = false;
}
}
void LogStateWrapper::log_enable_internal(bool threadsafe)
{
if(threadsafe)
{
std::lock_guard<std::mutex> lock(_mutex);
log_enable_internal_unsafe();
}
else
{
log_enable_internal_unsafe();
}
}
void LogStateWrapper::log_enable_internal_unsafe()
{
if(!_enabled && _global_override_enabled != LogTriState::False)
{
_current_target.store (_stored_target);
_enabled = true;
}
}
bool LogStateWrapper::log_param_single_parse_internal(const std::string & param)
{
#ifdef LOG_WITH_TEST
if (param == "--log-test")
{
log_test();
return true;
}
#endif
if (param == "--log-disable")
{
{
std::lock_guard<std::mutex> lock(_mutex);
log_disable_internal_unsafe();
_global_override_enabled = LogTriState::False;
}
return true;
}
if (param == "--log-enable")
{
{
std::lock_guard<std::mutex> lock(_mutex);
log_enable_internal_unsafe();
_global_override_enabled = LogTriState::True;
}
return true;
}
return false;
}
bool LogStateWrapper::log_param_pair_parse_internal(bool parse, const std::string & param, const std::string & next)
{
if (param == "--log-file")
{
if (parse)
{
log_flush_all_targets_internal();
std::lock_guard<std::mutex> lock(_mutex);
auto t = new LogTargetWrapper(log_filename_generator(next.empty() ? "unnamed" : next, "log"));
if(_enabled)
{
_current_target.store(t);
}
else
{
_stored_target.store(t);
}
_targets.insert(t);
_global_override_target = LogTriState::True;
return t;
}
return true;
}
return false;
}
std::string LogStateWrapper::log_filename_generator_internal(const std::string & basename, const std::string & extension)
{
std::stringstream buf;
buf << basename;
buf << ".";
buf << log_get_pid_impl();
buf << ".";
buf << extension;
return buf.str();
}
std::string LogStateWrapper::log_get_pid_internal()
{
static std::string pid;
if (pid.empty())
{
// std::this_thread::get_id() is the most portable way of obtaining a "process id"
// it's not the same as "pid" but is unique enough to solve multiple instances
// trying to write to the same log.
std::stringstream ss;
ss << std::this_thread::get_id();
pid = ss.str();
}
return pid;
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_impl(const std::string && filename)
{
return log_set_target_impl(filename);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_impl(const std::string & filename)
{
return instance().log_set_target_internal(filename);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_impl(FILE * handle)
{
return instance().log_set_target_internal(handle);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_impl(LogTargetWrapper * target)
{
return instance().log_set_target_internal(target);
}
FILE * LogStateWrapper::log_handler_impl()
{
return instance().log_handler_internal();
}
FILE * LogStateWrapper::log_tee_handler_impl()
{
return instance().log_tee_handler_internal();
}
void LogStateWrapper::log_disable_impl()
{
instance().log_disable_internal();
}
void LogStateWrapper::log_enable_impl()
{
instance().log_enable_internal();
}
bool LogStateWrapper::log_param_single_parse_impl(const std::string & param)
{
return instance().log_param_single_parse_internal(param);
}
bool LogStateWrapper::log_param_pair_parse_impl(bool parse, const std::string & param, const std::string & next)
{
return instance().log_param_pair_parse_internal(parse, param, next);
}
std::string LogStateWrapper::log_filename_generator_impl(const std::string & basename, const std::string & extension)
{
return instance().log_filename_generator_internal(basename, extension);
}
std::string LogStateWrapper::log_get_pid_impl()
{
return instance().log_get_pid_internal();
}

376
common/log.h
View file

@ -73,23 +73,14 @@ class LogStateWrapper
{ {
protected: protected:
LogStateWrapper(){}; LogStateWrapper(){};
~LogStateWrapper();
~LogStateWrapper()
{
log_flush_all_targets();
for(auto t : _targets){ delete t; }
}
private: private:
LogStateWrapper(LogStateWrapper & other) = delete; LogStateWrapper(LogStateWrapper & other) = delete;
void operator=(const LogStateWrapper &) = delete; void operator=(const LogStateWrapper &) = delete;
private: private:
static LogStateWrapper & instance() static LogStateWrapper & instance();
{
static LogStateWrapper inst;
return inst;
}
class LogTargetWrapper class LogTargetWrapper
{ {
@ -98,35 +89,10 @@ class LogStateWrapper
void operator=(const LogTargetWrapper &) = delete; void operator=(const LogTargetWrapper &) = delete;
public: public:
LogTargetWrapper(FILE * handle) LogTargetWrapper(FILE * handle);
: _type(Type::Stream), LogTargetWrapper(const std::string && filename);
_opened(true), LogTargetWrapper(const std::string & filename);
_handle(handle) ~LogTargetWrapper();
{}
LogTargetWrapper(const std::string && filename)
: LogTargetWrapper(filename)
{}
LogTargetWrapper(const std::string & filename)
: _filename(filename)
{
if(_filename.empty())
{
_type = Type::Stream;
_handle = stderr;
_opened = true;
}
else
{
_type = Type::File;
}
}
~LogTargetWrapper()
{
if(_type == Type::File && _handle != nullptr) { std::fclose(_handle); }
}
enum class Type{ enum class Type{
Invalid, Invalid,
@ -135,44 +101,8 @@ class LogStateWrapper
}; };
public: public:
operator FILE * () operator FILE * ();
{ void flush();
if(!_opened)
{
while(_lock.test_and_set(std::memory_order_acquire)){ std::this_thread::yield(); }
if(!_opened && _type == Type::File) // check for opened again after acquiring a lock
{
auto result = std::fopen(_filename.c_str(), "w");
if(result)
{
_handle = result;
}
else
{
std::fprintf(
stderr,
"Failed to open logfile '%s' with error '%s'\n",
_filename.c_str(), std::strerror(errno)
);
std::fflush(stderr);
_handle = stderr;
}
_opened = true;
}
_lock.clear(std::memory_order_release);
}
return _handle;
}
void flush()
{
while(_lock.test_and_set(std::memory_order_acquire)){ std::this_thread::yield(); }
if(_opened && _type != Type::Invalid && _handle)
{
std::fflush(_handle);
}
_lock.clear(std::memory_order_release);
}
private: private:
std::atomic_flag _lock = ATOMIC_FLAG_INIT; std::atomic_flag _lock = ATOMIC_FLAG_INIT;
@ -180,9 +110,7 @@ class LogStateWrapper
std::string _filename; std::string _filename;
std::atomic<bool> _opened {false}; std::atomic<bool> _opened {false};
std::atomic<FILE*> _handle {nullptr}; std::atomic<FILE*> _handle {nullptr};
}; // class LogTargetWrapper
//friend class LogStateWrapper;
};
private: private:
std::mutex _mutex; std::mutex _mutex;
@ -196,249 +124,37 @@ class LogStateWrapper
std::atomic<LogTargetWrapper*> _current_target {&_null_target}; std::atomic<LogTargetWrapper*> _current_target {&_null_target};
std::atomic<LogTargetWrapper*> _stored_target {&_null_target}; std::atomic<LogTargetWrapper*> _stored_target {&_null_target};
LogTargetWrapper * log_set_target_impl(const std::string && filename) LogTargetWrapper * log_set_target_internal(const std::string && filename);
{ LogTargetWrapper * log_set_target_internal(const std::string & filename);
return log_set_target_impl(filename); LogTargetWrapper * log_set_target_internal(FILE * handle);
} LogTargetWrapper * log_set_target_internal(LogTargetWrapper * target);
LogTargetWrapper * log_add_select_target_internal(LogTargetWrapper * t, bool insert = false);
LogTargetWrapper * log_set_target_impl(const std::string & filename) void log_flush_all_targets_internal();
{ FILE * log_handler_internal();
return log_add_select_target(new LogTargetWrapper(filename), true); FILE * log_tee_handler_internal();
} void log_disable_internal(bool threadsafe = true);
void log_disable_internal_unsafe();
LogTargetWrapper * log_set_target_impl(FILE * handle) void log_enable_internal(bool threadsafe = true);
{ void log_enable_internal_unsafe();
return log_add_select_target(new LogTargetWrapper(handle), true); bool log_param_single_parse_internal(const std::string & param);
} bool log_param_pair_parse_internal(bool parse, const std::string & param, const std::string & next = "");
std::string log_filename_generator_internal(const std::string & basename, const std::string & extension);
LogTargetWrapper * log_set_target_impl(LogTargetWrapper * target) std::string log_get_pid_internal();
{
return log_add_select_target(target);
}
LogTargetWrapper * log_add_select_target(LogTargetWrapper * t, bool insert = false)
{
log_flush_all_targets();
std::lock_guard<std::mutex> lock(_mutex);
if(_global_override_target == LogTriState::True)
{
if(_enabled || _global_override_enabled == LogTriState::True) return _current_target;
return _stored_target;
}
if(_enabled || _global_override_enabled == LogTriState::True) _current_target.store(t);
else _stored_target.store(t);
if(insert) _targets.insert(t);
return t;
}
void log_flush_all_targets()
{
std::lock_guard<std::mutex> lock(_mutex);
for(auto t : _targets){ t->flush(); }
}
FILE * log_handler_impl() { return *_current_target; }
FILE * log_tee_handler_impl() { return _stderr_target; }
void log_disable_impl(bool threadsafe = true)
{
if(threadsafe)
{
std::lock_guard<std::mutex> lock(_mutex);
log_disable_impl_unsafe();
}
else
{
log_disable_impl_unsafe();
}
}
void log_disable_impl_unsafe()
{
if(_enabled && _global_override_enabled != LogTriState::True)
{
_stored_target.store (_current_target);
_current_target.store (&_null_target);
_enabled = false;
}
}
void log_enable_impl(bool threadsafe = true)
{
if(threadsafe)
{
std::lock_guard<std::mutex> lock(_mutex);
log_enable_impl_unsafe();
}
else
{
log_enable_impl_unsafe();
}
}
void log_enable_impl_unsafe()
{
if(!_enabled && _global_override_enabled != LogTriState::False)
{
_current_target.store (_stored_target);
_enabled = true;
}
}
bool log_param_single_parse_impl(const std::string & param)
{
#ifdef LOG_WITH_TEST
if (param == "--log-test")
{
log_test();
return true;
}
#endif
if (param == "--log-disable")
{
{
std::lock_guard<std::mutex> lock(_mutex);
log_disable_impl_unsafe();
_global_override_enabled = LogTriState::False;
}
return true;
}
if (param == "--log-enable")
{
{
std::lock_guard<std::mutex> lock(_mutex);
log_enable_impl_unsafe();
_global_override_enabled = LogTriState::True;
}
return true;
}
return false;
}
bool log_param_pair_parse_impl(bool parse, const std::string & param, const std::string & next = std::string())
{
if (param == "--log-file")
{
if (parse)
{
log_flush_all_targets();
std::lock_guard<std::mutex> lock(_mutex);
auto t = new LogTargetWrapper(log_filename_generator(next.empty() ? "unnamed" : next, "log"));
if(_enabled)
{
_current_target.store(t);
}
else
{
_stored_target.store(t);
}
_targets.insert(t);
_global_override_target = LogTriState::True;
return t;
}
return true;
}
return false;
}
std::string log_filename_generator_impl(const std::string & basename, const std::string & extension)
{
std::stringstream buf;
buf << basename;
buf << ".";
buf << log_get_pid();
buf << ".";
buf << extension;
return buf.str();
}
std::string log_get_pid_impl()
{
static std::string pid;
if (pid.empty())
{
// std::this_thread::get_id() is the most portable way of obtaining a "process id"
// it's not the same as "pid" but is unique enough to solve multiple instances
// trying to write to the same log.
std::stringstream ss;
ss << std::this_thread::get_id();
pid = ss.str();
}
return pid;
}
public: public:
static LogTargetWrapper * log_set_target(const std::string &&filename) static LogTargetWrapper * log_set_target_impl(const std::string && filename);
{ static LogTargetWrapper * log_set_target_impl(const std::string & filename);
return log_set_target(filename); static LogTargetWrapper * log_set_target_impl(FILE * handle);
} static LogTargetWrapper * log_set_target_impl(LogTargetWrapper * target);
static FILE * log_handler_impl();
static LogTargetWrapper * log_set_target(const std::string &filename) static FILE * log_tee_handler_impl();
{ static void log_disable_impl();
return instance().log_set_target_impl(filename); static void log_enable_impl();
} static bool log_param_single_parse_impl(const std::string & param);
static bool log_param_pair_parse_impl(bool parse, const std::string & param, const std::string & next = "");
static LogTargetWrapper * log_set_target(FILE * handle) static std::string log_filename_generator_impl(const std::string & basename, const std::string & extension);
{ static std::string log_get_pid_impl();
return instance().log_set_target_impl(handle); }; // class LogStateWrapper
}
static LogTargetWrapper * log_set_target(LogTargetWrapper * target)
{
return instance().log_set_target_impl(target);
}
static FILE * log_handler()
{
return instance().log_handler_impl();
}
static FILE * log_tee_handler()
{
return instance().log_tee_handler_impl();
}
static void log_disable()
{
instance().log_disable_impl();
}
static void log_enable()
{
instance().log_enable_impl();
}
static bool log_param_single_parse(const std::string &param)
{
return instance().log_param_single_parse_impl(param);
}
static bool log_param_pair_parse(bool parse, const std::string & param, const std::string & next = "")
{
return instance().log_param_pair_parse_impl(parse, param, next);
}
static std::string log_filename_generator(const std::string & basename, const std::string & extension)
{
return instance().log_filename_generator_impl(basename, extension);
}
static std::string log_get_pid()
{
return instance().log_get_pid_impl();
}
};
// Specifies a log target. // Specifies a log target.
// default uses log_handler() with "llama.log" log file // default uses log_handler() with "llama.log" log file
@ -483,11 +199,11 @@ class LogStateWrapper
// } // }
// //
#ifndef LOG_TARGET #ifndef LOG_TARGET
#define LOG_TARGET LogStateWrapper::log_handler() #define LOG_TARGET LogStateWrapper::log_handler_impl()
#endif #endif
#ifndef LOG_TEE_TARGET #ifndef LOG_TEE_TARGET
#define LOG_TEE_TARGET LogStateWrapper::log_tee_handler() #define LOG_TEE_TARGET LogStateWrapper::log_tee_handler_impl()
#endif #endif
// Utility function for generating log file names with unique id based on thread id. // Utility function for generating log file names with unique id based on thread id.
@ -495,7 +211,7 @@ class LogStateWrapper
// where the number is a runtime id of the current thread. // where the number is a runtime id of the current thread.
#define log_filename_generator(log_file_basename, log_file_extension) \ #define log_filename_generator(log_file_basename, log_file_extension) \
LogStateWrapper::log_filename_generator(log_file_basename, log_file_extension) LogStateWrapper::log_filename_generator_impl(log_file_basename, log_file_extension)
// #ifndef LOG_DEFAULT_FILE_NAME // #ifndef LOG_DEFAULT_FILE_NAME
// #define LOG_DEFAULT_FILE_NAME log_filename_generator("llama", "log") // #define LOG_DEFAULT_FILE_NAME log_filename_generator("llama", "log")
@ -782,7 +498,7 @@ inline FILE *log_handler2_impl(bool change = false, LogTriState disable = LogTri
// Makes LOG() and LOG_TEE() produce no output, // Makes LOG() and LOG_TEE() produce no output,
// untill enabled back. // untill enabled back.
//#define log_disable() log_disable_impl() //#define log_disable() log_disable_impl()
#define log_disable() LogStateWrapper::log_disable() #define log_disable() LogStateWrapper::log_disable_impl()
/* /*
// INTERNAL, DO NOT USE // INTERNAL, DO NOT USE
inline FILE *log_disable_impl() inline FILE *log_disable_impl()
@ -792,7 +508,7 @@ inline FILE *log_disable_impl()
*/ */
// Enables logs at runtime. // Enables logs at runtime.
//#define log_enable() log_enable_impl() //#define log_enable() log_enable_impl()
#define log_enable() LogStateWrapper::log_enable() #define log_enable() LogStateWrapper::log_enable_impl()
/* /*
// INTERNAL, DO NOT USE // INTERNAL, DO NOT USE
inline FILE *log_enable_impl() inline FILE *log_enable_impl()
@ -802,7 +518,7 @@ inline FILE *log_enable_impl()
*/ */
// Sets target fir logs, either by a file name or FILE* pointer (stdout, stderr, or any valid FILE*) // Sets target fir logs, either by a file name or FILE* pointer (stdout, stderr, or any valid FILE*)
//#define log_set_target(target) log_set_target_impl(target) //#define log_set_target(target) log_set_target_impl(target)
#define log_set_target(target) LogStateWrapper::log_set_target(target) #define log_set_target(target) LogStateWrapper::log_set_target_impl(target)
/* /*
// INTERNAL, DO NOT USE // INTERNAL, DO NOT USE
@ -859,8 +575,8 @@ inline void log_test()
} }
#endif #endif
#define log_param_single_parse(param) LogStateWrapper::log_param_single_parse(param) #define log_param_single_parse(param) LogStateWrapper::log_param_single_parse_impl(param)
#define log_param_pair_parse(...) LogStateWrapper::log_param_pair_parse(__VA_ARGS__) #define log_param_pair_parse(...) LogStateWrapper::log_param_pair_parse_impl(__VA_ARGS__)
inline void log_print_usage() inline void log_print_usage()
{ {