vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge e283c20285 into 7ddf185537

Browse source
This commit is contained in:
staviq 2023-09-18 00:43:47 +00:00 • committed by GitHub
commit f80820f047
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 606 additions and 177 deletions
Download patch file Download diff file Expand all files Collapse all files

51
Makefile
View file

@ -489,6 +489,9 @@ OBJS += ggml-alloc.o
llama.o: llama.cpp ggml.h ggml-alloc.h ggml-cuda.h ggml-metal.h llama.h llama.o: llama.cpp ggml.h ggml-alloc.h ggml-cuda.h ggml-metal.h llama.h
$(CXX) $(CXXFLAGS) -c $< -o $@ $(CXX) $(CXXFLAGS) -c $< -o $@
log.o: common/log.cpp common/log.h
$(CXX) $(CXXFLAGS) -c $< -o $@
common.o: common/common.cpp common/common.h build-info.h common/log.h common.o: common/common.cpp common/common.h build-info.h common/log.h
$(CXX) $(CXXFLAGS) -c $< -o $@ $(CXX) $(CXXFLAGS) -c $< -o $@
@ -508,13 +511,13 @@ clean:
# Examples # Examples
# #
main: examples/main/main.cpp build-info.h ggml.o llama.o common.o console.o grammar-parser.o $(OBJS) main: examples/main/main.cpp build-info.h ggml.o llama.o common.o log.o console.o grammar-parser.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
@echo @echo
@echo '==== Run ./main -h for help. ====' @echo '==== Run ./main -h for help. ===='
@echo @echo
simple: examples/simple/simple.cpp ggml.o llama.o common.o $(OBJS) simple: examples/simple/simple.cpp ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
quantize: examples/quantize/quantize.cpp ggml.o llama.o $(OBJS) quantize: examples/quantize/quantize.cpp ggml.o llama.o $(OBJS)
@ -523,44 +526,44 @@ quantize: examples/quantize/quantize.cpp ggml.o llama.o $(O
quantize-stats: examples/quantize-stats/quantize-stats.cpp ggml.o llama.o $(OBJS) quantize-stats: examples/quantize-stats/quantize-stats.cpp ggml.o llama.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
perplexity: examples/perplexity/perplexity.cpp ggml.o llama.o common.o $(OBJS) perplexity: examples/perplexity/perplexity.cpp ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
embedding: examples/embedding/embedding.cpp ggml.o llama.o common.o $(OBJS) embedding: examples/embedding/embedding.cpp ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
save-load-state: examples/save-load-state/save-load-state.cpp ggml.o llama.o common.o $(OBJS) save-load-state: examples/save-load-state/save-load-state.cpp ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
server: examples/server/server.cpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp build-info.h ggml.o llama.o common.o grammar-parser.o $(OBJS) server: examples/server/server.cpp examples/server/httplib.h examples/server/json.hpp examples/server/index.html.hpp examples/server/index.js.hpp examples/server/completion.js.hpp build-info.h ggml.o llama.o common.o log.o grammar-parser.o $(OBJS)
$(CXX) $(CXXFLAGS) -Iexamples/server $(filter-out %.h,$(filter-out %.hpp,$^)) -o $@ $(LDFLAGS) $(LWINSOCK2) $(CXX) $(CXXFLAGS) -Iexamples/server $(filter-out %.h,$(filter-out %.hpp,$^)) -o $@ $(LDFLAGS) $(LWINSOCK2)
$(LIB_PRE)embdinput$(DSO_EXT): examples/embd-input/embd-input.h examples/embd-input/embd-input-lib.cpp build-info.h ggml.o llama.o common.o $(OBJS) $(LIB_PRE)embdinput$(DSO_EXT): examples/embd-input/embd-input.h examples/embd-input/embd-input-lib.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) --shared $(CXXFLAGS) $(filter-out %.h,$(filter-out %.hpp,$^)) -o $@ $(LDFLAGS) $(CXX) --shared $(CXXFLAGS) $(filter-out %.h,$(filter-out %.hpp,$^)) -o $@ $(LDFLAGS)
embd-input-test: $(LIB_PRE)embdinput$(DSO_EXT) examples/embd-input/embd-input-test.cpp build-info.h ggml.o llama.o common.o $(OBJS) embd-input-test: $(LIB_PRE)embdinput$(DSO_EXT) examples/embd-input/embd-input-test.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %$(DSO_EXT),$(filter-out %.h,$(filter-out %.hpp,$^))) -o $@ $(LDFLAGS) -L. -lembdinput $(CXX) $(CXXFLAGS) $(filter-out %$(DSO_EXT),$(filter-out %.h,$(filter-out %.hpp,$^))) -o $@ $(LDFLAGS) -L. -lembdinput
gguf: examples/gguf/gguf.cpp ggml.o llama.o $(OBJS) gguf: examples/gguf/gguf.cpp ggml.o llama.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
train-text-from-scratch: examples/train-text-from-scratch/train-text-from-scratch.cpp ggml.o llama.o common.o $(OBJS) train-text-from-scratch: examples/train-text-from-scratch/train-text-from-scratch.cpp ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(TTFS_CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(TTFS_CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
convert-llama2c-to-ggml: examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp ggml.o llama.o $(OBJS) convert-llama2c-to-ggml: examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp ggml.o llama.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
llama-bench: examples/llama-bench/llama-bench.cpp build-info.h ggml.o llama.o common.o $(OBJS) llama-bench: examples/llama-bench/llama-bench.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
baby-llama: examples/baby-llama/baby-llama.cpp ggml.o llama.o common.o $(OBJS) baby-llama: examples/baby-llama/baby-llama.cpp ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
beam-search: examples/beam-search/beam-search.cpp build-info.h ggml.o llama.o common.o $(OBJS) beam-search: examples/beam-search/beam-search.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
speculative: examples/speculative/speculative.cpp build-info.h ggml.o llama.o common.o grammar-parser.o $(OBJS) speculative: examples/speculative/speculative.cpp build-info.h ggml.o llama.o common.o log.o grammar-parser.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
ifdef LLAMA_METAL ifdef LLAMA_METAL
@ -589,37 +592,37 @@ benchmark-matmult: examples/benchmark/benchmark-matmult.cpp ggml.o $(OBJS)
vdot: pocs/vdot/vdot.cpp ggml.o $(OBJS) vdot: pocs/vdot/vdot.cpp ggml.o $(OBJS)
$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
tests/test-llama-grammar: tests/test-llama-grammar.cpp build-info.h ggml.o common.o grammar-parser.o $(OBJS) tests/test-llama-grammar: tests/test-llama-grammar.cpp build-info.h ggml.o common.o log.o grammar-parser.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-grammar-parser: tests/test-grammar-parser.cpp build-info.h ggml.o llama.o common.o grammar-parser.o $(OBJS) tests/test-grammar-parser: tests/test-grammar-parser.cpp build-info.h ggml.o llama.o common.o log.o grammar-parser.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-double-float: tests/test-double-float.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-double-float: tests/test-double-float.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-grad0: tests/test-grad0.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-grad0: tests/test-grad0.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-opt: tests/test-opt.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-opt: tests/test-opt.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-quantize-fns: tests/test-quantize-fns.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-quantize-fns: tests/test-quantize-fns.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-quantize-perf: tests/test-quantize-perf.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-quantize-perf: tests/test-quantize-perf.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-sampling: tests/test-sampling.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-sampling: tests/test-sampling.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-tokenizer-0-falcon: tests/test-tokenizer-0-falcon.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-tokenizer-0-falcon: tests/test-tokenizer-0-falcon.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-tokenizer-0-llama: tests/test-tokenizer-0-llama.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-tokenizer-0-llama: tests/test-tokenizer-0-llama.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-tokenizer-1-llama: tests/test-tokenizer-1-llama.cpp build-info.h ggml.o llama.o common.o $(OBJS) tests/test-tokenizer-1-llama: tests/test-tokenizer-1-llama.cpp build-info.h ggml.o llama.o common.o log.o $(OBJS)
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
tests/test-c.o: tests/test-c.c llama.h tests/test-c.o: tests/test-c.c llama.h

2
common/CMakeLists.txt
View file

@ -5,6 +5,8 @@ set(TARGET common)
add_library(${TARGET} OBJECT add_library(${TARGET} OBJECT
common.h common.h
common.cpp common.cpp
log.h
log.cpp
console.h console.h
console.cpp console.cpp
grammar-parser.h grammar-parser.h

4
common/common.cpp
View file

@ -550,7 +550,7 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
} else if ( log_param_single_parse( argv[i] ) ) { } else if ( log_param_single_parse( argv[i] ) ) {
// Do nothing, log_param_single_parse automatically does it's thing // Do nothing, log_param_single_parse automatically does it's thing
// and returns if a match was found and parsed. // and returns if a match was found and parsed.
} else if ( log_param_pair_parse( /*check_but_dont_parse*/ true, argv[i] ) ) { } else if ( log_param_pair_parse( /*parse*/ false, argv[i] ) ) {
// We have a matching known parameter requiring an argument, // We have a matching known parameter requiring an argument,
// now we need to check if there is anything after this argv // now we need to check if there is anything after this argv
// and flag invalid_param or parse it. // and flag invalid_param or parse it.
@ -558,7 +558,7 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
invalid_param = true; invalid_param = true;
break; break;
} }
if( !log_param_pair_parse( /*check_but_dont_parse*/ false, argv[i-1], argv[i]) ) { if( !log_param_pair_parse( /*parse*/ true, argv[i-1], argv[i]) ) {
invalid_param = true; invalid_param = true;
break; break;
} }

305
common/log.cpp Normal file
View file

@ -0,0 +1,305 @@
#include "log.h"
LogStateWrapper::~LogStateWrapper()
{
log_flush_all_targets_internal();
for(auto t : _targets){ delete t; }
}
LogStateWrapper::LogTargetWrapper::LogTargetWrapper(FILE * handle)
: _type(Type::Stream),
_opened(true),
_handle(handle)
{}
LogStateWrapper::LogTargetWrapper::LogTargetWrapper(const std::string && filename)
: LogTargetWrapper(filename)
{}
LogStateWrapper::LogTargetWrapper::LogTargetWrapper(const std::string & filename)
: _filename(filename)
{
if(_filename.empty())
{
_type = Type::Stream;
_handle = stderr;
_opened = true;
}
else
{
_type = Type::File;
}
}
LogStateWrapper::LogTargetWrapper::~LogTargetWrapper()
{
if(_type == Type::File && _handle != nullptr) { std::fclose(_handle); }
}
LogStateWrapper::LogTargetWrapper::operator FILE * ()
{
if(!_opened)
{
while(_lock.test_and_set(std::memory_order_acquire)){ std::this_thread::yield(); }
if(!_opened && _type == Type::File) // check for opened again after acquiring a lock
{
auto result = std::fopen(_filename.c_str(), "w");
if(result)
{
_handle = result;
}
else
{
std::fprintf(
stderr,
"Failed to open logfile '%s' with error '%s'\n",
_filename.c_str(), std::strerror(errno)
);
std::fflush(stderr);
_handle = stderr;
}
_opened = true;
}
_lock.clear(std::memory_order_release);
}
return _handle;
}
void LogStateWrapper::LogTargetWrapper::flush()
{
while(_lock.test_and_set(std::memory_order_acquire)){ std::this_thread::yield(); }
if(_opened && _type != Type::Invalid && _handle)
{
std::fflush(_handle);
}
_lock.clear(std::memory_order_release);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_internal(const std::string && filename)
{
return log_set_target_internal(filename);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_internal(const std::string & filename)
{
return log_add_select_target_internal(new LogTargetWrapper(filename), true);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_internal(FILE * handle)
{
return log_add_select_target_internal(new LogTargetWrapper(handle), true);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_internal(LogTargetWrapper * target)
{
return log_add_select_target_internal(target);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_add_select_target_internal(LogTargetWrapper * t, bool insert)
{
log_flush_all_targets_internal();
std::lock_guard<std::mutex> lock(_mutex);
if(_global_override_target == LogTriState::True)
{
if(_enabled || _global_override_enabled == LogTriState::True) return _current_target;
return _stored_target;
}
if(_enabled || _global_override_enabled == LogTriState::True) _current_target.store(t);
else _stored_target.store(t);
if(insert) _targets.insert(t);
return t;
}
void LogStateWrapper::log_flush_all_targets_internal()
{
std::lock_guard<std::mutex> lock(_mutex);
for(auto t : _targets){ t->flush(); }
}
void LogStateWrapper::log_disable_internal(bool threadsafe)
{
if(threadsafe)
{
std::lock_guard<std::mutex> lock(_mutex);
log_disable_internal_unsafe();
}
else
{
log_disable_internal_unsafe();
}
}
void LogStateWrapper::log_disable_internal_unsafe()
{
if(_enabled && _global_override_enabled != LogTriState::True)
{
_stored_target.store (_current_target);
_current_target.store (&_null_target);
_enabled = false;
}
}
void LogStateWrapper::log_enable_internal(bool threadsafe)
{
if(threadsafe)
{
std::lock_guard<std::mutex> lock(_mutex);
log_enable_internal_unsafe();
}
else
{
log_enable_internal_unsafe();
}
}
void LogStateWrapper::log_enable_internal_unsafe()
{
if(!_enabled && _global_override_enabled != LogTriState::False)
{
_current_target.store (_stored_target);
_enabled = true;
}
}
bool LogStateWrapper::log_param_single_parse_internal(const std::string & param)
{
#ifdef LOG_WITH_TEST
if (param == "--log-test")
{
log_test();
return true;
}
#endif
if (param == "--log-disable")
{
{
std::lock_guard<std::mutex> lock(_mutex);
log_disable_internal_unsafe();
_global_override_enabled = LogTriState::False;
}
return true;
}
if (param == "--log-enable")
{
{
std::lock_guard<std::mutex> lock(_mutex);
log_enable_internal_unsafe();
_global_override_enabled = LogTriState::True;
}
return true;
}
return false;
}
bool LogStateWrapper::log_param_pair_parse_internal(bool parse, const std::string & param, const std::string & next)
{
if (param == "--log-file")
{
if (parse)
{
log_flush_all_targets_internal();
std::lock_guard<std::mutex> lock(_mutex);
auto t = new LogTargetWrapper(log_filename_generator(next.empty() ? "unnamed" : next, "log"));
if(_enabled)
{
_current_target.store(t);
}
else
{
_stored_target.store(t);
}
_targets.insert(t);
_global_override_target = LogTriState::True;
return t;
}
return true;
}
return false;
}
std::string LogStateWrapper::log_filename_generator_internal(const std::string & basename, const std::string & extension)
{
std::stringstream buf;
buf << basename;
buf << ".";
buf << log_get_pid_impl();
buf << ".";
buf << extension;
return buf.str();
}
std::string LogStateWrapper::log_get_pid_internal()
{
static std::string pid;
if (pid.empty())
{
// std::this_thread::get_id() is the most portable way of obtaining a "process id"
// it's not the same as "pid" but is unique enough to solve multiple instances
// trying to write to the same log.
std::stringstream ss;
ss << std::this_thread::get_id();
pid = ss.str();
}
return pid;
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_impl(const std::string && filename)
{
return log_set_target_impl(filename);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_impl(const std::string & filename)
{
return instance().log_set_target_internal(filename);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_impl(FILE * handle)
{
return instance().log_set_target_internal(handle);
}
LogStateWrapper::LogTargetWrapper * LogStateWrapper::log_set_target_impl(LogTargetWrapper * target)
{
return instance().log_set_target_internal(target);
}
void LogStateWrapper::log_disable_impl()
{
instance().log_disable_internal();
}
void LogStateWrapper::log_enable_impl()
{
instance().log_enable_internal();
}
bool LogStateWrapper::log_param_single_parse_impl(const std::string & param)
{
return instance().log_param_single_parse_internal(param);
}
bool LogStateWrapper::log_param_pair_parse_impl(bool parse, const std::string & param, const std::string & next)
{
return instance().log_param_pair_parse_internal(parse, param, next);
}
std::string LogStateWrapper::log_filename_generator_impl(const std::string & basename, const std::string & extension)
{
return instance().log_filename_generator_internal(basename, extension);
}
std::string LogStateWrapper::log_get_pid_impl()
{
return instance().log_get_pid_internal();
}

303
common/log.h
View file

@ -5,7 +5,10 @@
#include <sstream> #include <sstream>
#include <iostream> #include <iostream>
#include <thread> #include <thread>
#include <mutex>
#include <atomic>
#include <vector> #include <vector>
#include <set>
#include <algorithm> #include <algorithm>
#include <cinttypes> #include <cinttypes>
@ -47,6 +50,132 @@
// //
// -------------------------------- // --------------------------------
#ifdef LOG_WITH_TEST
inline void log_test();
#endif
// Utility for synchronizing log configuration state
// since std::optional was introduced only in c++17
enum class LogTriState
{
Same,
False,
True
};
enum class LogTargetChannel
{
Logfile,
Screen
};
class LogStateWrapper
{
protected:
LogStateWrapper(){};
~LogStateWrapper();
private:
LogStateWrapper(LogStateWrapper & other) = delete;
void operator=(const LogStateWrapper &) = delete;
private:
static LogStateWrapper & instance()
{
static LogStateWrapper inst;
return inst;
}
class LogTargetWrapper
{
private:
LogTargetWrapper(LogTargetWrapper & other) = delete;
void operator=(const LogTargetWrapper &) = delete;
public:
LogTargetWrapper(FILE * handle);
LogTargetWrapper(const std::string && filename);
LogTargetWrapper(const std::string & filename);
~LogTargetWrapper();
enum class Type{
Invalid,
File,
Stream
};
public:
operator FILE * ();
void flush();
private:
std::atomic_flag _lock = ATOMIC_FLAG_INIT;
Type _type {Type::Invalid};
std::string _filename;
std::atomic<bool> _opened {false};
std::atomic<FILE*> _handle {nullptr};
}; // class LogTargetWrapper
private:
std::mutex _mutex;
std::atomic_flag _lock = ATOMIC_FLAG_INIT;
std::atomic<bool> _enabled {true};
std::atomic<LogTriState> _global_override_enabled {LogTriState::Same};
std::set<LogTargetWrapper*> _targets;
LogTargetWrapper _null_target {nullptr};
LogTargetWrapper _stderr_target {stderr};
std::atomic<LogTriState> _global_override_target {LogTriState::Same};
std::atomic<LogTargetWrapper*> _current_target {&_null_target};
std::atomic<LogTargetWrapper*> _stored_target {&_null_target};
LogTargetWrapper * log_set_target_internal(const std::string && filename);
LogTargetWrapper * log_set_target_internal(const std::string & filename);
LogTargetWrapper * log_set_target_internal(FILE * handle);
LogTargetWrapper * log_set_target_internal(LogTargetWrapper * target);
LogTargetWrapper * log_add_select_target_internal(LogTargetWrapper * t, bool insert = false);
void log_flush_all_targets_internal();
void log_disable_internal(bool threadsafe = true);
void log_disable_internal_unsafe();
void log_enable_internal(bool threadsafe = true);
void log_enable_internal_unsafe();
bool log_param_single_parse_internal(const std::string & param);
bool log_param_pair_parse_internal(bool parse, const std::string & param, const std::string & next = "");
std::string log_filename_generator_internal(const std::string & basename, const std::string & extension);
std::string log_get_pid_internal();
FILE * log_handler_internal()
{
return *_current_target;
}
FILE * log_tee_handler_internal()
{
return _stderr_target;
}
public:
static LogTargetWrapper * log_set_target_impl(const std::string && filename);
static LogTargetWrapper * log_set_target_impl(const std::string & filename);
static LogTargetWrapper * log_set_target_impl(FILE * handle);
static LogTargetWrapper * log_set_target_impl(LogTargetWrapper * target);
static void log_disable_impl();
static void log_enable_impl();
static bool log_param_single_parse_impl(const std::string & param);
static bool log_param_pair_parse_impl(bool parse, const std::string & param, const std::string & next = "");
static std::string log_filename_generator_impl(const std::string & basename, const std::string & extension);
static std::string log_get_pid_impl();
static FILE * log_handler_impl()
{
return instance().log_handler_internal();
}
static FILE * log_tee_handler_impl()
{
return instance().log_tee_handler_internal();
}
}; // class LogStateWrapper
// Specifies a log target. // Specifies a log target.
// default uses log_handler() with "llama.log" log file // default uses log_handler() with "llama.log" log file
// this can be changed, by defining LOG_TARGET // this can be changed, by defining LOG_TARGET
@ -90,53 +219,23 @@
// } // }
// //
#ifndef LOG_TARGET #ifndef LOG_TARGET
#define LOG_TARGET log_handler() #define LOG_TARGET LogStateWrapper::log_handler_impl()
#endif #endif
#ifndef LOG_TEE_TARGET #ifndef LOG_TEE_TARGET
#define LOG_TEE_TARGET stderr #define LOG_TEE_TARGET LogStateWrapper::log_tee_handler_impl()
#endif #endif
// Utility to obtain "pid" like unique process id and use it when creating log files.
inline std::string log_get_pid()
{
static std::string pid;
if (pid.empty())
{
// std::this_thread::get_id() is the most portable way of obtaining a "process id"
// it's not the same as "pid" but is unique enough to solve multiple instances
// trying to write to the same log.
std::stringstream ss;
ss << std::this_thread::get_id();
pid = ss.str();
}
return pid;
}
// Utility function for generating log file names with unique id based on thread id. // Utility function for generating log file names with unique id based on thread id.
// invocation with log_filename_generator( "llama", "log" ) creates a string "llama.<number>.log" // invocation with log_filename_generator( "llama", "log" ) creates a string "llama.<number>.log"
// where the number is a runtime id of the current thread. // where the number is a runtime id of the current thread.
#define log_filename_generator(log_file_basename, log_file_extension) log_filename_generator_impl(log_file_basename, log_file_extension) #define log_filename_generator(log_file_basename, log_file_extension) \
LogStateWrapper::log_filename_generator_impl(log_file_basename, log_file_extension)
// INTERNAL, DO NOT USE // #ifndef LOG_DEFAULT_FILE_NAME
inline std::string log_filename_generator_impl(const std::string & log_file_basename, const std::string & log_file_extension) // #define LOG_DEFAULT_FILE_NAME log_filename_generator("llama", "log")
{ // #endif
std::stringstream buf;
buf << log_file_basename;
buf << ".";
buf << log_get_pid();
buf << ".";
buf << log_file_extension;
return buf.str();
}
#ifndef LOG_DEFAULT_FILE_NAME
#define LOG_DEFAULT_FILE_NAME log_filename_generator("llama", "log")
#endif
// Utility for turning #define values into string literals // Utility for turning #define values into string literals
// so we can have a define for stderr and // so we can have a define for stderr and
@ -146,6 +245,13 @@ inline std::string log_filename_generator_impl(const std::string & log_file_base
#define LOG_TEE_TARGET_STRING LOG_STRINGIZE(LOG_TEE_TARGET) #define LOG_TEE_TARGET_STRING LOG_STRINGIZE(LOG_TEE_TARGET)
#define LOG_TIMESTAMP \
( \
std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>( \
std::chrono::system_clock::now().time_since_epoch() \
) \
).count()
// Allows disabling timestamps. // Allows disabling timestamps.
// in order to disable, define LOG_NO_TIMESTAMPS // in order to disable, define LOG_NO_TIMESTAMPS
// like so: // like so:
@ -156,10 +262,10 @@ inline std::string log_filename_generator_impl(const std::string & log_file_base
#ifndef LOG_NO_TIMESTAMPS #ifndef LOG_NO_TIMESTAMPS
#ifndef _MSC_VER #ifndef _MSC_VER
#define LOG_TIMESTAMP_FMT "[%" PRIu64 "] " #define LOG_TIMESTAMP_FMT "[%" PRIu64 "] "
#define LOG_TIMESTAMP_VAL , (std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>(std::chrono::system_clock::now().time_since_epoch())).count() #define LOG_TIMESTAMP_VAL , LOG_TIMESTAMP
#else #else
#define LOG_TIMESTAMP_FMT "[%" PRIu64 "] " #define LOG_TIMESTAMP_FMT "[%" PRIu64 "] "
#define LOG_TIMESTAMP_VAL , (std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>(std::chrono::system_clock::now().time_since_epoch())).count() #define LOG_TIMESTAMP_VAL , LOG_TIMESTAMP
#endif #endif
#else #else
#define LOG_TIMESTAMP_FMT "%s" #define LOG_TIMESTAMP_FMT "%s"
@ -169,10 +275,10 @@ inline std::string log_filename_generator_impl(const std::string & log_file_base
#ifdef LOG_TEE_TIMESTAMPS #ifdef LOG_TEE_TIMESTAMPS
#ifndef _MSC_VER #ifndef _MSC_VER
#define LOG_TEE_TIMESTAMP_FMT "[%" PRIu64 "] " #define LOG_TEE_TIMESTAMP_FMT "[%" PRIu64 "] "
#define LOG_TEE_TIMESTAMP_VAL , (std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>(std::chrono::system_clock::now().time_since_epoch())).count() #define LOG_TEE_TIMESTAMP_VAL , LOG_TIMESTAMP
#else #else
#define LOG_TEE_TIMESTAMP_FMT "[%" PRIu64 "] " #define LOG_TEE_TIMESTAMP_FMT "[%" PRIu64 "] "
#define LOG_TEE_TIMESTAMP_VAL , (std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>(std::chrono::system_clock::now().time_since_epoch())).count() #define LOG_TEE_TIMESTAMP_VAL , LOG_TIMESTAMP
#endif #endif
#else #else
#define LOG_TEE_TIMESTAMP_FMT "%s" #define LOG_TEE_TIMESTAMP_FMT "%s"
@ -212,24 +318,29 @@ inline std::string log_filename_generator_impl(const std::string & log_file_base
#define LOG_TEE_FLF_VAL ,"" #define LOG_TEE_FLF_VAL ,""
#endif #endif
// Utility for synchronizing log configuration state
// since std::optional was introduced only in c++17
enum LogTriState
{
LogTriStateSame,
LogTriStateFalse,
LogTriStateTrue
};
// INTERNAL, DO NOT USE // INTERNAL, DO NOT USE
// USE LOG() INSTEAD // USE LOG() INSTEAD
// //
#define LOG_FPRINTF_PARAMS_IMPL_GNU(str) \
LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL
#define LOG_FPRINTF_PARAMS_IMPL_MSVC(str) \
LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL ""
#define LOG_FPRINTF_PARAMS_TEE_IMPL_GNU(str) \
LOG_TEE_TIMESTAMP_FMT LOG_TEE_FLF_FMT str "%s" LOG_TEE_TIMESTAMP_VAL LOG_TEE_FLF_VAL
#define LOG_FPRINTF_PARAMS_TEE_IMPL_MSVC(str) \
LOG_TEE_TIMESTAMP_FMT LOG_TEE_FLF_FMT str "%s" LOG_TEE_TIMESTAMP_VAL LOG_TEE_FLF_VAL ""
#ifndef _MSC_VER #ifndef _MSC_VER
#define LOG_IMPL(str, ...) \ #define LOG_IMPL(str, ...) \
{ \ { \
if (LOG_TARGET != nullptr) \ if (LOG_TARGET != nullptr) \
{ \ { \
fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL, __VA_ARGS__); \ fprintf(LOG_TARGET, LOG_FPRINTF_PARAMS_IMPL_GNU(str), __VA_ARGS__); \
fflush(LOG_TARGET); \ fflush(LOG_TARGET); \
} \ } \
} }
@ -238,7 +349,7 @@ enum LogTriState
{ \ { \
if (LOG_TARGET != nullptr) \ if (LOG_TARGET != nullptr) \
{ \ { \
fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL "", ##__VA_ARGS__); \ fprintf(LOG_TARGET, LOG_FPRINTF_PARAMS_IMPL_MSVC(str), ##__VA_ARGS__); \
fflush(LOG_TARGET); \ fflush(LOG_TARGET); \
} \ } \
} }
@ -252,12 +363,12 @@ enum LogTriState
{ \ { \
if (LOG_TARGET != nullptr) \ if (LOG_TARGET != nullptr) \
{ \ { \
fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL, __VA_ARGS__); \ fprintf(LOG_TARGET, LOG_FPRINTF_PARAMS_IMPL_GNU(str), __VA_ARGS__); \
fflush(LOG_TARGET); \ fflush(LOG_TARGET); \
} \ } \
if (LOG_TARGET != nullptr && LOG_TARGET != stdout && LOG_TARGET != stderr && LOG_TEE_TARGET != nullptr) \ if (LOG_TARGET != stdout && LOG_TARGET != stderr && LOG_TEE_TARGET != nullptr) \
{ \ { \
fprintf(LOG_TEE_TARGET, LOG_TEE_TIMESTAMP_FMT LOG_TEE_FLF_FMT str "%s" LOG_TEE_TIMESTAMP_VAL LOG_TEE_FLF_VAL, __VA_ARGS__); \ fprintf(LOG_TEE_TARGET, LOG_FPRINTF_PARAMS_TEE_IMPL_GNU(str), __VA_ARGS__); \
fflush(LOG_TEE_TARGET); \ fflush(LOG_TEE_TARGET); \
} \ } \
} }
@ -266,12 +377,12 @@ enum LogTriState
{ \ { \
if (LOG_TARGET != nullptr) \ if (LOG_TARGET != nullptr) \
{ \ { \
fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL "", ##__VA_ARGS__); \ fprintf(LOG_TARGET, LOG_FPRINTF_PARAMS_IMPL_MSVC(str), ##__VA_ARGS__); \
fflush(LOG_TARGET); \ fflush(LOG_TARGET); \
} \ } \
if (LOG_TARGET != nullptr && LOG_TARGET != stdout && LOG_TARGET != stderr && LOG_TEE_TARGET != nullptr) \ if (LOG_TARGET != stdout && LOG_TARGET != stderr && LOG_TEE_TARGET != nullptr) \
{ \ { \
fprintf(LOG_TEE_TARGET, LOG_TEE_TIMESTAMP_FMT LOG_TEE_FLF_FMT str "%s" LOG_TEE_TIMESTAMP_VAL LOG_TEE_FLF_VAL "", ##__VA_ARGS__); \ fprintf(LOG_TEE_TARGET, LOG_FPRINTF_PARAMS_TEE_IMPL_MSVC(str), ##__VA_ARGS__); \
fflush(LOG_TEE_TARGET); \ fflush(LOG_TEE_TARGET); \
} \ } \
} }
@ -313,6 +424,7 @@ enum LogTriState
#define LOG_TEELN(str, ...) LOG_TEE_IMPL("%s" str, "", __VA_ARGS__, "\n") #define LOG_TEELN(str, ...) LOG_TEE_IMPL("%s" str, "", __VA_ARGS__, "\n")
#endif #endif
/*
// INTERNAL, DO NOT USE // INTERNAL, DO NOT USE
inline FILE *log_handler1_impl(bool change = false, LogTriState disable = LogTriStateSame, const std::string & filename = LOG_DEFAULT_FILE_NAME, FILE *target = nullptr) inline FILE *log_handler1_impl(bool change = false, LogTriState disable = LogTriStateSame, const std::string & filename = LOG_DEFAULT_FILE_NAME, FILE *target = nullptr)
{ {
@ -401,39 +513,46 @@ inline FILE *log_handler2_impl(bool change = false, LogTriState disable = LogTri
{ {
return log_handler1_impl(change, disable, filename, target); return log_handler1_impl(change, disable, filename, target);
} }
*/
// Disables logs entirely at runtime. // Disables logs entirely at runtime.
// Makes LOG() and LOG_TEE() produce no output, // Makes LOG() and LOG_TEE() produce no output,
// untill enabled back. // untill enabled back.
#define log_disable() log_disable_impl() //#define log_disable() log_disable_impl()
#define log_disable() LogStateWrapper::log_disable_impl()
/*
// INTERNAL, DO NOT USE // INTERNAL, DO NOT USE
inline FILE *log_disable_impl() inline FILE *log_disable_impl()
{ {
return log_handler1_impl(true, LogTriStateTrue); return log_handler1_impl(true, LogTriStateTrue);
} }
*/
// Enables logs at runtime. // Enables logs at runtime.
#define log_enable() log_enable_impl() //#define log_enable() log_enable_impl()
#define log_enable() LogStateWrapper::log_enable_impl()
/*
// INTERNAL, DO NOT USE // INTERNAL, DO NOT USE
inline FILE *log_enable_impl() inline FILE *log_enable_impl()
{ {
return log_handler1_impl(true, LogTriStateFalse); return log_handler1_impl(true, LogTriStateFalse);
} }
*/
// Sets target fir logs, either by a file name or FILE* pointer (stdout, stderr, or any valid FILE*) // Sets target fir logs, either by a file name or FILE* pointer (stdout, stderr, or any valid FILE*)
#define log_set_target(target) log_set_target_impl(target) //#define log_set_target(target) log_set_target_impl(target)
#define log_set_target(target) LogStateWrapper::log_set_target_impl(target)
/*
// INTERNAL, DO NOT USE // INTERNAL, DO NOT USE
inline FILE *log_set_target_impl(const std::string & filename) { return log_handler1_impl(true, LogTriStateSame, filename); } inline FILE *log_set_target_impl(const std::string & filename) { return log_handler1_impl(true, LogTriStateSame, filename); }
inline FILE *log_set_target_impl(FILE *target) { return log_handler2_impl(true, LogTriStateSame, target); } inline FILE *log_set_target_impl(FILE *target) { return log_handler2_impl(true, LogTriStateSame, target); }
// INTERNAL, DO NOT USE // INTERNAL, DO NOT USE
inline FILE *log_handler() { return log_handler1_impl(); } inline FILE *log_handler() { return log_handler1_impl(); }
*/
#ifdef LOG_WITH_TEST
inline void log_test() inline void log_test()
{ {
auto log_default_logfile = log_set_target(log_filename_generator("llama","log"));
log_disable(); log_disable();
LOG("01 Hello World to nobody, because logs are disabled!\n") LOG("01 Hello World to nobody, because logs are disabled!\n")
log_enable(); log_enable();
@ -442,11 +561,13 @@ inline void log_test()
log_set_target(stderr); log_set_target(stderr);
LOG("04 Hello World to stderr!\n") LOG("04 Hello World to stderr!\n")
LOG_TEE("05 Hello World TEE with double printing to stderr prevented!\n") LOG_TEE("05 Hello World TEE with double printing to stderr prevented!\n")
log_set_target(LOG_DEFAULT_FILE_NAME); //log_set_target(LOG_DEFAULT_FILE_NAME);
log_set_target(log_default_logfile);
LOG("06 Hello World to default log file!\n") LOG("06 Hello World to default log file!\n")
log_set_target(stdout); log_set_target(stdout);
LOG("07 Hello World to stdout!\n") LOG("07 Hello World to stdout!\n")
log_set_target(LOG_DEFAULT_FILE_NAME); //log_set_target(LOG_DEFAULT_FILE_NAME);
log_set_target(log_default_logfile);
LOG("08 Hello World to default log file again!\n") LOG("08 Hello World to default log file again!\n")
log_disable(); log_disable();
LOG("09 Hello World _1_ into the void!\n") LOG("09 Hello World _1_ into the void!\n")
@ -472,44 +593,10 @@ inline void log_test()
LOGLN("22 Hello msvc LOGLN with (%d)(%s) arguments\n", 1, "test") LOGLN("22 Hello msvc LOGLN with (%d)(%s) arguments\n", 1, "test")
#endif #endif
} }
#endif
inline bool log_param_single_parse(const std::string & param) #define log_param_single_parse(param) LogStateWrapper::log_param_single_parse_impl(param)
{ #define log_param_pair_parse(...) LogStateWrapper::log_param_pair_parse_impl(__VA_ARGS__)
if ( param == "--log-test")
{
log_test();
return true;
}
if ( param == "--log-disable")
{
log_disable();
return true;
}
if ( param == "--log-enable")
{
log_enable();
return true;
}
return false;
}
inline bool log_param_pair_parse(bool check_but_dont_parse, const std::string & param, const std::string & next = std::string())
{
if ( param == "--log-file")
{
if (!check_but_dont_parse)
{
log_set_target(log_filename_generator(next.empty() ? "unnamed" : next, "log"));
}
return true;
}
return false;
}
inline void log_print_usage() inline void log_print_usage()
{ {
@ -518,17 +605,19 @@ inline void log_print_usage()
printf(" -h, --help show this help message and exit\n");*/ printf(" -h, --help show this help message and exit\n");*/
/* spacing /* spacing
printf("__-param----------------Description\n");*/ printf("__-param----------------Description\n");*/
#ifdef LOG_WITH_TEST
printf(" --log-test Run simple logging test\n"); printf(" --log-test Run simple logging test\n");
#endif
printf(" --log-disable Disable trace logs\n"); printf(" --log-disable Disable trace logs\n");
printf(" --log-enable Enable trace logs\n"); printf(" --log-enable Enable trace logs\n");
printf(" --log-file Specify a log filename (without extension)\n"); printf(" --log-file Specify a log filename (without extension)\n");
printf(" Log file will be tagged with unique ID and written as \"<name>.<ID>.log\"\n"); /* */ printf(" Log file will be tagged with unique ID and written as \"<name>.<ID>.log\"\n");
} }
#define log_dump_cmdline(argc, argv) log_dump_cmdline_impl(argc, argv) #define log_dump_cmdline(argc, argv) log_dump_cmdline_impl(argc, argv)
// INTERNAL, DO NOT USE // INTERNAL, DO NOT USE
inline void log_dump_cmdline_impl(int argc, char **argv) inline void log_dump_cmdline_impl(int argc, char ** argv)
{ {
std::stringstream buf; std::stringstream buf;
for (int i = 0; i < argc; ++i) for (int i = 0; i < argc; ++i)

32
examples/main/main.cpp
View file

@ -298,8 +298,11 @@ int main(int argc, char ** argv) {
} else if (n_matching_session_tokens >= embd_inp.size()) { } else if (n_matching_session_tokens >= embd_inp.size()) {
LOG_TEE("%s: session file has exact match for prompt!\n", __func__); LOG_TEE("%s: session file has exact match for prompt!\n", __func__);
} else if (n_matching_session_tokens < (embd_inp.size() / 2)) { } else if (n_matching_session_tokens < (embd_inp.size() / 2)) {
LOG_TEE("%s: warning: session file has low similarity to prompt (%zu / %zu tokens); will mostly be reevaluated\n", LOG_TEE(
__func__, n_matching_session_tokens, embd_inp.size()); "%s: warning: session file has low similarity to prompt (%zu / %zu tokens);"
" will mostly be reevaluated\n",
__func__, n_matching_session_tokens, embd_inp.size()
);
} else { } else {
LOG_TEE("%s: session file matches %zu / %zu tokens of prompt\n", LOG_TEE("%s: session file matches %zu / %zu tokens of prompt\n",
__func__, n_matching_session_tokens, embd_inp.size()); __func__, n_matching_session_tokens, embd_inp.size());
@ -307,8 +310,10 @@ int main(int argc, char ** argv) {
} }
LOGLN( LOGLN(
"recalculate the cached logits (check): embd_inp.empty() %s, n_matching_session_tokens %zu, embd_inp.size() %zu, session_tokens.size() %zu, embd_inp.size() %zu", "recalculate the cached logits (check): embd_inp.empty() %s, n_matching_session_tokens %zu,"
log_tostr(embd_inp.empty()), n_matching_session_tokens, embd_inp.size(), session_tokens.size(), embd_inp.size()); " embd_inp.size() %zu, session_tokens.size() %zu, embd_inp.size() %zu",
log_tostr(embd_inp.empty()), n_matching_session_tokens, embd_inp.size(), session_tokens.size(), embd_inp.size()
);
// if we will use the cache for the full prompt without reaching the end of the cache, force // if we will use the cache for the full prompt without reaching the end of the cache, force
// reevaluation of the last token token to recalculate the cached logits // reevaluation of the last token token to recalculate the cached logits
@ -402,9 +407,17 @@ int main(int argc, char ** argv) {
LOG_TEE("Input suffix: '%s'\n", params.input_suffix.c_str()); LOG_TEE("Input suffix: '%s'\n", params.input_suffix.c_str());
} }
} }
LOG_TEE("sampling: repeat_last_n = %d, repeat_penalty = %f, presence_penalty = %f, frequency_penalty = %f, top_k = %d, tfs_z = %f, top_p = %f, typical_p = %f, temp = %f, mirostat = %d, mirostat_lr = %f, mirostat_ent = %f\n", LOG_TEE(
params.repeat_last_n, params.repeat_penalty, params.presence_penalty, params.frequency_penalty, params.top_k, params.tfs_z, params.top_p, params.typical_p, params.temp, params.mirostat, params.mirostat_eta, params.mirostat_tau); "sampling: repeat_last_n = %d, repeat_penalty = %f, presence_penalty = %f,"
LOG_TEE("generate: n_ctx = %d, n_batch = %d, n_predict = %d, n_keep = %d\n", n_ctx, params.n_batch, params.n_predict, params.n_keep); " frequency_penalty = %f, top_k = %d, tfs_z = %f, top_p = %f, typical_p = %f,"
" temp = %f, mirostat = %d, mirostat_lr = %f, mirostat_ent = %f\n",
params.repeat_last_n, params.repeat_penalty, params.presence_penalty,
params.frequency_penalty, params.top_k, params.tfs_z, params.top_p, params.typical_p,
params.temp, params.mirostat, params.mirostat_eta, params.mirostat_tau
);
LOG_TEE(
"generate: n_ctx = %d, n_batch = %d, n_predict = %d, n_keep = %d\n",
n_ctx, params.n_batch, params.n_predict, params.n_keep);
LOG_TEE("\n\n"); LOG_TEE("\n\n");
struct llama_grammar * grammar = NULL; struct llama_grammar * grammar = NULL;
@ -509,7 +522,10 @@ int main(int argc, char ** argv) {
} }
const int n_left = n_past - params.n_keep; const int n_left = n_past - params.n_keep;
LOG("context full, swapping: n_past = %d, n_left = %d, n_ctx = %d, n_keep = %d\n", n_past, n_left, n_ctx, params.n_keep); LOG(
"context full, swapping: n_past = %d, n_left = %d, n_ctx = %d, n_keep = %d\n",
n_past, n_left, n_ctx, params.n_keep
);
// always keep the first token - BOS // always keep the first token - BOS
n_past = std::max(1, params.n_keep); n_past = std::max(1, params.n_keep);

26
examples/speculative/speculative.cpp
View file

@ -154,7 +154,10 @@ int main(int argc, char ** argv) {
// check if the draft matches the target // check if the draft matches the target
if (i_dft < (int) drafted.size() && id == drafted[i_dft]) { if (i_dft < (int) drafted.size() && id == drafted[i_dft]) {
LOG("the sampled target token matches the %dth drafted token (%d, '%s') - accepted\n", i_dft, id, token_str.c_str()); LOG(
"the sampled target token matches the %dth drafted token (%d, '%s') - accepted\n",
i_dft, id, token_str.c_str()
);
++n_accept; ++n_accept;
++n_past_tgt; ++n_past_tgt;
++n_past_dft; ++n_past_dft;
@ -166,8 +169,10 @@ int main(int argc, char ** argv) {
// the drafted token was rejected or we are out of drafted tokens // the drafted token was rejected or we are out of drafted tokens
if (i_dft < (int) drafted.size()) { if (i_dft < (int) drafted.size()) {
LOG("the %dth drafted token (%d, '%s') does not match the sampled target token (%d, '%s') - rejected\n", LOG(
i_dft, drafted[i_dft], llama_token_to_piece(ctx_dft, drafted[i_dft]).c_str(), id, token_str.c_str()); "the %dth drafted token (%d, '%s') does not match the sampled target token (%d, '%s') - rejected\n",
i_dft, drafted[i_dft], llama_token_to_piece(ctx_dft, drafted[i_dft]).c_str(), id, token_str.c_str()
);
} else { } else {
LOG("out of drafted tokens\n"); LOG("out of drafted tokens\n");
} }
@ -235,7 +240,10 @@ int main(int argc, char ** argv) {
llama_sample_softmax(ctx_dft, &cur_p); llama_sample_softmax(ctx_dft, &cur_p);
for (int i = 0; i < 3; ++i) { for (int i = 0; i < 3; ++i) {
LOG(" - draft candidate %3d: %6d (%8.3f) '%s'\n", i, cur_p.data[i].id, cur_p.data[i].p, llama_token_to_piece(ctx_dft, cur_p.data[i].id).c_str()); LOG(
" - draft candidate %3d: %6d (%8.3f) '%s'\n",
i, cur_p.data[i].id, cur_p.data[i].p, llama_token_to_piece(ctx_dft, cur_p.data[i].id).c_str()
);
} }
// TODO: better logic? // TODO: better logic?
@ -276,8 +284,14 @@ int main(int argc, char ** argv) {
LOG_TEE("\n\n"); LOG_TEE("\n\n");
LOG_TEE("encoded %4d tokens in %8.3f seconds, speed: %8.3f t/s\n", n_input, (t_enc_end - t_enc_start) / 1e6f, inp.size() / ((t_enc_end - t_enc_start) / 1e6f)); LOG_TEE(
LOG_TEE("decoded %4d tokens in %8.3f seconds, speed: %8.3f t/s\n", n_predict, (t_dec_end - t_dec_start) / 1e6f, n_predict / ((t_dec_end - t_dec_start) / 1e6f)); "encoded %4d tokens in %8.3f seconds, speed: %8.3f t/s\n",
n_input, (t_enc_end - t_enc_start) / 1e6f, inp.size() / ((t_enc_end - t_enc_start) / 1e6f)
);
LOG_TEE(
"decoded %4d tokens in %8.3f seconds, speed: %8.3f t/s\n",
n_predict, (t_dec_end - t_dec_start) / 1e6f, n_predict / ((t_dec_end - t_dec_start) / 1e6f)
);
// TODO: make sure these numbers are computed correctly // TODO: make sure these numbers are computed correctly
LOG_TEE("\n"); LOG_TEE("\n");