vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

main: remove --ctrl-token-fd-out in favor for fcntl() based detection

Browse source
This commit is contained in:
brian khuu 2024-05-21 04:44:50 +10:00
parent ad4b6097c0
commit c9ea9df7fb
3 changed files with 20 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

7
common/common.cpp
View file

@ -909,10 +909,6 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
params.ctrl_token_no_out = true;
return true;
}
if (arg == "--ctrl-token-fd-out") {
params.ctrl_token_fd_out = true;
return true;
}
if (arg == "--embedding") {
params.embedding = true;
return true;
@ -1442,9 +1438,6 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
printf(" -i, --interactive run in interactive mode\n");
printf(" --interactive-specials allow special tokens in user text, in interactive mode\n");
printf(" --ctrl-token-no-out control tokens output disabled\n");
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
printf(" --ctrl-token-fd-out control tokens sent to file descriptor 3 out of band\n");
#endif
printf(" -cnv, --conversation run in conversation mode (does not print special tokens and suffix/prefix)\n");
printf(" -ins, --instruct run in instruction mode (use with Alpaca models)\n");
printf(" -cml, --chatml run in chatml mode (use with ChatML-compatible models)\n");

1
common/common.h
View file

@ -143,7 +143,6 @@ struct gpt_params {
bool interactive = false; // interactive mode
bool interactive_specials = false; // whether to allow special tokens from user, during interactive mode
bool ctrl_token_no_out = false; // disable control token output
bool ctrl_token_fd_out = false; // enable control token output and redirect it to file descriptor 3
bool conversation = false; // conversation mode (does not print special tokens and suffix/prefix)
bool chatml = false; // chatml mode (used for models trained on chatml syntax)
bool prompt_cache_all = false; // save user input and generations to prompt cache

31
examples/main/main.cpp
View file

@ -18,7 +18,8 @@
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
#include <signal.h>
#include <unistd.h>
#define CONTROL_TOKEN_FILE_DESCRIPTOR (3)
#include <fcntl.h>
#define CONTROL_TOKEN_FILENO (3)
#elif defined (_WIN32)
#define WIN32_LEAN_AND_MEAN
#ifndef NOMINMAX
@ -529,6 +530,14 @@ int main(int argc, char ** argv) {
exit(1);
}
#ifndef _MSC_VER
if (fcntl(CONTROL_TOKEN_FILENO, F_GETFL) == -1) {
// Control Token File Descriptor has nothing attached to it
// make control token file descriptor be an alias of stdout
dup2(STDOUT_FILENO, CONTROL_TOKEN_FILENO);
}
#endif
while ((n_remain != 0 && !is_antiprompt) || params.interactive) {
// predict
if (!embd.empty()) {
@ -746,23 +755,23 @@ int main(int argc, char ** argv) {
// Console/Stream Output
if (!llama_token_is_control_token(llama_get_model(ctx), id)) {
// Stream Output Token To Standard Output
fflush(stdout);
fprintf(stdout, "%s", token_str.c_str());
} else if (!params.ctrl_token_no_out) {
#ifndef _MSC_VER
if (params.ctrl_token_fd_out) {
// Stream Control Token To Special Token Output. Useful for debugging control token behaviour
ssize_t result = write(CONTROL_TOKEN_FILE_DESCRIPTOR, token_str.c_str(), token_str.length());
(void) result;
}
else
#endif
if (!params.conversation && sparams.grammar.empty())
{
// Stream Control Token To Standard Output as long as we are not in a conversation or grammar output
// Stream Control Token To Special Token Output. Useful for debugging control token behaviour
fflush(stdout);
fprintf(stdout, "%s", token_str.c_str());
}
#ifndef _MSC_VER
else {
// Stream Control Token To Special Token Output. Useful for debugging control token behaviour
ssize_t result = write(CONTROL_TOKEN_FILENO, token_str.c_str(), token_str.length());
(void) result;
}
#endif
}
// Record Displayed Tokens To Log
// Note: Generated tokens are created one by one hence this check
if (embd.size() > 1) {