mirror of
https://github.com/jart/cosmopolitan.git
synced 2025-01-31 11:37:35 +00:00
8fdb31681a
llama.com can now load weights that use the new file format which was introduced a few weeks ago. Note that, unlike llama.cpp, we will keep support for old file formats in our tool so you don't need to convert your weights when the upstream project makes breaking changes. Please note that using ggjt v3 does make avx2 inference go 5% faster for me.
658 lines
27 KiB
C++
658 lines
27 KiB
C++
/*-*-mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8-*-│
|
|
│vi: set net ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi│
|
|
╚──────────────────────────────────────────────────────────────────────────────╝
|
|
│ │
|
|
│ radpajama.com │
|
|
│ Copyright (c) 2023 Ariel Núñez │
|
|
│ Copyright (c) 2023 Georgi Gerganov │
|
|
│ │
|
|
│ Permission is hereby granted, free of charge, to any person obtaining │
|
|
│ a copy of this software and associated documentation files (the │
|
|
│ "Software"), to deal in the Software without restriction, including │
|
|
│ without limitation the rights to use, copy, modify, merge, publish, │
|
|
│ distribute, sublicense, and/or sell copies of the Software, and to │
|
|
│ permit persons to whom the Software is furnished to do so, subject to │
|
|
│ the following conditions: │
|
|
│ │
|
|
│ The above copyright notice and this permission notice shall be │
|
|
│ included in all copies or substantial portions of the Software. │
|
|
│ │
|
|
│ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, │
|
|
│ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF │
|
|
│ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. │
|
|
│ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY │
|
|
│ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, │
|
|
│ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE │
|
|
│ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. │
|
|
│ │
|
|
╚─────────────────────────────────────────────────────────────────────────────*/
|
|
#include "libc/calls/calls.h"
|
|
#include "libc/calls/sigtimedwait.h"
|
|
#include "libc/calls/struct/sigaction.h"
|
|
#include "libc/calls/struct/siginfo.h"
|
|
#include "libc/calls/weirdtypes.h"
|
|
#include "libc/log/log.h"
|
|
#include "libc/runtime/pathconf.h"
|
|
#include "libc/runtime/runtime.h"
|
|
#include "libc/runtime/sysconf.h"
|
|
#include "libc/sysv/consts/f.h"
|
|
#include "libc/sysv/consts/fileno.h"
|
|
#include "libc/sysv/consts/o.h"
|
|
#include "libc/sysv/consts/ok.h"
|
|
#include "libc/sysv/consts/sa.h"
|
|
#include "libc/sysv/consts/sicode.h"
|
|
#include "libc/sysv/consts/ss.h"
|
|
#include "libc/time/time.h"
|
|
#include "third_party/getopt/getopt.h"
|
|
#include "third_party/libcxx/cassert"
|
|
#include "third_party/libcxx/cinttypes"
|
|
#include "third_party/libcxx/cmath"
|
|
#include "third_party/libcxx/cstdio"
|
|
#include "third_party/libcxx/cstring"
|
|
#include "third_party/libcxx/ctime"
|
|
#include "third_party/libcxx/fstream"
|
|
#include "third_party/libcxx/iostream"
|
|
#include "third_party/libcxx/string"
|
|
#include "third_party/libcxx/vector"
|
|
#include "third_party/musl/crypt.h"
|
|
#include "third_party/musl/lockf.h"
|
|
#include "third_party/radpajama/common-gptneox.h"
|
|
#include "third_party/radpajama/gptneox.h"
|
|
// clang-format off
|
|
|
|
static console_state con_st;
|
|
static gptneox_context ** g_ctx;
|
|
|
|
static bool is_interacting = false;
|
|
|
|
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
|
|
void sigint_handler(int signo) {
|
|
set_console_color(con_st, CONSOLE_COLOR_DEFAULT);
|
|
printf("\n"); // this also force flush stdout.
|
|
if (signo == SIGINT) {
|
|
if (!is_interacting) {
|
|
is_interacting=true;
|
|
} else {
|
|
gptneox_print_timings(*g_ctx);
|
|
_exit(130);
|
|
}
|
|
}
|
|
}
|
|
#endif
|
|
|
|
int main(int argc, char ** argv) {
|
|
gpt_params params;
|
|
params.model = "./examples/redpajama/models/pythia/ggml-RedPajama-INCITE-Instruct-3B-v1-f16.bin";
|
|
|
|
MakeProcessNice();
|
|
ShowCrashReports();
|
|
|
|
if (gpt_params_parse(argc, argv, params) == false) {
|
|
return 1;
|
|
}
|
|
|
|
// save choice to use color for later
|
|
// (note for later: this is a slightly awkward choice)
|
|
con_st.use_color = params.use_color;
|
|
|
|
#if defined (_WIN32)
|
|
win32_console_init(params.use_color);
|
|
#endif
|
|
|
|
if (params.perplexity) {
|
|
printf("\n************\n");
|
|
printf("%s: please use the 'perplexity' tool for perplexity calculations\n", __func__);
|
|
printf("************\n\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
if (params.embedding) {
|
|
printf("\n************\n");
|
|
printf("%s: please use the 'embedding' tool for embedding calculations\n", __func__);
|
|
printf("************\n\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
if (params.n_ctx > 2048) {
|
|
fprintf(stderr, "%s: warning: model does not support context sizes greater than 2048 tokens (%d specified);"
|
|
"expect poor results\n", __func__, params.n_ctx);
|
|
}
|
|
|
|
if (params.seed < 0) {
|
|
params.seed = time(NULL);
|
|
}
|
|
|
|
fprintf(stderr, "%s: seed = %d\n", __func__, params.seed);
|
|
|
|
std::mt19937 rng(params.seed);
|
|
if (params.random_prompt) {
|
|
params.prompt = gpt_random_prompt(rng);
|
|
}
|
|
|
|
// params.prompt = R"(// this function checks if the number n is prime
|
|
//bool is_prime(int n) {)";
|
|
|
|
gptneox_context * ctx;
|
|
g_ctx = &ctx;
|
|
|
|
// load the model
|
|
{
|
|
auto lparams = gptneox_context_default_params();
|
|
|
|
lparams.n_ctx = params.n_ctx;
|
|
lparams.n_parts = params.n_parts;
|
|
lparams.seed = params.seed;
|
|
lparams.f16_kv = params.memory_f16;
|
|
lparams.use_mmap = params.use_mmap;
|
|
lparams.use_mlock = params.use_mlock;
|
|
|
|
ctx = gptneox_init_from_file(params.model.c_str(), lparams);
|
|
|
|
if (ctx == NULL) {
|
|
fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, params.model.c_str());
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
if (!params.lora_adapter.empty()) {
|
|
int err = gptneox_apply_lora_from_file(ctx,
|
|
params.lora_adapter.c_str(),
|
|
params.lora_base.empty() ? NULL : params.lora_base.c_str(),
|
|
params.n_threads);
|
|
if (err != 0) {
|
|
fprintf(stderr, "%s: error: failed to apply lora adapter\n", __func__);
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
// print system information
|
|
{
|
|
fprintf(stderr, "\n");
|
|
fprintf(stderr, "system_info: n_threads = %d / %d | %s\n",
|
|
params.n_threads, std::thread::hardware_concurrency(), gptneox_print_system_info());
|
|
}
|
|
|
|
// determine the maximum memory usage needed to do inference for the given n_batch and n_predict parameters
|
|
// uncomment the "used_mem" line in llama.cpp to see the results
|
|
if (params.mem_test) {
|
|
{
|
|
const std::vector<gptneox_token> tmp(params.n_batch, 0);
|
|
gptneox_eval(ctx, tmp.data(), tmp.size(), 0, params.n_threads);
|
|
}
|
|
|
|
{
|
|
const std::vector<gptneox_token> tmp = { 0, };
|
|
gptneox_eval(ctx, tmp.data(), tmp.size(), params.n_predict - 1, params.n_threads);
|
|
}
|
|
|
|
gptneox_print_timings(ctx);
|
|
gptneox_free(ctx);
|
|
|
|
return 0;
|
|
}
|
|
|
|
std::string path_session = params.path_session;
|
|
std::vector<gptneox_token> session_tokens;
|
|
|
|
if (!path_session.empty()) {
|
|
fprintf(stderr, "%s: attempting to load saved session from %s..\n", __func__, path_session.c_str());
|
|
|
|
// REVIEW - fopen to check for existing session
|
|
FILE * fp = std::fopen(path_session.c_str(), "rb");
|
|
if (fp != NULL) {
|
|
std::fclose(fp);
|
|
|
|
session_tokens.resize(params.n_ctx);
|
|
size_t n_token_count_out = 0;
|
|
const size_t n_session_bytes = gptneox_load_session_file(ctx, path_session.c_str(), session_tokens.data(), session_tokens.capacity(), &n_token_count_out);
|
|
session_tokens.resize(n_token_count_out);
|
|
|
|
if (n_session_bytes > 0) {
|
|
fprintf(stderr, "%s: loaded %zu bytes of session data!\n", __func__, n_session_bytes);
|
|
} else {
|
|
fprintf(stderr, "%s: could not load session file, will recreate\n", __func__);
|
|
}
|
|
} else {
|
|
fprintf(stderr, "%s: session file does not exist, will create\n", __func__);
|
|
}
|
|
}
|
|
|
|
// tokenize the prompt
|
|
auto embd_inp = ::gptneox_tokenize(ctx, params.prompt, false); //true);
|
|
|
|
const int n_ctx = gptneox_n_ctx(ctx);
|
|
|
|
if ((int) embd_inp.size() > n_ctx - 4) {
|
|
fprintf(stderr, "%s: error: prompt is too long (%d tokens, max %d)\n", __func__, (int) embd_inp.size(), n_ctx - 4);
|
|
return 1;
|
|
}
|
|
|
|
// debug message about similarity of saved session, if applicable
|
|
size_t n_matching_session_tokens = 0;
|
|
if (session_tokens.size()) {
|
|
for (gptneox_token id : session_tokens) {
|
|
if (n_matching_session_tokens >= embd_inp.size() || id != embd_inp[n_matching_session_tokens]) {
|
|
break;
|
|
}
|
|
n_matching_session_tokens++;
|
|
}
|
|
if (n_matching_session_tokens >= embd_inp.size()) {
|
|
fprintf(stderr, "%s: session file has exact match for prompt!\n", __func__);
|
|
} else if (n_matching_session_tokens < (embd_inp.size() / 2)) {
|
|
fprintf(stderr, "%s: warning: session file has low similarity to prompt (%zu / %zu tokens); will mostly be reevaluated\n",
|
|
__func__, n_matching_session_tokens, embd_inp.size());
|
|
} else {
|
|
fprintf(stderr, "%s: session file matches %zu / %zu tokens of prompt\n",
|
|
__func__, n_matching_session_tokens, embd_inp.size());
|
|
}
|
|
}
|
|
|
|
// number of tokens to keep when resetting context
|
|
if (params.n_keep < 0 || params.n_keep > (int)embd_inp.size() || params.instruct) {
|
|
params.n_keep = (int)embd_inp.size();
|
|
}
|
|
|
|
// in instruct mode, we inject a prefix and a suffix to each input by the user
|
|
if (params.instruct) {
|
|
params.interactive_first = true;
|
|
params.antiprompt.push_back("<|prompter|>");
|
|
}
|
|
|
|
// enable interactive mode if reverse prompt or interactive start is specified
|
|
if (params.antiprompt.size() != 0 || params.interactive_first) {
|
|
params.interactive = true;
|
|
}
|
|
|
|
// determine newline token
|
|
auto gptneox_token_newline = ::gptneox_tokenize(ctx, "\n", false);
|
|
|
|
if (params.verbose_prompt) {
|
|
fprintf(stderr, "\n");
|
|
fprintf(stderr, "%s: prompt: '%s'\n", __func__, params.prompt.c_str());
|
|
fprintf(stderr, "%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size());
|
|
for (int i = 0; i < (int) embd_inp.size(); i++) {
|
|
fprintf(stderr, "%6d -> '%s'\n", embd_inp[i], gptneox_token_to_str(ctx, embd_inp[i]));
|
|
}
|
|
if (params.n_keep > 0) {
|
|
fprintf(stderr, "%s: static prompt based on n_keep: '", __func__);
|
|
for (int i = 0; i < params.n_keep; i++) {
|
|
fprintf(stderr, "%s", gptneox_token_to_str(ctx, embd_inp[i]));
|
|
}
|
|
fprintf(stderr, "'\n");
|
|
}
|
|
fprintf(stderr, "\n");
|
|
}
|
|
|
|
if (params.interactive) {
|
|
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
|
|
struct sigaction sigint_action;
|
|
sigint_action.sa_handler = sigint_handler;
|
|
sigemptyset (&sigint_action.sa_mask);
|
|
sigint_action.sa_flags = 0;
|
|
sigaction(SIGINT, &sigint_action, NULL);
|
|
#elif defined (_WIN32)
|
|
signal(SIGINT, sigint_handler);
|
|
#endif
|
|
|
|
fprintf(stderr, "%s: interactive mode on.\n", __func__);
|
|
|
|
if (params.antiprompt.size()) {
|
|
for (auto antiprompt : params.antiprompt) {
|
|
fprintf(stderr, "Reverse prompt: '%s'\n", antiprompt.c_str());
|
|
}
|
|
}
|
|
|
|
if (!params.input_prefix.empty()) {
|
|
fprintf(stderr, "Input prefix: '%s'\n", params.input_prefix.c_str());
|
|
}
|
|
}
|
|
fprintf(stderr, "sampling: repeat_last_n = %d, repeat_penalty = %f, presence_penalty = %f, frequency_penalty = %f, top_k = %d, tfs_z = %f, top_p = %f, typical_p = %f, temp = %f, mirostat = %d, mirostat_lr = %f, mirostat_ent = %f\n",
|
|
params.repeat_last_n, params.repeat_penalty, params.presence_penalty, params.frequency_penalty, params.top_k, params.tfs_z, params.top_p, params.typical_p, params.temp, params.mirostat, params.mirostat_eta, params.mirostat_tau);
|
|
fprintf(stderr, "generate: n_ctx = %d, n_batch = %d, n_predict = %d, n_keep = %d\n", n_ctx, params.n_batch, params.n_predict, params.n_keep);
|
|
fprintf(stderr, "\n\n");
|
|
|
|
// TODO: replace with ring-buffer
|
|
std::vector<gptneox_token> last_n_tokens(n_ctx);
|
|
std::fill(last_n_tokens.begin(), last_n_tokens.end(), 0);
|
|
|
|
if (params.interactive) {
|
|
fprintf(stderr, "== Running in interactive mode. ==\n"
|
|
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
|
|
" - Press Ctrl+C to interject at any time.\n"
|
|
#endif
|
|
" - Press Return to return control to RedPajama.\n"
|
|
" - If you want to submit another line, end your input in '\\'.\n\n");
|
|
is_interacting = params.interactive_first;
|
|
}
|
|
|
|
bool input_noecho = false;
|
|
|
|
// HACK - because session saving incurs a non-negligible delay, for now skip re-saving session
|
|
// if we loaded a session with at least 75% similarity. It's currently just used to speed up the
|
|
// initial prompt so it doesn't need to be an exact match.
|
|
bool need_to_save_session = !path_session.empty() && n_matching_session_tokens < (embd_inp.size() * 3 / 4);
|
|
|
|
|
|
int n_past = 0;
|
|
int n_remain = params.n_predict;
|
|
int n_consumed = 0;
|
|
int n_session_consumed = 0;
|
|
|
|
// the first thing we will do is to output the prompt, so set color accordingly
|
|
set_console_color(con_st, CONSOLE_COLOR_PROMPT);
|
|
|
|
std::vector<gptneox_token> embd;
|
|
|
|
while (n_remain != 0 || params.interactive) {
|
|
// predict
|
|
if (embd.size() > 0) {
|
|
// infinite text generation via context swapping
|
|
// if we run out of context:
|
|
// - take the n_keep first tokens from the original prompt (via n_past)
|
|
// - take half of the last (n_ctx - n_keep) tokens and recompute the logits in batches
|
|
if (n_past + (int) embd.size() > n_ctx) {
|
|
const int n_left = n_past - params.n_keep;
|
|
|
|
n_past = params.n_keep;
|
|
|
|
// insert n_left/2 tokens at the start of embd from last_n_tokens
|
|
embd.insert(embd.begin(), last_n_tokens.begin() + n_ctx - n_left/2 - embd.size(), last_n_tokens.end() - embd.size());
|
|
|
|
// REVIEW - stop saving session if we run out of context
|
|
path_session = "";
|
|
|
|
//printf("\n---\n");
|
|
//printf("resetting: '");
|
|
//for (int i = 0; i < (int) embd.size(); i++) {
|
|
// printf("%s", gptneox_token_to_str(ctx, embd[i]));
|
|
//}
|
|
//printf("'\n");
|
|
//printf("\n---\n");
|
|
}
|
|
|
|
// try to reuse a matching prefix from the loaded session instead of re-eval (via n_past)
|
|
// REVIEW
|
|
if (n_session_consumed < (int) session_tokens.size()) {
|
|
size_t i = 0;
|
|
for ( ; i < embd.size(); i++) {
|
|
if (embd[i] != session_tokens[n_session_consumed]) {
|
|
session_tokens.resize(n_session_consumed);
|
|
break;
|
|
}
|
|
|
|
n_past++;
|
|
n_session_consumed++;
|
|
|
|
if (n_session_consumed >= (int) session_tokens.size()) {
|
|
break;
|
|
}
|
|
}
|
|
if (i > 0) {
|
|
embd.erase(embd.begin(), embd.begin() + i);
|
|
}
|
|
}
|
|
|
|
// evaluate tokens in batches
|
|
// embd is typically prepared beforehand to fit within a batch, but not always
|
|
for (int i = 0; i < (int) embd.size(); i += params.n_batch) {
|
|
int n_eval = (int) embd.size() - i;
|
|
if (n_eval > params.n_batch) {
|
|
n_eval = params.n_batch;
|
|
}
|
|
if (gptneox_eval(ctx, &embd[i], n_eval, n_past, params.n_threads)) {
|
|
fprintf(stderr, "%s : failed to eval\n", __func__);
|
|
return 1;
|
|
}
|
|
n_past += n_eval;
|
|
}
|
|
|
|
if (embd.size() > 0 && !path_session.empty()) {
|
|
session_tokens.insert(session_tokens.end(), embd.begin(), embd.end());
|
|
n_session_consumed = session_tokens.size();
|
|
}
|
|
}
|
|
|
|
embd.clear();
|
|
|
|
if ((int) embd_inp.size() <= n_consumed && !is_interacting) {
|
|
// out of user input, sample next token
|
|
const float temp = params.temp;
|
|
const int32_t top_k = params.top_k <= 0 ? gptneox_n_vocab(ctx) : params.top_k;
|
|
const float top_p = params.top_p;
|
|
const float tfs_z = params.tfs_z;
|
|
const float typical_p = params.typical_p;
|
|
const int32_t repeat_last_n = params.repeat_last_n < 0 ? n_ctx : params.repeat_last_n;
|
|
const float repeat_penalty = params.repeat_penalty;
|
|
const float alpha_presence = params.presence_penalty;
|
|
const float alpha_frequency = params.frequency_penalty;
|
|
const int mirostat = params.mirostat;
|
|
const float mirostat_tau = params.mirostat_tau;
|
|
const float mirostat_eta = params.mirostat_eta;
|
|
const bool penalize_nl = params.penalize_nl;
|
|
|
|
// optionally save the session on first sample (for faster prompt loading next time)
|
|
if (!path_session.empty() && need_to_save_session) {
|
|
need_to_save_session = false;
|
|
gptneox_save_session_file(ctx, path_session.c_str(), session_tokens.data(), session_tokens.size());
|
|
}
|
|
|
|
gptneox_token id = 0;
|
|
|
|
{
|
|
auto logits = gptneox_get_logits(ctx);
|
|
auto n_vocab = gptneox_n_vocab(ctx);
|
|
|
|
// Apply params.logit_bias map
|
|
for (auto it = params.logit_bias.begin(); it != params.logit_bias.end(); it++) {
|
|
logits[it->first] += it->second;
|
|
}
|
|
|
|
std::vector<gptneox_token_data> candidates;
|
|
candidates.reserve(n_vocab);
|
|
for (gptneox_token token_id = 0; token_id < n_vocab; token_id++) {
|
|
candidates.emplace_back(gptneox_token_data{token_id, logits[token_id], 0.0f});
|
|
}
|
|
|
|
gptneox_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
|
|
|
|
// Apply penalties
|
|
gptneox_token nl_token = gptneox_str_to_token(ctx, "\n");
|
|
float nl_logit = logits[nl_token];
|
|
auto last_n_repeat = std::min(std::min((int)last_n_tokens.size(), repeat_last_n), n_ctx);
|
|
gptneox_sample_repetition_penalty(ctx, &candidates_p,
|
|
last_n_tokens.data() + last_n_tokens.size() - last_n_repeat,
|
|
last_n_repeat, repeat_penalty);
|
|
gptneox_sample_frequency_and_presence_penalties(ctx, &candidates_p,
|
|
last_n_tokens.data() + last_n_tokens.size() - last_n_repeat,
|
|
last_n_repeat, alpha_frequency, alpha_presence);
|
|
if (!penalize_nl) {
|
|
logits[nl_token] = nl_logit;
|
|
}
|
|
|
|
if (temp <= 0) {
|
|
// Greedy sampling
|
|
id = gptneox_sample_token_greedy(ctx, &candidates_p);
|
|
} else {
|
|
if (mirostat == 1) {
|
|
static float mirostat_mu = 2.0f * mirostat_tau;
|
|
const int mirostat_m = 100;
|
|
gptneox_sample_temperature(ctx, &candidates_p, temp);
|
|
id = gptneox_sample_token_mirostat(ctx, &candidates_p, mirostat_tau, mirostat_eta, mirostat_m, &mirostat_mu);
|
|
} else if (mirostat == 2) {
|
|
static float mirostat_mu = 2.0f * mirostat_tau;
|
|
gptneox_sample_temperature(ctx, &candidates_p, temp);
|
|
id = gptneox_sample_token_mirostat_v2(ctx, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu);
|
|
} else {
|
|
// Temperature sampling
|
|
gptneox_sample_top_k(ctx, &candidates_p, top_k, 1);
|
|
gptneox_sample_tail_free(ctx, &candidates_p, tfs_z, 1);
|
|
gptneox_sample_typical(ctx, &candidates_p, typical_p, 1);
|
|
gptneox_sample_top_p(ctx, &candidates_p, top_p, 1);
|
|
gptneox_sample_temperature(ctx, &candidates_p, temp);
|
|
id = gptneox_sample_token(ctx, &candidates_p);
|
|
}
|
|
}
|
|
// printf("`%d`", candidates_p.size);
|
|
|
|
last_n_tokens.erase(last_n_tokens.begin());
|
|
last_n_tokens.push_back(id);
|
|
}
|
|
|
|
// replace end of text token with newline token when in interactive mode
|
|
if (id == gptneox_token_eos() && params.interactive && !params.instruct) {
|
|
id = gptneox_token_newline.front();
|
|
if (params.antiprompt.size() != 0) {
|
|
// tokenize and inject first reverse prompt
|
|
const auto first_antiprompt = ::gptneox_tokenize(ctx, params.antiprompt.front(), false);
|
|
embd_inp.insert(embd_inp.end(), first_antiprompt.begin(), first_antiprompt.end());
|
|
}
|
|
}
|
|
|
|
// add it to the context
|
|
embd.push_back(id);
|
|
|
|
// echo this to console
|
|
input_noecho = false;
|
|
|
|
// decrement remaining sampling budget
|
|
--n_remain;
|
|
} else {
|
|
// some user input remains from prompt or interaction, forward it to processing
|
|
while ((int) embd_inp.size() > n_consumed) {
|
|
embd.push_back(embd_inp[n_consumed]);
|
|
last_n_tokens.erase(last_n_tokens.begin());
|
|
last_n_tokens.push_back(embd_inp[n_consumed]);
|
|
++n_consumed;
|
|
if ((int) embd.size() >= params.n_batch) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// display text
|
|
if (!input_noecho) {
|
|
for (auto id : embd) {
|
|
printf("%s", gptneox_token_to_str(ctx, id));
|
|
}
|
|
fflush(stdout);
|
|
}
|
|
// reset color to default if we there is no pending user input
|
|
if (!input_noecho && (int)embd_inp.size() == n_consumed) {
|
|
set_console_color(con_st, CONSOLE_COLOR_DEFAULT);
|
|
}
|
|
|
|
// in interactive mode, and not currently processing queued inputs;
|
|
// check if we should prompt the user for more
|
|
if (params.interactive && (int) embd_inp.size() <= n_consumed) {
|
|
|
|
// check for reverse prompt
|
|
if (params.antiprompt.size()) {
|
|
std::string last_output;
|
|
for (auto id : last_n_tokens) {
|
|
last_output += gptneox_token_to_str(ctx, id);
|
|
}
|
|
|
|
// Check if each of the reverse prompts appears at the end of the output.
|
|
for (std::string & antiprompt : params.antiprompt) {
|
|
if (last_output.find(antiprompt.c_str(), last_output.length() - antiprompt.length(), antiprompt.length()) != std::string::npos) {
|
|
is_interacting = true;
|
|
set_console_color(con_st, CONSOLE_COLOR_USER_INPUT);
|
|
fflush(stdout);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (n_past > 0 && is_interacting) {
|
|
// potentially set color to indicate we are taking user input
|
|
set_console_color(con_st, CONSOLE_COLOR_USER_INPUT);
|
|
|
|
#if defined (_WIN32)
|
|
// Windows: must reactivate sigint handler after each signal
|
|
signal(SIGINT, sigint_handler);
|
|
#endif
|
|
|
|
if (params.instruct) {
|
|
printf("\n> ");
|
|
}
|
|
|
|
std::string buffer;
|
|
if (!params.input_prefix.empty()) {
|
|
buffer += params.input_prefix;
|
|
printf("%s", buffer.c_str());
|
|
}
|
|
|
|
std::string line;
|
|
bool another_line = true;
|
|
do {
|
|
#if defined(_WIN32)
|
|
std::wstring wline;
|
|
if (!std::getline(std::wcin, wline)) {
|
|
// input stream is bad or EOF received
|
|
return 0;
|
|
}
|
|
win32_utf8_encode(wline, line);
|
|
#else
|
|
if (!std::getline(std::cin, line)) {
|
|
// input stream is bad or EOF received
|
|
return 0;
|
|
}
|
|
#endif
|
|
if (line.empty() || line.back() != '\\') {
|
|
another_line = false;
|
|
} else {
|
|
line.pop_back(); // Remove the continue character
|
|
}
|
|
buffer += line + '\n'; // Append the line to the result
|
|
} while (another_line);
|
|
|
|
// done taking input, reset color
|
|
set_console_color(con_st, CONSOLE_COLOR_DEFAULT);
|
|
|
|
// Add tokens to embd only if the input buffer is non-empty
|
|
// Entering a empty line lets the user pass control back
|
|
if (buffer.length() > 1) {
|
|
|
|
auto line_inp = ::gptneox_tokenize(ctx, buffer, false);
|
|
embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end());
|
|
n_remain -= line_inp.size();
|
|
}
|
|
|
|
input_noecho = true; // do not echo this again
|
|
}
|
|
|
|
if (n_past > 0) {
|
|
is_interacting = false;
|
|
}
|
|
}
|
|
|
|
// end of text token
|
|
if (!embd.empty() && embd.back() == gptneox_token_eos()) {
|
|
if (params.instruct) {
|
|
is_interacting = true;
|
|
} else {
|
|
fprintf(stderr, " [end of text]\n");
|
|
break;
|
|
}
|
|
}
|
|
|
|
// In interactive mode, respect the maximum number of tokens and drop back to user input when reached.
|
|
if (params.interactive && n_remain <= 0 && params.n_predict != -1) {
|
|
n_remain = params.n_predict;
|
|
is_interacting = true;
|
|
}
|
|
}
|
|
|
|
#if defined (_WIN32)
|
|
signal(SIGINT, SIG_DFL);
|
|
#endif
|
|
printf("\n\n");
|
|
gptneox_print_timings(ctx);
|
|
gptneox_free(ctx);
|
|
|
|
set_console_color(con_st, CONSOLE_COLOR_DEFAULT);
|
|
|
|
return 0;
|
|
}
|