From 4fd3e29297e3246a7be291932c115636fadb0f52 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 20 May 2023 10:13:19 +0300 Subject: [PATCH 1/4] ggml : fix scalar implementation of Q4_1 dot --- ggml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml.c b/ggml.c index 1cb89636a..101cb733b 100644 --- a/ggml.c +++ b/ggml.c @@ -2481,7 +2481,7 @@ static void ggml_vec_dot_q4_1_q8_1(const int n, float * restrict s, const void * sumi += (v0 * y[i].qs[j]) + (v1 * y[i].qs[j + qk/2]); } - sumf += (GGML_FP16_TO_FP32(x[i]).d*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s; + sumf += (GGML_FP16_TO_FP32(x[i].d)*y[i].d)*sumi + GGML_FP16_TO_FP32(x[i].m)*y[i].s; } *s = sumf; From 8a203f9fa1b24e010be8f35ebbbd6786293684cb Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 20 May 2023 10:14:31 +0300 Subject: [PATCH 2/4] llama : fix compile warnings in llama_set_state_data() --- llama.cpp | 4 ++-- llama.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llama.cpp b/llama.cpp index 6ebe85d0f..68e3bec13 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2618,8 +2618,8 @@ size_t llama_copy_state_data(struct llama_context * ctx, uint8_t * dst) { } // Sets the state reading from the specified source address -size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src) { - const uint8_t * inp = src; +size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) { + uint8_t * inp = src; // set rng { diff --git a/llama.h b/llama.h index fd3f21e5f..8623e08ce 100644 --- a/llama.h +++ b/llama.h @@ -138,7 +138,7 @@ extern "C" { // Set the state reading from the specified address // Returns the number of bytes read - LLAMA_API size_t llama_set_state_data(struct llama_context * ctx, const uint8_t * src); + LLAMA_API size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src); // Save/load session file LLAMA_API bool llama_load_session_file(struct llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out); From 503db28849d8641d66244385e7e9649608a2e4d0 Mon Sep 17 00:00:00 2001 From: Maxime <672982+maximegmd@users.noreply.github.com> Date: Sat, 20 May 2023 09:22:37 +0200 Subject: [PATCH 3/4] llama : fix name shadowing and C4146 (#1526) * Fix name shadowing and C4146 * Fix if macros not using defined when required * Update llama-util.h Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update llama-util.h Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Code style Co-authored-by: Georgi Gerganov --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Georgi Gerganov --- ggml.c | 4 ++-- llama-util.h | 40 ++++++++++++++++++++-------------------- llama.cpp | 7 ++++--- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/ggml.c b/ggml.c index 101cb733b..939ab4d62 100644 --- a/ggml.c +++ b/ggml.c @@ -512,7 +512,7 @@ static inline int hsum_i32_4(const __m128i a) { return _mm_cvtsi128_si32(_mm_add_epi32(sum64, hi32)); } -#if __AVX2__ || __AVX512F__ +#if defined(__AVX2__) || defined(__AVX512F__) // spread 32 bits to 32 bytes { 0x00, 0xFF } static inline __m256i bytes_from_bits_32(const uint8_t * x) { uint32_t x32; @@ -688,7 +688,7 @@ static inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128 #endif // __AVX__ || __AVX2__ || __AVX512F__ #endif // defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__) -#if __ARM_NEON +#if defined(__ARM_NEON) #if !defined(__aarch64__) diff --git a/llama-util.h b/llama-util.h index 88ec28dca..a79c5dadd 100644 --- a/llama-util.h +++ b/llama-util.h @@ -101,12 +101,12 @@ struct llama_file { LLAMA_ASSERT(ret == 0); // same } - void read_raw(void * ptr, size_t size) { - if (size == 0) { + void read_raw(void * ptr, size_t len) const { + if (len == 0) { return; } errno = 0; - std::size_t ret = std::fread(ptr, size, 1, fp); + std::size_t ret = std::fread(ptr, len, 1, fp); if (ferror(fp)) { throw std::runtime_error(format("read error: %s", strerror(errno))); } @@ -127,12 +127,12 @@ struct llama_file { return std::string(chars.data(), len); } - void write_raw(const void * ptr, size_t size) { - if (size == 0) { + void write_raw(const void * ptr, size_t len) const { + if (len == 0) { return; } errno = 0; - size_t ret = std::fwrite(ptr, size, 1, fp); + size_t ret = std::fwrite(ptr, len, 1, fp); if (ret != 1) { throw std::runtime_error(format("write error: %s", strerror(errno))); } @@ -267,9 +267,9 @@ struct llama_mlock { } } - void init(void * addr) { - LLAMA_ASSERT(this->addr == NULL && this->size == 0); - this->addr = addr; + void init(void * ptr) { + LLAMA_ASSERT(addr == NULL && size == 0); + addr = ptr; } void grow_to(size_t target_size) { @@ -340,14 +340,14 @@ struct llama_mlock { return (size_t) si.dwPageSize; } - bool raw_lock(void * addr, size_t size) { + bool raw_lock(void * ptr, size_t len) { for (int tries = 1; ; tries++) { - if (VirtualLock(addr, size)) { + if (VirtualLock(ptr, len)) { return true; } if (tries == 2) { fprintf(stderr, "warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n", - size, this->size, llama_format_win_err(GetLastError()).c_str()); + len, size, llama_format_win_err(GetLastError()).c_str()); return false; } @@ -363,7 +363,7 @@ struct llama_mlock { // is equal to the number of pages in its minimum working set minus // a small overhead." // Hopefully a megabyte is enough overhead: - size_t increment = size + 1048576; + size_t increment = len + 1048576; // The minimum must be <= the maximum, so we need to increase both: min_ws_size += increment; max_ws_size += increment; @@ -375,8 +375,8 @@ struct llama_mlock { } } - void raw_unlock(void * addr, size_t size) { - if (!VirtualUnlock(addr, size)) { + void raw_unlock(void * ptr, size_t len) { + if (!VirtualUnlock(ptr, len)) { fprintf(stderr, "warning: failed to VirtualUnlock buffer: %s\n", llama_format_win_err(GetLastError()).c_str()); } @@ -388,12 +388,12 @@ struct llama_mlock { return (size_t) 65536; } - bool raw_lock(const void * addr, size_t size) { + bool raw_lock(const void * addr, size_t len) { fprintf(stderr, "warning: mlock not supported on this system\n"); return false; } - void raw_unlock(const void * addr, size_t size) {} + void raw_unlock(const void * addr, size_t len) {} #endif }; @@ -404,10 +404,10 @@ struct llama_buffer { llama_buffer() = default; - void resize(size_t size) { + void resize(size_t len) { delete[] addr; - addr = new uint8_t[size]; - this->size = size; + addr = new uint8_t[len]; + size = len; } ~llama_buffer() { diff --git a/llama.cpp b/llama.cpp index 68e3bec13..dd449592a 100644 --- a/llama.cpp +++ b/llama.cpp @@ -45,6 +45,7 @@ enum e_model { MODEL_65B, }; + static const size_t MB = 1024*1024; // computed for n_ctx == 2048 @@ -110,7 +111,7 @@ struct llama_hparams { enum llama_ftype ftype = LLAMA_FTYPE_MOSTLY_F16; bool operator!=(const llama_hparams & other) const { - return memcmp(this, &other, sizeof(llama_hparams)); + return static_cast(memcmp(this, &other, sizeof(llama_hparams))); } }; @@ -502,7 +503,7 @@ struct llama_file_loader { if (file_version >= LLAMA_FILE_VERSION_GGJT_V1) { // skip to the next multiple of 32 bytes - file.seek(-file.tell() & 31, SEEK_CUR); + file.seek(-static_cast(file.tell()) & 31, SEEK_CUR); } shard.file_idx = file_idx; shard.file_off = file.tell(); @@ -577,7 +578,7 @@ struct llama_file_saver { file.write_u32(new_type); file.write_raw(tensor.ne.data(), sizeof(tensor.ne[0]) * tensor.ne.size()); file.write_raw(tensor.name.data(), tensor.name.size()); - file.seek(-file.tell() & 31, SEEK_CUR); + file.seek(-static_cast(file.tell()) & 31, SEEK_CUR); LLAMA_ASSERT(new_size == llama_calc_tensor_size(tensor.ne, new_type)); file.write_raw(new_data, new_size); } From d2c59b8ba498ab01e65203dde6fe95236d20f6e7 Mon Sep 17 00:00:00 2001 From: DannyDaemonic Date: Sat, 20 May 2023 00:40:02 -0700 Subject: [PATCH 4/4] Fix for mingw (#1462) --- examples/common.cpp | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/examples/common.cpp b/examples/common.cpp index e89df537e..1308f8410 100644 --- a/examples/common.cpp +++ b/examples/common.cpp @@ -578,6 +578,37 @@ void console_set_color(console_state & con_st, console_color_t color) { } char32_t getchar32() { +#if defined(_WIN32) + HANDLE hConsole = GetStdHandle(STD_INPUT_HANDLE); + wchar_t high_surrogate = 0; + + while (true) { + INPUT_RECORD record; + DWORD count; + if (!ReadConsoleInputW(hConsole, &record, 1, &count) || count == 0) { + return WEOF; + } + + if (record.EventType == KEY_EVENT && record.Event.KeyEvent.bKeyDown) { + wchar_t wc = record.Event.KeyEvent.uChar.UnicodeChar; + if (wc == 0) { + continue; + } + + if ((wc >= 0xD800) && (wc <= 0xDBFF)) { // Check if wc is a high surrogate + high_surrogate = wc; + continue; + } else if ((wc >= 0xDC00) && (wc <= 0xDFFF)) { // Check if wc is a low surrogate + if (high_surrogate != 0) { // Check if we have a high surrogate + return ((high_surrogate - 0xD800) << 10) + (wc - 0xDC00) + 0x10000; + } + } + + high_surrogate = 0; // Reset the high surrogate + return static_cast(wc); + } + } +#else wchar_t wc = getwchar(); if (static_cast(wc) == WEOF) { return WEOF; @@ -596,6 +627,7 @@ char32_t getchar32() { #endif return static_cast(wc); +#endif } void pop_cursor(console_state & con_st) {