From 34ab5268432fd287caa68d60bdd8aef411def3fa Mon Sep 17 00:00:00 2001 From: anzz1 Date: Sat, 25 Mar 2023 22:29:22 +0200 Subject: [PATCH 1/4] (Windows) Set console to UTF-8 on init (#420) Sets console codepage to 65001 (CP_UTF8) on start for both input and output, should fix problems with UTF-8 characters. --- examples/main/main.cpp | 55 ++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 7bb2b6bc4..9af8a7405 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -23,6 +23,8 @@ extern "C" __declspec(dllimport) void* __stdcall GetStdHandle(unsigned long nStdHandle); extern "C" __declspec(dllimport) int __stdcall GetConsoleMode(void* hConsoleHandle, unsigned long* lpMode); extern "C" __declspec(dllimport) int __stdcall SetConsoleMode(void* hConsoleHandle, unsigned long dwMode); +extern "C" __declspec(dllimport) int __stdcall SetConsoleCP(unsigned int wCodePageID); +extern "C" __declspec(dllimport) int __stdcall SetConsoleOutputCP(unsigned int wCodePageID); #endif #define ANSI_COLOR_RED "\x1b[31m" @@ -44,17 +46,6 @@ enum console_state { static console_state con_st = CONSOLE_STATE_DEFAULT; static bool con_use_color = false; -void enable_console_colors() { -#if defined (_WIN32) - // Enable ANSI colors on Windows 10+ - unsigned long dwMode = 0; - void* hConOut = GetStdHandle((unsigned long)-11); // STD_OUTPUT_HANDLE (-11) - if (hConOut && hConOut != (void*)-1 && GetConsoleMode(hConOut, &dwMode) && !(dwMode & 0x4)) { - SetConsoleMode(hConOut, dwMode | 0x4); // ENABLE_VIRTUAL_TERMINAL_PROCESSING (0x4) - } -#endif -} - void set_console_state(console_state new_st) { if (!con_use_color) return; // only emit color code if state changed @@ -90,6 +81,32 @@ void sigint_handler(int signo) { } #endif +#if defined (_WIN32) +void win32_console_init(void) { + unsigned long dwMode = 0; + void* hConOut = GetStdHandle((unsigned long)-11); // STD_OUTPUT_HANDLE (-11) + if (!hConOut || hConOut == (void*)-1 || !GetConsoleMode(hConOut, &dwMode)) { + hConOut = GetStdHandle((unsigned long)-12); // STD_ERROR_HANDLE (-12) + if (hConOut && (hConOut == (void*)-1 || !GetConsoleMode(hConOut, &dwMode))) { + hConOut = 0; + } + } + if (hConOut) { + // Enable ANSI colors on Windows 10+ + if (con_use_color && !(dwMode & 0x4)) { + SetConsoleMode(hConOut, dwMode | 0x4); // ENABLE_VIRTUAL_TERMINAL_PROCESSING (0x4) + } + // Set console output codepage to UTF8 + SetConsoleOutputCP(65001); // CP_UTF8 + } + void* hConIn = GetStdHandle((unsigned long)-10); // STD_INPUT_HANDLE (-10) + if (hConIn && hConIn != (void*)-1 && GetConsoleMode(hConIn, &dwMode)) { + // Set console input codepage to UTF8 + SetConsoleCP(65001); // CP_UTF8 + } +} +#endif + int main(int argc, char ** argv) { gpt_params params; params.model = "models/llama-7B/ggml-model.bin"; @@ -98,6 +115,15 @@ int main(int argc, char ** argv) { return 1; } + + // save choice to use color for later + // (note for later: this is a slightly awkward choice) + con_use_color = params.use_color; + +#if defined (_WIN32) + win32_console_init(); +#endif + if (params.perplexity) { printf("\n************\n"); printf("%s: please use the 'perplexity' tool for perplexity calculations\n", __func__); @@ -130,10 +156,6 @@ int main(int argc, char ** argv) { params.prompt = gpt_random_prompt(rng); } - // save choice to use color for later - // (note for later: this is a slightly awkward choice) - con_use_color = params.use_color; - // params.prompt = R"(// this function checks if the number n is prime //bool is_prime(int n) {)"; @@ -285,9 +307,6 @@ int main(int argc, char ** argv) { int n_consumed = 0; // the first thing we will do is to output the prompt, so set color accordingly - if (params.use_color) { - enable_console_colors(); - } set_console_state(CONSOLE_STATE_PROMPT); std::vector embd; From 2f7bf7dd7cd7299874d582f7f34834418abf4057 Mon Sep 17 00:00:00 2001 From: anzz1 Date: Sat, 25 Mar 2023 23:38:11 +0200 Subject: [PATCH 2/4] CMake / CI additions (#497) * CMake: Add AVX512 option * CI: Add AVX/AVX512 builds (Windows) (AVX512 tests can only be run when the worker happens to support it, building works anyway) * CMake: Fix sanitizer linkage ( merged #468 ) * CI: Add sanitizer builds (Ubuntu) * CI: Fix release tagging (change @zendesk/action-create-release to @anzz1/action-create-release until upstream PR Added commitish as input zendesk/action-create-release#32 is merged) --- .github/workflows/build.yml | 113 ++++++++++++++++++++++++++++++++---- CMakeLists.txt | 14 ++++- 2 files changed, 114 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e9826a735..126e1e66d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,6 +64,38 @@ jobs: cd build ctest --output-on-failure + ubuntu-latest-cmake-sanitizer: + runs-on: ubuntu-latest + + strategy: + matrix: + sanitizer: [ADDRESS, THREAD, UNDEFINED] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v1 + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install build-essential + + - name: Build + id: cmake_build + run: | + mkdir build + cd build + cmake .. -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON + cmake --build . --config Release + + - name: Test + id: cmake_test + run: | + cd build + ctest --output-on-failure + macOS-latest-make: runs-on: macos-latest @@ -112,6 +144,16 @@ jobs: windows-latest-cmake: runs-on: windows-latest + strategy: + matrix: + include: + - build: 'avx2' + defines: '' + - build: 'avx' + defines: '-DLLAMA_AVX2=OFF' + - build: 'avx512' + defines: '-DLLAMA_AVX512=ON' + steps: - name: Clone id: checkout @@ -122,11 +164,21 @@ jobs: run: | mkdir build cd build - cmake .. + cmake .. ${{ matrix.defines }} cmake --build . --config Release + - name: Check AVX512F support + id: check_avx512f + if: ${{ matrix.build == 'avx512' }} + continue-on-error: true + run: | + cd build + Set-Content -Path .\avx512f.exe -Value ([Convert]::FromBase64String('TVqQAAMAAAAEAAAA//8AALgAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAyAAAAA4fug4AtAnNIbgBTM0hVGhpcyBwcm9ncmFtIGNhbm5vdCBiZSBydW4gaW4gRE9TIG1vZGUuDQ0KJAAAAAAAAAClmfXY4fibi+H4m4vh+JuL4fiai+P4m4si98aL4vibi7Xbq4vg+JuLUmljaOH4m4sAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABQRQAATAEBAGo6H2QAAAAAAAAAAOAADwELAQYAAAIAAAAAAAAAAAAADBAAAAAQAAAAIAAAAABAAAAQAAAAAgAABAAAAAAAAAAEAAAAAAAAAAAgAAAAAgAAAAAAAAMAAAAAABAAABAAAAAAEAAAEAAAAAAAABAAAAAAAAAAAAAAAFQQAAAoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC50ZXh0AAAAsgAAAAAQAAAAAgAAAAIAAAAAAAAAAAAAAAAAACAAAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACUEAAAiBAAAAAAAABVi+xRUVNTuAcAAAAPosHrEGaD4wGJXfxbg0X8MI1F+GoAUI1F/GoBUGr1/xUAEEAAUP8VBBBAAItF/FuDwND32BvAQMnDzMx8EAAAAAAAAAAAAACkEAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlBAAAIgQAAAAAAAApANXcml0ZUZpbGUAuQFHZXRTdGRIYW5kbGUAAEtFUk5FTDMyLmRsbAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==')) -AsByteStream + .\avx512f.exe && echo " AVX512F: YES" && ( echo HAS_AVX512F=1 >> $env:GITHUB_ENV ) || echo " AVX512F: NO" + - name: Test id: cmake_test + if: ${{ matrix.build != 'avx512' || env.HAS_AVX512F == '1' }} # Test AVX-512 only when possible run: | cd build ctest -C Release --output-on-failure @@ -140,12 +192,39 @@ jobs: id: pack_artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} run: | - 7z a llama-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-x64.zip .\build\bin\Release\* + 7z a llama-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip .\build\bin\Release\* + + - name: Upload artifacts + if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} + uses: actions/upload-artifact@v3 + with: + path: | + llama-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip + + release: + if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} + + runs-on: ubuntu-latest + + needs: + - ubuntu-latest-make + - ubuntu-latest-cmake + - macOS-latest-make + - macOS-latest-cmake + - windows-latest-cmake + + steps: + - name: Download artifacts + id: download-artifact + uses: actions/download-artifact@v3 + + - name: Get commit hash + id: commit + uses: pr-mpt/actions-commit-hash@v2 - name: Create release id: create_release - if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: zendesk/action-create-release@v1 + uses: anzz1/action-create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -153,15 +232,25 @@ jobs: - name: Upload release id: upload_release - if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: actions/github-script@v3 with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: .\llama-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-x64.zip - asset_name: llama-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-x64.zip - asset_content_type: application/octet-stream + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const path = require('path'); + const fs = require('fs'); + const release_id = '${{ steps.create_release.outputs.id }}'; + for (let file of await fs.readdirSync('./artifact')) { + if (path.extname(file) === '.zip') { + console.log('uploadReleaseAsset', file); + await github.repos.uploadReleaseAsset({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: release_id, + name: file, + data: await fs.readFileSync(`./artifact/${file}`) + }); + } + } # ubuntu-latest-gcc: # runs-on: ubuntu-latest diff --git a/CMakeLists.txt b/CMakeLists.txt index a1ff5a44e..27a222a16 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,6 +54,7 @@ option(LLAMA_SANITIZE_UNDEFINED "llama: enable undefined sanitizer" # instruction set specific option(LLAMA_AVX "llama: enable AVX" ON) option(LLAMA_AVX2 "llama: enable AVX2" ON) +option(LLAMA_AVX512 "llama: enable AVX512" OFF) option(LLAMA_FMA "llama: enable FMA" ON) # 3rd party libs @@ -75,14 +76,17 @@ find_package(Threads REQUIRED) if (NOT MSVC) if (LLAMA_SANITIZE_THREAD) add_compile_options(-fsanitize=thread) + link_libraries(-fsanitize=thread) endif() if (LLAMA_SANITIZE_ADDRESS) add_compile_options(-fsanitize=address -fno-omit-frame-pointer) + link_libraries(-fsanitize=address) endif() if (LLAMA_SANITIZE_UNDEFINED) add_compile_options(-fsanitize=undefined) + link_libraries(-fsanitize=undefined) endif() endif() @@ -185,7 +189,9 @@ if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$") message(STATUS "x86 detected") if (MSVC) - if (LLAMA_AVX2) + if (LLAMA_AVX512) + add_compile_options(/arch:AVX512) + elseif (LLAMA_AVX2) add_compile_options(/arch:AVX2) elseif (LLAMA_AVX) add_compile_options(/arch:AVX) @@ -201,6 +207,12 @@ elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "^(x86_64|i686|AMD64)$") if (LLAMA_AVX2) add_compile_options(-mavx2) endif() + if (LLAMA_AVX512) + add_compile_options(-mavx512f) + # add_compile_options(-mavx512cd) + # add_compile_options(-mavx512dq) + # add_compile_options(-mavx512bw) + endif() endif() else() # TODO: support PowerPC From f732695cd57fb41e3a1be625cec4edf5be45b40a Mon Sep 17 00:00:00 2001 From: jp-x-g Date: Sat, 25 Mar 2023 14:53:55 -0700 Subject: [PATCH 3/4] Clarify console output in convert-pth-to-ggml.py (#512) "Processing part 1 of 3" instead of "Processing part 0" --- convert-pth-to-ggml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert-pth-to-ggml.py b/convert-pth-to-ggml.py index f0f6b0ec4..ccf2c57b1 100644 --- a/convert-pth-to-ggml.py +++ b/convert-pth-to-ggml.py @@ -161,7 +161,7 @@ def main(): for p in range(n_parts): - print(f"Processing part {p}\n") + print(f"Processing part {p+1} of {n_parts}\n") fname_model = f"{dir_model}/consolidated.0{p}.pth" fname_out = f"{dir_model}/ggml-model-{ftype_str[ftype]}.bin{'' if p == 0 else '.' + str(p)}" From 19726169b379bebc96189673a19b89ab1d307659 Mon Sep 17 00:00:00 2001 From: anzz1 Date: Sun, 26 Mar 2023 00:13:28 +0200 Subject: [PATCH 4/4] CI: Run other sanitizer builds even if one fails (#511) applies only to sanitizer builds so they wont be cancelled --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 126e1e66d..2538d8566 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -67,6 +67,8 @@ jobs: ubuntu-latest-cmake-sanitizer: runs-on: ubuntu-latest + continue-on-error: true + strategy: matrix: sanitizer: [ADDRESS, THREAD, UNDEFINED]