diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..81ccc731a --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +models/ggml-vocab-*.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2d747e688..4b41a4a41 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,6 +33,7 @@ jobs: id: checkout uses: actions/checkout@v4 with: + lfs: true fetch-depth: 0 - name: Dependencies @@ -91,6 +92,7 @@ jobs: id: checkout uses: actions/checkout@v4 with: + lfs: true fetch-depth: 0 - name: Dependencies @@ -153,6 +155,8 @@ jobs: - name: Clone id: checkout uses: actions/checkout@v4 + with: + lfs: true - name: Dependencies id: depends @@ -188,6 +192,8 @@ jobs: - name: Clone id: checkout uses: actions/checkout@v4 + with: + lfs: true - name: Dependencies id: depends @@ -211,6 +217,7 @@ jobs: id: checkout uses: actions/checkout@v4 with: + lfs: true fetch-depth: 0 - name: Dependencies @@ -285,6 +292,8 @@ jobs: # - name: Clone # id: checkout # uses: actions/checkout@v4 +# with: +# lfs: true # # - name: Dependencies # id: depends @@ -319,6 +328,8 @@ jobs: - name: Clone id: checkout uses: actions/checkout@v4 + with: + lfs: true - name: Dependencies id: depends @@ -347,6 +358,8 @@ jobs: - name: Clone id: checkout uses: actions/checkout@v4 + with: + lfs: true - name: Dependencies id: depends @@ -369,6 +382,8 @@ jobs: steps: - uses: actions/checkout@v2 + with: + lfs: true - name: add oneAPI to apt shell: bash @@ -393,6 +408,8 @@ jobs: - name: Clone id: checkout uses: actions/checkout@v4 + with: + lfs: true - name: Build id: cmake_build @@ -410,6 +427,8 @@ jobs: steps: - uses: actions/checkout@v2 + with: + lfs: true - name: add oneAPI to apt shell: bash @@ -434,6 +453,8 @@ jobs: - name: Clone id: checkout uses: actions/checkout@v4 + with: + lfs: true - name: Build id: cmake_build @@ -454,6 +475,8 @@ jobs: - name: Clone id: checkout uses: actions/checkout@v4 + with: + lfs: true - name: Dependencies id: depends @@ -485,6 +508,8 @@ jobs: - name: Clone id: checkout uses: actions/checkout@v4 + with: + lfs: true - name: Dependencies id: depends @@ -514,6 +539,8 @@ jobs: - name: Clone id: checkout uses: actions/checkout@v1 + with: + lfs: true - name: Dependencies id: depends @@ -543,6 +570,8 @@ jobs: - name: Clone id: checkout uses: actions/checkout@v1 + with: + lfs: true - name: Dependencies id: depends @@ -576,6 +605,8 @@ jobs: - name: Clone id: checkout uses: actions/checkout@v1 + with: + lfs: true - name: Dependencies id: depends @@ -606,6 +637,8 @@ jobs: steps: - name: Clone uses: actions/checkout@v4 + with: + lfs: true - name: Setup ${{ matrix.sys }} uses: msys2/setup-msys2@v2 @@ -687,6 +720,7 @@ jobs: id: checkout uses: actions/checkout@v4 with: + lfs: true fetch-depth: 0 - name: Clone Kompute submodule @@ -833,6 +867,7 @@ jobs: id: checkout uses: actions/checkout@v4 with: + lfs: true fetch-depth: 0 - uses: Jimver/cuda-toolkit@v0.2.11 @@ -906,6 +941,7 @@ jobs: id: checkout uses: actions/checkout@v4 with: + lfs: true fetch-depth: 0 - name: Install @@ -947,6 +983,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + lfs: true - name: Build Xcode project run: xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' build @@ -957,6 +995,8 @@ jobs: steps: - name: Clone uses: actions/checkout@v4 + with: + lfs: true - name: Set up JDK uses: actions/setup-java@v3 @@ -979,7 +1019,9 @@ jobs: # runs-on: macos-12 # steps: # - name: Clone -# uses: actions/checkout@v4 +# uses: actions/checkout@#v4 +# with: +# lfs: true # # - name: Build # uses: cross-platform-actions/action@v0.19.0 @@ -1012,6 +1054,7 @@ jobs: id: checkout uses: actions/checkout@v4 with: + lfs: true fetch-depth: 0 - name: Determine tag name @@ -1077,6 +1120,8 @@ jobs: # steps: # - name: Clone # uses: actions/checkout@v4 +# with: +# lfs: true # # - name: Dependencies # run: | @@ -1101,6 +1146,8 @@ jobs: # steps: # - name: Clone # uses: actions/checkout@v4 +# with: +# lfs: true # # - name: Dependencies # run: | @@ -1125,6 +1172,8 @@ jobs: # steps: # - name: Clone # uses: actions/checkout@v4 +# with: +# lfs: true # # - name: Dependencies # run: | @@ -1155,6 +1204,8 @@ jobs: # steps: # - name: Clone # uses: actions/checkout@v4 +# with: +# lfs: true # # - name: Add msbuild to PATH # uses: microsoft/setup-msbuild@v1 @@ -1194,6 +1245,8 @@ jobs: # steps: # - name: Clone # uses: actions/checkout@v4 +# with: +# lfs: true # # - name: Add msbuild to PATH # uses: microsoft/setup-msbuild@v1 @@ -1240,6 +1293,8 @@ jobs: # steps: # - name: Clone # uses: actions/checkout@v4 +# with: +# lfs: true # # - name: Dependencies # run: | diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml index f12c558f8..e6f57793a 100644 --- a/.github/workflows/code-coverage.yml +++ b/.github/workflows/code-coverage.yml @@ -13,14 +13,16 @@ jobs: run: runs-on: ubuntu-20.04 steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Dependencies run: | sudo apt-get update sudo apt-get install build-essential gcc-8 lcov + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: true + - name: Build run: CC=gcc-8 make -j LLAMA_CODE_COVERAGE=1 tests diff --git a/models/ggml-vocab-aquila.gguf b/models/ggml-vocab-aquila.gguf index 7a9abb122..1d28649d9 100644 Binary files a/models/ggml-vocab-aquila.gguf and b/models/ggml-vocab-aquila.gguf differ diff --git a/models/ggml-vocab-baichuan.gguf b/models/ggml-vocab-baichuan.gguf index 7caaf8239..024492cfd 100644 Binary files a/models/ggml-vocab-baichuan.gguf and b/models/ggml-vocab-baichuan.gguf differ diff --git a/models/ggml-vocab-bert-bge.gguf b/models/ggml-vocab-bert-bge.gguf index b2cbd5df6..0c12abe39 100644 Binary files a/models/ggml-vocab-bert-bge.gguf and b/models/ggml-vocab-bert-bge.gguf differ diff --git a/models/ggml-vocab-command-r.gguf b/models/ggml-vocab-command-r.gguf index b553eab33..91656898f 100644 Binary files a/models/ggml-vocab-command-r.gguf and b/models/ggml-vocab-command-r.gguf differ diff --git a/models/ggml-vocab-deepseek-coder.gguf b/models/ggml-vocab-deepseek-coder.gguf index 6728cd747..b04617a31 100644 Binary files a/models/ggml-vocab-deepseek-coder.gguf and b/models/ggml-vocab-deepseek-coder.gguf differ diff --git a/models/ggml-vocab-deepseek-llm.gguf b/models/ggml-vocab-deepseek-llm.gguf index 5d66091c4..8b821b95e 100644 Binary files a/models/ggml-vocab-deepseek-llm.gguf and b/models/ggml-vocab-deepseek-llm.gguf differ diff --git a/models/ggml-vocab-falcon.gguf b/models/ggml-vocab-falcon.gguf index 334d50da5..1916c29e2 100644 Binary files a/models/ggml-vocab-falcon.gguf and b/models/ggml-vocab-falcon.gguf differ diff --git a/models/ggml-vocab-gpt-2.gguf b/models/ggml-vocab-gpt-2.gguf index 5ea85cf52..2745d9f7a 100644 Binary files a/models/ggml-vocab-gpt-2.gguf and b/models/ggml-vocab-gpt-2.gguf differ diff --git a/models/ggml-vocab-gpt-neox.gguf b/models/ggml-vocab-gpt-neox.gguf index b9af16845..62b552628 100644 Binary files a/models/ggml-vocab-gpt-neox.gguf and b/models/ggml-vocab-gpt-neox.gguf differ diff --git a/models/ggml-vocab-gpt2.gguf b/models/ggml-vocab-gpt2.gguf index 1fbc72c1e..14b52a993 100644 Binary files a/models/ggml-vocab-gpt2.gguf and b/models/ggml-vocab-gpt2.gguf differ diff --git a/models/ggml-vocab-llama-bpe.gguf b/models/ggml-vocab-llama-bpe.gguf index e51a99118..0c6826d8a 100644 Binary files a/models/ggml-vocab-llama-bpe.gguf and b/models/ggml-vocab-llama-bpe.gguf differ diff --git a/models/ggml-vocab-llama-spm.gguf b/models/ggml-vocab-llama-spm.gguf index 658295a5d..209328595 100644 Binary files a/models/ggml-vocab-llama-spm.gguf and b/models/ggml-vocab-llama-spm.gguf differ diff --git a/models/ggml-vocab-mpt.gguf b/models/ggml-vocab-mpt.gguf index f42f56dec..188bfa128 100644 Binary files a/models/ggml-vocab-mpt.gguf and b/models/ggml-vocab-mpt.gguf differ diff --git a/models/ggml-vocab-phi-3.gguf b/models/ggml-vocab-phi-3.gguf index f8022a385..17de5a738 100644 Binary files a/models/ggml-vocab-phi-3.gguf and b/models/ggml-vocab-phi-3.gguf differ diff --git a/models/ggml-vocab-refact.gguf b/models/ggml-vocab-refact.gguf index 52afcf01a..ba9082911 100644 Binary files a/models/ggml-vocab-refact.gguf and b/models/ggml-vocab-refact.gguf differ diff --git a/models/ggml-vocab-stablelm.gguf b/models/ggml-vocab-stablelm.gguf index ebb0cdb7d..f65062192 100644 Binary files a/models/ggml-vocab-stablelm.gguf and b/models/ggml-vocab-stablelm.gguf differ diff --git a/models/ggml-vocab-starcoder.gguf b/models/ggml-vocab-starcoder.gguf index 7a7e7742a..66c113ad7 100644 Binary files a/models/ggml-vocab-starcoder.gguf and b/models/ggml-vocab-starcoder.gguf differ