Merge branch 'ggerganov:master' into master
This commit is contained in:
commit
79aad2ffa9
19 changed files with 2816 additions and 1434 deletions
1
.github/workflows/close-issue.yml
vendored
1
.github/workflows/close-issue.yml
vendored
|
@ -12,6 +12,7 @@ jobs:
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/stale@v5
|
- uses: actions/stale@v5
|
||||||
with:
|
with:
|
||||||
|
exempt-issue-labels: "refactor,help wanted,good first issue,research"
|
||||||
days-before-issue-stale: 30
|
days-before-issue-stale: 30
|
||||||
days-before-issue-close: 14
|
days-before-issue-close: 14
|
||||||
stale-issue-label: "stale"
|
stale-issue-label: "stale"
|
||||||
|
|
2
.gitignore
vendored
2
.gitignore
vendored
|
@ -12,6 +12,8 @@
|
||||||
*.dot
|
*.dot
|
||||||
*.bat
|
*.bat
|
||||||
*.metallib
|
*.metallib
|
||||||
|
*.etag
|
||||||
|
*.lastModified
|
||||||
.DS_Store
|
.DS_Store
|
||||||
.build/
|
.build/
|
||||||
.cache/
|
.cache/
|
||||||
|
|
4
Makefile
4
Makefile
|
@ -753,6 +753,10 @@ gguf: examples/gguf/gguf.cpp ggml.o $(OBJS)
|
||||||
$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
|
$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
|
||||||
$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
|
$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
|
||||||
|
|
||||||
|
gguf-split: examples/gguf-split/gguf-split.cpp ggml.o llama.o $(COMMON_DEPS) $(OBJS)
|
||||||
|
$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
|
||||||
|
$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
|
||||||
|
|
||||||
train-text-from-scratch: examples/train-text-from-scratch/train-text-from-scratch.cpp ggml.o llama.o $(COMMON_DEPS) train.o $(OBJS)
|
train-text-from-scratch: examples/train-text-from-scratch/train-text-from-scratch.cpp ggml.o llama.o $(COMMON_DEPS) train.o $(OBJS)
|
||||||
$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
|
$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
|
||||||
$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
|
$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
|
||||||
|
|
129
README-sycl.md
129
README-sycl.md
|
@ -29,6 +29,7 @@ For Intel CPU, recommend to use llama.cpp for X86 (Intel MKL building).
|
||||||
## News
|
## News
|
||||||
|
|
||||||
- 2024.3
|
- 2024.3
|
||||||
|
- New base line is ready: [tag b2437](https://github.com/ggerganov/llama.cpp/tree/b2437).
|
||||||
- Support multiple cards: **--split-mode**: [none|layer]; not support [row], it's on developing.
|
- Support multiple cards: **--split-mode**: [none|layer]; not support [row], it's on developing.
|
||||||
- Support to assign main GPU by **--main-gpu**, replace $GGML_SYCL_DEVICE.
|
- Support to assign main GPU by **--main-gpu**, replace $GGML_SYCL_DEVICE.
|
||||||
- Support detecting all GPUs with level-zero and same top **Max compute units**.
|
- Support detecting all GPUs with level-zero and same top **Max compute units**.
|
||||||
|
@ -81,7 +82,7 @@ For dGPU, please make sure the device memory is enough. For llama-2-7b.Q4_0, rec
|
||||||
|-|-|-|
|
|-|-|-|
|
||||||
|Ampere Series| Support| A100|
|
|Ampere Series| Support| A100|
|
||||||
|
|
||||||
### oneMKL
|
### oneMKL for CUDA
|
||||||
|
|
||||||
The current oneMKL release does not contain the oneMKL cuBlas backend.
|
The current oneMKL release does not contain the oneMKL cuBlas backend.
|
||||||
As a result for Nvidia GPU's oneMKL must be built from source.
|
As a result for Nvidia GPU's oneMKL must be built from source.
|
||||||
|
@ -254,16 +255,16 @@ Run without parameter:
|
||||||
Check the ID in startup log, like:
|
Check the ID in startup log, like:
|
||||||
|
|
||||||
```
|
```
|
||||||
found 4 SYCL devices:
|
found 6 SYCL devices:
|
||||||
Device 0: Intel(R) Arc(TM) A770 Graphics, compute capability 1.3,
|
| | | |Compute |Max compute|Max work|Max sub| |
|
||||||
max compute_units 512, max work group size 1024, max sub group size 32, global mem size 16225243136
|
|ID| Device Type| Name|capability|units |group |group |Global mem size|
|
||||||
Device 1: Intel(R) FPGA Emulation Device, compute capability 1.2,
|
|--|------------------|---------------------------------------------|----------|-----------|--------|-------|---------------|
|
||||||
max compute_units 24, max work group size 67108864, max sub group size 64, global mem size 67065057280
|
| 0|[level_zero:gpu:0]| Intel(R) Arc(TM) A770 Graphics| 1.3| 512| 1024| 32| 16225243136|
|
||||||
Device 2: 13th Gen Intel(R) Core(TM) i7-13700K, compute capability 3.0,
|
| 1|[level_zero:gpu:1]| Intel(R) UHD Graphics 770| 1.3| 32| 512| 32| 53651849216|
|
||||||
max compute_units 24, max work group size 8192, max sub group size 64, global mem size 67065057280
|
| 2| [opencl:gpu:0]| Intel(R) Arc(TM) A770 Graphics| 3.0| 512| 1024| 32| 16225243136|
|
||||||
Device 3: Intel(R) Arc(TM) A770 Graphics, compute capability 3.0,
|
| 3| [opencl:gpu:1]| Intel(R) UHD Graphics 770| 3.0| 32| 512| 32| 53651849216|
|
||||||
max compute_units 512, max work group size 1024, max sub group size 32, global mem size 16225243136
|
| 4| [opencl:cpu:0]| 13th Gen Intel(R) Core(TM) i7-13700K| 3.0| 24| 8192| 64| 67064815616|
|
||||||
|
| 5| [opencl:acc:0]| Intel(R) FPGA Emulation Device| 1.2| 24|67108864| 64| 67064815616|
|
||||||
```
|
```
|
||||||
|
|
||||||
|Attribute|Note|
|
|Attribute|Note|
|
||||||
|
@ -271,12 +272,35 @@ found 4 SYCL devices:
|
||||||
|compute capability 1.3|Level-zero running time, recommended |
|
|compute capability 1.3|Level-zero running time, recommended |
|
||||||
|compute capability 3.0|OpenCL running time, slower than level-zero in most cases|
|
|compute capability 3.0|OpenCL running time, slower than level-zero in most cases|
|
||||||
|
|
||||||
4. Set device ID and execute llama.cpp
|
4. Device selection and execution of llama.cpp
|
||||||
|
|
||||||
Set device ID = 0 by **GGML_SYCL_DEVICE=0**
|
There are two device selection modes:
|
||||||
|
|
||||||
|
- Single device: Use one device assigned by user.
|
||||||
|
- Multiple devices: Automatically choose the devices with the same biggest Max compute units.
|
||||||
|
|
||||||
|
|Device selection|Parameter|
|
||||||
|
|-|-|
|
||||||
|
|Single device|--split-mode none --main-gpu DEVICE_ID |
|
||||||
|
|Multiple devices|--split-mode layer (default)|
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
- Use device 0:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
GGML_SYCL_DEVICE=0 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33
|
ZES_ENABLE_SYSMAN=1 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33 -sm none -mg 0
|
||||||
|
```
|
||||||
|
or run by script:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./examples/sycl/run_llama2.sh 0
|
||||||
|
```
|
||||||
|
|
||||||
|
- Use multiple devices:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
ZES_ENABLE_SYSMAN=1 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33 -sm layer
|
||||||
```
|
```
|
||||||
or run by script:
|
or run by script:
|
||||||
|
|
||||||
|
@ -289,12 +313,18 @@ Note:
|
||||||
- By default, mmap is used to read model file. In some cases, it leads to the hang issue. Recommend to use parameter **--no-mmap** to disable mmap() to skip this issue.
|
- By default, mmap is used to read model file. In some cases, it leads to the hang issue. Recommend to use parameter **--no-mmap** to disable mmap() to skip this issue.
|
||||||
|
|
||||||
|
|
||||||
5. Check the device ID in output
|
5. Verify the device ID in output
|
||||||
|
|
||||||
|
Verify to see if the selected GPU is shown in the output, like:
|
||||||
|
|
||||||
Like:
|
|
||||||
```
|
```
|
||||||
Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device
|
detect 1 SYCL GPUs: [0] with top Max compute units:512
|
||||||
```
|
```
|
||||||
|
Or
|
||||||
|
```
|
||||||
|
use 1 SYCL GPUs: [0] with Max compute units:512
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
## Windows
|
## Windows
|
||||||
|
|
||||||
|
@ -355,7 +385,7 @@ a. Download & install cmake for Windows: https://cmake.org/download/
|
||||||
|
|
||||||
b. Download & install mingw-w64 make for Windows provided by w64devkit
|
b. Download & install mingw-w64 make for Windows provided by w64devkit
|
||||||
|
|
||||||
- Download the latest fortran version of [w64devkit](https://github.com/skeeto/w64devkit/releases).
|
- Download the 1.19.0 version of [w64devkit](https://github.com/skeeto/w64devkit/releases/download/v1.19.0/w64devkit-1.19.0.zip).
|
||||||
|
|
||||||
- Extract `w64devkit` on your pc.
|
- Extract `w64devkit` on your pc.
|
||||||
|
|
||||||
|
@ -430,15 +460,16 @@ build\bin\main.exe
|
||||||
Check the ID in startup log, like:
|
Check the ID in startup log, like:
|
||||||
|
|
||||||
```
|
```
|
||||||
found 4 SYCL devices:
|
found 6 SYCL devices:
|
||||||
Device 0: Intel(R) Arc(TM) A770 Graphics, compute capability 1.3,
|
| | | |Compute |Max compute|Max work|Max sub| |
|
||||||
max compute_units 512, max work group size 1024, max sub group size 32, global mem size 16225243136
|
|ID| Device Type| Name|capability|units |group |group |Global mem size|
|
||||||
Device 1: Intel(R) FPGA Emulation Device, compute capability 1.2,
|
|--|------------------|---------------------------------------------|----------|-----------|--------|-------|---------------|
|
||||||
max compute_units 24, max work group size 67108864, max sub group size 64, global mem size 67065057280
|
| 0|[level_zero:gpu:0]| Intel(R) Arc(TM) A770 Graphics| 1.3| 512| 1024| 32| 16225243136|
|
||||||
Device 2: 13th Gen Intel(R) Core(TM) i7-13700K, compute capability 3.0,
|
| 1|[level_zero:gpu:1]| Intel(R) UHD Graphics 770| 1.3| 32| 512| 32| 53651849216|
|
||||||
max compute_units 24, max work group size 8192, max sub group size 64, global mem size 67065057280
|
| 2| [opencl:gpu:0]| Intel(R) Arc(TM) A770 Graphics| 3.0| 512| 1024| 32| 16225243136|
|
||||||
Device 3: Intel(R) Arc(TM) A770 Graphics, compute capability 3.0,
|
| 3| [opencl:gpu:1]| Intel(R) UHD Graphics 770| 3.0| 32| 512| 32| 53651849216|
|
||||||
max compute_units 512, max work group size 1024, max sub group size 32, global mem size 16225243136
|
| 4| [opencl:cpu:0]| 13th Gen Intel(R) Core(TM) i7-13700K| 3.0| 24| 8192| 64| 67064815616|
|
||||||
|
| 5| [opencl:acc:0]| Intel(R) FPGA Emulation Device| 1.2| 24|67108864| 64| 67064815616|
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
@ -447,13 +478,31 @@ found 4 SYCL devices:
|
||||||
|compute capability 1.3|Level-zero running time, recommended |
|
|compute capability 1.3|Level-zero running time, recommended |
|
||||||
|compute capability 3.0|OpenCL running time, slower than level-zero in most cases|
|
|compute capability 3.0|OpenCL running time, slower than level-zero in most cases|
|
||||||
|
|
||||||
4. Set device ID and execute llama.cpp
|
|
||||||
|
|
||||||
Set device ID = 0 by **set GGML_SYCL_DEVICE=0**
|
4. Device selection and execution of llama.cpp
|
||||||
|
|
||||||
|
There are two device selection modes:
|
||||||
|
|
||||||
|
- Single device: Use one device assigned by user.
|
||||||
|
- Multiple devices: Automatically choose the devices with the same biggest Max compute units.
|
||||||
|
|
||||||
|
|Device selection|Parameter|
|
||||||
|
|-|-|
|
||||||
|
|Single device|--split-mode none --main-gpu DEVICE_ID |
|
||||||
|
|Multiple devices|--split-mode layer (default)|
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
- Use device 0:
|
||||||
|
|
||||||
```
|
```
|
||||||
set GGML_SYCL_DEVICE=0
|
build\bin\main.exe -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 33 -s 0 -sm none -mg 0
|
||||||
build\bin\main.exe -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 33 -s 0
|
```
|
||||||
|
|
||||||
|
- Use multiple devices:
|
||||||
|
|
||||||
|
```
|
||||||
|
build\bin\main.exe -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 33 -s 0 -sm layer
|
||||||
```
|
```
|
||||||
or run by script:
|
or run by script:
|
||||||
|
|
||||||
|
@ -466,11 +515,17 @@ Note:
|
||||||
- By default, mmap is used to read model file. In some cases, it leads to the hang issue. Recommend to use parameter **--no-mmap** to disable mmap() to skip this issue.
|
- By default, mmap is used to read model file. In some cases, it leads to the hang issue. Recommend to use parameter **--no-mmap** to disable mmap() to skip this issue.
|
||||||
|
|
||||||
|
|
||||||
5. Check the device ID in output
|
|
||||||
|
|
||||||
Like:
|
5. Verify the device ID in output
|
||||||
|
|
||||||
|
Verify to see if the selected GPU is shown in the output, like:
|
||||||
|
|
||||||
```
|
```
|
||||||
Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device
|
detect 1 SYCL GPUs: [0] with top Max compute units:512
|
||||||
|
```
|
||||||
|
Or
|
||||||
|
```
|
||||||
|
use 1 SYCL GPUs: [0] with Max compute units:512
|
||||||
```
|
```
|
||||||
|
|
||||||
## Environment Variable
|
## Environment Variable
|
||||||
|
@ -489,7 +544,6 @@ Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device
|
||||||
|
|
||||||
|Name|Value|Function|
|
|Name|Value|Function|
|
||||||
|-|-|-|
|
|-|-|-|
|
||||||
|GGML_SYCL_DEVICE|0 (default) or 1|Set the device id used. Check the device ids by default running output|
|
|
||||||
|GGML_SYCL_DEBUG|0 (default) or 1|Enable log function by macro: GGML_SYCL_DEBUG|
|
|GGML_SYCL_DEBUG|0 (default) or 1|Enable log function by macro: GGML_SYCL_DEBUG|
|
||||||
|ZES_ENABLE_SYSMAN| 0 (default) or 1|Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.<br>Recommended to use when --split-mode = layer|
|
|ZES_ENABLE_SYSMAN| 0 (default) or 1|Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.<br>Recommended to use when --split-mode = layer|
|
||||||
|
|
||||||
|
@ -507,6 +561,9 @@ Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device
|
||||||
|
|
||||||
## Q&A
|
## Q&A
|
||||||
|
|
||||||
|
Note: please add prefix **[SYCL]** in issue title, so that we will check it as soon as possible.
|
||||||
|
|
||||||
|
|
||||||
- Error: `error while loading shared libraries: libsycl.so.7: cannot open shared object file: No such file or directory`.
|
- Error: `error while loading shared libraries: libsycl.so.7: cannot open shared object file: No such file or directory`.
|
||||||
|
|
||||||
Miss to enable oneAPI running environment.
|
Miss to enable oneAPI running environment.
|
||||||
|
@ -538,4 +595,4 @@ Using device **0** (Intel(R) Arc(TM) A770 Graphics) as main device
|
||||||
|
|
||||||
## Todo
|
## Todo
|
||||||
|
|
||||||
- Support multiple cards.
|
- Support row layer split for multiple card runs.
|
||||||
|
|
|
@ -1056,7 +1056,8 @@ static bool gpt_params_find_arg(int argc, char ** argv, gpt_params & params, int
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if (arg == "-h" || arg == "--help") {
|
if (arg == "-h" || arg == "--help") {
|
||||||
return false;
|
gpt_print_usage(argc, argv, gpt_params());
|
||||||
|
exit(0);
|
||||||
}
|
}
|
||||||
if (arg == "--version") {
|
if (arg == "--version") {
|
||||||
fprintf(stderr, "version: %d (%s)\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT);
|
fprintf(stderr, "version: %d (%s)\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT);
|
||||||
|
|
|
@ -32,13 +32,13 @@ typedef struct llama_sampling_params {
|
||||||
float dynatemp_range = 0.00f; // 0.0 = disabled
|
float dynatemp_range = 0.00f; // 0.0 = disabled
|
||||||
float dynatemp_exponent = 1.00f; // controls how entropy maps to temperature in dynamic temperature sampler
|
float dynatemp_exponent = 1.00f; // controls how entropy maps to temperature in dynamic temperature sampler
|
||||||
int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty, -1 = context size)
|
int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty, -1 = context size)
|
||||||
float penalty_repeat = 1.10f; // 1.0 = disabled
|
float penalty_repeat = 1.00f; // 1.0 = disabled
|
||||||
float penalty_freq = 0.00f; // 0.0 = disabled
|
float penalty_freq = 0.00f; // 0.0 = disabled
|
||||||
float penalty_present = 0.00f; // 0.0 = disabled
|
float penalty_present = 0.00f; // 0.0 = disabled
|
||||||
int32_t mirostat = 0; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0
|
int32_t mirostat = 0; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0
|
||||||
float mirostat_tau = 5.00f; // target entropy
|
float mirostat_tau = 5.00f; // target entropy
|
||||||
float mirostat_eta = 0.10f; // learning rate
|
float mirostat_eta = 0.10f; // learning rate
|
||||||
bool penalize_nl = true; // consider newlines as a repeatable token
|
bool penalize_nl = false; // consider newlines as a repeatable token
|
||||||
|
|
||||||
std::vector<llama_sampler_type> samplers_sequence = {
|
std::vector<llama_sampler_type> samplers_sequence = {
|
||||||
llama_sampler_type::TOP_K,
|
llama_sampler_type::TOP_K,
|
||||||
|
|
|
@ -21,6 +21,7 @@ else()
|
||||||
add_subdirectory(embedding)
|
add_subdirectory(embedding)
|
||||||
add_subdirectory(finetune)
|
add_subdirectory(finetune)
|
||||||
add_subdirectory(gritlm)
|
add_subdirectory(gritlm)
|
||||||
|
add_subdirectory(gguf-split)
|
||||||
add_subdirectory(infill)
|
add_subdirectory(infill)
|
||||||
add_subdirectory(llama-bench)
|
add_subdirectory(llama-bench)
|
||||||
add_subdirectory(llava)
|
add_subdirectory(llava)
|
||||||
|
|
5
examples/gguf-split/CMakeLists.txt
Normal file
5
examples/gguf-split/CMakeLists.txt
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
set(TARGET gguf-split)
|
||||||
|
add_executable(${TARGET} gguf-split.cpp)
|
||||||
|
install(TARGETS ${TARGET} RUNTIME)
|
||||||
|
target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
|
||||||
|
target_compile_features(${TARGET} PRIVATE cxx_std_11)
|
9
examples/gguf-split/README.md
Normal file
9
examples/gguf-split/README.md
Normal file
|
@ -0,0 +1,9 @@
|
||||||
|
## GGUF split Example
|
||||||
|
|
||||||
|
CLI to split / merge GGUF files.
|
||||||
|
|
||||||
|
**Command line options:**
|
||||||
|
|
||||||
|
- `--split`: split GGUF to multiple GGUF, default operation.
|
||||||
|
- `--split-max-tensors`: maximum tensors in each split: default(128)
|
||||||
|
- `--merge`: merge multiple GGUF to a single GGUF.
|
489
examples/gguf-split/gguf-split.cpp
Normal file
489
examples/gguf-split/gguf-split.cpp
Normal file
|
@ -0,0 +1,489 @@
|
||||||
|
#include "llama.h"
|
||||||
|
#include "ggml.h"
|
||||||
|
#include "common.h"
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <cmath>
|
||||||
|
#include <cstdint>
|
||||||
|
#include <cstdlib>
|
||||||
|
#include <fstream>
|
||||||
|
#include <ios>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
enum split_operation : uint8_t {
|
||||||
|
SPLIT_OP_SPLIT,
|
||||||
|
SPLIT_OP_MERGE,
|
||||||
|
};
|
||||||
|
|
||||||
|
static const char * const LLM_KV_GENERAL_SPLIT_I_SPLIT = "general.split";
|
||||||
|
static const char * const LLM_KV_GENERAL_SPLIT_N_SPLIT = "general.split_count";
|
||||||
|
|
||||||
|
static const int SPLIT_FILENAME_MAX = 256;
|
||||||
|
|
||||||
|
static const char * const SPLIT_FILENAME_FORMAT = "%s-%05d-of-%05d.gguf";
|
||||||
|
|
||||||
|
struct split_params {
|
||||||
|
split_operation operation = SPLIT_OP_SPLIT;
|
||||||
|
int n_split_tensors = 128;
|
||||||
|
std::string input;
|
||||||
|
std::string output;
|
||||||
|
};
|
||||||
|
|
||||||
|
static void split_print_usage(const char * executable) {
|
||||||
|
const split_params default_params;
|
||||||
|
printf("\n");
|
||||||
|
printf("usage: %s [options] GGUF_IN GGUF_OUT\n", executable);
|
||||||
|
printf("\n");
|
||||||
|
printf("Apply a GGUF operation on IN to OUT.");
|
||||||
|
printf("\n");
|
||||||
|
printf("options:\n");
|
||||||
|
printf(" -h, --help show this help message and exit\n");
|
||||||
|
printf(" --version show version and build info\n");
|
||||||
|
printf(" --split split GGUF to multiple GGUF (default)\n");
|
||||||
|
printf(" --split-max-tensors max tensors in each split: default(%d)\n", default_params.n_split_tensors);
|
||||||
|
printf(" --merge merge multiple GGUF to a single GGUF\n");
|
||||||
|
printf("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool split_params_parse_ex(int argc, const char ** argv, split_params & params) {
|
||||||
|
std::string arg;
|
||||||
|
const std::string arg_prefix = "--";
|
||||||
|
bool invalid_param = false;
|
||||||
|
|
||||||
|
int arg_idx = 1;
|
||||||
|
for (; arg_idx < argc && strncmp(argv[arg_idx], "--", 2) == 0; arg_idx++) {
|
||||||
|
arg = argv[arg_idx];
|
||||||
|
if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {
|
||||||
|
std::replace(arg.begin(), arg.end(), '_', '-');
|
||||||
|
}
|
||||||
|
|
||||||
|
bool arg_found = false;
|
||||||
|
if (arg == "-h" || arg == "--help") {
|
||||||
|
split_print_usage(argv[0]);
|
||||||
|
exit(0);
|
||||||
|
}
|
||||||
|
if (arg == "--version") {
|
||||||
|
fprintf(stderr, "version: %d (%s)\n", LLAMA_BUILD_NUMBER, LLAMA_COMMIT);
|
||||||
|
fprintf(stderr, "built with %s for %s\n", LLAMA_COMPILER, LLAMA_BUILD_TARGET);
|
||||||
|
exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (arg == "--merge") {
|
||||||
|
arg_found = true;
|
||||||
|
params.operation = SPLIT_OP_MERGE;
|
||||||
|
}
|
||||||
|
if (arg == "--split") {
|
||||||
|
arg_found = true;
|
||||||
|
params.operation = SPLIT_OP_SPLIT;
|
||||||
|
}
|
||||||
|
if (arg == "--split-max-tensors") {
|
||||||
|
if (++arg_idx >= argc) {
|
||||||
|
invalid_param = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
arg_found = true;
|
||||||
|
params.n_split_tensors = atoi(argv[arg_idx]);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!arg_found) {
|
||||||
|
throw std::invalid_argument("error: unknown argument: " + arg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (invalid_param) {
|
||||||
|
throw std::invalid_argument("error: invalid parameter for argument: " + arg);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (argc - arg_idx < 2) {
|
||||||
|
printf("%s: bad arguments\n", argv[0]);
|
||||||
|
split_print_usage(argv[0]);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
params.input = argv[arg_idx++];
|
||||||
|
params.output = argv[arg_idx++];
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool split_params_parse(int argc, const char ** argv, split_params & params) {
|
||||||
|
bool result = true;
|
||||||
|
try {
|
||||||
|
if (!split_params_parse_ex(argc, argv, params)) {
|
||||||
|
split_print_usage(argv[0]);
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (const std::invalid_argument & ex) {
|
||||||
|
fprintf(stderr, "%s\n", ex.what());
|
||||||
|
split_print_usage(argv[0]);
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void zeros(std::ofstream & file, size_t n) {
|
||||||
|
char zero = 0;
|
||||||
|
for (size_t i = 0; i < n; ++i) {
|
||||||
|
file.write(&zero, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static std::string split_file_name(const std::string & path, int i_split, int n_split) {
|
||||||
|
char f_split[SPLIT_FILENAME_MAX] = {0};
|
||||||
|
snprintf(f_split, sizeof(f_split), SPLIT_FILENAME_FORMAT, path.c_str(), i_split + 1, n_split);
|
||||||
|
return std::string(f_split);
|
||||||
|
}
|
||||||
|
|
||||||
|
struct split_strategy {
|
||||||
|
const split_params params;
|
||||||
|
std::ifstream & f_input;
|
||||||
|
struct gguf_context * ctx_gguf;
|
||||||
|
struct ggml_context * ctx_meta = NULL;
|
||||||
|
const int n_tensors;
|
||||||
|
|
||||||
|
const int n_split;
|
||||||
|
int i_split = 0;
|
||||||
|
|
||||||
|
int i_tensor = 0;
|
||||||
|
|
||||||
|
std::vector<uint8_t> read_data;
|
||||||
|
|
||||||
|
struct gguf_context * ctx_out;
|
||||||
|
std::ofstream fout;
|
||||||
|
|
||||||
|
split_strategy(const split_params & params,
|
||||||
|
std::ifstream & f_input,
|
||||||
|
struct gguf_context * ctx_gguf,
|
||||||
|
struct ggml_context * ctx_meta) :
|
||||||
|
params(params),
|
||||||
|
f_input(f_input),
|
||||||
|
ctx_gguf(ctx_gguf),
|
||||||
|
ctx_meta(ctx_meta),
|
||||||
|
n_tensors(gguf_get_n_tensors(ctx_gguf)),
|
||||||
|
n_split(std::ceil(1. * n_tensors / params.n_split_tensors)) {
|
||||||
|
}
|
||||||
|
|
||||||
|
bool should_split() const {
|
||||||
|
return i_tensor < n_tensors && i_tensor % params.n_split_tensors == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void split_start() {
|
||||||
|
ctx_out = gguf_init_empty();
|
||||||
|
|
||||||
|
// Save all metadata in first split only
|
||||||
|
if (i_split == 0) {
|
||||||
|
gguf_set_kv(ctx_out, ctx_gguf);
|
||||||
|
}
|
||||||
|
gguf_set_val_u8(ctx_out, LLM_KV_GENERAL_SPLIT_I_SPLIT, i_split);
|
||||||
|
gguf_set_val_u8(ctx_out, LLM_KV_GENERAL_SPLIT_N_SPLIT, n_split);
|
||||||
|
|
||||||
|
// populate the original tensors, so we get an initial metadata
|
||||||
|
for (int i = i_split * params.n_split_tensors; i < n_tensors && i < (i_split + 1) * params.n_split_tensors; ++i) {
|
||||||
|
struct ggml_tensor * meta = ggml_get_tensor(ctx_meta, gguf_get_tensor_name(ctx_gguf, i));
|
||||||
|
gguf_add_tensor(ctx_out, meta);
|
||||||
|
}
|
||||||
|
|
||||||
|
auto split_name = split_file_name(params.output, i_split, n_split);
|
||||||
|
|
||||||
|
fprintf(stderr, "%s: %s ...", __func__, split_name.c_str());
|
||||||
|
fout = std::ofstream(split_name, std::ios::binary);
|
||||||
|
fout.exceptions(std::ofstream::failbit); // fail fast on write errors
|
||||||
|
|
||||||
|
auto meta_size = gguf_get_meta_size(ctx_out);
|
||||||
|
|
||||||
|
// placeholder for the meta data
|
||||||
|
::zeros(fout, meta_size);
|
||||||
|
|
||||||
|
i_split++;
|
||||||
|
}
|
||||||
|
|
||||||
|
void next_tensor() {
|
||||||
|
const char * t_name = gguf_get_tensor_name(ctx_gguf, i_tensor);
|
||||||
|
struct ggml_tensor * t = ggml_get_tensor(ctx_meta, t_name);
|
||||||
|
auto n_bytes = ggml_nbytes(t);
|
||||||
|
|
||||||
|
if (read_data.size() < n_bytes) {
|
||||||
|
read_data.resize(n_bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
auto offset = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i_tensor);
|
||||||
|
f_input.seekg(offset);
|
||||||
|
f_input.read((char *)read_data.data(), n_bytes);
|
||||||
|
|
||||||
|
t->data = read_data.data();
|
||||||
|
|
||||||
|
// write tensor data + padding
|
||||||
|
fout.write((const char *)t->data, n_bytes);
|
||||||
|
zeros(fout, GGML_PAD(n_bytes, GGUF_DEFAULT_ALIGNMENT) - n_bytes);
|
||||||
|
|
||||||
|
i_tensor++;
|
||||||
|
}
|
||||||
|
|
||||||
|
void split_end() {
|
||||||
|
// go back to beginning of file and write the updated metadata
|
||||||
|
fout.seekp(0);
|
||||||
|
std::vector<uint8_t> data(gguf_get_meta_size(ctx_out));
|
||||||
|
gguf_get_meta_data(ctx_out, data.data());
|
||||||
|
fout.write((const char *)data.data(), data.size());
|
||||||
|
|
||||||
|
fout.close();
|
||||||
|
gguf_free(ctx_out);
|
||||||
|
|
||||||
|
fprintf(stderr, "\033[3Ddone\n");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
static void gguf_split(const split_params & split_params) {
|
||||||
|
struct ggml_context * ctx_meta = NULL;
|
||||||
|
|
||||||
|
struct gguf_init_params params = {
|
||||||
|
/*.no_alloc = */ true,
|
||||||
|
/*.ctx = */ &ctx_meta,
|
||||||
|
};
|
||||||
|
|
||||||
|
std::ifstream f_input(split_params.input.c_str(), std::ios::binary);
|
||||||
|
if (!f_input.is_open()) {
|
||||||
|
fprintf(stderr, "%s: failed to open input GGUF from %s\n", __func__, split_params.input.c_str());
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
auto * ctx_gguf = gguf_init_from_file(split_params.input.c_str(), params);
|
||||||
|
if (!ctx_gguf) {
|
||||||
|
fprintf(stderr, "%s: failed to load input GGUF from %s\n", __func__, split_params.input.c_str());
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
split_strategy strategy(split_params, f_input, ctx_gguf, ctx_meta);
|
||||||
|
fprintf(stderr, "%s: %s -> %s (%d tensors per file)\n",
|
||||||
|
__func__, split_params.input.c_str(),
|
||||||
|
split_file_name(split_params.output, strategy.i_split, strategy.n_split).c_str(),
|
||||||
|
split_params.n_split_tensors);
|
||||||
|
|
||||||
|
strategy.split_start();
|
||||||
|
|
||||||
|
while (strategy.i_tensor < strategy.n_tensors) {
|
||||||
|
strategy.next_tensor();
|
||||||
|
if (strategy.should_split()) {
|
||||||
|
strategy.split_end();
|
||||||
|
strategy.split_start();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
strategy.split_end();
|
||||||
|
|
||||||
|
gguf_free(ctx_gguf);
|
||||||
|
f_input.close();
|
||||||
|
|
||||||
|
fprintf(stderr, "%s: %d gguf split written with a total of %d tensors.\n",
|
||||||
|
__func__, strategy.n_split, strategy.n_tensors);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void gguf_merge(const split_params & split_params) {
|
||||||
|
fprintf(stderr, "%s: %s -> %s\n",
|
||||||
|
__func__, split_params.input.c_str(),
|
||||||
|
split_params.output.c_str());
|
||||||
|
int n_split = 1;
|
||||||
|
int total_tensors = 0;
|
||||||
|
|
||||||
|
auto * ctx_out = gguf_init_empty();
|
||||||
|
std::ofstream fout(split_params.output.c_str(), std::ios::binary);
|
||||||
|
fout.exceptions(std::ofstream::failbit); // fail fast on write errors
|
||||||
|
|
||||||
|
std::vector<uint8_t> read_data;
|
||||||
|
std::vector<ggml_context *> ctx_metas;
|
||||||
|
std::vector<gguf_context *> ctx_ggufs;
|
||||||
|
|
||||||
|
std::string split_prefix;
|
||||||
|
|
||||||
|
// First pass to find KV and tensors metadata
|
||||||
|
for (int i_split = 0; i_split < n_split; i_split++) {
|
||||||
|
struct ggml_context * ctx_meta = NULL;
|
||||||
|
|
||||||
|
struct gguf_init_params params = {
|
||||||
|
/*.no_alloc = */ true,
|
||||||
|
/*.ctx = */ &ctx_meta,
|
||||||
|
};
|
||||||
|
|
||||||
|
auto split_name = split_params.input;
|
||||||
|
if (i_split > 0) {
|
||||||
|
split_name = split_file_name(split_prefix, i_split, n_split);
|
||||||
|
}
|
||||||
|
fprintf(stderr, "%s: reading metadata %s ...", __func__, split_name.c_str());
|
||||||
|
|
||||||
|
auto * ctx_gguf = gguf_init_from_file(split_name.c_str(), params);
|
||||||
|
if (!ctx_gguf) {
|
||||||
|
fprintf(stderr, "\n%s: failed to load input GGUF from %s\n", __func__, split_params.input.c_str());
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
ctx_ggufs.push_back(ctx_gguf);
|
||||||
|
ctx_metas.push_back(ctx_meta);
|
||||||
|
|
||||||
|
if (i_split == 0) {
|
||||||
|
auto key_n_split = gguf_find_key(ctx_gguf, LLM_KV_GENERAL_SPLIT_N_SPLIT);
|
||||||
|
if (key_n_split < 0) {
|
||||||
|
fprintf(stderr,
|
||||||
|
"\n%s: input file does not contain %s metadata\n",
|
||||||
|
__func__,
|
||||||
|
LLM_KV_GENERAL_SPLIT_N_SPLIT);
|
||||||
|
gguf_free(ctx_gguf);
|
||||||
|
gguf_free(ctx_out);
|
||||||
|
fout.close();
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
n_split = gguf_get_val_u8(ctx_gguf, key_n_split);
|
||||||
|
if (n_split < 1) {
|
||||||
|
fprintf(stderr,
|
||||||
|
"\n%s: input file does not contain a valid split count %d\n",
|
||||||
|
__func__,
|
||||||
|
n_split);
|
||||||
|
gguf_free(ctx_gguf);
|
||||||
|
gguf_free(ctx_out);
|
||||||
|
fout.close();
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Do not trigger merge if we try to merge again the output
|
||||||
|
gguf_set_val_u8(ctx_out, LLM_KV_GENERAL_SPLIT_N_SPLIT, 0);
|
||||||
|
|
||||||
|
// Set metadata from the first split
|
||||||
|
gguf_set_kv(ctx_out, ctx_gguf);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the file naming
|
||||||
|
{
|
||||||
|
int i_split_file = 0;
|
||||||
|
int n_split_file = 0;
|
||||||
|
const char * i_split_format = "-00000-of-00000.gguf";
|
||||||
|
|
||||||
|
if (split_name.size() < strlen(i_split_format)) {
|
||||||
|
fprintf(stderr, "\n%s: unexpected input file name: %s\n", __func__, split_params.input.c_str());
|
||||||
|
for (auto * _ctx_gguf : ctx_ggufs) {
|
||||||
|
gguf_free(_ctx_gguf);
|
||||||
|
}
|
||||||
|
gguf_free(ctx_out);
|
||||||
|
fout.close();
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
split_prefix = split_name.substr(0, split_name.size() - strlen(i_split_format));
|
||||||
|
|
||||||
|
const char * split_name_c_str = split_name.c_str();
|
||||||
|
int n_part = sscanf(&split_name_c_str[0] + split_prefix.size(), "-%d-of-%d", &i_split_file, &n_split_file);
|
||||||
|
|
||||||
|
if (n_part != 2 || i_split_file - 1 != i_split || n_split_file != n_split) {
|
||||||
|
fprintf(stderr, "\n%s: unexpected input file name: %s"
|
||||||
|
" i_split=%d i_split_file=%d"
|
||||||
|
" n_split=%d n_split_file=%d\n", __func__,
|
||||||
|
split_params.input.c_str(),
|
||||||
|
i_split, i_split_file,
|
||||||
|
n_split, n_split_file);
|
||||||
|
for (auto * _ctx_gguf : ctx_ggufs) {
|
||||||
|
gguf_free(_ctx_gguf);
|
||||||
|
}
|
||||||
|
gguf_free(ctx_out);
|
||||||
|
fout.close();
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
auto n_tensors = gguf_get_n_tensors(ctx_gguf);
|
||||||
|
for (int i_tensor = 0; i_tensor < n_tensors; i_tensor++) {
|
||||||
|
const char * t_name = gguf_get_tensor_name(ctx_gguf, i_tensor);
|
||||||
|
struct ggml_tensor * t = ggml_get_tensor(ctx_meta, t_name);
|
||||||
|
gguf_add_tensor(ctx_out, t);
|
||||||
|
}
|
||||||
|
total_tensors += n_tensors;
|
||||||
|
|
||||||
|
fprintf(stderr, "\033[3Ddone\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
// placeholder for the meta data
|
||||||
|
{
|
||||||
|
auto meta_size = gguf_get_meta_size(ctx_out);
|
||||||
|
::zeros(fout, meta_size);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write tensors data
|
||||||
|
for (int i_split = 0; i_split < n_split; i_split++) {
|
||||||
|
auto split_name = split_file_name(split_prefix, i_split, n_split);
|
||||||
|
std::ifstream f_input(split_name.c_str(), std::ios::binary);
|
||||||
|
if (!f_input.is_open()) {
|
||||||
|
fprintf(stderr, "%s: failed to open input GGUF from %s\n", __func__, split_name.c_str());
|
||||||
|
for (auto * _ctx_gguf : ctx_ggufs) {
|
||||||
|
gguf_free(_ctx_gguf);
|
||||||
|
}
|
||||||
|
gguf_free(ctx_out);
|
||||||
|
fout.close();
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
fprintf(stderr, "%s: writing tensors %s ...", __func__, split_name.c_str());
|
||||||
|
|
||||||
|
auto * ctx_gguf = ctx_ggufs[i_split];
|
||||||
|
auto * ctx_meta = ctx_metas[i_split];
|
||||||
|
|
||||||
|
auto n_tensors = gguf_get_n_tensors(ctx_gguf);
|
||||||
|
for (int i_tensor = 0; i_tensor < n_tensors; i_tensor++) {
|
||||||
|
const char * t_name = gguf_get_tensor_name(ctx_gguf, i_tensor);
|
||||||
|
struct ggml_tensor * t = ggml_get_tensor(ctx_meta, t_name);
|
||||||
|
|
||||||
|
auto n_bytes = ggml_nbytes(t);
|
||||||
|
|
||||||
|
if (read_data.size() < n_bytes) {
|
||||||
|
read_data.resize(n_bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
auto offset = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i_tensor);
|
||||||
|
f_input.seekg(offset);
|
||||||
|
f_input.read((char *)read_data.data(), n_bytes);
|
||||||
|
|
||||||
|
// write tensor data + padding
|
||||||
|
fout.write((const char *)read_data.data(), n_bytes);
|
||||||
|
zeros(fout, GGML_PAD(n_bytes, GGUF_DEFAULT_ALIGNMENT) - n_bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
gguf_free(ctx_gguf);
|
||||||
|
ggml_free(ctx_meta);
|
||||||
|
f_input.close();
|
||||||
|
fprintf(stderr, "\033[3Ddone\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
// go back to beginning of file and write the updated metadata
|
||||||
|
fout.seekp(0);
|
||||||
|
std::vector<uint8_t> data(gguf_get_meta_size(ctx_out));
|
||||||
|
gguf_get_meta_data(ctx_out, data.data());
|
||||||
|
fout.write((const char *)data.data(), data.size());
|
||||||
|
|
||||||
|
fout.close();
|
||||||
|
gguf_free(ctx_out);
|
||||||
|
}
|
||||||
|
|
||||||
|
fprintf(stderr, "%s: %s merged from %d split with %d tensors.\n",
|
||||||
|
__func__, split_params.output.c_str(), n_split, total_tensors);
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, const char ** argv) {
|
||||||
|
if (argc < 3) {
|
||||||
|
split_print_usage(argv[0]);
|
||||||
|
}
|
||||||
|
|
||||||
|
split_params params;
|
||||||
|
split_params_parse(argc, argv, params);
|
||||||
|
|
||||||
|
switch (params.operation) {
|
||||||
|
case SPLIT_OP_SPLIT: gguf_split(params);
|
||||||
|
break;
|
||||||
|
case SPLIT_OP_MERGE: gguf_merge(params);
|
||||||
|
break;
|
||||||
|
default:split_print_usage(argv[0]);
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
@ -5,15 +5,14 @@ import sys
|
||||||
import time
|
import time
|
||||||
import traceback
|
import traceback
|
||||||
from contextlib import closing
|
from contextlib import closing
|
||||||
|
from subprocess import TimeoutExpired
|
||||||
import psutil
|
|
||||||
|
|
||||||
|
|
||||||
def before_scenario(context, scenario):
|
def before_scenario(context, scenario):
|
||||||
context.debug = 'DEBUG' in os.environ and os.environ['DEBUG'] == 'ON'
|
context.debug = 'DEBUG' in os.environ and os.environ['DEBUG'] == 'ON'
|
||||||
if context.debug:
|
if context.debug:
|
||||||
print("DEBUG=ON\n")
|
print("DEBUG=ON")
|
||||||
print(f"\x1b[33;42mStarting new scenario: {scenario.name}!\x1b[0m\n")
|
print(f"\x1b[33;42mStarting new scenario: {scenario.name}!\x1b[0m")
|
||||||
port = 8080
|
port = 8080
|
||||||
if 'PORT' in os.environ:
|
if 'PORT' in os.environ:
|
||||||
port = int(os.environ['PORT'])
|
port = int(os.environ['PORT'])
|
||||||
|
@ -27,60 +26,40 @@ def after_scenario(context, scenario):
|
||||||
return
|
return
|
||||||
if scenario.status == "failed":
|
if scenario.status == "failed":
|
||||||
if 'GITHUB_ACTIONS' in os.environ:
|
if 'GITHUB_ACTIONS' in os.environ:
|
||||||
print(f"\x1b[33;101mSCENARIO FAILED: {scenario.name} server logs:\x1b[0m\n\n")
|
print(f"\x1b[33;101mSCENARIO FAILED: {scenario.name} server logs:\x1b[0m\n")
|
||||||
if os.path.isfile('llama.log'):
|
if os.path.isfile('llama.log'):
|
||||||
with closing(open('llama.log', 'r')) as f:
|
with closing(open('llama.log', 'r')) as f:
|
||||||
for line in f:
|
for line in f:
|
||||||
print(line)
|
print(line)
|
||||||
if not is_server_listening(context.server_fqdn, context.server_port):
|
if not is_server_listening(context.server_fqdn, context.server_port):
|
||||||
print("\x1b[33;101mERROR: Server stopped listening\x1b[0m\n")
|
print("\x1b[33;101mERROR: Server stopped listening\x1b[0m")
|
||||||
|
|
||||||
if not pid_exists(context.server_process.pid):
|
if context.server_process.poll() is not None:
|
||||||
assert False, f"Server not running pid={context.server_process.pid} ..."
|
assert False, f"Server not running pid={context.server_process.pid} ..."
|
||||||
|
|
||||||
server_graceful_shutdown(context)
|
server_graceful_shutdown(context) # SIGINT
|
||||||
|
|
||||||
# Wait few for socket to free up
|
try:
|
||||||
time.sleep(0.05)
|
context.server_process.wait(0.5)
|
||||||
|
except TimeoutExpired:
|
||||||
|
print(f"server still alive after 500ms, force-killing pid={context.server_process.pid} ...")
|
||||||
|
context.server_process.kill() # SIGKILL
|
||||||
|
context.server_process.wait()
|
||||||
|
|
||||||
attempts = 0
|
while is_server_listening(context.server_fqdn, context.server_port):
|
||||||
while pid_exists(context.server_process.pid) or is_server_listening(context.server_fqdn, context.server_port):
|
|
||||||
server_kill(context)
|
|
||||||
time.sleep(0.1)
|
time.sleep(0.1)
|
||||||
attempts += 1
|
except Exception:
|
||||||
if attempts > 5:
|
print("ignoring error in after_scenario:")
|
||||||
server_kill_hard(context)
|
traceback.print_exc(file=sys.stdout)
|
||||||
except:
|
|
||||||
exc = sys.exception()
|
|
||||||
print("error in after scenario: \n")
|
|
||||||
print(exc)
|
|
||||||
print("*** print_tb: \n")
|
|
||||||
traceback.print_tb(exc.__traceback__, file=sys.stdout)
|
|
||||||
|
|
||||||
|
|
||||||
def server_graceful_shutdown(context):
|
def server_graceful_shutdown(context):
|
||||||
print(f"shutting down server pid={context.server_process.pid} ...\n")
|
print(f"shutting down server pid={context.server_process.pid} ...")
|
||||||
if os.name == 'nt':
|
if os.name == 'nt':
|
||||||
os.kill(context.server_process.pid, signal.CTRL_C_EVENT)
|
interrupt = signal.CTRL_C_EVENT
|
||||||
else:
|
else:
|
||||||
os.kill(context.server_process.pid, signal.SIGINT)
|
interrupt = signal.SIGINT
|
||||||
|
context.server_process.send_signal(interrupt)
|
||||||
|
|
||||||
def server_kill(context):
|
|
||||||
print(f"killing server pid={context.server_process.pid} ...\n")
|
|
||||||
context.server_process.kill()
|
|
||||||
|
|
||||||
|
|
||||||
def server_kill_hard(context):
|
|
||||||
pid = context.server_process.pid
|
|
||||||
path = context.server_path
|
|
||||||
|
|
||||||
print(f"Server dangling exits, hard killing force {pid}={path}...\n")
|
|
||||||
try:
|
|
||||||
psutil.Process(pid).kill()
|
|
||||||
except psutil.NoSuchProcess:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def is_server_listening(server_fqdn, server_port):
|
def is_server_listening(server_fqdn, server_port):
|
||||||
|
@ -88,14 +67,5 @@ def is_server_listening(server_fqdn, server_port):
|
||||||
result = sock.connect_ex((server_fqdn, server_port))
|
result = sock.connect_ex((server_fqdn, server_port))
|
||||||
_is_server_listening = result == 0
|
_is_server_listening = result == 0
|
||||||
if _is_server_listening:
|
if _is_server_listening:
|
||||||
print(f"server is listening on {server_fqdn}:{server_port}...\n")
|
print(f"server is listening on {server_fqdn}:{server_port}...")
|
||||||
return _is_server_listening
|
return _is_server_listening
|
||||||
|
|
||||||
|
|
||||||
def pid_exists(pid):
|
|
||||||
try:
|
|
||||||
psutil.Process(pid)
|
|
||||||
except psutil.NoSuchProcess:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
|
@ -35,9 +35,9 @@ Feature: llama.cpp server
|
||||||
And metric llamacpp:tokens_predicted is <n_predicted>
|
And metric llamacpp:tokens_predicted is <n_predicted>
|
||||||
|
|
||||||
Examples: Prompts
|
Examples: Prompts
|
||||||
| prompt | n_predict | re_content | n_prompt | n_predicted | truncated |
|
| prompt | n_predict | re_content | n_prompt | n_predicted | truncated |
|
||||||
| I believe the meaning of life is | 8 | (read\|going)+ | 18 | 8 | not |
|
| I believe the meaning of life is | 8 | (read\|going)+ | 18 | 8 | not |
|
||||||
| Write a joke about AI from a very long prompt which will not be truncated | 256 | (princesses\|everyone\|kids)+ | 46 | 64 | not |
|
| Write a joke about AI from a very long prompt which will not be truncated | 256 | (princesses\|everyone\|kids\|Anna\|forest)+ | 46 | 64 | not |
|
||||||
|
|
||||||
Scenario: Completion prompt truncated
|
Scenario: Completion prompt truncated
|
||||||
Given a prompt:
|
Given a prompt:
|
||||||
|
@ -48,7 +48,7 @@ Feature: llama.cpp server
|
||||||
Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
|
Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
|
||||||
"""
|
"""
|
||||||
And a completion request with no api error
|
And a completion request with no api error
|
||||||
Then 64 tokens are predicted matching fun|Annaks|popcorns|pictry
|
Then 64 tokens are predicted matching fun|Annaks|popcorns|pictry|bowl
|
||||||
And the completion is truncated
|
And the completion is truncated
|
||||||
And 109 prompt tokens are processed
|
And 109 prompt tokens are processed
|
||||||
|
|
||||||
|
@ -65,9 +65,9 @@ Feature: llama.cpp server
|
||||||
And the completion is <truncated> truncated
|
And the completion is <truncated> truncated
|
||||||
|
|
||||||
Examples: Prompts
|
Examples: Prompts
|
||||||
| model | system_prompt | user_prompt | max_tokens | re_content | n_prompt | n_predicted | enable_streaming | truncated |
|
| model | system_prompt | user_prompt | max_tokens | re_content | n_prompt | n_predicted | enable_streaming | truncated |
|
||||||
| llama-2 | Book | What is the best book | 8 | (Here\|what)+ | 77 | 8 | disabled | not |
|
| llama-2 | Book | What is the best book | 8 | (Here\|what)+ | 77 | 8 | disabled | not |
|
||||||
| codellama70b | You are a coding assistant. | Write the fibonacci function in c++. | 128 | (thanks\|happy\|bird)+ | -1 | 64 | enabled | |
|
| codellama70b | You are a coding assistant. | Write the fibonacci function in c++. | 128 | (thanks\|happy\|bird\|Annabyear)+ | -1 | 64 | enabled | |
|
||||||
|
|
||||||
|
|
||||||
Scenario: Tokenize / Detokenize
|
Scenario: Tokenize / Detokenize
|
||||||
|
|
|
@ -24,12 +24,16 @@ from prometheus_client import parser
|
||||||
def step_server_config(context, server_fqdn, server_port):
|
def step_server_config(context, server_fqdn, server_port):
|
||||||
context.server_fqdn = server_fqdn
|
context.server_fqdn = server_fqdn
|
||||||
context.server_port = int(server_port)
|
context.server_port = int(server_port)
|
||||||
|
context.n_gpu_layer = None
|
||||||
if 'PORT' in os.environ:
|
if 'PORT' in os.environ:
|
||||||
context.server_port = int(os.environ['PORT'])
|
context.server_port = int(os.environ['PORT'])
|
||||||
print(f"$PORT set, overriding server port with to {context.server_port}")
|
print(f"$PORT set, overriding server port with to {context.server_port}")
|
||||||
if 'FQDN' in os.environ:
|
if 'FQDN' in os.environ:
|
||||||
context.server_fqdn = os.environ['FQDN']
|
context.server_fqdn = os.environ['FQDN']
|
||||||
print(f"$FQDN set, overriding server fqdn with to {context.server_fqdn}")
|
print(f"$FQDN set, overriding server fqdn with to {context.server_fqdn}")
|
||||||
|
if 'N_GPU_LAYERS' in os.environ:
|
||||||
|
context.n_gpu_layer = int(os.environ['N_GPU_LAYERS'])
|
||||||
|
print(f"$N_GPU_LAYERS set, overriding n_gpu_layer with to {context.n_gpu_layer}")
|
||||||
|
|
||||||
context.base_url = f'http://{context.server_fqdn}:{context.server_port}'
|
context.base_url = f'http://{context.server_fqdn}:{context.server_port}'
|
||||||
|
|
||||||
|
@ -41,7 +45,6 @@ def step_server_config(context, server_fqdn, server_port):
|
||||||
context.n_ctx = None
|
context.n_ctx = None
|
||||||
context.n_ga = None
|
context.n_ga = None
|
||||||
context.n_ga_w = None
|
context.n_ga_w = None
|
||||||
context.n_gpu_layer = None
|
|
||||||
context.n_predict = None
|
context.n_predict = None
|
||||||
context.n_prompts = 0
|
context.n_prompts = 0
|
||||||
context.n_server_predict = None
|
context.n_server_predict = None
|
||||||
|
@ -66,7 +69,7 @@ def step_server_config(context, server_fqdn, server_port):
|
||||||
def step_download_hf_model(context, hf_file, hf_repo):
|
def step_download_hf_model(context, hf_file, hf_repo):
|
||||||
context.model_file = hf_hub_download(repo_id=hf_repo, filename=hf_file)
|
context.model_file = hf_hub_download(repo_id=hf_repo, filename=hf_file)
|
||||||
if context.debug:
|
if context.debug:
|
||||||
print(f"model file: {context.model_file}\n")
|
print(f"model file: {context.model_file}")
|
||||||
|
|
||||||
|
|
||||||
@step('a model file {model_file}')
|
@step('a model file {model_file}')
|
||||||
|
@ -137,9 +140,12 @@ def step_start_server(context):
|
||||||
if 'GITHUB_ACTIONS' in os.environ:
|
if 'GITHUB_ACTIONS' in os.environ:
|
||||||
max_attempts *= 2
|
max_attempts *= 2
|
||||||
|
|
||||||
|
addrs = socket.getaddrinfo(context.server_fqdn, context.server_port, type=socket.SOCK_STREAM)
|
||||||
|
family, typ, proto, _, sockaddr = addrs[0]
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
|
with closing(socket.socket(family, typ, proto)) as sock:
|
||||||
result = sock.connect_ex((context.server_fqdn, context.server_port))
|
result = sock.connect_ex(sockaddr)
|
||||||
if result == 0:
|
if result == 0:
|
||||||
print("\x1b[33;46mserver started!\x1b[0m")
|
print("\x1b[33;46mserver started!\x1b[0m")
|
||||||
return
|
return
|
||||||
|
@ -209,7 +215,7 @@ async def step_request_completion(context, api_error):
|
||||||
user_api_key=context.user_api_key)
|
user_api_key=context.user_api_key)
|
||||||
context.tasks_result.append(completion)
|
context.tasks_result.append(completion)
|
||||||
if context.debug:
|
if context.debug:
|
||||||
print(f"Completion response: {completion}\n")
|
print(f"Completion response: {completion}")
|
||||||
if expect_api_error:
|
if expect_api_error:
|
||||||
assert completion == 401, f"completion must be an 401 status code: {completion}"
|
assert completion == 401, f"completion must be an 401 status code: {completion}"
|
||||||
|
|
||||||
|
@ -354,7 +360,7 @@ def step_prompt_passkey(context, passkey, i_pos):
|
||||||
prompt += context.prompt_junk_suffix
|
prompt += context.prompt_junk_suffix
|
||||||
if context.debug:
|
if context.debug:
|
||||||
passkey_highlight = "\x1b[33m" + passkey + "\x1b[0m"
|
passkey_highlight = "\x1b[33m" + passkey + "\x1b[0m"
|
||||||
print(f"Passkey challenge:\n```{prompt.replace(passkey, passkey_highlight)}```\n")
|
print(f"Passkey challenge:\n```{prompt.replace(passkey, passkey_highlight)}```")
|
||||||
context.prompts.append(context.prompt_prefix + prompt + context.prompt_suffix)
|
context.prompts.append(context.prompt_prefix + prompt + context.prompt_suffix)
|
||||||
context.n_prompts = len(context.prompts)
|
context.n_prompts = len(context.prompts)
|
||||||
|
|
||||||
|
@ -363,7 +369,7 @@ def step_prompt_passkey(context, passkey, i_pos):
|
||||||
@async_run_until_complete
|
@async_run_until_complete
|
||||||
async def step_oai_chat_completions(context, api_error):
|
async def step_oai_chat_completions(context, api_error):
|
||||||
if context.debug:
|
if context.debug:
|
||||||
print(f"Submitting OAI compatible completions request...\n")
|
print(f"Submitting OAI compatible completions request...")
|
||||||
expect_api_error = api_error == 'raised'
|
expect_api_error = api_error == 'raised'
|
||||||
completion = await oai_chat_completions(context.prompts.pop(),
|
completion = await oai_chat_completions(context.prompts.pop(),
|
||||||
context.system_prompt,
|
context.system_prompt,
|
||||||
|
@ -508,12 +514,12 @@ async def step_all_embeddings_are_the_same(context):
|
||||||
embedding1 = np.array(embeddings[i])
|
embedding1 = np.array(embeddings[i])
|
||||||
embedding2 = np.array(embeddings[j])
|
embedding2 = np.array(embeddings[j])
|
||||||
if context.debug:
|
if context.debug:
|
||||||
print(f"embedding1: {embedding1[-8:]}\n")
|
print(f"embedding1: {embedding1[-8:]}")
|
||||||
print(f"embedding2: {embedding2[-8:]}\n")
|
print(f"embedding2: {embedding2[-8:]}")
|
||||||
similarity = np.dot(embedding1, embedding2) / (np.linalg.norm(embedding1) * np.linalg.norm(embedding2))
|
similarity = np.dot(embedding1, embedding2) / (np.linalg.norm(embedding1) * np.linalg.norm(embedding2))
|
||||||
msg = f"Similarity between {i} and {j}: {similarity:.10f}"
|
msg = f"Similarity between {i} and {j}: {similarity:.10f}"
|
||||||
if context.debug:
|
if context.debug:
|
||||||
print(f"{msg}\n")
|
print(f"{msg}")
|
||||||
assert np.isclose(similarity, 1.0, rtol=1e-05, atol=1e-08, equal_nan=False), msg
|
assert np.isclose(similarity, 1.0, rtol=1e-05, atol=1e-08, equal_nan=False), msg
|
||||||
|
|
||||||
|
|
||||||
|
@ -630,7 +636,7 @@ async def step_prometheus_metrics_exported(context):
|
||||||
metrics_raw = await metrics_response.text()
|
metrics_raw = await metrics_response.text()
|
||||||
metric_exported = False
|
metric_exported = False
|
||||||
if context.debug:
|
if context.debug:
|
||||||
print(f"/metrics answer:\n{metrics_raw}\n")
|
print(f"/metrics answer:\n{metrics_raw}")
|
||||||
context.metrics = {}
|
context.metrics = {}
|
||||||
for metric in parser.text_string_to_metric_families(metrics_raw):
|
for metric in parser.text_string_to_metric_families(metrics_raw):
|
||||||
match metric.name:
|
match metric.name:
|
||||||
|
@ -932,7 +938,7 @@ def assert_n_tokens_predicted(completion_response, expected_predicted_n=None, re
|
||||||
last_match = end
|
last_match = end
|
||||||
highlighted += content[last_match:]
|
highlighted += content[last_match:]
|
||||||
if 'DEBUG' in os.environ and os.environ['DEBUG'] == 'ON':
|
if 'DEBUG' in os.environ and os.environ['DEBUG'] == 'ON':
|
||||||
print(f"Checking completion response: {highlighted}\n")
|
print(f"Checking completion response: {highlighted}")
|
||||||
assert last_match > 0, f'/{re_content}/ must match ```{highlighted}```'
|
assert last_match > 0, f'/{re_content}/ must match ```{highlighted}```'
|
||||||
if expected_predicted_n and expected_predicted_n > 0:
|
if expected_predicted_n and expected_predicted_n > 0:
|
||||||
assert n_predicted == expected_predicted_n, (f'invalid number of tokens predicted:'
|
assert n_predicted == expected_predicted_n, (f'invalid number of tokens predicted:'
|
||||||
|
@ -942,7 +948,7 @@ def assert_n_tokens_predicted(completion_response, expected_predicted_n=None, re
|
||||||
async def gather_tasks_results(context):
|
async def gather_tasks_results(context):
|
||||||
n_tasks = len(context.concurrent_tasks)
|
n_tasks = len(context.concurrent_tasks)
|
||||||
if context.debug:
|
if context.debug:
|
||||||
print(f"Waiting for all {n_tasks} tasks results...\n")
|
print(f"Waiting for all {n_tasks} tasks results...")
|
||||||
for task_no in range(n_tasks):
|
for task_no in range(n_tasks):
|
||||||
context.tasks_result.append(await context.concurrent_tasks.pop())
|
context.tasks_result.append(await context.concurrent_tasks.pop())
|
||||||
n_completions = len(context.tasks_result)
|
n_completions = len(context.tasks_result)
|
||||||
|
@ -959,7 +965,7 @@ async def wait_for_health_status(context,
|
||||||
slots_processing=None,
|
slots_processing=None,
|
||||||
expected_slots=None):
|
expected_slots=None):
|
||||||
if context.debug:
|
if context.debug:
|
||||||
print(f"Starting checking for health for expected_health_status={expected_health_status}\n")
|
print(f"Starting checking for health for expected_health_status={expected_health_status}")
|
||||||
interval = 0.5
|
interval = 0.5
|
||||||
counter = 0
|
counter = 0
|
||||||
if 'GITHUB_ACTIONS' in os.environ:
|
if 'GITHUB_ACTIONS' in os.environ:
|
||||||
|
@ -1048,8 +1054,6 @@ def start_server_background(context):
|
||||||
if 'LLAMA_SERVER_BIN_PATH' in os.environ:
|
if 'LLAMA_SERVER_BIN_PATH' in os.environ:
|
||||||
context.server_path = os.environ['LLAMA_SERVER_BIN_PATH']
|
context.server_path = os.environ['LLAMA_SERVER_BIN_PATH']
|
||||||
server_listen_addr = context.server_fqdn
|
server_listen_addr = context.server_fqdn
|
||||||
if os.name == 'nt':
|
|
||||||
server_listen_addr = '0.0.0.0'
|
|
||||||
server_args = [
|
server_args = [
|
||||||
'--host', server_listen_addr,
|
'--host', server_listen_addr,
|
||||||
'--port', context.server_port,
|
'--port', context.server_port,
|
||||||
|
@ -1088,7 +1092,7 @@ def start_server_background(context):
|
||||||
server_args.append('--verbose')
|
server_args.append('--verbose')
|
||||||
if 'SERVER_LOG_FORMAT_JSON' not in os.environ:
|
if 'SERVER_LOG_FORMAT_JSON' not in os.environ:
|
||||||
server_args.extend(['--log-format', "text"])
|
server_args.extend(['--log-format', "text"])
|
||||||
print(f"starting server with: {context.server_path} {server_args}\n")
|
print(f"starting server with: {context.server_path} {server_args}")
|
||||||
flags = 0
|
flags = 0
|
||||||
if 'nt' == os.name:
|
if 'nt' == os.name:
|
||||||
flags |= subprocess.DETACHED_PROCESS
|
flags |= subprocess.DETACHED_PROCESS
|
||||||
|
|
|
@ -3,5 +3,4 @@ behave~=1.2.6
|
||||||
huggingface_hub~=0.20.3
|
huggingface_hub~=0.20.3
|
||||||
numpy~=1.24.4
|
numpy~=1.24.4
|
||||||
openai~=0.25.0
|
openai~=0.25.0
|
||||||
psutil~=5.9.8
|
|
||||||
prometheus-client~=0.20.0
|
prometheus-client~=0.20.0
|
||||||
|
|
|
@ -371,6 +371,7 @@ static json oaicompat_completion_params_parse(
|
||||||
llama_params["repeat_last_n"] = json_value(body, "repeat_last_n", default_sparams.penalty_last_n);
|
llama_params["repeat_last_n"] = json_value(body, "repeat_last_n", default_sparams.penalty_last_n);
|
||||||
llama_params["ignore_eos"] = json_value(body, "ignore_eos", false);
|
llama_params["ignore_eos"] = json_value(body, "ignore_eos", false);
|
||||||
llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z);
|
llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z);
|
||||||
|
llama_params["n_keep"] = json_value(body, "n_keep", 0);
|
||||||
|
|
||||||
if (body.count("grammar") != 0) {
|
if (body.count("grammar") != 0) {
|
||||||
llama_params["grammar"] = json_value(body, "grammar", json::object());
|
llama_params["grammar"] = json_value(body, "grammar", json::object());
|
||||||
|
|
|
@ -6,8 +6,6 @@ set INPUT2="Building a website can be done in 10 simple steps:\nStep 1:"
|
||||||
@call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force
|
@call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force
|
||||||
|
|
||||||
|
|
||||||
set GGML_SYCL_DEVICE=0
|
|
||||||
rem set GGML_SYCL_DEBUG=1
|
|
||||||
.\build\bin\main.exe -m models\llama-2-7b.Q4_0.gguf -p %INPUT2% -n 400 -e -ngl 33 -s 0
|
.\build\bin\main.exe -m models\llama-2-7b.Q4_0.gguf -p %INPUT2% -n 400 -e -ngl 33 -s 0
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -13,7 +13,7 @@
|
||||||
extern "C" {
|
extern "C" {
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#define GGML_SYCL_MAX_DEVICES 16
|
#define GGML_SYCL_MAX_DEVICES 48
|
||||||
#define GGML_SYCL_NAME "SYCL"
|
#define GGML_SYCL_NAME "SYCL"
|
||||||
|
|
||||||
GGML_API void ggml_init_sycl(void);
|
GGML_API void ggml_init_sycl(void);
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue