vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

implement llama_max_devices() for RPC

Browse source
This commit is contained in:
Radoslav Gerganov 2024-04-30 14:34:09 +03:00
parent 95c16c263c
commit 654c1cc279
2 changed files with 5 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
ggml-rpc.h
View file

@ -37,6 +37,8 @@ enum rpc_cmd {
GRAPH_COMPUTE, GRAPH_COMPUTE,
}; };
#define GGML_RPC_MAX_SERVERS 16
// backend API // backend API
GGML_API GGML_CALL ggml_backend_t ggml_backend_rpc_init(const std::string & endpoint); GGML_API GGML_CALL ggml_backend_t ggml_backend_rpc_init(const std::string & endpoint);
GGML_API GGML_CALL bool ggml_backend_is_rpc(ggml_backend_t backend); GGML_API GGML_CALL bool ggml_backend_is_rpc(ggml_backend_t backend);

4
llama.cpp
View file

@ -15480,7 +15480,9 @@ struct llama_model_quantize_params llama_model_quantize_default_params() {
} }
size_t llama_max_devices(void) { size_t llama_max_devices(void) {
#if defined(GGML_USE_METAL) #if defined(GGML_USE_RPC)
return GGML_RPC_MAX_SERVERS;
#elif defined(GGML_USE_METAL)
return 1; return 1;
#elif defined(GGML_USE_CUDA) #elif defined(GGML_USE_CUDA)
return GGML_CUDA_MAX_DEVICES; return GGML_CUDA_MAX_DEVICES;