diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt
index 19b36a163..9571e153e 100644
--- a/ggml/src/CMakeLists.txt
+++ b/ggml/src/CMakeLists.txt
@@ -275,19 +275,6 @@ if (GGML_SYCL)
     endif()
 endif()
 
-if (GGML_RPC)
-    message(STATUS "RPC found")
-
-    list(APPEND GGML_CDEF_PUBLIC GGML_USE_RPC)
-
-    if (WIN32)
-        list(APPEND GGML_EXTRA_LIBS_PRIVATE ws2_32)
-    endif()
-
-    set(GGML_HEADERS_RPC ../include/ggml-rpc.h)
-    set(GGML_SOURCES_RPC ggml-rpc.cpp)
-endif()
-
 if (GGML_VULKAN)
     find_package(Vulkan COMPONENTS glslc REQUIRED)
 
@@ -822,6 +809,11 @@ if (GGML_BLAS)
     target_link_libraries(ggml PUBLIC ggml-blas)
 endif()
 
+if (GGML_RPC)
+    add_subdirectory(ggml-rpc)
+    target_link_libraries(ggml PUBLIC ggml-rpc)
+endif()
+
 if (EMSCRIPTEN)
     set_target_properties(ggml PROPERTIES COMPILE_FLAGS "-msimd128")
 endif()
diff --git a/ggml/src/ggml-rpc/CMakeLists.txt b/ggml/src/ggml-rpc/CMakeLists.txt
new file mode 100644
index 000000000..bd977f46a
--- /dev/null
+++ b/ggml/src/ggml-rpc/CMakeLists.txt
@@ -0,0 +1,13 @@
+message(STATUS "Using RPC backend")
+
+set(GGML_CDEF_PUBLIC ${GGML_CDEF_PUBLIC} GGML_USE_RPC PARENT_SCOPE)
+
+add_library(ggml-rpc
+            ggml-rpc.cpp)
+
+target_link_libraries(ggml-rpc PRIVATE ggml-base)
+target_include_directories(ggml-rpc PRIVATE . ..)
+
+if (WIN32)
+    target_link_libraries(ggml-rpc PRIVATE ws2_32)
+endif()