vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improve ClBlast implementation, avoid recreating buffers, remove redundant transfers

Browse source
This commit is contained in:
0cc4m 2023-04-11 21:53:50 +02:00
parent f4257a8eef
commit c7e5c4f7b2
1 changed files with 63 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

93
ggml_blas_adapter.c
View file

@ -18,36 +18,45 @@ cl_platform_id platform;
cl_device_id device; cl_device_id device;
cl_context context; cl_context context;
cl_command_queue queue; cl_command_queue queue;
cl_event event;
bool cl_initialized = false; bool cl_initialized = false;
size_t cl_size_a = 0, cl_size_b = 0, cl_size_c = 0;
cl_mem cl_buffer_a, cl_buffer_b, cl_buffer_c;
static void ggml_cl_sgemm_wrapper(const enum CBLAS_ORDER order, const enum CBLAS_TRANSPOSE trans_a, const enum CBLAS_TRANSPOSE trans_b, const int m, const int n, const int k, const float alpha, const float *host_a, const int lda, const float *host_b, const int ldb, const float beta, float *host_c, const int ldc) { static void ggml_cl_sgemm_wrapper(const enum CBLAS_ORDER order, const enum CBLAS_TRANSPOSE trans_a, const enum CBLAS_TRANSPOSE trans_b, const int m, const int n, const int k, const float alpha, const float *host_a, const int lda, const float *host_b, const int ldb, const float beta, float *host_c, const int ldc) {
cl_int err = 0; cl_int err = 0;
cl_event events[2];
events[0] = NULL;
events[1] = NULL;
if (!cl_initialized) { if (!cl_initialized) {
char * KCPP_CLBLAST_PLATFORM = getenv("KCPP_CLBLAST_PLATFORM"); char * KCPP_CLBLAST_PLATFORM = getenv("KCPP_CLBLAST_PLATFORM");
char * KCPP_CLBLAST_DEVICES = getenv("KCPP_CLBLAST_DEVICES"); char * KCPP_CLBLAST_DEVICES = getenv("KCPP_CLBLAST_DEVICES");
int plat_num = (KCPP_CLBLAST_PLATFORM == NULL ? 0 : atoi(KCPP_CLBLAST_PLATFORM)); int plat_num = (KCPP_CLBLAST_PLATFORM == NULL ? 0 : atoi(KCPP_CLBLAST_PLATFORM));
int dev_num = (KCPP_CLBLAST_DEVICES == NULL ? 0 : atoi(KCPP_CLBLAST_DEVICES)); int dev_num = (KCPP_CLBLAST_DEVICES == NULL ? 0 : atoi(KCPP_CLBLAST_DEVICES));
printf("\nInitializing CLBlast (First Run)..."); printf("\nInitializing CLBlast (First Run)...");
printf("\nSelected: Platform=%d, Device=%d (If invalid, program will crash)\n",plat_num,dev_num); printf("\nAttempting to use: Platform=%d, Device=%d (If invalid, program will crash)\n",plat_num,dev_num);
cl_uint num_platforms; cl_uint num_platforms;
clGetPlatformIDs(0, NULL, &num_platforms); clGetPlatformIDs(0, NULL, &num_platforms);
cl_platform_id* platforms = (cl_platform_id*)malloc(num_platforms*sizeof(cl_platform_id)); cl_platform_id* platforms = (cl_platform_id*)malloc(num_platforms*sizeof(cl_platform_id));
clGetPlatformIDs(num_platforms, platforms, NULL); clGetPlatformIDs(num_platforms, platforms, NULL);
platform = platforms[plat_num]; platform = platforms[plat_num];
char platform_buffer[1024];
clGetPlatformInfo(platform, CL_PLATFORM_NAME, sizeof(platform_buffer), &platform_buffer, NULL);
cl_uint num_devices; cl_uint num_devices;
clGetDeviceIDs(platform, CL_DEVICE_TYPE_ALL, 0, NULL, &num_devices); clGetDeviceIDs(platform, CL_DEVICE_TYPE_ALL, 0, NULL, &num_devices);
cl_device_id* devices = (cl_device_id*)malloc(num_devices*sizeof(cl_device_id)); cl_device_id* devices = (cl_device_id*)malloc(num_devices*sizeof(cl_device_id));
clGetDeviceIDs(platform, CL_DEVICE_TYPE_ALL, num_devices, devices, NULL); clGetDeviceIDs(platform, CL_DEVICE_TYPE_ALL, num_devices, devices, NULL);
device = devices[dev_num]; device = devices[dev_num];
char device_buffer[1024];
clGetDeviceInfo(device, CL_DEVICE_NAME, sizeof(device_buffer), &device_buffer, NULL);
printf("Using Platform: %s Device: %s\n", platform_buffer, device_buffer);
context = clCreateContext(NULL, 1, &device, NULL, NULL, &err); context = clCreateContext(NULL, 1, &device, NULL, NULL, &err);
if (err != CL_SUCCESS) { if (err != CL_SUCCESS) {
printf("Error creating OpenCL context: %d\n", err); printf("Error creating OpenCL context: %d\n", err);
fflush(stdout); fflush(stdout);
} }
queue = clCreateCommandQueue(context, device, 0, &err); queue = clCreateCommandQueue(context, device, 0, &err);
event = NULL;
if (err != CL_SUCCESS) { if (err != CL_SUCCESS) {
printf("Error creating OpenCL Command Queue: %d\n", err); printf("Error creating OpenCL Command Queue: %d\n", err);
@ -56,29 +65,56 @@ static void ggml_cl_sgemm_wrapper(const enum CBLAS_ORDER order, const enum CBLAS
free(platforms); free(platforms);
free(devices); free(devices);
cl_size_a = m * k * sizeof(float);
cl_size_b = n * k * sizeof(float);
cl_size_c = m * n * sizeof(float);
// Prepare buffers
cl_buffer_a = clCreateBuffer(context, CL_MEM_READ_ONLY, cl_size_a, NULL, &err);
if (err != CL_SUCCESS) {
printf("Error creating OpenCL Buffer A: %d\n", err);
fflush(stdout);
}
cl_buffer_b = clCreateBuffer(context, CL_MEM_READ_ONLY, cl_size_b, NULL, &err);
if (err != CL_SUCCESS) {
printf("Error creating OpenCL Buffer B: %d\n", err);
fflush(stdout);
}
cl_buffer_c = clCreateBuffer(context, CL_MEM_READ_WRITE, cl_size_c, NULL, &err);
if (err != CL_SUCCESS) {
printf("Error creating OpenCL Buffer C: %d\n", err);
fflush(stdout);
}
cl_initialized = true; cl_initialized = true;
} }
// Prepare buffers // Resize buffers if too small
cl_mem cl_buffer_a = clCreateBuffer(context, CL_MEM_READ_WRITE, m*k*sizeof(float), NULL, &err); if (m * k * sizeof(float) > cl_size_a) {
if (err != CL_SUCCESS) { clReleaseMemObject(cl_buffer_a);
printf("Error creating OpenCL Buffer A: %d\n", err); cl_size_a = m * k * sizeof(float);
fflush(stdout); cl_buffer_a = clCreateBuffer(context, CL_MEM_READ_ONLY, cl_size_a, NULL, NULL);
} }
cl_mem cl_buffer_b = clCreateBuffer(context, CL_MEM_READ_WRITE, n*k*sizeof(float), NULL, &err); if (n * k * sizeof(float) > cl_size_b) {
if (err != CL_SUCCESS) { clReleaseMemObject(cl_buffer_b);
printf("Error creating OpenCL Buffer B: %d\n", err); cl_size_b = n * k * sizeof(float);
fflush(stdout); cl_buffer_b = clCreateBuffer(context, CL_MEM_READ_ONLY, cl_size_b, NULL, NULL);
} }
cl_mem cl_buffer_c = clCreateBuffer(context, CL_MEM_READ_WRITE, m*n*sizeof(float), NULL, &err); if (m * n * sizeof(float) > cl_size_c) {
if (err != CL_SUCCESS) { clReleaseMemObject(cl_buffer_c);
printf("Error creating OpenCL Buffer C: %d\n", err); cl_size_c = m * n * sizeof(float);
fflush(stdout); cl_buffer_c = clCreateBuffer(context, CL_MEM_READ_WRITE, cl_size_c, NULL, NULL);
} }
clEnqueueWriteBuffer(queue, cl_buffer_a, CL_TRUE, 0, m*k*sizeof(float), host_a, 0, NULL, NULL); clEnqueueWriteBuffer(queue, cl_buffer_a, CL_TRUE, 0, cl_size_a, host_a, 0, NULL, events);
clEnqueueWriteBuffer(queue, cl_buffer_b, CL_TRUE, 0, n*k*sizeof(float), host_b, 0, NULL, NULL); clEnqueueWriteBuffer(queue, cl_buffer_b, CL_TRUE, 0, cl_size_b, host_b, 0, NULL, events + 1);
clEnqueueWriteBuffer(queue, cl_buffer_c, CL_TRUE, 0, m*n*sizeof(float), host_c, 0, NULL, NULL); // buffer c is not required for this use case
// clEnqueueWriteBuffer(queue, cl_buffer_c, CL_TRUE, 0, cl_size_c, host_c, 0, NULL, NULL);
clWaitForEvents(2, events);
clReleaseEvent(events[0]);
clReleaseEvent(events[1]);
// Call the SGEMM routine. // Call the SGEMM routine.
CLBlastStatusCode status = CLBlastSgemm(order, CLBlastStatusCode status = CLBlastSgemm(order,
@ -89,19 +125,16 @@ static void ggml_cl_sgemm_wrapper(const enum CBLAS_ORDER order, const enum CBLAS
cl_buffer_b, 0, ldb, cl_buffer_b, 0, ldb,
beta, beta,
cl_buffer_c, 0, ldc, cl_buffer_c, 0, ldc,
&queue, &event); &queue, events);
clEnqueueReadBuffer(queue, cl_buffer_c, CL_TRUE, 0, m*n*sizeof(float), host_c, 1, events, events + 1);
// Wait for completion // Wait for completion
if (status == CLBlastSuccess) { if (status == CLBlastSuccess) {
clWaitForEvents(1, &event); clWaitForEvents(2, events);
clReleaseEvent(event); clReleaseEvent(events[0]);
clReleaseEvent(events[1]);
} }
clEnqueueReadBuffer(queue, cl_buffer_c, CL_TRUE, 0, m*n*sizeof(float), host_c, 0, NULL, NULL);
clReleaseMemObject(cl_buffer_a);
clReleaseMemObject(cl_buffer_b);
clReleaseMemObject(cl_buffer_c);
} }
#endif #endif
@ -117,4 +150,4 @@ ggml_cl_sgemm_wrapper(Order, TransA, TransB, M, N, K, alpha, A, lda, B, ldb, bet
cblas_sgemm(Order, TransA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc);\ cblas_sgemm(Order, TransA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc);\
}) })
#endif #endif
#endif #endif