vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix ggml_vk_h2d_tensor throwing on second call

Browse source
This commit is contained in:
niansa 2023-06-23 16:50:37 +02:00
parent 09b0b3a49b
commit 98e588c6eb
2 changed files with 21 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

8
ggml-vulkan.cpp
View file

@ -91,6 +91,13 @@ void ggml_vk_h2d_tensor(struct ggml_kompute_context * ctx, struct ggml_tensor *
auto data = t->data; auto data = t->data;
auto size = ggml_nbytes(t); auto size = ggml_nbytes(t);
auto res = ctx->tensors.find(t);
if (res != ctx->tensors.end()) {
assert(res->second->size() != size);
res->second->setRawData(data);
mgr.sequence()->eval<kp::OpTensorSyncDevice>({res->second});
} else {
std::vector<byte> vec(size); std::vector<byte> vec(size);
memcpy(vec.data(), data, size); memcpy(vec.data(), data, size);
@ -98,6 +105,7 @@ void ggml_vk_h2d_tensor(struct ggml_kompute_context * ctx, struct ggml_tensor *
mgr.sequence()->eval<kp::OpTensorSyncDevice>({tensor}); mgr.sequence()->eval<kp::OpTensorSyncDevice>({tensor});
ctx->tensors.emplace(t, std::move(tensor)); ctx->tensors.emplace(t, std::move(tensor));
} }
}
void ggml_vk_d2h_tensor(struct ggml_kompute_context * ctx, struct ggml_tensor * t) { void ggml_vk_d2h_tensor(struct ggml_kompute_context * ctx, struct ggml_tensor * t) {
if (t->backend != GGML_BACKEND_GPU) { if (t->backend != GGML_BACKEND_GPU) {

16
llama.cpp
View file

@ -2769,7 +2769,7 @@ struct llama_context * llama_init_from_file(
} }
#elif defined(GGML_USE_KOMPUTE) #elif defined(GGML_USE_KOMPUTE)
if (params.n_gpu_layers > 0) { if (params.n_gpu_layers > 0) {
// this allocates all Metal resources and memory buffers // this allocates all Vulkan resources and memory buffers
ctx->ctx_kompute = ggml_vk_init(); ctx->ctx_kompute = ggml_vk_init();
void * data_ptr = NULL; void * data_ptr = NULL;
@ -2787,21 +2787,21 @@ struct llama_context * llama_init_from_file(
printf("%s: max tensor size = %8.2f MB\n", __func__, max_size/1024.0/1024.0); printf("%s: max tensor size = %8.2f MB\n", __func__, max_size/1024.0/1024.0);
#define LLAMA_METAL_CHECK_BUF(result) \ #define LLAMA_VK_CHECK_BUF(result) \
if (!(result)) { \ if (!(result)) { \
fprintf(stderr, "%s: failed to add buffer\n", __func__); \ fprintf(stderr, "%s: failed to add buffer\n", __func__); \
llama_free(ctx); \ llama_free(ctx); \
return NULL; \ return NULL; \
} }
LLAMA_METAL_CHECK_BUF(ggml_vk_add_buffer(ctx->ctx_kompute, "data", data_ptr, data_size, max_size)); LLAMA_VK_CHECK_BUF(ggml_vk_add_buffer(ctx->ctx_kompute, "data", data_ptr, data_size, max_size));
LLAMA_METAL_CHECK_BUF(ggml_vk_add_buffer(ctx->ctx_kompute, "eval", ctx->buf_compute.addr, ctx->buf_compute.size, 0)); LLAMA_VK_CHECK_BUF(ggml_vk_add_buffer(ctx->ctx_kompute, "eval", ctx->buf_compute.addr, ctx->buf_compute.size, 0));
LLAMA_METAL_CHECK_BUF(ggml_vk_add_buffer(ctx->ctx_kompute, "kv", ctx->model.kv_self.buf.addr, ctx->model.kv_self.buf.size, 0)); LLAMA_VK_CHECK_BUF(ggml_vk_add_buffer(ctx->ctx_kompute, "kv", ctx->model.kv_self.buf.addr, ctx->model.kv_self.buf.size, 0));
LLAMA_METAL_CHECK_BUF(ggml_vk_add_buffer(ctx->ctx_kompute, "scr0", ctx->buf_scratch[0].addr, ctx->buf_scratch[0].size, 0)); LLAMA_VK_CHECK_BUF(ggml_vk_add_buffer(ctx->ctx_kompute, "scr0", ctx->buf_scratch[0].addr, ctx->buf_scratch[0].size, 0));
LLAMA_METAL_CHECK_BUF(ggml_vk_add_buffer(ctx->ctx_kompute, "scr1", ctx->buf_scratch[1].addr, ctx->buf_scratch[1].size, 0)); LLAMA_VK_CHECK_BUF(ggml_vk_add_buffer(ctx->ctx_kompute, "scr1", ctx->buf_scratch[1].addr, ctx->buf_scratch[1].size, 0));
#undef LLAMA_METAL_CHECK_BUF #undef LLAMA_VK_CHECK_BUF
} }
#endif #endif