diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp
index 9e2846ee4..9a1c68a7d 100644
--- a/ggml-vulkan.cpp
+++ b/ggml-vulkan.cpp
@@ -744,6 +744,8 @@ static vk_buffer ggml_vk_create_buffer(ggml_backend_vk_context * ctx, size_t siz
     }
 
     if (memory_type_index >= mem_props.memoryTypeCount) {
+        ctx->device.lock()->device.destroyBuffer(buf->buffer);
+        buf->size = 0;
         throw vk::OutOfDeviceMemoryError("No suitable memory type found");
     }