diff --git a/CMakeLists.txt b/CMakeLists.txt
index e6242dc31..e1e37a2dc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -392,6 +392,10 @@ if (LLAMA_HIPBLAS)
         set_source_files_properties(ggml-cuda.cu PROPERTIES LANGUAGE CXX)
         target_link_libraries(ggml-rocm PRIVATE hip::device PUBLIC hip::host roc::rocblas roc::hipblas)
 
+        if (LLAMA_CUDA_F16)
+            add_compile_definitions(GGML_CUDA_F16)
+        endif()
+
         if (LLAMA_STATIC)
             message(FATAL_ERROR "Static linking not supported for HIP/ROCm")
         endif()
diff --git a/Makefile b/Makefile
index a774dc50f..d5d2b7d65 100644
--- a/Makefile
+++ b/Makefile
@@ -412,6 +412,9 @@ ifdef LLAMA_HIPBLAS
 ifdef LLAMA_CUDA_FORCE_DMMV
 	HIPFLAGS 	+= -DGGML_CUDA_FORCE_DMMV
 endif # LLAMA_CUDA_FORCE_DMMV
+ifdef LLAMA_CUDA_F16
+	HIPFLAGS	+= -DGGML_CUDA_F16
+endif # LLAMA_CUDA_F16
 	OBJS        += ggml-cuda.o
 ggml-cuda.o: ggml-cuda.cu ggml-cuda.h
 	$(HIPCC) $(CXXFLAGS) $(HIPFLAGS) -x hip -c -o $@ $<