vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add clang format file and reformating

Browse source
This commit is contained in:
hongruichen 2024-07-04 22:18:45 +08:00
parent 38f88d5fb1
commit 000240cf62
12 changed files with 1514 additions and 1809 deletions
Download patch file Download diff file Expand all files Collapse all files

25
ggml/include/ggml-qnn.h
View file

@ -1,41 +1,48 @@
#pragma once #pragma once
#include "ggml.h"
#include "ggml-backend.h" #include "ggml-backend.h"
#include "ggml.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#define GGML_QNN_MAX_DEVICES 3 #define GGML_QNN_MAX_DEVICES 3
enum QNNBackend { enum QNNBackend {
QNN_BACKEND_CPU, QNN_BACKEND_CPU,
QNN_BACKEND_GPU, QNN_BACKEND_GPU,
QNN_BACKEND_NPU, QNN_BACKEND_NPU,
QNN_BACKEND_GGML, //"fake" QNN backend, used for compare performance between QNN and original GGML QNN_BACKEND_GGML, //"fake" QNN backend, used for compare performance between
// QNN and original GGML
}; };
GGML_API int ggml_backend_qnn_reg_devices(void); GGML_API int ggml_backend_qnn_reg_devices(void);
/** /**
* *
* @param device 0: QNN_BACKEND_CPU 1: QNN_BACKEND_GPU 2: QNN_BACKEND_NPU * @param device 0: QNN_BACKEND_CPU 1: QNN_BACKEND_GPU 2:
* @param qnn_lib_path qnn library path, such as "/data/local/tmp/" on Android or specified in JNI layer * QNN_BACKEND_NPU
* @param qnn_lib_path qnn library path, such as "/data/local/tmp/" on
* Android or specified in JNI layer
* @return * @return
*/ */
GGML_API ggml_backend_t ggml_backend_qnn_init(size_t dev_num, const char * qnn_lib_path); GGML_API ggml_backend_t ggml_backend_qnn_init(size_t dev_num,
const char* qnn_lib_path);
GGML_API bool ggml_backend_is_qnn(ggml_backend_t backend); GGML_API bool ggml_backend_is_qnn(ggml_backend_t backend);
GGML_API void ggml_backend_qnn_set_n_threads(ggml_backend_t backend, int thread_counts); GGML_API void ggml_backend_qnn_set_n_threads(ggml_backend_t backend,
int thread_counts);
GGML_API int ggml_backend_qnn_get_device_count(void); GGML_API int ggml_backend_qnn_get_device_count(void);
GGML_API void ggml_backend_qnn_get_device_description(size_t dev_num, char * description, size_t description_size); GGML_API void ggml_backend_qnn_get_device_description(size_t dev_num,
char* description,
size_t description_size);
GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_qnn_buffer_type(size_t dev_num); GGML_API GGML_CALL ggml_backend_buffer_type_t
ggml_backend_qnn_buffer_type(size_t dev_num);
#ifdef __cplusplus #ifdef __cplusplus
} }

31
ggml/src/ggml-qnn/.clang-format Normal file
View file

@ -0,0 +1,31 @@
---
BasedOnStyle: Google
IndentWidth: 4
AccessModifierOffset: -4
AlignAfterOpenBracket: Align
AlignOperands: true
AlignTrailingComments: true
BinPackArguments: true
BinPackParameters: true
BreakBeforeBraces: Custom
BreakConstructorInitializers: AfterColon
ColumnLimit: 120
Cpp11BracedListStyle: false
DerivePointerAlignment: false
IncludeCategories:
- Regex: '^<.*\.h>'
Priority: 1
- Regex: '^<.*'
Priority: 2
- Regex: '^"ggml\.h"'
Priority: 3
- Regex: '^"ggml-.+\.h"'
Priority: 4
- Regex: '.*'
Priority: 5
KeepEmptyLinesAtTheStartOfBlocks: true
MaxEmptyLinesToKeep: 1
PointerAlignment: Right
SortIncludes: true
SpacesBeforeTrailingComments: 1
UseTab: Never

394
ggml/src/ggml-qnn/backend-ops.cpp
View file

@ -1,13 +1,12 @@
#include "backend-ops.hpp" #include "backend-ops.hpp"
#include "utils.hpp"
#include "logger.hpp" #include "logger.hpp"
#include "tensor.hpp" #include "tensor.hpp"
#include "utils.hpp"
static bool qnn_is_valid_params(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
static bool qnn_is_valid_params(ggml_backend_qnn_context* ctx, const ggml_tensor* src0, ggml_tensor *dst) {
const ggml_tensor* src1, ggml_tensor* dst) {
if (!ctx || !src0 || !src1 || !dst) { if (!ctx || !src0 || !src1 || !dst) {
QNN_LOG_WARN("invalid params\n"); QNN_LOG_WARN("invalid params\n");
return false; return false;
@ -39,8 +38,8 @@ static bool qnn_is_valid_params(ggml_backend_qnn_context* ctx, const ggml_tensor
// TODO: this function can be removed later because there are duplicated codes with ggml_qnn_mul_mat // TODO: this function can be removed later because there are duplicated codes with ggml_qnn_mul_mat
// keep it for illustrate how to implement a specified GGMPL OP using QNN API + QNN RPC // keep it for illustrate how to implement a specified GGMPL OP using QNN API + QNN RPC
static void ggml_qnn_add(ggml_backend_qnn_context* ctx, const ggml_tensor* src0, static void ggml_qnn_add(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
const ggml_tensor* src1, ggml_tensor* dst) { ggml_tensor *dst) {
Qnn_ErrorHandle_t error = QNN_SUCCESS; Qnn_ErrorHandle_t error = QNN_SUCCESS;
bool graph_initialized = false; bool graph_initialized = false;
qnn::qnn_instance *instance = nullptr; qnn::qnn_instance *instance = nullptr;
@ -57,16 +56,14 @@ static void ggml_qnn_add(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
perf.start(); perf.start();
std::string map_entry(ggml_op_name(ggmlop)); std::string map_entry(ggml_op_name(ggmlop));
if (instance->_qnn_graph_map.find(map_entry) != if (instance->_qnn_graph_map.find(map_entry) != instance->_qnn_graph_map.end()) {
instance->_qnn_graph_map.end()) {
graph_initialized = true; graph_initialized = true;
auto &graph_item = instance->_qnn_graph_map[map_entry]; auto &graph_item = instance->_qnn_graph_map[map_entry];
graph_handle = std::get<0>(graph_item); graph_handle = std::get<0>(graph_item);
} }
if (!graph_initialized) { if (!graph_initialized) {
graph_name = graph_name + "_" + std::to_string(ctx->threads) + graph_name = graph_name + "_" + std::to_string(ctx->threads) + "_" + src0->name + "_" + src1->name;
"_" + src0->name + "_" + src1->name;
QNN_LOG_INFO("graph name %s", graph_name.c_str()); QNN_LOG_INFO("graph name %s", graph_name.c_str());
if (ctx->device == QNN_BACKEND_NPU) { if (ctx->device == QNN_BACKEND_NPU) {
QnnHtpGraph_CustomConfig_t hvx_config; QnnHtpGraph_CustomConfig_t hvx_config;
@ -98,28 +95,22 @@ static void ggml_qnn_add(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM;
graph_vtcm_config.customConfig = &vtcm_config; graph_vtcm_config.customConfig = &vtcm_config;
const QnnGraph_Config_t* p_graphconfig[] = { &graph_hvx_config, const QnnGraph_Config_t *p_graphconfig[] = { &graph_hvx_config, &graph_dlbc_config, &graph_vtcm_config,
&graph_dlbc_config, &graph_opt_config, NULL };
&graph_vtcm_config, error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), p_graphconfig,
&graph_opt_config,
NULL };
error = qnn_raw_interface.graphCreate(
instance->get_qnn_context_handle(), graph_name.c_str(), p_graphconfig,
&graph_handle); &graph_handle);
} } else {
else { error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), nullptr,
error = qnn_raw_interface.graphCreate(
instance->get_qnn_context_handle(), graph_name.c_str(), nullptr,
&graph_handle); &graph_handle);
} }
if (QNN_SUCCESS != error) { if (QNN_SUCCESS != error) {
QNN_LOG_INFO("can't create qnn graph handle with graph name %s, " QNN_LOG_INFO(
"can't create qnn graph handle with graph name %s, "
"error = %d\n", "error = %d\n",
graph_name.c_str(), error); graph_name.c_str(), error);
goto failure; goto failure;
} } else {
else {
QNN_LOG_INFO("create qnn graph handle with graph name %s ok\n", graph_name.c_str()); QNN_LOG_INFO("create qnn graph handle with graph name %s ok\n", graph_name.c_str());
} }
@ -139,30 +130,20 @@ static void ggml_qnn_add(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
Qnn_Tensor_t tensor_inputs[] = { *tensor_input0.get_qnn_tensor(), *tensor_input1.get_qnn_tensor() }; Qnn_Tensor_t tensor_inputs[] = { *tensor_input0.get_qnn_tensor(), *tensor_input1.get_qnn_tensor() };
Qnn_Tensor_t tensor_outputs[] = { *tensor_output.get_qnn_tensor() }; Qnn_Tensor_t tensor_outputs[] = { *tensor_output.get_qnn_tensor() };
Qnn_OpConfig_t op_config = { Qnn_OpConfig_t op_config = { (Qnn_OpConfigVersion_t)1,
(Qnn_OpConfigVersion_t)1, .v1 = { "ggml_op_add", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_ELEMENT_WISE_ADD, 0,
.v1 = {"ggml_op_add", qnn_params, 2, tensor_inputs, 1, tensor_outputs } };
QNN_OP_PACKAGE_NAME_QTI_AISW,
QNN_OP_ELEMENT_WISE_ADD,
0, qnn_params,
2, tensor_inputs,
1,tensor_outputs}
};
error = qnn_raw_interface.graphAddNode(graph_handle, op_config); error = qnn_raw_interface.graphAddNode(graph_handle, op_config);
if (QNN_SUCCESS != error) { if (QNN_SUCCESS != error) {
QNN_LOG_INFO("error = %d\n", error); QNN_LOG_INFO("error = %d\n", error);
goto failure; goto failure;
} }
error = qnn_raw_interface.graphFinalize(graph_handle, error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr);
nullptr, nullptr);
if (QNN_SUCCESS != error) { if (QNN_SUCCESS != error) {
QNN_LOG_INFO("error = %d\n", error); QNN_LOG_INFO("error = %d\n", error);
goto failure; goto failure;
} }
error = qnn_raw_interface.graphExecute(graph_handle, error = qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, nullptr, nullptr);
tensor_inputs, 2,
tensor_outputs, 1,
nullptr, nullptr);
if (ctx->device == QNN_BACKEND_NPU) { if (ctx->device == QNN_BACKEND_NPU) {
if (QNN_COMMON_ERROR_SYSTEM_COMMUNICATION == error) { if (QNN_COMMON_ERROR_SYSTEM_COMMUNICATION == error) {
QNN_LOG_WARN("NPU crashed. SSR detected. Caused QNN graph execute error\n"); QNN_LOG_WARN("NPU crashed. SSR detected. Caused QNN graph execute error\n");
@ -173,13 +154,10 @@ static void ggml_qnn_add(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
goto failure; goto failure;
} }
auto graph_item = std::make_tuple(graph_handle, auto graph_item = std::make_tuple(graph_handle, tensor_input0.get_qnn_tensor(), tensor_input1.get_qnn_tensor(),
tensor_input0.get_qnn_tensor(),
tensor_input1.get_qnn_tensor(),
tensor_output.get_qnn_tensor()); tensor_output.get_qnn_tensor());
instance->_qnn_graph_map[map_entry] = graph_item; instance->_qnn_graph_map[map_entry] = graph_item;
} } else {
else {
auto &graph_item = instance->_qnn_graph_map[map_entry]; auto &graph_item = instance->_qnn_graph_map[map_entry];
qnn::ggml_qnn_tensor_input tensor_input0(src0, std::get<1>(graph_item), ctx); qnn::ggml_qnn_tensor_input tensor_input0(src0, std::get<1>(graph_item), ctx);
qnn::ggml_qnn_tensor_input tensor_input1(src1, std::get<2>(graph_item), ctx); qnn::ggml_qnn_tensor_input tensor_input1(src1, std::get<2>(graph_item), ctx);
@ -187,10 +165,7 @@ static void ggml_qnn_add(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
Qnn_Tensor_t tensor_inputs[] = { *tensor_input0.get_qnn_tensor(), *tensor_input1.get_qnn_tensor() }; Qnn_Tensor_t tensor_inputs[] = { *tensor_input0.get_qnn_tensor(), *tensor_input1.get_qnn_tensor() };
Qnn_Tensor_t tensor_outputs[] = { *tensor_output.get_qnn_tensor() }; Qnn_Tensor_t tensor_outputs[] = { *tensor_output.get_qnn_tensor() };
error = qnn_raw_interface.graphExecute(graph_handle, error = qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, nullptr, nullptr);
tensor_inputs, 2,
tensor_outputs, 1,
nullptr, nullptr);
if (ctx->device == QNN_BACKEND_NPU) { if (ctx->device == QNN_BACKEND_NPU) {
if (QNN_COMMON_ERROR_SYSTEM_COMMUNICATION == error) { if (QNN_COMMON_ERROR_SYSTEM_COMMUNICATION == error) {
QNN_LOG_WARN("NPU crashed. SSR detected. Caused QNN graph execute error\n"); QNN_LOG_WARN("NPU crashed. SSR detected. Caused QNN graph execute error\n");
@ -204,20 +179,17 @@ static void ggml_qnn_add(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
failure: failure:
if (QNN_SUCCESS != error) { if (QNN_SUCCESS != error) {
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64
" x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", ", nb = (%5zi, %5zi, %5zi)\n",
src0->name, src0->type, ggml_type_name(src0->type), src0->name, src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2],
src0->ne[0], src0->ne[1], src0->ne[2], src0->nb[0], src0->nb[0], src0->nb[1], src0->nb[2]);
src0->nb[1], src0->nb[2]); QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n",
" x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", src1->name, src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2],
src1->name, src1->type, ggml_type_name(src1->type), src1->nb[0], src1->nb[1], src1->nb[2]);
src1->ne[0], src1->ne[1], src1->ne[2], src1->nb[0], QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64
src1->nb[1], src1->nb[2]); ", nb = (%5zi, %5zi, %5zi)\n",
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 dst->name, dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0],
" x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n",
dst->name, dst->type, ggml_type_name(dst->type),
dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0],
dst->nb[1], dst->nb[2]); dst->nb[1], dst->nb[2]);
} }
@ -235,8 +207,7 @@ failure:
* mul_mat_f16_f32: src0 is F16 and src1 is F32. * mul_mat_f16_f32: src0 is F16 and src1 is F32.
* mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, ...), and src1 is F32. * mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, ...), and src1 is F32.
*/ */
static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx, static void ggml_qnn_mul_mat(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor *dst) { ggml_tensor *dst) {
Qnn_ErrorHandle_t error = QNN_SUCCESS; Qnn_ErrorHandle_t error = QNN_SUCCESS;
bool graph_initialized = false; bool graph_initialized = false;
@ -254,8 +225,7 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx,
perf.start(); perf.start();
std::string map_entry = std::string(ggml_op_name(ggmlop)); std::string map_entry = std::string(ggml_op_name(ggmlop));
if (instance->_qnn_graph_map.find(map_entry) != if (instance->_qnn_graph_map.find(map_entry) != instance->_qnn_graph_map.end()) {
instance->_qnn_graph_map.end()) {
graph_initialized = true; graph_initialized = true;
auto &graph_item = instance->_qnn_graph_map[map_entry]; auto &graph_item = instance->_qnn_graph_map[map_entry];
graph_handle = std::get<0>(graph_item); graph_handle = std::get<0>(graph_item);
@ -267,8 +237,7 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx,
// the performance gains is worth although there is performance loss in pass-1 // the performance gains is worth although there is performance loss in pass-1
if (!graph_initialized) { if (!graph_initialized) {
graph_name = graph_name + "_" + std::to_string(ctx->threads) + graph_name = graph_name + "_" + std::to_string(ctx->threads) + "_" + src0->name + "_" + src1->name;
"_" + src0->name + "_" + src1->name;
QNN_LOG_INFO("graph name %s", graph_name.c_str()); QNN_LOG_INFO("graph name %s", graph_name.c_str());
if (ctx->device == QNN_BACKEND_NPU) { if (ctx->device == QNN_BACKEND_NPU) {
QnnHtpGraph_CustomConfig_t hvx_config; QnnHtpGraph_CustomConfig_t hvx_config;
@ -300,22 +269,17 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx,
graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM; graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM;
graph_vtcm_config.customConfig = &vtcm_config; graph_vtcm_config.customConfig = &vtcm_config;
const QnnGraph_Config_t* p_graphconfig[] = { &graph_hvx_config, const QnnGraph_Config_t *p_graphconfig[] = { &graph_hvx_config, &graph_dlbc_config, &graph_vtcm_config,
&graph_dlbc_config, &graph_opt_config, NULL };
&graph_vtcm_config, error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), p_graphconfig,
&graph_opt_config,
NULL };
error = qnn_raw_interface.graphCreate(
instance->get_qnn_context_handle(), graph_name.c_str(), p_graphconfig,
&graph_handle); &graph_handle);
} } else {
else { error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), nullptr,
error = qnn_raw_interface.graphCreate(
instance->get_qnn_context_handle(), graph_name.c_str(), nullptr,
&graph_handle); &graph_handle);
} }
if (QNN_SUCCESS != error) { if (QNN_SUCCESS != error) {
QNN_LOG_INFO("can't create qnn graph handle with graph name %s, " QNN_LOG_INFO(
"can't create qnn graph handle with graph name %s, "
"error = %d\n", "error = %d\n",
graph_name.c_str(), error); graph_name.c_str(), error);
goto failure; goto failure;
@ -336,30 +300,20 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx,
Qnn_Tensor_t tensor_inputs[] = { *tensor_input0.get_qnn_tensor(), *tensor_input1.get_qnn_tensor() }; Qnn_Tensor_t tensor_inputs[] = { *tensor_input0.get_qnn_tensor(), *tensor_input1.get_qnn_tensor() };
Qnn_Tensor_t tensor_outputs[] = { *tensor_output.get_qnn_tensor() }; Qnn_Tensor_t tensor_outputs[] = { *tensor_output.get_qnn_tensor() };
Qnn_OpConfig_t op_config = { Qnn_OpConfig_t op_config = { (Qnn_OpConfigVersion_t)1,
(Qnn_OpConfigVersion_t)1, .v1 = { "ggml_op_mul_mat", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, 0,
.v1 = {"ggml_op_mul_mat", qnn_params, 2, tensor_inputs, 1, tensor_outputs } };
QNN_OP_PACKAGE_NAME_QTI_AISW,
QNN_OP_MAT_MUL,
0, qnn_params,
2, tensor_inputs,
1, tensor_outputs}
};
error = qnn_raw_interface.graphAddNode(graph_handle, op_config); error = qnn_raw_interface.graphAddNode(graph_handle, op_config);
if (QNN_SUCCESS != error) { if (QNN_SUCCESS != error) {
QNN_LOG_INFO("error = %d\n", error); QNN_LOG_INFO("error = %d\n", error);
goto failure; goto failure;
} }
error = qnn_raw_interface.graphFinalize(graph_handle, error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr);
nullptr, nullptr);
if (QNN_SUCCESS != error) { if (QNN_SUCCESS != error) {
QNN_LOG_INFO("error = %d\n", error); QNN_LOG_INFO("error = %d\n", error);
goto failure; goto failure;
} }
error = qnn_raw_interface.graphExecute(graph_handle, error = qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, nullptr, nullptr);
tensor_inputs, 2,
tensor_outputs, 1,
nullptr, nullptr);
if (ctx->device == QNN_BACKEND_NPU) { if (ctx->device == QNN_BACKEND_NPU) {
if (QNN_COMMON_ERROR_SYSTEM_COMMUNICATION == error) { if (QNN_COMMON_ERROR_SYSTEM_COMMUNICATION == error) {
QNN_LOG_WARN("NPU crashed. SSR detected. Caused QNN graph execute error\n"); QNN_LOG_WARN("NPU crashed. SSR detected. Caused QNN graph execute error\n");
@ -370,13 +324,10 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx,
goto failure; goto failure;
} }
auto graph_item = std::make_tuple(graph_handle, auto graph_item = std::make_tuple(graph_handle, tensor_input0.get_qnn_tensor(), tensor_input1.get_qnn_tensor(),
tensor_input0.get_qnn_tensor(),
tensor_input1.get_qnn_tensor(),
tensor_output.get_qnn_tensor()); tensor_output.get_qnn_tensor());
instance->_qnn_graph_map[map_entry] = graph_item; instance->_qnn_graph_map[map_entry] = graph_item;
} } else {
else {
auto &graph_item = instance->_qnn_graph_map[map_entry]; auto &graph_item = instance->_qnn_graph_map[map_entry];
qnn::ggml_qnn_tensor_input tensor_input0(src0, std::get<1>(graph_item), ctx); qnn::ggml_qnn_tensor_input tensor_input0(src0, std::get<1>(graph_item), ctx);
qnn::ggml_qnn_tensor_input tensor_input1(src1, std::get<2>(graph_item), ctx); qnn::ggml_qnn_tensor_input tensor_input1(src1, std::get<2>(graph_item), ctx);
@ -384,10 +335,7 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx,
Qnn_Tensor_t tensor_inputs[] = { *tensor_input0.get_qnn_tensor(), *tensor_input1.get_qnn_tensor() }; Qnn_Tensor_t tensor_inputs[] = { *tensor_input0.get_qnn_tensor(), *tensor_input1.get_qnn_tensor() };
Qnn_Tensor_t tensor_outputs[] = { *tensor_output.get_qnn_tensor() }; Qnn_Tensor_t tensor_outputs[] = { *tensor_output.get_qnn_tensor() };
error = qnn_raw_interface.graphExecute(graph_handle, error = qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, nullptr, nullptr);
tensor_inputs, 2,
tensor_outputs, 1,
nullptr, nullptr);
if (ctx->device == QNN_BACKEND_NPU) { if (ctx->device == QNN_BACKEND_NPU) {
if (QNN_COMMON_ERROR_SYSTEM_COMMUNICATION == error) { if (QNN_COMMON_ERROR_SYSTEM_COMMUNICATION == error) {
QNN_LOG_WARN("NPU crashed. SSR detected. Caused QNN graph execute error\n"); QNN_LOG_WARN("NPU crashed. SSR detected. Caused QNN graph execute error\n");
@ -401,181 +349,127 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx,
failure: failure:
if (QNN_SUCCESS != error) { if (QNN_SUCCESS != error) {
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64
" x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", ", nb = (%5zi, %5zi, %5zi)\n",
src0->name, src0->type, ggml_type_name(src0->type), src0->name, src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2],
src0->ne[0], src0->ne[1], src0->ne[2], src0->nb[0], src0->nb[0], src0->nb[1], src0->nb[2]);
src0->nb[1], src0->nb[2]); QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n",
" x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", src1->name, src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2],
src1->name, src1->type, ggml_type_name(src1->type), src1->nb[0], src1->nb[1], src1->nb[2]);
src1->ne[0], src1->ne[1], src1->ne[2], src1->nb[0], QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64
src1->nb[1], src1->nb[2]); ", nb = (%5zi, %5zi, %5zi)\n",
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 dst->name, dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0],
" x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n", dst->nb[1], dst->nb[2]);
dst->name, dst->type, ggml_type_name(dst->type), dst->ne[0],
dst->ne[1], dst->ne[2], dst->nb[0], dst->nb[1], dst->nb[2]);
} }
perf.info(); perf.info();
} }
static void ggml_qnn_repeat(ggml_backend_qnn_context* ctx, static void ggml_qnn_repeat(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
const ggml_tensor* src0, const ggml_tensor* src1, ggml_tensor *dst) {}
static void ggml_qnn_get_rows(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_acc(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_div(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_gelu(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_silu(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_gelu_quick(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_tanh(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_relu(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_hardsigmoid(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_hardswish(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_leaky_relu(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_sqr(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_norm(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_group_norm(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_concat(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_upscale(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_pad(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_rms_norm(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_cpy(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_dup(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) { ggml_tensor *dst) {
}
static void ggml_qnn_get_rows(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_acc(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_div(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_gelu(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_silu(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_gelu_quick(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_tanh(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_relu(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_hardsigmoid(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_hardswish(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_leaky_relu(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_sqr(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_norm(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_group_norm(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_concat(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_upscale(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_pad(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_rms_norm(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_cpy(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_dup(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
ggml_qnn_cpy(ctx, src0, dst, nullptr); ggml_qnn_cpy(ctx, src0, dst, nullptr);
(void)src1; (void)src1;
} }
static void ggml_qnn_mul_mat_id(ggml_backend_qnn_context* ctx, static void ggml_qnn_mul_mat_id(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
const ggml_tensor* src0, ggml_tensor *dst) {}
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_scale(ggml_backend_qnn_context* ctx, static void ggml_qnn_scale(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
const ggml_tensor* src0, const ggml_tensor* src1, ggml_tensor *dst) {}
ggml_tensor* dst) {
}
static void ggml_qnn_clamp(ggml_backend_qnn_context* ctx, static void ggml_qnn_clamp(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
const ggml_tensor* src0, const ggml_tensor* src1, ggml_tensor *dst) {}
ggml_tensor* dst) {
}
static void ggml_qnn_diag_mask_inf(ggml_backend_qnn_context* ctx, static void ggml_qnn_diag_mask_inf(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
const ggml_tensor* src0, ggml_tensor *dst) {}
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_soft_max(ggml_backend_qnn_context* ctx, static void ggml_qnn_soft_max(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
const ggml_tensor* src0, const ggml_tensor* src1, ggml_tensor *dst) {}
ggml_tensor* dst) {
}
static void ggml_qnn_rope(ggml_backend_qnn_context* ctx, static void ggml_qnn_rope(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor *dst) { ggml_tensor *dst) {
GGML_ASSERT(ggml_is_contiguous(src0)); GGML_ASSERT(ggml_is_contiguous(src0));
} }
static void ggml_qnn_pool2d(ggml_backend_qnn_context* ctx, static void ggml_qnn_pool2d(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
const ggml_tensor* src0, const ggml_tensor* src1, ggml_tensor *dst) {}
ggml_tensor* dst) {
}
static void ggml_qnn_im2col(ggml_backend_qnn_context* ctx, static void ggml_qnn_im2col(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
const ggml_tensor* src0, const ggml_tensor* src1, ggml_tensor *dst) {}
ggml_tensor* dst) {
}
static void ggml_qnn_sum_rows(ggml_backend_qnn_context* ctx, static void ggml_qnn_sum_rows(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor *dst) { ggml_tensor *dst) {
GGML_ASSERT(ggml_is_contiguous(src0)); GGML_ASSERT(ggml_is_contiguous(src0));
} }
static void ggml_qnn_argsort(ggml_backend_qnn_context* ctx, static void ggml_qnn_argsort(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor *dst) { ggml_tensor *dst) {
GGML_ASSERT(ggml_is_contiguous(src0)); GGML_ASSERT(ggml_is_contiguous(src0));
} }
static void ggml_qnn_nop(ggml_backend_qnn_context* ctx, const ggml_tensor* src0, static void ggml_qnn_nop(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
const ggml_tensor* src1, ggml_tensor* dst) { ggml_tensor *dst) {
(void)src0; (void)src0;
(void)src1; (void)src1;
(void)dst; (void)dst;

7
ggml/src/ggml-qnn/backend-ops.hpp
View file

@ -1,17 +1,16 @@
#pragma once #pragma once
#include "ggml.h" #include "ggml.h"
#include "backend.hpp" #include "backend.hpp"
namespace qnn { namespace qnn {
typedef void (*ggml_qnn_op_t)(ggml_backend_qnn_context* ctx, typedef void (*ggml_qnn_op_t)(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
const ggml_tensor* src0,
const ggml_tensor* src1,
ggml_tensor *dst); ggml_tensor *dst);
typedef const ggml_qnn_op_t (&ggml_qnn_op_array_t)[GGML_OP_COUNT]; typedef const ggml_qnn_op_t (&ggml_qnn_op_array_t)[GGML_OP_COUNT];
ggml_qnn_op_array_t ggml_qnn_op_array(); ggml_qnn_op_array_t ggml_qnn_op_array();
} } // namespace qnn

1
ggml/src/ggml-qnn/backend.hpp
View file

@ -2,6 +2,7 @@
#pragma once #pragma once
#include "ggml.h" #include "ggml.h"
#include "ggml-backend.h" #include "ggml-backend.h"
#include "qnn.hpp" #include "qnn.hpp"

15
ggml/src/ggml-qnn/logger.cpp
View file

@ -2,6 +2,7 @@
#include "logger.hpp" #include "logger.hpp"
#include <stdio.h> #include <stdio.h>
#include <mutex> #include <mutex>
#if (defined __ANDROID__) || (defined ANDROID) #if (defined __ANDROID__) || (defined ANDROID)
@ -10,9 +11,7 @@
#define QNN_LOGBUF_LEN 4096 #define QNN_LOGBUF_LEN 4096
void qnn::internal_log(ggml_log_level level, const char* file, void qnn::internal_log(ggml_log_level level, const char *file, const char *func, int line, const char *format, ...) {
const char* func, int line,
const char* format, ...) {
static std::mutex qnn_internal_log_mutex; static std::mutex qnn_internal_log_mutex;
static char s_qnn_internal_log_buf[QNN_LOGBUF_LEN]; static char s_qnn_internal_log_buf[QNN_LOGBUF_LEN];
@ -21,11 +20,8 @@ void qnn::internal_log(ggml_log_level level, const char* file,
va_list args; va_list args;
va_start(args, format); va_start(args, format);
int len_prefix = int len_prefix = snprintf(s_qnn_internal_log_buf, QNN_LOGBUF_LEN, "[%s, %d]: ", func, line);
snprintf(s_qnn_internal_log_buf, QNN_LOGBUF_LEN, int len = vsnprintf(s_qnn_internal_log_buf + len_prefix, QNN_LOGBUF_LEN - len_prefix, format, args);
"[%s, %d]: ", func, line);
int len = vsnprintf(s_qnn_internal_log_buf + len_prefix,
QNN_LOGBUF_LEN - len_prefix, format, args);
if (len < (QNN_LOGBUF_LEN - len_prefix)) { if (len < (QNN_LOGBUF_LEN - len_prefix)) {
#if (defined __ANDROID__) || (defined ANDROID) #if (defined __ANDROID__) || (defined ANDROID)
// for Android APK // for Android APK
@ -38,8 +34,7 @@ void qnn::internal_log(ggml_log_level level, const char* file,
} }
} }
void qnn::sdk_logcallback(const char* fmt, QnnLog_Level_t level, void qnn::sdk_logcallback(const char *fmt, QnnLog_Level_t level, uint64_t timestamp, va_list argp) {
uint64_t timestamp, va_list argp) {
#if ENABLE_QNNSDK_LOG #if ENABLE_QNNSDK_LOG
static std::mutex log_mutex; static std::mutex log_mutex;
static unsigned char s_ggml_qnn_logbuf[QNN_LOGBUF_LEN]; static unsigned char s_ggml_qnn_logbuf[QNN_LOGBUF_LEN];

32
ggml/src/ggml-qnn/logger.hpp
View file

@ -2,36 +2,29 @@
#include <stdint.h> #include <stdint.h>
#include "QnnTypes.h"
#include "QnnCommon.h"
#include "QnnInterface.h"
#include "System/QnnSystemInterface.h"
#include "ggml.h" #include "ggml.h"
#include "QnnCommon.h"
#include "QnnInterface.h"
#include "QnnTypes.h"
#include "System/QnnSystemInterface.h"
namespace qnn { namespace qnn {
void internal_log(ggml_log_level level, const char* file, void internal_log(ggml_log_level level, const char *file, const char *func, int line, const char *format, ...);
const char* func, int line,
const char* format, ...);
void sdk_logcallback(const char *fmt, QnnLog_Level_t level, uint64_t timestamp, va_list argp);
void sdk_logcallback(const char* fmt, QnnLog_Level_t level, } // namespace qnn
uint64_t timestamp, va_list argp);
}
// ================================================================================================= // =================================================================================================
// //
// QNN backend internal log function // QNN backend internal log function
// //
// ================================================================================================= // =================================================================================================
#define QNN_LOG_ERROR(...) \ #define QNN_LOG_ERROR(...) qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
#define QNN_LOG_WARN(...) \ #define QNN_LOG_WARN(...) qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
#define QNN_LOG_INFO(...) \ #define QNN_LOG_INFO(...) qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
#ifdef NDEBUG #ifdef NDEBUG
#define ENABLE_QNNBACKEND_DEBUG 0 // for troubleshooting QNN backend #define ENABLE_QNNBACKEND_DEBUG 0 // for troubleshooting QNN backend
@ -42,8 +35,7 @@ namespace qnn {
#endif #endif
#if ENABLE_QNNBACKEND_DEBUG #if ENABLE_QNNBACKEND_DEBUG
#define QNN_LOG_DEBUG(...) \ #define QNN_LOG_DEBUG(...) qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
#else #else
#define QNN_LOG_DEBUG(...) #define QNN_LOG_DEBUG(...)
#endif #endif

10
ggml/src/ggml-qnn/qnn-types.hpp
View file

@ -1,9 +1,9 @@
#pragma once #pragma once
#include "QnnTypes.h"
#include "QnnCommon.h" #include "QnnCommon.h"
#include "QnnInterface.h" #include "QnnInterface.h"
#include "QnnTypes.h"
#include "Saver/QnnSaver.h" #include "Saver/QnnSaver.h"
#include "System/QnnSystemInterface.h" #include "System/QnnSystemInterface.h"
@ -14,11 +14,7 @@ namespace qnn {
// Qualcomm QNN(Qualcomm Neural Network, aka Qualcomm AI Engine Direct) SDK // Qualcomm QNN(Qualcomm Neural Network, aka Qualcomm AI Engine Direct) SDK
// ref:https://github.com/pytorch/executorch/tree/main/backends/qualcomm // ref:https://github.com/pytorch/executorch/tree/main/backends/qualcomm
// ================================================================================================= // =================================================================================================
enum sdk_profile_level { enum sdk_profile_level { profile_off = 0, profile_basic = 1, profile_detail = 2 };
profile_off = 0,
profile_basic = 1,
profile_detail = 2
};
enum qcom_htp_arch { enum qcom_htp_arch {
NONE = 0, NONE = 0,
@ -51,7 +47,7 @@ namespace qnn {
using pfn_qnnsaver_initialize = decltype(QnnSaver_initialize); using pfn_qnnsaver_initialize = decltype(QnnSaver_initialize);
using pfn_qnninterface_getproviders = decltype(QnnInterface_getProviders); using pfn_qnninterface_getproviders = decltype(QnnInterface_getProviders);
using pfn_qnnsysteminterface_getproviders = decltype(QnnSystemInterface_getProviders); using pfn_qnnsysteminterface_getproviders = decltype(QnnSystemInterface_getProviders);
} } // namespace qnn
#define QNN_VER_PTR(x) (&((x).v1)) // TODO: remove this macro after we have a separate header for QNN #define QNN_VER_PTR(x) (&((x).v1)) // TODO: remove this macro after we have a separate header for QNN

460
ggml/src/ggml-qnn/qnn.hpp
View file

@ -1,24 +1,25 @@
#pragma once #pragma once
#include <math.h> #include <math.h>
#include <map>
#include <mutex> #include <mutex>
#include <string> #include <string>
#include <unordered_map> #include <unordered_map>
#include <map>
// header file of Qualcomm QNN(Qualcomm Neural Network, aka Qualcomm AI Engine Direct) SDK // header file of Qualcomm QNN(Qualcomm Neural Network, aka Qualcomm AI Engine Direct) SDK
// https://qpm.qualcomm.com/#/main/tools/details/qualcomm_ai_engine_direct // https://qpm.qualcomm.com/#/main/tools/details/qualcomm_ai_engine_direct
#include "QnnTypes.h" #include <HTP/QnnHtpDevice.h>
#include "QnnCommon.h" #include <HTP/QnnHtpGraph.h>
#include "QnnInterface.h" #include <QnnBackend.h>
#include "QnnContext.h" #include <QnnCommon.h>
#include "QnnBackend.h" #include <QnnContext.h>
#include "QnnGraph.h" #include <QnnGraph.h>
#include "QnnProperty.h" #include <QnnInterface.h>
#include "QnnTensor.h" #include <QnnProperty.h>
#include "System/QnnSystemInterface.h" #include <QnnTensor.h>
#include "HTP/QnnHtpDevice.h" #include <QnnTypes.h>
#include "HTP/QnnHtpGraph.h" #include <System/QnnSystemInterface.h>
#include "qnn-types.hpp" #include "qnn-types.hpp"
#include "utils.hpp" #include "utils.hpp"
@ -33,16 +34,15 @@ namespace qnn {
class qnn_interface { class qnn_interface {
#define DEFINE_SHIM_FUNCTION_INTERFACE(F, pointer_name) \ #define DEFINE_SHIM_FUNCTION_INTERFACE(F, pointer_name) \
template <typename... Args> inline auto qnn_##F(Args... args) const { \ template <typename... Args> \
return (_qnn_interface->QNN_INTERFACE_VER_NAME.pointer_name)( \ inline auto qnn_##F(Args... args) const { \
std::forward<Args>(args)...); \ return (_qnn_interface->QNN_INTERFACE_VER_NAME.pointer_name)(std::forward<Args>(args)...); \
} }
#define DEFINE_SHIM_FUNCTION_SYS_INTERFACE(F, pointer_name) \ #define DEFINE_SHIM_FUNCTION_SYS_INTERFACE(F, pointer_name) \
template <typename... Args> inline auto qnn_##F(Args... args) const { \ template <typename... Args> \
return ( \ inline auto qnn_##F(Args... args) const { \
_qnn_sys_interface->QNN_SYSTEM_INTERFACE_VER_NAME.pointer_name)( \ return (_qnn_sys_interface->QNN_SYSTEM_INTERFACE_VER_NAME.pointer_name)(std::forward<Args>(args)...); \
std::forward<Args>(args)...); \
} }
friend class qnn_instance; friend class qnn_instance;
@ -55,38 +55,31 @@ namespace qnn {
DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree); DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree);
DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage);
backendRegisterOpPackage);
DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig);
backendValidateOpConfig);
DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion);
backendGetApiVersion);
// QnnDevice // QnnDevice
DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate); DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate);
DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree); DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree);
DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure);
deviceGetInfrastructure);
DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo);
deviceGetPlatformInfo);
DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo); DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo);
// QnnContext // QnnContext
DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate); DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate);
DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize);
contextGetBinarySize);
DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary); DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary);
DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary);
contextCreateFromBinary);
DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree); DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree);
@ -125,39 +118,29 @@ namespace qnn {
DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister); DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister);
// QnnProperty // QnnProperty
DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability);
propertyHasCapability);
// QnnTensor // QnnTensor
DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor);
tensorCreateContextTensor);
DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor);
tensorCreateGraphTensor);
// QnnSystem // QnnSystem
DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate);
systemContextCreate);
DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo);
systemContextGetBinaryInfo);
DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree); DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree);
void set_qnn_interface(const QnnInterface_t* qnn_interface) { void set_qnn_interface(const QnnInterface_t *qnn_interface) { _qnn_interface = qnn_interface; }
_qnn_interface = qnn_interface;
}
void set_qnn_system_interface( void set_qnn_system_interface(const QnnSystemInterface_t *qnn_sys_interface) {
const QnnSystemInterface_t* qnn_sys_interface) {
_qnn_sys_interface = qnn_sys_interface; _qnn_sys_interface = qnn_sys_interface;
} }
uint32_t get_backend_id() const { return _qnn_interface->backendId; } uint32_t get_backend_id() const { return _qnn_interface->backendId; }
bool is_loaded() const { bool is_loaded() const { return ((_qnn_sys_interface != nullptr) && (_qnn_interface != nullptr)); }
return ((_qnn_sys_interface != nullptr) && (_qnn_interface != nullptr));
}
private: private:
const QnnInterface_t *_qnn_interface = nullptr; const QnnInterface_t *_qnn_interface = nullptr;
@ -165,17 +148,12 @@ namespace qnn {
const QnnSystemInterface_t *_qnn_sys_interface = nullptr; const QnnSystemInterface_t *_qnn_sys_interface = nullptr;
}; };
class qnn_instance { class qnn_instance {
public: public:
using BackendIdType = decltype(QnnInterface_t{}.backendId); using BackendIdType = decltype(QnnInterface_t{}.backendId);
explicit qnn_instance(const std::string& lib_path, explicit qnn_instance(const std::string &lib_path, const std::string &backend_name, const std::string &model_name) :
const std::string& backend_name, _lib_path(std::move(lib_path)), _backend_name(std::move(backend_name)), _model_name(std::move(model_name)) {};
const std::string& model_name)
: _lib_path(std::move(lib_path))
, _backend_name(std::move(backend_name))
, _model_name(std::move(model_name)) {};
~qnn_instance() {} ~qnn_instance() {}
@ -188,8 +166,7 @@ namespace qnn {
if (0 != load_system()) { if (0 != load_system()) {
QNN_LOG_WARN("can not load QNN system lib, pls check why?\n"); QNN_LOG_WARN("can not load QNN system lib, pls check why?\n");
return 1; return 1;
} } else {
else {
QNN_LOG_DEBUG("load QNN system lib successfully\n"); QNN_LOG_DEBUG("load QNN system lib successfully\n");
} }
@ -203,12 +180,11 @@ namespace qnn {
} }
backend_id = _lib_path_to_backend_id[backend_lib_path]; backend_id = _lib_path_to_backend_id[backend_lib_path];
if (0 == _loaded_backend.count(backend_id) || if (0 == _loaded_backend.count(backend_id) || 0 == _loaded_lib_handle.count(backend_id)) {
0 == _loaded_lib_handle.count(backend_id)) { QNN_LOG_WARN(
QNN_LOG_WARN("library %s is loaded but loaded backend count=%zu, " "library %s is loaded but loaded backend count=%zu, "
"loaded lib_handle count=%zu\n", "loaded lib_handle count=%zu\n",
backend_lib_path.c_str(), _loaded_backend.count(backend_id), backend_lib_path.c_str(), _loaded_backend.count(backend_id), _loaded_lib_handle.count(backend_id));
_loaded_lib_handle.count(backend_id));
return 3; return 3;
} }
@ -219,27 +195,22 @@ namespace qnn {
// NPU backend not work on Qualcomm SoC equipped low-end phone // NPU backend not work on Qualcomm SoC equipped low-end phone
QNN_LOG_WARN("why failed to initialize qnn log\n"); QNN_LOG_WARN("why failed to initialize qnn log\n");
return 4; return 4;
} } else {
else {
QNN_LOG_DEBUG("initialize qnn log successfully\n"); QNN_LOG_DEBUG("initialize qnn log successfully\n");
} }
std::vector<const QnnBackend_Config_t *> temp_backend_config; std::vector<const QnnBackend_Config_t *> temp_backend_config;
_qnn_interface.qnn_backend_create( _qnn_interface.qnn_backend_create(
_qnn_log_handle, _qnn_log_handle, temp_backend_config.empty() ? nullptr : temp_backend_config.data(), &_qnn_backend_handle);
temp_backend_config.empty() ? nullptr : temp_backend_config.data(),
&_qnn_backend_handle);
if (nullptr == _qnn_backend_handle) { if (nullptr == _qnn_backend_handle) {
QNN_LOG_WARN("why failed to initialize qnn backend\n"); QNN_LOG_WARN("why failed to initialize qnn backend\n");
return 5; return 5;
} } else {
else {
QNN_LOG_DEBUG("initialize qnn backend successfully\n"); QNN_LOG_DEBUG("initialize qnn backend successfully\n");
} }
if (nullptr != _qnn_raw_interface.propertyHasCapability) { if (nullptr != _qnn_raw_interface.propertyHasCapability) {
Qnn_ErrorHandle_t qnn_status = Qnn_ErrorHandle_t qnn_status = _qnn_raw_interface.propertyHasCapability(QNN_PROPERTY_GROUP_DEVICE);
_qnn_raw_interface.propertyHasCapability(QNN_PROPERTY_GROUP_DEVICE);
if (QNN_PROPERTY_NOT_SUPPORTED == qnn_status) { if (QNN_PROPERTY_NOT_SUPPORTED == qnn_status) {
QNN_LOG_WARN("device property is not supported\n"); QNN_LOG_WARN("device property is not supported\n");
} }
@ -256,15 +227,16 @@ namespace qnn {
QnnDevice_HardwareDeviceInfo_t *infos = p_info->v1.hwDevices; QnnDevice_HardwareDeviceInfo_t *infos = p_info->v1.hwDevices;
QnnHtpDevice_OnChipDeviceInfoExtension_t chipinfo = {}; QnnHtpDevice_OnChipDeviceInfoExtension_t chipinfo = {};
for (int i = 0; i < p_info->v1.numHwDevices; i++) { for (int i = 0; i < p_info->v1.numHwDevices; i++) {
QNN_LOG_INFO("deviceID:%d, deviceType:%d, numCores %d", infos[i].v1.deviceId, QNN_LOG_INFO("deviceID:%d, deviceType:%d, numCores %d", infos[i].v1.deviceId, infos[i].v1.deviceType,
infos[i].v1.deviceType, infos[i].v1.numCores); infos[i].v1.numCores);
QnnDevice_DeviceInfoExtension_t devinfo = infos[i].v1.deviceInfoExtension; QnnDevice_DeviceInfoExtension_t devinfo = infos[i].v1.deviceInfoExtension;
chipinfo = devinfo->onChipDevice; chipinfo = devinfo->onChipDevice;
QnnHtpDevice_Arch_t htp_arch = chipinfo.arch; QnnHtpDevice_Arch_t htp_arch = chipinfo.arch;
QNN_LOG_INFO("htp_type:%d(%s)", devinfo->devType, (devinfo->devType == QNN_HTP_DEVICE_TYPE_ON_CHIP) ? "ON_CHIP" : ""); QNN_LOG_INFO("htp_type:%d(%s)", devinfo->devType,
QNN_LOG_INFO("qualcomm soc_model:%d(%s), htp_arch:%d(%s), vtcm_size:%d MB", (devinfo->devType == QNN_HTP_DEVICE_TYPE_ON_CHIP) ? "ON_CHIP" : "");
chipinfo.socModel, qnn::get_chipset_desc(chipinfo.socModel), QNN_LOG_INFO("qualcomm soc_model:%d(%s), htp_arch:%d(%s), vtcm_size:%d MB", chipinfo.socModel,
htp_arch, qnn::get_htparch_desc(htp_arch), chipinfo.vtcmSize); qnn::get_chipset_desc(chipinfo.socModel), htp_arch, qnn::get_htparch_desc(htp_arch),
chipinfo.vtcmSize);
_soc_info = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize }; _soc_info = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize };
} }
_qnn_raw_interface.deviceFreePlatformInfo(nullptr, p_info); _qnn_raw_interface.deviceFreePlatformInfo(nullptr, p_info);
@ -286,15 +258,12 @@ namespace qnn {
const QnnDevice_Config_t *p_deviceconfig[] = { &soc_devconfig, &arch_devconfig, nullptr }; const QnnDevice_Config_t *p_deviceconfig[] = { &soc_devconfig, &arch_devconfig, nullptr };
qnn_status = _qnn_raw_interface.deviceCreate(_qnn_log_handle, p_deviceconfig, &_qnn_device_handle); qnn_status = _qnn_raw_interface.deviceCreate(_qnn_log_handle, p_deviceconfig, &_qnn_device_handle);
} } else {
else {
qnn_status = _qnn_raw_interface.deviceCreate(_qnn_log_handle, nullptr, &_qnn_device_handle); qnn_status = _qnn_raw_interface.deviceCreate(_qnn_log_handle, nullptr, &_qnn_device_handle);
} }
if (QNN_SUCCESS != qnn_status && if (QNN_SUCCESS != qnn_status && QNN_DEVICE_ERROR_UNSUPPORTED_FEATURE != qnn_status) {
QNN_DEVICE_ERROR_UNSUPPORTED_FEATURE != qnn_status) {
QNN_LOG_WARN("failed to create QNN device\n"); QNN_LOG_WARN("failed to create QNN device\n");
} } else {
else {
QNN_LOG_INFO("create QNN device successfully\n"); QNN_LOG_INFO("create QNN device successfully\n");
} }
@ -302,27 +271,21 @@ namespace qnn {
QNN_LOG_INFO("profiling turned on; level = %d", _profile_level); QNN_LOG_INFO("profiling turned on; level = %d", _profile_level);
if (qnn::sdk_profile_level::profile_basic == _profile_level) { if (qnn::sdk_profile_level::profile_basic == _profile_level) {
QNN_LOG_INFO("basic profiling requested. creating Qnn Profile object\n"); QNN_LOG_INFO("basic profiling requested. creating Qnn Profile object\n");
if (QNN_PROFILE_NO_ERROR != if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate(
_qnn_raw_interface.profileCreate(_qnn_backend_handle, _qnn_backend_handle, QNN_PROFILE_LEVEL_BASIC, &_qnn_profile_handle)) {
QNN_PROFILE_LEVEL_BASIC,
&_qnn_profile_handle)) {
QNN_LOG_WARN("unable to create profile handle in the backend\n"); QNN_LOG_WARN("unable to create profile handle in the backend\n");
return 6; return 6;
} } else {
else {
QNN_LOG_DEBUG("initialize qnn profile successfully\n"); QNN_LOG_DEBUG("initialize qnn profile successfully\n");
} }
} } else if (qnn::sdk_profile_level::profile_detail == _profile_level) {
else if (qnn::sdk_profile_level::profile_detail == _profile_level) {
QNN_LOG_INFO("detailed profiling requested. Creating Qnn Profile object\n"); QNN_LOG_INFO("detailed profiling requested. Creating Qnn Profile object\n");
if (QNN_PROFILE_NO_ERROR != if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate(_qnn_backend_handle,
_qnn_raw_interface.profileCreate(_qnn_backend_handle,
QNN_PROFILE_LEVEL_DETAILED, QNN_PROFILE_LEVEL_DETAILED,
&_qnn_profile_handle)) { &_qnn_profile_handle)) {
QNN_LOG_WARN("unable to create profile handle in the backend\n"); QNN_LOG_WARN("unable to create profile handle in the backend\n");
return 7; return 7;
} } else {
else {
QNN_LOG_DEBUG("initialize qnn profile successfully\n"); QNN_LOG_DEBUG("initialize qnn profile successfully\n");
} }
} }
@ -332,23 +295,16 @@ namespace qnn {
if (nullptr == _rpc_lib_handle) { if (nullptr == _rpc_lib_handle) {
QNN_LOG_WARN("failed to load qualcomm's rpc lib, error:%s\n", dlerror()); QNN_LOG_WARN("failed to load qualcomm's rpc lib, error:%s\n", dlerror());
return 8; return 8;
} } else {
else {
QNN_LOG_DEBUG("load rpcmem lib successfully\n"); QNN_LOG_DEBUG("load rpcmem lib successfully\n");
set_rpcmem_initialized(true); set_rpcmem_initialized(true);
} }
_pfn_rpc_mem_init = reinterpret_cast<qnn::pfn_rpc_mem_init>( _pfn_rpc_mem_init = reinterpret_cast<qnn::pfn_rpc_mem_init>(dlsym(_rpc_lib_handle, "rpcmem_init"));
dlsym(_rpc_lib_handle, "rpcmem_init")); _pfn_rpc_mem_deinit = reinterpret_cast<qnn::pfn_rpc_mem_deinit>(dlsym(_rpc_lib_handle, "rpcmem_deinit"));
_pfn_rpc_mem_deinit = reinterpret_cast<qnn::pfn_rpc_mem_deinit>( _pfn_rpc_mem_alloc = reinterpret_cast<qnn::pfn_rpc_mem_alloc>(dlsym(_rpc_lib_handle, "rpcmem_alloc"));
dlsym(_rpc_lib_handle, "rpcmem_deinit")); _pfn_rpc_mem_free = reinterpret_cast<qnn::pfn_rpc_mem_free>(dlsym(_rpc_lib_handle, "rpcmem_free"));
_pfn_rpc_mem_alloc = reinterpret_cast<qnn::pfn_rpc_mem_alloc>( _pfn_rpc_mem_to_fd = reinterpret_cast<qnn::pfn_rpc_mem_to_fd>(dlsym(_rpc_lib_handle, "rpcmem_to_fd"));
dlsym(_rpc_lib_handle, "rpcmem_alloc")); if (nullptr == _pfn_rpc_mem_alloc || nullptr == _pfn_rpc_mem_free || nullptr == _pfn_rpc_mem_to_fd) {
_pfn_rpc_mem_free = reinterpret_cast<qnn::pfn_rpc_mem_free>(
dlsym(_rpc_lib_handle, "rpcmem_free"));
_pfn_rpc_mem_to_fd = reinterpret_cast<qnn::pfn_rpc_mem_to_fd>(
dlsym(_rpc_lib_handle, "rpcmem_to_fd"));
if (nullptr == _pfn_rpc_mem_alloc || nullptr == _pfn_rpc_mem_free ||
nullptr == _pfn_rpc_mem_to_fd) {
QNN_LOG_WARN("unable to access symbols in QNN RPC lib. dlerror(): %s", dlerror()); QNN_LOG_WARN("unable to access symbols in QNN RPC lib. dlerror(): %s", dlerror());
dlclose(_rpc_lib_handle); dlclose(_rpc_lib_handle);
return 9; return 9;
@ -363,15 +319,11 @@ namespace qnn {
qnn_context_config.priority = QNN_PRIORITY_DEFAULT; qnn_context_config.priority = QNN_PRIORITY_DEFAULT;
const QnnContext_Config_t * context_configs[] = {&qnn_context_config, nullptr}; const QnnContext_Config_t * context_configs[] = {&qnn_context_config, nullptr};
*/ */
_qnn_interface.qnn_context_create( _qnn_interface.qnn_context_create(_qnn_backend_handle, _qnn_device_handle, nullptr, &_qnn_context_handle);
_qnn_backend_handle, _qnn_device_handle,
nullptr,
&_qnn_context_handle);
if (nullptr == _qnn_context_handle) { if (nullptr == _qnn_context_handle) {
QNN_LOG_WARN("why failed to initialize qnn context\n"); QNN_LOG_WARN("why failed to initialize qnn context\n");
return 10; return 10;
} } else {
else {
QNN_LOG_DEBUG("initialize qnn context successfully\n"); QNN_LOG_DEBUG("initialize qnn context successfully\n");
} }
@ -383,21 +335,17 @@ namespace qnn {
size_t probe_slots[] = { 1024, 1536, 2048 - 48, 2048 }; size_t probe_slots[] = { 1024, 1536, 2048 - 48, 2048 };
size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); size_t probe_counts = sizeof(probe_slots) / sizeof(size_t);
for (size_t idx = 0; idx < probe_counts; idx++) { for (size_t idx = 0; idx < probe_counts; idx++) {
rpc_buffer = static_cast<uint8_t*>(alloc_rpcmem( rpc_buffer = static_cast<uint8_t *>(alloc_rpcmem(probe_slots[idx] * size_in_mb, 4));
probe_slots[idx] * size_in_mb, 4));
if (nullptr == rpc_buffer) { if (nullptr == rpc_buffer) {
QNN_LOG_INFO("alloc rpcmem %d (MB) failure, %s\n", QNN_LOG_INFO("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], strerror(errno));
probe_slots[idx], strerror(errno));
break; break;
} } else {
else {
candidate_size = probe_slots[idx]; candidate_size = probe_slots[idx];
free_rpcmem(rpc_buffer); free_rpcmem(rpc_buffer);
rpc_buffer = nullptr; rpc_buffer = nullptr;
} }
} }
if (candidate_size > _rpcmem_capacity) if (candidate_size > _rpcmem_capacity) _rpcmem_capacity = candidate_size;
_rpcmem_capacity = candidate_size;
QNN_LOG_INFO("capacity of QNN rpc ion memory is about %d MB\n", _rpcmem_capacity); QNN_LOG_INFO("capacity of QNN rpc ion memory is about %d MB\n", _rpcmem_capacity);
if (0 != init_htp_perfinfra()) { if (0 != init_htp_perfinfra()) {
@ -425,8 +373,7 @@ namespace qnn {
if (dlclose(_rpc_lib_handle) != 0) { if (dlclose(_rpc_lib_handle) != 0) {
QNN_LOG_WARN("failed to unload qualcomm's rpc lib, error:%s\n", dlerror()); QNN_LOG_WARN("failed to unload qualcomm's rpc lib, error:%s\n", dlerror());
} } else {
else {
QNN_LOG_DEBUG("succeed to close rpcmem lib\n"); QNN_LOG_DEBUG("succeed to close rpcmem lib\n");
} }
@ -435,11 +382,9 @@ namespace qnn {
} }
if (nullptr != _qnn_context_handle) { if (nullptr != _qnn_context_handle) {
error = _qnn_interface.qnn_context_free(_qnn_context_handle, error = _qnn_interface.qnn_context_free(_qnn_context_handle, _qnn_profile_handle);
_qnn_profile_handle);
if (error != QNN_SUCCESS) { if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to free QNN context_handle: ID %u, error %d\n", QNN_LOG_WARN("failed to free QNN context_handle: ID %u, error %d\n", _qnn_interface.get_backend_id(),
_qnn_interface.get_backend_id(),
QNN_GET_ERROR_CODE(error)); QNN_GET_ERROR_CODE(error));
} }
_qnn_context_handle = nullptr; _qnn_context_handle = nullptr;
@ -448,8 +393,7 @@ namespace qnn {
if (nullptr != _qnn_profile_handle) { if (nullptr != _qnn_profile_handle) {
error = _qnn_interface.qnn_profile_free(_qnn_profile_handle); error = _qnn_interface.qnn_profile_free(_qnn_profile_handle);
if (error != QNN_SUCCESS) { if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to free QNN profile_handle: ID %u, error %d\n", QNN_LOG_WARN("failed to free QNN profile_handle: ID %u, error %d\n", _qnn_interface.get_backend_id(),
_qnn_interface.get_backend_id(),
QNN_GET_ERROR_CODE(error)); QNN_GET_ERROR_CODE(error));
} }
_qnn_profile_handle = nullptr; _qnn_profile_handle = nullptr;
@ -458,8 +402,7 @@ namespace qnn {
if (nullptr != _qnn_device_handle) { if (nullptr != _qnn_device_handle) {
error = _qnn_interface.qnn_device_free(_qnn_device_handle); error = _qnn_interface.qnn_device_free(_qnn_device_handle);
if (error != QNN_SUCCESS) { if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to free QNN device_handle: ID %u, error %d\n", QNN_LOG_WARN("failed to free QNN device_handle: ID %u, error %d\n", _qnn_interface.get_backend_id(),
_qnn_interface.get_backend_id(),
QNN_GET_ERROR_CODE(error)); QNN_GET_ERROR_CODE(error));
} }
_qnn_device_handle = nullptr; _qnn_device_handle = nullptr;
@ -468,8 +411,7 @@ namespace qnn {
if (nullptr != _qnn_backend_handle) { if (nullptr != _qnn_backend_handle) {
error = _qnn_interface.qnn_backend_free(_qnn_backend_handle); error = _qnn_interface.qnn_backend_free(_qnn_backend_handle);
if (error != QNN_SUCCESS) { if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to free QNN backend_handle: ID %u, error %d\n", QNN_LOG_WARN("failed to free QNN backend_handle: ID %u, error %d\n", _qnn_interface.get_backend_id(),
_qnn_interface.get_backend_id(),
QNN_GET_ERROR_CODE(error)); QNN_GET_ERROR_CODE(error));
} }
_qnn_backend_handle = nullptr; _qnn_backend_handle = nullptr;
@ -478,8 +420,7 @@ namespace qnn {
if (nullptr != _qnn_log_handle) { if (nullptr != _qnn_log_handle) {
error = _qnn_interface.qnn_log_free(_qnn_log_handle); error = _qnn_interface.qnn_log_free(_qnn_log_handle);
if (error != QNN_SUCCESS) { if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to free QNN log_handle: ID %u, error %d\n", QNN_LOG_WARN("failed to free QNN log_handle: ID %u, error %d\n", _qnn_interface.get_backend_id(),
_qnn_interface.get_backend_id(),
QNN_GET_ERROR_CODE(error)); QNN_GET_ERROR_CODE(error));
} }
_qnn_log_handle = nullptr; _qnn_log_handle = nullptr;
@ -492,62 +433,6 @@ namespace qnn {
return ret_status; return ret_status;
} }
//TODO:keep it for further usage of offload the entire cgraph to a single QNN DAG directly
// which was used in Qualcomm's dedicated AI technology
#if 0
int init_qnn_graph(const char* graph_name, bool debug,
uint8_t do_node_validation = true,
const QnnGraph_Config_t** graph_configs = nullptr) {
int result = 0;
if (nullptr == graph_name) {
QNN_LOG_WARN("graph name is null\n");
return 1;
}
if (!_graph_name.empty()) {
QNN_LOG_WARN("qnn model for graph %s already initialized\n", graph_name);
return 2;
}
if (!do_node_validation) {
QNN_LOG_WARN("node validation disabled, backend will not perform op "
"validation prior to adding node\n");
}
_graph_name = graph_name;
_debug_tensor = debug;
_do_node_validations = do_node_validation;
result = _qnn_raw_interface.graphCreate(_qnn_context_handle, graph_name,
graph_configs, &_qnn_graph_handle);
if (result != QNN_GRAPH_NO_ERROR || nullptr == _qnn_graph_handle) {
QNN_LOG_WARN("failed to create graph in qnn context\n");
return 3;
}
else {
QNN_LOG_INFO("succeed to create graph %s, %p\n", graph_name, _qnn_graph_handle);
}
return 0;
}
int finalize_qnn_graph() {
if (nullptr != _qnn_graph_handle) {
if (_qnn_raw_interface.graphFinalize(_qnn_graph_handle,
_qnn_profile_handle,
nullptr) != QNN_GRAPH_NO_ERROR) {
QNN_LOG_WARN("finalizing graph failure\n");
}
}
else {
QNN_LOG_DEBUG("qnn graph handle is null\n");
}
return 0;
}
#endif
const qnn_interface &get_qnn_interface() { const qnn_interface &get_qnn_interface() {
if (!_qnn_interface.is_loaded()) { if (!_qnn_interface.is_loaded()) {
QNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); QNN_LOG_WARN("pls check why _qnn_interface is not loaded\n");
@ -571,25 +456,15 @@ namespace qnn {
const Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } const Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; }
const Qnn_ProfileHandle_t get_qnn_profile_handle() { const Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; }
return _qnn_profile_handle;
}
const Qnn_DeviceHandle_t get_qnn_device_handle() { const Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; }
return _qnn_device_handle;
}
const Qnn_BackendHandle_t get_qnn_backend_handle() { const Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; }
return _qnn_backend_handle;
}
const Qnn_ContextHandle_t get_qnn_context_handle() { const Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; }
return _qnn_context_handle;
}
const QnnSystemContext_Handle_t get_qnn_system_handle() { const QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; }
return _qnn_system_handle;
}
const Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } const Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; }
@ -599,8 +474,7 @@ namespace qnn {
if (error != QNN_SUCCESS) { if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to get qnn device infra\n"); QNN_LOG_WARN("failed to get qnn device infra\n");
return 1; return 1;
} } else {
else {
QNN_LOG_INFO("HTP backend perf_infrastructure creation ok\n"); QNN_LOG_INFO("HTP backend perf_infrastructure creation ok\n");
} }
@ -612,8 +486,7 @@ namespace qnn {
htp_perfinfra->createPowerConfigId(device_id, core_id, &power_configid); htp_perfinfra->createPowerConfigId(device_id, core_id, &power_configid);
if (htp_infra->infraType != QNN_HTP_DEVICE_INFRASTRUCTURE_TYPE_PERF) { if (htp_infra->infraType != QNN_HTP_DEVICE_INFRASTRUCTURE_TYPE_PERF) {
QNN_LOG_INFO("HTP infra type = %d, which is not perf infra type", htp_infra->infraType); QNN_LOG_INFO("HTP infra type = %d, which is not perf infra type", htp_infra->infraType);
} } else {
else {
QNN_LOG_INFO("HTP infra type = %d, which is perf infra type\n", htp_infra->infraType); QNN_LOG_INFO("HTP infra type = %d, which is perf infra type\n", htp_infra->infraType);
} }
_qnn_htp_perfinfra = htp_perfinfra; _qnn_htp_perfinfra = htp_perfinfra;
@ -636,21 +509,15 @@ namespace qnn {
// use rpc control latency recommended 100 us, refer hexagon sdk // use rpc control latency recommended 100 us, refer hexagon sdk
rpc_control_latency.rpcControlLatencyConfig = 100; rpc_control_latency.rpcControlLatencyConfig = 100;
const QnnHtpPerfInfrastructure_PowerConfig_t* power_configs[] = { const QnnHtpPerfInfrastructure_PowerConfig_t *power_configs[] = { &rpc_polling_time, &rpc_control_latency,
&rpc_polling_time,
&rpc_control_latency,
nullptr }; nullptr };
Qnn_ErrorHandle_t qnn_status = _qnn_htp_perfinfra->setPowerConfig( Qnn_ErrorHandle_t qnn_status = _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs);
_qnn_power_configid,
power_configs);
if (qnn_status != QNN_SUCCESS) { if (qnn_status != QNN_SUCCESS) {
QNN_LOG_WARN("set htp perf failed\n"); QNN_LOG_WARN("set htp perf failed\n");
} } else {
else {
QNN_LOG_INFO("set htp perf ok\n"); QNN_LOG_INFO("set htp perf ok\n");
} }
} } else {
else {
QNN_LOG_WARN("can't set htp perf\n"); QNN_LOG_WARN("can't set htp perf\n");
} }
@ -671,41 +538,29 @@ namespace qnn {
power_config.dcvsV3Config.dcvsEnable = 0; power_config.dcvsV3Config.dcvsEnable = 0;
power_config.dcvsV3Config.contextId = _qnn_power_configid; power_config.dcvsV3Config.contextId = _qnn_power_configid;
power_config.dcvsV3Config.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE; power_config.dcvsV3Config.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE;
power_config.dcvsV3Config.setSleepLatency = power_config.dcvsV3Config.setSleepLatency = 1; // true to consider Latency parameter otherwise false
1; // true to consider Latency parameter otherwise false
power_config.dcvsV3Config.sleepLatency = 40; power_config.dcvsV3Config.sleepLatency = 40;
power_config.dcvsV3Config.setBusParams = power_config.dcvsV3Config.setBusParams = 1; // true to consider Bus parameter otherwise false
1; // true to consider Bus parameter otherwise false power_config.dcvsV3Config.setCoreParams = 1; // true to consider Core parameter otherwise false
power_config.dcvsV3Config.setCoreParams = power_config.dcvsV3Config.sleepDisable = 1; // true to consider sleep/LPM modes, false to enable
1; // true to consider Core parameter otherwise false
power_config.dcvsV3Config.sleepDisable =
1; // true to consider sleep/LPM modes, false to enable
power_config.dcvsV3Config.setSleepDisable = power_config.dcvsV3Config.setSleepDisable =
1; // true to consider sleep disable/enable parameter otherwise false set sleep latency parameter 1; // true to consider sleep disable/enable parameter otherwise false set sleep latency parameter
// set Bus Clock Parameters // set Bus Clock Parameters
power_config.dcvsV3Config.busVoltageCornerMin = power_config.dcvsV3Config.busVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; power_config.dcvsV3Config.busVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
power_config.dcvsV3Config.busVoltageCornerTarget = power_config.dcvsV3Config.busVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
power_config.dcvsV3Config.busVoltageCornerMax =
DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
// set Core Clock Parameters // set Core Clock Parameters
power_config.dcvsV3Config.coreVoltageCornerMin = power_config.dcvsV3Config.coreVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; power_config.dcvsV3Config.coreVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
power_config.dcvsV3Config.coreVoltageCornerTarget = power_config.dcvsV3Config.coreVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
power_config.dcvsV3Config.coreVoltageCornerMax =
DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
// set power config with different performance parameters // set power config with different performance parameters
const QnnHtpPerfInfrastructure_PowerConfig_t* power_configs[] = { const QnnHtpPerfInfrastructure_PowerConfig_t *power_configs[] = { &power_config, nullptr };
&power_config, nullptr };
Qnn_ErrorHandle_t qnn_status = QNN_SUCCESS; Qnn_ErrorHandle_t qnn_status = QNN_SUCCESS;
qnn_status = _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); qnn_status = _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs);
if (qnn_status != QNN_SUCCESS) { if (qnn_status != QNN_SUCCESS) {
QNN_LOG_WARN("set htp high performance mode failed\n"); QNN_LOG_WARN("set htp high performance mode failed\n");
} } else {
else {
QNN_LOG_INFO("set htp high performance mode ok\n"); QNN_LOG_INFO("set htp high performance mode ok\n");
} }
@ -716,15 +571,11 @@ namespace qnn {
bool is_rpcmem_initialized() { return _rpcmem_initialized; } bool is_rpcmem_initialized() { return _rpcmem_initialized; }
void set_rpcmem_initialized(bool initialized) { void set_rpcmem_initialized(bool initialized) { _rpcmem_initialized = initialized; }
_rpcmem_initialized = initialized;
}
size_t get_rpcmem_capacity() { return _rpcmem_capacity; } size_t get_rpcmem_capacity() { return _rpcmem_capacity; }
bool is_rpcmem_registered(Qnn_MemHandle_t handle) { bool is_rpcmem_registered(Qnn_MemHandle_t handle) { return _qnn_mem_set.count(handle) != 0U; }
return _qnn_mem_set.count(handle) != 0U;
}
void *alloc_rpcmem(size_t bytes, size_t alignment) { void *alloc_rpcmem(size_t bytes, size_t alignment) {
if (!_rpcmem_initialized) { if (!_rpcmem_initialized) {
@ -733,17 +584,14 @@ namespace qnn {
} }
auto allocate_bytes = static_cast<int32_t>(bytes + alignment); auto allocate_bytes = static_cast<int32_t>(bytes + alignment);
void* buf = _pfn_rpc_mem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, void *buf = _pfn_rpc_mem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, allocate_bytes);
allocate_bytes);
if (buf == nullptr) { if (buf == nullptr) {
QNN_LOG_WARN("failed to allocate rpc memory\n"); QNN_LOG_WARN("failed to allocate rpc memory\n");
return nullptr; return nullptr;
} }
auto aligned_buf = reinterpret_cast<void*>( auto aligned_buf = reinterpret_cast<void *>(qnn::align_to(alignment, reinterpret_cast<intptr_t>(buf)));
qnn::align_to(alignment, reinterpret_cast<intptr_t>(buf))); bool status = _rpcmem_store_map.insert(std::pair<void *, void *>(aligned_buf, buf)).second;
bool status =
_rpcmem_store_map.insert(std::pair<void*, void*>(aligned_buf, buf)).second;
if (!status) { if (!status) {
QNN_LOG_WARN("failed to allocate rpc memory\n"); QNN_LOG_WARN("failed to allocate rpc memory\n");
_pfn_rpc_mem_free(buf); _pfn_rpc_mem_free(buf);
@ -755,11 +603,9 @@ namespace qnn {
void free_rpcmem(void *buf) { void free_rpcmem(void *buf) {
if (!_rpcmem_initialized) { if (!_rpcmem_initialized) {
QNN_LOG_WARN("rpc memory not initialized\n"); QNN_LOG_WARN("rpc memory not initialized\n");
} } else if (0 == _rpcmem_store_map.count(buf)) {
else if (0 == _rpcmem_store_map.count(buf)) {
QNN_LOG_WARN("no allocated tensor\n"); QNN_LOG_WARN("no allocated tensor\n");
} } else {
else {
_pfn_rpc_mem_free(_rpcmem_store_map[buf]); _pfn_rpc_mem_free(_rpcmem_store_map[buf]);
_rpcmem_store_map.erase(buf); _rpcmem_store_map.erase(buf);
} }
@ -769,8 +615,7 @@ namespace qnn {
int32_t mem_fd = -1; int32_t mem_fd = -1;
if (!is_rpcmem_initialized()) { if (!is_rpcmem_initialized()) {
QNN_LOG_WARN("rpc memory not initialized\n"); QNN_LOG_WARN("rpc memory not initialized\n");
} } else {
else {
mem_fd = _pfn_rpc_mem_to_fd(buf); mem_fd = _pfn_rpc_mem_to_fd(buf);
} }
@ -794,8 +639,7 @@ namespace qnn {
} }
if (is_rpcmem_registered((QNN_VER_PTR(*p_tensor)->memHandle))) { if (is_rpcmem_registered((QNN_VER_PTR(*p_tensor)->memHandle))) {
QNN_LOG_WARN("tensor %s has been registered shared memory\n", QNN_LOG_WARN("tensor %s has been registered shared memory\n", (QNN_VER_PTR(*p_tensor)->name));
(QNN_VER_PTR(*p_tensor)->name));
return 4; return 4;
} }
@ -805,8 +649,7 @@ namespace qnn {
return 5; return 5;
} }
QNN_LOG_INFO("mem_fd %d\n", mem_fd); QNN_LOG_INFO("mem_fd %d\n", mem_fd);
Qnn_MemDescriptor_t descriptor = { {QNN_VER_PTR(*p_tensor)->rank, Qnn_MemDescriptor_t descriptor = { { QNN_VER_PTR(*p_tensor)->rank, QNN_VER_PTR(*p_tensor)->dimensions,
QNN_VER_PTR(*p_tensor)->dimensions,
nullptr }, nullptr },
QNN_VER_PTR(*p_tensor)->dataType, QNN_VER_PTR(*p_tensor)->dataType,
QNN_MEM_TYPE_ION, QNN_MEM_TYPE_ION,
@ -816,13 +659,11 @@ namespace qnn {
error = _qnn_interface.qnn_mem_register(_qnn_context_handle, &descriptor, error = _qnn_interface.qnn_mem_register(_qnn_context_handle, &descriptor,
/*numDescriptors=*/1, &handle); /*numDescriptors=*/1, &handle);
if (error != QNN_SUCCESS) { if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to register shared memory, error %d, %s\n", QNN_LOG_WARN("failed to register shared memory, error %d, %s\n", QNN_GET_ERROR_CODE(error),
QNN_GET_ERROR_CODE(error), strerror(error)); strerror(error));
return 6; return 6;
} } else {
else { QNN_LOG_INFO("tensor %s successfully register shared memory\n", (QNN_VER_PTR(*p_tensor)->name));
QNN_LOG_INFO("tensor %s successfully register shared memory\n",
(QNN_VER_PTR(*p_tensor)->name));
} }
QNN_VER_PTR(*p_tensor)->memHandle = handle; QNN_VER_PTR(*p_tensor)->memHandle = handle;
_qnn_mem_set.insert((std::pair<void *, Qnn_MemHandle_t>(p_data, handle))); _qnn_mem_set.insert((std::pair<void *, Qnn_MemHandle_t>(p_data, handle)));
@ -831,8 +672,7 @@ namespace qnn {
} }
void *get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle) { void *get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle) {
for (std::unordered_map<void*, Qnn_MemHandle_t>::iterator it = _qnn_mem_set.begin(); for (std::unordered_map<void *, Qnn_MemHandle_t>::iterator it = _qnn_mem_set.begin(); it != _qnn_mem_set.end();
it != _qnn_mem_set.end();
it++) { it++) {
Qnn_MemHandle_t mem_handle = it->second; Qnn_MemHandle_t mem_handle = it->second;
if (it->second == mem_handle) { if (it->second == mem_handle) {
@ -850,28 +690,23 @@ namespace qnn {
QNN_LOG_WARN("no rpcmem registered\n"); QNN_LOG_WARN("no rpcmem registered\n");
} }
for (std::unordered_map<void*, Qnn_MemHandle_t>::iterator it = _qnn_mem_set.begin(); for (std::unordered_map<void *, Qnn_MemHandle_t>::iterator it = _qnn_mem_set.begin(); it != _qnn_mem_set.end();
it != _qnn_mem_set.end();
it++) { it++) {
Qnn_MemHandle_t mem_handle = it->second; Qnn_MemHandle_t mem_handle = it->second;
error = _qnn_interface.qnn_mem_de_register(&mem_handle, 1); error = _qnn_interface.qnn_mem_de_register(&mem_handle, 1);
if (error != QNN_SUCCESS) { if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to unregister shared memory, error %d\n", QNN_LOG_WARN("failed to unregister shared memory, error %d\n", QNN_GET_ERROR_CODE(error));
QNN_GET_ERROR_CODE(error));
} }
} }
_qnn_mem_set.clear(); _qnn_mem_set.clear();
} }
bool is_rpcmem_allocated(void* buf) { bool is_rpcmem_allocated(void *buf) { return _qnn_mem_set.count(buf) != 0U; }
return _qnn_mem_set.count(buf) != 0U;
}
const qnn::qcom_socinfo &get_soc_info() { return _soc_info; } const qnn::qcom_socinfo &get_soc_info() { return _soc_info; }
public: public:
std::map<std::string, std::map<std::string, std::tuple<Qnn_GraphHandle_t, Qnn_Tensor_t *, Qnn_Tensor_t *, Qnn_Tensor_t *>> _qnn_graph_map;
std::tuple<Qnn_GraphHandle_t, Qnn_Tensor_t*, Qnn_Tensor_t*, Qnn_Tensor_t*>> _qnn_graph_map;
private: private:
int load_system() { int load_system() {
@ -882,18 +717,14 @@ namespace qnn {
_system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL);
if (nullptr == _system_lib_handle) { if (nullptr == _system_lib_handle) {
QNN_LOG_WARN("can not open QNN library %s, error: %s\n", QNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror());
system_lib_path.c_str(), dlerror());
return 1; return 1;
} }
auto* get_providers = auto *get_providers = reinterpret_cast<qnn::pfn_qnnsysteminterface_getproviders *>(
reinterpret_cast<qnn::pfn_qnnsysteminterface_getproviders*>(
dlsym(_system_lib_handle, "QnnSystemInterface_getProviders")); dlsym(_system_lib_handle, "QnnSystemInterface_getProviders"));
if (nullptr == get_providers) { if (nullptr == get_providers) {
QNN_LOG_WARN( QNN_LOG_WARN("can not load QNN symbol QnnSystemInterface_getProviders: %s\n", dlerror());
"can not load QNN symbol QnnSystemInterface_getProviders: %s\n",
dlerror());
return 2; return 2;
} }
@ -901,14 +732,12 @@ namespace qnn {
const QnnSystemInterface_t **provider_list = nullptr; const QnnSystemInterface_t **provider_list = nullptr;
error = get_providers(&provider_list, &num_providers); error = get_providers(&provider_list, &num_providers);
if (error != QNN_SUCCESS) { if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to get providers, error %d\n", QNN_LOG_WARN("failed to get providers, error %d\n", QNN_GET_ERROR_CODE(error));
QNN_GET_ERROR_CODE(error));
return 3; return 3;
} }
if (num_providers != _required_num_providers) { if (num_providers != _required_num_providers) {
QNN_LOG_WARN("providers is %d instead of required %d\n", num_providers, QNN_LOG_WARN("providers is %d instead of required %d\n", num_providers, _required_num_providers);
_required_num_providers);
return 4; return 4;
} }
@ -920,21 +749,17 @@ namespace qnn {
QNN_SYSTEM_INTERFACE_VER_TYPE qnn_system_interface; QNN_SYSTEM_INTERFACE_VER_TYPE qnn_system_interface;
bool found_valid_system_interface = false; bool found_valid_system_interface = false;
for (size_t idx = 0; idx < num_providers; idx++) { for (size_t idx = 0; idx < num_providers; idx++) {
if (QNN_SYSTEM_API_VERSION_MAJOR == if (QNN_SYSTEM_API_VERSION_MAJOR == provider_list[idx]->systemApiVersion.major &&
provider_list[idx]->systemApiVersion.major && QNN_SYSTEM_API_VERSION_MINOR <= provider_list[idx]->systemApiVersion.minor) {
QNN_SYSTEM_API_VERSION_MINOR <=
provider_list[idx]->systemApiVersion.minor) {
found_valid_system_interface = true; found_valid_system_interface = true;
qnn_system_interface = qnn_system_interface = provider_list[idx]->QNN_SYSTEM_INTERFACE_VER_NAME;
provider_list[idx]->QNN_SYSTEM_INTERFACE_VER_NAME;
break; break;
} }
} }
if (!found_valid_system_interface) { if (!found_valid_system_interface) {
QNN_LOG_WARN("unable to find a valid qnn system interface\n"); QNN_LOG_WARN("unable to find a valid qnn system interface\n");
return 6; return 6;
} } else {
else {
QNN_LOG_INFO("find a valid qnn system interface\n"); QNN_LOG_INFO("find a valid qnn system interface\n");
} }
set_qnn_raw_system_interface(qnn_system_interface); set_qnn_raw_system_interface(qnn_system_interface);
@ -944,8 +769,7 @@ namespace qnn {
_qnn_interface.qnn_system_context_create(&_qnn_system_handle); _qnn_interface.qnn_system_context_create(&_qnn_system_handle);
if (nullptr == _qnn_system_handle) { if (nullptr == _qnn_system_handle) {
QNN_LOG_WARN("can not create QNN system contenxt\n"); QNN_LOG_WARN("can not create QNN system contenxt\n");
} } else {
else {
QNN_LOG_INFO("initialize qnn system successfully\n"); QNN_LOG_INFO("initialize qnn system successfully\n");
} }
@ -989,8 +813,7 @@ namespace qnn {
return 1; return 1;
} }
auto get_providers = auto get_providers = qnn::load_qnn_functionpointers<qnn::pfn_qnninterface_getproviders *>(
qnn::load_qnn_functionpointers<qnn::pfn_qnninterface_getproviders*>(
lib_handle, "QnnInterface_getProviders"); lib_handle, "QnnInterface_getProviders");
if (nullptr == get_providers) { if (nullptr == get_providers) {
QNN_LOG_WARN("can not load symbol QnnInterface_getProviders : %s", dlerror()); QNN_LOG_WARN("can not load symbol QnnInterface_getProviders : %s", dlerror());
@ -1017,10 +840,8 @@ namespace qnn {
bool found_valid_interface = false; bool found_valid_interface = false;
QNN_INTERFACE_VER_TYPE qnn_interface; QNN_INTERFACE_VER_TYPE qnn_interface;
for (size_t idx = 0; idx < num_providers; idx++) { for (size_t idx = 0; idx < num_providers; idx++) {
if (QNN_API_VERSION_MAJOR == if (QNN_API_VERSION_MAJOR == provider_list[idx]->apiVersion.coreApiVersion.major &&
provider_list[idx]->apiVersion.coreApiVersion.major && QNN_API_VERSION_MINOR <= provider_list[idx]->apiVersion.coreApiVersion.minor) {
QNN_API_VERSION_MINOR <=
provider_list[idx]->apiVersion.coreApiVersion.minor) {
found_valid_interface = true; found_valid_interface = true;
qnn_interface = provider_list[idx]->QNN_INTERFACE_VER_NAME; qnn_interface = provider_list[idx]->QNN_INTERFACE_VER_NAME;
break; break;
@ -1030,8 +851,7 @@ namespace qnn {
if (!found_valid_interface) { if (!found_valid_interface) {
QNN_LOG_WARN("unable to find a valid qnn interface\n"); QNN_LOG_WARN("unable to find a valid qnn interface\n");
return 6; return 6;
} } else {
else {
QNN_LOG_INFO("find a valid qnn interface\n"); QNN_LOG_INFO("find a valid qnn interface\n");
} }
set_qnn_raw_interface(qnn_interface); set_qnn_raw_interface(qnn_interface);
@ -1071,9 +891,7 @@ namespace qnn {
return 0; return 0;
} }
void set_qnn_raw_interface(QNN_INTERFACE_VER_TYPE& raw_interface) { void set_qnn_raw_interface(QNN_INTERFACE_VER_TYPE &raw_interface) { _qnn_raw_interface = raw_interface; }
_qnn_raw_interface = raw_interface;
}
void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE &raw_interface) { void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE &raw_interface) {
_qnn_raw_system_interface = raw_interface; _qnn_raw_system_interface = raw_interface;
@ -1140,4 +958,4 @@ namespace qnn {
qnn::qcom_socinfo _soc_info = {}; qnn::qcom_socinfo _soc_info = {};
}; };
} } // namespace qnn

77
ggml/src/ggml-qnn/tensor.hpp
View file

@ -1,23 +1,21 @@
#pragma once #pragma once
#include "ggml-qnn.h"
#include "QnnTensor.h" #include "QnnTensor.h"
#include "System/QnnSystemInterface.h" #include "System/QnnSystemInterface.h"
#include "ggml-qnn.h"
#include "backend.hpp" #include "backend.hpp"
#include "qnn.hpp" #include "qnn.hpp"
namespace qnn { namespace qnn {
template <Qnn_TensorType_t _tensorType> class ggml_qnn_tensor_readwrite { template <Qnn_TensorType_t _tensorType>
class ggml_qnn_tensor_readwrite {
public: public:
ggml_qnn_tensor_readwrite(const ggml_tensor* tensor, explicit ggml_qnn_tensor_readwrite(const ggml_tensor *tensor, Qnn_GraphHandle_t graph_handle,
Qnn_GraphHandle_t graph_handle, ggml_backend_qnn_context *ctx) :
ggml_backend_qnn_context* ctx) _tensor(tensor), _qnn_tensor(reinterpret_cast<Qnn_Tensor_t *>(tensor->extra)), _context(ctx) {
: _tensor(tensor),
_qnn_tensor(reinterpret_cast<Qnn_Tensor_t*>(tensor->extra)),
_context(ctx) {
_old_dimensions = QNN_VER_PTR(*_qnn_tensor)->dimensions; _old_dimensions = QNN_VER_PTR(*_qnn_tensor)->dimensions;
const auto qnn_data_type = datatype_from_ggml_datatype(tensor->type); const auto qnn_data_type = datatype_from_ggml_datatype(tensor->type);
const bool is_npu = ctx->device == QNN_BACKEND_NPU; const bool is_npu = ctx->device == QNN_BACKEND_NPU;
@ -27,12 +25,10 @@ namespace qnn {
QNN_VER_PTR(*_qnn_tensor)->clientBuf = { .data = nullptr, .dataSize = 0 }; QNN_VER_PTR(*_qnn_tensor)->clientBuf = { .data = nullptr, .dataSize = 0 };
} }
auto err = auto err = ctx->raw_interface.tensorCreateGraphTensor(graph_handle, _qnn_tensor);
ctx->raw_interface.tensorCreateGraphTensor(graph_handle, _qnn_tensor);
if (err != QNN_SUCCESS) { if (err != QNN_SUCCESS) {
QNN_LOG_INFO("error = %d\n", err); QNN_LOG_INFO("error = %d\n", err);
QNN_LOG_DEBUG("tensor%p name %s", _qnn_tensor, QNN_LOG_DEBUG("tensor%p name %s", _qnn_tensor, QNN_TENSOR_GET_NAME(*_qnn_tensor));
QNN_TENSOR_GET_NAME(*_qnn_tensor));
_context = nullptr; _context = nullptr;
return; return;
} }
@ -47,35 +43,29 @@ namespace qnn {
if (is_npu) { if (is_npu) {
auto *instance = ctx->instance; auto *instance = ctx->instance;
uint8_t* qnn_buffer = static_cast<uint8_t*>( uint8_t *qnn_buffer = static_cast<uint8_t *>(instance->alloc_rpcmem(ggml_nbytes(tensor), alignof(void *)));
instance->alloc_rpcmem(ggml_nbytes(tensor), alignof(void*)));
if (!qnn_buffer) { if (!qnn_buffer) {
QNN_LOG_WARN("alloc rpcmem failure, %s\n", strerror(errno)); QNN_LOG_WARN("alloc rpcmem failure, %s\n", strerror(errno));
QNN_LOG_DEBUG("tensor%p name %s", _qnn_tensor, QNN_LOG_DEBUG("tensor%p name %s", _qnn_tensor, QNN_TENSOR_GET_NAME(*_qnn_tensor));
QNN_TENSOR_GET_NAME(*_qnn_tensor));
_context = nullptr; _context = nullptr;
// No free for _qnn_tensor, because it's not registered. // No free for _qnn_tensor, because it's not registered.
return; return;
} } else {
else {
QNN_LOG_INFO("alloc rpcmem successfully\n"); QNN_LOG_INFO("alloc rpcmem successfully\n");
} }
instance->register_rpcmem(qnn_buffer, _qnn_tensor); instance->register_rpcmem(qnn_buffer, _qnn_tensor);
if (_tensorType == QNN_TENSOR_TYPE_APP_WRITE || if (_tensorType == QNN_TENSOR_TYPE_APP_WRITE || _tensorType == QNN_TENSOR_TYPE_APP_READWRITE) {
_tensorType == QNN_TENSOR_TYPE_APP_READWRITE) {
memcpy(qnn_buffer, tensor->data, ggml_nbytes(tensor)); memcpy(qnn_buffer, tensor->data, ggml_nbytes(tensor));
} }
} } else {
else { QNN_VER_PTR(*_qnn_tensor)->clientBuf = { tensor->data, get_ggml_tensor_data_size(tensor) };
QNN_VER_PTR(*_qnn_tensor)->clientBuf = {
tensor->data, get_ggml_tensor_data_size(tensor) };
} }
} }
ggml_qnn_tensor_readwrite(const ggml_tensor* tensor, Qnn_Tensor_t* qnn_tensor, explicit ggml_qnn_tensor_readwrite(const ggml_tensor *tensor, Qnn_Tensor_t *qnn_tensor,
ggml_backend_qnn_context* ctx) ggml_backend_qnn_context *ctx) :
: _tensor(tensor), _qnn_tensor(qnn_tensor), _context(ctx) { _tensor(tensor), _qnn_tensor(qnn_tensor), _context(ctx) {
_old_dimensions = QNN_VER_PTR(*_qnn_tensor)->dimensions; _old_dimensions = QNN_VER_PTR(*_qnn_tensor)->dimensions;
const auto qnn_data_type = qnn::datatype_from_ggml_datatype(tensor->type); const auto qnn_data_type = qnn::datatype_from_ggml_datatype(tensor->type);
const bool is_npu = ctx->device == QNN_BACKEND_NPU; const bool is_npu = ctx->device == QNN_BACKEND_NPU;
@ -90,32 +80,25 @@ namespace qnn {
if (is_npu) { if (is_npu) {
uint8_t *qnn_buffer = uint8_t *qnn_buffer =
static_cast<uint8_t*>(ctx->instance->get_rpcmem_from_memhandle( static_cast<uint8_t *>(ctx->instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*_qnn_tensor)->memHandle));
QNN_VER_PTR(*_qnn_tensor)->memHandle));
if (qnn_buffer) { if (qnn_buffer) {
memcpy(qnn_buffer, tensor->data, ggml_nbytes(tensor)); memcpy(qnn_buffer, tensor->data, ggml_nbytes(tensor));
} } else {
else {
QNN_LOG_WARN("can't find rpcmem from qnn mem handle\n"); QNN_LOG_WARN("can't find rpcmem from qnn mem handle\n");
QNN_LOG_DEBUG("tensor%p name %s", _qnn_tensor, QNN_LOG_DEBUG("tensor%p name %s", _qnn_tensor, QNN_TENSOR_GET_NAME(*_qnn_tensor));
QNN_TENSOR_GET_NAME(*_qnn_tensor));
_context = nullptr; _context = nullptr;
return; return;
} }
} } else {
else { QNN_VER_PTR(*_qnn_tensor)->clientBuf = { tensor->data, get_ggml_tensor_data_size(tensor) };
QNN_VER_PTR(*_qnn_tensor)->clientBuf = {
tensor->data, get_ggml_tensor_data_size(tensor) };
} }
} }
~ggml_qnn_tensor_readwrite() { ~ggml_qnn_tensor_readwrite() {
if ((_tensorType == QNN_TENSOR_TYPE_APP_READWRITE || if ((_tensorType == QNN_TENSOR_TYPE_APP_READWRITE || _tensorType == QNN_TENSOR_TYPE_APP_READ) && _context &&
_tensorType == QNN_TENSOR_TYPE_APP_READ) && _context->device == QNN_BACKEND_NPU) {
_context && _context->device == QNN_BACKEND_NPU) { uint8_t *qnn_buffer = static_cast<uint8_t *>(
uint8_t* qnn_buffer = _context->instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*_qnn_tensor)->memHandle));
static_cast<uint8_t*>(_context->instance->get_rpcmem_from_memhandle(
QNN_VER_PTR(*_qnn_tensor)->memHandle));
memcpy(_tensor->data, qnn_buffer, ggml_nbytes(_tensor)); memcpy(_tensor->data, qnn_buffer, ggml_nbytes(_tensor));
} }
@ -138,9 +121,7 @@ namespace qnn {
void operator=(ggml_qnn_tensor_readwrite &&) = delete; void operator=(ggml_qnn_tensor_readwrite &&) = delete;
}; };
using ggml_qnn_tensor_output = using ggml_qnn_tensor_output = ggml_qnn_tensor_readwrite<QNN_TENSOR_TYPE_APP_READ>;
ggml_qnn_tensor_readwrite<QNN_TENSOR_TYPE_APP_READ>; using ggml_qnn_tensor_input = ggml_qnn_tensor_readwrite<QNN_TENSOR_TYPE_APP_WRITE>;
using ggml_qnn_tensor_input =
ggml_qnn_tensor_readwrite<QNN_TENSOR_TYPE_APP_WRITE>;
} // namespace qnn } // namespace qnn

8
ggml/src/ggml-qnn/utils.cpp
View file

@ -2,6 +2,7 @@
#include "utils.hpp" #include "utils.hpp"
#include "ggml-qnn.h" #include "ggml-qnn.h"
#include "qnn-types.hpp" #include "qnn-types.hpp"
namespace qnn { namespace qnn {
@ -26,7 +27,6 @@ namespace qnn {
return QNN_DATATYPE_UNDEFINED; return QNN_DATATYPE_UNDEFINED;
} }
uint32_t get_ggml_tensor_rank(const ggml_tensor *tensor) { uint32_t get_ggml_tensor_rank(const ggml_tensor *tensor) {
uint32_t rank = 0; uint32_t rank = 0;
for (int i = 0; i < GGML_MAX_DIMS; i++) { for (int i = 0; i < GGML_MAX_DIMS; i++) {
@ -37,7 +37,6 @@ namespace qnn {
return rank; return rank;
} }
const char *get_backend_name(int n_backend_type) { const char *get_backend_name(int n_backend_type) {
switch (n_backend_type) { switch (n_backend_type) {
case QNN_BACKEND_CPU: case QNN_BACKEND_CPU:
@ -86,8 +85,7 @@ namespace qnn {
intptr_t align_to(size_t alignment, intptr_t offset) { intptr_t align_to(size_t alignment, intptr_t offset) {
return offset % alignment == 0 return offset % alignment == 0
? offset ? offset
: offset + (static_cast<intptr_t>(alignment) - : offset + (static_cast<intptr_t>(alignment) - offset % static_cast<intptr_t>(alignment));
offset % static_cast<intptr_t>(alignment));
} }
uint32_t get_ggml_tensor_data_size(const ggml_tensor *tensor) { uint32_t get_ggml_tensor_data_size(const ggml_tensor *tensor) {
@ -123,4 +121,4 @@ namespace qnn {
return nullptr; return nullptr;
} }
} } // namespace qnn

35
ggml/src/ggml-qnn/utils.hpp
View file

@ -1,16 +1,16 @@
#pragma once #pragma once
#include <stdint.h>
#include <stddef.h>
#include <inttypes.h>
#include <dlfcn.h> #include <dlfcn.h>
#include <fcntl.h> #include <fcntl.h>
#include <string> #include <inttypes.h>
#include <stddef.h>
#include <stdint.h>
#include "QnnTypes.h" #include <string>
#include "ggml.h" #include "ggml.h"
#include "QnnTypes.h"
#include "logger.hpp" #include "logger.hpp"
namespace qnn { namespace qnn {
@ -25,15 +25,15 @@ namespace qnn {
const char *opname_from_ggmlop(enum ggml_op ggmlop); const char *opname_from_ggmlop(enum ggml_op ggmlop);
template <typename Fn> Fn load_qnn_functionpointers(void* handle, const char* function_name) { template <typename Fn>
Fn load_qnn_functionpointers(void *handle, const char *function_name) {
return reinterpret_cast<Fn>(dlsym(handle, function_name)); return reinterpret_cast<Fn>(dlsym(handle, function_name));
} }
inline int validate_tensor_version(Qnn_Tensor_t tensor) { inline int validate_tensor_version(Qnn_Tensor_t tensor) {
if (tensor.version != QNN_TENSOR_VERSION_1) { if (tensor.version != QNN_TENSOR_VERSION_1) {
QNN_LOG_WARN( QNN_LOG_WARN("validate_tensor_version() tensor %s, got unsupported version %d\n", tensor.v1.name,
"validate_tensor_version() tensor %s, got unsupported version %d\n", tensor.version);
tensor.v1.name, tensor.version);
return 1; return 1;
} }
return 0; return 0;
@ -61,24 +61,21 @@ namespace qnn {
return QNN_TENSOR_TYPE_UNDEFINED; return QNN_TENSOR_TYPE_UNDEFINED;
} }
inline Qnn_TensorDataFormat_t inline Qnn_TensorDataFormat_t get_qnn_tensor_dataformat(const Qnn_Tensor_t &tensor) {
get_qnn_tensor_dataformat(const Qnn_Tensor_t& tensor) {
if (tensor.version == QNN_TENSOR_VERSION_1) { if (tensor.version == QNN_TENSOR_VERSION_1) {
return tensor.v1.dataFormat; return tensor.v1.dataFormat;
} }
return QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER; return QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER;
} }
inline Qnn_DataType_t inline Qnn_DataType_t get_qnn_tensor_datatype(const Qnn_Tensor_t &tensor) {
get_qnn_tensor_datatype(const Qnn_Tensor_t& tensor) {
if (tensor.version == QNN_TENSOR_VERSION_1) { if (tensor.version == QNN_TENSOR_VERSION_1) {
return tensor.v1.dataType; return tensor.v1.dataType;
} }
return QNN_DATATYPE_UNDEFINED; return QNN_DATATYPE_UNDEFINED;
} }
inline Qnn_QuantizeParams_t inline Qnn_QuantizeParams_t get_qnn_tensor_quantparams(const Qnn_Tensor_t &tensor) {
get_qnn_tensor_quantparams(const Qnn_Tensor_t& tensor) {
if (tensor.version == QNN_TENSOR_VERSION_1) { if (tensor.version == QNN_TENSOR_VERSION_1) {
return tensor.v1.quantizeParams; return tensor.v1.quantizeParams;
} }
@ -172,7 +169,6 @@ namespace qnn {
} }
} }
#if ENABLE_QNNBACKEND_PERF #if ENABLE_QNNBACKEND_PERF
class qnn_perf { class qnn_perf {
public: public:
@ -181,9 +177,7 @@ namespace qnn {
qnn_perf(const qnn_perf &) = delete; qnn_perf(const qnn_perf &) = delete;
qnn_perf &operator=(const qnn_perf &) = delete; qnn_perf &operator=(const qnn_perf &) = delete;
void start() { void start() { _begin_time = ggml_time_us(); }
_begin_time = ggml_time_us();
}
void info() { void info() {
_end_time = ggml_time_us(); _end_time = ggml_time_us();
@ -210,8 +204,7 @@ namespace qnn {
}; };
#endif #endif
} } // namespace qnn
#define VALIDATE(value, status) \ #define VALIDATE(value, status) \
do { \ do { \