vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add clang format file and reformating

Browse source
This commit is contained in:
hongruichen 2024-07-04 22:18:45 +08:00
parent 38f88d5fb1
commit 000240cf62
12 changed files with 1514 additions and 1809 deletions
Download patch file Download diff file Expand all files Collapse all files

25
ggml/include/ggml-qnn.h
View file

@ -1,41 +1,48 @@
#pragma once
#include "ggml.h"
#include "ggml-backend.h"
#include "ggml.h"
#ifdef __cplusplus
extern "C" {
#endif
#define GGML_QNN_MAX_DEVICES 3
enum QNNBackend {
QNN_BACKEND_CPU,
QNN_BACKEND_GPU,
QNN_BACKEND_NPU,
QNN_BACKEND_GGML, //"fake" QNN backend, used for compare performance between QNN and original GGML
QNN_BACKEND_GGML, //"fake" QNN backend, used for compare performance between
// QNN and original GGML
};
GGML_API int ggml_backend_qnn_reg_devices(void);
/**
*
* @param device 0: QNN_BACKEND_CPU 1: QNN_BACKEND_GPU 2: QNN_BACKEND_NPU
* @param qnn_lib_path qnn library path, such as "/data/local/tmp/" on Android or specified in JNI layer
* @param device 0: QNN_BACKEND_CPU 1: QNN_BACKEND_GPU 2:
* QNN_BACKEND_NPU
* @param qnn_lib_path qnn library path, such as "/data/local/tmp/" on
* Android or specified in JNI layer
* @return
*/
GGML_API ggml_backend_t ggml_backend_qnn_init(size_t dev_num, const char * qnn_lib_path);
GGML_API ggml_backend_t ggml_backend_qnn_init(size_t dev_num,
const char* qnn_lib_path);
GGML_API bool ggml_backend_is_qnn(ggml_backend_t backend);
GGML_API void ggml_backend_qnn_set_n_threads(ggml_backend_t backend, int thread_counts);
GGML_API void ggml_backend_qnn_set_n_threads(ggml_backend_t backend,
int thread_counts);
GGML_API int ggml_backend_qnn_get_device_count(void);
GGML_API void ggml_backend_qnn_get_device_description(size_t dev_num, char * description, size_t description_size);
GGML_API void ggml_backend_qnn_get_device_description(size_t dev_num,
char* description,
size_t description_size);
GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_qnn_buffer_type(size_t dev_num);
GGML_API GGML_CALL ggml_backend_buffer_type_t
ggml_backend_qnn_buffer_type(size_t dev_num);
#ifdef __cplusplus
}

31
ggml/src/ggml-qnn/.clang-format Normal file
View file

@ -0,0 +1,31 @@
---
BasedOnStyle: Google
IndentWidth: 4
AccessModifierOffset: -4
AlignAfterOpenBracket: Align
AlignOperands: true
AlignTrailingComments: true
BinPackArguments: true
BinPackParameters: true
BreakBeforeBraces: Custom
BreakConstructorInitializers: AfterColon
ColumnLimit: 120
Cpp11BracedListStyle: false
DerivePointerAlignment: false
IncludeCategories:
- Regex: '^<.*\.h>'
Priority: 1
- Regex: '^<.*'
Priority: 2
- Regex: '^"ggml\.h"'
Priority: 3
- Regex: '^"ggml-.+\.h"'
Priority: 4
- Regex: '.*'
Priority: 5
KeepEmptyLinesAtTheStartOfBlocks: true
MaxEmptyLinesToKeep: 1
PointerAlignment: Right
SortIncludes: true
SpacesBeforeTrailingComments: 1
UseTab: Never

394
ggml/src/ggml-qnn/backend-ops.cpp
View file

@ -1,13 +1,12 @@
#include "backend-ops.hpp"
#include "utils.hpp"
#include "logger.hpp"
#include "tensor.hpp"
#include "utils.hpp"
static bool qnn_is_valid_params(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
static bool qnn_is_valid_params(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {
if (!ctx || !src0 || !src1 || !dst) {
QNN_LOG_WARN("invalid params\n");
return false;
@ -39,8 +38,8 @@ static bool qnn_is_valid_params(ggml_backend_qnn_context* ctx, const ggml_tensor
// TODO: this function can be removed later because there are duplicated codes with ggml_qnn_mul_mat
// keep it for illustrate how to implement a specified GGMPL OP using QNN API + QNN RPC
static void ggml_qnn_add(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
static void ggml_qnn_add(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {
Qnn_ErrorHandle_t error = QNN_SUCCESS;
bool graph_initialized = false;
qnn::qnn_instance *instance = nullptr;
@ -57,16 +56,14 @@ static void ggml_qnn_add(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
perf.start();
std::string map_entry(ggml_op_name(ggmlop));
if (instance->_qnn_graph_map.find(map_entry) !=
instance->_qnn_graph_map.end()) {
if (instance->_qnn_graph_map.find(map_entry) != instance->_qnn_graph_map.end()) {
graph_initialized = true;
auto &graph_item = instance->_qnn_graph_map[map_entry];
graph_handle = std::get<0>(graph_item);
}
if (!graph_initialized) {
graph_name = graph_name + "_" + std::to_string(ctx->threads) +
"_" + src0->name + "_" + src1->name;
graph_name = graph_name + "_" + std::to_string(ctx->threads) + "_" + src0->name + "_" + src1->name;
QNN_LOG_INFO("graph name %s", graph_name.c_str());
if (ctx->device == QNN_BACKEND_NPU) {
QnnHtpGraph_CustomConfig_t hvx_config;
@ -98,28 +95,22 @@ static void ggml_qnn_add(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM;
graph_vtcm_config.customConfig = &vtcm_config;
const QnnGraph_Config_t* p_graphconfig[] = { &graph_hvx_config,
&graph_dlbc_config,
&graph_vtcm_config,
&graph_opt_config,
NULL };
error = qnn_raw_interface.graphCreate(
instance->get_qnn_context_handle(), graph_name.c_str(), p_graphconfig,
const QnnGraph_Config_t *p_graphconfig[] = { &graph_hvx_config, &graph_dlbc_config, &graph_vtcm_config,
&graph_opt_config, NULL };
error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), p_graphconfig,
&graph_handle);
}
else {
error = qnn_raw_interface.graphCreate(
instance->get_qnn_context_handle(), graph_name.c_str(), nullptr,
} else {
error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), nullptr,
&graph_handle);
}
if (QNN_SUCCESS != error) {
QNN_LOG_INFO("can't create qnn graph handle with graph name %s, "
QNN_LOG_INFO(
"can't create qnn graph handle with graph name %s, "
"error = %d\n",
graph_name.c_str(), error);
goto failure;
}
else {
} else {
QNN_LOG_INFO("create qnn graph handle with graph name %s ok\n", graph_name.c_str());
}
@ -139,30 +130,20 @@ static void ggml_qnn_add(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
Qnn_Tensor_t tensor_inputs[] = { *tensor_input0.get_qnn_tensor(), *tensor_input1.get_qnn_tensor() };
Qnn_Tensor_t tensor_outputs[] = { *tensor_output.get_qnn_tensor() };
Qnn_OpConfig_t op_config = {
(Qnn_OpConfigVersion_t)1,
.v1 = {"ggml_op_add",
QNN_OP_PACKAGE_NAME_QTI_AISW,
QNN_OP_ELEMENT_WISE_ADD,
0, qnn_params,
2, tensor_inputs,
1,tensor_outputs}
};
Qnn_OpConfig_t op_config = { (Qnn_OpConfigVersion_t)1,
.v1 = { "ggml_op_add", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_ELEMENT_WISE_ADD, 0,
qnn_params, 2, tensor_inputs, 1, tensor_outputs } };
error = qnn_raw_interface.graphAddNode(graph_handle, op_config);
if (QNN_SUCCESS != error) {
QNN_LOG_INFO("error = %d\n", error);
goto failure;
}
error = qnn_raw_interface.graphFinalize(graph_handle,
nullptr, nullptr);
error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr);
if (QNN_SUCCESS != error) {
QNN_LOG_INFO("error = %d\n", error);
goto failure;
}
error = qnn_raw_interface.graphExecute(graph_handle,
tensor_inputs, 2,
tensor_outputs, 1,
nullptr, nullptr);
error = qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, nullptr, nullptr);
if (ctx->device == QNN_BACKEND_NPU) {
if (QNN_COMMON_ERROR_SYSTEM_COMMUNICATION == error) {
QNN_LOG_WARN("NPU crashed. SSR detected. Caused QNN graph execute error\n");
@ -173,13 +154,10 @@ static void ggml_qnn_add(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
goto failure;
}
auto graph_item = std::make_tuple(graph_handle,
tensor_input0.get_qnn_tensor(),
tensor_input1.get_qnn_tensor(),
auto graph_item = std::make_tuple(graph_handle, tensor_input0.get_qnn_tensor(), tensor_input1.get_qnn_tensor(),
tensor_output.get_qnn_tensor());
instance->_qnn_graph_map[map_entry] = graph_item;
}
else {
} else {
auto &graph_item = instance->_qnn_graph_map[map_entry];
qnn::ggml_qnn_tensor_input tensor_input0(src0, std::get<1>(graph_item), ctx);
qnn::ggml_qnn_tensor_input tensor_input1(src1, std::get<2>(graph_item), ctx);
@ -187,10 +165,7 @@ static void ggml_qnn_add(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
Qnn_Tensor_t tensor_inputs[] = { *tensor_input0.get_qnn_tensor(), *tensor_input1.get_qnn_tensor() };
Qnn_Tensor_t tensor_outputs[] = { *tensor_output.get_qnn_tensor() };
error = qnn_raw_interface.graphExecute(graph_handle,
tensor_inputs, 2,
tensor_outputs, 1,
nullptr, nullptr);
error = qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, nullptr, nullptr);
if (ctx->device == QNN_BACKEND_NPU) {
if (QNN_COMMON_ERROR_SYSTEM_COMMUNICATION == error) {
QNN_LOG_WARN("NPU crashed. SSR detected. Caused QNN graph execute error\n");
@ -204,20 +179,17 @@ static void ggml_qnn_add(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
failure:
if (QNN_SUCCESS != error) {
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64
" x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n",
src0->name, src0->type, ggml_type_name(src0->type),
src0->ne[0], src0->ne[1], src0->ne[2], src0->nb[0],
src0->nb[1], src0->nb[2]);
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64
" x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n",
src1->name, src1->type, ggml_type_name(src1->type),
src1->ne[0], src1->ne[1], src1->ne[2], src1->nb[0],
src1->nb[1], src1->nb[2]);
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64
" x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n",
dst->name, dst->type, ggml_type_name(dst->type),
dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0],
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64
", nb = (%5zi, %5zi, %5zi)\n",
src0->name, src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2],
src0->nb[0], src0->nb[1], src0->nb[2]);
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64
", nb = (%5zi, %5zi, %5zi)\n",
src1->name, src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2],
src1->nb[0], src1->nb[1], src1->nb[2]);
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64
", nb = (%5zi, %5zi, %5zi)\n",
dst->name, dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0],
dst->nb[1], dst->nb[2]);
}
@ -235,8 +207,7 @@ failure:
* mul_mat_f16_f32: src0 is F16 and src1 is F32.
* mul_mat_q_f32: src0 is quantized (Q4_0, Q4_1, ...), and src1 is F32.
*/
static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
static void ggml_qnn_mul_mat(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {
Qnn_ErrorHandle_t error = QNN_SUCCESS;
bool graph_initialized = false;
@ -254,8 +225,7 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx,
perf.start();
std::string map_entry = std::string(ggml_op_name(ggmlop));
if (instance->_qnn_graph_map.find(map_entry) !=
instance->_qnn_graph_map.end()) {
if (instance->_qnn_graph_map.find(map_entry) != instance->_qnn_graph_map.end()) {
graph_initialized = true;
auto &graph_item = instance->_qnn_graph_map[map_entry];
graph_handle = std::get<0>(graph_item);
@ -267,8 +237,7 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx,
// the performance gains is worth although there is performance loss in pass-1
if (!graph_initialized) {
graph_name = graph_name + "_" + std::to_string(ctx->threads) +
"_" + src0->name + "_" + src1->name;
graph_name = graph_name + "_" + std::to_string(ctx->threads) + "_" + src0->name + "_" + src1->name;
QNN_LOG_INFO("graph name %s", graph_name.c_str());
if (ctx->device == QNN_BACKEND_NPU) {
QnnHtpGraph_CustomConfig_t hvx_config;
@ -300,22 +269,17 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx,
graph_vtcm_config.option = QNN_GRAPH_CONFIG_OPTION_CUSTOM;
graph_vtcm_config.customConfig = &vtcm_config;
const QnnGraph_Config_t* p_graphconfig[] = { &graph_hvx_config,
&graph_dlbc_config,
&graph_vtcm_config,
&graph_opt_config,
NULL };
error = qnn_raw_interface.graphCreate(
instance->get_qnn_context_handle(), graph_name.c_str(), p_graphconfig,
const QnnGraph_Config_t *p_graphconfig[] = { &graph_hvx_config, &graph_dlbc_config, &graph_vtcm_config,
&graph_opt_config, NULL };
error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), p_graphconfig,
&graph_handle);
}
else {
error = qnn_raw_interface.graphCreate(
instance->get_qnn_context_handle(), graph_name.c_str(), nullptr,
} else {
error = qnn_raw_interface.graphCreate(instance->get_qnn_context_handle(), graph_name.c_str(), nullptr,
&graph_handle);
}
if (QNN_SUCCESS != error) {
QNN_LOG_INFO("can't create qnn graph handle with graph name %s, "
QNN_LOG_INFO(
"can't create qnn graph handle with graph name %s, "
"error = %d\n",
graph_name.c_str(), error);
goto failure;
@ -336,30 +300,20 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx,
Qnn_Tensor_t tensor_inputs[] = { *tensor_input0.get_qnn_tensor(), *tensor_input1.get_qnn_tensor() };
Qnn_Tensor_t tensor_outputs[] = { *tensor_output.get_qnn_tensor() };
Qnn_OpConfig_t op_config = {
(Qnn_OpConfigVersion_t)1,
.v1 = {"ggml_op_mul_mat",
QNN_OP_PACKAGE_NAME_QTI_AISW,
QNN_OP_MAT_MUL,
0, qnn_params,
2, tensor_inputs,
1, tensor_outputs}
};
Qnn_OpConfig_t op_config = { (Qnn_OpConfigVersion_t)1,
.v1 = { "ggml_op_mul_mat", QNN_OP_PACKAGE_NAME_QTI_AISW, QNN_OP_MAT_MUL, 0,
qnn_params, 2, tensor_inputs, 1, tensor_outputs } };
error = qnn_raw_interface.graphAddNode(graph_handle, op_config);
if (QNN_SUCCESS != error) {
QNN_LOG_INFO("error = %d\n", error);
goto failure;
}
error = qnn_raw_interface.graphFinalize(graph_handle,
nullptr, nullptr);
error = qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr);
if (QNN_SUCCESS != error) {
QNN_LOG_INFO("error = %d\n", error);
goto failure;
}
error = qnn_raw_interface.graphExecute(graph_handle,
tensor_inputs, 2,
tensor_outputs, 1,
nullptr, nullptr);
error = qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, nullptr, nullptr);
if (ctx->device == QNN_BACKEND_NPU) {
if (QNN_COMMON_ERROR_SYSTEM_COMMUNICATION == error) {
QNN_LOG_WARN("NPU crashed. SSR detected. Caused QNN graph execute error\n");
@ -370,13 +324,10 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx,
goto failure;
}
auto graph_item = std::make_tuple(graph_handle,
tensor_input0.get_qnn_tensor(),
tensor_input1.get_qnn_tensor(),
auto graph_item = std::make_tuple(graph_handle, tensor_input0.get_qnn_tensor(), tensor_input1.get_qnn_tensor(),
tensor_output.get_qnn_tensor());
instance->_qnn_graph_map[map_entry] = graph_item;
}
else {
} else {
auto &graph_item = instance->_qnn_graph_map[map_entry];
qnn::ggml_qnn_tensor_input tensor_input0(src0, std::get<1>(graph_item), ctx);
qnn::ggml_qnn_tensor_input tensor_input1(src1, std::get<2>(graph_item), ctx);
@ -384,10 +335,7 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx,
Qnn_Tensor_t tensor_inputs[] = { *tensor_input0.get_qnn_tensor(), *tensor_input1.get_qnn_tensor() };
Qnn_Tensor_t tensor_outputs[] = { *tensor_output.get_qnn_tensor() };
error = qnn_raw_interface.graphExecute(graph_handle,
tensor_inputs, 2,
tensor_outputs, 1,
nullptr, nullptr);
error = qnn_raw_interface.graphExecute(graph_handle, tensor_inputs, 2, tensor_outputs, 1, nullptr, nullptr);
if (ctx->device == QNN_BACKEND_NPU) {
if (QNN_COMMON_ERROR_SYSTEM_COMMUNICATION == error) {
QNN_LOG_WARN("NPU crashed. SSR detected. Caused QNN graph execute error\n");
@ -401,181 +349,127 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context* ctx,
failure:
if (QNN_SUCCESS != error) {
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64
" x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n",
src0->name, src0->type, ggml_type_name(src0->type),
src0->ne[0], src0->ne[1], src0->ne[2], src0->nb[0],
src0->nb[1], src0->nb[2]);
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64
" x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n",
src1->name, src1->type, ggml_type_name(src1->type),
src1->ne[0], src1->ne[1], src1->ne[2], src1->nb[0],
src1->nb[1], src1->nb[2]);
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64
" x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n",
dst->name, dst->type, ggml_type_name(dst->type), dst->ne[0],
dst->ne[1], dst->ne[2], dst->nb[0], dst->nb[1], dst->nb[2]);
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64
", nb = (%5zi, %5zi, %5zi)\n",
src0->name, src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2],
src0->nb[0], src0->nb[1], src0->nb[2]);
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64
", nb = (%5zi, %5zi, %5zi)\n",
src1->name, src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2],
src1->nb[0], src1->nb[1], src1->nb[2]);
QNN_LOG_DEBUG("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64
", nb = (%5zi, %5zi, %5zi)\n",
dst->name, dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->nb[0],
dst->nb[1], dst->nb[2]);
}
perf.info();
}
static void ggml_qnn_repeat(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
static void ggml_qnn_repeat(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_get_rows(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_acc(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_div(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_gelu(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_silu(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_gelu_quick(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_tanh(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_relu(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_hardsigmoid(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_hardswish(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_leaky_relu(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_sqr(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_norm(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_group_norm(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_concat(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_upscale(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_pad(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_rms_norm(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_cpy(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_dup(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {
}
static void ggml_qnn_get_rows(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_acc(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_div(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_gelu(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_silu(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_gelu_quick(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_tanh(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_relu(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_hardsigmoid(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_hardswish(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_leaky_relu(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_sqr(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_norm(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_group_norm(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_concat(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_upscale(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_pad(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_rms_norm(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_cpy(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_dup(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
ggml_qnn_cpy(ctx, src0, dst, nullptr);
(void)src1;
}
static void ggml_qnn_mul_mat_id(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_mul_mat_id(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_scale(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_scale(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_clamp(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_clamp(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_diag_mask_inf(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
}
static void ggml_qnn_diag_mask_inf(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_soft_max(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_soft_max(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_rope(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
static void ggml_qnn_rope(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {
GGML_ASSERT(ggml_is_contiguous(src0));
}
static void ggml_qnn_pool2d(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_pool2d(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_im2col(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
ggml_tensor* dst) {
}
static void ggml_qnn_im2col(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {}
static void ggml_qnn_sum_rows(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
static void ggml_qnn_sum_rows(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {
GGML_ASSERT(ggml_is_contiguous(src0));
}
static void ggml_qnn_argsort(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0, const ggml_tensor* src1,
static void ggml_qnn_argsort(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {
GGML_ASSERT(ggml_is_contiguous(src0));
}
static void ggml_qnn_nop(ggml_backend_qnn_context* ctx, const ggml_tensor* src0,
const ggml_tensor* src1, ggml_tensor* dst) {
static void ggml_qnn_nop(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst) {
(void)src0;
(void)src1;
(void)dst;

7
ggml/src/ggml-qnn/backend-ops.hpp
View file

@ -1,17 +1,16 @@
#pragma once
#include "ggml.h"
#include "backend.hpp"
namespace qnn {
typedef void (*ggml_qnn_op_t)(ggml_backend_qnn_context* ctx,
const ggml_tensor* src0,
const ggml_tensor* src1,
typedef void (*ggml_qnn_op_t)(ggml_backend_qnn_context *ctx, const ggml_tensor *src0, const ggml_tensor *src1,
ggml_tensor *dst);
typedef const ggml_qnn_op_t (&ggml_qnn_op_array_t)[GGML_OP_COUNT];
ggml_qnn_op_array_t ggml_qnn_op_array();
}
} // namespace qnn

1
ggml/src/ggml-qnn/backend.hpp
View file

@ -2,6 +2,7 @@
#pragma once
#include "ggml.h"
#include "ggml-backend.h"
#include "qnn.hpp"

15
ggml/src/ggml-qnn/logger.cpp
View file

@ -2,6 +2,7 @@
#include "logger.hpp"
#include <stdio.h>
#include <mutex>
#if (defined __ANDROID__) || (defined ANDROID)
@ -10,9 +11,7 @@
#define QNN_LOGBUF_LEN 4096
void qnn::internal_log(ggml_log_level level, const char* file,
const char* func, int line,
const char* format, ...) {
void qnn::internal_log(ggml_log_level level, const char *file, const char *func, int line, const char *format, ...) {
static std::mutex qnn_internal_log_mutex;
static char s_qnn_internal_log_buf[QNN_LOGBUF_LEN];
@ -21,11 +20,8 @@ void qnn::internal_log(ggml_log_level level, const char* file,
va_list args;
va_start(args, format);
int len_prefix =
snprintf(s_qnn_internal_log_buf, QNN_LOGBUF_LEN,
"[%s, %d]: ", func, line);
int len = vsnprintf(s_qnn_internal_log_buf + len_prefix,
QNN_LOGBUF_LEN - len_prefix, format, args);
int len_prefix = snprintf(s_qnn_internal_log_buf, QNN_LOGBUF_LEN, "[%s, %d]: ", func, line);
int len = vsnprintf(s_qnn_internal_log_buf + len_prefix, QNN_LOGBUF_LEN - len_prefix, format, args);
if (len < (QNN_LOGBUF_LEN - len_prefix)) {
#if (defined __ANDROID__) || (defined ANDROID)
// for Android APK
@ -38,8 +34,7 @@ void qnn::internal_log(ggml_log_level level, const char* file,
}
}
void qnn::sdk_logcallback(const char* fmt, QnnLog_Level_t level,
uint64_t timestamp, va_list argp) {
void qnn::sdk_logcallback(const char *fmt, QnnLog_Level_t level, uint64_t timestamp, va_list argp) {
#if ENABLE_QNNSDK_LOG
static std::mutex log_mutex;
static unsigned char s_ggml_qnn_logbuf[QNN_LOGBUF_LEN];

32
ggml/src/ggml-qnn/logger.hpp
View file

@ -2,36 +2,29 @@
#include <stdint.h>
#include "QnnTypes.h"
#include "QnnCommon.h"
#include "QnnInterface.h"
#include "System/QnnSystemInterface.h"
#include "ggml.h"
#include "QnnCommon.h"
#include "QnnInterface.h"
#include "QnnTypes.h"
#include "System/QnnSystemInterface.h"
namespace qnn {
void internal_log(ggml_log_level level, const char* file,
const char* func, int line,
const char* format, ...);
void internal_log(ggml_log_level level, const char *file, const char *func, int line, const char *format, ...);
void sdk_logcallback(const char* fmt, QnnLog_Level_t level,
uint64_t timestamp, va_list argp);
}
void sdk_logcallback(const char *fmt, QnnLog_Level_t level, uint64_t timestamp, va_list argp);
} // namespace qnn
// =================================================================================================
//
// QNN backend internal log function
//
// =================================================================================================
#define QNN_LOG_ERROR(...) \
qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
#define QNN_LOG_ERROR(...) qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
#define QNN_LOG_WARN(...) \
qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
#define QNN_LOG_WARN(...) qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
#define QNN_LOG_INFO(...) \
qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
#define QNN_LOG_INFO(...) qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
#ifdef NDEBUG
#define ENABLE_QNNBACKEND_DEBUG 0 // for troubleshooting QNN backend
@ -42,8 +35,7 @@ namespace qnn {
#endif
#if ENABLE_QNNBACKEND_DEBUG
#define QNN_LOG_DEBUG(...) \
qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
#define QNN_LOG_DEBUG(...) qnn::internal_log(GGML_LOG_LEVEL_DEBUG, __FILE__, __FUNCTION__, __LINE__, __VA_ARGS__)
#else
#define QNN_LOG_DEBUG(...)
#endif

10
ggml/src/ggml-qnn/qnn-types.hpp
View file

@ -1,9 +1,9 @@
#pragma once
#include "QnnTypes.h"
#include "QnnCommon.h"
#include "QnnInterface.h"
#include "QnnTypes.h"
#include "Saver/QnnSaver.h"
#include "System/QnnSystemInterface.h"
@ -14,11 +14,7 @@ namespace qnn {
// Qualcomm QNN(Qualcomm Neural Network, aka Qualcomm AI Engine Direct) SDK
// ref:https://github.com/pytorch/executorch/tree/main/backends/qualcomm
// =================================================================================================
enum sdk_profile_level {
profile_off = 0,
profile_basic = 1,
profile_detail = 2
};
enum sdk_profile_level { profile_off = 0, profile_basic = 1, profile_detail = 2 };
enum qcom_htp_arch {
NONE = 0,
@ -51,7 +47,7 @@ namespace qnn {
using pfn_qnnsaver_initialize = decltype(QnnSaver_initialize);
using pfn_qnninterface_getproviders = decltype(QnnInterface_getProviders);
using pfn_qnnsysteminterface_getproviders = decltype(QnnSystemInterface_getProviders);
}
} // namespace qnn
#define QNN_VER_PTR(x) (&((x).v1)) // TODO: remove this macro after we have a separate header for QNN

460
ggml/src/ggml-qnn/qnn.hpp
View file

@ -1,24 +1,25 @@
#pragma once
#include <math.h>
#include <map>
#include <mutex>
#include <string>
#include <unordered_map>
#include <map>
// header file of Qualcomm QNN(Qualcomm Neural Network, aka Qualcomm AI Engine Direct) SDK
// https://qpm.qualcomm.com/#/main/tools/details/qualcomm_ai_engine_direct
#include "QnnTypes.h"
#include "QnnCommon.h"
#include "QnnInterface.h"
#include "QnnContext.h"
#include "QnnBackend.h"
#include "QnnGraph.h"
#include "QnnProperty.h"
#include "QnnTensor.h"
#include "System/QnnSystemInterface.h"
#include "HTP/QnnHtpDevice.h"
#include "HTP/QnnHtpGraph.h"
#include <HTP/QnnHtpDevice.h>
#include <HTP/QnnHtpGraph.h>
#include <QnnBackend.h>
#include <QnnCommon.h>
#include <QnnContext.h>
#include <QnnGraph.h>
#include <QnnInterface.h>
#include <QnnProperty.h>
#include <QnnTensor.h>
#include <QnnTypes.h>
#include <System/QnnSystemInterface.h>
#include "qnn-types.hpp"
#include "utils.hpp"
@ -33,16 +34,15 @@ namespace qnn {
class qnn_interface {
#define DEFINE_SHIM_FUNCTION_INTERFACE(F, pointer_name) \
template <typename... Args> inline auto qnn_##F(Args... args) const { \
return (_qnn_interface->QNN_INTERFACE_VER_NAME.pointer_name)( \
std::forward<Args>(args)...); \
template <typename... Args> \
inline auto qnn_##F(Args... args) const { \
return (_qnn_interface->QNN_INTERFACE_VER_NAME.pointer_name)(std::forward<Args>(args)...); \
}
#define DEFINE_SHIM_FUNCTION_SYS_INTERFACE(F, pointer_name) \
template <typename... Args> inline auto qnn_##F(Args... args) const { \
return ( \
_qnn_sys_interface->QNN_SYSTEM_INTERFACE_VER_NAME.pointer_name)( \
std::forward<Args>(args)...); \
template <typename... Args> \
inline auto qnn_##F(Args... args) const { \
return (_qnn_sys_interface->QNN_SYSTEM_INTERFACE_VER_NAME.pointer_name)(std::forward<Args>(args)...); \
}
friend class qnn_instance;
@ -55,38 +55,31 @@ namespace qnn {
DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree);
DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package,
backendRegisterOpPackage);
DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage);
DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config,
backendValidateOpConfig);
DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig);
DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version,
backendGetApiVersion);
DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion);
// QnnDevice
DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate);
DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree);
DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure,
deviceGetInfrastructure);
DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure);
DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info,
deviceGetPlatformInfo);
DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo);
DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo);
// QnnContext
DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate);
DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size,
contextGetBinarySize);
DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize);
DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary);
DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary,
contextCreateFromBinary);
DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary);
DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree);
@ -125,39 +118,29 @@ namespace qnn {
DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister);
// QnnProperty
DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability,
propertyHasCapability);
DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability);
// QnnTensor
DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor,
tensorCreateContextTensor);
DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor);
DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor,
tensorCreateGraphTensor);
DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor);
// QnnSystem
DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create,
systemContextCreate);
DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate);
DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info,
systemContextGetBinaryInfo);
DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo);
DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree);
void set_qnn_interface(const QnnInterface_t* qnn_interface) {
_qnn_interface = qnn_interface;
}
void set_qnn_interface(const QnnInterface_t *qnn_interface) { _qnn_interface = qnn_interface; }
void set_qnn_system_interface(
const QnnSystemInterface_t* qnn_sys_interface) {
void set_qnn_system_interface(const QnnSystemInterface_t *qnn_sys_interface) {
_qnn_sys_interface = qnn_sys_interface;
}
uint32_t get_backend_id() const { return _qnn_interface->backendId; }
bool is_loaded() const {
return ((_qnn_sys_interface != nullptr) && (_qnn_interface != nullptr));
}
bool is_loaded() const { return ((_qnn_sys_interface != nullptr) && (_qnn_interface != nullptr)); }
private:
const QnnInterface_t *_qnn_interface = nullptr;
@ -165,17 +148,12 @@ namespace qnn {
const QnnSystemInterface_t *_qnn_sys_interface = nullptr;
};
class qnn_instance {
public:
using BackendIdType = decltype(QnnInterface_t{}.backendId);
explicit qnn_instance(const std::string& lib_path,
const std::string& backend_name,
const std::string& model_name)
: _lib_path(std::move(lib_path))
, _backend_name(std::move(backend_name))
, _model_name(std::move(model_name)) {};
explicit qnn_instance(const std::string &lib_path, const std::string &backend_name, const std::string &model_name) :
_lib_path(std::move(lib_path)), _backend_name(std::move(backend_name)), _model_name(std::move(model_name)) {};
~qnn_instance() {}
@ -188,8 +166,7 @@ namespace qnn {
if (0 != load_system()) {
QNN_LOG_WARN("can not load QNN system lib, pls check why?\n");
return 1;
}
else {
} else {
QNN_LOG_DEBUG("load QNN system lib successfully\n");
}
@ -203,12 +180,11 @@ namespace qnn {
}
backend_id = _lib_path_to_backend_id[backend_lib_path];
if (0 == _loaded_backend.count(backend_id) ||
0 == _loaded_lib_handle.count(backend_id)) {
QNN_LOG_WARN("library %s is loaded but loaded backend count=%zu, "
if (0 == _loaded_backend.count(backend_id) || 0 == _loaded_lib_handle.count(backend_id)) {
QNN_LOG_WARN(
"library %s is loaded but loaded backend count=%zu, "
"loaded lib_handle count=%zu\n",
backend_lib_path.c_str(), _loaded_backend.count(backend_id),
_loaded_lib_handle.count(backend_id));
backend_lib_path.c_str(), _loaded_backend.count(backend_id), _loaded_lib_handle.count(backend_id));
return 3;
}
@ -219,27 +195,22 @@ namespace qnn {
// NPU backend not work on Qualcomm SoC equipped low-end phone
QNN_LOG_WARN("why failed to initialize qnn log\n");
return 4;
}
else {
} else {
QNN_LOG_DEBUG("initialize qnn log successfully\n");
}
std::vector<const QnnBackend_Config_t *> temp_backend_config;
_qnn_interface.qnn_backend_create(
_qnn_log_handle,
temp_backend_config.empty() ? nullptr : temp_backend_config.data(),
&_qnn_backend_handle);
_qnn_log_handle, temp_backend_config.empty() ? nullptr : temp_backend_config.data(), &_qnn_backend_handle);
if (nullptr == _qnn_backend_handle) {
QNN_LOG_WARN("why failed to initialize qnn backend\n");
return 5;
}
else {
} else {
QNN_LOG_DEBUG("initialize qnn backend successfully\n");
}
if (nullptr != _qnn_raw_interface.propertyHasCapability) {
Qnn_ErrorHandle_t qnn_status =
_qnn_raw_interface.propertyHasCapability(QNN_PROPERTY_GROUP_DEVICE);
Qnn_ErrorHandle_t qnn_status = _qnn_raw_interface.propertyHasCapability(QNN_PROPERTY_GROUP_DEVICE);
if (QNN_PROPERTY_NOT_SUPPORTED == qnn_status) {
QNN_LOG_WARN("device property is not supported\n");
}
@ -256,15 +227,16 @@ namespace qnn {
QnnDevice_HardwareDeviceInfo_t *infos = p_info->v1.hwDevices;
QnnHtpDevice_OnChipDeviceInfoExtension_t chipinfo = {};
for (int i = 0; i < p_info->v1.numHwDevices; i++) {
QNN_LOG_INFO("deviceID:%d, deviceType:%d, numCores %d", infos[i].v1.deviceId,
infos[i].v1.deviceType, infos[i].v1.numCores);
QNN_LOG_INFO("deviceID:%d, deviceType:%d, numCores %d", infos[i].v1.deviceId, infos[i].v1.deviceType,
infos[i].v1.numCores);
QnnDevice_DeviceInfoExtension_t devinfo = infos[i].v1.deviceInfoExtension;
chipinfo = devinfo->onChipDevice;
QnnHtpDevice_Arch_t htp_arch = chipinfo.arch;
QNN_LOG_INFO("htp_type:%d(%s)", devinfo->devType, (devinfo->devType == QNN_HTP_DEVICE_TYPE_ON_CHIP) ? "ON_CHIP" : "");
QNN_LOG_INFO("qualcomm soc_model:%d(%s), htp_arch:%d(%s), vtcm_size:%d MB",
chipinfo.socModel, qnn::get_chipset_desc(chipinfo.socModel),
htp_arch, qnn::get_htparch_desc(htp_arch), chipinfo.vtcmSize);
QNN_LOG_INFO("htp_type:%d(%s)", devinfo->devType,
(devinfo->devType == QNN_HTP_DEVICE_TYPE_ON_CHIP) ? "ON_CHIP" : "");
QNN_LOG_INFO("qualcomm soc_model:%d(%s), htp_arch:%d(%s), vtcm_size:%d MB", chipinfo.socModel,
qnn::get_chipset_desc(chipinfo.socModel), htp_arch, qnn::get_htparch_desc(htp_arch),
chipinfo.vtcmSize);
_soc_info = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize };
}
_qnn_raw_interface.deviceFreePlatformInfo(nullptr, p_info);
@ -286,15 +258,12 @@ namespace qnn {
const QnnDevice_Config_t *p_deviceconfig[] = { &soc_devconfig, &arch_devconfig, nullptr };
qnn_status = _qnn_raw_interface.deviceCreate(_qnn_log_handle, p_deviceconfig, &_qnn_device_handle);
}
else {
} else {
qnn_status = _qnn_raw_interface.deviceCreate(_qnn_log_handle, nullptr, &_qnn_device_handle);
}
if (QNN_SUCCESS != qnn_status &&
QNN_DEVICE_ERROR_UNSUPPORTED_FEATURE != qnn_status) {
if (QNN_SUCCESS != qnn_status && QNN_DEVICE_ERROR_UNSUPPORTED_FEATURE != qnn_status) {
QNN_LOG_WARN("failed to create QNN device\n");
}
else {
} else {
QNN_LOG_INFO("create QNN device successfully\n");
}
@ -302,27 +271,21 @@ namespace qnn {
QNN_LOG_INFO("profiling turned on; level = %d", _profile_level);
if (qnn::sdk_profile_level::profile_basic == _profile_level) {
QNN_LOG_INFO("basic profiling requested. creating Qnn Profile object\n");
if (QNN_PROFILE_NO_ERROR !=
_qnn_raw_interface.profileCreate(_qnn_backend_handle,
QNN_PROFILE_LEVEL_BASIC,
&_qnn_profile_handle)) {
if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate(
_qnn_backend_handle, QNN_PROFILE_LEVEL_BASIC, &_qnn_profile_handle)) {
QNN_LOG_WARN("unable to create profile handle in the backend\n");
return 6;
}
else {
} else {
QNN_LOG_DEBUG("initialize qnn profile successfully\n");
}
}
else if (qnn::sdk_profile_level::profile_detail == _profile_level) {
} else if (qnn::sdk_profile_level::profile_detail == _profile_level) {
QNN_LOG_INFO("detailed profiling requested. Creating Qnn Profile object\n");
if (QNN_PROFILE_NO_ERROR !=
_qnn_raw_interface.profileCreate(_qnn_backend_handle,
if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate(_qnn_backend_handle,
QNN_PROFILE_LEVEL_DETAILED,
&_qnn_profile_handle)) {
QNN_LOG_WARN("unable to create profile handle in the backend\n");
return 7;
}
else {
} else {
QNN_LOG_DEBUG("initialize qnn profile successfully\n");
}
}
@ -332,23 +295,16 @@ namespace qnn {
if (nullptr == _rpc_lib_handle) {
QNN_LOG_WARN("failed to load qualcomm's rpc lib, error:%s\n", dlerror());
return 8;
}
else {
} else {
QNN_LOG_DEBUG("load rpcmem lib successfully\n");
set_rpcmem_initialized(true);
}
_pfn_rpc_mem_init = reinterpret_cast<qnn::pfn_rpc_mem_init>(
dlsym(_rpc_lib_handle, "rpcmem_init"));
_pfn_rpc_mem_deinit = reinterpret_cast<qnn::pfn_rpc_mem_deinit>(
dlsym(_rpc_lib_handle, "rpcmem_deinit"));
_pfn_rpc_mem_alloc = reinterpret_cast<qnn::pfn_rpc_mem_alloc>(
dlsym(_rpc_lib_handle, "rpcmem_alloc"));
_pfn_rpc_mem_free = reinterpret_cast<qnn::pfn_rpc_mem_free>(
dlsym(_rpc_lib_handle, "rpcmem_free"));
_pfn_rpc_mem_to_fd = reinterpret_cast<qnn::pfn_rpc_mem_to_fd>(
dlsym(_rpc_lib_handle, "rpcmem_to_fd"));
if (nullptr == _pfn_rpc_mem_alloc || nullptr == _pfn_rpc_mem_free ||
nullptr == _pfn_rpc_mem_to_fd) {
_pfn_rpc_mem_init = reinterpret_cast<qnn::pfn_rpc_mem_init>(dlsym(_rpc_lib_handle, "rpcmem_init"));
_pfn_rpc_mem_deinit = reinterpret_cast<qnn::pfn_rpc_mem_deinit>(dlsym(_rpc_lib_handle, "rpcmem_deinit"));
_pfn_rpc_mem_alloc = reinterpret_cast<qnn::pfn_rpc_mem_alloc>(dlsym(_rpc_lib_handle, "rpcmem_alloc"));
_pfn_rpc_mem_free = reinterpret_cast<qnn::pfn_rpc_mem_free>(dlsym(_rpc_lib_handle, "rpcmem_free"));
_pfn_rpc_mem_to_fd = reinterpret_cast<qnn::pfn_rpc_mem_to_fd>(dlsym(_rpc_lib_handle, "rpcmem_to_fd"));
if (nullptr == _pfn_rpc_mem_alloc || nullptr == _pfn_rpc_mem_free || nullptr == _pfn_rpc_mem_to_fd) {
QNN_LOG_WARN("unable to access symbols in QNN RPC lib. dlerror(): %s", dlerror());
dlclose(_rpc_lib_handle);
return 9;
@ -363,15 +319,11 @@ namespace qnn {
qnn_context_config.priority = QNN_PRIORITY_DEFAULT;
const QnnContext_Config_t * context_configs[] = {&qnn_context_config, nullptr};
*/
_qnn_interface.qnn_context_create(
_qnn_backend_handle, _qnn_device_handle,
nullptr,
&_qnn_context_handle);
_qnn_interface.qnn_context_create(_qnn_backend_handle, _qnn_device_handle, nullptr, &_qnn_context_handle);
if (nullptr == _qnn_context_handle) {
QNN_LOG_WARN("why failed to initialize qnn context\n");
return 10;
}
else {
} else {
QNN_LOG_DEBUG("initialize qnn context successfully\n");
}
@ -383,21 +335,17 @@ namespace qnn {
size_t probe_slots[] = { 1024, 1536, 2048 - 48, 2048 };
size_t probe_counts = sizeof(probe_slots) / sizeof(size_t);
for (size_t idx = 0; idx < probe_counts; idx++) {
rpc_buffer = static_cast<uint8_t*>(alloc_rpcmem(
probe_slots[idx] * size_in_mb, 4));
rpc_buffer = static_cast<uint8_t *>(alloc_rpcmem(probe_slots[idx] * size_in_mb, 4));
if (nullptr == rpc_buffer) {
QNN_LOG_INFO("alloc rpcmem %d (MB) failure, %s\n",
probe_slots[idx], strerror(errno));
QNN_LOG_INFO("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], strerror(errno));
break;
}
else {
} else {
candidate_size = probe_slots[idx];
free_rpcmem(rpc_buffer);
rpc_buffer = nullptr;
}
}
if (candidate_size > _rpcmem_capacity)
_rpcmem_capacity = candidate_size;
if (candidate_size > _rpcmem_capacity) _rpcmem_capacity = candidate_size;
QNN_LOG_INFO("capacity of QNN rpc ion memory is about %d MB\n", _rpcmem_capacity);
if (0 != init_htp_perfinfra()) {
@ -425,8 +373,7 @@ namespace qnn {
if (dlclose(_rpc_lib_handle) != 0) {
QNN_LOG_WARN("failed to unload qualcomm's rpc lib, error:%s\n", dlerror());
}
else {
} else {
QNN_LOG_DEBUG("succeed to close rpcmem lib\n");
}
@ -435,11 +382,9 @@ namespace qnn {
}
if (nullptr != _qnn_context_handle) {
error = _qnn_interface.qnn_context_free(_qnn_context_handle,
_qnn_profile_handle);
error = _qnn_interface.qnn_context_free(_qnn_context_handle, _qnn_profile_handle);
if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to free QNN context_handle: ID %u, error %d\n",
_qnn_interface.get_backend_id(),
QNN_LOG_WARN("failed to free QNN context_handle: ID %u, error %d\n", _qnn_interface.get_backend_id(),
QNN_GET_ERROR_CODE(error));
}
_qnn_context_handle = nullptr;
@ -448,8 +393,7 @@ namespace qnn {
if (nullptr != _qnn_profile_handle) {
error = _qnn_interface.qnn_profile_free(_qnn_profile_handle);
if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to free QNN profile_handle: ID %u, error %d\n",
_qnn_interface.get_backend_id(),
QNN_LOG_WARN("failed to free QNN profile_handle: ID %u, error %d\n", _qnn_interface.get_backend_id(),
QNN_GET_ERROR_CODE(error));
}
_qnn_profile_handle = nullptr;
@ -458,8 +402,7 @@ namespace qnn {
if (nullptr != _qnn_device_handle) {
error = _qnn_interface.qnn_device_free(_qnn_device_handle);
if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to free QNN device_handle: ID %u, error %d\n",
_qnn_interface.get_backend_id(),
QNN_LOG_WARN("failed to free QNN device_handle: ID %u, error %d\n", _qnn_interface.get_backend_id(),
QNN_GET_ERROR_CODE(error));
}
_qnn_device_handle = nullptr;
@ -468,8 +411,7 @@ namespace qnn {
if (nullptr != _qnn_backend_handle) {
error = _qnn_interface.qnn_backend_free(_qnn_backend_handle);
if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to free QNN backend_handle: ID %u, error %d\n",
_qnn_interface.get_backend_id(),
QNN_LOG_WARN("failed to free QNN backend_handle: ID %u, error %d\n", _qnn_interface.get_backend_id(),
QNN_GET_ERROR_CODE(error));
}
_qnn_backend_handle = nullptr;
@ -478,8 +420,7 @@ namespace qnn {
if (nullptr != _qnn_log_handle) {
error = _qnn_interface.qnn_log_free(_qnn_log_handle);
if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to free QNN log_handle: ID %u, error %d\n",
_qnn_interface.get_backend_id(),
QNN_LOG_WARN("failed to free QNN log_handle: ID %u, error %d\n", _qnn_interface.get_backend_id(),
QNN_GET_ERROR_CODE(error));
}
_qnn_log_handle = nullptr;
@ -492,62 +433,6 @@ namespace qnn {
return ret_status;
}
//TODO:keep it for further usage of offload the entire cgraph to a single QNN DAG directly
// which was used in Qualcomm's dedicated AI technology
#if 0
int init_qnn_graph(const char* graph_name, bool debug,
uint8_t do_node_validation = true,
const QnnGraph_Config_t** graph_configs = nullptr) {
int result = 0;
if (nullptr == graph_name) {
QNN_LOG_WARN("graph name is null\n");
return 1;
}
if (!_graph_name.empty()) {
QNN_LOG_WARN("qnn model for graph %s already initialized\n", graph_name);
return 2;
}
if (!do_node_validation) {
QNN_LOG_WARN("node validation disabled, backend will not perform op "
"validation prior to adding node\n");
}
_graph_name = graph_name;
_debug_tensor = debug;
_do_node_validations = do_node_validation;
result = _qnn_raw_interface.graphCreate(_qnn_context_handle, graph_name,
graph_configs, &_qnn_graph_handle);
if (result != QNN_GRAPH_NO_ERROR || nullptr == _qnn_graph_handle) {
QNN_LOG_WARN("failed to create graph in qnn context\n");
return 3;
}
else {
QNN_LOG_INFO("succeed to create graph %s, %p\n", graph_name, _qnn_graph_handle);
}
return 0;
}
int finalize_qnn_graph() {
if (nullptr != _qnn_graph_handle) {
if (_qnn_raw_interface.graphFinalize(_qnn_graph_handle,
_qnn_profile_handle,
nullptr) != QNN_GRAPH_NO_ERROR) {
QNN_LOG_WARN("finalizing graph failure\n");
}
}
else {
QNN_LOG_DEBUG("qnn graph handle is null\n");
}
return 0;
}
#endif
const qnn_interface &get_qnn_interface() {
if (!_qnn_interface.is_loaded()) {
QNN_LOG_WARN("pls check why _qnn_interface is not loaded\n");
@ -571,25 +456,15 @@ namespace qnn {
const Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; }
const Qnn_ProfileHandle_t get_qnn_profile_handle() {
return _qnn_profile_handle;
}
const Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; }
const Qnn_DeviceHandle_t get_qnn_device_handle() {
return _qnn_device_handle;
}
const Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; }
const Qnn_BackendHandle_t get_qnn_backend_handle() {
return _qnn_backend_handle;
}
const Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; }
const Qnn_ContextHandle_t get_qnn_context_handle() {
return _qnn_context_handle;
}
const Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; }
const QnnSystemContext_Handle_t get_qnn_system_handle() {
return _qnn_system_handle;
}
const QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; }
const Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; }
@ -599,8 +474,7 @@ namespace qnn {
if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to get qnn device infra\n");
return 1;
}
else {
} else {
QNN_LOG_INFO("HTP backend perf_infrastructure creation ok\n");
}
@ -612,8 +486,7 @@ namespace qnn {
htp_perfinfra->createPowerConfigId(device_id, core_id, &power_configid);
if (htp_infra->infraType != QNN_HTP_DEVICE_INFRASTRUCTURE_TYPE_PERF) {
QNN_LOG_INFO("HTP infra type = %d, which is not perf infra type", htp_infra->infraType);
}
else {
} else {
QNN_LOG_INFO("HTP infra type = %d, which is perf infra type\n", htp_infra->infraType);
}
_qnn_htp_perfinfra = htp_perfinfra;
@ -636,21 +509,15 @@ namespace qnn {
// use rpc control latency recommended 100 us, refer hexagon sdk
rpc_control_latency.rpcControlLatencyConfig = 100;
const QnnHtpPerfInfrastructure_PowerConfig_t* power_configs[] = {
&rpc_polling_time,
&rpc_control_latency,
const QnnHtpPerfInfrastructure_PowerConfig_t *power_configs[] = { &rpc_polling_time, &rpc_control_latency,
nullptr };
Qnn_ErrorHandle_t qnn_status = _qnn_htp_perfinfra->setPowerConfig(
_qnn_power_configid,
power_configs);
Qnn_ErrorHandle_t qnn_status = _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs);
if (qnn_status != QNN_SUCCESS) {
QNN_LOG_WARN("set htp perf failed\n");
}
else {
} else {
QNN_LOG_INFO("set htp perf ok\n");
}
}
else {
} else {
QNN_LOG_WARN("can't set htp perf\n");
}
@ -671,41 +538,29 @@ namespace qnn {
power_config.dcvsV3Config.dcvsEnable = 0;
power_config.dcvsV3Config.contextId = _qnn_power_configid;
power_config.dcvsV3Config.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE;
power_config.dcvsV3Config.setSleepLatency =
1; // true to consider Latency parameter otherwise false
power_config.dcvsV3Config.setSleepLatency = 1; // true to consider Latency parameter otherwise false
power_config.dcvsV3Config.sleepLatency = 40;
power_config.dcvsV3Config.setBusParams =
1; // true to consider Bus parameter otherwise false
power_config.dcvsV3Config.setCoreParams =
1; // true to consider Core parameter otherwise false
power_config.dcvsV3Config.sleepDisable =
1; // true to consider sleep/LPM modes, false to enable
power_config.dcvsV3Config.setBusParams = 1; // true to consider Bus parameter otherwise false
power_config.dcvsV3Config.setCoreParams = 1; // true to consider Core parameter otherwise false
power_config.dcvsV3Config.sleepDisable = 1; // true to consider sleep/LPM modes, false to enable
power_config.dcvsV3Config.setSleepDisable =
1; // true to consider sleep disable/enable parameter otherwise false set sleep latency parameter
// set Bus Clock Parameters
power_config.dcvsV3Config.busVoltageCornerMin =
DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
power_config.dcvsV3Config.busVoltageCornerTarget =
DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
power_config.dcvsV3Config.busVoltageCornerMax =
DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
power_config.dcvsV3Config.busVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
power_config.dcvsV3Config.busVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
power_config.dcvsV3Config.busVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
// set Core Clock Parameters
power_config.dcvsV3Config.coreVoltageCornerMin =
DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
power_config.dcvsV3Config.coreVoltageCornerTarget =
DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
power_config.dcvsV3Config.coreVoltageCornerMax =
DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
power_config.dcvsV3Config.coreVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
power_config.dcvsV3Config.coreVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
power_config.dcvsV3Config.coreVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER;
// set power config with different performance parameters
const QnnHtpPerfInfrastructure_PowerConfig_t* power_configs[] = {
&power_config, nullptr };
const QnnHtpPerfInfrastructure_PowerConfig_t *power_configs[] = { &power_config, nullptr };
Qnn_ErrorHandle_t qnn_status = QNN_SUCCESS;
qnn_status = _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs);
if (qnn_status != QNN_SUCCESS) {
QNN_LOG_WARN("set htp high performance mode failed\n");
}
else {
} else {
QNN_LOG_INFO("set htp high performance mode ok\n");
}
@ -716,15 +571,11 @@ namespace qnn {
bool is_rpcmem_initialized() { return _rpcmem_initialized; }
void set_rpcmem_initialized(bool initialized) {
_rpcmem_initialized = initialized;
}
void set_rpcmem_initialized(bool initialized) { _rpcmem_initialized = initialized; }
size_t get_rpcmem_capacity() { return _rpcmem_capacity; }
bool is_rpcmem_registered(Qnn_MemHandle_t handle) {
return _qnn_mem_set.count(handle) != 0U;
}
bool is_rpcmem_registered(Qnn_MemHandle_t handle) { return _qnn_mem_set.count(handle) != 0U; }
void *alloc_rpcmem(size_t bytes, size_t alignment) {
if (!_rpcmem_initialized) {
@ -733,17 +584,14 @@ namespace qnn {
}
auto allocate_bytes = static_cast<int32_t>(bytes + alignment);
void* buf = _pfn_rpc_mem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS,
allocate_bytes);
void *buf = _pfn_rpc_mem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, allocate_bytes);
if (buf == nullptr) {
QNN_LOG_WARN("failed to allocate rpc memory\n");
return nullptr;
}
auto aligned_buf = reinterpret_cast<void*>(
qnn::align_to(alignment, reinterpret_cast<intptr_t>(buf)));
bool status =
_rpcmem_store_map.insert(std::pair<void*, void*>(aligned_buf, buf)).second;
auto aligned_buf = reinterpret_cast<void *>(qnn::align_to(alignment, reinterpret_cast<intptr_t>(buf)));
bool status = _rpcmem_store_map.insert(std::pair<void *, void *>(aligned_buf, buf)).second;
if (!status) {
QNN_LOG_WARN("failed to allocate rpc memory\n");
_pfn_rpc_mem_free(buf);
@ -755,11 +603,9 @@ namespace qnn {
void free_rpcmem(void *buf) {
if (!_rpcmem_initialized) {
QNN_LOG_WARN("rpc memory not initialized\n");
}
else if (0 == _rpcmem_store_map.count(buf)) {
} else if (0 == _rpcmem_store_map.count(buf)) {
QNN_LOG_WARN("no allocated tensor\n");
}
else {
} else {
_pfn_rpc_mem_free(_rpcmem_store_map[buf]);
_rpcmem_store_map.erase(buf);
}
@ -769,8 +615,7 @@ namespace qnn {
int32_t mem_fd = -1;
if (!is_rpcmem_initialized()) {
QNN_LOG_WARN("rpc memory not initialized\n");
}
else {
} else {
mem_fd = _pfn_rpc_mem_to_fd(buf);
}
@ -794,8 +639,7 @@ namespace qnn {
}
if (is_rpcmem_registered((QNN_VER_PTR(*p_tensor)->memHandle))) {
QNN_LOG_WARN("tensor %s has been registered shared memory\n",
(QNN_VER_PTR(*p_tensor)->name));
QNN_LOG_WARN("tensor %s has been registered shared memory\n", (QNN_VER_PTR(*p_tensor)->name));
return 4;
}
@ -805,8 +649,7 @@ namespace qnn {
return 5;
}
QNN_LOG_INFO("mem_fd %d\n", mem_fd);
Qnn_MemDescriptor_t descriptor = { {QNN_VER_PTR(*p_tensor)->rank,
QNN_VER_PTR(*p_tensor)->dimensions,
Qnn_MemDescriptor_t descriptor = { { QNN_VER_PTR(*p_tensor)->rank, QNN_VER_PTR(*p_tensor)->dimensions,
nullptr },
QNN_VER_PTR(*p_tensor)->dataType,
QNN_MEM_TYPE_ION,
@ -816,13 +659,11 @@ namespace qnn {
error = _qnn_interface.qnn_mem_register(_qnn_context_handle, &descriptor,
/*numDescriptors=*/1, &handle);
if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to register shared memory, error %d, %s\n",
QNN_GET_ERROR_CODE(error), strerror(error));
QNN_LOG_WARN("failed to register shared memory, error %d, %s\n", QNN_GET_ERROR_CODE(error),
strerror(error));
return 6;
}
else {
QNN_LOG_INFO("tensor %s successfully register shared memory\n",
(QNN_VER_PTR(*p_tensor)->name));
} else {
QNN_LOG_INFO("tensor %s successfully register shared memory\n", (QNN_VER_PTR(*p_tensor)->name));
}
QNN_VER_PTR(*p_tensor)->memHandle = handle;
_qnn_mem_set.insert((std::pair<void *, Qnn_MemHandle_t>(p_data, handle)));
@ -831,8 +672,7 @@ namespace qnn {
}
void *get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle) {
for (std::unordered_map<void*, Qnn_MemHandle_t>::iterator it = _qnn_mem_set.begin();
it != _qnn_mem_set.end();
for (std::unordered_map<void *, Qnn_MemHandle_t>::iterator it = _qnn_mem_set.begin(); it != _qnn_mem_set.end();
it++) {
Qnn_MemHandle_t mem_handle = it->second;
if (it->second == mem_handle) {
@ -850,28 +690,23 @@ namespace qnn {
QNN_LOG_WARN("no rpcmem registered\n");
}
for (std::unordered_map<void*, Qnn_MemHandle_t>::iterator it = _qnn_mem_set.begin();
it != _qnn_mem_set.end();
for (std::unordered_map<void *, Qnn_MemHandle_t>::iterator it = _qnn_mem_set.begin(); it != _qnn_mem_set.end();
it++) {
Qnn_MemHandle_t mem_handle = it->second;
error = _qnn_interface.qnn_mem_de_register(&mem_handle, 1);
if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to unregister shared memory, error %d\n",
QNN_GET_ERROR_CODE(error));
QNN_LOG_WARN("failed to unregister shared memory, error %d\n", QNN_GET_ERROR_CODE(error));
}
}
_qnn_mem_set.clear();
}
bool is_rpcmem_allocated(void* buf) {
return _qnn_mem_set.count(buf) != 0U;
}
bool is_rpcmem_allocated(void *buf) { return _qnn_mem_set.count(buf) != 0U; }
const qnn::qcom_socinfo &get_soc_info() { return _soc_info; }
public:
std::map<std::string,
std::tuple<Qnn_GraphHandle_t, Qnn_Tensor_t*, Qnn_Tensor_t*, Qnn_Tensor_t*>> _qnn_graph_map;
std::map<std::string, std::tuple<Qnn_GraphHandle_t, Qnn_Tensor_t *, Qnn_Tensor_t *, Qnn_Tensor_t *>> _qnn_graph_map;
private:
int load_system() {
@ -882,18 +717,14 @@ namespace qnn {
_system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL);
if (nullptr == _system_lib_handle) {
QNN_LOG_WARN("can not open QNN library %s, error: %s\n",
system_lib_path.c_str(), dlerror());
QNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror());
return 1;
}
auto* get_providers =
reinterpret_cast<qnn::pfn_qnnsysteminterface_getproviders*>(
auto *get_providers = reinterpret_cast<qnn::pfn_qnnsysteminterface_getproviders *>(
dlsym(_system_lib_handle, "QnnSystemInterface_getProviders"));
if (nullptr == get_providers) {
QNN_LOG_WARN(
"can not load QNN symbol QnnSystemInterface_getProviders: %s\n",
dlerror());
QNN_LOG_WARN("can not load QNN symbol QnnSystemInterface_getProviders: %s\n", dlerror());
return 2;
}
@ -901,14 +732,12 @@ namespace qnn {
const QnnSystemInterface_t **provider_list = nullptr;
error = get_providers(&provider_list, &num_providers);
if (error != QNN_SUCCESS) {
QNN_LOG_WARN("failed to get providers, error %d\n",
QNN_GET_ERROR_CODE(error));
QNN_LOG_WARN("failed to get providers, error %d\n", QNN_GET_ERROR_CODE(error));
return 3;
}
if (num_providers != _required_num_providers) {
QNN_LOG_WARN("providers is %d instead of required %d\n", num_providers,
_required_num_providers);
QNN_LOG_WARN("providers is %d instead of required %d\n", num_providers, _required_num_providers);
return 4;
}
@ -920,21 +749,17 @@ namespace qnn {
QNN_SYSTEM_INTERFACE_VER_TYPE qnn_system_interface;
bool found_valid_system_interface = false;
for (size_t idx = 0; idx < num_providers; idx++) {
if (QNN_SYSTEM_API_VERSION_MAJOR ==
provider_list[idx]->systemApiVersion.major &&
QNN_SYSTEM_API_VERSION_MINOR <=
provider_list[idx]->systemApiVersion.minor) {
if (QNN_SYSTEM_API_VERSION_MAJOR == provider_list[idx]->systemApiVersion.major &&
QNN_SYSTEM_API_VERSION_MINOR <= provider_list[idx]->systemApiVersion.minor) {
found_valid_system_interface = true;
qnn_system_interface =
provider_list[idx]->QNN_SYSTEM_INTERFACE_VER_NAME;
qnn_system_interface = provider_list[idx]->QNN_SYSTEM_INTERFACE_VER_NAME;
break;
}
}
if (!found_valid_system_interface) {
QNN_LOG_WARN("unable to find a valid qnn system interface\n");
return 6;
}
else {
} else {
QNN_LOG_INFO("find a valid qnn system interface\n");
}
set_qnn_raw_system_interface(qnn_system_interface);
@ -944,8 +769,7 @@ namespace qnn {
_qnn_interface.qnn_system_context_create(&_qnn_system_handle);
if (nullptr == _qnn_system_handle) {
QNN_LOG_WARN("can not create QNN system contenxt\n");
}
else {
} else {
QNN_LOG_INFO("initialize qnn system successfully\n");
}
@ -989,8 +813,7 @@ namespace qnn {
return 1;
}
auto get_providers =
qnn::load_qnn_functionpointers<qnn::pfn_qnninterface_getproviders*>(
auto get_providers = qnn::load_qnn_functionpointers<qnn::pfn_qnninterface_getproviders *>(
lib_handle, "QnnInterface_getProviders");
if (nullptr == get_providers) {
QNN_LOG_WARN("can not load symbol QnnInterface_getProviders : %s", dlerror());
@ -1017,10 +840,8 @@ namespace qnn {
bool found_valid_interface = false;
QNN_INTERFACE_VER_TYPE qnn_interface;
for (size_t idx = 0; idx < num_providers; idx++) {
if (QNN_API_VERSION_MAJOR ==
provider_list[idx]->apiVersion.coreApiVersion.major &&
QNN_API_VERSION_MINOR <=
provider_list[idx]->apiVersion.coreApiVersion.minor) {
if (QNN_API_VERSION_MAJOR == provider_list[idx]->apiVersion.coreApiVersion.major &&
QNN_API_VERSION_MINOR <= provider_list[idx]->apiVersion.coreApiVersion.minor) {
found_valid_interface = true;
qnn_interface = provider_list[idx]->QNN_INTERFACE_VER_NAME;
break;
@ -1030,8 +851,7 @@ namespace qnn {
if (!found_valid_interface) {
QNN_LOG_WARN("unable to find a valid qnn interface\n");
return 6;
}
else {
} else {
QNN_LOG_INFO("find a valid qnn interface\n");
}
set_qnn_raw_interface(qnn_interface);
@ -1071,9 +891,7 @@ namespace qnn {
return 0;
}
void set_qnn_raw_interface(QNN_INTERFACE_VER_TYPE& raw_interface) {
_qnn_raw_interface = raw_interface;
}
void set_qnn_raw_interface(QNN_INTERFACE_VER_TYPE &raw_interface) { _qnn_raw_interface = raw_interface; }
void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE &raw_interface) {
_qnn_raw_system_interface = raw_interface;
@ -1140,4 +958,4 @@ namespace qnn {
qnn::qcom_socinfo _soc_info = {};
};
}
} // namespace qnn

77
ggml/src/ggml-qnn/tensor.hpp
View file

@ -1,23 +1,21 @@
#pragma once
#include "ggml-qnn.h"
#include "QnnTensor.h"
#include "System/QnnSystemInterface.h"
#include "ggml-qnn.h"
#include "backend.hpp"
#include "qnn.hpp"
namespace qnn {
template <Qnn_TensorType_t _tensorType> class ggml_qnn_tensor_readwrite {
template <Qnn_TensorType_t _tensorType>
class ggml_qnn_tensor_readwrite {
public:
ggml_qnn_tensor_readwrite(const ggml_tensor* tensor,
Qnn_GraphHandle_t graph_handle,
ggml_backend_qnn_context* ctx)
: _tensor(tensor),
_qnn_tensor(reinterpret_cast<Qnn_Tensor_t*>(tensor->extra)),
_context(ctx) {
explicit ggml_qnn_tensor_readwrite(const ggml_tensor *tensor, Qnn_GraphHandle_t graph_handle,
ggml_backend_qnn_context *ctx) :
_tensor(tensor), _qnn_tensor(reinterpret_cast<Qnn_Tensor_t *>(tensor->extra)), _context(ctx) {
_old_dimensions = QNN_VER_PTR(*_qnn_tensor)->dimensions;
const auto qnn_data_type = datatype_from_ggml_datatype(tensor->type);
const bool is_npu = ctx->device == QNN_BACKEND_NPU;
@ -27,12 +25,10 @@ namespace qnn {
QNN_VER_PTR(*_qnn_tensor)->clientBuf = { .data = nullptr, .dataSize = 0 };
}
auto err =
ctx->raw_interface.tensorCreateGraphTensor(graph_handle, _qnn_tensor);
auto err = ctx->raw_interface.tensorCreateGraphTensor(graph_handle, _qnn_tensor);
if (err != QNN_SUCCESS) {
QNN_LOG_INFO("error = %d\n", err);
QNN_LOG_DEBUG("tensor%p name %s", _qnn_tensor,
QNN_TENSOR_GET_NAME(*_qnn_tensor));
QNN_LOG_DEBUG("tensor%p name %s", _qnn_tensor, QNN_TENSOR_GET_NAME(*_qnn_tensor));
_context = nullptr;
return;
}
@ -47,35 +43,29 @@ namespace qnn {
if (is_npu) {
auto *instance = ctx->instance;
uint8_t* qnn_buffer = static_cast<uint8_t*>(
instance->alloc_rpcmem(ggml_nbytes(tensor), alignof(void*)));
uint8_t *qnn_buffer = static_cast<uint8_t *>(instance->alloc_rpcmem(ggml_nbytes(tensor), alignof(void *)));
if (!qnn_buffer) {
QNN_LOG_WARN("alloc rpcmem failure, %s\n", strerror(errno));
QNN_LOG_DEBUG("tensor%p name %s", _qnn_tensor,
QNN_TENSOR_GET_NAME(*_qnn_tensor));
QNN_LOG_DEBUG("tensor%p name %s", _qnn_tensor, QNN_TENSOR_GET_NAME(*_qnn_tensor));
_context = nullptr;
// No free for _qnn_tensor, because it's not registered.
return;
}
else {
} else {
QNN_LOG_INFO("alloc rpcmem successfully\n");
}
instance->register_rpcmem(qnn_buffer, _qnn_tensor);
if (_tensorType == QNN_TENSOR_TYPE_APP_WRITE ||
_tensorType == QNN_TENSOR_TYPE_APP_READWRITE) {
if (_tensorType == QNN_TENSOR_TYPE_APP_WRITE || _tensorType == QNN_TENSOR_TYPE_APP_READWRITE) {
memcpy(qnn_buffer, tensor->data, ggml_nbytes(tensor));
}
}
else {
QNN_VER_PTR(*_qnn_tensor)->clientBuf = {
tensor->data, get_ggml_tensor_data_size(tensor) };
} else {
QNN_VER_PTR(*_qnn_tensor)->clientBuf = { tensor->data, get_ggml_tensor_data_size(tensor) };
}
}
ggml_qnn_tensor_readwrite(const ggml_tensor* tensor, Qnn_Tensor_t* qnn_tensor,
ggml_backend_qnn_context* ctx)
: _tensor(tensor), _qnn_tensor(qnn_tensor), _context(ctx) {
explicit ggml_qnn_tensor_readwrite(const ggml_tensor *tensor, Qnn_Tensor_t *qnn_tensor,
ggml_backend_qnn_context *ctx) :
_tensor(tensor), _qnn_tensor(qnn_tensor), _context(ctx) {
_old_dimensions = QNN_VER_PTR(*_qnn_tensor)->dimensions;
const auto qnn_data_type = qnn::datatype_from_ggml_datatype(tensor->type);
const bool is_npu = ctx->device == QNN_BACKEND_NPU;
@ -90,32 +80,25 @@ namespace qnn {
if (is_npu) {
uint8_t *qnn_buffer =
static_cast<uint8_t*>(ctx->instance->get_rpcmem_from_memhandle(
QNN_VER_PTR(*_qnn_tensor)->memHandle));
static_cast<uint8_t *>(ctx->instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*_qnn_tensor)->memHandle));
if (qnn_buffer) {
memcpy(qnn_buffer, tensor->data, ggml_nbytes(tensor));
}
else {
} else {
QNN_LOG_WARN("can't find rpcmem from qnn mem handle\n");
QNN_LOG_DEBUG("tensor%p name %s", _qnn_tensor,
QNN_TENSOR_GET_NAME(*_qnn_tensor));
QNN_LOG_DEBUG("tensor%p name %s", _qnn_tensor, QNN_TENSOR_GET_NAME(*_qnn_tensor));
_context = nullptr;
return;
}
}
else {
QNN_VER_PTR(*_qnn_tensor)->clientBuf = {
tensor->data, get_ggml_tensor_data_size(tensor) };
} else {
QNN_VER_PTR(*_qnn_tensor)->clientBuf = { tensor->data, get_ggml_tensor_data_size(tensor) };
}
}
~ggml_qnn_tensor_readwrite() {
if ((_tensorType == QNN_TENSOR_TYPE_APP_READWRITE ||
_tensorType == QNN_TENSOR_TYPE_APP_READ) &&
_context && _context->device == QNN_BACKEND_NPU) {
uint8_t* qnn_buffer =
static_cast<uint8_t*>(_context->instance->get_rpcmem_from_memhandle(
QNN_VER_PTR(*_qnn_tensor)->memHandle));
if ((_tensorType == QNN_TENSOR_TYPE_APP_READWRITE || _tensorType == QNN_TENSOR_TYPE_APP_READ) && _context &&
_context->device == QNN_BACKEND_NPU) {
uint8_t *qnn_buffer = static_cast<uint8_t *>(
_context->instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*_qnn_tensor)->memHandle));
memcpy(_tensor->data, qnn_buffer, ggml_nbytes(_tensor));
}
@ -138,9 +121,7 @@ namespace qnn {
void operator=(ggml_qnn_tensor_readwrite &&) = delete;
};
using ggml_qnn_tensor_output =
ggml_qnn_tensor_readwrite<QNN_TENSOR_TYPE_APP_READ>;
using ggml_qnn_tensor_input =
ggml_qnn_tensor_readwrite<QNN_TENSOR_TYPE_APP_WRITE>;
using ggml_qnn_tensor_output = ggml_qnn_tensor_readwrite<QNN_TENSOR_TYPE_APP_READ>;
using ggml_qnn_tensor_input = ggml_qnn_tensor_readwrite<QNN_TENSOR_TYPE_APP_WRITE>;
} // namespace qnn

8
ggml/src/ggml-qnn/utils.cpp
View file

@ -2,6 +2,7 @@
#include "utils.hpp"
#include "ggml-qnn.h"
#include "qnn-types.hpp"
namespace qnn {
@ -26,7 +27,6 @@ namespace qnn {
return QNN_DATATYPE_UNDEFINED;
}
uint32_t get_ggml_tensor_rank(const ggml_tensor *tensor) {
uint32_t rank = 0;
for (int i = 0; i < GGML_MAX_DIMS; i++) {
@ -37,7 +37,6 @@ namespace qnn {
return rank;
}
const char *get_backend_name(int n_backend_type) {
switch (n_backend_type) {
case QNN_BACKEND_CPU:
@ -86,8 +85,7 @@ namespace qnn {
intptr_t align_to(size_t alignment, intptr_t offset) {
return offset % alignment == 0
? offset
: offset + (static_cast<intptr_t>(alignment) -
offset % static_cast<intptr_t>(alignment));
: offset + (static_cast<intptr_t>(alignment) - offset % static_cast<intptr_t>(alignment));
}
uint32_t get_ggml_tensor_data_size(const ggml_tensor *tensor) {
@ -123,4 +121,4 @@ namespace qnn {
return nullptr;
}
}
} // namespace qnn

35
ggml/src/ggml-qnn/utils.hpp
View file

@ -1,16 +1,16 @@
#pragma once
#include <stdint.h>
#include <stddef.h>
#include <inttypes.h>
#include <dlfcn.h>
#include <fcntl.h>
#include <string>
#include <inttypes.h>
#include <stddef.h>
#include <stdint.h>
#include "QnnTypes.h"
#include <string>
#include "ggml.h"
#include "QnnTypes.h"
#include "logger.hpp"
namespace qnn {
@ -25,15 +25,15 @@ namespace qnn {
const char *opname_from_ggmlop(enum ggml_op ggmlop);
template <typename Fn> Fn load_qnn_functionpointers(void* handle, const char* function_name) {
template <typename Fn>
Fn load_qnn_functionpointers(void *handle, const char *function_name) {
return reinterpret_cast<Fn>(dlsym(handle, function_name));
}
inline int validate_tensor_version(Qnn_Tensor_t tensor) {
if (tensor.version != QNN_TENSOR_VERSION_1) {
QNN_LOG_WARN(
"validate_tensor_version() tensor %s, got unsupported version %d\n",
tensor.v1.name, tensor.version);
QNN_LOG_WARN("validate_tensor_version() tensor %s, got unsupported version %d\n", tensor.v1.name,
tensor.version);
return 1;
}
return 0;
@ -61,24 +61,21 @@ namespace qnn {
return QNN_TENSOR_TYPE_UNDEFINED;
}
inline Qnn_TensorDataFormat_t
get_qnn_tensor_dataformat(const Qnn_Tensor_t& tensor) {
inline Qnn_TensorDataFormat_t get_qnn_tensor_dataformat(const Qnn_Tensor_t &tensor) {
if (tensor.version == QNN_TENSOR_VERSION_1) {
return tensor.v1.dataFormat;
}
return QNN_TENSOR_DATA_FORMAT_FLAT_BUFFER;
}
inline Qnn_DataType_t
get_qnn_tensor_datatype(const Qnn_Tensor_t& tensor) {
inline Qnn_DataType_t get_qnn_tensor_datatype(const Qnn_Tensor_t &tensor) {
if (tensor.version == QNN_TENSOR_VERSION_1) {
return tensor.v1.dataType;
}
return QNN_DATATYPE_UNDEFINED;
}
inline Qnn_QuantizeParams_t
get_qnn_tensor_quantparams(const Qnn_Tensor_t& tensor) {
inline Qnn_QuantizeParams_t get_qnn_tensor_quantparams(const Qnn_Tensor_t &tensor) {
if (tensor.version == QNN_TENSOR_VERSION_1) {
return tensor.v1.quantizeParams;
}
@ -172,7 +169,6 @@ namespace qnn {
}
}
#if ENABLE_QNNBACKEND_PERF
class qnn_perf {
public:
@ -181,9 +177,7 @@ namespace qnn {
qnn_perf(const qnn_perf &) = delete;
qnn_perf &operator=(const qnn_perf &) = delete;
void start() {
_begin_time = ggml_time_us();
}
void start() { _begin_time = ggml_time_us(); }
void info() {
_end_time = ggml_time_us();
@ -210,8 +204,7 @@ namespace qnn {
};
#endif
}
} // namespace qnn
#define VALIDATE(value, status) \
do { \