vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

move graph map to backend object

Browse source
This commit is contained in:
hongruichen 2024-07-05 11:56:31 +08:00
parent ca0d999c2a
commit 4b2ee61f62
4 changed files with 16 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

15
ggml/src/ggml-qnn.cpp
View file

@ -529,18 +529,13 @@ GGML_CALL static void ggml_backend_qnn_free(ggml_backend_t backend) {
auto *instance = g_qnn_mgr[ctx->device].instance; auto *instance = g_qnn_mgr[ctx->device].instance;
if (instance != nullptr) { if (instance != nullptr) {
// TODO: this should be done inside the destructor for (const auto &graph_item: ctx->qnn_graph_map) {
std::map<std::string, Qnn_GraphHandle_t graph_handle = std::get<0>(graph_item.second);
std::tuple<Qnn_GraphHandle_t, Qnn_Tensor_t *, Qnn_Tensor_t *,
Qnn_Tensor_t *>>::iterator graph_it;
for (graph_it = instance->_qnn_graph_map.begin();
graph_it != instance->_qnn_graph_map.end(); graph_it++) {
auto & graph_item = graph_it->second;
Qnn_GraphHandle_t & graph_handle = std::get<0>(graph_item);
GGML_UNUSED(graph_handle); GGML_UNUSED(graph_handle);
QNN_LOG_INFO("graph type:%s", graph_it->first.c_str()); QNN_LOG_INFO("graph type:%s", graph_item.first.c_str());
} }
instance->_qnn_graph_map.clear();
ctx->qnn_graph_map.clear();
instance->qnn_finalize(); instance->qnn_finalize();
delete instance; delete instance;

16
ggml/src/ggml-qnn/backend-ops.cpp
View file

@ -57,9 +57,9 @@ static void ggml_qnn_add(ggml_backend_qnn_context *ctx, const ggml_tensor *src0,
perf.start(); perf.start();
std::string map_entry(ggml_op_name(ggmlop)); std::string map_entry(ggml_op_name(ggmlop));
if (instance->_qnn_graph_map.find(map_entry) != instance->_qnn_graph_map.end()) { if (ctx->qnn_graph_map.find(map_entry) != ctx->qnn_graph_map.end()) {
graph_initialized = true; graph_initialized = true;
auto &graph_item = instance->_qnn_graph_map[map_entry]; auto &graph_item = ctx->qnn_graph_map[map_entry];
graph_handle = std::get<0>(graph_item); graph_handle = std::get<0>(graph_item);
} }
@ -157,9 +157,9 @@ static void ggml_qnn_add(ggml_backend_qnn_context *ctx, const ggml_tensor *src0,
auto graph_item = std::make_tuple(graph_handle, tensor_input0.get_qnn_tensor(), tensor_input1.get_qnn_tensor(), auto graph_item = std::make_tuple(graph_handle, tensor_input0.get_qnn_tensor(), tensor_input1.get_qnn_tensor(),
tensor_output.get_qnn_tensor()); tensor_output.get_qnn_tensor());
instance->_qnn_graph_map[map_entry] = graph_item; ctx->qnn_graph_map[map_entry] = graph_item;
} else { } else {
auto &graph_item = instance->_qnn_graph_map[map_entry]; auto &graph_item = ctx->qnn_graph_map[map_entry];
qnn::ggml_qnn_tensor_input tensor_input0(src0, std::get<1>(graph_item), ctx); qnn::ggml_qnn_tensor_input tensor_input0(src0, std::get<1>(graph_item), ctx);
qnn::ggml_qnn_tensor_input tensor_input1(src1, std::get<2>(graph_item), ctx); qnn::ggml_qnn_tensor_input tensor_input1(src1, std::get<2>(graph_item), ctx);
qnn::ggml_qnn_tensor_output tensor_output(dst, std::get<3>(graph_item), ctx); qnn::ggml_qnn_tensor_output tensor_output(dst, std::get<3>(graph_item), ctx);
@ -226,9 +226,9 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context *ctx, const ggml_tensor *s
perf.start(); perf.start();
std::string map_entry = std::string(ggml_op_name(ggmlop)); std::string map_entry = std::string(ggml_op_name(ggmlop));
if (instance->_qnn_graph_map.find(map_entry) != instance->_qnn_graph_map.end()) { if (ctx->qnn_graph_map.find(map_entry) != ctx->qnn_graph_map.end()) {
graph_initialized = true; graph_initialized = true;
auto &graph_item = instance->_qnn_graph_map[map_entry]; auto &graph_item = ctx->qnn_graph_map[map_entry];
graph_handle = std::get<0>(graph_item); graph_handle = std::get<0>(graph_item);
} }
@ -327,9 +327,9 @@ static void ggml_qnn_mul_mat(ggml_backend_qnn_context *ctx, const ggml_tensor *s
auto graph_item = std::make_tuple(graph_handle, tensor_input0.get_qnn_tensor(), tensor_input1.get_qnn_tensor(), auto graph_item = std::make_tuple(graph_handle, tensor_input0.get_qnn_tensor(), tensor_input1.get_qnn_tensor(),
tensor_output.get_qnn_tensor()); tensor_output.get_qnn_tensor());
instance->_qnn_graph_map[map_entry] = graph_item; ctx->qnn_graph_map[map_entry] = graph_item;
} else { } else {
auto &graph_item = instance->_qnn_graph_map[map_entry]; auto &graph_item = ctx->qnn_graph_map[map_entry];
qnn::ggml_qnn_tensor_input tensor_input0(src0, std::get<1>(graph_item), ctx); qnn::ggml_qnn_tensor_input tensor_input0(src0, std::get<1>(graph_item), ctx);
qnn::ggml_qnn_tensor_input tensor_input1(src1, std::get<2>(graph_item), ctx); qnn::ggml_qnn_tensor_input tensor_input1(src1, std::get<2>(graph_item), ctx);
qnn::ggml_qnn_tensor_output tensor_output(dst, std::get<3>(graph_item), ctx); qnn::ggml_qnn_tensor_output tensor_output(dst, std::get<3>(graph_item), ctx);

3
ggml/src/ggml-qnn/backend.hpp
View file

@ -1,6 +1,8 @@
#pragma once #pragma once
#include <unordered_map>
#include "ggml.h" #include "ggml.h"
#include "ggml-backend.h" #include "ggml-backend.h"
@ -17,4 +19,5 @@ struct ggml_backend_qnn_context {
QNN_INTERFACE_VER_TYPE raw_interface; QNN_INTERFACE_VER_TYPE raw_interface;
QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface; QNN_SYSTEM_INTERFACE_VER_TYPE raw_system_interface;
qnn::qcom_socinfo socinfo; qnn::qcom_socinfo socinfo;
std::unordered_map<std::string, std::tuple<Qnn_GraphHandle_t, Qnn_Tensor_t *, Qnn_Tensor_t *, Qnn_Tensor_t *>> qnn_graph_map;
}; };

4
ggml/src/ggml-qnn/qnn.hpp
View file

@ -2,7 +2,6 @@
#include <math.h> #include <math.h>
#include <map>
#include <mutex> #include <mutex>
#include <string> #include <string>
#include <unordered_map> #include <unordered_map>
@ -705,9 +704,6 @@ public:
const qnn::qcom_socinfo &get_soc_info() { return _soc_info; } const qnn::qcom_socinfo &get_soc_info() { return _soc_info; }
public:
std::map<std::string, std::tuple<Qnn_GraphHandle_t, Qnn_Tensor_t *, Qnn_Tensor_t *, Qnn_Tensor_t *>> _qnn_graph_map;
private: private:
int load_system() { int load_system() {
Qnn_ErrorHandle_t error = QNN_SUCCESS; Qnn_ErrorHandle_t error = QNN_SUCCESS;