vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ggml : move CPU backend to a separate file (#10144)

Browse source
This commit is contained in:
Diego Devesa 2024-11-03 19:34:08 +01:00 • committed by GitHub
parent 08828a6d7d
commit 9f40989351
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
32 changed files with 14747 additions and 19345 deletions
Download patch file Download diff file Expand all files Collapse all files

3
pocs/vdot/q8dot.cpp
View file

@ -11,6 +11,7 @@
#include <type_traits>
#include <ggml.h>
#include <ggml-cpu.h>
constexpr int kVecSize = 1 << 16;
@ -136,7 +137,7 @@ int main(int argc, char** argv) {
auto ggml_type = type == 0 ? GGML_TYPE_Q4_0 : GGML_TYPE_Q4_1;
const auto * funcs = ggml_get_type_traits(ggml_type);
const auto * funcs = ggml_get_type_traits_cpu(ggml_type);
Stat simple, ggml;

10
pocs/vdot/vdot.cpp
View file

@ -9,6 +9,7 @@
#include <array>
#include <ggml.h>
#include <ggml-cpu.h>
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
@ -236,7 +237,8 @@ int main(int argc, char** argv) {
int n4 = useQ4_1 ? kVecSize / QK4_1 : kVecSize / QK4_0; n4 = 64*((n4 + 63)/64);
int n8 = kVecSize / QK8_0; n8 = 64*((n8 + 63)/64);
const auto * funcs = useQ4_1 ? ggml_get_type_traits(GGML_TYPE_Q4_1) : ggml_get_type_traits(GGML_TYPE_Q4_0);
const auto * funcs = ggml_get_type_traits(useQ4_1 ? GGML_TYPE_Q4_1 : GGML_TYPE_Q4_0);
const auto * funcs_cpu = ggml_get_type_traits_cpu(useQ4_1 ? GGML_TYPE_Q4_1 : GGML_TYPE_Q4_0);
std::vector<block_q4_0> q40;
std::vector<block_q4_1> q41;
@ -282,10 +284,10 @@ int main(int argc, char** argv) {
dot_q4_q8(kVecSize, &result, q40.data(), q8.data());
}
else {
const auto * vdot = ggml_get_type_traits(funcs->vec_dot_type);
const auto * vdot = ggml_get_type_traits(funcs_cpu->vec_dot_type);
vdot->from_float(y1.data(), q8.data(), kVecSize);
if (useQ4_1) funcs->vec_dot(kVecSize, &result, 0, q41.data(), 0, q8.data(), 0, 1);
else funcs->vec_dot(kVecSize, &result, 0, q40.data(), 0, q8.data(), 0, 1);
if (useQ4_1) funcs_cpu->vec_dot(kVecSize, &result, 0, q41.data(), 0, q8.data(), 0, 1);
else funcs_cpu->vec_dot(kVecSize, &result, 0, q40.data(), 0, q8.data(), 0, 1);
}
sumq += result;
t2 = std::chrono::high_resolution_clock::now();