diff --git a/examples/ggml-debug/README.md b/examples/ggml-debug/README.md index 825facd4d..1d76347a0 100644 --- a/examples/ggml-debug/README.md +++ b/examples/ggml-debug/README.md @@ -26,80 +26,69 @@ llama_new_context_with_model: CUDA0 compute buffer size = 105.00 MiB llama_new_context_with_model: CUDA_Host compute buffer size = 6.01 MiB llama_new_context_with_model: graph nodes = 1225 llama_new_context_with_model: graph splits = 2 - -system_info: n_threads = 6 / 12 | AVX = 1 | AVX_VNNI = 0 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | MATMUL_INT8 = 0 | -ggml_debug: inp_embd = GET_ROWS(token_embd.weight{2560, 51200, 1, 1}, inp_tokens{1, 1, 1, 1}}) = {2560, 1, 1, 1} +ggml_debug: inp_embd = (f32) GET_ROWS(token_embd.weight{2560, 51200, 1, 1}, inp_tokens{1, 1, 1, 1}}) = {2560, 1, 1, 1} [ [ - [ -0.0181, -0.0181, 0.0453, ...], + [ -0.0181, 0.0272, 0.0272, ...], ], ] -ggml_debug: norm-0 = NORM(CUDA0#inp_embd#0{2560, 1, 1, 1}, }) = {2560, 1, 1, 1} +ggml_debug: norm-0 = (f32) NORM(CUDA0#inp_embd#0{2560, 1, 1, 1}, }) = {2560, 1, 1, 1} [ [ - [ -0.6989, -0.6989, 1.7686, ...], + [ -0.6989, 1.0636, 1.0636, ...], ], ] -ggml_debug: norm_w-0 = MUL(norm-0{2560, 1, 1, 1}, blk.0.attn_norm.weight{2560, 1, 1, 1}}) = {2560, 1, 1, 1} +ggml_debug: norm_w-0 = (f32) MUL(norm-0{2560, 1, 1, 1}, blk.0.attn_norm.weight{2560, 1, 1, 1}}) = {2560, 1, 1, 1} [ [ - [ -0.1800, -0.1788, 0.4663, ...], + [ -0.1800, 0.2817, 0.2632, ...], ], ] -ggml_debug: attn_norm-0 = ADD(norm_w-0{2560, 1, 1, 1}, blk.0.attn_norm.bias{2560, 1, 1, 1}}) = {2560, 1, 1, 1} +ggml_debug: attn_norm-0 = (f32) ADD(norm_w-0{2560, 1, 1, 1}, blk.0.attn_norm.bias{2560, 1, 1, 1}}) = {2560, 1, 1, 1} [ [ - [ -0.1863, -0.1712, 0.4750, ...], + [ -0.1863, 0.2970, 0.2604, ...], ], ] -ggml_debug: wqkv-0 = MUL_MAT(blk.0.attn_qkv.weight{2560, 7680, 1, 1}, attn_norm-0{2560, 1, 1, 1}}) = {7680, 1, 1, 1} +ggml_debug: wqkv-0 = (f32) MUL_MAT(blk.0.attn_qkv.weight{2560, 7680, 1, 1}, attn_norm-0{2560, 1, 1, 1}}) = {7680, 1, 1, 1} [ [ - [ -1.1238, -2.3523, -1.6938, ...], + [ -1.1238, 1.2876, -1.8086, ...], ], ] -ggml_debug: bqkv-0 = ADD(wqkv-0{7680, 1, 1, 1}, blk.0.attn_qkv.bias{7680, 1, 1, 1}}) = {7680, 1, 1, 1} +ggml_debug: bqkv-0 = (f32) ADD(wqkv-0{7680, 1, 1, 1}, blk.0.attn_qkv.bias{7680, 1, 1, 1}}) = {7680, 1, 1, 1} [ [ - [ -1.1135, -2.5451, -1.8321, ...], + [ -1.1135, 1.4604, -1.9226, ...], ], ] -ggml_debug: bqkv-0 (view) = VIEW(bqkv-0{7680, 1, 1, 1}, }) = {2560, 1, 1, 1} +ggml_debug: bqkv-0 (view) = (f32) VIEW(bqkv-0{7680, 1, 1, 1}, }) = {2560, 1, 1, 1} [ [ - [ -1.1135, -2.5451, -1.8321, ...], + [ -1.1135, 1.4604, -1.9226, ...], ], ] -ggml_debug: Qcur-0 = CONT(bqkv-0 (view){2560, 1, 1, 1}, }) = {2560, 1, 1, 1} +ggml_debug: Qcur-0 = (f32) CONT(bqkv-0 (view){2560, 1, 1, 1}, }) = {2560, 1, 1, 1} [ [ - [ -1.1135, -2.5451, -1.8321, ...], + [ -1.1135, 1.4604, -1.9226, ...], ], ] -ggml_debug: Qcur-0 (reshaped) = RESHAPE(Qcur-0{2560, 1, 1, 1}, }) = {80, 32, 1, 1} +ggml_debug: Qcur-0 (reshaped) = (f32) RESHAPE(Qcur-0{2560, 1, 1, 1}, }) = {80, 32, 1, 1} [ [ - [ -1.1135, 0.8348, 0.8010, ...], - [ -2.5451, -1.1920, 0.0546, ...], - [ -1.8321, -0.0515, 0.8186, ...], + [ -1.1135, 1.4604, -1.9226, ...], + [ -0.3608, 0.5076, -1.8866, ...], + [ 1.7643, 0.0273, -2.1065, ...], ... ], ] -ggml_debug: Qcur-0 = ROPE(Qcur-0 (reshaped){80, 32, 1, 1}, CUDA0#inp_pos#0{1, 1, 1, 1}}) = {80, 32, 1, 1} +ggml_debug: Qcur-0 = (f32) ROPE(Qcur-0 (reshaped){80, 32, 1, 1}, CUDA0#inp_pos#0{1, 1, 1, 1}}) = {80, 32, 1, 1} [ [ - [ -1.1135, 0.8348, 0.8010, ...], - [ -2.5451, -1.1920, 0.0546, ...], - [ -1.8321, -0.0515, 0.8186, ...], - ... - ], - ] -ggml_debug: Qcur-0 = SCALE(Qcur-0{80, 32, 1, 1}, }) = {80, 32, 1, 1} - [ - [ - [ -0.1245, 0.0933, 0.0896, ...], - [ -0.2845, -0.1333, 0.0061, ...], - [ -0.2048, -0.0058, 0.0915, ...], + [ -1.1135, 1.4604, -1.9226, ...], + [ -0.3608, 0.5076, -1.8866, ...], + [ 1.7643, 0.0273, -2.1065, ...], ... ], ] diff --git a/examples/ggml-debug/ggml-debug.cpp b/examples/ggml-debug/ggml-debug.cpp index 168725a7d..145e7d53a 100644 --- a/examples/ggml-debug/ggml-debug.cpp +++ b/examples/ggml-debug/ggml-debug.cpp @@ -3,13 +3,14 @@ #include "ggml.h" #include -#include #include -#include #include +/** + * This the arbitrary data which will be passed to each callback. + * Later on we can for example add operation or tensor name filter from the CLI arg, or a file descriptor to dump the tensor. + */ struct callback_data { - std::mutex m_mutex; std::vector data; }; @@ -24,25 +25,28 @@ static std::string ggml_ne_string(const ggml_tensor * t) { return str; } -static void ggml_print_tensor(const float * data, const int64_t * ne) { - int i, j, k; - printf(" [\n"); - for (i = 0; i < ne[2] && i < 3; i++) { - printf(" [\n"); - for (j = 0; j < ne[1] && j < 3; j++) { - printf(" ["); - for (k = 0; k < ne[0] && k < 3; k++) { - printf("%8.4f", data[k * ne[1] * ne[2] + j * ne[2] + i]); - if (k < ne[0] - 1 && k < 2) printf(", "); +static void ggml_print_tensor(const float * data, const int64_t * ne, const size_t * nb, int64_t n) { + for (int64_t i3 = 0; i3 < ne[3]; i3++) { + printf(" [\n"); + for (int64_t i2 = 0; i2 < ne[2] && i2 < n; i2++) { + printf(" [\n"); + for (int64_t i1 = 0; i1 < ne[1] && i1 < n; i1++) { + printf(" ["); + for (int64_t i0 = 0; i0 < ne[0] && i0 < n; i0++) { + size_t i = i3 * nb[3] + i2 * nb[2] + i1 * nb[1] + i0 * nb[0]; + float v = *(data + i); + printf("%8.4f", v); + if (i0 < ne[0] - 1 && i0 < n - 1) printf(", "); + } + if (ne[0] > n) printf(", ..."); + printf("],\n"); } - if (ne[0] > 3) printf(", ..."); - printf("],\n"); + if (ne[1] > n) printf(" ...\n"); + printf(" ],\n"); } - if (ne[1] > 3) printf(" ...\n"); - printf(" ],\n"); + if (ne[2] > n) printf(" ...\n"); + printf(" ]\n"); } - if (ne[2] > 3) printf(" ...\n"); - printf(" ]\n"); } /** @@ -64,15 +68,13 @@ static bool ggml_debug(struct ggml_tensor * t, bool ask, void * user_data) { return true; // Always retrieve data } - std::lock_guard lock(cb_data->m_mutex); - char src1_str[128] = {0}; if (src1) { sprintf(src1_str, "%s{%s}", src1->name, ggml_ne_string(src1).c_str()); } - printf("%s: %24s = %10s(%s{%s}, %s}) = {%s} \n", __func__, - t->name, ggml_op_name(t->op), + printf("%s: %24s = (%s) %10s(%s{%s}, %s}) = {%s} \n", __func__, + t->name, ggml_type_name(t->type), ggml_op_name(t->op), src0->name, ggml_ne_string(src0).c_str(), src1 ? src1_str : "", ggml_ne_string(t).c_str()); @@ -87,8 +89,10 @@ static bool ggml_debug(struct ggml_tensor * t, bool ask, void * user_data) { ggml_backend_tensor_get(t, cb_data->data.data(), 0, n_bytes); } - const float * data = is_host ? (const float *) t->data : cb_data->data.data(); - ggml_print_tensor(data, t->ne); + if (t->type == GGML_TYPE_F32 || t->type == GGML_TYPE_F16) { + const float * data = is_host ? (const float *) t->data : cb_data->data.data(); + ggml_print_tensor(data, t->ne, t->nb, 3); + } return true; }