From 2acb281105f5f6e2c94137d1fa31a5e339f84974 Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Sat, 2 Mar 2024 09:40:01 +0100 Subject: [PATCH] Add soft_max alibi support --- ggml-vulkan-shaders.hpp | 663 +++++++++++++++++++++--------------- ggml-vulkan.cpp | 200 +++++++++-- ggml_vk_generate_shaders.py | 39 ++- llama.cpp | 2 +- 4 files changed, 583 insertions(+), 321 deletions(-) diff --git a/ggml-vulkan-shaders.hpp b/ggml-vulkan-shaders.hpp index e5bc1d137..339784002 100644 --- a/ggml-vulkan-shaders.hpp +++ b/ggml-vulkan-shaders.hpp @@ -32232,50 +32232,65 @@ const uint64_t silu_f32_len = 1264; unsigned char soft_max_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x0e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x5c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, 0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, 0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00,0x05,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0d,0x00,0x05,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x11,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x5c,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, 0x17,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x3a,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x3b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x3b,0x00,0x00,0x00, +0x72,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x73,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x73,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x3b,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3d,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x73,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x75,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x75,0x00,0x00,0x00, 0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x53,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x54,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x8a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x8a,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x54,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x56,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x8a,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x8c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8c,0x00,0x00,0x00, 0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xbf,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xc0,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xa0,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xa1,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xa1,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa3,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa3,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x07,0x01,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xc0,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xc2,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xc2,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xff,0x00,0x00,0x00, +0x07,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x09,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x09,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x45,0x01,0x00,0x00, 0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, 0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, @@ -32288,264 +32303,346 @@ unsigned char soft_max_f32_data[] = { 0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, 0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x16,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x06,0x00, +0x16,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x0a,0x00, 0x17,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x18,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x1a,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x24,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0x27,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x1a,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x1c,0x00,0x04,0x00, -0x21,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x22,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x22,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x27,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x34,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x3a,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x3c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x3c,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x44,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x48,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x53,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x54,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x55,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0x1c,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x5b,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x5b,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x60,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x1a,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x72,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x73,0x00,0x00,0x00,0x72,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x74,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x7c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x89,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x8a,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x8b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x8b,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xa0,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xa1,0x00,0x00,0x00, +0xa0,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa2,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xa2,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, 0x08,0x01,0x00,0x00,0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x1d,0x00,0x03,0x00, -0xbf,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0xc0,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xc1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xc1,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2c,0x00,0x06,0x00, -0x09,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0xfe,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x00,0x00,0x80,0xff, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x12,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x1d,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x28,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2b,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x0f,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x2e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x34,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x33,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x44,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x3d,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x43,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x46,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x48,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x49,0x00,0x00,0x00,0xac,0x00,0x05,0x00,0x34,0x00,0x00,0x00, -0x4e,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x52,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x5f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x51,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x44,0x00,0x00,0x00, -0x5d,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x5d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x52,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x52,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x52,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x5e,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x60,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, -0x16,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x09,0x01,0x00,0x00,0x0c,0x00,0x07,0x00,0x16,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x28,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2e,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x20,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x2d,0x00,0x00,0x00, -0xe0,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6c,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x1a,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x6b,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0xad,0x00,0x05,0x00,0x34,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x01,0x01,0x00,0x00,0x31,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x6e,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x72,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x6d,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x01,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0x34,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x78,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x76,0x00,0x00,0x00,0x77,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x83,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x28,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x78,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x78,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, -0x67,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6f,0x00,0x00,0x00,0xc3,0x00,0x05,0x00,0x1a,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x1b,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x6c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x27,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x88,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x67,0x00,0x00,0x00, -0x67,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x28,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x8e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x8e,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x02,0x01,0x00,0x00, -0x0f,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x34,0x00,0x00,0x00, -0x96,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x33,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x90,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x96,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x44,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x48,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0xac,0x00,0x05,0x00,0x34,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x1d,0x00,0x03,0x00, +0x06,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x08,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x07,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x5b,0x01,0x00,0x00,0x00,0x00,0x80,0xff,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0xba,0x00,0x05,0x00, +0x27,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x2a,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x28,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x29,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x38,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x35,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x38,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x3c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x38,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x38,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, +0x46,0x01,0x00,0x00,0x3b,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x48,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x35,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x47,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x48,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4c,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x52,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x48,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x48,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x4b,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x4c,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x57,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x0c,0x00,0x07,0x00, +0x16,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x57,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, +0x54,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x60,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x61,0x00,0x00,0x00, +0x5b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x64,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x64,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x0f,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x67,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x66,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x6d,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x7c,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0xac,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x85,0x00,0x00,0x00, 0x1e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xaa,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xa7,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x02,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x44,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, -0xb1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xaa,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xaa,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xaa,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0xb2,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x60,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x16,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x83,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x0c,0x00,0x06,0x00, -0x16,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x28,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x44,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc5,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x91,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x91,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x20,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x8e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x90,0x00,0x00,0x00, -0xe0,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc9,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x1a,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x6b,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0xad,0x00,0x05,0x00,0x34,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0x31,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xcb,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xcf,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0x34,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xd5,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xd3,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x27,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0xda,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0xdc,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x28,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x88,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x85,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x48,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x7c,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x88,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x95,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x88,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x88,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x51,0x01,0x00,0x00, +0x94,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x16,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x51,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0xac,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x9e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x9b,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9d,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x7c,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x54,0x01,0x00,0x00,0xa6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x9e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9e,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0xa7,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x81,0x00,0x05,0x00, +0x16,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0x0c,0x00,0x07,0x00,0x16,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x61,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x67,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x67,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x64,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x66,0x00,0x00,0x00, +0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x1a,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0xb2,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, +0xad,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb5,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xb5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xbd,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xbe,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x5c,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x0c,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x61,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xbf,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbf,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb6,0x00,0x00,0x00,0xc3,0x00,0x05,0x00,0x1a,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x1b,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb5,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x60,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x61,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, 0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xe0,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xcc,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcc,0x00,0x00,0x00,0xc3,0x00,0x05,0x00, -0x1a,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0x1b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe8,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0x0f,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0x34,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x33,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xea,0x00,0x00,0x00, -0xe9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xf0,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xea,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf6,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x44,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x0f,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0xd8,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdd,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x7c,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, +0xac,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0xee,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xf1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xee,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf0,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x7c,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, 0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x88,0x00,0x05,0x00,0x16,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xf8,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0x20,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xf1,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0xf9,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x16,0x00,0x00,0x00, +0xfc,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0xfc,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x0c,0x00,0x06,0x00, +0x16,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x61,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x04,0x01,0x00,0x00, +0x03,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x61,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x7c,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x6a,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x0c,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd8,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x10,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x10,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x1a,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0xb2,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x13,0x01,0x00,0x00, +0xad,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x16,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x12,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x16,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x11,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00, +0x1a,0x01,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0xf7,0x00,0x03,0x00,0x1c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x1a,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, +0x1c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x0f,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x60,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x5c,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x61,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x26,0x01,0x00,0x00, +0x25,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x61,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1c,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x13,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x13,0x01,0x00,0x00,0xc3,0x00,0x05,0x00, +0x1a,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x1b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x10,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x12,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x4c,0x01,0x00,0x00,0x0f,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0x27,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x31,0x01,0x00,0x00, +0x30,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x37,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x31,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x30,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x13,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3d,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x7c,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x3f,0x01,0x00,0x00,0x88,0x00,0x05,0x00,0x16,0x00,0x00,0x00, +0x41,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x3f,0x01,0x00,0x00,0x41,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x4c,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x31,0x01,0x00,0x00, 0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t soft_max_f32_len = 3752; +const uint64_t soft_max_f32_len = 4916; unsigned char split_k_reduce_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 9d73a69db..f04c93cba 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -269,6 +269,17 @@ struct vk_op_rope_neox_push_constants { float inv_ndims; }; +struct vk_op_soft_max_push_constants { + uint32_t KX; + uint32_t KY; + uint32_t KZ; + float scale; + float max_bias; + float m0; + float m1; + uint32_t n_head_log2; +}; + struct vk_op_argsort_push_constants { uint32_t ncols; bool ascending; @@ -1104,7 +1115,7 @@ static void ggml_vk_load_shaders(ggml_backend_vk_context * ctx) { ggml_vk_create_pipeline(ctx, ctx->device->pipeline_diag_mask_inf_f32, "diag_mask_inf_f32", diag_mask_inf_f32_len, diag_mask_inf_f32_data, "main", 2, sizeof(vk_op_diag_mask_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->device->pipeline_soft_max_f32, "soft_max_f32", soft_max_f32_len, soft_max_f32_data, "main", 3, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_soft_max_f32, "soft_max_f32", soft_max_f32_len, soft_max_f32_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(ctx, ctx->device->pipeline_rope_f32, "rope_f32", rope_f32_len, rope_f32_data, "main", 3, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); ggml_vk_create_pipeline(ctx, ctx->device->pipeline_rope_f16, "rope_f16", rope_f16_len, rope_f16_data, "main", 3, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); @@ -2968,7 +2979,7 @@ static void ggml_vk_op_repeat(ggml_backend_vk_context * ctx, vk_context * subctx } -static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_op op) { +static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst, ggml_op op) { switch (op) { case GGML_OP_ADD: if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { @@ -3045,7 +3056,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const } return nullptr; case GGML_OP_SOFT_MAX: - if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + if (src0->type == GGML_TYPE_F32 && (src1 == nullptr || src1->type == GGML_TYPE_F32) && (src2 == nullptr || src2->type == GGML_TYPE_F32) && dst->type == GGML_TYPE_F32) { return ctx->device->pipeline_soft_max_f32; } return nullptr; @@ -3096,12 +3107,15 @@ static ggml_vk_func_t ggml_vk_op_get_func(ggml_op op) { } template -static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_op op, const PC&& pc) { +static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst, ggml_op op, const PC&& pc) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_op_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; if (src1 != nullptr) { std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; } + if (src2 != nullptr) { + std::cerr << "), (" << src2 << ", name=" << src2->name << ", type=" << src2->type << ", backend=" << src2->backend << ", ne0=" << src2->ne[0] << ", ne1=" << src2->ne[1] << ", ne2=" << src2->ne[2] << ", ne3=" << src2->ne[3] << ", nb0=" << src2->nb[0] << ", nb1=" << src2->nb[1] << ", nb2=" << src2->nb[2] << ", nb3=" << src2->nb[3]; + } std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", backend=" << dst->backend << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "), " << ggml_op_name(op) << ")" << std::endl; #endif GGML_ASSERT(!ggml_is_quantized(src0->type) && (src1 == nullptr || !ggml_is_quantized(src1->type))); // NOLINT @@ -3122,7 +3136,10 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c const uint64_t nb2 = dst->nb[2]; const uint64_t nb3 = dst->nb[3]; - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, dst, op); + const bool use_src2 = src2 != nullptr; + const uint64_t ne2 = use_src2 ? src2->ne[0] * src2->ne[1] : 0; + + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, src2, dst, op); ggml_vk_func_t op_func; if (pipeline == nullptr) { @@ -3143,14 +3160,18 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) dst->extra; ggml_tensor_extra_gpu * extra_src0 = (ggml_tensor_extra_gpu *) src0->extra; ggml_tensor_extra_gpu * extra_src1 = use_src1 ? (ggml_tensor_extra_gpu *) src1->extra : nullptr; + ggml_tensor_extra_gpu * extra_src2 = use_src2 ? (ggml_tensor_extra_gpu *) src2->extra : nullptr; vk_buffer d_X = nullptr; size_t x_buf_offset = 0; vk_buffer d_Y = nullptr; size_t y_buf_offset = 0; + vk_buffer d_Z = nullptr; + size_t z_buf_offset = 0; bool src0_uma = false; bool src1_uma = false; + bool src2_uma = false; if (ctx->device->uma) { ggml_vk_host_get(ctx, src0->data, d_X, x_buf_offset); @@ -3159,13 +3180,19 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c ggml_vk_host_get(ctx, src1->data, d_Y, y_buf_offset); src1_uma = d_Y != nullptr; } + if (use_src2) { + ggml_vk_host_get(ctx, src1->data, d_Z, z_buf_offset); + src2_uma = d_Z != nullptr; + } } const bool transfer_src0 = src0->backend != GGML_BACKEND_TYPE_GPU && !src0_uma; const bool transfer_src1 = use_src1 && src1->backend != GGML_BACKEND_TYPE_GPU && !src1_uma; + const bool transfer_src2 = use_src2 && src2->backend != GGML_BACKEND_TYPE_GPU && !src2_uma; uint64_t x_sz = ggml_vk_align_size(ggml_type_size(src0->type) * ne0, ctx->device->properties.limits.minStorageBufferOffsetAlignment); uint64_t y_sz = use_src1 ? ggml_vk_align_size(ggml_type_size(src1->type) * ne1, ctx->device->properties.limits.minStorageBufferOffsetAlignment) : 0; + uint64_t z_sz = use_src2 ? ggml_vk_align_size(ggml_type_size(src2->type) * ne2, ctx->device->properties.limits.minStorageBufferOffsetAlignment) : 0; uint64_t d_sz = ggml_type_size(dst->type) * ne0; vk_buffer d_D = extra->buffer_gpu.lock(); @@ -3193,6 +3220,13 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c GGML_ASSERT(d_Y != nullptr); } + GGML_ASSERT(!transfer_src2); + if (use_src2 && !src2_uma) { + d_Z = extra_src2->buffer_gpu.lock(); + z_buf_offset = extra_src2->offset; + GGML_ASSERT(d_Z != nullptr); + } + if (op == GGML_OP_CPY) { GGML_ASSERT(!transfer_src0); GGML_ASSERT(!transfer_src1); @@ -3249,10 +3283,24 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c } } - if (!use_src1 && op == GGML_OP_SOFT_MAX) { - // Empty src1 is possible on soft_max, but the shader needs a buffer + if (op == GGML_OP_SOFT_MAX) { + // Empty src1 and src2 are possible on soft_max, but the shader needs buffers + vk_subbuffer subbuf_y; + if (use_src1) { + subbuf_y = { d_Y, y_buf_offset, y_sz }; + } else { + subbuf_y = { ctx->prealloc_y, 0, ctx->prealloc_y->size }; + } + + vk_subbuffer subbuf_z; + if (use_src2) { + subbuf_z = { d_Z, z_buf_offset, z_sz }; + } else { + subbuf_z = { ctx->prealloc_y, 0, ctx->prealloc_y->size }; + } + ggml_vk_sync_buffers(subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, { ctx->prealloc_y, 0, ctx->prealloc_y->size }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, subbuf_y, subbuf_z, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); } else if (use_src1) { ggml_vk_sync_buffers(subctx); ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, { d_Y, y_buf_offset, y_sz }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); @@ -3268,7 +3316,8 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c ggml_vk_buffer_read_async(ctx, subctx, d_D, 0, d, d_sz); } } else { - std::cerr << "oopsie" << std::endl; + GGML_ASSERT(op != GGML_OP_SOFT_MAX); + ggml_pipeline_allocate_descriptor_sets(ctx, pipeline, ne02 * ne03); switch (dst->op) { @@ -3294,11 +3343,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c const uint32_t y_offset = y_sz * it_idx1; const uint32_t d_offset = d_sz * it_idx0; - if (!use_src1 && op == GGML_OP_SOFT_MAX) { - // Empty src1 is possible on soft_max, but the shader needs a buffer - ggml_vk_sync_buffers(subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, { ctx->prealloc_y, 0, ctx->prealloc_y->size }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); - } else if (use_src1) { + if (use_src1) { ggml_vk_sync_buffers(subctx); ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset + x_offset, x_sz }, { d_Y, y_buf_offset + y_offset, y_sz }, { d_D, d_buf_offset + d_offset, d_sz } }, sizeof(PC), &pc, elements); } else { @@ -3315,11 +3360,11 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c } static void ggml_vk_repeat(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_REPEAT, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_REPEAT, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); } static void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_GET_ROWS, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_GET_ROWS, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); } static void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -3327,7 +3372,7 @@ static void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context * subctx, cons const uint32_t src1_type_size = ggml_type_size(src1->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_ADD, { + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_ADD, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, @@ -3342,7 +3387,7 @@ static void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context * subctx, cons const uint32_t src1_type_size = ggml_type_size(src1->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_MUL, { + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_MUL, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, @@ -3357,7 +3402,7 @@ static void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context * subctx, co const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_SCALE, { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_SCALE, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, @@ -3370,7 +3415,7 @@ static void ggml_vk_sqr(ggml_backend_vk_context * ctx, vk_context * subctx, cons const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_SQR, { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_SQR, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, @@ -3384,7 +3429,7 @@ static void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context * subctx, co const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_CLAMP, { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_CLAMP, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, @@ -3399,7 +3444,7 @@ static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context * subctx, cons const uint32_t dst_type_size = ggml_type_size(dst->type); const uint32_t d_offset = (extra->offset % ctx->device->properties.limits.minStorageBufferOffsetAlignment) / dst_type_size; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_CPY, { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_CPY, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, @@ -3409,26 +3454,47 @@ static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context * subctx, cons } static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], 0.0f, 0.0f }); } static void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_RMS_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_RMS_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }); } static void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_UNARY, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_UNARY, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }); } static void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { int32_t * op_params = (int32_t *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] }); } -static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_SOFT_MAX, { (uint32_t)src0->ne[0], (uint32_t)(src1 != nullptr ? ggml_nrows(src1) : 0), op_params[0], 0.0f }); + + float scale = op_params[0]; + float max_bias = op_params[1]; + + const uint32_t ncols = (uint32_t)src0->ne[0]; + const uint32_t nrows_x = (uint32_t)ggml_nrows(src0); + const uint32_t nrows_y = (uint32_t)src0->ne[1]; + + const uint32_t n_head_kv = nrows_x/nrows_y; + const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head_kv)); + + const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + + ggml_vk_op_f32(ctx, subctx, src0, src1, src2, dst, GGML_OP_SOFT_MAX, { + ncols, + nrows_y, + src2 != nullptr ? (uint32_t)1 : (uint32_t)0, + scale, max_bias, + m0, m1, + n_head_log2, + }); } static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -3454,15 +3520,15 @@ static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context * subctx, con if (is_neox) { const float theta_scale = powf(freq_base, -2.0f/n_dims); const float inv_ndims = -1.0f / n_dims; - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], (uint32_t)n_dims, freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, corr_dims[0], corr_dims[1], 0.0f, 0.0f, theta_scale, inv_ndims }); + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], (uint32_t)n_dims, freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, corr_dims[0], corr_dims[1], 0.0f, 0.0f, theta_scale, inv_ndims }); } else { - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, corr_dims[0], corr_dims[1], 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, corr_dims[0], corr_dims[1], 0.0f, 0.0f }); } } static void ggml_vk_argsort(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { int32_t * op_params = (int32_t *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_ARGSORT, { (uint32_t)src0->ne[0], ((ggml_sort_order) op_params[0]) == GGML_SORT_ORDER_ASC }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_ARGSORT, { (uint32_t)src0->ne[0], ((ggml_sort_order) op_params[0]) == GGML_SORT_ORDER_ASC }); } static void ggml_vk_nop(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { @@ -4564,6 +4630,7 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod const ggml_tensor * src0 = node->src[0]; const ggml_tensor * src1 = node->src[1]; + const ggml_tensor * src2 = node->src[2]; ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) node->extra; @@ -4682,7 +4749,7 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod break; case GGML_OP_SOFT_MAX: - ggml_vk_soft_max(ctx, ctx->compute_ctx, src0, src1, node); + ggml_vk_soft_max(ctx, ctx->compute_ctx, src0, src1, src2, node); break; case GGML_OP_ROPE: @@ -5836,6 +5903,7 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ ggml_tensor * src0 = tensor->src[0]; ggml_tensor * src1 = tensor->src[1]; + ggml_tensor * src2 = tensor->src[2]; struct ggml_init_params iparams = { /*.mem_size =*/ 1024*1024*1024, @@ -5847,13 +5915,16 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ struct ggml_tensor * src0_clone = nullptr; struct ggml_tensor * src1_clone = nullptr; + struct ggml_tensor * src2_clone = nullptr; struct ggml_tensor * tensor_clone = nullptr; size_t src0_size; size_t src1_size; + size_t src2_size; void * src0_buffer; void * src1_buffer; + void * src2_buffer; if (src0 != nullptr) { src0_clone = ggml_dup_tensor(ggml_ctx, src0); @@ -5959,6 +6030,66 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ ggml_vk_check_tensor(std::string(ggml_op_name(tensor->op)) + "->src1", src1_clone); } + if (src2 != nullptr) { + src2_clone = ggml_dup_tensor(ggml_ctx, src2); + + src2_size = ggml_nbytes(src2); + + src2_buffer = malloc(src2_size); + src2_clone->data = src2_buffer; + if (src2->backend == GGML_BACKEND_TYPE_CPU) { + memcpy(src2_clone->data, src2->data, src2_size); + memcpy(src2_clone->nb, src2->nb, sizeof(size_t) * GGML_MAX_DIMS); + } else if (src2->backend == GGML_BACKEND_TYPE_GPU) { + ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src2->extra; + vk_buffer buf = extra->buffer_gpu.lock(); + uint64_t offset = extra->offset; + if (!ggml_is_contiguous(src2) && ggml_vk_dim01_contiguous(src2)) { + for (int i3 = 0; i3 < src2->ne[3]; i3++) { + for (int i2 = 0; i2 < src2->ne[2]; i2++) { + const int idx = i3*src2->ne[2] + i2; + ggml_vk_buffer_read(ctx, buf, offset + idx * src2->nb[2], ((char *)src2_clone->data + idx * src2_clone->nb[2]), src2->ne[1] * src2->nb[1]); + } + } + + src2_clone->nb[0] = src2->nb[0]; + src2_clone->nb[1] = src2->nb[1]; + for (int i = 2; i < GGML_MAX_DIMS; i++) { + src2_clone->nb[i] = src2_clone->nb[i - 1]*src2_clone->ne[i - 1]; + } + } else { + if (offset + src2_size >= buf->size) { + src2_size = buf->size - offset; + } + ggml_vk_buffer_read(ctx, buf, offset, src2_clone->data, src2_size); + memcpy(src2_clone->nb, src2->nb, sizeof(size_t) * GGML_MAX_DIMS); + } + } else { + GGML_ASSERT(false); + } + + if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { + ggml_vk_print_tensor(ctx, src2, "src2"); + std::cerr << "TENSOR CHECK: " << ggml_op_name(src2_clone->op) << " (check " << check_counter << ")" << std::endl; + std::cerr << "src2_clone=" << tensor << " src2_clone->backend: " << src2_clone->backend << " src2_clone->type: " << ggml_type_name(src2_clone->type) << " ne0=" << src2_clone->ne[0] << " nb0=" << src2_clone->nb[0] << " ne1=" << src2_clone->ne[1] << " nb1=" << src2_clone->nb[1] << " ne2=" << src2_clone->ne[2] << " nb2=" << src2_clone->nb[2] << " ne3=" << src2_clone->ne[3] << " nb3=" << src2_clone->nb[3] << std::endl; + if (src2->src[0] != nullptr) { + std::cerr << "src2->src[0]=" << src2->src[0] << " op=" << ggml_op_name(src2->src[0]->op) << " type=" << ggml_type_name(src2->src[0]->type) << " backend=" << src2->src[0]->backend << " ne0=" << src2->src[0]->ne[0] << " nb0=" << src2->src[0]->nb[0] << " ne1=" << src2->src[0]->ne[1] << " nb1=" << src2->src[0]->nb[1] << " ne2=" << src2->src[0]->ne[2] << " nb2=" << src2->src[0]->nb[2] << " ne3=" << src2->src[0]->ne[3] << " nb3=" << src2->src[0]->nb[3] << std::endl; + } + if (src2->src[1] != nullptr) { + std::cerr << "src2->src[1]=" << src2->src[1] << " op=" << ggml_op_name(src2->src[1]->op) << " type=" << ggml_type_name(src2->src[1]->type) << " backend=" << src2->src[1]->backend << " ne0=" << src2->src[1]->ne[0] << " nb0=" << src2->src[1]->nb[0] << " ne1=" << src2->src[1]->ne[1] << " nb1=" << src2->src[1]->nb[1] << " ne2=" << src2->src[1]->ne[2] << " nb2=" << src2->src[1]->nb[2] << " ne3=" << src2->src[1]->ne[3] << " nb3=" << src2->src[1]->nb[3] << std::endl; + } + std::cerr << std::endl << "Result:" << std::endl; + ggml_vk_print_tensor_area(src2_clone, src2_clone->data, 5, 5, 0, 0); + std::cerr << std::endl; + std::cerr << std::endl << "Result:" << std::endl; + ggml_vk_print_tensor_area(src2_clone, src2_clone->data, 5, 5, 1, 0); + std::cerr << std::endl; + std::vector done; + ggml_vk_print_graph_origin(src2_clone, done); + } + + ggml_vk_check_tensor(std::string(ggml_op_name(tensor->op)) + "->src2", src2_clone); + } if (tensor->op == GGML_OP_MUL_MAT) { tensor_clone = ggml_mul_mat(ggml_ctx, src0_clone, src1_clone); @@ -5978,7 +6109,7 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ tensor_clone = ggml_rms_norm(ggml_ctx, src0_clone, *(float *)tensor->op_params); } else if (tensor->op == GGML_OP_SOFT_MAX) { if (src1 != nullptr) { - tensor_clone = ggml_soft_max_ext(ggml_ctx, src0_clone, src1_clone, nullptr, ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]); + tensor_clone = ggml_soft_max_ext(ggml_ctx, src0_clone, src1_clone, src2_clone, ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]); } else { tensor_clone = ggml_soft_max(ggml_ctx, src0_clone); } @@ -6061,6 +6192,9 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ if (src1 != nullptr) { free(src1_buffer); } + if (src2 != nullptr) { + free(src1_buffer); + } ggml_free(ggml_ctx); } diff --git a/ggml_vk_generate_shaders.py b/ggml_vk_generate_shaders.py index c1d1fc3c9..9ff775612 100644 --- a/ggml_vk_generate_shaders.py +++ b/ggml_vk_generate_shaders.py @@ -1901,6 +1901,24 @@ void main() { """ # SOFT_MAX +soft_max_head = """ +#version 450 + +#extension GL_EXT_shader_16bit_storage : require + +layout (push_constant) uniform parameter +{ + uint KX; + uint KY; + uint KZ; + float scale; + float max_bias; + float m0; + float m1; + uint n_head_log2; +} p; +""" + soft_max_body = """ #extension GL_EXT_control_flow_attributes : enable #define BLOCK_SIZE 512 @@ -1909,7 +1927,8 @@ layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; layout (binding = 1) readonly buffer Y {B_TYPE data_b[];}; -layout (binding = 2) buffer D {D_TYPE data_d[];}; +layout (binding = 2) readonly buffer Z {C_TYPE data_c[];}; +layout (binding = 3) buffer D {D_TYPE data_d[];}; shared FLOAT_TYPE vals[BLOCK_SIZE]; @@ -1918,11 +1937,23 @@ void main() { const uint rowx = gl_WorkGroupID.x; const uint rowy = rowx % p.KY; + float slope = 0.0f; + + // ALiBi + if (p.max_bias > 0.0f) { + const uint h = rowx/p.KY; // head index + + const float base = h < p.n_head_log2 ? p.m0 : p.m1; + const uint exp = h < p.n_head_log2 ? h + 1 : 2*(h - p.n_head_log2) + 1; + + slope = pow(base, exp); + } + // Find max vals[tid] = uintBitsToFloat(0xFF800000); [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { - vals[tid] = max(vals[tid], FLOAT_TYPE(data_a[rowx * p.KX + col]) * p.param1 + (p.KY > 0 ? FLOAT_TYPE(data_b[rowy * p.KX + col]) : FLOAT_TYPE(0.0f))); + vals[tid] = max(vals[tid], FLOAT_TYPE(data_a[rowx * p.KX + col]) * p.scale + (p.KY > 0 ? FLOAT_TYPE(data_b[rowy * p.KX + col]) : FLOAT_TYPE(0.0f)) + (p.KZ > 0 ? slope * data_c[col] : 0.0f)); } barrier(); @@ -1941,7 +1972,7 @@ void main() { [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { const uint i = rowx * p.KX + col; - const FLOAT_TYPE val = exp(FLOAT_TYPE(data_a[i]) * p.param1 + (p.KY > 0 ? FLOAT_TYPE(data_b[rowy * p.KX + col]) : FLOAT_TYPE(0.0f)) - max_val); + const FLOAT_TYPE val = exp(FLOAT_TYPE(data_a[i]) * p.scale + (p.KY > 0 ? FLOAT_TYPE(data_b[rowy * p.KX + col]) : FLOAT_TYPE(0.0f)) - max_val); vals[tid] += val; data_d[i] = D_TYPE(val); } @@ -2423,7 +2454,7 @@ async def main(): tasks.append(string_to_spv("diag_mask_inf_f32", f"{diag_mask_inf_head}\n{shader_f32}\n{diag_mask_inf_body}", {"A_TYPE": "float", "D_TYPE": "float"})) - tasks.append(string_to_spv("soft_max_f32", f"{generic_head}\n{shader_f32}\n{soft_max_body}", {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float"})) + tasks.append(string_to_spv("soft_max_f32", f"{soft_max_head}\n{shader_f32}\n{soft_max_body}", {"A_TYPE": "float", "B_TYPE": "float", "C_TYPE": "float", "D_TYPE": "float"})) tasks.append(string_to_spv("rope_f32", rope_src, {"A_TYPE": "float", "D_TYPE": "float"})) tasks.append(string_to_spv("rope_f16", rope_src, {"A_TYPE": "float16_t", "D_TYPE": "float16_t"})) diff --git a/llama.cpp b/llama.cpp index acd9be08a..a4746acb0 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4899,7 +4899,7 @@ static struct ggml_tensor * llm_build_kqv( ggml_mul_mat_set_prec(kq, GGML_PREC_F32); } -#if defined(GGML_USE_VULKAN) || defined(GGML_USE_KOMPUTE) || defined(GGML_USE_SYCL) +#if defined(GGML_USE_KOMPUTE) || defined(GGML_USE_SYCL) #pragma message("TODO: ALiBi support in ggml_soft_max_ext is not implemented for Vulkan, Kompute, and SYCL") #pragma message(" Falling back to ggml_alibi(). Will become an error in Mar 2024") #pragma message("ref: https://github.com/ggerganov/llama.cpp/pull/5488")