Initial working stuff
This commit is contained in:
niansa
parent
bbca06e269
commit
4f598dd973
7 changed files with 643 additions and 1 deletions
3
.gitmodules
vendored
Normal file
3
.gitmodules
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
[submodule "kompute"]
|
||||
path = kompute
|
||||
url = https://github.com/KomputeProject/kompute.git
|
||||
|
|
@ -73,6 +73,7 @@ set(LLAMA_CUDA_DMMV_Y "1" CACHE STRING "llama: y block size for dmmv CUDA
|
|||
option(LLAMA_CUDA_DMMV_F16 "llama: use 16 bit floats for dmmv CUDA kernels" OFF)
|
||||
set(LLAMA_CUDA_KQUANTS_ITER "2" CACHE STRING "llama: iters./thread per block for Q2_K/Q6_K")
|
||||
option(LLAMA_CLBLAST "llama: use CLBlast" OFF)
|
||||
option(LLAMA_KOMPUTE "llama: use Kompute" OFF)
|
||||
option(LLAMA_METAL "llama: use Metal" OFF)
|
||||
option(LLAMA_K_QUANTS "llama: use k-quants" ON)
|
||||
|
||||
|
|
@ -309,6 +310,22 @@ if (LLAMA_CLBLAST)
|
|||
endif()
|
||||
endif()
|
||||
|
||||
if (LLAMA_KOMPUTE)
|
||||
if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/kompute/CMakeLists.txt")
|
||||
message(STATUS "Kompute found")
|
||||
|
||||
add_subdirectory(kompute)
|
||||
|
||||
set(GGML_SOURCES_KOMPUTE ggml-vulkan.cpp ggml-vulkan.h)
|
||||
|
||||
add_compile_definitions(GGML_USE_KOMPUTE)
|
||||
|
||||
set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} kompute)
|
||||
else()
|
||||
message(WARNING "Kompute not found")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (LLAMA_ALL_WARNINGS)
|
||||
if (NOT MSVC)
|
||||
set(c_flags
|
||||
|
|
@ -466,6 +483,7 @@ add_library(ggml OBJECT
|
|||
ggml.h
|
||||
${GGML_SOURCES_CUDA}
|
||||
${GGML_SOURCES_OPENCL}
|
||||
${GGML_SOURCES_KOMPUTE}
|
||||
${GGML_SOURCES_METAL}
|
||||
${GGML_SOURCES_EXTRA}
|
||||
)
|
||||
|
|
|
|||
454
CMakeLists.txt.user
Normal file
454
CMakeLists.txt.user
Normal file
|
|
@ -0,0 +1,454 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE QtCreatorProject>
|
||||
<!-- Written by QtCreator 9.0.2, 2023-06-22T12:36:05. -->
|
||||
<qtcreator>
|
||||
<data>
|
||||
<variable>EnvironmentId</variable>
|
||||
<value type="QByteArray">{f3929b0b-3d39-4fa3-8d2d-2b329b63b30c}</value>
|
||||
</data>
|
||||
<data>
|
||||
<variable>ProjectExplorer.Project.ActiveTarget</variable>
|
||||
<value type="qlonglong">0</value>
|
||||
</data>
|
||||
<data>
|
||||
<variable>ProjectExplorer.Project.EditorSettings</variable>
|
||||
<valuemap type="QVariantMap">
|
||||
<value type="bool" key="EditorConfiguration.AutoIndent">true</value>
|
||||
<value type="bool" key="EditorConfiguration.AutoSpacesForTabs">false</value>
|
||||
<value type="bool" key="EditorConfiguration.CamelCaseNavigation">true</value>
|
||||
<valuemap type="QVariantMap" key="EditorConfiguration.CodeStyle.0">
|
||||
<value type="QString" key="language">Cpp</value>
|
||||
<valuemap type="QVariantMap" key="value">
|
||||
<value type="QByteArray" key="CurrentPreferences">CppGlobal</value>
|
||||
</valuemap>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="EditorConfiguration.CodeStyle.1">
|
||||
<value type="QString" key="language">QmlJS</value>
|
||||
<valuemap type="QVariantMap" key="value">
|
||||
<value type="QByteArray" key="CurrentPreferences">QmlJSGlobal</value>
|
||||
</valuemap>
|
||||
</valuemap>
|
||||
<value type="qlonglong" key="EditorConfiguration.CodeStyle.Count">2</value>
|
||||
<value type="QByteArray" key="EditorConfiguration.Codec">UTF-8</value>
|
||||
<value type="bool" key="EditorConfiguration.ConstrainTooltips">false</value>
|
||||
<value type="int" key="EditorConfiguration.IndentSize">4</value>
|
||||
<value type="bool" key="EditorConfiguration.KeyboardTooltips">false</value>
|
||||
<value type="int" key="EditorConfiguration.MarginColumn">80</value>
|
||||
<value type="bool" key="EditorConfiguration.MouseHiding">true</value>
|
||||
<value type="bool" key="EditorConfiguration.MouseNavigation">true</value>
|
||||
<value type="int" key="EditorConfiguration.PaddingMode">1</value>
|
||||
<value type="bool" key="EditorConfiguration.PreferSingleLineComments">false</value>
|
||||
<value type="bool" key="EditorConfiguration.ScrollWheelZooming">true</value>
|
||||
<value type="bool" key="EditorConfiguration.ShowMargin">false</value>
|
||||
<value type="int" key="EditorConfiguration.SmartBackspaceBehavior">0</value>
|
||||
<value type="bool" key="EditorConfiguration.SmartSelectionChanging">true</value>
|
||||
<value type="bool" key="EditorConfiguration.SpacesForTabs">true</value>
|
||||
<value type="int" key="EditorConfiguration.TabKeyBehavior">0</value>
|
||||
<value type="int" key="EditorConfiguration.TabSize">8</value>
|
||||
<value type="bool" key="EditorConfiguration.UseGlobal">true</value>
|
||||
<value type="bool" key="EditorConfiguration.UseIndenter">false</value>
|
||||
<value type="int" key="EditorConfiguration.Utf8BomBehavior">1</value>
|
||||
<value type="bool" key="EditorConfiguration.addFinalNewLine">true</value>
|
||||
<value type="bool" key="EditorConfiguration.cleanIndentation">true</value>
|
||||
<value type="bool" key="EditorConfiguration.cleanWhitespace">true</value>
|
||||
<value type="QString" key="EditorConfiguration.ignoreFileTypes">*.md, *.MD, Makefile</value>
|
||||
<value type="bool" key="EditorConfiguration.inEntireDocument">false</value>
|
||||
<value type="bool" key="EditorConfiguration.skipTrailingWhitespace">true</value>
|
||||
<value type="bool" key="EditorConfiguration.tintMarginArea">true</value>
|
||||
</valuemap>
|
||||
</data>
|
||||
<data>
|
||||
<variable>ProjectExplorer.Project.PluginSettings</variable>
|
||||
<valuemap type="QVariantMap">
|
||||
<valuemap type="QVariantMap" key="AutoTest.ActiveFrameworks">
|
||||
<value type="bool" key="AutoTest.Framework.Boost">true</value>
|
||||
<value type="bool" key="AutoTest.Framework.CTest">false</value>
|
||||
<value type="bool" key="AutoTest.Framework.Catch">true</value>
|
||||
<value type="bool" key="AutoTest.Framework.GTest">true</value>
|
||||
<value type="bool" key="AutoTest.Framework.QtQuickTest">true</value>
|
||||
<value type="bool" key="AutoTest.Framework.QtTest">true</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="AutoTest.CheckStates"/>
|
||||
<value type="int" key="AutoTest.RunAfterBuild">0</value>
|
||||
<value type="bool" key="AutoTest.UseGlobal">true</value>
|
||||
<valuemap type="QVariantMap" key="ClangTools">
|
||||
<value type="bool" key="ClangTools.AnalyzeOpenFiles">true</value>
|
||||
<value type="bool" key="ClangTools.BuildBeforeAnalysis">true</value>
|
||||
<value type="QString" key="ClangTools.DiagnosticConfig">Builtin.DefaultTidyAndClazy</value>
|
||||
<value type="int" key="ClangTools.ParallelJobs">6</value>
|
||||
<valuelist type="QVariantList" key="ClangTools.SelectedDirs"/>
|
||||
<valuelist type="QVariantList" key="ClangTools.SelectedFiles"/>
|
||||
<valuelist type="QVariantList" key="ClangTools.SuppressedDiagnostics"/>
|
||||
<value type="bool" key="ClangTools.UseGlobalSettings">true</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="CppEditor.QuickFix">
|
||||
<value type="bool" key="UseGlobalSettings">true</value>
|
||||
</valuemap>
|
||||
</valuemap>
|
||||
</data>
|
||||
<data>
|
||||
<variable>ProjectExplorer.Project.Target.0</variable>
|
||||
<valuemap type="QVariantMap">
|
||||
<value type="QString" key="DeviceType">Desktop</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">Clang</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">Clang</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">{913660d6-ca1c-4b66-a4da-64108a3258a2}</value>
|
||||
<value type="qlonglong" key="ProjectExplorer.Target.ActiveBuildConfiguration">0</value>
|
||||
<value type="qlonglong" key="ProjectExplorer.Target.ActiveDeployConfiguration">0</value>
|
||||
<value type="qlonglong" key="ProjectExplorer.Target.ActiveRunConfiguration">0</value>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.BuildConfiguration.0">
|
||||
<value type="QString" key="CMake.Build.Type">Release</value>
|
||||
<value type="bool" key="CMake.Configure.ClearSystemEnvironment">false</value>
|
||||
<valuelist type="QVariantList" key="CMake.Configure.UserEnvironmentChanges"/>
|
||||
<value type="QString" key="CMake.Initial.Parameters">-DCMAKE_GENERATOR:STRING=Unix Makefiles
|
||||
-DCMAKE_BUILD_TYPE:STRING=Release
|
||||
-DQT_QMAKE_EXECUTABLE:STRING=%{Qt:qmakeExecutable}
|
||||
-DCMAKE_PREFIX_PATH:STRING=%{Qt:QT_INSTALL_PREFIX}
|
||||
-DCMAKE_C_COMPILER:STRING=%{Compiler:Executable:C}
|
||||
-DCMAKE_CXX_COMPILER:STRING=%{Compiler:Executable:Cxx}
|
||||
-DCMAKE_CXX_FLAGS_INIT:STRING=%{Qt:QML_DEBUG_FLAG}</value>
|
||||
<value type="QString" key="ProjectExplorer.BuildConfiguration.BuildDirectory">/mnt/hhdd/Programme/OSS/llama.cpp/../build-llama.cpp-Clang-Release</value>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.BuildConfiguration.BuildStepList.0">
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.BuildStepList.Step.0">
|
||||
<value type="QString" key="CMakeProjectManager.MakeStep.BuildPreset"></value>
|
||||
<valuelist type="QVariantList" key="CMakeProjectManager.MakeStep.BuildTargets">
|
||||
<value type="QString">all</value>
|
||||
</valuelist>
|
||||
<value type="bool" key="CMakeProjectManager.MakeStep.ClearSystemEnvironment">false</value>
|
||||
<valuelist type="QVariantList" key="CMakeProjectManager.MakeStep.UserEnvironmentChanges"/>
|
||||
<value type="bool" key="ProjectExplorer.BuildStep.Enabled">true</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">Build</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.MakeStep</value>
|
||||
</valuemap>
|
||||
<value type="qlonglong" key="ProjectExplorer.BuildStepList.StepsCount">1</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">Build</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">Build</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">ProjectExplorer.BuildSteps.Build</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.BuildConfiguration.BuildStepList.1">
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.BuildStepList.Step.0">
|
||||
<value type="QString" key="CMakeProjectManager.MakeStep.BuildPreset"></value>
|
||||
<valuelist type="QVariantList" key="CMakeProjectManager.MakeStep.BuildTargets">
|
||||
<value type="QString">clean</value>
|
||||
</valuelist>
|
||||
<value type="bool" key="CMakeProjectManager.MakeStep.ClearSystemEnvironment">false</value>
|
||||
<valuelist type="QVariantList" key="CMakeProjectManager.MakeStep.UserEnvironmentChanges"/>
|
||||
<value type="bool" key="ProjectExplorer.BuildStep.Enabled">true</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">Build</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.MakeStep</value>
|
||||
</valuemap>
|
||||
<value type="qlonglong" key="ProjectExplorer.BuildStepList.StepsCount">1</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">Clean</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">Clean</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">ProjectExplorer.BuildSteps.Clean</value>
|
||||
</valuemap>
|
||||
<value type="int" key="ProjectExplorer.BuildConfiguration.BuildStepListCount">2</value>
|
||||
<value type="bool" key="ProjectExplorer.BuildConfiguration.ClearSystemEnvironment">false</value>
|
||||
<valuelist type="QVariantList" key="ProjectExplorer.BuildConfiguration.CustomParsers"/>
|
||||
<value type="bool" key="ProjectExplorer.BuildConfiguration.ParseStandardOutput">false</value>
|
||||
<valuelist type="QVariantList" key="ProjectExplorer.BuildConfiguration.UserEnvironmentChanges"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">Release</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeBuildConfiguration</value>
|
||||
</valuemap>
|
||||
<value type="qlonglong" key="ProjectExplorer.Target.BuildConfigurationCount">1</value>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.DeployConfiguration.0">
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.BuildConfiguration.BuildStepList.0">
|
||||
<value type="qlonglong" key="ProjectExplorer.BuildStepList.StepsCount">0</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DefaultDisplayName">Deploy</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">Deploy</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">ProjectExplorer.BuildSteps.Deploy</value>
|
||||
</valuemap>
|
||||
<value type="int" key="ProjectExplorer.BuildConfiguration.BuildStepListCount">1</value>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.DeployConfiguration.CustomData"/>
|
||||
<value type="bool" key="ProjectExplorer.DeployConfiguration.CustomDataEnabled">false</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">ProjectExplorer.DefaultDeployConfiguration</value>
|
||||
</valuemap>
|
||||
<value type="qlonglong" key="ProjectExplorer.Target.DeployConfigurationCount">1</value>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.0">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">baby-llama</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.baby-llama</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">baby-llama</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.1">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">benchmark</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.benchmark</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">benchmark</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.10">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">test-quantize-perf</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.test-quantize-perf</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">test-quantize-perf</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.11">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">test-sampling</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.test-sampling</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">test-sampling</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.12">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">test-tokenizer-0</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.test-tokenizer-0</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">test-tokenizer-0</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.13">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">train-text-from-scratch</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.train-text-from-scratch</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">train-text-from-scratch</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.14">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">vdot</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.vdot</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">vdot</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.15">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">simple</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.simple</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">simple</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.2">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">embedding</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.embedding</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">embedding</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.3">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">main</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.main</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">main</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.4">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">perplexity</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.perplexity</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">perplexity</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.5">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">q8dot</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.q8dot</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">q8dot</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.6">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">quantize</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.quantize</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">quantize</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.7">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">quantize-stats</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.quantize-stats</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">quantize-stats</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.8">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">save-load-state</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.save-load-state</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">save-load-state</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<valuemap type="QVariantMap" key="ProjectExplorer.Target.RunConfiguration.9">
|
||||
<value type="bool" key="Analyzer.Perf.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.QmlProfiler.Settings.UseGlobalSettings">true</value>
|
||||
<value type="bool" key="Analyzer.Valgrind.Settings.UseGlobalSettings">true</value>
|
||||
<valuelist type="QVariantList" key="CustomOutputParsers"/>
|
||||
<value type="int" key="PE.EnvironmentAspect.Base">2</value>
|
||||
<valuelist type="QVariantList" key="PE.EnvironmentAspect.Changes"/>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.DisplayName">test-quantize-fns</value>
|
||||
<value type="QString" key="ProjectExplorer.ProjectConfiguration.Id">CMakeProjectManager.CMakeRunConfiguration.test-quantize-fns</value>
|
||||
<value type="QString" key="ProjectExplorer.RunConfiguration.BuildKey">test-quantize-fns</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseCppDebuggerAuto">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseLibrarySearchPath">true</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebugger">false</value>
|
||||
<value type="bool" key="RunConfiguration.UseQmlDebuggerAuto">true</value>
|
||||
<value type="QString" key="RunConfiguration.WorkingDirectory.default">/mnt/hhdd/Programme/OSS/build-llama.cpp-Clang-Release/bin</value>
|
||||
</valuemap>
|
||||
<value type="qlonglong" key="ProjectExplorer.Target.RunConfigurationCount">16</value>
|
||||
</valuemap>
|
||||
</data>
|
||||
<data>
|
||||
<variable>ProjectExplorer.Project.TargetCount</variable>
|
||||
<value type="qlonglong">1</value>
|
||||
</data>
|
||||
<data>
|
||||
<variable>ProjectExplorer.Project.Updater.FileVersion</variable>
|
||||
<value type="int">22</value>
|
||||
</data>
|
||||
<data>
|
||||
<variable>Version</variable>
|
||||
<value type="int">22</value>
|
||||
</data>
|
||||
</qtcreator>
|
||||
151
ggml-vulkan.cpp
Normal file
151
ggml-vulkan.cpp
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
#include "ggml-vulkan.h"
|
||||
#include "ggml.h"
|
||||
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <kompute/Kompute.hpp>
|
||||
|
||||
typedef ggml_fp16_t half;
|
||||
|
||||
#define MULTILINE_QUOTE(...) #__VA_ARGS__
|
||||
#define STRINGIFY(x) STRINGIFY2(x)
|
||||
#define STRINGIFY2(x) #x
|
||||
|
||||
#define QK4_0 32
|
||||
#define QR4_0 2
|
||||
#define QK4_1 32
|
||||
|
||||
typedef struct {
|
||||
half d;
|
||||
uint8_t qs[QK4_0 / 2];
|
||||
} block_q4_0;
|
||||
|
||||
typedef struct {
|
||||
half d;
|
||||
half m;
|
||||
uint8_t qs[QK4_1 / 2];
|
||||
} block_q4_1;
|
||||
|
||||
|
||||
kp::Manager mgr;
|
||||
|
||||
|
||||
|
||||
static const std::string program_source_head = R"(
|
||||
#version 450
|
||||
#extension GL_EXT_shader_explicit_arithmetic_types_float16: enable
|
||||
#extension GL_EXT_shader_explicit_arithmetic_types_int8: enable
|
||||
#define QK4_0 32
|
||||
#define QR4_0 2
|
||||
#define QK4_1 32
|
||||
layout (local_size_x = 1) in;
|
||||
)";
|
||||
|
||||
|
||||
static const std::string kernel_dequantize_row_q4_0 =
|
||||
program_source_head+'\n'+MULTILINE_QUOTE(
|
||||
// Tensors
|
||||
layout(binding = 0) buffer tensorBlockQ4_0D { float16_t x_d[]; };
|
||||
layout(binding = 1) buffer tensorBlockQ4_0QS { uint8_t x_qs[]; };
|
||||
layout(binding = 2) buffer tensorY { float y[]; };
|
||||
|
||||
// Push constants
|
||||
layout(push_constant) uniform PushConstants {
|
||||
int k;
|
||||
} pcs;
|
||||
|
||||
void main() {
|
||||
const int qk = QK4_0;
|
||||
|
||||
const int i = int(gl_GlobalInvocationID.x);
|
||||
const int j = int(gl_GlobalInvocationID.y);
|
||||
|
||||
const float16_t d = x_d[i];
|
||||
const uint8_t qs = x_qs[i * (QK4_0 / 2) + j];
|
||||
|
||||
const int x0 = (qs & 0x0F) - 8;
|
||||
const int x1 = (qs >> 4) - 8;
|
||||
|
||||
y[i*qk + j + 0 ] = float16_t(x0)*d;
|
||||
y[i*qk + j + qk/2] = float16_t(x1)*d;
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
std::vector<uint32_t> compileSource(const std::string& source) {
|
||||
//FIXME: Terrible solution!!!!
|
||||
std::ofstream fileOut("tmp_kp_shader.comp");
|
||||
fileOut << source;
|
||||
fileOut.close();
|
||||
if (system(std::string("glslangValidator -V tmp_kp_shader.comp -o tmp_kp_shader.comp.spv").c_str()))
|
||||
throw std::runtime_error("Error running glslangValidator command");
|
||||
std::ifstream fileStream("tmp_kp_shader.comp.spv", std::ios::binary);
|
||||
std::vector<char> buffer;
|
||||
buffer.insert(buffer.begin(), std::istreambuf_iterator<char>(fileStream), {});
|
||||
return {(uint32_t*)buffer.data(), (uint32_t*)(buffer.data() + buffer.size())};
|
||||
}
|
||||
|
||||
void ggml_vk_dequantize_row_q4_0(const void *x_, float *y, int k) {
|
||||
static const int qk = QK4_0;
|
||||
static const unsigned nb = k / qk;
|
||||
static const unsigned y_size = qk*2*nb;
|
||||
const static auto spirv = compileSource(kernel_dequantize_row_q4_0);
|
||||
|
||||
const auto x = reinterpret_cast<const block_q4_0*>(x_);
|
||||
|
||||
auto getVecBlockQ4_0D = [] (const block_q4_0 *x) {
|
||||
std::vector<half> fres;
|
||||
fres.reserve(nb);
|
||||
for (unsigned it = 0; it != nb; it++) {
|
||||
fres.push_back(x[it].d);
|
||||
}
|
||||
return fres;
|
||||
};
|
||||
auto getVecBlockQ4_0QS = [] (const block_q4_0 *x) {
|
||||
std::vector<uint8_t> fres;
|
||||
fres.resize(nb*(qk/2));
|
||||
for (unsigned x_it = 0; x_it != nb; x_it++) {
|
||||
for (unsigned qs_it = 0; qs_it != qk / 2; qs_it++) {
|
||||
fres.push_back(x[x_it].qs[qs_it]);
|
||||
}
|
||||
}
|
||||
return fres;
|
||||
};
|
||||
|
||||
const auto tensorBlockQ4_0D = mgr.tensorT<half>(getVecBlockQ4_0D(x));
|
||||
const auto tensorBlockQ4_0QS = mgr.tensorT<uint8_t>(getVecBlockQ4_0QS(x));
|
||||
const auto tensorY = mgr.tensor(std::vector<float>(y, y+y_size));
|
||||
|
||||
struct PushConsts {
|
||||
int k;
|
||||
} pushConsts {
|
||||
k
|
||||
};
|
||||
|
||||
mgr.sequence()
|
||||
->record<kp::OpTensorSyncDevice>({tensorBlockQ4_0D, tensorBlockQ4_0QS, tensorY})
|
||||
->record<kp::OpAlgoDispatch>(mgr.algorithm({tensorBlockQ4_0D, tensorBlockQ4_0QS, tensorY}, spirv, {nb, qk/2, 0}, {}, {0}), std::vector<PushConsts>{pushConsts})
|
||||
->record<kp::OpTensorSyncLocal>({tensorY})
|
||||
->eval();
|
||||
|
||||
std::memcpy(y, tensorY->data(), tensorY->size());
|
||||
}
|
||||
|
||||
|
||||
template<>
|
||||
kp::Tensor::TensorDataTypes
|
||||
kp::TensorT<half>::dataType()
|
||||
{
|
||||
return TensorDataTypes::eFloat;
|
||||
}
|
||||
|
||||
template<>
|
||||
kp::Tensor::TensorDataTypes
|
||||
kp::TensorT<uint8_t>::dataType()
|
||||
{
|
||||
return TensorDataTypes::eUnsignedInt;
|
||||
}
|
||||
13
ggml-vulkan.h
Normal file
13
ggml-vulkan.h
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
#pragma once
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
void ggml_vk_init(void);
|
||||
|
||||
void ggml_vk_dequantize_row_q4_0(const void * x, float * y, int k);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
4
ggml.c
4
ggml.c
|
|
@ -161,6 +161,8 @@ inline static void* ggml_aligned_malloc(size_t size) {
|
|||
#endif
|
||||
#elif defined(GGML_USE_OPENBLAS)
|
||||
#include <cblas.h>
|
||||
#elif defined(GGML_USE_KOMPUTE)
|
||||
#include "ggml-vulkan.h"
|
||||
#elif defined(GGML_USE_CUBLAS)
|
||||
#include "ggml-cuda.h"
|
||||
#elif defined(GGML_USE_CLBLAST)
|
||||
|
|
@ -1548,7 +1550,7 @@ static void ggml_vec_dot_q8_0_q8_0(const int n, float * restrict s, const void *
|
|||
|
||||
static const quantize_fns_t quantize_fns[GGML_TYPE_COUNT] = {
|
||||
[GGML_TYPE_Q4_0] = {
|
||||
.dequantize_row_q = (dequantize_row_q_t) dequantize_row_q4_0,
|
||||
.dequantize_row_q = (dequantize_row_q_t) ggml_vk_dequantize_row_q4_0,
|
||||
.quantize_row_q = quantize_row_q4_0,
|
||||
.quantize_row_q_reference = (quantize_row_q_t) quantize_row_q4_0_reference,
|
||||
.quantize_row_q_dot = quantize_row_q8_0,
|
||||
|
|
|
|||
1
kompute
Submodule
1
kompute
Submodule
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 63567a72be6b26f79da92becaffa7cd55f46642b
|
||||
Loading…
Add table
Add a link
Reference in a new issue