vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

style: format with nixfmt/rfc101-style

Browse source
This commit is contained in:
ditsuke 2024-02-22 23:55:55 +05:30
parent c6d4cb4655
commit 0126788271
No known key found for this signature in database
GPG key ID: 71B6C31C8A5A9D21
5 changed files with 213 additions and 224 deletions
Download patch file Download diff file Expand all files Collapse all files

11
.devops/nix/devshells.nix
View file

@ -2,12 +2,9 @@
perSystem = perSystem =
{ config, lib, ... }: { config, lib, ... }:
{ {
devShells = devShells = lib.concatMapAttrs (name: package: {
lib.concatMapAttrs ${name} = package.passthru.shell;
(name: package: { ${name + "-extra"} = package.passthru.shell-extra;
${name} = package.passthru.shell; }) config.packages;
${name + "-extra"} = package.passthru.shell-extra;
})
config.packages;
}; };
} }

18
.devops/nix/nixpkgs-instances.nix
View file

@ -26,16 +26,14 @@
config.cudaSupport = true; config.cudaSupport = true;
config.allowUnfreePredicate = config.allowUnfreePredicate =
p: p:
builtins.all builtins.all (
( license:
license: license.free
license.free || builtins.elem license.shortName [
|| builtins.elem license.shortName [ "CUDA EULA"
"CUDA EULA" "cuDNN EULA"
"cuDNN EULA" ]
] ) (p.meta.licenses or [ p.meta.license ]);
)
(p.meta.licenses or [ p.meta.license ]);
}; };
# Ensure dependencies use ROCm consistently # Ensure dependencies use ROCm consistently
pkgsRocm = import inputs.nixpkgs { pkgsRocm = import inputs.nixpkgs {

388
.devops/nix/package.nix
View file

@ -20,12 +20,14 @@
vulkan-loader, vulkan-loader,
curl, curl,
shaderc, shaderc,
useBlas ? builtins.all (x: !x) [ useBlas ?
useCuda builtins.all (x: !x) [
useMetalKit useCuda
useRocm useMetalKit
useVulkan useRocm
] && blas.meta.available, useVulkan
]
&& blas.meta.available,
useCuda ? config.cudaSupport, useCuda ? config.cudaSupport,
useMetalKit ? stdenv.isAarch64 && stdenv.isDarwin, useMetalKit ? stdenv.isAarch64 && stdenv.isDarwin,
useMpi ? false, # Increases the runtime closure size by ~700M useMpi ? false, # Increases the runtime closure size by ~700M
@ -38,7 +40,7 @@
# otherwise we get libstdc++ errors downstream. # otherwise we get libstdc++ errors downstream.
effectiveStdenv ? if useCuda then cudaPackages.backendStdenv else stdenv, effectiveStdenv ? if useCuda then cudaPackages.backendStdenv else stdenv,
enableStatic ? effectiveStdenv.hostPlatform.isStatic, enableStatic ? effectiveStdenv.hostPlatform.isStatic,
precompileMetalShaders ? false precompileMetalShaders ? false,
}@inputs: }@inputs:
let let
@ -63,9 +65,9 @@ let
pnameSuffix = pnameSuffix =
strings.optionalString (suffices != [ ]) strings.optionalString (suffices != [ ])
"-${strings.concatMapStringsSep "-" strings.toLower suffices}"; "-${strings.concatMapStringsSep "-" strings.toLower suffices}";
descriptionSuffix = descriptionSuffix = strings.optionalString (
strings.optionalString (suffices != [ ]) suffices != [ ]
", accelerated with ${strings.concatStringsSep ", " suffices}"; ) ", accelerated with ${strings.concatStringsSep ", " suffices}";
executableSuffix = effectiveStdenv.hostPlatform.extensions.executable; executableSuffix = effectiveStdenv.hostPlatform.extensions.executable;
@ -76,41 +78,37 @@ let
# #
# TODO: Package up each Python script or service appropriately, by making # TODO: Package up each Python script or service appropriately, by making
# them into "entrypoints" # them into "entrypoints"
llama-python = python3.withPackages ( llama-python = python3.withPackages (ps: [
ps: [ ps.numpy
ps.numpy ps.sentencepiece
ps.sentencepiece ]);
]
);
# TODO(Green-Sky): find a better way to opt-into the heavy ml python runtime # TODO(Green-Sky): find a better way to opt-into the heavy ml python runtime
llama-python-extra = python3.withPackages ( llama-python-extra = python3.withPackages (ps: [
ps: [ ps.numpy
ps.numpy ps.sentencepiece
ps.sentencepiece ps.tiktoken
ps.tiktoken ps.torchWithoutCuda
ps.torchWithoutCuda ps.transformers
ps.transformers
# server bench # server bench
ps.matplotlib ps.matplotlib
# server tests # server tests
ps.openai ps.openai
ps.behave ps.behave
ps.prometheus-client ps.prometheus-client
# for examples/pydantic-models-to-grammar-examples.py # for examples/pydantic-models-to-grammar-examples.py
ps.docstring-parser ps.docstring-parser
ps.pydantic ps.pydantic
# for scripts/compare-llama-bench.py # for scripts/compare-llama-bench.py
ps.gitpython ps.gitpython
ps.tabulate ps.tabulate
] ]);
);
xcrunHost = runCommand "xcrunHost" {} '' xcrunHost = runCommand "xcrunHost" { } ''
mkdir -p $out/bin mkdir -p $out/bin
ln -s /usr/bin/xcrun $out/bin ln -s /usr/bin/xcrun $out/bin
''; '';
@ -145,178 +143,174 @@ let
]; ];
in in
effectiveStdenv.mkDerivation ( effectiveStdenv.mkDerivation (finalAttrs: {
finalAttrs: { pname = "llama-cpp${pnameSuffix}";
pname = "llama-cpp${pnameSuffix}"; version = llamaVersion;
version = llamaVersion;
# Note: none of the files discarded here are visible in the sandbox or # Note: none of the files discarded here are visible in the sandbox or
# affect the output hash. This also means they can be modified without # affect the output hash. This also means they can be modified without
# triggering a rebuild. # triggering a rebuild.
src = lib.cleanSourceWith { src = lib.cleanSourceWith {
filter = filter =
name: type: name: type:
let let
noneOf = builtins.all (x: !x); noneOf = builtins.all (x: !x);
baseName = baseNameOf name; baseName = baseNameOf name;
in in
noneOf [ noneOf [
(lib.hasSuffix ".nix" name) # Ignore *.nix files when computing outPaths (lib.hasSuffix ".nix" name) # Ignore *.nix files when computing outPaths
(lib.hasSuffix ".md" name) # Ignore *.md changes whe computing outPaths (lib.hasSuffix ".md" name) # Ignore *.md changes whe computing outPaths
(lib.hasPrefix "." baseName) # Skip hidden files and directories (lib.hasPrefix "." baseName) # Skip hidden files and directories
(baseName == "flake.lock") (baseName == "flake.lock")
];
src = lib.cleanSource ../../.;
};
postPatch = ''
substituteInPlace ./ggml/src/ggml-metal.m \
--replace '[bundle pathForResource:@"ggml-metal" ofType:@"metal"];' "@\"$out/bin/ggml-metal.metal\";"
substituteInPlace ./ggml/src/ggml-metal.m \
--replace '[bundle pathForResource:@"default" ofType:@"metallib"];' "@\"$out/bin/default.metallib\";"
'';
# With PR#6015 https://github.com/ggerganov/llama.cpp/pull/6015,
# `default.metallib` may be compiled with Metal compiler from XCode
# and we need to escape sandbox on MacOS to access Metal compiler.
# `xcrun` is used find the path of the Metal compiler, which is varible
# and not on $PATH
# see https://github.com/ggerganov/llama.cpp/pull/6118 for discussion
__noChroot = effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders;
nativeBuildInputs =
[
cmake
ninja
pkg-config
git
]
++ optionals useCuda [
cudaPackages.cuda_nvcc
autoAddDriverRunpath
]
++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [
glibc.static
] ++ optionals (effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders) [
xcrunHost
]; ];
src = lib.cleanSource ../../.;
};
buildInputs = postPatch = ''
optionals effectiveStdenv.isDarwin darwinBuildInputs substituteInPlace ./ggml/src/ggml-metal.m \
++ optionals useCuda cudaBuildInputs --replace '[bundle pathForResource:@"ggml-metal" ofType:@"metal"];' "@\"$out/bin/ggml-metal.metal\";"
++ optionals useMpi [ mpi ] substituteInPlace ./ggml/src/ggml-metal.m \
++ optionals useRocm rocmBuildInputs --replace '[bundle pathForResource:@"default" ofType:@"metallib"];' "@\"$out/bin/default.metallib\";"
++ optionals useBlas [ blas ] '';
++ optionals useVulkan vulkanBuildInputs
++ optionals enableCurl [ curl ];
cmakeFlags = # With PR#6015 https://github.com/ggerganov/llama.cpp/pull/6015,
[ # `default.metallib` may be compiled with Metal compiler from XCode
(cmakeBool "LLAMA_BUILD_SERVER" true) # and we need to escape sandbox on MacOS to access Metal compiler.
(cmakeBool "BUILD_SHARED_LIBS" (!enableStatic)) # `xcrun` is used find the path of the Metal compiler, which is varible
(cmakeBool "CMAKE_SKIP_BUILD_RPATH" true) # and not on $PATH
(cmakeBool "LLAMA_CURL" enableCurl) # see https://github.com/ggerganov/llama.cpp/pull/6118 for discussion
(cmakeBool "GGML_NATIVE" false) __noChroot = effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders;
(cmakeBool "GGML_BLAS" useBlas)
(cmakeBool "GGML_CUDA" useCuda) nativeBuildInputs =
(cmakeBool "GGML_HIPBLAS" useRocm) [
(cmakeBool "GGML_METAL" useMetalKit) cmake
(cmakeBool "GGML_VULKAN" useVulkan) ninja
(cmakeBool "GGML_STATIC" enableStatic) pkg-config
] git
++ optionals useCuda [ ]
( ++ optionals useCuda [
with cudaPackages.flags; cudaPackages.cuda_nvcc
cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
builtins.concatStringsSep ";" (map dropDot cudaCapabilities) autoAddDriverRunpath
) ]
++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static ]
++ optionals (effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];
buildInputs =
optionals effectiveStdenv.isDarwin darwinBuildInputs
++ optionals useCuda cudaBuildInputs
++ optionals useMpi [ mpi ]
++ optionals useRocm rocmBuildInputs
++ optionals useBlas [ blas ]
++ optionals useVulkan vulkanBuildInputs
++ optionals enableCurl [ curl ];
cmakeFlags =
[
(cmakeBool "LLAMA_BUILD_SERVER" true)
(cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))
(cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
(cmakeBool "LLAMA_CURL" enableCurl)
(cmakeBool "GGML_NATIVE" false)
(cmakeBool "GGML_BLAS" useBlas)
(cmakeBool "GGML_CUDA" useCuda)
(cmakeBool "GGML_HIPBLAS" useRocm)
(cmakeBool "GGML_METAL" useMetalKit)
(cmakeBool "GGML_VULKAN" useVulkan)
(cmakeBool "GGML_STATIC" enableStatic)
]
++ optionals useCuda [
(
with cudaPackages.flags;
cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
builtins.concatStringsSep ";" (map dropDot cudaCapabilities)
) )
] )
++ optionals useRocm [ ]
(cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang") ++ optionals useRocm [
(cmakeFeature "CMAKE_HIP_ARCHITECTURES" (builtins.concatStringsSep ";" rocmPackages.clr.gpuTargets)) (cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")
] (cmakeFeature "CMAKE_HIP_ARCHITECTURES" (builtins.concatStringsSep ";" rocmPackages.clr.gpuTargets))
++ optionals useMetalKit [ ]
(lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1") ++ optionals useMetalKit [
(cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders)) (lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")
]; (cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders))
];
# Environment variables needed for ROCm # Environment variables needed for ROCm
env = optionals useRocm { env = optionals useRocm {
ROCM_PATH = "${rocmPackages.clr}"; ROCM_PATH = "${rocmPackages.clr}";
HIP_DEVICE_LIB_PATH = "${rocmPackages.rocm-device-libs}/amdgcn/bitcode"; HIP_DEVICE_LIB_PATH = "${rocmPackages.rocm-device-libs}/amdgcn/bitcode";
};
# TODO(SomeoneSerge): It's better to add proper install targets at the CMake level,
# if they haven't been added yet.
postInstall = ''
mkdir -p $out/include
cp $src/include/llama.h $out/include/
'';
# Define the shells here, but don't add in the inputsFrom to avoid recursion.
passthru = {
inherit
useBlas
useCuda
useMetalKit
useMpi
useRocm
useVulkan
;
shell = mkShell {
name = "shell-${finalAttrs.finalPackage.name}";
description = "contains numpy and sentencepiece";
buildInputs = [ llama-python ];
inputsFrom = [ finalAttrs.finalPackage ];
shellHook = ''
addToSearchPath "LD_LIBRARY_PATH" "${lib.getLib effectiveStdenv.cc.cc}/lib"
'';
}; };
# TODO(SomeoneSerge): It's better to add proper install targets at the CMake level, shell-extra = mkShell {
# if they haven't been added yet. name = "shell-extra-${finalAttrs.finalPackage.name}";
postInstall = '' description = "contains numpy, sentencepiece, torchWithoutCuda, and transformers";
mkdir -p $out/include buildInputs = [ llama-python-extra ];
cp $src/include/llama.h $out/include/ inputsFrom = [ finalAttrs.finalPackage ];
'';
# Define the shells here, but don't add in the inputsFrom to avoid recursion.
passthru = {
inherit
useBlas
useCuda
useMetalKit
useMpi
useRocm
useVulkan
;
shell = mkShell {
name = "shell-${finalAttrs.finalPackage.name}";
description = "contains numpy and sentencepiece";
buildInputs = [ llama-python ];
inputsFrom = [ finalAttrs.finalPackage ];
shellHook = ''
addToSearchPath "LD_LIBRARY_PATH" "${lib.getLib effectiveStdenv.cc.cc}/lib"
'';
};
shell-extra = mkShell {
name = "shell-extra-${finalAttrs.finalPackage.name}";
description = "contains numpy, sentencepiece, torchWithoutCuda, and transformers";
buildInputs = [ llama-python-extra ];
inputsFrom = [ finalAttrs.finalPackage ];
};
}; };
};
meta = { meta = {
# Configurations we don't want even the CI to evaluate. Results in the # Configurations we don't want even the CI to evaluate. Results in the
# "unsupported platform" messages. This is mostly a no-op, because # "unsupported platform" messages. This is mostly a no-op, because
# cudaPackages would've refused to evaluate anyway. # cudaPackages would've refused to evaluate anyway.
badPlatforms = optionals useCuda lib.platforms.darwin; badPlatforms = optionals useCuda lib.platforms.darwin;
# Configurations that are known to result in build failures. Can be # Configurations that are known to result in build failures. Can be
# overridden by importing Nixpkgs with `allowBroken = true`. # overridden by importing Nixpkgs with `allowBroken = true`.
broken = (useMetalKit && !effectiveStdenv.isDarwin); broken = (useMetalKit && !effectiveStdenv.isDarwin);
description = "Inference of LLaMA model in pure C/C++${descriptionSuffix}"; description = "Inference of LLaMA model in pure C/C++${descriptionSuffix}";
homepage = "https://github.com/ggerganov/llama.cpp/"; homepage = "https://github.com/ggerganov/llama.cpp/";
license = lib.licenses.mit; license = lib.licenses.mit;
# Accommodates `nix run` and `lib.getExe` # Accommodates `nix run` and `lib.getExe`
mainProgram = "llama-cli"; mainProgram = "llama-cli";
# These people might respond, on the best effort basis, if you ping them # These people might respond, on the best effort basis, if you ping them
# in case of Nix-specific regressions or for reviewing Nix-specific PRs. # in case of Nix-specific regressions or for reviewing Nix-specific PRs.
# Consider adding yourself to this list if you want to ensure this flake # Consider adding yourself to this list if you want to ensure this flake
# stays maintained and you're willing to invest your time. Do not add # stays maintained and you're willing to invest your time. Do not add
# other people without their consent. Consider removing people after # other people without their consent. Consider removing people after
# they've been unreachable for long periods of time. # they've been unreachable for long periods of time.
# Note that lib.maintainers is defined in Nixpkgs, but you may just add # Note that lib.maintainers is defined in Nixpkgs, but you may just add
# an attrset following the same format as in # an attrset following the same format as in
# https://github.com/NixOS/nixpkgs/blob/f36a80e54da29775c78d7eff0e628c2b4e34d1d7/maintainers/maintainer-list.nix # https://github.com/NixOS/nixpkgs/blob/f36a80e54da29775c78d7eff0e628c2b4e34d1d7/maintainers/maintainer-list.nix
maintainers = with lib.maintainers; [ maintainers = with lib.maintainers; [
philiptaron philiptaron
SomeoneSerge SomeoneSerge
]; ];
# Extend `badPlatforms` instead # Extend `badPlatforms` instead
platforms = lib.platforms.all; platforms = lib.platforms.all;
}; };
} })
)

16
.devops/nix/scope.nix
View file

@ -8,12 +8,10 @@
# because it allows users to apply overlays later using `overrideScope'`. # because it allows users to apply overlays later using `overrideScope'`.
# Cf. https://noogle.dev/f/lib/makeScope # Cf. https://noogle.dev/f/lib/makeScope
lib.makeScope newScope ( lib.makeScope newScope (self: {
self: { inherit llamaVersion;
inherit llamaVersion; llama-cpp = self.callPackage ./package.nix { };
llama-cpp = self.callPackage ./package.nix { }; docker = self.callPackage ./docker.nix { };
docker = self.callPackage ./docker.nix { }; docker-min = self.callPackage ./docker.nix { interactive = false; };
docker-min = self.callPackage ./docker.nix { interactive = false; }; sif = self.callPackage ./sif.nix { };
sif = self.callPackage ./sif.nix { }; })
}
)

4
flake.nix
View file

@ -145,7 +145,9 @@
# the same path you would with an overlay. # the same path you would with an overlay.
legacyPackages = { legacyPackages = {
llamaPackages = pkgs.callPackage .devops/nix/scope.nix { inherit llamaVersion; }; llamaPackages = pkgs.callPackage .devops/nix/scope.nix { inherit llamaVersion; };
llamaPackagesWindows = pkgs.pkgsCross.mingwW64.callPackage .devops/nix/scope.nix { inherit llamaVersion; }; llamaPackagesWindows = pkgs.pkgsCross.mingwW64.callPackage .devops/nix/scope.nix {
inherit llamaVersion;
};
llamaPackagesCuda = pkgsCuda.callPackage .devops/nix/scope.nix { inherit llamaVersion; }; llamaPackagesCuda = pkgsCuda.callPackage .devops/nix/scope.nix { inherit llamaVersion; };
llamaPackagesRocm = pkgsRocm.callPackage .devops/nix/scope.nix { inherit llamaVersion; }; llamaPackagesRocm = pkgsRocm.callPackage .devops/nix/scope.nix { inherit llamaVersion; };
}; };