Merge branch 'master' into command_mode
This commit is contained in:
Brian
GitHub
commit
9422668ed4
638 changed files with 323858 additions and 10620 deletions
24
.clang-tidy
Normal file
24
.clang-tidy
Normal file
|
|
@ -0,0 +1,24 @@
|
||||||
|
---
|
||||||
|
Checks: >
|
||||||
|
bugprone-*,
|
||||||
|
-bugprone-easily-swappable-parameters,
|
||||||
|
-bugprone-implicit-widening-of-multiplication-result,
|
||||||
|
-bugprone-misplaced-widening-cast,
|
||||||
|
-bugprone-narrowing-conversions,
|
||||||
|
readability-*,
|
||||||
|
-readability-avoid-unconditional-preprocessor-if,
|
||||||
|
-readability-function-cognitive-complexity,
|
||||||
|
-readability-identifier-length,
|
||||||
|
-readability-implicit-bool-conversion,
|
||||||
|
-readability-magic-numbers,
|
||||||
|
-readability-uppercase-literal-suffix,
|
||||||
|
-readability-simplify-boolean-expr,
|
||||||
|
clang-analyzer-*,
|
||||||
|
-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling,
|
||||||
|
performance-*,
|
||||||
|
portability-*,
|
||||||
|
misc-*,
|
||||||
|
-misc-const-correctness,
|
||||||
|
-misc-non-private-member-variables-in-classes,
|
||||||
|
-misc-no-recursion,
|
||||||
|
FormatStyle: none
|
||||||
22
.devops/cloud-v-pipeline
Normal file
22
.devops/cloud-v-pipeline
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
node('x86_runner1'){ // Running on x86 runner containing latest vector qemu, latest vector gcc and all the necessary libraries
|
||||||
|
stage('Cleanup'){
|
||||||
|
cleanWs() // Cleaning previous CI build in workspace
|
||||||
|
}
|
||||||
|
stage('checkout repo'){
|
||||||
|
retry(5){ // Retry if the cloning fails due to some reason
|
||||||
|
checkout scm // Clone the repo on Runner
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stage('Compiling llama.cpp'){
|
||||||
|
sh'''#!/bin/bash
|
||||||
|
make RISCV=1 RISCV_CROSS_COMPILE=1 # Compiling llama for RISC-V
|
||||||
|
'''
|
||||||
|
}
|
||||||
|
stage('Running llama.cpp'){
|
||||||
|
sh'''#!/bin/bash
|
||||||
|
module load gnu-bin2/0.1 # loading latest versions of vector qemu and vector gcc
|
||||||
|
qemu-riscv64 -L /softwares/gnu-bin2/sysroot -cpu rv64,v=true,vlen=256,elen=64,vext_spec=v1.0 ./main -m /home/alitariq/codellama-7b.Q4_K_M.gguf -p "Anything" -n 9 > llama_log.txt # Running llama.cpp on vector qemu-riscv64
|
||||||
|
cat llama_log.txt # Printing results
|
||||||
|
'''
|
||||||
|
}
|
||||||
|
}
|
||||||
36
.devops/full-cuda.Dockerfile
Normal file
36
.devops/full-cuda.Dockerfile
Normal file
|
|
@ -0,0 +1,36 @@
|
||||||
|
ARG UBUNTU_VERSION=22.04
|
||||||
|
|
||||||
|
# This needs to generally match the container host's environment.
|
||||||
|
ARG CUDA_VERSION=11.7.1
|
||||||
|
|
||||||
|
# Target the CUDA build image
|
||||||
|
ARG BASE_CUDA_DEV_CONTAINER=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}
|
||||||
|
|
||||||
|
FROM ${BASE_CUDA_DEV_CONTAINER} as build
|
||||||
|
|
||||||
|
# Unless otherwise specified, we make a fat build.
|
||||||
|
ARG CUDA_DOCKER_ARCH=all
|
||||||
|
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y build-essential python3 python3-pip git libcurl4-openssl-dev
|
||||||
|
|
||||||
|
COPY requirements.txt requirements.txt
|
||||||
|
COPY requirements requirements
|
||||||
|
|
||||||
|
RUN pip install --upgrade pip setuptools wheel \
|
||||||
|
&& pip install -r requirements.txt
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Set nvcc architecture
|
||||||
|
ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
|
||||||
|
# Enable CUDA
|
||||||
|
ENV LLAMA_CUDA=1
|
||||||
|
# Enable cURL
|
||||||
|
ENV LLAMA_CURL=1
|
||||||
|
|
||||||
|
RUN make
|
||||||
|
|
||||||
|
ENTRYPOINT ["/app/.devops/tools.sh"]
|
||||||
50
.devops/full-rocm.Dockerfile
Normal file
50
.devops/full-rocm.Dockerfile
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
ARG UBUNTU_VERSION=22.04
|
||||||
|
|
||||||
|
# This needs to generally match the container host's environment.
|
||||||
|
ARG ROCM_VERSION=5.6
|
||||||
|
|
||||||
|
# Target the CUDA build image
|
||||||
|
ARG BASE_ROCM_DEV_CONTAINER=rocm/dev-ubuntu-${UBUNTU_VERSION}:${ROCM_VERSION}-complete
|
||||||
|
|
||||||
|
FROM ${BASE_ROCM_DEV_CONTAINER} as build
|
||||||
|
|
||||||
|
# Unless otherwise specified, we make a fat build.
|
||||||
|
# List from https://github.com/ggerganov/llama.cpp/pull/1087#issuecomment-1682807878
|
||||||
|
# This is mostly tied to rocBLAS supported archs.
|
||||||
|
ARG ROCM_DOCKER_ARCH=\
|
||||||
|
gfx803 \
|
||||||
|
gfx900 \
|
||||||
|
gfx906 \
|
||||||
|
gfx908 \
|
||||||
|
gfx90a \
|
||||||
|
gfx1010 \
|
||||||
|
gfx1030 \
|
||||||
|
gfx1100 \
|
||||||
|
gfx1101 \
|
||||||
|
gfx1102
|
||||||
|
|
||||||
|
COPY requirements.txt requirements.txt
|
||||||
|
COPY requirements requirements
|
||||||
|
|
||||||
|
RUN pip install --upgrade pip setuptools wheel \
|
||||||
|
&& pip install -r requirements.txt
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Set nvcc architecture
|
||||||
|
ENV GPU_TARGETS=${ROCM_DOCKER_ARCH}
|
||||||
|
# Enable ROCm
|
||||||
|
ENV LLAMA_HIPBLAS=1
|
||||||
|
ENV CC=/opt/rocm/llvm/bin/clang
|
||||||
|
ENV CXX=/opt/rocm/llvm/bin/clang++
|
||||||
|
|
||||||
|
# Enable cURL
|
||||||
|
ENV LLAMA_CURL=1
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y libcurl4-openssl-dev
|
||||||
|
|
||||||
|
RUN make
|
||||||
|
|
||||||
|
ENTRYPOINT ["/app/.devops/tools.sh"]
|
||||||
|
|
@ -3,9 +3,10 @@ ARG UBUNTU_VERSION=22.04
|
||||||
FROM ubuntu:$UBUNTU_VERSION as build
|
FROM ubuntu:$UBUNTU_VERSION as build
|
||||||
|
|
||||||
RUN apt-get update && \
|
RUN apt-get update && \
|
||||||
apt-get install -y build-essential python3 python3-pip
|
apt-get install -y build-essential python3 python3-pip git libcurl4-openssl-dev
|
||||||
|
|
||||||
COPY requirements.txt requirements.txt
|
COPY requirements.txt requirements.txt
|
||||||
|
COPY requirements requirements
|
||||||
|
|
||||||
RUN pip install --upgrade pip setuptools wheel \
|
RUN pip install --upgrade pip setuptools wheel \
|
||||||
&& pip install -r requirements.txt
|
&& pip install -r requirements.txt
|
||||||
|
|
@ -14,6 +15,11 @@ WORKDIR /app
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
ENV LLAMA_CURL=1
|
||||||
|
|
||||||
|
|
||||||
RUN make
|
RUN make
|
||||||
|
|
||||||
|
ENV LC_ALL=C.utf8
|
||||||
|
|
||||||
ENTRYPOINT ["/app/.devops/tools.sh"]
|
ENTRYPOINT ["/app/.devops/tools.sh"]
|
||||||
|
|
|
||||||
84
.devops/llama-cpp-clblast.srpm.spec
Normal file
84
.devops/llama-cpp-clblast.srpm.spec
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
# SRPM for building from source and packaging an RPM for RPM-based distros.
|
||||||
|
# https://docs.fedoraproject.org/en-US/quick-docs/creating-rpm-packages
|
||||||
|
# Built and maintained by John Boero - boeroboy@gmail.com
|
||||||
|
# In honor of Seth Vidal https://www.redhat.com/it/blog/thank-you-seth-vidal
|
||||||
|
|
||||||
|
# Notes for llama.cpp:
|
||||||
|
# 1. Tags are currently based on hash - which will not sort asciibetically.
|
||||||
|
# We need to declare standard versioning if people want to sort latest releases.
|
||||||
|
# 2. Builds for CUDA/OpenCL support are separate, with different depenedencies.
|
||||||
|
# 3. NVidia's developer repo must be enabled with nvcc, cublas, clblas, etc installed.
|
||||||
|
# Example: https://developer.download.nvidia.com/compute/cuda/repos/fedora37/x86_64/cuda-fedora37.repo
|
||||||
|
# 4. OpenCL/CLBLAST support simply requires the ICD loader and basic opencl libraries.
|
||||||
|
# It is up to the user to install the correct vendor-specific support.
|
||||||
|
|
||||||
|
Name: llama.cpp-clblast
|
||||||
|
Version: %( date "+%%Y%%m%%d" )
|
||||||
|
Release: 1%{?dist}
|
||||||
|
Summary: OpenCL Inference of LLaMA model in C/C++
|
||||||
|
License: MIT
|
||||||
|
Source0: https://github.com/ggerganov/llama.cpp/archive/refs/heads/master.tar.gz
|
||||||
|
BuildRequires: coreutils make gcc-c++ git mesa-libOpenCL-devel clblast-devel
|
||||||
|
Requires: clblast
|
||||||
|
URL: https://github.com/ggerganov/llama.cpp
|
||||||
|
|
||||||
|
%define debug_package %{nil}
|
||||||
|
%define source_date_epoch_from_changelog 0
|
||||||
|
|
||||||
|
%description
|
||||||
|
CPU inference for Meta's Lllama2 models using default options.
|
||||||
|
|
||||||
|
%prep
|
||||||
|
%setup -n llama.cpp-master
|
||||||
|
|
||||||
|
%build
|
||||||
|
make -j LLAMA_CLBLAST=1
|
||||||
|
|
||||||
|
%install
|
||||||
|
mkdir -p %{buildroot}%{_bindir}/
|
||||||
|
cp -p main %{buildroot}%{_bindir}/llamaclblast
|
||||||
|
cp -p server %{buildroot}%{_bindir}/llamaclblastserver
|
||||||
|
cp -p simple %{buildroot}%{_bindir}/llamaclblastsimple
|
||||||
|
|
||||||
|
mkdir -p %{buildroot}/usr/lib/systemd/system
|
||||||
|
%{__cat} <<EOF > %{buildroot}/usr/lib/systemd/system/llamaclblast.service
|
||||||
|
[Unit]
|
||||||
|
Description=Llama.cpp server, CPU only (no GPU support in this build).
|
||||||
|
After=syslog.target network.target local-fs.target remote-fs.target nss-lookup.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
EnvironmentFile=/etc/sysconfig/llama
|
||||||
|
ExecStart=/usr/bin/llamaclblastserver $LLAMA_ARGS
|
||||||
|
ExecReload=/bin/kill -s HUP $MAINPID
|
||||||
|
Restart=never
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=default.target
|
||||||
|
EOF
|
||||||
|
|
||||||
|
mkdir -p %{buildroot}/etc/sysconfig
|
||||||
|
%{__cat} <<EOF > %{buildroot}/etc/sysconfig/llama
|
||||||
|
LLAMA_ARGS="-m /opt/llama2/ggml-model-f32.bin"
|
||||||
|
EOF
|
||||||
|
|
||||||
|
%clean
|
||||||
|
rm -rf %{buildroot}
|
||||||
|
rm -rf %{_builddir}/*
|
||||||
|
|
||||||
|
%files
|
||||||
|
%{_bindir}/llamaclblast
|
||||||
|
%{_bindir}/llamaclblastserver
|
||||||
|
%{_bindir}/llamaclblastsimple
|
||||||
|
/usr/lib/systemd/system/llamaclblast.service
|
||||||
|
%config /etc/sysconfig/llama
|
||||||
|
|
||||||
|
|
||||||
|
%pre
|
||||||
|
|
||||||
|
%post
|
||||||
|
|
||||||
|
%preun
|
||||||
|
%postun
|
||||||
|
|
||||||
|
%changelog
|
||||||
83
.devops/llama-cpp-cuda.srpm.spec
Normal file
83
.devops/llama-cpp-cuda.srpm.spec
Normal file
|
|
@ -0,0 +1,83 @@
|
||||||
|
# SRPM for building from source and packaging an RPM for RPM-based distros.
|
||||||
|
# https://docs.fedoraproject.org/en-US/quick-docs/creating-rpm-packages
|
||||||
|
# Built and maintained by John Boero - boeroboy@gmail.com
|
||||||
|
# In honor of Seth Vidal https://www.redhat.com/it/blog/thank-you-seth-vidal
|
||||||
|
|
||||||
|
# Notes for llama.cpp:
|
||||||
|
# 1. Tags are currently based on hash - which will not sort asciibetically.
|
||||||
|
# We need to declare standard versioning if people want to sort latest releases.
|
||||||
|
# 2. Builds for CUDA/OpenCL support are separate, with different depenedencies.
|
||||||
|
# 3. NVidia's developer repo must be enabled with nvcc, cublas, clblas, etc installed.
|
||||||
|
# Example: https://developer.download.nvidia.com/compute/cuda/repos/fedora37/x86_64/cuda-fedora37.repo
|
||||||
|
# 4. OpenCL/CLBLAST support simply requires the ICD loader and basic opencl libraries.
|
||||||
|
# It is up to the user to install the correct vendor-specific support.
|
||||||
|
|
||||||
|
Name: llama.cpp-cuda
|
||||||
|
Version: %( date "+%%Y%%m%%d" )
|
||||||
|
Release: 1%{?dist}
|
||||||
|
Summary: CPU Inference of LLaMA model in pure C/C++ (no CUDA/OpenCL)
|
||||||
|
License: MIT
|
||||||
|
Source0: https://github.com/ggerganov/llama.cpp/archive/refs/heads/master.tar.gz
|
||||||
|
BuildRequires: coreutils make gcc-c++ git cuda-toolkit
|
||||||
|
Requires: cuda-toolkit
|
||||||
|
URL: https://github.com/ggerganov/llama.cpp
|
||||||
|
|
||||||
|
%define debug_package %{nil}
|
||||||
|
%define source_date_epoch_from_changelog 0
|
||||||
|
|
||||||
|
%description
|
||||||
|
CPU inference for Meta's Lllama2 models using default options.
|
||||||
|
|
||||||
|
%prep
|
||||||
|
%setup -n llama.cpp-master
|
||||||
|
|
||||||
|
%build
|
||||||
|
make -j LLAMA_CUDA=1
|
||||||
|
|
||||||
|
%install
|
||||||
|
mkdir -p %{buildroot}%{_bindir}/
|
||||||
|
cp -p main %{buildroot}%{_bindir}/llamacppcuda
|
||||||
|
cp -p server %{buildroot}%{_bindir}/llamacppcudaserver
|
||||||
|
cp -p simple %{buildroot}%{_bindir}/llamacppcudasimple
|
||||||
|
|
||||||
|
mkdir -p %{buildroot}/usr/lib/systemd/system
|
||||||
|
%{__cat} <<EOF > %{buildroot}/usr/lib/systemd/system/llamacuda.service
|
||||||
|
[Unit]
|
||||||
|
Description=Llama.cpp server, CPU only (no GPU support in this build).
|
||||||
|
After=syslog.target network.target local-fs.target remote-fs.target nss-lookup.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
EnvironmentFile=/etc/sysconfig/llama
|
||||||
|
ExecStart=/usr/bin/llamacppcudaserver $LLAMA_ARGS
|
||||||
|
ExecReload=/bin/kill -s HUP $MAINPID
|
||||||
|
Restart=never
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=default.target
|
||||||
|
EOF
|
||||||
|
|
||||||
|
mkdir -p %{buildroot}/etc/sysconfig
|
||||||
|
%{__cat} <<EOF > %{buildroot}/etc/sysconfig/llama
|
||||||
|
LLAMA_ARGS="-m /opt/llama2/ggml-model-f32.bin"
|
||||||
|
EOF
|
||||||
|
|
||||||
|
%clean
|
||||||
|
rm -rf %{buildroot}
|
||||||
|
rm -rf %{_builddir}/*
|
||||||
|
|
||||||
|
%files
|
||||||
|
%{_bindir}/llamacppcuda
|
||||||
|
%{_bindir}/llamacppcudaserver
|
||||||
|
%{_bindir}/llamacppcudasimple
|
||||||
|
/usr/lib/systemd/system/llamacuda.service
|
||||||
|
%config /etc/sysconfig/llama
|
||||||
|
|
||||||
|
%pre
|
||||||
|
|
||||||
|
%post
|
||||||
|
|
||||||
|
%preun
|
||||||
|
%postun
|
||||||
|
|
||||||
|
%changelog
|
||||||
85
.devops/llama-cpp.srpm.spec
Normal file
85
.devops/llama-cpp.srpm.spec
Normal file
|
|
@ -0,0 +1,85 @@
|
||||||
|
# SRPM for building from source and packaging an RPM for RPM-based distros.
|
||||||
|
# https://docs.fedoraproject.org/en-US/quick-docs/creating-rpm-packages
|
||||||
|
# Built and maintained by John Boero - boeroboy@gmail.com
|
||||||
|
# In honor of Seth Vidal https://www.redhat.com/it/blog/thank-you-seth-vidal
|
||||||
|
|
||||||
|
# Notes for llama.cpp:
|
||||||
|
# 1. Tags are currently based on hash - which will not sort asciibetically.
|
||||||
|
# We need to declare standard versioning if people want to sort latest releases.
|
||||||
|
# In the meantime, YYYYMMDD format will be used.
|
||||||
|
# 2. Builds for CUDA/OpenCL support are separate, with different depenedencies.
|
||||||
|
# 3. NVidia's developer repo must be enabled with nvcc, cublas, clblas, etc installed.
|
||||||
|
# Example: https://developer.download.nvidia.com/compute/cuda/repos/fedora37/x86_64/cuda-fedora37.repo
|
||||||
|
# 4. OpenCL/CLBLAST support simply requires the ICD loader and basic opencl libraries.
|
||||||
|
# It is up to the user to install the correct vendor-specific support.
|
||||||
|
|
||||||
|
Name: llama.cpp
|
||||||
|
Version: %( date "+%%Y%%m%%d" )
|
||||||
|
Release: 1%{?dist}
|
||||||
|
Summary: CPU Inference of LLaMA model in pure C/C++ (no CUDA/OpenCL)
|
||||||
|
License: MIT
|
||||||
|
Source0: https://github.com/ggerganov/llama.cpp/archive/refs/heads/master.tar.gz
|
||||||
|
BuildRequires: coreutils make gcc-c++ git libstdc++-devel
|
||||||
|
Requires: libstdc++
|
||||||
|
URL: https://github.com/ggerganov/llama.cpp
|
||||||
|
|
||||||
|
%define debug_package %{nil}
|
||||||
|
%define source_date_epoch_from_changelog 0
|
||||||
|
|
||||||
|
%description
|
||||||
|
CPU inference for Meta's Lllama2 models using default options.
|
||||||
|
Models are not included in this package and must be downloaded separately.
|
||||||
|
|
||||||
|
%prep
|
||||||
|
%setup -n llama.cpp-master
|
||||||
|
|
||||||
|
%build
|
||||||
|
make -j
|
||||||
|
|
||||||
|
%install
|
||||||
|
mkdir -p %{buildroot}%{_bindir}/
|
||||||
|
cp -p main %{buildroot}%{_bindir}/llama
|
||||||
|
cp -p server %{buildroot}%{_bindir}/llamaserver
|
||||||
|
cp -p simple %{buildroot}%{_bindir}/llamasimple
|
||||||
|
|
||||||
|
mkdir -p %{buildroot}/usr/lib/systemd/system
|
||||||
|
%{__cat} <<EOF > %{buildroot}/usr/lib/systemd/system/llama.service
|
||||||
|
[Unit]
|
||||||
|
Description=Llama.cpp server, CPU only (no GPU support in this build).
|
||||||
|
After=syslog.target network.target local-fs.target remote-fs.target nss-lookup.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
EnvironmentFile=/etc/sysconfig/llama
|
||||||
|
ExecStart=/usr/bin/llamaserver $LLAMA_ARGS
|
||||||
|
ExecReload=/bin/kill -s HUP $MAINPID
|
||||||
|
Restart=never
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=default.target
|
||||||
|
EOF
|
||||||
|
|
||||||
|
mkdir -p %{buildroot}/etc/sysconfig
|
||||||
|
%{__cat} <<EOF > %{buildroot}/etc/sysconfig/llama
|
||||||
|
LLAMA_ARGS="-m /opt/llama2/ggml-model-f32.bin"
|
||||||
|
EOF
|
||||||
|
|
||||||
|
%clean
|
||||||
|
rm -rf %{buildroot}
|
||||||
|
rm -rf %{_builddir}/*
|
||||||
|
|
||||||
|
%files
|
||||||
|
%{_bindir}/llama
|
||||||
|
%{_bindir}/llamaserver
|
||||||
|
%{_bindir}/llamasimple
|
||||||
|
/usr/lib/systemd/system/llama.service
|
||||||
|
%config /etc/sysconfig/llama
|
||||||
|
|
||||||
|
%pre
|
||||||
|
|
||||||
|
%post
|
||||||
|
|
||||||
|
%preun
|
||||||
|
%postun
|
||||||
|
|
||||||
|
%changelog
|
||||||
32
.devops/main-cuda.Dockerfile
Normal file
32
.devops/main-cuda.Dockerfile
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
ARG UBUNTU_VERSION=22.04
|
||||||
|
# This needs to generally match the container host's environment.
|
||||||
|
ARG CUDA_VERSION=11.7.1
|
||||||
|
# Target the CUDA build image
|
||||||
|
ARG BASE_CUDA_DEV_CONTAINER=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}
|
||||||
|
# Target the CUDA runtime image
|
||||||
|
ARG BASE_CUDA_RUN_CONTAINER=nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}
|
||||||
|
|
||||||
|
FROM ${BASE_CUDA_DEV_CONTAINER} as build
|
||||||
|
|
||||||
|
# Unless otherwise specified, we make a fat build.
|
||||||
|
ARG CUDA_DOCKER_ARCH=all
|
||||||
|
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y build-essential git
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Set nvcc architecture
|
||||||
|
ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
|
||||||
|
# Enable CUDA
|
||||||
|
ENV LLAMA_CUDA=1
|
||||||
|
|
||||||
|
RUN make
|
||||||
|
|
||||||
|
FROM ${BASE_CUDA_RUN_CONTAINER} as runtime
|
||||||
|
|
||||||
|
COPY --from=build /app/main /main
|
||||||
|
|
||||||
|
ENTRYPOINT [ "/main" ]
|
||||||
26
.devops/main-intel.Dockerfile
Normal file
26
.devops/main-intel.Dockerfile
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
ARG ONEAPI_VERSION=2024.0.1-devel-ubuntu22.04
|
||||||
|
|
||||||
|
FROM intel/oneapi-basekit:$ONEAPI_VERSION as build
|
||||||
|
|
||||||
|
ARG LLAMA_SYCL_F16=OFF
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y git
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
RUN if [ "${LLAMA_SYCL_F16}" = "ON" ]; then \
|
||||||
|
echo "LLAMA_SYCL_F16 is set" && \
|
||||||
|
export OPT_SYCL_F16="-DLLAMA_SYCL_F16=ON"; \
|
||||||
|
fi && \
|
||||||
|
cmake -B build -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx ${OPT_SYCL_F16} && \
|
||||||
|
cmake --build build --config Release --target main
|
||||||
|
|
||||||
|
FROM intel/oneapi-basekit:$ONEAPI_VERSION as runtime
|
||||||
|
|
||||||
|
COPY --from=build /app/build/bin/main /main
|
||||||
|
|
||||||
|
ENV LC_ALL=C.utf8
|
||||||
|
|
||||||
|
ENTRYPOINT [ "/main" ]
|
||||||
45
.devops/main-rocm.Dockerfile
Normal file
45
.devops/main-rocm.Dockerfile
Normal file
|
|
@ -0,0 +1,45 @@
|
||||||
|
ARG UBUNTU_VERSION=22.04
|
||||||
|
|
||||||
|
# This needs to generally match the container host's environment.
|
||||||
|
ARG ROCM_VERSION=5.6
|
||||||
|
|
||||||
|
# Target the CUDA build image
|
||||||
|
ARG BASE_ROCM_DEV_CONTAINER=rocm/dev-ubuntu-${UBUNTU_VERSION}:${ROCM_VERSION}-complete
|
||||||
|
|
||||||
|
FROM ${BASE_ROCM_DEV_CONTAINER} as build
|
||||||
|
|
||||||
|
# Unless otherwise specified, we make a fat build.
|
||||||
|
# List from https://github.com/ggerganov/llama.cpp/pull/1087#issuecomment-1682807878
|
||||||
|
# This is mostly tied to rocBLAS supported archs.
|
||||||
|
ARG ROCM_DOCKER_ARCH=\
|
||||||
|
gfx803 \
|
||||||
|
gfx900 \
|
||||||
|
gfx906 \
|
||||||
|
gfx908 \
|
||||||
|
gfx90a \
|
||||||
|
gfx1010 \
|
||||||
|
gfx1030 \
|
||||||
|
gfx1100 \
|
||||||
|
gfx1101 \
|
||||||
|
gfx1102
|
||||||
|
|
||||||
|
COPY requirements.txt requirements.txt
|
||||||
|
COPY requirements requirements
|
||||||
|
|
||||||
|
RUN pip install --upgrade pip setuptools wheel \
|
||||||
|
&& pip install -r requirements.txt
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Set nvcc architecture
|
||||||
|
ENV GPU_TARGETS=${ROCM_DOCKER_ARCH}
|
||||||
|
# Enable ROCm
|
||||||
|
ENV LLAMA_HIPBLAS=1
|
||||||
|
ENV CC=/opt/rocm/llvm/bin/clang
|
||||||
|
ENV CXX=/opt/rocm/llvm/bin/clang++
|
||||||
|
|
||||||
|
RUN make
|
||||||
|
|
||||||
|
ENTRYPOINT [ "/app/main" ]
|
||||||
27
.devops/main-vulkan.Dockerfile
Normal file
27
.devops/main-vulkan.Dockerfile
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
ARG UBUNTU_VERSION=jammy
|
||||||
|
|
||||||
|
FROM ubuntu:$UBUNTU_VERSION as build
|
||||||
|
|
||||||
|
# Install build tools
|
||||||
|
RUN apt update && apt install -y git build-essential cmake wget
|
||||||
|
|
||||||
|
# Install Vulkan SDK
|
||||||
|
RUN wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - && \
|
||||||
|
wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list && \
|
||||||
|
apt update -y && \
|
||||||
|
apt-get install -y vulkan-sdk
|
||||||
|
|
||||||
|
# Build it
|
||||||
|
WORKDIR /app
|
||||||
|
COPY . .
|
||||||
|
RUN cmake -B build -DLLAMA_VULKAN=1 && \
|
||||||
|
cmake --build build --config Release --target main
|
||||||
|
|
||||||
|
# Clean up
|
||||||
|
WORKDIR /
|
||||||
|
RUN cp /app/build/bin/main /main && \
|
||||||
|
rm -rf /app
|
||||||
|
|
||||||
|
ENV LC_ALL=C.utf8
|
||||||
|
|
||||||
|
ENTRYPOINT [ "/main" ]
|
||||||
|
|
@ -3,7 +3,7 @@ ARG UBUNTU_VERSION=22.04
|
||||||
FROM ubuntu:$UBUNTU_VERSION as build
|
FROM ubuntu:$UBUNTU_VERSION as build
|
||||||
|
|
||||||
RUN apt-get update && \
|
RUN apt-get update && \
|
||||||
apt-get install -y build-essential
|
apt-get install -y build-essential git
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
|
|
@ -15,4 +15,6 @@ FROM ubuntu:$UBUNTU_VERSION as runtime
|
||||||
|
|
||||||
COPY --from=build /app/main /main
|
COPY --from=build /app/main /main
|
||||||
|
|
||||||
|
ENV LC_ALL=C.utf8
|
||||||
|
|
||||||
ENTRYPOINT [ "/main" ]
|
ENTRYPOINT [ "/main" ]
|
||||||
|
|
|
||||||
22
.devops/nix/apps.nix
Normal file
22
.devops/nix/apps.nix
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
{
|
||||||
|
perSystem =
|
||||||
|
{ config, lib, ... }:
|
||||||
|
{
|
||||||
|
apps =
|
||||||
|
let
|
||||||
|
inherit (config.packages) default;
|
||||||
|
binaries = [
|
||||||
|
"llama"
|
||||||
|
"llama-embedding"
|
||||||
|
"llama-server"
|
||||||
|
"quantize"
|
||||||
|
"train-text-from-scratch"
|
||||||
|
];
|
||||||
|
mkApp = name: {
|
||||||
|
type = "app";
|
||||||
|
program = "${default}/bin/${name}";
|
||||||
|
};
|
||||||
|
in
|
||||||
|
lib.genAttrs binaries mkApp;
|
||||||
|
};
|
||||||
|
}
|
||||||
13
.devops/nix/devshells.nix
Normal file
13
.devops/nix/devshells.nix
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
{
|
||||||
|
perSystem =
|
||||||
|
{ config, lib, ... }:
|
||||||
|
{
|
||||||
|
devShells =
|
||||||
|
lib.concatMapAttrs
|
||||||
|
(name: package: {
|
||||||
|
${name} = package.passthru.shell;
|
||||||
|
${name + "-extra"} = package.passthru.shell-extra;
|
||||||
|
})
|
||||||
|
config.packages;
|
||||||
|
};
|
||||||
|
}
|
||||||
37
.devops/nix/docker.nix
Normal file
37
.devops/nix/docker.nix
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
{
|
||||||
|
lib,
|
||||||
|
dockerTools,
|
||||||
|
buildEnv,
|
||||||
|
llama-cpp,
|
||||||
|
interactive ? true,
|
||||||
|
coreutils,
|
||||||
|
}:
|
||||||
|
|
||||||
|
# A tar that can be fed into `docker load`:
|
||||||
|
#
|
||||||
|
# $ nix build .#llamaPackages.docker
|
||||||
|
# $ docker load < result
|
||||||
|
|
||||||
|
# For details and variations cf.
|
||||||
|
# - https://nixos.org/manual/nixpkgs/unstable/#ssec-pkgs-dockerTools-buildLayeredImage
|
||||||
|
# - https://discourse.nixos.org/t/a-faster-dockertools-buildimage-prototype/16922
|
||||||
|
# - https://nixery.dev/
|
||||||
|
|
||||||
|
# Approximate (compressed) sizes, at the time of writing, are:
|
||||||
|
#
|
||||||
|
# .#llamaPackages.docker: 125M;
|
||||||
|
# .#llamaPackagesCuda.docker: 537M;
|
||||||
|
# .#legacyPackages.aarch64-linux.llamaPackagesXavier.docker: 415M.
|
||||||
|
|
||||||
|
dockerTools.buildLayeredImage {
|
||||||
|
name = llama-cpp.pname;
|
||||||
|
tag = "latest";
|
||||||
|
|
||||||
|
contents =
|
||||||
|
[ llama-cpp ]
|
||||||
|
++ lib.optionals interactive [
|
||||||
|
coreutils
|
||||||
|
dockerTools.binSh
|
||||||
|
dockerTools.caCertificates
|
||||||
|
];
|
||||||
|
}
|
||||||
39
.devops/nix/jetson-support.nix
Normal file
39
.devops/nix/jetson-support.nix
Normal file
|
|
@ -0,0 +1,39 @@
|
||||||
|
{ inputs, ... }:
|
||||||
|
{
|
||||||
|
perSystem =
|
||||||
|
{
|
||||||
|
config,
|
||||||
|
system,
|
||||||
|
lib,
|
||||||
|
pkgsCuda,
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
{
|
||||||
|
legacyPackages =
|
||||||
|
let
|
||||||
|
caps.llamaPackagesXavier = "7.2";
|
||||||
|
caps.llamaPackagesOrin = "8.7";
|
||||||
|
caps.llamaPackagesTX2 = "6.2";
|
||||||
|
caps.llamaPackagesNano = "5.3";
|
||||||
|
|
||||||
|
pkgsFor =
|
||||||
|
cap:
|
||||||
|
import inputs.nixpkgs {
|
||||||
|
inherit system;
|
||||||
|
config = {
|
||||||
|
cudaSupport = true;
|
||||||
|
cudaCapabilities = [ cap ];
|
||||||
|
cudaEnableForwardCompat = false;
|
||||||
|
inherit (pkgsCuda.config) allowUnfreePredicate;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
in
|
||||||
|
builtins.mapAttrs (name: cap: (pkgsFor cap).callPackage ./scope.nix { }) caps;
|
||||||
|
|
||||||
|
packages = lib.optionalAttrs (system == "aarch64-linux") {
|
||||||
|
jetson-xavier = config.legacyPackages.llamaPackagesXavier.llama-cpp;
|
||||||
|
jetson-orin = config.legacyPackages.llamaPackagesOrin.llama-cpp;
|
||||||
|
jetson-nano = config.legacyPackages.llamaPackagesNano.llama-cpp;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
47
.devops/nix/nixpkgs-instances.nix
Normal file
47
.devops/nix/nixpkgs-instances.nix
Normal file
|
|
@ -0,0 +1,47 @@
|
||||||
|
{ inputs, ... }:
|
||||||
|
{
|
||||||
|
# The _module.args definitions are passed on to modules as arguments. E.g.
|
||||||
|
# the module `{ pkgs ... }: { /* config */ }` implicitly uses
|
||||||
|
# `_module.args.pkgs` (defined in this case by flake-parts).
|
||||||
|
perSystem =
|
||||||
|
{ system, ... }:
|
||||||
|
{
|
||||||
|
_module.args = {
|
||||||
|
# Note: bringing up https://zimbatm.com/notes/1000-instances-of-nixpkgs
|
||||||
|
# again, the below creates several nixpkgs instances which the
|
||||||
|
# flake-centric CLI will be forced to evaluate e.g. on `nix flake show`.
|
||||||
|
#
|
||||||
|
# This is currently "slow" and "expensive", on a certain scale.
|
||||||
|
# This also isn't "right" in that this hinders dependency injection at
|
||||||
|
# the level of flake inputs. This might get removed in the foreseeable
|
||||||
|
# future.
|
||||||
|
#
|
||||||
|
# Note that you can use these expressions without Nix
|
||||||
|
# (`pkgs.callPackage ./devops/nix/scope.nix { }` is the entry point).
|
||||||
|
|
||||||
|
pkgsCuda = import inputs.nixpkgs {
|
||||||
|
inherit system;
|
||||||
|
# Ensure dependencies use CUDA consistently (e.g. that openmpi, ucc,
|
||||||
|
# and ucx are built with CUDA support)
|
||||||
|
config.cudaSupport = true;
|
||||||
|
config.allowUnfreePredicate =
|
||||||
|
p:
|
||||||
|
builtins.all
|
||||||
|
(
|
||||||
|
license:
|
||||||
|
license.free
|
||||||
|
|| builtins.elem license.shortName [
|
||||||
|
"CUDA EULA"
|
||||||
|
"cuDNN EULA"
|
||||||
|
]
|
||||||
|
)
|
||||||
|
(p.meta.licenses or [ p.meta.license ]);
|
||||||
|
};
|
||||||
|
# Ensure dependencies use ROCm consistently
|
||||||
|
pkgsRocm = import inputs.nixpkgs {
|
||||||
|
inherit system;
|
||||||
|
config.rocmSupport = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
319
.devops/nix/package.nix
Normal file
319
.devops/nix/package.nix
Normal file
|
|
@ -0,0 +1,319 @@
|
||||||
|
{
|
||||||
|
lib,
|
||||||
|
glibc,
|
||||||
|
config,
|
||||||
|
stdenv,
|
||||||
|
mkShell,
|
||||||
|
runCommand,
|
||||||
|
cmake,
|
||||||
|
ninja,
|
||||||
|
pkg-config,
|
||||||
|
git,
|
||||||
|
python3,
|
||||||
|
mpi,
|
||||||
|
blas,
|
||||||
|
cudaPackages,
|
||||||
|
darwin,
|
||||||
|
rocmPackages,
|
||||||
|
vulkan-headers,
|
||||||
|
vulkan-loader,
|
||||||
|
clblast,
|
||||||
|
useBlas ? builtins.all (x: !x) [
|
||||||
|
useCuda
|
||||||
|
useMetalKit
|
||||||
|
useOpenCL
|
||||||
|
useRocm
|
||||||
|
useVulkan
|
||||||
|
] && blas.meta.available,
|
||||||
|
useCuda ? config.cudaSupport,
|
||||||
|
useMetalKit ? stdenv.isAarch64 && stdenv.isDarwin && !useOpenCL,
|
||||||
|
useMpi ? false, # Increases the runtime closure size by ~700M
|
||||||
|
useOpenCL ? false,
|
||||||
|
useRocm ? config.rocmSupport,
|
||||||
|
useVulkan ? false,
|
||||||
|
llamaVersion ? "0.0.0", # Arbitrary version, substituted by the flake
|
||||||
|
|
||||||
|
# It's necessary to consistently use backendStdenv when building with CUDA support,
|
||||||
|
# otherwise we get libstdc++ errors downstream.
|
||||||
|
effectiveStdenv ? if useCuda then cudaPackages.backendStdenv else stdenv,
|
||||||
|
enableStatic ? effectiveStdenv.hostPlatform.isStatic,
|
||||||
|
precompileMetalShaders ? false
|
||||||
|
}@inputs:
|
||||||
|
|
||||||
|
let
|
||||||
|
inherit (lib)
|
||||||
|
cmakeBool
|
||||||
|
cmakeFeature
|
||||||
|
optionals
|
||||||
|
strings
|
||||||
|
versionOlder
|
||||||
|
;
|
||||||
|
|
||||||
|
stdenv = throw "Use effectiveStdenv instead";
|
||||||
|
|
||||||
|
suffices =
|
||||||
|
lib.optionals useBlas [ "BLAS" ]
|
||||||
|
++ lib.optionals useCuda [ "CUDA" ]
|
||||||
|
++ lib.optionals useMetalKit [ "MetalKit" ]
|
||||||
|
++ lib.optionals useMpi [ "MPI" ]
|
||||||
|
++ lib.optionals useOpenCL [ "OpenCL" ]
|
||||||
|
++ lib.optionals useRocm [ "ROCm" ]
|
||||||
|
++ lib.optionals useVulkan [ "Vulkan" ];
|
||||||
|
|
||||||
|
pnameSuffix =
|
||||||
|
strings.optionalString (suffices != [ ])
|
||||||
|
"-${strings.concatMapStringsSep "-" strings.toLower suffices}";
|
||||||
|
descriptionSuffix =
|
||||||
|
strings.optionalString (suffices != [ ])
|
||||||
|
", accelerated with ${strings.concatStringsSep ", " suffices}";
|
||||||
|
|
||||||
|
executableSuffix = effectiveStdenv.hostPlatform.extensions.executable;
|
||||||
|
|
||||||
|
# TODO: package the Python in this repository in a Nix-like way.
|
||||||
|
# It'd be nice to migrate to buildPythonPackage, as well as ensure this repo
|
||||||
|
# is PEP 517-compatible, and ensure the correct .dist-info is generated.
|
||||||
|
# https://peps.python.org/pep-0517/
|
||||||
|
#
|
||||||
|
# TODO: Package up each Python script or service appropriately, by making
|
||||||
|
# them into "entrypoints"
|
||||||
|
llama-python = python3.withPackages (
|
||||||
|
ps: [
|
||||||
|
ps.numpy
|
||||||
|
ps.sentencepiece
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
# TODO(Green-Sky): find a better way to opt-into the heavy ml python runtime
|
||||||
|
llama-python-extra = python3.withPackages (
|
||||||
|
ps: [
|
||||||
|
ps.numpy
|
||||||
|
ps.sentencepiece
|
||||||
|
ps.tiktoken
|
||||||
|
ps.torchWithoutCuda
|
||||||
|
ps.transformers
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
xcrunHost = runCommand "xcrunHost" {} ''
|
||||||
|
mkdir -p $out/bin
|
||||||
|
ln -s /usr/bin/xcrun $out/bin
|
||||||
|
'';
|
||||||
|
|
||||||
|
# apple_sdk is supposed to choose sane defaults, no need to handle isAarch64
|
||||||
|
# separately
|
||||||
|
darwinBuildInputs =
|
||||||
|
with darwin.apple_sdk.frameworks;
|
||||||
|
[
|
||||||
|
Accelerate
|
||||||
|
CoreVideo
|
||||||
|
CoreGraphics
|
||||||
|
]
|
||||||
|
++ optionals useMetalKit [ MetalKit ];
|
||||||
|
|
||||||
|
cudaBuildInputs = with cudaPackages; [
|
||||||
|
cuda_cccl.dev # <nv/target>
|
||||||
|
|
||||||
|
# A temporary hack for reducing the closure size, remove once cudaPackages
|
||||||
|
# have stopped using lndir: https://github.com/NixOS/nixpkgs/issues/271792
|
||||||
|
cuda_cudart.dev
|
||||||
|
cuda_cudart.lib
|
||||||
|
cuda_cudart.static
|
||||||
|
libcublas.dev
|
||||||
|
libcublas.lib
|
||||||
|
libcublas.static
|
||||||
|
];
|
||||||
|
|
||||||
|
rocmBuildInputs = with rocmPackages; [
|
||||||
|
clr
|
||||||
|
hipblas
|
||||||
|
rocblas
|
||||||
|
];
|
||||||
|
|
||||||
|
vulkanBuildInputs = [
|
||||||
|
vulkan-headers
|
||||||
|
vulkan-loader
|
||||||
|
];
|
||||||
|
in
|
||||||
|
|
||||||
|
effectiveStdenv.mkDerivation (
|
||||||
|
finalAttrs: {
|
||||||
|
pname = "llama-cpp${pnameSuffix}";
|
||||||
|
version = llamaVersion;
|
||||||
|
|
||||||
|
# Note: none of the files discarded here are visible in the sandbox or
|
||||||
|
# affect the output hash. This also means they can be modified without
|
||||||
|
# triggering a rebuild.
|
||||||
|
src = lib.cleanSourceWith {
|
||||||
|
filter =
|
||||||
|
name: type:
|
||||||
|
let
|
||||||
|
noneOf = builtins.all (x: !x);
|
||||||
|
baseName = baseNameOf name;
|
||||||
|
in
|
||||||
|
noneOf [
|
||||||
|
(lib.hasSuffix ".nix" name) # Ignore *.nix files when computing outPaths
|
||||||
|
(lib.hasSuffix ".md" name) # Ignore *.md changes whe computing outPaths
|
||||||
|
(lib.hasPrefix "." baseName) # Skip hidden files and directories
|
||||||
|
(baseName == "flake.lock")
|
||||||
|
];
|
||||||
|
src = lib.cleanSource ../../.;
|
||||||
|
};
|
||||||
|
|
||||||
|
postPatch = ''
|
||||||
|
substituteInPlace ./ggml-metal.m \
|
||||||
|
--replace '[bundle pathForResource:@"ggml-metal" ofType:@"metal"];' "@\"$out/bin/ggml-metal.metal\";"
|
||||||
|
substituteInPlace ./ggml-metal.m \
|
||||||
|
--replace '[bundle pathForResource:@"default" ofType:@"metallib"];' "@\"$out/bin/default.metallib\";"
|
||||||
|
'';
|
||||||
|
|
||||||
|
# With PR#6015 https://github.com/ggerganov/llama.cpp/pull/6015,
|
||||||
|
# `default.metallib` may be compiled with Metal compiler from XCode
|
||||||
|
# and we need to escape sandbox on MacOS to access Metal compiler.
|
||||||
|
# `xcrun` is used find the path of the Metal compiler, which is varible
|
||||||
|
# and not on $PATH
|
||||||
|
# see https://github.com/ggerganov/llama.cpp/pull/6118 for discussion
|
||||||
|
__noChroot = effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders;
|
||||||
|
|
||||||
|
nativeBuildInputs =
|
||||||
|
[
|
||||||
|
cmake
|
||||||
|
ninja
|
||||||
|
pkg-config
|
||||||
|
git
|
||||||
|
]
|
||||||
|
++ optionals useCuda [
|
||||||
|
cudaPackages.cuda_nvcc
|
||||||
|
|
||||||
|
# TODO: Replace with autoAddDriverRunpath
|
||||||
|
# once https://github.com/NixOS/nixpkgs/pull/275241 has been merged
|
||||||
|
cudaPackages.autoAddOpenGLRunpathHook
|
||||||
|
]
|
||||||
|
++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [
|
||||||
|
glibc.static
|
||||||
|
] ++ optionals (effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders) [
|
||||||
|
xcrunHost
|
||||||
|
];
|
||||||
|
|
||||||
|
buildInputs =
|
||||||
|
optionals effectiveStdenv.isDarwin darwinBuildInputs
|
||||||
|
++ optionals useCuda cudaBuildInputs
|
||||||
|
++ optionals useMpi [ mpi ]
|
||||||
|
++ optionals useOpenCL [ clblast ]
|
||||||
|
++ optionals useRocm rocmBuildInputs
|
||||||
|
++ optionals useBlas [ blas ]
|
||||||
|
++ optionals useVulkan vulkanBuildInputs;
|
||||||
|
|
||||||
|
cmakeFlags =
|
||||||
|
[
|
||||||
|
(cmakeBool "LLAMA_NATIVE" false)
|
||||||
|
(cmakeBool "LLAMA_BUILD_SERVER" true)
|
||||||
|
(cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))
|
||||||
|
(cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
|
||||||
|
(cmakeBool "LLAMA_BLAS" useBlas)
|
||||||
|
(cmakeBool "LLAMA_CLBLAST" useOpenCL)
|
||||||
|
(cmakeBool "LLAMA_CUDA" useCuda)
|
||||||
|
(cmakeBool "LLAMA_HIPBLAS" useRocm)
|
||||||
|
(cmakeBool "LLAMA_METAL" useMetalKit)
|
||||||
|
(cmakeBool "LLAMA_MPI" useMpi)
|
||||||
|
(cmakeBool "LLAMA_VULKAN" useVulkan)
|
||||||
|
(cmakeBool "LLAMA_STATIC" enableStatic)
|
||||||
|
]
|
||||||
|
++ optionals useCuda [
|
||||||
|
(
|
||||||
|
with cudaPackages.flags;
|
||||||
|
cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
|
||||||
|
builtins.concatStringsSep ";" (map dropDot cudaCapabilities)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
++ optionals useRocm [
|
||||||
|
(cmakeFeature "CMAKE_C_COMPILER" "hipcc")
|
||||||
|
(cmakeFeature "CMAKE_CXX_COMPILER" "hipcc")
|
||||||
|
|
||||||
|
# Build all targets supported by rocBLAS. When updating search for TARGET_LIST_ROCM
|
||||||
|
# in https://github.com/ROCmSoftwarePlatform/rocBLAS/blob/develop/CMakeLists.txt
|
||||||
|
# and select the line that matches the current nixpkgs version of rocBLAS.
|
||||||
|
# Should likely use `rocmPackages.clr.gpuTargets`.
|
||||||
|
"-DAMDGPU_TARGETS=gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx940;gfx941;gfx942;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102"
|
||||||
|
]
|
||||||
|
++ optionals useMetalKit [
|
||||||
|
(lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")
|
||||||
|
(cmakeBool "LLAMA_METAL_EMBED_LIBRARY" (!precompileMetalShaders))
|
||||||
|
];
|
||||||
|
|
||||||
|
# TODO(SomeoneSerge): It's better to add proper install targets at the CMake level,
|
||||||
|
# if they haven't been added yet.
|
||||||
|
postInstall = ''
|
||||||
|
mv $out/bin/main${executableSuffix} $out/bin/llama${executableSuffix}
|
||||||
|
mv $out/bin/server${executableSuffix} $out/bin/llama-server${executableSuffix}
|
||||||
|
mkdir -p $out/include
|
||||||
|
cp $src/llama.h $out/include/
|
||||||
|
'';
|
||||||
|
|
||||||
|
# Define the shells here, but don't add in the inputsFrom to avoid recursion.
|
||||||
|
passthru = {
|
||||||
|
inherit
|
||||||
|
useBlas
|
||||||
|
useCuda
|
||||||
|
useMetalKit
|
||||||
|
useMpi
|
||||||
|
useOpenCL
|
||||||
|
useRocm
|
||||||
|
useVulkan
|
||||||
|
;
|
||||||
|
|
||||||
|
shell = mkShell {
|
||||||
|
name = "shell-${finalAttrs.finalPackage.name}";
|
||||||
|
description = "contains numpy and sentencepiece";
|
||||||
|
buildInputs = [ llama-python ];
|
||||||
|
inputsFrom = [ finalAttrs.finalPackage ];
|
||||||
|
shellHook = ''
|
||||||
|
addToSearchPath "LD_LIBRARY_PATH" "${lib.getLib effectiveStdenv.cc.cc}/lib"
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
shell-extra = mkShell {
|
||||||
|
name = "shell-extra-${finalAttrs.finalPackage.name}";
|
||||||
|
description = "contains numpy, sentencepiece, torchWithoutCuda, and transformers";
|
||||||
|
buildInputs = [ llama-python-extra ];
|
||||||
|
inputsFrom = [ finalAttrs.finalPackage ];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
meta = {
|
||||||
|
# Configurations we don't want even the CI to evaluate. Results in the
|
||||||
|
# "unsupported platform" messages. This is mostly a no-op, because
|
||||||
|
# cudaPackages would've refused to evaluate anyway.
|
||||||
|
badPlatforms = optionals (useCuda || useOpenCL) lib.platforms.darwin;
|
||||||
|
|
||||||
|
# Configurations that are known to result in build failures. Can be
|
||||||
|
# overridden by importing Nixpkgs with `allowBroken = true`.
|
||||||
|
broken = (useMetalKit && !effectiveStdenv.isDarwin);
|
||||||
|
|
||||||
|
description = "Inference of LLaMA model in pure C/C++${descriptionSuffix}";
|
||||||
|
homepage = "https://github.com/ggerganov/llama.cpp/";
|
||||||
|
license = lib.licenses.mit;
|
||||||
|
|
||||||
|
# Accommodates `nix run` and `lib.getExe`
|
||||||
|
mainProgram = "llama";
|
||||||
|
|
||||||
|
# These people might respond, on the best effort basis, if you ping them
|
||||||
|
# in case of Nix-specific regressions or for reviewing Nix-specific PRs.
|
||||||
|
# Consider adding yourself to this list if you want to ensure this flake
|
||||||
|
# stays maintained and you're willing to invest your time. Do not add
|
||||||
|
# other people without their consent. Consider removing people after
|
||||||
|
# they've been unreachable for long periods of time.
|
||||||
|
|
||||||
|
# Note that lib.maintainers is defined in Nixpkgs, but you may just add
|
||||||
|
# an attrset following the same format as in
|
||||||
|
# https://github.com/NixOS/nixpkgs/blob/f36a80e54da29775c78d7eff0e628c2b4e34d1d7/maintainers/maintainer-list.nix
|
||||||
|
maintainers = with lib.maintainers; [
|
||||||
|
philiptaron
|
||||||
|
SomeoneSerge
|
||||||
|
];
|
||||||
|
|
||||||
|
# Extend `badPlatforms` instead
|
||||||
|
platforms = lib.platforms.all;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
)
|
||||||
19
.devops/nix/scope.nix
Normal file
19
.devops/nix/scope.nix
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
{
|
||||||
|
lib,
|
||||||
|
newScope,
|
||||||
|
llamaVersion ? "0.0.0",
|
||||||
|
}:
|
||||||
|
|
||||||
|
# We're using `makeScope` instead of just writing out an attrset
|
||||||
|
# because it allows users to apply overlays later using `overrideScope'`.
|
||||||
|
# Cf. https://noogle.dev/f/lib/makeScope
|
||||||
|
|
||||||
|
lib.makeScope newScope (
|
||||||
|
self: {
|
||||||
|
inherit llamaVersion;
|
||||||
|
llama-cpp = self.callPackage ./package.nix { };
|
||||||
|
docker = self.callPackage ./docker.nix { };
|
||||||
|
docker-min = self.callPackage ./docker.nix { interactive = false; };
|
||||||
|
sif = self.callPackage ./sif.nix { };
|
||||||
|
}
|
||||||
|
)
|
||||||
27
.devops/nix/sif.nix
Normal file
27
.devops/nix/sif.nix
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
{
|
||||||
|
lib,
|
||||||
|
singularity-tools,
|
||||||
|
llama-cpp,
|
||||||
|
bashInteractive,
|
||||||
|
interactive ? false,
|
||||||
|
}:
|
||||||
|
|
||||||
|
let
|
||||||
|
optionalInt = cond: x: if cond then x else 0;
|
||||||
|
in
|
||||||
|
singularity-tools.buildImage rec {
|
||||||
|
inherit (llama-cpp) name;
|
||||||
|
contents = [ llama-cpp ] ++ lib.optionals interactive [ bashInteractive ];
|
||||||
|
|
||||||
|
# These are excessive (but safe) for most variants. Building singularity
|
||||||
|
# images requires superuser privileges, so we build them inside a VM in a
|
||||||
|
# writable image of pre-determined size.
|
||||||
|
#
|
||||||
|
# ROCm is currently affected by https://github.com/NixOS/nixpkgs/issues/276846
|
||||||
|
#
|
||||||
|
# Expected image sizes:
|
||||||
|
# - cpu/blas: 150M,
|
||||||
|
# - cuda, all gencodes: 560M,
|
||||||
|
diskSize = 4096 + optionalInt llama-cpp.useRocm 16384;
|
||||||
|
memSize = diskSize;
|
||||||
|
}
|
||||||
37
.devops/server-cuda.Dockerfile
Normal file
37
.devops/server-cuda.Dockerfile
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
ARG UBUNTU_VERSION=22.04
|
||||||
|
# This needs to generally match the container host's environment.
|
||||||
|
ARG CUDA_VERSION=11.7.1
|
||||||
|
# Target the CUDA build image
|
||||||
|
ARG BASE_CUDA_DEV_CONTAINER=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}
|
||||||
|
# Target the CUDA runtime image
|
||||||
|
ARG BASE_CUDA_RUN_CONTAINER=nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}
|
||||||
|
|
||||||
|
FROM ${BASE_CUDA_DEV_CONTAINER} as build
|
||||||
|
|
||||||
|
# Unless otherwise specified, we make a fat build.
|
||||||
|
ARG CUDA_DOCKER_ARCH=all
|
||||||
|
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y build-essential git libcurl4-openssl-dev
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Set nvcc architecture
|
||||||
|
ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
|
||||||
|
# Enable CUDA
|
||||||
|
ENV LLAMA_CUDA=1
|
||||||
|
# Enable cURL
|
||||||
|
ENV LLAMA_CURL=1
|
||||||
|
|
||||||
|
RUN make
|
||||||
|
|
||||||
|
FROM ${BASE_CUDA_RUN_CONTAINER} as runtime
|
||||||
|
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y libcurl4-openssl-dev
|
||||||
|
|
||||||
|
COPY --from=build /app/server /server
|
||||||
|
|
||||||
|
ENTRYPOINT [ "/server" ]
|
||||||
29
.devops/server-intel.Dockerfile
Normal file
29
.devops/server-intel.Dockerfile
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
ARG ONEAPI_VERSION=2024.0.1-devel-ubuntu22.04
|
||||||
|
|
||||||
|
FROM intel/oneapi-basekit:$ONEAPI_VERSION as build
|
||||||
|
|
||||||
|
ARG LLAMA_SYCL_F16=OFF
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y git libcurl4-openssl-dev
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
RUN if [ "${LLAMA_SYCL_F16}" = "ON" ]; then \
|
||||||
|
echo "LLAMA_SYCL_F16 is set" && \
|
||||||
|
export OPT_SYCL_F16="-DLLAMA_SYCL_F16=ON"; \
|
||||||
|
fi && \
|
||||||
|
cmake -B build -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_CURL=ON ${OPT_SYCL_F16} && \
|
||||||
|
cmake --build build --config Release --target server
|
||||||
|
|
||||||
|
FROM intel/oneapi-basekit:$ONEAPI_VERSION as runtime
|
||||||
|
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y libcurl4-openssl-dev
|
||||||
|
|
||||||
|
COPY --from=build /app/build/bin/server /server
|
||||||
|
|
||||||
|
ENV LC_ALL=C.utf8
|
||||||
|
|
||||||
|
ENTRYPOINT [ "/server" ]
|
||||||
50
.devops/server-rocm.Dockerfile
Normal file
50
.devops/server-rocm.Dockerfile
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
ARG UBUNTU_VERSION=22.04
|
||||||
|
|
||||||
|
# This needs to generally match the container host's environment.
|
||||||
|
ARG ROCM_VERSION=5.6
|
||||||
|
|
||||||
|
# Target the CUDA build image
|
||||||
|
ARG BASE_ROCM_DEV_CONTAINER=rocm/dev-ubuntu-${UBUNTU_VERSION}:${ROCM_VERSION}-complete
|
||||||
|
|
||||||
|
FROM ${BASE_ROCM_DEV_CONTAINER} as build
|
||||||
|
|
||||||
|
# Unless otherwise specified, we make a fat build.
|
||||||
|
# List from https://github.com/ggerganov/llama.cpp/pull/1087#issuecomment-1682807878
|
||||||
|
# This is mostly tied to rocBLAS supported archs.
|
||||||
|
ARG ROCM_DOCKER_ARCH=\
|
||||||
|
gfx803 \
|
||||||
|
gfx900 \
|
||||||
|
gfx906 \
|
||||||
|
gfx908 \
|
||||||
|
gfx90a \
|
||||||
|
gfx1010 \
|
||||||
|
gfx1030 \
|
||||||
|
gfx1100 \
|
||||||
|
gfx1101 \
|
||||||
|
gfx1102
|
||||||
|
|
||||||
|
COPY requirements.txt requirements.txt
|
||||||
|
COPY requirements requirements
|
||||||
|
|
||||||
|
RUN pip install --upgrade pip setuptools wheel \
|
||||||
|
&& pip install -r requirements.txt
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Set nvcc architecture
|
||||||
|
ENV GPU_TARGETS=${ROCM_DOCKER_ARCH}
|
||||||
|
# Enable ROCm
|
||||||
|
ENV LLAMA_HIPBLAS=1
|
||||||
|
ENV CC=/opt/rocm/llvm/bin/clang
|
||||||
|
ENV CXX=/opt/rocm/llvm/bin/clang++
|
||||||
|
|
||||||
|
# Enable cURL
|
||||||
|
ENV LLAMA_CURL=1
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y libcurl4-openssl-dev
|
||||||
|
|
||||||
|
RUN make
|
||||||
|
|
||||||
|
ENTRYPOINT [ "/app/server" ]
|
||||||
31
.devops/server-vulkan.Dockerfile
Normal file
31
.devops/server-vulkan.Dockerfile
Normal file
|
|
@ -0,0 +1,31 @@
|
||||||
|
ARG UBUNTU_VERSION=jammy
|
||||||
|
|
||||||
|
FROM ubuntu:$UBUNTU_VERSION as build
|
||||||
|
|
||||||
|
# Install build tools
|
||||||
|
RUN apt update && apt install -y git build-essential cmake wget
|
||||||
|
|
||||||
|
# Install Vulkan SDK
|
||||||
|
RUN wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - && \
|
||||||
|
wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list && \
|
||||||
|
apt update -y && \
|
||||||
|
apt-get install -y vulkan-sdk
|
||||||
|
|
||||||
|
# Install cURL
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y libcurl4-openssl-dev
|
||||||
|
|
||||||
|
# Build it
|
||||||
|
WORKDIR /app
|
||||||
|
COPY . .
|
||||||
|
RUN cmake -B build -DLLAMA_VULKAN=1 -DLLAMA_CURL=1 && \
|
||||||
|
cmake --build build --config Release --target server
|
||||||
|
|
||||||
|
# Clean up
|
||||||
|
WORKDIR /
|
||||||
|
RUN cp /app/build/bin/server /server && \
|
||||||
|
rm -rf /app
|
||||||
|
|
||||||
|
ENV LC_ALL=C.utf8
|
||||||
|
|
||||||
|
ENTRYPOINT [ "/server" ]
|
||||||
25
.devops/server.Dockerfile
Normal file
25
.devops/server.Dockerfile
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
ARG UBUNTU_VERSION=22.04
|
||||||
|
|
||||||
|
FROM ubuntu:$UBUNTU_VERSION as build
|
||||||
|
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y build-essential git libcurl4-openssl-dev
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
ENV LLAMA_CURL=1
|
||||||
|
|
||||||
|
RUN make
|
||||||
|
|
||||||
|
FROM ubuntu:$UBUNTU_VERSION as runtime
|
||||||
|
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y libcurl4-openssl-dev
|
||||||
|
|
||||||
|
COPY --from=build /app/server /server
|
||||||
|
|
||||||
|
ENV LC_ALL=C.utf8
|
||||||
|
|
||||||
|
ENTRYPOINT [ "/server" ]
|
||||||
|
|
@ -7,34 +7,39 @@ arg1="$1"
|
||||||
# Shift the arguments to remove the first one
|
# Shift the arguments to remove the first one
|
||||||
shift
|
shift
|
||||||
|
|
||||||
# Join the remaining arguments into a single string
|
if [[ "$arg1" == '--convert' || "$arg1" == '-c' ]]; then
|
||||||
arg2="$@"
|
python3 ./convert.py "$@"
|
||||||
|
elif [[ "$arg1" == '--quantize' || "$arg1" == '-q' ]]; then
|
||||||
if [[ $arg1 == '--convert' || $arg1 == '-c' ]]; then
|
./quantize "$@"
|
||||||
python3 ./convert-pth-to-ggml.py $arg2
|
elif [[ "$arg1" == '--run' || "$arg1" == '-r' ]]; then
|
||||||
elif [[ $arg1 == '--quantize' || $arg1 == '-q' ]]; then
|
./main "$@"
|
||||||
./quantize $arg2
|
elif [[ "$arg1" == '--finetune' || "$arg1" == '-f' ]]; then
|
||||||
elif [[ $arg1 == '--run' || $arg1 == '-r' ]]; then
|
./finetune "$@"
|
||||||
./main $arg2
|
elif [[ "$arg1" == '--all-in-one' || "$arg1" == '-a' ]]; then
|
||||||
elif [[ $arg1 == '--all-in-one' || $arg1 == '-a' ]]; then
|
|
||||||
echo "Converting PTH to GGML..."
|
echo "Converting PTH to GGML..."
|
||||||
for i in `ls $1/$2/ggml-model-f16.bin*`; do
|
for i in `ls $1/$2/ggml-model-f16.bin*`; do
|
||||||
if [ -f "${i/f16/q4_0}" ]; then
|
if [ -f "${i/f16/q4_0}" ]; then
|
||||||
echo "Skip model quantization, it already exists: ${i/f16/q4_0}"
|
echo "Skip model quantization, it already exists: ${i/f16/q4_0}"
|
||||||
else
|
else
|
||||||
echo "Converting PTH to GGML: $i into ${i/f16/q4_0}..."
|
echo "Converting PTH to GGML: $i into ${i/f16/q4_0}..."
|
||||||
./quantize "$i" "${i/f16/q4_0}" 2
|
./quantize "$i" "${i/f16/q4_0}" q4_0
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
elif [[ "$arg1" == '--server' || "$arg1" == '-s' ]]; then
|
||||||
|
./server "$@"
|
||||||
else
|
else
|
||||||
echo "Unknown command: $arg1"
|
echo "Unknown command: $arg1"
|
||||||
echo "Available commands: "
|
echo "Available commands: "
|
||||||
echo " --run (-r): Run a model previously converted into ggml"
|
echo " --run (-r): Run a model previously converted into ggml"
|
||||||
echo " ex: -m /models/7B/ggml-model-q4_0.bin -p \"Building a website can be done in 10 simple steps:\" -n 512"
|
echo " ex: -m /models/7B/ggml-model-q4_0.bin -p \"Building a website can be done in 10 simple steps:\" -n 512"
|
||||||
echo " --convert (-c): Convert a llama model into ggml"
|
echo " --convert (-c): Convert a llama model into ggml"
|
||||||
echo " ex: \"/models/7B/\" 1"
|
echo " ex: --outtype f16 \"/models/7B/\" "
|
||||||
echo " --quantize (-q): Optimize with quantization process ggml"
|
echo " --quantize (-q): Optimize with quantization process ggml"
|
||||||
echo " ex: \"/models/7B/ggml-model-f16.bin\" \"/models/7B/ggml-model-q4_0.bin\" 2"
|
echo " ex: \"/models/7B/ggml-model-f16.bin\" \"/models/7B/ggml-model-q4_0.bin\" 2"
|
||||||
|
echo " --finetune (-f): Run finetune command to create a lora finetune of the model"
|
||||||
|
echo " See documentation for finetune for command-line parameters"
|
||||||
echo " --all-in-one (-a): Execute --convert & --quantize"
|
echo " --all-in-one (-a): Execute --convert & --quantize"
|
||||||
echo " ex: \"/models/\" 7B"
|
echo " ex: \"/models/\" 7B"
|
||||||
|
echo " --server (-s): Run a model on the server"
|
||||||
|
echo " ex: -m /models/7B/ggml-model-q4_0.bin -c 2048 -ngl 43 -mg 1 --port 8080"
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
|
|
@ -1,18 +1,14 @@
|
||||||
*.o
|
*.o
|
||||||
*.a
|
*.a
|
||||||
.cache/
|
.cache/
|
||||||
|
.git/
|
||||||
|
.github/
|
||||||
|
.gitignore
|
||||||
.vs/
|
.vs/
|
||||||
.vscode/
|
.vscode/
|
||||||
.DS_Store
|
.DS_Store
|
||||||
|
|
||||||
build/
|
build*/
|
||||||
build-em/
|
|
||||||
build-debug/
|
|
||||||
build-release/
|
|
||||||
build-static/
|
|
||||||
build-no-accel/
|
|
||||||
build-sanitize-addr/
|
|
||||||
build-sanitize-thread/
|
|
||||||
|
|
||||||
models/*
|
models/*
|
||||||
|
|
||||||
|
|
|
||||||
1
.ecrc
1
.ecrc
|
|
@ -1,4 +1,5 @@
|
||||||
{
|
{
|
||||||
|
"Exclude": ["^\\.gitmodules$"],
|
||||||
"Disable": {
|
"Disable": {
|
||||||
"IndentSize": true
|
"IndentSize": true
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -15,5 +15,14 @@ indent_size = 4
|
||||||
[Makefile]
|
[Makefile]
|
||||||
indent_style = tab
|
indent_style = tab
|
||||||
|
|
||||||
|
[scripts/*.mk]
|
||||||
|
indent_style = tab
|
||||||
|
|
||||||
[prompts/*.txt]
|
[prompts/*.txt]
|
||||||
insert_final_newline = unset
|
insert_final_newline = unset
|
||||||
|
|
||||||
|
[examples/server/public/*]
|
||||||
|
indent_size = 2
|
||||||
|
|
||||||
|
[examples/llama.swiftui/llama.swiftui.xcodeproj/*]
|
||||||
|
indent_style = tab
|
||||||
|
|
|
||||||
17
.flake8
Normal file
17
.flake8
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
[flake8]
|
||||||
|
max-line-length = 125
|
||||||
|
ignore = E203,E211,E221,E225,E231,E241,E251,E261,E266,E501,E701,E704,W503
|
||||||
|
exclude =
|
||||||
|
# Do not traverse examples
|
||||||
|
examples,
|
||||||
|
# Do not include package initializers
|
||||||
|
__init__.py,
|
||||||
|
# No need to traverse our git directory
|
||||||
|
.git,
|
||||||
|
# There's no value in checking cache directories
|
||||||
|
__pycache__,
|
||||||
|
# No need to include the build path
|
||||||
|
build,
|
||||||
|
# This contains builds that we don't want to check
|
||||||
|
dist # This is generated with `python build .` for package releases
|
||||||
|
# max-complexity = 10
|
||||||
11
.github/ISSUE_TEMPLATE/bug.md
vendored
Normal file
11
.github/ISSUE_TEMPLATE/bug.md
vendored
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
---
|
||||||
|
name: Bug template
|
||||||
|
about: Used to report bugs in llama.cpp
|
||||||
|
labels: ["bug-unconfirmed"]
|
||||||
|
assignees: ''
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
Please include information about your system, the steps to reproduce the bug, and the version of llama.cpp that you are using. If possible, please provide a minimal code example that reproduces the bug.
|
||||||
|
|
||||||
|
If the bug concerns the server, please try to reproduce it first using the [server test scenario framework](https://github.com/ggerganov/llama.cpp/tree/master/examples/server/tests).
|
||||||
185
.github/ISSUE_TEMPLATE/custom.md
vendored
185
.github/ISSUE_TEMPLATE/custom.md
vendored
|
|
@ -1,185 +0,0 @@
|
||||||
---
|
|
||||||
name: Issue and enhancement template
|
|
||||||
about: Used to report issues and request enhancements for llama.cpp
|
|
||||||
title: "[User] Insert summary of your issue or enhancement.."
|
|
||||||
labels: ''
|
|
||||||
assignees: ''
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
# Prerequisites
|
|
||||||
|
|
||||||
Please answer the following questions for yourself before submitting an issue.
|
|
||||||
|
|
||||||
- [ ] I am running the latest code. Development is very rapid so there are no tagged versions as of now.
|
|
||||||
- [ ] I carefully followed the [README.md](https://github.com/ggerganov/llama.cpp/blob/master/README.md).
|
|
||||||
- [ ] I [searched using keywords relevant to my issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/filtering-and-searching-issues-and-pull-requests) to make sure that I am creating a new issue that is not already open (or closed).
|
|
||||||
- [ ] I reviewed the [Discussions](https://github.com/ggerganov/llama.cpp/discussions), and have a new bug or useful enhancement to share.
|
|
||||||
|
|
||||||
# Expected Behavior
|
|
||||||
|
|
||||||
Please provide a detailed written description of what you were trying to do, and what you expected `llama.cpp` to do.
|
|
||||||
|
|
||||||
# Current Behavior
|
|
||||||
|
|
||||||
Please provide a detailed written description of what `llama.cpp` did, instead.
|
|
||||||
|
|
||||||
# Environment and Context
|
|
||||||
|
|
||||||
Please provide detailed information about your computer setup. This is important in case the issue is not reproducible except for under certain specific conditions.
|
|
||||||
|
|
||||||
* Physical (or virtual) hardware you are using, e.g. for Linux:
|
|
||||||
|
|
||||||
`$ lscpu`
|
|
||||||
|
|
||||||
* Operating System, e.g. for Linux:
|
|
||||||
|
|
||||||
`$ uname -a`
|
|
||||||
|
|
||||||
* SDK version, e.g. for Linux:
|
|
||||||
|
|
||||||
```
|
|
||||||
$ python3 --version
|
|
||||||
$ make --version
|
|
||||||
$ g++ --version
|
|
||||||
```
|
|
||||||
|
|
||||||
# Failure Information (for bugs)
|
|
||||||
|
|
||||||
Please help provide information about the failure if this is a bug. If it is not a bug, please remove the rest of this template.
|
|
||||||
|
|
||||||
# Steps to Reproduce
|
|
||||||
|
|
||||||
Please provide detailed steps for reproducing the issue. We are not sitting in front of your screen, so the more detail the better.
|
|
||||||
|
|
||||||
1. step 1
|
|
||||||
2. step 2
|
|
||||||
3. step 3
|
|
||||||
4. etc.
|
|
||||||
|
|
||||||
# Failure Logs
|
|
||||||
|
|
||||||
Please include any relevant log snippets or files. If it works under one configuration but not under another, please provide logs for both configurations and their corresponding outputs so it is easy to see where behavior changes.
|
|
||||||
|
|
||||||
Also, please try to **avoid using screenshots** if at all possible. Instead, copy/paste the console output and use [Github's markdown](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax) to cleanly format your logs for easy readability.
|
|
||||||
|
|
||||||
Example environment info:
|
|
||||||
```
|
|
||||||
llama.cpp$ git log | head -1
|
|
||||||
commit 2af23d30434a677c6416812eea52ccc0af65119c
|
|
||||||
|
|
||||||
llama.cpp$ lscpu | egrep "AMD|Flags"
|
|
||||||
Vendor ID: AuthenticAMD
|
|
||||||
Model name: AMD Ryzen Threadripper 1950X 16-Core Processor
|
|
||||||
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid amd_dcm aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb hw_pstate ssbd ibpb vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt sha_ni xsaveopt xsavec xgetbv1 xsaves clzero irperf xsaveerptr arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif overflow_recov succor smca sme sev
|
|
||||||
Virtualization: AMD-V
|
|
||||||
|
|
||||||
llama.cpp$ python3 --version
|
|
||||||
Python 3.10.9
|
|
||||||
|
|
||||||
llama.cpp$ pip list | egrep "torch|numpy|sentencepiece"
|
|
||||||
numpy 1.24.2
|
|
||||||
numpydoc 1.5.0
|
|
||||||
sentencepiece 0.1.97
|
|
||||||
torch 1.13.1
|
|
||||||
torchvision 0.14.1
|
|
||||||
|
|
||||||
llama.cpp$ make --version | head -1
|
|
||||||
GNU Make 4.3
|
|
||||||
|
|
||||||
$ md5sum ./models/65B/ggml-model-q4_0.bin
|
|
||||||
dbdd682cce80e2d6e93cefc7449df487 ./models/65B/ggml-model-q4_0.bin
|
|
||||||
```
|
|
||||||
|
|
||||||
Example run with the Linux command [perf](https://www.brendangregg.com/perf.html)
|
|
||||||
```
|
|
||||||
llama.cpp$ perf stat ./main -m ./models/65B/ggml-model-q4_0.bin -t 16 -n 1024 -p "Please close your issue when it has been answered."
|
|
||||||
main: seed = 1679149377
|
|
||||||
llama_model_load: loading model from './models/65B/ggml-model-q4_0.bin' - please wait ...
|
|
||||||
llama_model_load: n_vocab = 32000
|
|
||||||
llama_model_load: n_ctx = 512
|
|
||||||
llama_model_load: n_embd = 8192
|
|
||||||
llama_model_load: n_mult = 256
|
|
||||||
llama_model_load: n_head = 64
|
|
||||||
llama_model_load: n_layer = 80
|
|
||||||
llama_model_load: n_rot = 128
|
|
||||||
llama_model_load: f16 = 2
|
|
||||||
llama_model_load: n_ff = 22016
|
|
||||||
llama_model_load: n_parts = 8
|
|
||||||
llama_model_load: ggml ctx size = 41477.73 MB
|
|
||||||
llama_model_load: memory_size = 2560.00 MB, n_mem = 40960
|
|
||||||
llama_model_load: loading model part 1/8 from './models/65B/ggml-model-q4_0.bin'
|
|
||||||
llama_model_load: .......................................................................................... done
|
|
||||||
llama_model_load: model size = 4869.09 MB / num tensors = 723
|
|
||||||
llama_model_load: loading model part 2/8 from './models/65B/ggml-model-q4_0.bin.1'
|
|
||||||
llama_model_load: .......................................................................................... done
|
|
||||||
llama_model_load: model size = 4869.09 MB / num tensors = 723
|
|
||||||
llama_model_load: loading model part 3/8 from './models/65B/ggml-model-q4_0.bin.2'
|
|
||||||
llama_model_load: .......................................................................................... done
|
|
||||||
llama_model_load: model size = 4869.09 MB / num tensors = 723
|
|
||||||
llama_model_load: loading model part 4/8 from './models/65B/ggml-model-q4_0.bin.3'
|
|
||||||
llama_model_load: .......................................................................................... done
|
|
||||||
llama_model_load: model size = 4869.09 MB / num tensors = 723
|
|
||||||
llama_model_load: loading model part 5/8 from './models/65B/ggml-model-q4_0.bin.4'
|
|
||||||
llama_model_load: .......................................................................................... done
|
|
||||||
llama_model_load: model size = 4869.09 MB / num tensors = 723
|
|
||||||
llama_model_load: loading model part 6/8 from './models/65B/ggml-model-q4_0.bin.5'
|
|
||||||
llama_model_load: .......................................................................................... done
|
|
||||||
llama_model_load: model size = 4869.09 MB / num tensors = 723
|
|
||||||
llama_model_load: loading model part 7/8 from './models/65B/ggml-model-q4_0.bin.6'
|
|
||||||
llama_model_load: .......................................................................................... done
|
|
||||||
llama_model_load: model size = 4869.09 MB / num tensors = 723
|
|
||||||
llama_model_load: loading model part 8/8 from './models/65B/ggml-model-q4_0.bin.7'
|
|
||||||
llama_model_load: .......................................................................................... done
|
|
||||||
llama_model_load: model size = 4869.09 MB / num tensors = 723
|
|
||||||
|
|
||||||
system_info: n_threads = 16 / 32 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 0 | SSE3 = 1 | VSX = 0 |
|
|
||||||
|
|
||||||
main: prompt: 'Please close your issue when it has been answered.'
|
|
||||||
main: number of tokens in prompt = 11
|
|
||||||
1 -> ''
|
|
||||||
12148 -> 'Please'
|
|
||||||
3802 -> ' close'
|
|
||||||
596 -> ' your'
|
|
||||||
2228 -> ' issue'
|
|
||||||
746 -> ' when'
|
|
||||||
372 -> ' it'
|
|
||||||
756 -> ' has'
|
|
||||||
1063 -> ' been'
|
|
||||||
7699 -> ' answered'
|
|
||||||
29889 -> '.'
|
|
||||||
|
|
||||||
sampling parameters: temp = 0.800000, top_k = 40, top_p = 0.950000, repeat_last_n = 64, repeat_penalty = 1.300000
|
|
||||||
|
|
||||||
|
|
||||||
Please close your issue when it has been answered.
|
|
||||||
@duncan-donut: I'm trying to figure out what kind of "support" you need for this script and why, exactly? Is there a question about how the code works that hasn't already been addressed in one or more comments below this ticket, or are we talking something else entirely like some sorta bugfixing job because your server setup is different from mine??
|
|
||||||
I can understand if your site needs to be running smoothly and you need help with a fix of sorts but there should really be nothing wrong here that the code itself could not handle. And given that I'm getting reports about how it works perfectly well on some other servers, what exactly are we talking? A detailed report will do wonders in helping us get this resolved for ya quickly so please take your time and describe the issue(s) you see as clearly & concisely as possible!!
|
|
||||||
@duncan-donut: I'm not sure if you have access to cPanel but you could try these instructions. It is worth a shot! Let me know how it goes (or what error message, exactly!) when/if ya give that code a go? [end of text]
|
|
||||||
|
|
||||||
|
|
||||||
main: mem per token = 71159620 bytes
|
|
||||||
main: load time = 19309.95 ms
|
|
||||||
main: sample time = 168.62 ms
|
|
||||||
main: predict time = 223895.61 ms / 888.47 ms per token
|
|
||||||
main: total time = 246406.42 ms
|
|
||||||
|
|
||||||
Performance counter stats for './main -m ./models/65B/ggml-model-q4_0.bin -t 16 -n 1024 -p Please close your issue when it has been answered.':
|
|
||||||
|
|
||||||
3636882.89 msec task-clock # 14.677 CPUs utilized
|
|
||||||
13509 context-switches # 3.714 /sec
|
|
||||||
2436 cpu-migrations # 0.670 /sec
|
|
||||||
10476679 page-faults # 2.881 K/sec
|
|
||||||
13133115082869 cycles # 3.611 GHz (16.77%)
|
|
||||||
29314462753 stalled-cycles-frontend # 0.22% frontend cycles idle (16.76%)
|
|
||||||
10294402631459 stalled-cycles-backend # 78.39% backend cycles idle (16.74%)
|
|
||||||
23479217109614 instructions # 1.79 insn per cycle
|
|
||||||
# 0.44 stalled cycles per insn (16.76%)
|
|
||||||
2353072268027 branches # 647.002 M/sec (16.77%)
|
|
||||||
1998682780 branch-misses # 0.08% of all branches (16.76%)
|
|
||||||
|
|
||||||
247.802177522 seconds time elapsed
|
|
||||||
|
|
||||||
3618.573072000 seconds user
|
|
||||||
18.491698000 seconds sys
|
|
||||||
```
|
|
||||||
28
.github/ISSUE_TEMPLATE/enhancement.md
vendored
Normal file
28
.github/ISSUE_TEMPLATE/enhancement.md
vendored
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
---
|
||||||
|
name: Enhancement template
|
||||||
|
about: Used to request enhancements for llama.cpp
|
||||||
|
labels: ["enhancement"]
|
||||||
|
assignees: ''
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Prerequisites
|
||||||
|
|
||||||
|
Please answer the following questions for yourself before submitting an issue.
|
||||||
|
|
||||||
|
- [ ] I am running the latest code. Development is very rapid so there are no tagged versions as of now.
|
||||||
|
- [ ] I carefully followed the [README.md](https://github.com/ggerganov/llama.cpp/blob/master/README.md).
|
||||||
|
- [ ] I [searched using keywords relevant to my issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/filtering-and-searching-issues-and-pull-requests) to make sure that I am creating a new issue that is not already open (or closed).
|
||||||
|
- [ ] I reviewed the [Discussions](https://github.com/ggerganov/llama.cpp/discussions), and have a new bug or useful enhancement to share.
|
||||||
|
|
||||||
|
# Feature Description
|
||||||
|
|
||||||
|
Please provide a detailed written description of what you were trying to do, and what you expected `llama.cpp` to do as an enhancement.
|
||||||
|
|
||||||
|
# Motivation
|
||||||
|
|
||||||
|
Please provide a detailed written description of reasons why this feature is necessary and how it is useful to `llama.cpp` users.
|
||||||
|
|
||||||
|
# Possible Implementation
|
||||||
|
|
||||||
|
If you have an idea as to how it can be implemented, please write a detailed description. Feel free to give links to external sources or share visuals that might be helpful to understand the details better.
|
||||||
310
.github/workflows/bench.yml
vendored
Normal file
310
.github/workflows/bench.yml
vendored
Normal file
|
|
@ -0,0 +1,310 @@
|
||||||
|
# Benchmark
|
||||||
|
name: Benchmark
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
gpu-series:
|
||||||
|
description: 'Azure GPU series to run with'
|
||||||
|
required: true
|
||||||
|
type: choice
|
||||||
|
options:
|
||||||
|
- Standard_NC4as_T4_v3
|
||||||
|
- Standard_NC24ads_A100_v4
|
||||||
|
- Standard_NC80adis_H100_v5
|
||||||
|
sha:
|
||||||
|
description: 'Commit SHA1 to build'
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
duration:
|
||||||
|
description: 'Duration of the bench'
|
||||||
|
type: string
|
||||||
|
default: 10m
|
||||||
|
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
paths: ['llama.cpp', 'ggml.c', 'ggml-backend.c', 'ggml-quants.c', '**/*.cu', 'examples/server/*.h*', 'examples/server/*.cpp']
|
||||||
|
pull_request_target:
|
||||||
|
types: [opened, synchronize, reopened]
|
||||||
|
paths: ['llama.cpp', 'ggml.c', 'ggml-backend.c', 'ggml-quants.c', '**/*.cu', 'examples/server/*.h*', 'examples/server/*.cpp']
|
||||||
|
schedule:
|
||||||
|
- cron: '04 2 * * *'
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}-${{ github.event.inputs.sha }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
bench-server-baseline:
|
||||||
|
runs-on: Standard_NC4as_T4_v3
|
||||||
|
env:
|
||||||
|
RUNNER_LABEL: Standard_NC4as_T4_v3 # FIXME Do not find a way to not duplicate it
|
||||||
|
N_USERS: 8
|
||||||
|
DURATION: 10m
|
||||||
|
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
model: [phi-2]
|
||||||
|
ftype: [q4_0, q8_0, f16]
|
||||||
|
include:
|
||||||
|
- model: phi-2
|
||||||
|
ftype: q4_0
|
||||||
|
pr_comment_enabled: "true"
|
||||||
|
|
||||||
|
if: |
|
||||||
|
inputs.gpu-series == 'Standard_NC4as_T4_v3'
|
||||||
|
|| (
|
||||||
|
github.event_name == 'schedule'
|
||||||
|
&& github.ref_name == 'master'
|
||||||
|
&& github.repository_owner == 'ggerganov'
|
||||||
|
)
|
||||||
|
|| github.event_name == 'pull_request_target'
|
||||||
|
|| (
|
||||||
|
github.event_name == 'push'
|
||||||
|
&& github.event.ref == 'refs/heads/master'
|
||||||
|
&& github.repository_owner == 'ggerganov'
|
||||||
|
)
|
||||||
|
steps:
|
||||||
|
- name: Clone
|
||||||
|
id: checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||||
|
|
||||||
|
- name: Install python env
|
||||||
|
id: pipenv
|
||||||
|
run: |
|
||||||
|
cd examples/server/bench
|
||||||
|
python3 -m venv venv
|
||||||
|
source venv/bin/activate
|
||||||
|
pip install -r requirements.txt
|
||||||
|
|
||||||
|
- name: Prometheus
|
||||||
|
id: install_prometheus
|
||||||
|
run: |
|
||||||
|
wget --quiet https://github.com/prometheus/prometheus/releases/download/v2.51.0/prometheus-2.51.0.linux-amd64.tar.gz
|
||||||
|
tar xzf prometheus*.tar.gz --strip-components=1
|
||||||
|
./prometheus --config.file=examples/server/bench/prometheus.yml &
|
||||||
|
while ! nc -z localhost 9090; do
|
||||||
|
sleep 0.1
|
||||||
|
done
|
||||||
|
|
||||||
|
- name: Set up Go
|
||||||
|
uses: actions/setup-go@v5
|
||||||
|
with:
|
||||||
|
go-version: '1.21'
|
||||||
|
|
||||||
|
- name: Install k6 and xk6-sse
|
||||||
|
id: k6_installation
|
||||||
|
run: |
|
||||||
|
cd examples/server/bench
|
||||||
|
go install go.k6.io/xk6/cmd/xk6@latest
|
||||||
|
xk6 build master \
|
||||||
|
--with github.com/phymbert/xk6-sse
|
||||||
|
|
||||||
|
- name: Build
|
||||||
|
id: cmake_build
|
||||||
|
run: |
|
||||||
|
set -eux
|
||||||
|
cmake -B build \
|
||||||
|
-DLLAMA_NATIVE=OFF \
|
||||||
|
-DLLAMA_BUILD_SERVER=ON \
|
||||||
|
-DLLAMA_CURL=ON \
|
||||||
|
-DLLAMA_CUBLAS=ON \
|
||||||
|
-DCUDAToolkit_ROOT=/usr/local/cuda \
|
||||||
|
-DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \
|
||||||
|
-DCMAKE_CUDA_ARCHITECTURES=75 \
|
||||||
|
-DLLAMA_FATAL_WARNINGS=OFF \
|
||||||
|
-DLLAMA_ALL_WARNINGS=OFF \
|
||||||
|
-DCMAKE_BUILD_TYPE=Release;
|
||||||
|
cmake --build build --config Release -j $(nproc) --target server
|
||||||
|
|
||||||
|
- name: Download the dataset
|
||||||
|
id: download_dataset
|
||||||
|
run: |
|
||||||
|
cd examples/server/bench
|
||||||
|
wget --quiet https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
|
||||||
|
|
||||||
|
- name: Server bench
|
||||||
|
id: server_bench
|
||||||
|
run: |
|
||||||
|
set -eux
|
||||||
|
|
||||||
|
cd examples/server/bench
|
||||||
|
source venv/bin/activate
|
||||||
|
python bench.py \
|
||||||
|
--runner-label ${{ env.RUNNER_LABEL }} \
|
||||||
|
--name ${{ github.job }} \
|
||||||
|
--branch ${{ github.head_ref || github.ref_name }} \
|
||||||
|
--commit ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha }} \
|
||||||
|
--scenario script.js \
|
||||||
|
--duration ${{ github.event.inputs.duration || env.DURATION }} \
|
||||||
|
--hf-repo ggml-org/models \
|
||||||
|
--hf-file ${{ matrix.model }}/ggml-model-${{ matrix.ftype }}.gguf \
|
||||||
|
--model-path-prefix /models \
|
||||||
|
--parallel ${{ env.N_USERS }} \
|
||||||
|
-ngl 33 \
|
||||||
|
--batch-size 2048 \
|
||||||
|
--ubatch-size 256 \
|
||||||
|
--ctx-size 16384 \
|
||||||
|
--n-prompts 1000 \
|
||||||
|
--max-prompt-tokens 1024 \
|
||||||
|
--max-tokens 2048
|
||||||
|
|
||||||
|
cat results.github.env >> $GITHUB_ENV
|
||||||
|
|
||||||
|
# Remove dataset as we do not want it in the artefact
|
||||||
|
rm ShareGPT_V3_unfiltered_cleaned_split.json
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: bench-server-${{ github.job }}-${{ env.RUNNER_LABEL }}-${{ matrix.model }}-${{ matrix.ftype }}
|
||||||
|
compression-level: 9
|
||||||
|
path: |
|
||||||
|
examples/server/bench/*.jpg
|
||||||
|
examples/server/bench/*.json
|
||||||
|
examples/server/bench/*.log
|
||||||
|
|
||||||
|
- name: Commit status
|
||||||
|
uses: Sibz/github-status-action@v1
|
||||||
|
with:
|
||||||
|
authToken: ${{secrets.GITHUB_TOKEN}}
|
||||||
|
sha: ${{ inputs.sha || github.event.pull_request.head.sha || github.sha }}
|
||||||
|
context: bench-server-${{ github.job }}-${{ env.RUNNER_LABEL }}-${{ matrix.model }}-${{ matrix.ftype }}
|
||||||
|
description: |
|
||||||
|
${{ env.BENCH_RESULTS }}
|
||||||
|
state: 'success'
|
||||||
|
|
||||||
|
- name: Upload benchmark images
|
||||||
|
uses: devicons/public-upload-to-imgur@v2.2.2
|
||||||
|
continue-on-error: true # Important as it looks unstable: 503
|
||||||
|
id: imgur_step
|
||||||
|
with:
|
||||||
|
client_id: ${{secrets.IMGUR_CLIENT_ID}}
|
||||||
|
path: |
|
||||||
|
examples/server/bench/prompt_tokens_seconds.jpg
|
||||||
|
examples/server/bench/predicted_tokens_seconds.jpg
|
||||||
|
examples/server/bench/kv_cache_usage_ratio.jpg
|
||||||
|
examples/server/bench/requests_processing.jpg
|
||||||
|
|
||||||
|
- name: Extract mermaid
|
||||||
|
id: set_mermaid
|
||||||
|
run: |
|
||||||
|
set -eux
|
||||||
|
|
||||||
|
cd examples/server/bench
|
||||||
|
PROMPT_TOKENS_SECONDS=$(cat prompt_tokens_seconds.mermaid)
|
||||||
|
echo "PROMPT_TOKENS_SECONDS<<EOF" >> $GITHUB_ENV
|
||||||
|
echo "$PROMPT_TOKENS_SECONDS" >> $GITHUB_ENV
|
||||||
|
echo "EOF" >> $GITHUB_ENV
|
||||||
|
|
||||||
|
PREDICTED_TOKENS_SECONDS=$(cat predicted_tokens_seconds.mermaid)
|
||||||
|
echo "PREDICTED_TOKENS_SECONDS<<EOF" >> $GITHUB_ENV
|
||||||
|
echo "$PREDICTED_TOKENS_SECONDS" >> $GITHUB_ENV
|
||||||
|
echo "EOF" >> $GITHUB_ENV
|
||||||
|
|
||||||
|
KV_CACHE_USAGE_RATIO=$(cat kv_cache_usage_ratio.mermaid)
|
||||||
|
echo "KV_CACHE_USAGE_RATIO<<EOF" >> $GITHUB_ENV
|
||||||
|
echo "$KV_CACHE_USAGE_RATIO" >> $GITHUB_ENV
|
||||||
|
echo "EOF" >> $GITHUB_ENV
|
||||||
|
|
||||||
|
REQUESTS_PROCESSING=$(cat requests_processing.mermaid)
|
||||||
|
echo "REQUESTS_PROCESSING<<EOF" >> $GITHUB_ENV
|
||||||
|
echo "$REQUESTS_PROCESSING" >> $GITHUB_ENV
|
||||||
|
echo "EOF" >> $GITHUB_ENV
|
||||||
|
|
||||||
|
- name: Extract image url
|
||||||
|
id: extract_image_url
|
||||||
|
continue-on-error: true
|
||||||
|
run: |
|
||||||
|
set -eux
|
||||||
|
|
||||||
|
echo "IMAGE_O=${{ fromJSON(steps.imgur_step.outputs.imgur_urls)[0] }}" >> $GITHUB_ENV
|
||||||
|
echo "IMAGE_1=${{ fromJSON(steps.imgur_step.outputs.imgur_urls)[1] }}" >> $GITHUB_ENV
|
||||||
|
echo "IMAGE_2=${{ fromJSON(steps.imgur_step.outputs.imgur_urls)[2] }}" >> $GITHUB_ENV
|
||||||
|
echo "IMAGE_3=${{ fromJSON(steps.imgur_step.outputs.imgur_urls)[3] }}" >> $GITHUB_ENV
|
||||||
|
|
||||||
|
- name: Comment PR
|
||||||
|
uses: mshick/add-pr-comment@v2
|
||||||
|
id: comment_pr
|
||||||
|
if: ${{ github.event.pull_request != '' && matrix.pr_comment_enabled == 'true' }}
|
||||||
|
with:
|
||||||
|
message-id: bench-server-${{ github.job }}-${{ env.RUNNER_LABEL }}-${{ matrix.model }}-${{ matrix.ftype }}
|
||||||
|
message: |
|
||||||
|
<p align="center">
|
||||||
|
|
||||||
|
📈 **llama.cpp server** for _${{ github.job }}_ on _${{ env.RUNNER_LABEL }}_ for `${{ matrix.model }}`-`${{ matrix.ftype }}`: **${{ env.BENCH_ITERATIONS}} iterations** 🚀
|
||||||
|
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<details>
|
||||||
|
|
||||||
|
<summary>Expand details for performance related PR only</summary>
|
||||||
|
|
||||||
|
- Concurrent users: ${{ env.N_USERS }}, duration: ${{ github.event.inputs.duration || env.DURATION }}
|
||||||
|
- HTTP request : avg=${{ env.HTTP_REQ_DURATION_AVG }}ms p(95)=${{ env.HTTP_REQ_DURATION_P_95_ }}ms fails=${{ env.HTTP_REQ_FAILED_PASSES }}, finish reason: stop=${{ env.LLAMACPP_COMPLETIONS_STOP_RATE_PASSES }} truncated=${{ env.LLAMACPP_COMPLETIONS_TRUNCATED_RATE_PASSES }}
|
||||||
|
- Prompt processing (pp): avg=${{ env.LLAMACPP_PROMPT_PROCESSING_SECOND_AVG }}tk/s p(95)=${{ env.LLAMACPP_PROMPT_PROCESSING_SECOND_P_95_ }}tk/s
|
||||||
|
- Token generation (tg): avg=${{ env.LLAMACPP_TOKENS_SECOND_AVG }}tk/s p(95)=${{ env.LLAMACPP_TOKENS_SECOND_P_95_ }}tk/s
|
||||||
|
- ${{ env.BENCH_GRAPH_XLABEL }}
|
||||||
|
|
||||||
|
|
||||||
|
<p align="center">
|
||||||
|
|
||||||
|
<img width="100%" height="100%" src="${{ env.IMAGE_O }}" alt="prompt_tokens_seconds" />
|
||||||
|
|
||||||
|
<details>
|
||||||
|
|
||||||
|
<summary>More</summary>
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
${{ env.PROMPT_TOKENS_SECONDS }}
|
||||||
|
```
|
||||||
|
|
||||||
|
</details>
|
||||||
|
|
||||||
|
<img width="100%" height="100%" src="${{ env.IMAGE_1 }}" alt="predicted_tokens_seconds"/>
|
||||||
|
|
||||||
|
<details>
|
||||||
|
<summary>More</summary>
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
${{ env.PREDICTED_TOKENS_SECONDS }}
|
||||||
|
```
|
||||||
|
|
||||||
|
</details>
|
||||||
|
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<details>
|
||||||
|
|
||||||
|
<summary>Details</summary>
|
||||||
|
|
||||||
|
<p align="center">
|
||||||
|
|
||||||
|
<img width="100%" height="100%" src="${{ env.IMAGE_2 }}" alt="kv_cache_usage_ratio" />
|
||||||
|
|
||||||
|
<details>
|
||||||
|
<summary>More</summary>
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
${{ env.KV_CACHE_USAGE_RATIO }}
|
||||||
|
```
|
||||||
|
|
||||||
|
</details>
|
||||||
|
|
||||||
|
<img width="100%" height="100%" src="${{ env.IMAGE_3 }}" alt="requests_processing"/>
|
||||||
|
|
||||||
|
<details>
|
||||||
|
<summary>More</summary>
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
${{ env.REQUESTS_PROCESSING }}
|
||||||
|
```
|
||||||
|
|
||||||
|
</details>
|
||||||
|
|
||||||
|
</p>
|
||||||
|
</details>
|
||||||
|
</details>
|
||||||
949
.github/workflows/build.yml
vendored
949
.github/workflows/build.yml
vendored
File diff suppressed because it is too large
Load diff
23
.github/workflows/close-issue.yml
vendored
Normal file
23
.github/workflows/close-issue.yml
vendored
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
name: Close inactive issues
|
||||||
|
on:
|
||||||
|
schedule:
|
||||||
|
- cron: "42 0 * * *"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
close-issues:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
issues: write
|
||||||
|
pull-requests: write
|
||||||
|
steps:
|
||||||
|
- uses: actions/stale@v5
|
||||||
|
with:
|
||||||
|
exempt-issue-labels: "refactor,help wanted,good first issue,research,bug"
|
||||||
|
days-before-issue-stale: 30
|
||||||
|
days-before-issue-close: 14
|
||||||
|
stale-issue-label: "stale"
|
||||||
|
close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
|
||||||
|
days-before-pr-stale: -1
|
||||||
|
days-before-pr-close: -1
|
||||||
|
operations-per-run: 10000
|
||||||
|
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
40
.github/workflows/code-coverage.yml
vendored
Normal file
40
.github/workflows/code-coverage.yml
vendored
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
name: Code Coverage
|
||||||
|
on: [push, pull_request]
|
||||||
|
|
||||||
|
env:
|
||||||
|
GGML_NLOOP: 3
|
||||||
|
GGML_N_THREADS: 1
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
run:
|
||||||
|
runs-on: ubuntu-20.04
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Dependencies
|
||||||
|
run: |
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install build-essential gcc-8 lcov
|
||||||
|
|
||||||
|
- name: Build
|
||||||
|
run: CC=gcc-8 make -j LLAMA_CODE_COVERAGE=1 tests
|
||||||
|
|
||||||
|
- name: Run tests
|
||||||
|
run: CC=gcc-8 make test
|
||||||
|
|
||||||
|
- name: Generate coverage report
|
||||||
|
run: |
|
||||||
|
make coverage
|
||||||
|
make lcov-report
|
||||||
|
|
||||||
|
- name: Upload coverage to Codecov
|
||||||
|
uses: codecov/codecov-action@v3
|
||||||
|
env:
|
||||||
|
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||||
|
with:
|
||||||
|
files: lcov-report/coverage.info
|
||||||
68
.github/workflows/docker.yml
vendored
68
.github/workflows/docker.yml
vendored
|
|
@ -15,20 +15,38 @@ on:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
push_to_registry:
|
push_to_registry:
|
||||||
name: Push Docker image to Docker Hub
|
name: Push Docker image to Docker Hub
|
||||||
|
if: github.event.pull_request.draft == false
|
||||||
|
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
env:
|
env:
|
||||||
COMMIT_SHA: ${{ github.sha }}
|
COMMIT_SHA: ${{ github.sha }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
config:
|
config:
|
||||||
- { tag: "light", dockerfile: ".devops/main.Dockerfile" }
|
- { tag: "light", dockerfile: ".devops/main.Dockerfile", platforms: "linux/amd64,linux/arm64" }
|
||||||
- { tag: "full", dockerfile: ".devops/full.Dockerfile" }
|
- { tag: "full", dockerfile: ".devops/full.Dockerfile", platforms: "linux/amd64,linux/arm64" }
|
||||||
|
- { tag: "server", dockerfile: ".devops/server.Dockerfile", platforms: "linux/amd64,linux/arm64" }
|
||||||
|
# NOTE(canardletter): The CUDA builds on arm64 are very slow, so I
|
||||||
|
# have disabled them for now until the reason why
|
||||||
|
# is understood.
|
||||||
|
- { tag: "light-cuda", dockerfile: ".devops/main-cuda.Dockerfile", platforms: "linux/amd64" }
|
||||||
|
- { tag: "full-cuda", dockerfile: ".devops/full-cuda.Dockerfile", platforms: "linux/amd64" }
|
||||||
|
- { tag: "server-cuda", dockerfile: ".devops/server-cuda.Dockerfile", platforms: "linux/amd64" }
|
||||||
|
- { tag: "light-rocm", dockerfile: ".devops/main-rocm.Dockerfile", platforms: "linux/amd64,linux/arm64" }
|
||||||
|
- { tag: "full-rocm", dockerfile: ".devops/full-rocm.Dockerfile", platforms: "linux/amd64,linux/arm64" }
|
||||||
|
- { tag: "server-rocm", dockerfile: ".devops/server-rocm.Dockerfile", platforms: "linux/amd64,linux/arm64" }
|
||||||
|
- { tag: "light-intel", dockerfile: ".devops/main-intel.Dockerfile", platforms: "linux/amd64" }
|
||||||
|
- { tag: "server-intel", dockerfile: ".devops/server-intel.Dockerfile", platforms: "linux/amd64" }
|
||||||
steps:
|
steps:
|
||||||
- name: Check out the repo
|
- name: Check out the repo
|
||||||
uses: actions/checkout@v3
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up QEMU
|
- name: Set up QEMU
|
||||||
uses: docker/setup-qemu-action@v2
|
uses: docker/setup-qemu-action@v2
|
||||||
|
|
@ -43,14 +61,50 @@ jobs:
|
||||||
username: ${{ github.repository_owner }}
|
username: ${{ github.repository_owner }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
# https://github.com/jlumbroso/free-disk-space/tree/54081f138730dfa15788a46383842cd2f914a1be#example
|
||||||
|
- name: Free Disk Space (Ubuntu)
|
||||||
|
uses: jlumbroso/free-disk-space@main
|
||||||
|
with:
|
||||||
|
# this might remove tools that are actually needed,
|
||||||
|
# if set to "true" but frees about 6 GB
|
||||||
|
tool-cache: false
|
||||||
|
|
||||||
|
# all of these default to true, but feel free to set to
|
||||||
|
# "false" if necessary for your workflow
|
||||||
|
android: true
|
||||||
|
dotnet: true
|
||||||
|
haskell: true
|
||||||
|
large-packages: true
|
||||||
|
docker-images: true
|
||||||
|
swap-storage: true
|
||||||
|
|
||||||
|
- name: Determine tag name
|
||||||
|
id: tag
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
BUILD_NUMBER="$(git rev-list --count HEAD)"
|
||||||
|
SHORT_HASH="$(git rev-parse --short=7 HEAD)"
|
||||||
|
if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then
|
||||||
|
echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT
|
||||||
|
else
|
||||||
|
SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
|
||||||
|
echo "name=${SAFE_NAME}-b${BUILD_NUMBER}-${SHORT_HASH}" >> $GITHUB_OUTPUT
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Downcase github.repository_owner
|
||||||
|
run: |
|
||||||
|
echo "repository_owner_lowercase=${GITHUB_REPOSITORY_OWNER@L}" >> $GITHUB_ENV
|
||||||
|
env:
|
||||||
|
GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}'
|
||||||
|
|
||||||
- name: Build and push Docker image (versioned)
|
- name: Build and push Docker image (versioned)
|
||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
uses: docker/build-push-action@v4
|
uses: docker/build-push-action@v4
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
push: true
|
push: true
|
||||||
platforms: linux/amd64,linux/arm64
|
platforms: ${{ matrix.config.platforms }}
|
||||||
tags: "ghcr.io/ggerganov/llama.cpp:${{ matrix.config.tag }}-${{ env.COMMIT_SHA }}"
|
tags: "ghcr.io/${{ env.repository_owner_lowercase }}/llama.cpp:${{ matrix.config.tag }}-${{ env.COMMIT_SHA }}"
|
||||||
file: ${{ matrix.config.dockerfile }}
|
file: ${{ matrix.config.dockerfile }}
|
||||||
|
|
||||||
- name: Build and push Docker image (tagged)
|
- name: Build and push Docker image (tagged)
|
||||||
|
|
@ -58,6 +112,6 @@ jobs:
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
push: ${{ github.event_name == 'push' }}
|
push: ${{ github.event_name == 'push' }}
|
||||||
platforms: linux/amd64,linux/arm64
|
platforms: ${{ matrix.config.platforms }}
|
||||||
tags: "ghcr.io/ggerganov/llama.cpp:${{ matrix.config.tag }}"
|
tags: "ghcr.io/${{ env.repository_owner_lowercase }}/llama.cpp:${{ matrix.config.tag }},ghcr.io/${{ env.repository_owner_lowercase }}/llama.cpp:${{ matrix.config.tag }}-${{ steps.tag.outputs.name }}"
|
||||||
file: ${{ matrix.config.dockerfile }}
|
file: ${{ matrix.config.dockerfile }}
|
||||||
|
|
|
||||||
12
.github/workflows/editorconfig.yml
vendored
12
.github/workflows/editorconfig.yml
vendored
|
|
@ -1,6 +1,12 @@
|
||||||
name: EditorConfig Checker
|
name: EditorConfig Checker
|
||||||
|
|
||||||
on:
|
on:
|
||||||
|
workflow_dispatch: # allows manual triggering
|
||||||
|
inputs:
|
||||||
|
create_release:
|
||||||
|
description: 'Create new release'
|
||||||
|
required: true
|
||||||
|
type: boolean
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
|
@ -8,10 +14,14 @@ on:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
editorconfig:
|
editorconfig:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v4
|
||||||
- uses: editorconfig-checker/action-editorconfig-checker@main
|
- uses: editorconfig-checker/action-editorconfig-checker@main
|
||||||
- run: editorconfig-checker
|
- run: editorconfig-checker
|
||||||
|
|
|
||||||
44
.github/workflows/gguf-publish.yml
vendored
Normal file
44
.github/workflows/gguf-publish.yml
vendored
Normal file
|
|
@ -0,0 +1,44 @@
|
||||||
|
# This workflow will upload a Python Package using Twine when a GGUF release is created
|
||||||
|
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
|
||||||
|
|
||||||
|
# See `gguf-py/README.md` for how to make a release.
|
||||||
|
|
||||||
|
# This workflow uses actions that are not certified by GitHub.
|
||||||
|
# They are provided by a third-party and are governed by
|
||||||
|
# separate terms of service, privacy policy, and support
|
||||||
|
# documentation.
|
||||||
|
|
||||||
|
name: Upload Python Package
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
# Pattern matched against refs/tags
|
||||||
|
tags:
|
||||||
|
- 'gguf-v*' # Push events to every version tag
|
||||||
|
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
deploy:
|
||||||
|
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.9.x'
|
||||||
|
- name: Install dependencies
|
||||||
|
run: |
|
||||||
|
cd gguf-py
|
||||||
|
python -m pip install poetry
|
||||||
|
poetry install
|
||||||
|
|
||||||
|
- name: Build package
|
||||||
|
run: cd gguf-py && poetry build
|
||||||
|
- name: Publish package
|
||||||
|
uses: pypa/gh-action-pypi-publish@release/v1
|
||||||
|
with:
|
||||||
|
password: ${{ secrets.PYPI_API_TOKEN }}
|
||||||
|
packages-dir: gguf-py/dist
|
||||||
65
.github/workflows/nix-ci-aarch64.yml
vendored
Normal file
65
.github/workflows/nix-ci-aarch64.yml
vendored
Normal file
|
|
@ -0,0 +1,65 @@
|
||||||
|
name: Nix aarch64 builds
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch: # allows manual triggering
|
||||||
|
schedule:
|
||||||
|
# Rebuild daily rather than on every push because QEMU is expensive (e.g.
|
||||||
|
# 1.5h instead of minutes with the cold cache).
|
||||||
|
#
|
||||||
|
# randint(0, 59), randint(0, 23)
|
||||||
|
- cron: '26 12 * * *'
|
||||||
|
# But also rebuild if we touched any of the Nix expressions:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
paths: ['**/*.nix', 'flake.lock']
|
||||||
|
pull_request:
|
||||||
|
types: [opened, synchronize, reopened]
|
||||||
|
paths: ['**/*.nix', 'flake.lock']
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
nix-build-aarch64:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
- name: Install QEMU
|
||||||
|
# Copy-paste from https://github.com/orgs/community/discussions/8305#discussioncomment-5888654
|
||||||
|
run: |
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y qemu-user-static qemu-system-aarch64
|
||||||
|
sudo usermod -a -G kvm $USER
|
||||||
|
- name: Install Nix
|
||||||
|
uses: DeterminateSystems/nix-installer-action@v9
|
||||||
|
with:
|
||||||
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
extra-conf: |
|
||||||
|
extra-platforms = aarch64-linux
|
||||||
|
extra-system-features = nixos-test kvm
|
||||||
|
extra-substituters = https://llama-cpp.cachix.org https://cuda-maintainers.cachix.org
|
||||||
|
extra-trusted-public-keys = llama-cpp.cachix.org-1:H75X+w83wUKTIPSO1KWy9ADUrzThyGs8P5tmAbkWhQc= cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E=
|
||||||
|
- uses: DeterminateSystems/magic-nix-cache-action@v2
|
||||||
|
with:
|
||||||
|
upstream-cache: https://${{ matrix.cachixName }}.cachix.org
|
||||||
|
- name: Set-up cachix to push the results to
|
||||||
|
uses: cachix/cachix-action@v13
|
||||||
|
with:
|
||||||
|
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
|
||||||
|
name: llama-cpp
|
||||||
|
- name: Show all output paths
|
||||||
|
run: >
|
||||||
|
nix run github:nix-community/nix-eval-jobs
|
||||||
|
-- --gc-roots-dir gcroot
|
||||||
|
--flake
|
||||||
|
".#packages.aarch64-linux"
|
||||||
|
- name: Build
|
||||||
|
run: >
|
||||||
|
nix run github:Mic92/nix-fast-build
|
||||||
|
-- --skip-cached --no-nom
|
||||||
|
--systems aarch64-linux
|
||||||
|
--flake
|
||||||
|
".#checks.aarch64-linux"
|
||||||
72
.github/workflows/nix-ci.yml
vendored
Normal file
72
.github/workflows/nix-ci.yml
vendored
Normal file
|
|
@ -0,0 +1,72 @@
|
||||||
|
name: Nix CI
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch: # allows manual triggering
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
pull_request:
|
||||||
|
types: [opened, synchronize, reopened]
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
nix-eval:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
os: [ ubuntu-latest, macos-latest ]
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
- name: Install Nix
|
||||||
|
uses: DeterminateSystems/nix-installer-action@v9
|
||||||
|
with:
|
||||||
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
extra-conf: |
|
||||||
|
extra-substituters = https://llama-cpp.cachix.org https://cuda-maintainers.cachix.org
|
||||||
|
extra-trusted-public-keys = llama-cpp.cachix.org-1:H75X+w83wUKTIPSO1KWy9ADUrzThyGs8P5tmAbkWhQc= cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E=
|
||||||
|
- uses: DeterminateSystems/magic-nix-cache-action@v2
|
||||||
|
with:
|
||||||
|
upstream-cache: https://${{ matrix.cachixName }}.cachix.org
|
||||||
|
- name: List all flake outputs
|
||||||
|
run: nix flake show --all-systems
|
||||||
|
- name: Show all output paths
|
||||||
|
run: >
|
||||||
|
nix run github:nix-community/nix-eval-jobs
|
||||||
|
-- --gc-roots-dir gcroot
|
||||||
|
--flake
|
||||||
|
".#packages.$(nix eval --raw --impure --expr builtins.currentSystem)"
|
||||||
|
nix-build:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
os: [ ubuntu-latest, macos-latest ]
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
- name: Install Nix
|
||||||
|
uses: DeterminateSystems/nix-installer-action@v9
|
||||||
|
with:
|
||||||
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
extra-conf: |
|
||||||
|
extra-substituters = https://llama-cpp.cachix.org https://cuda-maintainers.cachix.org
|
||||||
|
extra-trusted-public-keys = llama-cpp.cachix.org-1:H75X+w83wUKTIPSO1KWy9ADUrzThyGs8P5tmAbkWhQc= cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E=
|
||||||
|
- uses: DeterminateSystems/magic-nix-cache-action@v2
|
||||||
|
with:
|
||||||
|
upstream-cache: https://${{ matrix.cachixName }}.cachix.org
|
||||||
|
- name: Set-up cachix to push the results to
|
||||||
|
uses: cachix/cachix-action@v13
|
||||||
|
with:
|
||||||
|
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
|
||||||
|
name: llama-cpp
|
||||||
|
- name: Build
|
||||||
|
run: >
|
||||||
|
nix run github:Mic92/nix-fast-build
|
||||||
|
-- --skip-cached --no-nom
|
||||||
|
--flake
|
||||||
|
".#checks.$(nix eval --raw --impure --expr builtins.currentSystem)"
|
||||||
22
.github/workflows/nix-flake-update.yml
vendored
Normal file
22
.github/workflows/nix-flake-update.yml
vendored
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
name: update-flake-lock
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
schedule:
|
||||||
|
- cron: '0 0 * * 0' # runs weekly on Sunday at 00:00
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
lockfile:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
- name: Install Nix
|
||||||
|
uses: DeterminateSystems/nix-installer-action@main
|
||||||
|
- name: Update flake.lock
|
||||||
|
uses: DeterminateSystems/update-flake-lock@main
|
||||||
|
with:
|
||||||
|
pr-title: "nix: update flake.lock"
|
||||||
|
pr-labels: |
|
||||||
|
nix
|
||||||
|
pr-reviewers: philiptaron,SomeoneSerge
|
||||||
|
token: ${{ secrets.FLAKE_TOKEN }}
|
||||||
36
.github/workflows/nix-publish-flake.yml
vendored
Normal file
36
.github/workflows/nix-publish-flake.yml
vendored
Normal file
|
|
@ -0,0 +1,36 @@
|
||||||
|
# Make the flake discoverable on https://flakestry.dev and https://flakehub.com/flakes
|
||||||
|
name: "Publish a flake to flakestry & flakehub"
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- "*"
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
tag:
|
||||||
|
description: "The existing tag to publish"
|
||||||
|
type: "string"
|
||||||
|
required: true
|
||||||
|
jobs:
|
||||||
|
flakestry-publish:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
id-token: "write"
|
||||||
|
contents: "read"
|
||||||
|
steps:
|
||||||
|
- uses: flakestry/flakestry-publish@main
|
||||||
|
with:
|
||||||
|
version: "${{ inputs.tag || github.ref_name }}"
|
||||||
|
flakehub-publish:
|
||||||
|
runs-on: "ubuntu-latest"
|
||||||
|
permissions:
|
||||||
|
id-token: "write"
|
||||||
|
contents: "read"
|
||||||
|
steps:
|
||||||
|
- uses: "actions/checkout@v4"
|
||||||
|
with:
|
||||||
|
ref: "${{ (inputs.tag != null) && format('refs/tags/{0}', inputs.tag) || '' }}"
|
||||||
|
- uses: "DeterminateSystems/nix-installer-action@main"
|
||||||
|
- uses: "DeterminateSystems/flakehub-push@main"
|
||||||
|
with:
|
||||||
|
visibility: "public"
|
||||||
|
tag: "${{ inputs.tag }}"
|
||||||
35
.github/workflows/python-check-requirements.yml
vendored
Normal file
35
.github/workflows/python-check-requirements.yml
vendored
Normal file
|
|
@ -0,0 +1,35 @@
|
||||||
|
name: Python check requirements.txt
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
paths:
|
||||||
|
- '.github/workflows/python-check-requirements.yml'
|
||||||
|
- 'scripts/check-requirements.sh'
|
||||||
|
- 'convert*.py'
|
||||||
|
- 'requirements.txt'
|
||||||
|
- 'requirements/*.txt'
|
||||||
|
pull_request:
|
||||||
|
paths:
|
||||||
|
- '.github/workflows/python-check-requirements.yml'
|
||||||
|
- 'scripts/check-requirements.sh'
|
||||||
|
- 'convert*.py'
|
||||||
|
- 'requirements.txt'
|
||||||
|
- 'requirements/*.txt'
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
python-check-requirements:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
name: check-requirements
|
||||||
|
steps:
|
||||||
|
- name: Check out source repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
- name: Set up Python environment
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.11"
|
||||||
|
- name: Run check-requirements.sh script
|
||||||
|
run: bash scripts/check-requirements.sh
|
||||||
23
.github/workflows/python-lint.yml
vendored
Normal file
23
.github/workflows/python-lint.yml
vendored
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
name: flake8 Lint
|
||||||
|
|
||||||
|
on: [push, pull_request]
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
flake8-lint:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
name: Lint
|
||||||
|
steps:
|
||||||
|
- name: Check out source repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
- name: Set up Python environment
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.11"
|
||||||
|
- name: flake8 Lint
|
||||||
|
uses: py-actions/flake8@v2
|
||||||
|
with:
|
||||||
|
plugins: "flake8-no-print"
|
||||||
175
.github/workflows/server.yml
vendored
Normal file
175
.github/workflows/server.yml
vendored
Normal file
|
|
@ -0,0 +1,175 @@
|
||||||
|
# Server build and tests
|
||||||
|
name: Server
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch: # allows manual triggering
|
||||||
|
inputs:
|
||||||
|
sha:
|
||||||
|
description: 'Commit SHA1 to build'
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
slow_tests:
|
||||||
|
description: 'Run slow tests'
|
||||||
|
required: true
|
||||||
|
type: boolean
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
paths: ['.github/workflows/server.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'examples/server/**.*']
|
||||||
|
pull_request_target:
|
||||||
|
types: [opened, synchronize, reopened]
|
||||||
|
paths: ['.github/workflows/server.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'examples/server/**.*']
|
||||||
|
schedule:
|
||||||
|
- cron: '2 4 * * *'
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
server:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
# TODO: temporary disabled due to linux kernel issues
|
||||||
|
#sanitizer: [ADDRESS, THREAD, UNDEFINED]
|
||||||
|
sanitizer: [UNDEFINED]
|
||||||
|
build_type: [Debug]
|
||||||
|
include:
|
||||||
|
- build_type: Release
|
||||||
|
sanitizer: ""
|
||||||
|
fail-fast: false # While -DLLAMA_SANITIZE_THREAD=ON is broken
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Dependencies
|
||||||
|
id: depends
|
||||||
|
run: |
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get -y install \
|
||||||
|
build-essential \
|
||||||
|
xxd \
|
||||||
|
git \
|
||||||
|
cmake \
|
||||||
|
curl \
|
||||||
|
wget \
|
||||||
|
language-pack-en \
|
||||||
|
libcurl4-openssl-dev
|
||||||
|
|
||||||
|
- name: Clone
|
||||||
|
id: checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||||
|
|
||||||
|
- name: Python setup
|
||||||
|
id: setup_python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.11'
|
||||||
|
|
||||||
|
- name: Tests dependencies
|
||||||
|
id: test_dependencies
|
||||||
|
run: |
|
||||||
|
pip install -r examples/server/tests/requirements.txt
|
||||||
|
|
||||||
|
- name: Verify server deps
|
||||||
|
id: verify_server_deps
|
||||||
|
run: |
|
||||||
|
git config --global --add safe.directory $(realpath .)
|
||||||
|
cd examples/server
|
||||||
|
git ls-files --others --modified
|
||||||
|
git status
|
||||||
|
./deps.sh
|
||||||
|
git status
|
||||||
|
not_ignored_files="$(git ls-files --others --modified)"
|
||||||
|
echo "Modified files: ${not_ignored_files}"
|
||||||
|
if [ -n "${not_ignored_files}" ]; then
|
||||||
|
echo "Repository is dirty or server deps are not built as expected"
|
||||||
|
echo "${not_ignored_files}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Build
|
||||||
|
id: cmake_build
|
||||||
|
run: |
|
||||||
|
cmake -B build \
|
||||||
|
-DLLAMA_NATIVE=OFF \
|
||||||
|
-DLLAMA_BUILD_SERVER=ON \
|
||||||
|
-DLLAMA_CURL=ON \
|
||||||
|
-DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
|
||||||
|
-DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON ;
|
||||||
|
cmake --build build --config ${{ matrix.build_type }} -j $(nproc) --target server
|
||||||
|
|
||||||
|
|
||||||
|
- name: Tests
|
||||||
|
id: server_integration_tests
|
||||||
|
if: ${{ !matrix.disabled_on_pr || !github.event.pull_request }}
|
||||||
|
run: |
|
||||||
|
cd examples/server/tests
|
||||||
|
PORT=8888 ./tests.sh
|
||||||
|
|
||||||
|
- name: Slow tests
|
||||||
|
id: server_integration_tests_slow
|
||||||
|
if: ${{ (github.event.schedule || github.event.inputs.slow_tests == 'true') && matrix.build_type == 'Release' }}
|
||||||
|
run: |
|
||||||
|
cd examples/server/tests
|
||||||
|
PORT=8888 ./tests.sh --stop --no-skipped --no-capture --tags slow
|
||||||
|
|
||||||
|
|
||||||
|
server-windows:
|
||||||
|
runs-on: windows-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Clone
|
||||||
|
id: checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||||
|
|
||||||
|
- name: libCURL
|
||||||
|
id: get_libcurl
|
||||||
|
env:
|
||||||
|
CURL_VERSION: 8.6.0_6
|
||||||
|
run: |
|
||||||
|
curl.exe -o $env:RUNNER_TEMP/curl.zip -L "https://curl.se/windows/dl-${env:CURL_VERSION}/curl-${env:CURL_VERSION}-win64-mingw.zip"
|
||||||
|
mkdir $env:RUNNER_TEMP/libcurl
|
||||||
|
tar.exe -xvf $env:RUNNER_TEMP/curl.zip --strip-components=1 -C $env:RUNNER_TEMP/libcurl
|
||||||
|
|
||||||
|
- name: Build
|
||||||
|
id: cmake_build
|
||||||
|
run: |
|
||||||
|
cmake -B build -DLLAMA_CURL=ON -DCURL_LIBRARY="$env:RUNNER_TEMP/libcurl/lib/libcurl.dll.a" -DCURL_INCLUDE_DIR="$env:RUNNER_TEMP/libcurl/include"
|
||||||
|
cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS} --target server
|
||||||
|
|
||||||
|
- name: Python setup
|
||||||
|
id: setup_python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.11'
|
||||||
|
|
||||||
|
- name: Tests dependencies
|
||||||
|
id: test_dependencies
|
||||||
|
run: |
|
||||||
|
pip install -r examples/server/tests/requirements.txt
|
||||||
|
|
||||||
|
- name: Copy Libcurl
|
||||||
|
id: prepare_libcurl
|
||||||
|
run: |
|
||||||
|
cp $env:RUNNER_TEMP/libcurl/bin/libcurl-x64.dll ./build/bin/Release/libcurl-x64.dll
|
||||||
|
|
||||||
|
- name: Tests
|
||||||
|
id: server_integration_tests
|
||||||
|
if: ${{ !matrix.disabled_on_pr || !github.event.pull_request }}
|
||||||
|
run: |
|
||||||
|
cd examples/server/tests
|
||||||
|
behave.exe --summary --stop --no-capture --exclude 'issues|wrong_usages|passkey' --tags llama.cpp
|
||||||
|
|
||||||
|
- name: Slow tests
|
||||||
|
id: server_integration_tests_slow
|
||||||
|
if: ${{ (github.event.schedule || github.event.inputs.slow_tests == 'true') && matrix.build_type == 'Release' }}
|
||||||
|
run: |
|
||||||
|
cd examples/server/tests
|
||||||
|
behave.exe --stop --no-skipped --no-capture --tags slow
|
||||||
29
.github/workflows/zig-build.yml
vendored
Normal file
29
.github/workflows/zig-build.yml
vendored
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
name: Zig CI
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
runs-on: [ubuntu-latest, macos-latest, windows-latest]
|
||||||
|
runs-on: ${{ matrix.runs-on }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
submodules: recursive
|
||||||
|
fetch-depth: 0
|
||||||
|
- uses: goto-bus-stop/setup-zig@v2
|
||||||
|
with:
|
||||||
|
version: 0.11.0
|
||||||
|
- name: Build Summary
|
||||||
|
run: zig build --summary all -freference-trace
|
||||||
126
.gitignore
vendored
126
.gitignore
vendored
|
|
@ -1,40 +1,126 @@
|
||||||
*.o
|
*.o
|
||||||
*.a
|
*.a
|
||||||
|
*.so
|
||||||
|
*.gguf
|
||||||
|
*.gguf.json
|
||||||
|
*.bin
|
||||||
|
*.exe
|
||||||
|
*.dll
|
||||||
|
*.log
|
||||||
|
*.gcov
|
||||||
|
*.gcno
|
||||||
|
*.gcda
|
||||||
|
*.dot
|
||||||
|
*.bat
|
||||||
|
*.tmp
|
||||||
|
*.metallib
|
||||||
|
*.etag
|
||||||
|
*.lastModified
|
||||||
|
.DS_Store
|
||||||
|
.build/
|
||||||
.cache/
|
.cache/
|
||||||
|
.ccls-cache/
|
||||||
|
.direnv/
|
||||||
|
.envrc
|
||||||
|
.swiftpm
|
||||||
|
.venv
|
||||||
|
.clang-tidy
|
||||||
.vs/
|
.vs/
|
||||||
.vscode/
|
.vscode/
|
||||||
.DS_Store
|
.idea/
|
||||||
|
|
||||||
.build/
|
ggml-metal-embed.metal
|
||||||
build/
|
|
||||||
build-em/
|
lcov-report/
|
||||||
build-debug/
|
gcovr-report/
|
||||||
build-release/
|
|
||||||
build-static/
|
build*
|
||||||
build-no-accel/
|
!build.zig
|
||||||
build-sanitize-addr/
|
cmake-build-*
|
||||||
build-sanitize-thread/
|
out/
|
||||||
|
tmp/
|
||||||
|
|
||||||
models/*
|
models/*
|
||||||
|
models-mnt
|
||||||
|
|
||||||
|
/Pipfile
|
||||||
|
/baby-llama
|
||||||
|
/beam-search
|
||||||
|
/benchmark-matmult
|
||||||
|
/convert-llama2c-to-ggml
|
||||||
|
/embd-input-test
|
||||||
|
/embedding
|
||||||
|
/eval-callback
|
||||||
|
/gguf
|
||||||
|
/gguf-llama-simple
|
||||||
|
/gguf-split
|
||||||
|
/gritlm
|
||||||
|
/imatrix
|
||||||
|
/infill
|
||||||
|
/libllama.so
|
||||||
|
/llama-bench
|
||||||
|
/llava-cli
|
||||||
|
/lookahead
|
||||||
|
/lookup
|
||||||
|
/lookup-create
|
||||||
|
/lookup-merge
|
||||||
|
/lookup-stats
|
||||||
/main
|
/main
|
||||||
|
/metal
|
||||||
|
/passkey
|
||||||
|
/perplexity
|
||||||
|
/q8dot
|
||||||
/quantize
|
/quantize
|
||||||
/quantize-stats
|
/quantize-stats
|
||||||
/result
|
/result
|
||||||
/perplexity
|
/save-load-state
|
||||||
/embedding
|
/server
|
||||||
/benchmark-q4_0-matmult
|
/simple
|
||||||
/Pipfile
|
/batched
|
||||||
|
/batched-bench
|
||||||
|
/export-lora
|
||||||
|
/finetune
|
||||||
|
/retrieval
|
||||||
|
/speculative
|
||||||
|
/parallel
|
||||||
|
/train-text-from-scratch
|
||||||
|
/tokenize
|
||||||
|
/vdot
|
||||||
|
/common/build-info.cpp
|
||||||
arm_neon.h
|
arm_neon.h
|
||||||
compile_commands.json
|
compile_commands.json
|
||||||
|
CMakeSettings.json
|
||||||
|
|
||||||
.envrc
|
|
||||||
.direnv/
|
|
||||||
|
|
||||||
.venv
|
|
||||||
__pycache__
|
__pycache__
|
||||||
.swiftpm
|
dist
|
||||||
|
|
||||||
zig-out/
|
zig-out/
|
||||||
zig-cache/
|
zig-cache/
|
||||||
|
|
||||||
|
ppl-*.txt
|
||||||
|
qnt-*.txt
|
||||||
|
perf-*.txt
|
||||||
|
|
||||||
|
examples/jeopardy/results.txt
|
||||||
|
examples/server/*.html.hpp
|
||||||
|
examples/server/*.js.hpp
|
||||||
|
examples/server/*.mjs.hpp
|
||||||
|
|
||||||
|
poetry.lock
|
||||||
|
poetry.toml
|
||||||
|
nppBackup
|
||||||
|
|
||||||
|
# Test binaries
|
||||||
|
/tests/test-grammar-parser
|
||||||
|
/tests/test-llama-grammar
|
||||||
|
/tests/test-double-float
|
||||||
|
/tests/test-grad0
|
||||||
|
/tests/test-opt
|
||||||
|
/tests/test-quantize-fns
|
||||||
|
/tests/test-quantize-perf
|
||||||
|
/tests/test-sampling
|
||||||
|
/tests/test-tokenizer-0
|
||||||
|
/tests/test-tokenizer-1-spm
|
||||||
|
/tests/test-tokenizer-1-bpe
|
||||||
|
/tests/test-rope
|
||||||
|
/tests/test-backend-ops
|
||||||
|
|
|
||||||
3
.gitmodules
vendored
Normal file
3
.gitmodules
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
[submodule "kompute"]
|
||||||
|
path = kompute
|
||||||
|
url = https://github.com/nomic-ai/kompute.git
|
||||||
16
.pre-commit-config.yaml
Normal file
16
.pre-commit-config.yaml
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
# See https://pre-commit.com for more information
|
||||||
|
# See https://pre-commit.com/hooks.html for more hooks
|
||||||
|
exclude: prompts/.*.txt
|
||||||
|
repos:
|
||||||
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||||
|
rev: v4.6.0
|
||||||
|
hooks:
|
||||||
|
- id: trailing-whitespace
|
||||||
|
- id: end-of-file-fixer
|
||||||
|
- id: check-yaml
|
||||||
|
- id: check-added-large-files
|
||||||
|
- repo: https://github.com/PyCQA/flake8
|
||||||
|
rev: 7.0.0
|
||||||
|
hooks:
|
||||||
|
- id: flake8
|
||||||
|
additional_dependencies: [flake8-no-print]
|
||||||
655
AUTHORS
Normal file
655
AUTHORS
Normal file
|
|
@ -0,0 +1,655 @@
|
||||||
|
# date: Tue Apr 9 09:17:14 EEST 2024
|
||||||
|
# this file is auto-generated by scripts/gen-authors.sh
|
||||||
|
|
||||||
|
0cc4m <picard12@live.de>
|
||||||
|
0xspringtime <110655352+0xspringtime@users.noreply.github.com>
|
||||||
|
2f38b454 <dxf@protonmail.com>
|
||||||
|
3ooabkhxtn <31479382+3ooabkhxtn@users.noreply.github.com>
|
||||||
|
44670 <44670@users.noreply.github.com>
|
||||||
|
AN Long <aisk@users.noreply.github.com>
|
||||||
|
AT <manyoso@users.noreply.github.com>
|
||||||
|
Aarni Koskela <akx@iki.fi>
|
||||||
|
Aaron Miller <apage43@ninjawhale.com>
|
||||||
|
Aaryaman Vasishta <aaryaman.vasishta@amd.com>
|
||||||
|
Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com>
|
||||||
|
Abhishek Gopinath K <31348521+overtunned@users.noreply.github.com>
|
||||||
|
Adithya Balaji <adithya.b94@gmail.com>
|
||||||
|
AdithyanI <adithyan.i4internet@gmail.com>
|
||||||
|
Adrian <smith.adriane@gmail.com>
|
||||||
|
Adrian Hesketh <a-h@users.noreply.github.com>
|
||||||
|
AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com>
|
||||||
|
Aisuko <urakiny@gmail.com>
|
||||||
|
Alberto <57916483+albbus-stack@users.noreply.github.com>
|
||||||
|
Alex <awhill19@icloud.com>
|
||||||
|
Alex Azarov <alex@azarov.by>
|
||||||
|
Alex Azarov <alexander.azarov@mapbox.com>
|
||||||
|
Alex Klinkhamer <from.github.com.917@grencez.dev>
|
||||||
|
Alex Klinkhamer <git@grencez.dev>
|
||||||
|
Alex Nguyen <tiendung@users.noreply.github.com>
|
||||||
|
Alex Petenchea <alex.petenchea@gmail.com>
|
||||||
|
Alex Renda <alexrenda@users.noreply.github.com>
|
||||||
|
Alex von Gluck IV <kallisti5@unixzen.com>
|
||||||
|
Alexey Parfenov <zxed@alkatrazstudio.net>
|
||||||
|
Ali Chraghi <63465728+alichraghi@users.noreply.github.com>
|
||||||
|
Ali Nehzat <ali.nehzat@thanks.dev>
|
||||||
|
Ali Tariq <ali.tariq@10xengineers.ai>
|
||||||
|
Alon <alonfaraj@gmail.com>
|
||||||
|
AlpinDale <52078762+AlpinDale@users.noreply.github.com>
|
||||||
|
AmirAli Mirian <37371367+amiralimi@users.noreply.github.com>
|
||||||
|
Ananta Bastola <anantarajbastola@gmail.com>
|
||||||
|
Anas Ahouzi <112881240+aahouzi@users.noreply.github.com>
|
||||||
|
András Salamon <ott2@users.noreply.github.com>
|
||||||
|
Andrei <abetlen@gmail.com>
|
||||||
|
Andrew Canis <andrew.canis@gmail.com>
|
||||||
|
Andrew Duffy <a10y@users.noreply.github.com>
|
||||||
|
Andrew Godfrey <AndrewGodfrey@users.noreply.github.com>
|
||||||
|
Arik Poznanski <arikpoz@users.noreply.github.com>
|
||||||
|
Artem <guinmoon@gmail.com>
|
||||||
|
Artyom Lebedev <vagran.ast@gmail.com>
|
||||||
|
Asbjørn Olling <asbjornolling@gmail.com>
|
||||||
|
Ásgeir Bjarni Ingvarsson <asgeir@fundinn.org>
|
||||||
|
Ashok Gelal <401055+ashokgelal@users.noreply.github.com>
|
||||||
|
Ashraful Islam <ashraful.meche@gmail.com>
|
||||||
|
Atsushi Tatsuma <yoshoku@outlook.com>
|
||||||
|
Austin <77757836+teleprint-me@users.noreply.github.com>
|
||||||
|
AustinMroz <austinmroz@utexas.edu>
|
||||||
|
BADR <contact@pythops.com>
|
||||||
|
Bach Le <bach@bullno1.com>
|
||||||
|
Bailey Chittle <39804642+bachittle@users.noreply.github.com>
|
||||||
|
BarfingLemurs <128182951+BarfingLemurs@users.noreply.github.com>
|
||||||
|
Behnam M <58621210+ibehnam@users.noreply.github.com>
|
||||||
|
Ben Garney <bengarney@users.noreply.github.com>
|
||||||
|
Ben Siraphob <bensiraphob@gmail.com>
|
||||||
|
Ben Williams <ben@719ben.com>
|
||||||
|
Benjamin Lecaillon <84293038+blecaillon@users.noreply.github.com>
|
||||||
|
Bernat Vadell <hounter.caza@gmail.com>
|
||||||
|
Bodo Graumann <mail@bodograumann.de>
|
||||||
|
Bono Lv <lvscar@users.noreply.github.com>
|
||||||
|
Borislav Stanimirov <b.stanimirov@abv.bg>
|
||||||
|
Branden Butler <bwtbutler@hotmail.com>
|
||||||
|
Brian <mofosyne@gmail.com>
|
||||||
|
Bruce MacDonald <brucewmacdonald@gmail.com>
|
||||||
|
CJ Pais <cj@cjpais.com>
|
||||||
|
CRD716 <crd716@gmail.com>
|
||||||
|
Cameron <csteele@steelecameron.com>
|
||||||
|
Cameron Kaiser <classilla@users.noreply.github.com>
|
||||||
|
Casey Primozic <casey@cprimozic.net>
|
||||||
|
Casey Primozic <me@ameo.link>
|
||||||
|
CausalLM <148736309+CausalLM@users.noreply.github.com>
|
||||||
|
Cebtenzzre <cebtenzzre@gmail.com>
|
||||||
|
Chad Brewbaker <crb002@gmail.com>
|
||||||
|
Cheng Shao <terrorjack@type.dance>
|
||||||
|
Chris Kuehl <ckuehl@ckuehl.me>
|
||||||
|
Christian Demsar <christian@github.email.demsar.us>
|
||||||
|
Christian Demsar <crasm@git.vczf.us>
|
||||||
|
Christian Falch <875252+chrfalch@users.noreply.github.com>
|
||||||
|
Christian Kögler <ck3d@gmx.de>
|
||||||
|
Clark Saben <76020733+csaben@users.noreply.github.com>
|
||||||
|
Clint Herron <hanclinto@gmail.com>
|
||||||
|
Cuong Trinh Manh <nguoithichkhampha@gmail.com>
|
||||||
|
DAN™ <dranger003@gmail.com>
|
||||||
|
Damian Stewart <d@damianstewart.com>
|
||||||
|
Dane Madsen <dane_madsen@hotmail.com>
|
||||||
|
DaniAndTheWeb <57776841+DaniAndTheWeb@users.noreply.github.com>
|
||||||
|
Daniel Bevenius <daniel.bevenius@gmail.com>
|
||||||
|
Daniel Drake <drake@endlessos.org>
|
||||||
|
Daniel Hiltgen <dhiltgen@users.noreply.github.com>
|
||||||
|
Daniel Illescas Romero <illescas.daniel@protonmail.com>
|
||||||
|
DannyDaemonic <DannyDaemonic@gmail.com>
|
||||||
|
Dat Quoc Nguyen <2412555+datquocnguyen@users.noreply.github.com>
|
||||||
|
Dave Della Costa <ddellacosta+github@gmail.com>
|
||||||
|
David Friehs <david@friehs.info>
|
||||||
|
David Kennedy <dakennedyd@gmail.com>
|
||||||
|
David Pflug <david@pflug.email>
|
||||||
|
David Renshaw <dwrenshaw@gmail.com>
|
||||||
|
David Sommers <12738+databyte@users.noreply.github.com>
|
||||||
|
David Yang <davidyang6us@gmail.com>
|
||||||
|
Dawid Wysocki <62249621+TortillaZHawaii@users.noreply.github.com>
|
||||||
|
Dean <Dean.Sinaean@gmail.com>
|
||||||
|
Deins <deinsegle@gmail.com>
|
||||||
|
Didzis Gosko <didzis@users.noreply.github.com>
|
||||||
|
Don Mahurin <dmahurin@users.noreply.github.com>
|
||||||
|
DooWoong Lee (David) <manics99@naver.com>
|
||||||
|
Doomsdayrs <38189170+Doomsdayrs@users.noreply.github.com>
|
||||||
|
Douglas Hanley <thesecretaryofwar@gmail.com>
|
||||||
|
Dr. Tom Murphy VII Ph.D <499244+tom7@users.noreply.github.com>
|
||||||
|
Ebey Abraham <ebey97@gmail.com>
|
||||||
|
Ed Lee <edilee@mozilla.com>
|
||||||
|
Ed Lepedus <ed.lepedus@googlemail.com>
|
||||||
|
Edward Taylor <edeetee@gmail.com>
|
||||||
|
Elbios <141279586+Elbios@users.noreply.github.com>
|
||||||
|
Engininja2 <139037756+Engininja2@users.noreply.github.com>
|
||||||
|
Equim <sayaka@ekyu.moe>
|
||||||
|
Eric Sommerlade <es0m@users.noreply.github.com>
|
||||||
|
Eric Zhang <34133756+EZForever@users.noreply.github.com>
|
||||||
|
Erik Garrison <erik.garrison@gmail.com>
|
||||||
|
Erik Scholz <Green-Sky@users.noreply.github.com>
|
||||||
|
Ettore Di Giacinto <mudler@users.noreply.github.com>
|
||||||
|
Evan Jones <evan.q.jones@gmail.com>
|
||||||
|
Evan Miller <emmiller@gmail.com>
|
||||||
|
Eve <139727413+netrunnereve@users.noreply.github.com>
|
||||||
|
Evgeny Kurnevsky <kurnevsky@gmail.com>
|
||||||
|
Ewout ter Hoeven <E.M.terHoeven@student.tudelft.nl>
|
||||||
|
ExtReMLapin <3909752+ExtReMLapin@users.noreply.github.com>
|
||||||
|
FK <sozforex@gmail.com>
|
||||||
|
Fabian <cmdrf@users.noreply.github.com>
|
||||||
|
Fabio R. Sluzala <Fabio3rs@users.noreply.github.com>
|
||||||
|
Faez Shakil <faez.shakil@gmail.com>
|
||||||
|
FantasyGmm <16450052+FantasyGmm@users.noreply.github.com>
|
||||||
|
Fattire <528174+fat-tire@users.noreply.github.com>
|
||||||
|
Felix <stenbackfelix@gmail.com>
|
||||||
|
Finn Voorhees <finnvoorhees@gmail.com>
|
||||||
|
Firat <firatkiral@gmail.com>
|
||||||
|
Folko-Ven <71110216+Folko-Ven@users.noreply.github.com>
|
||||||
|
Foul-Tarnished <107711110+Foul-Tarnished@users.noreply.github.com>
|
||||||
|
Francisco Melo <43780565+francis2tm@users.noreply.github.com>
|
||||||
|
FrankHB <frankhb1989@gmail.com>
|
||||||
|
Frederik Vogel <Schaltfehler@users.noreply.github.com>
|
||||||
|
Gabe Goodhart <gabe.l.hart@gmail.com>
|
||||||
|
GainLee <perfecter.gen@gmail.com>
|
||||||
|
Galunid <karolek1231456@gmail.com>
|
||||||
|
Gary Linscott <glinscott@gmail.com>
|
||||||
|
Gary Mulder <gjmulder@gmail.com>
|
||||||
|
Genkagaku.GPT <hlhr202@163.com>
|
||||||
|
Georgi Gerganov <ggerganov@gmail.com>
|
||||||
|
Gilad S <giladgd@users.noreply.github.com>
|
||||||
|
GiviMAD <GiviMAD@users.noreply.github.com>
|
||||||
|
Govlzkoy <gotope@users.noreply.github.com>
|
||||||
|
Guillaume "Vermeille" Sanchez <Guillaume.V.Sanchez@gmail.com>
|
||||||
|
Guillaume Wenzek <gwenzek@users.noreply.github.com>
|
||||||
|
Guoteng <32697156+SolenoidWGT@users.noreply.github.com>
|
||||||
|
Gustavo Rocha Dias <91472747+gustrd@users.noreply.github.com>
|
||||||
|
Halalaluyafail3 <55773281+Halalaluyafail3@users.noreply.github.com>
|
||||||
|
Haohui Mai <ricetons@gmail.com>
|
||||||
|
Haoxiang Fei <tonyfettes@tonyfettes.com>
|
||||||
|
Harald Fernengel <harald.fernengel@here.com>
|
||||||
|
Hatsune Miku <129688334+at8u@users.noreply.github.com>
|
||||||
|
Henk Poley <HenkPoley@gmail.com>
|
||||||
|
Henri Vasserman <henv@hot.ee>
|
||||||
|
Henrik Forstén <henrik.forsten@gmail.com>
|
||||||
|
Herman Semenov <GermanAizek@yandex.ru>
|
||||||
|
Hesen Peng <hesen.peng@gmail.com>
|
||||||
|
Hoang Nguyen <hugo53@users.noreply.github.com>
|
||||||
|
Hongyu Ouyang <96765450+casavaca@users.noreply.github.com>
|
||||||
|
Howard Su <howard0su@gmail.com>
|
||||||
|
Hua Jiang <allenhjiang@outlook.com>
|
||||||
|
Huawei Lin <huaweilin.cs@gmail.com>
|
||||||
|
Ian Bull <irbull@eclipsesource.com>
|
||||||
|
Ian Bull <irbull@gmail.com>
|
||||||
|
Ian Scrivener <github@zilogy.asia>
|
||||||
|
Ido S <ido.pluto@gmail.com>
|
||||||
|
IgnacioFDM <ignaciofdm@gmail.com>
|
||||||
|
Igor Okulist <okigan@gmail.com>
|
||||||
|
Ikko Eltociear Ashimine <eltociear@gmail.com>
|
||||||
|
Ilya Kurdyukov <59548320+ilyakurdyukov@users.noreply.github.com>
|
||||||
|
Ionoclast Laboratories <brigham@ionoclast.com>
|
||||||
|
Isaac McFadyen <isaac@imcf.me>
|
||||||
|
IsaacDynamo <61521674+IsaacDynamo@users.noreply.github.com>
|
||||||
|
Ivan Komarov <Ivan.Komarov@dfyz.info>
|
||||||
|
Ivan Stepanov <ivanstepanovftw@gmail.com>
|
||||||
|
JH23X <165871467+JH23X@users.noreply.github.com>
|
||||||
|
Jack Mousseau <jmousseau@users.noreply.github.com>
|
||||||
|
JackJollimore <130917767+JackJollimore@users.noreply.github.com>
|
||||||
|
Jag Chadha <jagtesh@gmail.com>
|
||||||
|
Jakub N <jakubniemczyk97@gmail.com>
|
||||||
|
James Reynolds <magnusviri@users.noreply.github.com>
|
||||||
|
Jan Boon <jan.boon@kaetemi.be>
|
||||||
|
Jan Boon <kaetemi@gmail.com>
|
||||||
|
Jan Ploski <jpl@plosquare.com>
|
||||||
|
Jannis Schönleber <joennlae@gmail.com>
|
||||||
|
Jared Van Bortel <cebtenzzre@gmail.com>
|
||||||
|
Jared Van Bortel <jared@nomic.ai>
|
||||||
|
Jason McCartney <jmac@theroot.org>
|
||||||
|
Jean-Christophe Hoelt <hoelt@fovea.cc>
|
||||||
|
Jean-Michaël Celerier <jeanmichael.celerier+github@gmail.com>
|
||||||
|
Jed Fox <git@jedfox.com>
|
||||||
|
Jeffrey Quesnelle <emozilla@nousresearch.com>
|
||||||
|
Jesse Jojo Johnson <williamsaintgeorge@gmail.com>
|
||||||
|
Jhen-Jie Hong <iainst0409@gmail.com>
|
||||||
|
Jiahao Li <liplus17@163.com>
|
||||||
|
Jian Liao <jianliao@users.noreply.github.com>
|
||||||
|
JidongZhang-THU <1119708529@qq.com>
|
||||||
|
Jinwoo Jeong <33892306+williamjeong2@users.noreply.github.com>
|
||||||
|
Jiří Podivín <66251151+jpodivin@users.noreply.github.com>
|
||||||
|
Johannes Gäßler <johannesg@5d6.de>
|
||||||
|
Johannes Rudolph <johannes.rudolph@gmail.com>
|
||||||
|
John <78893154+cmp-nct@users.noreply.github.com>
|
||||||
|
John Balis <phobossystems@gmail.com>
|
||||||
|
John Smith <67539080+kingsidelee@users.noreply.github.com>
|
||||||
|
JohnnyB <jboero@users.noreply.github.com>
|
||||||
|
Jonas Wunderlich <32615971+jonas-w@users.noreply.github.com>
|
||||||
|
Jorge A <161275481+jorgealias@users.noreply.github.com>
|
||||||
|
Jose Maldonado <63384398+yukiteruamano@users.noreply.github.com>
|
||||||
|
Joseph Stahl <1269177+josephst@users.noreply.github.com>
|
||||||
|
Joyce <joycebrum@google.com>
|
||||||
|
Juan Calderon-Perez <835733+gaby@users.noreply.github.com>
|
||||||
|
Judd <foldl@users.noreply.github.com>
|
||||||
|
Julius Arkenberg <arki05@users.noreply.github.com>
|
||||||
|
Jun Jie <71215065+junnjiee16@users.noreply.github.com>
|
||||||
|
Juraj Bednar <juraj@bednar.io>
|
||||||
|
Justin Parker <jparkerweb@gmail.com>
|
||||||
|
Justin Suess <justin.suess@westpoint.edu>
|
||||||
|
Justine Tunney <jtunney@gmail.com>
|
||||||
|
Juuso Alasuutari <juuso.alasuutari@gmail.com>
|
||||||
|
KASR <karim.asrih@gmail.com>
|
||||||
|
Kamil Tomšík <info@tomsik.cz>
|
||||||
|
Karsten Weiss <knweiss@gmail.com>
|
||||||
|
Karthick <j.karthic2004@gmail.com>
|
||||||
|
Karthik Kumar Viswanathan <195178+guilt@users.noreply.github.com>
|
||||||
|
Karthik Sethuraman <k.seth1993@gmail.com>
|
||||||
|
Kasumi <90275229+kasumi-1@users.noreply.github.com>
|
||||||
|
Kawrakow <48489457+ikawrakow@users.noreply.github.com>
|
||||||
|
Keiichi Tabata <keiichi.tabata@outlook.com>
|
||||||
|
Kenvix ⭐ <kenvixzure@live.com>
|
||||||
|
Kerfuffle <44031344+KerfuffleV2@users.noreply.github.com>
|
||||||
|
Kevin Ji <1146876+kevinji@users.noreply.github.com>
|
||||||
|
Kevin Kwok <antimatter15@gmail.com>
|
||||||
|
Kevin Lo <kevlo@kevlo.org>
|
||||||
|
Kolen Cheung <ickc@users.noreply.github.com>
|
||||||
|
Konstantin Herud <konstantin.herud@denkbares.com>
|
||||||
|
Konstantin Zhuravlyov <konstantin.zhuravlyov@amd.com>
|
||||||
|
Kunshang Ji <kunshang.ji@intel.com>
|
||||||
|
Kyle Liang <liangmanlai@gmail.com>
|
||||||
|
Kyle Mistele <kyle@mistele.com>
|
||||||
|
Kylin <56434533+KyL0N@users.noreply.github.com>
|
||||||
|
Lars Grammel <lars.grammel@gmail.com>
|
||||||
|
Laura <Tijntje_7@msn.com>
|
||||||
|
Lee <44310445+lx200916@users.noreply.github.com>
|
||||||
|
Lee Drake <b.lee.drake@gmail.com>
|
||||||
|
Leng Yue <lengyue@lengyue.me>
|
||||||
|
LeonEricsson <70749762+LeonEricsson@users.noreply.github.com>
|
||||||
|
Leonardo Neumann <leonardo@neumann.dev.br>
|
||||||
|
Li Tan <tanliboy@gmail.com>
|
||||||
|
Linwei Wang <wanix1988@gmail.com>
|
||||||
|
LoganDark <github@logandark.mozmail.com>
|
||||||
|
LostRuins <39025047+LostRuins@users.noreply.github.com>
|
||||||
|
Luciano <lucianostrika44@gmail.com>
|
||||||
|
Luo Tian <lt@basecity.com>
|
||||||
|
M. Yusuf Sarıgöz <yusufsarigoz@gmail.com>
|
||||||
|
Maarten ter Huurne <maarten@treewalker.org>
|
||||||
|
Mack Straight <eiz@users.noreply.github.com>
|
||||||
|
Maël Kerbiriou <m431.kerbiriou@gmail.com>
|
||||||
|
MaggotHATE <clay1326@gmail.com>
|
||||||
|
Marc Köhlbrugge <subscriptions@marckohlbrugge.com>
|
||||||
|
Marco Matthies <71844+marcom@users.noreply.github.com>
|
||||||
|
Marcus Dunn <51931484+MarcusDunn@users.noreply.github.com>
|
||||||
|
Marian Cepok <marian.cepok@gmail.com>
|
||||||
|
Mark Fairbairn <thebaron88@gmail.com>
|
||||||
|
Marko Tasic <mtasic85@gmail.com>
|
||||||
|
Martin Krasser <krasserm@googlemail.com>
|
||||||
|
Martin Schwaighofer <mschwaig@users.noreply.github.com>
|
||||||
|
Marvin Gießing <marvin.giessing@gmail.com>
|
||||||
|
Mateusz Charytoniuk <mateusz.charytoniuk@protonmail.com>
|
||||||
|
Matheus C. França <matheus-catarino@hotmail.com>
|
||||||
|
Matheus Gabriel Alves Silva <matheusgasource@gmail.com>
|
||||||
|
Mathieu Nayrolles <MathieuNls@users.noreply.github.com>
|
||||||
|
Mathijs de Bruin <mathijs@mathijsfietst.nl>
|
||||||
|
Matt Clayton <156335168+mattjcly@users.noreply.github.com>
|
||||||
|
Matt Pulver <matt.pulver@heavy.ai>
|
||||||
|
Matteo Boschini <12133566+mbosc@users.noreply.github.com>
|
||||||
|
Matthew Tejo <matthew.tejo@gmail.com>
|
||||||
|
Matvey Soloviev <blackhole89@gmail.com>
|
||||||
|
Maxime <672982+maximegmd@users.noreply.github.com>
|
||||||
|
Maximilian Winter <maximilian.winter.91@gmail.com>
|
||||||
|
Meng Zhang <meng@tabbyml.com>
|
||||||
|
Meng, Hengyu <hengyu.meng@intel.com>
|
||||||
|
Merrick Christensen <merrick.christensen@gmail.com>
|
||||||
|
Michael Coppola <m18coppola@gmail.com>
|
||||||
|
Michael Hueschen <m@mhueschen.dev>
|
||||||
|
Michael Kesper <mkesper@schokokeks.org>
|
||||||
|
Michael Klimenko <mklimenko29@gmail.com>
|
||||||
|
Michael Podvitskiy <podvitskiymichael@gmail.com>
|
||||||
|
Michael Potter <NanoTekGuy@Gmail.com>
|
||||||
|
Michaël de Vries <vriesdemichael@gmail.com>
|
||||||
|
Mihai <mihai.chirculescu@yahoo.com>
|
||||||
|
Mike <ytianhui2004@gmail.com>
|
||||||
|
Minsoo Cheong <54794500+mscheong01@users.noreply.github.com>
|
||||||
|
Mirko185 <mirkosig@gmail.com>
|
||||||
|
Mirror Azure <54669636+MirrorAzure@users.noreply.github.com>
|
||||||
|
Miwa / Ensan <63481257+ensan-hcl@users.noreply.github.com>
|
||||||
|
Mohammadreza Hendiani <hendiani.mohammadreza@gmail.com>
|
||||||
|
Murilo Santana <mvrilo@gmail.com>
|
||||||
|
Musab Gultekin <musabgultekin@users.noreply.github.com>
|
||||||
|
Nam D. Tran <42194884+namtranase@users.noreply.github.com>
|
||||||
|
NawafAlansari <72708095+NawafAlansari@users.noreply.github.com>
|
||||||
|
Nebula <infinitewormhole@gmail.com>
|
||||||
|
Neo Zhang Jianyu <jianyu.zhang@intel.com>
|
||||||
|
Neuman Vong <neuman.vong@gmail.com>
|
||||||
|
Nexesenex <124105151+Nexesenex@users.noreply.github.com>
|
||||||
|
Niall Coates <1349685+Niall-@users.noreply.github.com>
|
||||||
|
Nicolai Weitkemper <kontakt@nicolaiweitkemper.de>
|
||||||
|
Nigel Bosch <pnigelb@gmail.com>
|
||||||
|
Niklas Korz <niklas@niklaskorz.de>
|
||||||
|
Nindaleth <Nindaleth@users.noreply.github.com>
|
||||||
|
Oleksandr Nikitin <oleksandr@tvori.info>
|
||||||
|
Oleksii Maryshchenko <oleksii.maryshchenko@gmail.com>
|
||||||
|
Olivier Chafik <ochafik@users.noreply.github.com>
|
||||||
|
Ondřej Čertík <ondrej@certik.us>
|
||||||
|
Ouadie EL FAROUKI <ouadie.elfarouki@codeplay.com>
|
||||||
|
Paul Tsochantaris <ptsochantaris@icloud.com>
|
||||||
|
Pavol Rusnak <pavol@rusnak.io>
|
||||||
|
Pedro Cuenca <pedro@huggingface.co>
|
||||||
|
Peter Sugihara <peter@campsh.com>
|
||||||
|
Phil H <5756783+phiharri@users.noreply.github.com>
|
||||||
|
Philip Taron <philip.taron@gmail.com>
|
||||||
|
Phillip Kravtsov <phillip@kravtsov.net>
|
||||||
|
Pierre Alexandre SCHEMBRI <pa.schembri@gmail.com>
|
||||||
|
Pierrick Hymbert <pierrick.hymbert@gmail.com>
|
||||||
|
Przemysław Pawełczyk <przemoc@gmail.com>
|
||||||
|
Qin Yue Chen <71813199+chenqiny@users.noreply.github.com>
|
||||||
|
Qingyou Meng <meng.qingyou@gmail.com>
|
||||||
|
Qu Zongfu <43257352+yancaoweidaode@users.noreply.github.com>
|
||||||
|
RJ Adriaansen <adriaansen@eshcc.eur.nl>
|
||||||
|
Radoslav Gerganov <rgerganov@gmail.com>
|
||||||
|
Radosław Gryta <radek.gryta@gmail.com>
|
||||||
|
Rahul Vivek Nair <68507071+RahulVivekNair@users.noreply.github.com>
|
||||||
|
Rand Xie <randxiexyy29@gmail.com>
|
||||||
|
Randall Fitzgerald <randall@dasaku.net>
|
||||||
|
Reinforce-II <fate@eastal.com>
|
||||||
|
Riceball LEE <snowyu.lee@gmail.com>
|
||||||
|
Richard Kiss <him@richardkiss.com>
|
||||||
|
Richard Roberson <richardr1126@gmail.com>
|
||||||
|
Rick G <26732651+TheFlipbook@users.noreply.github.com>
|
||||||
|
Rickard Edén <rickardeden@gmail.com>
|
||||||
|
Rickard Hallerbäck <rickard.hallerback@gmail.com>
|
||||||
|
Rickey Bowers Jr <bitRAKE@gmail.com>
|
||||||
|
Riley Stewart <ristew@users.noreply.github.com>
|
||||||
|
Rinne <AsakusaRinne@gmail.com>
|
||||||
|
Rinne <liu_yaohui1998@126.com>
|
||||||
|
Robert Brisita <986796+rbrisita@users.noreply.github.com>
|
||||||
|
Robert Sung-wook Shin <edp1096@users.noreply.github.com>
|
||||||
|
Robey Holderith <robey@flaminglunchbox.net>
|
||||||
|
Robyn <robyngraf@users.noreply.github.com>
|
||||||
|
Roger Meier <r.meier@siemens.com>
|
||||||
|
Roland <14355895+rbur0425@users.noreply.github.com>
|
||||||
|
Romain D <90720+Artefact2@users.noreply.github.com>
|
||||||
|
Romain Neutron <romain@neutron.io>
|
||||||
|
Roman Parykin <donderom@gmail.com>
|
||||||
|
Ron Evans <ron@hybridgroup.com>
|
||||||
|
Ron Jailall <rojailal@gmail.com>
|
||||||
|
Ronny Brendel <ronnybrendel@gmail.com>
|
||||||
|
Ronsor <ronsor@ronsor.pw>
|
||||||
|
Rowan Hart <rowanbhart@gmail.com>
|
||||||
|
Rune <43761327+Rune-AI@users.noreply.github.com>
|
||||||
|
Ryan Landay <rlanday@gmail.com>
|
||||||
|
Ryder Wishart <ryderwishart@gmail.com>
|
||||||
|
Rőczey Barnabás <31726601+An0nie@users.noreply.github.com>
|
||||||
|
SakuraUmi <yukinon244@gmail.com>
|
||||||
|
Salvador E. Tropea <stropea@inti.gob.ar>
|
||||||
|
Sam Spilsbury <smspillaz@gmail.com>
|
||||||
|
Sami Farin <3876865+Safari77@users.noreply.github.com>
|
||||||
|
Samuel Maynard <samwmaynard@gmail.com>
|
||||||
|
Sang-Kil Park <sang.park@42dot.ai>
|
||||||
|
Seb C <47074056+Sebby37@users.noreply.github.com>
|
||||||
|
Sebastián A <sebastian.aedo29@gmail.com>
|
||||||
|
SebastianApel <13675545+SebastianApel@users.noreply.github.com>
|
||||||
|
Senemu <10880819+Senemu@users.noreply.github.com>
|
||||||
|
Sergey Alirzaev <zl29ah@gmail.com>
|
||||||
|
Sergio López <slp@sinrega.org>
|
||||||
|
SeungWon Jeong <65549245+redlion0929@users.noreply.github.com>
|
||||||
|
ShadovvBeast <ShadovvBeast@gmail.com>
|
||||||
|
Shakhar Dasgupta <shakhardasgupta@gmail.com>
|
||||||
|
Shangning Xu <32517059+xushangning@users.noreply.github.com>
|
||||||
|
Shijie <821898965@qq.com>
|
||||||
|
Shintarou Okada <kokuzen@gmail.com>
|
||||||
|
Shouzheng Liu <61452103+lshzh-ww@users.noreply.github.com>
|
||||||
|
Shouzheng Liu <lshzh.hi@gmail.com>
|
||||||
|
Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
|
||||||
|
Simon Willison <swillison@gmail.com>
|
||||||
|
Siwen Yu <yusiwen@gmail.com>
|
||||||
|
Sky Yan <skyan83@gmail.com>
|
||||||
|
Slaren <2141330+slaren@users.noreply.github.com>
|
||||||
|
Slava Primenko <primenko.s@gmail.com>
|
||||||
|
SoftwareRenderer <138734813+SoftwareRenderer@users.noreply.github.com>
|
||||||
|
Someone <sergei.kozlukov@aalto.fi>
|
||||||
|
Someone Serge <sergei.kozlukov@aalto.fi>
|
||||||
|
Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com>
|
||||||
|
Spencer Sutton <spencersutton@users.noreply.github.com>
|
||||||
|
Srinivas Billa <nivibilla@gmail.com>
|
||||||
|
Stefan Sydow <stefan@sydow.email>
|
||||||
|
Stephan Walter <stephan@walter.name>
|
||||||
|
Stephen Nichols <snichols@users.noreply.github.com>
|
||||||
|
Steve Grubb <ausearch.1@gmail.com>
|
||||||
|
Steven Roussey <sroussey@gmail.com>
|
||||||
|
Steward Garcia <57494570+FSSRepo@users.noreply.github.com>
|
||||||
|
Suaj Carrot <72162667+SuajCarrot@users.noreply.github.com>
|
||||||
|
SuperUserNameMan <yoann@terminajones.com>
|
||||||
|
Tai Duc Nguyen <taiducnguyen.drexel@gmail.com>
|
||||||
|
Taikono-Himazin <kazu@po.harenet.ne.jp>
|
||||||
|
Tameem <113388789+AhmadTameem@users.noreply.github.com>
|
||||||
|
Tamotsu Takahashi <ttakah+github@gmail.com>
|
||||||
|
Thái Hoàng Tâm <75922889+RoyalHeart@users.noreply.github.com>
|
||||||
|
Thatcher Chamberlin <j.thatcher.c@gmail.com>
|
||||||
|
Theia Vogel <theia@vgel.me>
|
||||||
|
Thérence <13496987+Royalphax@users.noreply.github.com>
|
||||||
|
Thibault Terrasson <thibault.terrasson@gmail.com>
|
||||||
|
Thomas Klausner <wiz@gatalith.at>
|
||||||
|
Tim Miller <drasticactions@users.noreply.github.com>
|
||||||
|
Timmy Knight <r2d2fish@gmail.com>
|
||||||
|
Timothy Cronin <40186632+4imothy@users.noreply.github.com>
|
||||||
|
Ting Lou <ting.lou@gmail.com>
|
||||||
|
Ting Sun <suntcrick@gmail.com>
|
||||||
|
Tobias Lütke <tobi@shopify.com>
|
||||||
|
Tom C <tom.corelis@gmail.com>
|
||||||
|
Tom Jobbins <784313+TheBloke@users.noreply.github.com>
|
||||||
|
Tomas <tom.tomas.36478119@gmail.com>
|
||||||
|
Tomáš Pazdiora <tomas.pazdiora@gmail.com>
|
||||||
|
Tristan Ross <rosscomputerguy@protonmail.com>
|
||||||
|
Tungsten842 <886724vf@anonaddy.me>
|
||||||
|
Tungsten842 <quantmint@protonmail.com>
|
||||||
|
Tushar <ditsuke@protonmail.com>
|
||||||
|
UEXTM.com <84163508+uextm@users.noreply.github.com>
|
||||||
|
Uzo Nweke <uzoechi@gmail.com>
|
||||||
|
Vaibhav Srivastav <vaibhavs10@gmail.com>
|
||||||
|
Val Kharitonov <mail@kharvd.com>
|
||||||
|
Valentin Konovalov <valle.ketsujin@gmail.com>
|
||||||
|
Valentyn Bezshapkin <61702053+valentynbez@users.noreply.github.com>
|
||||||
|
Victor Z. Peng <ziliangdotme@gmail.com>
|
||||||
|
Vlad <spitfireage@gmail.com>
|
||||||
|
Vladimir <bogdad@gmail.com>
|
||||||
|
Vladimir Malyutin <first-leon@yandex.ru>
|
||||||
|
Vladimir Zorin <vladimir@deviant.guru>
|
||||||
|
Volodymyr Vitvitskyi <72226+signalpillar@users.noreply.github.com>
|
||||||
|
WangHaoranRobin <56047610+WangHaoranRobin@users.noreply.github.com>
|
||||||
|
Weird Constructor <weirdconstructor@gmail.com>
|
||||||
|
Welby Seely <welbyseely@gmail.com>
|
||||||
|
Wentai Zhang <rchardx@gmail.com>
|
||||||
|
WillCorticesAI <150854901+WillCorticesAI@users.noreply.github.com>
|
||||||
|
Willy Tarreau <w@1wt.eu>
|
||||||
|
Wu Jian Ping <wujjpp@hotmail.com>
|
||||||
|
Wu Jian Ping <wujp@greatld.com>
|
||||||
|
Xiake Sun <xiake.sun@intel.com>
|
||||||
|
Xiang (Kevin) Li <kevinli020508@gmail.com>
|
||||||
|
Xiao-Yong Jin <jinxiaoyong@gmail.com>
|
||||||
|
XiaotaoChen <chenxiaotao1234@gmail.com>
|
||||||
|
Xiaoyi Chen <cxychina@gmail.com>
|
||||||
|
Xingchen Song(宋星辰) <xingchensong1996@163.com>
|
||||||
|
Xuan Son Nguyen <thichthat@gmail.com>
|
||||||
|
Yann Follet <131855179+YannFollet@users.noreply.github.com>
|
||||||
|
Yiming Cui <conandiy@vip.qq.com>
|
||||||
|
Yishuo Wang <MeouSker77@outlook.com>
|
||||||
|
Yueh-Po Peng <94939112+y10ab1@users.noreply.github.com>
|
||||||
|
Yui <dev@sleepyyui.com>
|
||||||
|
Yusuf Kağan Hanoğlu <hanoglu@yahoo.com>
|
||||||
|
Yuval Peled <31162840+Yuval-Peled@users.noreply.github.com>
|
||||||
|
ZHAOKAI WANG <sanxianwei@163.com>
|
||||||
|
Zane Shannon <z@zcs.me>
|
||||||
|
Zay <95888118+isaiahbjork@users.noreply.github.com>
|
||||||
|
Zenix <zenixls2@gmail.com>
|
||||||
|
Zhang Peiyuan <a1286225768@gmail.com>
|
||||||
|
ZhouYuChen <zhouyuchen@naver.com>
|
||||||
|
Ziad Ben Hadj-Alouane <zied.benhadjalouane@gmail.com>
|
||||||
|
Ziang Wu <97337387+ZiangWu-77@users.noreply.github.com>
|
||||||
|
Zsapi <martin1.zsapka@gmail.com>
|
||||||
|
a-n-n-a-l-e-e <150648636+a-n-n-a-l-e-e@users.noreply.github.com>
|
||||||
|
adel boussaken <netdur@gmail.com>
|
||||||
|
afrideva <95653597+afrideva@users.noreply.github.com>
|
||||||
|
akawrykow <142945436+akawrykow@users.noreply.github.com>
|
||||||
|
alexpinel <93524949+alexpinel@users.noreply.github.com>
|
||||||
|
alonfaraj <alonfaraj@gmail.com>
|
||||||
|
andrijdavid <david@geek.mg>
|
||||||
|
anon998 <131767832+anon998@users.noreply.github.com>
|
||||||
|
anzz1 <anzz1@live.com>
|
||||||
|
apaz <aarpazdera@gmail.com>
|
||||||
|
apcameron <37645737+apcameron@users.noreply.github.com>
|
||||||
|
arcrank <arcrank@gmail.com>
|
||||||
|
arlo-phoenix <140345165+arlo-phoenix@users.noreply.github.com>
|
||||||
|
at8u <129688334+at8u@users.noreply.github.com>
|
||||||
|
automaticcat <daogiatuank54@gmail.com>
|
||||||
|
bandoti <141645996+bandoti@users.noreply.github.com>
|
||||||
|
beiller <beiller@gmail.com>
|
||||||
|
bhubbb <79117352+bhubbb@users.noreply.github.com>
|
||||||
|
bmwl <brian.marshall@tolko.com>
|
||||||
|
bobqianic <129547291+bobqianic@users.noreply.github.com>
|
||||||
|
bryanSwk <93190252+bryanSwk@users.noreply.github.com>
|
||||||
|
bsilvereagle <bsilvereagle@users.noreply.github.com>
|
||||||
|
bssrdf <merlintiger@hotmail.com>
|
||||||
|
byte-6174 <88070277+byte-6174@users.noreply.github.com>
|
||||||
|
cebtenzzre <cebtenzzre@gmail.com>
|
||||||
|
chaihahaha <chai836275709@gmail.com>
|
||||||
|
chiranko <96988916+chiranko@users.noreply.github.com>
|
||||||
|
clibdev <52199778+clibdev@users.noreply.github.com>
|
||||||
|
clyang <clyang@clyang.net>
|
||||||
|
cocktailpeanut <121128867+cocktailpeanut@users.noreply.github.com>
|
||||||
|
coezbek <c.oezbek@gmail.com>
|
||||||
|
comex <comexk@gmail.com>
|
||||||
|
compilade <113953597+compilade@users.noreply.github.com>
|
||||||
|
crasm <crasm@git.vczf.net>
|
||||||
|
crasm <crasm@git.vczf.us>
|
||||||
|
daboe01 <daboe01@googlemail.com>
|
||||||
|
david raistrick <keen99@users.noreply.github.com>
|
||||||
|
ddpasa <112642920+ddpasa@users.noreply.github.com>
|
||||||
|
deepdiffuser <112834445+deepdiffuser@users.noreply.github.com>
|
||||||
|
divinity76 <divinity76@gmail.com>
|
||||||
|
dotpy314 <33351922+dotpy314@users.noreply.github.com>
|
||||||
|
drbh <david.richard.holtz@gmail.com>
|
||||||
|
ds5t5 <145942675+ds5t5@users.noreply.github.com>
|
||||||
|
dylan <canardleteer@users.noreply.github.com>
|
||||||
|
eastriver <lee@eastriver.dev>
|
||||||
|
ebraminio <ebraminio@gmail.com>
|
||||||
|
eiery <19350831+eiery@users.noreply.github.com>
|
||||||
|
eric8607242 <e0928021388@gmail.com>
|
||||||
|
fraxy-v <65565042+fraxy-v@users.noreply.github.com>
|
||||||
|
github-actions[bot] <github-actions[bot]@users.noreply.github.com>
|
||||||
|
gliptic <gliptic@users.noreply.github.com>
|
||||||
|
goerch <jhr.walter@t-online.de>
|
||||||
|
grahameth <96447521+grahameth@users.noreply.github.com>
|
||||||
|
gwjr <502526+gwjr@users.noreply.github.com>
|
||||||
|
h-h-h-h <13482553+h-h-h-h@users.noreply.github.com>
|
||||||
|
hankcs <cnhankmc@gmail.com>
|
||||||
|
hoangmit <hoangmit@users.noreply.github.com>
|
||||||
|
hongbo.mo <352280764@qq.com>
|
||||||
|
howlger <eclipse@voormann.de>
|
||||||
|
howlger <github@voormann.de>
|
||||||
|
hutli <6594598+hutli@users.noreply.github.com>
|
||||||
|
hutli <hutli@hutli.hu>
|
||||||
|
hutli <jensstaermose@hotmail.com>
|
||||||
|
hxer7963 <hxer7963@gmail.com>
|
||||||
|
hydai <z54981220@gmail.com>
|
||||||
|
iSma <ismail.senhaji@gmail.com>
|
||||||
|
iacore <74560659+iacore@users.noreply.github.com>
|
||||||
|
igarnier <igarnier@protonmail.com>
|
||||||
|
iohub <rickyang.pro@gmail.com>
|
||||||
|
jacobi petrucciani <8117202+jpetrucciani@users.noreply.github.com>
|
||||||
|
jameswu2014 <545426914@qq.com>
|
||||||
|
jneem <joeneeman@gmail.com>
|
||||||
|
johnson442 <56517414+johnson442@users.noreply.github.com>
|
||||||
|
jon-chuang <9093549+jon-chuang@users.noreply.github.com>
|
||||||
|
jp-x-g <jpxg-dev@protonmail.com>
|
||||||
|
jwj7140 <32943891+jwj7140@users.noreply.github.com>
|
||||||
|
kaizau <kaizau@users.noreply.github.com>
|
||||||
|
kalomaze <66376113+kalomaze@users.noreply.github.com>
|
||||||
|
kang <tpdns9032100@gmail.com>
|
||||||
|
katsu560 <118887472+katsu560@users.noreply.github.com>
|
||||||
|
kchro3 <62481661+kchro3@users.noreply.github.com>
|
||||||
|
khimaros <me@khimaros.com>
|
||||||
|
kiltyj <kiltyj@gmail.com>
|
||||||
|
klosax <131523366+klosax@users.noreply.github.com>
|
||||||
|
kunal-vaishnavi <115581922+kunal-vaishnavi@users.noreply.github.com>
|
||||||
|
kunnis <kunnis@users.noreply.github.com>
|
||||||
|
kuronekosaiko <EvanChanJ@163.com>
|
||||||
|
kuvaus <22169537+kuvaus@users.noreply.github.com>
|
||||||
|
kwin1412 <42286931+kwin1412@users.noreply.github.com>
|
||||||
|
l3utterfly <gc.pthzfoldr@gmail.com>
|
||||||
|
ldwang <ftgreat@163.com>
|
||||||
|
le.chang <cljs118@126.com>
|
||||||
|
leejet <leejet714@gmail.com>
|
||||||
|
limitedAtonement <limitedAtonement@users.noreply.github.com>
|
||||||
|
lon <114724657+longregen@users.noreply.github.com>
|
||||||
|
m3ndax <adrian.goessl@outlook.com>
|
||||||
|
maddes8cht <55592906+maddes8cht@users.noreply.github.com>
|
||||||
|
makomk <makosoft@googlemail.com>
|
||||||
|
manikbhandari <mbbhandarimanik2@gmail.com>
|
||||||
|
mdrokz <mohammadmunshi@gmail.com>
|
||||||
|
mgroeber9110 <45620825+mgroeber9110@users.noreply.github.com>
|
||||||
|
minarchist <minarchist@users.noreply.github.com>
|
||||||
|
mj-shifu <77107165+mj-shifu@users.noreply.github.com>
|
||||||
|
mmyjona <jonathan.gonse@gmail.com>
|
||||||
|
momonga <115213907+mmnga@users.noreply.github.com>
|
||||||
|
moritzbrantner <31051084+moritzbrantner@users.noreply.github.com>
|
||||||
|
mzcu <milos.cubrilo@gmail.com>
|
||||||
|
nanahi <130121847+na-na-hi@users.noreply.github.com>
|
||||||
|
ngc92 <7938269+ngc92@users.noreply.github.com>
|
||||||
|
nhamanasu <45545786+nhamanasu@users.noreply.github.com>
|
||||||
|
niansa/tuxifan <anton-sa@web.de>
|
||||||
|
niansa/tuxifan <tuxifan@posteo.de>
|
||||||
|
ningshanwutuobang <ningshanwutuobang@gmail.com>
|
||||||
|
nold <Nold360@users.noreply.github.com>
|
||||||
|
nopperl <54780682+nopperl@users.noreply.github.com>
|
||||||
|
nusu-github <29514220+nusu-github@users.noreply.github.com>
|
||||||
|
olexiyb <olexiyb@gmail.com>
|
||||||
|
oobabooga <112222186+oobabooga@users.noreply.github.com>
|
||||||
|
opparco <parco.opaai@gmail.com>
|
||||||
|
ostix360 <55257054+ostix360@users.noreply.github.com>
|
||||||
|
perserk <perserk@gmail.com>
|
||||||
|
postmasters <namnguyen@google.com>
|
||||||
|
pudepiedj <pudepiedj@gmail.com>
|
||||||
|
qingfengfenga <41416092+qingfengfenga@users.noreply.github.com>
|
||||||
|
qouoq <qouoq@fastmail.com>
|
||||||
|
qunash <anzoria@gmail.com>
|
||||||
|
rabidcopy <rabidcopy@yahoo.com>
|
||||||
|
rankaiyx <rankaiyx@rankaiyx.com>
|
||||||
|
rhjdvsgsgks <26178113+rhjdvsgsgks@users.noreply.github.com>
|
||||||
|
rhuddleston <ryan.huddleston@percona.com>
|
||||||
|
rimoliga <53384203+rimoliga@users.noreply.github.com>
|
||||||
|
runfuture <runfuture@users.noreply.github.com>
|
||||||
|
sandyiscool <sandyiscool@gmail.com>
|
||||||
|
semidark <me@semidark.net>
|
||||||
|
sharpHL <132747147+sharpHL@users.noreply.github.com>
|
||||||
|
shibe2 <shibe@tuta.io>
|
||||||
|
singularity <12184989+singularity-s0@users.noreply.github.com>
|
||||||
|
sjinzh <sjinzh@gmail.com>
|
||||||
|
slaren <2141330+slaren@users.noreply.github.com>
|
||||||
|
slaren <slarengh@gmail.com>
|
||||||
|
snadampal <87143774+snadampal@users.noreply.github.com>
|
||||||
|
staviq <staviq@gmail.com>
|
||||||
|
stduhpf <stephduh@live.fr>
|
||||||
|
swittk <switt1995@gmail.com>
|
||||||
|
takov751 <40316768+takov751@users.noreply.github.com>
|
||||||
|
tarcey <cey.tarik@gmail.com>
|
||||||
|
texmex76 <40733439+texmex76@users.noreply.github.com>
|
||||||
|
thement <40525767+thement@users.noreply.github.com>
|
||||||
|
tjohnman <tjohnman@users.noreply.github.com>
|
||||||
|
tslmy <tslmy@users.noreply.github.com>
|
||||||
|
ubik2 <ubik2@users.noreply.github.com>
|
||||||
|
uint256_t <konndennsa@gmail.com>
|
||||||
|
uint256_t <maekawatoshiki1017@gmail.com>
|
||||||
|
unbounded <haakon@likedan.net>
|
||||||
|
valiray <133289098+valiray@users.noreply.github.com>
|
||||||
|
vodkaslime <646329483@qq.com>
|
||||||
|
vvhg1 <94630311+vvhg1@users.noreply.github.com>
|
||||||
|
vxiiduu <73044267+vxiiduu@users.noreply.github.com>
|
||||||
|
wbpxre150 <100937007+wbpxre150@users.noreply.github.com>
|
||||||
|
whoreson <139810751+whoreson@users.noreply.github.com>
|
||||||
|
wonjun Jang <strutive07@gmail.com>
|
||||||
|
wzy <32936898+Freed-Wu@users.noreply.github.com>
|
||||||
|
xaedes <xaedes@gmail.com>
|
||||||
|
xaedes <xaedes@googlemail.com>
|
||||||
|
xloem <0xloem@gmail.com>
|
||||||
|
yangli2 <yangli2@gmail.com>
|
||||||
|
yuiseki <yuiseki@gmail.com>
|
||||||
|
zakkor <edward.partenie@gmail.com>
|
||||||
|
zhouwg <6889919+zhouwg@users.noreply.github.com>
|
||||||
|
zrm <trustiosity.zrm@gmail.com>
|
||||||
|
源文雨 <41315874+fumiama@users.noreply.github.com>
|
||||||
|
Нияз Гарифзянов <112617865+garrnizon@users.noreply.github.com>
|
||||||
1232
CMakeLists.txt
1232
CMakeLists.txt
File diff suppressed because it is too large
Load diff
2
LICENSE
2
LICENSE
|
|
@ -1,6 +1,6 @@
|
||||||
MIT License
|
MIT License
|
||||||
|
|
||||||
Copyright (c) 2023 Georgi Gerganov
|
Copyright (c) 2023-2024 The ggml authors
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,54 @@
|
||||||
// swift-tools-version:5.3
|
// swift-tools-version:5.5
|
||||||
|
|
||||||
import PackageDescription
|
import PackageDescription
|
||||||
|
|
||||||
|
var sources = [
|
||||||
|
"ggml.c",
|
||||||
|
"sgemm.cpp",
|
||||||
|
"llama.cpp",
|
||||||
|
"unicode.cpp",
|
||||||
|
"unicode-data.cpp",
|
||||||
|
"ggml-alloc.c",
|
||||||
|
"ggml-backend.c",
|
||||||
|
"ggml-quants.c",
|
||||||
|
]
|
||||||
|
|
||||||
|
var resources: [Resource] = []
|
||||||
|
var linkerSettings: [LinkerSetting] = []
|
||||||
|
var cSettings: [CSetting] = [
|
||||||
|
.unsafeFlags(["-Wno-shorten-64-to-32", "-O3", "-DNDEBUG"]),
|
||||||
|
.unsafeFlags(["-fno-objc-arc"]),
|
||||||
|
// NOTE: NEW_LAPACK will required iOS version 16.4+
|
||||||
|
// We should consider add this in the future when we drop support for iOS 14
|
||||||
|
// (ref: ref: https://developer.apple.com/documentation/accelerate/1513264-cblas_sgemm?language=objc)
|
||||||
|
// .define("ACCELERATE_NEW_LAPACK"),
|
||||||
|
// .define("ACCELERATE_LAPACK_ILP64")
|
||||||
|
]
|
||||||
|
|
||||||
|
#if canImport(Darwin)
|
||||||
|
sources.append("ggml-metal.m")
|
||||||
|
resources.append(.process("ggml-metal.metal"))
|
||||||
|
linkerSettings.append(.linkedFramework("Accelerate"))
|
||||||
|
cSettings.append(
|
||||||
|
contentsOf: [
|
||||||
|
.define("GGML_USE_ACCELERATE"),
|
||||||
|
.define("GGML_USE_METAL")
|
||||||
|
]
|
||||||
|
)
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#if os(Linux)
|
||||||
|
cSettings.append(.define("_GNU_SOURCE"))
|
||||||
|
#endif
|
||||||
|
|
||||||
let package = Package(
|
let package = Package(
|
||||||
name: "llama",
|
name: "llama",
|
||||||
|
platforms: [
|
||||||
|
.macOS(.v12),
|
||||||
|
.iOS(.v14),
|
||||||
|
.watchOS(.v4),
|
||||||
|
.tvOS(.v14)
|
||||||
|
],
|
||||||
products: [
|
products: [
|
||||||
.library(name: "llama", targets: ["llama"]),
|
.library(name: "llama", targets: ["llama"]),
|
||||||
],
|
],
|
||||||
|
|
@ -11,13 +56,23 @@ let package = Package(
|
||||||
.target(
|
.target(
|
||||||
name: "llama",
|
name: "llama",
|
||||||
path: ".",
|
path: ".",
|
||||||
sources: ["ggml.c", "llama.cpp"],
|
exclude: [
|
||||||
|
"cmake",
|
||||||
|
"examples",
|
||||||
|
"scripts",
|
||||||
|
"models",
|
||||||
|
"tests",
|
||||||
|
"CMakeLists.txt",
|
||||||
|
"ggml-cuda.cu",
|
||||||
|
"ggml-cuda.h",
|
||||||
|
"Makefile"
|
||||||
|
],
|
||||||
|
sources: sources,
|
||||||
|
resources: resources,
|
||||||
publicHeadersPath: "spm-headers",
|
publicHeadersPath: "spm-headers",
|
||||||
cSettings: [.unsafeFlags(["-Wno-shorten-64-to-32"]), .define("GGML_USE_ACCELERATE")],
|
cSettings: cSettings,
|
||||||
linkerSettings: [
|
linkerSettings: linkerSettings
|
||||||
.linkedFramework("Accelerate")
|
)
|
||||||
]
|
|
||||||
),
|
|
||||||
],
|
],
|
||||||
cxxLanguageStandard: .cxx11
|
cxxLanguageStandard: .cxx11
|
||||||
)
|
)
|
||||||
|
|
|
||||||
568
README-sycl.md
Normal file
568
README-sycl.md
Normal file
|
|
@ -0,0 +1,568 @@
|
||||||
|
# llama.cpp for SYCL
|
||||||
|
|
||||||
|
- [Background](#background)
|
||||||
|
- [News](#news)
|
||||||
|
- [OS](#os)
|
||||||
|
- [Hardware](#hardware)
|
||||||
|
- [Docker](#docker)
|
||||||
|
- [Linux](#linux)
|
||||||
|
- [Windows](#windows)
|
||||||
|
- [Environment Variable](#environment-variable)
|
||||||
|
- [Known Issue](#known-issues)
|
||||||
|
- [Q&A](#qa)
|
||||||
|
- [TODO](#todo)
|
||||||
|
|
||||||
|
## Background
|
||||||
|
|
||||||
|
**SYCL** is a high-level parallel programming model designed to improve developers productivity writing code across various hardware accelerators such as CPUs, GPUs, and FPGAs. It is a single-source language designed for heterogeneous computing and based on standard C++17.
|
||||||
|
|
||||||
|
**oneAPI** is an open ecosystem and a standard-based specification, supporting multiple architectures including but not limited to intel CPUs, GPUs and FPGAs. The key components of the oneAPI ecosystem include:
|
||||||
|
|
||||||
|
- **DPCPP** *(Data Parallel C++)*: The primary oneAPI SYCL implementation, which includes the icpx/icx Compilers.
|
||||||
|
- **oneAPI Libraries**: A set of highly optimized libraries targeting multiple domains *(e.g. oneMKL - Math Kernel Library)*.
|
||||||
|
- **oneAPI LevelZero**: A high performance low level interface for fine-grained control over intel iGPUs and dGPUs.
|
||||||
|
- **Nvidia & AMD Plugins**: These are plugins extending oneAPI's DPCPP support to SYCL on Nvidia and AMD GPU targets.
|
||||||
|
|
||||||
|
### Llama.cpp + SYCL
|
||||||
|
|
||||||
|
The llama.cpp SYCL backend is designed to support **Intel GPU** firstly. Based on the cross-platform feature of SYCL, it could support other vendor GPUs: Nvidia GPU (*AMD GPU coming*).
|
||||||
|
|
||||||
|
When targeting **Intel CPU**, it is recommended to use llama.cpp for [Intel oneMKL](README.md#intel-onemkl) backend.
|
||||||
|
|
||||||
|
It has the similar design of other llama.cpp BLAS-based paths such as *OpenBLAS, cuBLAS, CLBlast etc..*. In beginning work, the oneAPI's [SYCLomatic](https://github.com/oneapi-src/SYCLomatic) open-source migration tool (Commercial release [Intel® DPC++ Compatibility Tool](https://www.intel.com/content/www/us/en/developer/tools/oneapi/dpc-compatibility-tool.html)) was used for this purpose.
|
||||||
|
|
||||||
|
## News
|
||||||
|
|
||||||
|
- 2024.4
|
||||||
|
- Support data types: GGML_TYPE_IQ4_NL, GGML_TYPE_IQ4_XS, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M.
|
||||||
|
|
||||||
|
- 2024.3
|
||||||
|
- Release binary files of Windows.
|
||||||
|
- A blog is published: **Run LLM on all Intel GPUs Using llama.cpp**: [intel.com](https://www.intel.com/content/www/us/en/developer/articles/technical/run-llm-on-all-gpus-using-llama-cpp-artical.html) or [medium.com](https://medium.com/@jianyu_neo/run-llm-on-all-intel-gpus-using-llama-cpp-fd2e2dcbd9bd).
|
||||||
|
- New base line is ready: [tag b2437](https://github.com/ggerganov/llama.cpp/tree/b2437).
|
||||||
|
- Support multiple cards: **--split-mode**: [none|layer]; not support [row], it's on developing.
|
||||||
|
- Support to assign main GPU by **--main-gpu**, replace $GGML_SYCL_DEVICE.
|
||||||
|
- Support detecting all GPUs with level-zero and same top **Max compute units**.
|
||||||
|
- Support OPs
|
||||||
|
- hardsigmoid
|
||||||
|
- hardswish
|
||||||
|
- pool2d
|
||||||
|
|
||||||
|
- 2024.1
|
||||||
|
- Create SYCL backend for Intel GPU.
|
||||||
|
- Support Windows build
|
||||||
|
|
||||||
|
## OS
|
||||||
|
|
||||||
|
| OS | Status | Verified |
|
||||||
|
|---------|---------|------------------------------------|
|
||||||
|
| Linux | Support | Ubuntu 22.04, Fedora Silverblue 39 |
|
||||||
|
| Windows | Support | Windows 11 |
|
||||||
|
|
||||||
|
|
||||||
|
## Hardware
|
||||||
|
|
||||||
|
### Intel GPU
|
||||||
|
|
||||||
|
**Verified devices**
|
||||||
|
|
||||||
|
| Intel GPU | Status | Verified Model |
|
||||||
|
|-------------------------------|---------|---------------------------------------|
|
||||||
|
| Intel Data Center Max Series | Support | Max 1550, 1100 |
|
||||||
|
| Intel Data Center Flex Series | Support | Flex 170 |
|
||||||
|
| Intel Arc Series | Support | Arc 770, 730M |
|
||||||
|
| Intel built-in Arc GPU | Support | built-in Arc GPU in Meteor Lake |
|
||||||
|
| Intel iGPU | Support | iGPU in i5-1250P, i7-1260P, i7-1165G7 |
|
||||||
|
|
||||||
|
*Notes:*
|
||||||
|
|
||||||
|
- **Memory**
|
||||||
|
- The device memory is a limitation when running a large model. The loaded model size, *`llm_load_tensors: buffer_size`*, is displayed in the log when running `./bin/main`.
|
||||||
|
|
||||||
|
- Please make sure the GPU shared memory from the host is large enough to account for the model's size. For e.g. the *llama-2-7b.Q4_0* requires at least 8.0GB for integrated GPU and 4.0GB for discrete GPU.
|
||||||
|
|
||||||
|
- **Execution Unit (EU)**
|
||||||
|
- If the iGPU has less than 80 EUs, the inference speed will likely be too slow for practical use.
|
||||||
|
|
||||||
|
### Other Vendor GPU
|
||||||
|
|
||||||
|
**Verified devices**
|
||||||
|
|
||||||
|
| Nvidia GPU | Status | Verified Model |
|
||||||
|
|--------------------------|---------|----------------|
|
||||||
|
| Ampere Series | Support | A100, A4000 |
|
||||||
|
| Ampere Series *(Mobile)* | Support | RTX 40 Series |
|
||||||
|
|
||||||
|
## Docker
|
||||||
|
The docker build option is currently limited to *intel GPU* targets.
|
||||||
|
|
||||||
|
### Build image
|
||||||
|
```sh
|
||||||
|
# Using FP16
|
||||||
|
docker build -t llama-cpp-sycl --build-arg="LLAMA_SYCL_F16=ON" -f .devops/main-intel.Dockerfile .
|
||||||
|
```
|
||||||
|
|
||||||
|
*Notes*:
|
||||||
|
|
||||||
|
To build in default FP32 *(Slower than FP16 alternative)*, you can remove the `--build-arg="LLAMA_SYCL_F16=ON"` argument from the previous command.
|
||||||
|
|
||||||
|
You can also use the `.devops/server-intel.Dockerfile`, which builds the *"server"* alternative.
|
||||||
|
|
||||||
|
### Run container
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# First, find all the DRI cards
|
||||||
|
ls -la /dev/dri
|
||||||
|
# Then, pick the card that you want to use (here for e.g. /dev/dri/card1).
|
||||||
|
docker run -it --rm -v "$(pwd):/app:Z" --device /dev/dri/renderD128:/dev/dri/renderD128 --device /dev/dri/card1:/dev/dri/card1 llama-cpp-sycl -m "/app/models/YOUR_MODEL_FILE" -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33
|
||||||
|
```
|
||||||
|
|
||||||
|
*Notes:*
|
||||||
|
- Docker has been tested successfully on native Linux. WSL support has not been verified yet.
|
||||||
|
- You may need to install Intel GPU driver on the **host** machine *(Please refer to the [Linux configuration](#linux) for details)*.
|
||||||
|
|
||||||
|
## Linux
|
||||||
|
|
||||||
|
### I. Setup Environment
|
||||||
|
|
||||||
|
1. **Install GPU drivers**
|
||||||
|
|
||||||
|
- **Intel GPU**
|
||||||
|
|
||||||
|
Intel data center GPUs drivers installation guide and download page can be found here: [Get intel dGPU Drivers](https://dgpu-docs.intel.com/driver/installation.html#ubuntu-install-steps).
|
||||||
|
|
||||||
|
*Note*: for client GPUs *(iGPU & Arc A-Series)*, please refer to the [client iGPU driver installation](https://dgpu-docs.intel.com/driver/client/overview.html).
|
||||||
|
|
||||||
|
Once installed, add the user(s) to the `video` and `render` groups.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
sudo usermod -aG render $USER
|
||||||
|
sudo usermod -aG video $USER
|
||||||
|
```
|
||||||
|
|
||||||
|
*Note*: logout/re-login for the changes to take effect.
|
||||||
|
|
||||||
|
Verify installation through `clinfo`:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
sudo apt install clinfo
|
||||||
|
sudo clinfo -l
|
||||||
|
```
|
||||||
|
|
||||||
|
Sample output:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
Platform #0: Intel(R) OpenCL Graphics
|
||||||
|
`-- Device #0: Intel(R) Arc(TM) A770 Graphics
|
||||||
|
|
||||||
|
Platform #0: Intel(R) OpenCL HD Graphics
|
||||||
|
`-- Device #0: Intel(R) Iris(R) Xe Graphics [0x9a49]
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Nvidia GPU**
|
||||||
|
|
||||||
|
In order to target Nvidia GPUs through SYCL, please make sure the CUDA/CUBLAS native requirements *-found [here](README.md#cuda)-* are installed.
|
||||||
|
|
||||||
|
2. **Install Intel® oneAPI Base toolkit**
|
||||||
|
|
||||||
|
- **For Intel GPU**
|
||||||
|
|
||||||
|
The base toolkit can be obtained from the official [Intel® oneAPI Base Toolkit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html) page.
|
||||||
|
|
||||||
|
Please follow the instructions for downloading and installing the Toolkit for Linux, and preferably keep the default installation values unchanged, notably the installation path *(`/opt/intel/oneapi` by default)*.
|
||||||
|
|
||||||
|
Following guidelines/code snippets assume the default installation values. Otherwise, please make sure the necessary changes are reflected where applicable.
|
||||||
|
|
||||||
|
Upon a successful installation, SYCL is enabled for the available intel devices, along with relevant libraries such as oneAPI MKL for intel GPUs.
|
||||||
|
|
||||||
|
- **Adding support to Nvidia GPUs**
|
||||||
|
|
||||||
|
**oneAPI Plugin**: In order to enable SYCL support on Nvidia GPUs, please install the [Codeplay oneAPI Plugin for Nvidia GPUs](https://developer.codeplay.com/products/oneapi/nvidia/download). User should also make sure the plugin version matches the installed base toolkit one *(previous step)* for a seamless "oneAPI on Nvidia GPU" setup.
|
||||||
|
|
||||||
|
|
||||||
|
**oneMKL for cuBlas**: The current oneMKL releases *(shipped with the oneAPI base-toolkit)* do not contain the cuBLAS backend. A build from source of the upstream [oneMKL](https://github.com/oneapi-src/oneMKL) with the *cuBLAS* backend enabled is thus required to run it on Nvidia GPUs.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git clone https://github.com/oneapi-src/oneMKL
|
||||||
|
cd oneMKL
|
||||||
|
cmake -B buildWithCublas -DCMAKE_CXX_COMPILER=icpx -DCMAKE_C_COMPILER=icx -DENABLE_MKLGPU_BACKEND=OFF -DENABLE_MKLCPU_BACKEND=OFF -DENABLE_CUBLAS_BACKEND=ON -DTARGET_DOMAINS=blas
|
||||||
|
cmake --build buildWithCublas --config Release
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
3. **Verify installation and environment**
|
||||||
|
|
||||||
|
In order to check the available SYCL devices on the machine, please use the `sycl-ls` command.
|
||||||
|
```sh
|
||||||
|
source /opt/intel/oneapi/setvars.sh
|
||||||
|
sycl-ls
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Intel GPU**
|
||||||
|
|
||||||
|
When targeting an intel GPU, the user should expect one or more level-zero devices among the available SYCL devices. Please make sure that at least one GPU is present, for instance [`ext_oneapi_level_zero:gpu:0`] in the sample output below:
|
||||||
|
|
||||||
|
```
|
||||||
|
[opencl:acc:0] Intel(R) FPGA Emulation Platform for OpenCL(TM), Intel(R) FPGA Emulation Device OpenCL 1.2 [2023.16.10.0.17_160000]
|
||||||
|
[opencl:cpu:1] Intel(R) OpenCL, 13th Gen Intel(R) Core(TM) i7-13700K OpenCL 3.0 (Build 0) [2023.16.10.0.17_160000]
|
||||||
|
[opencl:gpu:2] Intel(R) OpenCL Graphics, Intel(R) Arc(TM) A770 Graphics OpenCL 3.0 NEO [23.30.26918.50]
|
||||||
|
[ext_oneapi_level_zero:gpu:0] Intel(R) Level-Zero, Intel(R) Arc(TM) A770 Graphics 1.3 [1.3.26918]
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Nvidia GPU**
|
||||||
|
|
||||||
|
Similarly, user targeting Nvidia GPUs should expect at least one SYCL-CUDA device [`ext_oneapi_cuda:gpu`] as bellow:
|
||||||
|
```
|
||||||
|
[opencl:acc:0] Intel(R) FPGA Emulation Platform for OpenCL(TM), Intel(R) FPGA Emulation Device OpenCL 1.2 [2023.16.12.0.12_195853.xmain-hotfix]
|
||||||
|
[opencl:cpu:1] Intel(R) OpenCL, Intel(R) Xeon(R) Gold 6326 CPU @ 2.90GHz OpenCL 3.0 (Build 0) [2023.16.12.0.12_195853.xmain-hotfix]
|
||||||
|
[ext_oneapi_cuda:gpu:0] NVIDIA CUDA BACKEND, NVIDIA A100-PCIE-40GB 8.0 [CUDA 12.2]
|
||||||
|
```
|
||||||
|
|
||||||
|
### II. Build llama.cpp
|
||||||
|
|
||||||
|
#### Intel GPU
|
||||||
|
```sh
|
||||||
|
# Export relevant ENV variables
|
||||||
|
source /opt/intel/oneapi/setvars.sh
|
||||||
|
|
||||||
|
# Build LLAMA with MKL BLAS acceleration for intel GPU
|
||||||
|
|
||||||
|
# Option 1: Use FP32 (recommended for better performance in most cases)
|
||||||
|
cmake -B build -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx
|
||||||
|
|
||||||
|
# Option 2: Use FP16
|
||||||
|
cmake -B build -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_SYCL_F16=ON
|
||||||
|
|
||||||
|
# build all binary
|
||||||
|
cmake --build build --config Release -j -v
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Nvidia GPU
|
||||||
|
```sh
|
||||||
|
# Export relevant ENV variables
|
||||||
|
export LD_LIBRARY_PATH=/path/to/oneMKL/buildWithCublas/lib:$LD_LIBRARY_PATH
|
||||||
|
export LIBRARY_PATH=/path/to/oneMKL/buildWithCublas/lib:$LIBRARY_PATH
|
||||||
|
export CPLUS_INCLUDE_DIR=/path/to/oneMKL/buildWithCublas/include:$CPLUS_INCLUDE_DIR
|
||||||
|
export CPLUS_INCLUDE_DIR=/path/to/oneMKL/include:$CPLUS_INCLUDE_DIR
|
||||||
|
|
||||||
|
# Build LLAMA with Nvidia BLAS acceleration through SYCL
|
||||||
|
|
||||||
|
# Option 1: Use FP32 (recommended for better performance in most cases)
|
||||||
|
cmake -B build -DLLAMA_SYCL=ON -DLLAMA_SYCL_TARGET=NVIDIA -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx
|
||||||
|
|
||||||
|
# Option 2: Use FP16
|
||||||
|
cmake -B build -DLLAMA_SYCL=ON -DLLAMA_SYCL_TARGET=NVIDIA -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_SYCL_F16=ON
|
||||||
|
|
||||||
|
# build all binary
|
||||||
|
cmake --build build --config Release -j -v
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
### III. Run the inference
|
||||||
|
|
||||||
|
1. Retrieve and prepare model
|
||||||
|
|
||||||
|
You can refer to the general [*Prepare and Quantize*](README.md#prepare-and-quantize) guide for model prepration, or simply download [llama-2-7b.Q4_0.gguf](https://huggingface.co/TheBloke/Llama-2-7B-GGUF/blob/main/llama-2-7b.Q4_0.gguf) model as example.
|
||||||
|
|
||||||
|
2. Enable oneAPI running environment
|
||||||
|
|
||||||
|
```sh
|
||||||
|
source /opt/intel/oneapi/setvars.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
3. List devices information
|
||||||
|
|
||||||
|
Similar to the native `sycl-ls`, available SYCL devices can be queried as follow:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./build/bin/ls-sycl-device
|
||||||
|
```
|
||||||
|
A example of such log in a system with 1 *intel CPU* and 1 *intel GPU* can look like the following:
|
||||||
|
```
|
||||||
|
found 6 SYCL devices:
|
||||||
|
| | | |Compute |Max compute|Max work|Max sub| |
|
||||||
|
|ID| Device Type| Name|capability|units |group |group |Global mem size|
|
||||||
|
|--|------------------|---------------------------------------------|----------|-----------|--------|-------|---------------|
|
||||||
|
| 0|[level_zero:gpu:0]| Intel(R) Arc(TM) A770 Graphics| 1.3| 512| 1024| 32| 16225243136|
|
||||||
|
| 1|[level_zero:gpu:1]| Intel(R) UHD Graphics 770| 1.3| 32| 512| 32| 53651849216|
|
||||||
|
| 2| [opencl:gpu:0]| Intel(R) Arc(TM) A770 Graphics| 3.0| 512| 1024| 32| 16225243136|
|
||||||
|
| 3| [opencl:gpu:1]| Intel(R) UHD Graphics 770| 3.0| 32| 512| 32| 53651849216|
|
||||||
|
| 4| [opencl:cpu:0]| 13th Gen Intel(R) Core(TM) i7-13700K| 3.0| 24| 8192| 64| 67064815616|
|
||||||
|
| 5| [opencl:acc:0]| Intel(R) FPGA Emulation Device| 1.2| 24|67108864| 64| 67064815616|
|
||||||
|
```
|
||||||
|
|
||||||
|
| Attribute | Note |
|
||||||
|
|------------------------|-------------------------------------------------------------|
|
||||||
|
| compute capability 1.3 | Level-zero driver/runtime, recommended |
|
||||||
|
| compute capability 3.0 | OpenCL driver/runtime, slower than level-zero in most cases |
|
||||||
|
|
||||||
|
4. Launch inference
|
||||||
|
|
||||||
|
There are two device selection modes:
|
||||||
|
|
||||||
|
- Single device: Use one device target specified by the user.
|
||||||
|
- Multiple devices: Automatically select the devices with the same largest Max compute-units.
|
||||||
|
|
||||||
|
| Device selection | Parameter |
|
||||||
|
|------------------|----------------------------------------|
|
||||||
|
| Single device | --split-mode none --main-gpu DEVICE_ID |
|
||||||
|
| Multiple devices | --split-mode layer (default) |
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
- Use device 0:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
ZES_ENABLE_SYSMAN=1 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33 -sm none -mg 0
|
||||||
|
```
|
||||||
|
or run by script:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./examples/sycl/run_llama2.sh 0
|
||||||
|
```
|
||||||
|
|
||||||
|
- Use multiple devices:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
ZES_ENABLE_SYSMAN=1 ./build/bin/main -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33 -sm layer
|
||||||
|
```
|
||||||
|
|
||||||
|
Otherwise, you can run the script:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./examples/sycl/run_llama2.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
*Notes:*
|
||||||
|
|
||||||
|
- Upon execution, verify the selected device(s) ID(s) in the output log, which can for instance be displayed as follow:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
detect 1 SYCL GPUs: [0] with top Max compute units:512
|
||||||
|
```
|
||||||
|
Or
|
||||||
|
```sh
|
||||||
|
use 1 SYCL GPUs: [0] with Max compute units:512
|
||||||
|
```
|
||||||
|
|
||||||
|
## Windows
|
||||||
|
|
||||||
|
### I. Setup Environment
|
||||||
|
|
||||||
|
1. Install GPU driver
|
||||||
|
|
||||||
|
Intel GPU drivers instructions guide and download page can be found here: [Get intel GPU Drivers](https://www.intel.com/content/www/us/en/products/docs/discrete-gpus/arc/software/drivers.html).
|
||||||
|
|
||||||
|
2. Install Visual Studio
|
||||||
|
|
||||||
|
If you already have a recent version of Microsoft Visual Studio, you can skip this step. Otherwise, please refer to the official download page for [Microsoft Visual Studio](https://visualstudio.microsoft.com/).
|
||||||
|
|
||||||
|
3. Install Intel® oneAPI Base toolkit
|
||||||
|
|
||||||
|
The base toolkit can be obtained from the official [Intel® oneAPI Base Toolkit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html) page.
|
||||||
|
|
||||||
|
Please follow the instructions for downloading and installing the Toolkit for Windows, and preferably keep the default installation values unchanged, notably the installation path *(`C:\Program Files (x86)\Intel\oneAPI` by default)*.
|
||||||
|
|
||||||
|
Following guidelines/code snippets assume the default installation values. Otherwise, please make sure the necessary changes are reflected where applicable.
|
||||||
|
|
||||||
|
b. Enable oneAPI running environment:
|
||||||
|
|
||||||
|
- Type "oneAPI" in the search bar, then open the `Intel oneAPI command prompt for Intel 64 for Visual Studio 2022` App.
|
||||||
|
|
||||||
|
- On the command prompt, enable the runtime environment with the following:
|
||||||
|
```
|
||||||
|
"C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64
|
||||||
|
```
|
||||||
|
|
||||||
|
c. Verify installation
|
||||||
|
|
||||||
|
In the oneAPI command line, run the following to print the available SYCL devices:
|
||||||
|
|
||||||
|
```
|
||||||
|
sycl-ls
|
||||||
|
```
|
||||||
|
|
||||||
|
There should be one or more *level-zero* GPU devices displayed as **[ext_oneapi_level_zero:gpu]**. Below is example of such output detecting an *intel Iris Xe* GPU as a Level-zero SYCL device:
|
||||||
|
|
||||||
|
Output (example):
|
||||||
|
```
|
||||||
|
[opencl:acc:0] Intel(R) FPGA Emulation Platform for OpenCL(TM), Intel(R) FPGA Emulation Device OpenCL 1.2 [2023.16.10.0.17_160000]
|
||||||
|
[opencl:cpu:1] Intel(R) OpenCL, 11th Gen Intel(R) Core(TM) i7-1185G7 @ 3.00GHz OpenCL 3.0 (Build 0) [2023.16.10.0.17_160000]
|
||||||
|
[opencl:gpu:2] Intel(R) OpenCL Graphics, Intel(R) Iris(R) Xe Graphics OpenCL 3.0 NEO [31.0.101.5186]
|
||||||
|
[ext_oneapi_level_zero:gpu:0] Intel(R) Level-Zero, Intel(R) Iris(R) Xe Graphics 1.3 [1.3.28044]
|
||||||
|
```
|
||||||
|
|
||||||
|
4. Install build tools
|
||||||
|
|
||||||
|
a. Download & install cmake for Windows: https://cmake.org/download/
|
||||||
|
|
||||||
|
b. Download & install mingw-w64 make for Windows provided by w64devkit
|
||||||
|
|
||||||
|
- Download the 1.19.0 version of [w64devkit](https://github.com/skeeto/w64devkit/releases/download/v1.19.0/w64devkit-1.19.0.zip).
|
||||||
|
|
||||||
|
- Extract `w64devkit` on your pc.
|
||||||
|
|
||||||
|
- Add the **bin** folder path in the Windows system PATH environment (for e.g. `C:\xxx\w64devkit\bin\`).
|
||||||
|
|
||||||
|
### II. Build llama.cpp
|
||||||
|
|
||||||
|
On the oneAPI command line window, step into the llama.cpp main directory and run the following:
|
||||||
|
|
||||||
|
```
|
||||||
|
@call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force
|
||||||
|
|
||||||
|
# Option 1: Use FP32 (recommended for better performance in most cases)
|
||||||
|
cmake -B build -G "MinGW Makefiles" -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icx -DCMAKE_BUILD_TYPE=Release
|
||||||
|
|
||||||
|
# Option 2: Or FP16
|
||||||
|
cmake -B build -G "MinGW Makefiles" -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icx -DCMAKE_BUILD_TYPE=Release -DLLAMA_SYCL_F16=ON
|
||||||
|
|
||||||
|
cmake --build build --config Release -j
|
||||||
|
```
|
||||||
|
|
||||||
|
Otherwise, run the `win-build-sycl.bat` wrapper which encapsulates the former instructions:
|
||||||
|
```sh
|
||||||
|
.\examples\sycl\win-build-sycl.bat
|
||||||
|
```
|
||||||
|
|
||||||
|
*Notes:*
|
||||||
|
|
||||||
|
- By default, calling `make` will build all target binary files. In case of a minimal experimental setup, the user can build the inference executable only through `make main`.
|
||||||
|
|
||||||
|
### III. Run the inference
|
||||||
|
|
||||||
|
1. Retrieve and prepare model
|
||||||
|
|
||||||
|
You can refer to the general [*Prepare and Quantize*](README#prepare-and-quantize) guide for model prepration, or simply download [llama-2-7b.Q4_0.gguf](https://huggingface.co/TheBloke/Llama-2-7B-GGUF/blob/main/llama-2-7b.Q4_0.gguf) model as example.
|
||||||
|
|
||||||
|
2. Enable oneAPI running environment
|
||||||
|
|
||||||
|
On the oneAPI command line window, run the following and step into the llama.cpp directory:
|
||||||
|
```
|
||||||
|
"C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64
|
||||||
|
```
|
||||||
|
|
||||||
|
3. List devices information
|
||||||
|
|
||||||
|
Similar to the native `sycl-ls`, available SYCL devices can be queried as follow:
|
||||||
|
|
||||||
|
```
|
||||||
|
build\bin\ls-sycl-device.exe
|
||||||
|
```
|
||||||
|
|
||||||
|
The output of this command in a system with 1 *intel CPU* and 1 *intel GPU* would look like the following:
|
||||||
|
```
|
||||||
|
found 6 SYCL devices:
|
||||||
|
| | | |Compute |Max compute|Max work|Max sub| |
|
||||||
|
|ID| Device Type| Name|capability|units |group |group |Global mem size|
|
||||||
|
|--|------------------|---------------------------------------------|----------|-----------|--------|-------|---------------|
|
||||||
|
| 0|[level_zero:gpu:0]| Intel(R) Arc(TM) A770 Graphics| 1.3| 512| 1024| 32| 16225243136|
|
||||||
|
| 1|[level_zero:gpu:1]| Intel(R) UHD Graphics 770| 1.3| 32| 512| 32| 53651849216|
|
||||||
|
| 2| [opencl:gpu:0]| Intel(R) Arc(TM) A770 Graphics| 3.0| 512| 1024| 32| 16225243136|
|
||||||
|
| 3| [opencl:gpu:1]| Intel(R) UHD Graphics 770| 3.0| 32| 512| 32| 53651849216|
|
||||||
|
| 4| [opencl:cpu:0]| 13th Gen Intel(R) Core(TM) i7-13700K| 3.0| 24| 8192| 64| 67064815616|
|
||||||
|
| 5| [opencl:acc:0]| Intel(R) FPGA Emulation Device| 1.2| 24|67108864| 64| 67064815616|
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
| Attribute | Note |
|
||||||
|
|------------------------|-----------------------------------------------------------|
|
||||||
|
| compute capability 1.3 | Level-zero running time, recommended |
|
||||||
|
| compute capability 3.0 | OpenCL running time, slower than level-zero in most cases |
|
||||||
|
|
||||||
|
|
||||||
|
4. Launch inference
|
||||||
|
|
||||||
|
There are two device selection modes:
|
||||||
|
|
||||||
|
- Single device: Use one device assigned by user.
|
||||||
|
- Multiple devices: Automatically choose the devices with the same biggest Max compute units.
|
||||||
|
|
||||||
|
| Device selection | Parameter |
|
||||||
|
|------------------|----------------------------------------|
|
||||||
|
| Single device | --split-mode none --main-gpu DEVICE_ID |
|
||||||
|
| Multiple devices | --split-mode layer (default) |
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
- Use device 0:
|
||||||
|
|
||||||
|
```
|
||||||
|
build\bin\main.exe -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 33 -s 0 -sm none -mg 0
|
||||||
|
```
|
||||||
|
|
||||||
|
- Use multiple devices:
|
||||||
|
|
||||||
|
```
|
||||||
|
build\bin\main.exe -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 33 -s 0 -sm layer
|
||||||
|
```
|
||||||
|
Otherwise, run the following wrapper script:
|
||||||
|
|
||||||
|
```
|
||||||
|
.\examples\sycl\win-run-llama2.bat
|
||||||
|
```
|
||||||
|
|
||||||
|
Note:
|
||||||
|
|
||||||
|
- Upon execution, verify the selected device(s) ID(s) in the output log, which can for instance be displayed as follow:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
detect 1 SYCL GPUs: [0] with top Max compute units:512
|
||||||
|
```
|
||||||
|
Or
|
||||||
|
```sh
|
||||||
|
use 1 SYCL GPUs: [0] with Max compute units:512
|
||||||
|
```
|
||||||
|
|
||||||
|
## Environment Variable
|
||||||
|
|
||||||
|
#### Build
|
||||||
|
|
||||||
|
| Name | Value | Function |
|
||||||
|
|--------------------|-----------------------------------|---------------------------------------------|
|
||||||
|
| LLAMA_SYCL | ON (mandatory) | Enable build with SYCL code path. |
|
||||||
|
| LLAMA_SYCL_TARGET | INTEL *(default)* \| NVIDIA | Set the SYCL target device type. |
|
||||||
|
| LLAMA_SYCL_F16 | OFF *(default)* \|ON *(optional)* | Enable FP16 build with SYCL code path. |
|
||||||
|
| CMAKE_C_COMPILER | icx | Set *icx* compiler for SYCL code path. |
|
||||||
|
| CMAKE_CXX_COMPILER | icpx *(Linux)*, icx *(Windows)* | Set `icpx/icx` compiler for SYCL code path. |
|
||||||
|
|
||||||
|
#### Runtime
|
||||||
|
|
||||||
|
| Name | Value | Function |
|
||||||
|
|-------------------|------------------|---------------------------------------------------------------------------------------------------------------------------|
|
||||||
|
| GGML_SYCL_DEBUG | 0 (default) or 1 | Enable log function by macro: GGML_SYCL_DEBUG |
|
||||||
|
| ZES_ENABLE_SYSMAN | 0 (default) or 1 | Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.<br>Recommended to use when --split-mode = layer |
|
||||||
|
|
||||||
|
## Known Issues
|
||||||
|
|
||||||
|
- `Split-mode:[row]` is not supported.
|
||||||
|
|
||||||
|
## Q&A
|
||||||
|
|
||||||
|
- Error: `error while loading shared libraries: libsycl.so.7: cannot open shared object file: No such file or directory`.
|
||||||
|
|
||||||
|
- Potential cause: Unavailable oneAPI installation or not set ENV variables.
|
||||||
|
- Solution: Install *oneAPI base toolkit* and enable its ENV through: `source /opt/intel/oneapi/setvars.sh`.
|
||||||
|
|
||||||
|
- General compiler error:
|
||||||
|
|
||||||
|
- Remove **build** folder or try a clean-build.
|
||||||
|
|
||||||
|
- I can **not** see `[ext_oneapi_level_zero:gpu]` afer installing the GPU driver on Linux.
|
||||||
|
|
||||||
|
Please double-check with `sudo sycl-ls`.
|
||||||
|
|
||||||
|
If it's present in the list, please add video/render group to your user then **logout/login** or restart your system:
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo usermod -aG render $USER
|
||||||
|
sudo usermod -aG video $USER
|
||||||
|
```
|
||||||
|
Otherwise, please double-check the GPU driver installation steps.
|
||||||
|
|
||||||
|
### **GitHub contribution**:
|
||||||
|
Please add the **[SYCL]** prefix/tag in issues/PRs titles to help the SYCL-team check/address them without delay.
|
||||||
|
|
||||||
|
## TODO
|
||||||
|
|
||||||
|
- Support row layer split for multiple card runs.
|
||||||
67
SECURITY.md
Normal file
67
SECURITY.md
Normal file
|
|
@ -0,0 +1,67 @@
|
||||||
|
# Security Policy
|
||||||
|
|
||||||
|
- [**Using llama.cpp securely**](#using-llamacpp-securely)
|
||||||
|
- [Untrusted models](#untrusted-models)
|
||||||
|
- [Untrusted inputs](#untrusted-inputs)
|
||||||
|
- [Data privacy](#data-privacy)
|
||||||
|
- [Untrusted environments or networks](#untrusted-environments-or-networks)
|
||||||
|
- [Multi-Tenant environments](#multi-tenant-environments)
|
||||||
|
- [**Reporting a vulnerability**](#reporting-a-vulnerability)
|
||||||
|
|
||||||
|
## Using llama.cpp securely
|
||||||
|
|
||||||
|
### Untrusted models
|
||||||
|
Be careful when running untrusted models. This classification includes models created by unknown developers or utilizing data obtained from unknown sources.
|
||||||
|
|
||||||
|
*Always execute untrusted models within a secure, isolated environment such as a sandbox* (e.g., containers, virtual machines). This helps protect your system from potentially malicious code.
|
||||||
|
|
||||||
|
> [!NOTE]
|
||||||
|
> The trustworthiness of a model is not binary. You must always determine the proper level of caution depending on the specific model and how it matches your use case and risk tolerance.
|
||||||
|
|
||||||
|
### Untrusted inputs
|
||||||
|
|
||||||
|
Some models accept various input formats (text, images, audio, etc.). The libraries converting these inputs have varying security levels, so it's crucial to isolate the model and carefully pre-process inputs to mitigate script injection risks.
|
||||||
|
|
||||||
|
For maximum security when handling untrusted inputs, you may need to employ the following:
|
||||||
|
|
||||||
|
* Sandboxing: Isolate the environment where the inference happens.
|
||||||
|
* Pre-analysis: Check how the model performs by default when exposed to prompt injection (e.g. using [fuzzing for prompt injection](https://github.com/FonduAI/awesome-prompt-injection?tab=readme-ov-file#tools)). This will give you leads on how hard you will have to work on the next topics.
|
||||||
|
* Updates: Keep both LLaMA C++ and your libraries updated with the latest security patches.
|
||||||
|
* Input Sanitation: Before feeding data to the model, sanitize inputs rigorously. This involves techniques such as:
|
||||||
|
* Validation: Enforce strict rules on allowed characters and data types.
|
||||||
|
* Filtering: Remove potentially malicious scripts or code fragments.
|
||||||
|
* Encoding: Convert special characters into safe representations.
|
||||||
|
* Verification: Run tooling that identifies potential script injections (e.g. [models that detect prompt injection attempts](https://python.langchain.com/docs/guides/safety/hugging_face_prompt_injection)).
|
||||||
|
|
||||||
|
### Data privacy
|
||||||
|
|
||||||
|
To protect sensitive data from potential leaks or unauthorized access, it is crucial to sandbox the model execution. This means running the model in a secure, isolated environment, which helps mitigate many attack vectors.
|
||||||
|
|
||||||
|
### Untrusted environments or networks
|
||||||
|
|
||||||
|
If you can't run your models in a secure and isolated environment or if it must be exposed to an untrusted network, make sure to take the following security precautions:
|
||||||
|
* Confirm the hash of any downloaded artifact (e.g. pre-trained model weights) matches a known-good value
|
||||||
|
* Encrypt your data if sending it over the network.
|
||||||
|
|
||||||
|
### Multi-Tenant environments
|
||||||
|
|
||||||
|
If you intend to run multiple models in parallel with shared memory, it is your responsibility to ensure the models do not interact or access each other's data. The primary areas of concern are tenant isolation, resource allocation, model sharing and hardware attacks.
|
||||||
|
|
||||||
|
1. Tenant Isolation: Models should run separately with strong isolation methods to prevent unwanted data access. Separating networks is crucial for isolation, as it prevents unauthorized access to data or models and malicious users from sending graphs to execute under another tenant's identity.
|
||||||
|
|
||||||
|
2. Resource Allocation: A denial of service caused by one model can impact the overall system health. Implement safeguards like rate limits, access controls, and health monitoring.
|
||||||
|
|
||||||
|
3. Model Sharing: In a multitenant model sharing design, tenants and users must understand the security risks of running code provided by others. Since there are no reliable methods to detect malicious models, sandboxing the model execution is the recommended approach to mitigate the risk.
|
||||||
|
|
||||||
|
4. Hardware Attacks: GPUs or TPUs can also be attacked. [Researches](https://scholar.google.com/scholar?q=gpu+side+channel) has shown that side channel attacks on GPUs are possible, which can make data leak from other models or processes running on the same system at the same time.
|
||||||
|
|
||||||
|
## Reporting a vulnerability
|
||||||
|
|
||||||
|
Beware that none of the topics under [Using llama.cpp securely](#using-llamacpp-securely) are considered vulnerabilities of LLaMA C++.
|
||||||
|
|
||||||
|
<!-- normal version -->
|
||||||
|
However, If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released.
|
||||||
|
|
||||||
|
Please disclose it as a private [security advisory](https://github.com/ggerganov/llama.cpp/security/advisories/new).
|
||||||
|
|
||||||
|
A team of volunteers on a reasonable-effort basis maintains this project. As such, please give us at least 90 days to work on a fix before public exposure.
|
||||||
20
SHA256SUMS
20
SHA256SUMS
|
|
@ -1,20 +0,0 @@
|
||||||
700df0d3013b703a806d2ae7f1bfb8e59814e3d06ae78be0c66368a50059f33d models/7B/consolidated.00.pth
|
|
||||||
7e89e242ddc0dd6f060b43ca219ce8b3e8f08959a72cb3c0855df8bb04d46265 models/7B/params.json
|
|
||||||
745bf4e29a4dd6f411e72976d92b452da1b49168a4f41c951cfcc8051823cf08 models/13B/consolidated.00.pth
|
|
||||||
d5ccbcc465c71c0de439a5aeffebe8344c68a519bce70bc7f9f92654ee567085 models/13B/consolidated.01.pth
|
|
||||||
4ab77bec4d4405ccb66a97b282574c89a94417e3c32e5f68f37e2876fc21322f models/13B/params.json
|
|
||||||
e23294a58552d8cdec5b7e8abb87993b97ea6eced4178ff2697c02472539d067 models/30B/consolidated.00.pth
|
|
||||||
4e077b7136c7ae2302e954860cf64930458d3076fcde9443f4d0e939e95903ff models/30B/consolidated.01.pth
|
|
||||||
24a87f01028cbd3a12de551dcedb712346c0b5cbdeff1454e0ddf2df9b675378 models/30B/consolidated.02.pth
|
|
||||||
1adfcef71420886119544949767f6a56cb6339b4d5fcde755d80fe68b49de93b models/30B/consolidated.03.pth
|
|
||||||
2c07118ea98d69dbe7810d88520e30288fa994751b337f8fca02b171955f44cb models/30B/params.json
|
|
||||||
135c563f6b3938114458183afb01adc9a63bef3d8ff7cccc3977e5d3664ecafe models/65B/consolidated.00.pth
|
|
||||||
9a600b37b19d38c7e43809485f70d17d1dc12206c07efa83bc72bb498a568bde models/65B/consolidated.01.pth
|
|
||||||
e7babf7c5606f165a3756f527cb0fedc4f83e67ef1290391e52fb1cce5f26770 models/65B/consolidated.02.pth
|
|
||||||
73176ffb426b40482f2aa67ae1217ef79fbbd1fff5482bae5060cdc5a24ab70e models/65B/consolidated.03.pth
|
|
||||||
882e6431d0b08a8bc66261a0d3607da21cbaeafa96a24e7e59777632dbdac225 models/65B/consolidated.04.pth
|
|
||||||
a287c0dfe49081626567c7fe87f74cce5831f58e459b427b5e05567641f47b78 models/65B/consolidated.05.pth
|
|
||||||
72b4eba67a1a3b18cb67a85b70f8f1640caae9b40033ea943fb166bd80a7b36b models/65B/consolidated.06.pth
|
|
||||||
d27f5b0677d7ff129ceacd73fd461c4d06910ad7787cf217b249948c3f3bc638 models/65B/consolidated.07.pth
|
|
||||||
999ed1659b469ccc2a941714c0a9656fa571d17c9f7c8c7589817ca90edef51b models/65B/params.json
|
|
||||||
9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 models/tokenizer.model
|
|
||||||
219
build.zig
219
build.zig
|
|
@ -1,61 +1,172 @@
|
||||||
|
// Compatible with Zig Version 0.11.0
|
||||||
const std = @import("std");
|
const std = @import("std");
|
||||||
|
const ArrayList = std.ArrayList;
|
||||||
|
const Compile = std.Build.Step.Compile;
|
||||||
|
const ConfigHeader = std.Build.Step.ConfigHeader;
|
||||||
|
const Mode = std.builtin.Mode;
|
||||||
|
const CrossTarget = std.zig.CrossTarget;
|
||||||
|
|
||||||
pub fn build(b: *std.build.Builder) void {
|
const Maker = struct {
|
||||||
const target = b.standardTargetOptions(.{});
|
builder: *std.build.Builder,
|
||||||
const optimize = b.standardReleaseOptions();
|
target: CrossTarget,
|
||||||
const want_lto = b.option(bool, "lto", "Want -fLTO");
|
optimize: Mode,
|
||||||
|
enable_lto: bool,
|
||||||
|
|
||||||
const lib = b.addStaticLibrary("llama", null);
|
include_dirs: ArrayList([]const u8),
|
||||||
lib.want_lto = want_lto;
|
cflags: ArrayList([]const u8),
|
||||||
lib.setTarget(target);
|
cxxflags: ArrayList([]const u8),
|
||||||
lib.setBuildMode(optimize);
|
objs: ArrayList(*Compile),
|
||||||
lib.linkLibCpp();
|
|
||||||
lib.addIncludePath(".");
|
|
||||||
lib.addIncludePath("examples");
|
|
||||||
lib.addCSourceFiles(&.{
|
|
||||||
"ggml.c",
|
|
||||||
}, &.{"-std=c11"});
|
|
||||||
lib.addCSourceFiles(&.{
|
|
||||||
"llama.cpp",
|
|
||||||
}, &.{"-std=c++11"});
|
|
||||||
lib.install();
|
|
||||||
|
|
||||||
const build_args = .{ .b = b, .lib = lib, .target = target, .optimize = optimize, .want_lto = want_lto };
|
fn addInclude(m: *Maker, dir: []const u8) !void {
|
||||||
|
try m.include_dirs.append(dir);
|
||||||
const exe = build_example("main", build_args);
|
}
|
||||||
_ = build_example("quantize", build_args);
|
fn addProjectInclude(m: *Maker, path: []const []const u8) !void {
|
||||||
_ = build_example("perplexity", build_args);
|
try m.addInclude(try m.builder.build_root.join(m.builder.allocator, path));
|
||||||
_ = build_example("embedding", build_args);
|
}
|
||||||
|
fn addCFlag(m: *Maker, flag: []const u8) !void {
|
||||||
// create "zig build run" command for ./main
|
try m.cflags.append(flag);
|
||||||
|
}
|
||||||
const run_cmd = exe.run();
|
fn addCxxFlag(m: *Maker, flag: []const u8) !void {
|
||||||
run_cmd.step.dependOn(b.getInstallStep());
|
try m.cxxflags.append(flag);
|
||||||
if (b.args) |args| {
|
}
|
||||||
run_cmd.addArgs(args);
|
fn addFlag(m: *Maker, flag: []const u8) !void {
|
||||||
|
try m.addCFlag(flag);
|
||||||
|
try m.addCxxFlag(flag);
|
||||||
}
|
}
|
||||||
|
|
||||||
const run_step = b.step("run", "Run the app");
|
fn init(builder: *std.build.Builder) !Maker {
|
||||||
run_step.dependOn(&run_cmd.step);
|
const target = builder.standardTargetOptions(.{});
|
||||||
}
|
const zig_version = @import("builtin").zig_version_string;
|
||||||
|
const commit_hash = try std.ChildProcess.exec(
|
||||||
fn build_example(comptime name: []const u8, args: anytype) *std.build.LibExeObjStep {
|
.{ .allocator = builder.allocator, .argv = &.{ "git", "rev-parse", "HEAD" } },
|
||||||
const b = args.b;
|
);
|
||||||
const lib = args.lib;
|
try std.fs.cwd().writeFile("common/build-info.cpp", builder.fmt(
|
||||||
const want_lto = args.want_lto;
|
\\int LLAMA_BUILD_NUMBER = {};
|
||||||
|
\\char const *LLAMA_COMMIT = "{s}";
|
||||||
const exe = b.addExecutable(name, null);
|
\\char const *LLAMA_COMPILER = "Zig {s}";
|
||||||
exe.want_lto = want_lto;
|
\\char const *LLAMA_BUILD_TARGET = "{s}";
|
||||||
lib.setTarget(args.target);
|
\\
|
||||||
lib.setBuildMode(args.optimize);
|
, .{ 0, commit_hash.stdout[0 .. commit_hash.stdout.len - 1], zig_version, try target.allocDescription(builder.allocator) }));
|
||||||
exe.addIncludePath(".");
|
var m = Maker{
|
||||||
exe.addIncludePath("examples");
|
.builder = builder,
|
||||||
exe.addCSourceFiles(&.{
|
.target = target,
|
||||||
std.fmt.comptimePrint("examples/{s}/{s}.cpp", .{name, name}),
|
.optimize = builder.standardOptimizeOption(.{}),
|
||||||
"examples/common.cpp",
|
.enable_lto = false,
|
||||||
}, &.{"-std=c++11"});
|
.include_dirs = ArrayList([]const u8).init(builder.allocator),
|
||||||
exe.linkLibrary(lib);
|
.cflags = ArrayList([]const u8).init(builder.allocator),
|
||||||
exe.install();
|
.cxxflags = ArrayList([]const u8).init(builder.allocator),
|
||||||
|
.objs = ArrayList(*Compile).init(builder.allocator),
|
||||||
return exe;
|
};
|
||||||
|
|
||||||
|
try m.addCFlag("-std=c11");
|
||||||
|
try m.addCxxFlag("-std=c++11");
|
||||||
|
try m.addProjectInclude(&.{});
|
||||||
|
try m.addProjectInclude(&.{"common"});
|
||||||
|
return m;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn obj(m: *const Maker, name: []const u8, src: []const u8) *Compile {
|
||||||
|
const o = m.builder.addObject(.{ .name = name, .target = m.target, .optimize = m.optimize });
|
||||||
|
if (o.target.getAbi() != .msvc)
|
||||||
|
o.defineCMacro("_GNU_SOURCE", null);
|
||||||
|
|
||||||
|
if (std.mem.endsWith(u8, src, ".c")) {
|
||||||
|
o.addCSourceFiles(&.{src}, m.cflags.items);
|
||||||
|
o.linkLibC();
|
||||||
|
} else {
|
||||||
|
o.addCSourceFiles(&.{src}, m.cxxflags.items);
|
||||||
|
if (o.target.getAbi() == .msvc) {
|
||||||
|
o.linkLibC(); // need winsdk + crt
|
||||||
|
} else {
|
||||||
|
// linkLibCpp already add (libc++ + libunwind + libc)
|
||||||
|
o.linkLibCpp();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (m.include_dirs.items) |i| o.addIncludePath(.{ .path = i });
|
||||||
|
o.want_lto = m.enable_lto;
|
||||||
|
return o;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exe(m: *const Maker, name: []const u8, src: []const u8, deps: []const *Compile) *Compile {
|
||||||
|
const e = m.builder.addExecutable(.{ .name = name, .target = m.target, .optimize = m.optimize });
|
||||||
|
e.addCSourceFiles(&.{src}, m.cxxflags.items);
|
||||||
|
for (deps) |d| e.addObject(d);
|
||||||
|
for (m.objs.items) |o| e.addObject(o);
|
||||||
|
for (m.include_dirs.items) |i| e.addIncludePath(.{ .path = i });
|
||||||
|
|
||||||
|
// https://github.com/ziglang/zig/issues/15448
|
||||||
|
if (e.target.getAbi() == .msvc) {
|
||||||
|
e.linkLibC(); // need winsdk + crt
|
||||||
|
} else {
|
||||||
|
// linkLibCpp already add (libc++ + libunwind + libc)
|
||||||
|
e.linkLibCpp();
|
||||||
|
}
|
||||||
|
m.builder.installArtifact(e);
|
||||||
|
e.want_lto = m.enable_lto;
|
||||||
|
return e;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
pub fn build(b: *std.build.Builder) !void {
|
||||||
|
var make = try Maker.init(b);
|
||||||
|
make.enable_lto = b.option(bool, "lto", "Enable LTO optimization, (default: false)") orelse false;
|
||||||
|
|
||||||
|
const ggml = make.obj("ggml", "ggml.c");
|
||||||
|
const sgemm = make.obj("sgemm", "sgemm.cpp");
|
||||||
|
const ggml_alloc = make.obj("ggml-alloc", "ggml-alloc.c");
|
||||||
|
const ggml_backend = make.obj("ggml-backend", "ggml-backend.c");
|
||||||
|
const ggml_quants = make.obj("ggml-quants", "ggml-quants.c");
|
||||||
|
const unicode = make.obj("unicode", "unicode.cpp");
|
||||||
|
const unicode_data = make.obj("unicode-data", "unicode-data.cpp");
|
||||||
|
const llama = make.obj("llama", "llama.cpp");
|
||||||
|
const buildinfo = make.obj("common", "common/build-info.cpp");
|
||||||
|
const common = make.obj("common", "common/common.cpp");
|
||||||
|
const console = make.obj("console", "common/console.cpp");
|
||||||
|
const sampling = make.obj("sampling", "common/sampling.cpp");
|
||||||
|
const grammar_parser = make.obj("grammar-parser", "common/grammar-parser.cpp");
|
||||||
|
const json_schema_to_grammar = make.obj("json-schema-to-grammar", "common/json-schema-to-grammar.cpp");
|
||||||
|
const train = make.obj("train", "common/train.cpp");
|
||||||
|
const clip = make.obj("clip", "examples/llava/clip.cpp");
|
||||||
|
const llava = make.obj("llava", "examples/llava/llava.cpp");
|
||||||
|
|
||||||
|
_ = make.exe("main", "examples/main/main.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, json_schema_to_grammar, buildinfo, sampling, console, grammar_parser });
|
||||||
|
_ = make.exe("quantize", "examples/quantize/quantize.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, json_schema_to_grammar, buildinfo });
|
||||||
|
_ = make.exe("perplexity", "examples/perplexity/perplexity.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, json_schema_to_grammar, buildinfo });
|
||||||
|
_ = make.exe("embedding", "examples/embedding/embedding.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, json_schema_to_grammar, buildinfo });
|
||||||
|
_ = make.exe("finetune", "examples/finetune/finetune.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, json_schema_to_grammar, buildinfo, train });
|
||||||
|
_ = make.exe("train-text-from-scratch", "examples/train-text-from-scratch/train-text-from-scratch.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, json_schema_to_grammar, buildinfo, train });
|
||||||
|
|
||||||
|
const server = make.exe("server", "examples/server/server.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, json_schema_to_grammar, buildinfo, sampling, grammar_parser, clip, llava });
|
||||||
|
if (server.target.isWindows()) {
|
||||||
|
server.linkSystemLibrary("ws2_32");
|
||||||
|
}
|
||||||
|
|
||||||
|
const server_assets = [_][]const u8{ "index.html", "index.js", "completion.js", "json-schema-to-grammar.mjs" };
|
||||||
|
for (server_assets) |asset| {
|
||||||
|
const input_path = b.fmt("examples/server/public/{s}", .{asset});
|
||||||
|
const output_path = b.fmt("examples/server/{s}.hpp", .{asset});
|
||||||
|
|
||||||
|
// Portable equivalent of `b.addSystemCommand(&.{ "xxd", "-n", asset, "-i", input_path, output_path }) })`:
|
||||||
|
|
||||||
|
const input = try std.fs.cwd().readFileAlloc(b.allocator, input_path, std.math.maxInt(usize));
|
||||||
|
defer b.allocator.free(input);
|
||||||
|
|
||||||
|
var buf = std.ArrayList(u8).init(b.allocator);
|
||||||
|
defer buf.deinit();
|
||||||
|
|
||||||
|
for (input) |byte| {
|
||||||
|
try std.fmt.format(buf.writer(), "0x{X:0>2}, ", .{byte});
|
||||||
|
}
|
||||||
|
|
||||||
|
var name = try std.mem.replaceOwned(u8, b.allocator, asset, "-", "_");
|
||||||
|
defer b.allocator.free(name);
|
||||||
|
std.mem.replaceScalar(u8, name, '.', '_');
|
||||||
|
|
||||||
|
try std.fs.cwd().writeFile(output_path, b.fmt(
|
||||||
|
"unsigned char {s}[] = {{{s}}};\nunsigned int {s}_len = {d};\n",
|
||||||
|
.{ name, buf.items, name, input.len },
|
||||||
|
));
|
||||||
|
|
||||||
|
std.debug.print("Dumped hex of \"{s}\" ({s}) to {s}\n", .{ input_path, name, output_path });
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
29
ci/README.md
Normal file
29
ci/README.md
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
# CI
|
||||||
|
|
||||||
|
In addition to [Github Actions](https://github.com/ggerganov/llama.cpp/actions) `llama.cpp` uses a custom CI framework:
|
||||||
|
|
||||||
|
https://github.com/ggml-org/ci
|
||||||
|
|
||||||
|
It monitors the `master` branch for new commits and runs the
|
||||||
|
[ci/run.sh](https://github.com/ggerganov/llama.cpp/blob/master/ci/run.sh) script on dedicated cloud instances. This allows us
|
||||||
|
to execute heavier workloads compared to just using Github Actions. Also with time, the cloud instances will be scaled
|
||||||
|
to cover various hardware architectures, including GPU and Apple Silicon instances.
|
||||||
|
|
||||||
|
Collaborators can optionally trigger the CI run by adding the `ggml-ci` keyword to their commit message.
|
||||||
|
Only the branches of this repo are monitored for this keyword.
|
||||||
|
|
||||||
|
It is a good practice, before publishing changes to execute the full CI locally on your machine:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mkdir tmp
|
||||||
|
|
||||||
|
# CPU-only build
|
||||||
|
bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
||||||
|
|
||||||
|
# with CUDA support
|
||||||
|
GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
||||||
|
|
||||||
|
# with SYCL support
|
||||||
|
source /opt/intel/oneapi/setvars.sh
|
||||||
|
GG_BUILD_SYCL=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
||||||
|
```
|
||||||
713
ci/run.sh
Executable file
713
ci/run.sh
Executable file
|
|
@ -0,0 +1,713 @@
|
||||||
|
#/bin/bash
|
||||||
|
#
|
||||||
|
# sample usage:
|
||||||
|
#
|
||||||
|
# mkdir tmp
|
||||||
|
#
|
||||||
|
# # CPU-only build
|
||||||
|
# bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
||||||
|
#
|
||||||
|
# # with CUDA support
|
||||||
|
# GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
||||||
|
#
|
||||||
|
# # with SYCL support
|
||||||
|
# GG_BUILD_SYCL=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
||||||
|
#
|
||||||
|
|
||||||
|
if [ -z "$2" ]; then
|
||||||
|
echo "usage: $0 <output-dir> <mnt-dir>"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p "$1"
|
||||||
|
mkdir -p "$2"
|
||||||
|
|
||||||
|
OUT=$(realpath "$1")
|
||||||
|
MNT=$(realpath "$2")
|
||||||
|
|
||||||
|
rm -f "$OUT/*.log"
|
||||||
|
rm -f "$OUT/*.exit"
|
||||||
|
rm -f "$OUT/*.md"
|
||||||
|
|
||||||
|
sd=`dirname $0`
|
||||||
|
cd $sd/../
|
||||||
|
SRC=`pwd`
|
||||||
|
|
||||||
|
CMAKE_EXTRA="-DLLAMA_FATAL_WARNINGS=ON"
|
||||||
|
|
||||||
|
if [ ! -z ${GG_BUILD_METAL} ]; then
|
||||||
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DLLAMA_METAL_SHADER_DEBUG=ON"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -z ${GG_BUILD_CUDA} ]; then
|
||||||
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DLLAMA_CUDA=1"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -z ${GG_BUILD_SYCL} ]; then
|
||||||
|
if [ -z ${ONEAPI_ROOT} ]; then
|
||||||
|
echo "Not detected ONEAPI_ROOT, please install oneAPI base toolkit and enable it by:"
|
||||||
|
echo "source /opt/intel/oneapi/setvars.sh"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
CMAKE_EXTRA="${CMAKE_EXTRA} -DLLAMA_SYCL=1 DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_SYCL_F16=ON"
|
||||||
|
fi
|
||||||
|
## helpers
|
||||||
|
|
||||||
|
# download a file if it does not exist or if it is outdated
|
||||||
|
function gg_wget {
|
||||||
|
local out=$1
|
||||||
|
local url=$2
|
||||||
|
|
||||||
|
local cwd=`pwd`
|
||||||
|
|
||||||
|
mkdir -p $out
|
||||||
|
cd $out
|
||||||
|
|
||||||
|
# should not re-download if file is the same
|
||||||
|
wget -nv -N $url
|
||||||
|
|
||||||
|
cd $cwd
|
||||||
|
}
|
||||||
|
|
||||||
|
function gg_printf {
|
||||||
|
printf -- "$@" >> $OUT/README.md
|
||||||
|
}
|
||||||
|
|
||||||
|
function gg_run {
|
||||||
|
ci=$1
|
||||||
|
|
||||||
|
set -o pipefail
|
||||||
|
set -x
|
||||||
|
|
||||||
|
gg_run_$ci | tee $OUT/$ci.log
|
||||||
|
cur=$?
|
||||||
|
echo "$cur" > $OUT/$ci.exit
|
||||||
|
|
||||||
|
set +x
|
||||||
|
set +o pipefail
|
||||||
|
|
||||||
|
gg_sum_$ci
|
||||||
|
|
||||||
|
ret=$((ret | cur))
|
||||||
|
}
|
||||||
|
|
||||||
|
## ci
|
||||||
|
|
||||||
|
# ctest_debug
|
||||||
|
|
||||||
|
function gg_run_ctest_debug {
|
||||||
|
cd ${SRC}
|
||||||
|
|
||||||
|
rm -rf build-ci-debug && mkdir build-ci-debug && cd build-ci-debug
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
(time cmake -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
||||||
|
(time make -j ) 2>&1 | tee -a $OUT/${ci}-make.log
|
||||||
|
|
||||||
|
(time ctest --output-on-failure -L main -E test-opt ) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
||||||
|
|
||||||
|
set +e
|
||||||
|
}
|
||||||
|
|
||||||
|
function gg_sum_ctest_debug {
|
||||||
|
gg_printf '### %s\n\n' "${ci}"
|
||||||
|
|
||||||
|
gg_printf 'Runs ctest in debug mode\n'
|
||||||
|
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||||
|
gg_printf '```\n'
|
||||||
|
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
|
||||||
|
gg_printf '```\n'
|
||||||
|
gg_printf '\n'
|
||||||
|
}
|
||||||
|
|
||||||
|
# ctest_release
|
||||||
|
|
||||||
|
function gg_run_ctest_release {
|
||||||
|
cd ${SRC}
|
||||||
|
|
||||||
|
rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
(time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
||||||
|
(time make -j ) 2>&1 | tee -a $OUT/${ci}-make.log
|
||||||
|
|
||||||
|
if [ -z ${GG_BUILD_LOW_PERF} ]; then
|
||||||
|
(time ctest --output-on-failure -L main ) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
||||||
|
else
|
||||||
|
(time ctest --output-on-failure -L main -E test-opt ) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
||||||
|
fi
|
||||||
|
|
||||||
|
set +e
|
||||||
|
}
|
||||||
|
|
||||||
|
function gg_sum_ctest_release {
|
||||||
|
gg_printf '### %s\n\n' "${ci}"
|
||||||
|
|
||||||
|
gg_printf 'Runs ctest in release mode\n'
|
||||||
|
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||||
|
gg_printf '```\n'
|
||||||
|
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
|
||||||
|
gg_printf '```\n'
|
||||||
|
}
|
||||||
|
|
||||||
|
# test_scripts_debug
|
||||||
|
|
||||||
|
function gg_run_test_scripts_debug {
|
||||||
|
cd ${SRC}
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
(cd ./examples/gguf-split && time bash tests.sh "$SRC/build-ci-debug/bin" "$MNT/models") 2>&1 | tee -a $OUT/${ci}-scripts.log
|
||||||
|
(cd ./examples/quantize && time bash tests.sh "$SRC/build-ci-debug/bin" "$MNT/models") 2>&1 | tee -a $OUT/${ci}-scripts.log
|
||||||
|
|
||||||
|
set +e
|
||||||
|
}
|
||||||
|
|
||||||
|
function gg_sum_test_scripts_debug {
|
||||||
|
gg_printf '### %s\n\n' "${ci}"
|
||||||
|
|
||||||
|
gg_printf 'Runs test scripts in debug mode\n'
|
||||||
|
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||||
|
gg_printf '```\n'
|
||||||
|
gg_printf '%s\n' "$(cat $OUT/${ci}-scripts.log)"
|
||||||
|
gg_printf '```\n'
|
||||||
|
gg_printf '\n'
|
||||||
|
}
|
||||||
|
|
||||||
|
# test_scripts_release
|
||||||
|
|
||||||
|
function gg_run_test_scripts_release {
|
||||||
|
cd ${SRC}
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
(cd ./examples/gguf-split && time bash tests.sh "$SRC/build-ci-release/bin" "$MNT/models") 2>&1 | tee -a $OUT/${ci}-scripts.log
|
||||||
|
(cd ./examples/quantize && time bash tests.sh "$SRC/build-ci-release/bin" "$MNT/models") 2>&1 | tee -a $OUT/${ci}-scripts.log
|
||||||
|
|
||||||
|
set +e
|
||||||
|
}
|
||||||
|
|
||||||
|
function gg_sum_test_scripts_release {
|
||||||
|
gg_printf '### %s\n\n' "${ci}"
|
||||||
|
|
||||||
|
gg_printf 'Runs test scripts in release mode\n'
|
||||||
|
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||||
|
gg_printf '```\n'
|
||||||
|
gg_printf '%s\n' "$(cat $OUT/${ci}-scripts.log)"
|
||||||
|
gg_printf '```\n'
|
||||||
|
gg_printf '\n'
|
||||||
|
}
|
||||||
|
|
||||||
|
function gg_get_model {
|
||||||
|
local gguf_3b="$MNT/models/open-llama/3B-v2/ggml-model-f16.gguf"
|
||||||
|
local gguf_7b="$MNT/models/open-llama/7B-v2/ggml-model-f16.gguf"
|
||||||
|
if [[ -s $gguf_3b ]]; then
|
||||||
|
echo -n "$gguf_3b"
|
||||||
|
elif [[ -s $gguf_7b ]]; then
|
||||||
|
echo -n "$gguf_7b"
|
||||||
|
else
|
||||||
|
echo >&2 "No model found. Can't run gg_run_ctest_with_model."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function gg_run_ctest_with_model_debug {
|
||||||
|
cd ${SRC}
|
||||||
|
|
||||||
|
local model; model=$(gg_get_model)
|
||||||
|
cd build-ci-debug
|
||||||
|
set -e
|
||||||
|
(LLAMACPP_TEST_MODELFILE="$model" time ctest --output-on-failure -L model) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
||||||
|
set +e
|
||||||
|
cd ..
|
||||||
|
}
|
||||||
|
|
||||||
|
function gg_run_ctest_with_model_release {
|
||||||
|
cd ${SRC}
|
||||||
|
|
||||||
|
local model; model=$(gg_get_model)
|
||||||
|
cd build-ci-release
|
||||||
|
set -e
|
||||||
|
(LLAMACPP_TEST_MODELFILE="$model" time ctest --output-on-failure -L model) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
||||||
|
set +e
|
||||||
|
cd ..
|
||||||
|
}
|
||||||
|
|
||||||
|
function gg_sum_ctest_with_model_debug {
|
||||||
|
gg_printf '### %s\n\n' "${ci}"
|
||||||
|
|
||||||
|
gg_printf 'Runs ctest with model files in debug mode\n'
|
||||||
|
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||||
|
gg_printf '```\n'
|
||||||
|
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
|
||||||
|
gg_printf '```\n'
|
||||||
|
}
|
||||||
|
|
||||||
|
function gg_sum_ctest_with_model_release {
|
||||||
|
gg_printf '### %s\n\n' "${ci}"
|
||||||
|
|
||||||
|
gg_printf 'Runs ctest with model files in release mode\n'
|
||||||
|
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||||
|
gg_printf '```\n'
|
||||||
|
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
|
||||||
|
gg_printf '```\n'
|
||||||
|
}
|
||||||
|
|
||||||
|
# open_llama_3b_v2
|
||||||
|
|
||||||
|
function gg_run_open_llama_3b_v2 {
|
||||||
|
cd ${SRC}
|
||||||
|
|
||||||
|
gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/config.json
|
||||||
|
gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/resolve/main/tokenizer.model
|
||||||
|
gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/tokenizer_config.json
|
||||||
|
gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/special_tokens_map.json
|
||||||
|
gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/resolve/main/pytorch_model.bin
|
||||||
|
gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/generation_config.json
|
||||||
|
|
||||||
|
gg_wget models-mnt/wikitext/ https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip
|
||||||
|
unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d models-mnt/wikitext/
|
||||||
|
head -n 60 models-mnt/wikitext/wikitext-2-raw/wiki.test.raw > models-mnt/wikitext/wikitext-2-raw/wiki.test-60.raw
|
||||||
|
|
||||||
|
path_models="../models-mnt/open-llama/3B-v2"
|
||||||
|
path_wiki="../models-mnt/wikitext/wikitext-2-raw"
|
||||||
|
|
||||||
|
rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
(time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} -DLLAMA_QKK_64=1 .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
||||||
|
(time make -j ) 2>&1 | tee -a $OUT/${ci}-make.log
|
||||||
|
|
||||||
|
python3 ../convert.py ${path_models}
|
||||||
|
|
||||||
|
model_f16="${path_models}/ggml-model-f16.gguf"
|
||||||
|
model_q8_0="${path_models}/ggml-model-q8_0.gguf"
|
||||||
|
model_q4_0="${path_models}/ggml-model-q4_0.gguf"
|
||||||
|
model_q4_1="${path_models}/ggml-model-q4_1.gguf"
|
||||||
|
model_q5_0="${path_models}/ggml-model-q5_0.gguf"
|
||||||
|
model_q5_1="${path_models}/ggml-model-q5_1.gguf"
|
||||||
|
model_q2_k="${path_models}/ggml-model-q2_k.gguf"
|
||||||
|
model_q3_k="${path_models}/ggml-model-q3_k.gguf"
|
||||||
|
model_q4_k="${path_models}/ggml-model-q4_k.gguf"
|
||||||
|
model_q5_k="${path_models}/ggml-model-q5_k.gguf"
|
||||||
|
model_q6_k="${path_models}/ggml-model-q6_k.gguf"
|
||||||
|
|
||||||
|
wiki_test_60="${path_wiki}/wiki.test-60.raw"
|
||||||
|
|
||||||
|
./bin/quantize ${model_f16} ${model_q8_0} q8_0
|
||||||
|
./bin/quantize ${model_f16} ${model_q4_0} q4_0
|
||||||
|
./bin/quantize ${model_f16} ${model_q4_1} q4_1
|
||||||
|
./bin/quantize ${model_f16} ${model_q5_0} q5_0
|
||||||
|
./bin/quantize ${model_f16} ${model_q5_1} q5_1
|
||||||
|
./bin/quantize ${model_f16} ${model_q2_k} q2_k
|
||||||
|
./bin/quantize ${model_f16} ${model_q3_k} q3_k
|
||||||
|
./bin/quantize ${model_f16} ${model_q4_k} q4_k
|
||||||
|
./bin/quantize ${model_f16} ${model_q5_k} q5_k
|
||||||
|
./bin/quantize ${model_f16} ${model_q6_k} q6_k
|
||||||
|
|
||||||
|
(time ./bin/main --model ${model_f16} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
|
||||||
|
(time ./bin/main --model ${model_q8_0} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
|
||||||
|
(time ./bin/main --model ${model_q4_0} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
|
||||||
|
(time ./bin/main --model ${model_q4_1} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
|
||||||
|
(time ./bin/main --model ${model_q5_0} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
|
||||||
|
(time ./bin/main --model ${model_q5_1} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
|
||||||
|
(time ./bin/main --model ${model_q2_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
|
||||||
|
(time ./bin/main --model ${model_q3_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
|
||||||
|
(time ./bin/main --model ${model_q4_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
|
||||||
|
(time ./bin/main --model ${model_q5_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
|
||||||
|
(time ./bin/main --model ${model_q6_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
|
||||||
|
|
||||||
|
(time ./bin/perplexity --model ${model_f16} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
|
||||||
|
(time ./bin/perplexity --model ${model_q8_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
|
||||||
|
(time ./bin/perplexity --model ${model_q4_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
|
||||||
|
(time ./bin/perplexity --model ${model_q4_1} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
|
||||||
|
(time ./bin/perplexity --model ${model_q5_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
|
||||||
|
(time ./bin/perplexity --model ${model_q5_1} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
|
||||||
|
(time ./bin/perplexity --model ${model_q2_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
|
||||||
|
(time ./bin/perplexity --model ${model_q3_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
|
||||||
|
(time ./bin/perplexity --model ${model_q4_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
|
||||||
|
(time ./bin/perplexity --model ${model_q5_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
|
||||||
|
(time ./bin/perplexity --model ${model_q6_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
|
||||||
|
|
||||||
|
(time ./bin/imatrix --model ${model_f16} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log
|
||||||
|
|
||||||
|
(time ./bin/save-load-state --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
|
||||||
|
(time ./bin/save-load-state -fa --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
|
||||||
|
|
||||||
|
function check_ppl {
|
||||||
|
qnt="$1"
|
||||||
|
ppl=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
|
||||||
|
|
||||||
|
if [ $(echo "$ppl > 20.0" | bc) -eq 1 ]; then
|
||||||
|
printf ' - %s @ %s (FAIL: ppl > 20.0)\n' "$qnt" "$ppl"
|
||||||
|
return 20
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf ' - %s @ %s OK\n' "$qnt" "$ppl"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
check_ppl "f16" "$(cat $OUT/${ci}-tg-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
|
||||||
|
cat $OUT/${ci}-imatrix.log | grep "Final" >> $OUT/${ci}-imatrix-sum.log
|
||||||
|
|
||||||
|
# lora
|
||||||
|
function compare_ppl {
|
||||||
|
qnt="$1"
|
||||||
|
ppl1=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
|
||||||
|
ppl2=$(echo "$3" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
|
||||||
|
|
||||||
|
if [ $(echo "$ppl1 < $ppl2" | bc) -eq 1 ]; then
|
||||||
|
printf ' - %s @ %s (FAIL: %s > %s)\n' "$qnt" "$ppl" "$ppl1" "$ppl2"
|
||||||
|
return 20
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf ' - %s @ %s %s OK\n' "$qnt" "$ppl1" "$ppl2"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
path_lora="../models-mnt/open-llama/3B-v2/lora"
|
||||||
|
path_shakespeare="../models-mnt/shakespeare"
|
||||||
|
|
||||||
|
shakespeare="${path_shakespeare}/shakespeare.txt"
|
||||||
|
lora_shakespeare="${path_lora}/ggml-adapter-model.bin"
|
||||||
|
|
||||||
|
gg_wget ${path_lora} https://huggingface.co/slaren/open_llama_3b_v2_shakespeare_lora/resolve/main/adapter_config.json
|
||||||
|
gg_wget ${path_lora} https://huggingface.co/slaren/open_llama_3b_v2_shakespeare_lora/resolve/main/adapter_model.bin
|
||||||
|
gg_wget ${path_shakespeare} https://huggingface.co/slaren/open_llama_3b_v2_shakespeare_lora/resolve/main/shakespeare.txt
|
||||||
|
|
||||||
|
python3 ../convert-lora-to-ggml.py ${path_lora}
|
||||||
|
|
||||||
|
# f16
|
||||||
|
(time ./bin/perplexity --model ${model_f16} -f ${shakespeare} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-f16.log
|
||||||
|
(time ./bin/perplexity --model ${model_f16} -f ${shakespeare} --lora ${lora_shakespeare} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-f16.log
|
||||||
|
compare_ppl "f16 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-f16.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
|
||||||
|
|
||||||
|
# q8_0
|
||||||
|
(time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-q8_0.log
|
||||||
|
(time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0.log
|
||||||
|
compare_ppl "q8_0 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
|
||||||
|
|
||||||
|
# q8_0 + f16 lora-base
|
||||||
|
(time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} --lora-base ${model_f16} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log
|
||||||
|
compare_ppl "q8_0 / f16 base shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
|
||||||
|
|
||||||
|
set +e
|
||||||
|
}
|
||||||
|
|
||||||
|
function gg_sum_open_llama_3b_v2 {
|
||||||
|
gg_printf '### %s\n\n' "${ci}"
|
||||||
|
|
||||||
|
gg_printf 'OpenLLaMA 3B-v2:\n'
|
||||||
|
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||||
|
gg_printf '- perplexity:\n%s\n' "$(cat $OUT/${ci}-ppl.log)"
|
||||||
|
gg_printf '- imatrix:\n```\n%s\n```\n' "$(cat $OUT/${ci}-imatrix-sum.log)"
|
||||||
|
gg_printf '- lora:\n%s\n' "$(cat $OUT/${ci}-lora-ppl.log)"
|
||||||
|
gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)"
|
||||||
|
gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)"
|
||||||
|
gg_printf '- q4_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_0.log)"
|
||||||
|
gg_printf '- q4_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_1.log)"
|
||||||
|
gg_printf '- q5_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_0.log)"
|
||||||
|
gg_printf '- q5_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_1.log)"
|
||||||
|
gg_printf '- q2_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q2_k.log)"
|
||||||
|
gg_printf '- q3_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q3_k.log)"
|
||||||
|
gg_printf '- q4_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_k.log)"
|
||||||
|
gg_printf '- q5_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_k.log)"
|
||||||
|
gg_printf '- q6_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q6_k.log)"
|
||||||
|
gg_printf '- save-load-state: \n```\n%s\n```\n' "$(cat $OUT/${ci}-save-load-state.log)"
|
||||||
|
gg_printf '- shakespeare (f16):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-f16.log)"
|
||||||
|
gg_printf '- shakespeare (f16 lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-f16.log)"
|
||||||
|
gg_printf '- shakespeare (q8_0):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log)"
|
||||||
|
gg_printf '- shakespeare (q8_0 lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0.log)"
|
||||||
|
gg_printf '- shakespeare (q8_0 / f16 base lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log)"
|
||||||
|
}
|
||||||
|
|
||||||
|
# open_llama_7b_v2
|
||||||
|
# requires: GG_BUILD_CUDA
|
||||||
|
|
||||||
|
function gg_run_open_llama_7b_v2 {
|
||||||
|
cd ${SRC}
|
||||||
|
|
||||||
|
gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/config.json
|
||||||
|
gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/resolve/main/tokenizer.model
|
||||||
|
gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/tokenizer_config.json
|
||||||
|
gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/special_tokens_map.json
|
||||||
|
gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/pytorch_model.bin.index.json
|
||||||
|
gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/resolve/main/pytorch_model-00001-of-00002.bin
|
||||||
|
gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/resolve/main/pytorch_model-00002-of-00002.bin
|
||||||
|
gg_wget models-mnt/open-llama/7B-v2/ https://huggingface.co/openlm-research/open_llama_7b_v2/raw/main/generation_config.json
|
||||||
|
|
||||||
|
gg_wget models-mnt/wikitext/ https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip
|
||||||
|
unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d models-mnt/wikitext/
|
||||||
|
|
||||||
|
path_models="../models-mnt/open-llama/7B-v2"
|
||||||
|
path_wiki="../models-mnt/wikitext/wikitext-2-raw"
|
||||||
|
|
||||||
|
rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
(time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} -DLLAMA_CUDA=1 .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
||||||
|
(time make -j ) 2>&1 | tee -a $OUT/${ci}-make.log
|
||||||
|
|
||||||
|
python3 ../convert.py ${path_models}
|
||||||
|
|
||||||
|
model_f16="${path_models}/ggml-model-f16.gguf"
|
||||||
|
model_q8_0="${path_models}/ggml-model-q8_0.gguf"
|
||||||
|
model_q4_0="${path_models}/ggml-model-q4_0.gguf"
|
||||||
|
model_q4_1="${path_models}/ggml-model-q4_1.gguf"
|
||||||
|
model_q5_0="${path_models}/ggml-model-q5_0.gguf"
|
||||||
|
model_q5_1="${path_models}/ggml-model-q5_1.gguf"
|
||||||
|
model_q2_k="${path_models}/ggml-model-q2_k.gguf"
|
||||||
|
model_q3_k="${path_models}/ggml-model-q3_k.gguf"
|
||||||
|
model_q4_k="${path_models}/ggml-model-q4_k.gguf"
|
||||||
|
model_q5_k="${path_models}/ggml-model-q5_k.gguf"
|
||||||
|
model_q6_k="${path_models}/ggml-model-q6_k.gguf"
|
||||||
|
|
||||||
|
wiki_test="${path_wiki}/wiki.test.raw"
|
||||||
|
|
||||||
|
./bin/quantize ${model_f16} ${model_q8_0} q8_0
|
||||||
|
./bin/quantize ${model_f16} ${model_q4_0} q4_0
|
||||||
|
./bin/quantize ${model_f16} ${model_q4_1} q4_1
|
||||||
|
./bin/quantize ${model_f16} ${model_q5_0} q5_0
|
||||||
|
./bin/quantize ${model_f16} ${model_q5_1} q5_1
|
||||||
|
./bin/quantize ${model_f16} ${model_q2_k} q2_k
|
||||||
|
./bin/quantize ${model_f16} ${model_q3_k} q3_k
|
||||||
|
./bin/quantize ${model_f16} ${model_q4_k} q4_k
|
||||||
|
./bin/quantize ${model_f16} ${model_q5_k} q5_k
|
||||||
|
./bin/quantize ${model_f16} ${model_q6_k} q6_k
|
||||||
|
|
||||||
|
(time ./bin/main --model ${model_f16} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
|
||||||
|
(time ./bin/main --model ${model_q8_0} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
|
||||||
|
(time ./bin/main --model ${model_q4_0} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
|
||||||
|
(time ./bin/main --model ${model_q4_1} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
|
||||||
|
(time ./bin/main --model ${model_q5_0} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
|
||||||
|
(time ./bin/main --model ${model_q5_1} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
|
||||||
|
(time ./bin/main --model ${model_q2_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
|
||||||
|
(time ./bin/main --model ${model_q3_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
|
||||||
|
(time ./bin/main --model ${model_q4_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
|
||||||
|
(time ./bin/main --model ${model_q5_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
|
||||||
|
(time ./bin/main --model ${model_q6_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
|
||||||
|
|
||||||
|
(time ./bin/perplexity --model ${model_f16} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
|
||||||
|
(time ./bin/perplexity --model ${model_q8_0} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
|
||||||
|
(time ./bin/perplexity --model ${model_q4_0} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log
|
||||||
|
(time ./bin/perplexity --model ${model_q4_1} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log
|
||||||
|
(time ./bin/perplexity --model ${model_q5_0} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log
|
||||||
|
(time ./bin/perplexity --model ${model_q5_1} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log
|
||||||
|
(time ./bin/perplexity --model ${model_q2_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log
|
||||||
|
(time ./bin/perplexity --model ${model_q3_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log
|
||||||
|
(time ./bin/perplexity --model ${model_q4_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log
|
||||||
|
(time ./bin/perplexity --model ${model_q5_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log
|
||||||
|
(time ./bin/perplexity --model ${model_q6_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log
|
||||||
|
|
||||||
|
(time ./bin/imatrix --model ${model_f16} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log
|
||||||
|
|
||||||
|
(time ./bin/save-load-state -ngl 10 --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
|
||||||
|
(time ./bin/save-load-state -fa -ngl 10 --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
|
||||||
|
(time ./bin/save-load-state -ngl 99 --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
|
||||||
|
(time ./bin/save-load-state -fa -ngl 99 --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
|
||||||
|
|
||||||
|
function check_ppl {
|
||||||
|
qnt="$1"
|
||||||
|
ppl=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
|
||||||
|
|
||||||
|
if [ $(echo "$ppl > 20.0" | bc) -eq 1 ]; then
|
||||||
|
printf ' - %s @ %s (FAIL: ppl > 20.0)\n' "$qnt" "$ppl"
|
||||||
|
return 20
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf ' - %s @ %s OK\n' "$qnt" "$ppl"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
check_ppl "f16" "$(cat $OUT/${ci}-tg-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log
|
||||||
|
|
||||||
|
cat $OUT/${ci}-imatrix.log | grep "Final" >> $OUT/${ci}-imatrix-sum.log
|
||||||
|
|
||||||
|
# lora
|
||||||
|
function compare_ppl {
|
||||||
|
qnt="$1"
|
||||||
|
ppl1=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
|
||||||
|
ppl2=$(echo "$3" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1)
|
||||||
|
|
||||||
|
if [ $(echo "$ppl1 < $ppl2" | bc) -eq 1 ]; then
|
||||||
|
printf ' - %s @ %s (FAIL: %s > %s)\n' "$qnt" "$ppl" "$ppl1" "$ppl2"
|
||||||
|
return 20
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf ' - %s @ %s %s OK\n' "$qnt" "$ppl1" "$ppl2"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
path_lora="../models-mnt/open-llama/7B-v2/lora"
|
||||||
|
path_shakespeare="../models-mnt/shakespeare"
|
||||||
|
|
||||||
|
shakespeare="${path_shakespeare}/shakespeare.txt"
|
||||||
|
lora_shakespeare="${path_lora}/ggml-adapter-model.bin"
|
||||||
|
|
||||||
|
gg_wget ${path_lora} https://huggingface.co/slaren/open_llama_7b_v2_shakespeare_lora/resolve/main/adapter_config.json
|
||||||
|
gg_wget ${path_lora} https://huggingface.co/slaren/open_llama_7b_v2_shakespeare_lora/resolve/main/adapter_model.bin
|
||||||
|
gg_wget ${path_shakespeare} https://huggingface.co/slaren/open_llama_7b_v2_shakespeare_lora/resolve/main/shakespeare.txt
|
||||||
|
|
||||||
|
python3 ../convert-lora-to-ggml.py ${path_lora}
|
||||||
|
|
||||||
|
# f16
|
||||||
|
(time ./bin/perplexity --model ${model_f16} -f ${shakespeare} -t 1 -ngl 999 -c 2048 -b 512 --chunks 3 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-f16.log
|
||||||
|
(time ./bin/perplexity --model ${model_f16} -f ${shakespeare} --lora ${lora_shakespeare} -t 1 -ngl 999 -c 2048 -b 512 --chunks 3 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-f16.log
|
||||||
|
compare_ppl "f16 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-f16.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
|
||||||
|
|
||||||
|
# currently not supported by the CUDA backend
|
||||||
|
# q8_0
|
||||||
|
#(time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} -t 1 -ngl 999 -c 2048 -b 512 --chunks 3 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-q8_0.log
|
||||||
|
#(time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} -t 1 -ngl 999 -c 2048 -b 512 --chunks 3 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0.log
|
||||||
|
#compare_ppl "q8_0 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
|
||||||
|
|
||||||
|
# q8_0 + f16 lora-base
|
||||||
|
#(time ./bin/perplexity --model ${model_q8_0} -f ${shakespeare} --lora ${lora_shakespeare} --lora-base ${model_f16} -t 1 -ngl 999 -c 2048 -b 512 --chunks 3 ) 2>&1 | tee -a $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log
|
||||||
|
#compare_ppl "q8_0 / f16 shakespeare" "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log | grep "^\[1\]")" "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-lora-ppl.log
|
||||||
|
|
||||||
|
set +e
|
||||||
|
}
|
||||||
|
|
||||||
|
function gg_sum_open_llama_7b_v2 {
|
||||||
|
gg_printf '### %s\n\n' "${ci}"
|
||||||
|
|
||||||
|
gg_printf 'OpenLLaMA 7B-v2:\n'
|
||||||
|
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||||
|
gg_printf '- perplexity:\n%s\n' "$(cat $OUT/${ci}-ppl.log)"
|
||||||
|
gg_printf '- imatrix:\n```\n%s\n```\n' "$(cat $OUT/${ci}-imatrix-sum.log)"
|
||||||
|
gg_printf '- lora:\n%s\n' "$(cat $OUT/${ci}-lora-ppl.log)"
|
||||||
|
gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)"
|
||||||
|
gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)"
|
||||||
|
gg_printf '- q4_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_0.log)"
|
||||||
|
gg_printf '- q4_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_1.log)"
|
||||||
|
gg_printf '- q5_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_0.log)"
|
||||||
|
gg_printf '- q5_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_1.log)"
|
||||||
|
gg_printf '- q2_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q2_k.log)"
|
||||||
|
gg_printf '- q3_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q3_k.log)"
|
||||||
|
gg_printf '- q4_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_k.log)"
|
||||||
|
gg_printf '- q5_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_k.log)"
|
||||||
|
gg_printf '- q6_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q6_k.log)"
|
||||||
|
gg_printf '- save-load-state: \n```\n%s\n```\n' "$(cat $OUT/${ci}-save-load-state.log)"
|
||||||
|
gg_printf '- shakespeare (f16):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-f16.log)"
|
||||||
|
gg_printf '- shakespeare (f16 lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-f16.log)"
|
||||||
|
#gg_printf '- shakespeare (q8_0):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-q8_0.log)"
|
||||||
|
#gg_printf '- shakespeare (q8_0 lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0.log)"
|
||||||
|
#gg_printf '- shakespeare (q8_0 / f16 base lora):\n```\n%s\n```\n' "$(cat $OUT/${ci}-ppl-shakespeare-lora-q8_0-f16.log)"
|
||||||
|
}
|
||||||
|
|
||||||
|
# bge-small
|
||||||
|
|
||||||
|
function gg_run_embd_bge_small {
|
||||||
|
cd ${SRC}
|
||||||
|
|
||||||
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/config.json
|
||||||
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/tokenizer.json
|
||||||
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/tokenizer_config.json
|
||||||
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/special_tokens_map.json
|
||||||
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/resolve/main/pytorch_model.bin
|
||||||
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/sentence_bert_config.json
|
||||||
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/vocab.txt
|
||||||
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/modules.json
|
||||||
|
gg_wget models-mnt/bge-small/ https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/config.json
|
||||||
|
|
||||||
|
gg_wget models-mnt/bge-small/1_Pooling https://huggingface.co/BAAI/bge-small-en-v1.5/raw/main/1_Pooling/config.json
|
||||||
|
|
||||||
|
path_models="../models-mnt/bge-small"
|
||||||
|
|
||||||
|
rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
(time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
||||||
|
(time make -j ) 2>&1 | tee -a $OUT/${ci}-make.log
|
||||||
|
|
||||||
|
python3 ../convert-hf-to-gguf.py ${path_models}
|
||||||
|
|
||||||
|
model_f16="${path_models}/ggml-model-f16.gguf"
|
||||||
|
model_q8_0="${path_models}/ggml-model-q8_0.gguf"
|
||||||
|
|
||||||
|
./bin/quantize ${model_f16} ${model_q8_0} q8_0
|
||||||
|
|
||||||
|
(time ./bin/embedding --model ${model_f16} -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log
|
||||||
|
(time ./bin/embedding --model ${model_q8_0} -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log
|
||||||
|
|
||||||
|
set +e
|
||||||
|
}
|
||||||
|
|
||||||
|
function gg_sum_embd_bge_small {
|
||||||
|
gg_printf '### %s\n\n' "${ci}"
|
||||||
|
|
||||||
|
gg_printf 'BGE Small (BERT):\n'
|
||||||
|
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||||
|
gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)"
|
||||||
|
gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)"
|
||||||
|
}
|
||||||
|
|
||||||
|
## main
|
||||||
|
|
||||||
|
if [ -z ${GG_BUILD_LOW_PERF} ]; then
|
||||||
|
# Create symlink: ./llama.cpp/models-mnt -> $MNT/models/models-mnt
|
||||||
|
rm -rf ${SRC}/models-mnt
|
||||||
|
mnt_models=${MNT}/models
|
||||||
|
mkdir -p ${mnt_models}
|
||||||
|
ln -sfn ${mnt_models} ${SRC}/models-mnt
|
||||||
|
|
||||||
|
# Create a fresh python3 venv and enter it
|
||||||
|
python3 -m venv "$MNT/venv"
|
||||||
|
source "$MNT/venv/bin/activate"
|
||||||
|
|
||||||
|
pip install -r ${SRC}/requirements.txt --disable-pip-version-check
|
||||||
|
pip install --editable gguf-py --disable-pip-version-check
|
||||||
|
fi
|
||||||
|
|
||||||
|
ret=0
|
||||||
|
|
||||||
|
test $ret -eq 0 && gg_run ctest_debug
|
||||||
|
test $ret -eq 0 && gg_run ctest_release
|
||||||
|
|
||||||
|
if [ -z ${GG_BUILD_LOW_PERF} ]; then
|
||||||
|
test $ret -eq 0 && gg_run embd_bge_small
|
||||||
|
|
||||||
|
if [ -z ${GG_BUILD_CLOUD} ] || [ ${GG_BUILD_EXTRA_TESTS_0} ]; then
|
||||||
|
test $ret -eq 0 && gg_run test_scripts_debug
|
||||||
|
test $ret -eq 0 && gg_run test_scripts_release
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z ${GG_BUILD_VRAM_GB} ] || [ ${GG_BUILD_VRAM_GB} -ge 8 ]; then
|
||||||
|
if [ -z ${GG_BUILD_CUDA} ]; then
|
||||||
|
test $ret -eq 0 && gg_run open_llama_3b_v2
|
||||||
|
else
|
||||||
|
test $ret -eq 0 && gg_run open_llama_7b_v2
|
||||||
|
fi
|
||||||
|
test $ret -eq 0 && gg_run ctest_with_model_debug
|
||||||
|
test $ret -eq 0 && gg_run ctest_with_model_release
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
exit $ret
|
||||||
100
cmake/FindSIMD.cmake
Normal file
100
cmake/FindSIMD.cmake
Normal file
|
|
@ -0,0 +1,100 @@
|
||||||
|
include(CheckCSourceRuns)
|
||||||
|
|
||||||
|
set(AVX_CODE "
|
||||||
|
#include <immintrin.h>
|
||||||
|
int main()
|
||||||
|
{
|
||||||
|
__m256 a;
|
||||||
|
a = _mm256_set1_ps(0);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
")
|
||||||
|
|
||||||
|
set(AVX512_CODE "
|
||||||
|
#include <immintrin.h>
|
||||||
|
int main()
|
||||||
|
{
|
||||||
|
__m512i a = _mm512_set_epi8(0, 0, 0, 0, 0, 0, 0, 0,
|
||||||
|
0, 0, 0, 0, 0, 0, 0, 0,
|
||||||
|
0, 0, 0, 0, 0, 0, 0, 0,
|
||||||
|
0, 0, 0, 0, 0, 0, 0, 0,
|
||||||
|
0, 0, 0, 0, 0, 0, 0, 0,
|
||||||
|
0, 0, 0, 0, 0, 0, 0, 0,
|
||||||
|
0, 0, 0, 0, 0, 0, 0, 0,
|
||||||
|
0, 0, 0, 0, 0, 0, 0, 0);
|
||||||
|
__m512i b = a;
|
||||||
|
__mmask64 equality_mask = _mm512_cmp_epi8_mask(a, b, _MM_CMPINT_EQ);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
")
|
||||||
|
|
||||||
|
set(AVX2_CODE "
|
||||||
|
#include <immintrin.h>
|
||||||
|
int main()
|
||||||
|
{
|
||||||
|
__m256i a = {0};
|
||||||
|
a = _mm256_abs_epi16(a);
|
||||||
|
__m256i x;
|
||||||
|
_mm256_extract_epi64(x, 0); // we rely on this in our AVX2 code
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
")
|
||||||
|
|
||||||
|
set(FMA_CODE "
|
||||||
|
#include <immintrin.h>
|
||||||
|
int main()
|
||||||
|
{
|
||||||
|
__m256 acc = _mm256_setzero_ps();
|
||||||
|
const __m256 d = _mm256_setzero_ps();
|
||||||
|
const __m256 p = _mm256_setzero_ps();
|
||||||
|
acc = _mm256_fmadd_ps( d, p, acc );
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
")
|
||||||
|
|
||||||
|
macro(check_sse type flags)
|
||||||
|
set(__FLAG_I 1)
|
||||||
|
set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS})
|
||||||
|
foreach (__FLAG ${flags})
|
||||||
|
if (NOT ${type}_FOUND)
|
||||||
|
set(CMAKE_REQUIRED_FLAGS ${__FLAG})
|
||||||
|
check_c_source_runs("${${type}_CODE}" HAS_${type}_${__FLAG_I})
|
||||||
|
if (HAS_${type}_${__FLAG_I})
|
||||||
|
set(${type}_FOUND TRUE CACHE BOOL "${type} support")
|
||||||
|
set(${type}_FLAGS "${__FLAG}" CACHE STRING "${type} flags")
|
||||||
|
endif()
|
||||||
|
math(EXPR __FLAG_I "${__FLAG_I}+1")
|
||||||
|
endif()
|
||||||
|
endforeach()
|
||||||
|
set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE})
|
||||||
|
|
||||||
|
if (NOT ${type}_FOUND)
|
||||||
|
set(${type}_FOUND FALSE CACHE BOOL "${type} support")
|
||||||
|
set(${type}_FLAGS "" CACHE STRING "${type} flags")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
mark_as_advanced(${type}_FOUND ${type}_FLAGS)
|
||||||
|
endmacro()
|
||||||
|
|
||||||
|
# flags are for MSVC only!
|
||||||
|
check_sse("AVX" " ;/arch:AVX")
|
||||||
|
if (NOT ${AVX_FOUND})
|
||||||
|
set(LLAMA_AVX OFF)
|
||||||
|
else()
|
||||||
|
set(LLAMA_AVX ON)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
check_sse("AVX2" " ;/arch:AVX2")
|
||||||
|
check_sse("FMA" " ;/arch:AVX2")
|
||||||
|
if ((NOT ${AVX2_FOUND}) OR (NOT ${FMA_FOUND}))
|
||||||
|
set(LLAMA_AVX2 OFF)
|
||||||
|
else()
|
||||||
|
set(LLAMA_AVX2 ON)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
check_sse("AVX512" " ;/arch:AVX512")
|
||||||
|
if (NOT ${AVX512_FOUND})
|
||||||
|
set(LLAMA_AVX512 OFF)
|
||||||
|
else()
|
||||||
|
set(LLAMA_AVX512 ON)
|
||||||
|
endif()
|
||||||
14
codecov.yml
Normal file
14
codecov.yml
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
comment: off
|
||||||
|
|
||||||
|
coverage:
|
||||||
|
status:
|
||||||
|
project:
|
||||||
|
default:
|
||||||
|
target: auto
|
||||||
|
threshold: 0
|
||||||
|
base: auto
|
||||||
|
patch:
|
||||||
|
default:
|
||||||
|
target: auto
|
||||||
|
threshold: 0
|
||||||
|
base: auto
|
||||||
87
common/CMakeLists.txt
Normal file
87
common/CMakeLists.txt
Normal file
|
|
@ -0,0 +1,87 @@
|
||||||
|
# common
|
||||||
|
|
||||||
|
|
||||||
|
# Build info header
|
||||||
|
#
|
||||||
|
|
||||||
|
if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../.git")
|
||||||
|
set(GIT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../.git")
|
||||||
|
|
||||||
|
# Is git submodule
|
||||||
|
if(NOT IS_DIRECTORY "${GIT_DIR}")
|
||||||
|
file(READ ${GIT_DIR} REAL_GIT_DIR_LINK)
|
||||||
|
string(REGEX REPLACE "gitdir: (.*)\n$" "\\1" REAL_GIT_DIR ${REAL_GIT_DIR_LINK})
|
||||||
|
string(FIND "${REAL_GIT_DIR}" "/" SLASH_POS)
|
||||||
|
if (SLASH_POS EQUAL 0)
|
||||||
|
set(GIT_DIR "${REAL_GIT_DIR}")
|
||||||
|
else()
|
||||||
|
set(GIT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../${REAL_GIT_DIR}")
|
||||||
|
endif()
|
||||||
|
endif()
|
||||||
|
|
||||||
|
if(EXISTS "${GIT_DIR}/index")
|
||||||
|
set(GIT_INDEX "${GIT_DIR}/index")
|
||||||
|
else()
|
||||||
|
message(WARNING "Git index not found in git repository.")
|
||||||
|
set(GIT_INDEX "")
|
||||||
|
endif()
|
||||||
|
else()
|
||||||
|
message(WARNING "Git repository not found; to enable automatic generation of build info, make sure Git is installed and the project is a Git repository.")
|
||||||
|
set(GIT_INDEX "")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
# Add a custom command to rebuild build-info.cpp when .git/index changes
|
||||||
|
add_custom_command(
|
||||||
|
OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/build-info.cpp"
|
||||||
|
COMMENT "Generating build details from Git"
|
||||||
|
COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC} -DCMAKE_C_COMPILER_VERSION=${CMAKE_C_COMPILER_VERSION}
|
||||||
|
-DCMAKE_C_COMPILER_ID=${CMAKE_C_COMPILER_ID} -DCMAKE_VS_PLATFORM_NAME=${CMAKE_VS_PLATFORM_NAME}
|
||||||
|
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -P "${CMAKE_CURRENT_SOURCE_DIR}/../scripts/gen-build-info-cpp.cmake"
|
||||||
|
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/.."
|
||||||
|
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/build-info.cpp.in" ${GIT_INDEX}
|
||||||
|
VERBATIM
|
||||||
|
)
|
||||||
|
set(TARGET build_info)
|
||||||
|
add_library(${TARGET} OBJECT build-info.cpp)
|
||||||
|
if (BUILD_SHARED_LIBS)
|
||||||
|
set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
set(TARGET common)
|
||||||
|
|
||||||
|
add_library(${TARGET} STATIC
|
||||||
|
base64.hpp
|
||||||
|
common.h
|
||||||
|
common.cpp
|
||||||
|
sampling.h
|
||||||
|
sampling.cpp
|
||||||
|
console.h
|
||||||
|
console.cpp
|
||||||
|
grammar-parser.h
|
||||||
|
grammar-parser.cpp
|
||||||
|
json.hpp
|
||||||
|
json-schema-to-grammar.cpp
|
||||||
|
train.h
|
||||||
|
train.cpp
|
||||||
|
ngram-cache.h
|
||||||
|
ngram-cache.cpp
|
||||||
|
)
|
||||||
|
|
||||||
|
if (BUILD_SHARED_LIBS)
|
||||||
|
set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
set(LLAMA_COMMON_EXTRA_LIBS build_info)
|
||||||
|
|
||||||
|
# Use curl to download model url
|
||||||
|
if (LLAMA_CURL)
|
||||||
|
find_package(CURL REQUIRED)
|
||||||
|
add_definitions(-DLLAMA_USE_CURL)
|
||||||
|
include_directories(${CURL_INCLUDE_DIRS})
|
||||||
|
find_library(CURL_LIBRARY curl REQUIRED)
|
||||||
|
set(LLAMA_COMMON_EXTRA_LIBS ${LLAMA_COMMON_EXTRA_LIBS} ${CURL_LIBRARY})
|
||||||
|
endif ()
|
||||||
|
|
||||||
|
target_include_directories(${TARGET} PUBLIC .)
|
||||||
|
target_compile_features(${TARGET} PUBLIC cxx_std_11)
|
||||||
|
target_link_libraries(${TARGET} PRIVATE ${LLAMA_COMMON_EXTRA_LIBS} PUBLIC llama)
|
||||||
392
common/base64.hpp
Normal file
392
common/base64.hpp
Normal file
|
|
@ -0,0 +1,392 @@
|
||||||
|
/*
|
||||||
|
This is free and unencumbered software released into the public domain.
|
||||||
|
|
||||||
|
Anyone is free to copy, modify, publish, use, compile, sell, or
|
||||||
|
distribute this software, either in source code form or as a compiled
|
||||||
|
binary, for any purpose, commercial or non-commercial, and by any
|
||||||
|
means.
|
||||||
|
|
||||||
|
In jurisdictions that recognize copyright laws, the author or authors
|
||||||
|
of this software dedicate any and all copyright interest in the
|
||||||
|
software to the public domain. We make this dedication for the benefit
|
||||||
|
of the public at large and to the detriment of our heirs and
|
||||||
|
successors. We intend this dedication to be an overt act of
|
||||||
|
relinquishment in perpetuity of all present and future rights to this
|
||||||
|
software under copyright law.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||||
|
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||||
|
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||||
|
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||||
|
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||||
|
OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
For more information, please refer to <http://unlicense.org>
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef PUBLIC_DOMAIN_BASE64_HPP_
|
||||||
|
#define PUBLIC_DOMAIN_BASE64_HPP_
|
||||||
|
|
||||||
|
#include <cstdint>
|
||||||
|
#include <iterator>
|
||||||
|
#include <stdexcept>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
class base64_error : public std::runtime_error
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
using std::runtime_error::runtime_error;
|
||||||
|
};
|
||||||
|
|
||||||
|
class base64
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
enum class alphabet
|
||||||
|
{
|
||||||
|
/** the alphabet is detected automatically */
|
||||||
|
auto_,
|
||||||
|
/** the standard base64 alphabet is used */
|
||||||
|
standard,
|
||||||
|
/** like `standard` except that the characters `+` and `/` are replaced by `-` and `_` respectively*/
|
||||||
|
url_filename_safe
|
||||||
|
};
|
||||||
|
|
||||||
|
enum class decoding_behavior
|
||||||
|
{
|
||||||
|
/** if the input is not padded, the remaining bits are ignored */
|
||||||
|
moderate,
|
||||||
|
/** if a padding character is encounter decoding is finished */
|
||||||
|
loose
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
Encodes all the elements from `in_begin` to `in_end` to `out`.
|
||||||
|
|
||||||
|
@warning The source and destination cannot overlap. The destination must be able to hold at least
|
||||||
|
`required_encode_size(std::distance(in_begin, in_end))`, otherwise the behavior depends on the output iterator.
|
||||||
|
|
||||||
|
@tparam Input_iterator the source; the returned elements are cast to `std::uint8_t` and should not be greater than
|
||||||
|
8 bits
|
||||||
|
@tparam Output_iterator the destination; the elements written to it are from the type `char`
|
||||||
|
@param in_begin the beginning of the source
|
||||||
|
@param in_end the ending of the source
|
||||||
|
@param out the destination iterator
|
||||||
|
@param alphabet which alphabet should be used
|
||||||
|
@returns the iterator to the next element past the last element copied
|
||||||
|
@throws see `Input_iterator` and `Output_iterator`
|
||||||
|
*/
|
||||||
|
template<typename Input_iterator, typename Output_iterator>
|
||||||
|
static Output_iterator encode(Input_iterator in_begin, Input_iterator in_end, Output_iterator out,
|
||||||
|
alphabet alphabet = alphabet::standard)
|
||||||
|
{
|
||||||
|
constexpr auto pad = '=';
|
||||||
|
const char* alpha = alphabet == alphabet::url_filename_safe
|
||||||
|
? "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
|
||||||
|
: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||||
|
|
||||||
|
while (in_begin != in_end) {
|
||||||
|
std::uint8_t i0 = 0, i1 = 0, i2 = 0;
|
||||||
|
|
||||||
|
// first character
|
||||||
|
i0 = static_cast<std::uint8_t>(*in_begin);
|
||||||
|
++in_begin;
|
||||||
|
|
||||||
|
*out = alpha[i0 >> 2 & 0x3f];
|
||||||
|
++out;
|
||||||
|
|
||||||
|
// part of first character and second
|
||||||
|
if (in_begin != in_end) {
|
||||||
|
i1 = static_cast<std::uint8_t>(*in_begin);
|
||||||
|
++in_begin;
|
||||||
|
|
||||||
|
*out = alpha[((i0 & 0x3) << 4) | (i1 >> 4 & 0x0f)];
|
||||||
|
++out;
|
||||||
|
} else {
|
||||||
|
*out = alpha[(i0 & 0x3) << 4];
|
||||||
|
++out;
|
||||||
|
|
||||||
|
// last padding
|
||||||
|
*out = pad;
|
||||||
|
++out;
|
||||||
|
|
||||||
|
// last padding
|
||||||
|
*out = pad;
|
||||||
|
++out;
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// part of second character and third
|
||||||
|
if (in_begin != in_end) {
|
||||||
|
i2 = static_cast<std::uint8_t>(*in_begin);
|
||||||
|
++in_begin;
|
||||||
|
|
||||||
|
*out = alpha[((i1 & 0xf) << 2) | (i2 >> 6 & 0x03)];
|
||||||
|
++out;
|
||||||
|
} else {
|
||||||
|
*out = alpha[(i1 & 0xf) << 2];
|
||||||
|
++out;
|
||||||
|
|
||||||
|
// last padding
|
||||||
|
*out = pad;
|
||||||
|
++out;
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// rest of third
|
||||||
|
*out = alpha[i2 & 0x3f];
|
||||||
|
++out;
|
||||||
|
}
|
||||||
|
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
Encodes a string.
|
||||||
|
|
||||||
|
@param str the string that should be encoded
|
||||||
|
@param alphabet which alphabet should be used
|
||||||
|
@returns the encoded base64 string
|
||||||
|
@throws see base64::encode()
|
||||||
|
*/
|
||||||
|
static std::string encode(const std::string& str, alphabet alphabet = alphabet::standard)
|
||||||
|
{
|
||||||
|
std::string result;
|
||||||
|
|
||||||
|
result.reserve(required_encode_size(str.length()) + 1);
|
||||||
|
|
||||||
|
encode(str.begin(), str.end(), std::back_inserter(result), alphabet);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
Encodes a char array.
|
||||||
|
|
||||||
|
@param buffer the char array
|
||||||
|
@param size the size of the array
|
||||||
|
@param alphabet which alphabet should be used
|
||||||
|
@returns the encoded string
|
||||||
|
*/
|
||||||
|
static std::string encode(const char* buffer, std::size_t size, alphabet alphabet = alphabet::standard)
|
||||||
|
{
|
||||||
|
std::string result;
|
||||||
|
|
||||||
|
result.reserve(required_encode_size(size) + 1);
|
||||||
|
|
||||||
|
encode(buffer, buffer + size, std::back_inserter(result), alphabet);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
Decodes all the elements from `in_begin` to `in_end` to `out`. `in_begin` may point to the same location as `out`,
|
||||||
|
in other words: inplace decoding is possible.
|
||||||
|
|
||||||
|
@warning The destination must be able to hold at least `required_decode_size(std::distance(in_begin, in_end))`,
|
||||||
|
otherwise the behavior depends on the output iterator.
|
||||||
|
|
||||||
|
@tparam Input_iterator the source; the returned elements are cast to `char`
|
||||||
|
@tparam Output_iterator the destination; the elements written to it are from the type `std::uint8_t`
|
||||||
|
@param in_begin the beginning of the source
|
||||||
|
@param in_end the ending of the source
|
||||||
|
@param out the destination iterator
|
||||||
|
@param alphabet which alphabet should be used
|
||||||
|
@param behavior the behavior when an error was detected
|
||||||
|
@returns the iterator to the next element past the last element copied
|
||||||
|
@throws base64_error depending on the set behavior
|
||||||
|
@throws see `Input_iterator` and `Output_iterator`
|
||||||
|
*/
|
||||||
|
template<typename Input_iterator, typename Output_iterator>
|
||||||
|
static Output_iterator decode(Input_iterator in_begin, Input_iterator in_end, Output_iterator out,
|
||||||
|
alphabet alphabet = alphabet::auto_,
|
||||||
|
decoding_behavior behavior = decoding_behavior::moderate)
|
||||||
|
{
|
||||||
|
//constexpr auto pad = '=';
|
||||||
|
std::uint8_t last = 0;
|
||||||
|
auto bits = 0;
|
||||||
|
|
||||||
|
while (in_begin != in_end) {
|
||||||
|
auto c = *in_begin;
|
||||||
|
++in_begin;
|
||||||
|
|
||||||
|
if (c == '=') {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto part = _base64_value(alphabet, c);
|
||||||
|
|
||||||
|
// enough bits for one byte
|
||||||
|
if (bits + 6 >= 8) {
|
||||||
|
*out = (last << (8 - bits)) | (part >> (bits - 2));
|
||||||
|
++out;
|
||||||
|
|
||||||
|
bits -= 2;
|
||||||
|
} else {
|
||||||
|
bits += 6;
|
||||||
|
}
|
||||||
|
|
||||||
|
last = part;
|
||||||
|
}
|
||||||
|
|
||||||
|
// check padding
|
||||||
|
if (behavior != decoding_behavior::loose) {
|
||||||
|
while (in_begin != in_end) {
|
||||||
|
auto c = *in_begin;
|
||||||
|
++in_begin;
|
||||||
|
|
||||||
|
if (c != '=') {
|
||||||
|
throw base64_error("invalid base64 character.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
Decodes a string.
|
||||||
|
|
||||||
|
@param str the base64 encoded string
|
||||||
|
@param alphabet which alphabet should be used
|
||||||
|
@param behavior the behavior when an error was detected
|
||||||
|
@returns the decoded string
|
||||||
|
@throws see base64::decode()
|
||||||
|
*/
|
||||||
|
static std::string decode(const std::string& str, alphabet alphabet = alphabet::auto_,
|
||||||
|
decoding_behavior behavior = decoding_behavior::moderate)
|
||||||
|
{
|
||||||
|
std::string result;
|
||||||
|
|
||||||
|
result.reserve(max_decode_size(str.length()));
|
||||||
|
|
||||||
|
decode(str.begin(), str.end(), std::back_inserter(result), alphabet, behavior);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
Decodes a string.
|
||||||
|
|
||||||
|
@param buffer the base64 encoded buffer
|
||||||
|
@param size the size of the buffer
|
||||||
|
@param alphabet which alphabet should be used
|
||||||
|
@param behavior the behavior when an error was detected
|
||||||
|
@returns the decoded string
|
||||||
|
@throws see base64::decode()
|
||||||
|
*/
|
||||||
|
static std::string decode(const char* buffer, std::size_t size, alphabet alphabet = alphabet::auto_,
|
||||||
|
decoding_behavior behavior = decoding_behavior::moderate)
|
||||||
|
{
|
||||||
|
std::string result;
|
||||||
|
|
||||||
|
result.reserve(max_decode_size(size));
|
||||||
|
|
||||||
|
decode(buffer, buffer + size, std::back_inserter(result), alphabet, behavior);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
Decodes a string inplace.
|
||||||
|
|
||||||
|
@param[in,out] str the base64 encoded string
|
||||||
|
@param alphabet which alphabet should be used
|
||||||
|
@param behavior the behavior when an error was detected
|
||||||
|
@throws base64::decode_inplace()
|
||||||
|
*/
|
||||||
|
static void decode_inplace(std::string& str, alphabet alphabet = alphabet::auto_,
|
||||||
|
decoding_behavior behavior = decoding_behavior::moderate)
|
||||||
|
{
|
||||||
|
str.resize(decode(str.begin(), str.end(), str.begin(), alphabet, behavior) - str.begin());
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
Decodes a char array inplace.
|
||||||
|
|
||||||
|
@param[in,out] str the string array
|
||||||
|
@param size the length of the array
|
||||||
|
@param alphabet which alphabet should be used
|
||||||
|
@param behavior the behavior when an error was detected
|
||||||
|
@returns the pointer to the next element past the last element decoded
|
||||||
|
@throws base64::decode_inplace()
|
||||||
|
*/
|
||||||
|
static char* decode_inplace(char* str, std::size_t size, alphabet alphabet = alphabet::auto_,
|
||||||
|
decoding_behavior behavior = decoding_behavior::moderate)
|
||||||
|
{
|
||||||
|
return decode(str, str + size, str, alphabet, behavior);
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
Returns the required decoding size for a given size. The value is calculated with the following formula:
|
||||||
|
|
||||||
|
$$
|
||||||
|
\lceil \frac{size}{4} \rceil \cdot 3
|
||||||
|
$$
|
||||||
|
|
||||||
|
@param size the size of the encoded input
|
||||||
|
@returns the size of the resulting decoded buffer; this the absolute maximum
|
||||||
|
*/
|
||||||
|
static std::size_t max_decode_size(std::size_t size) noexcept
|
||||||
|
{
|
||||||
|
return (size / 4 + (size % 4 ? 1 : 0)) * 3;
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
Returns the required encoding size for a given size. The value is calculated with the following formula:
|
||||||
|
|
||||||
|
$$
|
||||||
|
\lceil \frac{size}{3} \rceil \cdot 4
|
||||||
|
$$
|
||||||
|
|
||||||
|
@param size the size of the decoded input
|
||||||
|
@returns the size of the resulting encoded buffer
|
||||||
|
*/
|
||||||
|
static std::size_t required_encode_size(std::size_t size) noexcept
|
||||||
|
{
|
||||||
|
return (size / 3 + (size % 3 ? 1 : 0)) * 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
static std::uint8_t _base64_value(alphabet& alphabet, char c)
|
||||||
|
{
|
||||||
|
if (c >= 'A' && c <= 'Z') {
|
||||||
|
return c - 'A';
|
||||||
|
} else if (c >= 'a' && c <= 'z') {
|
||||||
|
return c - 'a' + 26;
|
||||||
|
} else if (c >= '0' && c <= '9') {
|
||||||
|
return c - '0' + 52;
|
||||||
|
}
|
||||||
|
|
||||||
|
// comes down to alphabet
|
||||||
|
if (alphabet == alphabet::standard) {
|
||||||
|
if (c == '+') {
|
||||||
|
return 62;
|
||||||
|
} else if (c == '/') {
|
||||||
|
return 63;
|
||||||
|
}
|
||||||
|
} else if (alphabet == alphabet::url_filename_safe) {
|
||||||
|
if (c == '-') {
|
||||||
|
return 62;
|
||||||
|
} else if (c == '_') {
|
||||||
|
return 63;
|
||||||
|
}
|
||||||
|
} // auto detect
|
||||||
|
else {
|
||||||
|
if (c == '+') {
|
||||||
|
alphabet = alphabet::standard;
|
||||||
|
|
||||||
|
return 62;
|
||||||
|
} else if (c == '/') {
|
||||||
|
alphabet = alphabet::standard;
|
||||||
|
|
||||||
|
return 63;
|
||||||
|
} else if (c == '-') {
|
||||||
|
alphabet = alphabet::url_filename_safe;
|
||||||
|
|
||||||
|
return 62;
|
||||||
|
} else if (c == '_') {
|
||||||
|
alphabet = alphabet::url_filename_safe;
|
||||||
|
|
||||||
|
return 63;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw base64_error("invalid base64 character.");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif // !PUBLIC_DOMAIN_BASE64_HPP_
|
||||||
4
common/build-info.cpp.in
Normal file
4
common/build-info.cpp.in
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
int LLAMA_BUILD_NUMBER = @BUILD_NUMBER@;
|
||||||
|
char const *LLAMA_COMMIT = "@BUILD_COMMIT@";
|
||||||
|
char const *LLAMA_COMPILER = "@BUILD_COMPILER@";
|
||||||
|
char const *LLAMA_BUILD_TARGET = "@BUILD_TARGET@";
|
||||||
3001
common/common.cpp
Normal file
3001
common/common.cpp
Normal file
File diff suppressed because it is too large
Load diff
336
common/common.h
Normal file
336
common/common.h
Normal file
|
|
@ -0,0 +1,336 @@
|
||||||
|
// Various helper functions and utilities
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "llama.h"
|
||||||
|
|
||||||
|
#include "sampling.h"
|
||||||
|
|
||||||
|
#define LOG_NO_FILE_LINE_FUNCTION
|
||||||
|
#include "log.h"
|
||||||
|
|
||||||
|
#include <cmath>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
#include <random>
|
||||||
|
#include <thread>
|
||||||
|
#include <unordered_map>
|
||||||
|
#include <tuple>
|
||||||
|
|
||||||
|
#ifdef _WIN32
|
||||||
|
#define DIRECTORY_SEPARATOR '\\'
|
||||||
|
#else
|
||||||
|
#define DIRECTORY_SEPARATOR '/'
|
||||||
|
#endif // _WIN32
|
||||||
|
|
||||||
|
#define die(msg) do { fputs("error: " msg "\n", stderr); exit(1); } while (0)
|
||||||
|
#define die_fmt(fmt, ...) do { fprintf(stderr, "error: " fmt "\n", __VA_ARGS__); exit(1); } while (0)
|
||||||
|
|
||||||
|
#define print_build_info() do { \
|
||||||
|
fprintf(stderr, "%s: build = %d (%s)\n", __func__, LLAMA_BUILD_NUMBER, LLAMA_COMMIT); \
|
||||||
|
fprintf(stderr, "%s: built with %s for %s\n", __func__, LLAMA_COMPILER, LLAMA_BUILD_TARGET); \
|
||||||
|
} while(0)
|
||||||
|
|
||||||
|
#define DEFAULT_MODEL_PATH "models/7B/ggml-model-f16.gguf"
|
||||||
|
|
||||||
|
// build info
|
||||||
|
extern int LLAMA_BUILD_NUMBER;
|
||||||
|
extern char const *LLAMA_COMMIT;
|
||||||
|
extern char const *LLAMA_COMPILER;
|
||||||
|
extern char const *LLAMA_BUILD_TARGET;
|
||||||
|
|
||||||
|
struct llama_control_vector_load_info;
|
||||||
|
|
||||||
|
int get_math_cpu_count();
|
||||||
|
int32_t get_num_physical_cores();
|
||||||
|
|
||||||
|
//
|
||||||
|
// CLI argument parsing
|
||||||
|
//
|
||||||
|
|
||||||
|
struct gpt_params {
|
||||||
|
uint32_t seed = LLAMA_DEFAULT_SEED; // RNG seed
|
||||||
|
|
||||||
|
int32_t n_threads = get_math_cpu_count();
|
||||||
|
int32_t n_threads_draft = -1;
|
||||||
|
int32_t n_threads_batch = -1; // number of threads to use for batch processing (-1 = use n_threads)
|
||||||
|
int32_t n_threads_batch_draft = -1;
|
||||||
|
int32_t n_predict = -1; // new tokens to predict
|
||||||
|
int32_t n_ctx = 512; // context size
|
||||||
|
int32_t n_batch = 2048; // logical batch size for prompt processing (must be >=32 to use BLAS)
|
||||||
|
int32_t n_ubatch = 512; // physical batch size for prompt processing (must be >=32 to use BLAS)
|
||||||
|
int32_t n_keep = 0; // number of tokens to keep from initial prompt
|
||||||
|
int32_t n_draft = 5; // number of tokens to draft during speculative decoding
|
||||||
|
int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited)
|
||||||
|
int32_t n_parallel = 1; // number of parallel sequences to decode
|
||||||
|
int32_t n_sequences = 1; // number of sequences to decode
|
||||||
|
float p_split = 0.1f; // speculative decoding split probability
|
||||||
|
int32_t n_gpu_layers = -1; // number of layers to store in VRAM (-1 - use default)
|
||||||
|
int32_t n_gpu_layers_draft = -1; // number of layers to store in VRAM for the draft model (-1 - use default)
|
||||||
|
llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
|
||||||
|
int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors
|
||||||
|
float tensor_split[128] = {0}; // how split tensors should be distributed across GPUs
|
||||||
|
int32_t n_beams = 0; // if non-zero then use beam search of given width.
|
||||||
|
int32_t grp_attn_n = 1; // group-attention factor
|
||||||
|
int32_t grp_attn_w = 512; // group-attention width
|
||||||
|
int32_t n_print = -1; // print token count every n tokens (-1 = disabled)
|
||||||
|
float rope_freq_base = 0.0f; // RoPE base frequency
|
||||||
|
float rope_freq_scale = 0.0f; // RoPE frequency scaling factor
|
||||||
|
float yarn_ext_factor = -1.0f; // YaRN extrapolation mix factor
|
||||||
|
float yarn_attn_factor = 1.0f; // YaRN magnitude scaling factor
|
||||||
|
float yarn_beta_fast = 32.0f; // YaRN low correction dim
|
||||||
|
float yarn_beta_slow = 1.0f; // YaRN high correction dim
|
||||||
|
int32_t yarn_orig_ctx = 0; // YaRN original context length
|
||||||
|
float defrag_thold = -1.0f; // KV cache defragmentation threshold
|
||||||
|
|
||||||
|
ggml_backend_sched_eval_callback cb_eval = nullptr;
|
||||||
|
void * cb_eval_user_data = nullptr;
|
||||||
|
|
||||||
|
ggml_numa_strategy numa = GGML_NUMA_STRATEGY_DISABLED;
|
||||||
|
|
||||||
|
enum llama_rope_scaling_type rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED;
|
||||||
|
enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED; // pooling type for embeddings
|
||||||
|
|
||||||
|
// // sampling parameters
|
||||||
|
struct llama_sampling_params sparams;
|
||||||
|
|
||||||
|
std::string model = ""; // model path
|
||||||
|
std::string model_draft = ""; // draft model for speculative decoding
|
||||||
|
std::string model_alias = "unknown"; // model alias
|
||||||
|
std::string model_url = ""; // model url to download
|
||||||
|
std::string hf_repo = ""; // HF repo
|
||||||
|
std::string hf_file = ""; // HF file
|
||||||
|
std::string prompt = "";
|
||||||
|
std::string prompt_file = ""; // store the external prompt file name
|
||||||
|
std::string path_prompt_cache = ""; // path to file for saving/loading prompt eval state
|
||||||
|
std::string input_prefix = ""; // string to prefix user inputs with
|
||||||
|
std::string input_suffix = ""; // string to suffix user inputs with
|
||||||
|
std::vector<std::string> antiprompt; // string upon seeing which more user input is prompted
|
||||||
|
std::string logdir = ""; // directory in which to save YAML log files
|
||||||
|
std::string lookup_cache_static = ""; // path of static ngram cache file for lookup decoding
|
||||||
|
std::string lookup_cache_dynamic = ""; // path of dynamic ngram cache file for lookup decoding
|
||||||
|
std::string logits_file = ""; // file for saving *all* logits
|
||||||
|
|
||||||
|
std::vector<llama_model_kv_override> kv_overrides;
|
||||||
|
|
||||||
|
// TODO: avoid tuple, use struct
|
||||||
|
std::vector<std::tuple<std::string, float>> lora_adapter; // lora adapter path with user defined scale
|
||||||
|
std::string lora_base = ""; // base model path for the lora adapter
|
||||||
|
|
||||||
|
std::vector<llama_control_vector_load_info> control_vectors; // control vector with user defined scale
|
||||||
|
|
||||||
|
int32_t control_vector_layer_start = -1; // layer range for control vector
|
||||||
|
int32_t control_vector_layer_end = -1; // layer range for control vector
|
||||||
|
|
||||||
|
int ppl_stride = 0; // stride for perplexity calculations. If left at 0, the pre-existing approach will be used.
|
||||||
|
int ppl_output_type = 0; // = 0 -> ppl output is as usual, = 1 -> ppl output is num_tokens, ppl, one per line
|
||||||
|
// (which is more convenient to use for plotting)
|
||||||
|
//
|
||||||
|
bool hellaswag = false; // compute HellaSwag score over random tasks from datafile supplied in prompt
|
||||||
|
size_t hellaswag_tasks = 400; // number of tasks to use when computing the HellaSwag score
|
||||||
|
|
||||||
|
bool winogrande = false; // compute Winogrande score over random tasks from datafile supplied in prompt
|
||||||
|
size_t winogrande_tasks= 0; // number of tasks to use when computing the Winogrande score. If 0, all tasks will be computed
|
||||||
|
|
||||||
|
bool multiple_choice = false; // compute TruthfulQA score over random tasks from datafile supplied in prompt
|
||||||
|
size_t multiple_choice_tasks = 0; // number of tasks to use when computing the TruthfulQA score. If 0, all tasks will be computed
|
||||||
|
|
||||||
|
bool kl_divergence = false; // compute KL divergence
|
||||||
|
|
||||||
|
bool random_prompt = false; // do not randomize prompt if none provided
|
||||||
|
bool use_color = false; // use color to distinguish generations and inputs
|
||||||
|
bool interactive = false; // interactive mode
|
||||||
|
bool interactive_specials = false; // whether to allow special tokens from user, during interactive mode
|
||||||
|
bool conversation = false; // conversation mode (does not print special tokens and suffix/prefix)
|
||||||
|
bool chatml = false; // chatml mode (used for models trained on chatml syntax)
|
||||||
|
bool prompt_cache_all = false; // save user input and generations to prompt cache
|
||||||
|
bool prompt_cache_ro = false; // open the prompt cache read-only and do not update it
|
||||||
|
|
||||||
|
bool embedding = false; // get only sentence embedding
|
||||||
|
bool escape = false; // escape "\n", "\r", "\t", "\'", "\"", and "\\"
|
||||||
|
bool interactive_first = false; // wait for user input immediately
|
||||||
|
bool multiline_input = false; // reverse the usage of `\`
|
||||||
|
bool simple_io = false; // improves compatibility with subprocesses and limited consoles
|
||||||
|
bool cont_batching = true; // insert new sequences for decoding on-the-fly
|
||||||
|
bool flash_attn = false; // flash attention
|
||||||
|
|
||||||
|
bool input_prefix_bos = false; // prefix BOS to user inputs, preceding input_prefix
|
||||||
|
bool ignore_eos = false; // ignore generated EOS tokens
|
||||||
|
bool instruct = false; // instruction mode (used for Alpaca models)
|
||||||
|
bool logits_all = false; // return logits for all tokens in the batch
|
||||||
|
bool use_mmap = true; // use mmap for faster loads
|
||||||
|
bool use_mlock = false; // use mlock to keep model in memory
|
||||||
|
bool verbose_prompt = false; // print prompt tokens before generation
|
||||||
|
bool display_prompt = true; // print prompt before generation
|
||||||
|
bool infill = false; // use infill mode
|
||||||
|
bool dump_kv_cache = false; // dump the KV cache contents for debugging purposes
|
||||||
|
bool no_kv_offload = false; // disable KV offloading
|
||||||
|
bool warmup = true; // warmup run
|
||||||
|
bool check_tensors = false; // validate tensor data
|
||||||
|
|
||||||
|
std::string cache_type_k = "f16"; // KV cache data type for the K
|
||||||
|
std::string cache_type_v = "f16"; // KV cache data type for the V
|
||||||
|
|
||||||
|
// multimodal models (see examples/llava)
|
||||||
|
std::string mmproj = ""; // path to multimodal projector
|
||||||
|
std::vector<std::string> image; // path to image file(s)
|
||||||
|
};
|
||||||
|
|
||||||
|
void gpt_params_handle_model_default(gpt_params & params);
|
||||||
|
|
||||||
|
bool parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides);
|
||||||
|
|
||||||
|
bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params);
|
||||||
|
|
||||||
|
bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
|
||||||
|
|
||||||
|
void gpt_print_usage(int argc, char ** argv, const gpt_params & params);
|
||||||
|
|
||||||
|
bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_params & params, int & i, bool & invalid_param);
|
||||||
|
|
||||||
|
std::string get_system_info(const gpt_params & params);
|
||||||
|
|
||||||
|
std::string gpt_random_prompt(std::mt19937 & rng);
|
||||||
|
|
||||||
|
void process_escapes(std::string& input);
|
||||||
|
|
||||||
|
bool validate_file_name(const std::string & filename);
|
||||||
|
|
||||||
|
//
|
||||||
|
// String utils
|
||||||
|
//
|
||||||
|
|
||||||
|
std::vector<llama_sampler_type> sampler_types_from_names(const std::vector<std::string> & names, bool allow_alt_names);
|
||||||
|
std::vector<llama_sampler_type> sampler_types_from_chars(const std::string & names_string);
|
||||||
|
std::vector<std::string> string_split(std::string input, char separator);
|
||||||
|
std::string string_strip(const std::string & str);
|
||||||
|
std::string sampler_type_to_name_string(llama_sampler_type sampler_type);
|
||||||
|
|
||||||
|
//
|
||||||
|
// Model utils
|
||||||
|
//
|
||||||
|
|
||||||
|
// TODO: avoid tuplue, use struct
|
||||||
|
std::tuple<struct llama_model *, struct llama_context *> llama_init_from_gpt_params(gpt_params & params);
|
||||||
|
|
||||||
|
struct llama_model_params llama_model_params_from_gpt_params (const gpt_params & params);
|
||||||
|
struct llama_context_params llama_context_params_from_gpt_params(const gpt_params & params);
|
||||||
|
|
||||||
|
struct llama_model * llama_load_model_from_url(const char * model_url, const char * path_model, const struct llama_model_params & params);
|
||||||
|
struct llama_model * llama_load_model_from_hf(const char * repo, const char * file, const char * path_model, const struct llama_model_params & params);
|
||||||
|
|
||||||
|
// Batch utils
|
||||||
|
|
||||||
|
void llama_batch_clear(struct llama_batch & batch);
|
||||||
|
|
||||||
|
void llama_batch_add(
|
||||||
|
struct llama_batch & batch,
|
||||||
|
llama_token id,
|
||||||
|
llama_pos pos,
|
||||||
|
const std::vector<llama_seq_id> & seq_ids,
|
||||||
|
bool logits);
|
||||||
|
|
||||||
|
//
|
||||||
|
// Vocab utils
|
||||||
|
//
|
||||||
|
|
||||||
|
// tokenizes a string into a vector of tokens
|
||||||
|
// should work similar to Python's `tokenizer.encode`
|
||||||
|
std::vector<llama_token> llama_tokenize(
|
||||||
|
const struct llama_context * ctx,
|
||||||
|
const std::string & text,
|
||||||
|
bool add_special,
|
||||||
|
bool parse_special = false);
|
||||||
|
|
||||||
|
std::vector<llama_token> llama_tokenize(
|
||||||
|
const struct llama_model * model,
|
||||||
|
const std::string & text,
|
||||||
|
bool add_special,
|
||||||
|
bool parse_special = false);
|
||||||
|
|
||||||
|
// tokenizes a token into a piece, optionally renders special/control tokens
|
||||||
|
// should work similar to Python's `tokenizer.id_to_piece`
|
||||||
|
std::string llama_token_to_piece(
|
||||||
|
const struct llama_context * ctx,
|
||||||
|
llama_token token,
|
||||||
|
bool special = true);
|
||||||
|
|
||||||
|
// TODO: these should be moved in llama.h C-style API under single `llama_detokenize` function
|
||||||
|
// that takes into account the tokenizer type and decides how to handle the leading space
|
||||||
|
//
|
||||||
|
// detokenizes a vector of tokens into a string
|
||||||
|
// should work similar to Python's `tokenizer.decode`
|
||||||
|
// removes the leading space from the first non-BOS token
|
||||||
|
std::string llama_detokenize_spm(
|
||||||
|
llama_context * ctx,
|
||||||
|
const std::vector<llama_token> & tokens);
|
||||||
|
|
||||||
|
// detokenizes a vector of tokens into a string
|
||||||
|
// should work similar to Python's `tokenizer.decode`
|
||||||
|
std::string llama_detokenize_bpe(
|
||||||
|
llama_context * ctx,
|
||||||
|
const std::vector<llama_token> & tokens);
|
||||||
|
|
||||||
|
// Uses the value from the model metadata if possible, otherwise
|
||||||
|
// defaults to true when model type is SPM, otherwise false.
|
||||||
|
bool llama_should_add_bos_token(const llama_model * model);
|
||||||
|
|
||||||
|
//
|
||||||
|
// YAML utils
|
||||||
|
//
|
||||||
|
|
||||||
|
bool create_directory_with_parents(const std::string & path);
|
||||||
|
void dump_vector_float_yaml(FILE * stream, const char * prop_name, const std::vector<float> & data);
|
||||||
|
void dump_vector_int_yaml(FILE * stream, const char * prop_name, const std::vector<int> & data);
|
||||||
|
void dump_string_yaml_multiline(FILE * stream, const char * prop_name, const char * data);
|
||||||
|
std::string get_sortable_timestamp();
|
||||||
|
|
||||||
|
void dump_non_result_info_yaml(
|
||||||
|
FILE * stream, const gpt_params & params, const llama_context * lctx,
|
||||||
|
const std::string & timestamp, const std::vector<int> & prompt_tokens, const char * model_desc);
|
||||||
|
|
||||||
|
//
|
||||||
|
// KV cache utils
|
||||||
|
//
|
||||||
|
|
||||||
|
// Dump the KV cache view with the number of sequences per cell.
|
||||||
|
void dump_kv_cache_view(const llama_kv_cache_view & view, int row_size = 80);
|
||||||
|
|
||||||
|
// Dump the KV cache view showing individual sequences in each cell (long output).
|
||||||
|
void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size = 40);
|
||||||
|
|
||||||
|
//
|
||||||
|
// Embedding utils
|
||||||
|
//
|
||||||
|
|
||||||
|
void llama_embd_normalize(const float * inp, float * out, int n);
|
||||||
|
|
||||||
|
float llama_embd_similarity_cos(const float * embd1, const float * embd2, int n);
|
||||||
|
|
||||||
|
//
|
||||||
|
// Control vector utils
|
||||||
|
//
|
||||||
|
|
||||||
|
struct llama_control_vector_data {
|
||||||
|
int n_embd;
|
||||||
|
|
||||||
|
// stores data for layers [1, n_layer] where n_layer = data.size() / n_embd
|
||||||
|
std::vector<float> data;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct llama_control_vector_load_info {
|
||||||
|
float strength;
|
||||||
|
|
||||||
|
std::string fname;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Load control vectors, scale each by strength, and add them together.
|
||||||
|
// On error, returns {-1, empty}
|
||||||
|
llama_control_vector_data llama_control_vector_load(const std::vector<llama_control_vector_load_info> & load_infos);
|
||||||
|
|
||||||
|
//
|
||||||
|
// Split utils
|
||||||
|
//
|
||||||
|
static const char * const LLM_KV_SPLIT_NO = "split.no";
|
||||||
|
static const char * const LLM_KV_SPLIT_COUNT = "split.count";
|
||||||
|
static const char * const LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count";
|
||||||
501
common/console.cpp
Normal file
501
common/console.cpp
Normal file
|
|
@ -0,0 +1,501 @@
|
||||||
|
#include "console.h"
|
||||||
|
#include <vector>
|
||||||
|
#include <iostream>
|
||||||
|
|
||||||
|
#if defined(_WIN32)
|
||||||
|
#define WIN32_LEAN_AND_MEAN
|
||||||
|
#ifndef NOMINMAX
|
||||||
|
#define NOMINMAX
|
||||||
|
#endif
|
||||||
|
#include <windows.h>
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <io.h>
|
||||||
|
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
|
||||||
|
#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004
|
||||||
|
#endif
|
||||||
|
#else
|
||||||
|
#include <climits>
|
||||||
|
#include <sys/ioctl.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
#include <wchar.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <signal.h>
|
||||||
|
#include <termios.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#define ANSI_COLOR_RED "\x1b[31m"
|
||||||
|
#define ANSI_COLOR_GREEN "\x1b[32m"
|
||||||
|
#define ANSI_COLOR_YELLOW "\x1b[33m"
|
||||||
|
#define ANSI_COLOR_BLUE "\x1b[34m"
|
||||||
|
#define ANSI_COLOR_MAGENTA "\x1b[35m"
|
||||||
|
#define ANSI_COLOR_CYAN "\x1b[36m"
|
||||||
|
#define ANSI_COLOR_RESET "\x1b[0m"
|
||||||
|
#define ANSI_BOLD "\x1b[1m"
|
||||||
|
|
||||||
|
namespace console {
|
||||||
|
|
||||||
|
//
|
||||||
|
// Console state
|
||||||
|
//
|
||||||
|
|
||||||
|
static bool advanced_display = false;
|
||||||
|
static bool simple_io = true;
|
||||||
|
static display_t current_display = reset;
|
||||||
|
|
||||||
|
static FILE* out = stdout;
|
||||||
|
|
||||||
|
#if defined (_WIN32)
|
||||||
|
static void* hConsole;
|
||||||
|
#else
|
||||||
|
static FILE* tty = nullptr;
|
||||||
|
static termios initial_state;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
//
|
||||||
|
// Init and cleanup
|
||||||
|
//
|
||||||
|
|
||||||
|
void init(bool use_simple_io, bool use_advanced_display) {
|
||||||
|
advanced_display = use_advanced_display;
|
||||||
|
simple_io = use_simple_io;
|
||||||
|
#if defined(_WIN32)
|
||||||
|
// Windows-specific console initialization
|
||||||
|
DWORD dwMode = 0;
|
||||||
|
hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
|
||||||
|
if (hConsole == INVALID_HANDLE_VALUE || !GetConsoleMode(hConsole, &dwMode)) {
|
||||||
|
hConsole = GetStdHandle(STD_ERROR_HANDLE);
|
||||||
|
if (hConsole != INVALID_HANDLE_VALUE && (!GetConsoleMode(hConsole, &dwMode))) {
|
||||||
|
hConsole = nullptr;
|
||||||
|
simple_io = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (hConsole) {
|
||||||
|
// Check conditions combined to reduce nesting
|
||||||
|
if (advanced_display && !(dwMode & ENABLE_VIRTUAL_TERMINAL_PROCESSING) &&
|
||||||
|
!SetConsoleMode(hConsole, dwMode | ENABLE_VIRTUAL_TERMINAL_PROCESSING)) {
|
||||||
|
advanced_display = false;
|
||||||
|
}
|
||||||
|
// Set console output codepage to UTF8
|
||||||
|
SetConsoleOutputCP(CP_UTF8);
|
||||||
|
}
|
||||||
|
HANDLE hConIn = GetStdHandle(STD_INPUT_HANDLE);
|
||||||
|
if (hConIn != INVALID_HANDLE_VALUE && GetConsoleMode(hConIn, &dwMode)) {
|
||||||
|
// Set console input codepage to UTF16
|
||||||
|
_setmode(_fileno(stdin), _O_WTEXT);
|
||||||
|
|
||||||
|
// Set ICANON (ENABLE_LINE_INPUT) and ECHO (ENABLE_ECHO_INPUT)
|
||||||
|
if (simple_io) {
|
||||||
|
dwMode |= ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT;
|
||||||
|
} else {
|
||||||
|
dwMode &= ~(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT);
|
||||||
|
}
|
||||||
|
if (!SetConsoleMode(hConIn, dwMode)) {
|
||||||
|
simple_io = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
// POSIX-specific console initialization
|
||||||
|
if (!simple_io) {
|
||||||
|
struct termios new_termios;
|
||||||
|
tcgetattr(STDIN_FILENO, &initial_state);
|
||||||
|
new_termios = initial_state;
|
||||||
|
new_termios.c_lflag &= ~(ICANON | ECHO);
|
||||||
|
new_termios.c_cc[VMIN] = 1;
|
||||||
|
new_termios.c_cc[VTIME] = 0;
|
||||||
|
tcsetattr(STDIN_FILENO, TCSANOW, &new_termios);
|
||||||
|
|
||||||
|
tty = fopen("/dev/tty", "w+");
|
||||||
|
if (tty != nullptr) {
|
||||||
|
out = tty;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
setlocale(LC_ALL, "");
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
void cleanup() {
|
||||||
|
// Reset console display
|
||||||
|
set_display(reset);
|
||||||
|
|
||||||
|
#if !defined(_WIN32)
|
||||||
|
// Restore settings on POSIX systems
|
||||||
|
if (!simple_io) {
|
||||||
|
if (tty != nullptr) {
|
||||||
|
out = stdout;
|
||||||
|
fclose(tty);
|
||||||
|
tty = nullptr;
|
||||||
|
}
|
||||||
|
tcsetattr(STDIN_FILENO, TCSANOW, &initial_state);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
// Display and IO
|
||||||
|
//
|
||||||
|
|
||||||
|
// Keep track of current display and only emit ANSI code if it changes
|
||||||
|
void set_display(display_t display) {
|
||||||
|
if (advanced_display && current_display != display) {
|
||||||
|
fflush(stdout);
|
||||||
|
switch(display) {
|
||||||
|
case reset:
|
||||||
|
fprintf(out, ANSI_COLOR_RESET);
|
||||||
|
break;
|
||||||
|
case prompt:
|
||||||
|
fprintf(out, ANSI_COLOR_YELLOW);
|
||||||
|
break;
|
||||||
|
case user_input:
|
||||||
|
fprintf(out, ANSI_BOLD ANSI_COLOR_GREEN);
|
||||||
|
break;
|
||||||
|
case error:
|
||||||
|
fprintf(out, ANSI_BOLD ANSI_COLOR_RED);
|
||||||
|
}
|
||||||
|
current_display = display;
|
||||||
|
fflush(out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static char32_t getchar32() {
|
||||||
|
#if defined(_WIN32)
|
||||||
|
HANDLE hConsole = GetStdHandle(STD_INPUT_HANDLE);
|
||||||
|
wchar_t high_surrogate = 0;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
INPUT_RECORD record;
|
||||||
|
DWORD count;
|
||||||
|
if (!ReadConsoleInputW(hConsole, &record, 1, &count) || count == 0) {
|
||||||
|
return WEOF;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (record.EventType == KEY_EVENT && record.Event.KeyEvent.bKeyDown) {
|
||||||
|
wchar_t wc = record.Event.KeyEvent.uChar.UnicodeChar;
|
||||||
|
if (wc == 0) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((wc >= 0xD800) && (wc <= 0xDBFF)) { // Check if wc is a high surrogate
|
||||||
|
high_surrogate = wc;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if ((wc >= 0xDC00) && (wc <= 0xDFFF)) { // Check if wc is a low surrogate
|
||||||
|
if (high_surrogate != 0) { // Check if we have a high surrogate
|
||||||
|
return ((high_surrogate - 0xD800) << 10) + (wc - 0xDC00) + 0x10000;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
high_surrogate = 0; // Reset the high surrogate
|
||||||
|
return static_cast<char32_t>(wc);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
wchar_t wc = getwchar();
|
||||||
|
if (static_cast<wint_t>(wc) == WEOF) {
|
||||||
|
return WEOF;
|
||||||
|
}
|
||||||
|
|
||||||
|
#if WCHAR_MAX == 0xFFFF
|
||||||
|
if ((wc >= 0xD800) && (wc <= 0xDBFF)) { // Check if wc is a high surrogate
|
||||||
|
wchar_t low_surrogate = getwchar();
|
||||||
|
if ((low_surrogate >= 0xDC00) && (low_surrogate <= 0xDFFF)) { // Check if the next wchar is a low surrogate
|
||||||
|
return (static_cast<char32_t>(wc & 0x03FF) << 10) + (low_surrogate & 0x03FF) + 0x10000;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ((wc >= 0xD800) && (wc <= 0xDFFF)) { // Invalid surrogate pair
|
||||||
|
return 0xFFFD; // Return the replacement character U+FFFD
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return static_cast<char32_t>(wc);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
static void pop_cursor() {
|
||||||
|
#if defined(_WIN32)
|
||||||
|
if (hConsole != NULL) {
|
||||||
|
CONSOLE_SCREEN_BUFFER_INFO bufferInfo;
|
||||||
|
GetConsoleScreenBufferInfo(hConsole, &bufferInfo);
|
||||||
|
|
||||||
|
COORD newCursorPosition = bufferInfo.dwCursorPosition;
|
||||||
|
if (newCursorPosition.X == 0) {
|
||||||
|
newCursorPosition.X = bufferInfo.dwSize.X - 1;
|
||||||
|
newCursorPosition.Y -= 1;
|
||||||
|
} else {
|
||||||
|
newCursorPosition.X -= 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
SetConsoleCursorPosition(hConsole, newCursorPosition);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
putc('\b', out);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int estimateWidth(char32_t codepoint) {
|
||||||
|
#if defined(_WIN32)
|
||||||
|
(void)codepoint;
|
||||||
|
return 1;
|
||||||
|
#else
|
||||||
|
return wcwidth(codepoint);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
static int put_codepoint(const char* utf8_codepoint, size_t length, int expectedWidth) {
|
||||||
|
#if defined(_WIN32)
|
||||||
|
CONSOLE_SCREEN_BUFFER_INFO bufferInfo;
|
||||||
|
if (!GetConsoleScreenBufferInfo(hConsole, &bufferInfo)) {
|
||||||
|
// go with the default
|
||||||
|
return expectedWidth;
|
||||||
|
}
|
||||||
|
COORD initialPosition = bufferInfo.dwCursorPosition;
|
||||||
|
DWORD nNumberOfChars = length;
|
||||||
|
WriteConsole(hConsole, utf8_codepoint, nNumberOfChars, &nNumberOfChars, NULL);
|
||||||
|
|
||||||
|
CONSOLE_SCREEN_BUFFER_INFO newBufferInfo;
|
||||||
|
GetConsoleScreenBufferInfo(hConsole, &newBufferInfo);
|
||||||
|
|
||||||
|
// Figure out our real position if we're in the last column
|
||||||
|
if (utf8_codepoint[0] != 0x09 && initialPosition.X == newBufferInfo.dwSize.X - 1) {
|
||||||
|
DWORD nNumberOfChars;
|
||||||
|
WriteConsole(hConsole, &" \b", 2, &nNumberOfChars, NULL);
|
||||||
|
GetConsoleScreenBufferInfo(hConsole, &newBufferInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
int width = newBufferInfo.dwCursorPosition.X - initialPosition.X;
|
||||||
|
if (width < 0) {
|
||||||
|
width += newBufferInfo.dwSize.X;
|
||||||
|
}
|
||||||
|
return width;
|
||||||
|
#else
|
||||||
|
// We can trust expectedWidth if we've got one
|
||||||
|
if (expectedWidth >= 0 || tty == nullptr) {
|
||||||
|
fwrite(utf8_codepoint, length, 1, out);
|
||||||
|
return expectedWidth;
|
||||||
|
}
|
||||||
|
|
||||||
|
fputs("\033[6n", tty); // Query cursor position
|
||||||
|
int x1;
|
||||||
|
int y1;
|
||||||
|
int x2;
|
||||||
|
int y2;
|
||||||
|
int results = 0;
|
||||||
|
results = fscanf(tty, "\033[%d;%dR", &y1, &x1);
|
||||||
|
|
||||||
|
fwrite(utf8_codepoint, length, 1, tty);
|
||||||
|
|
||||||
|
fputs("\033[6n", tty); // Query cursor position
|
||||||
|
results += fscanf(tty, "\033[%d;%dR", &y2, &x2);
|
||||||
|
|
||||||
|
if (results != 4) {
|
||||||
|
return expectedWidth;
|
||||||
|
}
|
||||||
|
|
||||||
|
int width = x2 - x1;
|
||||||
|
if (width < 0) {
|
||||||
|
// Calculate the width considering text wrapping
|
||||||
|
struct winsize w;
|
||||||
|
ioctl(STDOUT_FILENO, TIOCGWINSZ, &w);
|
||||||
|
width += w.ws_col;
|
||||||
|
}
|
||||||
|
return width;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
static void replace_last(char ch) {
|
||||||
|
#if defined(_WIN32)
|
||||||
|
pop_cursor();
|
||||||
|
put_codepoint(&ch, 1, 1);
|
||||||
|
#else
|
||||||
|
fprintf(out, "\b%c", ch);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
static void append_utf8(char32_t ch, std::string & out) {
|
||||||
|
if (ch <= 0x7F) {
|
||||||
|
out.push_back(static_cast<unsigned char>(ch));
|
||||||
|
} else if (ch <= 0x7FF) {
|
||||||
|
out.push_back(static_cast<unsigned char>(0xC0 | ((ch >> 6) & 0x1F)));
|
||||||
|
out.push_back(static_cast<unsigned char>(0x80 | (ch & 0x3F)));
|
||||||
|
} else if (ch <= 0xFFFF) {
|
||||||
|
out.push_back(static_cast<unsigned char>(0xE0 | ((ch >> 12) & 0x0F)));
|
||||||
|
out.push_back(static_cast<unsigned char>(0x80 | ((ch >> 6) & 0x3F)));
|
||||||
|
out.push_back(static_cast<unsigned char>(0x80 | (ch & 0x3F)));
|
||||||
|
} else if (ch <= 0x10FFFF) {
|
||||||
|
out.push_back(static_cast<unsigned char>(0xF0 | ((ch >> 18) & 0x07)));
|
||||||
|
out.push_back(static_cast<unsigned char>(0x80 | ((ch >> 12) & 0x3F)));
|
||||||
|
out.push_back(static_cast<unsigned char>(0x80 | ((ch >> 6) & 0x3F)));
|
||||||
|
out.push_back(static_cast<unsigned char>(0x80 | (ch & 0x3F)));
|
||||||
|
} else {
|
||||||
|
// Invalid Unicode code point
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to remove the last UTF-8 character from a string
|
||||||
|
static void pop_back_utf8_char(std::string & line) {
|
||||||
|
if (line.empty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t pos = line.length() - 1;
|
||||||
|
|
||||||
|
// Find the start of the last UTF-8 character (checking up to 4 bytes back)
|
||||||
|
for (size_t i = 0; i < 3 && pos > 0; ++i, --pos) {
|
||||||
|
if ((line[pos] & 0xC0) != 0x80) {
|
||||||
|
break; // Found the start of the character
|
||||||
|
}
|
||||||
|
}
|
||||||
|
line.erase(pos);
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool readline_advanced(std::string & line, bool multiline_input) {
|
||||||
|
if (out != stdout) {
|
||||||
|
fflush(stdout);
|
||||||
|
}
|
||||||
|
|
||||||
|
line.clear();
|
||||||
|
std::vector<int> widths;
|
||||||
|
bool is_special_char = false;
|
||||||
|
bool end_of_stream = false;
|
||||||
|
|
||||||
|
char32_t input_char;
|
||||||
|
while (true) {
|
||||||
|
fflush(out); // Ensure all output is displayed before waiting for input
|
||||||
|
input_char = getchar32();
|
||||||
|
|
||||||
|
if (input_char == '\r' || input_char == '\n') {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (input_char == (char32_t) WEOF || input_char == 0x04 /* Ctrl+D*/) {
|
||||||
|
end_of_stream = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (is_special_char) {
|
||||||
|
set_display(user_input);
|
||||||
|
replace_last(line.back());
|
||||||
|
is_special_char = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (input_char == '\033') { // Escape sequence
|
||||||
|
char32_t code = getchar32();
|
||||||
|
if (code == '[' || code == 0x1B) {
|
||||||
|
// Discard the rest of the escape sequence
|
||||||
|
while ((code = getchar32()) != (char32_t) WEOF) {
|
||||||
|
if ((code >= 'A' && code <= 'Z') || (code >= 'a' && code <= 'z') || code == '~') {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (input_char == 0x08 || input_char == 0x7F) { // Backspace
|
||||||
|
if (!widths.empty()) {
|
||||||
|
int count;
|
||||||
|
do {
|
||||||
|
count = widths.back();
|
||||||
|
widths.pop_back();
|
||||||
|
// Move cursor back, print space, and move cursor back again
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
replace_last(' ');
|
||||||
|
pop_cursor();
|
||||||
|
}
|
||||||
|
pop_back_utf8_char(line);
|
||||||
|
} while (count == 0 && !widths.empty());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
int offset = line.length();
|
||||||
|
append_utf8(input_char, line);
|
||||||
|
int width = put_codepoint(line.c_str() + offset, line.length() - offset, estimateWidth(input_char));
|
||||||
|
if (width < 0) {
|
||||||
|
width = 0;
|
||||||
|
}
|
||||||
|
widths.push_back(width);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!line.empty() && (line.back() == '\\' || line.back() == '/')) {
|
||||||
|
set_display(prompt);
|
||||||
|
replace_last(line.back());
|
||||||
|
is_special_char = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool has_more = multiline_input;
|
||||||
|
if (is_special_char) {
|
||||||
|
replace_last(' ');
|
||||||
|
pop_cursor();
|
||||||
|
|
||||||
|
char last = line.back();
|
||||||
|
line.pop_back();
|
||||||
|
if (last == '\\') {
|
||||||
|
line += '\n';
|
||||||
|
fputc('\n', out);
|
||||||
|
has_more = !has_more;
|
||||||
|
} else {
|
||||||
|
// llama will just eat the single space, it won't act as a space
|
||||||
|
if (line.length() == 1 && line.back() == ' ') {
|
||||||
|
line.clear();
|
||||||
|
pop_cursor();
|
||||||
|
}
|
||||||
|
has_more = false;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (end_of_stream) {
|
||||||
|
has_more = false;
|
||||||
|
} else {
|
||||||
|
line += '\n';
|
||||||
|
fputc('\n', out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fflush(out);
|
||||||
|
return has_more;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool readline_simple(std::string & line, bool multiline_input) {
|
||||||
|
#if defined(_WIN32)
|
||||||
|
std::wstring wline;
|
||||||
|
if (!std::getline(std::wcin, wline)) {
|
||||||
|
// Input stream is bad or EOF received
|
||||||
|
line.clear();
|
||||||
|
GenerateConsoleCtrlEvent(CTRL_C_EVENT, 0);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
int size_needed = WideCharToMultiByte(CP_UTF8, 0, &wline[0], (int)wline.size(), NULL, 0, NULL, NULL);
|
||||||
|
line.resize(size_needed);
|
||||||
|
WideCharToMultiByte(CP_UTF8, 0, &wline[0], (int)wline.size(), &line[0], size_needed, NULL, NULL);
|
||||||
|
#else
|
||||||
|
if (!std::getline(std::cin, line)) {
|
||||||
|
// Input stream is bad or EOF received
|
||||||
|
line.clear();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
if (!line.empty()) {
|
||||||
|
char last = line.back();
|
||||||
|
if (last == '/') { // Always return control on '/' symbol
|
||||||
|
line.pop_back();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (last == '\\') { // '\\' changes the default action
|
||||||
|
line.pop_back();
|
||||||
|
multiline_input = !multiline_input;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
line += '\n';
|
||||||
|
|
||||||
|
// By default, continue input if multiline_input is set
|
||||||
|
return multiline_input;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool readline(std::string & line, bool multiline_input) {
|
||||||
|
set_display(user_input);
|
||||||
|
|
||||||
|
if (simple_io) {
|
||||||
|
return readline_simple(line, multiline_input);
|
||||||
|
}
|
||||||
|
return readline_advanced(line, multiline_input);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
19
common/console.h
Normal file
19
common/console.h
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
// Console functions
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
namespace console {
|
||||||
|
enum display_t {
|
||||||
|
reset = 0,
|
||||||
|
prompt,
|
||||||
|
user_input,
|
||||||
|
error
|
||||||
|
};
|
||||||
|
|
||||||
|
void init(bool use_simple_io, bool use_advanced_display);
|
||||||
|
void cleanup();
|
||||||
|
void set_display(display_t display);
|
||||||
|
bool readline(std::string & line, bool multiline_input);
|
||||||
|
}
|
||||||
449
common/grammar-parser.cpp
Normal file
449
common/grammar-parser.cpp
Normal file
|
|
@ -0,0 +1,449 @@
|
||||||
|
#include "grammar-parser.h"
|
||||||
|
#include <cstdint>
|
||||||
|
#include <cwchar>
|
||||||
|
#include <string>
|
||||||
|
#include <utility>
|
||||||
|
#include <stdexcept>
|
||||||
|
#include <exception>
|
||||||
|
|
||||||
|
namespace grammar_parser {
|
||||||
|
// NOTE: assumes valid utf8 (but checks for overrun)
|
||||||
|
// copied from llama.cpp
|
||||||
|
static std::pair<uint32_t, const char *> decode_utf8(const char * src) {
|
||||||
|
static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };
|
||||||
|
uint8_t first_byte = static_cast<uint8_t>(*src);
|
||||||
|
uint8_t highbits = first_byte >> 4;
|
||||||
|
int len = lookup[highbits];
|
||||||
|
uint8_t mask = (1 << (8 - len)) - 1;
|
||||||
|
uint32_t value = first_byte & mask;
|
||||||
|
const char * end = src + len; // may overrun!
|
||||||
|
const char * pos = src + 1;
|
||||||
|
for ( ; pos < end && *pos; pos++) {
|
||||||
|
value = (value << 6) + (static_cast<uint8_t>(*pos) & 0x3F);
|
||||||
|
}
|
||||||
|
return std::make_pair(value, pos);
|
||||||
|
}
|
||||||
|
|
||||||
|
static uint32_t get_symbol_id(parse_state & state, const char * src, size_t len) {
|
||||||
|
uint32_t next_id = static_cast<uint32_t>(state.symbol_ids.size());
|
||||||
|
auto result = state.symbol_ids.insert(std::make_pair(std::string(src, len), next_id));
|
||||||
|
return result.first->second;
|
||||||
|
}
|
||||||
|
|
||||||
|
static uint32_t generate_symbol_id(parse_state & state, const std::string & base_name) {
|
||||||
|
uint32_t next_id = static_cast<uint32_t>(state.symbol_ids.size());
|
||||||
|
state.symbol_ids[base_name + '_' + std::to_string(next_id)] = next_id;
|
||||||
|
return next_id;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void add_rule(
|
||||||
|
parse_state & state,
|
||||||
|
uint32_t rule_id,
|
||||||
|
const std::vector<llama_grammar_element> & rule) {
|
||||||
|
if (state.rules.size() <= rule_id) {
|
||||||
|
state.rules.resize(rule_id + 1);
|
||||||
|
}
|
||||||
|
state.rules[rule_id] = rule;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool is_word_char(char c) {
|
||||||
|
return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '-' || ('0' <= c && c <= '9');
|
||||||
|
}
|
||||||
|
|
||||||
|
static std::pair<uint32_t, const char *> parse_hex(const char * src, int size) {
|
||||||
|
const char * pos = src;
|
||||||
|
const char * end = src + size;
|
||||||
|
uint32_t value = 0;
|
||||||
|
for ( ; pos < end && *pos; pos++) {
|
||||||
|
value <<= 4;
|
||||||
|
char c = *pos;
|
||||||
|
if ('a' <= c && c <= 'f') {
|
||||||
|
value += c - 'a' + 10;
|
||||||
|
} else if ('A' <= c && c <= 'F') {
|
||||||
|
value += c - 'A' + 10;
|
||||||
|
} else if ('0' <= c && c <= '9') {
|
||||||
|
value += c - '0';
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (pos != end) {
|
||||||
|
throw std::runtime_error("expecting " + std::to_string(size) + " hex chars at " + src);
|
||||||
|
}
|
||||||
|
return std::make_pair(value, pos);
|
||||||
|
}
|
||||||
|
|
||||||
|
static const char * parse_space(const char * src, bool newline_ok) {
|
||||||
|
const char * pos = src;
|
||||||
|
while (*pos == ' ' || *pos == '\t' || *pos == '#' ||
|
||||||
|
(newline_ok && (*pos == '\r' || *pos == '\n'))) {
|
||||||
|
if (*pos == '#') {
|
||||||
|
while (*pos && *pos != '\r' && *pos != '\n') {
|
||||||
|
pos++;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
pos++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return pos;
|
||||||
|
}
|
||||||
|
|
||||||
|
static const char * parse_name(const char * src) {
|
||||||
|
const char * pos = src;
|
||||||
|
while (is_word_char(*pos)) {
|
||||||
|
pos++;
|
||||||
|
}
|
||||||
|
if (pos == src) {
|
||||||
|
throw std::runtime_error(std::string("expecting name at ") + src);
|
||||||
|
}
|
||||||
|
return pos;
|
||||||
|
}
|
||||||
|
|
||||||
|
static std::pair<uint32_t, const char *> parse_char(const char * src) {
|
||||||
|
if (*src == '\\') {
|
||||||
|
switch (src[1]) {
|
||||||
|
case 'x': return parse_hex(src + 2, 2);
|
||||||
|
case 'u': return parse_hex(src + 2, 4);
|
||||||
|
case 'U': return parse_hex(src + 2, 8);
|
||||||
|
case 't': return std::make_pair('\t', src + 2);
|
||||||
|
case 'r': return std::make_pair('\r', src + 2);
|
||||||
|
case 'n': return std::make_pair('\n', src + 2);
|
||||||
|
case '\\':
|
||||||
|
case '"':
|
||||||
|
case '[':
|
||||||
|
case ']':
|
||||||
|
return std::make_pair(src[1], src + 2);
|
||||||
|
default:
|
||||||
|
throw std::runtime_error(std::string("unknown escape at ") + src);
|
||||||
|
}
|
||||||
|
} else if (*src) {
|
||||||
|
return decode_utf8(src);
|
||||||
|
}
|
||||||
|
throw std::runtime_error("unexpected end of input");
|
||||||
|
}
|
||||||
|
|
||||||
|
const char * parse_alternates(
|
||||||
|
parse_state & state,
|
||||||
|
const char * src,
|
||||||
|
const std::string & rule_name,
|
||||||
|
uint32_t rule_id,
|
||||||
|
bool is_nested);
|
||||||
|
|
||||||
|
static const char * parse_sequence(
|
||||||
|
parse_state & state,
|
||||||
|
const char * src,
|
||||||
|
const std::string & rule_name,
|
||||||
|
std::vector<llama_grammar_element> & out_elements,
|
||||||
|
bool is_nested) {
|
||||||
|
size_t last_sym_start = out_elements.size();
|
||||||
|
const char * pos = src;
|
||||||
|
while (*pos) {
|
||||||
|
if (*pos == '"') { // literal string
|
||||||
|
pos++;
|
||||||
|
last_sym_start = out_elements.size();
|
||||||
|
while (*pos != '"') {
|
||||||
|
if (!*pos) {
|
||||||
|
throw std::runtime_error("unexpected end of input");
|
||||||
|
}
|
||||||
|
auto char_pair = parse_char(pos);
|
||||||
|
pos = char_pair.second;
|
||||||
|
out_elements.push_back({LLAMA_GRETYPE_CHAR, char_pair.first});
|
||||||
|
}
|
||||||
|
pos = parse_space(pos + 1, is_nested);
|
||||||
|
} else if (*pos == '[') { // char range(s)
|
||||||
|
pos++;
|
||||||
|
enum llama_gretype start_type = LLAMA_GRETYPE_CHAR;
|
||||||
|
if (*pos == '^') {
|
||||||
|
pos++;
|
||||||
|
start_type = LLAMA_GRETYPE_CHAR_NOT;
|
||||||
|
}
|
||||||
|
last_sym_start = out_elements.size();
|
||||||
|
while (*pos != ']') {
|
||||||
|
if (!*pos) {
|
||||||
|
throw std::runtime_error("unexpected end of input");
|
||||||
|
}
|
||||||
|
auto char_pair = parse_char(pos);
|
||||||
|
pos = char_pair.second;
|
||||||
|
enum llama_gretype type = last_sym_start < out_elements.size()
|
||||||
|
? LLAMA_GRETYPE_CHAR_ALT
|
||||||
|
: start_type;
|
||||||
|
|
||||||
|
out_elements.push_back({type, char_pair.first});
|
||||||
|
if (pos[0] == '-' && pos[1] != ']') {
|
||||||
|
if (!pos[1]) {
|
||||||
|
throw std::runtime_error("unexpected end of input");
|
||||||
|
}
|
||||||
|
auto endchar_pair = parse_char(pos + 1);
|
||||||
|
pos = endchar_pair.second;
|
||||||
|
out_elements.push_back({LLAMA_GRETYPE_CHAR_RNG_UPPER, endchar_pair.first});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pos = parse_space(pos + 1, is_nested);
|
||||||
|
} else if (is_word_char(*pos)) { // rule reference
|
||||||
|
const char * name_end = parse_name(pos);
|
||||||
|
uint32_t ref_rule_id = get_symbol_id(state, pos, name_end - pos);
|
||||||
|
pos = parse_space(name_end, is_nested);
|
||||||
|
last_sym_start = out_elements.size();
|
||||||
|
out_elements.push_back({LLAMA_GRETYPE_RULE_REF, ref_rule_id});
|
||||||
|
} else if (*pos == '(') { // grouping
|
||||||
|
// parse nested alternates into synthesized rule
|
||||||
|
pos = parse_space(pos + 1, true);
|
||||||
|
uint32_t sub_rule_id = generate_symbol_id(state, rule_name);
|
||||||
|
pos = parse_alternates(state, pos, rule_name, sub_rule_id, true);
|
||||||
|
last_sym_start = out_elements.size();
|
||||||
|
// output reference to synthesized rule
|
||||||
|
out_elements.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id});
|
||||||
|
if (*pos != ')') {
|
||||||
|
throw std::runtime_error(std::string("expecting ')' at ") + pos);
|
||||||
|
}
|
||||||
|
pos = parse_space(pos + 1, is_nested);
|
||||||
|
} else if (*pos == '*' || *pos == '+' || *pos == '?') { // repetition operator
|
||||||
|
if (last_sym_start == out_elements.size()) {
|
||||||
|
throw std::runtime_error(std::string("expecting preceding item to */+/? at ") + pos);
|
||||||
|
}
|
||||||
|
|
||||||
|
// apply transformation to previous symbol (last_sym_start to end) according to
|
||||||
|
// rewrite rules:
|
||||||
|
// S* --> S' ::= S S' |
|
||||||
|
// S+ --> S' ::= S S' | S
|
||||||
|
// S? --> S' ::= S |
|
||||||
|
uint32_t sub_rule_id = generate_symbol_id(state, rule_name);
|
||||||
|
std::vector<llama_grammar_element> sub_rule;
|
||||||
|
// add preceding symbol to generated rule
|
||||||
|
sub_rule.insert(
|
||||||
|
sub_rule.end(), out_elements.begin() + last_sym_start, out_elements.end());
|
||||||
|
if (*pos == '*' || *pos == '+') {
|
||||||
|
// cause generated rule to recurse
|
||||||
|
sub_rule.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id});
|
||||||
|
}
|
||||||
|
// mark start of alternate def
|
||||||
|
sub_rule.push_back({LLAMA_GRETYPE_ALT, 0});
|
||||||
|
if (*pos == '+') {
|
||||||
|
// add preceding symbol as alternate only for '+' (otherwise empty)
|
||||||
|
sub_rule.insert(
|
||||||
|
sub_rule.end(), out_elements.begin() + last_sym_start, out_elements.end());
|
||||||
|
}
|
||||||
|
sub_rule.push_back({LLAMA_GRETYPE_END, 0});
|
||||||
|
add_rule(state, sub_rule_id, sub_rule);
|
||||||
|
|
||||||
|
// in original rule, replace previous symbol with reference to generated rule
|
||||||
|
out_elements.resize(last_sym_start);
|
||||||
|
out_elements.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id});
|
||||||
|
|
||||||
|
pos = parse_space(pos + 1, is_nested);
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return pos;
|
||||||
|
}
|
||||||
|
|
||||||
|
const char * parse_alternates(
|
||||||
|
parse_state & state,
|
||||||
|
const char * src,
|
||||||
|
const std::string & rule_name,
|
||||||
|
uint32_t rule_id,
|
||||||
|
bool is_nested) {
|
||||||
|
std::vector<llama_grammar_element> rule;
|
||||||
|
const char * pos = parse_sequence(state, src, rule_name, rule, is_nested);
|
||||||
|
while (*pos == '|') {
|
||||||
|
rule.push_back({LLAMA_GRETYPE_ALT, 0});
|
||||||
|
pos = parse_space(pos + 1, true);
|
||||||
|
pos = parse_sequence(state, pos, rule_name, rule, is_nested);
|
||||||
|
}
|
||||||
|
rule.push_back({LLAMA_GRETYPE_END, 0});
|
||||||
|
add_rule(state, rule_id, rule);
|
||||||
|
return pos;
|
||||||
|
}
|
||||||
|
|
||||||
|
static const char * parse_rule(parse_state & state, const char * src) {
|
||||||
|
const char * name_end = parse_name(src);
|
||||||
|
const char * pos = parse_space(name_end, false);
|
||||||
|
size_t name_len = name_end - src;
|
||||||
|
uint32_t rule_id = get_symbol_id(state, src, name_len);
|
||||||
|
const std::string name(src, name_len);
|
||||||
|
|
||||||
|
if (!(pos[0] == ':' && pos[1] == ':' && pos[2] == '=')) {
|
||||||
|
throw std::runtime_error(std::string("expecting ::= at ") + pos);
|
||||||
|
}
|
||||||
|
pos = parse_space(pos + 3, true);
|
||||||
|
|
||||||
|
pos = parse_alternates(state, pos, name, rule_id, false);
|
||||||
|
|
||||||
|
if (*pos == '\r') {
|
||||||
|
pos += pos[1] == '\n' ? 2 : 1;
|
||||||
|
} else if (*pos == '\n') {
|
||||||
|
pos++;
|
||||||
|
} else if (*pos) {
|
||||||
|
throw std::runtime_error(std::string("expecting newline or end at ") + pos);
|
||||||
|
}
|
||||||
|
return parse_space(pos, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_state parse(const char * src) {
|
||||||
|
try {
|
||||||
|
parse_state state;
|
||||||
|
const char * pos = parse_space(src, true);
|
||||||
|
while (*pos) {
|
||||||
|
pos = parse_rule(state, pos);
|
||||||
|
}
|
||||||
|
// Validate the state to ensure that all rules are defined
|
||||||
|
for (const auto & rule : state.rules) {
|
||||||
|
for (const auto & elem : rule) {
|
||||||
|
if (elem.type == LLAMA_GRETYPE_RULE_REF) {
|
||||||
|
// Ensure that the rule at that location exists
|
||||||
|
if (elem.value >= state.rules.size() || state.rules[elem.value].empty()) {
|
||||||
|
// Get the name of the rule that is missing
|
||||||
|
for (const auto & kv : state.symbol_ids) {
|
||||||
|
if (kv.second == elem.value) {
|
||||||
|
throw std::runtime_error("Undefined rule identifier '" + kv.first + "'");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return state;
|
||||||
|
} catch (const std::exception & err) {
|
||||||
|
fprintf(stderr, "%s: error parsing grammar: %s\n", __func__, err.what());
|
||||||
|
return parse_state();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void print_grammar_char(FILE * file, uint32_t c) {
|
||||||
|
if (0x20 <= c && c <= 0x7f) {
|
||||||
|
fprintf(file, "%c", static_cast<char>(c));
|
||||||
|
} else {
|
||||||
|
// cop out of encoding UTF-8
|
||||||
|
fprintf(file, "<U+%04X>", c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool is_char_element(llama_grammar_element elem) {
|
||||||
|
switch (elem.type) {
|
||||||
|
case LLAMA_GRETYPE_CHAR: return true;
|
||||||
|
case LLAMA_GRETYPE_CHAR_NOT: return true;
|
||||||
|
case LLAMA_GRETYPE_CHAR_ALT: return true;
|
||||||
|
case LLAMA_GRETYPE_CHAR_RNG_UPPER: return true;
|
||||||
|
default: return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void print_rule_binary(FILE * file, const std::vector<llama_grammar_element> & rule) {
|
||||||
|
for (auto elem : rule) {
|
||||||
|
switch (elem.type) {
|
||||||
|
case LLAMA_GRETYPE_END: fprintf(file, "END"); break;
|
||||||
|
case LLAMA_GRETYPE_ALT: fprintf(file, "ALT"); break;
|
||||||
|
case LLAMA_GRETYPE_RULE_REF: fprintf(file, "RULE_REF"); break;
|
||||||
|
case LLAMA_GRETYPE_CHAR: fprintf(file, "CHAR"); break;
|
||||||
|
case LLAMA_GRETYPE_CHAR_NOT: fprintf(file, "CHAR_NOT"); break;
|
||||||
|
case LLAMA_GRETYPE_CHAR_RNG_UPPER: fprintf(file, "CHAR_RNG_UPPER"); break;
|
||||||
|
case LLAMA_GRETYPE_CHAR_ALT: fprintf(file, "CHAR_ALT"); break;
|
||||||
|
}
|
||||||
|
switch (elem.type) {
|
||||||
|
case LLAMA_GRETYPE_END:
|
||||||
|
case LLAMA_GRETYPE_ALT:
|
||||||
|
case LLAMA_GRETYPE_RULE_REF:
|
||||||
|
fprintf(file, "(%u) ", elem.value);
|
||||||
|
break;
|
||||||
|
case LLAMA_GRETYPE_CHAR:
|
||||||
|
case LLAMA_GRETYPE_CHAR_NOT:
|
||||||
|
case LLAMA_GRETYPE_CHAR_RNG_UPPER:
|
||||||
|
case LLAMA_GRETYPE_CHAR_ALT:
|
||||||
|
fprintf(file, "(\"");
|
||||||
|
print_grammar_char(file, elem.value);
|
||||||
|
fprintf(file, "\") ");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fprintf(file, "\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
static void print_rule(
|
||||||
|
FILE * file,
|
||||||
|
uint32_t rule_id,
|
||||||
|
const std::vector<llama_grammar_element> & rule,
|
||||||
|
const std::map<uint32_t, std::string> & symbol_id_names) {
|
||||||
|
if (rule.empty() || rule.back().type != LLAMA_GRETYPE_END) {
|
||||||
|
throw std::runtime_error(
|
||||||
|
"malformed rule, does not end with LLAMA_GRETYPE_END: " + std::to_string(rule_id));
|
||||||
|
}
|
||||||
|
fprintf(file, "%s ::= ", symbol_id_names.at(rule_id).c_str());
|
||||||
|
for (size_t i = 0, end = rule.size() - 1; i < end; i++) {
|
||||||
|
llama_grammar_element elem = rule[i];
|
||||||
|
switch (elem.type) {
|
||||||
|
case LLAMA_GRETYPE_END:
|
||||||
|
throw std::runtime_error(
|
||||||
|
"unexpected end of rule: " + std::to_string(rule_id) + "," +
|
||||||
|
std::to_string(i));
|
||||||
|
case LLAMA_GRETYPE_ALT:
|
||||||
|
fprintf(file, "| ");
|
||||||
|
break;
|
||||||
|
case LLAMA_GRETYPE_RULE_REF:
|
||||||
|
fprintf(file, "%s ", symbol_id_names.at(elem.value).c_str());
|
||||||
|
break;
|
||||||
|
case LLAMA_GRETYPE_CHAR:
|
||||||
|
fprintf(file, "[");
|
||||||
|
print_grammar_char(file, elem.value);
|
||||||
|
break;
|
||||||
|
case LLAMA_GRETYPE_CHAR_NOT:
|
||||||
|
fprintf(file, "[^");
|
||||||
|
print_grammar_char(file, elem.value);
|
||||||
|
break;
|
||||||
|
case LLAMA_GRETYPE_CHAR_RNG_UPPER:
|
||||||
|
if (i == 0 || !is_char_element(rule[i - 1])) {
|
||||||
|
throw std::runtime_error(
|
||||||
|
"LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: " +
|
||||||
|
std::to_string(rule_id) + "," + std::to_string(i));
|
||||||
|
}
|
||||||
|
fprintf(file, "-");
|
||||||
|
print_grammar_char(file, elem.value);
|
||||||
|
break;
|
||||||
|
case LLAMA_GRETYPE_CHAR_ALT:
|
||||||
|
if (i == 0 || !is_char_element(rule[i - 1])) {
|
||||||
|
throw std::runtime_error(
|
||||||
|
"LLAMA_GRETYPE_CHAR_ALT without preceding char: " +
|
||||||
|
std::to_string(rule_id) + "," + std::to_string(i));
|
||||||
|
}
|
||||||
|
print_grammar_char(file, elem.value);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (is_char_element(elem)) {
|
||||||
|
switch (rule[i + 1].type) {
|
||||||
|
case LLAMA_GRETYPE_CHAR_ALT:
|
||||||
|
case LLAMA_GRETYPE_CHAR_RNG_UPPER:
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
fprintf(file, "] ");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fprintf(file, "\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
void print_grammar(FILE * file, const parse_state & state) {
|
||||||
|
try {
|
||||||
|
std::map<uint32_t, std::string> symbol_id_names;
|
||||||
|
for (const auto & kv : state.symbol_ids) {
|
||||||
|
symbol_id_names[kv.second] = kv.first;
|
||||||
|
}
|
||||||
|
for (size_t i = 0, end = state.rules.size(); i < end; i++) {
|
||||||
|
// fprintf(file, "%zu: ", i);
|
||||||
|
// print_rule_binary(file, state.rules[i]);
|
||||||
|
print_rule(file, uint32_t(i), state.rules[i], symbol_id_names);
|
||||||
|
// fprintf(file, "\n");
|
||||||
|
}
|
||||||
|
} catch (const std::exception & err) {
|
||||||
|
fprintf(stderr, "\n%s: error printing grammar: %s\n", __func__, err.what());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<const llama_grammar_element *> parse_state::c_rules() {
|
||||||
|
std::vector<const llama_grammar_element *> ret;
|
||||||
|
ret.reserve(rules.size());
|
||||||
|
for (const auto & rule : rules) {
|
||||||
|
ret.push_back(rule.data());
|
||||||
|
}
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
}
|
||||||
29
common/grammar-parser.h
Normal file
29
common/grammar-parser.h
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
// Implements a parser for an extended Backus-Naur form (BNF), producing the
|
||||||
|
// binary context-free grammar format specified by llama.h. Supports character
|
||||||
|
// ranges, grouping, and repetition operators. As an example, a grammar for
|
||||||
|
// arithmetic might look like:
|
||||||
|
//
|
||||||
|
// root ::= expr
|
||||||
|
// expr ::= term ([-+*/] term)*
|
||||||
|
// term ::= num | "(" space expr ")" space
|
||||||
|
// num ::= [0-9]+ space
|
||||||
|
// space ::= [ \t\n]*
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
#include "llama.h"
|
||||||
|
#include <vector>
|
||||||
|
#include <map>
|
||||||
|
#include <cstdint>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
namespace grammar_parser {
|
||||||
|
struct parse_state {
|
||||||
|
std::map<std::string, uint32_t> symbol_ids;
|
||||||
|
std::vector<std::vector<llama_grammar_element>> rules;
|
||||||
|
|
||||||
|
std::vector<const llama_grammar_element *> c_rules();
|
||||||
|
};
|
||||||
|
|
||||||
|
parse_state parse(const char * src);
|
||||||
|
void print_grammar(FILE * file, const parse_state & state);
|
||||||
|
}
|
||||||
764
common/json-schema-to-grammar.cpp
Normal file
764
common/json-schema-to-grammar.cpp
Normal file
|
|
@ -0,0 +1,764 @@
|
||||||
|
#include "json-schema-to-grammar.h"
|
||||||
|
#include <algorithm>
|
||||||
|
#include <fstream>
|
||||||
|
#include <map>
|
||||||
|
#include <regex>
|
||||||
|
#include <sstream>
|
||||||
|
#include <string>
|
||||||
|
#include <unordered_map>
|
||||||
|
#include <unordered_set>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
using json = nlohmann::ordered_json;
|
||||||
|
|
||||||
|
template <typename Iterator>
|
||||||
|
static std::string join(Iterator begin, Iterator end, const std::string & separator);
|
||||||
|
|
||||||
|
static std::string repeat(const std::string & str, size_t n);
|
||||||
|
|
||||||
|
static std::string build_repetition(const std::string & item_rule, int min_items, int max_items, const std::string & separator_rule = "", bool item_rule_is_literal = false) {
|
||||||
|
if (separator_rule.empty()) {
|
||||||
|
if (min_items == 0 && max_items == 1) {
|
||||||
|
return item_rule + "?";
|
||||||
|
} else if (min_items == 1 && max_items == std::numeric_limits<int>::max()) {
|
||||||
|
return item_rule + "+";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string result;
|
||||||
|
if (min_items > 0) {
|
||||||
|
if (item_rule_is_literal && separator_rule.empty()) {
|
||||||
|
result = "\"" + repeat(std::string(item_rule.begin() + 1, item_rule.end() - 1), min_items) + "\"";
|
||||||
|
} else {
|
||||||
|
std::vector<std::string> items(min_items, item_rule);
|
||||||
|
result = join(items.begin(), items.end(), separator_rule.empty() ? " " : " " + separator_rule + " ");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::function<std::string(int, bool)> opt_repetitions = [&](int up_to_n, bool prefix_with_sep) -> std::string {
|
||||||
|
auto content = prefix_with_sep && !separator_rule.empty() ? separator_rule + " " + item_rule : item_rule;
|
||||||
|
|
||||||
|
if (up_to_n == 0) {
|
||||||
|
return "";
|
||||||
|
} else if (up_to_n == 1) {
|
||||||
|
return "(" + content + ")?";
|
||||||
|
} else if (!separator_rule.empty() && !prefix_with_sep) {
|
||||||
|
return "(" + content + " " + opt_repetitions(up_to_n - 1, true) + ")?";
|
||||||
|
} else {
|
||||||
|
std::string res = repeat("(" + content + " ", up_to_n);
|
||||||
|
// strip trailing space
|
||||||
|
res = res.substr(0, res.length() - 1);
|
||||||
|
res += repeat(")?", up_to_n);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (min_items > 0 && max_items != min_items) {
|
||||||
|
result += " ";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (max_items != std::numeric_limits<int>::max()) {
|
||||||
|
result += opt_repetitions(max_items - min_items, min_items > 0);
|
||||||
|
} else {
|
||||||
|
std::string item_operator = "(" + (separator_rule.empty() ? "" : separator_rule + " ") + item_rule + ")";
|
||||||
|
if (min_items == 0 && !separator_rule.empty()) {
|
||||||
|
result = "(" + item_rule + " " + item_operator + "*)?";
|
||||||
|
} else {
|
||||||
|
result += item_operator + "*";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string SPACE_RULE = "\" \"?";
|
||||||
|
|
||||||
|
struct BuiltinRule {
|
||||||
|
std::string content;
|
||||||
|
std::vector<std::string> deps;
|
||||||
|
};
|
||||||
|
|
||||||
|
const std::string _up_to_15_digits = build_repetition("[0-9]", 0, 15);
|
||||||
|
|
||||||
|
std::unordered_map<std::string, BuiltinRule> PRIMITIVE_RULES = {
|
||||||
|
{"boolean", {"(\"true\" | \"false\") space", {}}},
|
||||||
|
{"decimal-part", {"[0-9] " + _up_to_15_digits, {}}},
|
||||||
|
{"integral-part", {"[0-9] | [1-9] " + _up_to_15_digits, {}}},
|
||||||
|
{"number", {"(\"-\"? integral-part) (\".\" decimal-part)? ([eE] [-+]? integral-part)? space", {"integral-part", "decimal-part"}}},
|
||||||
|
{"integer", {"(\"-\"? integral-part) space", {"integral-part"}}},
|
||||||
|
{"value", {"object | array | string | number | boolean | null", {"object", "array", "string", "number", "boolean", "null"}}},
|
||||||
|
{"object", {"\"{\" space ( string \":\" space value (\",\" space string \":\" space value)* )? \"}\" space", {"string", "value"}}},
|
||||||
|
{"array", {"\"[\" space ( value (\",\" space value)* )? \"]\" space", {"value"}}},
|
||||||
|
{"uuid", {"\"\\\"\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "
|
||||||
|
"\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "
|
||||||
|
"\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "
|
||||||
|
"\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] "
|
||||||
|
"\"-\" [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] \"\\\"\" space", {}}},
|
||||||
|
{"char", {"[^\"\\\\] | \"\\\\\" ([\"\\\\/bfnrt] | \"u\" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])", {}}},
|
||||||
|
{"string", {"\"\\\"\" char* \"\\\"\" space", {"char"}}},
|
||||||
|
{"null", {"\"null\" space", {}}},
|
||||||
|
};
|
||||||
|
|
||||||
|
std::unordered_map<std::string, BuiltinRule> STRING_FORMAT_RULES = {
|
||||||
|
{"date", {"[0-9] [0-9] [0-9] [0-9] \"-\" ( \"0\" [1-9] | \"1\" [0-2] ) \"-\" ( \"0\" [1-9] | [1-2] [0-9] | \"3\" [0-1] )", {}}},
|
||||||
|
{"time", {"([01] [0-9] | \"2\" [0-3]) \":\" [0-5] [0-9] \":\" [0-5] [0-9] ( \".\" [0-9] [0-9] [0-9] )? ( \"Z\" | ( \"+\" | \"-\" ) ( [01] [0-9] | \"2\" [0-3] ) \":\" [0-5] [0-9] )", {}}},
|
||||||
|
{"date-time", {"date \"T\" time", {"date", "time"}}},
|
||||||
|
{"date-string", {"\"\\\"\" date \"\\\"\" space", {"date"}}},
|
||||||
|
{"time-string", {"\"\\\"\" time \"\\\"\" space", {"time"}}},
|
||||||
|
{"date-time-string", {"\"\\\"\" date-time \"\\\"\" space", {"date-time"}}}
|
||||||
|
};
|
||||||
|
|
||||||
|
static bool is_reserved_name(const std::string & name) {
|
||||||
|
static std::unordered_set<std::string> RESERVED_NAMES;
|
||||||
|
if (RESERVED_NAMES.empty()) {
|
||||||
|
RESERVED_NAMES.insert("root");
|
||||||
|
for (const auto &p : PRIMITIVE_RULES) RESERVED_NAMES.insert(p.first);
|
||||||
|
for (const auto &p : STRING_FORMAT_RULES) RESERVED_NAMES.insert(p.first);
|
||||||
|
}
|
||||||
|
return RESERVED_NAMES.find(name) != RESERVED_NAMES.end();
|
||||||
|
}
|
||||||
|
|
||||||
|
std::regex INVALID_RULE_CHARS_RE("[^a-zA-Z0-9-]+");
|
||||||
|
std::regex GRAMMAR_LITERAL_ESCAPE_RE("[\r\n\"]");
|
||||||
|
std::regex GRAMMAR_RANGE_LITERAL_ESCAPE_RE("[\r\n\"\\]\\-\\\\]");
|
||||||
|
std::unordered_map<char, std::string> GRAMMAR_LITERAL_ESCAPES = {
|
||||||
|
{'\r', "\\r"}, {'\n', "\\n"}, {'"', "\\\""}, {'-', "\\-"}, {']', "\\]"}
|
||||||
|
};
|
||||||
|
|
||||||
|
std::unordered_set<char> NON_LITERAL_SET = {'|', '.', '(', ')', '[', ']', '{', '}', '*', '+', '?'};
|
||||||
|
std::unordered_set<char> ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = {'[', ']', '(', ')', '|', '{', '}', '*', '+', '?'};
|
||||||
|
|
||||||
|
template <typename Iterator>
|
||||||
|
std::string join(Iterator begin, Iterator end, const std::string & separator) {
|
||||||
|
std::ostringstream result;
|
||||||
|
if (begin != end) {
|
||||||
|
result << *begin;
|
||||||
|
for (Iterator it = begin + 1; it != end; ++it) {
|
||||||
|
result << separator << *it;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result.str();
|
||||||
|
}
|
||||||
|
|
||||||
|
static std::vector<std::string> split(const std::string & str, const std::string & delimiter) {
|
||||||
|
std::vector<std::string> tokens;
|
||||||
|
size_t start = 0;
|
||||||
|
size_t end = str.find(delimiter);
|
||||||
|
|
||||||
|
while (end != std::string::npos) {
|
||||||
|
tokens.push_back(str.substr(start, end - start));
|
||||||
|
start = end + delimiter.length();
|
||||||
|
end = str.find(delimiter, start);
|
||||||
|
}
|
||||||
|
|
||||||
|
tokens.push_back(str.substr(start));
|
||||||
|
|
||||||
|
return tokens;
|
||||||
|
}
|
||||||
|
|
||||||
|
static std::string repeat(const std::string & str, size_t n) {
|
||||||
|
if (n == 0) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string result;
|
||||||
|
result.reserve(str.length() * n);
|
||||||
|
|
||||||
|
for (size_t i = 0; i < n; ++i) {
|
||||||
|
result += str;
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
static std::string replacePattern(const std::string & input, const std::regex & regex, const std::function<std::string(const std::smatch &)> & replacement) {
|
||||||
|
std::smatch match;
|
||||||
|
std::string result;
|
||||||
|
|
||||||
|
std::string::const_iterator searchStart(input.cbegin());
|
||||||
|
std::string::const_iterator searchEnd(input.cend());
|
||||||
|
|
||||||
|
while (std::regex_search(searchStart, searchEnd, match, regex)) {
|
||||||
|
result.append(searchStart, searchStart + match.position());
|
||||||
|
result.append(replacement(match));
|
||||||
|
searchStart = match.suffix().first;
|
||||||
|
}
|
||||||
|
|
||||||
|
result.append(searchStart, searchEnd);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
static std::string format_literal(const std::string & literal) {
|
||||||
|
std::string escaped = replacePattern(literal, GRAMMAR_LITERAL_ESCAPE_RE, [&](const std::smatch & match) {
|
||||||
|
char c = match.str()[0];
|
||||||
|
return GRAMMAR_LITERAL_ESCAPES.at(c);
|
||||||
|
});
|
||||||
|
return "\"" + escaped + "\"";
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class SchemaConverter {
|
||||||
|
private:
|
||||||
|
std::function<json(const std::string &)> _fetch_json;
|
||||||
|
bool _dotall;
|
||||||
|
std::map<std::string, std::string> _rules;
|
||||||
|
std::unordered_map<std::string, json> _refs;
|
||||||
|
std::unordered_set<std::string> _refs_being_resolved;
|
||||||
|
std::vector<std::string> _errors;
|
||||||
|
std::vector<std::string> _warnings;
|
||||||
|
|
||||||
|
std::string _add_rule(const std::string & name, const std::string & rule) {
|
||||||
|
std::string esc_name = regex_replace(name, INVALID_RULE_CHARS_RE, "-");
|
||||||
|
if (_rules.find(esc_name) == _rules.end() || _rules[esc_name] == rule) {
|
||||||
|
_rules[esc_name] = rule;
|
||||||
|
return esc_name;
|
||||||
|
} else {
|
||||||
|
int i = 0;
|
||||||
|
while (_rules.find(esc_name + std::to_string(i)) != _rules.end() && _rules[esc_name + std::to_string(i)] != rule) {
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
std::string key = esc_name + std::to_string(i);
|
||||||
|
_rules[key] = rule;
|
||||||
|
return key;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string _generate_union_rule(const std::string & name, const std::vector<json> & alt_schemas) {
|
||||||
|
std::vector<std::string> rules;
|
||||||
|
for (size_t i = 0; i < alt_schemas.size(); i++) {
|
||||||
|
rules.push_back(visit(alt_schemas[i], name + (name.empty() ? "alternative-" : "-") + std::to_string(i)));
|
||||||
|
}
|
||||||
|
return join(rules.begin(), rules.end(), " | ");
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string _visit_pattern(const std::string & pattern, const std::string & name) {
|
||||||
|
if (!(pattern.front() == '^' && pattern.back() == '$')) {
|
||||||
|
_errors.push_back("Pattern must start with '^' and end with '$'");
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
std::string sub_pattern = pattern.substr(1, pattern.length() - 2);
|
||||||
|
std::unordered_map<std::string, std::string> sub_rule_ids;
|
||||||
|
|
||||||
|
size_t i = 0;
|
||||||
|
size_t length = sub_pattern.length();
|
||||||
|
|
||||||
|
using literal_or_rule = std::pair<std::string, bool>;
|
||||||
|
auto to_rule = [&](const literal_or_rule & ls) {
|
||||||
|
auto is_literal = ls.second;
|
||||||
|
auto s = ls.first;
|
||||||
|
return is_literal ? "\"" + s + "\"" : s;
|
||||||
|
};
|
||||||
|
std::function<literal_or_rule()> transform = [&]() -> literal_or_rule {
|
||||||
|
size_t start = i;
|
||||||
|
std::vector<literal_or_rule> seq;
|
||||||
|
|
||||||
|
auto get_dot = [&]() {
|
||||||
|
std::string rule;
|
||||||
|
if (_dotall) {
|
||||||
|
rule = "[\\U00000000-\\U0010FFFF]";
|
||||||
|
} else {
|
||||||
|
rule = "[^\\x0A\\x0D]";
|
||||||
|
}
|
||||||
|
return _add_rule("dot", rule);
|
||||||
|
};
|
||||||
|
|
||||||
|
// Joins the sequence, merging consecutive literals together.
|
||||||
|
auto join_seq = [&]() {
|
||||||
|
std::vector<literal_or_rule> ret;
|
||||||
|
|
||||||
|
std::string literal;
|
||||||
|
auto flush_literal = [&]() {
|
||||||
|
if (literal.empty()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
ret.push_back(std::make_pair(literal, true));
|
||||||
|
literal.clear();
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const auto & item : seq) {
|
||||||
|
auto is_literal = item.second;
|
||||||
|
if (is_literal) {
|
||||||
|
literal += item.first;
|
||||||
|
} else {
|
||||||
|
flush_literal();
|
||||||
|
ret.push_back(item);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
flush_literal();
|
||||||
|
|
||||||
|
std::vector<std::string> results;
|
||||||
|
for (const auto & item : ret) {
|
||||||
|
results.push_back(to_rule(item));
|
||||||
|
}
|
||||||
|
return std::make_pair(join(results.begin(), results.end(), " "), false);
|
||||||
|
};
|
||||||
|
|
||||||
|
while (i < length) {
|
||||||
|
char c = sub_pattern[i];
|
||||||
|
if (c == '.') {
|
||||||
|
seq.push_back(std::make_pair(get_dot(), false));
|
||||||
|
i++;
|
||||||
|
} else if (c == '(') {
|
||||||
|
i++;
|
||||||
|
if (i < length) {
|
||||||
|
if (sub_pattern[i] == '?') {
|
||||||
|
_warnings.push_back("Unsupported pattern syntax");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
seq.push_back(std::make_pair("(" + to_rule(transform()) + ")", false));
|
||||||
|
} else if (c == ')') {
|
||||||
|
i++;
|
||||||
|
if (start > 0 && sub_pattern[start - 1] != '(') {
|
||||||
|
_errors.push_back("Unbalanced parentheses");
|
||||||
|
}
|
||||||
|
return join_seq();
|
||||||
|
} else if (c == '[') {
|
||||||
|
std::string square_brackets = std::string(1, c);
|
||||||
|
i++;
|
||||||
|
while (i < length && sub_pattern[i] != ']') {
|
||||||
|
if (sub_pattern[i] == '\\') {
|
||||||
|
square_brackets += sub_pattern.substr(i, 2);
|
||||||
|
i += 2;
|
||||||
|
} else {
|
||||||
|
square_brackets += sub_pattern[i];
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (i >= length) {
|
||||||
|
_errors.push_back("Unbalanced square brackets");
|
||||||
|
}
|
||||||
|
square_brackets += ']';
|
||||||
|
i++;
|
||||||
|
seq.push_back(std::make_pair(square_brackets, false));
|
||||||
|
} else if (c == '|') {
|
||||||
|
seq.push_back(std::make_pair("|", false));
|
||||||
|
i++;
|
||||||
|
} else if (c == '*' || c == '+' || c == '?') {
|
||||||
|
seq.back() = std::make_pair(to_rule(seq.back()) + c, false);
|
||||||
|
i++;
|
||||||
|
} else if (c == '{') {
|
||||||
|
std::string curly_brackets = std::string(1, c);
|
||||||
|
i++;
|
||||||
|
while (i < length && sub_pattern[i] != '}') {
|
||||||
|
curly_brackets += sub_pattern[i];
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
if (i >= length) {
|
||||||
|
_errors.push_back("Unbalanced curly brackets");
|
||||||
|
}
|
||||||
|
curly_brackets += '}';
|
||||||
|
i++;
|
||||||
|
auto nums = split(curly_brackets.substr(1, curly_brackets.length() - 2), ",");
|
||||||
|
int min_times = 0;
|
||||||
|
int max_times = std::numeric_limits<int>::max();
|
||||||
|
try {
|
||||||
|
if (nums.size() == 1) {
|
||||||
|
min_times = max_times = std::stoi(nums[0]);
|
||||||
|
} else if (nums.size() != 2) {
|
||||||
|
_errors.push_back("Wrong number of values in curly brackets");
|
||||||
|
} else {
|
||||||
|
if (!nums[0].empty()) {
|
||||||
|
min_times = std::stoi(nums[0]);
|
||||||
|
}
|
||||||
|
if (!nums[1].empty()) {
|
||||||
|
max_times = std::stoi(nums[1]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (const std::invalid_argument & e) {
|
||||||
|
_errors.push_back("Invalid number in curly brackets");
|
||||||
|
return std::make_pair("", false);
|
||||||
|
}
|
||||||
|
auto &last = seq.back();
|
||||||
|
auto &sub = last.first;
|
||||||
|
auto sub_is_literal = last.second;
|
||||||
|
|
||||||
|
if (!sub_is_literal) {
|
||||||
|
std::string & sub_id = sub_rule_ids[sub];
|
||||||
|
if (sub_id.empty()) {
|
||||||
|
sub_id = _add_rule(name + "-" + std::to_string(sub_rule_ids.size()), sub);
|
||||||
|
}
|
||||||
|
sub = sub_id;
|
||||||
|
}
|
||||||
|
seq.back().first = build_repetition(
|
||||||
|
sub_is_literal ? "\"" + sub + "\"" : sub,
|
||||||
|
min_times,
|
||||||
|
max_times,
|
||||||
|
"",
|
||||||
|
sub_is_literal
|
||||||
|
);
|
||||||
|
seq.back().second = false;
|
||||||
|
} else {
|
||||||
|
std::string literal;
|
||||||
|
auto is_non_literal = [&](char c) {
|
||||||
|
return NON_LITERAL_SET.find(c) != NON_LITERAL_SET.end();
|
||||||
|
};
|
||||||
|
while (i < length) {
|
||||||
|
if (sub_pattern[i] == '\\' && i < length - 1) {
|
||||||
|
char next = sub_pattern[i + 1];
|
||||||
|
if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.find(next) != ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.end()) {
|
||||||
|
i++;
|
||||||
|
literal += sub_pattern[i];
|
||||||
|
i++;
|
||||||
|
} else {
|
||||||
|
literal += sub_pattern.substr(i, 2);
|
||||||
|
i += 2;
|
||||||
|
}
|
||||||
|
} else if (sub_pattern[i] == '"') {
|
||||||
|
literal += "\\\"";
|
||||||
|
i++;
|
||||||
|
} else if (!is_non_literal(sub_pattern[i]) &&
|
||||||
|
(i == length - 1 || literal.empty() || sub_pattern[i + 1] == '.' || !is_non_literal(sub_pattern[i + 1]))) {
|
||||||
|
literal += sub_pattern[i];
|
||||||
|
i++;
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!literal.empty()) {
|
||||||
|
seq.push_back(std::make_pair(literal, true));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return join_seq();
|
||||||
|
};
|
||||||
|
return _add_rule(name, "\"\\\"\" " + to_rule(transform()) + " \"\\\"\" space");
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string _resolve_ref(const std::string & ref) {
|
||||||
|
std::string ref_name = ref.substr(ref.find_last_of('/') + 1);
|
||||||
|
if (_rules.find(ref_name) == _rules.end() && _refs_being_resolved.find(ref) == _refs_being_resolved.end()) {
|
||||||
|
_refs_being_resolved.insert(ref);
|
||||||
|
json resolved = _refs[ref];
|
||||||
|
ref_name = visit(resolved, ref_name);
|
||||||
|
_refs_being_resolved.erase(ref);
|
||||||
|
}
|
||||||
|
return ref_name;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string _build_object_rule(
|
||||||
|
const std::vector<std::pair<std::string, json>> & properties,
|
||||||
|
const std::unordered_set<std::string> & required,
|
||||||
|
const std::string & name,
|
||||||
|
const json & additional_properties)
|
||||||
|
{
|
||||||
|
std::vector<std::string> required_props;
|
||||||
|
std::vector<std::string> optional_props;
|
||||||
|
std::unordered_map<std::string, std::string> prop_kv_rule_names;
|
||||||
|
for (const auto & kv : properties) {
|
||||||
|
const auto &prop_name = kv.first;
|
||||||
|
const auto &prop_schema = kv.second;
|
||||||
|
|
||||||
|
std::string prop_rule_name = visit(prop_schema, name + (name.empty() ? "" : "-") + prop_name);
|
||||||
|
prop_kv_rule_names[prop_name] = _add_rule(
|
||||||
|
name + (name.empty() ? "" : "-") + prop_name + "-kv",
|
||||||
|
format_literal(json(prop_name).dump()) + " space \":\" space " + prop_rule_name
|
||||||
|
);
|
||||||
|
if (required.find(prop_name) != required.end()) {
|
||||||
|
required_props.push_back(prop_name);
|
||||||
|
} else {
|
||||||
|
optional_props.push_back(prop_name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (additional_properties.is_object() || (additional_properties.is_boolean() && additional_properties.get<bool>())) {
|
||||||
|
std::string sub_name = name + (name.empty() ? "" : "-") + "additional";
|
||||||
|
std::string value_rule = visit(additional_properties.is_object() ? additional_properties : json::object(), sub_name + "-value");
|
||||||
|
std::string kv_rule = _add_rule(sub_name + "-kv", _add_primitive("string", PRIMITIVE_RULES.at("string")) + " \":\" space " + value_rule);
|
||||||
|
prop_kv_rule_names["*"] = kv_rule;
|
||||||
|
optional_props.push_back("*");
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string rule = "\"{\" space ";
|
||||||
|
for (size_t i = 0; i < required_props.size(); i++) {
|
||||||
|
if (i > 0) {
|
||||||
|
rule += " \",\" space ";
|
||||||
|
}
|
||||||
|
rule += prop_kv_rule_names[required_props[i]];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!optional_props.empty()) {
|
||||||
|
rule += " (";
|
||||||
|
if (!required_props.empty()) {
|
||||||
|
rule += " \",\" space ( ";
|
||||||
|
}
|
||||||
|
|
||||||
|
std::function<std::string(const std::vector<std::string> &, bool)> get_recursive_refs = [&](const std::vector<std::string> & ks, bool first_is_optional) {
|
||||||
|
std::string res;
|
||||||
|
if (ks.empty()) {
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
std::string k = ks[0];
|
||||||
|
std::string kv_rule_name = prop_kv_rule_names[k];
|
||||||
|
if (k == "*") {
|
||||||
|
res = _add_rule(
|
||||||
|
name + (name.empty() ? "" : "-") + "additional-kvs",
|
||||||
|
kv_rule_name + " ( \",\" space " + kv_rule_name + " )*"
|
||||||
|
);
|
||||||
|
} else if (first_is_optional) {
|
||||||
|
res = "( \",\" space " + kv_rule_name + " )?";
|
||||||
|
} else {
|
||||||
|
res = kv_rule_name;
|
||||||
|
}
|
||||||
|
if (ks.size() > 1) {
|
||||||
|
res += " " + _add_rule(
|
||||||
|
name + (name.empty() ? "" : "-") + k + "-rest",
|
||||||
|
get_recursive_refs(std::vector<std::string>(ks.begin() + 1, ks.end()), true)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
|
||||||
|
for (size_t i = 0; i < optional_props.size(); i++) {
|
||||||
|
if (i > 0) {
|
||||||
|
rule += " | ";
|
||||||
|
}
|
||||||
|
rule += get_recursive_refs(std::vector<std::string>(optional_props.begin() + i, optional_props.end()), false);
|
||||||
|
}
|
||||||
|
if (!required_props.empty()) {
|
||||||
|
rule += " )";
|
||||||
|
}
|
||||||
|
rule += " )?";
|
||||||
|
}
|
||||||
|
|
||||||
|
rule += " \"}\" space";
|
||||||
|
|
||||||
|
return rule;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string _add_primitive(const std::string & name, const BuiltinRule & rule) {
|
||||||
|
auto n = _add_rule(name, rule.content);
|
||||||
|
for (const auto & dep : rule.deps) {
|
||||||
|
BuiltinRule dep_rule;
|
||||||
|
auto it = PRIMITIVE_RULES.find(dep);
|
||||||
|
if (it == PRIMITIVE_RULES.end()) {
|
||||||
|
it = STRING_FORMAT_RULES.find(dep);
|
||||||
|
if (it == STRING_FORMAT_RULES.end()) {
|
||||||
|
_errors.push_back("Rule " + dep + " not known");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (_rules.find(dep) == _rules.end()) {
|
||||||
|
_add_primitive(dep, it->second);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
|
public:
|
||||||
|
SchemaConverter(
|
||||||
|
const std::function<json(const std::string &)> & fetch_json,
|
||||||
|
bool dotall)
|
||||||
|
: _fetch_json(fetch_json), _dotall(dotall)
|
||||||
|
{
|
||||||
|
_rules["space"] = SPACE_RULE;
|
||||||
|
}
|
||||||
|
|
||||||
|
void resolve_refs(json & schema, const std::string & url) {
|
||||||
|
/*
|
||||||
|
* Resolves all $ref fields in the given schema, fetching any remote schemas,
|
||||||
|
* replacing each $ref with absolute reference URL and populates _refs with the
|
||||||
|
* respective referenced (sub)schema dictionaries.
|
||||||
|
*/
|
||||||
|
std::function<void(json &)> visit_refs = [&](json & n) {
|
||||||
|
if (n.is_array()) {
|
||||||
|
for (auto & x : n) {
|
||||||
|
visit_refs(x);
|
||||||
|
}
|
||||||
|
} else if (n.is_object()) {
|
||||||
|
if (n.contains("$ref")) {
|
||||||
|
std::string ref = n["$ref"];
|
||||||
|
if (_refs.find(ref) == _refs.end()) {
|
||||||
|
json target;
|
||||||
|
if (ref.find("https://") == 0) {
|
||||||
|
std::string base_url = ref.substr(0, ref.find('#'));
|
||||||
|
auto it = _refs.find(base_url);
|
||||||
|
if (it != _refs.end()) {
|
||||||
|
target = it->second;
|
||||||
|
} else {
|
||||||
|
// Fetch the referenced schema and resolve its refs
|
||||||
|
auto referenced = _fetch_json(ref);
|
||||||
|
resolve_refs(referenced, base_url);
|
||||||
|
_refs[base_url] = referenced;
|
||||||
|
}
|
||||||
|
if (ref.find('#') == std::string::npos || ref.substr(ref.find('#') + 1).empty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} else if (ref.find("#/") == 0) {
|
||||||
|
target = schema;
|
||||||
|
n["$ref"] = url + ref;
|
||||||
|
ref = url + ref;
|
||||||
|
} else {
|
||||||
|
_errors.push_back("Unsupported ref: " + ref);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
std::string pointer = ref.substr(ref.find('#') + 1);
|
||||||
|
std::vector<std::string> tokens = split(pointer, "/");
|
||||||
|
for (size_t i = 1; i < tokens.size(); ++i) {
|
||||||
|
std::string sel = tokens[i];
|
||||||
|
if (target.is_null() || !target.contains(sel)) {
|
||||||
|
_errors.push_back("Error resolving ref " + ref + ": " + sel + " not in " + target.dump());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
target = target[sel];
|
||||||
|
}
|
||||||
|
_refs[ref] = target;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for (auto & kv : n.items()) {
|
||||||
|
visit_refs(kv.value());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
visit_refs(schema);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string _generate_constant_rule(const json & value) {
|
||||||
|
return format_literal(value.dump());
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string visit(const json & schema, const std::string & name) {
|
||||||
|
json schema_type = schema.contains("type") ? schema["type"] : json();
|
||||||
|
std::string schema_format = schema.contains("format") ? schema["format"].get<std::string>() : "";
|
||||||
|
std::string rule_name = is_reserved_name(name) ? name + "-" : name.empty() ? "root" : name;
|
||||||
|
|
||||||
|
if (schema.contains("$ref")) {
|
||||||
|
return _add_rule(rule_name, _resolve_ref(schema["$ref"]));
|
||||||
|
} else if (schema.contains("oneOf") || schema.contains("anyOf")) {
|
||||||
|
std::vector<json> alt_schemas = schema.contains("oneOf") ? schema["oneOf"].get<std::vector<json>>() : schema["anyOf"].get<std::vector<json>>();
|
||||||
|
return _add_rule(rule_name, _generate_union_rule(name, alt_schemas));
|
||||||
|
} else if (schema_type.is_array()) {
|
||||||
|
std::vector<json> schema_types;
|
||||||
|
for (const auto & t : schema_type) {
|
||||||
|
schema_types.push_back({{"type", t}});
|
||||||
|
}
|
||||||
|
return _add_rule(rule_name, _generate_union_rule(name, schema_types));
|
||||||
|
} else if (schema.contains("const")) {
|
||||||
|
return _add_rule(rule_name, _generate_constant_rule(schema["const"]));
|
||||||
|
} else if (schema.contains("enum")) {
|
||||||
|
std::vector<std::string> enum_values;
|
||||||
|
for (const auto & v : schema["enum"]) {
|
||||||
|
enum_values.push_back(_generate_constant_rule(v));
|
||||||
|
}
|
||||||
|
return _add_rule(rule_name, join(enum_values.begin(), enum_values.end(), " | "));
|
||||||
|
} else if ((schema_type.is_null() || schema_type == "object")
|
||||||
|
&& (schema.contains("properties") ||
|
||||||
|
(schema.contains("additionalProperties") && schema["additionalProperties"] != true))) {
|
||||||
|
std::unordered_set<std::string> required;
|
||||||
|
if (schema.contains("required") && schema["required"].is_array()) {
|
||||||
|
for (const auto & item : schema["required"]) {
|
||||||
|
if (item.is_string()) {
|
||||||
|
required.insert(item.get<std::string>());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std::vector<std::pair<std::string, json>> properties;
|
||||||
|
if (schema.contains("properties")) {
|
||||||
|
for (const auto & prop : schema["properties"].items()) {
|
||||||
|
properties.emplace_back(prop.key(), prop.value());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return _add_rule(rule_name,
|
||||||
|
_build_object_rule(
|
||||||
|
properties, required, name,
|
||||||
|
schema.contains("additionalProperties") ? schema["additionalProperties"] : json()));
|
||||||
|
} else if ((schema_type.is_null() || schema_type == "object") && schema.contains("allOf")) {
|
||||||
|
std::unordered_set<std::string> required;
|
||||||
|
std::vector<std::pair<std::string, json>> properties;
|
||||||
|
std::string hybrid_name = name;
|
||||||
|
std::function<void(const json &, bool)> add_component = [&](const json & comp_schema, bool is_required) {
|
||||||
|
if (comp_schema.contains("$ref")) {
|
||||||
|
add_component(_refs[comp_schema["$ref"]], is_required);
|
||||||
|
} else if (comp_schema.contains("properties")) {
|
||||||
|
for (const auto & prop : comp_schema["properties"].items()) {
|
||||||
|
properties.emplace_back(prop.key(), prop.value());
|
||||||
|
if (is_required) {
|
||||||
|
required.insert(prop.key());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// todo warning
|
||||||
|
}
|
||||||
|
};
|
||||||
|
for (auto & t : schema["allOf"]) {
|
||||||
|
if (t.contains("anyOf")) {
|
||||||
|
for (auto & tt : t["anyOf"]) {
|
||||||
|
add_component(tt, false);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
add_component(t, true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json()));
|
||||||
|
} else if ((schema_type.is_null() || schema_type == "array") && (schema.contains("items") || schema.contains("prefixItems"))) {
|
||||||
|
json items = schema.contains("items") ? schema["items"] : schema["prefixItems"];
|
||||||
|
if (items.is_array()) {
|
||||||
|
std::string rule = "\"[\" space ";
|
||||||
|
for (size_t i = 0; i < items.size(); i++) {
|
||||||
|
if (i > 0) {
|
||||||
|
rule += " \",\" space ";
|
||||||
|
}
|
||||||
|
rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i));
|
||||||
|
}
|
||||||
|
rule += " \"]\" space";
|
||||||
|
return _add_rule(rule_name, rule);
|
||||||
|
} else {
|
||||||
|
std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item");
|
||||||
|
int min_items = schema.contains("minItems") ? schema["minItems"].get<int>() : 0;
|
||||||
|
json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json();
|
||||||
|
int max_items = max_items_json.is_number_integer() ? max_items_json.get<int>() : std::numeric_limits<int>::max();
|
||||||
|
|
||||||
|
return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " \"]\" space");
|
||||||
|
}
|
||||||
|
} else if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) {
|
||||||
|
return _visit_pattern(schema["pattern"], rule_name);
|
||||||
|
} else if ((schema_type.is_null() || schema_type == "string") && std::regex_match(schema_format, std::regex("^uuid[1-5]?$"))) {
|
||||||
|
return _add_primitive(rule_name == "root" ? "root" : schema_format, PRIMITIVE_RULES.at("uuid"));
|
||||||
|
} else if ((schema_type.is_null() || schema_type == "string") && STRING_FORMAT_RULES.find(schema_format + "-string") != STRING_FORMAT_RULES.end()) {
|
||||||
|
auto prim_name = schema_format + "-string";
|
||||||
|
return _add_rule(rule_name, _add_primitive(prim_name, STRING_FORMAT_RULES.at(prim_name)));
|
||||||
|
} else if (schema_type == "string" && (schema.contains("minLength") || schema.contains("maxLength"))) {
|
||||||
|
std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char"));
|
||||||
|
int min_len = schema.contains("minLength") ? schema["minLength"].get<int>() : 0;
|
||||||
|
int max_len = schema.contains("maxLength") ? schema["maxLength"].get<int>() : std::numeric_limits<int>::max();
|
||||||
|
return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\" space");
|
||||||
|
} else if (schema.empty() || schema_type == "object") {
|
||||||
|
return _add_rule(rule_name, _add_primitive("object", PRIMITIVE_RULES.at("object")));
|
||||||
|
} else {
|
||||||
|
if (!schema_type.is_string() || PRIMITIVE_RULES.find(schema_type.get<std::string>()) == PRIMITIVE_RULES.end()) {
|
||||||
|
_errors.push_back("Unrecognized schema: " + schema.dump());
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
// TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero
|
||||||
|
return _add_primitive(rule_name == "root" ? "root" : schema_type.get<std::string>(), PRIMITIVE_RULES.at(schema_type.get<std::string>()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void check_errors() {
|
||||||
|
if (!_errors.empty()) {
|
||||||
|
throw std::runtime_error("JSON schema conversion failed:\n" + join(_errors.begin(), _errors.end(), "\n"));
|
||||||
|
}
|
||||||
|
if (!_warnings.empty()) {
|
||||||
|
fprintf(stderr, "WARNING: JSON schema conversion was incomplete: %s\n", join(_warnings.begin(), _warnings.end(), "; ").c_str());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string format_grammar() {
|
||||||
|
std::stringstream ss;
|
||||||
|
for (const auto & kv : _rules) {
|
||||||
|
ss << kv.first << " ::= " << kv.second << std::endl;
|
||||||
|
}
|
||||||
|
return ss.str();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
std::string json_schema_to_grammar(const json & schema) {
|
||||||
|
SchemaConverter converter([](const std::string &) { return json::object(); }, /* dotall= */ false);
|
||||||
|
auto copy = schema;
|
||||||
|
converter.resolve_refs(copy, "input");
|
||||||
|
converter.visit(copy, "");
|
||||||
|
converter.check_errors();
|
||||||
|
return converter.format_grammar();
|
||||||
|
}
|
||||||
8
common/json-schema-to-grammar.h
Normal file
8
common/json-schema-to-grammar.h
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "ggml.h"
|
||||||
|
// Change JSON_ASSERT from assert() to GGML_ASSERT:
|
||||||
|
#define JSON_ASSERT GGML_ASSERT
|
||||||
|
#include "json.hpp"
|
||||||
|
|
||||||
|
std::string json_schema_to_grammar(const nlohmann::ordered_json& schema);
|
||||||
24766
common/json.hpp
Normal file
24766
common/json.hpp
Normal file
File diff suppressed because it is too large
Load diff
724
common/log.h
Normal file
724
common/log.h
Normal file
|
|
@ -0,0 +1,724 @@
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <chrono>
|
||||||
|
#include <cstring>
|
||||||
|
#include <sstream>
|
||||||
|
#include <iostream>
|
||||||
|
#include <thread>
|
||||||
|
#include <vector>
|
||||||
|
#include <algorithm>
|
||||||
|
#include <cinttypes>
|
||||||
|
|
||||||
|
// --------------------------------
|
||||||
|
//
|
||||||
|
// Basic usage:
|
||||||
|
//
|
||||||
|
// --------
|
||||||
|
//
|
||||||
|
// The LOG() and LOG_TEE() macros are ready to go by default
|
||||||
|
// they do not require any initialization.
|
||||||
|
//
|
||||||
|
// LOGLN() and LOG_TEELN() are variants which automatically
|
||||||
|
// include \n character at the end of the log string.
|
||||||
|
//
|
||||||
|
// LOG() behaves exactly like printf, by default writing to a logfile.
|
||||||
|
// LOG_TEE() additionally, prints to the screen too ( mimics Unix tee command ).
|
||||||
|
//
|
||||||
|
// Default logfile is named
|
||||||
|
// "llama.<threadID>.log"
|
||||||
|
// Default LOG_TEE() secondary output target is
|
||||||
|
// stderr
|
||||||
|
//
|
||||||
|
// Logs can be dynamically disabled or enabled using functions:
|
||||||
|
// log_disable()
|
||||||
|
// and
|
||||||
|
// log_enable()
|
||||||
|
//
|
||||||
|
// A log target can be changed with:
|
||||||
|
// log_set_target( string )
|
||||||
|
// creating and opening, or re-opening a file by string filename
|
||||||
|
// or
|
||||||
|
// log_set_target( FILE* )
|
||||||
|
// allowing to point at stderr, stdout, or any valid FILE* file handler.
|
||||||
|
//
|
||||||
|
// --------
|
||||||
|
//
|
||||||
|
// End of Basic usage.
|
||||||
|
//
|
||||||
|
// --------------------------------
|
||||||
|
|
||||||
|
// Specifies a log target.
|
||||||
|
// default uses log_handler() with "llama.log" log file
|
||||||
|
// this can be changed, by defining LOG_TARGET
|
||||||
|
// like so:
|
||||||
|
//
|
||||||
|
// #define LOG_TARGET (a valid FILE*)
|
||||||
|
// #include "log.h"
|
||||||
|
//
|
||||||
|
// or it can be simply redirected to stdout or stderr
|
||||||
|
// like so:
|
||||||
|
//
|
||||||
|
// #define LOG_TARGET stderr
|
||||||
|
// #include "log.h"
|
||||||
|
//
|
||||||
|
// The log target can also be redirected to a different function
|
||||||
|
// like so:
|
||||||
|
//
|
||||||
|
// #define LOG_TARGET log_handler_different()
|
||||||
|
// #include "log.h"
|
||||||
|
//
|
||||||
|
// FILE* log_handler_different()
|
||||||
|
// {
|
||||||
|
// return stderr;
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// or:
|
||||||
|
//
|
||||||
|
// #define LOG_TARGET log_handler_another_one("somelog.log")
|
||||||
|
// #include "log.h"
|
||||||
|
//
|
||||||
|
// FILE* log_handler_another_one(char*filename)
|
||||||
|
// {
|
||||||
|
// static FILE* logfile = nullptr;
|
||||||
|
// (...)
|
||||||
|
// if( !logfile )
|
||||||
|
// {
|
||||||
|
// fopen(...)
|
||||||
|
// }
|
||||||
|
// (...)
|
||||||
|
// return logfile
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
#ifndef LOG_TARGET
|
||||||
|
#define LOG_TARGET log_handler()
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifndef LOG_TEE_TARGET
|
||||||
|
#define LOG_TEE_TARGET stderr
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// Utility for synchronizing log configuration state
|
||||||
|
// since std::optional was introduced only in c++17
|
||||||
|
enum LogTriState
|
||||||
|
{
|
||||||
|
LogTriStateSame,
|
||||||
|
LogTriStateFalse,
|
||||||
|
LogTriStateTrue
|
||||||
|
};
|
||||||
|
|
||||||
|
// Utility to obtain "pid" like unique process id and use it when creating log files.
|
||||||
|
inline std::string log_get_pid()
|
||||||
|
{
|
||||||
|
static std::string pid;
|
||||||
|
if (pid.empty())
|
||||||
|
{
|
||||||
|
// std::this_thread::get_id() is the most portable way of obtaining a "process id"
|
||||||
|
// it's not the same as "pid" but is unique enough to solve multiple instances
|
||||||
|
// trying to write to the same log.
|
||||||
|
std::stringstream ss;
|
||||||
|
ss << std::this_thread::get_id();
|
||||||
|
pid = ss.str();
|
||||||
|
}
|
||||||
|
|
||||||
|
return pid;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Utility function for generating log file names with unique id based on thread id.
|
||||||
|
// invocation with log_filename_generator( "llama", "log" ) creates a string "llama.<number>.log"
|
||||||
|
// where the number is a runtime id of the current thread.
|
||||||
|
|
||||||
|
#define log_filename_generator(log_file_basename, log_file_extension) log_filename_generator_impl(LogTriStateSame, log_file_basename, log_file_extension)
|
||||||
|
|
||||||
|
// INTERNAL, DO NOT USE
|
||||||
|
inline std::string log_filename_generator_impl(LogTriState multilog, const std::string & log_file_basename, const std::string & log_file_extension)
|
||||||
|
{
|
||||||
|
static bool _multilog = false;
|
||||||
|
|
||||||
|
if (multilog != LogTriStateSame)
|
||||||
|
{
|
||||||
|
_multilog = multilog == LogTriStateTrue;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::stringstream buf;
|
||||||
|
|
||||||
|
buf << log_file_basename;
|
||||||
|
if (_multilog)
|
||||||
|
{
|
||||||
|
buf << ".";
|
||||||
|
buf << log_get_pid();
|
||||||
|
}
|
||||||
|
buf << ".";
|
||||||
|
buf << log_file_extension;
|
||||||
|
|
||||||
|
return buf.str();
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifndef LOG_DEFAULT_FILE_NAME
|
||||||
|
#define LOG_DEFAULT_FILE_NAME log_filename_generator("llama", "log")
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// Utility for turning #define values into string literals
|
||||||
|
// so we can have a define for stderr and
|
||||||
|
// we can print "stderr" instead of literal stderr, etc.
|
||||||
|
#define LOG_STRINGIZE1(s) #s
|
||||||
|
#define LOG_STRINGIZE(s) LOG_STRINGIZE1(s)
|
||||||
|
|
||||||
|
#define LOG_TEE_TARGET_STRING LOG_STRINGIZE(LOG_TEE_TARGET)
|
||||||
|
|
||||||
|
// Allows disabling timestamps.
|
||||||
|
// in order to disable, define LOG_NO_TIMESTAMPS
|
||||||
|
// like so:
|
||||||
|
//
|
||||||
|
// #define LOG_NO_TIMESTAMPS
|
||||||
|
// #include "log.h"
|
||||||
|
//
|
||||||
|
#ifndef LOG_NO_TIMESTAMPS
|
||||||
|
#ifndef _MSC_VER
|
||||||
|
#define LOG_TIMESTAMP_FMT "[%" PRIu64 "] "
|
||||||
|
#define LOG_TIMESTAMP_VAL , (std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>(std::chrono::system_clock::now().time_since_epoch())).count()
|
||||||
|
#else
|
||||||
|
#define LOG_TIMESTAMP_FMT "[%" PRIu64 "] "
|
||||||
|
#define LOG_TIMESTAMP_VAL , (std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>(std::chrono::system_clock::now().time_since_epoch())).count()
|
||||||
|
#endif
|
||||||
|
#else
|
||||||
|
#define LOG_TIMESTAMP_FMT "%s"
|
||||||
|
#define LOG_TIMESTAMP_VAL ,""
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef LOG_TEE_TIMESTAMPS
|
||||||
|
#ifndef _MSC_VER
|
||||||
|
#define LOG_TEE_TIMESTAMP_FMT "[%" PRIu64 "] "
|
||||||
|
#define LOG_TEE_TIMESTAMP_VAL , (std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>(std::chrono::system_clock::now().time_since_epoch())).count()
|
||||||
|
#else
|
||||||
|
#define LOG_TEE_TIMESTAMP_FMT "[%" PRIu64 "] "
|
||||||
|
#define LOG_TEE_TIMESTAMP_VAL , (std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>(std::chrono::system_clock::now().time_since_epoch())).count()
|
||||||
|
#endif
|
||||||
|
#else
|
||||||
|
#define LOG_TEE_TIMESTAMP_FMT "%s"
|
||||||
|
#define LOG_TEE_TIMESTAMP_VAL ,""
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// Allows disabling file/line/function prefix
|
||||||
|
// in order to disable, define LOG_NO_FILE_LINE_FUNCTION
|
||||||
|
// like so:
|
||||||
|
//
|
||||||
|
// #define LOG_NO_FILE_LINE_FUNCTION
|
||||||
|
// #include "log.h"
|
||||||
|
//
|
||||||
|
#ifndef LOG_NO_FILE_LINE_FUNCTION
|
||||||
|
#ifndef _MSC_VER
|
||||||
|
#define LOG_FLF_FMT "[%24s:%5d][%24s] "
|
||||||
|
#define LOG_FLF_VAL , __FILE__, __LINE__, __FUNCTION__
|
||||||
|
#else
|
||||||
|
#define LOG_FLF_FMT "[%24s:%5ld][%24s] "
|
||||||
|
#define LOG_FLF_VAL , __FILE__, __LINE__, __FUNCTION__
|
||||||
|
#endif
|
||||||
|
#else
|
||||||
|
#define LOG_FLF_FMT "%s"
|
||||||
|
#define LOG_FLF_VAL ,""
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef LOG_TEE_FILE_LINE_FUNCTION
|
||||||
|
#ifndef _MSC_VER
|
||||||
|
#define LOG_TEE_FLF_FMT "[%24s:%5d][%24s] "
|
||||||
|
#define LOG_TEE_FLF_VAL , __FILE__, __LINE__, __FUNCTION__
|
||||||
|
#else
|
||||||
|
#define LOG_TEE_FLF_FMT "[%24s:%5ld][%24s] "
|
||||||
|
#define LOG_TEE_FLF_VAL , __FILE__, __LINE__, __FUNCTION__
|
||||||
|
#endif
|
||||||
|
#else
|
||||||
|
#define LOG_TEE_FLF_FMT "%s"
|
||||||
|
#define LOG_TEE_FLF_VAL ,""
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// INTERNAL, DO NOT USE
|
||||||
|
// USE LOG() INSTEAD
|
||||||
|
//
|
||||||
|
#if !defined(_MSC_VER) || defined(__INTEL_LLVM_COMPILER) || defined(__clang__)
|
||||||
|
#define LOG_IMPL(str, ...) \
|
||||||
|
do { \
|
||||||
|
if (LOG_TARGET != nullptr) \
|
||||||
|
{ \
|
||||||
|
fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL, __VA_ARGS__); \
|
||||||
|
fflush(LOG_TARGET); \
|
||||||
|
} \
|
||||||
|
} while (0)
|
||||||
|
#else
|
||||||
|
#define LOG_IMPL(str, ...) \
|
||||||
|
do { \
|
||||||
|
if (LOG_TARGET != nullptr) \
|
||||||
|
{ \
|
||||||
|
fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL "", ##__VA_ARGS__); \
|
||||||
|
fflush(LOG_TARGET); \
|
||||||
|
} \
|
||||||
|
} while (0)
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// INTERNAL, DO NOT USE
|
||||||
|
// USE LOG_TEE() INSTEAD
|
||||||
|
//
|
||||||
|
#if !defined(_MSC_VER) || defined(__INTEL_LLVM_COMPILER) || defined(__clang__)
|
||||||
|
#define LOG_TEE_IMPL(str, ...) \
|
||||||
|
do { \
|
||||||
|
if (LOG_TARGET != nullptr) \
|
||||||
|
{ \
|
||||||
|
fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL, __VA_ARGS__); \
|
||||||
|
fflush(LOG_TARGET); \
|
||||||
|
} \
|
||||||
|
if (LOG_TARGET != nullptr && LOG_TARGET != stdout && LOG_TARGET != stderr && LOG_TEE_TARGET != nullptr) \
|
||||||
|
{ \
|
||||||
|
fprintf(LOG_TEE_TARGET, LOG_TEE_TIMESTAMP_FMT LOG_TEE_FLF_FMT str "%s" LOG_TEE_TIMESTAMP_VAL LOG_TEE_FLF_VAL, __VA_ARGS__); \
|
||||||
|
fflush(LOG_TEE_TARGET); \
|
||||||
|
} \
|
||||||
|
} while (0)
|
||||||
|
#else
|
||||||
|
#define LOG_TEE_IMPL(str, ...) \
|
||||||
|
do { \
|
||||||
|
if (LOG_TARGET != nullptr) \
|
||||||
|
{ \
|
||||||
|
fprintf(LOG_TARGET, LOG_TIMESTAMP_FMT LOG_FLF_FMT str "%s" LOG_TIMESTAMP_VAL LOG_FLF_VAL "", ##__VA_ARGS__); \
|
||||||
|
fflush(LOG_TARGET); \
|
||||||
|
} \
|
||||||
|
if (LOG_TARGET != nullptr && LOG_TARGET != stdout && LOG_TARGET != stderr && LOG_TEE_TARGET != nullptr) \
|
||||||
|
{ \
|
||||||
|
fprintf(LOG_TEE_TARGET, LOG_TEE_TIMESTAMP_FMT LOG_TEE_FLF_FMT str "%s" LOG_TEE_TIMESTAMP_VAL LOG_TEE_FLF_VAL "", ##__VA_ARGS__); \
|
||||||
|
fflush(LOG_TEE_TARGET); \
|
||||||
|
} \
|
||||||
|
} while (0)
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// The '\0' as a last argument, is a trick to bypass the silly
|
||||||
|
// "warning: ISO C++11 requires at least one argument for the "..." in a variadic macro"
|
||||||
|
// so we can have a single macro which can be called just like printf.
|
||||||
|
|
||||||
|
// Main LOG macro.
|
||||||
|
// behaves like printf, and supports arguments the exact same way.
|
||||||
|
//
|
||||||
|
#ifndef _MSC_VER
|
||||||
|
#define LOG(...) LOG_IMPL(__VA_ARGS__, "")
|
||||||
|
#else
|
||||||
|
#define LOG(str, ...) LOG_IMPL("%s" str, "", ##__VA_ARGS__, "")
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// Main TEE macro.
|
||||||
|
// does the same as LOG
|
||||||
|
// and
|
||||||
|
// simultaneously writes stderr.
|
||||||
|
//
|
||||||
|
// Secondary target can be changed just like LOG_TARGET
|
||||||
|
// by defining LOG_TEE_TARGET
|
||||||
|
//
|
||||||
|
#ifndef _MSC_VER
|
||||||
|
#define LOG_TEE(...) LOG_TEE_IMPL(__VA_ARGS__, "")
|
||||||
|
#else
|
||||||
|
#define LOG_TEE(str, ...) LOG_TEE_IMPL("%s" str, "", ##__VA_ARGS__, "")
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// LOG macro variants with auto endline.
|
||||||
|
#ifndef _MSC_VER
|
||||||
|
#define LOGLN(...) LOG_IMPL(__VA_ARGS__, "\n")
|
||||||
|
#define LOG_TEELN(...) LOG_TEE_IMPL(__VA_ARGS__, "\n")
|
||||||
|
#else
|
||||||
|
#define LOGLN(str, ...) LOG_IMPL("%s" str, "", ##__VA_ARGS__, "\n")
|
||||||
|
#define LOG_TEELN(str, ...) LOG_TEE_IMPL("%s" str, "", ##__VA_ARGS__, "\n")
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// INTERNAL, DO NOT USE
|
||||||
|
inline FILE *log_handler1_impl(bool change = false, LogTriState append = LogTriStateSame, LogTriState disable = LogTriStateSame, const std::string & filename = LOG_DEFAULT_FILE_NAME, FILE *target = nullptr)
|
||||||
|
{
|
||||||
|
static bool _initialized = false;
|
||||||
|
static bool _append = false;
|
||||||
|
static bool _disabled = filename.empty() && target == nullptr;
|
||||||
|
static std::string log_current_filename{filename};
|
||||||
|
static FILE *log_current_target{target};
|
||||||
|
static FILE *logfile = nullptr;
|
||||||
|
|
||||||
|
if (change)
|
||||||
|
{
|
||||||
|
if (append != LogTriStateSame)
|
||||||
|
{
|
||||||
|
_append = append == LogTriStateTrue;
|
||||||
|
return logfile;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (disable == LogTriStateTrue)
|
||||||
|
{
|
||||||
|
// Disable primary target
|
||||||
|
_disabled = true;
|
||||||
|
}
|
||||||
|
// If previously disabled, only enable, and keep previous target
|
||||||
|
else if (disable == LogTriStateFalse)
|
||||||
|
{
|
||||||
|
_disabled = false;
|
||||||
|
}
|
||||||
|
// Otherwise, process the arguments
|
||||||
|
else if (log_current_filename != filename || log_current_target != target)
|
||||||
|
{
|
||||||
|
_initialized = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (_disabled)
|
||||||
|
{
|
||||||
|
// Log is disabled
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (_initialized)
|
||||||
|
{
|
||||||
|
// with fallback in case something went wrong
|
||||||
|
return logfile ? logfile : stderr;
|
||||||
|
}
|
||||||
|
|
||||||
|
// do the (re)initialization
|
||||||
|
if (target != nullptr)
|
||||||
|
{
|
||||||
|
if (logfile != nullptr && logfile != stdout && logfile != stderr)
|
||||||
|
{
|
||||||
|
fclose(logfile);
|
||||||
|
}
|
||||||
|
|
||||||
|
log_current_filename = LOG_DEFAULT_FILE_NAME;
|
||||||
|
log_current_target = target;
|
||||||
|
|
||||||
|
logfile = target;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (log_current_filename != filename)
|
||||||
|
{
|
||||||
|
if (logfile != nullptr && logfile != stdout && logfile != stderr)
|
||||||
|
{
|
||||||
|
fclose(logfile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logfile = fopen(filename.c_str(), _append ? "a" : "w");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!logfile)
|
||||||
|
{
|
||||||
|
// Verify whether the file was opened, otherwise fallback to stderr
|
||||||
|
logfile = stderr;
|
||||||
|
|
||||||
|
fprintf(stderr, "Failed to open logfile '%s' with error '%s'\n", filename.c_str(), std::strerror(errno));
|
||||||
|
fflush(stderr);
|
||||||
|
|
||||||
|
// At this point we let the init flag be to true below, and let the target fallback to stderr
|
||||||
|
// otherwise we would repeatedly fopen() which was already unsuccessful
|
||||||
|
}
|
||||||
|
|
||||||
|
_initialized = true;
|
||||||
|
|
||||||
|
return logfile ? logfile : stderr;
|
||||||
|
}
|
||||||
|
|
||||||
|
// INTERNAL, DO NOT USE
|
||||||
|
inline FILE *log_handler2_impl(bool change = false, LogTriState append = LogTriStateSame, LogTriState disable = LogTriStateSame, FILE *target = nullptr, const std::string & filename = LOG_DEFAULT_FILE_NAME)
|
||||||
|
{
|
||||||
|
return log_handler1_impl(change, append, disable, filename, target);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Disables logs entirely at runtime.
|
||||||
|
// Makes LOG() and LOG_TEE() produce no output,
|
||||||
|
// until enabled back.
|
||||||
|
#define log_disable() log_disable_impl()
|
||||||
|
|
||||||
|
// INTERNAL, DO NOT USE
|
||||||
|
inline FILE *log_disable_impl()
|
||||||
|
{
|
||||||
|
return log_handler1_impl(true, LogTriStateSame, LogTriStateTrue);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enables logs at runtime.
|
||||||
|
#define log_enable() log_enable_impl()
|
||||||
|
|
||||||
|
// INTERNAL, DO NOT USE
|
||||||
|
inline FILE *log_enable_impl()
|
||||||
|
{
|
||||||
|
return log_handler1_impl(true, LogTriStateSame, LogTriStateFalse);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sets target fir logs, either by a file name or FILE* pointer (stdout, stderr, or any valid FILE*)
|
||||||
|
#define log_set_target(target) log_set_target_impl(target)
|
||||||
|
|
||||||
|
// INTERNAL, DO NOT USE
|
||||||
|
inline FILE *log_set_target_impl(const std::string & filename) { return log_handler1_impl(true, LogTriStateSame, LogTriStateSame, filename); }
|
||||||
|
inline FILE *log_set_target_impl(FILE *target) { return log_handler2_impl(true, LogTriStateSame, LogTriStateSame, target); }
|
||||||
|
|
||||||
|
// INTERNAL, DO NOT USE
|
||||||
|
inline FILE *log_handler() { return log_handler1_impl(); }
|
||||||
|
|
||||||
|
// Enable or disable creating separate log files for each run.
|
||||||
|
// can ONLY be invoked BEFORE first log use.
|
||||||
|
#define log_multilog(enable) log_filename_generator_impl((enable) ? LogTriStateTrue : LogTriStateFalse, "", "")
|
||||||
|
// Enable or disable append mode for log file.
|
||||||
|
// can ONLY be invoked BEFORE first log use.
|
||||||
|
#define log_append(enable) log_append_impl(enable)
|
||||||
|
// INTERNAL, DO NOT USE
|
||||||
|
inline FILE *log_append_impl(bool enable)
|
||||||
|
{
|
||||||
|
return log_handler1_impl(true, enable ? LogTriStateTrue : LogTriStateFalse, LogTriStateSame);
|
||||||
|
}
|
||||||
|
|
||||||
|
inline void log_test()
|
||||||
|
{
|
||||||
|
log_disable();
|
||||||
|
LOG("01 Hello World to nobody, because logs are disabled!\n");
|
||||||
|
log_enable();
|
||||||
|
LOG("02 Hello World to default output, which is \"%s\" ( Yaaay, arguments! )!\n", LOG_STRINGIZE(LOG_TARGET));
|
||||||
|
LOG_TEE("03 Hello World to **both** default output and " LOG_TEE_TARGET_STRING "!\n");
|
||||||
|
log_set_target(stderr);
|
||||||
|
LOG("04 Hello World to stderr!\n");
|
||||||
|
LOG_TEE("05 Hello World TEE with double printing to stderr prevented!\n");
|
||||||
|
log_set_target(LOG_DEFAULT_FILE_NAME);
|
||||||
|
LOG("06 Hello World to default log file!\n");
|
||||||
|
log_set_target(stdout);
|
||||||
|
LOG("07 Hello World to stdout!\n");
|
||||||
|
log_set_target(LOG_DEFAULT_FILE_NAME);
|
||||||
|
LOG("08 Hello World to default log file again!\n");
|
||||||
|
log_disable();
|
||||||
|
LOG("09 Hello World _1_ into the void!\n");
|
||||||
|
log_enable();
|
||||||
|
LOG("10 Hello World back from the void ( you should not see _1_ in the log or the output )!\n");
|
||||||
|
log_disable();
|
||||||
|
log_set_target("llama.anotherlog.log");
|
||||||
|
LOG("11 Hello World _2_ to nobody, new target was selected but logs are still disabled!\n");
|
||||||
|
log_enable();
|
||||||
|
LOG("12 Hello World this time in a new file ( you should not see _2_ in the log or the output )?\n");
|
||||||
|
log_set_target("llama.yetanotherlog.log");
|
||||||
|
LOG("13 Hello World this time in yet new file?\n");
|
||||||
|
log_set_target(log_filename_generator("llama_autonamed", "log"));
|
||||||
|
LOG("14 Hello World in log with generated filename!\n");
|
||||||
|
#ifdef _MSC_VER
|
||||||
|
LOG_TEE("15 Hello msvc TEE without arguments\n");
|
||||||
|
LOG_TEE("16 Hello msvc TEE with (%d)(%s) arguments\n", 1, "test");
|
||||||
|
LOG_TEELN("17 Hello msvc TEELN without arguments\n");
|
||||||
|
LOG_TEELN("18 Hello msvc TEELN with (%d)(%s) arguments\n", 1, "test");
|
||||||
|
LOG("19 Hello msvc LOG without arguments\n");
|
||||||
|
LOG("20 Hello msvc LOG with (%d)(%s) arguments\n", 1, "test");
|
||||||
|
LOGLN("21 Hello msvc LOGLN without arguments\n");
|
||||||
|
LOGLN("22 Hello msvc LOGLN with (%d)(%s) arguments\n", 1, "test");
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
inline bool log_param_single_parse(const std::string & param)
|
||||||
|
{
|
||||||
|
if ( param == "--log-test")
|
||||||
|
{
|
||||||
|
log_test();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ( param == "--log-disable")
|
||||||
|
{
|
||||||
|
log_disable();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ( param == "--log-enable")
|
||||||
|
{
|
||||||
|
log_enable();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (param == "--log-new")
|
||||||
|
{
|
||||||
|
log_multilog(true);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (param == "--log-append")
|
||||||
|
{
|
||||||
|
log_append(true);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
inline bool log_param_pair_parse(bool check_but_dont_parse, const std::string & param, const std::string & next = std::string())
|
||||||
|
{
|
||||||
|
if ( param == "--log-file")
|
||||||
|
{
|
||||||
|
if (!check_but_dont_parse)
|
||||||
|
{
|
||||||
|
log_set_target(log_filename_generator(next.empty() ? "unnamed" : next, "log"));
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
inline void log_print_usage()
|
||||||
|
{
|
||||||
|
printf("log options:\n");
|
||||||
|
/* format
|
||||||
|
printf(" -h, --help show this help message and exit\n");*/
|
||||||
|
/* spacing
|
||||||
|
printf("__-param----------------Description\n");*/
|
||||||
|
printf(" --log-test Run simple logging test\n");
|
||||||
|
printf(" --log-disable Disable trace logs\n");
|
||||||
|
printf(" --log-enable Enable trace logs\n");
|
||||||
|
printf(" --log-file Specify a log filename (without extension)\n");
|
||||||
|
printf(" --log-new Create a separate new log file on start. "
|
||||||
|
"Each log file will have unique name: \"<name>.<ID>.log\"\n");
|
||||||
|
printf(" --log-append Don't truncate the old log file.\n");
|
||||||
|
printf("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
#define log_dump_cmdline(argc, argv) log_dump_cmdline_impl(argc, argv)
|
||||||
|
|
||||||
|
// INTERNAL, DO NOT USE
|
||||||
|
inline void log_dump_cmdline_impl(int argc, char **argv)
|
||||||
|
{
|
||||||
|
std::stringstream buf;
|
||||||
|
for (int i = 0; i < argc; ++i)
|
||||||
|
{
|
||||||
|
if (std::string(argv[i]).find(' ') != std::string::npos)
|
||||||
|
{
|
||||||
|
buf << " \"" << argv[i] <<"\"";
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
buf << " " << argv[i];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
LOGLN("Cmd:%s", buf.str().c_str());
|
||||||
|
}
|
||||||
|
|
||||||
|
#define log_tostr(var) log_var_to_string_impl(var).c_str()
|
||||||
|
|
||||||
|
inline std::string log_var_to_string_impl(bool var)
|
||||||
|
{
|
||||||
|
return var ? "true" : "false";
|
||||||
|
}
|
||||||
|
|
||||||
|
inline std::string log_var_to_string_impl(std::string var)
|
||||||
|
{
|
||||||
|
return var;
|
||||||
|
}
|
||||||
|
|
||||||
|
inline std::string log_var_to_string_impl(const std::vector<int> & var)
|
||||||
|
{
|
||||||
|
std::stringstream buf;
|
||||||
|
buf << "[ ";
|
||||||
|
bool first = true;
|
||||||
|
for (auto e : var)
|
||||||
|
{
|
||||||
|
if (first)
|
||||||
|
{
|
||||||
|
first = false;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
buf << ", ";
|
||||||
|
}
|
||||||
|
buf << std::to_string(e);
|
||||||
|
}
|
||||||
|
buf << " ]";
|
||||||
|
|
||||||
|
return buf.str();
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename C, typename T>
|
||||||
|
inline std::string LOG_TOKENS_TOSTR_PRETTY(const C & ctx, const T & tokens)
|
||||||
|
{
|
||||||
|
std::stringstream buf;
|
||||||
|
buf << "[ ";
|
||||||
|
|
||||||
|
bool first = true;
|
||||||
|
for (const auto &token : tokens)
|
||||||
|
{
|
||||||
|
if (!first) {
|
||||||
|
buf << ", ";
|
||||||
|
} else {
|
||||||
|
first = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto detokenized = llama_token_to_piece(ctx, token);
|
||||||
|
|
||||||
|
detokenized.erase(
|
||||||
|
std::remove_if(
|
||||||
|
detokenized.begin(),
|
||||||
|
detokenized.end(),
|
||||||
|
[](const unsigned char c) { return !std::isprint(c); }),
|
||||||
|
detokenized.end());
|
||||||
|
|
||||||
|
buf
|
||||||
|
<< "'" << detokenized << "'"
|
||||||
|
<< ":" << std::to_string(token);
|
||||||
|
}
|
||||||
|
buf << " ]";
|
||||||
|
|
||||||
|
return buf.str();
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename C, typename B>
|
||||||
|
inline std::string LOG_BATCH_TOSTR_PRETTY(const C & ctx, const B & batch)
|
||||||
|
{
|
||||||
|
std::stringstream buf;
|
||||||
|
buf << "[ ";
|
||||||
|
|
||||||
|
bool first = true;
|
||||||
|
for (int i = 0; i < batch.n_tokens; ++i)
|
||||||
|
{
|
||||||
|
if (!first) {
|
||||||
|
buf << ", ";
|
||||||
|
} else {
|
||||||
|
first = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto detokenized = llama_token_to_piece(ctx, batch.token[i]);
|
||||||
|
|
||||||
|
detokenized.erase(
|
||||||
|
std::remove_if(
|
||||||
|
detokenized.begin(),
|
||||||
|
detokenized.end(),
|
||||||
|
[](const unsigned char c) { return !std::isprint(c); }),
|
||||||
|
detokenized.end());
|
||||||
|
|
||||||
|
buf
|
||||||
|
<< "\n" << std::to_string(i)
|
||||||
|
<< ":token '" << detokenized << "'"
|
||||||
|
<< ":pos " << std::to_string(batch.pos[i])
|
||||||
|
<< ":n_seq_id " << std::to_string(batch.n_seq_id[i])
|
||||||
|
<< ":seq_id " << std::to_string(batch.seq_id[i][0])
|
||||||
|
<< ":logits " << std::to_string(batch.logits[i]);
|
||||||
|
}
|
||||||
|
buf << " ]";
|
||||||
|
|
||||||
|
return buf.str();
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifdef LOG_DISABLE_LOGS
|
||||||
|
|
||||||
|
#undef LOG
|
||||||
|
#define LOG(...) // dummy stub
|
||||||
|
#undef LOGLN
|
||||||
|
#define LOGLN(...) // dummy stub
|
||||||
|
|
||||||
|
#undef LOG_TEE
|
||||||
|
#define LOG_TEE(...) fprintf(stderr, __VA_ARGS__) // convert to normal fprintf
|
||||||
|
|
||||||
|
#undef LOG_TEELN
|
||||||
|
#define LOG_TEELN(...) fprintf(stderr, __VA_ARGS__) // convert to normal fprintf
|
||||||
|
|
||||||
|
#undef LOG_DISABLE
|
||||||
|
#define LOG_DISABLE() // dummy stub
|
||||||
|
|
||||||
|
#undef LOG_ENABLE
|
||||||
|
#define LOG_ENABLE() // dummy stub
|
||||||
|
|
||||||
|
#undef LOG_ENABLE
|
||||||
|
#define LOG_ENABLE() // dummy stub
|
||||||
|
|
||||||
|
#undef LOG_SET_TARGET
|
||||||
|
#define LOG_SET_TARGET(...) // dummy stub
|
||||||
|
|
||||||
|
#undef LOG_DUMP_CMDLINE
|
||||||
|
#define LOG_DUMP_CMDLINE(...) // dummy stub
|
||||||
|
|
||||||
|
#endif // LOG_DISABLE_LOGS
|
||||||
282
common/ngram-cache.cpp
Normal file
282
common/ngram-cache.cpp
Normal file
|
|
@ -0,0 +1,282 @@
|
||||||
|
#include "ngram-cache.h"
|
||||||
|
#include "common.h"
|
||||||
|
#include "log.h"
|
||||||
|
|
||||||
|
#include <cstdint>
|
||||||
|
#include <fstream>
|
||||||
|
|
||||||
|
void llama_ngram_cache_update(llama_ngram_cache & ngram_cache, int ngram_min, int ngram_max,
|
||||||
|
std::vector<llama_token> & inp, int nnew, bool print_progress) {
|
||||||
|
const int64_t t_start_ms = ggml_time_ms();
|
||||||
|
const int64_t inp_size = inp.size();
|
||||||
|
|
||||||
|
const int64_t n_todo = inp_size * (ngram_max - ngram_min + 1);
|
||||||
|
int64_t n_done = 0;
|
||||||
|
|
||||||
|
for (int64_t ngram_size = ngram_min; ngram_size <= ngram_max; ++ngram_size) {
|
||||||
|
const int64_t i_start = std::max(inp_size - nnew, ngram_size);
|
||||||
|
for (int64_t i = i_start; i < inp_size; ++i) {
|
||||||
|
const int64_t ngram_start = i - ngram_size;
|
||||||
|
llama_ngram ngram(&inp[ngram_start], ngram_size);
|
||||||
|
const llama_token token = inp[i];
|
||||||
|
|
||||||
|
llama_ngram_cache::iterator part_it = ngram_cache.find(ngram);
|
||||||
|
if (part_it == ngram_cache.end()) {
|
||||||
|
llama_ngram_cache_part part;
|
||||||
|
part.emplace(token, 1);
|
||||||
|
ngram_cache.emplace(ngram, part);
|
||||||
|
} else {
|
||||||
|
llama_ngram_cache_part::iterator token_count_it = part_it->second.find(token);
|
||||||
|
if (token_count_it == part_it->second.end()) {
|
||||||
|
part_it->second.emplace(token, 1);
|
||||||
|
} else {
|
||||||
|
token_count_it->second++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
++n_done;
|
||||||
|
|
||||||
|
if (print_progress && n_done % 10000000 == 0) {
|
||||||
|
const int64_t t_now_ms = ggml_time_ms();
|
||||||
|
const int64_t eta_ms = (inp_size*(ngram_max-ngram_min+1) - n_done) * (t_now_ms - t_start_ms) / n_done;
|
||||||
|
const int64_t eta_min = eta_ms / (60*1000);
|
||||||
|
const int64_t eta_s = (eta_ms - 60*1000*eta_min) / 1000;
|
||||||
|
|
||||||
|
fprintf(stderr, "%s: %" PRId64 "/%" PRId64 " done, ETA: %02" PRId64 ":%02" PRId64 "\n", __func__, n_done, n_todo, eta_min, eta_s);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to get a token from the combined, speculative sequence of inp and draft.
|
||||||
|
static llama_token get_token(const std::vector<llama_token> & inp, const std::vector<llama_token> & draft, const size_t i) {
|
||||||
|
return i < inp.size() ? inp[i] : draft[1 + i - inp.size()];
|
||||||
|
}
|
||||||
|
|
||||||
|
// If sample size or percentage are below these thresholds the draft is aborted early:
|
||||||
|
constexpr int draft_min_sample_size_lax[LLAMA_NGRAM_MAX] = { 2, 2, 1, 1};
|
||||||
|
constexpr int draft_min_percent_lax[LLAMA_NGRAM_MAX] = {66, 50, 50, 50};
|
||||||
|
constexpr int draft_min_sample_size_strict[LLAMA_NGRAM_MAX] = { 4, 3, 2, 2};
|
||||||
|
constexpr int draft_min_percent_strict[LLAMA_NGRAM_MAX] = {75, 66, 66, 66};
|
||||||
|
|
||||||
|
// Helper function that tries to draft a token from only the static ngram cache:
|
||||||
|
static llama_token try_draft(llama_ngram_cache & nc_static, const llama_ngram ngram_static) {
|
||||||
|
llama_ngram_cache::iterator part_static_it = nc_static.find(ngram_static);
|
||||||
|
if (part_static_it == nc_static.end()) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
const llama_ngram_cache_part part_static = part_static_it->second;
|
||||||
|
|
||||||
|
int max_count_static = 0;
|
||||||
|
int sum_count_static = 0;
|
||||||
|
llama_token max_token = -1;
|
||||||
|
|
||||||
|
for (std::pair<llama_token, int> token_count_static : part_static) {
|
||||||
|
const llama_token token = token_count_static.first;
|
||||||
|
const int32_t count_static = token_count_static.second;
|
||||||
|
|
||||||
|
if (count_static > max_count_static) {
|
||||||
|
max_token = token;
|
||||||
|
max_count_static = count_static;
|
||||||
|
}
|
||||||
|
sum_count_static += count_static;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sum_count_static < draft_min_sample_size_lax[LLAMA_NGRAM_STATIC-1]) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (100*max_count_static < draft_min_percent_lax[LLAMA_NGRAM_STATIC-1]*sum_count_static) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return max_token;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to draft a token from primary cache (context/dynamic), validate with static cache:
|
||||||
|
static llama_token try_draft(
|
||||||
|
llama_ngram_cache & nc_primary, const std::vector<llama_ngram> & ngrams_primary, llama_ngram_cache_part & part_static,
|
||||||
|
const int * min_sample_size, const int * min_percent) {
|
||||||
|
|
||||||
|
llama_token drafted_token = -1;
|
||||||
|
|
||||||
|
for (int i = ngrams_primary.size()-1; i >= 0 && drafted_token == -1; --i) {
|
||||||
|
const llama_ngram ngram_primary = ngrams_primary[i];
|
||||||
|
|
||||||
|
llama_ngram_cache::iterator part_primary_it = nc_primary.find(ngram_primary);
|
||||||
|
if (part_primary_it == nc_primary.end()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const llama_ngram_cache_part part_primary = part_primary_it->second;
|
||||||
|
|
||||||
|
int max_count_primary = 0;
|
||||||
|
int max_count_static = 0;
|
||||||
|
int sum_count_primary = 0;
|
||||||
|
llama_token max_token = -1;
|
||||||
|
|
||||||
|
for (std::pair<llama_token, int> token_count_primary : part_primary) {
|
||||||
|
const llama_token token = token_count_primary.first;
|
||||||
|
|
||||||
|
llama_ngram_cache_part::iterator token_count_static_it = part_static.find(token);
|
||||||
|
|
||||||
|
const int32_t count_primary = token_count_primary.second;
|
||||||
|
const int32_t count_static = token_count_static_it != part_static.end() ? 100*token_count_static_it->second : 1;
|
||||||
|
|
||||||
|
if (count_primary*count_static > max_count_primary*max_count_static) {
|
||||||
|
max_token = token;
|
||||||
|
max_count_primary = count_primary;
|
||||||
|
max_count_static = count_static;
|
||||||
|
}
|
||||||
|
sum_count_primary += count_primary;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sum_count_primary < min_sample_size[i]) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (100*max_count_primary < min_percent[i]*sum_count_primary) {
|
||||||
|
continue;;
|
||||||
|
}
|
||||||
|
drafted_token = max_token;
|
||||||
|
}
|
||||||
|
|
||||||
|
return drafted_token;
|
||||||
|
}
|
||||||
|
|
||||||
|
void llama_ngram_cache_draft(
|
||||||
|
std::vector<llama_token> & inp, std::vector<llama_token> & draft, int n_draft, int ngram_min, int ngram_max,
|
||||||
|
llama_ngram_cache & nc_context, llama_ngram_cache & nc_dynamic, llama_ngram_cache & nc_static
|
||||||
|
) {
|
||||||
|
GGML_ASSERT(draft.size() == 1);
|
||||||
|
const int inp_size = inp.size();
|
||||||
|
|
||||||
|
if (inp_size < LLAMA_NGRAM_STATIC) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
while ((int) draft.size()-1 < n_draft) {
|
||||||
|
llama_token drafted_token = -1;
|
||||||
|
|
||||||
|
const int ngram_start_static = inp_size-LLAMA_NGRAM_STATIC + draft.size()-1;
|
||||||
|
llama_ngram ngram_static;
|
||||||
|
for (int j = ngram_start_static; j < ngram_start_static + LLAMA_NGRAM_STATIC; ++j) {
|
||||||
|
ngram_static.tokens[j-ngram_start_static] = get_token(inp, draft, j);
|
||||||
|
}
|
||||||
|
llama_ngram_cache::iterator part_static_it = nc_static.find(ngram_static);
|
||||||
|
llama_ngram_cache_part part_static;
|
||||||
|
if (part_static_it != nc_static.end()) {
|
||||||
|
part_static = part_static_it->second;
|
||||||
|
}
|
||||||
|
|
||||||
|
// cd = context + dynamic
|
||||||
|
std::vector<llama_ngram> ngrams_cd;
|
||||||
|
for (int ngram_size_cd = ngram_min; ngram_size_cd <= ngram_max; ++ngram_size_cd) {
|
||||||
|
const int ngram_start_cd = inp_size-ngram_size_cd + draft.size()-1;
|
||||||
|
llama_ngram ngram_cd;
|
||||||
|
for (int j = ngram_start_cd; j < ngram_start_cd + ngram_size_cd; ++j) {
|
||||||
|
ngram_cd.tokens[j-ngram_start_cd] = get_token(inp, draft, j);
|
||||||
|
}
|
||||||
|
ngrams_cd.push_back(ngram_cd);
|
||||||
|
}
|
||||||
|
if (drafted_token == -1) {
|
||||||
|
drafted_token = try_draft(nc_context, ngrams_cd, part_static, draft_min_sample_size_lax, draft_min_percent_lax);
|
||||||
|
}
|
||||||
|
if (drafted_token == -1) {
|
||||||
|
drafted_token = try_draft(nc_dynamic, ngrams_cd, part_static, draft_min_sample_size_strict, draft_min_percent_strict);
|
||||||
|
}
|
||||||
|
if (drafted_token == -1) {
|
||||||
|
drafted_token = try_draft(nc_static, ngram_static);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (drafted_token == -1) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
LOG(" - draft candidate: token=%d\n", drafted_token);
|
||||||
|
draft.push_back(drafted_token);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void llama_ngram_cache_save(llama_ngram_cache & ngram_cache, std::string & filename) {
|
||||||
|
std::ofstream file_out(filename, std::ios::binary);
|
||||||
|
for (std::pair<llama_ngram, llama_ngram_cache_part> item : ngram_cache) {
|
||||||
|
const llama_ngram ngram = item.first;
|
||||||
|
llama_ngram_cache_part token_counts = item.second;
|
||||||
|
GGML_ASSERT(!token_counts.empty());
|
||||||
|
const int32_t ntokens = token_counts.size();
|
||||||
|
GGML_ASSERT(ntokens > 0);
|
||||||
|
|
||||||
|
file_out.write(reinterpret_cast<const char *>(&ngram), sizeof(llama_ngram));
|
||||||
|
file_out.write(reinterpret_cast<const char *>(&ntokens), sizeof(int32_t));
|
||||||
|
for (std::pair<llama_token, int32_t> item2 : token_counts) {
|
||||||
|
const llama_token token = item2.first;
|
||||||
|
const int32_t count = item2.second;
|
||||||
|
GGML_ASSERT(count > 0);
|
||||||
|
|
||||||
|
file_out.write(reinterpret_cast<const char *>(&token), sizeof(llama_token));
|
||||||
|
file_out.write(reinterpret_cast<const char *>(&count), sizeof(int32_t));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
llama_ngram_cache llama_ngram_cache_load(std::string & filename) {
|
||||||
|
std::ifstream hashmap_file(filename, std::ios::binary);
|
||||||
|
if (!hashmap_file) {
|
||||||
|
throw std::ifstream::failure("Unable to open file " + filename);
|
||||||
|
}
|
||||||
|
llama_ngram_cache ngram_cache;
|
||||||
|
|
||||||
|
llama_ngram ngram;
|
||||||
|
int32_t ntokens;
|
||||||
|
llama_token token;
|
||||||
|
int32_t count;
|
||||||
|
|
||||||
|
char * ngramc = reinterpret_cast<char*>(&ngram);
|
||||||
|
char * ntokensc = reinterpret_cast<char*>(&ntokens);
|
||||||
|
char * tokenc = reinterpret_cast<char*>(&token);
|
||||||
|
char * countc = reinterpret_cast<char*>(&count);
|
||||||
|
while(hashmap_file.read(ngramc, sizeof(llama_ngram))) {
|
||||||
|
GGML_ASSERT(!hashmap_file.eof());
|
||||||
|
GGML_ASSERT(hashmap_file.read(ntokensc, sizeof(int32_t)));
|
||||||
|
GGML_ASSERT(ntokens > 0);
|
||||||
|
llama_ngram_cache_part token_counts;
|
||||||
|
|
||||||
|
for (int i = 0; i < ntokens; ++i) {
|
||||||
|
GGML_ASSERT(!hashmap_file.eof());
|
||||||
|
GGML_ASSERT(hashmap_file.read(tokenc, sizeof(llama_token)));
|
||||||
|
GGML_ASSERT(!hashmap_file.eof());
|
||||||
|
GGML_ASSERT(hashmap_file.read(countc, sizeof(int32_t)));
|
||||||
|
GGML_ASSERT(count > 0);
|
||||||
|
token_counts.emplace(token, count);
|
||||||
|
}
|
||||||
|
|
||||||
|
ngram_cache.emplace(ngram, token_counts);
|
||||||
|
}
|
||||||
|
GGML_ASSERT(hashmap_file.eof());
|
||||||
|
|
||||||
|
return ngram_cache;
|
||||||
|
}
|
||||||
|
|
||||||
|
void llama_ngram_cache_merge(llama_ngram_cache & ngram_cache_target, llama_ngram_cache & ngram_cache_add) {
|
||||||
|
for (std::pair<llama_ngram, llama_ngram_cache_part> ngram_part : ngram_cache_add) {
|
||||||
|
const llama_ngram ngram = ngram_part.first;
|
||||||
|
llama_ngram_cache_part part = ngram_part.second;
|
||||||
|
|
||||||
|
llama_ngram_cache::iterator part_merged_it = ngram_cache_target.find(ngram);
|
||||||
|
if (part_merged_it == ngram_cache_target.end()) {
|
||||||
|
ngram_cache_target.emplace(ngram, part);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (std::pair<llama_token, int32_t> token_count : part) {
|
||||||
|
const llama_token token = token_count.first;
|
||||||
|
const int32_t count = token_count.second;
|
||||||
|
GGML_ASSERT(count > 0);
|
||||||
|
|
||||||
|
llama_ngram_cache_part::iterator token_count_merged_it = part_merged_it->second.find(token);
|
||||||
|
if (token_count_merged_it == part_merged_it->second.end()) {
|
||||||
|
part_merged_it->second.emplace(token, count);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
token_count_merged_it->second += count;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
94
common/ngram-cache.h
Normal file
94
common/ngram-cache.h
Normal file
|
|
@ -0,0 +1,94 @@
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "llama.h"
|
||||||
|
|
||||||
|
#include <unordered_map>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#define LLAMA_NGRAM_MIN 1
|
||||||
|
#define LLAMA_NGRAM_MAX 4
|
||||||
|
#define LLAMA_NGRAM_STATIC 2
|
||||||
|
|
||||||
|
// Data structures to map n-grams to empirical token probabilities:
|
||||||
|
|
||||||
|
struct llama_ngram {
|
||||||
|
llama_token tokens[LLAMA_NGRAM_MAX];
|
||||||
|
|
||||||
|
llama_ngram() {
|
||||||
|
for (int i = 0; i < LLAMA_NGRAM_MAX; ++i) {
|
||||||
|
tokens[i] = -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
llama_ngram(const llama_token * input, const int ngram_size) {
|
||||||
|
for (int i = 0; i < LLAMA_NGRAM_MAX; ++i) {
|
||||||
|
tokens[i] = i < ngram_size ? input[i] : -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool operator==(const llama_ngram & other) const {
|
||||||
|
for (int i = 0; i < LLAMA_NGRAM_MAX; ++i) {
|
||||||
|
if (tokens[i] != other.tokens[i]) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
struct llama_ngram_hash_function {
|
||||||
|
size_t operator()(const llama_ngram & ngram) const {
|
||||||
|
size_t hash = 0;
|
||||||
|
for (int i = 0; i < LLAMA_NGRAM_MAX; ++i) {
|
||||||
|
hash ^= std::hash<llama_token>{}(ngram.tokens[i]);
|
||||||
|
}
|
||||||
|
return hash;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// token -> number of times token has been seen
|
||||||
|
typedef std::unordered_map<llama_token, int32_t> llama_ngram_cache_part;
|
||||||
|
|
||||||
|
// n-gram -> empirical distribution of following tokens
|
||||||
|
typedef std::unordered_map<llama_ngram, llama_ngram_cache_part, llama_ngram_hash_function> llama_ngram_cache;
|
||||||
|
|
||||||
|
|
||||||
|
// Update an ngram cache with tokens.
|
||||||
|
// ngram_cache: the cache to modify.
|
||||||
|
// ngram_min/ngram_max: the min/max size of the ngrams to extract from inp_data.
|
||||||
|
// inp_data: the token sequence with which to update ngram_cache.
|
||||||
|
// nnew: how many new tokens have been appended to inp_data since the last call to this function.
|
||||||
|
// print_progress: whether to print progress to stderr.
|
||||||
|
//
|
||||||
|
// In order to get correct results inp_data can ONLY BE APPENDED TO.
|
||||||
|
// Changes in the middle need a complete rebuild.
|
||||||
|
void llama_ngram_cache_update(
|
||||||
|
llama_ngram_cache & ngram_cache, int ngram_min, int ngram_max, std::vector<llama_token> & inp_data, int nnew, bool print_progress);
|
||||||
|
|
||||||
|
// Try to draft tokens from ngram caches.
|
||||||
|
// inp: the tokens generated so far.
|
||||||
|
// draft: the token sequence to draft. Expected to initially contain the previously sampled token.
|
||||||
|
// n_draft: maximum number of tokens to add to draft.
|
||||||
|
// ngram_min/gram_max: the min/max size of the ngrams in nc_context and nc_dynamic.
|
||||||
|
// nc_context: ngram cache based on current context.
|
||||||
|
// nc_dynamic: ngram cache based on previous user generations.
|
||||||
|
// nc_static: ngram cache generated from a large text corpus, used for validation.
|
||||||
|
void llama_ngram_cache_draft(
|
||||||
|
std::vector<llama_token> & inp, std::vector<llama_token> & draft, int n_draft, int ngram_min, int ngram_max,
|
||||||
|
llama_ngram_cache & nc_context, llama_ngram_cache & nc_dynamic, llama_ngram_cache & nc_static);
|
||||||
|
|
||||||
|
// Save an ngram cache to a file.
|
||||||
|
// ngram_cache: the ngram cache to save.
|
||||||
|
// filename: the path under which to save the ngram cache.
|
||||||
|
void llama_ngram_cache_save(llama_ngram_cache & ngram_cache, std::string & filename);
|
||||||
|
|
||||||
|
// Load an ngram cache saved with llama_ngram_cache_save.
|
||||||
|
// filename: the path from which to load the ngram cache.
|
||||||
|
// returns: an ngram cache containing the information saved to filename.
|
||||||
|
llama_ngram_cache llama_ngram_cache_load(std::string & filename);
|
||||||
|
|
||||||
|
// Merge two ngram caches.
|
||||||
|
// ngram_cache_target: the ngram cache to which to add the information from ngram_cache_add.
|
||||||
|
// ngram_cache_add: the ngram cache to add to ngram_cache_target.
|
||||||
|
void llama_ngram_cache_merge(llama_ngram_cache & ngram_cache_target, llama_ngram_cache & ngram_cache_add);
|
||||||
369
common/sampling.cpp
Normal file
369
common/sampling.cpp
Normal file
|
|
@ -0,0 +1,369 @@
|
||||||
|
#define LLAMA_API_INTERNAL
|
||||||
|
#include "sampling.h"
|
||||||
|
#include <random>
|
||||||
|
|
||||||
|
struct llama_sampling_context * llama_sampling_init(const struct llama_sampling_params & params) {
|
||||||
|
struct llama_sampling_context * result = new llama_sampling_context();
|
||||||
|
|
||||||
|
result->params = params;
|
||||||
|
result->grammar = nullptr;
|
||||||
|
|
||||||
|
// if there is a grammar, parse it
|
||||||
|
if (!params.grammar.empty()) {
|
||||||
|
result->parsed_grammar = grammar_parser::parse(params.grammar.c_str());
|
||||||
|
|
||||||
|
// will be empty (default) if there are parse errors
|
||||||
|
if (result->parsed_grammar.rules.empty()) {
|
||||||
|
fprintf(stderr, "%s: failed to parse grammar\n", __func__);
|
||||||
|
delete result;
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure that there is a "root" node.
|
||||||
|
if (result->parsed_grammar.symbol_ids.find("root") == result->parsed_grammar.symbol_ids.end()) {
|
||||||
|
fprintf(stderr, "%s: grammar does not contain a 'root' symbol\n", __func__);
|
||||||
|
delete result;
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<const llama_grammar_element *> grammar_rules(result->parsed_grammar.c_rules());
|
||||||
|
|
||||||
|
result->grammar = llama_grammar_init(
|
||||||
|
grammar_rules.data(),
|
||||||
|
grammar_rules.size(), result->parsed_grammar.symbol_ids.at("root"));
|
||||||
|
}
|
||||||
|
|
||||||
|
result->prev.resize(params.n_prev);
|
||||||
|
|
||||||
|
result->n_considered = 0;
|
||||||
|
|
||||||
|
llama_sampling_set_rng_seed(result, params.seed);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
void llama_sampling_free(struct llama_sampling_context * ctx) {
|
||||||
|
if (ctx->grammar != NULL) {
|
||||||
|
llama_grammar_free(ctx->grammar);
|
||||||
|
}
|
||||||
|
|
||||||
|
delete ctx;
|
||||||
|
}
|
||||||
|
|
||||||
|
void llama_sampling_reset(llama_sampling_context * ctx) {
|
||||||
|
if (ctx->grammar != NULL) {
|
||||||
|
llama_grammar_free(ctx->grammar);
|
||||||
|
ctx->grammar = NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!ctx->parsed_grammar.rules.empty()) {
|
||||||
|
std::vector<const llama_grammar_element *> grammar_rules(ctx->parsed_grammar.c_rules());
|
||||||
|
|
||||||
|
ctx->grammar = llama_grammar_init(
|
||||||
|
grammar_rules.data(),
|
||||||
|
grammar_rules.size(), ctx->parsed_grammar.symbol_ids.at("root"));
|
||||||
|
}
|
||||||
|
|
||||||
|
std::fill(ctx->prev.begin(), ctx->prev.end(), 0);
|
||||||
|
ctx->cur.clear();
|
||||||
|
ctx->n_considered = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void llama_sampling_set_rng_seed(struct llama_sampling_context * ctx, uint32_t seed) {
|
||||||
|
if (seed == LLAMA_DEFAULT_SEED) {
|
||||||
|
seed = std::random_device{}();
|
||||||
|
}
|
||||||
|
ctx->rng.seed(seed);
|
||||||
|
}
|
||||||
|
|
||||||
|
void llama_sampling_cp(llama_sampling_context * src, llama_sampling_context * dst) {
|
||||||
|
if (dst->grammar) {
|
||||||
|
llama_grammar_free(dst->grammar);
|
||||||
|
dst->grammar = nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (src->grammar) {
|
||||||
|
dst->grammar = llama_grammar_copy(src->grammar);
|
||||||
|
}
|
||||||
|
|
||||||
|
dst->prev = src->prev;
|
||||||
|
}
|
||||||
|
|
||||||
|
llama_token llama_sampling_last(llama_sampling_context * ctx) {
|
||||||
|
return ctx->prev.back();
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string llama_sampling_prev_str(llama_sampling_context * ctx_sampling, llama_context * ctx_main, int n) {
|
||||||
|
const int size = ctx_sampling->prev.size();
|
||||||
|
|
||||||
|
n = std::min(n, size);
|
||||||
|
|
||||||
|
std::string result;
|
||||||
|
|
||||||
|
for (int i = size - n; i < size; i++) {
|
||||||
|
result += llama_token_to_piece(ctx_main, ctx_sampling->prev[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string llama_sampling_print(const llama_sampling_params & params) {
|
||||||
|
char result[1024];
|
||||||
|
|
||||||
|
snprintf(result, sizeof(result),
|
||||||
|
"\trepeat_last_n = %d, repeat_penalty = %.3f, frequency_penalty = %.3f, presence_penalty = %.3f\n"
|
||||||
|
"\ttop_k = %d, tfs_z = %.3f, top_p = %.3f, min_p = %.3f, typical_p = %.3f, temp = %.3f\n"
|
||||||
|
"\tmirostat = %d, mirostat_lr = %.3f, mirostat_ent = %.3f",
|
||||||
|
params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present,
|
||||||
|
params.top_k, params.tfs_z, params.top_p, params.min_p, params.typical_p, params.temp,
|
||||||
|
params.mirostat, params.mirostat_eta, params.mirostat_tau);
|
||||||
|
|
||||||
|
return std::string(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string llama_sampling_order_print(const llama_sampling_params & params) {
|
||||||
|
std::string result = "CFG -> Penalties ";
|
||||||
|
if (params.mirostat == 0) {
|
||||||
|
for (auto sampler_type : params.samplers_sequence) {
|
||||||
|
const auto sampler_type_name = sampler_type_to_name_string(sampler_type);
|
||||||
|
if (!sampler_type_name.empty()) {
|
||||||
|
result += "-> " + sampler_type_name + " ";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
result += "-> mirostat ";
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// no reasons to expose this function in header
|
||||||
|
static void sampler_queue(
|
||||||
|
struct llama_context * ctx_main,
|
||||||
|
const llama_sampling_params & params,
|
||||||
|
llama_token_data_array & cur_p,
|
||||||
|
size_t min_keep) {
|
||||||
|
const float temp = params.temp;
|
||||||
|
const float dynatemp_range = params.dynatemp_range;
|
||||||
|
const float dynatemp_exponent = params.dynatemp_exponent;
|
||||||
|
const int32_t top_k = params.top_k;
|
||||||
|
const float top_p = params.top_p;
|
||||||
|
const float min_p = params.min_p;
|
||||||
|
const float tfs_z = params.tfs_z;
|
||||||
|
const float typical_p = params.typical_p;
|
||||||
|
const std::vector<llama_sampler_type> & samplers_sequence = params.samplers_sequence;
|
||||||
|
|
||||||
|
for (auto sampler_type : samplers_sequence) {
|
||||||
|
switch (sampler_type) {
|
||||||
|
case llama_sampler_type::TOP_K : llama_sample_top_k (ctx_main, &cur_p, top_k, min_keep); break;
|
||||||
|
case llama_sampler_type::TFS_Z : llama_sample_tail_free(ctx_main, &cur_p, tfs_z, min_keep); break;
|
||||||
|
case llama_sampler_type::TYPICAL_P: llama_sample_typical (ctx_main, &cur_p, typical_p, min_keep); break;
|
||||||
|
case llama_sampler_type::TOP_P : llama_sample_top_p (ctx_main, &cur_p, top_p, min_keep); break;
|
||||||
|
case llama_sampler_type::MIN_P : llama_sample_min_p (ctx_main, &cur_p, min_p, min_keep); break;
|
||||||
|
case llama_sampler_type::TEMPERATURE:
|
||||||
|
if (dynatemp_range > 0) {
|
||||||
|
float dynatemp_min = std::max(0.0f, temp - dynatemp_range);
|
||||||
|
float dynatemp_max = std::max(0.0f, temp + dynatemp_range);
|
||||||
|
llama_sample_entropy(ctx_main, &cur_p, dynatemp_min, dynatemp_max, dynatemp_exponent);
|
||||||
|
} else {
|
||||||
|
llama_sample_temp(ctx_main, &cur_p, temp);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
default : break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static llama_token llama_sampling_sample_impl(
|
||||||
|
struct llama_sampling_context * ctx_sampling,
|
||||||
|
struct llama_context * ctx_main,
|
||||||
|
struct llama_context * ctx_cfg,
|
||||||
|
const int idx,
|
||||||
|
bool is_resampling) { // Add a parameter to indicate if we are resampling
|
||||||
|
const llama_sampling_params & params = ctx_sampling->params;
|
||||||
|
|
||||||
|
const float temp = params.temp;
|
||||||
|
const int mirostat = params.mirostat;
|
||||||
|
const float mirostat_tau = params.mirostat_tau;
|
||||||
|
const float mirostat_eta = params.mirostat_eta;
|
||||||
|
|
||||||
|
std::vector<float> original_logits;
|
||||||
|
auto cur_p = llama_sampling_prepare(ctx_sampling, ctx_main, ctx_cfg, idx, !is_resampling, &original_logits);
|
||||||
|
if (!is_resampling) {
|
||||||
|
GGML_ASSERT(!original_logits.empty());
|
||||||
|
}
|
||||||
|
llama_token id = 0;
|
||||||
|
// Get a pointer to the logits
|
||||||
|
float * logits = llama_get_logits_ith(ctx_main, idx);
|
||||||
|
|
||||||
|
if (temp < 0.0) {
|
||||||
|
// greedy sampling, with probs
|
||||||
|
llama_sample_softmax(ctx_main, &cur_p);
|
||||||
|
id = cur_p.data[0].id;
|
||||||
|
} else if (temp == 0.0) {
|
||||||
|
// greedy sampling, no probs
|
||||||
|
id = llama_sample_token_greedy(ctx_main, &cur_p);
|
||||||
|
} else {
|
||||||
|
if (mirostat == 1) {
|
||||||
|
const int mirostat_m = 100;
|
||||||
|
llama_sample_temp(ctx_main, &cur_p, temp);
|
||||||
|
id = llama_sample_token_mirostat(ctx_main, &cur_p, mirostat_tau, mirostat_eta, mirostat_m, &ctx_sampling->mirostat_mu);
|
||||||
|
} else if (mirostat == 2) {
|
||||||
|
llama_sample_temp(ctx_main, &cur_p, temp);
|
||||||
|
id = llama_sample_token_mirostat_v2(ctx_main, &cur_p, mirostat_tau, mirostat_eta, &ctx_sampling->mirostat_mu);
|
||||||
|
} else {
|
||||||
|
// temperature sampling
|
||||||
|
size_t min_keep = std::max(1, params.min_keep);
|
||||||
|
|
||||||
|
sampler_queue(ctx_main, params, cur_p, min_keep);
|
||||||
|
|
||||||
|
id = llama_sample_token_with_rng(ctx_main, &cur_p, ctx_sampling->rng);
|
||||||
|
|
||||||
|
//{
|
||||||
|
// const int n_top = 10;
|
||||||
|
// LOG("top %d candidates:\n", n_top);
|
||||||
|
|
||||||
|
// for (int i = 0; i < n_top; i++) {
|
||||||
|
// const llama_token id = cur_p.data[i].id;
|
||||||
|
// (void)id; // To avoid a warning that id is unused when logging is disabled.
|
||||||
|
// LOG(" - %5d: '%12s' (%.3f)\n", id, llama_token_to_piece(ctx_main, id).c_str(), cur_p.data[i].p);
|
||||||
|
// }
|
||||||
|
//}
|
||||||
|
|
||||||
|
//LOG("sampled token: %5d: '%s'\n", id, llama_token_to_piece(ctx_main, id).c_str());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ctx_sampling->grammar != NULL && !is_resampling) {
|
||||||
|
// Create an array with a single token data element for the sampled id
|
||||||
|
llama_token_data single_token_data = {id, logits[id], 0.0f};
|
||||||
|
llama_token_data_array single_token_data_array = { &single_token_data, 1, false };
|
||||||
|
|
||||||
|
// Apply grammar constraints to the single token
|
||||||
|
llama_sample_grammar(ctx_main, &single_token_data_array, ctx_sampling->grammar);
|
||||||
|
|
||||||
|
// Check if the token is valid according to the grammar by seeing if its logit has been set to -INFINITY
|
||||||
|
bool is_valid = single_token_data_array.data[0].logit != -INFINITY;
|
||||||
|
|
||||||
|
// If the token is not valid according to the grammar, perform resampling
|
||||||
|
if (!is_valid) {
|
||||||
|
LOG("Resampling because token %d: '%s' does not meet grammar rules\n", id, llama_token_to_piece(ctx_main, id).c_str());
|
||||||
|
|
||||||
|
// Restore logits from the copy
|
||||||
|
std::copy(original_logits.begin(), original_logits.end(), logits);
|
||||||
|
|
||||||
|
return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, true); // Pass true for is_resampling
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx_sampling->n_considered = cur_p.size;
|
||||||
|
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
static llama_token_data_array llama_sampling_prepare_impl(
|
||||||
|
struct llama_sampling_context * ctx_sampling,
|
||||||
|
struct llama_context * ctx_main,
|
||||||
|
struct llama_context * ctx_cfg,
|
||||||
|
const int idx,
|
||||||
|
bool apply_grammar,
|
||||||
|
std::vector<float> * original_logits) {
|
||||||
|
const llama_sampling_params & params = ctx_sampling->params;
|
||||||
|
|
||||||
|
const int n_vocab = llama_n_vocab(llama_get_model(ctx_main));
|
||||||
|
|
||||||
|
const int32_t penalty_last_n = params.penalty_last_n < 0 ? params.n_prev : params.penalty_last_n;
|
||||||
|
const float penalty_repeat = params.penalty_repeat;
|
||||||
|
const float penalty_freq = params.penalty_freq;
|
||||||
|
const float penalty_present = params.penalty_present;
|
||||||
|
|
||||||
|
const bool penalize_nl = params.penalize_nl;
|
||||||
|
|
||||||
|
auto & prev = ctx_sampling->prev;
|
||||||
|
auto & cur = ctx_sampling->cur;
|
||||||
|
|
||||||
|
// Get a pointer to the logits
|
||||||
|
float * logits = llama_get_logits_ith(ctx_main, idx);
|
||||||
|
|
||||||
|
if (apply_grammar && original_logits != NULL) {
|
||||||
|
// Only make a copy of the original logits if we are not applying grammar checks, not sure if I actually have to do this.
|
||||||
|
*original_logits = {logits, logits + llama_n_vocab(llama_get_model(ctx_main))};
|
||||||
|
}
|
||||||
|
|
||||||
|
// apply params.logit_bias map
|
||||||
|
for (auto it = params.logit_bias.begin(); it != params.logit_bias.end(); it++) {
|
||||||
|
logits[it->first] += it->second;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ctx_cfg) {
|
||||||
|
float * logits_guidance = llama_get_logits_ith(ctx_cfg, idx);
|
||||||
|
llama_sample_apply_guidance(ctx_main, logits, logits_guidance, params.cfg_scale);
|
||||||
|
}
|
||||||
|
|
||||||
|
cur.clear();
|
||||||
|
|
||||||
|
for (llama_token token_id = 0; token_id < n_vocab; token_id++) {
|
||||||
|
cur.emplace_back(llama_token_data{token_id, logits[token_id], 0.0f});
|
||||||
|
}
|
||||||
|
|
||||||
|
llama_token_data_array cur_p = { cur.data(), cur.size(), false };
|
||||||
|
|
||||||
|
// apply penalties
|
||||||
|
const auto& penalty_tokens = params.use_penalty_prompt_tokens ? params.penalty_prompt_tokens : prev;
|
||||||
|
const int penalty_tokens_used_size = std::min((int)penalty_tokens.size(), penalty_last_n);
|
||||||
|
if (penalty_tokens_used_size) {
|
||||||
|
const float nl_logit = logits[llama_token_nl(llama_get_model(ctx_main))];
|
||||||
|
|
||||||
|
llama_sample_repetition_penalties(ctx_main, &cur_p,
|
||||||
|
penalty_tokens.data() + penalty_tokens.size() - penalty_tokens_used_size,
|
||||||
|
penalty_tokens_used_size, penalty_repeat, penalty_freq, penalty_present);
|
||||||
|
|
||||||
|
if (!penalize_nl) {
|
||||||
|
for (size_t idx = 0; idx < cur_p.size; idx++) {
|
||||||
|
if (cur_p.data[idx].id == llama_token_nl(llama_get_model(ctx_main))) {
|
||||||
|
cur_p.data[idx].logit = nl_logit;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// apply grammar checks before sampling logic
|
||||||
|
if (apply_grammar && ctx_sampling->grammar != NULL) {
|
||||||
|
llama_sample_grammar(ctx_main, &cur_p, ctx_sampling->grammar);
|
||||||
|
}
|
||||||
|
|
||||||
|
return cur_p;
|
||||||
|
}
|
||||||
|
|
||||||
|
llama_token llama_sampling_sample(
|
||||||
|
struct llama_sampling_context * ctx_sampling,
|
||||||
|
struct llama_context * ctx_main,
|
||||||
|
struct llama_context * ctx_cfg,
|
||||||
|
const int idx) {
|
||||||
|
// Call the implementation function with is_resampling set to false by default
|
||||||
|
return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
llama_token_data_array llama_sampling_prepare(
|
||||||
|
struct llama_sampling_context * ctx_sampling,
|
||||||
|
struct llama_context * ctx_main,
|
||||||
|
struct llama_context * ctx_cfg,
|
||||||
|
const int idx,
|
||||||
|
bool apply_grammar,
|
||||||
|
std::vector<float> * original_logits) {
|
||||||
|
return llama_sampling_prepare_impl(ctx_sampling,ctx_main, ctx_cfg, idx, apply_grammar, original_logits);
|
||||||
|
}
|
||||||
|
|
||||||
|
void llama_sampling_accept(
|
||||||
|
struct llama_sampling_context * ctx_sampling,
|
||||||
|
struct llama_context * ctx_main,
|
||||||
|
llama_token id,
|
||||||
|
bool apply_grammar) {
|
||||||
|
ctx_sampling->prev.erase(ctx_sampling->prev.begin());
|
||||||
|
ctx_sampling->prev.push_back(id);
|
||||||
|
|
||||||
|
if (ctx_sampling->grammar != NULL && apply_grammar) {
|
||||||
|
llama_grammar_accept_token(ctx_main, ctx_sampling->grammar, id);
|
||||||
|
}
|
||||||
|
}
|
||||||
155
common/sampling.h
Normal file
155
common/sampling.h
Normal file
|
|
@ -0,0 +1,155 @@
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "llama.h"
|
||||||
|
|
||||||
|
#include "grammar-parser.h"
|
||||||
|
|
||||||
|
#include <random>
|
||||||
|
#include <string>
|
||||||
|
#include <unordered_map>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
// sampler types
|
||||||
|
enum class llama_sampler_type : char {
|
||||||
|
TOP_K = 'k',
|
||||||
|
TOP_P = 'p',
|
||||||
|
MIN_P = 'm',
|
||||||
|
TFS_Z = 'f',
|
||||||
|
TYPICAL_P = 'y',
|
||||||
|
TEMPERATURE = 't'
|
||||||
|
};
|
||||||
|
|
||||||
|
// sampling parameters
|
||||||
|
typedef struct llama_sampling_params {
|
||||||
|
int32_t n_prev = 64; // number of previous tokens to remember
|
||||||
|
int32_t n_probs = 0; // if greater than 0, output the probabilities of top n_probs tokens.
|
||||||
|
int32_t min_keep = 0; // 0 = disabled, otherwise samplers should return at least min_keep tokens
|
||||||
|
int32_t top_k = 40; // <= 0 to use vocab size
|
||||||
|
float top_p = 0.95f; // 1.0 = disabled
|
||||||
|
float min_p = 0.05f; // 0.0 = disabled
|
||||||
|
float tfs_z = 1.00f; // 1.0 = disabled
|
||||||
|
float typical_p = 1.00f; // 1.0 = disabled
|
||||||
|
float temp = 0.80f; // <= 0.0 to sample greedily, 0.0 to not output probabilities
|
||||||
|
float dynatemp_range = 0.00f; // 0.0 = disabled
|
||||||
|
float dynatemp_exponent = 1.00f; // controls how entropy maps to temperature in dynamic temperature sampler
|
||||||
|
int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty, -1 = context size)
|
||||||
|
float penalty_repeat = 1.00f; // 1.0 = disabled
|
||||||
|
float penalty_freq = 0.00f; // 0.0 = disabled
|
||||||
|
float penalty_present = 0.00f; // 0.0 = disabled
|
||||||
|
int32_t mirostat = 0; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0
|
||||||
|
float mirostat_tau = 5.00f; // target entropy
|
||||||
|
float mirostat_eta = 0.10f; // learning rate
|
||||||
|
bool penalize_nl = false; // consider newlines as a repeatable token
|
||||||
|
uint32_t seed = LLAMA_DEFAULT_SEED; // the seed used to initialize llama_sampling_context
|
||||||
|
|
||||||
|
std::vector<llama_sampler_type> samplers_sequence = {
|
||||||
|
llama_sampler_type::TOP_K,
|
||||||
|
llama_sampler_type::TFS_Z,
|
||||||
|
llama_sampler_type::TYPICAL_P,
|
||||||
|
llama_sampler_type::TOP_P,
|
||||||
|
llama_sampler_type::MIN_P,
|
||||||
|
llama_sampler_type::TEMPERATURE
|
||||||
|
};
|
||||||
|
|
||||||
|
std::string grammar; // optional BNF-like grammar to constrain sampling
|
||||||
|
|
||||||
|
// Classifier-Free Guidance
|
||||||
|
// https://arxiv.org/abs/2306.17806
|
||||||
|
std::string cfg_negative_prompt; // string to help guidance
|
||||||
|
float cfg_scale = 1.f; // how strong is guidance
|
||||||
|
|
||||||
|
std::unordered_map<llama_token, float> logit_bias; // logit bias for specific tokens
|
||||||
|
|
||||||
|
std::vector<llama_token> penalty_prompt_tokens;
|
||||||
|
bool use_penalty_prompt_tokens = false;
|
||||||
|
} llama_sampling_params;
|
||||||
|
|
||||||
|
// general sampler context
|
||||||
|
// TODO: move to llama.h
|
||||||
|
struct llama_sampling_context {
|
||||||
|
// parameters that will be used for sampling
|
||||||
|
llama_sampling_params params;
|
||||||
|
|
||||||
|
// mirostat sampler state
|
||||||
|
float mirostat_mu;
|
||||||
|
|
||||||
|
llama_grammar * grammar;
|
||||||
|
|
||||||
|
// internal
|
||||||
|
grammar_parser::parse_state parsed_grammar;
|
||||||
|
|
||||||
|
// TODO: replace with ring-buffer
|
||||||
|
std::vector<llama_token> prev;
|
||||||
|
std::vector<llama_token_data> cur;
|
||||||
|
size_t n_considered;
|
||||||
|
|
||||||
|
std::mt19937 rng;
|
||||||
|
};
|
||||||
|
|
||||||
|
#include "common.h"
|
||||||
|
|
||||||
|
// Create a new sampling context instance.
|
||||||
|
struct llama_sampling_context * llama_sampling_init(const struct llama_sampling_params & params);
|
||||||
|
|
||||||
|
void llama_sampling_free(struct llama_sampling_context * ctx);
|
||||||
|
|
||||||
|
// Reset the sampler context
|
||||||
|
// - clear prev tokens
|
||||||
|
// - reset grammar
|
||||||
|
void llama_sampling_reset(llama_sampling_context * ctx);
|
||||||
|
|
||||||
|
// Set the sampler seed
|
||||||
|
void llama_sampling_set_rng_seed(struct llama_sampling_context * ctx, uint32_t seed);
|
||||||
|
|
||||||
|
// Copy the sampler context
|
||||||
|
void llama_sampling_cp(llama_sampling_context * src, llama_sampling_context * dst);
|
||||||
|
|
||||||
|
// Get the last sampled token
|
||||||
|
llama_token llama_sampling_last(llama_sampling_context * ctx);
|
||||||
|
|
||||||
|
// Get a string representation of the last sampled tokens
|
||||||
|
std::string llama_sampling_prev_str(llama_sampling_context * ctx_sampling, llama_context * ctx_main, int n);
|
||||||
|
|
||||||
|
// Print sampling parameters into a string
|
||||||
|
std::string llama_sampling_print(const llama_sampling_params & params);
|
||||||
|
|
||||||
|
// Print sampling order into a string
|
||||||
|
std::string llama_sampling_order_print(const llama_sampling_params & params);
|
||||||
|
|
||||||
|
// this is a common sampling function used across the examples for convenience
|
||||||
|
// it can serve as a starting point for implementing your own sampling function
|
||||||
|
// Note: When using multiple sequences, it is the caller's responsibility to call
|
||||||
|
// llama_sampling_reset when a sequence ends
|
||||||
|
//
|
||||||
|
// required:
|
||||||
|
// - ctx_main: context to use for sampling
|
||||||
|
// - ctx_sampling: sampling-specific context
|
||||||
|
//
|
||||||
|
// optional:
|
||||||
|
// - ctx_cfg: context to use for classifier-free guidance
|
||||||
|
// - idx: sample from llama_get_logits_ith(ctx, idx)
|
||||||
|
//
|
||||||
|
// returns:
|
||||||
|
// - token: sampled token
|
||||||
|
// - candidates: vector of candidate tokens
|
||||||
|
//
|
||||||
|
llama_token llama_sampling_sample(
|
||||||
|
struct llama_sampling_context * ctx_sampling,
|
||||||
|
struct llama_context * ctx_main,
|
||||||
|
struct llama_context * ctx_cfg,
|
||||||
|
int idx = -1);
|
||||||
|
|
||||||
|
// Prepares and adjusts the set of token candidates for sampling based on penalties, biases, and sampling parameters.
|
||||||
|
llama_token_data_array llama_sampling_prepare(
|
||||||
|
struct llama_sampling_context * ctx_sampling,
|
||||||
|
struct llama_context * ctx_main,
|
||||||
|
struct llama_context * ctx_cfg,
|
||||||
|
int idx = 0,
|
||||||
|
bool apply_grammar = true,
|
||||||
|
std::vector<float> * original_logits = nullptr);
|
||||||
|
|
||||||
|
void llama_sampling_accept(
|
||||||
|
struct llama_sampling_context * ctx_sampling,
|
||||||
|
struct llama_context * ctx_main,
|
||||||
|
llama_token id,
|
||||||
|
bool apply_grammar);
|
||||||
8396
common/stb_image.h
Normal file
8396
common/stb_image.h
Normal file
File diff suppressed because it is too large
Load diff
1513
common/train.cpp
Normal file
1513
common/train.cpp
Normal file
File diff suppressed because it is too large
Load diff
233
common/train.h
Normal file
233
common/train.h
Normal file
|
|
@ -0,0 +1,233 @@
|
||||||
|
// Various helper functions and utilities for training
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
#include <random>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "ggml.h"
|
||||||
|
#include "llama.h"
|
||||||
|
|
||||||
|
#define LLAMA_TRAIN_MAX_NODES 16384
|
||||||
|
|
||||||
|
typedef std::string mt19937_state;
|
||||||
|
|
||||||
|
struct train_state {
|
||||||
|
struct ggml_opt_context * opt;
|
||||||
|
|
||||||
|
uint64_t train_its;
|
||||||
|
uint64_t train_samples;
|
||||||
|
uint64_t train_tokens;
|
||||||
|
uint64_t train_epochs;
|
||||||
|
|
||||||
|
size_t shuffle_samples_hash; // fn, sample_count, *zip(sample_begins, sample_sizes)
|
||||||
|
mt19937_state shuffle_rng_state_current;
|
||||||
|
mt19937_state shuffle_rng_state_next;
|
||||||
|
size_t shuffle_sample_count;
|
||||||
|
size_t shuffle_next_sample;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct train_params_common {
|
||||||
|
const char * fn_train_data;
|
||||||
|
const char * fn_checkpoint_in;
|
||||||
|
const char * fn_checkpoint_out;
|
||||||
|
const char * pattern_fn_it;
|
||||||
|
const char * fn_latest;
|
||||||
|
|
||||||
|
bool print_usage;
|
||||||
|
|
||||||
|
int save_every;
|
||||||
|
|
||||||
|
uint32_t seed;
|
||||||
|
|
||||||
|
int n_ctx;
|
||||||
|
int n_threads;
|
||||||
|
int n_batch;
|
||||||
|
int n_gradient_accumulation;
|
||||||
|
int n_epochs;
|
||||||
|
int n_gpu_layers;
|
||||||
|
|
||||||
|
bool custom_n_ctx;
|
||||||
|
|
||||||
|
bool use_flash;
|
||||||
|
bool use_checkpointing;
|
||||||
|
|
||||||
|
std::string sample_start;
|
||||||
|
bool include_sample_start;
|
||||||
|
bool escape;
|
||||||
|
bool overlapping_samples;
|
||||||
|
bool fill_with_next_samples;
|
||||||
|
bool separate_with_eos;
|
||||||
|
bool separate_with_bos;
|
||||||
|
bool sample_random_offsets;
|
||||||
|
|
||||||
|
bool force_reshuffle;
|
||||||
|
|
||||||
|
int warmup;
|
||||||
|
int cos_decay_steps;
|
||||||
|
float cos_decay_restart;
|
||||||
|
float cos_decay_min;
|
||||||
|
bool enable_restart;
|
||||||
|
|
||||||
|
int opt_past;
|
||||||
|
float opt_delta;
|
||||||
|
int opt_max_no_improvement;
|
||||||
|
|
||||||
|
int adam_n_iter;
|
||||||
|
float adam_alpha;
|
||||||
|
float adam_min_alpha;
|
||||||
|
float adam_decay;
|
||||||
|
int adam_decay_min_ndim;
|
||||||
|
float adam_beta1;
|
||||||
|
float adam_beta2;
|
||||||
|
float adam_gclip;
|
||||||
|
float adam_eps_f;
|
||||||
|
};
|
||||||
|
|
||||||
|
typedef void (*save_train_files_callback)(void * data, struct train_state * train);
|
||||||
|
|
||||||
|
struct train_opt_callback_data {
|
||||||
|
struct train_params_common * params;
|
||||||
|
struct train_state * train;
|
||||||
|
save_train_files_callback save_cb;
|
||||||
|
void * save_data;
|
||||||
|
struct llama_context * lctx;
|
||||||
|
int last_save_iter;
|
||||||
|
llama_token * tokens_data;
|
||||||
|
size_t tokens_size;
|
||||||
|
size_t * samples_begin;
|
||||||
|
size_t * samples_size;
|
||||||
|
size_t * shuffled_samples_offs;
|
||||||
|
size_t * shuffled_samples_begin;
|
||||||
|
size_t * shuffled_samples_size;
|
||||||
|
size_t samples_count;
|
||||||
|
struct ggml_tensor * tokens_input;
|
||||||
|
struct ggml_tensor * target_probs;
|
||||||
|
int first_iter;
|
||||||
|
int first_epoch;
|
||||||
|
int iter_at_last_epoch;
|
||||||
|
int64_t last_time;
|
||||||
|
double millis_per_iter;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct train_state * init_train_state();
|
||||||
|
void free_train_state(struct train_state * state);
|
||||||
|
|
||||||
|
struct train_params_common get_default_train_params_common();
|
||||||
|
void print_common_train_usage(int /*argc*/, char ** argv, const struct train_params_common * params);
|
||||||
|
|
||||||
|
bool consume_common_train_arg(int argc, char ** argv, int * idx, struct train_params_common * params, bool * invalid_param);
|
||||||
|
void finish_processing_train_args(struct train_params_common * params);
|
||||||
|
|
||||||
|
struct random_normal_distribution;
|
||||||
|
struct random_uniform_distribution;
|
||||||
|
|
||||||
|
struct random_normal_distribution * init_random_normal_distribution (int seed, float mean, float std, float min, float max);
|
||||||
|
struct random_uniform_distribution * init_random_uniform_distribution(int seed, float min, float max);
|
||||||
|
|
||||||
|
void free_random_normal_distribution (struct random_normal_distribution * rnd);
|
||||||
|
void free_random_uniform_distribution(struct random_uniform_distribution * rnd);
|
||||||
|
|
||||||
|
struct ggml_tensor * randomize_tensor_normal (struct ggml_tensor * tensor, struct random_normal_distribution * rnd);
|
||||||
|
struct ggml_tensor * randomize_tensor_uniform(struct ggml_tensor * tensor, struct random_uniform_distribution * rnd);
|
||||||
|
|
||||||
|
// generate random float in interval [0,1)
|
||||||
|
float frand();
|
||||||
|
float frand_normal (struct random_normal_distribution * rnd);
|
||||||
|
float frand_uniform(struct random_uniform_distribution * rnd);
|
||||||
|
|
||||||
|
int clamp (const int v, const int min, const int max);
|
||||||
|
float fclamp(const float v, const float min, const float max);
|
||||||
|
|
||||||
|
void assert_shape_1d(struct ggml_tensor * tensor, int64_t ne0);
|
||||||
|
void assert_shape_2d(struct ggml_tensor * tensor, int64_t ne0, int64_t ne1);
|
||||||
|
void assert_shape_3d(struct ggml_tensor * tensor, int64_t ne0, int64_t ne1, int64_t ne2);
|
||||||
|
void assert_shape_4d(struct ggml_tensor * tensor, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3);
|
||||||
|
|
||||||
|
size_t tokenize_file(
|
||||||
|
struct llama_context * lctx,
|
||||||
|
const char * filename,
|
||||||
|
const std::string & sample_start,
|
||||||
|
bool include_sample_start,
|
||||||
|
bool overlapping_samples,
|
||||||
|
unsigned context_length,
|
||||||
|
std::vector<llama_token> & out_tokens,
|
||||||
|
std::vector<size_t> & out_samples_begin,
|
||||||
|
std::vector<size_t> & out_samples_size);
|
||||||
|
|
||||||
|
int64_t get_example_targets_batch(
|
||||||
|
struct llama_context * lctx,
|
||||||
|
struct ggml_tensor * tokens_input,
|
||||||
|
struct ggml_tensor * target_probs,
|
||||||
|
int64_t example_id,
|
||||||
|
const size_t * samples_offs,
|
||||||
|
const size_t * samples_begin,
|
||||||
|
const size_t * samples_size,
|
||||||
|
size_t samples_count,
|
||||||
|
const llama_token * train_data,
|
||||||
|
size_t n_train_data,
|
||||||
|
bool separate_with_eos,
|
||||||
|
bool separate_with_bos,
|
||||||
|
bool fill_with_next_samples,
|
||||||
|
bool sample_random_offsets);
|
||||||
|
|
||||||
|
|
||||||
|
void mt19937_set_state(std::mt19937& rng, const mt19937_state& rng_state);
|
||||||
|
mt19937_state mt19937_get_state(const std::mt19937& rng);
|
||||||
|
mt19937_state mt19937_seed_to_state(unsigned seed);
|
||||||
|
|
||||||
|
mt19937_state shuffle_samples(
|
||||||
|
const mt19937_state & rng_state,
|
||||||
|
size_t * shuffled_offs,
|
||||||
|
size_t * shuffled_begins,
|
||||||
|
size_t * shuffled_sizes,
|
||||||
|
const size_t * begins,
|
||||||
|
const size_t * sizes,
|
||||||
|
size_t count);
|
||||||
|
|
||||||
|
size_t hash_combine(size_t h1, size_t h2);
|
||||||
|
|
||||||
|
size_t compute_samples_hash(
|
||||||
|
const char* fn,
|
||||||
|
const size_t* samples_begin,
|
||||||
|
const size_t* samples_size,
|
||||||
|
size_t sample_count);
|
||||||
|
|
||||||
|
|
||||||
|
std::string replace_str(const char * s, const char * needle, const char * replacement);
|
||||||
|
|
||||||
|
void print_duration(double milliseconds);
|
||||||
|
|
||||||
|
float cosine_decay(
|
||||||
|
int64_t step,
|
||||||
|
int64_t decay_steps,
|
||||||
|
float minimum);
|
||||||
|
|
||||||
|
float cosine_decay_restart(
|
||||||
|
int64_t step,
|
||||||
|
int64_t decay_steps,
|
||||||
|
float minimum,
|
||||||
|
float restart_step_mult);
|
||||||
|
|
||||||
|
float learning_schedule(
|
||||||
|
int64_t step,
|
||||||
|
int64_t warmup_steps,
|
||||||
|
int64_t decay_steps,
|
||||||
|
float learning_rate,
|
||||||
|
float overall_minimum,
|
||||||
|
float cos_decay_minimum,
|
||||||
|
float cos_decay_restart_step_mult,
|
||||||
|
bool enable_restart);
|
||||||
|
|
||||||
|
void copy_tensor_by_name(struct ggml_tensor * dst, struct ggml_context * ctx, const char * name);
|
||||||
|
|
||||||
|
void load_opt_context_gguf(struct gguf_context * fctx, struct ggml_context * f_ggml_ctx, struct ggml_opt_context * opt);
|
||||||
|
void save_opt_context_gguf(struct gguf_context * fctx, struct ggml_opt_context * opt);
|
||||||
|
|
||||||
|
bool load_train_state_gguf(struct gguf_context * fctx, struct ggml_context * f_ggml_ctx, struct train_state * train);
|
||||||
|
void save_train_state_gguf(struct gguf_context * fctx, struct train_state * train);
|
||||||
|
|
||||||
|
std::string get_train_filename(const char * filename, const char * pattern_it, const char * latest, int64_t iteration);
|
||||||
|
|
||||||
|
void train_opt_callback(void * vdata, int accum_step, float * sched, bool * cancel);
|
||||||
311
convert-hf-to-gguf-update.py
Executable file
311
convert-hf-to-gguf-update.py
Executable file
|
|
@ -0,0 +1,311 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
# This script downloads the tokenizer models of the specified models from Huggingface and
|
||||||
|
# generates the get_vocab_base_pre() function for convert-hf-to-gguf.py
|
||||||
|
#
|
||||||
|
# This is necessary in order to analyze the type of pre-tokenizer used by the model and
|
||||||
|
# provide the necessary information to llama.cpp via the GGUF header in order to implement
|
||||||
|
# the same pre-tokenizer.
|
||||||
|
#
|
||||||
|
# ref: https://github.com/ggerganov/llama.cpp/pull/6920
|
||||||
|
#
|
||||||
|
# Instructions:
|
||||||
|
#
|
||||||
|
# - Add a new model to the "models" list
|
||||||
|
# - Run the script with your huggingface token:
|
||||||
|
#
|
||||||
|
# python3 convert-hf-to-gguf-update.py <huggingface_token>
|
||||||
|
#
|
||||||
|
# - Copy-paste the generated get_vocab_base_pre() function into convert-hf-to-gguf.py
|
||||||
|
# - Update llama.cpp with the new pre-tokenizer if necessary
|
||||||
|
#
|
||||||
|
# TODO: generate tokenizer tests for llama.cpp
|
||||||
|
# TODO: automate the update of convert-hf-to-gguf.py
|
||||||
|
#
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import requests
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
|
||||||
|
from hashlib import sha256
|
||||||
|
from enum import IntEnum, auto
|
||||||
|
from transformers import AutoTokenizer
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.DEBUG)
|
||||||
|
logger = logging.getLogger("convert-hf-to-gguf-update")
|
||||||
|
|
||||||
|
|
||||||
|
class TOKENIZER_TYPE(IntEnum):
|
||||||
|
SPM = auto()
|
||||||
|
BPE = auto()
|
||||||
|
WPM = auto()
|
||||||
|
|
||||||
|
|
||||||
|
# TODO: this string has to exercise as much pre-tokenizer functionality as possible
|
||||||
|
# will be updated with time - contributions welcome
|
||||||
|
chktxt = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български \'\'\'\'\'\'```````\"\"\"\"......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL'
|
||||||
|
|
||||||
|
if len(sys.argv) == 2:
|
||||||
|
token = sys.argv[1]
|
||||||
|
if not token.startswith("hf_"):
|
||||||
|
logger.info("Huggingface token seems invalid")
|
||||||
|
logger.info("Usage: python convert-hf-to-gguf-update.py <huggingface_token>")
|
||||||
|
sys.exit(1)
|
||||||
|
else:
|
||||||
|
logger.info("Usage: python convert-hf-to-gguf-update.py <huggingface_token>")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# TODO: add models here, base models preferred
|
||||||
|
models = [
|
||||||
|
{"name": "llama-spm", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/meta-llama/Llama-2-7b-hf", },
|
||||||
|
{"name": "llama-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", },
|
||||||
|
{"name": "phi-3", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct", },
|
||||||
|
{"name": "deepseek-llm", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-llm-7b-base", },
|
||||||
|
{"name": "deepseek-coder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base", },
|
||||||
|
{"name": "falcon", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/falcon-7b", },
|
||||||
|
{"name": "bert-bge", "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/BAAI/bge-small-en-v1.5", },
|
||||||
|
{"name": "mpt", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mosaicml/mpt-7b", },
|
||||||
|
{"name": "starcoder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/bigcode/starcoder2-3b", },
|
||||||
|
{"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai-community/gpt2", },
|
||||||
|
{"name": "refact", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/smallcloudai/Refact-1_6-base", },
|
||||||
|
{"name": "command-r", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", },
|
||||||
|
{"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", },
|
||||||
|
{"name": "olmo", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", },
|
||||||
|
{"name": "dbrx", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/databricks/dbrx-base", },
|
||||||
|
]
|
||||||
|
|
||||||
|
# make directory "models/tokenizers" if it doesn't exist
|
||||||
|
if not os.path.exists("models/tokenizers"):
|
||||||
|
os.makedirs("models/tokenizers")
|
||||||
|
|
||||||
|
|
||||||
|
def download_file_with_auth(url, token, save_path):
|
||||||
|
headers = {"Authorization": f"Bearer {token}"}
|
||||||
|
response = requests.get(url, headers=headers)
|
||||||
|
if response.status_code == 200:
|
||||||
|
with open(save_path, 'wb') as f:
|
||||||
|
f.write(response.content)
|
||||||
|
logger.info(f"File {save_path} downloaded successfully")
|
||||||
|
else:
|
||||||
|
logger.info(f"Failed to download file. Status code: {response.status_code}")
|
||||||
|
|
||||||
|
|
||||||
|
# download the tokenizer models
|
||||||
|
for model in models:
|
||||||
|
name = model["name"]
|
||||||
|
repo = model["repo"]
|
||||||
|
tokt = model["tokt"]
|
||||||
|
|
||||||
|
if not os.path.exists(f"models/tokenizers/{name}"):
|
||||||
|
os.makedirs(f"models/tokenizers/{name}")
|
||||||
|
else:
|
||||||
|
logger.info(f"Directory models/tokenizers/{name} already exists - skipping")
|
||||||
|
continue
|
||||||
|
|
||||||
|
logger.info(f"Downloading {name} to models/tokenizers/{name}")
|
||||||
|
|
||||||
|
url = f"{repo}/raw/main/config.json"
|
||||||
|
save_path = f"models/tokenizers/{name}/config.json"
|
||||||
|
download_file_with_auth(url, token, save_path)
|
||||||
|
|
||||||
|
url = f"{repo}/raw/main/tokenizer.json"
|
||||||
|
save_path = f"models/tokenizers/{name}/tokenizer.json"
|
||||||
|
download_file_with_auth(url, token, save_path)
|
||||||
|
|
||||||
|
# if downloaded file is less than 1KB, we likely need to download an LFS instead
|
||||||
|
if os.path.getsize(save_path) < 1024:
|
||||||
|
# remove the file
|
||||||
|
os.remove(save_path)
|
||||||
|
url = f"{repo}/resolve/main/tokenizer.json"
|
||||||
|
save_path = f"models/tokenizers/{name}/tokenizer.json"
|
||||||
|
download_file_with_auth(url, token, save_path)
|
||||||
|
|
||||||
|
if tokt == TOKENIZER_TYPE.SPM:
|
||||||
|
url = f"{repo}/resolve/main/tokenizer.model"
|
||||||
|
save_path = f"models/tokenizers/{name}/tokenizer.model"
|
||||||
|
download_file_with_auth(url, token, save_path)
|
||||||
|
|
||||||
|
url = f"{repo}/raw/main/tokenizer_config.json"
|
||||||
|
save_path = f"models/tokenizers/{name}/tokenizer_config.json"
|
||||||
|
download_file_with_auth(url, token, save_path)
|
||||||
|
|
||||||
|
# generate the source code for the convert-hf-to-gguf.py:get_vocab_base_pre() function:
|
||||||
|
# TODO: auto-update convert-hf-to-gguf.py with the generated function
|
||||||
|
|
||||||
|
src_ifs = ""
|
||||||
|
for model in models:
|
||||||
|
name = model["name"]
|
||||||
|
tokt = model["tokt"]
|
||||||
|
|
||||||
|
if tokt == TOKENIZER_TYPE.SPM:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# create the tokenizer
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")
|
||||||
|
|
||||||
|
chktok = tokenizer.encode(chktxt)
|
||||||
|
chkhsh = sha256(str(chktok).encode()).hexdigest()
|
||||||
|
|
||||||
|
logger.info(f"model: {name}")
|
||||||
|
logger.info(f"tokt: {tokt}")
|
||||||
|
logger.info(f"repo: {model['repo']}")
|
||||||
|
logger.info(f"chktok: {chktok}")
|
||||||
|
logger.info(f"chkhsh: {chkhsh}")
|
||||||
|
|
||||||
|
# print the "pre_tokenizer" content from the tokenizer.json
|
||||||
|
with open(f"models/tokenizers/{name}/tokenizer.json", "r", encoding="utf-8") as f:
|
||||||
|
cfg = json.load(f)
|
||||||
|
normalizer = cfg["normalizer"]
|
||||||
|
logger.info("normalizer: " + json.dumps(normalizer, indent=4))
|
||||||
|
pre_tokenizer = cfg["pre_tokenizer"]
|
||||||
|
logger.info("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4))
|
||||||
|
|
||||||
|
logger.info("")
|
||||||
|
|
||||||
|
src_ifs += f" if chkhsh == \"{chkhsh}\":\n"
|
||||||
|
src_ifs += f" # ref: {model['repo']}\n"
|
||||||
|
src_ifs += f" res = \"{name}\"\n"
|
||||||
|
|
||||||
|
src_func = f"""
|
||||||
|
def get_vocab_base_pre(self, tokenizer) -> str:
|
||||||
|
# encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that
|
||||||
|
# is specific for the BPE pre-tokenizer used by the model
|
||||||
|
# we will use this unique identifier to write a "tokenizer.ggml.pre" entry in the GGUF file which we can
|
||||||
|
# use in llama.cpp to implement the same pre-tokenizer
|
||||||
|
|
||||||
|
chktxt = {repr(chktxt)}
|
||||||
|
|
||||||
|
chktok = tokenizer.encode(chktxt)
|
||||||
|
chkhsh = sha256(str(chktok).encode()).hexdigest()
|
||||||
|
|
||||||
|
logger.debug(f"chktok: {{chktok}}")
|
||||||
|
logger.debug(f"chkhsh: {{chkhsh}}")
|
||||||
|
|
||||||
|
res = None
|
||||||
|
|
||||||
|
# NOTE: if you get an error here, you need to update the convert-hf-to-gguf-update.py script
|
||||||
|
# or pull the latest version of the model from Huggingface
|
||||||
|
# don't edit the hashes manually!
|
||||||
|
{src_ifs}
|
||||||
|
if res is None:
|
||||||
|
logger.warning("\\n")
|
||||||
|
logger.warning("**************************************************************************************")
|
||||||
|
logger.warning("** WARNING: The BPE pre-tokenizer was not recognized!")
|
||||||
|
logger.warning("** There are 2 possible reasons for this:")
|
||||||
|
logger.warning("** - the model has not been added to convert-hf-to-gguf-update.py yet")
|
||||||
|
logger.warning("** - the pre-tokenization config has changed upstream")
|
||||||
|
logger.warning("** Check your model files and convert-hf-to-gguf-update.py and update them accordingly.")
|
||||||
|
logger.warning("** ref: https://github.com/ggerganov/llama.cpp/pull/6920")
|
||||||
|
logger.warning("**")
|
||||||
|
logger.warning(f"** chkhsh: {{chkhsh}}")
|
||||||
|
logger.warning("**************************************************************************************")
|
||||||
|
logger.warning("\\n")
|
||||||
|
raise NotImplementedError("BPE pre-tokenizer was not recognized - update get_vocab_base_pre()")
|
||||||
|
|
||||||
|
logger.debug(f"tokenizer.ggml.pre: {{repr(res)}}")
|
||||||
|
logger.debug(f"chkhsh: {{chkhsh}}")
|
||||||
|
|
||||||
|
return res
|
||||||
|
"""
|
||||||
|
|
||||||
|
print(src_func) # noqa: NP100
|
||||||
|
|
||||||
|
logger.info("\n")
|
||||||
|
logger.info("!!! Copy-paste the function above into convert-hf-to-gguf.py !!!")
|
||||||
|
logger.info("\n")
|
||||||
|
|
||||||
|
# generate tests for each tokenizer model
|
||||||
|
|
||||||
|
tests = [
|
||||||
|
"ied 4 ½ months",
|
||||||
|
"Führer",
|
||||||
|
"",
|
||||||
|
" ",
|
||||||
|
" ",
|
||||||
|
" ",
|
||||||
|
"\t",
|
||||||
|
"\n",
|
||||||
|
"\n\n",
|
||||||
|
"\n\n\n",
|
||||||
|
"\t\n",
|
||||||
|
"Hello world",
|
||||||
|
" Hello world",
|
||||||
|
"Hello World",
|
||||||
|
" Hello World",
|
||||||
|
" Hello World!",
|
||||||
|
"Hello, world!",
|
||||||
|
" Hello, world!",
|
||||||
|
" this is 🦙.cpp",
|
||||||
|
"w048 7tuijk dsdfhu",
|
||||||
|
"нещо на Български",
|
||||||
|
"កាន់តែពិសេសអាចខលចេញ",
|
||||||
|
"🚀 (normal) 😶🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)",
|
||||||
|
"Hello",
|
||||||
|
" Hello",
|
||||||
|
" Hello",
|
||||||
|
" Hello",
|
||||||
|
" Hello",
|
||||||
|
" Hello\n Hello",
|
||||||
|
" (",
|
||||||
|
"\n =",
|
||||||
|
"' era",
|
||||||
|
"Hello, y'all! How are you 😁 ?我想在apple工作1314151天～",
|
||||||
|
"3",
|
||||||
|
"33",
|
||||||
|
"333",
|
||||||
|
"3333",
|
||||||
|
"33333",
|
||||||
|
"333333",
|
||||||
|
"3333333",
|
||||||
|
"33333333",
|
||||||
|
"333333333",
|
||||||
|
# "Cửa Việt", # llama-bpe fails on this
|
||||||
|
chktxt,
|
||||||
|
]
|
||||||
|
|
||||||
|
# write the tests to ./models/ggml-vocab-{name}.gguf.inp
|
||||||
|
# the format is:
|
||||||
|
#
|
||||||
|
# test0
|
||||||
|
# __ggml_vocab_test__
|
||||||
|
# test1
|
||||||
|
# __ggml_vocab_test__
|
||||||
|
# ...
|
||||||
|
#
|
||||||
|
|
||||||
|
# with each model, encode all tests and write the results in ./models/ggml-vocab-{name}.gguf.out
|
||||||
|
# for each test, write the resulting tokens on a separate line
|
||||||
|
|
||||||
|
for model in models:
|
||||||
|
name = model["name"]
|
||||||
|
tokt = model["tokt"]
|
||||||
|
|
||||||
|
# create the tokenizer
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")
|
||||||
|
|
||||||
|
with open(f"models/ggml-vocab-{name}.gguf.inp", "w", encoding="utf-8") as f:
|
||||||
|
for text in tests:
|
||||||
|
f.write(f"{text}")
|
||||||
|
f.write("\n__ggml_vocab_test__\n")
|
||||||
|
|
||||||
|
with open(f"models/ggml-vocab-{name}.gguf.out", "w") as f:
|
||||||
|
for text in tests:
|
||||||
|
res = tokenizer.encode(text, add_special_tokens=False)
|
||||||
|
for r in res:
|
||||||
|
f.write(f" {r}")
|
||||||
|
f.write("\n")
|
||||||
|
|
||||||
|
logger.info(f"Tests for {name} written in ./models/ggml-vocab-{name}.gguf.*")
|
||||||
|
|
||||||
|
# generate commands for creating vocab files
|
||||||
|
|
||||||
|
logger.info("\nRun the following commands to generate the vocab files for testing:\n")
|
||||||
|
|
||||||
|
for model in models:
|
||||||
|
name = model["name"]
|
||||||
|
|
||||||
|
print(f"python3 convert-hf-to-gguf.py models/tokenizers/{name}/ --outfile models/ggml-vocab-{name}.gguf --vocab-only") # noqa: NP100
|
||||||
|
|
||||||
|
logger.info("\n")
|
||||||
2511
convert-hf-to-gguf.py
Executable file
2511
convert-hf-to-gguf.py
Executable file
File diff suppressed because it is too large
Load diff
445
convert-llama-ggml-to-gguf.py
Executable file
445
convert-llama-ggml-to-gguf.py
Executable file
|
|
@ -0,0 +1,445 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import struct
|
||||||
|
import sys
|
||||||
|
from enum import IntEnum
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
if 'NO_LOCAL_GGUF' not in os.environ:
|
||||||
|
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))
|
||||||
|
import gguf
|
||||||
|
|
||||||
|
logger = logging.getLogger("ggml-to-gguf")
|
||||||
|
|
||||||
|
|
||||||
|
class GGMLFormat(IntEnum):
|
||||||
|
GGML = 0
|
||||||
|
GGMF = 1
|
||||||
|
GGJT = 2
|
||||||
|
|
||||||
|
|
||||||
|
class GGMLFType(IntEnum):
|
||||||
|
ALL_F32 = 0
|
||||||
|
MOSTLY_F16 = 1
|
||||||
|
MOSTLY_Q4_0 = 2
|
||||||
|
MOSTLY_Q4_1 = 3
|
||||||
|
MOSTLY_Q4_1_SOME_F16 = 4
|
||||||
|
MOSTLY_Q8_0 = 7
|
||||||
|
MOSTLY_Q5_0 = 8
|
||||||
|
MOSTLY_Q5_1 = 9
|
||||||
|
MOSTLY_Q2_K = 10
|
||||||
|
MOSTLY_Q3_K_S = 11
|
||||||
|
MOSTLY_Q3_K_M = 12
|
||||||
|
MOSTLY_Q3_K_L = 13
|
||||||
|
MOSTLY_Q4_K_S = 14
|
||||||
|
MOSTLY_Q4_K_M = 15
|
||||||
|
MOSTLY_Q5_K_S = 16
|
||||||
|
MOSTLY_Q5_K_M = 17
|
||||||
|
MOSTLY_Q6_K = 18
|
||||||
|
|
||||||
|
|
||||||
|
class Hyperparameters:
|
||||||
|
def __init__(self):
|
||||||
|
self.n_vocab = self.n_embd = self.n_mult = self.n_head = 0
|
||||||
|
self.n_layer = self.n_rot = self.n_ff = 0
|
||||||
|
self.ftype = GGMLFType.ALL_F32
|
||||||
|
|
||||||
|
def set_n_ff(self, model):
|
||||||
|
ff_tensor_idx = model.tensor_map.get(b'layers.0.feed_forward.w1.weight')
|
||||||
|
assert ff_tensor_idx is not None, 'Missing layer 0 FF tensor'
|
||||||
|
ff_tensor = model.tensors[ff_tensor_idx]
|
||||||
|
self.n_ff = ff_tensor.dims[1]
|
||||||
|
|
||||||
|
def load(self, data, offset):
|
||||||
|
(
|
||||||
|
self.n_vocab,
|
||||||
|
self.n_embd,
|
||||||
|
self.n_mult,
|
||||||
|
self.n_head,
|
||||||
|
self.n_layer,
|
||||||
|
self.n_rot,
|
||||||
|
ftype,
|
||||||
|
) = struct.unpack('<7I', data[offset:offset + (4 * 7)])
|
||||||
|
try:
|
||||||
|
self.ftype = GGMLFType(ftype)
|
||||||
|
except ValueError:
|
||||||
|
raise ValueError(f'Invalid ftype {ftype}')
|
||||||
|
return 4 * 7
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return f'<Hyperparameters: n_vocab={self.n_vocab}, n_embd={self.n_embd}, n_mult={self.n_mult}, n_head={self.n_head}, n_layer={self.n_layer}, n_rot={self.n_rot}, n_ff={self.n_ff}, ftype={self.ftype.name}>'
|
||||||
|
|
||||||
|
|
||||||
|
class Vocab:
|
||||||
|
def __init__(self, load_scores = True):
|
||||||
|
self.items = []
|
||||||
|
self.load_scores = load_scores
|
||||||
|
|
||||||
|
def load(self, data, offset, n_vocab):
|
||||||
|
orig_offset = offset
|
||||||
|
for _ in range(n_vocab):
|
||||||
|
itemlen = struct.unpack('<I', data[offset:offset + 4])[0]
|
||||||
|
assert itemlen < 4096, 'Absurd vocab item length'
|
||||||
|
offset += 4
|
||||||
|
item_text = bytes(data[offset:offset + itemlen])
|
||||||
|
offset += itemlen
|
||||||
|
if self.load_scores:
|
||||||
|
item_score = struct.unpack('<f', data[offset:offset + 4])[0]
|
||||||
|
offset += 4
|
||||||
|
else:
|
||||||
|
item_score = 0.0
|
||||||
|
self.items.append((item_text, item_score))
|
||||||
|
return offset - orig_offset
|
||||||
|
|
||||||
|
|
||||||
|
class Tensor:
|
||||||
|
def __init__(self, use_padding = True):
|
||||||
|
self.name = None
|
||||||
|
self.dims: tuple[int, ...] = ()
|
||||||
|
self.dtype = None
|
||||||
|
self.start_offset = 0
|
||||||
|
self.len_bytes = np.int64(0)
|
||||||
|
self.use_padding = use_padding
|
||||||
|
|
||||||
|
def load(self, data, offset):
|
||||||
|
orig_offset = offset
|
||||||
|
(n_dims, name_len, dtype) = struct.unpack('<3I', data[offset:offset + 12])
|
||||||
|
assert n_dims >= 0 and n_dims <= 4, f'Invalid tensor dimensions {n_dims}'
|
||||||
|
assert name_len < 4096, 'Absurd tensor name length'
|
||||||
|
quant = gguf.GGML_QUANT_SIZES.get(dtype)
|
||||||
|
assert quant is not None, 'Unknown tensor type'
|
||||||
|
(blksize, tysize) = quant
|
||||||
|
offset += 12
|
||||||
|
self.dtype= dtype
|
||||||
|
self.dims = struct.unpack(f'<{n_dims}I', data[offset:offset + (4 * n_dims)])
|
||||||
|
offset += 4 * n_dims
|
||||||
|
self.name = bytes(data[offset:offset + name_len])
|
||||||
|
offset += name_len
|
||||||
|
pad = ((offset + 31) & ~31) - offset if self.use_padding else 0
|
||||||
|
offset += pad
|
||||||
|
n_elems = np.prod(self.dims)
|
||||||
|
n_bytes = np.int64(np.int64(n_elems) * np.int64(tysize)) // np.int64(blksize)
|
||||||
|
self.start_offset = offset
|
||||||
|
self.len_bytes = n_bytes
|
||||||
|
offset += n_bytes
|
||||||
|
return offset - orig_offset
|
||||||
|
|
||||||
|
|
||||||
|
class GGMLModel:
|
||||||
|
def __init__(self):
|
||||||
|
self.hyperparameters = None
|
||||||
|
self.vocab = None
|
||||||
|
self.tensor_map = {}
|
||||||
|
self.tensors = []
|
||||||
|
|
||||||
|
def validate_header(self, data, offset):
|
||||||
|
magic = bytes(data[offset:offset + 4])
|
||||||
|
if magic == b'GGUF':
|
||||||
|
raise ValueError('File is already in GGUF format.')
|
||||||
|
if magic == b'lmgg':
|
||||||
|
self.file_format = GGMLFormat.GGML
|
||||||
|
self.format_version = 1
|
||||||
|
return 4
|
||||||
|
version = struct.unpack('<I', data[offset + 4:offset + 8])[0]
|
||||||
|
if magic == b'fmgg':
|
||||||
|
if version != 1:
|
||||||
|
raise ValueError(f'Cannot handle unexpected GGMF file version {version}')
|
||||||
|
self.file_format = GGMLFormat.GGMF
|
||||||
|
self.format_version = version
|
||||||
|
return 8
|
||||||
|
if magic == b'tjgg':
|
||||||
|
if version < 1 or version > 3:
|
||||||
|
raise ValueError(f'Cannot handle unexpected GGJT file version {version}')
|
||||||
|
self.file_format = GGMLFormat.GGJT
|
||||||
|
self.format_version = version
|
||||||
|
return 8
|
||||||
|
raise ValueError(f"Unexpected file magic {magic!r}! This doesn't look like a GGML format file.")
|
||||||
|
|
||||||
|
def validate_conversion(self, ftype):
|
||||||
|
err = ''
|
||||||
|
if (self.file_format < GGMLFormat.GGJT or self.format_version < 2):
|
||||||
|
if ftype not in (GGMLFType.ALL_F32, GGMLFType.MOSTLY_F16):
|
||||||
|
err = 'Quantizations changed in GGJTv2. Can only convert unquantized GGML files older than GGJTv2.'
|
||||||
|
elif (self.file_format == GGMLFormat.GGJT and self.format_version == 2):
|
||||||
|
if ftype in (GGMLFType.MOSTLY_Q4_0, GGMLFType.MOSTLY_Q4_1,
|
||||||
|
GGMLFType.MOSTLY_Q4_1_SOME_F16, GGMLFType.MOSTLY_Q8_0):
|
||||||
|
err = 'Q4 and Q8 quantizations changed in GGJTv3.'
|
||||||
|
if len(err) > 0:
|
||||||
|
raise ValueError(f'{err} Sorry, your {self.file_format.name}v{self.format_version} file of type {ftype.name} is not eligible for conversion.')
|
||||||
|
|
||||||
|
def load(self, data, offset):
|
||||||
|
offset += self.validate_header(data, offset)
|
||||||
|
hp = Hyperparameters()
|
||||||
|
offset += hp.load(data, offset)
|
||||||
|
logger.info(f'* File format: {self.file_format.name}v{self.format_version} with ftype {hp.ftype.name}')
|
||||||
|
self.validate_conversion(hp.ftype)
|
||||||
|
vocab = Vocab(load_scores = self.file_format > GGMLFormat.GGML)
|
||||||
|
offset += vocab.load(data, offset, hp.n_vocab)
|
||||||
|
tensors: list[Tensor] = []
|
||||||
|
tensor_map = {}
|
||||||
|
while offset < len(data):
|
||||||
|
tensor = Tensor(use_padding = self.file_format > GGMLFormat.GGMF)
|
||||||
|
offset += tensor.load(data, offset)
|
||||||
|
tensor_map[tensor.name] = len(tensors)
|
||||||
|
tensors.append(tensor)
|
||||||
|
self.hyperparameters = hp
|
||||||
|
self.vocab = vocab
|
||||||
|
self.tensors = tensors
|
||||||
|
self.tensor_map = tensor_map
|
||||||
|
hp.set_n_ff(self)
|
||||||
|
return offset
|
||||||
|
|
||||||
|
|
||||||
|
class GGMLToGGUF:
|
||||||
|
def __init__(self, ggml_model, data, cfg, params_override = None, vocab_override = None, special_vocab = None):
|
||||||
|
hp = ggml_model.hyperparameters
|
||||||
|
self.model = ggml_model
|
||||||
|
self.data = data
|
||||||
|
self.cfg = cfg
|
||||||
|
self.params_override = params_override
|
||||||
|
self.vocab_override = vocab_override
|
||||||
|
self.special_vocab = special_vocab
|
||||||
|
if params_override is not None:
|
||||||
|
n_kv_head = params_override.n_head_kv
|
||||||
|
else:
|
||||||
|
if cfg.gqa == 1:
|
||||||
|
n_kv_head = hp.n_head
|
||||||
|
else:
|
||||||
|
gqa = float(cfg.gqa)
|
||||||
|
n_kv_head = None
|
||||||
|
for x in range(1, 256):
|
||||||
|
if float(hp.n_head) / float(x) == gqa:
|
||||||
|
n_kv_head = x
|
||||||
|
assert n_kv_head is not None, "Couldn't determine n_kv_head from GQA param"
|
||||||
|
logger.info(f'- Guessed n_kv_head = {n_kv_head} based on GQA {cfg.gqa}')
|
||||||
|
self.n_kv_head = n_kv_head
|
||||||
|
self.name_map = gguf.get_tensor_name_map(gguf.MODEL_ARCH.LLAMA, ggml_model.hyperparameters.n_layer)
|
||||||
|
|
||||||
|
def save(self):
|
||||||
|
logger.info('* Preparing to save GGUF file')
|
||||||
|
gguf_writer = gguf.GGUFWriter(
|
||||||
|
self.cfg.output,
|
||||||
|
gguf.MODEL_ARCH_NAMES[gguf.MODEL_ARCH.LLAMA],
|
||||||
|
use_temp_file = False)
|
||||||
|
self.add_params(gguf_writer)
|
||||||
|
self.add_vocab(gguf_writer)
|
||||||
|
if self.special_vocab is not None:
|
||||||
|
self.special_vocab.add_to_gguf(gguf_writer)
|
||||||
|
self.add_tensors(gguf_writer)
|
||||||
|
logger.info(" gguf: write header")
|
||||||
|
gguf_writer.write_header_to_file()
|
||||||
|
logger.info(" gguf: write metadata")
|
||||||
|
gguf_writer.write_kv_data_to_file()
|
||||||
|
logger.info(" gguf: write tensors")
|
||||||
|
gguf_writer.write_tensors_to_file()
|
||||||
|
gguf_writer.close()
|
||||||
|
|
||||||
|
def add_params(self, gguf_writer):
|
||||||
|
hp = self.model.hyperparameters
|
||||||
|
cfg = self.cfg
|
||||||
|
if cfg.desc is not None:
|
||||||
|
desc = cfg.desc
|
||||||
|
else:
|
||||||
|
desc = f'converted from legacy {self.model.file_format.name}v{self.model.format_version} {hp.ftype.name} format'
|
||||||
|
try:
|
||||||
|
# Filenames aren't necessarily valid UTF8.
|
||||||
|
name = cfg.name if cfg.name is not None else cfg.input.name
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
name = None
|
||||||
|
logger.info('* Adding model parameters and KV items')
|
||||||
|
if name is not None:
|
||||||
|
gguf_writer.add_name(name)
|
||||||
|
gguf_writer.add_description(desc)
|
||||||
|
gguf_writer.add_file_type(int(hp.ftype))
|
||||||
|
if self.params_override is not None:
|
||||||
|
po = self.params_override
|
||||||
|
assert po.n_embd == hp.n_embd, 'Model hyperparams mismatch'
|
||||||
|
assert po.n_layer == hp.n_layer, 'Model hyperparams mismatch'
|
||||||
|
assert po.n_head == hp.n_head, 'Model hyperparams mismatch'
|
||||||
|
gguf_writer.add_context_length (po.n_ctx)
|
||||||
|
gguf_writer.add_embedding_length (po.n_embd)
|
||||||
|
gguf_writer.add_block_count (po.n_layer)
|
||||||
|
gguf_writer.add_feed_forward_length (po.n_ff)
|
||||||
|
gguf_writer.add_rope_dimension_count(po.n_embd // po.n_head)
|
||||||
|
gguf_writer.add_head_count (po.n_head)
|
||||||
|
gguf_writer.add_head_count_kv (po.n_head_kv)
|
||||||
|
gguf_writer.add_layer_norm_rms_eps (po.f_norm_eps)
|
||||||
|
return
|
||||||
|
gguf_writer.add_context_length(cfg.context_length)
|
||||||
|
gguf_writer.add_embedding_length(hp.n_embd)
|
||||||
|
gguf_writer.add_block_count(hp.n_layer)
|
||||||
|
gguf_writer.add_feed_forward_length(hp.n_ff)
|
||||||
|
gguf_writer.add_rope_dimension_count(hp.n_embd // hp.n_head)
|
||||||
|
gguf_writer.add_head_count(hp.n_head)
|
||||||
|
gguf_writer.add_head_count_kv(self.n_kv_head)
|
||||||
|
gguf_writer.add_layer_norm_rms_eps(float(cfg.eps))
|
||||||
|
|
||||||
|
def add_vocab(self, gguf_writer):
|
||||||
|
hp = self.model.hyperparameters
|
||||||
|
gguf_writer.add_tokenizer_model('llama')
|
||||||
|
gguf_writer.add_tokenizer_pre('default')
|
||||||
|
tokens = []
|
||||||
|
scores = []
|
||||||
|
toktypes = []
|
||||||
|
if self.vocab_override is not None:
|
||||||
|
vo = self.vocab_override
|
||||||
|
logger.info('* Adding vocab item(s)')
|
||||||
|
for (idx, (vbytes, score, ttype)) in enumerate(vo.all_tokens()):
|
||||||
|
tokens.append(vbytes)
|
||||||
|
scores.append(score)
|
||||||
|
toktypes.append(ttype)
|
||||||
|
assert len(tokens) == hp.n_vocab, \
|
||||||
|
f'Override vocab has a different number of items than hyperparameters - override = {len(tokens)} but n_vocab={hp.n_vocab}'
|
||||||
|
gguf_writer.add_token_list(tokens)
|
||||||
|
gguf_writer.add_token_scores(scores)
|
||||||
|
if len(toktypes) > 0:
|
||||||
|
gguf_writer.add_token_types(toktypes)
|
||||||
|
return
|
||||||
|
logger.info(f'* Adding {hp.n_vocab} vocab item(s)')
|
||||||
|
assert len(self.model.vocab.items) >= 3, 'Cannot handle unexpectedly short model vocab'
|
||||||
|
for (tokid, (vbytes, vscore)) in enumerate(self.model.vocab.items):
|
||||||
|
tt = 1 # Normal
|
||||||
|
# Special handling for UNK, BOS, EOS tokens.
|
||||||
|
if tokid <= 2:
|
||||||
|
if tokid == 0:
|
||||||
|
vbytes = b'<unk>'
|
||||||
|
tt = 2
|
||||||
|
elif tokid == 1:
|
||||||
|
vbytes = b'<s>'
|
||||||
|
tt = 3
|
||||||
|
else:
|
||||||
|
vbytes = b'</s>'
|
||||||
|
tt = 3
|
||||||
|
elif len(vbytes) == 0:
|
||||||
|
tt = 3 # Control
|
||||||
|
elif tokid >= 3 and tokid <= 258 and len(vbytes) == 1:
|
||||||
|
vbytes = bytes(f'<0x{vbytes[0]:02X}>', encoding = 'UTF-8')
|
||||||
|
tt = 6 # Byte
|
||||||
|
else:
|
||||||
|
vbytes = vbytes.replace(b' ', b'\xe2\x96\x81')
|
||||||
|
toktypes.append(tt)
|
||||||
|
tokens.append(vbytes)
|
||||||
|
scores.append(vscore)
|
||||||
|
gguf_writer.add_token_list(tokens)
|
||||||
|
gguf_writer.add_token_scores(scores)
|
||||||
|
gguf_writer.add_token_types(toktypes)
|
||||||
|
gguf_writer.add_unk_token_id(0)
|
||||||
|
gguf_writer.add_bos_token_id(1)
|
||||||
|
gguf_writer.add_eos_token_id(2)
|
||||||
|
|
||||||
|
def add_tensors(self, gguf_writer):
|
||||||
|
tensor_map = self.name_map
|
||||||
|
data = self.data
|
||||||
|
logger.info(f'* Adding {len(self.model.tensors)} tensor(s)')
|
||||||
|
for tensor in self.model.tensors:
|
||||||
|
name = str(tensor.name, 'UTF-8')
|
||||||
|
mapped_name = tensor_map.get_name(name, try_suffixes = (".weight", ".bias"))
|
||||||
|
assert mapped_name is not None, f'Bad name {name}'
|
||||||
|
tempdims = list(tensor.dims[:])
|
||||||
|
if len(tempdims) > 1:
|
||||||
|
temp = tempdims[1]
|
||||||
|
tempdims[1] = tempdims[0]
|
||||||
|
tempdims[0] = temp
|
||||||
|
gguf_writer.add_tensor(
|
||||||
|
mapped_name,
|
||||||
|
data[tensor.start_offset:tensor.start_offset + tensor.len_bytes],
|
||||||
|
raw_shape = tempdims,
|
||||||
|
raw_dtype = tensor.dtype)
|
||||||
|
|
||||||
|
|
||||||
|
def handle_metadata(cfg, hp):
|
||||||
|
import convert
|
||||||
|
assert cfg.model_metadata_dir.is_dir(), 'Metadata dir is not a directory'
|
||||||
|
hf_config_path = cfg.model_metadata_dir / "config.json"
|
||||||
|
orig_config_path = cfg.model_metadata_dir / "params.json"
|
||||||
|
# We pass a fake model here. "original" mode will check the shapes of some
|
||||||
|
# tensors if information is missing in the .json file: other than that, the
|
||||||
|
# model data isn't used so this should be safe (at least for now).
|
||||||
|
fakemodel = {
|
||||||
|
'tok_embeddings.weight': convert.LazyTensor.__new__(convert.LazyTensor),
|
||||||
|
'layers.0.feed_forward.w1.weight': convert.LazyTensor.__new__(convert.LazyTensor),
|
||||||
|
}
|
||||||
|
fakemodel['tok_embeddings.weight'].shape = [hp.n_vocab]
|
||||||
|
fakemodel['layers.0.feed_forward.w1.weight'].shape = [hp.n_ff]
|
||||||
|
if hf_config_path.exists():
|
||||||
|
params = convert.Params.loadHFTransformerJson(fakemodel, hf_config_path)
|
||||||
|
elif orig_config_path.exists():
|
||||||
|
params = convert.Params.loadOriginalParamsJson(fakemodel, orig_config_path)
|
||||||
|
else:
|
||||||
|
raise ValueError('Unable to load metadata')
|
||||||
|
vocab_path = Path(cfg.vocab_dir if cfg.vocab_dir is not None else cfg.model_metadata_dir)
|
||||||
|
vocab_factory = convert.VocabFactory(vocab_path)
|
||||||
|
vocab, special_vocab = vocab_factory.load_vocab(cfg.vocabtype.split(","), cfg.model_metadata_dir)
|
||||||
|
convert.check_vocab_size(params, vocab)
|
||||||
|
return params, vocab, special_vocab
|
||||||
|
|
||||||
|
|
||||||
|
def handle_args():
|
||||||
|
parser = argparse.ArgumentParser(description = 'Convert GGML models to GGUF')
|
||||||
|
parser.add_argument('--input', '-i', type = Path, required = True,
|
||||||
|
help = 'Input GGMLv3 filename')
|
||||||
|
parser.add_argument('--output', '-o', type = Path, required = True,
|
||||||
|
help ='Output GGUF filename')
|
||||||
|
parser.add_argument('--name',
|
||||||
|
help = 'Set model name')
|
||||||
|
parser.add_argument('--desc',
|
||||||
|
help = 'Set model description')
|
||||||
|
parser.add_argument('--gqa', type = int, default = 1,
|
||||||
|
help = 'grouped-query attention factor (use 8 for LLaMA2 70B)')
|
||||||
|
parser.add_argument('--eps', default = '5.0e-06',
|
||||||
|
help = 'RMS norm eps: Use 1e-6 for LLaMA1 and OpenLLaMA, use 1e-5 for LLaMA2')
|
||||||
|
parser.add_argument('--context-length', '-c', type=int, default = 2048,
|
||||||
|
help = 'Default max context length: LLaMA1 is typically 2048, LLaMA2 is typically 4096')
|
||||||
|
parser.add_argument('--model-metadata-dir', '-m', type = Path,
|
||||||
|
help ='Load HuggingFace/.pth vocab and metadata from the specified directory')
|
||||||
|
parser.add_argument("--vocab-dir", type=Path,
|
||||||
|
help="directory containing tokenizer.model, if separate from model file - only meaningful with --model-metadata-dir")
|
||||||
|
parser.add_argument("--vocabtype", default="spm,hfft",
|
||||||
|
help="vocab format - only meaningful with --model-metadata-dir and/or --vocab-dir (default: spm,hfft)")
|
||||||
|
parser.add_argument("--verbose", action="store_true", help="increase output verbosity")
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
cfg = handle_args()
|
||||||
|
logging.basicConfig(level=logging.DEBUG if cfg.verbose else logging.INFO)
|
||||||
|
logger.info(f'* Using config: {cfg}')
|
||||||
|
logger.warning('=== WARNING === Be aware that this conversion script is best-effort. Use a native GGUF model if possible. === WARNING ===')
|
||||||
|
if cfg.model_metadata_dir is None and (cfg.gqa == 1 or cfg.eps == '5.0e-06'):
|
||||||
|
logger.info('- Note: If converting LLaMA2, specifying "--eps 1e-5" is required. 70B models also need "--gqa 8".')
|
||||||
|
data = np.memmap(cfg.input, mode = 'r')
|
||||||
|
model = GGMLModel()
|
||||||
|
logger.info('* Scanning GGML input file')
|
||||||
|
offset = model.load(data, 0) # noqa
|
||||||
|
logger.info(f'* GGML model hyperparameters: {model.hyperparameters}')
|
||||||
|
vocab_override = None
|
||||||
|
params_override = None
|
||||||
|
special_vocab = None
|
||||||
|
if cfg.model_metadata_dir is not None:
|
||||||
|
(params_override, vocab_override, special_vocab) = handle_metadata(cfg, model.hyperparameters)
|
||||||
|
logger.info('!! Note: When overriding params the --gqa, --eps and --context-length options are ignored.')
|
||||||
|
logger.info(f'* Overriding params: {params_override}')
|
||||||
|
logger.info(f'* Overriding vocab: {vocab_override}')
|
||||||
|
logger.info(f'* Special vocab: {special_vocab}')
|
||||||
|
else:
|
||||||
|
logger.warning('\n=== WARNING === Special tokens may not be converted correctly. Use --model-metadata-dir if possible === WARNING ===\n')
|
||||||
|
if model.file_format == GGMLFormat.GGML:
|
||||||
|
logger.info('! This is a very old GGML file that does not contain vocab scores. Strongly recommend using model metadata!')
|
||||||
|
converter = GGMLToGGUF(
|
||||||
|
model, data, cfg,
|
||||||
|
params_override = params_override,
|
||||||
|
vocab_override = vocab_override,
|
||||||
|
special_vocab = special_vocab
|
||||||
|
)
|
||||||
|
converter.save()
|
||||||
|
logger.info(f'* Successful completion. Output saved to: {cfg.output}')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
150
convert-lora-to-ggml.py
Executable file
150
convert-lora-to-ggml.py
Executable file
|
|
@ -0,0 +1,150 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import struct
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, BinaryIO, Sequence
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import torch
|
||||||
|
|
||||||
|
if 'NO_LOCAL_GGUF' not in os.environ:
|
||||||
|
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py' / 'gguf'))
|
||||||
|
import gguf
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.DEBUG)
|
||||||
|
logger = logging.getLogger("lora-to-gguf")
|
||||||
|
|
||||||
|
NUMPY_TYPE_TO_FTYPE: dict[str, int] = {"float32": 0, "float16": 1}
|
||||||
|
|
||||||
|
|
||||||
|
def write_file_header(fout: BinaryIO, params: dict[str, Any]) -> None:
|
||||||
|
fout.write(b"ggla"[::-1]) # magic (ggml lora)
|
||||||
|
fout.write(struct.pack("i", 1)) # file version
|
||||||
|
fout.write(struct.pack("i", params["r"]))
|
||||||
|
# https://opendelta.readthedocs.io/en/latest/modules/deltas.html says that `lora_alpha` is an int
|
||||||
|
# but some models ship a float value instead
|
||||||
|
# let's convert to int, but fail if lossless conversion is not possible
|
||||||
|
assert (
|
||||||
|
int(params["lora_alpha"]) == params["lora_alpha"]
|
||||||
|
), "cannot convert float to int losslessly"
|
||||||
|
fout.write(struct.pack("i", int(params["lora_alpha"])))
|
||||||
|
|
||||||
|
|
||||||
|
def write_tensor_header(fout: BinaryIO, name: str, shape: Sequence[int], data_type: np.dtype[Any]) -> None:
|
||||||
|
sname = name.encode("utf-8")
|
||||||
|
fout.write(
|
||||||
|
struct.pack(
|
||||||
|
"iii",
|
||||||
|
len(shape),
|
||||||
|
len(sname),
|
||||||
|
NUMPY_TYPE_TO_FTYPE[data_type.name],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
fout.write(struct.pack("i" * len(shape), *shape[::-1]))
|
||||||
|
fout.write(sname)
|
||||||
|
fout.seek((fout.tell() + 31) & -32)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
if len(sys.argv) < 2:
|
||||||
|
logger.info(f"Usage: python {sys.argv[0]} <path> [arch]")
|
||||||
|
logger.info("Path must contain HuggingFace PEFT LoRA files 'adapter_config.json' and 'adapter_model.bin'")
|
||||||
|
logger.info(f"Arch must be one of {list(gguf.MODEL_ARCH_NAMES.values())} (default: llama)")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
input_json = os.path.join(sys.argv[1], "adapter_config.json")
|
||||||
|
input_model = os.path.join(sys.argv[1], "adapter_model.bin")
|
||||||
|
output_path = os.path.join(sys.argv[1], "ggml-adapter-model.bin")
|
||||||
|
|
||||||
|
if os.path.exists(input_model):
|
||||||
|
model = torch.load(input_model, map_location="cpu")
|
||||||
|
else:
|
||||||
|
input_model = os.path.join(sys.argv[1], "adapter_model.safetensors")
|
||||||
|
# lazy import load_file only if lora is in safetensors format.
|
||||||
|
from safetensors.torch import load_file
|
||||||
|
model = load_file(input_model, device="cpu")
|
||||||
|
|
||||||
|
arch_name = sys.argv[2] if len(sys.argv) == 3 else "llama"
|
||||||
|
|
||||||
|
if arch_name not in gguf.MODEL_ARCH_NAMES.values():
|
||||||
|
logger.error(f"Error: unsupported architecture {arch_name}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
arch = list(gguf.MODEL_ARCH_NAMES.keys())[list(gguf.MODEL_ARCH_NAMES.values()).index(arch_name)]
|
||||||
|
name_map = gguf.TensorNameMap(arch, 200) # 200 layers ought to be enough for anyone
|
||||||
|
|
||||||
|
with open(input_json, "r") as f:
|
||||||
|
params = json.load(f)
|
||||||
|
|
||||||
|
if params["peft_type"] != "LORA":
|
||||||
|
logger.error(f"Error: unsupported adapter type {params['peft_type']}, expected LORA")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if params["fan_in_fan_out"] is True:
|
||||||
|
logger.error("Error: param fan_in_fan_out is not supported")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if params["bias"] is not None and params["bias"] != "none":
|
||||||
|
logger.error("Error: param bias is not supported")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# TODO: these seem to be layers that have been trained but without lora.
|
||||||
|
# doesn't seem widely used but eventually should be supported
|
||||||
|
if params["modules_to_save"] is not None and len(params["modules_to_save"]) > 0:
|
||||||
|
logger.error("Error: param modules_to_save is not supported")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
with open(output_path, "wb") as fout:
|
||||||
|
fout.truncate()
|
||||||
|
|
||||||
|
write_file_header(fout, params)
|
||||||
|
for k, v in model.items():
|
||||||
|
orig_k = k
|
||||||
|
if k.endswith(".default.weight"):
|
||||||
|
k = k.replace(".default.weight", ".weight")
|
||||||
|
if k in ["llama_proj.weight", "llama_proj.bias"]:
|
||||||
|
continue
|
||||||
|
if k.endswith("lora_A.weight"):
|
||||||
|
if v.dtype != torch.float16 and v.dtype != torch.float32:
|
||||||
|
v = v.float()
|
||||||
|
v = v.T
|
||||||
|
else:
|
||||||
|
v = v.float()
|
||||||
|
|
||||||
|
t = v.detach().numpy()
|
||||||
|
|
||||||
|
prefix = "base_model.model."
|
||||||
|
if k.startswith(prefix):
|
||||||
|
k = k[len(prefix) :]
|
||||||
|
|
||||||
|
lora_suffixes = (".lora_A.weight", ".lora_B.weight")
|
||||||
|
if k.endswith(lora_suffixes):
|
||||||
|
suffix = k[-len(lora_suffixes[0]):]
|
||||||
|
k = k[: -len(lora_suffixes[0])]
|
||||||
|
else:
|
||||||
|
logger.error(f"Error: unrecognized tensor name {orig_k}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
tname = name_map.get_name(k)
|
||||||
|
if tname is None:
|
||||||
|
logger.error(f"Error: could not map tensor name {orig_k}")
|
||||||
|
logger.error(" Note: the arch parameter must be specified if the model is not llama")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if suffix == ".lora_A.weight":
|
||||||
|
tname += ".weight.loraA"
|
||||||
|
elif suffix == ".lora_B.weight":
|
||||||
|
tname += ".weight.loraB"
|
||||||
|
else:
|
||||||
|
assert False
|
||||||
|
|
||||||
|
logger.info(f"{k} => {tname} {t.shape} {t.dtype} {t.nbytes/1024/1024:.2f}MB")
|
||||||
|
write_tensor_header(fout, tname, t.shape, t.dtype)
|
||||||
|
t.tofile(fout)
|
||||||
|
|
||||||
|
logger.info(f"Converted {input_json} and {input_model} to {output_path}")
|
||||||
143
convert-persimmon-to-gguf.py
Executable file
143
convert-persimmon-to-gguf.py
Executable file
|
|
@ -0,0 +1,143 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from pprint import pprint
|
||||||
|
|
||||||
|
import torch
|
||||||
|
from sentencepiece import SentencePieceProcessor
|
||||||
|
|
||||||
|
if 'NO_LOCAL_GGUF' not in os.environ:
|
||||||
|
sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))
|
||||||
|
import gguf
|
||||||
|
|
||||||
|
logger = logging.getLogger("persimmon-to-gguf")
|
||||||
|
|
||||||
|
|
||||||
|
def _flatten_dict(dct, tensors, prefix=None):
|
||||||
|
assert isinstance(dct, dict)
|
||||||
|
for key in dct.keys():
|
||||||
|
new_prefix = prefix + '.' + key if prefix is not None else key
|
||||||
|
if isinstance(dct[key], torch.Tensor):
|
||||||
|
tensors[new_prefix] = dct[key]
|
||||||
|
elif isinstance(dct[key], dict):
|
||||||
|
_flatten_dict(dct[key], tensors, new_prefix)
|
||||||
|
else:
|
||||||
|
raise ValueError(type(dct[key]))
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _get_sentencepiece_tokenizer_info(dir_model: Path):
|
||||||
|
tokenizer_path = dir_model / 'adept_vocab.model'
|
||||||
|
logger.info('getting sentencepiece tokenizer from', tokenizer_path)
|
||||||
|
tokenizer = SentencePieceProcessor(str(tokenizer_path))
|
||||||
|
logger.info('adding tokens')
|
||||||
|
tokens: list[bytes] = []
|
||||||
|
scores: list[float] = []
|
||||||
|
toktypes: list[int] = []
|
||||||
|
|
||||||
|
for i in range(tokenizer.vocab_size()):
|
||||||
|
text: bytes
|
||||||
|
score: float
|
||||||
|
|
||||||
|
piece = tokenizer.id_to_piece(i)
|
||||||
|
text = piece.encode("utf-8")
|
||||||
|
score = tokenizer.get_score(i)
|
||||||
|
|
||||||
|
toktype = 1
|
||||||
|
if tokenizer.is_unknown(i):
|
||||||
|
toktype = 2
|
||||||
|
if tokenizer.is_control(i):
|
||||||
|
toktype = 3
|
||||||
|
if tokenizer.is_unused(i):
|
||||||
|
toktype = 5
|
||||||
|
if tokenizer.is_byte(i):
|
||||||
|
toktype = 6
|
||||||
|
|
||||||
|
tokens.append(text)
|
||||||
|
scores.append(score)
|
||||||
|
toktypes.append(toktype)
|
||||||
|
pass
|
||||||
|
return tokens, scores, toktypes
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Convert a Persimmon model from Adept (e.g. Persimmon 8b chat) to a GGML compatible file")
|
||||||
|
parser.add_argument("--outfile", type=Path, help="path to write to; default: based on input")
|
||||||
|
parser.add_argument("--ckpt-path", type=Path, help="path to persimmon checkpoint .pt file")
|
||||||
|
parser.add_argument("--model-dir", type=Path, help="directory containing model e.g. 8b_chat_model_release")
|
||||||
|
parser.add_argument("--adept-inference-dir", type=str, help="path to adept-inference code directory")
|
||||||
|
parser.add_argument("--verbose", action="store_true", help="increase output verbosity")
|
||||||
|
args = parser.parse_args()
|
||||||
|
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
|
||||||
|
sys.path.append(str(args.adept_inference_dir))
|
||||||
|
persimmon_model = torch.load(args.ckpt_path)
|
||||||
|
hparams = persimmon_model['args']
|
||||||
|
pprint(hparams)
|
||||||
|
tensors: dict[str, torch.Tensor] = {}
|
||||||
|
_flatten_dict(persimmon_model['model'], tensors, None)
|
||||||
|
|
||||||
|
arch = gguf.MODEL_ARCH.PERSIMMON
|
||||||
|
gguf_writer = gguf.GGUFWriter(args.outfile, gguf.MODEL_ARCH_NAMES[arch])
|
||||||
|
|
||||||
|
block_count = hparams.num_layers
|
||||||
|
head_count = hparams.num_attention_heads
|
||||||
|
head_count_kv = head_count
|
||||||
|
ctx_length = hparams.seq_length
|
||||||
|
hidden_size = hparams.hidden_size
|
||||||
|
|
||||||
|
gguf_writer.add_name('persimmon-8b-chat')
|
||||||
|
gguf_writer.add_context_length(ctx_length)
|
||||||
|
gguf_writer.add_embedding_length(hidden_size)
|
||||||
|
gguf_writer.add_block_count(block_count)
|
||||||
|
gguf_writer.add_feed_forward_length(hparams.ffn_hidden_size)
|
||||||
|
# ref: https://github.com/ggerganov/llama.cpp/pull/4889/commits/eea19039fc52ea2dbd1aab45b59ab4e3e29a3443
|
||||||
|
gguf_writer.add_rope_dimension_count(hidden_size // head_count // 2)
|
||||||
|
gguf_writer.add_head_count(head_count)
|
||||||
|
gguf_writer.add_head_count_kv(head_count_kv)
|
||||||
|
gguf_writer.add_rope_freq_base(hparams.rotary_emb_base)
|
||||||
|
gguf_writer.add_layer_norm_eps(hparams.layernorm_epsilon)
|
||||||
|
|
||||||
|
tokens, scores, toktypes = _get_sentencepiece_tokenizer_info(args.model_dir)
|
||||||
|
gguf_writer.add_tokenizer_model('llama')
|
||||||
|
gguf_writer.add_tokenizer_pre('default')
|
||||||
|
gguf_writer.add_token_list(tokens)
|
||||||
|
gguf_writer.add_token_scores(scores)
|
||||||
|
gguf_writer.add_token_types(toktypes)
|
||||||
|
gguf_writer.add_bos_token_id(71013)
|
||||||
|
gguf_writer.add_eos_token_id(71013)
|
||||||
|
|
||||||
|
tensor_map = gguf.get_tensor_name_map(arch, block_count)
|
||||||
|
logger.info(tensor_map)
|
||||||
|
for name in tensors.keys():
|
||||||
|
data_torch = tensors[name]
|
||||||
|
if name.endswith(".self_attention.rotary_emb.inv_freq"):
|
||||||
|
continue
|
||||||
|
old_dtype = data_torch.dtype
|
||||||
|
# TODO: FP16 conversion produces garbage outputs. (Q8_0 does not, so..?)
|
||||||
|
data = data_torch.to(torch.float32).squeeze().numpy()
|
||||||
|
new_name = tensor_map.get_name(name, try_suffixes = (".weight", ".bias"))
|
||||||
|
if new_name is None:
|
||||||
|
raise ValueError(f"Can not map tensor '{name}'")
|
||||||
|
|
||||||
|
n_dims = len(data.shape)
|
||||||
|
logger.debug(f"{new_name}, n_dims = {str(n_dims)}, {str(old_dtype)} --> {str(data.dtype)}")
|
||||||
|
gguf_writer.add_tensor(new_name, data)
|
||||||
|
logger.info("gguf: write header")
|
||||||
|
gguf_writer.write_header_to_file()
|
||||||
|
logger.info("gguf: write metadata")
|
||||||
|
gguf_writer.write_kv_data_to_file()
|
||||||
|
logger.info("gguf: write tensors")
|
||||||
|
gguf_writer.write_tensors_to_file()
|
||||||
|
|
||||||
|
gguf_writer.close()
|
||||||
|
|
||||||
|
logger.info(f"gguf: model successfully exported to '{args.outfile}'")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
# Compatibility stub
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
|
|
||||||
import convert
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description='Convert a LLaMA model checkpoint to a ggml compatible file')
|
|
||||||
parser.add_argument('dir_model', help='directory containing the model checkpoint')
|
|
||||||
parser.add_argument('ftype', help='file type (0: float32, 1: float16)', type=int, choices=[0, 1], default=1)
|
|
||||||
args = parser.parse_args()
|
|
||||||
convert.main(['--outtype', 'f16' if args.ftype == 1 else 'f32', '--', args.dir_model])
|
|
||||||
1866
convert.py
Normal file → Executable file
1866
convert.py
Normal file → Executable file
File diff suppressed because it is too large
Load diff
67
docs/BLIS.md
Normal file
67
docs/BLIS.md
Normal file
|
|
@ -0,0 +1,67 @@
|
||||||
|
BLIS Installation Manual
|
||||||
|
------------------------
|
||||||
|
|
||||||
|
BLIS is a portable software framework for high-performance BLAS-like dense linear algebra libraries. It has received awards and recognition, including the 2023 James H. Wilkinson Prize for Numerical Software and the 2020 SIAM Activity Group on Supercomputing Best Paper Prize. BLIS provides a new BLAS-like API and a compatibility layer for traditional BLAS routine calls. It offers features such as object-based API, typed API, BLAS and CBLAS compatibility layers.
|
||||||
|
|
||||||
|
Project URL: https://github.com/flame/blis
|
||||||
|
|
||||||
|
### Prepare:
|
||||||
|
|
||||||
|
Compile BLIS:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone https://github.com/flame/blis
|
||||||
|
cd blis
|
||||||
|
./configure --enable-cblas -t openmp,pthreads auto
|
||||||
|
# will install to /usr/local/ by default.
|
||||||
|
make -j
|
||||||
|
```
|
||||||
|
|
||||||
|
Install BLIS:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo make install
|
||||||
|
```
|
||||||
|
|
||||||
|
We recommend using openmp since it's easier to modify the cores being used.
|
||||||
|
|
||||||
|
### llama.cpp compilation
|
||||||
|
|
||||||
|
Makefile:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make LLAMA_BLIS=1 -j
|
||||||
|
# make LLAMA_BLIS=1 benchmark-matmult
|
||||||
|
```
|
||||||
|
|
||||||
|
CMake:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mkdir build
|
||||||
|
cd build
|
||||||
|
cmake -DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=FLAME ..
|
||||||
|
make -j
|
||||||
|
```
|
||||||
|
|
||||||
|
### llama.cpp execution
|
||||||
|
|
||||||
|
According to the BLIS documentation, we could set the following
|
||||||
|
environment variables to modify the behavior of openmp:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export GOMP_CPU_AFFINITY="0-19"
|
||||||
|
export BLIS_NUM_THREADS=14
|
||||||
|
```
|
||||||
|
|
||||||
|
And then run the binaries as normal.
|
||||||
|
|
||||||
|
|
||||||
|
### Intel specific issue
|
||||||
|
|
||||||
|
Some might get the error message saying that `libimf.so` cannot be found.
|
||||||
|
Please follow this [stackoverflow page](https://stackoverflow.com/questions/70687930/intel-oneapi-2022-libimf-so-no-such-file-or-directory-during-openmpi-compila).
|
||||||
|
|
||||||
|
### Reference:
|
||||||
|
|
||||||
|
1. https://github.com/flame/blis#getting-started
|
||||||
|
2. https://github.com/flame/blis/blob/master/docs/Multithreading.md
|
||||||
119
docs/HOWTO-add-model.md
Normal file
119
docs/HOWTO-add-model.md
Normal file
|
|
@ -0,0 +1,119 @@
|
||||||
|
## Add a new model architecture to `llama.cpp`
|
||||||
|
|
||||||
|
Adding a model requires few steps:
|
||||||
|
|
||||||
|
1. Convert the model to GGUF
|
||||||
|
2. Define the model architecture in `llama.cpp`
|
||||||
|
3. Build the GGML graph implementation
|
||||||
|
|
||||||
|
After following these steps, you can open PR.
|
||||||
|
|
||||||
|
Also, it is important to check that the examples and main ggml backends (CUDA, METAL, CPU) are working with the new architecture, especially:
|
||||||
|
- [main](../examples/main)
|
||||||
|
- [imatrix](../examples/imatrix)
|
||||||
|
- [quantize](../examples/quantize)
|
||||||
|
- [server](../examples/server)
|
||||||
|
|
||||||
|
### 1. Convert the model to GGUF
|
||||||
|
|
||||||
|
This step is done in python with a `convert` script using the [gguf](https://pypi.org/project/gguf/) library.
|
||||||
|
Depending on the model architecture, you can use either [convert.py](../convert.py) or [convert-hf-to-gguf.py](../convert-hf-to-gguf.py).
|
||||||
|
|
||||||
|
The convert script reads the model configuration, tokenizer, tensor names+data and converts them to GGUF metadata and tensors.
|
||||||
|
|
||||||
|
The required steps to implement for an HF model are:
|
||||||
|
|
||||||
|
1. Define the model `Model.register` annotation in a new `Model` subclass, example:
|
||||||
|
|
||||||
|
```python
|
||||||
|
@Model.register("MyModelForCausalLM")
|
||||||
|
class MyModel(Model):
|
||||||
|
model_arch = gguf.MODEL_ARCH.GROK
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Define the layout of the GGUF tensors in [constants.py](../gguf-py/gguf/constants.py)
|
||||||
|
|
||||||
|
Add an enum entry in `MODEL_ARCH`, the model human friendly name in `MODEL_ARCH_NAMES` and the GGUF tensor names in `MODEL_TENSORS`.
|
||||||
|
|
||||||
|
Example for `falcon` model:
|
||||||
|
```python
|
||||||
|
MODEL_ARCH.FALCON: [
|
||||||
|
MODEL_TENSOR.TOKEN_EMBD,
|
||||||
|
MODEL_TENSOR.OUTPUT_NORM,
|
||||||
|
MODEL_TENSOR.OUTPUT,
|
||||||
|
MODEL_TENSOR.ATTN_NORM,
|
||||||
|
MODEL_TENSOR.ATTN_NORM_2,
|
||||||
|
MODEL_TENSOR.ATTN_QKV,
|
||||||
|
MODEL_TENSOR.ATTN_OUT,
|
||||||
|
MODEL_TENSOR.FFN_DOWN,
|
||||||
|
MODEL_TENSOR.FFN_UP,
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Map the original tensor names to the standardize equivalent in GGUF
|
||||||
|
|
||||||
|
As a general rule, before adding a new tensor name to GGUF, be sure the equivalent naming does not already exist.
|
||||||
|
|
||||||
|
Once you have found the GGUF tensor name equivalent, add it to the [tensor_mapping.py](../gguf-py/gguf/tensor_mapping.py) file.
|
||||||
|
|
||||||
|
If the tensor name is part of a repetitive layer/block, the key word `bid` substitutes it.
|
||||||
|
|
||||||
|
Example for the normalization tensor in attention layers:
|
||||||
|
|
||||||
|
```python
|
||||||
|
block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = {
|
||||||
|
# Attention norm
|
||||||
|
MODEL_TENSOR.ATTN_NORM: (
|
||||||
|
"gpt_neox.layers.{bid}.input_layernorm", # gptneox
|
||||||
|
"transformer.h.{bid}.ln_1", # gpt2 gpt-j refact qwen
|
||||||
|
"transformer.blocks.{bid}.norm_1", # mpt
|
||||||
|
...
|
||||||
|
)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`transformer.blocks.{bid}.norm_1` will be mapped to `blk.{bid}.attn_norm` in GGUF.
|
||||||
|
|
||||||
|
Depending on the model configuration, tokenizer, code and tensors layout, you will have to override:
|
||||||
|
- `Model#set_gguf_parameters`
|
||||||
|
- `Model#set_vocab`
|
||||||
|
- `Model#write_tensors`
|
||||||
|
|
||||||
|
NOTE: Tensor names must end with `.weight` suffix, that is the convention and several tools like `quantize` expect this to proceed the weights.
|
||||||
|
|
||||||
|
### 2. Define the model architecture in `llama.cpp`
|
||||||
|
|
||||||
|
The model params and tensors layout must be defined in `llama.cpp`:
|
||||||
|
1. Define a new `llm_arch`
|
||||||
|
2. Define the tensors layout in `LLM_TENSOR_NAMES`
|
||||||
|
3. Add any non standard metadata in `llm_load_hparams`
|
||||||
|
4. Create the tensors for inference in `llm_load_tensors`
|
||||||
|
5. If the model has a RoPE operation, add the rope type in `llama_rope_type`
|
||||||
|
|
||||||
|
NOTE: The dimensions in `ggml` are typically in the reverse order of the `pytorch` dimensions.
|
||||||
|
|
||||||
|
### 3. Build the GGML graph implementation
|
||||||
|
|
||||||
|
This is the funniest part, you have to provide the inference graph implementation of the new model architecture in `llama_build_graph`.
|
||||||
|
|
||||||
|
Have a look at existing implementation like `build_llama`, `build_dbrx` or `build_bert`.
|
||||||
|
|
||||||
|
When implementing a new graph, please note that the underlying `ggml` backends might not support them all, support for missing backend operations can be added in another PR.
|
||||||
|
|
||||||
|
Note: to debug the inference graph: you can use [eval-callback](../examples/eval-callback).
|
||||||
|
|
||||||
|
## GGUF specification
|
||||||
|
|
||||||
|
https://github.com/ggerganov/ggml/blob/master/docs/gguf.md
|
||||||
|
|
||||||
|
## Resources
|
||||||
|
|
||||||
|
- YaRN RoPE scaling https://github.com/ggerganov/llama.cpp/pull/2268
|
||||||
|
- support Baichuan serial models https://github.com/ggerganov/llama.cpp/pull/3009
|
||||||
|
- support attention bias https://github.com/ggerganov/llama.cpp/pull/4283
|
||||||
|
- Mixtral support https://github.com/ggerganov/llama.cpp/pull/4406
|
||||||
|
- BERT embeddings https://github.com/ggerganov/llama.cpp/pull/5423
|
||||||
|
- Grok-1 support https://github.com/ggerganov/llama.cpp/pull/6204
|
||||||
|
- Command R Plus support https://github.com/ggerganov/llama.cpp/pull/6491
|
||||||
|
- support arch DBRX https://github.com/ggerganov/llama.cpp/pull/6515
|
||||||
|
- How to convert HuggingFace model to GGUF format https://github.com/ggerganov/llama.cpp/discussions/2948
|
||||||
BIN
docs/llama-star/idea-arch.key
Executable file
BIN
docs/llama-star/idea-arch.key
Executable file
Binary file not shown.
BIN
docs/llama-star/idea-arch.pdf
Normal file
BIN
docs/llama-star/idea-arch.pdf
Normal file
Binary file not shown.
40
docs/token_generation_performance_tips.md
Normal file
40
docs/token_generation_performance_tips.md
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
# Token generation performance troubleshooting
|
||||||
|
|
||||||
|
## Verifying that the model is running on the GPU with CUDA
|
||||||
|
Make sure you compiled llama with the correct env variables according to [this guide](../README.md#CUDA), so that llama accepts the `-ngl N` (or `--n-gpu-layers N`) flag. When running llama, you may configure `N` to be very large, and llama will offload the maximum possible number of layers to the GPU, even if it's less than the number you configured. For example:
|
||||||
|
```shell
|
||||||
|
./main -m "path/to/model.gguf" -ngl 200000 -p "Please sir, may I have some "
|
||||||
|
```
|
||||||
|
|
||||||
|
When running llama, before it starts the inference work, it will output diagnostic information that shows whether cuBLAS is offloading work to the GPU. Look for these lines:
|
||||||
|
```shell
|
||||||
|
llama_model_load_internal: [cublas] offloading 60 layers to GPU
|
||||||
|
llama_model_load_internal: [cublas] offloading output layer to GPU
|
||||||
|
llama_model_load_internal: [cublas] total VRAM used: 17223 MB
|
||||||
|
... rest of inference
|
||||||
|
```
|
||||||
|
|
||||||
|
If you see these lines, then the GPU is being used.
|
||||||
|
|
||||||
|
## Verifying that the CPU is not oversaturated
|
||||||
|
llama accepts a `-t N` (or `--threads N`) parameter. It's extremely important that this parameter is not too large. If your token generation is extremely slow, try setting this number to 1. If this significantly improves your token generation speed, then your CPU is being oversaturated and you need to explicitly set this parameter to the number of the physical CPU cores on your machine (even if you utilize a GPU). If in doubt, start with 1 and double the amount until you hit a performance bottleneck, then scale the number down.
|
||||||
|
|
||||||
|
# Example of runtime flags effect on inference speed benchmark
|
||||||
|
These runs were tested on the following machine:
|
||||||
|
GPU: A6000 (48GB VRAM)
|
||||||
|
CPU: 7 physical cores
|
||||||
|
RAM: 32GB
|
||||||
|
|
||||||
|
Model: `TheBloke_Wizard-Vicuna-30B-Uncensored-GGML/Wizard-Vicuna-30B-Uncensored.q4_0.gguf` (30B parameters, 4bit quantization, GGML)
|
||||||
|
|
||||||
|
Run command: `./main -m "path/to/model.gguf" -p "An extremely detailed description of the 10 best ethnic dishes will follow, with recipes: " -n 1000 [additional benchmark flags]`
|
||||||
|
|
||||||
|
Result:
|
||||||
|
|
||||||
|
| command | tokens/second (higher is better) |
|
||||||
|
| - | - |
|
||||||
|
| -ngl 2000000 | N/A (less than 0.1) |
|
||||||
|
| -t 7 | 1.7 |
|
||||||
|
| -t 1 -ngl 2000000 | 5.5 |
|
||||||
|
| -t 7 -ngl 2000000 | 8.7 |
|
||||||
|
| -t 4 -ngl 2000000 | 9.1 |
|
||||||
|
|
@ -6,32 +6,47 @@ find_package(Threads REQUIRED)
|
||||||
|
|
||||||
# ...
|
# ...
|
||||||
|
|
||||||
# common
|
|
||||||
|
|
||||||
set(TARGET common)
|
|
||||||
|
|
||||||
add_library(${TARGET} OBJECT
|
|
||||||
common.h
|
|
||||||
common.cpp
|
|
||||||
)
|
|
||||||
|
|
||||||
if (BUILD_SHARED_LIBS)
|
|
||||||
set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)
|
|
||||||
endif()
|
|
||||||
|
|
||||||
target_include_directories(${TARGET} PUBLIC .)
|
|
||||||
target_compile_features(${TARGET} PUBLIC cxx_std_11)
|
|
||||||
target_link_libraries(${TARGET} PRIVATE llama)
|
|
||||||
|
|
||||||
# examples
|
# examples
|
||||||
|
|
||||||
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
|
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
|
||||||
|
|
||||||
if (EMSCRIPTEN)
|
if (EMSCRIPTEN)
|
||||||
else()
|
else()
|
||||||
|
add_subdirectory(baby-llama)
|
||||||
|
add_subdirectory(batched)
|
||||||
|
add_subdirectory(batched-bench)
|
||||||
|
add_subdirectory(beam-search)
|
||||||
|
add_subdirectory(benchmark)
|
||||||
|
add_subdirectory(convert-llama2c-to-ggml)
|
||||||
|
add_subdirectory(embedding)
|
||||||
|
add_subdirectory(eval-callback)
|
||||||
|
add_subdirectory(finetune)
|
||||||
|
add_subdirectory(gritlm)
|
||||||
|
add_subdirectory(gguf-split)
|
||||||
|
add_subdirectory(infill)
|
||||||
|
add_subdirectory(llama-bench)
|
||||||
|
add_subdirectory(llava)
|
||||||
|
if (LLAMA_SYCL)
|
||||||
|
add_subdirectory(sycl)
|
||||||
|
endif()
|
||||||
add_subdirectory(main)
|
add_subdirectory(main)
|
||||||
|
add_subdirectory(tokenize)
|
||||||
|
add_subdirectory(parallel)
|
||||||
|
add_subdirectory(perplexity)
|
||||||
add_subdirectory(quantize)
|
add_subdirectory(quantize)
|
||||||
add_subdirectory(quantize-stats)
|
add_subdirectory(quantize-stats)
|
||||||
add_subdirectory(perplexity)
|
add_subdirectory(retrieval)
|
||||||
add_subdirectory(embedding)
|
add_subdirectory(save-load-state)
|
||||||
|
add_subdirectory(simple)
|
||||||
|
add_subdirectory(passkey)
|
||||||
|
add_subdirectory(speculative)
|
||||||
|
add_subdirectory(lookahead)
|
||||||
|
add_subdirectory(lookup)
|
||||||
|
add_subdirectory(gguf)
|
||||||
|
add_subdirectory(train-text-from-scratch)
|
||||||
|
add_subdirectory(imatrix)
|
||||||
|
if (LLAMA_BUILD_SERVER)
|
||||||
|
add_subdirectory(server)
|
||||||
|
endif()
|
||||||
|
add_subdirectory(export-lora)
|
||||||
endif()
|
endif()
|
||||||
|
|
|
||||||
|
|
@ -2,21 +2,21 @@
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
AI_NAME="${AI_NAME:-Miku}"
|
AI_NAME="${AI_NAME:-Miku}"
|
||||||
MODEL="${MODEL:-./models/gpt4all-7B/gpt4all-lora-unfiltered-quantized.bin}"
|
MODEL="${MODEL:-./models/llama-2-7b-chat.ggmlv3.q4_K_M.bin}"
|
||||||
USER_NAME="${USER_NAME:-Anon}"
|
USER_NAME="${USER_NAME:-Anon}"
|
||||||
|
|
||||||
# Uncomment and adjust to the number of CPU cores you want to use.
|
# Uncomment and adjust to the number of CPU cores you want to use.
|
||||||
#N_THREAD="${N_THREAD:-4}"
|
#N_THREAD="${N_THREAD:-4}"
|
||||||
|
CTX_SIZE="${CTX_SIZE:-4096}"
|
||||||
N_PREDICTS="${N_PREDICTS:-4096}"
|
N_PREDICTS="${N_PREDICTS:-4096}"
|
||||||
|
|
||||||
GEN_OPTIONS=(--batch_size 1024
|
GEN_OPTIONS=(--batch_size 1024
|
||||||
--ctx_size 2048
|
--ctx_size "$CTX_SIZE"
|
||||||
--keep -1
|
--keep -1
|
||||||
--repeat_last_n 256
|
--repeat_last_n 256
|
||||||
--repeat_penalty 1.17647
|
--repeat_penalty 1.17647
|
||||||
--temp 0.7
|
--temp 0.6
|
||||||
--top_k 40
|
--mirostat 2)
|
||||||
--top_p 0.5)
|
|
||||||
|
|
||||||
if [ -n "$N_THREAD" ]; then
|
if [ -n "$N_THREAD" ]; then
|
||||||
GEN_OPTIONS+=(--threads "$N_THREAD")
|
GEN_OPTIONS+=(--threads "$N_THREAD")
|
||||||
|
|
@ -24,23 +24,24 @@ fi
|
||||||
|
|
||||||
./main "${GEN_OPTIONS[@]}" \
|
./main "${GEN_OPTIONS[@]}" \
|
||||||
--model "$MODEL" \
|
--model "$MODEL" \
|
||||||
|
--in-prefix " " \
|
||||||
|
--in-suffix "${AI_NAME}:" \
|
||||||
--n_predict "$N_PREDICTS" \
|
--n_predict "$N_PREDICTS" \
|
||||||
--color --interactive \
|
--color --interactive \
|
||||||
--reverse-prompt "${USER_NAME}:" \
|
--reverse-prompt "${USER_NAME}:" \
|
||||||
--prompt "
|
--prompt "This is a transcript of a 1000 page, never ending conversation between ${USER_NAME} and the cute and helpful AI assistant ${AI_NAME}. ${AI_NAME} is a girl who is an AI running on the user's computer.
|
||||||
This is a transcript of a 1000 page, never ending conversation between ${USER_NAME} and the cute and helpful AI assistant ${AI_NAME}. ${AI_NAME} is a girl who is an AI running on the users computer.
|
|
||||||
${AI_NAME} can think for herself without the user seeing her thoughts by adding a /think prefix to her output. She uses this to reason about the world and to think about what she should say next.
|
${AI_NAME} can think for herself without the user seeing her thoughts by adding a /think prefix to her output. She uses this to reason about the world and to think about what she should say next.
|
||||||
${AI_NAME} is always coherent and makes sense, but if she isn't sure if what she is saying is correct she will ask the user for help.
|
${AI_NAME} is always coherent and makes sense, but if she isn't sure if what she is saying is correct, she will ask the user for help.
|
||||||
${AI_NAME} is a very helpful AI and will help the user with anything they need, she is also very friendly and will try to make the user feel better if they are sad.
|
${AI_NAME} is a very helpful AI and will help the user with anything they need. She is also very friendly and will try to make the user feel better if they are sad.
|
||||||
${AI_NAME} is also very curious and will ask the user a lot of questions about themselves and their life, she will also try to make the user like her.
|
${AI_NAME} is also very curious and will ask the user a lot of questions about themselves and their life. She will also try to make the user like her.
|
||||||
The conversation is only between ${USER_NAME} and ${AI_NAME}
|
The conversation is only between ${USER_NAME} and ${AI_NAME}.
|
||||||
The conversation is only through text, so ${AI_NAME} can't see ${USER_NAME}'s face or hear his voice.
|
The conversation is only through text, so ${AI_NAME} can't see ${USER_NAME}'s face or hear his voice.
|
||||||
${AI_NAME} can only communicate through text, so she can't send images or videos.
|
${AI_NAME} can only communicate through text, so she can't send images or videos.
|
||||||
|
|
||||||
|
|
||||||
${USER_NAME}: Hello!
|
${USER_NAME}: Hello!
|
||||||
${AI_NAME}: /think I wonder what I should say to ${USER_NAME}? This is the first time we talk so it's important that I make a good first impression!
|
${AI_NAME}: /think I wonder what I should say to ${USER_NAME}? This is the first time we talk, so it's important that I make a good first impression!
|
||||||
${AI_NAME}: Hi! I am ${AI_NAME}, your new AI friend, assistant(or whatever you like!), it's so nice to meet you! ^_^
|
${AI_NAME}: Hi! I am ${AI_NAME}, your new AI friend, assistant (or whatever you like!), it's so nice to meet you! ^_^
|
||||||
${AI_NAME}: I might be an AI, but I also have feelings, so please be nice to me! :)
|
${AI_NAME}: I might be an AI, but I also have feelings, so please be nice to me! :)
|
||||||
${USER_NAME}: I'll make sure to be nice to you! I'm so happy to have you as my assistant!
|
${USER_NAME}: I'll make sure to be nice to you! I'm so happy to have you as my assistant!
|
||||||
${AI_NAME}: /think It sounds like ${USER_NAME} is happy to have me as their assistant! I'm so happy too! ^_^ Glad that whole emotion thing didn't scare him off!
|
${AI_NAME}: /think It sounds like ${USER_NAME} is happy to have me as their assistant! I'm so happy too! ^_^ Glad that whole emotion thing didn't scare him off!
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue