diff --git a/scripts/check-requirements.sh b/scripts/check-requirements.sh index 947df25aa..af7bab753 100755 --- a/scripts/check-requirements.sh +++ b/scripts/check-requirements.sh @@ -1,12 +1,5 @@ #!/bin/bash -#### BEGIN SETUP ##### set -euo pipefail -this=$(realpath -- "$0"); readonly this -cd "$(dirname "$this")" -shellcheck --external-sources "$this" -# shellcheck source=lib.sh -source 'lib.sh' -#### END SETUP #### # # check-requirements.sh checks all requirements files for each top-level @@ -33,9 +26,27 @@ source 'lib.sh' # finally imports the python script to check for `ImportError`. # -Cleanup() { - if _IsSet workdir && [[ -d $workdir && -w $workdir ]]; then - _LogInfo "Removing $workdir" +log() { + local level=$1 msg=$2 + printf >&2 '%s: %s\n' "$level" "$msg" +} + +debug() { + log DEBUG "$@" +} + +info() { + log INFO "$@" +} + +fatal() { + log FATAL "$@" + exit 1 +} + +cleanup() { + if [[ -n ${workdir+x} && -d $workdir && -w $workdir ]]; then + info "Removing $workdir" local count=0 rm -rfv -- "$workdir" | while read -r; do if (( count++ > 750 )); then @@ -44,7 +55,7 @@ Cleanup() { fi done printf '\n' - _LogInfo "Removed $workdir" + info "Removed $workdir" fi } @@ -55,43 +66,46 @@ fi if (( do_cleanup )); then trap exit INT TERM - trap Cleanup EXIT + trap cleanup EXIT fi -cd .. # PWD should be llama.cpp project directory +this=$(realpath -- "$0"); readonly this +cd "$(dirname "$this")/.." # PWD should stay in llama.cpp project directory + +shellcheck "$this" readonly reqs_dir=requirements if [[ ${1+x} ]]; then tmp_dir=$(realpath -- "$1") if [[ ! ( -d $tmp_dir && -w $tmp_dir ) ]]; then - _LogFatal "$tmp_dir is not a writable directory" + fatal "$tmp_dir is not a writable directory" fi else tmp_dir=/tmp fi workdir=$(mktemp -d "$tmp_dir/check-requirements.XXXX"); readonly workdir -_LogInfo "Working directory: $workdir" +info "Working directory: $workdir" -CheckRequirements() { +check_requirements() { local reqs=$1 - _LogInfo "$reqs: beginning check" + info "$reqs: beginning check" pip --disable-pip-version-check install -qr "$reqs" - _LogInfo "$reqs: OK" + info "$reqs: OK" } -CheckConvertScript() { +check_convert_script() { local py=$1 # e.g. ./convert-hf-to-gguf.py local pyname=${py##*/} # e.g. convert-hf-to-gguf.py pyname=${pyname%.py} # e.g. convert-hf-to-gguf - _LogInfo "$py: beginning check" + info "$py: beginning check" local reqs="$reqs_dir/requirements-$pyname.txt" if [[ ! -r $reqs ]]; then - _LogFatal "$py missing requirements. Expected: $reqs" + fatal "$py missing requirements. Expected: $reqs" fi local venv="$workdir/$pyname-venv" @@ -101,7 +115,7 @@ CheckConvertScript() { # shellcheck source=/dev/null source "$venv/bin/activate" - CheckRequirements "$reqs" + check_requirements "$reqs" python - "$py" "$pyname" <<'EOF' import sys @@ -115,7 +129,7 @@ EOF rm -rf -- "$venv" fi - _LogInfo "$py: imports OK" + info "$py: imports OK" } readonly ignore_eq_eq='check_requirements: ignore "=="' @@ -123,7 +137,7 @@ readonly ignore_eq_eq='check_requirements: ignore "=="' for req in "$reqs_dir"/*; do # Check that all sub-requirements are added to top-level requirements.txt if ! grep -qF "$req" requirements.txt; then - _LogFatal "$req needs to be added to requirements.txt" + fatal "$req needs to be added to requirements.txt" fi # Make sure exact release versions aren't being pinned in the requirements @@ -145,16 +159,16 @@ python3 -m venv "$all_venv" ( # shellcheck source=/dev/null source "$all_venv/bin/activate" - CheckRequirements requirements.txt + check_requirements requirements.txt ) if (( do_cleanup )); then rm -rf -- "$all_venv" fi -CheckConvertScript convert.py +check_convert_script convert.py for py in convert-*.py; do - CheckConvertScript "$py" + check_convert_script "$py" done -_LogInfo 'Done! No issues found.' +info 'Done! No issues found.' diff --git a/scripts/ci-run.sh b/scripts/ci-run.sh deleted file mode 100755 index 934d9f77b..000000000 --- a/scripts/ci-run.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash -#### BEGIN SETUP ##### -set -euo pipefail -this=$(realpath -- "$0"); readonly this -cd "$(dirname "$this")" -shellcheck --external-sources "$this" -# shellcheck source=lib.sh -source 'lib.sh' -#### END SETUP #### - -# TODO: send model to ctest_model tests using env variable -# GG_CI_CTEST_MODELFILE= - -IsValidTestTarget() { - case "$1" in - cmake | ctest_main | model_3b | model_7b | test_cpu | test_cuda | test_metal | ctest_model) - return $_OK;; - *) - return $_ERR;; - esac -} - -declare -a targets -if (( $# > 0 )); then - targets=("$@") -elif _IsSet GG_CI_TARGETS; then - read -r -a targets <<< "$GG_CI_TARGETS" -else - cat >&2 <<'EOF' -usage: - ci-run.sh [targets...] - -config variables: - GG_CI_TARGETS : Space delimited sequence of targets. - Overridden by commandline arguments. - GG_CI_WORKDIR : Build files and results. - Defaults to /tmp. - GG_CI_DATADIR : Persistent model files and datasets, unchanged between runs. - Defaults to ~/.cache/llama.cpp/ci-data. - GG_CI_TEMPDIR : Scratch directory for quantized model files. - Defaults to ~/.cache/llama.cpp/ci-temp - -examples: - # A run on a low-spec VM without a dedicated GPU. - ci-run.sh cmake ctest_main model_3b test_cpu ctest_model - - # A run on a Mac Studio with a ramdisk at ~/tmp - GG_CI_WORKDIR=~/tmp/ci-work \ - GG_CI_TEMPDIR=~/tmp/ci-temp \ - ci-run.sh cmake ctest_main model_7b test_cpu test_metal ctest_model - -test targets: - cmake : Run cmake to produce debug and release builds. - ctest_main : Run main ctest tests for debug and release. - model_3b, - model_7b : Download and quantize openllama_3b_v2 and/or openllama_7b_v2. - test_cpu, - test_cuda, - test_metal : Test CPU inference, perplexity tests, etc. - ctest_model : Run ctest tests that require the openllama model. -EOF - exit $_ERR -fi - -for target in "${targets[@]}"; do - if IsValidTestTarget "$target"; then - _LogInfo "Received test target: $target" - else - _LogFatal "Invalid test target: $target" - fi -done - -cd .. -[[ -d .git && -x .git ]] || _LogFatal 'Could not cd to llama.cpp root direcory' - -TargetCmake() { - echo hello -} - -for target in "${targets[@]}"; do - pascal_target="$(_SnakeToPascalCase "$target")" - "Target$pascal_target" -done diff --git a/scripts/lib.sh b/scripts/lib.sh deleted file mode 100644 index d4d240736..000000000 --- a/scripts/lib.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -if [[ ${BASH_SOURCE[0]} -ef $0 ]]; then - echo >&2 "This script should be sourced, not executed!" - exit 1 -fi - -readonly _OK=0 -readonly _ERR=1 - -_Log() { - local level=$1 msg=$2 - printf >&2 '%s: %s\n' "$level" "$msg" -} - -_LogDebug() { - _Log DEBUG "$@" -} - -_LogInfo() { - _Log INFO "$@" -} - -_LogFatal() { - _Log FATAL "$@" - exit 1 -} - -# Return true if the variable with name $1 is set -_IsSet() { - (( $# != 1 )) && return $_ERR - if [[ -n ${!1+x} ]]; then - return $_OK - else - return $_ERR - fi -} - -_IsNotSet() { - ! _IsSet "$@" -} - -_SnakeToPascalCase() { - (( $# != 1 )) && return $_ERR - local IFS='_' - local pascal='' - for word in $1; do - local head; head=$(tr '[:lower:]' '[:upper:]' <<< "${word:0:1}") - local tail=${word:1} - pascal+="$head$tail" - done - echo -n "$pascal" -} diff --git a/scripts/lib_test.sh b/scripts/lib_test.sh deleted file mode 100755 index 7cb9693e7..000000000 --- a/scripts/lib_test.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -if [[ ! ${BASH_SOURCE[0]} -ef $0 ]]; then - echo >&2 "This script should be executed, not sourced!" - exit 1 -fi -#### BEGIN SETUP ##### -set -euo pipefail -this=$(realpath -- "$0"); readonly this -cd "$(dirname "$this")" -shellcheck --external-sources "$this" -# shellcheck source=lib.sh -source 'lib.sh' -#### END SETUP #### - -shellcheck 'lib.sh' - -Pass() { - local test_func="${FUNCNAME[1]}" - _Log 'PASSED' "$test_func" -} - -Fail() { - local test_func="${FUNCNAME[1]}" - _Log 'FAILED' "$test_func: $1" -} - -TestLibShExecution() { - if bash lib.sh 2>/dev/null; then - Fail 'lib.sh should fail execution, but did not' - else Pass; fi -}; TestLibShExecution - -TestIsSet() { - # shellcheck disable=SC2034 - local foo=1 - if ! _IsSet 'foo'; then - Fail 'foo was not detecting as set' - elif _IsSet 'bar'; then - Fail 'bar was detected as set' - else Pass; fi -}; TestIsSet - -TestIsNotSet() { - # shellcheck disable=SC2034 - local foo=1 - if _IsNotSet 'foo'; then - Fail 'foo was detected as not set' - elif ! _IsNotSet 'bar'; then - Fail 'bar was detected as set' - else Pass; fi -}; TestIsNotSet - -TestSnakeToPascalCase() { - local id; id=$(_SnakeToPascalCase this_is_an_id) - if [[ $id != ThisIsAnId ]]; then - Fail "this_is_an_id was converted to $id" - else Pass; fi -}; TestSnakeToPascalCase