Revert scripts work

scripts/check-requirements.sh

@@ -1,12 +1,5 @@
 #!/bin/bash
-#### BEGIN SETUP #####
 set -euo pipefail
-this=$(realpath -- "$0"); readonly this
-cd "$(dirname "$this")"
-shellcheck --external-sources "$this"
-# shellcheck source=lib.sh
-source 'lib.sh'
-#### END SETUP ####
 
 #
 # check-requirements.sh checks all requirements files for each top-level
@@ -33,9 +26,27 @@ source 'lib.sh'
 # finally imports the python script to check for `ImportError`.
 #
 
-Cleanup() {
+log() {
-    if _IsSet workdir && [[ -d $workdir && -w $workdir ]]; then
+    local level=$1 msg=$2
-        _LogInfo "Removing $workdir"
+    printf >&2 '%s: %s\n' "$level" "$msg"
+}
+
+debug() {
+    log DEBUG "$@"
+}
+
+info() {
+    log INFO "$@"
+}
+
+fatal() {
+    log FATAL "$@"
+    exit 1
+}
+
+cleanup() {
+    if [[ -n ${workdir+x} && -d $workdir && -w $workdir ]]; then
+        info "Removing $workdir"
         local count=0
         rm -rfv -- "$workdir" | while read -r; do
             if (( count++ > 750 )); then
@@ -44,7 +55,7 @@ Cleanup() {
             fi
         done
         printf '\n'
-        _LogInfo "Removed $workdir"
+        info "Removed $workdir"
     fi
 }
 
@@ -55,43 +66,46 @@ fi
 
 if (( do_cleanup )); then
     trap exit INT TERM
-    trap Cleanup EXIT
+    trap cleanup EXIT
 fi
 
-cd .. # PWD should be llama.cpp project directory
+this=$(realpath -- "$0"); readonly this
+cd "$(dirname "$this")/.." # PWD should stay in llama.cpp project directory
+
+shellcheck "$this"
 
 readonly reqs_dir=requirements
 
 if [[ ${1+x} ]]; then
     tmp_dir=$(realpath -- "$1")
     if [[ ! ( -d $tmp_dir && -w $tmp_dir ) ]]; then
-        _LogFatal "$tmp_dir is not a writable directory"
+        fatal "$tmp_dir is not a writable directory"
     fi
 else
     tmp_dir=/tmp
 fi
 
 workdir=$(mktemp -d "$tmp_dir/check-requirements.XXXX"); readonly workdir
-_LogInfo "Working directory: $workdir"
+info "Working directory: $workdir"
 
-CheckRequirements() {
+check_requirements() {
     local reqs=$1
 
-    _LogInfo "$reqs: beginning check"
+    info "$reqs: beginning check"
     pip --disable-pip-version-check install -qr "$reqs"
-    _LogInfo "$reqs: OK"
+    info "$reqs: OK"
 }
 
-CheckConvertScript() {
+check_convert_script() {
     local py=$1             # e.g. ./convert-hf-to-gguf.py
     local pyname=${py##*/}  # e.g. convert-hf-to-gguf.py
     pyname=${pyname%.py}    # e.g. convert-hf-to-gguf
 
-    _LogInfo "$py: beginning check"
+    info "$py: beginning check"
 
     local reqs="$reqs_dir/requirements-$pyname.txt"
     if [[ ! -r $reqs ]]; then
-        _LogFatal "$py missing requirements. Expected: $reqs"
+        fatal "$py missing requirements. Expected: $reqs"
     fi
 
     local venv="$workdir/$pyname-venv"
@@ -101,7 +115,7 @@ CheckConvertScript() {
         # shellcheck source=/dev/null
         source "$venv/bin/activate"
 
-        CheckRequirements "$reqs"
+        check_requirements "$reqs"
 
         python - "$py" "$pyname" <<'EOF'
 import sys
@@ -115,7 +129,7 @@ EOF
         rm -rf -- "$venv"
     fi
 
-    _LogInfo "$py: imports OK"
+    info "$py: imports OK"
 }
 
 readonly ignore_eq_eq='check_requirements: ignore "=="'
@@ -123,7 +137,7 @@ readonly ignore_eq_eq='check_requirements: ignore "=="'
 for req in "$reqs_dir"/*; do
     # Check that all sub-requirements are added to top-level requirements.txt
     if ! grep -qF "$req" requirements.txt; then
-        _LogFatal "$req needs to be added to requirements.txt"
+        fatal "$req needs to be added to requirements.txt"
     fi
 
     # Make sure exact release versions aren't being pinned in the requirements
@@ -145,16 +159,16 @@ python3 -m venv "$all_venv"
 (
     # shellcheck source=/dev/null
     source "$all_venv/bin/activate"
-    CheckRequirements requirements.txt
+    check_requirements requirements.txt
 )
 
 if (( do_cleanup )); then
     rm -rf -- "$all_venv"
 fi
 
-CheckConvertScript convert.py
+check_convert_script convert.py
 for py in convert-*.py; do
-    CheckConvertScript "$py"
+    check_convert_script "$py"
 done
 
-_LogInfo 'Done! No issues found.'
+info 'Done! No issues found.'

scripts/ci-run.sh

@@ -1,83 +0,0 @@
-#!/bin/bash
-#### BEGIN SETUP #####
-set -euo pipefail
-this=$(realpath -- "$0"); readonly this
-cd "$(dirname "$this")"
-shellcheck --external-sources "$this"
-# shellcheck source=lib.sh
-source 'lib.sh'
-#### END SETUP ####
-
-# TODO: send model to ctest_model tests using env variable
-# GG_CI_CTEST_MODELFILE=<snip>
-
-IsValidTestTarget() {
-    case "$1" in
-        cmake | ctest_main | model_3b | model_7b | test_cpu | test_cuda | test_metal | ctest_model)
-            return $_OK;;
-        *)
-            return $_ERR;;
-    esac
-}
-
-declare -a targets
-if (( $# > 0 )); then
-    targets=("$@")
-elif _IsSet GG_CI_TARGETS; then
-    read -r -a targets <<< "$GG_CI_TARGETS"
-else
-    cat >&2 <<'EOF'
-usage:
-    ci-run.sh [targets...]
-
-config variables:
-    GG_CI_TARGETS : Space delimited sequence of targets.
-                      Overridden by commandline arguments.
-    GG_CI_WORKDIR : Build files and results.
-                      Defaults to /tmp.
-    GG_CI_DATADIR : Persistent model files and datasets, unchanged between runs.
-                      Defaults to ~/.cache/llama.cpp/ci-data.
-    GG_CI_TEMPDIR : Scratch directory for quantized model files.
-                      Defaults to ~/.cache/llama.cpp/ci-temp
-
-examples:
-    # A run on a low-spec VM without a dedicated GPU.
-    ci-run.sh cmake ctest_main model_3b test_cpu ctest_model
-
-    # A run on a Mac Studio with a ramdisk at ~/tmp
-    GG_CI_WORKDIR=~/tmp/ci-work \
-        GG_CI_TEMPDIR=~/tmp/ci-temp \
-        ci-run.sh cmake ctest_main model_7b test_cpu test_metal ctest_model
-
-test targets:
-    cmake           : Run cmake to produce debug and release builds.
-    ctest_main      : Run main ctest tests for debug and release.
-    model_3b,
-        model_7b    : Download and quantize openllama_3b_v2 and/or openllama_7b_v2.
-    test_cpu,
-        test_cuda,
-        test_metal  : Test CPU inference, perplexity tests, etc.
-    ctest_model     : Run ctest tests that require the openllama model.
-EOF
-    exit $_ERR
-fi
-
-for target in "${targets[@]}"; do
-    if IsValidTestTarget "$target"; then
-        _LogInfo "Received test target: $target"
-    else
-        _LogFatal "Invalid test target: $target"
-    fi
-done
-
-cd ..
-[[ -d .git && -x .git ]] || _LogFatal 'Could not cd to llama.cpp root direcory'
-
-TargetCmake() {
-    echo hello
-}
-
-for target in "${targets[@]}"; do
-    pascal_target="$(_SnakeToPascalCase "$target")"
-    "Target$pascal_target"
-done

scripts/lib.sh

@@ -1,53 +0,0 @@
-#!/bin/bash
-
-if [[ ${BASH_SOURCE[0]} -ef $0 ]]; then
-   echo >&2 "This script should be sourced, not executed!"
-   exit 1
-fi
-
-readonly _OK=0
-readonly _ERR=1
-
-_Log() {
-    local level=$1 msg=$2
-    printf >&2 '%s: %s\n' "$level" "$msg"
-}
-
-_LogDebug() {
-    _Log DEBUG "$@"
-}
-
-_LogInfo() {
-    _Log INFO "$@"
-}
-
-_LogFatal() {
-    _Log FATAL "$@"
-    exit 1
-}
-
-# Return true if the variable with name $1 is set
-_IsSet() {
-    (( $# != 1 )) && return $_ERR
-    if [[ -n ${!1+x} ]]; then
-        return $_OK
-    else
-        return $_ERR
-    fi
-}
-
-_IsNotSet() {
-    ! _IsSet "$@"
-}
-
-_SnakeToPascalCase() {
-    (( $# != 1 )) && return $_ERR
-    local IFS='_'
-    local pascal=''
-    for word in $1; do
-        local head; head=$(tr '[:lower:]' '[:upper:]' <<< "${word:0:1}")
-        local tail=${word:1}
-        pascal+="$head$tail"
-    done
-    echo -n "$pascal"
-}

scripts/lib_test.sh

@@ -1,58 +0,0 @@
-#!/bin/bash
-if [[ ! ${BASH_SOURCE[0]} -ef $0 ]]; then
-   echo >&2 "This script should be executed, not sourced!"
-   exit 1
-fi
-#### BEGIN SETUP #####
-set -euo pipefail
-this=$(realpath -- "$0"); readonly this
-cd "$(dirname "$this")"
-shellcheck --external-sources "$this"
-# shellcheck source=lib.sh
-source 'lib.sh'
-#### END SETUP ####
-
-shellcheck 'lib.sh'
-
-Pass() {
-    local test_func="${FUNCNAME[1]}"
-    _Log 'PASSED' "$test_func"
-}
-
-Fail() {
-    local test_func="${FUNCNAME[1]}"
-    _Log 'FAILED' "$test_func: $1"
-}
-
-TestLibShExecution() {
-    if bash lib.sh 2>/dev/null; then
-        Fail 'lib.sh should fail execution, but did not'
-    else Pass; fi
-}; TestLibShExecution
-
-TestIsSet() {
-    # shellcheck disable=SC2034
-    local foo=1
-    if ! _IsSet 'foo'; then
-        Fail 'foo was not detecting as set'
-    elif _IsSet 'bar'; then
-        Fail 'bar was detected as set'
-    else Pass; fi
-}; TestIsSet
-
-TestIsNotSet() {
-    # shellcheck disable=SC2034
-    local foo=1
-    if _IsNotSet 'foo'; then
-        Fail 'foo was detected as not set'
-    elif ! _IsNotSet 'bar'; then
-        Fail 'bar was detected as set'
-    else Pass; fi
-}; TestIsNotSet
-
-TestSnakeToPascalCase() {
-    local id; id=$(_SnakeToPascalCase this_is_an_id)
-    if [[ $id != ThisIsAnId ]]; then
-        Fail "this_is_an_id was converted to $id"
-    else Pass; fi
-}; TestSnakeToPascalCase