Revert scripts work
This commit is contained in:
crasm
parent
66510528f7
commit
09f5d3c4ab
4 changed files with 42 additions and 222 deletions
|
|
@ -1,12 +1,5 @@
|
|||
#!/bin/bash
|
||||
#### BEGIN SETUP #####
|
||||
set -euo pipefail
|
||||
this=$(realpath -- "$0"); readonly this
|
||||
cd "$(dirname "$this")"
|
||||
shellcheck --external-sources "$this"
|
||||
# shellcheck source=lib.sh
|
||||
source 'lib.sh'
|
||||
#### END SETUP ####
|
||||
|
||||
#
|
||||
# check-requirements.sh checks all requirements files for each top-level
|
||||
|
|
@ -33,9 +26,27 @@ source 'lib.sh'
|
|||
# finally imports the python script to check for `ImportError`.
|
||||
#
|
||||
|
||||
Cleanup() {
|
||||
if _IsSet workdir && [[ -d $workdir && -w $workdir ]]; then
|
||||
_LogInfo "Removing $workdir"
|
||||
log() {
|
||||
local level=$1 msg=$2
|
||||
printf >&2 '%s: %s\n' "$level" "$msg"
|
||||
}
|
||||
|
||||
debug() {
|
||||
log DEBUG "$@"
|
||||
}
|
||||
|
||||
info() {
|
||||
log INFO "$@"
|
||||
}
|
||||
|
||||
fatal() {
|
||||
log FATAL "$@"
|
||||
exit 1
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [[ -n ${workdir+x} && -d $workdir && -w $workdir ]]; then
|
||||
info "Removing $workdir"
|
||||
local count=0
|
||||
rm -rfv -- "$workdir" | while read -r; do
|
||||
if (( count++ > 750 )); then
|
||||
|
|
@ -44,7 +55,7 @@ Cleanup() {
|
|||
fi
|
||||
done
|
||||
printf '\n'
|
||||
_LogInfo "Removed $workdir"
|
||||
info "Removed $workdir"
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -55,43 +66,46 @@ fi
|
|||
|
||||
if (( do_cleanup )); then
|
||||
trap exit INT TERM
|
||||
trap Cleanup EXIT
|
||||
trap cleanup EXIT
|
||||
fi
|
||||
|
||||
cd .. # PWD should be llama.cpp project directory
|
||||
this=$(realpath -- "$0"); readonly this
|
||||
cd "$(dirname "$this")/.." # PWD should stay in llama.cpp project directory
|
||||
|
||||
shellcheck "$this"
|
||||
|
||||
readonly reqs_dir=requirements
|
||||
|
||||
if [[ ${1+x} ]]; then
|
||||
tmp_dir=$(realpath -- "$1")
|
||||
if [[ ! ( -d $tmp_dir && -w $tmp_dir ) ]]; then
|
||||
_LogFatal "$tmp_dir is not a writable directory"
|
||||
fatal "$tmp_dir is not a writable directory"
|
||||
fi
|
||||
else
|
||||
tmp_dir=/tmp
|
||||
fi
|
||||
|
||||
workdir=$(mktemp -d "$tmp_dir/check-requirements.XXXX"); readonly workdir
|
||||
_LogInfo "Working directory: $workdir"
|
||||
info "Working directory: $workdir"
|
||||
|
||||
CheckRequirements() {
|
||||
check_requirements() {
|
||||
local reqs=$1
|
||||
|
||||
_LogInfo "$reqs: beginning check"
|
||||
info "$reqs: beginning check"
|
||||
pip --disable-pip-version-check install -qr "$reqs"
|
||||
_LogInfo "$reqs: OK"
|
||||
info "$reqs: OK"
|
||||
}
|
||||
|
||||
CheckConvertScript() {
|
||||
check_convert_script() {
|
||||
local py=$1 # e.g. ./convert-hf-to-gguf.py
|
||||
local pyname=${py##*/} # e.g. convert-hf-to-gguf.py
|
||||
pyname=${pyname%.py} # e.g. convert-hf-to-gguf
|
||||
|
||||
_LogInfo "$py: beginning check"
|
||||
info "$py: beginning check"
|
||||
|
||||
local reqs="$reqs_dir/requirements-$pyname.txt"
|
||||
if [[ ! -r $reqs ]]; then
|
||||
_LogFatal "$py missing requirements. Expected: $reqs"
|
||||
fatal "$py missing requirements. Expected: $reqs"
|
||||
fi
|
||||
|
||||
local venv="$workdir/$pyname-venv"
|
||||
|
|
@ -101,7 +115,7 @@ CheckConvertScript() {
|
|||
# shellcheck source=/dev/null
|
||||
source "$venv/bin/activate"
|
||||
|
||||
CheckRequirements "$reqs"
|
||||
check_requirements "$reqs"
|
||||
|
||||
python - "$py" "$pyname" <<'EOF'
|
||||
import sys
|
||||
|
|
@ -115,7 +129,7 @@ EOF
|
|||
rm -rf -- "$venv"
|
||||
fi
|
||||
|
||||
_LogInfo "$py: imports OK"
|
||||
info "$py: imports OK"
|
||||
}
|
||||
|
||||
readonly ignore_eq_eq='check_requirements: ignore "=="'
|
||||
|
|
@ -123,7 +137,7 @@ readonly ignore_eq_eq='check_requirements: ignore "=="'
|
|||
for req in "$reqs_dir"/*; do
|
||||
# Check that all sub-requirements are added to top-level requirements.txt
|
||||
if ! grep -qF "$req" requirements.txt; then
|
||||
_LogFatal "$req needs to be added to requirements.txt"
|
||||
fatal "$req needs to be added to requirements.txt"
|
||||
fi
|
||||
|
||||
# Make sure exact release versions aren't being pinned in the requirements
|
||||
|
|
@ -145,16 +159,16 @@ python3 -m venv "$all_venv"
|
|||
(
|
||||
# shellcheck source=/dev/null
|
||||
source "$all_venv/bin/activate"
|
||||
CheckRequirements requirements.txt
|
||||
check_requirements requirements.txt
|
||||
)
|
||||
|
||||
if (( do_cleanup )); then
|
||||
rm -rf -- "$all_venv"
|
||||
fi
|
||||
|
||||
CheckConvertScript convert.py
|
||||
check_convert_script convert.py
|
||||
for py in convert-*.py; do
|
||||
CheckConvertScript "$py"
|
||||
check_convert_script "$py"
|
||||
done
|
||||
|
||||
_LogInfo 'Done! No issues found.'
|
||||
info 'Done! No issues found.'
|
||||
|
|
|
|||
|
|
@ -1,83 +0,0 @@
|
|||
#!/bin/bash
|
||||
#### BEGIN SETUP #####
|
||||
set -euo pipefail
|
||||
this=$(realpath -- "$0"); readonly this
|
||||
cd "$(dirname "$this")"
|
||||
shellcheck --external-sources "$this"
|
||||
# shellcheck source=lib.sh
|
||||
source 'lib.sh'
|
||||
#### END SETUP ####
|
||||
|
||||
# TODO: send model to ctest_model tests using env variable
|
||||
# GG_CI_CTEST_MODELFILE=<snip>
|
||||
|
||||
IsValidTestTarget() {
|
||||
case "$1" in
|
||||
cmake | ctest_main | model_3b | model_7b | test_cpu | test_cuda | test_metal | ctest_model)
|
||||
return $_OK;;
|
||||
*)
|
||||
return $_ERR;;
|
||||
esac
|
||||
}
|
||||
|
||||
declare -a targets
|
||||
if (( $# > 0 )); then
|
||||
targets=("$@")
|
||||
elif _IsSet GG_CI_TARGETS; then
|
||||
read -r -a targets <<< "$GG_CI_TARGETS"
|
||||
else
|
||||
cat >&2 <<'EOF'
|
||||
usage:
|
||||
ci-run.sh [targets...]
|
||||
|
||||
config variables:
|
||||
GG_CI_TARGETS : Space delimited sequence of targets.
|
||||
Overridden by commandline arguments.
|
||||
GG_CI_WORKDIR : Build files and results.
|
||||
Defaults to /tmp.
|
||||
GG_CI_DATADIR : Persistent model files and datasets, unchanged between runs.
|
||||
Defaults to ~/.cache/llama.cpp/ci-data.
|
||||
GG_CI_TEMPDIR : Scratch directory for quantized model files.
|
||||
Defaults to ~/.cache/llama.cpp/ci-temp
|
||||
|
||||
examples:
|
||||
# A run on a low-spec VM without a dedicated GPU.
|
||||
ci-run.sh cmake ctest_main model_3b test_cpu ctest_model
|
||||
|
||||
# A run on a Mac Studio with a ramdisk at ~/tmp
|
||||
GG_CI_WORKDIR=~/tmp/ci-work \
|
||||
GG_CI_TEMPDIR=~/tmp/ci-temp \
|
||||
ci-run.sh cmake ctest_main model_7b test_cpu test_metal ctest_model
|
||||
|
||||
test targets:
|
||||
cmake : Run cmake to produce debug and release builds.
|
||||
ctest_main : Run main ctest tests for debug and release.
|
||||
model_3b,
|
||||
model_7b : Download and quantize openllama_3b_v2 and/or openllama_7b_v2.
|
||||
test_cpu,
|
||||
test_cuda,
|
||||
test_metal : Test CPU inference, perplexity tests, etc.
|
||||
ctest_model : Run ctest tests that require the openllama model.
|
||||
EOF
|
||||
exit $_ERR
|
||||
fi
|
||||
|
||||
for target in "${targets[@]}"; do
|
||||
if IsValidTestTarget "$target"; then
|
||||
_LogInfo "Received test target: $target"
|
||||
else
|
||||
_LogFatal "Invalid test target: $target"
|
||||
fi
|
||||
done
|
||||
|
||||
cd ..
|
||||
[[ -d .git && -x .git ]] || _LogFatal 'Could not cd to llama.cpp root direcory'
|
||||
|
||||
TargetCmake() {
|
||||
echo hello
|
||||
}
|
||||
|
||||
for target in "${targets[@]}"; do
|
||||
pascal_target="$(_SnakeToPascalCase "$target")"
|
||||
"Target$pascal_target"
|
||||
done
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [[ ${BASH_SOURCE[0]} -ef $0 ]]; then
|
||||
echo >&2 "This script should be sourced, not executed!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
readonly _OK=0
|
||||
readonly _ERR=1
|
||||
|
||||
_Log() {
|
||||
local level=$1 msg=$2
|
||||
printf >&2 '%s: %s\n' "$level" "$msg"
|
||||
}
|
||||
|
||||
_LogDebug() {
|
||||
_Log DEBUG "$@"
|
||||
}
|
||||
|
||||
_LogInfo() {
|
||||
_Log INFO "$@"
|
||||
}
|
||||
|
||||
_LogFatal() {
|
||||
_Log FATAL "$@"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Return true if the variable with name $1 is set
|
||||
_IsSet() {
|
||||
(( $# != 1 )) && return $_ERR
|
||||
if [[ -n ${!1+x} ]]; then
|
||||
return $_OK
|
||||
else
|
||||
return $_ERR
|
||||
fi
|
||||
}
|
||||
|
||||
_IsNotSet() {
|
||||
! _IsSet "$@"
|
||||
}
|
||||
|
||||
_SnakeToPascalCase() {
|
||||
(( $# != 1 )) && return $_ERR
|
||||
local IFS='_'
|
||||
local pascal=''
|
||||
for word in $1; do
|
||||
local head; head=$(tr '[:lower:]' '[:upper:]' <<< "${word:0:1}")
|
||||
local tail=${word:1}
|
||||
pascal+="$head$tail"
|
||||
done
|
||||
echo -n "$pascal"
|
||||
}
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
#!/bin/bash
|
||||
if [[ ! ${BASH_SOURCE[0]} -ef $0 ]]; then
|
||||
echo >&2 "This script should be executed, not sourced!"
|
||||
exit 1
|
||||
fi
|
||||
#### BEGIN SETUP #####
|
||||
set -euo pipefail
|
||||
this=$(realpath -- "$0"); readonly this
|
||||
cd "$(dirname "$this")"
|
||||
shellcheck --external-sources "$this"
|
||||
# shellcheck source=lib.sh
|
||||
source 'lib.sh'
|
||||
#### END SETUP ####
|
||||
|
||||
shellcheck 'lib.sh'
|
||||
|
||||
Pass() {
|
||||
local test_func="${FUNCNAME[1]}"
|
||||
_Log 'PASSED' "$test_func"
|
||||
}
|
||||
|
||||
Fail() {
|
||||
local test_func="${FUNCNAME[1]}"
|
||||
_Log 'FAILED' "$test_func: $1"
|
||||
}
|
||||
|
||||
TestLibShExecution() {
|
||||
if bash lib.sh 2>/dev/null; then
|
||||
Fail 'lib.sh should fail execution, but did not'
|
||||
else Pass; fi
|
||||
}; TestLibShExecution
|
||||
|
||||
TestIsSet() {
|
||||
# shellcheck disable=SC2034
|
||||
local foo=1
|
||||
if ! _IsSet 'foo'; then
|
||||
Fail 'foo was not detecting as set'
|
||||
elif _IsSet 'bar'; then
|
||||
Fail 'bar was detected as set'
|
||||
else Pass; fi
|
||||
}; TestIsSet
|
||||
|
||||
TestIsNotSet() {
|
||||
# shellcheck disable=SC2034
|
||||
local foo=1
|
||||
if _IsNotSet 'foo'; then
|
||||
Fail 'foo was detected as not set'
|
||||
elif ! _IsNotSet 'bar'; then
|
||||
Fail 'bar was detected as set'
|
||||
else Pass; fi
|
||||
}; TestIsNotSet
|
||||
|
||||
TestSnakeToPascalCase() {
|
||||
local id; id=$(_SnakeToPascalCase this_is_an_id)
|
||||
if [[ $id != ThisIsAnId ]]; then
|
||||
Fail "this_is_an_id was converted to $id"
|
||||
else Pass; fi
|
||||
}; TestSnakeToPascalCase
|
||||
Loading…
Add table
Add a link
Reference in a new issue