debug-test.sh: combined execute and gdb test mode via -g flag
This commit is contained in:
brian khuu
parent
d75ceb168b
commit
f274de5386
3 changed files with 101 additions and 200 deletions
|
|
@ -1,41 +1,6 @@
|
||||||
# Debugging Tests Tips
|
# Debugging Tests Tips
|
||||||
|
|
||||||
## How to run & execute a specific test without anything else to keep the feedback loop short?
|
## How to run & execute or debug a specific test without anything else to keep the feedback loop short?
|
||||||
|
|
||||||
There is a script called run-single-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number.
|
|
||||||
|
|
||||||
For example, running the following command will output an interactive list from which you can select a test. It takes this form:
|
|
||||||
|
|
||||||
`run-single-test.sh [OPTION]... <test_regex> <test_number>`
|
|
||||||
|
|
||||||
It will then build & run in the debugger for you.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./scripts/run-single-test.sh test-tokenizer
|
|
||||||
```
|
|
||||||
|
|
||||||
An example of a single test output is shown below. You will get either a green TEST PASS or a red TEST FAIL if a particular test is working or not. This shorter feedback loop will hopefully make it easier for you to figure out the problem you are trying to solve.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
$ ./scripts/run-single-test.sh test 24
|
|
||||||
~/gitextern/llama.cpp ~/gitextern/llama.cpp
|
|
||||||
|
|
||||||
... prepping cmake environment ...
|
|
||||||
... building test binaries ...
|
|
||||||
... running test ...
|
|
||||||
|
|
||||||
Ran Test #24: test-eval-callback
|
|
||||||
Command: /home/mofosyne/gitextern/llama.cpp/build-ci-debug/bin/eval-callback "--hf-repo" "ggml-org/models" "--hf-file" "tinyllamas/stories260K.gguf" "--model" "stories260K.gguf" "--prompt" "hello" "--seed" "42" "-ngl" "0"
|
|
||||||
TEST PASS
|
|
||||||
```
|
|
||||||
|
|
||||||
For further reference use `run-single-test.sh -h` to print help.
|
|
||||||
|
|
||||||
### How does the script work?
|
|
||||||
|
|
||||||
This is similar to `debug-test.sh` so you can follow the similar guide in this page for similar process. Just run the command directly rather than though gdb.
|
|
||||||
|
|
||||||
## How to run & debug a specific test without anything else to keep the feedback loop short?
|
|
||||||
|
|
||||||
There is a script called debug-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number.
|
There is a script called debug-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number.
|
||||||
|
|
||||||
|
|
@ -45,13 +10,27 @@ For example, running the following command will output an interactive list from
|
||||||
|
|
||||||
It will then build & run in the debugger for you.
|
It will then build & run in the debugger for you.
|
||||||
|
|
||||||
|
To just execute a test and get back a PASS or FAIL message run:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./scripts/debug-test.sh test-tokenizer
|
./scripts/debug-test.sh test-tokenizer
|
||||||
|
```
|
||||||
|
|
||||||
|
To test in GDB use the `-g` flag to enable gdb test mode.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/debug-test.sh -g test-tokenizer
|
||||||
|
|
||||||
# Once in the debugger, i.e. at the chevrons prompt, setting a breakpoint could be as follows:
|
# Once in the debugger, i.e. at the chevrons prompt, setting a breakpoint could be as follows:
|
||||||
>>> b main
|
>>> b main
|
||||||
```
|
```
|
||||||
|
|
||||||
|
To speed up the testing loop, if you know your test number you can just run it similar to below:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/debug-test.sh test 23
|
||||||
|
```
|
||||||
|
|
||||||
For further reference use `debug-test.sh -h` to print help.
|
For further reference use `debug-test.sh -h` to print help.
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,24 +5,62 @@ test_number=${2:-}
|
||||||
PROG=${0##*/}
|
PROG=${0##*/}
|
||||||
build_dir="build-ci-debug"
|
build_dir="build-ci-debug"
|
||||||
|
|
||||||
|
# Print Color Commands
|
||||||
|
red=$(tput setaf 1)
|
||||||
|
green=$(tput setaf 2)
|
||||||
|
yellow=$(tput setaf 3)
|
||||||
|
blue=$(tput setaf 4)
|
||||||
|
normal=$(tput sgr0)
|
||||||
|
|
||||||
|
print_full_help() {
|
||||||
|
cat << EOF
|
||||||
|
Usage: $PROG [OPTION]... <test_regex> (test_number)
|
||||||
|
Debug specific ctest program.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
-h, --help display this help and exit
|
||||||
|
-g run in gdb mode
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
<test_regex> (Mandatory) Supply one regex to the script to filter tests
|
||||||
|
(test_number) (Optional) Test number to run a specific test
|
||||||
|
|
||||||
|
Example:
|
||||||
|
$PROG test-tokenizer
|
||||||
|
$PROG test-tokenizer 3
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
abort() {
|
||||||
|
echo "Error: $1" >&2
|
||||||
|
cat << EOF >&2
|
||||||
|
Usage: $PROG [OPTION]... <test_regex> (test_number)
|
||||||
|
Debug specific ctest program.
|
||||||
|
Refer to --help for full instructions.
|
||||||
|
EOF
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then
|
if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then
|
||||||
echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
|
print_full_help >&2
|
||||||
echo "Debug specific ctest program."
|
exit 0
|
||||||
echo
|
|
||||||
echo "Options:"
|
|
||||||
echo " -h, --help Display this help and exit"
|
|
||||||
echo
|
|
||||||
echo "Arguments:"
|
|
||||||
echo " <test_regex> (Mandatory) Supply one regex to the script to filter tests"
|
|
||||||
echo " (test_number) (Optional) Test number to run a specific test"
|
|
||||||
echo
|
|
||||||
echo "Example:"
|
|
||||||
echo " $PROG test-tokenizer"
|
|
||||||
echo " $PROG test-tokenizer 3"
|
|
||||||
echo
|
|
||||||
exit 0
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Parse command-line options
|
||||||
|
gdb_mode=false
|
||||||
|
while getopts "hg" opt; do
|
||||||
|
case $opt in
|
||||||
|
h)
|
||||||
|
print_full_help >&2
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
g)
|
||||||
|
gdb_mode=true
|
||||||
|
echo "gdb_mode Mode Enabled"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
# Function to select and debug a test
|
# Function to select and debug a test
|
||||||
function select_test() {
|
function select_test() {
|
||||||
test_suite=${1:-test}
|
test_suite=${1:-test}
|
||||||
|
|
@ -65,31 +103,43 @@ function select_test() {
|
||||||
IFS=$'\n'
|
IFS=$'\n'
|
||||||
|
|
||||||
# Get test args
|
# Get test args
|
||||||
gdb_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' ))
|
test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' ))
|
||||||
IFS=$sIFS
|
IFS=$sIFS
|
||||||
printf "Debug arguments: ${gdb_args[test_number]}\n\n"
|
|
||||||
|
|
||||||
# Expand paths if needed
|
printf "${blue}Running Test #${test_number}: ${tests[test_number]}${normal}\n"
|
||||||
args=()
|
printf "${blue}test_args[test_number]: ${test_args[test_number]}${normal}\n"
|
||||||
for x in $(echo ${gdb_args[test_number]} | sed -e 's/"\/\<//' -e 's/\>"//')
|
|
||||||
do
|
|
||||||
args+=($(echo $x | sed -e 's/.*\/..\//..\//'))
|
|
||||||
done
|
|
||||||
|
|
||||||
# Print Command
|
if [ "$gdb_mode" = "true" ]; then
|
||||||
red=$(tput setaf 1)
|
# Expand paths if needed
|
||||||
green=$(tput setaf 2)
|
gdb_args=()
|
||||||
yellow=$(tput setaf 3)
|
for x in $(echo ${test_args[test_number]} | sed -e 's/"\/\<//' -e 's/\>"//')
|
||||||
blue=$(tput setaf 4)
|
do
|
||||||
normal=$(tput sgr0)
|
gdb_args+=($(echo $x | sed -e 's/.*\/..\//..\//'))
|
||||||
printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n"
|
done
|
||||||
printf "${yellow}GDB args: ${gdb_args[test_number]}${normal}\n"
|
|
||||||
printf "${yellow}GDB args @: ${args[@]}${normal}\n"
|
|
||||||
|
|
||||||
# Execute debugger
|
printf "${blue}gdb_args @: ${gdb_args[@]}${normal}\n"
|
||||||
pushd "$repo_root" || exit 1
|
|
||||||
gdb --args ${args[@]}
|
# Execute debugger
|
||||||
popd > /dev/null || exit 1
|
pushd "$repo_root" || exit 1
|
||||||
|
gdb --args ${gdb_args[@]}
|
||||||
|
popd > /dev/null || exit 1
|
||||||
|
else
|
||||||
|
|
||||||
|
# Execute Test
|
||||||
|
pushd "$repo_root" || exit 1
|
||||||
|
eval "${test_args[test_number]}"
|
||||||
|
exit_code=$?
|
||||||
|
popd > /dev/null || exit 1
|
||||||
|
|
||||||
|
# Print Result
|
||||||
|
printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n"
|
||||||
|
printf "${yellow}Command: ${test_args[test_number]}${normal}\n"
|
||||||
|
if [ $exit_code -eq 0 ]; then
|
||||||
|
printf "${green}TEST PASS${normal}\n"
|
||||||
|
else
|
||||||
|
printf "${red}TEST FAIL${normal}\n"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# Step 0: Check the args
|
# Step 0: Check the args
|
||||||
|
|
|
||||||
|
|
@ -1,128 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
test_suite=${1:-}
|
|
||||||
test_number=${2:-}
|
|
||||||
|
|
||||||
PROG=${0##*/}
|
|
||||||
build_dir="build-ci-debug"
|
|
||||||
|
|
||||||
if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then
|
|
||||||
echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
|
|
||||||
echo "Run a specific ctest program."
|
|
||||||
echo
|
|
||||||
echo "Options:"
|
|
||||||
echo " -h, --help Display this help and exit"
|
|
||||||
echo
|
|
||||||
echo "Arguments:"
|
|
||||||
echo " <test_regex> (Mandatory) Supply one regex to the script to filter tests"
|
|
||||||
echo " (test_number) (Optional) Test number to run a specific test"
|
|
||||||
echo
|
|
||||||
echo "Example:"
|
|
||||||
echo " $PROG test-tokenizer"
|
|
||||||
echo " $PROG test-tokenizer 3"
|
|
||||||
echo
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Function to select and debug a test
|
|
||||||
function select_test() {
|
|
||||||
test_suite=${1:-test}
|
|
||||||
test_number=${2:-}
|
|
||||||
repo_root=${3:-}
|
|
||||||
|
|
||||||
# Color
|
|
||||||
red=$(tput setaf 1)
|
|
||||||
green=$(tput setaf 2)
|
|
||||||
yellow=$(tput setaf 3)
|
|
||||||
blue=$(tput setaf 4)
|
|
||||||
normal=$(tput sgr0)
|
|
||||||
|
|
||||||
# Sanity Check If Tests Is Detected
|
|
||||||
printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n"
|
|
||||||
tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1'))
|
|
||||||
if [ ${#tests[@]} -eq 0 ]
|
|
||||||
then
|
|
||||||
echo "No tests avaliable... check your compliation process..."
|
|
||||||
echo "Exiting."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z $test_number ]
|
|
||||||
then
|
|
||||||
# List out avaliable tests
|
|
||||||
printf "Which test would you like to debug?\n"
|
|
||||||
id=0
|
|
||||||
for s in "${tests[@]}"
|
|
||||||
do
|
|
||||||
echo "Test# ${id}"
|
|
||||||
echo " $s"
|
|
||||||
((id++))
|
|
||||||
done
|
|
||||||
|
|
||||||
# Prompt user which test they wanted to run
|
|
||||||
printf "\nRun test#? "
|
|
||||||
read test_number
|
|
||||||
else
|
|
||||||
printf "\nUser Already Requested #${test_number}\n"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Find requested test binary and arguments
|
|
||||||
# Change IFS (Internal Field Separator)
|
|
||||||
sIFS=$IFS
|
|
||||||
IFS=$'\n'
|
|
||||||
|
|
||||||
# Get test args
|
|
||||||
test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' ))
|
|
||||||
IFS=$sIFS
|
|
||||||
|
|
||||||
# Execute Test
|
|
||||||
pushd "$repo_root" || exit 1
|
|
||||||
printf "${blue}Running Test #${test_number}: ${tests[test_number]}${normal}\n"
|
|
||||||
eval "${test_args[test_number]}"
|
|
||||||
exit_code=$?
|
|
||||||
popd
|
|
||||||
|
|
||||||
# Print Result
|
|
||||||
printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n"
|
|
||||||
printf "${yellow}Command: ${test_args[test_number]}${normal}\n"
|
|
||||||
if [ $exit_code -eq 0 ]; then
|
|
||||||
printf "${green}TEST PASS${normal}\n"
|
|
||||||
else
|
|
||||||
printf "${red}TEST FAIL${normal}\n"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Step 0: Check the args
|
|
||||||
if [ -z "$test_suite" ]
|
|
||||||
then
|
|
||||||
echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
|
|
||||||
echo "Supply one regex to the script to filter tests,"
|
|
||||||
echo "and optionally a test number to run a specific test."
|
|
||||||
echo "Use --help flag for full instructions"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Step 1: Reset and Setup folder context
|
|
||||||
## Sanity check that we are actually in a git repo
|
|
||||||
repo_root=$(git rev-parse --show-toplevel)
|
|
||||||
if [ ! -d "$repo_root" ]; then
|
|
||||||
echo "Error: Not in a Git repository."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
## Reset folder to root context of git repo
|
|
||||||
pushd "$repo_root" || exit 1
|
|
||||||
|
|
||||||
## Create and enter build directory
|
|
||||||
rm -rf "$build_dir" && mkdir "$build_dir" || exit 1
|
|
||||||
|
|
||||||
# Step 2: Setup Build Environment and Compile Test Binaries
|
|
||||||
# Note: test-eval-callback requires -DLLAMA_CURL=1
|
|
||||||
cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=1 || exit 1
|
|
||||||
pushd "$build_dir" && make -j || exit 1
|
|
||||||
|
|
||||||
# Step 3: Debug the Test
|
|
||||||
select_test "$test_suite" "$test_number" "$repo_root"
|
|
||||||
|
|
||||||
# Step 4: Return to the directory from which the user ran the command.
|
|
||||||
popd > /dev/null || exit 1
|
|
||||||
popd > /dev/null || exit 1
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue