From f274de538600451491b337edab5679dd51a9430f Mon Sep 17 00:00:00 2001 From: brian khuu Date: Tue, 14 May 2024 23:09:25 +1000 Subject: [PATCH] debug-test.sh: combined execute and gdb test mode via -g flag --- docs/debugging-tests.md | 51 +++++---------- scripts/debug-test.sh | 122 ++++++++++++++++++++++++----------- scripts/run-single-test.sh | 128 ------------------------------------- 3 files changed, 101 insertions(+), 200 deletions(-) delete mode 100755 scripts/run-single-test.sh diff --git a/docs/debugging-tests.md b/docs/debugging-tests.md index 57d0148f1..984da80e1 100644 --- a/docs/debugging-tests.md +++ b/docs/debugging-tests.md @@ -1,41 +1,6 @@ # Debugging Tests Tips -## How to run & execute a specific test without anything else to keep the feedback loop short? - -There is a script called run-single-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number. - -For example, running the following command will output an interactive list from which you can select a test. It takes this form: - -`run-single-test.sh [OPTION]... ` - -It will then build & run in the debugger for you. - -```bash -./scripts/run-single-test.sh test-tokenizer -``` - -An example of a single test output is shown below. You will get either a green TEST PASS or a red TEST FAIL if a particular test is working or not. This shorter feedback loop will hopefully make it easier for you to figure out the problem you are trying to solve. - -```bash -$ ./scripts/run-single-test.sh test 24 -~/gitextern/llama.cpp ~/gitextern/llama.cpp - -... prepping cmake environment ... -... building test binaries ... -... running test ... - -Ran Test #24: test-eval-callback -Command: /home/mofosyne/gitextern/llama.cpp/build-ci-debug/bin/eval-callback "--hf-repo" "ggml-org/models" "--hf-file" "tinyllamas/stories260K.gguf" "--model" "stories260K.gguf" "--prompt" "hello" "--seed" "42" "-ngl" "0" -TEST PASS -``` - -For further reference use `run-single-test.sh -h` to print help. - -### How does the script work? - -This is similar to `debug-test.sh` so you can follow the similar guide in this page for similar process. Just run the command directly rather than though gdb. - -## How to run & debug a specific test without anything else to keep the feedback loop short? +## How to run & execute or debug a specific test without anything else to keep the feedback loop short? There is a script called debug-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number. @@ -45,13 +10,27 @@ For example, running the following command will output an interactive list from It will then build & run in the debugger for you. +To just execute a test and get back a PASS or FAIL message run: + ```bash ./scripts/debug-test.sh test-tokenizer +``` + +To test in GDB use the `-g` flag to enable gdb test mode. + +```bash +./scripts/debug-test.sh -g test-tokenizer # Once in the debugger, i.e. at the chevrons prompt, setting a breakpoint could be as follows: >>> b main ``` +To speed up the testing loop, if you know your test number you can just run it similar to below: + +```bash +./scripts/debug-test.sh test 23 +``` + For further reference use `debug-test.sh -h` to print help.   diff --git a/scripts/debug-test.sh b/scripts/debug-test.sh index 806fcb7e2..497e44e57 100755 --- a/scripts/debug-test.sh +++ b/scripts/debug-test.sh @@ -5,24 +5,62 @@ test_number=${2:-} PROG=${0##*/} build_dir="build-ci-debug" +# Print Color Commands +red=$(tput setaf 1) +green=$(tput setaf 2) +yellow=$(tput setaf 3) +blue=$(tput setaf 4) +normal=$(tput sgr0) + +print_full_help() { + cat << EOF +Usage: $PROG [OPTION]... (test_number) +Debug specific ctest program. + +Options: + -h, --help display this help and exit + -g run in gdb mode + +Arguments: + (Mandatory) Supply one regex to the script to filter tests + (test_number) (Optional) Test number to run a specific test + +Example: + $PROG test-tokenizer + $PROG test-tokenizer 3 +EOF +} + +abort() { + echo "Error: $1" >&2 + cat << EOF >&2 +Usage: $PROG [OPTION]... (test_number) +Debug specific ctest program. +Refer to --help for full instructions. +EOF + exit 1 +} + if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then - echo "Usage: $PROG [OPTION]... (test_number)" - echo "Debug specific ctest program." - echo - echo "Options:" - echo " -h, --help Display this help and exit" - echo - echo "Arguments:" - echo " (Mandatory) Supply one regex to the script to filter tests" - echo " (test_number) (Optional) Test number to run a specific test" - echo - echo "Example:" - echo " $PROG test-tokenizer" - echo " $PROG test-tokenizer 3" - echo - exit 0 + print_full_help >&2 + exit 0 fi +# Parse command-line options +gdb_mode=false +while getopts "hg" opt; do + case $opt in + h) + print_full_help >&2 + exit 0 + ;; + g) + gdb_mode=true + echo "gdb_mode Mode Enabled" + ;; + esac +done + # Function to select and debug a test function select_test() { test_suite=${1:-test} @@ -65,31 +103,43 @@ function select_test() { IFS=$'\n' # Get test args - gdb_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' )) + test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' )) IFS=$sIFS - printf "Debug arguments: ${gdb_args[test_number]}\n\n" - # Expand paths if needed - args=() - for x in $(echo ${gdb_args[test_number]} | sed -e 's/"\/\"//') - do - args+=($(echo $x | sed -e 's/.*\/..\//..\//')) - done + printf "${blue}Running Test #${test_number}: ${tests[test_number]}${normal}\n" + printf "${blue}test_args[test_number]: ${test_args[test_number]}${normal}\n" - # Print Command - red=$(tput setaf 1) - green=$(tput setaf 2) - yellow=$(tput setaf 3) - blue=$(tput setaf 4) - normal=$(tput sgr0) - printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n" - printf "${yellow}GDB args: ${gdb_args[test_number]}${normal}\n" - printf "${yellow}GDB args @: ${args[@]}${normal}\n" + if [ "$gdb_mode" = "true" ]; then + # Expand paths if needed + gdb_args=() + for x in $(echo ${test_args[test_number]} | sed -e 's/"\/\"//') + do + gdb_args+=($(echo $x | sed -e 's/.*\/..\//..\//')) + done - # Execute debugger - pushd "$repo_root" || exit 1 - gdb --args ${args[@]} - popd > /dev/null || exit 1 + printf "${blue}gdb_args @: ${gdb_args[@]}${normal}\n" + + # Execute debugger + pushd "$repo_root" || exit 1 + gdb --args ${gdb_args[@]} + popd > /dev/null || exit 1 + else + + # Execute Test + pushd "$repo_root" || exit 1 + eval "${test_args[test_number]}" + exit_code=$? + popd > /dev/null || exit 1 + + # Print Result + printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n" + printf "${yellow}Command: ${test_args[test_number]}${normal}\n" + if [ $exit_code -eq 0 ]; then + printf "${green}TEST PASS${normal}\n" + else + printf "${red}TEST FAIL${normal}\n" + fi + fi } # Step 0: Check the args diff --git a/scripts/run-single-test.sh b/scripts/run-single-test.sh deleted file mode 100755 index 08d20515e..000000000 --- a/scripts/run-single-test.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/bin/bash -test_suite=${1:-} -test_number=${2:-} - -PROG=${0##*/} -build_dir="build-ci-debug" - -if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then - echo "Usage: $PROG [OPTION]... (test_number)" - echo "Run a specific ctest program." - echo - echo "Options:" - echo " -h, --help Display this help and exit" - echo - echo "Arguments:" - echo " (Mandatory) Supply one regex to the script to filter tests" - echo " (test_number) (Optional) Test number to run a specific test" - echo - echo "Example:" - echo " $PROG test-tokenizer" - echo " $PROG test-tokenizer 3" - echo - exit 0 -fi - -# Function to select and debug a test -function select_test() { - test_suite=${1:-test} - test_number=${2:-} - repo_root=${3:-} - - # Color - red=$(tput setaf 1) - green=$(tput setaf 2) - yellow=$(tput setaf 3) - blue=$(tput setaf 4) - normal=$(tput sgr0) - - # Sanity Check If Tests Is Detected - printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n" - tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1')) - if [ ${#tests[@]} -eq 0 ] - then - echo "No tests avaliable... check your compliation process..." - echo "Exiting." - exit 1 - fi - - if [ -z $test_number ] - then - # List out avaliable tests - printf "Which test would you like to debug?\n" - id=0 - for s in "${tests[@]}" - do - echo "Test# ${id}" - echo " $s" - ((id++)) - done - - # Prompt user which test they wanted to run - printf "\nRun test#? " - read test_number - else - printf "\nUser Already Requested #${test_number}\n" - fi - - # Find requested test binary and arguments - # Change IFS (Internal Field Separator) - sIFS=$IFS - IFS=$'\n' - - # Get test args - test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' )) - IFS=$sIFS - - # Execute Test - pushd "$repo_root" || exit 1 - printf "${blue}Running Test #${test_number}: ${tests[test_number]}${normal}\n" - eval "${test_args[test_number]}" - exit_code=$? - popd - - # Print Result - printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n" - printf "${yellow}Command: ${test_args[test_number]}${normal}\n" - if [ $exit_code -eq 0 ]; then - printf "${green}TEST PASS${normal}\n" - else - printf "${red}TEST FAIL${normal}\n" - fi -} - -# Step 0: Check the args -if [ -z "$test_suite" ] -then - echo "Usage: $PROG [OPTION]... (test_number)" - echo "Supply one regex to the script to filter tests," - echo "and optionally a test number to run a specific test." - echo "Use --help flag for full instructions" - exit 1 -fi - -# Step 1: Reset and Setup folder context -## Sanity check that we are actually in a git repo -repo_root=$(git rev-parse --show-toplevel) -if [ ! -d "$repo_root" ]; then - echo "Error: Not in a Git repository." - exit 1 -fi - -## Reset folder to root context of git repo -pushd "$repo_root" || exit 1 - -## Create and enter build directory -rm -rf "$build_dir" && mkdir "$build_dir" || exit 1 - -# Step 2: Setup Build Environment and Compile Test Binaries -# Note: test-eval-callback requires -DLLAMA_CURL=1 -cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=1 || exit 1 -pushd "$build_dir" && make -j || exit 1 - -# Step 3: Debug the Test -select_test "$test_suite" "$test_number" "$repo_root" - -# Step 4: Return to the directory from which the user ran the command. -popd > /dev/null || exit 1 -popd > /dev/null || exit 1