From d75ceb168bbb741b5a8c52ec05bdebbe65bad964 Mon Sep 17 00:00:00 2001 From: brian khuu Date: Tue, 14 May 2024 21:53:33 +1000 Subject: [PATCH] run-single-test.sh: added a single test function script and fix debug-test.sh to be more robust --- docs/debugging-tests.md | 35 ++++++++++ scripts/debug-test.sh | 26 ++++++-- scripts/run-single-test.sh | 128 +++++++++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+), 7 deletions(-) create mode 100755 scripts/run-single-test.sh diff --git a/docs/debugging-tests.md b/docs/debugging-tests.md index 51a125e19..57d0148f1 100644 --- a/docs/debugging-tests.md +++ b/docs/debugging-tests.md @@ -1,5 +1,40 @@ # Debugging Tests Tips +## How to run & execute a specific test without anything else to keep the feedback loop short? + +There is a script called run-single-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number. + +For example, running the following command will output an interactive list from which you can select a test. It takes this form: + +`run-single-test.sh [OPTION]... ` + +It will then build & run in the debugger for you. + +```bash +./scripts/run-single-test.sh test-tokenizer +``` + +An example of a single test output is shown below. You will get either a green TEST PASS or a red TEST FAIL if a particular test is working or not. This shorter feedback loop will hopefully make it easier for you to figure out the problem you are trying to solve. + +```bash +$ ./scripts/run-single-test.sh test 24 +~/gitextern/llama.cpp ~/gitextern/llama.cpp + +... prepping cmake environment ... +... building test binaries ... +... running test ... + +Ran Test #24: test-eval-callback +Command: /home/mofosyne/gitextern/llama.cpp/build-ci-debug/bin/eval-callback "--hf-repo" "ggml-org/models" "--hf-file" "tinyllamas/stories260K.gguf" "--model" "stories260K.gguf" "--prompt" "hello" "--seed" "42" "-ngl" "0" +TEST PASS +``` + +For further reference use `run-single-test.sh -h` to print help. + +### How does the script work? + +This is similar to `debug-test.sh` so you can follow the similar guide in this page for similar process. Just run the command directly rather than though gdb. + ## How to run & debug a specific test without anything else to keep the feedback loop short? There is a script called debug-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number. diff --git a/scripts/debug-test.sh b/scripts/debug-test.sh index 231a23d69..806fcb7e2 100755 --- a/scripts/debug-test.sh +++ b/scripts/debug-test.sh @@ -27,6 +27,7 @@ fi function select_test() { test_suite=${1:-test} test_number=${2:-} + repo_root=${3:-} # Sanity Check If Tests Is Detected printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n" @@ -54,7 +55,7 @@ function select_test() { printf "\nRun test#? " read test_number else - printf "\nUser Already Requested #${test_number}" + printf "\nUser Already Requested #${test_number}\n" fi # Start GDB with the requested test binary and arguments @@ -75,9 +76,20 @@ function select_test() { args+=($(echo $x | sed -e 's/.*\/..\//..\//')) done + # Print Command + red=$(tput setaf 1) + green=$(tput setaf 2) + yellow=$(tput setaf 3) + blue=$(tput setaf 4) + normal=$(tput sgr0) + printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n" + printf "${yellow}GDB args: ${gdb_args[test_number]}${normal}\n" + printf "${yellow}GDB args @: ${args[@]}${normal}\n" + # Execute debugger - echo "gdb args: ${args[@]}" + pushd "$repo_root" || exit 1 gdb --args ${args[@]} + popd > /dev/null || exit 1 } # Step 0: Check the args @@ -105,13 +117,13 @@ pushd "$repo_root" || exit 1 rm -rf "$build_dir" && mkdir "$build_dir" || exit 1 # Step 2: Setup Build Environment and Compile Test Binaries -cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON || exit 1 +# Note: test-eval-callback requires -DLLAMA_CURL +cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=1 || exit 1 pushd "$build_dir" && make -j || exit 1 # Step 3: Debug the Test -select_test "$test_suite" "$test_number" +select_test "$test_suite" "$test_number" "$repo_root" # Step 4: Return to the directory from which the user ran the command. -popd || exit 1 -popd || exit 1 -popd || exit 1 +popd > /dev/null || exit 1 +popd > /dev/null || exit 1 diff --git a/scripts/run-single-test.sh b/scripts/run-single-test.sh new file mode 100755 index 000000000..08d20515e --- /dev/null +++ b/scripts/run-single-test.sh @@ -0,0 +1,128 @@ +#!/bin/bash +test_suite=${1:-} +test_number=${2:-} + +PROG=${0##*/} +build_dir="build-ci-debug" + +if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then + echo "Usage: $PROG [OPTION]... (test_number)" + echo "Run a specific ctest program." + echo + echo "Options:" + echo " -h, --help Display this help and exit" + echo + echo "Arguments:" + echo " (Mandatory) Supply one regex to the script to filter tests" + echo " (test_number) (Optional) Test number to run a specific test" + echo + echo "Example:" + echo " $PROG test-tokenizer" + echo " $PROG test-tokenizer 3" + echo + exit 0 +fi + +# Function to select and debug a test +function select_test() { + test_suite=${1:-test} + test_number=${2:-} + repo_root=${3:-} + + # Color + red=$(tput setaf 1) + green=$(tput setaf 2) + yellow=$(tput setaf 3) + blue=$(tput setaf 4) + normal=$(tput sgr0) + + # Sanity Check If Tests Is Detected + printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n" + tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1')) + if [ ${#tests[@]} -eq 0 ] + then + echo "No tests avaliable... check your compliation process..." + echo "Exiting." + exit 1 + fi + + if [ -z $test_number ] + then + # List out avaliable tests + printf "Which test would you like to debug?\n" + id=0 + for s in "${tests[@]}" + do + echo "Test# ${id}" + echo " $s" + ((id++)) + done + + # Prompt user which test they wanted to run + printf "\nRun test#? " + read test_number + else + printf "\nUser Already Requested #${test_number}\n" + fi + + # Find requested test binary and arguments + # Change IFS (Internal Field Separator) + sIFS=$IFS + IFS=$'\n' + + # Get test args + test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' )) + IFS=$sIFS + + # Execute Test + pushd "$repo_root" || exit 1 + printf "${blue}Running Test #${test_number}: ${tests[test_number]}${normal}\n" + eval "${test_args[test_number]}" + exit_code=$? + popd + + # Print Result + printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n" + printf "${yellow}Command: ${test_args[test_number]}${normal}\n" + if [ $exit_code -eq 0 ]; then + printf "${green}TEST PASS${normal}\n" + else + printf "${red}TEST FAIL${normal}\n" + fi +} + +# Step 0: Check the args +if [ -z "$test_suite" ] +then + echo "Usage: $PROG [OPTION]... (test_number)" + echo "Supply one regex to the script to filter tests," + echo "and optionally a test number to run a specific test." + echo "Use --help flag for full instructions" + exit 1 +fi + +# Step 1: Reset and Setup folder context +## Sanity check that we are actually in a git repo +repo_root=$(git rev-parse --show-toplevel) +if [ ! -d "$repo_root" ]; then + echo "Error: Not in a Git repository." + exit 1 +fi + +## Reset folder to root context of git repo +pushd "$repo_root" || exit 1 + +## Create and enter build directory +rm -rf "$build_dir" && mkdir "$build_dir" || exit 1 + +# Step 2: Setup Build Environment and Compile Test Binaries +# Note: test-eval-callback requires -DLLAMA_CURL=1 +cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=1 || exit 1 +pushd "$build_dir" && make -j || exit 1 + +# Step 3: Debug the Test +select_test "$test_suite" "$test_number" "$repo_root" + +# Step 4: Return to the directory from which the user ran the command. +popd > /dev/null || exit 1 +popd > /dev/null || exit 1