run-single-test.sh: added a single test function script and fix debug-test.sh to be more robust

docs/debugging-tests.md

@@ -1,5 +1,40 @@
 # Debugging Tests Tips
 
+## How to run & execute a specific test without anything else to keep the feedback loop short?
+
+There is a script called run-single-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number.
+
+For example, running the following command will output an interactive list from which you can select a test. It takes this form:
+
+`run-single-test.sh [OPTION]... <test_regex> <test_number>`
+
+It will then build & run in the debugger for you.
+
+```bash
+./scripts/run-single-test.sh test-tokenizer
+```
+
+An example of a single test output is shown below. You will get either a green TEST PASS or a red TEST FAIL if a particular test is working or not. This shorter feedback loop will hopefully make it easier for you to figure out the problem you are trying to solve.
+
+```bash
+$ ./scripts/run-single-test.sh test 24
+~/gitextern/llama.cpp ~/gitextern/llama.cpp
+
+... prepping cmake environment ...
+... building test binaries ...
+... running test ...
+
+Ran Test #24: test-eval-callback
+Command: /home/mofosyne/gitextern/llama.cpp/build-ci-debug/bin/eval-callback "--hf-repo" "ggml-org/models" "--hf-file" "tinyllamas/stories260K.gguf" "--model" "stories260K.gguf" "--prompt" "hello" "--seed" "42" "-ngl" "0"
+TEST PASS
+```
+
+For further reference use `run-single-test.sh -h` to print help.
+
+### How does the script work?
+
+This is similar to `debug-test.sh` so you can follow the similar guide in this page for similar process. Just run the command directly rather than though gdb.
+
 ## How to run & debug a specific test without anything else to keep the feedback loop short?
 
 There is a script called debug-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number.

scripts/debug-test.sh

@@ -27,6 +27,7 @@ fi
 function select_test() {
     test_suite=${1:-test}
     test_number=${2:-}
+    repo_root=${3:-}
 
     # Sanity Check If Tests Is Detected
     printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n"
@@ -54,7 +55,7 @@ function select_test() {
         printf "\nRun test#? "
         read test_number
     else
-        printf "\nUser Already Requested #${test_number}"
+        printf "\nUser Already Requested #${test_number}\n"
     fi
 
     # Start GDB with the requested test binary and arguments
@@ -75,9 +76,20 @@ function select_test() {
         args+=($(echo $x | sed -e 's/.*\/..\//..\//'))
     done
 
+    # Print Command
+    red=$(tput setaf 1)
+    green=$(tput setaf 2)
+    yellow=$(tput setaf 3)
+    blue=$(tput setaf 4)
+    normal=$(tput sgr0)
+    printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n"
+    printf "${yellow}GDB args: ${gdb_args[test_number]}${normal}\n"
+    printf "${yellow}GDB args @: ${args[@]}${normal}\n"
+
     # Execute debugger
-    echo "gdb args: ${args[@]}"
+    pushd "$repo_root" || exit 1
     gdb --args ${args[@]}
+    popd > /dev/null || exit 1
 }
 
 # Step 0: Check the args
@@ -105,13 +117,13 @@ pushd "$repo_root" || exit 1
 rm -rf "$build_dir" && mkdir "$build_dir" || exit 1
 
 # Step 2: Setup Build Environment and Compile Test Binaries
-cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON || exit 1
+# Note: test-eval-callback requires -DLLAMA_CURL
+cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=1 || exit 1
 pushd "$build_dir" && make -j || exit 1
 
 # Step 3: Debug the Test
-select_test "$test_suite" "$test_number"
+select_test "$test_suite" "$test_number" "$repo_root"
 
 # Step 4: Return to the directory from which the user ran the command.
-popd || exit 1
+popd > /dev/null || exit 1
-popd || exit 1
+popd > /dev/null || exit 1
-popd || exit 1

scripts/run-single-test.sh

@@ -0,0 +1,128 @@
+#!/bin/bash
+test_suite=${1:-}
+test_number=${2:-}
+
+PROG=${0##*/}
+build_dir="build-ci-debug"
+
+if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then
+    echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
+    echo "Run a specific ctest program."
+    echo
+    echo "Options:"
+    echo "  -h, --help       Display this help and exit"
+    echo
+    echo "Arguments:"
+    echo "  <test_regex>     (Mandatory) Supply one regex to the script to filter tests"
+    echo "  (test_number)    (Optional) Test number to run a specific test"
+    echo
+    echo "Example:"
+    echo "  $PROG test-tokenizer"
+    echo "  $PROG test-tokenizer 3"
+    echo
+    exit 0
+fi
+
+# Function to select and debug a test
+function select_test() {
+    test_suite=${1:-test}
+    test_number=${2:-}
+    repo_root=${3:-}
+
+    # Color
+    red=$(tput setaf 1)
+    green=$(tput setaf 2)
+    yellow=$(tput setaf 3)
+    blue=$(tput setaf 4)
+    normal=$(tput sgr0)
+
+    # Sanity Check If Tests Is Detected
+    printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n"
+    tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1'))
+    if [ ${#tests[@]} -eq 0 ]
+    then
+        echo "No tests avaliable... check your compliation process..."
+        echo "Exiting."
+        exit 1
+    fi
+
+    if [ -z $test_number ]
+    then
+        # List out avaliable tests
+        printf "Which test would you like to debug?\n"
+        id=0
+        for s in "${tests[@]}"
+        do
+            echo "Test# ${id}"
+            echo "  $s"
+            ((id++))
+        done
+
+        # Prompt user which test they wanted to run
+        printf "\nRun test#? "
+        read test_number
+    else
+        printf "\nUser Already Requested #${test_number}\n"
+    fi
+
+    # Find requested test binary and arguments
+    # Change IFS (Internal Field Separator)
+    sIFS=$IFS
+    IFS=$'\n'
+
+    # Get test args
+    test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' ))
+    IFS=$sIFS
+
+    # Execute Test
+    pushd "$repo_root" || exit 1
+    printf "${blue}Running Test #${test_number}: ${tests[test_number]}${normal}\n"
+    eval "${test_args[test_number]}"
+    exit_code=$?
+    popd
+
+    # Print Result
+    printf "${blue}Ran Test #${test_number}: ${tests[test_number]}${normal}\n"
+    printf "${yellow}Command: ${test_args[test_number]}${normal}\n"
+    if [ $exit_code -eq 0 ]; then
+        printf "${green}TEST PASS${normal}\n"
+    else
+        printf "${red}TEST FAIL${normal}\n"
+    fi
+}
+
+# Step 0: Check the args
+if [ -z "$test_suite" ]
+then
+    echo "Usage: $PROG [OPTION]... <test_regex> (test_number)"
+    echo "Supply one regex to the script to filter tests,"
+    echo "and optionally a test number to run a specific test."
+    echo "Use --help flag for full instructions"
+    exit 1
+fi
+
+# Step 1: Reset and Setup folder context
+## Sanity check that we are actually in a git repo
+repo_root=$(git rev-parse --show-toplevel)
+if [ ! -d "$repo_root" ]; then
+    echo "Error: Not in a Git repository."
+    exit 1
+fi
+
+## Reset folder to root context of git repo
+pushd "$repo_root" || exit 1
+
+## Create and enter build directory
+rm -rf "$build_dir" && mkdir "$build_dir" || exit 1
+
+# Step 2: Setup Build Environment and Compile Test Binaries
+# Note: test-eval-callback requires -DLLAMA_CURL=1
+cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=1 || exit 1
+pushd "$build_dir" && make -j || exit 1
+
+# Step 3: Debug the Test
+select_test "$test_suite" "$test_number" "$repo_root"
+
+# Step 4: Return to the directory from which the user ran the command.
+popd > /dev/null || exit 1
+popd > /dev/null || exit 1