From 6fc804ffbb7d0163a3f6a73dbd2b6f7c5f954fff Mon Sep 17 00:00:00 2001 From: BVK Chaitanya Date: Fri, 8 Jan 2010 15:19:10 +0530 Subject: [PATCH] unit testing framework --- Makefile.in | 19 +++- conf/tests.rmk | 41 ++++++++ genmk.rb | 6 +- include/grub/test.h | 88 ++++++++++++++++++ tests/example_functional_test.c | 17 ++++ tests/example_grub_script_test.in | 3 + tests/example_scripted_test.in | 3 + tests/example_unit_test.c | 20 ++++ tests/lib/functional_test.c | 75 +++++++++++++++ tests/lib/test.c | 150 ++++++++++++++++++++++++++++++ tests/lib/unit_test.c | 122 ++++++++++++++++++++++++ tests/util/grub-shell-tester.in | 109 ++++++++++++++++++++++ tests/util/grub-shell.in | 125 +++++++++++++++++++++++++ 13 files changed, 774 insertions(+), 4 deletions(-) create mode 100644 conf/tests.rmk create mode 100644 include/grub/test.h create mode 100644 tests/example_functional_test.c create mode 100644 tests/example_grub_script_test.in create mode 100644 tests/example_scripted_test.in create mode 100644 tests/example_unit_test.c create mode 100644 tests/lib/functional_test.c create mode 100644 tests/lib/test.c create mode 100644 tests/lib/unit_test.c create mode 100644 tests/util/grub-shell-tester.in create mode 100644 tests/util/grub-shell.in diff --git a/Makefile.in b/Makefile.in index ffa8716ed..2358060f2 100644 --- a/Makefile.in +++ b/Makefile.in @@ -141,6 +141,7 @@ PROGRAMS = $(bin_UTILITIES) $(sbin_UTILITIES) SCRIPTS = $(bin_SCRIPTS) $(sbin_SCRIPTS) $(grub-mkconfig_SCRIPTS) \ $(lib_SCRIPTS) INFOS = $(info_INFOS) +TESTS = $(check_UNITTESTS) $(check_FUNCTIONALTESTS) $(check_SCRIPTEDTESTS) CLEANFILES = MOSTLYCLEANFILES = @@ -167,6 +168,8 @@ ifeq ($(platform), emu) include $(srcdir)/conf/any-emu.mk else include $(srcdir)/conf/$(target_cpu)-$(platform).mk +# For tests. +include $(srcdir)/conf/tests.mk # For external modules. -include $(wildcard $(GRUB_CONTRIB)/*/conf/common.mk) endif @@ -458,7 +461,21 @@ distcheck: dist @echo "$(distdir).tar.gz is ready for distribution" | \ sed 'h;s/./=/g;p;x;p;x' -check: +$(TESTS): $(check_SCRIPTS) $(check_MODULES) $(check_PROGRAMS) +check: all $(TESTS) + @list="$(check_UNITTESTS) $(check_SCRIPTEDTESTS)"; \ + for file in $$list; do \ + if $(builddir)/$$file; then \ + echo "$$file: PASS"; \ + else \ + echo "$$file: FAIL"; \ + fi; \ + done + @list="$(check_FUNCTIONALTESTS)"; \ + for test in $$list; do \ + echo "insmod functional_test; insmod $test; functional_test" \ + | $(builddir)/grub-shell; \ + done .SUFFIX: .SUFFIX: .c .o .S .d diff --git a/conf/tests.rmk b/conf/tests.rmk new file mode 100644 index 000000000..03d6acbd1 --- /dev/null +++ b/conf/tests.rmk @@ -0,0 +1,41 @@ +# -*- makefile -*- + +# For grub-shell +grub-shell: tests/util/grub-shell.in config.status + ./config.status --file=$@:$< + chmod +x $@ +check_SCRIPTS += grub-shell +CLEANFILES += grub-shell + +# For grub-shell-tester +grub-shell-tester: tests/util/grub-shell-tester.in config.status + ./config.status --file=$@:$< + chmod +x $@ +check_SCRIPTS += grub-shell-tester +CLEANFILES += grub-shell-tester + +check_MODULES += functional_test.mod +functional_test_mod_SOURCES = tests/lib/functional_test.c tests/lib/test.c +functional_test_mod_CFLAGS = $(COMMON_CFLAGS) +functional_test_mod_LDFLAGS = $(COMMON_LDFLAGS) + +# Unit tests + +check_UNITTESTS += example_unit_test +example_unit_test_SOURCES = tests/example_unit_test.c kern/list.c kern/misc.c tests/lib/test.c tests/lib/unit_test.c +example_unit_test_CFLAGS = -Wno-format + +# Functional tests + +check_FUNCTIONALTESTS += example_functional_test.mod +example_functional_test_mod_SOURCES = tests/example_functional_test.c +example_functional_test_mod_CFLAGS = -Wno-format $(COMMON_CFLAGS) +example_functional_test_mod_LDFLAGS = $(COMMON_LDFLAGS) + +# Scripted tests + +check_SCRIPTEDTESTS += example_scripted_test +example_scripted_test_SOURCES = tests/example_scripted_test.in + +check_SCRIPTEDTESTS += example_grub_script_test +example_grub_script_test_SOURCES = tests/example_grub_script_test.in diff --git a/genmk.rb b/genmk.rb index b1cce3831..006fc3d29 100644 --- a/genmk.rb +++ b/genmk.rb @@ -407,12 +407,12 @@ while l = gets Image.new(prefix, img) end - when 'MODULES' + when 'MODULES', 'FUNCTIONALTESTS' pmodules += args.split(/\s+/).collect do |pmod| PModule.new(prefix, pmod) end - when 'UTILITIES' + when 'UTILITIES', 'UNITTESTS' utils += args.split(/\s+/).collect do |util| Utility.new(prefix, util) end @@ -422,7 +422,7 @@ while l = gets Program.new(prefix, prog) end - when 'SCRIPTS' + when 'SCRIPTS', 'SCRIPTEDTESTS' scripts += args.split(/\s+/).collect do |script| Script.new(prefix, script) end diff --git a/include/grub/test.h b/include/grub/test.h new file mode 100644 index 000000000..fbc2bff8c --- /dev/null +++ b/include/grub/test.h @@ -0,0 +1,88 @@ +#ifndef GRUB_TEST_HEADER +#define GRUB_TEST_HEADER + +#include +#include +#include +#include +#include + +struct grub_test +{ + /* The next test. */ + struct grub_test *next; + + /* The test name. */ + char *name; + + /* The test main function. */ + void (*main) (void); +}; +typedef struct grub_test *grub_test_t; + +extern grub_test_t EXPORT_VAR (grub_test_list); + +void EXPORT_FUNC (grub_test_register) (const char *name, void (*test) (void)); + +void EXPORT_FUNC (grub_test_unregister) (const char *name); + +/* Execute a test and print results. */ +int grub_test_run (const char *name); + +/* Test `cond' for nonzero; log failure otherwise. */ +void grub_test_nonzero (int cond, const char *file, + const char *func, grub_uint32_t line, + const char *fmt, ...) + __attribute__ ((format (printf, 5, 6))); + +#ifdef __STDC_VERSION__ +#if __STDC_VERSION__ < 199901L +# if __GNUC__ >= 2 +# define __func__ __FUNCTION__ +# else +# define __func__ "" +# endif +#endif +#else +#define __func__ "" +#endif + +/* Macro to fill in location details and an optional error message. */ +#define grub_test_assert(cond, ...) \ + grub_test_nonzero(cond, __FILE__, __func__, __LINE__, \ + ## __VA_ARGS__, \ + "assert failed: %s", #cond) + +/* Macro to define a unit test. */ +#define GRUB_UNIT_TEST(name, funp) \ + void grub_unit_test_init (void) \ + { \ + grub_test_register (name, funp); \ + } \ + \ + void grub_unit_test_fini (void) \ + { \ + grub_test_unregister (name); \ + } + +/* Macro to define a functional test. */ +#define GRUB_FUNCTIONAL_TEST(name, funp) \ + GRUB_MOD_INIT(functional_test_##funp) \ + { \ + grub_test_register (name, funp); \ + } \ + \ + GRUB_MOD_FINI(functional_test_##funp) \ + { \ + grub_test_unregister (name); \ + } + +/* Functions that are defined differently for unit and functional tests. */ +void *grub_test_malloc (grub_size_t size); +void grub_test_free (void *ptr); +char *grub_test_strdup (const char *str); +int grub_test_vsprintf (char *str, const char *fmt, va_list args); +int grub_test_printf (const char *fmt, ...) + __attribute__ ((format (printf, 1, 2))); + +#endif /* ! GRUB_TEST_HEADER */ diff --git a/tests/example_functional_test.c b/tests/example_functional_test.c new file mode 100644 index 000000000..475d1c7f0 --- /dev/null +++ b/tests/example_functional_test.c @@ -0,0 +1,17 @@ +/* All tests need to include test.h for GRUB testing framework. */ +#include + +/* Functional test main method. */ +static void +example_test (void) +{ + /* Check if 1st argument is true and report with default error message. */ + grub_test_assert (1 == 1); + + /* Check if 1st argument is true and report with custom error message. */ + grub_test_assert (2 == 2, "2 equal 2 expected"); + grub_test_assert (2 == 3, "2 is not equal to %d", 3); +} + +/* Register example_test method as a functional test. */ +GRUB_FUNCTIONAL_TEST ("example_functional_test", example_test); diff --git a/tests/example_grub_script_test.in b/tests/example_grub_script_test.in new file mode 100644 index 000000000..93a90a18e --- /dev/null +++ b/tests/example_grub_script_test.in @@ -0,0 +1,3 @@ +#! @builddir@/grub-shell-tester --modules=echo + +echo "hello world" diff --git a/tests/example_scripted_test.in b/tests/example_scripted_test.in new file mode 100644 index 000000000..9ac0424c0 --- /dev/null +++ b/tests/example_scripted_test.in @@ -0,0 +1,3 @@ +#!/bin/sh -e + +true diff --git a/tests/example_unit_test.c b/tests/example_unit_test.c new file mode 100644 index 000000000..e2fad06ff --- /dev/null +++ b/tests/example_unit_test.c @@ -0,0 +1,20 @@ +/* Unit tests are normal programs, so they can include C library. */ +#include + +/* All tests need to include test.h for GRUB testing framework. */ +#include + +/* Unit test main method. */ +static void +example_test (void) +{ + /* Check if 1st argument is true and report with default error message. */ + grub_test_assert (1 == 1); + + /* Check if 1st argument is true and report with custom error message. */ + grub_test_assert (2 == 2, "2 equal 2 expected"); + grub_test_assert (2 == 3, "2 is not equal to %d", 3); +} + +/* Register example_test method as a unit test. */ +GRUB_UNIT_TEST ("example_unit_test", example_test); diff --git a/tests/lib/functional_test.c b/tests/lib/functional_test.c new file mode 100644 index 000000000..24e82932e --- /dev/null +++ b/tests/lib/functional_test.c @@ -0,0 +1,75 @@ +#include +#include +#include +#include + +void * +grub_test_malloc (grub_size_t size) +{ + return grub_malloc (size); +} + +void +grub_test_free (void *ptr) +{ + grub_free (ptr); +} + +int +grub_test_vsprintf (char *str, const char *fmt, va_list args) +{ + return grub_vsprintf (str, fmt, args); +} + +char * +grub_test_strdup (const char *str) +{ + return grub_strdup (str); +} + +int +grub_test_printf (const char *fmt, ...) +{ + int r; + va_list ap; + + va_start (ap, fmt); + r = grub_vprintf (fmt, ap); + va_end (ap); + + return r; +} + +static grub_err_t +grub_functional_test (struct grub_extcmd *cmd __attribute__ ((unused)), + int argc __attribute__ ((unused)), + char **args __attribute__ ((unused))) +{ + int i; + int status; + grub_test_t test; + + status = 0; + for (i = 0; i < argc; i++) + { + test = grub_named_list_find (GRUB_AS_NAMED_LIST (grub_test_list), + args[i]); + status = grub_test_run (test->name) ? : status; + } + + return status; +} + +static grub_extcmd_t cmd; + +GRUB_MOD_INIT (functional_test) +{ + cmd = grub_register_extcmd ("functional_test", grub_functional_test, + GRUB_COMMAND_FLAG_CMDLINE, 0, + "Run all functional tests.", 0); +} + +GRUB_MOD_FINI (functional_test) +{ + grub_unregister_extcmd (cmd); +} diff --git a/tests/lib/test.c b/tests/lib/test.c new file mode 100644 index 000000000..37f8fff72 --- /dev/null +++ b/tests/lib/test.c @@ -0,0 +1,150 @@ +#include +#include + +struct grub_test_failure +{ + /* The next failure. */ + struct grub_test_failure *next; + + /* The test source file name. */ + char *file; + + /* The test function name. */ + char *funp; + + /* The test call line number. */ + grub_uint32_t line; + + /* The test failure message. */ + char *message; +}; +typedef struct grub_test_failure *grub_test_failure_t; + +grub_test_t grub_test_list; +static grub_test_failure_t failure_list; + +static void +add_failure (const char *file, + const char *funp, + grub_uint32_t line, const char *fmt, va_list args) +{ + char buf[1024]; + grub_test_failure_t failure; + + failure = (grub_test_failure_t) grub_test_malloc (sizeof (*failure)); + if (!failure) + return; + + grub_test_vsprintf (buf, fmt, args); + + failure->file = grub_test_strdup (file ? : ""); + failure->funp = grub_test_strdup (funp ? : ""); + failure->line = line; + failure->message = grub_test_strdup (buf); + + grub_list_push (GRUB_AS_LIST_P (&failure_list), GRUB_AS_LIST (failure)); +} + +static void +free_failures (void) +{ + grub_test_failure_t item; + + while ((item = grub_list_pop (GRUB_AS_LIST_P (&failure_list))) != 0) + { + if (item->message) + grub_test_free (item->message); + + if (item->funp) + grub_test_free (item->funp); + + if (item->file) + grub_test_free (item->file); + + grub_test_free (item); + } + failure_list = 0; +} + +void +grub_test_nonzero (int cond, + const char *file, + const char *funp, grub_uint32_t line, const char *fmt, ...) +{ + va_list ap; + + if (cond) + return; + + va_start (ap, fmt); + add_failure (file, funp, line, fmt, ap); + va_end (ap); +} + +void +grub_test_register (const char *name, void (*test_main) (void)) +{ + grub_test_t test; + + test = (grub_test_t) grub_test_malloc (sizeof (*test)); + if (!test) + return; + + test->name = grub_test_strdup (name); + test->main = test_main; + + grub_list_push (GRUB_AS_LIST_P (&grub_test_list), GRUB_AS_LIST (test)); +} + +void +grub_test_unregister (const char *name) +{ + grub_test_t test; + + test = (grub_test_t) grub_named_list_find + (GRUB_AS_NAMED_LIST (grub_test_list), name); + + if (test) + { + grub_list_remove (GRUB_AS_LIST_P (&grub_test_list), + GRUB_AS_LIST (test)); + + if (test->name) + grub_test_free (test->name); + + grub_test_free (test); + } +} + +int +grub_test_run (const char *name) +{ + grub_test_t test; + + auto int print_failure (grub_test_failure_t item); + int print_failure (grub_test_failure_t item) + { + grub_test_failure_t failure = (grub_test_failure_t) item; + + grub_test_printf (" %s:%s:%u: %s\n", + (failure->file ? : ""), + (failure->funp ? : ""), + failure->line, (failure->message ? : "")); + return 0; + } + + test = grub_named_list_find (GRUB_AS_NAMED_LIST (grub_test_list), name); + if (!test) + return GRUB_ERR_FILE_NOT_FOUND; + + test->main (); + + if (!failure_list) + return GRUB_ERR_NONE; + + grub_test_printf ("%s:\n", test->name); + grub_list_iterate (GRUB_AS_LIST (failure_list), + (grub_list_hook_t) print_failure); + free_failures (); + return GRUB_ERR_TEST_FAILURE; +} diff --git a/tests/lib/unit_test.c b/tests/lib/unit_test.c new file mode 100644 index 000000000..15bb0da5d --- /dev/null +++ b/tests/lib/unit_test.c @@ -0,0 +1,122 @@ +#include +#include +#include +#include + +#include +#include +#include + +void * +grub_test_malloc (grub_size_t size) +{ + return malloc (size); +} + +void +grub_test_free (void *ptr) +{ + free (ptr); +} + +int +grub_test_vsprintf (char *str, const char *fmt, va_list args) +{ + return vsprintf (str, fmt, args); +} + +char * +grub_test_strdup (const char *str) +{ + return strdup (str); +} + +int +grub_test_printf (const char *fmt, ...) +{ + int r; + va_list ap; + + va_start (ap, fmt); + r = vprintf (fmt, ap); + va_end (ap); + + return r; +} + +int +main (int argc __attribute__ ((unused)), + char *argv[] __attribute__ ((unused))) +{ + int status = 0; + + extern void grub_unit_test_init (void); + extern void grub_unit_test_fini (void); + + auto int run_test (grub_test_t test); + int run_test (grub_test_t test) + { + status = grub_test_run (test->name) ? : status; + return 0; + } + + grub_unit_test_init (); + grub_list_iterate (GRUB_AS_LIST (grub_test_list), + (grub_list_hook_t) run_test); + grub_unit_test_fini (); + + exit (status); +} + +/* Other misc. functions necessary for successful linking. */ + +char * +grub_env_get (const char *name __attribute__ ((unused))) +{ + return NULL; +} + +grub_err_t +grub_error (grub_err_t n, const char *fmt, ...) +{ + va_list ap; + + va_start (ap, fmt); + vfprintf (stderr, fmt, ap); + va_end (ap); + + return n; +} + +void * +grub_malloc (grub_size_t size) +{ + return malloc (size); +} + +void +grub_refresh (void) +{ + fflush (stdout); +} + +void +grub_putchar (int c) +{ + putchar (c); +} + +int +grub_getkey (void) +{ + return -1; +} + +void +grub_exit (void) +{ + exit (1); +} + +struct grub_handler_class grub_term_input_class; +struct grub_handler_class grub_term_output_class; diff --git a/tests/util/grub-shell-tester.in b/tests/util/grub-shell-tester.in new file mode 100644 index 000000000..18f5e79b7 --- /dev/null +++ b/tests/util/grub-shell-tester.in @@ -0,0 +1,109 @@ +#! /bin/bash -e + +# Compares GRUB script output with BASH output. +# Copyright (C) 1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009 Free Software Foundation, Inc. +# +# GRUB is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# GRUB is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GRUB. If not, see . + +# Initialize some variables. +transform="@program_transform_name@" + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +bindir=@bindir@ +libdir=@libdir@ +builddir=@builddir@ +PACKAGE_NAME=@PACKAGE_NAME@ +PACKAGE_TARNAME=@PACKAGE_TARNAME@ +PACKAGE_VERSION=@PACKAGE_VERSION@ +target_cpu=@target_cpu@ + +# Force build directory components +PATH=${builddir}:$PATH +export PATH + +# Usage: usage +# Print the usage. +usage () { + cat <. +EOF +} + +# Check the arguments. +for option in "$@"; do + case "$option" in + -h | --help) + usage + exit 0 ;; + -v | --version) + echo "$0 (GNU GRUB ${PACKAGE_VERSION})" + exit 0 ;; + --modules=*) + ms=`echo "$option" | sed -e 's/--modules=//'` + modules="$modules,$ms" ;; + --qemu-opts=*) + qs=`echo "$option" | sed -e 's/--qemu-opts=//'` + qemuopts="$qemuopts $qs" ;; + -*) + echo "Unrecognized option \`$option'" 1>&2 + usage + exit 1 + ;; + *) + if [ "x${source}" != x ] ; then + echo "too many parameters at the end" 1>&2 + usage + exit 1 + fi + source="${option}" ;; + esac +done + +if [ "x${source}" = x ] ; then + tmpfile=`mktemp` + while read; do + echo $REPLY >> ${tmpfile} + done + source=${tmpfile} +fi + +outfile1=`mktemp` +@builddir@/grub-shell --qemu-opts="${qemuopts}" --modules=${modules} ${source} >${outfile1} + +outfile2=`mktemp` +bash ${source} >${outfile2} + +if ! diff -q ${outfile1} ${outfile2} >/dev/null +then + echo "$1: GRUB and BASH outputs (${outfile1}, ${outfile2}) did not match" + status=1 +else + rm -f ${outfile1} ${outfile2} +fi + +exit $status + + diff --git a/tests/util/grub-shell.in b/tests/util/grub-shell.in new file mode 100644 index 000000000..c1b2ef258 --- /dev/null +++ b/tests/util/grub-shell.in @@ -0,0 +1,125 @@ +#! /bin/bash -e + +# Run GRUB script in a Qemu instance +# Copyright (C) 1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009 Free Software Foundation, Inc. +# +# GRUB is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# GRUB is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GRUB. If not, see . + +# Initialize some variables. +transform="@program_transform_name@" + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +bindir=@bindir@ +libdir=@libdir@ +builddir=@builddir@ +PACKAGE_NAME=@PACKAGE_NAME@ +PACKAGE_TARNAME=@PACKAGE_TARNAME@ +PACKAGE_VERSION=@PACKAGE_VERSION@ +target_cpu=@target_cpu@ + +# Force build directory components +PATH=${builddir}:$PATH +export PATH + +# Usage: usage +# Print the usage. +usage () { + cat <. +EOF +} + +# Check the arguments. +for option in "$@"; do + case "$option" in + -h | --help) + usage + exit 0 ;; + -v | --version) + echo "$0 (GNU GRUB ${PACKAGE_VERSION})" + exit 0 ;; + --modules=*) + ms=`echo "$option" | sed -e 's/--modules=//' -e 's/,/ /g'` + modules="$modules $ms" ;; + --qemu-opts=*) + qs=`echo "$option" | sed -e 's/--qemu-opts=//'` + qemuopts="$qemuopts $qs" ;; + -*) + echo "Unrecognized option \`$option'" 1>&2 + usage + exit 1 + ;; + *) + if [ "x${source}" != x ] ; then + echo "too many parameters at the end" 1>&2 + usage + exit 1 + fi + source="${option}" ;; + esac +done + +if [ "x${source}" = x ] ; then + tmpfile=`mktemp` + while read; do + echo $REPLY >> ${tmpfile} + done + source=${tmpfile} +fi + +cfgfile=`mktemp` +cat <${cfgfile} +grubshell=yes +insmod serial +serial +terminal_input serial +terminal_output serial +EOF + +for mod in ${modules} +do + echo "insmod ${mod}" >> ${cfgfile} +done + +cat <>${cfgfile} +source /boot/grub/testcase.cfg +halt +EOF + +isofile=`mktemp` +grub-mkrescue --output=${isofile} --override-directory=${builddir} \ + /boot/grub/grub.cfg=${cfgfile} /boot/grub/testcase.cfg=${source} \ + >/dev/null 2>&1 + +outfile=`mktemp` +qemu ${qemuopts} -nographic -serial stdio -cdrom ${isofile} -boot d | tr -d "\r" >${outfile} + +cat $outfile + +rm -f ${tmpfile} ${outfile} ${cfgfile} ${isofile} +exit 0 + +