From 5a5d8e4844987b6d8d64d583eb33dd926973a270 Mon Sep 17 00:00:00 2001 From: Satoru Takeuchi Date: Sun, 1 Dec 2013 07:57:58 +0900 Subject: [PATCH 1/5] ktest: Make the signal to terminate the console configurable Currently ktest sends SIGINT to terminate the console. However, there are consoles which do not exit by this signal, for example, in my case, "virsh console ". In such case, ktest is blocked in close_console(). It prevents this automate test. This patch adds new option CLOSE_CONSOLE_SIGNAL which mean the signal to terminate the console. Since its default value is "INT", the original behavior isn't changed. Link: http://lkml.kernel.org/r/87zjol8pl5.wl%satoru.takeuchi@gmail.com Signed-off-by: Satoru Takeuchi Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 0d7fd8b51544..5dc5704d8680 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -25,6 +25,7 @@ my %default = ( "TEST_TYPE" => "build", "BUILD_TYPE" => "randconfig", "MAKE_CMD" => "make", + "CLOSE_CONSOLE_SIGNAL" => "INT", "TIMEOUT" => 120, "TMP_DIR" => "/tmp/ktest/\${MACHINE}", "SLEEP_TIME" => 60, # sleep time between tests @@ -163,6 +164,7 @@ my $timeout; my $booted_timeout; my $detect_triplefault; my $console; +my $close_console_signal; my $reboot_success_line; my $success_line; my $stop_after_success; @@ -285,6 +287,7 @@ my %option_map = ( "TIMEOUT" => \$timeout, "BOOTED_TIMEOUT" => \$booted_timeout, "CONSOLE" => \$console, + "CLOSE_CONSOLE_SIGNAL" => \$close_console_signal, "DETECT_TRIPLE_FAULT" => \$detect_triplefault, "SUCCESS_LINE" => \$success_line, "REBOOT_SUCCESS_LINE" => \$reboot_success_line, @@ -1296,7 +1299,7 @@ sub close_console { my ($fp, $pid) = @_; doprint "kill child process $pid\n"; - kill 2, $pid; + kill $close_console_signal, $pid; print "closing!\n"; close($fp); From 298a0d1d575026edec7fbb9b7e8cbb8fe18498cd Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Tue, 3 Dec 2013 11:57:23 -0500 Subject: [PATCH 2/5] ktest: Add documentation of CLOSE_CONSOLE_SIGNAL The sample.conf file needs to document all available options. With the new CLOSE_CONSOE_SIGNAL option, it too needs to be document. Cc: Satoru Takeuchi Signed-off-by: Steven Rostedt --- tools/testing/ktest/sample.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf index 0a290fb4cd5e..2eb4bd2f6ab4 100644 --- a/tools/testing/ktest/sample.conf +++ b/tools/testing/ktest/sample.conf @@ -328,6 +328,13 @@ # For a virtual machine with guest name "Guest". #CONSOLE = virsh console Guest +# Signal to send to kill console. +# ktest.pl will create a child process to monitor the console. +# When the console is finished, ktest will kill the child process +# with this signal. +# (default INT) +#CLOSE_CONSOLE_SIGNAL = HUP + # Required version ending to differentiate the test # from other linux builds on the system. #LOCALVERSION = -test From 8e80bf05ff7e7bda6f1683b1201ada56c4efa4c1 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Wed, 11 Dec 2013 15:40:46 -0500 Subject: [PATCH 3/5] ktest: Add special variable ${KERNEL_VERSION} Add a special variable that can be used in other variables called ${KERNEL_VERSION}. This will embed the current kernel version into the variable. For example: WARNINGS_FILE = ${OUTPUT_DIR}/warnings-${KERNEL_VERSION} If the current version is v3.8 then the WARNINGS_FILE will become ${OUTPUT_DIR}/warnings-v3.8 Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 5dc5704d8680..b285933873c7 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -1129,6 +1129,10 @@ sub __eval_option { } elsif (defined($opt{$var})) { $o = $opt{$var}; $retval = "$retval$o"; + } elsif ($var eq "KERNEL_VERSION" && defined($make)) { + # special option KERNEL_VERSION uses kernel version + get_version(); + $retval = "$retval$version"; } else { $retval = "$retval\$\{$var\}"; } @@ -3919,6 +3923,18 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) { my $makecmd = set_test_option("MAKE_CMD", $i); + $outputdir = set_test_option("OUTPUT_DIR", $i); + $builddir = set_test_option("BUILD_DIR", $i); + + chdir $builddir || die "can't change directory to $builddir"; + + if (!-d $outputdir) { + mkpath($outputdir) or + die "can't create $outputdir"; + } + + $make = "$makecmd O=$outputdir"; + # Load all the options into their mapped variable names foreach my $opt (keys %option_map) { ${$option_map{$opt}} = set_test_option($opt, $i); @@ -3943,13 +3959,9 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) { $start_minconfig = $minconfig; } - chdir $builddir || die "can't change directory to $builddir"; - - foreach my $dir ($tmpdir, $outputdir) { - if (!-d $dir) { - mkpath($dir) or - die "can't create $dir"; - } + if (!-d $tmpdir) { + mkpath($tmpdir) or + die "can't create $tmpdir"; } $ENV{"SSH_USER"} = $ssh_user; @@ -3958,7 +3970,6 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) { $buildlog = "$tmpdir/buildlog-$machine"; $testlog = "$tmpdir/testlog-$machine"; $dmesg = "$tmpdir/dmesg-$machine"; - $make = "$makecmd O=$outputdir"; $output_config = "$outputdir/.config"; if (!$buildonly) { From c75d22d9c675c4c77d87ff36de6e5023f14724ef Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Wed, 11 Dec 2013 21:16:59 -0500 Subject: [PATCH 4/5] ktest: Add eval '=~' command to modify variables in config file With the added variable ${KERNEL_VERSION}, it is useful to be able to use parts of it for other variables. For example, if you want to create a warnings file for each major kernel version to test sub versions against you can create your warnings file with like this: WARNINGS_FILE = warnings-file-${KERNEL_VERSION} But this may add 3.8.12 or something, and we want all 3.8.* to use the same file, and 3.10.* to use another file, and so on. With the eval command we can, by adding: WARNINGS_FILE =~ s/(-file-\d+\.\d+).*/$1/ Which will chop off the extra characters after the 3.8. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 102 +++++++++++++++++++++++++++-------- 1 file changed, 81 insertions(+), 21 deletions(-) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index b285933873c7..82006c2d88c9 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -18,6 +18,7 @@ $| = 1; my %opt; my %repeat_tests; my %repeats; +my %evals; #default opts my %default = ( @@ -448,6 +449,27 @@ $config_help{"REBOOT_SCRIPT"} = << "EOF" EOF ; +sub _logit { + if (defined($opt{"LOG_FILE"})) { + open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}"; + print OUT @_; + close(OUT); + } +} + +sub logit { + if (defined($opt{"LOG_FILE"})) { + _logit @_; + } else { + print @_; + } +} + +sub doprint { + print @_; + _logit @_; +} + sub read_prompt { my ($cancel, $prompt) = @_; @@ -665,6 +687,22 @@ sub set_value { } } +sub set_eval { + my ($lvalue, $rvalue, $name) = @_; + + my $prvalue = process_variables($rvalue); + my $arr; + + if (defined($evals{$lvalue})) { + $arr = $evals{$lvalue}; + } else { + $arr = []; + $evals{$lvalue} = $arr; + } + + push @{$arr}, $rvalue; +} + sub set_variable { my ($lvalue, $rvalue) = @_; @@ -950,6 +988,20 @@ sub __read_config { $test_case = 1; } + } elsif (/^\s*([A-Z_\[\]\d]+)\s*=~\s*(.*?)\s*$/) { + + next if ($skip); + + my $lvalue = $1; + my $rvalue = $2; + + if ($default || $lvalue =~ /\[\d+\]$/) { + set_eval($lvalue, $rvalue, $name); + } else { + my $val = "$lvalue\[$test_num\]"; + set_eval($val, $rvalue, $name); + } + } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) { next if ($skip); @@ -1147,6 +1199,33 @@ sub __eval_option { return $retval; } +sub process_evals { + my ($name, $option, $i) = @_; + + my $option_name = "$name\[$i\]"; + my $ev; + + my $old_option = $option; + + if (defined($evals{$option_name})) { + $ev = $evals{$option_name}; + } elsif (defined($evals{$name})) { + $ev = $evals{$name}; + } else { + return $option; + } + + for my $e (@{$ev}) { + eval "\$option =~ $e"; + } + + if ($option ne $old_option) { + doprint("$name changed from '$old_option' to '$option'\n"); + } + + return $option; +} + sub eval_option { my ($name, $option, $i) = @_; @@ -1167,30 +1246,11 @@ sub eval_option { $option = __eval_option($name, $option, $i); } + $option = process_evals($name, $option, $i); + return $option; } -sub _logit { - if (defined($opt{"LOG_FILE"})) { - open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}"; - print OUT @_; - close(OUT); - } -} - -sub logit { - if (defined($opt{"LOG_FILE"})) { - _logit @_; - } else { - print @_; - } -} - -sub doprint { - print @_; - _logit @_; -} - sub run_command; sub start_monitor; sub end_monitor; From 961d9caceea2d5350a15c17b7d3ffc24c08c9b09 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Sat, 18 Jan 2014 19:52:13 -0500 Subject: [PATCH 5/5] ktest: Add BISECT_TRIES to bisect test For those cases that it takes several tries to hit a bug, it would be useful for ktest.pl to try a test multiple times before it considers the test as a pass. To accomplish this, BISECT_TRIES ktest config option has been added. It is default to one, as most of the time a bisect only needs to try a test once. But the user can now up this to make ktest run a given test multiple times. The first failure that is detected will set a bisect bad. It only repeats on success. Note, as with all race bugs, there's no guarantee that if it succeeds, it is really a good bisect. But it helps in case the bug is somewhat reliable. You can set BISECT_TRIES to zero, and all tests will be considered good, unless you also set BISECT_MANUAL. Suggested-by: "Paul E. McKenney" Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 24 ++++++++++++++++++++++-- tools/testing/ktest/sample.conf | 14 ++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 82006c2d88c9..a511d4aae35d 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -41,6 +41,7 @@ my %default = ( "CLEAR_LOG" => 0, "BISECT_MANUAL" => 0, "BISECT_SKIP" => 1, + "BISECT_TRIES" => 1, "MIN_CONFIG_TYPE" => "boot", "SUCCESS_LINE" => "login:", "DETECT_TRIPLE_FAULT" => 1, @@ -139,6 +140,7 @@ my $bisect_bad_commit = ""; my $reverse_bisect; my $bisect_manual; my $bisect_skip; +my $bisect_tries; my $config_bisect_good; my $bisect_ret_good; my $bisect_ret_bad; @@ -276,6 +278,7 @@ my %option_map = ( "IGNORE_ERRORS" => \$ignore_errors, "BISECT_MANUAL" => \$bisect_manual, "BISECT_SKIP" => \$bisect_skip, + "BISECT_TRIES" => \$bisect_tries, "CONFIG_BISECT_GOOD" => \$config_bisect_good, "BISECT_RET_GOOD" => \$bisect_ret_good, "BISECT_RET_BAD" => \$bisect_ret_bad, @@ -2584,12 +2587,29 @@ sub run_bisect { $buildtype = "useconfig:$minconfig"; } - my $ret = run_bisect_test $type, $buildtype; + # If the user sets bisect_tries to less than 1, then no tries + # is a success. + my $ret = 1; - if ($bisect_manual) { + # Still let the user manually decide that though. + if ($bisect_tries < 1 && $bisect_manual) { $ret = answer_bisect; } + for (my $i = 0; $i < $bisect_tries; $i++) { + if ($bisect_tries > 1) { + my $t = $i + 1; + doprint("Running bisect trial $t of $bisect_tries:\n"); + } + $ret = run_bisect_test $type, $buildtype; + + if ($bisect_manual) { + $ret = answer_bisect; + } + + last if (!$ret); + } + # Are we looking for where it worked, not failed? if ($reverse_bisect && $ret >= 0) { $ret = !$ret; diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf index 2eb4bd2f6ab4..172eec4517fb 100644 --- a/tools/testing/ktest/sample.conf +++ b/tools/testing/ktest/sample.conf @@ -1028,6 +1028,20 @@ # BISECT_BAD with BISECT_CHECK = good or # BISECT_CHECK = bad, respectively. # +# BISECT_TRIES = 5 (optional, default 1) +# +# For those cases that it takes several tries to hit a bug, +# the BISECT_TRIES is useful. It is the number of times the +# test is ran before it says the kernel is good. The first failure +# will stop trying and mark the current SHA1 as bad. +# +# Note, as with all race bugs, there's no guarantee that if +# it succeeds, it is really a good bisect. But it helps in case +# the bug is some what reliable. +# +# You can set BISECT_TRIES to zero, and all tests will be considered +# good, unless you also set BISECT_MANUAL. +# # BISECT_RET_GOOD = 0 (optional, default undefined) # # In case the specificed test returns something other than just