2020-11-23 14:15:33 +00:00
|
|
|
#!/usr/bin/env python3
|
2019-09-23 09:02:43 +00:00
|
|
|
# SPDX-License-Identifier: GPL-2.0
|
|
|
|
#
|
|
|
|
# A thin wrapper on top of the KUnit Kernel
|
|
|
|
#
|
|
|
|
# Copyright (C) 2019, Google LLC.
|
|
|
|
# Author: Felix Guo <felixguoxiuping@gmail.com>
|
|
|
|
# Author: Brendan Higgins <brendanhiggins@google.com>
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
import os
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
import re
|
2022-05-18 17:01:24 +00:00
|
|
|
import shlex
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
import sys
|
2019-09-23 09:02:43 +00:00
|
|
|
import time
|
|
|
|
|
2021-07-12 19:52:58 +00:00
|
|
|
assert sys.version_info >= (3, 7), "Python version is too old"
|
|
|
|
|
2021-12-14 19:26:11 +00:00
|
|
|
from dataclasses import dataclass
|
2019-09-23 09:02:43 +00:00
|
|
|
from enum import Enum, auto
|
2022-01-18 19:09:18 +00:00
|
|
|
from typing import Iterable, List, Optional, Sequence, Tuple
|
2019-09-23 09:02:43 +00:00
|
|
|
|
2020-08-11 21:27:56 +00:00
|
|
|
import kunit_json
|
2019-09-23 09:02:43 +00:00
|
|
|
import kunit_kernel
|
|
|
|
import kunit_parser
|
2022-05-16 19:47:30 +00:00
|
|
|
from kunit_printer import stdout
|
2019-09-23 09:02:43 +00:00
|
|
|
|
|
|
|
class KunitStatus(Enum):
|
|
|
|
SUCCESS = auto()
|
|
|
|
CONFIG_FAILURE = auto()
|
|
|
|
BUILD_FAILURE = auto()
|
|
|
|
TEST_FAILURE = auto()
|
|
|
|
|
2021-12-14 19:26:11 +00:00
|
|
|
@dataclass
|
|
|
|
class KunitResult:
|
|
|
|
status: KunitStatus
|
|
|
|
elapsed_time: float
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
class KunitConfigRequest:
|
|
|
|
build_dir: str
|
|
|
|
make_options: Optional[List[str]]
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
class KunitBuildRequest(KunitConfigRequest):
|
|
|
|
jobs: int
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
class KunitParseRequest:
|
|
|
|
raw_output: Optional[str]
|
|
|
|
json: Optional[str]
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
class KunitExecRequest(KunitParseRequest):
|
2022-02-24 19:20:35 +00:00
|
|
|
build_dir: str
|
2021-12-14 19:26:11 +00:00
|
|
|
timeout: int
|
|
|
|
filter_glob: str
|
kunit: tool: Add command line interface to filter and report attributes
Add ability to kunit.py to filter attributes and report a list of tests
including attributes without running tests.
Add flag "--filter" to input filters on test attributes. Tests will be
filtered out if they do not match all inputted filters.
Example: --filter speed=slow (This filter would run only the tests that are
marked as slow)
Filters have operations: <, >, <=, >=, !=, and =. But note that the
characters < and > are often interpreted by the shell, so they may need to
be quoted or escaped.
Example: --filter "speed>slow" or --filter speed\>slow (This filter would
run only the tests that have the speed faster than slow.
Additionally, multiple filters can be used.
Example: --filter "speed=slow, module!=example" (This filter would run
only the tests that have the speed slow and are not in the "example"
module)
Note if the user wants to skip filtered tests instead of not
running/showing them use the "--filter_action=skip" flag instead.
Expose the output of kunit.action=list option with flag "--list_tests" to
output a list of tests. Additionally, add flag "--list_tests_attr" to
output a list of tests and their attributes. These flags are useful to see
tests and test attributes without needing to run tests.
Example of the output of "--list_tests_attr":
example
example.test_1
example.test_2
# example.test_2.speed: slow
This output includes a suite, example, with two test cases, test_1 and
test_2. And in this instance test_2 has been marked as slow.
Reviewed-by: David Gow <davidgow@google.com>
Signed-off-by: Rae Moar <rmoar@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2023-07-25 21:25:16 +00:00
|
|
|
filter: str
|
|
|
|
filter_action: Optional[str]
|
2021-12-14 19:26:11 +00:00
|
|
|
kernel_args: Optional[List[str]]
|
|
|
|
run_isolated: Optional[str]
|
kunit: tool: Add command line interface to filter and report attributes
Add ability to kunit.py to filter attributes and report a list of tests
including attributes without running tests.
Add flag "--filter" to input filters on test attributes. Tests will be
filtered out if they do not match all inputted filters.
Example: --filter speed=slow (This filter would run only the tests that are
marked as slow)
Filters have operations: <, >, <=, >=, !=, and =. But note that the
characters < and > are often interpreted by the shell, so they may need to
be quoted or escaped.
Example: --filter "speed>slow" or --filter speed\>slow (This filter would
run only the tests that have the speed faster than slow.
Additionally, multiple filters can be used.
Example: --filter "speed=slow, module!=example" (This filter would run
only the tests that have the speed slow and are not in the "example"
module)
Note if the user wants to skip filtered tests instead of not
running/showing them use the "--filter_action=skip" flag instead.
Expose the output of kunit.action=list option with flag "--list_tests" to
output a list of tests. Additionally, add flag "--list_tests_attr" to
output a list of tests and their attributes. These flags are useful to see
tests and test attributes without needing to run tests.
Example of the output of "--list_tests_attr":
example
example.test_1
example.test_2
# example.test_2.speed: slow
This output includes a suite, example, with two test cases, test_1 and
test_2. And in this instance test_2 has been marked as slow.
Reviewed-by: David Gow <davidgow@google.com>
Signed-off-by: Rae Moar <rmoar@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2023-07-25 21:25:16 +00:00
|
|
|
list_tests: bool
|
|
|
|
list_tests_attr: bool
|
2021-12-14 19:26:11 +00:00
|
|
|
|
|
|
|
@dataclass
|
|
|
|
class KunitRequest(KunitExecRequest, KunitBuildRequest):
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
2021-01-15 00:39:11 +00:00
|
|
|
def get_kernel_root_path() -> str:
|
|
|
|
path = sys.argv[0] if not __file__ else __file__
|
|
|
|
parts = os.path.realpath(path).split('tools/testing/kunit')
|
2020-02-18 22:19:16 +00:00
|
|
|
if len(parts) != 2:
|
|
|
|
sys.exit(1)
|
|
|
|
return parts[0]
|
|
|
|
|
2020-05-01 04:27:01 +00:00
|
|
|
def config_tests(linux: kunit_kernel.LinuxSourceTree,
|
|
|
|
request: KunitConfigRequest) -> KunitResult:
|
2022-05-16 19:47:30 +00:00
|
|
|
stdout.print_with_timestamp('Configuring KUnit Kernel ...')
|
2020-05-01 04:27:01 +00:00
|
|
|
|
2019-09-23 09:02:43 +00:00
|
|
|
config_start = time.time()
|
2020-03-23 19:04:59 +00:00
|
|
|
success = linux.build_reconfig(request.build_dir, request.make_options)
|
2019-09-23 09:02:43 +00:00
|
|
|
config_end = time.time()
|
2023-01-18 07:42:19 +00:00
|
|
|
status = KunitStatus.SUCCESS if success else KunitStatus.CONFIG_FAILURE
|
|
|
|
return KunitResult(status, config_end - config_start)
|
2019-09-23 09:02:43 +00:00
|
|
|
|
2020-05-01 04:27:01 +00:00
|
|
|
def build_tests(linux: kunit_kernel.LinuxSourceTree,
|
|
|
|
request: KunitBuildRequest) -> KunitResult:
|
2022-05-16 19:47:30 +00:00
|
|
|
stdout.print_with_timestamp('Building KUnit Kernel ...')
|
2019-09-23 09:02:43 +00:00
|
|
|
|
|
|
|
build_start = time.time()
|
2022-09-02 20:22:48 +00:00
|
|
|
success = linux.build_kernel(request.jobs,
|
2021-05-26 21:24:06 +00:00
|
|
|
request.build_dir,
|
|
|
|
request.make_options)
|
2019-09-23 09:02:43 +00:00
|
|
|
build_end = time.time()
|
2023-01-18 07:42:19 +00:00
|
|
|
status = KunitStatus.SUCCESS if success else KunitStatus.BUILD_FAILURE
|
|
|
|
return KunitResult(status, build_end - build_start)
|
2019-09-23 09:02:43 +00:00
|
|
|
|
2021-12-14 19:30:10 +00:00
|
|
|
def config_and_build_tests(linux: kunit_kernel.LinuxSourceTree,
|
|
|
|
request: KunitBuildRequest) -> KunitResult:
|
|
|
|
config_result = config_tests(linux, request)
|
|
|
|
if config_result.status != KunitStatus.SUCCESS:
|
|
|
|
return config_result
|
|
|
|
|
|
|
|
return build_tests(linux, request)
|
|
|
|
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> List[str]:
|
|
|
|
args = ['kunit.action=list']
|
kunit: tool: Add command line interface to filter and report attributes
Add ability to kunit.py to filter attributes and report a list of tests
including attributes without running tests.
Add flag "--filter" to input filters on test attributes. Tests will be
filtered out if they do not match all inputted filters.
Example: --filter speed=slow (This filter would run only the tests that are
marked as slow)
Filters have operations: <, >, <=, >=, !=, and =. But note that the
characters < and > are often interpreted by the shell, so they may need to
be quoted or escaped.
Example: --filter "speed>slow" or --filter speed\>slow (This filter would
run only the tests that have the speed faster than slow.
Additionally, multiple filters can be used.
Example: --filter "speed=slow, module!=example" (This filter would run
only the tests that have the speed slow and are not in the "example"
module)
Note if the user wants to skip filtered tests instead of not
running/showing them use the "--filter_action=skip" flag instead.
Expose the output of kunit.action=list option with flag "--list_tests" to
output a list of tests. Additionally, add flag "--list_tests_attr" to
output a list of tests and their attributes. These flags are useful to see
tests and test attributes without needing to run tests.
Example of the output of "--list_tests_attr":
example
example.test_1
example.test_2
# example.test_2.speed: slow
This output includes a suite, example, with two test cases, test_1 and
test_2. And in this instance test_2 has been marked as slow.
Reviewed-by: David Gow <davidgow@google.com>
Signed-off-by: Rae Moar <rmoar@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2023-07-25 21:25:16 +00:00
|
|
|
|
|
|
|
if request.kernel_args:
|
|
|
|
args.extend(request.kernel_args)
|
|
|
|
|
|
|
|
output = linux.run_kernel(args=args,
|
|
|
|
timeout=request.timeout,
|
|
|
|
filter_glob=request.filter_glob,
|
|
|
|
filter=request.filter,
|
|
|
|
filter_action=request.filter_action,
|
|
|
|
build_dir=request.build_dir)
|
|
|
|
lines = kunit_parser.extract_tap_lines(output)
|
|
|
|
# Hack! Drop the dummy TAP version header that the executor prints out.
|
|
|
|
lines.pop()
|
|
|
|
|
|
|
|
# Filter out any extraneous non-test output that might have gotten mixed in.
|
|
|
|
return [l for l in output if re.match(r'^[^\s.]+\.[^\s.]+$', l)]
|
|
|
|
|
|
|
|
def _list_tests_attr(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> Iterable[str]:
|
|
|
|
args = ['kunit.action=list_attr']
|
|
|
|
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
if request.kernel_args:
|
|
|
|
args.extend(request.kernel_args)
|
|
|
|
|
|
|
|
output = linux.run_kernel(args=args,
|
2022-09-02 20:22:48 +00:00
|
|
|
timeout=request.timeout,
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
filter_glob=request.filter_glob,
|
kunit: tool: Add command line interface to filter and report attributes
Add ability to kunit.py to filter attributes and report a list of tests
including attributes without running tests.
Add flag "--filter" to input filters on test attributes. Tests will be
filtered out if they do not match all inputted filters.
Example: --filter speed=slow (This filter would run only the tests that are
marked as slow)
Filters have operations: <, >, <=, >=, !=, and =. But note that the
characters < and > are often interpreted by the shell, so they may need to
be quoted or escaped.
Example: --filter "speed>slow" or --filter speed\>slow (This filter would
run only the tests that have the speed faster than slow.
Additionally, multiple filters can be used.
Example: --filter "speed=slow, module!=example" (This filter would run
only the tests that have the speed slow and are not in the "example"
module)
Note if the user wants to skip filtered tests instead of not
running/showing them use the "--filter_action=skip" flag instead.
Expose the output of kunit.action=list option with flag "--list_tests" to
output a list of tests. Additionally, add flag "--list_tests_attr" to
output a list of tests and their attributes. These flags are useful to see
tests and test attributes without needing to run tests.
Example of the output of "--list_tests_attr":
example
example.test_1
example.test_2
# example.test_2.speed: slow
This output includes a suite, example, with two test cases, test_1 and
test_2. And in this instance test_2 has been marked as slow.
Reviewed-by: David Gow <davidgow@google.com>
Signed-off-by: Rae Moar <rmoar@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2023-07-25 21:25:16 +00:00
|
|
|
filter=request.filter,
|
|
|
|
filter_action=request.filter_action,
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
build_dir=request.build_dir)
|
|
|
|
lines = kunit_parser.extract_tap_lines(output)
|
|
|
|
# Hack! Drop the dummy TAP version header that the executor prints out.
|
|
|
|
lines.pop()
|
|
|
|
|
|
|
|
# Filter out any extraneous non-test output that might have gotten mixed in.
|
kunit: tool: Add command line interface to filter and report attributes
Add ability to kunit.py to filter attributes and report a list of tests
including attributes without running tests.
Add flag "--filter" to input filters on test attributes. Tests will be
filtered out if they do not match all inputted filters.
Example: --filter speed=slow (This filter would run only the tests that are
marked as slow)
Filters have operations: <, >, <=, >=, !=, and =. But note that the
characters < and > are often interpreted by the shell, so they may need to
be quoted or escaped.
Example: --filter "speed>slow" or --filter speed\>slow (This filter would
run only the tests that have the speed faster than slow.
Additionally, multiple filters can be used.
Example: --filter "speed=slow, module!=example" (This filter would run
only the tests that have the speed slow and are not in the "example"
module)
Note if the user wants to skip filtered tests instead of not
running/showing them use the "--filter_action=skip" flag instead.
Expose the output of kunit.action=list option with flag "--list_tests" to
output a list of tests. Additionally, add flag "--list_tests_attr" to
output a list of tests and their attributes. These flags are useful to see
tests and test attributes without needing to run tests.
Example of the output of "--list_tests_attr":
example
example.test_1
example.test_2
# example.test_2.speed: slow
This output includes a suite, example, with two test cases, test_1 and
test_2. And in this instance test_2 has been marked as slow.
Reviewed-by: David Gow <davidgow@google.com>
Signed-off-by: Rae Moar <rmoar@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2023-07-25 21:25:16 +00:00
|
|
|
return lines
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
|
|
|
|
def _suites_from_test_list(tests: List[str]) -> List[str]:
|
|
|
|
"""Extracts all the suites from an ordered list of tests."""
|
|
|
|
suites = [] # type: List[str]
|
|
|
|
for t in tests:
|
|
|
|
parts = t.split('.', maxsplit=2)
|
|
|
|
if len(parts) != 2:
|
|
|
|
raise ValueError(f'internal KUnit error, test name should be of the form "<suite>.<test>", got "{t}"')
|
2023-03-16 22:06:37 +00:00
|
|
|
suite, _ = parts
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
if not suites or suites[-1] != suite:
|
|
|
|
suites.append(suite)
|
|
|
|
return suites
|
|
|
|
|
2021-12-14 19:26:11 +00:00
|
|
|
def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> KunitResult:
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
filter_globs = [request.filter_glob]
|
kunit: tool: Add command line interface to filter and report attributes
Add ability to kunit.py to filter attributes and report a list of tests
including attributes without running tests.
Add flag "--filter" to input filters on test attributes. Tests will be
filtered out if they do not match all inputted filters.
Example: --filter speed=slow (This filter would run only the tests that are
marked as slow)
Filters have operations: <, >, <=, >=, !=, and =. But note that the
characters < and > are often interpreted by the shell, so they may need to
be quoted or escaped.
Example: --filter "speed>slow" or --filter speed\>slow (This filter would
run only the tests that have the speed faster than slow.
Additionally, multiple filters can be used.
Example: --filter "speed=slow, module!=example" (This filter would run
only the tests that have the speed slow and are not in the "example"
module)
Note if the user wants to skip filtered tests instead of not
running/showing them use the "--filter_action=skip" flag instead.
Expose the output of kunit.action=list option with flag "--list_tests" to
output a list of tests. Additionally, add flag "--list_tests_attr" to
output a list of tests and their attributes. These flags are useful to see
tests and test attributes without needing to run tests.
Example of the output of "--list_tests_attr":
example
example.test_1
example.test_2
# example.test_2.speed: slow
This output includes a suite, example, with two test cases, test_1 and
test_2. And in this instance test_2 has been marked as slow.
Reviewed-by: David Gow <davidgow@google.com>
Signed-off-by: Rae Moar <rmoar@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2023-07-25 21:25:16 +00:00
|
|
|
if request.list_tests:
|
|
|
|
output = _list_tests(linux, request)
|
|
|
|
for line in output:
|
|
|
|
print(line.rstrip())
|
|
|
|
return KunitResult(status=KunitStatus.SUCCESS, elapsed_time=0.0)
|
|
|
|
if request.list_tests_attr:
|
|
|
|
attr_output = _list_tests_attr(linux, request)
|
|
|
|
for line in attr_output:
|
|
|
|
print(line.rstrip())
|
|
|
|
return KunitResult(status=KunitStatus.SUCCESS, elapsed_time=0.0)
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
if request.run_isolated:
|
|
|
|
tests = _list_tests(linux, request)
|
|
|
|
if request.run_isolated == 'test':
|
|
|
|
filter_globs = tests
|
2023-01-18 07:42:19 +00:00
|
|
|
elif request.run_isolated == 'suite':
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
filter_globs = _suites_from_test_list(tests)
|
|
|
|
# Apply the test-part of the user's glob, if present.
|
|
|
|
if '.' in request.filter_glob:
|
|
|
|
test_glob = request.filter_glob.split('.', maxsplit=2)[1]
|
|
|
|
filter_globs = [g + '.'+ test_glob for g in filter_globs]
|
|
|
|
|
kunit: tool: properly report the used arch for --json, or '' if not known
Before, kunit.py always printed "arch": "UM" in its json output, but...
1. With `kunit.py parse`, we could be parsing output from anywhere, so
we can't say that.
2. Capitalizing it is probably wrong, as it's `ARCH=um`
3. Commit 87c9c1631788 ("kunit: tool: add support for QEMU") made it so
kunit.py could knowingly run a different arch, yet we'd still always
claim "UM".
This patch addresses all of those. E.g.
1.
$ ./tools/testing/kunit/kunit.py parse .kunit/test.log --json | grep -o '"arch.*' | sort -u
"arch": "",
2.
$ ./tools/testing/kunit/kunit.py run --json | ...
"arch": "um",
3.
$ ./tools/testing/kunit/kunit.py run --json --arch=x86_64 | ...
"arch": "x86_64",
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2022-02-24 19:20:36 +00:00
|
|
|
metadata = kunit_json.Metadata(arch=linux.arch(), build_dir=request.build_dir, def_config='kunit_defconfig')
|
2022-02-24 19:20:35 +00:00
|
|
|
|
2021-10-11 21:50:37 +00:00
|
|
|
test_counts = kunit_parser.TestCounts()
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
exec_time = 0.0
|
|
|
|
for i, filter_glob in enumerate(filter_globs):
|
2022-05-16 19:47:30 +00:00
|
|
|
stdout.print_with_timestamp('Starting KUnit Kernel ({}/{})...'.format(i+1, len(filter_globs)))
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
|
|
|
|
test_start = time.time()
|
|
|
|
run_result = linux.run_kernel(
|
|
|
|
args=request.kernel_args,
|
2022-09-02 20:22:48 +00:00
|
|
|
timeout=request.timeout,
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
filter_glob=filter_glob,
|
kunit: tool: Add command line interface to filter and report attributes
Add ability to kunit.py to filter attributes and report a list of tests
including attributes without running tests.
Add flag "--filter" to input filters on test attributes. Tests will be
filtered out if they do not match all inputted filters.
Example: --filter speed=slow (This filter would run only the tests that are
marked as slow)
Filters have operations: <, >, <=, >=, !=, and =. But note that the
characters < and > are often interpreted by the shell, so they may need to
be quoted or escaped.
Example: --filter "speed>slow" or --filter speed\>slow (This filter would
run only the tests that have the speed faster than slow.
Additionally, multiple filters can be used.
Example: --filter "speed=slow, module!=example" (This filter would run
only the tests that have the speed slow and are not in the "example"
module)
Note if the user wants to skip filtered tests instead of not
running/showing them use the "--filter_action=skip" flag instead.
Expose the output of kunit.action=list option with flag "--list_tests" to
output a list of tests. Additionally, add flag "--list_tests_attr" to
output a list of tests and their attributes. These flags are useful to see
tests and test attributes without needing to run tests.
Example of the output of "--list_tests_attr":
example
example.test_1
example.test_2
# example.test_2.speed: slow
This output includes a suite, example, with two test cases, test_1 and
test_2. And in this instance test_2 has been marked as slow.
Reviewed-by: David Gow <davidgow@google.com>
Signed-off-by: Rae Moar <rmoar@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2023-07-25 21:25:16 +00:00
|
|
|
filter=request.filter,
|
|
|
|
filter_action=request.filter_action,
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
build_dir=request.build_dir)
|
|
|
|
|
2022-02-24 19:20:35 +00:00
|
|
|
_, test_result = parse_tests(request, metadata, run_result)
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
# run_kernel() doesn't block on the kernel exiting.
|
|
|
|
# That only happens after we get the last line of output from `run_result`.
|
|
|
|
# So exec_time here actually contains parsing + execution time, which is fine.
|
|
|
|
test_end = time.time()
|
|
|
|
exec_time += test_end - test_start
|
|
|
|
|
2022-01-18 19:09:18 +00:00
|
|
|
test_counts.add_subtest_counts(test_result.counts)
|
2020-05-01 04:27:01 +00:00
|
|
|
|
2021-11-20 03:24:01 +00:00
|
|
|
if len(filter_globs) == 1 and test_counts.crashed > 0:
|
|
|
|
bd = request.build_dir
|
|
|
|
print('The kernel seems to have crashed; you can decode the stack traces with:')
|
|
|
|
print('$ scripts/decode_stacktrace.sh {}/vmlinux {} < {} | tee {}/decoded.log | {} parse'.format(
|
|
|
|
bd, bd, kunit_kernel.get_outfile_path(bd), bd, sys.argv[0]))
|
|
|
|
|
2021-10-11 21:50:37 +00:00
|
|
|
kunit_status = _map_to_overall_status(test_counts.get_status())
|
2022-01-18 19:09:18 +00:00
|
|
|
return KunitResult(status=kunit_status, elapsed_time=exec_time)
|
2021-10-11 21:50:37 +00:00
|
|
|
|
|
|
|
def _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus:
|
|
|
|
if test_status in (kunit_parser.TestStatus.SUCCESS, kunit_parser.TestStatus.SKIPPED):
|
|
|
|
return KunitStatus.SUCCESS
|
2022-05-09 20:49:09 +00:00
|
|
|
return KunitStatus.TEST_FAILURE
|
2020-05-01 04:27:01 +00:00
|
|
|
|
2022-02-24 19:20:35 +00:00
|
|
|
def parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input_data: Iterable[str]) -> Tuple[KunitResult, kunit_parser.Test]:
|
2020-05-01 04:27:01 +00:00
|
|
|
parse_start = time.time()
|
|
|
|
|
2019-09-23 09:02:43 +00:00
|
|
|
if request.raw_output:
|
2021-10-11 21:50:37 +00:00
|
|
|
# Treat unparsed results as one passing test.
|
2022-11-21 19:55:26 +00:00
|
|
|
fake_test = kunit_parser.Test()
|
|
|
|
fake_test.status = kunit_parser.TestStatus.SUCCESS
|
|
|
|
fake_test.counts.passed = 1
|
2021-10-11 21:50:37 +00:00
|
|
|
|
2021-09-30 22:20:46 +00:00
|
|
|
output: Iterable[str] = input_data
|
2021-08-05 23:51:44 +00:00
|
|
|
if request.raw_output == 'all':
|
|
|
|
pass
|
|
|
|
elif request.raw_output == 'kunit':
|
2022-11-30 18:54:19 +00:00
|
|
|
output = kunit_parser.extract_tap_lines(output)
|
2021-08-05 23:51:44 +00:00
|
|
|
for line in output:
|
|
|
|
print(line.rstrip())
|
2022-11-21 19:55:26 +00:00
|
|
|
parse_time = time.time() - parse_start
|
|
|
|
return KunitResult(KunitStatus.SUCCESS, parse_time), fake_test
|
|
|
|
|
2021-08-05 23:51:44 +00:00
|
|
|
|
2022-11-21 19:55:26 +00:00
|
|
|
# Actually parse the test results.
|
|
|
|
test = kunit_parser.parse_run_tests(input_data)
|
|
|
|
parse_time = time.time() - parse_start
|
2020-05-01 04:27:01 +00:00
|
|
|
|
2020-08-11 21:27:56 +00:00
|
|
|
if request.json:
|
2022-01-18 19:09:19 +00:00
|
|
|
json_str = kunit_json.get_json_result(
|
2022-11-21 19:55:26 +00:00
|
|
|
test=test,
|
2022-02-24 19:20:35 +00:00
|
|
|
metadata=metadata)
|
2020-08-11 21:27:56 +00:00
|
|
|
if request.json == 'stdout':
|
2022-01-18 19:09:19 +00:00
|
|
|
print(json_str)
|
|
|
|
else:
|
|
|
|
with open(request.json, 'w') as f:
|
|
|
|
f.write(json_str)
|
2022-05-16 19:47:30 +00:00
|
|
|
stdout.print_with_timestamp("Test results stored in %s" %
|
2022-01-18 19:09:19 +00:00
|
|
|
os.path.abspath(request.json))
|
2020-08-11 21:27:56 +00:00
|
|
|
|
2022-11-21 19:55:26 +00:00
|
|
|
if test.status != kunit_parser.TestStatus.SUCCESS:
|
|
|
|
return KunitResult(KunitStatus.TEST_FAILURE, parse_time), test
|
2020-05-01 04:27:01 +00:00
|
|
|
|
2022-11-21 19:55:26 +00:00
|
|
|
return KunitResult(KunitStatus.SUCCESS, parse_time), test
|
2020-05-01 04:27:01 +00:00
|
|
|
|
|
|
|
def run_tests(linux: kunit_kernel.LinuxSourceTree,
|
|
|
|
request: KunitRequest) -> KunitResult:
|
|
|
|
run_start = time.time()
|
|
|
|
|
2021-12-14 19:26:11 +00:00
|
|
|
config_result = config_tests(linux, request)
|
2020-05-01 04:27:01 +00:00
|
|
|
if config_result.status != KunitStatus.SUCCESS:
|
|
|
|
return config_result
|
|
|
|
|
2021-12-14 19:26:11 +00:00
|
|
|
build_result = build_tests(linux, request)
|
2020-05-01 04:27:01 +00:00
|
|
|
if build_result.status != KunitStatus.SUCCESS:
|
|
|
|
return build_result
|
|
|
|
|
2021-12-14 19:26:11 +00:00
|
|
|
exec_result = exec_tests(linux, request)
|
2020-05-01 04:27:01 +00:00
|
|
|
|
|
|
|
run_end = time.time()
|
2019-09-23 09:02:43 +00:00
|
|
|
|
2022-05-16 19:47:30 +00:00
|
|
|
stdout.print_with_timestamp((
|
2019-09-23 09:02:43 +00:00
|
|
|
'Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' +
|
|
|
|
'building, %.3fs running\n') % (
|
2020-05-01 04:27:01 +00:00
|
|
|
run_end - run_start,
|
|
|
|
config_result.elapsed_time,
|
|
|
|
build_result.elapsed_time,
|
|
|
|
exec_result.elapsed_time))
|
2021-09-30 22:20:46 +00:00
|
|
|
return exec_result
|
2020-05-01 04:27:01 +00:00
|
|
|
|
2021-09-22 16:39:21 +00:00
|
|
|
# Problem:
|
|
|
|
# $ kunit.py run --json
|
|
|
|
# works as one would expect and prints the parsed test results as JSON.
|
|
|
|
# $ kunit.py run --json suite_name
|
|
|
|
# would *not* pass suite_name as the filter_glob and print as json.
|
|
|
|
# argparse will consider it to be another way of writing
|
|
|
|
# $ kunit.py run --json=suite_name
|
|
|
|
# i.e. it would run all tests, and dump the json to a `suite_name` file.
|
|
|
|
# So we hackily automatically rewrite --json => --json=stdout
|
|
|
|
pseudo_bool_flag_defaults = {
|
|
|
|
'--json': 'stdout',
|
|
|
|
'--raw_output': 'kunit',
|
|
|
|
}
|
|
|
|
def massage_argv(argv: Sequence[str]) -> Sequence[str]:
|
|
|
|
def massage_arg(arg: str) -> str:
|
|
|
|
if arg not in pseudo_bool_flag_defaults:
|
|
|
|
return arg
|
|
|
|
return f'{arg}={pseudo_bool_flag_defaults[arg]}'
|
|
|
|
return list(map(massage_arg, argv))
|
|
|
|
|
2021-12-15 20:50:14 +00:00
|
|
|
def get_default_jobs() -> int:
|
|
|
|
return len(os.sched_getaffinity(0))
|
|
|
|
|
2023-03-16 22:06:38 +00:00
|
|
|
def add_common_opts(parser: argparse.ArgumentParser) -> None:
|
2020-05-01 04:27:01 +00:00
|
|
|
parser.add_argument('--build_dir',
|
|
|
|
help='As in the make command, it specifies the build '
|
|
|
|
'directory.',
|
2022-02-26 21:23:25 +00:00
|
|
|
type=str, default='.kunit', metavar='DIR')
|
2020-05-01 04:27:01 +00:00
|
|
|
parser.add_argument('--make_options',
|
|
|
|
help='X=Y make option, can be repeated.',
|
2022-02-26 21:23:25 +00:00
|
|
|
action='append', metavar='X=Y')
|
2020-05-01 04:27:01 +00:00
|
|
|
parser.add_argument('--alltests',
|
2022-09-02 20:22:48 +00:00
|
|
|
help='Run all KUnit tests via tools/testing/kunit/configs/all_tests.config',
|
2020-05-01 04:27:01 +00:00
|
|
|
action='store_true')
|
2021-02-01 20:55:14 +00:00
|
|
|
parser.add_argument('--kunitconfig',
|
2021-02-22 22:52:41 +00:00
|
|
|
help='Path to Kconfig fragment that enables KUnit tests.'
|
|
|
|
' If given a directory, (e.g. lib/kunit), "/.kunitconfig" '
|
kunit: tool: make --kunitconfig repeatable, blindly concat
It's come up a few times that it would be useful to have --kunitconfig
be repeatable [1][2].
This could be done before with a bit of shell-fu, e.g.
$ find fs/ -name '.kunitconfig' -exec cat {} + | \
./tools/testing/kunit/kunit.py run --kunitconfig=/dev/stdin
or equivalently:
$ cat fs/ext4/.kunitconfig fs/fat/.kunitconfig | \
./tools/testing/kunit/kunit.py run --kunitconfig=/dev/stdin
But this can be fairly clunky to use in practice.
And having explicit support in kunit.py opens the door to having more
config fragments of interest, e.g. options for PCI on UML [1], UML
coverage [2], variants of tests [3].
There's another argument to be made that users can just use multiple
--kconfig_add's, but this gets very clunky very fast (e.g. [2]).
Note: there's a big caveat here that some kconfig options might be
incompatible. We try to give a clearish error message in the simple case
where the same option appears multiple times with conflicting values,
but more subtle ones (e.g. mutually exclusive options) will be
potentially very confusing for the user. I don't know we can do better.
Note 2: if you want to combine a --kunitconfig with the default, you
either have to do to specify the current build_dir
> --kunitconfig=.kunit --kunitconfig=additional.config
or
> --kunitconfig=tools/testing/kunit/configs/default.config --kunitconifg=additional.config
each of which have their downsides (former depends on --build_dir,
doesn't work if you don't have a .kunitconfig yet), etc.
Example with conflicting values:
> $ ./tools/testing/kunit/kunit.py config --kunitconfig=lib/kunit --kunitconfig=/dev/stdin <<EOF
> CONFIG_KUNIT_TEST=n
> CONFIG_KUNIT=m
> EOF
> ...
> kunit_kernel.ConfigError: Multiple values specified for 2 options in kunitconfig:
> CONFIG_KUNIT=y
> vs from /dev/stdin
> CONFIG_KUNIT=m
>
> CONFIG_KUNIT_TEST=y
> vs from /dev/stdin
> # CONFIG_KUNIT_TEST is not set
[1] https://lists.freedesktop.org/archives/dri-devel/2022-June/357616.html
[2] https://lore.kernel.org/linux-kselftest/CAFd5g45f3X3xF2vz2BkTHRqOC4uW6GZxtUUMaP5mwwbK8uNVtA@mail.gmail.com/
[3] https://lore.kernel.org/linux-kselftest/CANpmjNOdSy6DuO6CYZ4UxhGxqhjzx4tn0sJMbRqo2xRFv9kX6Q@mail.gmail.com/
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2022-07-08 01:36:32 +00:00
|
|
|
'will get automatically appended. If repeated, the files '
|
|
|
|
'blindly concatenated, which might not work in all cases.',
|
|
|
|
action='append', metavar='PATHS')
|
2021-11-06 01:30:58 +00:00
|
|
|
parser.add_argument('--kconfig_add',
|
|
|
|
help='Additional Kconfig options to append to the '
|
|
|
|
'.kunitconfig, e.g. CONFIG_KASAN=y. Can be repeated.',
|
2022-02-26 21:23:25 +00:00
|
|
|
action='append', metavar='CONFIG_X=Y')
|
2020-05-01 04:27:01 +00:00
|
|
|
|
2021-05-26 21:24:06 +00:00
|
|
|
parser.add_argument('--arch',
|
|
|
|
help=('Specifies the architecture to run tests under. '
|
|
|
|
'The architecture specified here must match the '
|
|
|
|
'string passed to the ARCH make param, '
|
|
|
|
'e.g. i386, x86_64, arm, um, etc. Non-UML '
|
|
|
|
'architectures run on QEMU.'),
|
2022-02-26 21:23:25 +00:00
|
|
|
type=str, default='um', metavar='ARCH')
|
2021-05-26 21:24:06 +00:00
|
|
|
|
|
|
|
parser.add_argument('--cross_compile',
|
|
|
|
help=('Sets make\'s CROSS_COMPILE variable; it should '
|
|
|
|
'be set to a toolchain path prefix (the prefix '
|
|
|
|
'of gcc and other tools in your toolchain, for '
|
|
|
|
'example `sparc64-linux-gnu-` if you have the '
|
|
|
|
'sparc toolchain installed on your system, or '
|
|
|
|
'`$HOME/toolchains/microblaze/gcc-9.2.0-nolibc/microblaze-linux/bin/microblaze-linux-` '
|
|
|
|
'if you have downloaded the microblaze toolchain '
|
|
|
|
'from the 0-day website to a directory in your '
|
|
|
|
'home directory called `toolchains`).'),
|
2022-02-26 21:23:25 +00:00
|
|
|
metavar='PREFIX')
|
2021-05-26 21:24:06 +00:00
|
|
|
|
|
|
|
parser.add_argument('--qemu_config',
|
|
|
|
help=('Takes a path to a path to a file containing '
|
|
|
|
'a QemuArchParams object.'),
|
2022-02-26 21:23:25 +00:00
|
|
|
type=str, metavar='FILE')
|
2021-05-26 21:24:06 +00:00
|
|
|
|
2022-05-18 17:01:24 +00:00
|
|
|
parser.add_argument('--qemu_args',
|
|
|
|
help='Additional QEMU arguments, e.g. "-smp 8"',
|
|
|
|
action='append', metavar='')
|
|
|
|
|
2023-03-16 22:06:38 +00:00
|
|
|
def add_build_opts(parser: argparse.ArgumentParser) -> None:
|
2020-05-01 04:27:01 +00:00
|
|
|
parser.add_argument('--jobs',
|
|
|
|
help='As in the make command, "Specifies the number of '
|
|
|
|
'jobs (commands) to run simultaneously."',
|
2022-02-26 21:23:25 +00:00
|
|
|
type=int, default=get_default_jobs(), metavar='N')
|
2020-05-01 04:27:01 +00:00
|
|
|
|
2023-03-16 22:06:38 +00:00
|
|
|
def add_exec_opts(parser: argparse.ArgumentParser) -> None:
|
2020-05-01 04:27:01 +00:00
|
|
|
parser.add_argument('--timeout',
|
|
|
|
help='maximum number of seconds to allow for all tests '
|
|
|
|
'to run. This does not include time taken to build the '
|
|
|
|
'tests.',
|
|
|
|
type=int,
|
|
|
|
default=300,
|
2022-02-26 21:23:25 +00:00
|
|
|
metavar='SECONDS')
|
2021-02-06 00:08:53 +00:00
|
|
|
parser.add_argument('filter_glob',
|
2021-09-14 21:03:48 +00:00
|
|
|
help='Filter which KUnit test suites/tests run at '
|
|
|
|
'boot-time, e.g. list* or list*.*del_test',
|
2021-02-06 00:08:53 +00:00
|
|
|
type=str,
|
|
|
|
nargs='?',
|
|
|
|
default='',
|
|
|
|
metavar='filter_glob')
|
kunit: tool: Add command line interface to filter and report attributes
Add ability to kunit.py to filter attributes and report a list of tests
including attributes without running tests.
Add flag "--filter" to input filters on test attributes. Tests will be
filtered out if they do not match all inputted filters.
Example: --filter speed=slow (This filter would run only the tests that are
marked as slow)
Filters have operations: <, >, <=, >=, !=, and =. But note that the
characters < and > are often interpreted by the shell, so they may need to
be quoted or escaped.
Example: --filter "speed>slow" or --filter speed\>slow (This filter would
run only the tests that have the speed faster than slow.
Additionally, multiple filters can be used.
Example: --filter "speed=slow, module!=example" (This filter would run
only the tests that have the speed slow and are not in the "example"
module)
Note if the user wants to skip filtered tests instead of not
running/showing them use the "--filter_action=skip" flag instead.
Expose the output of kunit.action=list option with flag "--list_tests" to
output a list of tests. Additionally, add flag "--list_tests_attr" to
output a list of tests and their attributes. These flags are useful to see
tests and test attributes without needing to run tests.
Example of the output of "--list_tests_attr":
example
example.test_1
example.test_2
# example.test_2.speed: slow
This output includes a suite, example, with two test cases, test_1 and
test_2. And in this instance test_2 has been marked as slow.
Reviewed-by: David Gow <davidgow@google.com>
Signed-off-by: Rae Moar <rmoar@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2023-07-25 21:25:16 +00:00
|
|
|
parser.add_argument('--filter',
|
|
|
|
help='Filter KUnit tests with attributes, '
|
|
|
|
'e.g. module=example or speed>slow',
|
|
|
|
type=str,
|
|
|
|
default='')
|
|
|
|
parser.add_argument('--filter_action',
|
|
|
|
help='If set to skip, filtered tests will be skipped, '
|
|
|
|
'e.g. --filter_action=skip. Otherwise they will not run.',
|
|
|
|
type=str,
|
|
|
|
choices=['skip'])
|
2021-07-15 16:08:19 +00:00
|
|
|
parser.add_argument('--kernel_args',
|
|
|
|
help='Kernel command-line parameters. Maybe be repeated',
|
2022-02-26 21:23:25 +00:00
|
|
|
action='append', metavar='')
|
kunit: tool: support running each suite/test separately
The new --run_isolated flag makes the tool boot the kernel once per
suite or test, preventing leftover state from one suite to impact the
other. This can be useful as a starting point to debugging test
hermeticity issues.
Note: it takes a lot longer, so people should not use it normally.
Consider the following very simplified example:
bool disable_something_for_test = false;
void function_being_tested() {
...
if (disable_something_for_test) return;
...
}
static void test_before(struct kunit *test)
{
disable_something_for_test = true;
function_being_tested();
/* oops, we forgot to reset it back to false */
}
static void test_after(struct kunit *test)
{
/* oops, now "fixing" test_before can cause test_after to fail! */
function_being_tested();
}
Presented like this, the issues are obvious, but it gets a lot more
complicated to track down as the amount of test setup and helper
functions increases.
Another use case is memory corruption. It might not be surfaced as a
failure/crash in the test case or suite that caused it. I've noticed in
kunit's own unit tests, the 3rd suite after might be the one to finally
crash after an out-of-bounds write, for example.
Example usage:
Per suite:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite
...
Starting KUnit Kernel (1/7)...
============================================================
======== [PASSED] kunit_executor_test ========
....
Testing complete. 5 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/7)...
============================================================
======== [PASSED] kunit-try-catch-test ========
...
Per test:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=test
Starting KUnit Kernel (1/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] parse_filter_test
============================================================
Testing complete. 1 tests run. 0 failed. 0 crashed. 0 skipped.
Starting KUnit Kernel (2/23)...
============================================================
======== [PASSED] kunit_executor_test ========
[PASSED] filter_subsuite_test
...
It works with filters as well:
$ ./tools/testing/kunit/kunit.py run --kunitconfig=lib/kunit --run_isolated=suite example
...
Starting KUnit Kernel (1/1)...
============================================================
======== [PASSED] example ========
...
It also handles test filters, '*.*skip*' runs these 3 tests:
kunit_status.kunit_status_mark_skipped_test
example.example_skip_test
example.example_mark_skipped_test
Fixed up merge conflict between:
d8c23ead708b ("kunit: tool: better handling of quasi-bool args (--json, --raw_output)") and
6710951ee039 ("kunit: tool: support running each suite/test separately")
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Shuah Khan <skhan@linuxfoundation.org>
Signed-off-by: Daniel Latypov <dlatypov@google.com>
Reviewed-by: David Gow <davidgow@google.com>
Reviewed-by: Brendan Higgins <brendanhiggins@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2021-09-30 22:20:48 +00:00
|
|
|
parser.add_argument('--run_isolated', help='If set, boot the kernel for each '
|
|
|
|
'individual suite/test. This is can be useful for debugging '
|
|
|
|
'a non-hermetic test, one that might pass/fail based on '
|
|
|
|
'what ran before it.',
|
|
|
|
type=str,
|
2022-05-09 20:49:09 +00:00
|
|
|
choices=['suite', 'test'])
|
kunit: tool: Add command line interface to filter and report attributes
Add ability to kunit.py to filter attributes and report a list of tests
including attributes without running tests.
Add flag "--filter" to input filters on test attributes. Tests will be
filtered out if they do not match all inputted filters.
Example: --filter speed=slow (This filter would run only the tests that are
marked as slow)
Filters have operations: <, >, <=, >=, !=, and =. But note that the
characters < and > are often interpreted by the shell, so they may need to
be quoted or escaped.
Example: --filter "speed>slow" or --filter speed\>slow (This filter would
run only the tests that have the speed faster than slow.
Additionally, multiple filters can be used.
Example: --filter "speed=slow, module!=example" (This filter would run
only the tests that have the speed slow and are not in the "example"
module)
Note if the user wants to skip filtered tests instead of not
running/showing them use the "--filter_action=skip" flag instead.
Expose the output of kunit.action=list option with flag "--list_tests" to
output a list of tests. Additionally, add flag "--list_tests_attr" to
output a list of tests and their attributes. These flags are useful to see
tests and test attributes without needing to run tests.
Example of the output of "--list_tests_attr":
example
example.test_1
example.test_2
# example.test_2.speed: slow
This output includes a suite, example, with two test cases, test_1 and
test_2. And in this instance test_2 has been marked as slow.
Reviewed-by: David Gow <davidgow@google.com>
Signed-off-by: Rae Moar <rmoar@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2023-07-25 21:25:16 +00:00
|
|
|
parser.add_argument('--list_tests', help='If set, list all tests that will be '
|
|
|
|
'run.',
|
|
|
|
action='store_true')
|
|
|
|
parser.add_argument('--list_tests_attr', help='If set, list all tests and test '
|
|
|
|
'attributes.',
|
|
|
|
action='store_true')
|
2020-05-01 04:27:01 +00:00
|
|
|
|
2023-03-16 22:06:38 +00:00
|
|
|
def add_parse_opts(parser: argparse.ArgumentParser) -> None:
|
2022-11-21 19:55:26 +00:00
|
|
|
parser.add_argument('--raw_output', help='If set don\'t parse output from kernel. '
|
|
|
|
'By default, filters to just KUnit output. Use '
|
|
|
|
'--raw_output=all to show everything',
|
2022-02-26 21:23:25 +00:00
|
|
|
type=str, nargs='?', const='all', default=None, choices=['all', 'kunit'])
|
2020-08-11 21:27:56 +00:00
|
|
|
parser.add_argument('--json',
|
|
|
|
nargs='?',
|
2022-11-21 19:55:26 +00:00
|
|
|
help='Prints parsed test results as JSON to stdout or a file if '
|
|
|
|
'a filename is specified. Does nothing if --raw_output is set.',
|
2022-02-26 21:23:25 +00:00
|
|
|
type=str, const='stdout', default=None, metavar='FILE')
|
2019-09-23 09:02:43 +00:00
|
|
|
|
2022-05-16 19:47:29 +00:00
|
|
|
|
|
|
|
def tree_from_args(cli_args: argparse.Namespace) -> kunit_kernel.LinuxSourceTree:
|
|
|
|
"""Returns a LinuxSourceTree based on the user's arguments."""
|
2022-05-18 17:01:24 +00:00
|
|
|
# Allow users to specify multiple arguments in one string, e.g. '-smp 8'
|
|
|
|
qemu_args: List[str] = []
|
|
|
|
if cli_args.qemu_args:
|
|
|
|
for arg in cli_args.qemu_args:
|
|
|
|
qemu_args.extend(shlex.split(arg))
|
|
|
|
|
2022-09-02 20:22:48 +00:00
|
|
|
kunitconfigs = cli_args.kunitconfig if cli_args.kunitconfig else []
|
|
|
|
if cli_args.alltests:
|
|
|
|
# Prepend so user-specified options take prio if we ever allow
|
|
|
|
# --kunitconfig options to have differing options.
|
|
|
|
kunitconfigs = [kunit_kernel.ALL_TESTS_CONFIG_PATH] + kunitconfigs
|
|
|
|
|
2022-05-16 19:47:29 +00:00
|
|
|
return kunit_kernel.LinuxSourceTree(cli_args.build_dir,
|
2022-09-02 20:22:48 +00:00
|
|
|
kunitconfig_paths=kunitconfigs,
|
2022-05-16 19:47:29 +00:00
|
|
|
kconfig_add=cli_args.kconfig_add,
|
|
|
|
arch=cli_args.arch,
|
|
|
|
cross_compile=cli_args.cross_compile,
|
2022-05-18 17:01:24 +00:00
|
|
|
qemu_config_path=cli_args.qemu_config,
|
|
|
|
extra_qemu_args=qemu_args)
|
2022-05-16 19:47:29 +00:00
|
|
|
|
|
|
|
|
2023-03-16 22:06:38 +00:00
|
|
|
def run_handler(cli_args: argparse.Namespace) -> None:
|
2023-01-21 21:27:17 +00:00
|
|
|
if not os.path.exists(cli_args.build_dir):
|
|
|
|
os.mkdir(cli_args.build_dir)
|
|
|
|
|
|
|
|
linux = tree_from_args(cli_args)
|
|
|
|
request = KunitRequest(build_dir=cli_args.build_dir,
|
|
|
|
make_options=cli_args.make_options,
|
|
|
|
jobs=cli_args.jobs,
|
|
|
|
raw_output=cli_args.raw_output,
|
|
|
|
json=cli_args.json,
|
|
|
|
timeout=cli_args.timeout,
|
|
|
|
filter_glob=cli_args.filter_glob,
|
kunit: tool: Add command line interface to filter and report attributes
Add ability to kunit.py to filter attributes and report a list of tests
including attributes without running tests.
Add flag "--filter" to input filters on test attributes. Tests will be
filtered out if they do not match all inputted filters.
Example: --filter speed=slow (This filter would run only the tests that are
marked as slow)
Filters have operations: <, >, <=, >=, !=, and =. But note that the
characters < and > are often interpreted by the shell, so they may need to
be quoted or escaped.
Example: --filter "speed>slow" or --filter speed\>slow (This filter would
run only the tests that have the speed faster than slow.
Additionally, multiple filters can be used.
Example: --filter "speed=slow, module!=example" (This filter would run
only the tests that have the speed slow and are not in the "example"
module)
Note if the user wants to skip filtered tests instead of not
running/showing them use the "--filter_action=skip" flag instead.
Expose the output of kunit.action=list option with flag "--list_tests" to
output a list of tests. Additionally, add flag "--list_tests_attr" to
output a list of tests and their attributes. These flags are useful to see
tests and test attributes without needing to run tests.
Example of the output of "--list_tests_attr":
example
example.test_1
example.test_2
# example.test_2.speed: slow
This output includes a suite, example, with two test cases, test_1 and
test_2. And in this instance test_2 has been marked as slow.
Reviewed-by: David Gow <davidgow@google.com>
Signed-off-by: Rae Moar <rmoar@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2023-07-25 21:25:16 +00:00
|
|
|
filter=cli_args.filter,
|
|
|
|
filter_action=cli_args.filter_action,
|
2023-01-21 21:27:17 +00:00
|
|
|
kernel_args=cli_args.kernel_args,
|
kunit: tool: Add command line interface to filter and report attributes
Add ability to kunit.py to filter attributes and report a list of tests
including attributes without running tests.
Add flag "--filter" to input filters on test attributes. Tests will be
filtered out if they do not match all inputted filters.
Example: --filter speed=slow (This filter would run only the tests that are
marked as slow)
Filters have operations: <, >, <=, >=, !=, and =. But note that the
characters < and > are often interpreted by the shell, so they may need to
be quoted or escaped.
Example: --filter "speed>slow" or --filter speed\>slow (This filter would
run only the tests that have the speed faster than slow.
Additionally, multiple filters can be used.
Example: --filter "speed=slow, module!=example" (This filter would run
only the tests that have the speed slow and are not in the "example"
module)
Note if the user wants to skip filtered tests instead of not
running/showing them use the "--filter_action=skip" flag instead.
Expose the output of kunit.action=list option with flag "--list_tests" to
output a list of tests. Additionally, add flag "--list_tests_attr" to
output a list of tests and their attributes. These flags are useful to see
tests and test attributes without needing to run tests.
Example of the output of "--list_tests_attr":
example
example.test_1
example.test_2
# example.test_2.speed: slow
This output includes a suite, example, with two test cases, test_1 and
test_2. And in this instance test_2 has been marked as slow.
Reviewed-by: David Gow <davidgow@google.com>
Signed-off-by: Rae Moar <rmoar@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2023-07-25 21:25:16 +00:00
|
|
|
run_isolated=cli_args.run_isolated,
|
|
|
|
list_tests=cli_args.list_tests,
|
|
|
|
list_tests_attr=cli_args.list_tests_attr)
|
2023-01-21 21:27:17 +00:00
|
|
|
result = run_tests(linux, request)
|
|
|
|
if result.status != KunitStatus.SUCCESS:
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
|
2023-03-16 22:06:38 +00:00
|
|
|
def config_handler(cli_args: argparse.Namespace) -> None:
|
2023-01-21 21:27:17 +00:00
|
|
|
if cli_args.build_dir and (
|
|
|
|
not os.path.exists(cli_args.build_dir)):
|
|
|
|
os.mkdir(cli_args.build_dir)
|
|
|
|
|
|
|
|
linux = tree_from_args(cli_args)
|
|
|
|
request = KunitConfigRequest(build_dir=cli_args.build_dir,
|
|
|
|
make_options=cli_args.make_options)
|
|
|
|
result = config_tests(linux, request)
|
|
|
|
stdout.print_with_timestamp((
|
|
|
|
'Elapsed time: %.3fs\n') % (
|
|
|
|
result.elapsed_time))
|
|
|
|
if result.status != KunitStatus.SUCCESS:
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
|
2023-03-16 22:06:38 +00:00
|
|
|
def build_handler(cli_args: argparse.Namespace) -> None:
|
2023-01-21 21:27:17 +00:00
|
|
|
linux = tree_from_args(cli_args)
|
|
|
|
request = KunitBuildRequest(build_dir=cli_args.build_dir,
|
|
|
|
make_options=cli_args.make_options,
|
|
|
|
jobs=cli_args.jobs)
|
|
|
|
result = config_and_build_tests(linux, request)
|
|
|
|
stdout.print_with_timestamp((
|
|
|
|
'Elapsed time: %.3fs\n') % (
|
|
|
|
result.elapsed_time))
|
|
|
|
if result.status != KunitStatus.SUCCESS:
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
|
2023-03-16 22:06:38 +00:00
|
|
|
def exec_handler(cli_args: argparse.Namespace) -> None:
|
2023-01-21 21:27:17 +00:00
|
|
|
linux = tree_from_args(cli_args)
|
|
|
|
exec_request = KunitExecRequest(raw_output=cli_args.raw_output,
|
|
|
|
build_dir=cli_args.build_dir,
|
|
|
|
json=cli_args.json,
|
|
|
|
timeout=cli_args.timeout,
|
|
|
|
filter_glob=cli_args.filter_glob,
|
kunit: tool: Add command line interface to filter and report attributes
Add ability to kunit.py to filter attributes and report a list of tests
including attributes without running tests.
Add flag "--filter" to input filters on test attributes. Tests will be
filtered out if they do not match all inputted filters.
Example: --filter speed=slow (This filter would run only the tests that are
marked as slow)
Filters have operations: <, >, <=, >=, !=, and =. But note that the
characters < and > are often interpreted by the shell, so they may need to
be quoted or escaped.
Example: --filter "speed>slow" or --filter speed\>slow (This filter would
run only the tests that have the speed faster than slow.
Additionally, multiple filters can be used.
Example: --filter "speed=slow, module!=example" (This filter would run
only the tests that have the speed slow and are not in the "example"
module)
Note if the user wants to skip filtered tests instead of not
running/showing them use the "--filter_action=skip" flag instead.
Expose the output of kunit.action=list option with flag "--list_tests" to
output a list of tests. Additionally, add flag "--list_tests_attr" to
output a list of tests and their attributes. These flags are useful to see
tests and test attributes without needing to run tests.
Example of the output of "--list_tests_attr":
example
example.test_1
example.test_2
# example.test_2.speed: slow
This output includes a suite, example, with two test cases, test_1 and
test_2. And in this instance test_2 has been marked as slow.
Reviewed-by: David Gow <davidgow@google.com>
Signed-off-by: Rae Moar <rmoar@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2023-07-25 21:25:16 +00:00
|
|
|
filter=cli_args.filter,
|
|
|
|
filter_action=cli_args.filter_action,
|
2023-01-21 21:27:17 +00:00
|
|
|
kernel_args=cli_args.kernel_args,
|
kunit: tool: Add command line interface to filter and report attributes
Add ability to kunit.py to filter attributes and report a list of tests
including attributes without running tests.
Add flag "--filter" to input filters on test attributes. Tests will be
filtered out if they do not match all inputted filters.
Example: --filter speed=slow (This filter would run only the tests that are
marked as slow)
Filters have operations: <, >, <=, >=, !=, and =. But note that the
characters < and > are often interpreted by the shell, so they may need to
be quoted or escaped.
Example: --filter "speed>slow" or --filter speed\>slow (This filter would
run only the tests that have the speed faster than slow.
Additionally, multiple filters can be used.
Example: --filter "speed=slow, module!=example" (This filter would run
only the tests that have the speed slow and are not in the "example"
module)
Note if the user wants to skip filtered tests instead of not
running/showing them use the "--filter_action=skip" flag instead.
Expose the output of kunit.action=list option with flag "--list_tests" to
output a list of tests. Additionally, add flag "--list_tests_attr" to
output a list of tests and their attributes. These flags are useful to see
tests and test attributes without needing to run tests.
Example of the output of "--list_tests_attr":
example
example.test_1
example.test_2
# example.test_2.speed: slow
This output includes a suite, example, with two test cases, test_1 and
test_2. And in this instance test_2 has been marked as slow.
Reviewed-by: David Gow <davidgow@google.com>
Signed-off-by: Rae Moar <rmoar@google.com>
Signed-off-by: Shuah Khan <skhan@linuxfoundation.org>
2023-07-25 21:25:16 +00:00
|
|
|
run_isolated=cli_args.run_isolated,
|
|
|
|
list_tests=cli_args.list_tests,
|
|
|
|
list_tests_attr=cli_args.list_tests_attr)
|
2023-01-21 21:27:17 +00:00
|
|
|
result = exec_tests(linux, exec_request)
|
|
|
|
stdout.print_with_timestamp((
|
|
|
|
'Elapsed time: %.3fs\n') % (result.elapsed_time))
|
|
|
|
if result.status != KunitStatus.SUCCESS:
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
|
2023-03-16 22:06:38 +00:00
|
|
|
def parse_handler(cli_args: argparse.Namespace) -> None:
|
2023-01-21 21:27:17 +00:00
|
|
|
if cli_args.file is None:
|
2023-03-16 22:06:38 +00:00
|
|
|
sys.stdin.reconfigure(errors='backslashreplace') # type: ignore
|
|
|
|
kunit_output = sys.stdin # type: Iterable[str]
|
2023-01-21 21:27:17 +00:00
|
|
|
else:
|
|
|
|
with open(cli_args.file, 'r', errors='backslashreplace') as f:
|
|
|
|
kunit_output = f.read().splitlines()
|
|
|
|
# We know nothing about how the result was created!
|
|
|
|
metadata = kunit_json.Metadata()
|
|
|
|
request = KunitParseRequest(raw_output=cli_args.raw_output,
|
|
|
|
json=cli_args.json)
|
|
|
|
result, _ = parse_tests(request, metadata, kunit_output)
|
|
|
|
if result.status != KunitStatus.SUCCESS:
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
|
|
|
|
subcommand_handlers_map = {
|
|
|
|
'run': run_handler,
|
|
|
|
'config': config_handler,
|
|
|
|
'build': build_handler,
|
|
|
|
'exec': exec_handler,
|
|
|
|
'parse': parse_handler
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2023-03-16 22:06:38 +00:00
|
|
|
def main(argv: Sequence[str]) -> None:
|
2019-09-23 09:02:43 +00:00
|
|
|
parser = argparse.ArgumentParser(
|
|
|
|
description='Helps writing and running KUnit tests.')
|
|
|
|
subparser = parser.add_subparsers(dest='subcommand')
|
|
|
|
|
2020-05-01 04:27:01 +00:00
|
|
|
# The 'run' command will config, build, exec, and parse in one go.
|
2019-09-23 09:02:43 +00:00
|
|
|
run_parser = subparser.add_parser('run', help='Runs KUnit tests.')
|
2020-05-01 04:27:01 +00:00
|
|
|
add_common_opts(run_parser)
|
|
|
|
add_build_opts(run_parser)
|
|
|
|
add_exec_opts(run_parser)
|
|
|
|
add_parse_opts(run_parser)
|
|
|
|
|
|
|
|
config_parser = subparser.add_parser('config',
|
|
|
|
help='Ensures that .config contains all of '
|
|
|
|
'the options in .kunitconfig')
|
|
|
|
add_common_opts(config_parser)
|
|
|
|
|
|
|
|
build_parser = subparser.add_parser('build', help='Builds a kernel with KUnit tests')
|
|
|
|
add_common_opts(build_parser)
|
|
|
|
add_build_opts(build_parser)
|
|
|
|
|
|
|
|
exec_parser = subparser.add_parser('exec', help='Run a kernel with KUnit tests')
|
|
|
|
add_common_opts(exec_parser)
|
|
|
|
add_exec_opts(exec_parser)
|
|
|
|
add_parse_opts(exec_parser)
|
|
|
|
|
|
|
|
# The 'parse' option is special, as it doesn't need the kernel source
|
|
|
|
# (therefore there is no need for a build_dir, hence no add_common_opts)
|
|
|
|
# and the '--file' argument is not relevant to 'run', so isn't in
|
|
|
|
# add_parse_opts()
|
|
|
|
parse_parser = subparser.add_parser('parse',
|
|
|
|
help='Parses KUnit results from a file, '
|
|
|
|
'and parses formatted results.')
|
|
|
|
add_parse_opts(parse_parser)
|
|
|
|
parse_parser.add_argument('file',
|
|
|
|
help='Specifies the file to read results from.',
|
|
|
|
type=str, nargs='?', metavar='input_file')
|
2020-03-23 19:04:59 +00:00
|
|
|
|
2021-09-22 16:39:21 +00:00
|
|
|
cli_args = parser.parse_args(massage_argv(argv))
|
2019-09-23 09:02:43 +00:00
|
|
|
|
2020-08-11 21:27:55 +00:00
|
|
|
if get_kernel_root_path():
|
|
|
|
os.chdir(get_kernel_root_path())
|
|
|
|
|
2023-01-21 21:27:17 +00:00
|
|
|
subcomand_handler = subcommand_handlers_map.get(cli_args.subcommand, None)
|
|
|
|
|
|
|
|
if subcomand_handler is None:
|
2019-09-23 09:02:43 +00:00
|
|
|
parser.print_help()
|
2023-01-21 21:27:17 +00:00
|
|
|
return
|
|
|
|
|
|
|
|
subcomand_handler(cli_args)
|
|
|
|
|
2019-09-23 09:02:43 +00:00
|
|
|
|
|
|
|
if __name__ == '__main__':
|
2019-09-23 09:02:44 +00:00
|
|
|
main(sys.argv[1:])
|