From 2eaa8a4a1b2c50bf152d12565aacb440b75ce877 Mon Sep 17 00:00:00 2001 From: ant31 <2t.antoine@gmail.com> Date: Thu, 24 Nov 2016 15:18:22 +0100 Subject: [PATCH] Add pytest and tox to run tests --- .dockerignore | 5 +++++ .gitignore | 6 ++++++ Dockerfile | 12 ++++++++++-- README.md | 42 +++++++++++++++++++++++++++++++++++++++++- requirements-tests.txt | 6 ++++++ setup.cfg | 11 +++++++++++ test/test_secscan.py | 4 ++-- tox.ini | 17 +++++++++++++++++ 8 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 requirements-tests.txt create mode 100644 setup.cfg create mode 100644 tox.ini diff --git a/.dockerignore b/.dockerignore index b4d35e3b9..cd4ba187b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,3 +13,8 @@ requirements-nover.txt run-local.sh .DS_Store *.pyc +.tox +htmlcov +.coverage +.cache +test/__pycache__ diff --git a/.gitignore b/.gitignore index f808caee1..e2aa3f78b 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,9 @@ test/data/registry/ typings GIT_HEAD .idea +.python-version +.pylintrc +.coverage +htmlcov +.tox +.cache diff --git a/Dockerfile b/Dockerfile index 57a62f46d..9724c846c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -142,13 +142,21 @@ ADD . . ARG RUN_TESTS=true ENV RUN_TESTS ${RUN_TESTS} ENV RUN_ACI_TESTS False +ADD requirements-tests.txt requirements-tests.txt + RUN if [ "$RUN_TESTS" = true ]; then \ - TEST=true venv/bin/python -m unittest discover -f; \ + venv/bin/pip install -r requirements-tests.txt ;\ fi + RUN if [ "$RUN_TESTS" = true ]; then \ - TEST=true venv/bin/python -m test.registry_tests -f; \ + TEST=true PYTHONPATH="." venv/bin/py.test --verbose --show-count -x --color=no test ; \ fi + +RUN if [ "$RUN_TESTS" = true ]; then \ + TEST=true PYTHONPATH="." venv/bin/py.test --verbose --show-count -x --color=no test/registry_tests.py ; \ + fi + RUN PYTHONPATH=. venv/bin/alembic heads | grep -E '^[0-9a-f]+ \(head\)$' > ALEMBIC_HEAD VOLUME ["/conf/stack", "/var/log", "/datastorage", "/tmp", "/conf/etcd"] diff --git a/README.md b/README.md index a73c2a815..fc980fddf 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,47 @@ TEST=true python -m test.test_api_usage -f # To run a specific test in a suite TEST=true python -m test.test_api_usage -f SuiteName ``` +#### Pytest +``` +# To run all tests +TEST=true PYTHONPATH="." py.test --verbose test/ + +# To run a specific test module +TEST=true PYTHONPATH="." py.test --verbose test/registry_tests.py + +# To run a specific test unique test +TEST=true PYTHONPATH="." py.test --verbose test/test_api_usage.py::TestDeleteNamespace + +# To retry only last failed (--lf): +TEST=true PYTHONPATH="." py.test --verbose --lf + +# To start pdb on failure: +TEST=true PYTHONPATH="." py.test --verbose --pdb + +# To run a coverage report (html pages in ./htmlcov): +TEST=true PYTHONPATH="." py.test --cov="." --cov-report=html --cov-report=term-missing --cov-config=.coverage.ini --verbose + +# Don't capture stdout (-s) +TEST=true PYTHONPATH="." py.test --verbose -s + +``` + +#### Tox +To create a virtualenv to run the tests. +It allows to test the code on multiple env like python2.x and python3.x or different library versions + +``` +# Test all tox env: +tox + +# Add extra parameters to the pytest command: +# tox -- [pytest ARGS] +tox -- -x + +# build a single env with -e: +tox -e py27-api +``` ### Running migrations @@ -193,7 +233,7 @@ TEST=true python -m test.test_api_usage -f SuiteName PYTHONPATH=. alembic upgrade head # You can also rebuild your local sqlite db image from initdb.py using -# And once you have a migration you should do this and check in the +# And once you have a migration you should do this and check in the # changes to share your migration with others. rm test/data/test.db python initdb.py diff --git a/requirements-tests.txt b/requirements-tests.txt new file mode 100644 index 000000000..4704613ab --- /dev/null +++ b/requirements-tests.txt @@ -0,0 +1,6 @@ +pytest +pytest-cov +python-coveralls +pytest-flask +pytest-runner +-e git+https://github.com/ant31/pytest-sugar.git#egg=pytest-sugar diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..7d9e70054 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,11 @@ +[tool:pytest] +norecursedirs = .* *.egg build dist docs +testpaths = test +confcutdir = test + +[coverage:run] +branch = True + +[coverage:report] +omit = + test/* diff --git a/test/test_secscan.py b/test/test_secscan.py index 67234bb2e..4a5fc44db 100644 --- a/test/test_secscan.py +++ b/test/test_secscan.py @@ -458,7 +458,7 @@ class TestSecurityScanner(unittest.TestCase): self.assertIsNotNone(queue_item) body = json.loads(queue_item.body) - self.assertEquals(['prod', 'latest'], body['event_data']['tags']) + self.assertEquals(sorted(['prod', 'latest']), sorted(body['event_data']['tags'])) self.assertEquals('CVE-TEST', body['event_data']['vulnerability']['id']) self.assertEquals('Low', body['event_data']['vulnerability']['priority']) self.assertTrue(body['event_data']['vulnerability']['has_fix']) @@ -530,7 +530,7 @@ class TestSecurityScanner(unittest.TestCase): self.assertIsNotNone(queue_item) body = json.loads(queue_item.body) - self.assertEquals(['prod', 'latest'], body['event_data']['tags']) + self.assertEquals(sorted(['prod', 'latest']), sorted(body['event_data']['tags'])) self.assertEquals('CVE-TEST', body['event_data']['vulnerability']['id']) self.assertEquals('Critical', body['event_data']['vulnerability']['priority']) self.assertTrue(body['event_data']['vulnerability']['has_fix']) diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..554cf9cf4 --- /dev/null +++ b/tox.ini @@ -0,0 +1,17 @@ +[tox] +envlist = {py27}-{unittest,api} +skipsdist = True + +[testenv] +basepython = + py27: python2.7 +deps = + -r{toxinidir}/requirements-tests.txt + -r{toxinidir}/requirements.txt +setenv = + PYTHONPATH = {toxinidir}:{toxinidir} + TEST=true + api: FILE="registry_tests.py" + unittest: FILE="" +commands = + py.test --verbose test/{env:FILE} -vv {posargs}