Merge pull request #2160 from ant31/use_pytest

Add pytest, tox  and code-coverage to run tests

.dockerignore

@@ -13,3 +13,8 @@ requirements-nover.txt
 run-local.sh
 .DS_Store
 *.pyc
+.tox
+htmlcov
+.coverage
+.cache
+test/__pycache__

.gitignore

@@ -14,3 +14,9 @@ test/data/registry/
 typings
 GIT_HEAD
 .idea
+.python-version
+.pylintrc
+.coverage
+htmlcov
+.tox
+.cache

Dockerfile

@@ -142,13 +142,21 @@ ADD . .
 ARG RUN_TESTS=true
 ENV RUN_TESTS ${RUN_TESTS}
 ENV RUN_ACI_TESTS False
+ADD requirements-tests.txt requirements-tests.txt
+
 
 RUN if [ "$RUN_TESTS" = true ]; then \
-      TEST=true venv/bin/python -m unittest discover -f; \
+      venv/bin/pip install -r requirements-tests.txt ;\
     fi
+
 RUN if [ "$RUN_TESTS" = true ]; then \
-      TEST=true venv/bin/python -m test.registry_tests -f; \
+      TEST=true PYTHONPATH="." venv/bin/py.test --verbose --show-count -x --color=no test ; \
     fi
+
+RUN if [ "$RUN_TESTS" = true ]; then \
+      TEST=true PYTHONPATH="." venv/bin/py.test --verbose --show-count -x --color=no test/registry_tests.py ; \
+    fi
+
 RUN PYTHONPATH=. venv/bin/alembic heads | grep -E '^[0-9a-f]+ \(head\)$' > ALEMBIC_HEAD
 
 VOLUME ["/conf/stack", "/var/log", "/datastorage", "/tmp", "/conf/etcd"]

README.md

@@ -175,7 +175,47 @@ TEST=true python -m test.test_api_usage -f
 # To run a specific test in a suite
 TEST=true python -m test.test_api_usage -f SuiteName
 ```
+#### Pytest
 
+```
+# To run all tests
+TEST=true PYTHONPATH="." py.test --verbose test/
+
+# To run a specific test module
+TEST=true PYTHONPATH="." py.test --verbose test/registry_tests.py
+
+# To run a specific test unique test
+TEST=true PYTHONPATH="." py.test --verbose test/test_api_usage.py::TestDeleteNamespace
+
+# To retry only last failed (--lf):
+TEST=true PYTHONPATH="." py.test --verbose --lf
+
+# To start pdb on failure:
+TEST=true PYTHONPATH="." py.test --verbose --pdb
+
+# To run a coverage report (html pages in ./htmlcov):
+TEST=true PYTHONPATH="." py.test --cov="." --cov-report=html --cov-report=term-missing  --cov-config=.coverage.ini --verbose
+
+# Don't capture stdout (-s)
+TEST=true PYTHONPATH="." py.test --verbose -s
+
+```
+
+#### Tox
+To create a virtualenv to run the tests.
+It allows to test the code on multiple env like python2.x and python3.x or different library versions
+
+```
+# Test all tox env:
+tox
+
+# Add extra parameters to the pytest command:
+# tox -- [pytest ARGS]
+tox -- -x
+
+# build a single env with -e:
+tox -e py27-api
+```
 
 ### Running migrations
 
@@ -193,7 +233,7 @@ TEST=true python -m test.test_api_usage -f SuiteName
 PYTHONPATH=. alembic upgrade head
 
 # You can also rebuild your local sqlite db image from initdb.py using
-# And once you have a migration you should do this and check in the 
+# And once you have a migration you should do this and check in the
 # changes to share your migration with others.
 rm test/data/test.db
 python initdb.py

requirements-tests.txt

@@ -0,0 +1,6 @@
+pytest
+pytest-cov
+python-coveralls
+pytest-flask
+pytest-runner
+-e git+https://github.com/ant31/pytest-sugar.git#egg=pytest-sugar

setup.cfg

@@ -0,0 +1,11 @@
+[tool:pytest]
+norecursedirs = .* *.egg build dist docs
+testpaths = test
+confcutdir = test
+
+[coverage:run]
+branch = True
+
+[coverage:report]
+omit =
+     test/*

test/test_secscan.py

@@ -458,7 +458,7 @@ class TestSecurityScanner(unittest.TestCase):
     self.assertIsNotNone(queue_item)
 
     body = json.loads(queue_item.body)
-    self.assertEquals(['prod', 'latest'], body['event_data']['tags'])
+    self.assertEquals(sorted(['prod', 'latest']), sorted(body['event_data']['tags']))
     self.assertEquals('CVE-TEST', body['event_data']['vulnerability']['id'])
     self.assertEquals('Low', body['event_data']['vulnerability']['priority'])
     self.assertTrue(body['event_data']['vulnerability']['has_fix'])
@@ -530,7 +530,7 @@ class TestSecurityScanner(unittest.TestCase):
     self.assertIsNotNone(queue_item)
 
     body = json.loads(queue_item.body)
-    self.assertEquals(['prod', 'latest'], body['event_data']['tags'])
+    self.assertEquals(sorted(['prod', 'latest']), sorted(body['event_data']['tags']))
     self.assertEquals('CVE-TEST', body['event_data']['vulnerability']['id'])
     self.assertEquals('Critical', body['event_data']['vulnerability']['priority'])
     self.assertTrue(body['event_data']['vulnerability']['has_fix'])

tox.ini

@@ -0,0 +1,17 @@
+[tox]
+envlist = {py27}-{unittest,api}
+skipsdist = True
+
+[testenv]
+basepython =
+    py27: python2.7
+deps =
+     -r{toxinidir}/requirements-tests.txt
+     -r{toxinidir}/requirements.txt
+setenv =
+    PYTHONPATH = {toxinidir}:{toxinidir}
+    TEST=true
+    api: FILE="registry_tests.py"
+    unittest: FILE=""
+commands =
+         py.test --verbose test/{env:FILE} -vv {posargs}