diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..8b1085fa3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,49 @@ +--- +language: python + +sudo: required + +services: + - docker + +install: true + +# Stop default database instances here to avoid port conflicts. +before_script: + - sudo service mysql stop + - sudo service postgresql stop + +cache: + directories: + - $HOME/docker + +stages: + - build + - test + - database + - clean + +# We should label the steps if Travis ever supports it: +# https://github.com/travis-ci/travis-ci/issues/5898 +jobs: + include: + - stage: build + script: scripts/ci build + + - stage: test + script: scripts/ci unit + + - stage: test + script: scripts/ci registry + + - stage: test + script: scripts/ci registry_old + + - stage: database + script: scripts/ci mysql + + - stage: database + script: scripts/ci postgres + + - stage: clean + script: scripts/ci clean diff --git a/Makefile b/Makefile index a0f187ce3..50f8109c7 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ SHELL := /bin/bash +export PATH := ./venv/bin:$(PATH) + SHA := $(shell git rev-parse --short HEAD ) REPO := quay.io/quay/quay TAG := $(REPO):$(SHA) @@ -11,7 +13,7 @@ MODIFIED_FILES = $(shell git diff --name-only $(GIT_MERGE_BASED) | grep -E .+\.p show-modified: echo $(MODIFIED_FILES) -.PHONY: all unit test pkgs build run clean +.PHONY: all unit-test registry-test registry-test-old test pkgs build run clean all: clean pkgs test build @@ -52,8 +54,25 @@ registry-test: --timeout=3600 --verbose --show-count -x \ test/registry/registry_tests.py +registry-test-old: + TEST=true PYTHONPATH="." py.test \ + --cov="." --cov-report=html --cov-report=term-missing \ + --timeout=3600 --verbose --show-count -x \ + ./test/registry_tests.py + +full-db-test: ensure-test-db + TEST=true PYTHONPATH=. alembic upgrade head + TEST=true PYTHONPATH=. SKIP_DB_SCHEMA=true py.test --timeout=7200 \ + --verbose --show-count -x --ignore=endpoints/appr/test/ \ + ./ + test: unit-test registry-test +ensure-test-db: + @if [ -z $(TEST_DATABASE_URI) ]; then \ + echo "TEST_DATABASE_URI is undefined"; \ + exit 1; \ + fi PG_PASSWORD := quay PG_USER := quay @@ -133,13 +152,6 @@ yapf-all: yapf-diff: if [ $(MODIFIED_FILES_COUNT) -ne 0 ]; then yapf -d -p $(MODIFIED_FILES) ; fi -yapf: -ifneq (0,$(shell git diff-index HEAD | wc -l)) - echo "Failed, git dirty" && false -else ifneq (0,$(shell yapf -d -p $(MODIFIED_FILES) | wc -l)) - yapf -i -p $(MODIFIED_FILES) - git commit -a -m "code-stye Yapf: $(MODIFIED_FILES_COUNT) files updated" -m "$(MODIFIED_FILES)" -endif yapf-test: if [ `yapf -d -p $(MODIFIED_FILES) | wc -l` -gt 0 ] ; then false ; else true ;fi diff --git a/requirements-tests.txt b/requirements-tests.txt index 435f7c1fe..bb613f9cc 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,3 +1,4 @@ +pytest pytest-cov python-coveralls pytest-flask @@ -5,4 +6,3 @@ pytest-runner pytest-xdist pytest-timeout -e git+https://github.com/ant31/pytest-sugar.git#egg=pytest-sugar --e git+https://github.com/ant31/pytest.git#egg=pytest diff --git a/scripts/ci b/scripts/ci new file mode 100755 index 000000000..681e76faa --- /dev/null +++ b/scripts/ci @@ -0,0 +1,159 @@ +#!/bin/bash + +set -euo pipefail + + +IMAGE="quay-ci" + +CACHE_DIR="${HOME}/docker" +SHORT_SHA="${TRAVIS_COMMIT:0:7}" +IMAGE_TAG="${SHORT_SHA}-${TRAVIS_BUILD_NUMBER}" +IMAGE_TAR="${CACHE_DIR}/${IMAGE}-${IMAGE_TAG}.tar.gz" + +MYSQL_IMAGE="mysql:5.6" +POSTGRES_IMAGE="postgres:9.6" + +export MYSQL_ROOT_PASSWORD="quay" +export MYSQL_USER="quay" +export MYSQL_PASSWORD="quay" +export MYSQL_DATABASE="quay_ci" + +export POSTGRES_USER="quay" +export POSTGRES_PASSWORD="quay" +export POSTGRES_DB="quay_ci" + + +build_image() { + # Build the image and save it to the shared cache. + docker build -t "${IMAGE}:${IMAGE_TAG}" . + + echo "Exporting Docker image to cache..." + time (docker save "${IMAGE}:${IMAGE_TAG}" | gzip -2 > "${IMAGE_TAR}") +} + + +load_image() { + # Load our cached Docker image. + echo "Loading Docker image from cache..." + time (zcat "${IMAGE_TAR}" | docker load) +} + + +clean_cache() { + rm "${IMAGE_TAR}" +} + + +quay_run() { + docker run --net=host -e TEST_DATABASE_URI -ti "${IMAGE}:${IMAGE_TAG}" "$@" +} + + +unit() { + load_image && quay_run make unit-test +} + + +registry() { + load_image && quay_run make registry-test +} + + +registry_old() { + load_image && quay_run make registry-test-old +} + + +mysql_ping() { + mysqladmin --connect-timeout=2 --wait=30 --host=127.0.0.1 \ + --user=root --password="${MYSQL_ROOT_PASSWORD}" ping +} + + +mysql_start() { + docker run --net=host -d -e MYSQL_ROOT_PASSWORD -e MYSQL_USER \ + -e MYSQL_PASSWORD -e MYSQL_DATABASE "${MYSQL_IMAGE}" + + if ! (sleep 10 && mysql_ping); then + echo "MySQL failed to respond in time." + exit 1 + fi +} + + +mysql() { + TEST_DATABASE_URI="mysql+pymysql://" + TEST_DATABASE_URI+="${MYSQL_USER}:${MYSQL_PASSWORD}" + TEST_DATABASE_URI+="@127.0.0.1/${MYSQL_DATABASE}" + + export TEST_DATABASE_URI + + load_image + mysql_start + quay_run make full-db-test +} + + +postgres_ping() { + pg_isready --timeout=30 --dbname="${TEST_DATABASE_URI}" +} + + +postgres_start() { + docker run --net=host -d -e POSTGRES_USER -e POSTGRES_PASSWORD \ + -e POSTGRES_DB "${POSTGRES_IMAGE}" + + if ! (sleep 10 && postgres_ping); then + echo "PostgreSQL failed to respond in time." + exit 1 + fi +} + + +postgres() { + TEST_DATABASE_URI="postgresql://" + TEST_DATABASE_URI+="${POSTGRES_USER}:${POSTGRES_PASSWORD}" + TEST_DATABASE_URI+="@127.0.0.1/${POSTGRES_DB}" + + export TEST_DATABASE_URI + + load_image + postgres_start + quay_run make full-db-test +} + + +case "$1" in + build) + build_image + ;; + + unit) + unit + ;; + + registry) + registry + ;; + + registry_old) + registry_old + ;; + + mysql) + mysql + ;; + + postgres) + postgres + ;; + + clean) + clean_cache + ;; + + *) + echo "Usage: $0 {build|unit|registry|registry_old|mysql|postgres|clean}" + exit 1 + ;; +esac