Merge pull request #3067 from bison/travis-ci

Add Travis CI build configuration

.travis.yml

@@ -0,0 +1,49 @@
+---
+language: python
+
+sudo: required
+
+services:
+  - docker
+
+install: true
+
+# Stop default database instances here to avoid port conflicts.
+before_script:
+  - sudo service mysql stop
+  - sudo service postgresql stop
+
+cache:
+  directories:
+    - $HOME/docker
+
+stages:
+  - build
+  - test
+  - database
+  - clean
+
+# We should label the steps if Travis ever supports it:
+# https://github.com/travis-ci/travis-ci/issues/5898
+jobs:
+  include:
+    - stage: build
+      script: scripts/ci build
+
+    - stage: test
+      script: scripts/ci unit
+
+    - stage: test
+      script: scripts/ci registry
+
+    - stage: test
+      script: scripts/ci registry_old
+
+    - stage: database
+      script: scripts/ci mysql
+
+    - stage: database
+      script: scripts/ci postgres
+
+    - stage: clean
+      script: scripts/ci clean

Dockerfile

@@ -1,13 +1,98 @@
 # vim:ft=dockerfile
 
-FROM quay.io/quay/quay-base:latest
+FROM phusion/baseimage:0.10.0
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV HOME /root
+ENV QUAYDIR /quay-registry
+ENV QUAYCONF /quay-registry/conf
+ENV QUAYPATH "."
+
+RUN mkdir $QUAYDIR
+WORKDIR $QUAYDIR
+
+# This is so we don't break http golang/go#17066
+# When Ubuntu has nginx >= 1.11.0 we can switch back.
+RUN  add-apt-repository ppa:nginx/development
+
+# Add Yarn repository until it is officially added to Ubuntu
+RUN curl -fsSL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
+    && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
+RUN curl -fsSL https://deb.nodesource.com/setup_8.x | bash -
+# Install system packages
+RUN apt-get update && apt-get upgrade -y \
+    && apt-get install -y \
+     dnsmasq           \
+     g++               \
+     gdb               \
+     gdebi-core        \
+     git               \
+     jpegoptim         \
+     libevent-2.0.5    \
+     libevent-dev      \
+     libffi-dev        \
+     libfreetype6-dev  \
+     libgpgme11        \
+     libgpgme11-dev    \
+     libjpeg62         \
+     libjpeg62-dev     \
+     libjpeg8          \
+     libldap-2.4-2     \
+     libldap2-dev      \
+     libmagic1         \
+     libpq-dev         \
+     libpq5            \
+     libsasl2-dev      \
+     libsasl2-modules  \
+     memcached         \
+     monit             \
+     nginx             \
+     nodejs            \
+     optipng           \
+     openssl           \
+     python-dbg        \
+     python-dev        \
+     python-pip        \
+     python-virtualenv \
+     yarn=0.22.0-1     \
+     w3m # 27MAR2018
+
+# Install cfssl
+RUN curl -fsSL -o /bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 \
+    && curl -fsSL -o /bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 \
+    && chmod 0755 /bin/cfssl /bin/cfssljson
+
+# Install jwtproxy
+RUN curl -fsSL -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 \
+    && chmod +x /usr/local/bin/jwtproxy
+
+# Install prometheus-aggregator
+RUN curl -fsSL -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator \
+    && chmod +x /usr/local/bin/prometheus-aggregator
+
+# Install python dependencies
+COPY requirements.txt requirements-tests.txt ./
+RUN virtualenv --distribute venv \
+    && venv/bin/pip install -r requirements.txt \
+    && venv/bin/pip install -r requirements-tests.txt \
+    && venv/bin/pip freeze
+
+# Install front-end dependencies
+COPY static/ package.json tsconfig.json webpack.config.js tslint.json yarn.lock ./
+RUN yarn install --ignore-engines
+
+
+RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit $QUAYDIR/static/fonts $QUAYDIR/static/ldn /usr/local/nginx/logs/
+
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache
+VOLUME ["/var/log", "/datastorage", "/tmp"]
+
+
 
 RUN adduser memcached --disabled-login --system
 
 WORKDIR $QUAYDIR
 
-COPY requirements.txt requirements-tests.txt ./
-
 # Check python dependencies for the GPL
 # Due to the following bug, pip results must be piped to a file before grepping:
 # https://github.com/pypa/pip/pull/3304
@@ -15,16 +100,6 @@ RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/
     test -z "$(cat pipinfo.txt | grep GPL | grep -v LGPL)" && \
     rm pipinfo.txt
 
-RUN virtualenv --distribute venv \
-    && venv/bin/pip install -r requirements.txt \
-    && venv/bin/pip install -r requirements-tests.txt \
-    && venv/bin/pip freeze
-
-# Install front-end dependencies
-# JS dependencies
-COPY yarn.lock package.json tsconfig.json webpack.config.js tslint.json ./
-RUN yarn install --ignore-engines
-
 # JS compile
 COPY static static
 RUN yarn build \
@@ -33,8 +108,10 @@ RUN yarn build \
 
 COPY . .
 
+RUN PYTHONPATH=$QUAYPATH venv/bin/python -m external_libraries
+
 # Update local copy of AWS IP Ranges.
-RUN curl https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json
+RUN curl -fsSL https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json
 
 # Set up the init system
 RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit static/fonts static/ldn /usr/local/nginx/logs/ \
@@ -59,4 +136,6 @@ RUN PYTHONPATH=$QUAYPATH venv/bin/alembic heads | grep -E '^[0-9a-f]+ \(head\)$'
 
 RUN ./scripts/detect-config.sh
 
+EXPOSE 443 8443 80
+
 CMD ./quay-entrypoint.sh

Makefile

@@ -1,5 +1,7 @@
 SHELL := /bin/bash
 
+export PATH := ./venv/bin:$(PATH)
+
 SHA  := $(shell git rev-parse --short HEAD )
 REPO := quay.io/quay/quay
 TAG  := $(REPO):$(SHA)
@@ -11,7 +13,7 @@ MODIFIED_FILES = $(shell git diff --name-only $(GIT_MERGE_BASED) | grep -E .+\.p
 show-modified:
 	echo $(MODIFIED_FILES)
 
-.PHONY: all unit test pkgs build run clean
+.PHONY: all unit-test registry-test registry-test-old test pkgs build run clean
 
 all: clean pkgs test build
 
@@ -52,8 +54,25 @@ registry-test:
 	--timeout=3600 --verbose --show-count -x \
 	test/registry/registry_tests.py
 
+registry-test-old:
+	TEST=true PYTHONPATH="." py.test  \
+	--cov="." --cov-report=html --cov-report=term-missing \
+	--timeout=3600 --verbose --show-count -x \
+	./test/registry_tests.py
+
+full-db-test: ensure-test-db
+	TEST=true PYTHONPATH=. alembic upgrade head
+	TEST=true PYTHONPATH=. SKIP_DB_SCHEMA=true py.test --timeout=7200 \
+	--verbose --show-count -x --ignore=endpoints/appr/test/ \
+	./
+
 test: unit-test registry-test
 
+ensure-test-db:
+	@if [ -z $(TEST_DATABASE_URI) ]; then \
+	  echo "TEST_DATABASE_URI is undefined"; \
+	  exit 1; \
+	fi
 
 PG_PASSWORD := quay
 PG_USER := quay
@@ -133,13 +152,6 @@ yapf-all:
 yapf-diff:
 	if [ $(MODIFIED_FILES_COUNT) -ne 0 ]; then yapf -d -p $(MODIFIED_FILES) ; fi
 
-yapf:
-ifneq (0,$(shell git diff-index HEAD | wc -l))
-	echo "Failed, git dirty" && false
-else ifneq (0,$(shell yapf -d -p $(MODIFIED_FILES) | wc -l))
-	yapf -i -p $(MODIFIED_FILES)
-	git commit -a -m "code-stye Yapf: $(MODIFIED_FILES_COUNT) files updated" -m "$(MODIFIED_FILES)"
-endif
 
 yapf-test:
 	if [ `yapf -d -p $(MODIFIED_FILES) | wc -l` -gt 0 ] ; then false ; else true ;fi

requirements-tests.txt

@@ -1,3 +1,4 @@
+pytest
 pytest-cov
 python-coveralls
 pytest-flask
@@ -5,4 +6,3 @@ pytest-runner
 pytest-xdist
 pytest-timeout
 -e git+https://github.com/ant31/pytest-sugar.git#egg=pytest-sugar
--e git+https://github.com/ant31/pytest.git#egg=pytest

scripts/ci

@@ -0,0 +1,159 @@
+#!/bin/bash
+
+set -euo pipefail
+
+
+IMAGE="quay-ci"
+
+CACHE_DIR="${HOME}/docker"
+SHORT_SHA="${TRAVIS_COMMIT:0:7}"
+IMAGE_TAG="${SHORT_SHA}-${TRAVIS_BUILD_NUMBER}"
+IMAGE_TAR="${CACHE_DIR}/${IMAGE}-${IMAGE_TAG}.tar.gz"
+
+MYSQL_IMAGE="mysql:5.6"
+POSTGRES_IMAGE="postgres:9.6"
+
+export MYSQL_ROOT_PASSWORD="quay"
+export MYSQL_USER="quay"
+export MYSQL_PASSWORD="quay"
+export MYSQL_DATABASE="quay_ci"
+
+export POSTGRES_USER="quay"
+export POSTGRES_PASSWORD="quay"
+export POSTGRES_DB="quay_ci"
+
+
+build_image() {
+    # Build the image and save it to the shared cache.
+    docker build -t "${IMAGE}:${IMAGE_TAG}" .
+
+    echo "Exporting Docker image to cache..."
+    time (docker save "${IMAGE}:${IMAGE_TAG}" | gzip -2 > "${IMAGE_TAR}")
+}
+
+
+load_image() {
+    # Load our cached Docker image.
+    echo "Loading Docker image from cache..."
+    time (zcat "${IMAGE_TAR}" | docker load)
+}
+
+
+clean_cache() {
+    rm "${IMAGE_TAR}"
+}
+
+
+quay_run() {
+    docker run --net=host -e TEST_DATABASE_URI -ti "${IMAGE}:${IMAGE_TAG}" "$@"
+}
+
+
+unit() {
+    load_image && quay_run make unit-test
+}
+
+
+registry() {
+    load_image && quay_run make registry-test
+}
+
+
+registry_old() {
+    load_image && quay_run make registry-test-old
+}
+
+
+mysql_ping() {
+    mysqladmin --connect-timeout=2 --wait=30 --host=127.0.0.1 \
+               --user=root --password="${MYSQL_ROOT_PASSWORD}" ping
+}
+
+
+mysql_start() {
+    docker run --net=host -d -e MYSQL_ROOT_PASSWORD -e MYSQL_USER \
+           -e MYSQL_PASSWORD -e MYSQL_DATABASE "${MYSQL_IMAGE}"
+
+    if ! (sleep 10 && mysql_ping); then
+        echo "MySQL failed to respond in time."
+        exit 1
+    fi
+}
+
+
+mysql() {
+    TEST_DATABASE_URI="mysql+pymysql://"
+    TEST_DATABASE_URI+="${MYSQL_USER}:${MYSQL_PASSWORD}"
+    TEST_DATABASE_URI+="@127.0.0.1/${MYSQL_DATABASE}"
+
+    export TEST_DATABASE_URI
+
+    load_image
+    mysql_start
+    quay_run make full-db-test
+}
+
+
+postgres_ping() {
+    pg_isready --timeout=30 --dbname="${TEST_DATABASE_URI}"
+}
+
+
+postgres_start() {
+    docker run --net=host -d -e POSTGRES_USER -e POSTGRES_PASSWORD \
+           -e POSTGRES_DB "${POSTGRES_IMAGE}"
+
+    if ! (sleep 10 && postgres_ping); then
+        echo "PostgreSQL failed to respond in time."
+        exit 1
+    fi    
+}
+
+
+postgres() {
+    TEST_DATABASE_URI="postgresql://"
+    TEST_DATABASE_URI+="${POSTGRES_USER}:${POSTGRES_PASSWORD}"
+    TEST_DATABASE_URI+="@127.0.0.1/${POSTGRES_DB}"
+
+    export TEST_DATABASE_URI
+
+    load_image
+    postgres_start
+    quay_run make full-db-test
+}
+
+
+case "$1" in
+    build)
+        build_image
+        ;;
+
+    unit)
+        unit
+        ;;
+
+    registry)
+        registry
+        ;;
+
+    registry_old)
+        registry_old
+        ;;
+
+    mysql)
+        mysql
+        ;;
+
+    postgres)
+        postgres
+        ;;
+
+    clean)
+        clean_cache
+        ;;
+
+    *)
+        echo "Usage: $0 {build|unit|registry|registry_old|mysql|postgres|clean}"
+        exit 1
+        ;;
+esac