Merge pull request #3067 from bison/travis-ci
Add Travis CI build configuration
This commit is contained in:
Brad Ison
GitHub
commit
33029ba9d0
5 changed files with 322 additions and 23 deletions
49
.travis.yml
Normal file
49
.travis.yml
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
---
|
||||
language: python
|
||||
|
||||
sudo: required
|
||||
|
||||
services:
|
||||
- docker
|
||||
|
||||
install: true
|
||||
|
||||
# Stop default database instances here to avoid port conflicts.
|
||||
before_script:
|
||||
- sudo service mysql stop
|
||||
- sudo service postgresql stop
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/docker
|
||||
|
||||
stages:
|
||||
- build
|
||||
- test
|
||||
- database
|
||||
- clean
|
||||
|
||||
# We should label the steps if Travis ever supports it:
|
||||
# https://github.com/travis-ci/travis-ci/issues/5898
|
||||
jobs:
|
||||
include:
|
||||
- stage: build
|
||||
script: scripts/ci build
|
||||
|
||||
- stage: test
|
||||
script: scripts/ci unit
|
||||
|
||||
- stage: test
|
||||
script: scripts/ci registry
|
||||
|
||||
- stage: test
|
||||
script: scripts/ci registry_old
|
||||
|
||||
- stage: database
|
||||
script: scripts/ci mysql
|
||||
|
||||
- stage: database
|
||||
script: scripts/ci postgres
|
||||
|
||||
- stage: clean
|
||||
script: scripts/ci clean
|
||||
107
Dockerfile
107
Dockerfile
|
|
@ -1,13 +1,98 @@
|
|||
# vim:ft=dockerfile
|
||||
|
||||
FROM quay.io/quay/quay-base:latest
|
||||
FROM phusion/baseimage:0.10.0
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
ENV HOME /root
|
||||
ENV QUAYDIR /quay-registry
|
||||
ENV QUAYCONF /quay-registry/conf
|
||||
ENV QUAYPATH "."
|
||||
|
||||
RUN mkdir $QUAYDIR
|
||||
WORKDIR $QUAYDIR
|
||||
|
||||
# This is so we don't break http golang/go#17066
|
||||
# When Ubuntu has nginx >= 1.11.0 we can switch back.
|
||||
RUN add-apt-repository ppa:nginx/development
|
||||
|
||||
# Add Yarn repository until it is officially added to Ubuntu
|
||||
RUN curl -fsSL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
|
||||
&& echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
|
||||
RUN curl -fsSL https://deb.nodesource.com/setup_8.x | bash -
|
||||
# Install system packages
|
||||
RUN apt-get update && apt-get upgrade -y \
|
||||
&& apt-get install -y \
|
||||
dnsmasq \
|
||||
g++ \
|
||||
gdb \
|
||||
gdebi-core \
|
||||
git \
|
||||
jpegoptim \
|
||||
libevent-2.0.5 \
|
||||
libevent-dev \
|
||||
libffi-dev \
|
||||
libfreetype6-dev \
|
||||
libgpgme11 \
|
||||
libgpgme11-dev \
|
||||
libjpeg62 \
|
||||
libjpeg62-dev \
|
||||
libjpeg8 \
|
||||
libldap-2.4-2 \
|
||||
libldap2-dev \
|
||||
libmagic1 \
|
||||
libpq-dev \
|
||||
libpq5 \
|
||||
libsasl2-dev \
|
||||
libsasl2-modules \
|
||||
memcached \
|
||||
monit \
|
||||
nginx \
|
||||
nodejs \
|
||||
optipng \
|
||||
openssl \
|
||||
python-dbg \
|
||||
python-dev \
|
||||
python-pip \
|
||||
python-virtualenv \
|
||||
yarn=0.22.0-1 \
|
||||
w3m # 27MAR2018
|
||||
|
||||
# Install cfssl
|
||||
RUN curl -fsSL -o /bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 \
|
||||
&& curl -fsSL -o /bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 \
|
||||
&& chmod 0755 /bin/cfssl /bin/cfssljson
|
||||
|
||||
# Install jwtproxy
|
||||
RUN curl -fsSL -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 \
|
||||
&& chmod +x /usr/local/bin/jwtproxy
|
||||
|
||||
# Install prometheus-aggregator
|
||||
RUN curl -fsSL -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator \
|
||||
&& chmod +x /usr/local/bin/prometheus-aggregator
|
||||
|
||||
# Install python dependencies
|
||||
COPY requirements.txt requirements-tests.txt ./
|
||||
RUN virtualenv --distribute venv \
|
||||
&& venv/bin/pip install -r requirements.txt \
|
||||
&& venv/bin/pip install -r requirements-tests.txt \
|
||||
&& venv/bin/pip freeze
|
||||
|
||||
# Install front-end dependencies
|
||||
COPY static/ package.json tsconfig.json webpack.config.js tslint.json yarn.lock ./
|
||||
RUN yarn install --ignore-engines
|
||||
|
||||
|
||||
RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit $QUAYDIR/static/fonts $QUAYDIR/static/ldn /usr/local/nginx/logs/
|
||||
|
||||
RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache
|
||||
VOLUME ["/var/log", "/datastorage", "/tmp"]
|
||||
|
||||
|
||||
|
||||
RUN adduser memcached --disabled-login --system
|
||||
|
||||
WORKDIR $QUAYDIR
|
||||
|
||||
COPY requirements.txt requirements-tests.txt ./
|
||||
|
||||
# Check python dependencies for the GPL
|
||||
# Due to the following bug, pip results must be piped to a file before grepping:
|
||||
# https://github.com/pypa/pip/pull/3304
|
||||
|
|
@ -15,16 +100,6 @@ RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/
|
|||
test -z "$(cat pipinfo.txt | grep GPL | grep -v LGPL)" && \
|
||||
rm pipinfo.txt
|
||||
|
||||
RUN virtualenv --distribute venv \
|
||||
&& venv/bin/pip install -r requirements.txt \
|
||||
&& venv/bin/pip install -r requirements-tests.txt \
|
||||
&& venv/bin/pip freeze
|
||||
|
||||
# Install front-end dependencies
|
||||
# JS dependencies
|
||||
COPY yarn.lock package.json tsconfig.json webpack.config.js tslint.json ./
|
||||
RUN yarn install --ignore-engines
|
||||
|
||||
# JS compile
|
||||
COPY static static
|
||||
RUN yarn build \
|
||||
|
|
@ -33,8 +108,10 @@ RUN yarn build \
|
|||
|
||||
COPY . .
|
||||
|
||||
RUN PYTHONPATH=$QUAYPATH venv/bin/python -m external_libraries
|
||||
|
||||
# Update local copy of AWS IP Ranges.
|
||||
RUN curl https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json
|
||||
RUN curl -fsSL https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json
|
||||
|
||||
# Set up the init system
|
||||
RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit static/fonts static/ldn /usr/local/nginx/logs/ \
|
||||
|
|
@ -59,4 +136,6 @@ RUN PYTHONPATH=$QUAYPATH venv/bin/alembic heads | grep -E '^[0-9a-f]+ \(head\)$'
|
|||
|
||||
RUN ./scripts/detect-config.sh
|
||||
|
||||
EXPOSE 443 8443 80
|
||||
|
||||
CMD ./quay-entrypoint.sh
|
||||
|
|
|
|||
28
Makefile
28
Makefile
|
|
@ -1,5 +1,7 @@
|
|||
SHELL := /bin/bash
|
||||
|
||||
export PATH := ./venv/bin:$(PATH)
|
||||
|
||||
SHA := $(shell git rev-parse --short HEAD )
|
||||
REPO := quay.io/quay/quay
|
||||
TAG := $(REPO):$(SHA)
|
||||
|
|
@ -11,7 +13,7 @@ MODIFIED_FILES = $(shell git diff --name-only $(GIT_MERGE_BASED) | grep -E .+\.p
|
|||
show-modified:
|
||||
echo $(MODIFIED_FILES)
|
||||
|
||||
.PHONY: all unit test pkgs build run clean
|
||||
.PHONY: all unit-test registry-test registry-test-old test pkgs build run clean
|
||||
|
||||
all: clean pkgs test build
|
||||
|
||||
|
|
@ -52,8 +54,25 @@ registry-test:
|
|||
--timeout=3600 --verbose --show-count -x \
|
||||
test/registry/registry_tests.py
|
||||
|
||||
registry-test-old:
|
||||
TEST=true PYTHONPATH="." py.test \
|
||||
--cov="." --cov-report=html --cov-report=term-missing \
|
||||
--timeout=3600 --verbose --show-count -x \
|
||||
./test/registry_tests.py
|
||||
|
||||
full-db-test: ensure-test-db
|
||||
TEST=true PYTHONPATH=. alembic upgrade head
|
||||
TEST=true PYTHONPATH=. SKIP_DB_SCHEMA=true py.test --timeout=7200 \
|
||||
--verbose --show-count -x --ignore=endpoints/appr/test/ \
|
||||
./
|
||||
|
||||
test: unit-test registry-test
|
||||
|
||||
ensure-test-db:
|
||||
@if [ -z $(TEST_DATABASE_URI) ]; then \
|
||||
echo "TEST_DATABASE_URI is undefined"; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
PG_PASSWORD := quay
|
||||
PG_USER := quay
|
||||
|
|
@ -133,13 +152,6 @@ yapf-all:
|
|||
yapf-diff:
|
||||
if [ $(MODIFIED_FILES_COUNT) -ne 0 ]; then yapf -d -p $(MODIFIED_FILES) ; fi
|
||||
|
||||
yapf:
|
||||
ifneq (0,$(shell git diff-index HEAD | wc -l))
|
||||
echo "Failed, git dirty" && false
|
||||
else ifneq (0,$(shell yapf -d -p $(MODIFIED_FILES) | wc -l))
|
||||
yapf -i -p $(MODIFIED_FILES)
|
||||
git commit -a -m "code-stye Yapf: $(MODIFIED_FILES_COUNT) files updated" -m "$(MODIFIED_FILES)"
|
||||
endif
|
||||
|
||||
yapf-test:
|
||||
if [ `yapf -d -p $(MODIFIED_FILES) | wc -l` -gt 0 ] ; then false ; else true ;fi
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
pytest
|
||||
pytest-cov
|
||||
python-coveralls
|
||||
pytest-flask
|
||||
|
|
@ -5,4 +6,3 @@ pytest-runner
|
|||
pytest-xdist
|
||||
pytest-timeout
|
||||
-e git+https://github.com/ant31/pytest-sugar.git#egg=pytest-sugar
|
||||
-e git+https://github.com/ant31/pytest.git#egg=pytest
|
||||
|
|
|
|||
159
scripts/ci
Executable file
159
scripts/ci
Executable file
|
|
@ -0,0 +1,159 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
|
||||
IMAGE="quay-ci"
|
||||
|
||||
CACHE_DIR="${HOME}/docker"
|
||||
SHORT_SHA="${TRAVIS_COMMIT:0:7}"
|
||||
IMAGE_TAG="${SHORT_SHA}-${TRAVIS_BUILD_NUMBER}"
|
||||
IMAGE_TAR="${CACHE_DIR}/${IMAGE}-${IMAGE_TAG}.tar.gz"
|
||||
|
||||
MYSQL_IMAGE="mysql:5.6"
|
||||
POSTGRES_IMAGE="postgres:9.6"
|
||||
|
||||
export MYSQL_ROOT_PASSWORD="quay"
|
||||
export MYSQL_USER="quay"
|
||||
export MYSQL_PASSWORD="quay"
|
||||
export MYSQL_DATABASE="quay_ci"
|
||||
|
||||
export POSTGRES_USER="quay"
|
||||
export POSTGRES_PASSWORD="quay"
|
||||
export POSTGRES_DB="quay_ci"
|
||||
|
||||
|
||||
build_image() {
|
||||
# Build the image and save it to the shared cache.
|
||||
docker build -t "${IMAGE}:${IMAGE_TAG}" .
|
||||
|
||||
echo "Exporting Docker image to cache..."
|
||||
time (docker save "${IMAGE}:${IMAGE_TAG}" | gzip -2 > "${IMAGE_TAR}")
|
||||
}
|
||||
|
||||
|
||||
load_image() {
|
||||
# Load our cached Docker image.
|
||||
echo "Loading Docker image from cache..."
|
||||
time (zcat "${IMAGE_TAR}" | docker load)
|
||||
}
|
||||
|
||||
|
||||
clean_cache() {
|
||||
rm "${IMAGE_TAR}"
|
||||
}
|
||||
|
||||
|
||||
quay_run() {
|
||||
docker run --net=host -e TEST_DATABASE_URI -ti "${IMAGE}:${IMAGE_TAG}" "$@"
|
||||
}
|
||||
|
||||
|
||||
unit() {
|
||||
load_image && quay_run make unit-test
|
||||
}
|
||||
|
||||
|
||||
registry() {
|
||||
load_image && quay_run make registry-test
|
||||
}
|
||||
|
||||
|
||||
registry_old() {
|
||||
load_image && quay_run make registry-test-old
|
||||
}
|
||||
|
||||
|
||||
mysql_ping() {
|
||||
mysqladmin --connect-timeout=2 --wait=30 --host=127.0.0.1 \
|
||||
--user=root --password="${MYSQL_ROOT_PASSWORD}" ping
|
||||
}
|
||||
|
||||
|
||||
mysql_start() {
|
||||
docker run --net=host -d -e MYSQL_ROOT_PASSWORD -e MYSQL_USER \
|
||||
-e MYSQL_PASSWORD -e MYSQL_DATABASE "${MYSQL_IMAGE}"
|
||||
|
||||
if ! (sleep 10 && mysql_ping); then
|
||||
echo "MySQL failed to respond in time."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
mysql() {
|
||||
TEST_DATABASE_URI="mysql+pymysql://"
|
||||
TEST_DATABASE_URI+="${MYSQL_USER}:${MYSQL_PASSWORD}"
|
||||
TEST_DATABASE_URI+="@127.0.0.1/${MYSQL_DATABASE}"
|
||||
|
||||
export TEST_DATABASE_URI
|
||||
|
||||
load_image
|
||||
mysql_start
|
||||
quay_run make full-db-test
|
||||
}
|
||||
|
||||
|
||||
postgres_ping() {
|
||||
pg_isready --timeout=30 --dbname="${TEST_DATABASE_URI}"
|
||||
}
|
||||
|
||||
|
||||
postgres_start() {
|
||||
docker run --net=host -d -e POSTGRES_USER -e POSTGRES_PASSWORD \
|
||||
-e POSTGRES_DB "${POSTGRES_IMAGE}"
|
||||
|
||||
if ! (sleep 10 && postgres_ping); then
|
||||
echo "PostgreSQL failed to respond in time."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
postgres() {
|
||||
TEST_DATABASE_URI="postgresql://"
|
||||
TEST_DATABASE_URI+="${POSTGRES_USER}:${POSTGRES_PASSWORD}"
|
||||
TEST_DATABASE_URI+="@127.0.0.1/${POSTGRES_DB}"
|
||||
|
||||
export TEST_DATABASE_URI
|
||||
|
||||
load_image
|
||||
postgres_start
|
||||
quay_run make full-db-test
|
||||
}
|
||||
|
||||
|
||||
case "$1" in
|
||||
build)
|
||||
build_image
|
||||
;;
|
||||
|
||||
unit)
|
||||
unit
|
||||
;;
|
||||
|
||||
registry)
|
||||
registry
|
||||
;;
|
||||
|
||||
registry_old)
|
||||
registry_old
|
||||
;;
|
||||
|
||||
mysql)
|
||||
mysql
|
||||
;;
|
||||
|
||||
postgres)
|
||||
postgres
|
||||
;;
|
||||
|
||||
clean)
|
||||
clean_cache
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Usage: $0 {build|unit|registry|registry_old|mysql|postgres|clean}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
Reference in a new issue