vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #3067 from bison/travis-ci

Browse source 
Add Travis CI build configuration
This commit is contained in:
Brad Ison 2018-06-05 14:32:03 -04:00 committed by GitHub
commit 33029ba9d0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 322 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
.travis.yml Normal file
View file

@ -0,0 +1,49 @@
---
language: python
sudo: required
services:
- docker
install: true
# Stop default database instances here to avoid port conflicts.
before_script:
- sudo service mysql stop
- sudo service postgresql stop
cache:
directories:
- $HOME/docker
stages:
- build
- test
- database
- clean
# We should label the steps if Travis ever supports it:
# https://github.com/travis-ci/travis-ci/issues/5898
jobs:
include:
- stage: build
script: scripts/ci build
- stage: test
script: scripts/ci unit
- stage: test
script: scripts/ci registry
- stage: test
script: scripts/ci registry_old
- stage: database
script: scripts/ci mysql
- stage: database
script: scripts/ci postgres
- stage: clean
script: scripts/ci clean

107
Dockerfile
View file

@ -1,13 +1,98 @@
# vim:ft=dockerfile
FROM quay.io/quay/quay-base:latest
FROM phusion/baseimage:0.10.0
ENV DEBIAN_FRONTEND noninteractive
ENV HOME /root
ENV QUAYDIR /quay-registry
ENV QUAYCONF /quay-registry/conf
ENV QUAYPATH "."
RUN mkdir $QUAYDIR
WORKDIR $QUAYDIR
# This is so we don't break http golang/go#17066
# When Ubuntu has nginx >= 1.11.0 we can switch back.
RUN add-apt-repository ppa:nginx/development
# Add Yarn repository until it is officially added to Ubuntu
RUN curl -fsSL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
&& echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
RUN curl -fsSL https://deb.nodesource.com/setup_8.x | bash -
# Install system packages
RUN apt-get update && apt-get upgrade -y \
&& apt-get install -y \
dnsmasq \
g++ \
gdb \
gdebi-core \
git \
jpegoptim \
libevent-2.0.5 \
libevent-dev \
libffi-dev \
libfreetype6-dev \
libgpgme11 \
libgpgme11-dev \
libjpeg62 \
libjpeg62-dev \
libjpeg8 \
libldap-2.4-2 \
libldap2-dev \
libmagic1 \
libpq-dev \
libpq5 \
libsasl2-dev \
libsasl2-modules \
memcached \
monit \
nginx \
nodejs \
optipng \
openssl \
python-dbg \
python-dev \
python-pip \
python-virtualenv \
yarn=0.22.0-1 \
w3m # 27MAR2018
# Install cfssl
RUN curl -fsSL -o /bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 \
&& curl -fsSL -o /bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 \
&& chmod 0755 /bin/cfssl /bin/cfssljson
# Install jwtproxy
RUN curl -fsSL -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 \
&& chmod +x /usr/local/bin/jwtproxy
# Install prometheus-aggregator
RUN curl -fsSL -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator \
&& chmod +x /usr/local/bin/prometheus-aggregator
# Install python dependencies
COPY requirements.txt requirements-tests.txt ./
RUN virtualenv --distribute venv \
&& venv/bin/pip install -r requirements.txt \
&& venv/bin/pip install -r requirements-tests.txt \
&& venv/bin/pip freeze
# Install front-end dependencies
COPY static/ package.json tsconfig.json webpack.config.js tslint.json yarn.lock ./
RUN yarn install --ignore-engines
RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit $QUAYDIR/static/fonts $QUAYDIR/static/ldn /usr/local/nginx/logs/
RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache
VOLUME ["/var/log", "/datastorage", "/tmp"]
RUN adduser memcached --disabled-login --system
WORKDIR $QUAYDIR
COPY requirements.txt requirements-tests.txt ./
# Check python dependencies for the GPL
# Due to the following bug, pip results must be piped to a file before grepping:
# https://github.com/pypa/pip/pull/3304
@ -15,16 +100,6 @@ RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/
test -z "$(cat pipinfo.txt | grep GPL | grep -v LGPL)" && \
rm pipinfo.txt
RUN virtualenv --distribute venv \
&& venv/bin/pip install -r requirements.txt \
&& venv/bin/pip install -r requirements-tests.txt \
&& venv/bin/pip freeze
# Install front-end dependencies
# JS dependencies
COPY yarn.lock package.json tsconfig.json webpack.config.js tslint.json ./
RUN yarn install --ignore-engines
# JS compile
COPY static static
RUN yarn build \
@ -33,8 +108,10 @@ RUN yarn build \
COPY . .
RUN PYTHONPATH=$QUAYPATH venv/bin/python -m external_libraries
# Update local copy of AWS IP Ranges.
RUN curl https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json
RUN curl -fsSL https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json
# Set up the init system
RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit static/fonts static/ldn /usr/local/nginx/logs/ \
@ -59,4 +136,6 @@ RUN PYTHONPATH=$QUAYPATH venv/bin/alembic heads | grep -E '^[0-9a-f]+ \(head\)$'
RUN ./scripts/detect-config.sh
EXPOSE 443 8443 80
CMD ./quay-entrypoint.sh

28
Makefile
View file

@ -1,5 +1,7 @@
SHELL := /bin/bash
export PATH := ./venv/bin:$(PATH)
SHA := $(shell git rev-parse --short HEAD )
REPO := quay.io/quay/quay
TAG := $(REPO):$(SHA)
@ -11,7 +13,7 @@ MODIFIED_FILES = $(shell git diff --name-only $(GIT_MERGE_BASED) | grep -E .+\.p
show-modified:
echo $(MODIFIED_FILES)
.PHONY: all unit test pkgs build run clean
.PHONY: all unit-test registry-test registry-test-old test pkgs build run clean
all: clean pkgs test build
@ -52,8 +54,25 @@ registry-test:
--timeout=3600 --verbose --show-count -x \
test/registry/registry_tests.py
registry-test-old:
TEST=true PYTHONPATH="." py.test \
--cov="." --cov-report=html --cov-report=term-missing \
--timeout=3600 --verbose --show-count -x \
./test/registry_tests.py
full-db-test: ensure-test-db
TEST=true PYTHONPATH=. alembic upgrade head
TEST=true PYTHONPATH=. SKIP_DB_SCHEMA=true py.test --timeout=7200 \
--verbose --show-count -x --ignore=endpoints/appr/test/ \
./
test: unit-test registry-test
ensure-test-db:
@if [ -z $(TEST_DATABASE_URI) ]; then \
echo "TEST_DATABASE_URI is undefined"; \
exit 1; \
fi
PG_PASSWORD := quay
PG_USER := quay
@ -133,13 +152,6 @@ yapf-all:
yapf-diff:
if [ $(MODIFIED_FILES_COUNT) -ne 0 ]; then yapf -d -p $(MODIFIED_FILES) ; fi
yapf:
ifneq (0,$(shell git diff-index HEAD | wc -l))
echo "Failed, git dirty" && false
else ifneq (0,$(shell yapf -d -p $(MODIFIED_FILES) | wc -l))
yapf -i -p $(MODIFIED_FILES)
git commit -a -m "code-stye Yapf: $(MODIFIED_FILES_COUNT) files updated" -m "$(MODIFIED_FILES)"
endif
yapf-test:
if [ `yapf -d -p $(MODIFIED_FILES) | wc -l` -gt 0 ] ; then false ; else true ;fi

2
requirements-tests.txt
View file

@ -1,3 +1,4 @@
pytest
pytest-cov
python-coveralls
pytest-flask
@ -5,4 +6,3 @@ pytest-runner
pytest-xdist
pytest-timeout
-e git+https://github.com/ant31/pytest-sugar.git#egg=pytest-sugar
-e git+https://github.com/ant31/pytest.git#egg=pytest

159
scripts/ci Executable file
View file

@ -0,0 +1,159 @@
#!/bin/bash
set -euo pipefail
IMAGE="quay-ci"
CACHE_DIR="${HOME}/docker"
SHORT_SHA="${TRAVIS_COMMIT:0:7}"
IMAGE_TAG="${SHORT_SHA}-${TRAVIS_BUILD_NUMBER}"
IMAGE_TAR="${CACHE_DIR}/${IMAGE}-${IMAGE_TAG}.tar.gz"
MYSQL_IMAGE="mysql:5.6"
POSTGRES_IMAGE="postgres:9.6"
export MYSQL_ROOT_PASSWORD="quay"
export MYSQL_USER="quay"
export MYSQL_PASSWORD="quay"
export MYSQL_DATABASE="quay_ci"
export POSTGRES_USER="quay"
export POSTGRES_PASSWORD="quay"
export POSTGRES_DB="quay_ci"
build_image() {
# Build the image and save it to the shared cache.
docker build -t "${IMAGE}:${IMAGE_TAG}" .
echo "Exporting Docker image to cache..."
time (docker save "${IMAGE}:${IMAGE_TAG}" | gzip -2 > "${IMAGE_TAR}")
}
load_image() {
# Load our cached Docker image.
echo "Loading Docker image from cache..."
time (zcat "${IMAGE_TAR}" | docker load)
}
clean_cache() {
rm "${IMAGE_TAR}"
}
quay_run() {
docker run --net=host -e TEST_DATABASE_URI -ti "${IMAGE}:${IMAGE_TAG}" "$@"
}
unit() {
load_image && quay_run make unit-test
}
registry() {
load_image && quay_run make registry-test
}
registry_old() {
load_image && quay_run make registry-test-old
}
mysql_ping() {
mysqladmin --connect-timeout=2 --wait=30 --host=127.0.0.1 \
--user=root --password="${MYSQL_ROOT_PASSWORD}" ping
}
mysql_start() {
docker run --net=host -d -e MYSQL_ROOT_PASSWORD -e MYSQL_USER \
-e MYSQL_PASSWORD -e MYSQL_DATABASE "${MYSQL_IMAGE}"
if ! (sleep 10 && mysql_ping); then
echo "MySQL failed to respond in time."
exit 1
fi
}
mysql() {
TEST_DATABASE_URI="mysql+pymysql://"
TEST_DATABASE_URI+="${MYSQL_USER}:${MYSQL_PASSWORD}"
TEST_DATABASE_URI+="@127.0.0.1/${MYSQL_DATABASE}"
export TEST_DATABASE_URI
load_image
mysql_start
quay_run make full-db-test
}
postgres_ping() {
pg_isready --timeout=30 --dbname="${TEST_DATABASE_URI}"
}
postgres_start() {
docker run --net=host -d -e POSTGRES_USER -e POSTGRES_PASSWORD \
-e POSTGRES_DB "${POSTGRES_IMAGE}"
if ! (sleep 10 && postgres_ping); then
echo "PostgreSQL failed to respond in time."
exit 1
fi
}
postgres() {
TEST_DATABASE_URI="postgresql://"
TEST_DATABASE_URI+="${POSTGRES_USER}:${POSTGRES_PASSWORD}"
TEST_DATABASE_URI+="@127.0.0.1/${POSTGRES_DB}"
export TEST_DATABASE_URI
load_image
postgres_start
quay_run make full-db-test
}
case "$1" in
build)
build_image
;;
unit)
unit
;;
registry)
registry
;;
registry_old)
registry_old
;;
mysql)
mysql
;;
postgres)
postgres
;;
clean)
clean_cache
;;
*)
echo "Usage: $0 {build|unit|registry|registry_old|mysql|postgres|clean}"
exit 1
;;
esac