From 467a95135cac62a7d88a601f87108da5b722e631 Mon Sep 17 00:00:00 2001 From: Brad Ison Date: Mon, 7 May 2018 15:34:01 -0700 Subject: [PATCH 1/2] Combine base and main Dockerfile --- Dockerfile | 107 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 93 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 30504e666..ef4cf3ee1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,98 @@ # vim:ft=dockerfile -FROM quay.io/quay/quay-base:latest +FROM phusion/baseimage:0.10.0 + +ENV DEBIAN_FRONTEND noninteractive +ENV HOME /root +ENV QUAYDIR /quay-registry +ENV QUAYCONF /quay-registry/conf +ENV QUAYPATH "." + +RUN mkdir $QUAYDIR +WORKDIR $QUAYDIR + +# This is so we don't break http golang/go#17066 +# When Ubuntu has nginx >= 1.11.0 we can switch back. +RUN add-apt-repository ppa:nginx/development + +# Add Yarn repository until it is officially added to Ubuntu +RUN curl -fsSL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \ + && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list +RUN curl -fsSL https://deb.nodesource.com/setup_8.x | bash - +# Install system packages +RUN apt-get update && apt-get upgrade -y \ + && apt-get install -y \ + dnsmasq \ + g++ \ + gdb \ + gdebi-core \ + git \ + jpegoptim \ + libevent-2.0.5 \ + libevent-dev \ + libffi-dev \ + libfreetype6-dev \ + libgpgme11 \ + libgpgme11-dev \ + libjpeg62 \ + libjpeg62-dev \ + libjpeg8 \ + libldap-2.4-2 \ + libldap2-dev \ + libmagic1 \ + libpq-dev \ + libpq5 \ + libsasl2-dev \ + libsasl2-modules \ + memcached \ + monit \ + nginx \ + nodejs \ + optipng \ + openssl \ + python-dbg \ + python-dev \ + python-pip \ + python-virtualenv \ + yarn=0.22.0-1 \ + w3m # 27MAR2018 + +# Install cfssl +RUN curl -fsSL -o /bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 \ + && curl -fsSL -o /bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 \ + && chmod 0755 /bin/cfssl /bin/cfssljson + +# Install jwtproxy +RUN curl -fsSL -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 \ + && chmod +x /usr/local/bin/jwtproxy + +# Install prometheus-aggregator +RUN curl -fsSL -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator \ + && chmod +x /usr/local/bin/prometheus-aggregator + +# Install python dependencies +COPY requirements.txt requirements-tests.txt ./ +RUN virtualenv --distribute venv \ + && venv/bin/pip install -r requirements.txt \ + && venv/bin/pip install -r requirements-tests.txt \ + && venv/bin/pip freeze + +# Install front-end dependencies +COPY static/ package.json tsconfig.json webpack.config.js tslint.json yarn.lock ./ +RUN yarn install --ignore-engines + + +RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit $QUAYDIR/static/fonts $QUAYDIR/static/ldn /usr/local/nginx/logs/ + +RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache +VOLUME ["/var/log", "/datastorage", "/tmp"] + + RUN adduser memcached --disabled-login --system WORKDIR $QUAYDIR -COPY requirements.txt requirements-tests.txt ./ - # Check python dependencies for the GPL # Due to the following bug, pip results must be piped to a file before grepping: # https://github.com/pypa/pip/pull/3304 @@ -15,16 +100,6 @@ RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/ test -z "$(cat pipinfo.txt | grep GPL | grep -v LGPL)" && \ rm pipinfo.txt -RUN virtualenv --distribute venv \ - && venv/bin/pip install -r requirements.txt \ - && venv/bin/pip install -r requirements-tests.txt \ - && venv/bin/pip freeze - -# Install front-end dependencies -# JS dependencies -COPY yarn.lock package.json tsconfig.json webpack.config.js tslint.json ./ -RUN yarn install --ignore-engines - # JS compile COPY static static RUN yarn build \ @@ -33,8 +108,10 @@ RUN yarn build \ COPY . . +RUN PYTHONPATH=$QUAYPATH venv/bin/python -m external_libraries + # Update local copy of AWS IP Ranges. -RUN curl https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json +RUN curl -fsSL https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json # Set up the init system RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit static/fonts static/ldn /usr/local/nginx/logs/ \ @@ -59,4 +136,6 @@ RUN PYTHONPATH=$QUAYPATH venv/bin/alembic heads | grep -E '^[0-9a-f]+ \(head\)$' RUN ./scripts/detect-config.sh +EXPOSE 443 8443 80 + CMD ./quay-entrypoint.sh From e306a375a5f5a10f7db5b3e75710d845e908022f Mon Sep 17 00:00:00 2001 From: Brad Ison Date: Fri, 4 May 2018 14:03:18 -0400 Subject: [PATCH 2/2] Add Travis CI build configuration --- .travis.yml | 49 +++++++++++++ Makefile | 28 +++++--- requirements-tests.txt | 2 +- scripts/ci | 159 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 229 insertions(+), 9 deletions(-) create mode 100644 .travis.yml create mode 100755 scripts/ci diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..8b1085fa3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,49 @@ +--- +language: python + +sudo: required + +services: + - docker + +install: true + +# Stop default database instances here to avoid port conflicts. +before_script: + - sudo service mysql stop + - sudo service postgresql stop + +cache: + directories: + - $HOME/docker + +stages: + - build + - test + - database + - clean + +# We should label the steps if Travis ever supports it: +# https://github.com/travis-ci/travis-ci/issues/5898 +jobs: + include: + - stage: build + script: scripts/ci build + + - stage: test + script: scripts/ci unit + + - stage: test + script: scripts/ci registry + + - stage: test + script: scripts/ci registry_old + + - stage: database + script: scripts/ci mysql + + - stage: database + script: scripts/ci postgres + + - stage: clean + script: scripts/ci clean diff --git a/Makefile b/Makefile index a0f187ce3..50f8109c7 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ SHELL := /bin/bash +export PATH := ./venv/bin:$(PATH) + SHA := $(shell git rev-parse --short HEAD ) REPO := quay.io/quay/quay TAG := $(REPO):$(SHA) @@ -11,7 +13,7 @@ MODIFIED_FILES = $(shell git diff --name-only $(GIT_MERGE_BASED) | grep -E .+\.p show-modified: echo $(MODIFIED_FILES) -.PHONY: all unit test pkgs build run clean +.PHONY: all unit-test registry-test registry-test-old test pkgs build run clean all: clean pkgs test build @@ -52,8 +54,25 @@ registry-test: --timeout=3600 --verbose --show-count -x \ test/registry/registry_tests.py +registry-test-old: + TEST=true PYTHONPATH="." py.test \ + --cov="." --cov-report=html --cov-report=term-missing \ + --timeout=3600 --verbose --show-count -x \ + ./test/registry_tests.py + +full-db-test: ensure-test-db + TEST=true PYTHONPATH=. alembic upgrade head + TEST=true PYTHONPATH=. SKIP_DB_SCHEMA=true py.test --timeout=7200 \ + --verbose --show-count -x --ignore=endpoints/appr/test/ \ + ./ + test: unit-test registry-test +ensure-test-db: + @if [ -z $(TEST_DATABASE_URI) ]; then \ + echo "TEST_DATABASE_URI is undefined"; \ + exit 1; \ + fi PG_PASSWORD := quay PG_USER := quay @@ -133,13 +152,6 @@ yapf-all: yapf-diff: if [ $(MODIFIED_FILES_COUNT) -ne 0 ]; then yapf -d -p $(MODIFIED_FILES) ; fi -yapf: -ifneq (0,$(shell git diff-index HEAD | wc -l)) - echo "Failed, git dirty" && false -else ifneq (0,$(shell yapf -d -p $(MODIFIED_FILES) | wc -l)) - yapf -i -p $(MODIFIED_FILES) - git commit -a -m "code-stye Yapf: $(MODIFIED_FILES_COUNT) files updated" -m "$(MODIFIED_FILES)" -endif yapf-test: if [ `yapf -d -p $(MODIFIED_FILES) | wc -l` -gt 0 ] ; then false ; else true ;fi diff --git a/requirements-tests.txt b/requirements-tests.txt index 435f7c1fe..bb613f9cc 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,3 +1,4 @@ +pytest pytest-cov python-coveralls pytest-flask @@ -5,4 +6,3 @@ pytest-runner pytest-xdist pytest-timeout -e git+https://github.com/ant31/pytest-sugar.git#egg=pytest-sugar --e git+https://github.com/ant31/pytest.git#egg=pytest diff --git a/scripts/ci b/scripts/ci new file mode 100755 index 000000000..681e76faa --- /dev/null +++ b/scripts/ci @@ -0,0 +1,159 @@ +#!/bin/bash + +set -euo pipefail + + +IMAGE="quay-ci" + +CACHE_DIR="${HOME}/docker" +SHORT_SHA="${TRAVIS_COMMIT:0:7}" +IMAGE_TAG="${SHORT_SHA}-${TRAVIS_BUILD_NUMBER}" +IMAGE_TAR="${CACHE_DIR}/${IMAGE}-${IMAGE_TAG}.tar.gz" + +MYSQL_IMAGE="mysql:5.6" +POSTGRES_IMAGE="postgres:9.6" + +export MYSQL_ROOT_PASSWORD="quay" +export MYSQL_USER="quay" +export MYSQL_PASSWORD="quay" +export MYSQL_DATABASE="quay_ci" + +export POSTGRES_USER="quay" +export POSTGRES_PASSWORD="quay" +export POSTGRES_DB="quay_ci" + + +build_image() { + # Build the image and save it to the shared cache. + docker build -t "${IMAGE}:${IMAGE_TAG}" . + + echo "Exporting Docker image to cache..." + time (docker save "${IMAGE}:${IMAGE_TAG}" | gzip -2 > "${IMAGE_TAR}") +} + + +load_image() { + # Load our cached Docker image. + echo "Loading Docker image from cache..." + time (zcat "${IMAGE_TAR}" | docker load) +} + + +clean_cache() { + rm "${IMAGE_TAR}" +} + + +quay_run() { + docker run --net=host -e TEST_DATABASE_URI -ti "${IMAGE}:${IMAGE_TAG}" "$@" +} + + +unit() { + load_image && quay_run make unit-test +} + + +registry() { + load_image && quay_run make registry-test +} + + +registry_old() { + load_image && quay_run make registry-test-old +} + + +mysql_ping() { + mysqladmin --connect-timeout=2 --wait=30 --host=127.0.0.1 \ + --user=root --password="${MYSQL_ROOT_PASSWORD}" ping +} + + +mysql_start() { + docker run --net=host -d -e MYSQL_ROOT_PASSWORD -e MYSQL_USER \ + -e MYSQL_PASSWORD -e MYSQL_DATABASE "${MYSQL_IMAGE}" + + if ! (sleep 10 && mysql_ping); then + echo "MySQL failed to respond in time." + exit 1 + fi +} + + +mysql() { + TEST_DATABASE_URI="mysql+pymysql://" + TEST_DATABASE_URI+="${MYSQL_USER}:${MYSQL_PASSWORD}" + TEST_DATABASE_URI+="@127.0.0.1/${MYSQL_DATABASE}" + + export TEST_DATABASE_URI + + load_image + mysql_start + quay_run make full-db-test +} + + +postgres_ping() { + pg_isready --timeout=30 --dbname="${TEST_DATABASE_URI}" +} + + +postgres_start() { + docker run --net=host -d -e POSTGRES_USER -e POSTGRES_PASSWORD \ + -e POSTGRES_DB "${POSTGRES_IMAGE}" + + if ! (sleep 10 && postgres_ping); then + echo "PostgreSQL failed to respond in time." + exit 1 + fi +} + + +postgres() { + TEST_DATABASE_URI="postgresql://" + TEST_DATABASE_URI+="${POSTGRES_USER}:${POSTGRES_PASSWORD}" + TEST_DATABASE_URI+="@127.0.0.1/${POSTGRES_DB}" + + export TEST_DATABASE_URI + + load_image + postgres_start + quay_run make full-db-test +} + + +case "$1" in + build) + build_image + ;; + + unit) + unit + ;; + + registry) + registry + ;; + + registry_old) + registry_old + ;; + + mysql) + mysql + ;; + + postgres) + postgres + ;; + + clean) + clean_cache + ;; + + *) + echo "Usage: $0 {build|unit|registry|registry_old|mysql|postgres|clean}" + exit 1 + ;; +esac