vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add gitlab-ci docker-build

Browse source
This commit is contained in:
Antoine Legrand 2017-05-30 23:06:09 +02:00
parent c568542d77
commit 7e85ac6f89
5 changed files with 116 additions and 182 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.dockerignore
View file

@ -23,3 +23,4 @@ coverage
.npm-debug.log .npm-debug.log
test/__pycache__ test/__pycache__
__pycache__ __pycache__
**/__pycache__

113
.gitlab-ci.yml
View file

@ -1,5 +1,6 @@
--- ---
stages: stages:
- docker-build
- unit-tests - unit-tests
- integration - integration
- release - release
@ -8,55 +9,75 @@ stages:
variables: variables:
FAILFASTCI_NAMESPACE: 'quay' FAILFASTCI_NAMESPACE: 'quay'
IMAGE: quay.io/quay/quay IMAGE: quay.io/quay/quay
PIP_CACHE_DIR: /pip-cache PIP_CACHE_DIR: pip-cache
PIP: /venv/bin/pip PIP: /venv/bin/pip
PYTEST: /venv/bin/py.test PYTEST: /venv/bin/py.test
TEST: "true" TEST: "true"
PYTHONPATH: "." PYTHONPATH: "."
cache:
paths:
- cache
- /pip-cache
- /apt-cache
key: "$CI_PROJECT_ID"
# STAGE 1: container build
.job: &job .docker: &docker
variables:
DOCKER_DRIVER: aufs
image: docker:git
before_script: before_script:
- mkdir -p /apt-cache - docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io
- apt-get update services:
- apt-get install -o dir::cache::archives="/apt-cache" -y python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev libgpgme11-dev nodejs jpegoptim optipng w3m - docker:dind
- pip install -r requirements.txt tags:
- pip install -r requirements-tests.txt - docker
container-base-build:
<<: *docker
stage: docker-build
script: script:
- echo test - docker build --cache-from quay.io/quay/quay-base:latest -t quay.io/quay/quay-base:latest -f quay-base.dockerfile .
- docker push quay.io/quay/quay-base:latest
when: manual
container-build:
<<: *docker
stage: docker-build
script:
- docker build -t quay.io/quay/quay-ci:$CI_COMMIT_REF_SLUG -f quay.dockerfile .
- docker push quay.io/quay/quay-ci:$CI_COMMIT_REF_SLUG
# STAGE 2: Unit tests & code-style
.job: &job
variables:
GIT_STRATEGY: none
image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
before_script:
- cd /
- source venv/bin/activate
tags: tags:
- kubernetes - kubernetes
image: quay.io/quay/quay:build
unit-tests: unit-tests:
<<: *job <<: *job
stage: unit-tests stage: unit-tests
script: script:
- py.test --timeout=7200 --verbose --show-count ./ --color=no - py.test --timeout=7200 --verbose --show-count ./ --color=no -x
registry-tests: registry-tests:
<<: *job <<: *job
stage: unit-tests stage: unit-tests
script: script:
- py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no - py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no -x
karma-tests: karma-tests:
<<: *job <<: *job
stage: unit-tests stage: unit-tests
script: script:
- yarn install --ignore-engines
- yarn test - yarn test
code-styles: code-styles:
<<: *job <<: *job
stage: unit-tests stage: unit-tests
@ -66,6 +87,7 @@ code-styles:
- echo "pylint" - echo "pylint"
# Stage 3: Integration/e2e tests
postgres: postgres:
<<: *job <<: *job
variables: variables:
@ -73,60 +95,51 @@ postgres:
SKIP_DB_SCHEMA: 'true' SKIP_DB_SCHEMA: 'true'
POSTGRES_PASSWORD: quay POSTGRES_PASSWORD: quay
POSTGRES_USER: quay POSTGRES_USER: quay
GIT_STRATEGY: none
stage: integration stage: integration
services: services:
- postgres:9.6 - postgres:9.6
script: script:
- sleep 30
- alembic upgrade head - alembic upgrade head
- py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ - PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x
mysql: mysql:
<<: *job <<: *job
variables: variables:
TEST_DATABASE_URI: mysql+pymysql://quay:quay@localhost/quay TEST_DATABASE_URI: 'mysql+pymysql://quay:quay@localhost/quay'
SKIP_DB_SCHEMA: 'true' SKIP_DB_SCHEMA: 'true'
MYSQL_ROOT_PASSWORD: quay MYSQL_ROOT_PASSWORD: quay
MYSQL_DATABASE: quay MYSQL_DATABASE: quay
MYSQL_USER: quay MYSQL_USER: quay
MYSQL_PASSWORD: quay MYSQL_PASSWORD: quay
GIT_STRATEGY: none
stage: integration stage: integration
services: services:
- mysql - mysql
script: script:
- sleep 30
- alembic upgrade head - alembic upgrade head
- py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ - PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x
# # TODO (@ant31) # e2e-demo:
# .docker: &docker # <<: *job
# image: python:2.7
# variables: # variables:
# DOCKER_HOST: tcp://localhost:2375 # TEST_DATABASE_URI: 'postgresql://quay:quay@localhost/quay'
# image: docker:git # SKIP_DB_SCHEMA: 'true'
# POSTGRES_PASSWORD: quay
# POSTGRES_USER: quay
# GIT_STRATEGY: none
# stage: integration
# before_script: # before_script:
# - docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io # - cd /
# services: # services:
# - docker:dind # - postgres:9.6
# tags: # - quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
# - kubernetes
# docker-push:
# <<: *docker
# stage: release
# script: # script:
# - echo "push container to the quay prod repo" # - sleep 240
# - docker build --no-cache -t quay.io/quay/quay-branches:$CI_BUILD_REF_NAME . # - curl localhost:80/cnr/version
# - docker push quay.io/quay/quay-branches:$CI_BUILD_REF_NAME # allow_failure: true
# helm-push:
# <<: *docker
# stage: release
# script:
# - echo "build chart"
# deploy-staging:
# when: manual
# stage: deploy
# script:
# - echo "deploy-staging"

58
quay-base.dockerfile
View file

@ -10,12 +10,12 @@ ENV HOME /root
RUN add-apt-repository ppa:nginx/development RUN add-apt-repository ppa:nginx/development
# Add Yarn repository until it is officially added to Ubuntu # Add Yarn repository until it is officially added to Ubuntu
RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
# Install system packages # Install system packages
RUN apt-get update && apt-get upgrade -y # 26MAY2017 RUN apt-get update && apt-get upgrade -y \
RUN apt-get install -y \ && apt-get install -y \
dnsmasq \ dnsmasq \
g++ \ g++ \
gdb \ gdb \
@ -48,20 +48,14 @@ RUN apt-get install -y \
python-pip \ python-pip \
python-virtualenv \ python-virtualenv \
yarn=0.22.0-1 \ yarn=0.22.0-1 \
w3m w3m # 26MAY2017
# Install python dependencies # Install python dependencies
ADD requirements.txt requirements.txt COPY requirements.txt requirements-tests.txt ./
RUN virtualenv --distribute venv RUN virtualenv --distribute venv \
RUN venv/bin/pip install -r requirements.txt # 07SEP2016 && venv/bin/pip install -r requirements.txt \
RUN venv/bin/pip freeze && venv/bin/pip install -r requirements-tests.txt \
&& venv/bin/pip freeze # 07SEP2016
# Check python dependencies for the GPL
# Due to the following bug, pip results must be piped to a file before grepping:
# https://github.com/pypa/pip/pull/3304
RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/bin/pip --disable-pip-version-check show > pipinfo.txt && \
test -z $(cat pipinfo.txt | grep GPL | grep -v LGPL) && \
rm pipinfo.txt
# Install cfssl # Install cfssl
RUN mkdir /gocode RUN mkdir /gocode
@ -77,34 +71,28 @@ RUN curl -O https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz && \
rm -rf /gocode && rm -rf /usr/local/go rm -rf /gocode && rm -rf /usr/local/go
# Install jwtproxy # Install jwtproxy
RUN curl -L -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 RUN curl -L -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 \
RUN chmod +x /usr/local/bin/jwtproxy && chmod +x /usr/local/bin/jwtproxy
# Install prometheus-aggregator # Install prometheus-aggregator
RUN curl -L -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator RUN curl -L -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator \
RUN chmod +x /usr/local/bin/prometheus-aggregator && chmod +x /usr/local/bin/prometheus-aggregator
# Install front-end dependencies # Install front-end dependencies
RUN ln -s /usr/bin/nodejs /usr/bin/node RUN ln -s /usr/bin/nodejs /usr/bin/node
ADD package.json package.json COPY static/ package.json tsconfig.json webpack.config.js typings.json yarn.lock ./
ADD tsconfig.json tsconfig.json
ADD webpack.config.js webpack.config.js
ADD typings.json typings.json
ADD yarn.lock yarn.lock
RUN yarn install --ignore-engines RUN yarn install --ignore-engines
# Add static files
ADD static static
# Run Webpack RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit static/fonts static/ldn /usr/local/nginx/logs/
RUN yarn build
# Optimize our images COPY external_libraries.py ./
ADD static/img static/img
RUN jpegoptim static/img/**/*.jpg
RUN optipng -clobber -quiet static/img/**/*.png
ADD external_libraries.py external_libraries.py
RUN venv/bin/python -m external_libraries RUN venv/bin/python -m external_libraries
ARG RUN_TESTS=false
ENV RUN_TESTS ${RUN_TESTS}
RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache
VOLUME ["/conf/stack", "/var/log", "/datastorage", "/tmp", "/conf/etcd"]
EXPOSE 443 8443 80 EXPOSE 443 8443 80

124
quay.dockerfile
View file

@ -2,15 +2,13 @@
FROM quay.io/quay/quay-base:latest FROM quay.io/quay/quay-base:latest
ENV DEBIAN_FRONTEND noninteractive COPY . .
ENV HOME /root
# Install python dependencies # Install python dependencies
ADD requirements.txt requirements.txt RUN virtualenv --distribute venv \
RUN virtualenv --distribute venv && venv/bin/pip install -r requirements.txt \
RUN venv/bin/pip install -r requirements.txt # 07SEP2016 && venv/bin/pip install -r requirements-tests.txt \
RUN venv/bin/pip freeze && venv/bin/pip freeze # 07SEP2016
# Check python dependencies for the GPL # Check python dependencies for the GPL
# Due to the following bug, pip results must be piped to a file before grepping: # Due to the following bug, pip results must be piped to a file before grepping:
@ -19,106 +17,40 @@ RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/
test -z $(cat pipinfo.txt | grep GPL | grep -v LGPL) && \ test -z $(cat pipinfo.txt | grep GPL | grep -v LGPL) && \
rm pipinfo.txt rm pipinfo.txt
# Install front-end dependencies # Install front-end dependencies and Optimize our images
RUN ln -s /usr/bin/nodejs /usr/bin/node RUN yarn install --ignore-engines \
ADD package.json package.json && yarn build \
ADD tsconfig.json tsconfig.json && jpegoptim static/img/**/*.jpg \
ADD webpack.config.js webpack.config.js && optipng -clobber -quiet static/img/**/*.png
ADD typings.json typings.json
ADD yarn.lock yarn.lock
RUN yarn install --ignore-engines
# Add static files
ADD static static
# Run Webpack
RUN yarn build
# Run front-end tests
ARG RUN_TESTS=false
ENV RUN_TESTS ${RUN_TESTS}
ADD karma.conf.js karma.conf.js
RUN if [ "$RUN_TESTS" = true ]; then \
yarn test; \
fi
# Optimize our images
ADD static/img static/img
RUN jpegoptim static/img/**/*.jpg
RUN optipng -clobber -quiet static/img/**/*.png
RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev libgpgme11-dev nodejs jpegoptim optipng w3m
RUN apt-get autoremove -y
RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# Set up the init system # Set up the init system
ADD conf/init/copy_config_files.sh /etc/my_init.d/ RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit static/fonts static/ldn /usr/local/nginx/logs/ \
ADD conf/init/doupdatelimits.sh /etc/my_init.d/ && cp conf/init/*.sh /etc/my_init.d/ \
ADD conf/init/copy_syslog_config.sh /etc/my_init.d/ && cp conf/init/syslog-ng.conf /etc/syslog-ng/ \
ADD conf/init/certs_create.sh /etc/my_init.d/ && cp -r conf/init/service/* /etc/service \
ADD conf/init/certs_install.sh /etc/my_init.d/ && cp conf/kill-buildmanager.sh /usr/local/bin/kill-buildmanager.sh \
ADD conf/init/nginx_conf_create.sh /etc/my_init.d/ && cp conf/monitrc /etc/monit/monitrc \
ADD conf/init/runmigration.sh /etc/my_init.d/ && chmod 0600 /etc/monit/monitrc \
ADD conf/init/syslog-ng.conf /etc/syslog-ng/ && cp conf/init/logrotate.conf /etc/logrotate.conf \
ADD conf/init/zz_boot.sh /etc/my_init.d/ && cp .git/HEAD GIT_HEAD \
ADD conf/init/service/ /etc/service/ && rm -rf /etc/service/syslog-forwarder
RUN rm -rf /etc/service/syslog-forwarder
ADD conf/kill-buildmanager.sh /usr/local/bin/kill-buildmanager.sh
ADD conf/monitrc /etc/monit/monitrc
RUN chmod 0600 /etc/monit/monitrc
# remove after phusion/baseimage-docker#338 is fixed
ADD conf/init/logrotate.conf /etc/logrotate.conf
# Download any external libs.
RUN mkdir static/fonts static/ldn
ADD external_libraries.py external_libraries.py
RUN venv/bin/python -m external_libraries
RUN mkdir -p /usr/local/nginx/logs/
# TODO(ssewell): only works on a detached head, make work with ref
ADD .git/HEAD GIT_HEAD
# Add all of the files!
ADD . .
RUN pyclean .
# Cleanup any NPM-related stuff.
RUN rm -rf /root/.npm
RUN rm -rf /.npm
RUN rm -rf /usr/local/lib/node_modules
RUN rm -rf /usr/share/yarn/node_modules
RUN rm -rf /root/node_modules
RUN rm -rf /node_modules
RUN rm -rf /grunt
RUN rm package.json yarn.lock
# Run the tests # Run the tests
ENV RUN_ACI_TESTS False
ADD requirements-tests.txt requirements-tests.txt
RUN if [ "$RUN_TESTS" = true ]; then \
venv/bin/pip install -r requirements-tests.txt ;\
fi
RUN if [ "$RUN_TESTS" = true ]; then \ RUN if [ "$RUN_TESTS" = true ]; then \
TEST=true PYTHONPATH="." venv/bin/py.test --timeout=7200 --verbose \ TEST=true PYTHONPATH="." venv/bin/py.test --timeout=7200 --verbose \
--show-count -x --color=no ./ && rm -rf /var/tmp/; \ --show-count -x --color=no ./ && rm -rf /var/tmp/; \
fi
RUN if [ "$RUN_TESTS" = true ]; then \
TEST=true PYTHONPATH="." venv/bin/py.test --timeout=7200 --verbose \ TEST=true PYTHONPATH="." venv/bin/py.test --timeout=7200 --verbose \
--show-count -x --color=no test/registry_tests.py && rm -rf /var/tmp/;\ --show-count -x --color=no test/registry_tests.py && rm -rf /var/tmp/;\
yarn test; \
fi fi
RUN rm -rf /root/.cache
# Cleanup any NPM-related stuff.
# RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev libgpgme11-dev nodejs jpegoptim optipng w3m \
# && apt-get autoremove -y \
# && apt-get clean
# && rm -rf /root/.npm /.npm /usr/local/lib/node_modules /usr/share/yarn/node_modules \
# /root/node_modules /node_modules /grunt
RUN PYTHONPATH=. venv/bin/alembic heads | grep -E '^[0-9a-f]+ \(head\)$' > ALEMBIC_HEAD RUN PYTHONPATH=. venv/bin/alembic heads | grep -E '^[0-9a-f]+ \(head\)$' > ALEMBIC_HEAD
VOLUME ["/conf/stack", "/var/log", "/datastorage", "/tmp", "/conf/etcd"]
EXPOSE 443 8443 80

2
requirements-tests.txt
View file

@ -1,4 +1,3 @@
pytest
pytest-cov pytest-cov
python-coveralls python-coveralls
pytest-flask pytest-flask
@ -6,3 +5,4 @@ pytest-runner
pytest-xdist pytest-xdist
pytest-timeout pytest-timeout
-e git+https://github.com/ant31/pytest-sugar.git#egg=pytest-sugar -e git+https://github.com/ant31/pytest-sugar.git#egg=pytest-sugar
-e git+https://github.com/ant31/pytest.git#egg=pytest