From 7e85ac6f89d9991366a2addf5baf668fe2a1dff7 Mon Sep 17 00:00:00 2001 From: Antoine Legrand <2t.antoine@gmail.com> Date: Tue, 30 May 2017 23:06:09 +0200 Subject: [PATCH] Add gitlab-ci docker-build --- .dockerignore | 1 + .gitlab-ci.yml | 113 ++++++++++++++++++++----------------- quay-base.dockerfile | 58 ++++++++----------- quay.dockerfile | 124 ++++++++++------------------------------- requirements-tests.txt | 2 +- 5 files changed, 116 insertions(+), 182 deletions(-) diff --git a/.dockerignore b/.dockerignore index b84c81eb2..911b9d41e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -23,3 +23,4 @@ coverage .npm-debug.log test/__pycache__ __pycache__ +**/__pycache__ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0b8336c30..85f91651f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,6 @@ --- stages: + - docker-build - unit-tests - integration - release @@ -8,55 +9,75 @@ stages: variables: FAILFASTCI_NAMESPACE: 'quay' IMAGE: quay.io/quay/quay - PIP_CACHE_DIR: /pip-cache + PIP_CACHE_DIR: pip-cache PIP: /venv/bin/pip PYTEST: /venv/bin/py.test TEST: "true" PYTHONPATH: "." -cache: - paths: - - cache - - /pip-cache - - /apt-cache - key: "$CI_PROJECT_ID" +# STAGE 1: container build -.job: &job +.docker: &docker + variables: + DOCKER_DRIVER: aufs + image: docker:git before_script: - - mkdir -p /apt-cache - - apt-get update - - apt-get install -o dir::cache::archives="/apt-cache" -y python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev libgpgme11-dev nodejs jpegoptim optipng w3m - - pip install -r requirements.txt - - pip install -r requirements-tests.txt + - docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io + services: + - docker:dind + tags: + - docker + +container-base-build: + <<: *docker + stage: docker-build script: - - echo test + - docker build --cache-from quay.io/quay/quay-base:latest -t quay.io/quay/quay-base:latest -f quay-base.dockerfile . + - docker push quay.io/quay/quay-base:latest + when: manual + +container-build: + <<: *docker + stage: docker-build + script: + - docker build -t quay.io/quay/quay-ci:$CI_COMMIT_REF_SLUG -f quay.dockerfile . + - docker push quay.io/quay/quay-ci:$CI_COMMIT_REF_SLUG + + + +# STAGE 2: Unit tests & code-style +.job: &job + variables: + GIT_STRATEGY: none + image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} + before_script: + - cd / + - source venv/bin/activate tags: - kubernetes - image: quay.io/quay/quay:build - unit-tests: <<: *job stage: unit-tests script: - - py.test --timeout=7200 --verbose --show-count ./ --color=no + - py.test --timeout=7200 --verbose --show-count ./ --color=no -x registry-tests: <<: *job stage: unit-tests script: - - py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no + - py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no -x karma-tests: <<: *job stage: unit-tests script: - - yarn install --ignore-engines - yarn test + code-styles: <<: *job stage: unit-tests @@ -66,6 +87,7 @@ code-styles: - echo "pylint" +# Stage 3: Integration/e2e tests postgres: <<: *job variables: @@ -73,60 +95,51 @@ postgres: SKIP_DB_SCHEMA: 'true' POSTGRES_PASSWORD: quay POSTGRES_USER: quay + GIT_STRATEGY: none stage: integration services: - postgres:9.6 script: + - sleep 30 - alembic upgrade head - - py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ - + - PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x mysql: <<: *job variables: - TEST_DATABASE_URI: mysql+pymysql://quay:quay@localhost/quay + TEST_DATABASE_URI: 'mysql+pymysql://quay:quay@localhost/quay' SKIP_DB_SCHEMA: 'true' MYSQL_ROOT_PASSWORD: quay MYSQL_DATABASE: quay MYSQL_USER: quay MYSQL_PASSWORD: quay + GIT_STRATEGY: none stage: integration services: - mysql script: + - sleep 30 - alembic upgrade head - - py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ + - PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x -# # TODO (@ant31) -# .docker: &docker +# e2e-demo: +# <<: *job +# image: python:2.7 # variables: -# DOCKER_HOST: tcp://localhost:2375 -# image: docker:git +# TEST_DATABASE_URI: 'postgresql://quay:quay@localhost/quay' +# SKIP_DB_SCHEMA: 'true' +# POSTGRES_PASSWORD: quay +# POSTGRES_USER: quay +# GIT_STRATEGY: none +# stage: integration # before_script: -# - docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io +# - cd / # services: -# - docker:dind -# tags: -# - kubernetes - -# docker-push: -# <<: *docker -# stage: release +# - postgres:9.6 +# - quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} # script: -# - echo "push container to the quay prod repo" -# - docker build --no-cache -t quay.io/quay/quay-branches:$CI_BUILD_REF_NAME . -# - docker push quay.io/quay/quay-branches:$CI_BUILD_REF_NAME - -# helm-push: -# <<: *docker -# stage: release -# script: -# - echo "build chart" - -# deploy-staging: -# when: manual -# stage: deploy -# script: -# - echo "deploy-staging" +# - sleep 240 +# - curl localhost:80/cnr/version +# allow_failure: true diff --git a/quay-base.dockerfile b/quay-base.dockerfile index 88bbf2e1c..8fde99c75 100644 --- a/quay-base.dockerfile +++ b/quay-base.dockerfile @@ -10,12 +10,12 @@ ENV HOME /root RUN add-apt-repository ppa:nginx/development # Add Yarn repository until it is officially added to Ubuntu -RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list +RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \ + && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list # Install system packages -RUN apt-get update && apt-get upgrade -y # 26MAY2017 -RUN apt-get install -y \ +RUN apt-get update && apt-get upgrade -y \ + && apt-get install -y \ dnsmasq \ g++ \ gdb \ @@ -48,20 +48,14 @@ RUN apt-get install -y \ python-pip \ python-virtualenv \ yarn=0.22.0-1 \ - w3m + w3m # 26MAY2017 # Install python dependencies -ADD requirements.txt requirements.txt -RUN virtualenv --distribute venv -RUN venv/bin/pip install -r requirements.txt # 07SEP2016 -RUN venv/bin/pip freeze - -# Check python dependencies for the GPL -# Due to the following bug, pip results must be piped to a file before grepping: -# https://github.com/pypa/pip/pull/3304 -RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/bin/pip --disable-pip-version-check show > pipinfo.txt && \ - test -z $(cat pipinfo.txt | grep GPL | grep -v LGPL) && \ - rm pipinfo.txt +COPY requirements.txt requirements-tests.txt ./ +RUN virtualenv --distribute venv \ + && venv/bin/pip install -r requirements.txt \ + && venv/bin/pip install -r requirements-tests.txt \ + && venv/bin/pip freeze # 07SEP2016 # Install cfssl RUN mkdir /gocode @@ -77,34 +71,28 @@ RUN curl -O https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz && \ rm -rf /gocode && rm -rf /usr/local/go # Install jwtproxy -RUN curl -L -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 -RUN chmod +x /usr/local/bin/jwtproxy +RUN curl -L -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 \ + && chmod +x /usr/local/bin/jwtproxy # Install prometheus-aggregator -RUN curl -L -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator -RUN chmod +x /usr/local/bin/prometheus-aggregator +RUN curl -L -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator \ + && chmod +x /usr/local/bin/prometheus-aggregator # Install front-end dependencies RUN ln -s /usr/bin/nodejs /usr/bin/node -ADD package.json package.json -ADD tsconfig.json tsconfig.json -ADD webpack.config.js webpack.config.js -ADD typings.json typings.json -ADD yarn.lock yarn.lock +COPY static/ package.json tsconfig.json webpack.config.js typings.json yarn.lock ./ RUN yarn install --ignore-engines -# Add static files -ADD static static -# Run Webpack -RUN yarn build +RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit static/fonts static/ldn /usr/local/nginx/logs/ -# Optimize our images -ADD static/img static/img -RUN jpegoptim static/img/**/*.jpg -RUN optipng -clobber -quiet static/img/**/*.png - -ADD external_libraries.py external_libraries.py +COPY external_libraries.py ./ RUN venv/bin/python -m external_libraries +ARG RUN_TESTS=false +ENV RUN_TESTS ${RUN_TESTS} +RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache + +VOLUME ["/conf/stack", "/var/log", "/datastorage", "/tmp", "/conf/etcd"] + EXPOSE 443 8443 80 diff --git a/quay.dockerfile b/quay.dockerfile index 3fe6fcb88..3fc330e27 100644 --- a/quay.dockerfile +++ b/quay.dockerfile @@ -2,15 +2,13 @@ FROM quay.io/quay/quay-base:latest -ENV DEBIAN_FRONTEND noninteractive -ENV HOME /root - +COPY . . # Install python dependencies -ADD requirements.txt requirements.txt -RUN virtualenv --distribute venv -RUN venv/bin/pip install -r requirements.txt # 07SEP2016 -RUN venv/bin/pip freeze +RUN virtualenv --distribute venv \ + && venv/bin/pip install -r requirements.txt \ + && venv/bin/pip install -r requirements-tests.txt \ + && venv/bin/pip freeze # 07SEP2016 # Check python dependencies for the GPL # Due to the following bug, pip results must be piped to a file before grepping: @@ -19,106 +17,40 @@ RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/ test -z $(cat pipinfo.txt | grep GPL | grep -v LGPL) && \ rm pipinfo.txt -# Install front-end dependencies -RUN ln -s /usr/bin/nodejs /usr/bin/node -ADD package.json package.json -ADD tsconfig.json tsconfig.json -ADD webpack.config.js webpack.config.js -ADD typings.json typings.json -ADD yarn.lock yarn.lock -RUN yarn install --ignore-engines - -# Add static files -ADD static static - -# Run Webpack -RUN yarn build - -# Run front-end tests -ARG RUN_TESTS=false -ENV RUN_TESTS ${RUN_TESTS} - -ADD karma.conf.js karma.conf.js -RUN if [ "$RUN_TESTS" = true ]; then \ - yarn test; \ - fi - -# Optimize our images -ADD static/img static/img -RUN jpegoptim static/img/**/*.jpg -RUN optipng -clobber -quiet static/img/**/*.png - -RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev libgpgme11-dev nodejs jpegoptim optipng w3m -RUN apt-get autoremove -y -RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +# Install front-end dependencies and Optimize our images +RUN yarn install --ignore-engines \ + && yarn build \ + && jpegoptim static/img/**/*.jpg \ + && optipng -clobber -quiet static/img/**/*.png # Set up the init system -ADD conf/init/copy_config_files.sh /etc/my_init.d/ -ADD conf/init/doupdatelimits.sh /etc/my_init.d/ -ADD conf/init/copy_syslog_config.sh /etc/my_init.d/ -ADD conf/init/certs_create.sh /etc/my_init.d/ -ADD conf/init/certs_install.sh /etc/my_init.d/ -ADD conf/init/nginx_conf_create.sh /etc/my_init.d/ -ADD conf/init/runmigration.sh /etc/my_init.d/ -ADD conf/init/syslog-ng.conf /etc/syslog-ng/ -ADD conf/init/zz_boot.sh /etc/my_init.d/ -ADD conf/init/service/ /etc/service/ -RUN rm -rf /etc/service/syslog-forwarder - -ADD conf/kill-buildmanager.sh /usr/local/bin/kill-buildmanager.sh -ADD conf/monitrc /etc/monit/monitrc -RUN chmod 0600 /etc/monit/monitrc - -# remove after phusion/baseimage-docker#338 is fixed -ADD conf/init/logrotate.conf /etc/logrotate.conf - -# Download any external libs. -RUN mkdir static/fonts static/ldn -ADD external_libraries.py external_libraries.py -RUN venv/bin/python -m external_libraries -RUN mkdir -p /usr/local/nginx/logs/ - -# TODO(ssewell): only works on a detached head, make work with ref -ADD .git/HEAD GIT_HEAD - -# Add all of the files! -ADD . . -RUN pyclean . - -# Cleanup any NPM-related stuff. -RUN rm -rf /root/.npm -RUN rm -rf /.npm -RUN rm -rf /usr/local/lib/node_modules -RUN rm -rf /usr/share/yarn/node_modules -RUN rm -rf /root/node_modules -RUN rm -rf /node_modules -RUN rm -rf /grunt -RUN rm package.json yarn.lock +RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit static/fonts static/ldn /usr/local/nginx/logs/ \ + && cp conf/init/*.sh /etc/my_init.d/ \ + && cp conf/init/syslog-ng.conf /etc/syslog-ng/ \ + && cp -r conf/init/service/* /etc/service \ + && cp conf/kill-buildmanager.sh /usr/local/bin/kill-buildmanager.sh \ + && cp conf/monitrc /etc/monit/monitrc \ + && chmod 0600 /etc/monit/monitrc \ + && cp conf/init/logrotate.conf /etc/logrotate.conf \ + && cp .git/HEAD GIT_HEAD \ + && rm -rf /etc/service/syslog-forwarder # Run the tests -ENV RUN_ACI_TESTS False -ADD requirements-tests.txt requirements-tests.txt - -RUN if [ "$RUN_TESTS" = true ]; then \ - venv/bin/pip install -r requirements-tests.txt ;\ - fi - - RUN if [ "$RUN_TESTS" = true ]; then \ TEST=true PYTHONPATH="." venv/bin/py.test --timeout=7200 --verbose \ --show-count -x --color=no ./ && rm -rf /var/tmp/; \ - fi -RUN if [ "$RUN_TESTS" = true ]; then \ TEST=true PYTHONPATH="." venv/bin/py.test --timeout=7200 --verbose \ --show-count -x --color=no test/registry_tests.py && rm -rf /var/tmp/;\ - + yarn test; \ fi -RUN rm -rf /root/.cache + +# Cleanup any NPM-related stuff. +# RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev libgpgme11-dev nodejs jpegoptim optipng w3m \ +# && apt-get autoremove -y \ +# && apt-get clean +# && rm -rf /root/.npm /.npm /usr/local/lib/node_modules /usr/share/yarn/node_modules \ +# /root/node_modules /node_modules /grunt RUN PYTHONPATH=. venv/bin/alembic heads | grep -E '^[0-9a-f]+ \(head\)$' > ALEMBIC_HEAD - -VOLUME ["/conf/stack", "/var/log", "/datastorage", "/tmp", "/conf/etcd"] - -EXPOSE 443 8443 80 diff --git a/requirements-tests.txt b/requirements-tests.txt index bb613f9cc..435f7c1fe 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,4 +1,3 @@ -pytest pytest-cov python-coveralls pytest-flask @@ -6,3 +5,4 @@ pytest-runner pytest-xdist pytest-timeout -e git+https://github.com/ant31/pytest-sugar.git#egg=pytest-sugar +-e git+https://github.com/ant31/pytest.git#egg=pytest