Merge pull request #2679 from coreos-inc/split-dockerfile

Split dockerfile - e2e CI
2017-06-06 14:25:22 -07:00 · 2017-06-06 14:25:22 -07:00 · ba713b624d
commit ba713b624d
parent 6321d08024 7e85ac6f89
5 changed files with 219 additions and 51 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -23,3 +23,4 @@ coverage
 .npm-debug.log
 test/__pycache__
 __pycache__
+**/__pycache__
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -1,5 +1,6 @@
 ---
 stages:
+  - docker-build
  - unit-tests
  - integration
  - release
@ -8,55 +9,75 @@ stages:
 variables:
  FAILFASTCI_NAMESPACE: 'quay'
  IMAGE: quay.io/quay/quay
-  PIP_CACHE_DIR: /pip-cache
+  PIP_CACHE_DIR: pip-cache
  PIP: /venv/bin/pip
  PYTEST: /venv/bin/py.test
  TEST: "true"
  PYTHONPATH: "."

-cache:
-  paths:
-    - cache
-    - /pip-cache
-    - /apt-cache
-  key: "$CI_PROJECT_ID"

+# STAGE 1: container build

-.job: &job
+.docker: &docker
+  variables:
+    DOCKER_DRIVER: aufs
+  image: docker:git
  before_script:
-    - mkdir -p /apt-cache
-    - apt-get update
-    - apt-get install -o dir::cache::archives="/apt-cache" -y python-dev g++ libjpeg62-dev libevent-dev libldap2-dev  libsasl2-dev libpq-dev libffi-dev libgpgme11-dev nodejs jpegoptim optipng w3m
-    - pip install -r requirements.txt
-    - pip install -r requirements-tests.txt
+    - docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io
+  services:
+     - docker:dind
+  tags:
+    - docker
+
+container-base-build:
+  <<: *docker
+  stage: docker-build
  script:
-    - echo test
+    - docker build --cache-from quay.io/quay/quay-base:latest -t quay.io/quay/quay-base:latest -f quay-base.dockerfile .
+    - docker push quay.io/quay/quay-base:latest
+  when: manual
+
+container-build:
+  <<: *docker
+  stage: docker-build
+  script:
+    - docker build -t quay.io/quay/quay-ci:$CI_COMMIT_REF_SLUG -f quay.dockerfile .
+    - docker push quay.io/quay/quay-ci:$CI_COMMIT_REF_SLUG
+
+
+
+# STAGE 2: Unit tests & code-style
+.job: &job
+  variables:
+    GIT_STRATEGY: none
+  image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
+  before_script:
+    - cd /
+    - source venv/bin/activate
  tags:
    - kubernetes
-  image: quay.io/quay/quay:build
-

 unit-tests:
  <<: *job
  stage: unit-tests
  script:
-    - py.test --timeout=7200 --verbose --show-count ./ --color=no
+    - py.test --timeout=7200 --verbose --show-count ./ --color=no -x


 registry-tests:
  <<: *job
  stage: unit-tests
  script:
-    - py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no
+    - py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no -x


 karma-tests:
  <<: *job
  stage: unit-tests
  script:
-    - yarn install --ignore-engines
    - yarn test

+
 code-styles:
  <<: *job
  stage: unit-tests
@ -66,6 +87,7 @@ code-styles:
    - echo "pylint"


+# Stage 3: Integration/e2e tests
 postgres:
  <<: *job
  variables:
@ -73,60 +95,51 @@ postgres:
    SKIP_DB_SCHEMA: 'true'
    POSTGRES_PASSWORD: quay
    POSTGRES_USER: quay
+    GIT_STRATEGY: none
  stage: integration
  services:
    - postgres:9.6
  script:
+    - sleep 30
    - alembic upgrade head
-    - py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/
-
+    - PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x


 mysql:
  <<: *job
  variables:
-    TEST_DATABASE_URI: mysql+pymysql://quay:quay@localhost/quay
+    TEST_DATABASE_URI: 'mysql+pymysql://quay:quay@localhost/quay'
    SKIP_DB_SCHEMA: 'true'
    MYSQL_ROOT_PASSWORD: quay
    MYSQL_DATABASE: quay
    MYSQL_USER: quay
    MYSQL_PASSWORD: quay
+    GIT_STRATEGY: none
  stage: integration
  services:
    - mysql
  script:
+    - sleep 30
    - alembic upgrade head
-    - py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/
+    - PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x


-# # TODO (@ant31)
-# .docker: &docker
+# e2e-demo:
+#   <<: *job
+#   image: python:2.7
 #   variables:
-#     DOCKER_HOST: tcp://localhost:2375
-#   image: docker:git
+#     TEST_DATABASE_URI: 'postgresql://quay:quay@localhost/quay'
+#     SKIP_DB_SCHEMA: 'true'
+#     POSTGRES_PASSWORD: quay
+#     POSTGRES_USER: quay
+#     GIT_STRATEGY: none
+#   stage: integration
 #   before_script:
-#     - docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io
+#     - cd /
 #   services:
-#      - docker:dind
-#   tags:
-#     - kubernetes
-
-# docker-push:
-#   <<: *docker
-#   stage: release
+#     - postgres:9.6
+#     - quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
 #   script:
-#     - echo "push container to the quay prod repo"
-#     - docker build --no-cache -t quay.io/quay/quay-branches:$CI_BUILD_REF_NAME .
-#     - docker push quay.io/quay/quay-branches:$CI_BUILD_REF_NAME
-
-# helm-push:
-#   <<: *docker
-#   stage: release
-#   script:
-#     - echo "build chart"
-
-# deploy-staging:
-#   when: manual
-#   stage: deploy
-#   script:
-#     - echo "deploy-staging"
+#     - sleep 240
+#     - curl localhost:80/cnr/version
+#   allow_failure: true
--- a/quay-base.dockerfile
+++ b/quay-base.dockerfile
@ -0,0 +1,98 @@
+# vim:ft=dockerfile
+
+FROM phusion/baseimage:0.9.19
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV HOME /root
+
+# This is so we don't break http golang/go#17066
+# When Ubuntu has nginx >= 1.11.0 we can switch back.
+RUN  add-apt-repository ppa:nginx/development
+
+# Add Yarn repository until it is officially added to Ubuntu
+RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
+    && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
+
+# Install system packages
+RUN apt-get update && apt-get upgrade -y \
+    && apt-get install -y \
+     dnsmasq           \
+     g++               \
+     gdb               \
+     gdebi-core        \
+     git               \
+     jpegoptim         \
+     libevent-2.0.5    \
+     libevent-dev      \
+     libffi-dev        \
+     libfreetype6-dev  \
+     libgpgme11        \
+     libgpgme11-dev    \
+     libjpeg62         \
+     libjpeg62-dev     \
+     libjpeg8          \
+     libldap-2.4-2     \
+     libldap2-dev      \
+     libmagic1         \
+     libpq-dev         \
+     libpq5            \
+     libsasl2-dev      \
+     libsasl2-modules  \
+     monit             \
+     nginx             \
+     nodejs            \
+     optipng           \
+     openssl           \
+     python-dbg        \
+     python-dev        \
+     python-pip        \
+     python-virtualenv \
+     yarn=0.22.0-1     \
+     w3m # 26MAY2017
+
+# Install python dependencies
+COPY requirements.txt requirements-tests.txt ./
+RUN virtualenv --distribute venv \
+    && venv/bin/pip install -r requirements.txt \
+    && venv/bin/pip install -r requirements-tests.txt \
+    && venv/bin/pip freeze # 07SEP2016
+
+# Install cfssl
+RUN mkdir /gocode
+ENV GOPATH /gocode
+RUN curl -O https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz && \
+    tar -xvf go1.6.linux-amd64.tar.gz && \
+    mv go /usr/local && \
+    rm -rf go1.6.linux-amd64.tar.gz && \
+    /usr/local/go/bin/go get -u github.com/cloudflare/cfssl/cmd/cfssl && \
+    /usr/local/go/bin/go get -u github.com/cloudflare/cfssl/cmd/cfssljson && \
+    cp /gocode/bin/cfssljson /bin/cfssljson && \
+    cp /gocode/bin/cfssl /bin/cfssl && \
+    rm -rf /gocode && rm -rf /usr/local/go
+
+# Install jwtproxy
+RUN curl -L -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 \
+    && chmod +x /usr/local/bin/jwtproxy
+
+# Install prometheus-aggregator
+RUN curl -L -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator \
+    && chmod +x /usr/local/bin/prometheus-aggregator
+
+# Install front-end dependencies
+RUN ln -s /usr/bin/nodejs /usr/bin/node
+COPY static/ package.json tsconfig.json webpack.config.js typings.json yarn.lock ./
+RUN yarn install --ignore-engines
+
+
+RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit static/fonts static/ldn /usr/local/nginx/logs/
+
+COPY external_libraries.py ./
+RUN venv/bin/python -m external_libraries
+
+ARG RUN_TESTS=false
+ENV RUN_TESTS ${RUN_TESTS}
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache
+
+VOLUME ["/conf/stack", "/var/log", "/datastorage", "/tmp", "/conf/etcd"]
+
+EXPOSE 443 8443 80
--- a/quay.dockerfile
+++ b/quay.dockerfile
@ -0,0 +1,56 @@
+# vim:ft=dockerfile
+
+FROM quay.io/quay/quay-base:latest
+
+COPY . .
+
+# Install python dependencies
+RUN virtualenv --distribute venv \
+     && venv/bin/pip install -r requirements.txt \
+     && venv/bin/pip install -r requirements-tests.txt \
+     && venv/bin/pip freeze # 07SEP2016
+
+# Check python dependencies for the GPL
+# Due to the following bug, pip results must be piped to a file before grepping:
+# https://github.com/pypa/pip/pull/3304
+RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/bin/pip --disable-pip-version-check show > pipinfo.txt && \
+    test -z $(cat pipinfo.txt | grep GPL | grep -v LGPL) && \
+    rm pipinfo.txt
+
+# Install front-end dependencies and Optimize our images
+RUN yarn install --ignore-engines \
+    && yarn build \
+    && jpegoptim static/img/**/*.jpg \
+    && optipng -clobber -quiet static/img/**/*.png
+
+# Set up the init system
+RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit static/fonts static/ldn /usr/local/nginx/logs/ \
+    && cp conf/init/*.sh /etc/my_init.d/ \
+    && cp conf/init/syslog-ng.conf /etc/syslog-ng/ \
+    && cp -r conf/init/service/* /etc/service \
+    && cp conf/kill-buildmanager.sh /usr/local/bin/kill-buildmanager.sh \
+    && cp conf/monitrc /etc/monit/monitrc \
+    && chmod 0600 /etc/monit/monitrc \
+    && cp conf/init/logrotate.conf /etc/logrotate.conf \
+    && cp .git/HEAD GIT_HEAD \
+    && rm -rf /etc/service/syslog-forwarder
+
+# Run the tests
+RUN if [ "$RUN_TESTS" = true ]; then \
+      TEST=true PYTHONPATH="." venv/bin/py.test --timeout=7200 --verbose \
+                --show-count -x --color=no ./ && rm -rf /var/tmp/; \
+
+      TEST=true PYTHONPATH="." venv/bin/py.test  --timeout=7200 --verbose \
+                --show-count -x --color=no test/registry_tests.py && rm -rf /var/tmp/;\
+      yarn test; \
+    fi
+
+
+# Cleanup any NPM-related stuff.
+# RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev libgpgme11-dev nodejs jpegoptim optipng w3m \
+#     && apt-get autoremove -y \
+#     && apt-get clean
+#     && rm -rf /root/.npm /.npm /usr/local/lib/node_modules /usr/share/yarn/node_modules \
+#            /root/node_modules /node_modules /grunt
+
+RUN PYTHONPATH=. venv/bin/alembic heads | grep -E '^[0-9a-f]+ \(head\)$' > ALEMBIC_HEAD
--- a/requirements-tests.txt
+++ b/requirements-tests.txt
@ -1,4 +1,3 @@
-pytest
 pytest-cov
 python-coveralls
 pytest-flask
@ -6,3 +5,4 @@ pytest-runner
 pytest-xdist
 pytest-timeout
 -e git+https://github.com/ant31/pytest-sugar.git#egg=pytest-sugar
+-e git+https://github.com/ant31/pytest.git#egg=pytest