diff --git a/.gitlab-ci.jsonnet b/.gitlab-ci.jsonnet index 7b4f63fff..f669df0a1 100644 --- a/.gitlab-ci.jsonnet +++ b/.gitlab-ci.jsonnet @@ -42,7 +42,7 @@ local jobs = { // Docker Tag is the branch/tag name stage: stages.docker_build, script: [ - 'docker build -t %s -f quay.dockerfile .' % images.quayci.name, + 'docker build -t %s -f Dockerfile .' % images.quayci.name, 'docker push %s' % images.quayci.name], }, diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79d9f6fc6..ca92ee270 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,7 +22,7 @@ container-build: - docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io image: docker:git script: - - docker build -t quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} -f quay.dockerfile . + - docker build -t quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} -f Dockerfile . - docker push quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} stage: docker_build tags: @@ -49,8 +49,8 @@ container-release: DOCKER_HOST: tcp://docker-host.gitlab-runner.svc.cluster.local:2375 karma-tests: before_script: - - cd / - - source venv/bin/activate + - cd $QUAYDIR + - source $QUAYDIR/venv/bin/activate image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} script: - curl -Ss https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - @@ -68,8 +68,8 @@ karma-tests: TEST: 'true' mysql: before_script: - - cd / - - source venv/bin/activate + - cd $QUAYDIR + - source $QUAYDIR/venv/bin/activate image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} script: - sleep 30 @@ -93,8 +93,8 @@ mysql: TEST_DATABASE_URI: mysql+pymysql://quay:quay@localhost/quay postgres: before_script: - - cd / - - source venv/bin/activate + - cd $QUAYDIR + - source $QUAYDIR/venv/bin/activate image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} script: - sleep 30 @@ -116,8 +116,8 @@ postgres: TEST_DATABASE_URI: postgresql://quay:quay@localhost/quay registry-tests: before_script: - - cd / - - source venv/bin/activate + - cd $QUAYDIR + - source $QUAYDIR/venv/bin/activate image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} script: - py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no -x @@ -138,8 +138,8 @@ stages: - teardown unit-tests: before_script: - - cd / - - source venv/bin/activate + - cd $QUAYDIR + - source $QUAYDIR/venv/bin/activate image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} script: - py.test --timeout=7200 --verbose --show-count ./ --color=no -x diff --git a/.gitlab-ci/base_jobs.libsonnet b/.gitlab-ci/base_jobs.libsonnet index 51d22a2a1..d90a3eef1 100644 --- a/.gitlab-ci/base_jobs.libsonnet +++ b/.gitlab-ci/base_jobs.libsonnet @@ -27,8 +27,8 @@ function(vars={}) GIT_STRATEGY: "none", }, before_script: [ - "cd /", - "source venv/bin/activate", + "cd $QUAYDIR", + "source $QUAYDIR/venv/bin/activate", ], tags: [ "kubernetes", diff --git a/Dockerfile b/Dockerfile index 191a5cafe..bef534802 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,67 +1,15 @@ # vim:ft=dockerfile -FROM phusion/baseimage:0.9.19 +FROM quay.io/quay/quay-base:latest -ENV DEBIAN_FRONTEND noninteractive -ENV HOME /root -ENV QUAYCONF /quay/conf -ENV QUAYDIR /quay -ENV QUAYPATH "." - -RUN mkdir $QUAYDIR WORKDIR $QUAYDIR - -# This is so we don't break http golang/go#17066 -# When Ubuntu has nginx >= 1.11.0 we can switch back. -RUN add-apt-repository ppa:nginx/development - -# Add Yarn repository until it is officially added to Ubuntu -RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list - -# Install system packages -RUN apt-get update && apt-get upgrade -y # 26MAY2017 -RUN apt-get install -y \ - dnsmasq \ - g++ \ - gdb \ - gdebi-core \ - git \ - jpegoptim \ - libevent-2.0.5 \ - libevent-dev \ - libffi-dev \ - libfreetype6-dev \ - libgpgme11 \ - libgpgme11-dev \ - libjpeg62 \ - libjpeg62-dev \ - libjpeg8 \ - libldap-2.4-2 \ - libldap2-dev \ - libmagic1 \ - libpq-dev \ - libpq5 \ - libsasl2-dev \ - libsasl2-modules \ - monit \ - nginx \ - nodejs \ - optipng \ - openssl \ - python-dbg \ - python-dev \ - python-pip \ - python-virtualenv \ - yarn=0.22.0-1 \ - w3m +COPY . . # Install python dependencies - -ADD requirements.txt requirements.txt -RUN virtualenv --distribute venv -RUN venv/bin/pip install -r requirements.txt # 07SEP2016 -RUN venv/bin/pip freeze +RUN virtualenv --distribute venv \ + && venv/bin/pip install -r requirements.txt \ + && venv/bin/pip install -r requirements-tests.txt \ + && venv/bin/pip freeze # Check python dependencies for the GPL # Due to the following bug, pip results must be piped to a file before grepping: @@ -70,120 +18,29 @@ RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/ test -z $(cat pipinfo.txt | grep GPL | grep -v LGPL) && \ rm pipinfo.txt -# Install cfssl -RUN mkdir /gocode -ENV GOPATH /gocode -RUN curl -O https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz && \ - tar -xvf go1.6.linux-amd64.tar.gz && \ - mv go /usr/local && \ - rm -rf go1.6.linux-amd64.tar.gz && \ - /usr/local/go/bin/go get -u github.com/cloudflare/cfssl/cmd/cfssl && \ - /usr/local/go/bin/go get -u github.com/cloudflare/cfssl/cmd/cfssljson && \ - cp /gocode/bin/cfssljson /bin/cfssljson && \ - cp /gocode/bin/cfssl /bin/cfssl && \ - rm -rf /gocode && rm -rf /usr/local/go - -# Install jwtproxy -RUN curl -L -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 -RUN chmod +x /usr/local/bin/jwtproxy - -# Install prometheus-aggregator -RUN curl -L -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator -RUN chmod +x /usr/local/bin/prometheus-aggregator - -# Install front-end dependencies -RUN ln -s /usr/bin/nodejs /usr/bin/node -ADD package.json package.json -ADD tsconfig.json tsconfig.json -ADD webpack.config.js webpack.config.js -ADD yarn.lock yarn.lock -RUN yarn install --ignore-engines - -# Add static files -ADD static static - -# Run Webpack -RUN yarn build - -# Optimize our images -ADD static/img static/img -RUN jpegoptim static/img/**/*.jpg -RUN optipng -clobber -quiet static/img/**/*.png - -RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev libgpgme11-dev nodejs jpegoptim optipng w3m -RUN apt-get autoremove -y -RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +# Install front-end dependencies and Optimize our images +RUN yarn install --ignore-engines \ + && yarn build \ + && jpegoptim static/img/**/*.jpg \ + && optipng -clobber -quiet static/img/**/*.png # Set up the init system -ADD conf/init/copy_config_files.sh /etc/my_init.d/ -ADD conf/init/doupdatelimits.sh /etc/my_init.d/ -ADD conf/init/copy_syslog_config.sh /etc/my_init.d/ -ADD conf/init/certs_create.sh /etc/my_init.d/ -ADD conf/init/certs_install.sh /etc/my_init.d/ -ADD conf/init/nginx_conf_create.sh /etc/my_init.d/ -ADD conf/init/runmigration.sh /etc/my_init.d/ -ADD conf/init/syslog-ng.conf /etc/syslog-ng/ -ADD conf/init/zz_boot.sh /etc/my_init.d/ -ADD conf/init/service/ /etc/service/ -RUN rm -rf /etc/service/syslog-forwarder - -ADD conf/kill-buildmanager.sh /usr/local/bin/kill-buildmanager.sh -ADD conf/monitrc /etc/monit/monitrc -RUN chmod 0600 /etc/monit/monitrc - -# remove after phusion/baseimage-docker#338 is fixed -ADD conf/init/logrotate.conf /etc/logrotate.conf - -# TODO(ssewell): only works on a detached head, make work with ref -ADD .git/HEAD GIT_HEAD - -# Add all of the files! -ADD . . -RUN mkdir static/fonts static/ldn - -# Download any external libs. -RUN venv/bin/python -m external_libraries -RUN mkdir -p /usr/local/nginx/logs/ - - -RUN pyclean . +RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit static/fonts static/ldn /usr/local/nginx/logs/ \ + && cp $QUAYCONF/init/*.sh /etc/my_init.d/ \ + && cp $QUAYCONF/init/syslog-ng.conf /etc/syslog-ng/ \ + && cp -r $QUAYCONF/init/service/* /etc/service \ + && cp $QUAYCONF/kill-buildmanager.sh /usr/local/bin/kill-buildmanager.sh \ + && cp $QUAYCONF/monitrc /etc/monit/monitrc \ + && chmod 0600 /etc/monit/monitrc \ + && cp $QUAYCONF/init/logrotate.conf /etc/logrotate.conf \ + && cp .git/HEAD GIT_HEAD \ + && rm -rf /etc/service/syslog-forwarder # Cleanup any NPM-related stuff. -RUN rm -rf /root/.npm -RUN rm -rf .npm -RUN rm -rf /usr/local/lib/node_modules -RUN rm -rf /usr/share/yarn/node_modules -RUN rm -rf /root/node_modules -RUN rm -rf node_modules -RUN rm -rf grunt -RUN rm package.json yarn.lock - -# Run the tests -ARG RUN_TESTS=true -ENV RUN_TESTS ${RUN_TESTS} - -ENV RUN_ACI_TESTS False -ADD requirements-tests.txt requirements-tests.txt - -RUN if [ "$RUN_TESTS" = true ]; then \ - venv/bin/pip install -r requirements-tests.txt ;\ - fi - -RUN if [ "$RUN_TESTS" = true ]; then \ - TEST=true PYTHONPATH="." venv/bin/py.test --timeout=7200 --verbose \ - --show-count -x --color=no ./ && rm -rf /var/tmp/; \ - fi - -RUN if [ "$RUN_TESTS" = true ]; then \ - TEST=true PYTHONPATH="." venv/bin/py.test --timeout=7200 --verbose \ - --show-count -x --color=no test/registry_tests.py && rm -rf /var/tmp/;\ - - fi - -RUN rm -rf /root/.cache - -RUN PYTHONPATH=. venv/bin/alembic heads | grep -E '^[0-9a-f]+ \(head\)$' > ALEMBIC_HEAD - -VOLUME ["/conf/stack", "/var/log", "/datastorage", "/tmp", "/conf/etcd"] - -EXPOSE 443 8443 80 +# RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev libgpgme11-dev nodejs jpegoptim optipng w3m \ +# && apt-get autoremove -y \ +# && apt-get clean +# && rm -rf /root/.npm /.npm /usr/local/lib/node_modules /usr/share/yarn/node_modules \ +# /root/node_modules /node_modules /grunt +RUN ln -s $QUAYCONF /conf +RUN PYTHONPATH=$QUAYPATH venv/bin/alembic heads | grep -E '^[0-9a-f]+ \(head\)$' > ALEMBIC_HEAD diff --git a/Dockerfile.old b/Dockerfile.old new file mode 100644 index 000000000..191a5cafe --- /dev/null +++ b/Dockerfile.old @@ -0,0 +1,189 @@ +# vim:ft=dockerfile + +FROM phusion/baseimage:0.9.19 + +ENV DEBIAN_FRONTEND noninteractive +ENV HOME /root +ENV QUAYCONF /quay/conf +ENV QUAYDIR /quay +ENV QUAYPATH "." + +RUN mkdir $QUAYDIR +WORKDIR $QUAYDIR + +# This is so we don't break http golang/go#17066 +# When Ubuntu has nginx >= 1.11.0 we can switch back. +RUN add-apt-repository ppa:nginx/development + +# Add Yarn repository until it is officially added to Ubuntu +RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - +RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list + +# Install system packages +RUN apt-get update && apt-get upgrade -y # 26MAY2017 +RUN apt-get install -y \ + dnsmasq \ + g++ \ + gdb \ + gdebi-core \ + git \ + jpegoptim \ + libevent-2.0.5 \ + libevent-dev \ + libffi-dev \ + libfreetype6-dev \ + libgpgme11 \ + libgpgme11-dev \ + libjpeg62 \ + libjpeg62-dev \ + libjpeg8 \ + libldap-2.4-2 \ + libldap2-dev \ + libmagic1 \ + libpq-dev \ + libpq5 \ + libsasl2-dev \ + libsasl2-modules \ + monit \ + nginx \ + nodejs \ + optipng \ + openssl \ + python-dbg \ + python-dev \ + python-pip \ + python-virtualenv \ + yarn=0.22.0-1 \ + w3m + +# Install python dependencies + +ADD requirements.txt requirements.txt +RUN virtualenv --distribute venv +RUN venv/bin/pip install -r requirements.txt # 07SEP2016 +RUN venv/bin/pip freeze + +# Check python dependencies for the GPL +# Due to the following bug, pip results must be piped to a file before grepping: +# https://github.com/pypa/pip/pull/3304 +RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/bin/pip --disable-pip-version-check show > pipinfo.txt && \ + test -z $(cat pipinfo.txt | grep GPL | grep -v LGPL) && \ + rm pipinfo.txt + +# Install cfssl +RUN mkdir /gocode +ENV GOPATH /gocode +RUN curl -O https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz && \ + tar -xvf go1.6.linux-amd64.tar.gz && \ + mv go /usr/local && \ + rm -rf go1.6.linux-amd64.tar.gz && \ + /usr/local/go/bin/go get -u github.com/cloudflare/cfssl/cmd/cfssl && \ + /usr/local/go/bin/go get -u github.com/cloudflare/cfssl/cmd/cfssljson && \ + cp /gocode/bin/cfssljson /bin/cfssljson && \ + cp /gocode/bin/cfssl /bin/cfssl && \ + rm -rf /gocode && rm -rf /usr/local/go + +# Install jwtproxy +RUN curl -L -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 +RUN chmod +x /usr/local/bin/jwtproxy + +# Install prometheus-aggregator +RUN curl -L -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator +RUN chmod +x /usr/local/bin/prometheus-aggregator + +# Install front-end dependencies +RUN ln -s /usr/bin/nodejs /usr/bin/node +ADD package.json package.json +ADD tsconfig.json tsconfig.json +ADD webpack.config.js webpack.config.js +ADD yarn.lock yarn.lock +RUN yarn install --ignore-engines + +# Add static files +ADD static static + +# Run Webpack +RUN yarn build + +# Optimize our images +ADD static/img static/img +RUN jpegoptim static/img/**/*.jpg +RUN optipng -clobber -quiet static/img/**/*.png + +RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev libgpgme11-dev nodejs jpegoptim optipng w3m +RUN apt-get autoremove -y +RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Set up the init system +ADD conf/init/copy_config_files.sh /etc/my_init.d/ +ADD conf/init/doupdatelimits.sh /etc/my_init.d/ +ADD conf/init/copy_syslog_config.sh /etc/my_init.d/ +ADD conf/init/certs_create.sh /etc/my_init.d/ +ADD conf/init/certs_install.sh /etc/my_init.d/ +ADD conf/init/nginx_conf_create.sh /etc/my_init.d/ +ADD conf/init/runmigration.sh /etc/my_init.d/ +ADD conf/init/syslog-ng.conf /etc/syslog-ng/ +ADD conf/init/zz_boot.sh /etc/my_init.d/ +ADD conf/init/service/ /etc/service/ +RUN rm -rf /etc/service/syslog-forwarder + +ADD conf/kill-buildmanager.sh /usr/local/bin/kill-buildmanager.sh +ADD conf/monitrc /etc/monit/monitrc +RUN chmod 0600 /etc/monit/monitrc + +# remove after phusion/baseimage-docker#338 is fixed +ADD conf/init/logrotate.conf /etc/logrotate.conf + +# TODO(ssewell): only works on a detached head, make work with ref +ADD .git/HEAD GIT_HEAD + +# Add all of the files! +ADD . . +RUN mkdir static/fonts static/ldn + +# Download any external libs. +RUN venv/bin/python -m external_libraries +RUN mkdir -p /usr/local/nginx/logs/ + + +RUN pyclean . + +# Cleanup any NPM-related stuff. +RUN rm -rf /root/.npm +RUN rm -rf .npm +RUN rm -rf /usr/local/lib/node_modules +RUN rm -rf /usr/share/yarn/node_modules +RUN rm -rf /root/node_modules +RUN rm -rf node_modules +RUN rm -rf grunt +RUN rm package.json yarn.lock + +# Run the tests +ARG RUN_TESTS=true +ENV RUN_TESTS ${RUN_TESTS} + +ENV RUN_ACI_TESTS False +ADD requirements-tests.txt requirements-tests.txt + +RUN if [ "$RUN_TESTS" = true ]; then \ + venv/bin/pip install -r requirements-tests.txt ;\ + fi + +RUN if [ "$RUN_TESTS" = true ]; then \ + TEST=true PYTHONPATH="." venv/bin/py.test --timeout=7200 --verbose \ + --show-count -x --color=no ./ && rm -rf /var/tmp/; \ + fi + +RUN if [ "$RUN_TESTS" = true ]; then \ + TEST=true PYTHONPATH="." venv/bin/py.test --timeout=7200 --verbose \ + --show-count -x --color=no test/registry_tests.py && rm -rf /var/tmp/;\ + + fi + +RUN rm -rf /root/.cache + +RUN PYTHONPATH=. venv/bin/alembic heads | grep -E '^[0-9a-f]+ \(head\)$' > ALEMBIC_HEAD + +VOLUME ["/conf/stack", "/var/log", "/datastorage", "/tmp", "/conf/etcd"] + +EXPOSE 443 8443 80 diff --git a/quay-base.dockerfile b/quay-base.dockerfile index 20bf89e63..c2332353f 100644 --- a/quay-base.dockerfile +++ b/quay-base.dockerfile @@ -98,7 +98,6 @@ COPY external_libraries.py _init.py ./ RUN venv/bin/python -m external_libraries RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache - -VOLUME ["$QUAYCONF/stack", "/var/log", "/datastorage", "/tmp", "$QUAYCONF/etcd"] +VOLUME ["$QUAYCONF/stack", "/conf/stack", "/conf/etcd", "/var/log", "/datastorage", "/tmp", "$QUAYCONF/etcd"] EXPOSE 443 8443 80 diff --git a/quay.dockerfile b/quay.dockerfile deleted file mode 100644 index b5aad69d3..000000000 --- a/quay.dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -# vim:ft=dockerfile - -FROM quay.io/quay/quay-base:absolute - -WORKDIR $QUAYDIR -COPY . . - -# Install python dependencies -RUN virtualenv --distribute venv \ - && venv/bin/pip install -r requirements.txt \ - && venv/bin/pip install -r requirements-tests.txt \ - && venv/bin/pip freeze - -# Check python dependencies for the GPL -# Due to the following bug, pip results must be piped to a file before grepping: -# https://github.com/pypa/pip/pull/3304 -RUN cat requirements.txt | grep -v "^-e" | awk -F'==' '{print $1}' | xargs venv/bin/pip --disable-pip-version-check show > pipinfo.txt && \ - test -z $(cat pipinfo.txt | grep GPL | grep -v LGPL) && \ - rm pipinfo.txt - -# Install front-end dependencies and Optimize our images -RUN yarn install --ignore-engines \ - && yarn build \ - && jpegoptim static/img/**/*.jpg \ - && optipng -clobber -quiet static/img/**/*.png - -# Set up the init system -RUN mkdir -p /etc/my_init.d /etc/systlog-ng /usr/local/bin /etc/monit static/fonts static/ldn /usr/local/nginx/logs/ \ - && cp $QUAYCONF/init/*.sh /etc/my_init.d/ \ - && cp $QUAYCONF/init/syslog-ng.conf /etc/syslog-ng/ \ - && cp -r $QUAYCONF/init/service/* /etc/service \ - && cp $QUAYCONF/kill-buildmanager.sh /usr/local/bin/kill-buildmanager.sh \ - && cp $QUAYCONF/monitrc /etc/monit/monitrc \ - && chmod 0600 /etc/monit/monitrc \ - && cp $QUAYCONF/init/logrotate.conf /etc/logrotate.conf \ - && cp .git/HEAD GIT_HEAD \ - && rm -rf /etc/service/syslog-forwarder - -# Cleanup any NPM-related stuff. -# RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev libgpgme11-dev nodejs jpegoptim optipng w3m \ -# && apt-get autoremove -y \ -# && apt-get clean -# && rm -rf /root/.npm /.npm /usr/local/lib/node_modules /usr/share/yarn/node_modules \ -# /root/node_modules /node_modules /grunt - -RUN PYTHONPATH=$QUAYPATH venv/bin/alembic heads | grep -E '^[0-9a-f]+ \(head\)$' > ALEMBIC_HEAD