From 24044df94553186937f9484958959f00a2248c84 Mon Sep 17 00:00:00 2001 From: Antoine Legrand <2t.antoine@gmail.com> Date: Tue, 13 Jun 2017 05:55:12 +0200 Subject: [PATCH] Generate .gitlab.yaml via .gitlab.jsonnet --- .gitlab-ci.jsonnet | 98 ++++++++++++ .gitlab-ci.yml | 275 +++++++++++++++++---------------- .gitlab-ci/base_jobs.libsonnet | 49 ++++++ .gitlab-ci/utils.libsonnet | 66 ++++++++ .gitlab-ci/vars.libsonnet | 27 ++++ requirements-dev.txt | 1 + 6 files changed, 379 insertions(+), 137 deletions(-) create mode 100644 .gitlab-ci.jsonnet create mode 100644 .gitlab-ci/base_jobs.libsonnet create mode 100644 .gitlab-ci/utils.libsonnet create mode 100644 .gitlab-ci/vars.libsonnet diff --git a/.gitlab-ci.jsonnet b/.gitlab-ci.jsonnet new file mode 100644 index 000000000..38303cfd6 --- /dev/null +++ b/.gitlab-ci.jsonnet @@ -0,0 +1,98 @@ +local utils = import '.gitlab-ci/utils.libsonnet'; +local vars = import '.gitlab-ci/vars.libsonnet'; +local mergeJob = utils.ci.mergeJob; +local images = vars.images; +local baseJob = (import '.gitlab-ci/base_jobs.libsonnet')(vars); + +local stages_list = [ + // gitlab-ci stages + 'docker_base', + 'docker_build', + 'unit_tests', + 'integration', + 'docker_release', + 'teardown', +]; +local stages = utils.set(stages_list); + +// List CI jobs +local jobs = { + // Helpers + local onlyMaster = { + only: ['master', 'tags'], + }, + local onlyBranch = { + only: ['branches'], + }, + + 'container-base-build': baseJob.dockerBuild + onlyMaster { + // ! Only master/tags + // Update the base container + stage: stages.docker_base, + script: [ + 'docker build --cache-from quay.io/quay/quay-base:latest' + + ' -t %s -f quay-base.dockerfile .' % images.base.name, + 'docker push %s' % images.base.name, + ], + }, + + 'container-build': baseJob.dockerBuild { + // Build and push the quay container. + // Docker Tag is the branch/tag name + stage: stages.docker_build, + script: [ + 'docker build -t %s -f quay.dockerfile .' % images.quayci.name, + 'docker push %s' % images.quayci.name], + }, + + 'container-release': baseJob.dockerBuild + onlyMaster { + // ! Only master/tags + // push the container to the 'prod' repository + local repo_with_sha = images.release.name, + stage: stages.docker_release, + script: [ + 'docker pull %s' % images.quayci.name, + 'docker tag %s %s' % [images.quayci.name, repo_with_sha], + 'docker push %s' % [repo_with_sha], # @TODO(ant31) add signing + ], + }, + + // Unit-tests + local unittest_stage = baseJob.QuayTest { + stage: stages.unit_tests }, + 'unit-tests': unittest_stage { + script: [ + 'py.test --timeout=7200 --verbose --show-count ./ --color=no -x'] }, + + 'registry-tests': unittest_stage { + script: [ + 'py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no -x'] }, + + // UI tests + 'karma-tests': unittest_stage { + script: [ + 'curl -Ss https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -', + 'echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list', + 'apt-get update -yqqq', + 'apt-get install -y google-chrome-stable', + 'yarn test' + ] }, + + // Unit-tests with real databases + local db_stage = { stage: stages.unit_tests }, + local dbname = 'quay', + postgres: db_stage + baseJob.dbTest('postgresql', + image='postgres:9.6', + env={ POSTGRES_PASSWORD: dbname, POSTGRES_USER: dbname }), + + mysql: db_stage + baseJob.dbTest('mysql+pymysql', + image='mysql:latest', + env={ [key]: dbname for key in ['MYSQL_ROOT_PASSWORD', 'MYSQL_DATABASE', + 'MYSQL_USER', 'MYSQL_PASSWORD'] }), + +}; + +{ + stages: stages_list, + variables: vars.global, +} + jobs diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4adfef668..5374fd23c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,152 +1,153 @@ +# Generated from .gitlab-ci.jsonnet +# DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN --- -stages: - - docker-build - - unit-tests - - integration - - release - - deploy - -variables: - FAILFASTCI_NAMESPACE: 'quay' - IMAGE: quay.io/quay/quay - PIP_CACHE_DIR: pip-cache - PIP: /venv/bin/pip - PYTEST: /venv/bin/py.test - TEST: "true" - PYTHONPATH: "." - - -# STAGE 1: container build - -.docker: &docker +container-base-build: + before_script: + - docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io + image: docker:git + only: + - master + - tags + script: + - docker build --cache-from quay.io/quay/quay-base:latest -t quay.io/quay/quay-base:latest -f quay-base.dockerfile . + - docker push quay.io/quay/quay-base:latest + services: + - docker:dind + stage: docker_base + tags: + - docker variables: DOCKER_DRIVER: aufs - image: docker:git - before_script: - - docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io - services: - - docker:dind - tags: - - docker - -container-base-build: - <<: *docker - stage: docker-build - script: - - docker build --cache-from quay.io/quay/quay-base:latest -t quay.io/quay/quay-base:latest -f quay-base.dockerfile . - - docker push quay.io/quay/quay-base:latest - when: manual - container-build: - <<: *docker - stage: docker-build + before_script: + - docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io + image: docker:git script: - - docker build -t quay.io/quay/quay-ci:$CI_COMMIT_REF_SLUG -f quay.dockerfile . - - docker push quay.io/quay/quay-ci:$CI_COMMIT_REF_SLUG - - - -# STAGE 2: Unit tests & code-style -.job: &job + - docker build -t quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} -f quay.dockerfile . + - docker push quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} + services: + - docker:dind + stage: docker_build + tags: + - docker + variables: + DOCKER_DRIVER: aufs +container-release: + before_script: + - docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io + image: docker:git + only: + - master + - tags + script: + - docker pull quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} + - docker tag quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA} + - docker push quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA} + services: + - docker:dind + stage: docker_release + tags: + - docker + variables: + DOCKER_DRIVER: aufs +karma-tests: + before_script: + - cd / + - source venv/bin/activate + image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} + script: + - curl -Ss https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - + - echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list + - apt-get update -yqqq + - apt-get install -y google-chrome-stable + - yarn test + stage: unit_tests + tags: + - kubernetes variables: GIT_STRATEGY: none - image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} + PYTHONPATH: . + TEST: 'true' +mysql: before_script: - - cd / - - source venv/bin/activate + - cd / + - source venv/bin/activate + image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} + script: + - sleep 30 + - alembic upgrade head + - PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x + services: + - mysql:latest + stage: unit_tests tags: - - kubernetes - -unit-tests: - <<: *job - stage: unit-tests - script: - - py.test --timeout=7200 --verbose --show-count ./ --color=no -x - - -registry-tests: - <<: *job - stage: unit-tests - script: - - py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no -x - - -karma-tests: - <<: *job - stage: unit-tests - script: - # Install Chrome - - set -xe - - curl -Ss https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - - - echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list - - apt-get update -yqqq - - apt-get install -y google-chrome-stable - - - yarn test - - -code-styles: - <<: *job - stage: unit-tests - script: - - echo "yapf" - - echo "pycodestyle" - - echo "pylint" - - -# Stage 3: Integration/e2e tests -postgres: - <<: *job + - kubernetes variables: - TEST_DATABASE_URI: 'postgresql://quay:quay@localhost/quay' + GIT_STRATEGY: none + MYSQL_DATABASE: quay + MYSQL_PASSWORD: quay + MYSQL_ROOT_PASSWORD: quay + MYSQL_USER: quay + PYTHONPATH: . SKIP_DB_SCHEMA: 'true' + TEST: 'true' + TEST_DATABASE_URI: mysql+pymysql://quay:quay@localhost/quay +postgres: + before_script: + - cd / + - source venv/bin/activate + image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} + script: + - sleep 30 + - alembic upgrade head + - PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x + services: + - postgres:9.6 + stage: unit_tests + tags: + - kubernetes + variables: + GIT_STRATEGY: none POSTGRES_PASSWORD: quay POSTGRES_USER: quay - GIT_STRATEGY: none - stage: integration - services: - - postgres:9.6 - script: - - sleep 30 - - alembic upgrade head - - PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x - - -mysql: - <<: *job - variables: - TEST_DATABASE_URI: 'mysql+pymysql://quay:quay@localhost/quay' + PYTHONPATH: . SKIP_DB_SCHEMA: 'true' - MYSQL_ROOT_PASSWORD: quay - MYSQL_DATABASE: quay - MYSQL_USER: quay - MYSQL_PASSWORD: quay - GIT_STRATEGY: none - stage: integration - services: - - mysql + TEST: 'true' + TEST_DATABASE_URI: postgresql://quay:quay@localhost/quay +registry-tests: + before_script: + - cd / + - source venv/bin/activate + image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} script: - - sleep 30 - - alembic upgrade head - - PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x - - -# e2e-demo: -# <<: *job -# image: python:2.7 -# variables: -# TEST_DATABASE_URI: 'postgresql://quay:quay@localhost/quay' -# SKIP_DB_SCHEMA: 'true' -# POSTGRES_PASSWORD: quay -# POSTGRES_USER: quay -# GIT_STRATEGY: none -# stage: integration -# before_script: -# - cd / -# services: -# - postgres:9.6 -# - quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} -# script: -# - sleep 240 -# - curl localhost:80/cnr/version -# allow_failure: true + - py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no -x + stage: unit_tests + tags: + - kubernetes + variables: + GIT_STRATEGY: none + PYTHONPATH: . + TEST: 'true' +stages: +- docker_base +- docker_build +- unit_tests +- integration +- docker_release +- teardown +unit-tests: + before_script: + - cd / + - source venv/bin/activate + image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} + script: + - py.test --timeout=7200 --verbose --show-count ./ --color=no -x + stage: unit_tests + tags: + - kubernetes + variables: + GIT_STRATEGY: none + PYTHONPATH: . + TEST: 'true' +variables: + FAILFASTCI_NAMESPACE: quay diff --git a/.gitlab-ci/base_jobs.libsonnet b/.gitlab-ci/base_jobs.libsonnet new file mode 100644 index 000000000..d45227087 --- /dev/null +++ b/.gitlab-ci/base_jobs.libsonnet @@ -0,0 +1,49 @@ +function(vars={}) + { + dockerBuild: { + // base job to manage containers (build / push) + variables: { + DOCKER_DRIVER: "aufs", + }, + image: "docker:git", + before_script: [ + "docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io", + ], + services: [ + "docker:dind", + ], + tags: [ + "docker", + ], + }, + + QuayTest: { + // base job to test the container + image: vars.images.quayci.name, + variables: { + TEST: "true", + PYTHONPATH: ".", + GIT_STRATEGY: "none", + }, + before_script: [ + "cd /", + "source venv/bin/activate", + ], + tags: [ + "kubernetes", + ], + }, + + dbTest(scheme, image, env):: self.QuayTest { + variables+: { + SKIP_DB_SCHEMA: 'true', + TEST_DATABASE_URI: '%s://quay:quay@localhost/quay' % scheme, + } + env, + services: [image], + script: [ + "sleep 30", + "alembic upgrade head", + 'PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x', + ], + }, + } diff --git a/.gitlab-ci/utils.libsonnet b/.gitlab-ci/utils.libsonnet new file mode 100644 index 000000000..73801c928 --- /dev/null +++ b/.gitlab-ci/utils.libsonnet @@ -0,0 +1,66 @@ +{ + local topSelf = self, + # Generate a sequence array from 1 to i + seq(i):: ( + [x for x in std.range(1, i)] + ), + + objectFieldsHidden(obj):: ( + std.setDiff(std.objectFieldsAll(obj), std.objectFields(obj)) + ), + + objectFlatten(obj):: ( + // Merge 1 level dict depth into toplevel + local visible = { [k]: obj[j][k] + for j in std.objectFieldsAll(obj) + for k in std.objectFieldsAll(obj[j]) }; + + visible + ), + + compact(array):: ( + [x for x in array if x != null] + ), + + objectValues(obj):: ( + local fields = std.objectFields(obj); + [obj[key] for key in fields] + ), + + objectMap(func, obj):: ( + local fields = std.objectFields(obj); + { [key]: func(obj[key]) for key in fields } + ), + + capitalize(str):: ( + std.char(std.codepoint(str[0]) - 32) + str[1:] + ), + + test: self.capitalize("test"), + + set(array):: + { [key]: key for key in array }, + + containerName(repo, tag):: "%s:%s" % [repo, tag], + + ci: { + + mergeJob(base_job, jobs, stage=null):: { + [job_name]: base_job + jobs[job_name] + + if stage != null then { stage: stage } else {} + for job_name in std.objectFields(jobs) + }, + + only(key):: ( + if key == "master" + then { only: ['master', 'tags'] } + else { only: ['branches'] } + ), + + setManual(key, values):: ( + if std.objectHas(topSelf.set(values), key) + then { when: 'manual' } + else { only: ['branches'] } + ), + }, +} diff --git a/.gitlab-ci/vars.libsonnet b/.gitlab-ci/vars.libsonnet new file mode 100644 index 000000000..76a935919 --- /dev/null +++ b/.gitlab-ci/vars.libsonnet @@ -0,0 +1,27 @@ +local utils = import "utils.libsonnet"; + +{ + global: { + // .gitlab-ci.yaml top `variables` key + FAILFASTCI_NAMESPACE: "quay", + }, + + // internal variables + images: { + // Quay initial image, used in the FROM clause + base: { repo: "quay.io/quay/quay-base", tag: "latest", + name: utils.containerName(self.repo, self.tag), + }, + + // @TODO(ant31) release should use quay/quay + // release is a copy of the quayci image to the 'prod' repository + release: { repo: "quay.io/quay/quay-ci", + tag: "${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA}", + name: utils.containerName(self.repo, self.tag), + }, + + quayci: { repo: "quay.io/quay/quay-ci", tag: "${CI_COMMIT_REF_SLUG}", + name: utils.containerName(self.repo, self.tag), + }, + }, +} diff --git a/requirements-dev.txt b/requirements-dev.txt index 0cdbc4aea..6165eb863 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,4 @@ pylint ipdb tqdm yapf==0.15.2 +ffctl>=0.1.2