Generate .gitlab.yaml via .gitlab.jsonnet

.gitlab-ci.jsonnet

@@ -0,0 +1,98 @@
+local utils = import '.gitlab-ci/utils.libsonnet';
+local vars = import '.gitlab-ci/vars.libsonnet';
+local mergeJob = utils.ci.mergeJob;
+local images = vars.images;
+local baseJob = (import '.gitlab-ci/base_jobs.libsonnet')(vars);
+
+local stages_list = [
+  // gitlab-ci stages
+  'docker_base',
+  'docker_build',
+  'unit_tests',
+  'integration',
+  'docker_release',
+  'teardown',
+];
+local stages = utils.set(stages_list);
+
+// List CI jobs
+local jobs = {
+  // Helpers
+  local onlyMaster = {
+    only: ['master', 'tags'],
+  },
+  local onlyBranch = {
+    only: ['branches'],
+  },
+
+  'container-base-build': baseJob.dockerBuild + onlyMaster {
+    // ! Only master/tags
+    // Update the base container
+    stage: stages.docker_base,
+    script: [
+      'docker build --cache-from quay.io/quay/quay-base:latest' +
+      ' -t %s -f quay-base.dockerfile .' % images.base.name,
+      'docker push %s' % images.base.name,
+    ],
+  },
+
+  'container-build': baseJob.dockerBuild {
+    // Build and push the quay container.
+    // Docker Tag is the branch/tag name
+    stage: stages.docker_build,
+    script: [
+      'docker build -t %s -f quay.dockerfile .' % images.quayci.name,
+      'docker push %s' % images.quayci.name],
+  },
+
+  'container-release': baseJob.dockerBuild + onlyMaster {
+    // ! Only master/tags
+    // push the container to the 'prod' repository
+    local repo_with_sha = images.release.name,
+    stage: stages.docker_release,
+    script: [
+      'docker pull %s' % images.quayci.name,
+      'docker tag %s %s' % [images.quayci.name, repo_with_sha],
+      'docker push %s' % [repo_with_sha],  # @TODO(ant31) add signing
+    ],
+  },
+
+  // Unit-tests
+  local unittest_stage = baseJob.QuayTest {
+    stage: stages.unit_tests },
+  'unit-tests': unittest_stage {
+    script: [
+      'py.test --timeout=7200 --verbose --show-count ./ --color=no -x'] },
+
+  'registry-tests': unittest_stage {
+    script: [
+      'py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no -x'] },
+
+  // UI tests
+  'karma-tests': unittest_stage {
+    script: [
+     'curl -Ss https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -',
+     'echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list',
+     'apt-get update -yqqq',
+     'apt-get install -y google-chrome-stable',
+     'yarn test'
+     ] },
+
+  // Unit-tests with real databases
+  local db_stage = { stage: stages.unit_tests },
+  local dbname = 'quay',
+  postgres: db_stage + baseJob.dbTest('postgresql',
+                                      image='postgres:9.6',
+                                      env={ POSTGRES_PASSWORD: dbname, POSTGRES_USER: dbname }),
+
+  mysql: db_stage + baseJob.dbTest('mysql+pymysql',
+                                   image='mysql:latest',
+                                   env={ [key]: dbname for key in ['MYSQL_ROOT_PASSWORD', 'MYSQL_DATABASE',
+                                                                   'MYSQL_USER', 'MYSQL_PASSWORD'] }),
+
+};
+
+{
+  stages: stages_list,
+  variables: vars.global,
+} + jobs

.gitlab-ci.yml

@@ -1,152 +1,153 @@
+# Generated from .gitlab-ci.jsonnet 
+#     DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN 
 ---
-stages:
-  - docker-build
-  - unit-tests
-  - integration
-  - release
-  - deploy
-
-variables:
-  FAILFASTCI_NAMESPACE: 'quay'
-  IMAGE: quay.io/quay/quay
-  PIP_CACHE_DIR: pip-cache
-  PIP: /venv/bin/pip
-  PYTEST: /venv/bin/py.test
-  TEST: "true"
-  PYTHONPATH: "."
-
-
-# STAGE 1: container build
-
-.docker: &docker
-  variables:
-    DOCKER_DRIVER: aufs
-  image: docker:git
+container-base-build:
   before_script:
   - docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io
-  services:
-     - docker:dind
-  tags:
-    - docker
-
-container-base-build:
-  <<: *docker
-  stage: docker-build
+  image: docker:git
+  only:
+  - master
+  - tags
   script:
   - docker build --cache-from quay.io/quay/quay-base:latest -t quay.io/quay/quay-base:latest -f quay-base.dockerfile .
   - docker push quay.io/quay/quay-base:latest
-  when: manual
-
-container-build:
-  <<: *docker
-  stage: docker-build
-  script:
-    - docker build -t quay.io/quay/quay-ci:$CI_COMMIT_REF_SLUG -f quay.dockerfile .
-    - docker push quay.io/quay/quay-ci:$CI_COMMIT_REF_SLUG
-
-
-
-# STAGE 2: Unit tests & code-style
-.job: &job
+  services:
+  - docker:dind
+  stage: docker_base
+  tags:
+  - docker
   variables:
-    GIT_STRATEGY: none
-  image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
+    DOCKER_DRIVER: aufs
+container-build:
+  before_script:
+  - docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io
+  image: docker:git
+  script:
+  - docker build -t quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} -f quay.dockerfile .
+  - docker push quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
+  services:
+  - docker:dind
+  stage: docker_build
+  tags:
+  - docker
+  variables:
+    DOCKER_DRIVER: aufs
+container-release:
+  before_script:
+  - docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io
+  image: docker:git
+  only:
+  - master
+  - tags
+  script:
+  - docker pull quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
+  - docker tag quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA}
+  - docker push quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA}
+  services:
+  - docker:dind
+  stage: docker_release
+  tags:
+  - docker
+  variables:
+    DOCKER_DRIVER: aufs
+karma-tests:
   before_script:
   - cd /
   - source venv/bin/activate
-  tags:
-    - kubernetes
-
-unit-tests:
-  <<: *job
-  stage: unit-tests
+  image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
   script:
-    - py.test --timeout=7200 --verbose --show-count ./ --color=no -x
-
-
-registry-tests:
-  <<: *job
-  stage: unit-tests
-  script:
-    - py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no -x
-
-
-karma-tests:
-  <<: *job
-  stage: unit-tests
-  script:
-    # Install Chrome
-    - set -xe
   - curl -Ss https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
   - echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list
   - apt-get update -yqqq
   - apt-get install -y google-chrome-stable
-
   - yarn test
-
-
-code-styles:
-  <<: *job
-  stage: unit-tests
-  script:
-    - echo "yapf"
-    - echo "pycodestyle"
-    - echo "pylint"
-
-
-# Stage 3: Integration/e2e tests
-postgres:
-  <<: *job
+  stage: unit_tests
+  tags:
+  - kubernetes
   variables:
-    TEST_DATABASE_URI: 'postgresql://quay:quay@localhost/quay'
-    SKIP_DB_SCHEMA: 'true'
-    POSTGRES_PASSWORD: quay
-    POSTGRES_USER: quay
     GIT_STRATEGY: none
-  stage: integration
+    PYTHONPATH: .
+    TEST: 'true'
+mysql:
+  before_script:
+  - cd /
+  - source venv/bin/activate
+  image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
+  script:
+  - sleep 30
+  - alembic upgrade head
+  - PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x
+  services:
+  - mysql:latest
+  stage: unit_tests
+  tags:
+  - kubernetes
+  variables:
+    GIT_STRATEGY: none
+    MYSQL_DATABASE: quay
+    MYSQL_PASSWORD: quay
+    MYSQL_ROOT_PASSWORD: quay
+    MYSQL_USER: quay
+    PYTHONPATH: .
+    SKIP_DB_SCHEMA: 'true'
+    TEST: 'true'
+    TEST_DATABASE_URI: mysql+pymysql://quay:quay@localhost/quay
+postgres:
+  before_script:
+  - cd /
+  - source venv/bin/activate
+  image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
+  script:
+  - sleep 30
+  - alembic upgrade head
+  - PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x
   services:
   - postgres:9.6
-  script:
-    - sleep 30
-    - alembic upgrade head
-    - PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x
-
-
-mysql:
-  <<: *job
+  stage: unit_tests
+  tags:
+  - kubernetes
   variables:
-    TEST_DATABASE_URI: 'mysql+pymysql://quay:quay@localhost/quay'
-    SKIP_DB_SCHEMA: 'true'
-    MYSQL_ROOT_PASSWORD: quay
-    MYSQL_DATABASE: quay
-    MYSQL_USER: quay
-    MYSQL_PASSWORD: quay
     GIT_STRATEGY: none
-  stage: integration
-  services:
-    - mysql
+    POSTGRES_PASSWORD: quay
+    POSTGRES_USER: quay
+    PYTHONPATH: .
+    SKIP_DB_SCHEMA: 'true'
+    TEST: 'true'
+    TEST_DATABASE_URI: postgresql://quay:quay@localhost/quay
+registry-tests:
+  before_script:
+  - cd /
+  - source venv/bin/activate
+  image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
   script:
-    - sleep 30
-    - alembic upgrade head
-    - PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x
-
-
-# e2e-demo:
-#   <<: *job
-#   image: python:2.7
-#   variables:
-#     TEST_DATABASE_URI: 'postgresql://quay:quay@localhost/quay'
-#     SKIP_DB_SCHEMA: 'true'
-#     POSTGRES_PASSWORD: quay
-#     POSTGRES_USER: quay
-#     GIT_STRATEGY: none
-#   stage: integration
-#   before_script:
-#     - cd /
-#   services:
-#     - postgres:9.6
-#     - quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
-#   script:
-#     - sleep 240
-#     - curl localhost:80/cnr/version
-#   allow_failure: true
+  - py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no -x
+  stage: unit_tests
+  tags:
+  - kubernetes
+  variables:
+    GIT_STRATEGY: none
+    PYTHONPATH: .
+    TEST: 'true'
+stages:
+- docker_base
+- docker_build
+- unit_tests
+- integration
+- docker_release
+- teardown
+unit-tests:
+  before_script:
+  - cd /
+  - source venv/bin/activate
+  image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
+  script:
+  - py.test --timeout=7200 --verbose --show-count ./ --color=no -x
+  stage: unit_tests
+  tags:
+  - kubernetes
+  variables:
+    GIT_STRATEGY: none
+    PYTHONPATH: .
+    TEST: 'true'
+variables:
+  FAILFASTCI_NAMESPACE: quay

.gitlab-ci/base_jobs.libsonnet

@@ -0,0 +1,49 @@
+function(vars={})
+  {
+    dockerBuild: {
+      // base job to manage containers (build / push)
+      variables: {
+        DOCKER_DRIVER: "aufs",
+      },
+      image: "docker:git",
+      before_script: [
+        "docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io",
+      ],
+      services: [
+        "docker:dind",
+      ],
+      tags: [
+        "docker",
+      ],
+    },
+
+    QuayTest: {
+      // base job to test the container
+      image: vars.images.quayci.name,
+      variables: {
+        TEST: "true",
+        PYTHONPATH: ".",
+        GIT_STRATEGY: "none",
+      },
+      before_script: [
+        "cd /",
+        "source venv/bin/activate",
+      ],
+      tags: [
+        "kubernetes",
+      ],
+    },
+
+    dbTest(scheme, image, env):: self.QuayTest {
+      variables+: {
+        SKIP_DB_SCHEMA: 'true',
+        TEST_DATABASE_URI: '%s://quay:quay@localhost/quay' % scheme,
+      } + env,
+      services: [image],
+      script: [
+        "sleep 30",
+        "alembic upgrade head",
+        'PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x',
+      ],
+    },
+  }

.gitlab-ci/utils.libsonnet

@@ -0,0 +1,66 @@
+{
+  local topSelf = self,
+  # Generate a sequence array from 1 to i
+  seq(i):: (
+    [x for x in std.range(1, i)]
+  ),
+
+  objectFieldsHidden(obj):: (
+    std.setDiff(std.objectFieldsAll(obj), std.objectFields(obj))
+  ),
+
+  objectFlatten(obj):: (
+    // Merge 1 level dict depth into toplevel
+    local visible = { [k]: obj[j][k]
+                      for j in std.objectFieldsAll(obj)
+                      for k in std.objectFieldsAll(obj[j]) };
+
+    visible
+  ),
+
+  compact(array):: (
+    [x for x in array if x != null]
+  ),
+
+  objectValues(obj):: (
+    local fields = std.objectFields(obj);
+    [obj[key] for key in fields]
+  ),
+
+  objectMap(func, obj):: (
+    local fields = std.objectFields(obj);
+    { [key]: func(obj[key]) for key in fields }
+  ),
+
+  capitalize(str):: (
+    std.char(std.codepoint(str[0]) - 32) + str[1:]
+  ),
+
+  test: self.capitalize("test"),
+
+  set(array)::
+    { [key]: key for key in array },
+
+  containerName(repo, tag):: "%s:%s" % [repo, tag],
+
+  ci: {
+
+    mergeJob(base_job, jobs, stage=null):: {
+      [job_name]: base_job + jobs[job_name] +
+                  if stage != null then { stage: stage } else {}
+      for job_name in std.objectFields(jobs)
+    },
+
+    only(key):: (
+      if key == "master"
+      then { only: ['master', 'tags'] }
+      else { only: ['branches'] }
+    ),
+
+    setManual(key, values):: (
+      if std.objectHas(topSelf.set(values), key)
+      then { when: 'manual' }
+      else { only: ['branches'] }
+    ),
+  },
+}

.gitlab-ci/vars.libsonnet

@@ -0,0 +1,27 @@
+local utils = import "utils.libsonnet";
+
+{
+  global: {
+    // .gitlab-ci.yaml top `variables` key
+    FAILFASTCI_NAMESPACE: "quay",
+  },
+
+  // internal variables
+  images: {
+    // Quay initial image, used in the FROM clause
+    base: { repo: "quay.io/quay/quay-base", tag: "latest",
+            name: utils.containerName(self.repo, self.tag),
+    },
+
+    // @TODO(ant31) release should use quay/quay
+    // release is a copy of the quayci image to the 'prod' repository
+    release: { repo: "quay.io/quay/quay-ci",
+               tag: "${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA}",
+               name: utils.containerName(self.repo, self.tag),
+    },
+
+    quayci: { repo: "quay.io/quay/quay-ci", tag: "${CI_COMMIT_REF_SLUG}",
+              name: utils.containerName(self.repo, self.tag),
+    },
+  },
+}

requirements-dev.txt

@@ -3,3 +3,4 @@ pylint
 ipdb
 tqdm
 yapf==0.15.2
+ffctl>=0.1.2