Generate .gitlab.yaml via .gitlab.jsonnet
This commit is contained in:
Antoine Legrand
parent
165486180d
commit
24044df945
|
|
@ -0,0 +1,98 @@
|
|||
local utils = import '.gitlab-ci/utils.libsonnet';
|
||||
local vars = import '.gitlab-ci/vars.libsonnet';
|
||||
local mergeJob = utils.ci.mergeJob;
|
||||
local images = vars.images;
|
||||
local baseJob = (import '.gitlab-ci/base_jobs.libsonnet')(vars);
|
||||
|
||||
local stages_list = [
|
||||
// gitlab-ci stages
|
||||
'docker_base',
|
||||
'docker_build',
|
||||
'unit_tests',
|
||||
'integration',
|
||||
'docker_release',
|
||||
'teardown',
|
||||
];
|
||||
local stages = utils.set(stages_list);
|
||||
|
||||
// List CI jobs
|
||||
local jobs = {
|
||||
// Helpers
|
||||
local onlyMaster = {
|
||||
only: ['master', 'tags'],
|
||||
},
|
||||
local onlyBranch = {
|
||||
only: ['branches'],
|
||||
},
|
||||
|
||||
'container-base-build': baseJob.dockerBuild + onlyMaster {
|
||||
// ! Only master/tags
|
||||
// Update the base container
|
||||
stage: stages.docker_base,
|
||||
script: [
|
||||
'docker build --cache-from quay.io/quay/quay-base:latest' +
|
||||
' -t %s -f quay-base.dockerfile .' % images.base.name,
|
||||
'docker push %s' % images.base.name,
|
||||
],
|
||||
},
|
||||
|
||||
'container-build': baseJob.dockerBuild {
|
||||
// Build and push the quay container.
|
||||
// Docker Tag is the branch/tag name
|
||||
stage: stages.docker_build,
|
||||
script: [
|
||||
'docker build -t %s -f quay.dockerfile .' % images.quayci.name,
|
||||
'docker push %s' % images.quayci.name],
|
||||
},
|
||||
|
||||
'container-release': baseJob.dockerBuild + onlyMaster {
|
||||
// ! Only master/tags
|
||||
// push the container to the 'prod' repository
|
||||
local repo_with_sha = images.release.name,
|
||||
stage: stages.docker_release,
|
||||
script: [
|
||||
'docker pull %s' % images.quayci.name,
|
||||
'docker tag %s %s' % [images.quayci.name, repo_with_sha],
|
||||
'docker push %s' % [repo_with_sha], # @TODO(ant31) add signing
|
||||
],
|
||||
},
|
||||
|
||||
// Unit-tests
|
||||
local unittest_stage = baseJob.QuayTest {
|
||||
stage: stages.unit_tests },
|
||||
'unit-tests': unittest_stage {
|
||||
script: [
|
||||
'py.test --timeout=7200 --verbose --show-count ./ --color=no -x'] },
|
||||
|
||||
'registry-tests': unittest_stage {
|
||||
script: [
|
||||
'py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no -x'] },
|
||||
|
||||
// UI tests
|
||||
'karma-tests': unittest_stage {
|
||||
script: [
|
||||
'curl -Ss https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -',
|
||||
'echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list',
|
||||
'apt-get update -yqqq',
|
||||
'apt-get install -y google-chrome-stable',
|
||||
'yarn test'
|
||||
] },
|
||||
|
||||
// Unit-tests with real databases
|
||||
local db_stage = { stage: stages.unit_tests },
|
||||
local dbname = 'quay',
|
||||
postgres: db_stage + baseJob.dbTest('postgresql',
|
||||
image='postgres:9.6',
|
||||
env={ POSTGRES_PASSWORD: dbname, POSTGRES_USER: dbname }),
|
||||
|
||||
mysql: db_stage + baseJob.dbTest('mysql+pymysql',
|
||||
image='mysql:latest',
|
||||
env={ [key]: dbname for key in ['MYSQL_ROOT_PASSWORD', 'MYSQL_DATABASE',
|
||||
'MYSQL_USER', 'MYSQL_PASSWORD'] }),
|
||||
|
||||
};
|
||||
|
||||
{
|
||||
stages: stages_list,
|
||||
variables: vars.global,
|
||||
} + jobs
|
||||
275
.gitlab-ci.yml
275
.gitlab-ci.yml
|
|
@ -1,152 +1,153 @@
|
|||
# Generated from .gitlab-ci.jsonnet
|
||||
# DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN
|
||||
---
|
||||
stages:
|
||||
- docker-build
|
||||
- unit-tests
|
||||
- integration
|
||||
- release
|
||||
- deploy
|
||||
|
||||
variables:
|
||||
FAILFASTCI_NAMESPACE: 'quay'
|
||||
IMAGE: quay.io/quay/quay
|
||||
PIP_CACHE_DIR: pip-cache
|
||||
PIP: /venv/bin/pip
|
||||
PYTEST: /venv/bin/py.test
|
||||
TEST: "true"
|
||||
PYTHONPATH: "."
|
||||
|
||||
|
||||
# STAGE 1: container build
|
||||
|
||||
.docker: &docker
|
||||
container-base-build:
|
||||
before_script:
|
||||
- docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io
|
||||
image: docker:git
|
||||
only:
|
||||
- master
|
||||
- tags
|
||||
script:
|
||||
- docker build --cache-from quay.io/quay/quay-base:latest -t quay.io/quay/quay-base:latest -f quay-base.dockerfile .
|
||||
- docker push quay.io/quay/quay-base:latest
|
||||
services:
|
||||
- docker:dind
|
||||
stage: docker_base
|
||||
tags:
|
||||
- docker
|
||||
variables:
|
||||
DOCKER_DRIVER: aufs
|
||||
image: docker:git
|
||||
before_script:
|
||||
- docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io
|
||||
services:
|
||||
- docker:dind
|
||||
tags:
|
||||
- docker
|
||||
|
||||
container-base-build:
|
||||
<<: *docker
|
||||
stage: docker-build
|
||||
script:
|
||||
- docker build --cache-from quay.io/quay/quay-base:latest -t quay.io/quay/quay-base:latest -f quay-base.dockerfile .
|
||||
- docker push quay.io/quay/quay-base:latest
|
||||
when: manual
|
||||
|
||||
container-build:
|
||||
<<: *docker
|
||||
stage: docker-build
|
||||
before_script:
|
||||
- docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io
|
||||
image: docker:git
|
||||
script:
|
||||
- docker build -t quay.io/quay/quay-ci:$CI_COMMIT_REF_SLUG -f quay.dockerfile .
|
||||
- docker push quay.io/quay/quay-ci:$CI_COMMIT_REF_SLUG
|
||||
|
||||
|
||||
|
||||
# STAGE 2: Unit tests & code-style
|
||||
.job: &job
|
||||
- docker build -t quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} -f quay.dockerfile .
|
||||
- docker push quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
|
||||
services:
|
||||
- docker:dind
|
||||
stage: docker_build
|
||||
tags:
|
||||
- docker
|
||||
variables:
|
||||
DOCKER_DRIVER: aufs
|
||||
container-release:
|
||||
before_script:
|
||||
- docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io
|
||||
image: docker:git
|
||||
only:
|
||||
- master
|
||||
- tags
|
||||
script:
|
||||
- docker pull quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
|
||||
- docker tag quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG} quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA}
|
||||
- docker push quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA}
|
||||
services:
|
||||
- docker:dind
|
||||
stage: docker_release
|
||||
tags:
|
||||
- docker
|
||||
variables:
|
||||
DOCKER_DRIVER: aufs
|
||||
karma-tests:
|
||||
before_script:
|
||||
- cd /
|
||||
- source venv/bin/activate
|
||||
image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
|
||||
script:
|
||||
- curl -Ss https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
|
||||
- echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list
|
||||
- apt-get update -yqqq
|
||||
- apt-get install -y google-chrome-stable
|
||||
- yarn test
|
||||
stage: unit_tests
|
||||
tags:
|
||||
- kubernetes
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
|
||||
PYTHONPATH: .
|
||||
TEST: 'true'
|
||||
mysql:
|
||||
before_script:
|
||||
- cd /
|
||||
- source venv/bin/activate
|
||||
- cd /
|
||||
- source venv/bin/activate
|
||||
image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
|
||||
script:
|
||||
- sleep 30
|
||||
- alembic upgrade head
|
||||
- PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x
|
||||
services:
|
||||
- mysql:latest
|
||||
stage: unit_tests
|
||||
tags:
|
||||
- kubernetes
|
||||
|
||||
unit-tests:
|
||||
<<: *job
|
||||
stage: unit-tests
|
||||
script:
|
||||
- py.test --timeout=7200 --verbose --show-count ./ --color=no -x
|
||||
|
||||
|
||||
registry-tests:
|
||||
<<: *job
|
||||
stage: unit-tests
|
||||
script:
|
||||
- py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no -x
|
||||
|
||||
|
||||
karma-tests:
|
||||
<<: *job
|
||||
stage: unit-tests
|
||||
script:
|
||||
# Install Chrome
|
||||
- set -xe
|
||||
- curl -Ss https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
|
||||
- echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list
|
||||
- apt-get update -yqqq
|
||||
- apt-get install -y google-chrome-stable
|
||||
|
||||
- yarn test
|
||||
|
||||
|
||||
code-styles:
|
||||
<<: *job
|
||||
stage: unit-tests
|
||||
script:
|
||||
- echo "yapf"
|
||||
- echo "pycodestyle"
|
||||
- echo "pylint"
|
||||
|
||||
|
||||
# Stage 3: Integration/e2e tests
|
||||
postgres:
|
||||
<<: *job
|
||||
- kubernetes
|
||||
variables:
|
||||
TEST_DATABASE_URI: 'postgresql://quay:quay@localhost/quay'
|
||||
GIT_STRATEGY: none
|
||||
MYSQL_DATABASE: quay
|
||||
MYSQL_PASSWORD: quay
|
||||
MYSQL_ROOT_PASSWORD: quay
|
||||
MYSQL_USER: quay
|
||||
PYTHONPATH: .
|
||||
SKIP_DB_SCHEMA: 'true'
|
||||
TEST: 'true'
|
||||
TEST_DATABASE_URI: mysql+pymysql://quay:quay@localhost/quay
|
||||
postgres:
|
||||
before_script:
|
||||
- cd /
|
||||
- source venv/bin/activate
|
||||
image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
|
||||
script:
|
||||
- sleep 30
|
||||
- alembic upgrade head
|
||||
- PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x
|
||||
services:
|
||||
- postgres:9.6
|
||||
stage: unit_tests
|
||||
tags:
|
||||
- kubernetes
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
POSTGRES_PASSWORD: quay
|
||||
POSTGRES_USER: quay
|
||||
GIT_STRATEGY: none
|
||||
stage: integration
|
||||
services:
|
||||
- postgres:9.6
|
||||
script:
|
||||
- sleep 30
|
||||
- alembic upgrade head
|
||||
- PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x
|
||||
|
||||
|
||||
mysql:
|
||||
<<: *job
|
||||
variables:
|
||||
TEST_DATABASE_URI: 'mysql+pymysql://quay:quay@localhost/quay'
|
||||
PYTHONPATH: .
|
||||
SKIP_DB_SCHEMA: 'true'
|
||||
MYSQL_ROOT_PASSWORD: quay
|
||||
MYSQL_DATABASE: quay
|
||||
MYSQL_USER: quay
|
||||
MYSQL_PASSWORD: quay
|
||||
GIT_STRATEGY: none
|
||||
stage: integration
|
||||
services:
|
||||
- mysql
|
||||
TEST: 'true'
|
||||
TEST_DATABASE_URI: postgresql://quay:quay@localhost/quay
|
||||
registry-tests:
|
||||
before_script:
|
||||
- cd /
|
||||
- source venv/bin/activate
|
||||
image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
|
||||
script:
|
||||
- sleep 30
|
||||
- alembic upgrade head
|
||||
- PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x
|
||||
|
||||
|
||||
# e2e-demo:
|
||||
# <<: *job
|
||||
# image: python:2.7
|
||||
# variables:
|
||||
# TEST_DATABASE_URI: 'postgresql://quay:quay@localhost/quay'
|
||||
# SKIP_DB_SCHEMA: 'true'
|
||||
# POSTGRES_PASSWORD: quay
|
||||
# POSTGRES_USER: quay
|
||||
# GIT_STRATEGY: none
|
||||
# stage: integration
|
||||
# before_script:
|
||||
# - cd /
|
||||
# services:
|
||||
# - postgres:9.6
|
||||
# - quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
|
||||
# script:
|
||||
# - sleep 240
|
||||
# - curl localhost:80/cnr/version
|
||||
# allow_failure: true
|
||||
- py.test --timeout=7200 --verbose --show-count ./test/registry_tests.py --color=no -x
|
||||
stage: unit_tests
|
||||
tags:
|
||||
- kubernetes
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
PYTHONPATH: .
|
||||
TEST: 'true'
|
||||
stages:
|
||||
- docker_base
|
||||
- docker_build
|
||||
- unit_tests
|
||||
- integration
|
||||
- docker_release
|
||||
- teardown
|
||||
unit-tests:
|
||||
before_script:
|
||||
- cd /
|
||||
- source venv/bin/activate
|
||||
image: quay.io/quay/quay-ci:${CI_COMMIT_REF_SLUG}
|
||||
script:
|
||||
- py.test --timeout=7200 --verbose --show-count ./ --color=no -x
|
||||
stage: unit_tests
|
||||
tags:
|
||||
- kubernetes
|
||||
variables:
|
||||
GIT_STRATEGY: none
|
||||
PYTHONPATH: .
|
||||
TEST: 'true'
|
||||
variables:
|
||||
FAILFASTCI_NAMESPACE: quay
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
function(vars={})
|
||||
{
|
||||
dockerBuild: {
|
||||
// base job to manage containers (build / push)
|
||||
variables: {
|
||||
DOCKER_DRIVER: "aufs",
|
||||
},
|
||||
image: "docker:git",
|
||||
before_script: [
|
||||
"docker login -u $DOCKER_USER -p $DOCKER_PASS quay.io",
|
||||
],
|
||||
services: [
|
||||
"docker:dind",
|
||||
],
|
||||
tags: [
|
||||
"docker",
|
||||
],
|
||||
},
|
||||
|
||||
QuayTest: {
|
||||
// base job to test the container
|
||||
image: vars.images.quayci.name,
|
||||
variables: {
|
||||
TEST: "true",
|
||||
PYTHONPATH: ".",
|
||||
GIT_STRATEGY: "none",
|
||||
},
|
||||
before_script: [
|
||||
"cd /",
|
||||
"source venv/bin/activate",
|
||||
],
|
||||
tags: [
|
||||
"kubernetes",
|
||||
],
|
||||
},
|
||||
|
||||
dbTest(scheme, image, env):: self.QuayTest {
|
||||
variables+: {
|
||||
SKIP_DB_SCHEMA: 'true',
|
||||
TEST_DATABASE_URI: '%s://quay:quay@localhost/quay' % scheme,
|
||||
} + env,
|
||||
services: [image],
|
||||
script: [
|
||||
"sleep 30",
|
||||
"alembic upgrade head",
|
||||
'PYTHONPATH="." TEST="true" py.test --timeout=7200 --verbose --show-count ./ --color=no --ignore=endpoints/appr/test/ -x',
|
||||
],
|
||||
},
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
{
|
||||
local topSelf = self,
|
||||
# Generate a sequence array from 1 to i
|
||||
seq(i):: (
|
||||
[x for x in std.range(1, i)]
|
||||
),
|
||||
|
||||
objectFieldsHidden(obj):: (
|
||||
std.setDiff(std.objectFieldsAll(obj), std.objectFields(obj))
|
||||
),
|
||||
|
||||
objectFlatten(obj):: (
|
||||
// Merge 1 level dict depth into toplevel
|
||||
local visible = { [k]: obj[j][k]
|
||||
for j in std.objectFieldsAll(obj)
|
||||
for k in std.objectFieldsAll(obj[j]) };
|
||||
|
||||
visible
|
||||
),
|
||||
|
||||
compact(array):: (
|
||||
[x for x in array if x != null]
|
||||
),
|
||||
|
||||
objectValues(obj):: (
|
||||
local fields = std.objectFields(obj);
|
||||
[obj[key] for key in fields]
|
||||
),
|
||||
|
||||
objectMap(func, obj):: (
|
||||
local fields = std.objectFields(obj);
|
||||
{ [key]: func(obj[key]) for key in fields }
|
||||
),
|
||||
|
||||
capitalize(str):: (
|
||||
std.char(std.codepoint(str[0]) - 32) + str[1:]
|
||||
),
|
||||
|
||||
test: self.capitalize("test"),
|
||||
|
||||
set(array)::
|
||||
{ [key]: key for key in array },
|
||||
|
||||
containerName(repo, tag):: "%s:%s" % [repo, tag],
|
||||
|
||||
ci: {
|
||||
|
||||
mergeJob(base_job, jobs, stage=null):: {
|
||||
[job_name]: base_job + jobs[job_name] +
|
||||
if stage != null then { stage: stage } else {}
|
||||
for job_name in std.objectFields(jobs)
|
||||
},
|
||||
|
||||
only(key):: (
|
||||
if key == "master"
|
||||
then { only: ['master', 'tags'] }
|
||||
else { only: ['branches'] }
|
||||
),
|
||||
|
||||
setManual(key, values):: (
|
||||
if std.objectHas(topSelf.set(values), key)
|
||||
then { when: 'manual' }
|
||||
else { only: ['branches'] }
|
||||
),
|
||||
},
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
local utils = import "utils.libsonnet";
|
||||
|
||||
{
|
||||
global: {
|
||||
// .gitlab-ci.yaml top `variables` key
|
||||
FAILFASTCI_NAMESPACE: "quay",
|
||||
},
|
||||
|
||||
// internal variables
|
||||
images: {
|
||||
// Quay initial image, used in the FROM clause
|
||||
base: { repo: "quay.io/quay/quay-base", tag: "latest",
|
||||
name: utils.containerName(self.repo, self.tag),
|
||||
},
|
||||
|
||||
// @TODO(ant31) release should use quay/quay
|
||||
// release is a copy of the quayci image to the 'prod' repository
|
||||
release: { repo: "quay.io/quay/quay-ci",
|
||||
tag: "${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA}",
|
||||
name: utils.containerName(self.repo, self.tag),
|
||||
},
|
||||
|
||||
quayci: { repo: "quay.io/quay/quay-ci", tag: "${CI_COMMIT_REF_SLUG}",
|
||||
name: utils.containerName(self.repo, self.tag),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
@ -3,3 +3,4 @@ pylint
|
|||
ipdb
|
||||
tqdm
|
||||
yapf==0.15.2
|
||||
ffctl>=0.1.2
|
||||
|
|
|
|||
Reference in New Issue