09a8d1a751
Our CI tests on RHEL and Fedora and we want to test the systemd cgroup driver. However, kubelet needs to run in tests with systemd cgroup driver as well, or tests fail. This patch fixes broken CI because of not matching cgroup driver between CRI-O and the kubelet. Signed-off-by: Antonio Murdaca <runcom@redhat.com>
419 lines
13 KiB
YAML
419 lines
13 KiB
YAML
## This playbook expects --extra-vars "commit=<commit>"
|
|
## and either --extra-vars "pullrequest=<PR #>" or
|
|
## --skip-tags pr
|
|
- hosts: all
|
|
remote_user: root
|
|
vars:
|
|
xunit: false
|
|
cni_commit: dcf7368eeab15e2affc6256f0bb1e84dd46a34de
|
|
tasks:
|
|
- name: Update all packages
|
|
yum:
|
|
name: '*'
|
|
state: latest
|
|
async: 600
|
|
poll: 10
|
|
when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS')
|
|
ignore_errors: true
|
|
- name: Update all packages on Fedora
|
|
dnf:
|
|
name: '*'
|
|
state: latest
|
|
async: 600
|
|
poll: 10
|
|
when: ansible_distribution == 'Fedora'
|
|
- name: Make sure we have all required packages
|
|
yum:
|
|
name: "{{ item }}"
|
|
state: latest
|
|
with_items:
|
|
- wget
|
|
- git
|
|
- make
|
|
- gcc
|
|
- tar
|
|
- libseccomp-devel
|
|
- golang
|
|
- glib2-devel
|
|
- glibc-static
|
|
- container-selinux
|
|
- btrfs-progs-devel
|
|
- device-mapper-devel
|
|
- ostree-devel
|
|
- glibc-devel
|
|
- gpgme-devel
|
|
- libassuan-devel
|
|
- libgpg-error-devel
|
|
- pkgconfig
|
|
- skopeo-containers
|
|
- oci-systemd-hook
|
|
- oci-register-machine
|
|
- oci-umount
|
|
async: 600
|
|
poll: 10
|
|
when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS')
|
|
- name: Make sure we have all required packages on Fedora
|
|
dnf:
|
|
name: "{{ item }}"
|
|
state: latest
|
|
with_items:
|
|
- wget
|
|
- git
|
|
- make
|
|
- gcc
|
|
- tar
|
|
- libseccomp-devel
|
|
- golang
|
|
- glib2-devel
|
|
- glibc-static
|
|
- container-selinux
|
|
- btrfs-progs-devel
|
|
- device-mapper-devel
|
|
- ostree-devel
|
|
- glibc-devel
|
|
- gpgme-devel
|
|
- libassuan-devel
|
|
- libgpg-error-devel
|
|
- pkgconfig
|
|
- skopeo-containers
|
|
- oci-systemd-hook
|
|
- oci-register-machine
|
|
- oci-umount
|
|
async: 600
|
|
poll: 10
|
|
when: ansible_distribution == 'Fedora'
|
|
- name: Setup swap to prevent kernel firing off the OOM killer
|
|
shell: |
|
|
truncate -s 8G /root/swap && \
|
|
export SWAPDEV=$(losetup --show -f /root/swap | head -1) && \
|
|
mkswap $SWAPDEV && \
|
|
swapon $SWAPDEV && \
|
|
swapon --show
|
|
- name: Make testing directories to conform to testing standards
|
|
file:
|
|
path: "{{ item }}"
|
|
state: directory
|
|
with_items:
|
|
- /root/src/github.com/kubernetes-incubator
|
|
- /root/src/github.com/opencontainers
|
|
- /opt/cni/bin
|
|
- /etc/cni/net.d
|
|
- /usr/local/go
|
|
- name: install Golang upstream in CentOS
|
|
shell: |
|
|
curl -fsSL "https://golang.org/dl/go1.8.1.linux-amd64.tar.gz" \
|
|
| tar -xzC /usr/local
|
|
when: ansible_distribution == 'CentOS'
|
|
- name: Set custom Golang path for CentOS
|
|
lineinfile:
|
|
dest: /root/.bashrc
|
|
line: 'export PATH=/usr/local/go/bin:$PATH'
|
|
insertafter: 'EOF'
|
|
regexp: 'export PATH=/usr/local/go/bin:$PATH'
|
|
state: present
|
|
when: ansible_distribution == 'CentOS'
|
|
- name: set sysctl vm.overcommit_memory=1 for CentOS
|
|
shell: |
|
|
sysctl -w vm.overcommit_memory=1
|
|
when: ansible_distribution == 'CentOS'
|
|
- name: disable selinux on CentOS :(
|
|
shell: |
|
|
setenforce 0
|
|
when: ansible_distribution == 'CentOS'
|
|
- name: git clone bats repo
|
|
git:
|
|
repo: https://github.com/sstephenson/bats.git
|
|
dest: /root/src/bats
|
|
async: 600
|
|
poll: 10
|
|
- name: Fetch the xunit feature PR for bats
|
|
shell: "git fetch origin +refs/pull/161/head:refs/remotes/origin/pr/161"
|
|
args:
|
|
chdir: /root/src/bats
|
|
async: 600
|
|
poll: 10
|
|
when: xunit
|
|
- name: Git checkout the xunit PR for bats
|
|
shell: "git checkout origin/pr/161"
|
|
args:
|
|
chdir: /root/src/bats
|
|
async: 600
|
|
poll: 10
|
|
when: xunit
|
|
- name: git clone crictl repo
|
|
git:
|
|
repo: https://github.com/kubernetes-incubator/cri-tools
|
|
dest: /root/src/github.com/kubernetes-incubator/cri-tools
|
|
version: 16e6fe4d7199c5689db4630a9330e6a8a12cecd1
|
|
async: 600
|
|
poll: 10
|
|
- name: git clone runc repo
|
|
git:
|
|
repo: https://github.com/opencontainers/runc
|
|
dest: /root/src/github.com/opencontainers/runc
|
|
async: 600
|
|
poll: 10
|
|
- name: git clone cri-o repo
|
|
git:
|
|
repo: https://github.com/kubernetes-incubator/cri-o
|
|
dest: /root/src/github.com/kubernetes-incubator/cri-o
|
|
async: 600
|
|
poll: 10
|
|
- name: git clone cni repo
|
|
git:
|
|
repo: https://github.com/containernetworking/plugins
|
|
dest: /root/src/github.com/containernetworking/plugins
|
|
version: "{{ cni_commit }}"
|
|
- name: Git fetch the PR
|
|
shell: "git fetch origin +refs/pull/{{ pullrequest }}/head:refs/remotes/origin/pr/{{ pullrequest }}"
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o
|
|
tags:
|
|
- pr
|
|
async: 600
|
|
poll: 10
|
|
- name: Git checkout the commit into working branch
|
|
shell: "git checkout {{ commit }}"
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o
|
|
async: 600
|
|
poll: 10
|
|
- name: Install bats
|
|
command: bats/install.sh /usr/local
|
|
args:
|
|
chdir: /root/src
|
|
- name: Add go testing dir to bashrc files
|
|
lineinfile:
|
|
dest: /root/.bashrc
|
|
line: 'export GOPATH=/root'
|
|
insertafter: 'EOF'
|
|
regexp: 'export GOPATH=/root'
|
|
state: present
|
|
- name: Source the bashrc file
|
|
shell: source /root/.bashrc
|
|
- name: Build cni networking
|
|
shell: ./build.sh
|
|
args:
|
|
chdir: /root/src/github.com/containernetworking/plugins
|
|
- name: cp bin to cni bin dir
|
|
shell: cp /root/src/github.com/containernetworking/plugins/bin/* /opt/cni/bin
|
|
- name: curl crio bridge conf file for cni networking
|
|
get_url:
|
|
url: https://raw.githubusercontent.com/kubernetes-incubator/cri-o/{{ commit }}/contrib/cni/10-crio-bridge.conf
|
|
dest: /etc/cni/net.d/10-crio-bridge.conf
|
|
- name: curl loopback conf for cni networking
|
|
get_url:
|
|
url: https://raw.githubusercontent.com/kubernetes-incubator/cri-o/{{ commit }}/contrib/cni/99-loopback.conf
|
|
dest: /etc/cni/net.d/99-loopback.conf
|
|
- name: make clean
|
|
make:
|
|
target: clean
|
|
chdir: /root/src/github.com/opencontainers/runc
|
|
async: 600
|
|
poll: 10
|
|
- name: make crictl
|
|
shell: |
|
|
go install github.com/kubernetes-incubator/cri-tools/cmd/crictl && \
|
|
cp $GOPATH/bin/crictl /usr/bin/crictl
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o/
|
|
- name: make runc
|
|
make:
|
|
params: BUILDTAGS="seccomp selinux"
|
|
chdir: /root/src/github.com/opencontainers/runc
|
|
async: 600
|
|
poll: 10
|
|
- name: install runc
|
|
make:
|
|
target: install
|
|
chdir: /root/src/github.com/opencontainers/runc
|
|
async: 600
|
|
poll: 10
|
|
- name: Change test_runner.sh to use bats xunit output
|
|
lineinfile:
|
|
dest: /root/src/github.com/kubernetes-incubator/cri-o/test/test_runner.sh
|
|
line: 'execute time bats --tap --junit $TESTS'
|
|
regexp: 'execute time bats --tap \$TESTS'
|
|
state: present
|
|
when: xunit
|
|
- name: git clone cni test repo
|
|
git:
|
|
repo: https://github.com/runcom/plugins
|
|
dest: /root/src/github.com/containernetworking/plugins
|
|
version: "custom-bridge"
|
|
force: yes
|
|
- name: Build cni test networking
|
|
shell: ./build.sh
|
|
args:
|
|
chdir: /root/src/github.com/containernetworking/plugins
|
|
- name: cp custom-bridge to opt bin
|
|
shell: cp /root/src/github.com/containernetworking/plugins/bin/bridge /opt/cni/bin/bridge-custom
|
|
# k8s builds with go1.8.x, rhel, fedora don't have it yet
|
|
- name: install Golang upstream in Fedora/RHEL
|
|
shell: |
|
|
curl -fsSL "https://golang.org/dl/go1.8.3.linux-amd64.tar.gz" \
|
|
| tar -xzC /usr/local
|
|
when: ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat'
|
|
- name: Set custom Golang path for Fedora/RHEL
|
|
lineinfile:
|
|
dest: /root/.bashrc
|
|
line: 'export PATH=/usr/local/go/bin:$PATH'
|
|
insertafter: 'EOF'
|
|
regexp: 'export PATH=/usr/local/go/bin:$PATH'
|
|
state: present
|
|
when: ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat'
|
|
- name: run integration tests RHEL
|
|
shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2 --storage-opt overlay2.override_kernel_check=1" make localintegration 2>&1 > testout.txt'
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o
|
|
async: 3600
|
|
poll: 10
|
|
ignore_errors: yes
|
|
when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
|
|
- name: run integration tests RHEL with xunit results
|
|
shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2 --storage-opt overlay2.override_kernel_check=1" make localintegration'
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o
|
|
async: 3600
|
|
poll: 10
|
|
ignore_errors: yes
|
|
when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS') and xunit
|
|
- name: run integration tests Fedora
|
|
shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2" make localintegration 2>&1 > testout.txt'
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o
|
|
async: 3600
|
|
poll: 10
|
|
ignore_errors: yes
|
|
when: ansible_distribution == 'Fedora'
|
|
- name: run integration tests Fedora with xunit results
|
|
shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2" make localintegration'
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o
|
|
async: 3600
|
|
poll: 10
|
|
ignore_errors: yes
|
|
when: (ansible_distribution == 'Fedora' and xunit)
|
|
- name: Make testing output directory
|
|
file:
|
|
path: /root/src/github.com/kubernetes-incubator/cri-o/reports
|
|
state: directory
|
|
ignore_errors: yes
|
|
when: xunit
|
|
- name: Move all xunit files into one dir to scp
|
|
shell: 'mv /root/src/github.com/kubernetes-incubator/cri-o/test/TestReport-bats*.xml /root/src/github.com/kubernetes-incubator/cri-o/reports/'
|
|
when: xunit
|
|
# XXX: kube tests from now on
|
|
- name: git clone k8s repo
|
|
git:
|
|
repo: https://github.com/runcom/kubernetes
|
|
dest: /root/src/k8s.io/kubernetes
|
|
# based on kube upstream v1.7.4
|
|
version: cri-o-node-e2e-patched
|
|
force: yes
|
|
async: 600
|
|
poll: 10
|
|
- name: make and install CRI-O
|
|
shell: |
|
|
make install.tools && \
|
|
make && \
|
|
make install && \
|
|
make install.systemd && \
|
|
make install.config
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o
|
|
async: 600
|
|
poll: 10
|
|
- name: link runc
|
|
file: src=/usr/local/sbin/runc dest=/usr/bin/runc state=link
|
|
- name: run with overlay2
|
|
replace:
|
|
regexp: 'storage_driver = ""'
|
|
replace: 'storage_driver = "overlay2"'
|
|
name: /etc/crio/crio.conf
|
|
backup: yes
|
|
- name: run with systemd cgroup manager
|
|
replace:
|
|
regexp: 'cgroup_manager = "cgroupfs"'
|
|
replace: 'cgroup_manager = "systemd"'
|
|
name: /etc/crio/crio.conf
|
|
backup: yes
|
|
- name: add docker.io default registry
|
|
lineinfile:
|
|
dest: /etc/crio/crio.conf
|
|
line: '"docker.io"'
|
|
insertafter: 'registries = \['
|
|
regexp: 'docker\.io'
|
|
state: present
|
|
- name: add overlay2 storage opts on RHEL/CentOS
|
|
lineinfile:
|
|
dest: /etc/crio/crio.conf
|
|
line: '"overlay2.override_kernel_check=1"'
|
|
insertafter: 'storage_option = \['
|
|
regexp: 'overlay2\.override_kernel_check=1'
|
|
state: present
|
|
when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
|
|
- name: enable and start CRI-O
|
|
systemd:
|
|
name: crio
|
|
state: started
|
|
enabled: yes
|
|
daemon_reload: yes
|
|
async: 600
|
|
poll: 10
|
|
# see https://github.com/kubernetes-incubator/cri-o/issues/528
|
|
- name: disable selinux for k8s conformance tests
|
|
shell: |
|
|
setenforce 0
|
|
async: 600
|
|
poll: 10
|
|
- name: Go get the go-bindata file
|
|
shell: go get -u github.com/jteeuwen/go-bindata/go-bindata
|
|
args:
|
|
chdir: /root/src/k8s.io/kubernetes
|
|
async: 600
|
|
poll: 10
|
|
- name: Install etcd
|
|
command: hack/install-etcd.sh
|
|
args:
|
|
chdir: /root/src/k8s.io/kubernetes
|
|
async: 600
|
|
poll: 10
|
|
- name: Install necessary github go packages
|
|
shell: go get github.com/onsi/ginkgo/ginkgo ; go get github.com/onsi/gomega ; go get -u github.com/cloudflare/cfssl/cmd/...
|
|
args:
|
|
chdir: /root/src/k8s.io/kubernetes
|
|
async: 600
|
|
poll: 10
|
|
- name: Add path to bashrc files
|
|
lineinfile:
|
|
dest: /root/.bashrc
|
|
line: 'export PATH=$PATH:/root/src/k8s.io/kubernetes/third_party/etcd'
|
|
insertafter: 'EOF'
|
|
regexp: 'export PATH=\$PATH:/root/src/k8s.io/kubernetes/third_party/etcd'
|
|
state: present
|
|
- name: gather correct hostname
|
|
shell: |
|
|
cat /etc/hostname
|
|
register: hostname
|
|
- name: inject hostname into /etc/hosts
|
|
lineinfile:
|
|
dest: /etc/hosts
|
|
line: '127.0.0.1 {{ hostname.stdout }}'
|
|
insertafter: 'EOF'
|
|
regexp: '127\.0\.0\.1\s+{{ hostname.stdout }}'
|
|
state: present
|
|
- name: Flush the iptables
|
|
command: iptables -F
|
|
async: 600
|
|
poll: 10
|
|
- name: run k8s tests
|
|
shell: |
|
|
make test-e2e-node PARALLELISM=1 RUNTIME=remote CONTAINER_RUNTIME_ENDPOINT=/var/run/crio.sock IMAGE_SERVICE_ENDPOINT=/var/run/crio.sock TEST_ARGS='--prepull-images=true --kubelet-flags="--cgroup-driver=systemd"' FOCUS="\[Conformance\]" 2>&1 > node-e2e.log
|
|
args:
|
|
chdir: /root/src/k8s.io/kubernetes
|
|
async: 7200
|
|
poll: 10
|
|
ignore_errors: true
|
|
# XXX: tests on RHEL/CentOS are unreliable fow now
|
|
when: ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat'
|