Refactor the CRI-O test playbook to be more modular

Signed-off-by: Steve Kuznetsov <skuznets@redhat.com>
This commit is contained in:
Steve Kuznetsov 2017-09-19 15:50:37 -07:00
parent 89f18fa7b5
commit e160796d4e
No known key found for this signature in database
GPG key ID: 366E054B30FC03A2
12 changed files with 550 additions and 618 deletions

View file

@ -1,618 +0,0 @@
## This playbook expects --extra-vars "commit=<commit>"
## and either --extra-vars "pullrequest=<PR #>" or
## --skip-tags pr.
##
## Required: --extra-vars extras_repo_baseurl=<HTTP or HTTPS>
- hosts: all
remote_user: root
gather_facts: False # Ansible dependencies need installing first
vars:
xunit: false
cni_commit: dcf7368eeab15e2affc6256f0bb1e84dd46a34de
integration_timeout_seconds: "{{ 60 * 60 * 2 }}"
pre_tasks:
# Assert subscription stuff won't get in the way
- assert:
that:
- 'extras_repo_baseurl | default(False)'
- 'server_repo_baseurl | default(False)'
- name: Install pre-requesites for ansible on all platforms
raw: $(type -P dnf || type -P yum) install -y python libselinux-python yum-utils python3-pycurl
# Gather facts
- setup:
# Just one more thing...
- name: Install special ansible deps. for fedora
raw: dnf install -y python2-dnf python3-dnf
when: ansible_distribution == 'Fedora'
tasks:
- name: Enable EPEL on RHEL & CentOS
yum_repository:
name: epel
description: "Extra Packages for Enterprise Linux"
baseurl: "http://dl.fedoraproject.org/pub/epel/7/$basearch"
metalink: "https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=$basearch"
enabled: True
gpgcheck: True
gpgcakey: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7Server"
when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS')
- name: import EPEL GPG keys on RHEL & CentOS
command: 'rpm --import {{ item }}'
with_items:
- https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7Server
- https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
- https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL
when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS')
- name: Enable required RHEL repositories
yum_repository:
name: "{{ item.key }}"
description: Required repo. for RHEL
baseurl: "{{ item.value }}"
gpgcheck: False
enabled: True
with_dict:
extras: '{{ extras_repo_baseurl | mandatory }}'
server: '{{ server_repo_baseurl | mandatory }}'
when: (ansible_distribution == 'RedHat')
- name: Don't let old cache interfere with new efforts
shell: $(type -P dnf || type -P yum) clean all && rm -rf /var/cache/{yum,dnf}
- name: disable redhat yum plugin
ini_file:
path: "/etc/yum/pluginconf.d/subscription-manager.conf"
section: main
option: enabled
value: 0
when: (ansible_distribution == 'RedHat')
- name: Update all packages
yum:
name: '*'
state: latest
async: 600
poll: 10
when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS')
ignore_errors: true
- name: Update all packages on Fedora
dnf:
name: '*'
state: latest
async: 600
poll: 10
when: ansible_distribution == 'Fedora'
- name: Make sure we have all required packages
yum:
name: "{{ item }}"
state: latest
with_items:
- btrfs-progs-devel
- container-selinux
- skopeo-containers
- curl
- device-mapper-devel
- expect
- findutils
- gcc
- git
- glib2-devel
- glibc-devel
- glibc-static
- golang
- gpgme-devel
- hostname
- iproute
- iptables
- krb5-workstation
- libassuan-devel
- libffi-devel
- libgpg-error-devel
- libguestfs-tools
- libseccomp-devel
- libvirt-client
- libvirt-python
- libxml2-devel
- libxslt-devel
- make
- mlocate
- nfs-utils
- nmap-ncat
- npm
- oci-register-machine
- oci-systemd-hook
- oci-umount
- openssl
- openssl-devel
- ostree-devel
- pkgconfig
- python
- python2-boto
- python2-crypto
- python2-mock
- python-click
- python-devel
- python-virtualenv
- PyYAML
- redhat-rpm-config
- rpcbind
- rsync
- sed
- socat
async: 600
poll: 10
when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS')
- name: Make sure we have all required packages on Fedora
dnf:
name: "{{ item }}"
state: latest
with_items:
- nmap-ncat
- iptables
- wget
- git
- make
- gcc
- tar
- libseccomp-devel
- golang
- glib2-devel
- glibc-static
- container-selinux
- btrfs-progs-devel
- device-mapper-devel
- ostree-devel
- glibc-devel
- gpgme-devel
- libassuan-devel
- libgpg-error-devel
- pkgconfig
- skopeo-containers
- oci-systemd-hook
- oci-register-machine
- oci-umount
- socat
async: 600
poll: 10
when: ansible_distribution == 'Fedora'
- name: Setup swap to prevent kernel firing off the OOM killer
shell: |
truncate -s 8G /root/swap && \
export SWAPDEV=$(losetup --show -f /root/swap | head -1) && \
mkswap $SWAPDEV && \
swapon $SWAPDEV && \
swapon --show
- name: Make testing directories to conform to testing standards
file:
path: "{{ item }}"
state: directory
with_items:
- /root/src/github.com/kubernetes-incubator
- /root/src/github.com/opencontainers
- /opt/cni/bin
- /etc/cni/net.d
- /usr/local/go
- name: install Golang upstream in CentOS
shell: |
curl -fsSL "https://golang.org/dl/go1.8.3.linux-amd64.tar.gz" \
| tar -xzC /usr/local
when: ansible_distribution == 'CentOS'
- name: Set custom Golang path for CentOS
lineinfile:
dest: /root/.bashrc
line: 'export PATH=/usr/local/go/bin:$PATH'
insertafter: 'EOF'
regexp: 'export PATH=/usr/local/go/bin:$PATH'
state: present
when: ansible_distribution == 'CentOS'
- name: set sysctl vm.overcommit_memory=1 for CentOS
shell: |
sysctl -w vm.overcommit_memory=1
when: ansible_distribution == 'CentOS'
- name: disable selinux on CentOS :(
shell: |
setenforce 0
when: ansible_distribution == 'CentOS'
- name: git clone bats repo
git:
repo: https://github.com/sstephenson/bats.git
dest: /root/src/bats
async: 600
poll: 10
- name: Fetch the xunit feature PR for bats
shell: "git fetch origin +refs/pull/161/head:refs/remotes/origin/pr/161"
args:
chdir: /root/src/bats
async: 600
poll: 10
when: xunit
- name: Git checkout the xunit PR for bats
shell: "git checkout origin/pr/161"
args:
chdir: /root/src/bats
async: 600
poll: 10
when: xunit
- name: git clone crictl repo
git:
repo: https://github.com/kubernetes-incubator/cri-tools
dest: /root/src/github.com/kubernetes-incubator/cri-tools
version: 16e6fe4d7199c5689db4630a9330e6a8a12cecd1
async: 600
poll: 10
- name: git clone runc repo
git:
repo: https://github.com/opencontainers/runc
dest: /root/src/github.com/opencontainers/runc
async: 600
poll: 10
- name: git clone cri-o repo
git:
repo: https://github.com/kubernetes-incubator/cri-o
dest: /root/src/github.com/kubernetes-incubator/cri-o
async: 600
poll: 10
- name: git clone cni repo
git:
repo: https://github.com/containernetworking/plugins
dest: /root/src/github.com/containernetworking/plugins
version: "{{ cni_commit }}"
- name: Git fetch the PR
shell: "git fetch origin +refs/pull/{{ pullrequest }}/head:refs/remotes/origin/pr/{{ pullrequest }}"
args:
chdir: /root/src/github.com/kubernetes-incubator/cri-o
tags:
- pr
async: 600
poll: 10
- name: Git checkout the commit into working branch
shell: "git checkout {{ commit }}"
args:
chdir: /root/src/github.com/kubernetes-incubator/cri-o
async: 600
poll: 10
- name: Install bats
command: bats/install.sh /usr/local
args:
chdir: /root/src
- name: Add go testing dir to bashrc files
lineinfile:
dest: /root/.bashrc
line: 'export GOPATH=/root'
insertafter: 'EOF'
regexp: 'export GOPATH=/root'
state: present
- name: Source the bashrc file
shell: source /root/.bashrc
- name: Build cni networking
shell: ./build.sh
args:
chdir: /root/src/github.com/containernetworking/plugins
- name: cp bin to cni bin dir
shell: cp /root/src/github.com/containernetworking/plugins/bin/* /opt/cni/bin
- name: curl crio bridge conf file for cni networking
get_url:
url: https://raw.githubusercontent.com/kubernetes-incubator/cri-o/{{ commit }}/contrib/cni/10-crio-bridge.conf
dest: /etc/cni/net.d/10-crio-bridge.conf
- name: curl loopback conf for cni networking
get_url:
url: https://raw.githubusercontent.com/kubernetes-incubator/cri-o/{{ commit }}/contrib/cni/99-loopback.conf
dest: /etc/cni/net.d/99-loopback.conf
- name: make clean
make:
target: clean
chdir: /root/src/github.com/opencontainers/runc
async: 600
poll: 10
- name: make crictl
shell: |
go install github.com/kubernetes-incubator/cri-tools/cmd/crictl && \
cp $GOPATH/bin/crictl /usr/bin/crictl
args:
chdir: /root/src/github.com/kubernetes-incubator/cri-o/
- name: make runc
make:
params: BUILDTAGS="seccomp selinux"
chdir: /root/src/github.com/opencontainers/runc
async: 600
poll: 10
- name: install runc
make:
target: install
chdir: /root/src/github.com/opencontainers/runc
async: 600
poll: 10
- name: Change test_runner.sh to use bats xunit output
lineinfile:
dest: /root/src/github.com/kubernetes-incubator/cri-o/test/test_runner.sh
line: 'execute time bats --tap --junit $TESTS'
regexp: 'execute time bats --tap \$TESTS'
state: present
when: xunit
- name: git clone cni test repo
git:
repo: https://github.com/runcom/plugins
dest: /root/src/github.com/containernetworking/plugins
version: "custom-bridge"
force: yes
- name: Build cni test networking
shell: ./build.sh
args:
chdir: /root/src/github.com/containernetworking/plugins
- name: cp custom-bridge to opt bin
shell: cp /root/src/github.com/containernetworking/plugins/bin/bridge /opt/cni/bin/bridge-custom
# k8s builds with go1.8.x, rhel, fedora don't have it yet
- name: install Golang upstream in Fedora/RHEL
shell: |
curl -fsSL "https://golang.org/dl/go1.8.3.linux-amd64.tar.gz" \
| tar -xzC /usr/local
when: ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat'
- name: Set custom Golang path for Fedora/RHEL
lineinfile:
dest: /root/.bashrc
line: 'export PATH=/usr/local/go/bin:$PATH'
insertafter: 'EOF'
regexp: 'export PATH=/usr/local/go/bin:$PATH'
state: present
when: ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat'
# dest dir depends on skopeo-containers package
- name: Copy redhat sigstore file to test host
copy:
remote_src: yes
src: /root/src/github.com/kubernetes-incubator/cri-o/test/redhat_sigstore.yaml
dest: /etc/containers/registries.d/registry.access.redhat.com.yaml
- name: Copy test policy.json file
copy:
remote_src: yes
src: /root/src/github.com/kubernetes-incubator/cri-o/test/policy.json
dest: /etc/containers/policy.json
- name: run integration tests RHEL
shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2 --storage-opt overlay2.override_kernel_check=1" make localintegration 2>&1 > testout.txt'
args:
chdir: /root/src/github.com/kubernetes-incubator/cri-o
async: "{{ integration_timeout_seconds | int }}"
poll: 10
ignore_errors: yes
when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
- name: run integration tests RHEL with xunit results
shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2 --storage-opt overlay2.override_kernel_check=1" make localintegration'
args:
chdir: /root/src/github.com/kubernetes-incubator/cri-o
async: "{{ integration_timeout_seconds | int }}"
poll: 10
ignore_errors: yes
when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS') and xunit
- name: run integration tests Fedora
shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2" make localintegration 2>&1 > testout.txt'
args:
chdir: /root/src/github.com/kubernetes-incubator/cri-o
async: "{{ integration_timeout_seconds | int }}"
poll: 10
ignore_errors: yes
when: ansible_distribution == 'Fedora'
- name: run integration tests Fedora with xunit results
shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2" make localintegration'
args:
chdir: /root/src/github.com/kubernetes-incubator/cri-o
async: "{{ integration_timeout_seconds | int }}"
poll: 10
ignore_errors: yes
when: (ansible_distribution == 'Fedora' and xunit)
- name: Make testing output directory
file:
path: /root/src/github.com/kubernetes-incubator/cri-o/reports
state: directory
ignore_errors: yes
when: xunit
- name: Move all xunit files into one dir to scp
shell: 'mv /root/src/github.com/kubernetes-incubator/cri-o/test/TestReport-bats*.xml /root/src/github.com/kubernetes-incubator/cri-o/reports/'
when: xunit
# XXX: kube tests from now on
- name: git clone k8s repo
git:
repo: https://github.com/runcom/kubernetes
dest: /root/src/k8s.io/kubernetes
# based on kube upstream v1.7.5
version: cri-o-node-e2e-patched
force: yes
async: 600
poll: 10
- name: Add custom cluster service file for the e2e testing
copy:
dest: /etc/systemd/system/customcluster.service
content: |
[Unit]
After=network-online.target
Wants=network-online.target
[Service]
ExecStart=/usr/local/bin/createcluster.sh
User=root
[Install]
WantedBy=multi-user.target
- name: Add create cluster background script for e2e testing
copy:
dest: /usr/local/bin/createcluster.sh
content: |
#!/bin/bash
export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:/usr/local/go/bin:/root/go-tools/bin:/root/src/k8s.io/kubernetes/third_party/etcd
export GOPATH=/root/go-tools
cd /root/src/k8s.io/kubernetes
export CONTAINER_RUNTIME=remote
export CGROUP_DRIVER=systemd
export CONTAINER_RUNTIME_ENDPOINT='/var/run/crio.sock --runtime-request-timeout=5m'
export ALLOW_SECURITY_CONTEXT=","
export ALLOW_PRIVILEGED=1
export DNS_SERVER_IP={{ ansible_default_ipv4.address }}
export API_HOST={{ ansible_default_ipv4.address }}
export API_HOST_IP={{ ansible_default_ipv4.address }}
export KUBE_ENABLE_CLUSTER_DNS=true
hack/local-up-cluster.sh
- name: Add path to bashrc files
lineinfile:
dest: /root/.bashrc
line: 'export PATH=/tmp/mybin:$PATH:/usr/local/go/bin:/root/go-tools/bin:/root/src/k8s.io/kubernetes/third_party/etcd'
insertafter: 'EOF'
regexp: 'export PATH=/tmp/mybin:\$PATH:/usr/local/go/bin:/root/go-tools/bin:/root/src/k8s.io/kubernetes/third_party/etcd'
state: present
- name: Add gopath to bashrc files
lineinfile:
dest: /root/.bashrc
line: 'export GOPATH=/root/go-tools'
insertafter: 'EOF'
regexp: 'export GOPATH=/root/go-tools'
state: present
- name: Source the bash rc file
shell: source /root/.bashrc
- name: Get godep
shell: go get -u github.com/tools/godep
async: 600
poll: 10
- name: Install etcd
command: hack/install-etcd.sh
args:
chdir: /root/src/k8s.io/kubernetes
async: 600
poll: 10
- name: Install necessary github go packages
shell: go get github.com/onsi/ginkgo/ginkgo ; go get github.com/onsi/gomega ; go get -u github.com/cloudflare/cfssl/cmd/...
args:
chdir: /root/src/k8s.io/kubernetes
async: 600
poll: 10
- name: Make testing output verbose so it can be converted to xunit
lineinfile:
dest: /root/src/k8s.io/kubernetes/hack/make-rules/test.sh
line: ' go test -v "${goflags[@]:+${goflags[@]}}" \'
regexp: ' go test \"\$'
state: present
- name: make and install CRI-O
shell: |
make install.tools && \
make && \
make install && \
make install.systemd && \
make install.config
args:
chdir: /root/src/github.com/kubernetes-incubator/cri-o
async: 600
poll: 10
- name: link runc
file: src=/usr/local/sbin/runc dest=/usr/bin/runc state=link
- name: run with overlay2
replace:
regexp: 'storage_driver = ""'
replace: 'storage_driver = "overlay2"'
name: /etc/crio/crio.conf
backup: yes
- name: run with systemd cgroup manager
replace:
regexp: 'cgroup_manager = "cgroupfs"'
replace: 'cgroup_manager = "systemd"'
name: /etc/crio/crio.conf
backup: yes
- name: add docker.io default registry
lineinfile:
dest: /etc/crio/crio.conf
line: '"docker.io"'
insertafter: 'registries = \['
regexp: 'docker\.io'
state: present
- name: add overlay2 storage opts on RHEL/CentOS
lineinfile:
dest: /etc/crio/crio.conf
line: '"overlay2.override_kernel_check=1"'
insertafter: 'storage_option = \['
regexp: 'overlay2\.override_kernel_check=1'
state: present
when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
- name: enable and start CRI-O
systemd:
name: crio
state: started
enabled: yes
daemon_reload: yes
async: 600
poll: 10
# see https://github.com/kubernetes-incubator/cri-o/issues/528
- name: disable selinux for k8s conformance tests
shell: |
setenforce 0
async: 600
poll: 10
- name: Go get the go-bindata file
shell: go get -u github.com/jteeuwen/go-bindata/go-bindata
args:
chdir: /root/src/k8s.io/kubernetes
async: 600
poll: 10
- name: Install necessary github go packages
shell: go get github.com/onsi/ginkgo/ginkgo ; go get github.com/onsi/gomega ; go get -u github.com/cloudflare/cfssl/cmd/...
args:
chdir: /root/src/k8s.io/kubernetes
async: 600
poll: 10
- name: Add path to bashrc files
lineinfile:
dest: /root/.bashrc
line: 'export PATH=$PATH:/root/src/k8s.io/kubernetes/third_party/etcd'
insertafter: 'EOF'
regexp: 'export PATH=\$PATH:/root/src/k8s.io/kubernetes/third_party/etcd'
state: present
- name: inject hostname into /etc/hosts
lineinfile:
dest: /etc/hosts
line: '{{ ansible_default_ipv4.address }} {{ ansible_nodename }}'
insertafter: 'EOF'
regexp: '{{ ansible_default_ipv4.address }}\s+{{ ansible_nodename }}'
state: present
- name: Flush the iptables
command: iptables -F
- name: Make clean
make:
target: clean
chdir: /root/src/k8s.io/kubernetes
- name: Make
make:
chdir: /root/src/k8s.io/kubernetes
- name: Set kubernetes_provider to be local
lineinfile:
dest: /root/.bashrc
line: 'export KUBERNETES_PROVIDER=local'
insertafter: 'EOF'
regexp: 'export KUBERNETES_PROVIDER=local'
state: present
- name: Set KUBECONFIG
lineinfile:
dest: /root/.bashrc
line: 'export KUBECONFIG=/var/run/kubernetes/admin.kubeconfig'
insertafter: 'EOF'
regexp: 'export KUBECONFIG=/var/run/kubernetes/admin.kubeconfig'
state: present
- name: Source the bash rc file
shell: source /root/.bashrc
- name: Make the create cluster script executable
file:
path: /usr/local/bin/createcluster.sh
mode: a+x
- name: Reload daemons to make custom cluster available
shell: systemctl daemon-reload
- name: Bring up the cluster
service:
name: customcluster.service
state: started
enabled: yes
- name: Give the cluster time to initialize
pause: minutes=5
- name: Flush the iptables
command: iptables -F
# XXX: The task below is running kube e2e not kube node-e2e!!!
# The reason behind calling the file node-e2e.log is because the internal
# RH CI looks for a file named like that and would require otherwise a
# change to the CI itself (not doable in this playbook) to look for just
# "e2e.log"..
# So, we'll change it to "e2e.log", some day...
- name: run e2e tests
shell: |
go run hack/e2e.go -v --test -test_args="-host=https://{{ ansible_default_ipv4.address }}:6443 --ginkgo.focus=\[Conformance\]" 2>&1 > node-e2e.log
args:
chdir: /root/src/k8s.io/kubernetes
async: 7200
poll: 10
ignore_errors: yes

View file

@ -0,0 +1,17 @@
---
- name: clone bats source repo
git:
repo: "https://github.com/sstephenson/bats.git"
dest: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats"
version: "{{ 'pull/161/head' if xunit else 'HEAD' }}"
- name: install bats
command: "./install.sh /usr/local"
args:
chdir: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats"
- name: link bats
file:
src: /usr/local/bin/bats
dest: /usr/bin/bats
state: link

View file

@ -0,0 +1,78 @@
---
- name: stat the expected cri-o directory
stat:
path: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
register: dir_stat
- name: expect cri-o to be cloned already
fail:
msg: "Expected cri-o to be cloned at {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o but it wasn't!"
when: not dir_stat.stat.exists
- name: install cri-o tools
make:
target: install.tools
chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
- name: build cri-o
make:
chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
- name: install cri-o
make:
target: install
chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
- name: install cri-o systemd files
make:
target: install.systemd
chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
- name: install cri-o config
make:
target: install.config
chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
- name: install configs
copy:
src: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/{{ item.src }}"
dest: "{{ item.dest }}"
remote_src: yes
with_items:
- src: contrib/cni/10-crio-bridge.conf
dest: /etc/cni/net.d/10-crio-bridge.conf
- src: contrib/cni/99-loopback.conf
dest: /etc/cni/net.d/99-loopback.conf
- src: test/redhat_sigstore.yaml
dest: /etc/containers/registries.d/registry.access.redhat.com.yaml
- name: run with overlay2
replace:
regexp: 'storage_driver = ""'
replace: 'storage_driver = "overlay2"'
name: /etc/crio/crio.conf
backup: yes
- name: run with systemd cgroup manager
replace:
regexp: 'cgroup_manager = "cgroupfs"'
replace: 'cgroup_manager = "systemd"'
name: /etc/crio/crio.conf
backup: yes
- name: add docker.io default registry
lineinfile:
dest: /etc/crio/crio.conf
line: '"docker.io"'
insertafter: 'registries = \['
regexp: 'docker\.io'
state: present
- name: add overlay2 storage opts on RHEL/CentOS
lineinfile:
dest: /etc/crio/crio.conf
line: '"overlay2.override_kernel_check=1"'
insertafter: 'storage_option = \['
regexp: 'overlay2\.override_kernel_check=1'
state: present
when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'

View file

@ -0,0 +1,15 @@
---
- name: clone cri-tools source repo
git:
repo: "https://github.com/kubernetes-incubator/cri-tools.git"
dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-tools"
version: "16e6fe4d7199c5689db4630a9330e6a8a12cecd1"
- name: install crictl
command: "go install github.com/kubernetes-incubator/cri-tools/cmd/crictl"
- name: link crictl
file:
src: "{{ ansible_env.GOPATH }}/bin/crictl"
dest: /usr/bin/crictl
state: link

View file

@ -0,0 +1,61 @@
---
- name: clone kubernetes source repo
git:
repo: "https://github.com/runcom/kubernetes.git"
dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
version: "cri-o-node-e2e-patched"
- name: install etcd
command: "hack/install-etcd.sh"
args:
chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
- name: build kubernetes
make:
chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
- name: Add custom cluster service file for the e2e testing
copy:
dest: /etc/systemd/system/customcluster.service
content: |
[Unit]
After=network-online.target
Wants=network-online.target
[Service]
ExecStart=/usr/local/bin/createcluster.sh
User=root
[Install]
WantedBy=multi-user.target
- name: Add create cluster background script for e2e testing
copy:
dest: /usr/local/bin/createcluster.sh
content: |
#!/bin/bash
export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ ansible_env.GOPATH }}/bin:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/third_party/etcd:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/
export CONTAINER_RUNTIME=remote
export CGROUP_DRIVER=systemd
export CONTAINER_RUNTIME_ENDPOINT='/var/run/crio.sock --runtime-request-timeout=5m'
export ALLOW_SECURITY_CONTEXT=","
export ALLOW_PRIVILEGED=1
export DNS_SERVER_IP={{ ansible_eth0.ipv4.address }}
export API_HOST={{ ansible_eth0.ipv4.address }}
export API_HOST_IP={{ ansible_eth0.ipv4.address }}
export KUBE_ENABLE_CLUSTER_DNS=true
{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/hack/local-up-cluster.sh
mode: "u=rwx,g=rwx,o=x"
- name: Set kubernetes_provider to be local
lineinfile:
dest: /etc/environment
line: 'KUBERNETES_PROVIDER=local'
regexp: 'KUBERNETES_PROVIDER='
state: present
- name: Set KUBECONFIG
lineinfile:
dest: /etc/environment
line: 'KUBECONFIG=/var/run/kubernetes/admin.kubeconfig'
regexp: 'KUBECONFIG='
state: present

View file

@ -0,0 +1,49 @@
---
- name: clone plugins source repo
git:
repo: "https://github.com/containernetworking/plugins.git"
dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
version: "dcf7368eeab15e2affc6256f0bb1e84dd46a34de"
- name: build plugins
command: "./build.sh"
args:
chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
- name: install plugins
copy:
src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/{{ item }}"
dest: "/opt/cni/bin"
mode: "o=rwx,g=rx,o=rx"
remote_src: yes
with_items:
- bridge
- dhcp
- flannel
- host-local
- ipvlan
- loopback
- macvlan
- ptp
- sample
- tuning
- vlan
- name: clone runcom plugins source repo
git:
repo: "https://github.com/runcom/plugins.git"
dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
version: "custom-bridge"
force: yes
- name: build plugins
command: "./build.sh"
args:
chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
- name: install custom bridge
copy:
src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/bridge"
dest: "/opt/cni/bin/bridge-custom"
mode: "o=rwx,g=rx,o=rx"
remote_src: yes

View file

@ -0,0 +1,21 @@
---
- name: clone runc source repo
git:
repo: "https://github.com/opencontainers/runc.git"
dest: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
- name: build runc
make:
params: BUILDTAGS="seccomp selinux"
chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
- name: install runc
make:
target: "install"
chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
- name: link runc
file:
src: /usr/local/sbin/runc
dest: /usr/bin/runc
state: link

View file

@ -0,0 +1,44 @@
---
- name: enable and start CRI-O
systemd:
name: crio
state: started
enabled: yes
daemon_reload: yes
- name: update the server address for the custom cluster
lineinfile:
dest: /usr/local/bin/createcluster.sh
line: "export {{ item }}={{ ansible_eth0.ipv4.address }}"
regexp: "^export {{ item }}="
state: present
with_items:
- DNS_SERVER_IP
- API_HOST
- API_HOST_IP
- name: enable and start the custom cluster
systemd:
name: customcluster.service
state: started
enabled: yes
daemon_reload: yes
- name: wait for the cluster to be running
command: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default"
register: kube_poll
until: kube_poll | succeeded
retries: 100
delay: 30
- name: ensure directory exists for e2e reports
file:
path: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/artifacts"
state: directory
- name: run e2e tests
shell: |
go run hack/e2e.go -v --test -test_args="-host=https://{{ ansible_default_ipv4.address }}:6443 --ginkgo.focus=\[Conformance\]" >e2e.log 2>&1
args:
chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
ignore_errors: yes

View file

@ -0,0 +1,40 @@
---
- name: set up GOPATH if it is not already set
lineinfile:
dest: /etc/environment
line: 'GOPATH={{ ansible_env.HOME }}/go'
regexp: 'GOPATH='
state: present
when: ansible_env.GOPATH is not defined
- name: ensure user profile exists
file:
path: "{{ ansible_user_dir }}/.profile"
state: touch
- name: set up PATH for Go binaries
lineinfile:
dest: "{{ ansible_user_dir }}/.profile"
line: 'PATH={{ ansible_env.PATH }}:{{ ansible_env.GOPATH }}/bin'
regexp: '^PATH='
state: present
- name: set up directories
file:
path: "{{ item }}"
state: directory
with_items:
- "{{ ansible_env.GOPATH }}/src/github.com/containernetworking"
- "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator"
- "{{ ansible_env.GOPATH }}/src/github.com/k8s.io"
- "{{ ansible_env.GOPATH }}/src/github.com/sstephenson"
- "{{ ansible_env.GOPATH }}/src/github.com/opencontainers"
- name: install Go tools and dependencies
shell: /usr/bin/go get -u "github.com/{{ item }}"
with_items:
- tools/godep
- onsi/ginkgo/ginkgo
- onsi/gomega
- cloudflare/cfssl/cmd/...
- jteeuwen/go-bindata/go-bindata

View file

@ -0,0 +1,56 @@
- hosts: all
remote_user: root
vars:
xunit: false
tags:
- setup
tasks:
- name: set up the system
include: system.yml
- name: install Golang tools
include: golang.yml
- name: clone build and install bats
include: "build/bats.yml"
- name: clone build and install cri-tools
include: "build/cri-tools.yml"
- name: clone build and install kubernetes
include: "build/kubernetes.yml"
- name: clone build and install runc
include: "build/runc.yml"
- name: clone build and install networking plugins
include: "build/plugins.yml"
- hosts: all
remote_user: root
vars:
xunit: false
tags:
- integration
- e2e
tasks:
- name: clone build and install cri-o
include: "build/cri-o.yml"
- hosts: all
remote_user: root
vars:
xunit: false
tags:
- integration
tasks:
- name: run cri-o integration tests
include: test.yml
- hosts: all
remote_user: root
tags:
- e2e
tasks:
- name: run k8s e2e tests
include: e2e.yml

View file

@ -0,0 +1,119 @@
---
- name: register a repo for Golang
yum_repository:
name: 'centos-paas-sig-openshift-origin37-rpms'
state: present
description: 'CentOS PaaS SIG Origin 3.7 Repository'
baseurl: 'https://cbs.centos.org/repos/paas7-openshift-origin37-candidate/x86_64/os/'
includepkgs: 'golang'
gpgcheck: no
sslverify: no
when: ansible_distribution in ['RedHat', 'CentOS']
- name: Make sure we have all required packages
package:
name: "{{ item }}"
state: present
with_items:
- btrfs-progs-devel
- container-selinux
- curl
- device-mapper-devel
- expect
- findutils
- gcc
- git
- glib2-devel
- glibc-devel
- glibc-static
- golang
- gpgme-devel
- hostname
- iproute
- iptables
- krb5-workstation
- libassuan-devel
- libffi-devel
- libgpg-error-devel
- libguestfs-tools
- libseccomp-devel
- libvirt-client
- libvirt-python
- libxml2-devel
- libxslt-devel
- make
- mlocate
- nfs-utils
- nmap-ncat
- npm
- oci-register-machine
- oci-systemd-hook
- oci-umount
- openssl
- openssl-devel
- ostree-devel
- pkgconfig
- python
- python2-boto
- python2-crypto
- python2-mock
- python-click
- python-devel
- python-virtualenv
- PyYAML
- redhat-rpm-config
- rpcbind
- rsync
- sed
- skopeo-containers
- socat
- tar
- wget
async: 600
poll: 10
- name: Update all packages
package:
name: '*'
state: latest
async: 600
poll: 10
- name: Setup swap to prevent kernel firing off the OOM killer
shell: |
truncate -s 8G /root/swap && \
export SWAPDEV=$(losetup --show -f /root/swap | head -1) && \
mkswap $SWAPDEV && \
swapon $SWAPDEV && \
swapon --show
- name: ensure directories exist as needed
file:
path: "{{ item }}"
state: directory
with_items:
- /opt/cni/bin
- /etc/cni/net.d
- name: set sysctl vm.overcommit_memory=1 for CentOS
sysctl:
name: vm.overcommit_memory
state: present
value: 1
when: ansible_distribution == 'CentOS'
- name: disable selinux see https://github.com/kubernetes-incubator/cri-o/issues/528
selinux:
policy: targeted
state: permissive
- name: inject hostname into /etc/hosts
lineinfile:
dest: /etc/hosts
line: '{{ ansible_default_ipv4.address }} {{ ansible_nodename }}'
insertafter: 'EOF'
regexp: '{{ ansible_default_ipv4.address }}\s+{{ ansible_nodename }}'
state: present
- name: Flush the iptables
command: iptables -F

View file

@ -0,0 +1,50 @@
---
- name: Change test_runner.sh to use bats xunit output
lineinfile:
dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/test/test_runner.sh"
line: 'execute time bats --tap --junit $TESTS'
regexp: 'execute time bats --tap \$TESTS'
state: present
when: xunit
- name: Make testing output verbose so it can be converted to xunit
lineinfile:
dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/hack/make-rules/test.sh"
line: ' go test -v "${goflags[@]:+${goflags[@]}}" \'
regexp: ' go test \"\$'
state: present
- name: set extra storage options
set_fact:
extra_storage_opts: " --storage-opt overlay2.override_kernel_check=1"
when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
- name: set extra shell for non-xunit tests
set_fact:
extra_shell_suffix: " >testout.txt 2>&1"
when: not xunit
- name: run integration tests
shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTS='--storage-driver=overlay2{{ extra_storage_opts | default('') }}' make localintegration{{ extra_shell_suffix }}"
args:
chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
ignore_errors: yes
register: integration_test
async: 5400
poll: 30
- name: Make testing output directory
file:
path: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/reports"
state: directory
ignore_errors: yes
when: xunit
- name: Move all xunit files into one dir to scp
shell: "mv {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/test/TestReport-bats*.xml {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/reports/"
when: xunit
- name: ensure we fail on bad tests
fail:
msg: Integration tests failed!
when: "'not ok' in integration_test.stdout"