530 lines
18 KiB
YAML
530 lines
18 KiB
YAML
## This playbook expects --extra-vars "commit=<commit>"
|
|
## and either --extra-vars "pullrequest=<PR #>" or
|
|
## --skip-tags pr
|
|
- hosts: all
|
|
remote_user: root
|
|
vars:
|
|
xunit: false
|
|
cni_commit: dcf7368eeab15e2affc6256f0bb1e84dd46a34de
|
|
tasks:
|
|
- name: Update all packages
|
|
yum:
|
|
name: '*'
|
|
state: latest
|
|
async: 600
|
|
poll: 10
|
|
when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS')
|
|
ignore_errors: true
|
|
- name: Update all packages on Fedora
|
|
dnf:
|
|
name: '*'
|
|
state: latest
|
|
async: 600
|
|
poll: 10
|
|
when: ansible_distribution == 'Fedora'
|
|
- name: Make sure we have all required packages
|
|
yum:
|
|
name: "{{ item }}"
|
|
state: latest
|
|
with_items:
|
|
- wget
|
|
- git
|
|
- make
|
|
- gcc
|
|
- tar
|
|
- libseccomp-devel
|
|
- golang
|
|
- glib2-devel
|
|
- glibc-static
|
|
- container-selinux
|
|
- btrfs-progs-devel
|
|
- device-mapper-devel
|
|
- ostree-devel
|
|
- glibc-devel
|
|
- gpgme-devel
|
|
- libassuan-devel
|
|
- libgpg-error-devel
|
|
- pkgconfig
|
|
- skopeo-containers
|
|
- oci-systemd-hook
|
|
- oci-register-machine
|
|
- oci-umount
|
|
async: 600
|
|
poll: 10
|
|
when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS')
|
|
- name: Make sure we have all required packages on Fedora
|
|
dnf:
|
|
name: "{{ item }}"
|
|
state: latest
|
|
with_items:
|
|
- wget
|
|
- git
|
|
- make
|
|
- gcc
|
|
- tar
|
|
- libseccomp-devel
|
|
- golang
|
|
- glib2-devel
|
|
- glibc-static
|
|
- container-selinux
|
|
- btrfs-progs-devel
|
|
- device-mapper-devel
|
|
- ostree-devel
|
|
- glibc-devel
|
|
- gpgme-devel
|
|
- libassuan-devel
|
|
- libgpg-error-devel
|
|
- pkgconfig
|
|
- skopeo-containers
|
|
- oci-systemd-hook
|
|
- oci-register-machine
|
|
- oci-umount
|
|
async: 600
|
|
poll: 10
|
|
when: ansible_distribution == 'Fedora'
|
|
- name: Setup swap to prevent kernel firing off the OOM killer
|
|
shell: |
|
|
truncate -s 8G /root/swap && \
|
|
export SWAPDEV=$(losetup --show -f /root/swap | head -1) && \
|
|
mkswap $SWAPDEV && \
|
|
swapon $SWAPDEV && \
|
|
swapon --show
|
|
- name: Make testing directories to conform to testing standards
|
|
file:
|
|
path: "{{ item }}"
|
|
state: directory
|
|
with_items:
|
|
- /root/src/github.com/kubernetes-incubator
|
|
- /root/src/github.com/opencontainers
|
|
- /opt/cni/bin
|
|
- /etc/cni/net.d
|
|
- /usr/local/go
|
|
- name: install Golang upstream in CentOS
|
|
shell: |
|
|
curl -fsSL "https://golang.org/dl/go1.8.3.linux-amd64.tar.gz" \
|
|
| tar -xzC /usr/local
|
|
when: ansible_distribution == 'CentOS'
|
|
- name: Set custom Golang path for CentOS
|
|
lineinfile:
|
|
dest: /root/.bashrc
|
|
line: 'export PATH=/usr/local/go/bin:$PATH'
|
|
insertafter: 'EOF'
|
|
regexp: 'export PATH=/usr/local/go/bin:$PATH'
|
|
state: present
|
|
when: ansible_distribution == 'CentOS'
|
|
- name: set sysctl vm.overcommit_memory=1 for CentOS
|
|
shell: |
|
|
sysctl -w vm.overcommit_memory=1
|
|
when: ansible_distribution == 'CentOS'
|
|
- name: disable selinux on CentOS :(
|
|
shell: |
|
|
setenforce 0
|
|
when: ansible_distribution == 'CentOS'
|
|
- name: git clone bats repo
|
|
git:
|
|
repo: https://github.com/sstephenson/bats.git
|
|
dest: /root/src/bats
|
|
async: 600
|
|
poll: 10
|
|
- name: Fetch the xunit feature PR for bats
|
|
shell: "git fetch origin +refs/pull/161/head:refs/remotes/origin/pr/161"
|
|
args:
|
|
chdir: /root/src/bats
|
|
async: 600
|
|
poll: 10
|
|
when: xunit
|
|
- name: Git checkout the xunit PR for bats
|
|
shell: "git checkout origin/pr/161"
|
|
args:
|
|
chdir: /root/src/bats
|
|
async: 600
|
|
poll: 10
|
|
when: xunit
|
|
- name: git clone crictl repo
|
|
git:
|
|
repo: https://github.com/kubernetes-incubator/cri-tools
|
|
dest: /root/src/github.com/kubernetes-incubator/cri-tools
|
|
version: 16e6fe4d7199c5689db4630a9330e6a8a12cecd1
|
|
async: 600
|
|
poll: 10
|
|
- name: git clone runc repo
|
|
git:
|
|
repo: https://github.com/opencontainers/runc
|
|
dest: /root/src/github.com/opencontainers/runc
|
|
async: 600
|
|
poll: 10
|
|
- name: git clone cri-o repo
|
|
git:
|
|
repo: https://github.com/kubernetes-incubator/cri-o
|
|
dest: /root/src/github.com/kubernetes-incubator/cri-o
|
|
async: 600
|
|
poll: 10
|
|
- name: git clone cni repo
|
|
git:
|
|
repo: https://github.com/containernetworking/plugins
|
|
dest: /root/src/github.com/containernetworking/plugins
|
|
version: "{{ cni_commit }}"
|
|
- name: Git fetch the PR
|
|
shell: "git fetch origin +refs/pull/{{ pullrequest }}/head:refs/remotes/origin/pr/{{ pullrequest }}"
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o
|
|
tags:
|
|
- pr
|
|
async: 600
|
|
poll: 10
|
|
- name: Git checkout the commit into working branch
|
|
shell: "git checkout {{ commit }}"
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o
|
|
async: 600
|
|
poll: 10
|
|
- name: Install bats
|
|
command: bats/install.sh /usr/local
|
|
args:
|
|
chdir: /root/src
|
|
- name: Add go testing dir to bashrc files
|
|
lineinfile:
|
|
dest: /root/.bashrc
|
|
line: 'export GOPATH=/root'
|
|
insertafter: 'EOF'
|
|
regexp: 'export GOPATH=/root'
|
|
state: present
|
|
- name: Source the bashrc file
|
|
shell: source /root/.bashrc
|
|
- name: Build cni networking
|
|
shell: ./build.sh
|
|
args:
|
|
chdir: /root/src/github.com/containernetworking/plugins
|
|
- name: cp bin to cni bin dir
|
|
shell: cp /root/src/github.com/containernetworking/plugins/bin/* /opt/cni/bin
|
|
- name: curl crio bridge conf file for cni networking
|
|
get_url:
|
|
url: https://raw.githubusercontent.com/kubernetes-incubator/cri-o/{{ commit }}/contrib/cni/10-crio-bridge.conf
|
|
dest: /etc/cni/net.d/10-crio-bridge.conf
|
|
- name: curl loopback conf for cni networking
|
|
get_url:
|
|
url: https://raw.githubusercontent.com/kubernetes-incubator/cri-o/{{ commit }}/contrib/cni/99-loopback.conf
|
|
dest: /etc/cni/net.d/99-loopback.conf
|
|
- name: make clean
|
|
make:
|
|
target: clean
|
|
chdir: /root/src/github.com/opencontainers/runc
|
|
async: 600
|
|
poll: 10
|
|
- name: make crictl
|
|
shell: |
|
|
go install github.com/kubernetes-incubator/cri-tools/cmd/crictl && \
|
|
cp $GOPATH/bin/crictl /usr/bin/crictl
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o/
|
|
- name: make runc
|
|
make:
|
|
params: BUILDTAGS="seccomp selinux"
|
|
chdir: /root/src/github.com/opencontainers/runc
|
|
async: 600
|
|
poll: 10
|
|
- name: install runc
|
|
make:
|
|
target: install
|
|
chdir: /root/src/github.com/opencontainers/runc
|
|
async: 600
|
|
poll: 10
|
|
- name: Change test_runner.sh to use bats xunit output
|
|
lineinfile:
|
|
dest: /root/src/github.com/kubernetes-incubator/cri-o/test/test_runner.sh
|
|
line: 'execute time bats --tap --junit $TESTS'
|
|
regexp: 'execute time bats --tap \$TESTS'
|
|
state: present
|
|
when: xunit
|
|
- name: git clone cni test repo
|
|
git:
|
|
repo: https://github.com/runcom/plugins
|
|
dest: /root/src/github.com/containernetworking/plugins
|
|
version: "custom-bridge"
|
|
force: yes
|
|
- name: Build cni test networking
|
|
shell: ./build.sh
|
|
args:
|
|
chdir: /root/src/github.com/containernetworking/plugins
|
|
- name: cp custom-bridge to opt bin
|
|
shell: cp /root/src/github.com/containernetworking/plugins/bin/bridge /opt/cni/bin/bridge-custom
|
|
# k8s builds with go1.8.x, rhel, fedora don't have it yet
|
|
- name: install Golang upstream in Fedora/RHEL
|
|
shell: |
|
|
curl -fsSL "https://golang.org/dl/go1.8.3.linux-amd64.tar.gz" \
|
|
| tar -xzC /usr/local
|
|
when: ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat'
|
|
- name: Set custom Golang path for Fedora/RHEL
|
|
lineinfile:
|
|
dest: /root/.bashrc
|
|
line: 'export PATH=/usr/local/go/bin:$PATH'
|
|
insertafter: 'EOF'
|
|
regexp: 'export PATH=/usr/local/go/bin:$PATH'
|
|
state: present
|
|
when: ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat'
|
|
- name: Copy redhat sigstore file to test host
|
|
copy:
|
|
remote_src: yes
|
|
src: /root/src/github.com/kubernetes-incubator/cri-o/test/redhat_sigstore.yaml
|
|
dest: /etc/containers/registries.d/registry.access.redhat.com.yaml
|
|
- name: Copy test policy.json file
|
|
copy:
|
|
remote_src: yes
|
|
src: /root/src/github.com/kubernetes-incubator/cri-o/test/policy.json
|
|
dest: /etc/containers/policy.json
|
|
- name: run integration tests RHEL
|
|
shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2 --storage-opt overlay2.override_kernel_check=1" make localintegration 2>&1 > testout.txt'
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o
|
|
async: 3600
|
|
poll: 10
|
|
ignore_errors: yes
|
|
when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
|
|
- name: run integration tests RHEL with xunit results
|
|
shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2 --storage-opt overlay2.override_kernel_check=1" make localintegration'
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o
|
|
async: 3600
|
|
poll: 10
|
|
ignore_errors: yes
|
|
when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS') and xunit
|
|
- name: run integration tests Fedora
|
|
shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2" make localintegration 2>&1 > testout.txt'
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o
|
|
async: 3600
|
|
poll: 10
|
|
ignore_errors: yes
|
|
when: ansible_distribution == 'Fedora'
|
|
- name: run integration tests Fedora with xunit results
|
|
shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2" make localintegration'
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o
|
|
async: 3600
|
|
poll: 10
|
|
ignore_errors: yes
|
|
when: (ansible_distribution == 'Fedora' and xunit)
|
|
- name: Make testing output directory
|
|
file:
|
|
path: /root/src/github.com/kubernetes-incubator/cri-o/reports
|
|
state: directory
|
|
ignore_errors: yes
|
|
when: xunit
|
|
- name: Move all xunit files into one dir to scp
|
|
shell: 'mv /root/src/github.com/kubernetes-incubator/cri-o/test/TestReport-bats*.xml /root/src/github.com/kubernetes-incubator/cri-o/reports/'
|
|
when: xunit
|
|
# XXX: kube tests from now on
|
|
- name: git clone k8s repo
|
|
git:
|
|
repo: https://github.com/runcom/kubernetes
|
|
dest: /root/src/k8s.io/kubernetes
|
|
# based on kube upstream v1.7.5
|
|
version: cri-o-node-e2e-patched
|
|
force: yes
|
|
async: 600
|
|
poll: 10
|
|
- name: Add custom cluster service file for the e2e testing
|
|
copy:
|
|
dest: /etc/systemd/system/customcluster.service
|
|
content: |
|
|
[Unit]
|
|
After=network-online.target
|
|
Wants=network-online.target
|
|
[Service]
|
|
ExecStart=/usr/local/bin/createcluster.sh
|
|
User=root
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
|
|
- name: Add create cluster background script for e2e testing
|
|
copy:
|
|
dest: /usr/local/bin/createcluster.sh
|
|
content: |
|
|
#!/bin/bash
|
|
|
|
export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:/usr/local/go/bin:/root/go-tools/bin:/root/src/k8s.io/kubernetes/third_party/etcd
|
|
export GOPATH=/root/go-tools
|
|
cd /root/src/k8s.io/kubernetes
|
|
export CONTAINER_RUNTIME=remote
|
|
export CGROUP_DRIVER=systemd
|
|
export CONTAINER_RUNTIME_ENDPOINT='/var/run/crio.sock --runtime-request-timeout=5m'
|
|
export ALLOW_SECURITY_CONTEXT=","
|
|
export ALLOW_PRIVILEGED=1
|
|
export DNS_SERVER_IP={{ ansible_default_ipv4.address }}
|
|
export API_HOST={{ ansible_default_ipv4.address }}
|
|
export API_HOST_IP={{ ansible_default_ipv4.address }}
|
|
export KUBE_ENABLE_CLUSTER_DNS=true
|
|
hack/local-up-cluster.sh
|
|
- name: Add path to bashrc files
|
|
lineinfile:
|
|
dest: /root/.bashrc
|
|
line: 'export PATH=/tmp/mybin:$PATH:/usr/local/go/bin:/root/go-tools/bin:/root/src/k8s.io/kubernetes/third_party/etcd'
|
|
insertafter: 'EOF'
|
|
regexp: 'export PATH=/tmp/mybin:\$PATH:/usr/local/go/bin:/root/go-tools/bin:/root/src/k8s.io/kubernetes/third_party/etcd'
|
|
state: present
|
|
- name: Add gopath to bashrc files
|
|
lineinfile:
|
|
dest: /root/.bashrc
|
|
line: 'export GOPATH=/root/go-tools'
|
|
insertafter: 'EOF'
|
|
regexp: 'export GOPATH=/root/go-tools'
|
|
state: present
|
|
- name: Source the bash rc file
|
|
shell: source /root/.bashrc
|
|
- name: Get godep
|
|
shell: go get -u github.com/tools/godep
|
|
async: 600
|
|
poll: 10
|
|
- name: Install etcd
|
|
command: hack/install-etcd.sh
|
|
args:
|
|
chdir: /root/src/k8s.io/kubernetes
|
|
async: 600
|
|
poll: 10
|
|
- name: Install necessary github go packages
|
|
shell: go get github.com/onsi/ginkgo/ginkgo ; go get github.com/onsi/gomega ; go get -u github.com/cloudflare/cfssl/cmd/...
|
|
args:
|
|
chdir: /root/src/k8s.io/kubernetes
|
|
async: 600
|
|
poll: 10
|
|
- name: Make testing output verbose so it can be converted to xunit
|
|
lineinfile:
|
|
dest: /root/src/k8s.io/kubernetes/hack/make-rules/test.sh
|
|
line: ' go test -v "${goflags[@]:+${goflags[@]}}" \'
|
|
regexp: ' go test \"\$'
|
|
state: present
|
|
- name: make and install CRI-O
|
|
shell: |
|
|
make install.tools && \
|
|
make && \
|
|
make install && \
|
|
make install.systemd && \
|
|
make install.config
|
|
args:
|
|
chdir: /root/src/github.com/kubernetes-incubator/cri-o
|
|
async: 600
|
|
poll: 10
|
|
- name: link runc
|
|
file: src=/usr/local/sbin/runc dest=/usr/bin/runc state=link
|
|
- name: run with overlay2
|
|
replace:
|
|
regexp: 'storage_driver = ""'
|
|
replace: 'storage_driver = "overlay2"'
|
|
name: /etc/crio/crio.conf
|
|
backup: yes
|
|
- name: run with systemd cgroup manager
|
|
replace:
|
|
regexp: 'cgroup_manager = "cgroupfs"'
|
|
replace: 'cgroup_manager = "systemd"'
|
|
name: /etc/crio/crio.conf
|
|
backup: yes
|
|
- name: add docker.io default registry
|
|
lineinfile:
|
|
dest: /etc/crio/crio.conf
|
|
line: '"docker.io"'
|
|
insertafter: 'registries = \['
|
|
regexp: 'docker\.io'
|
|
state: present
|
|
- name: add overlay2 storage opts on RHEL/CentOS
|
|
lineinfile:
|
|
dest: /etc/crio/crio.conf
|
|
line: '"overlay2.override_kernel_check=1"'
|
|
insertafter: 'storage_option = \['
|
|
regexp: 'overlay2\.override_kernel_check=1'
|
|
state: present
|
|
when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
|
|
- name: enable and start CRI-O
|
|
systemd:
|
|
name: crio
|
|
state: started
|
|
enabled: yes
|
|
daemon_reload: yes
|
|
async: 600
|
|
poll: 10
|
|
# see https://github.com/kubernetes-incubator/cri-o/issues/528
|
|
- name: disable selinux for k8s conformance tests
|
|
shell: |
|
|
setenforce 0
|
|
async: 600
|
|
poll: 10
|
|
- name: Go get the go-bindata file
|
|
shell: go get -u github.com/jteeuwen/go-bindata/go-bindata
|
|
args:
|
|
chdir: /root/src/k8s.io/kubernetes
|
|
async: 600
|
|
poll: 10
|
|
- name: Install necessary github go packages
|
|
shell: go get github.com/onsi/ginkgo/ginkgo ; go get github.com/onsi/gomega ; go get -u github.com/cloudflare/cfssl/cmd/...
|
|
args:
|
|
chdir: /root/src/k8s.io/kubernetes
|
|
async: 600
|
|
poll: 10
|
|
- name: Add path to bashrc files
|
|
lineinfile:
|
|
dest: /root/.bashrc
|
|
line: 'export PATH=$PATH:/root/src/k8s.io/kubernetes/third_party/etcd'
|
|
insertafter: 'EOF'
|
|
regexp: 'export PATH=\$PATH:/root/src/k8s.io/kubernetes/third_party/etcd'
|
|
state: present
|
|
- name: inject hostname into /etc/hosts
|
|
lineinfile:
|
|
dest: /etc/hosts
|
|
line: '{{ ansible_default_ipv4.address }} {{ ansible_nodename }}'
|
|
insertafter: 'EOF'
|
|
regexp: '{{ ansible_default_ipv4.address }}\s+{{ ansible_nodename }}'
|
|
state: present
|
|
- name: Flush the iptables
|
|
command: iptables -F
|
|
- name: Make clean
|
|
make:
|
|
target: clean
|
|
chdir: /root/src/k8s.io/kubernetes
|
|
- name: Make
|
|
make:
|
|
chdir: /root/src/k8s.io/kubernetes
|
|
- name: Set kubernetes_provider to be local
|
|
lineinfile:
|
|
dest: /root/.bashrc
|
|
line: 'export KUBERNETES_PROVIDER=local'
|
|
insertafter: 'EOF'
|
|
regexp: 'export KUBERNETES_PROVIDER=local'
|
|
state: present
|
|
- name: Set KUBECONFIG
|
|
lineinfile:
|
|
dest: /root/.bashrc
|
|
line: 'export KUBECONFIG=/var/run/kubernetes/admin.kubeconfig'
|
|
insertafter: 'EOF'
|
|
regexp: 'export KUBECONFIG=/var/run/kubernetes/admin.kubeconfig'
|
|
state: present
|
|
- name: Source the bash rc file
|
|
shell: source /root/.bashrc
|
|
- name: Make the create cluster script executable
|
|
file:
|
|
path: /usr/local/bin/createcluster.sh
|
|
mode: a+x
|
|
- name: Reload daemons to make custom cluster available
|
|
shell: systemctl daemon-reload
|
|
- name: Bring up the cluster
|
|
service:
|
|
name: customcluster.service
|
|
state: started
|
|
enabled: yes
|
|
- name: Give the cluster time to initialize
|
|
pause: minutes=5
|
|
- name: Flush the iptables
|
|
command: iptables -F
|
|
# XXX: The task below is running kube e2e not kube node-e2e!!!
|
|
# The reason behind calling the file node-e2e.log is because the internal
|
|
# RH CI looks for a file named like that and would require otherwise a
|
|
# change to the CI itself (not doable in this playbook) to look for just
|
|
# "e2e.log"..
|
|
# So, we'll change it to "e2e.log", some day...
|
|
- name: run e2e tests
|
|
shell: |
|
|
go run hack/e2e.go -v --test -test_args="-host=https://{{ ansible_default_ipv4.address }}:6443 --ginkgo.focus=\[Conformance\]" 2>&1 > node-e2e.log
|
|
args:
|
|
chdir: /root/src/k8s.io/kubernetes
|
|
async: 7200
|
|
poll: 10
|
|
ignore_errors: yes
|