## This playbook expects --extra-vars "commit=" ## and either --extra-vars "pullrequest=" or ## --skip-tags pr. ## ## Required: --extra-vars extras_repo_baseurl= - hosts: all remote_user: root gather_facts: False # Ansible dependencies need installing first vars: xunit: false cni_commit: dcf7368eeab15e2affc6256f0bb1e84dd46a34de integration_timeout_seconds: "{{ 60 * 60 * 2 }}" pre_tasks: # Assert subscription stuff won't get in the way - assert: that: - 'extras_repo_baseurl | default(False)' - 'server_repo_baseurl | default(False)' - name: Install pre-requesites for ansible on all platforms raw: $(type -P dnf || type -P yum) install -y python libselinux-python yum-utils python3-pycurl # Gather facts - setup: # Just one more thing... - name: Install special ansible deps. for fedora raw: dnf install -y python2-dnf python3-dnf when: ansible_distribution == 'Fedora' tasks: - name: Enable EPEL on RHEL & CentOS yum_repository: name: epel description: "Extra Packages for Enterprise Linux" baseurl: "http://dl.fedoraproject.org/pub/epel/7/$basearch" metalink: "https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=$basearch" enabled: True gpgcheck: True gpgcakey: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7Server" when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS') - name: import EPEL GPG keys on RHEL & CentOS command: 'rpm --import {{ item }}' with_items: - https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7Server - https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 - https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS') - name: Enable required RHEL repositories yum_repository: name: "{{ item.key }}" description: Required repo. for RHEL baseurl: "{{ item.value }}" gpgcheck: False enabled: True with_dict: extras: '{{ extras_repo_baseurl | mandatory }}' server: '{{ server_repo_baseurl | mandatory }}' when: (ansible_distribution == 'RedHat') - name: Don't let old cache interfere with new efforts shell: $(type -P dnf || type -P yum) clean all && rm -rf /var/cache/{yum,dnf} - name: disable redhat yum plugin ini_file: path: "/etc/yum/pluginconf.d/subscription-manager.conf" section: main option: enabled value: 0 when: (ansible_distribution == 'RedHat') - name: Update all packages yum: name: '*' state: latest async: 600 poll: 10 when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS') ignore_errors: true - name: Update all packages on Fedora dnf: name: '*' state: latest async: 600 poll: 10 when: ansible_distribution == 'Fedora' - name: Make sure we have all required packages yum: name: "{{ item }}" state: latest with_items: - btrfs-progs-devel - container-selinux - skopeo-containers - curl - device-mapper-devel - expect - findutils - gcc - git - glib2-devel - glibc-devel - glibc-static - golang - gpgme-devel - hostname - iproute - iptables - krb5-workstation - libassuan-devel - libffi-devel - libgpg-error-devel - libguestfs-tools - libseccomp-devel - libvirt-client - libvirt-python - libxml2-devel - libxslt-devel - make - mlocate - nfs-utils - nmap-ncat - npm - oci-register-machine - oci-systemd-hook - oci-umount - openssl - openssl-devel - ostree-devel - pkgconfig - python - python2-boto - python2-crypto - python2-mock - python-click - python-devel - python-virtualenv - PyYAML - redhat-rpm-config - rpcbind - rsync - sed - socat async: 600 poll: 10 when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS') - name: Make sure we have all required packages on Fedora dnf: name: "{{ item }}" state: latest with_items: - nmap-ncat - iptables - wget - git - make - gcc - tar - libseccomp-devel - golang - glib2-devel - glibc-static - container-selinux - btrfs-progs-devel - device-mapper-devel - ostree-devel - glibc-devel - gpgme-devel - libassuan-devel - libgpg-error-devel - pkgconfig - skopeo-containers - oci-systemd-hook - oci-register-machine - oci-umount - socat async: 600 poll: 10 when: ansible_distribution == 'Fedora' - name: Setup swap to prevent kernel firing off the OOM killer shell: | truncate -s 8G /root/swap && \ export SWAPDEV=$(losetup --show -f /root/swap | head -1) && \ mkswap $SWAPDEV && \ swapon $SWAPDEV && \ swapon --show - name: Make testing directories to conform to testing standards file: path: "{{ item }}" state: directory with_items: - /root/src/github.com/kubernetes-incubator - /root/src/github.com/opencontainers - /opt/cni/bin - /etc/cni/net.d - /usr/local/go - name: install Golang upstream in CentOS shell: | curl -fsSL "https://golang.org/dl/go1.8.3.linux-amd64.tar.gz" \ | tar -xzC /usr/local when: ansible_distribution == 'CentOS' - name: Set custom Golang path for CentOS lineinfile: dest: /root/.bashrc line: 'export PATH=/usr/local/go/bin:$PATH' insertafter: 'EOF' regexp: 'export PATH=/usr/local/go/bin:$PATH' state: present when: ansible_distribution == 'CentOS' - name: set sysctl vm.overcommit_memory=1 for CentOS shell: | sysctl -w vm.overcommit_memory=1 when: ansible_distribution == 'CentOS' - name: disable selinux on CentOS :( shell: | setenforce 0 when: ansible_distribution == 'CentOS' - name: git clone bats repo git: repo: https://github.com/sstephenson/bats.git dest: /root/src/bats async: 600 poll: 10 - name: Fetch the xunit feature PR for bats shell: "git fetch origin +refs/pull/161/head:refs/remotes/origin/pr/161" args: chdir: /root/src/bats async: 600 poll: 10 when: xunit - name: Git checkout the xunit PR for bats shell: "git checkout origin/pr/161" args: chdir: /root/src/bats async: 600 poll: 10 when: xunit - name: git clone crictl repo git: repo: https://github.com/kubernetes-incubator/cri-tools dest: /root/src/github.com/kubernetes-incubator/cri-tools version: 16e6fe4d7199c5689db4630a9330e6a8a12cecd1 async: 600 poll: 10 - name: git clone runc repo git: repo: https://github.com/opencontainers/runc dest: /root/src/github.com/opencontainers/runc async: 600 poll: 10 - name: git clone cri-o repo git: repo: https://github.com/kubernetes-incubator/cri-o dest: /root/src/github.com/kubernetes-incubator/cri-o async: 600 poll: 10 - name: git clone cni repo git: repo: https://github.com/containernetworking/plugins dest: /root/src/github.com/containernetworking/plugins version: "{{ cni_commit }}" - name: Git fetch the PR shell: "git fetch origin +refs/pull/{{ pullrequest }}/head:refs/remotes/origin/pr/{{ pullrequest }}" args: chdir: /root/src/github.com/kubernetes-incubator/cri-o tags: - pr async: 600 poll: 10 - name: Git checkout the commit into working branch shell: "git checkout {{ commit }}" args: chdir: /root/src/github.com/kubernetes-incubator/cri-o async: 600 poll: 10 - name: Install bats command: bats/install.sh /usr/local args: chdir: /root/src - name: Add go testing dir to bashrc files lineinfile: dest: /root/.bashrc line: 'export GOPATH=/root' insertafter: 'EOF' regexp: 'export GOPATH=/root' state: present - name: Source the bashrc file shell: source /root/.bashrc - name: Build cni networking shell: ./build.sh args: chdir: /root/src/github.com/containernetworking/plugins - name: cp bin to cni bin dir shell: cp /root/src/github.com/containernetworking/plugins/bin/* /opt/cni/bin - name: curl crio bridge conf file for cni networking get_url: url: https://raw.githubusercontent.com/kubernetes-incubator/cri-o/{{ commit }}/contrib/cni/10-crio-bridge.conf dest: /etc/cni/net.d/10-crio-bridge.conf - name: curl loopback conf for cni networking get_url: url: https://raw.githubusercontent.com/kubernetes-incubator/cri-o/{{ commit }}/contrib/cni/99-loopback.conf dest: /etc/cni/net.d/99-loopback.conf - name: make clean make: target: clean chdir: /root/src/github.com/opencontainers/runc async: 600 poll: 10 - name: make crictl shell: | go install github.com/kubernetes-incubator/cri-tools/cmd/crictl && \ cp $GOPATH/bin/crictl /usr/bin/crictl args: chdir: /root/src/github.com/kubernetes-incubator/cri-o/ - name: make runc make: params: BUILDTAGS="seccomp selinux" chdir: /root/src/github.com/opencontainers/runc async: 600 poll: 10 - name: install runc make: target: install chdir: /root/src/github.com/opencontainers/runc async: 600 poll: 10 - name: Change test_runner.sh to use bats xunit output lineinfile: dest: /root/src/github.com/kubernetes-incubator/cri-o/test/test_runner.sh line: 'execute time bats --tap --junit $TESTS' regexp: 'execute time bats --tap \$TESTS' state: present when: xunit - name: git clone cni test repo git: repo: https://github.com/runcom/plugins dest: /root/src/github.com/containernetworking/plugins version: "custom-bridge" force: yes - name: Build cni test networking shell: ./build.sh args: chdir: /root/src/github.com/containernetworking/plugins - name: cp custom-bridge to opt bin shell: cp /root/src/github.com/containernetworking/plugins/bin/bridge /opt/cni/bin/bridge-custom # k8s builds with go1.8.x, rhel, fedora don't have it yet - name: install Golang upstream in Fedora/RHEL shell: | curl -fsSL "https://golang.org/dl/go1.8.3.linux-amd64.tar.gz" \ | tar -xzC /usr/local when: ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat' - name: Set custom Golang path for Fedora/RHEL lineinfile: dest: /root/.bashrc line: 'export PATH=/usr/local/go/bin:$PATH' insertafter: 'EOF' regexp: 'export PATH=/usr/local/go/bin:$PATH' state: present when: ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat' # dest dir depends on skopeo-containers package - name: Copy redhat sigstore file to test host copy: remote_src: yes src: /root/src/github.com/kubernetes-incubator/cri-o/test/redhat_sigstore.yaml dest: /etc/containers/registries.d/registry.access.redhat.com.yaml - name: Copy test policy.json file copy: remote_src: yes src: /root/src/github.com/kubernetes-incubator/cri-o/test/policy.json dest: /etc/containers/policy.json - name: run integration tests RHEL shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2 --storage-opt overlay2.override_kernel_check=1" make localintegration 2>&1 > testout.txt' args: chdir: /root/src/github.com/kubernetes-incubator/cri-o async: "{{ integration_timeout_seconds | int }}" poll: 10 ignore_errors: yes when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' - name: run integration tests RHEL with xunit results shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2 --storage-opt overlay2.override_kernel_check=1" make localintegration' args: chdir: /root/src/github.com/kubernetes-incubator/cri-o async: "{{ integration_timeout_seconds | int }}" poll: 10 ignore_errors: yes when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS') and xunit - name: run integration tests Fedora shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2" make localintegration 2>&1 > testout.txt' args: chdir: /root/src/github.com/kubernetes-incubator/cri-o async: "{{ integration_timeout_seconds | int }}" poll: 10 ignore_errors: yes when: ansible_distribution == 'Fedora' - name: run integration tests Fedora with xunit results shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2" make localintegration' args: chdir: /root/src/github.com/kubernetes-incubator/cri-o async: "{{ integration_timeout_seconds | int }}" poll: 10 ignore_errors: yes when: (ansible_distribution == 'Fedora' and xunit) - name: Make testing output directory file: path: /root/src/github.com/kubernetes-incubator/cri-o/reports state: directory ignore_errors: yes when: xunit - name: Move all xunit files into one dir to scp shell: 'mv /root/src/github.com/kubernetes-incubator/cri-o/test/TestReport-bats*.xml /root/src/github.com/kubernetes-incubator/cri-o/reports/' when: xunit # XXX: kube tests from now on - name: git clone k8s repo git: repo: https://github.com/runcom/kubernetes dest: /root/src/k8s.io/kubernetes # based on kube upstream v1.7.5 version: cri-o-node-e2e-patched force: yes async: 600 poll: 10 - name: Add custom cluster service file for the e2e testing copy: dest: /etc/systemd/system/customcluster.service content: | [Unit] After=network-online.target Wants=network-online.target [Service] ExecStart=/usr/local/bin/createcluster.sh User=root [Install] WantedBy=multi-user.target - name: Add create cluster background script for e2e testing copy: dest: /usr/local/bin/createcluster.sh content: | #!/bin/bash export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:/usr/local/go/bin:/root/go-tools/bin:/root/src/k8s.io/kubernetes/third_party/etcd export GOPATH=/root/go-tools cd /root/src/k8s.io/kubernetes export CONTAINER_RUNTIME=remote export CGROUP_DRIVER=systemd export CONTAINER_RUNTIME_ENDPOINT='/var/run/crio.sock --runtime-request-timeout=5m' export ALLOW_SECURITY_CONTEXT="," export ALLOW_PRIVILEGED=1 export DNS_SERVER_IP={{ ansible_default_ipv4.address }} export API_HOST={{ ansible_default_ipv4.address }} export API_HOST_IP={{ ansible_default_ipv4.address }} export KUBE_ENABLE_CLUSTER_DNS=true hack/local-up-cluster.sh - name: Add path to bashrc files lineinfile: dest: /root/.bashrc line: 'export PATH=/tmp/mybin:$PATH:/usr/local/go/bin:/root/go-tools/bin:/root/src/k8s.io/kubernetes/third_party/etcd' insertafter: 'EOF' regexp: 'export PATH=/tmp/mybin:\$PATH:/usr/local/go/bin:/root/go-tools/bin:/root/src/k8s.io/kubernetes/third_party/etcd' state: present - name: Add gopath to bashrc files lineinfile: dest: /root/.bashrc line: 'export GOPATH=/root/go-tools' insertafter: 'EOF' regexp: 'export GOPATH=/root/go-tools' state: present - name: Source the bash rc file shell: source /root/.bashrc - name: Get godep shell: go get -u github.com/tools/godep async: 600 poll: 10 - name: Install etcd command: hack/install-etcd.sh args: chdir: /root/src/k8s.io/kubernetes async: 600 poll: 10 - name: Install necessary github go packages shell: go get github.com/onsi/ginkgo/ginkgo ; go get github.com/onsi/gomega ; go get -u github.com/cloudflare/cfssl/cmd/... args: chdir: /root/src/k8s.io/kubernetes async: 600 poll: 10 - name: Make testing output verbose so it can be converted to xunit lineinfile: dest: /root/src/k8s.io/kubernetes/hack/make-rules/test.sh line: ' go test -v "${goflags[@]:+${goflags[@]}}" \' regexp: ' go test \"\$' state: present - name: make and install CRI-O shell: | make install.tools && \ make && \ make install && \ make install.systemd && \ make install.config args: chdir: /root/src/github.com/kubernetes-incubator/cri-o async: 600 poll: 10 - name: link runc file: src=/usr/local/sbin/runc dest=/usr/bin/runc state=link - name: run with overlay2 replace: regexp: 'storage_driver = ""' replace: 'storage_driver = "overlay2"' name: /etc/crio/crio.conf backup: yes - name: run with systemd cgroup manager replace: regexp: 'cgroup_manager = "cgroupfs"' replace: 'cgroup_manager = "systemd"' name: /etc/crio/crio.conf backup: yes - name: add docker.io default registry lineinfile: dest: /etc/crio/crio.conf line: '"docker.io"' insertafter: 'registries = \[' regexp: 'docker\.io' state: present - name: add overlay2 storage opts on RHEL/CentOS lineinfile: dest: /etc/crio/crio.conf line: '"overlay2.override_kernel_check=1"' insertafter: 'storage_option = \[' regexp: 'overlay2\.override_kernel_check=1' state: present when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' - name: enable and start CRI-O systemd: name: crio state: started enabled: yes daemon_reload: yes async: 600 poll: 10 # see https://github.com/kubernetes-incubator/cri-o/issues/528 - name: disable selinux for k8s conformance tests shell: | setenforce 0 async: 600 poll: 10 - name: Go get the go-bindata file shell: go get -u github.com/jteeuwen/go-bindata/go-bindata args: chdir: /root/src/k8s.io/kubernetes async: 600 poll: 10 - name: Install necessary github go packages shell: go get github.com/onsi/ginkgo/ginkgo ; go get github.com/onsi/gomega ; go get -u github.com/cloudflare/cfssl/cmd/... args: chdir: /root/src/k8s.io/kubernetes async: 600 poll: 10 - name: Add path to bashrc files lineinfile: dest: /root/.bashrc line: 'export PATH=$PATH:/root/src/k8s.io/kubernetes/third_party/etcd' insertafter: 'EOF' regexp: 'export PATH=\$PATH:/root/src/k8s.io/kubernetes/third_party/etcd' state: present - name: inject hostname into /etc/hosts lineinfile: dest: /etc/hosts line: '{{ ansible_default_ipv4.address }} {{ ansible_nodename }}' insertafter: 'EOF' regexp: '{{ ansible_default_ipv4.address }}\s+{{ ansible_nodename }}' state: present - name: Flush the iptables command: iptables -F - name: Make clean make: target: clean chdir: /root/src/k8s.io/kubernetes - name: Make make: chdir: /root/src/k8s.io/kubernetes - name: Set kubernetes_provider to be local lineinfile: dest: /root/.bashrc line: 'export KUBERNETES_PROVIDER=local' insertafter: 'EOF' regexp: 'export KUBERNETES_PROVIDER=local' state: present - name: Set KUBECONFIG lineinfile: dest: /root/.bashrc line: 'export KUBECONFIG=/var/run/kubernetes/admin.kubeconfig' insertafter: 'EOF' regexp: 'export KUBECONFIG=/var/run/kubernetes/admin.kubeconfig' state: present - name: Source the bash rc file shell: source /root/.bashrc - name: Make the create cluster script executable file: path: /usr/local/bin/createcluster.sh mode: a+x - name: Reload daemons to make custom cluster available shell: systemctl daemon-reload - name: Bring up the cluster service: name: customcluster.service state: started enabled: yes - name: Give the cluster time to initialize pause: minutes=5 - name: Flush the iptables command: iptables -F # XXX: The task below is running kube e2e not kube node-e2e!!! # The reason behind calling the file node-e2e.log is because the internal # RH CI looks for a file named like that and would require otherwise a # change to the CI itself (not doable in this playbook) to look for just # "e2e.log".. # So, we'll change it to "e2e.log", some day... - name: run e2e tests shell: | go run hack/e2e.go -v --test -test_args="-host=https://{{ ansible_default_ipv4.address }}:6443 --ginkgo.focus=\[Conformance\]" 2>&1 > node-e2e.log args: chdir: /root/src/k8s.io/kubernetes async: 7200 poll: 10 ignore_errors: yes