From e160796d4ea42e9c4b3a911ae7c2b8e3d3241e8b Mon Sep 17 00:00:00 2001 From: Steve Kuznetsov Date: Tue, 19 Sep 2017 15:50:37 -0700 Subject: [PATCH] Refactor the CRI-O test playbook to be more modular Signed-off-by: Steve Kuznetsov --- contrib/test/crio-integration-playbook.yaml | 618 ------------------ contrib/test/integration/build/bats.yml | 17 + contrib/test/integration/build/cri-o.yml | 78 +++ contrib/test/integration/build/cri-tools.yml | 15 + contrib/test/integration/build/kubernetes.yml | 61 ++ contrib/test/integration/build/plugins.yml | 49 ++ contrib/test/integration/build/runc.yml | 21 + contrib/test/integration/e2e.yml | 44 ++ contrib/test/integration/golang.yml | 40 ++ contrib/test/integration/main.yml | 56 ++ contrib/test/integration/system.yml | 119 ++++ contrib/test/integration/test.yml | 50 ++ 12 files changed, 550 insertions(+), 618 deletions(-) delete mode 100644 contrib/test/crio-integration-playbook.yaml create mode 100644 contrib/test/integration/build/bats.yml create mode 100644 contrib/test/integration/build/cri-o.yml create mode 100644 contrib/test/integration/build/cri-tools.yml create mode 100644 contrib/test/integration/build/kubernetes.yml create mode 100644 contrib/test/integration/build/plugins.yml create mode 100644 contrib/test/integration/build/runc.yml create mode 100644 contrib/test/integration/e2e.yml create mode 100644 contrib/test/integration/golang.yml create mode 100644 contrib/test/integration/main.yml create mode 100644 contrib/test/integration/system.yml create mode 100644 contrib/test/integration/test.yml diff --git a/contrib/test/crio-integration-playbook.yaml b/contrib/test/crio-integration-playbook.yaml deleted file mode 100644 index c097f523..00000000 --- a/contrib/test/crio-integration-playbook.yaml +++ /dev/null @@ -1,618 +0,0 @@ -## This playbook expects --extra-vars "commit=" -## and either --extra-vars "pullrequest=" or -## --skip-tags pr. -## -## Required: --extra-vars extras_repo_baseurl= -- hosts: all - remote_user: root - gather_facts: False # Ansible dependencies need installing first - vars: - xunit: false - cni_commit: dcf7368eeab15e2affc6256f0bb1e84dd46a34de - integration_timeout_seconds: "{{ 60 * 60 * 2 }}" - pre_tasks: - # Assert subscription stuff won't get in the way - - assert: - that: - - 'extras_repo_baseurl | default(False)' - - 'server_repo_baseurl | default(False)' - - name: Install pre-requesites for ansible on all platforms - raw: $(type -P dnf || type -P yum) install -y python libselinux-python yum-utils python3-pycurl - # Gather facts - - setup: - # Just one more thing... - - name: Install special ansible deps. for fedora - raw: dnf install -y python2-dnf python3-dnf - when: ansible_distribution == 'Fedora' - tasks: - - name: Enable EPEL on RHEL & CentOS - yum_repository: - name: epel - description: "Extra Packages for Enterprise Linux" - baseurl: "http://dl.fedoraproject.org/pub/epel/7/$basearch" - metalink: "https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=$basearch" - enabled: True - gpgcheck: True - gpgcakey: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7Server" - when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS') - - name: import EPEL GPG keys on RHEL & CentOS - command: 'rpm --import {{ item }}' - with_items: - - https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7Server - - https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7 - - https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL - when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS') - - name: Enable required RHEL repositories - yum_repository: - name: "{{ item.key }}" - description: Required repo. for RHEL - baseurl: "{{ item.value }}" - gpgcheck: False - enabled: True - with_dict: - extras: '{{ extras_repo_baseurl | mandatory }}' - server: '{{ server_repo_baseurl | mandatory }}' - when: (ansible_distribution == 'RedHat') - - name: Don't let old cache interfere with new efforts - shell: $(type -P dnf || type -P yum) clean all && rm -rf /var/cache/{yum,dnf} - - name: disable redhat yum plugin - ini_file: - path: "/etc/yum/pluginconf.d/subscription-manager.conf" - section: main - option: enabled - value: 0 - when: (ansible_distribution == 'RedHat') - - name: Update all packages - yum: - name: '*' - state: latest - async: 600 - poll: 10 - when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS') - ignore_errors: true - - name: Update all packages on Fedora - dnf: - name: '*' - state: latest - async: 600 - poll: 10 - when: ansible_distribution == 'Fedora' - - name: Make sure we have all required packages - yum: - name: "{{ item }}" - state: latest - with_items: - - btrfs-progs-devel - - container-selinux - - skopeo-containers - - curl - - device-mapper-devel - - expect - - findutils - - gcc - - git - - glib2-devel - - glibc-devel - - glibc-static - - golang - - gpgme-devel - - hostname - - iproute - - iptables - - krb5-workstation - - libassuan-devel - - libffi-devel - - libgpg-error-devel - - libguestfs-tools - - libseccomp-devel - - libvirt-client - - libvirt-python - - libxml2-devel - - libxslt-devel - - make - - mlocate - - nfs-utils - - nmap-ncat - - npm - - oci-register-machine - - oci-systemd-hook - - oci-umount - - openssl - - openssl-devel - - ostree-devel - - pkgconfig - - python - - python2-boto - - python2-crypto - - python2-mock - - python-click - - python-devel - - python-virtualenv - - PyYAML - - redhat-rpm-config - - rpcbind - - rsync - - sed - - socat - async: 600 - poll: 10 - when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS') - - name: Make sure we have all required packages on Fedora - dnf: - name: "{{ item }}" - state: latest - with_items: - - nmap-ncat - - iptables - - wget - - git - - make - - gcc - - tar - - libseccomp-devel - - golang - - glib2-devel - - glibc-static - - container-selinux - - btrfs-progs-devel - - device-mapper-devel - - ostree-devel - - glibc-devel - - gpgme-devel - - libassuan-devel - - libgpg-error-devel - - pkgconfig - - skopeo-containers - - oci-systemd-hook - - oci-register-machine - - oci-umount - - socat - async: 600 - poll: 10 - when: ansible_distribution == 'Fedora' - - name: Setup swap to prevent kernel firing off the OOM killer - shell: | - truncate -s 8G /root/swap && \ - export SWAPDEV=$(losetup --show -f /root/swap | head -1) && \ - mkswap $SWAPDEV && \ - swapon $SWAPDEV && \ - swapon --show - - name: Make testing directories to conform to testing standards - file: - path: "{{ item }}" - state: directory - with_items: - - /root/src/github.com/kubernetes-incubator - - /root/src/github.com/opencontainers - - /opt/cni/bin - - /etc/cni/net.d - - /usr/local/go - - name: install Golang upstream in CentOS - shell: | - curl -fsSL "https://golang.org/dl/go1.8.3.linux-amd64.tar.gz" \ - | tar -xzC /usr/local - when: ansible_distribution == 'CentOS' - - name: Set custom Golang path for CentOS - lineinfile: - dest: /root/.bashrc - line: 'export PATH=/usr/local/go/bin:$PATH' - insertafter: 'EOF' - regexp: 'export PATH=/usr/local/go/bin:$PATH' - state: present - when: ansible_distribution == 'CentOS' - - name: set sysctl vm.overcommit_memory=1 for CentOS - shell: | - sysctl -w vm.overcommit_memory=1 - when: ansible_distribution == 'CentOS' - - name: disable selinux on CentOS :( - shell: | - setenforce 0 - when: ansible_distribution == 'CentOS' - - name: git clone bats repo - git: - repo: https://github.com/sstephenson/bats.git - dest: /root/src/bats - async: 600 - poll: 10 - - name: Fetch the xunit feature PR for bats - shell: "git fetch origin +refs/pull/161/head:refs/remotes/origin/pr/161" - args: - chdir: /root/src/bats - async: 600 - poll: 10 - when: xunit - - name: Git checkout the xunit PR for bats - shell: "git checkout origin/pr/161" - args: - chdir: /root/src/bats - async: 600 - poll: 10 - when: xunit - - name: git clone crictl repo - git: - repo: https://github.com/kubernetes-incubator/cri-tools - dest: /root/src/github.com/kubernetes-incubator/cri-tools - version: 16e6fe4d7199c5689db4630a9330e6a8a12cecd1 - async: 600 - poll: 10 - - name: git clone runc repo - git: - repo: https://github.com/opencontainers/runc - dest: /root/src/github.com/opencontainers/runc - async: 600 - poll: 10 - - name: git clone cri-o repo - git: - repo: https://github.com/kubernetes-incubator/cri-o - dest: /root/src/github.com/kubernetes-incubator/cri-o - async: 600 - poll: 10 - - name: git clone cni repo - git: - repo: https://github.com/containernetworking/plugins - dest: /root/src/github.com/containernetworking/plugins - version: "{{ cni_commit }}" - - name: Git fetch the PR - shell: "git fetch origin +refs/pull/{{ pullrequest }}/head:refs/remotes/origin/pr/{{ pullrequest }}" - args: - chdir: /root/src/github.com/kubernetes-incubator/cri-o - tags: - - pr - async: 600 - poll: 10 - - name: Git checkout the commit into working branch - shell: "git checkout {{ commit }}" - args: - chdir: /root/src/github.com/kubernetes-incubator/cri-o - async: 600 - poll: 10 - - name: Install bats - command: bats/install.sh /usr/local - args: - chdir: /root/src - - name: Add go testing dir to bashrc files - lineinfile: - dest: /root/.bashrc - line: 'export GOPATH=/root' - insertafter: 'EOF' - regexp: 'export GOPATH=/root' - state: present - - name: Source the bashrc file - shell: source /root/.bashrc - - name: Build cni networking - shell: ./build.sh - args: - chdir: /root/src/github.com/containernetworking/plugins - - name: cp bin to cni bin dir - shell: cp /root/src/github.com/containernetworking/plugins/bin/* /opt/cni/bin - - name: curl crio bridge conf file for cni networking - get_url: - url: https://raw.githubusercontent.com/kubernetes-incubator/cri-o/{{ commit }}/contrib/cni/10-crio-bridge.conf - dest: /etc/cni/net.d/10-crio-bridge.conf - - name: curl loopback conf for cni networking - get_url: - url: https://raw.githubusercontent.com/kubernetes-incubator/cri-o/{{ commit }}/contrib/cni/99-loopback.conf - dest: /etc/cni/net.d/99-loopback.conf - - name: make clean - make: - target: clean - chdir: /root/src/github.com/opencontainers/runc - async: 600 - poll: 10 - - name: make crictl - shell: | - go install github.com/kubernetes-incubator/cri-tools/cmd/crictl && \ - cp $GOPATH/bin/crictl /usr/bin/crictl - args: - chdir: /root/src/github.com/kubernetes-incubator/cri-o/ - - name: make runc - make: - params: BUILDTAGS="seccomp selinux" - chdir: /root/src/github.com/opencontainers/runc - async: 600 - poll: 10 - - name: install runc - make: - target: install - chdir: /root/src/github.com/opencontainers/runc - async: 600 - poll: 10 - - name: Change test_runner.sh to use bats xunit output - lineinfile: - dest: /root/src/github.com/kubernetes-incubator/cri-o/test/test_runner.sh - line: 'execute time bats --tap --junit $TESTS' - regexp: 'execute time bats --tap \$TESTS' - state: present - when: xunit - - name: git clone cni test repo - git: - repo: https://github.com/runcom/plugins - dest: /root/src/github.com/containernetworking/plugins - version: "custom-bridge" - force: yes - - name: Build cni test networking - shell: ./build.sh - args: - chdir: /root/src/github.com/containernetworking/plugins - - name: cp custom-bridge to opt bin - shell: cp /root/src/github.com/containernetworking/plugins/bin/bridge /opt/cni/bin/bridge-custom - # k8s builds with go1.8.x, rhel, fedora don't have it yet - - name: install Golang upstream in Fedora/RHEL - shell: | - curl -fsSL "https://golang.org/dl/go1.8.3.linux-amd64.tar.gz" \ - | tar -xzC /usr/local - when: ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat' - - name: Set custom Golang path for Fedora/RHEL - lineinfile: - dest: /root/.bashrc - line: 'export PATH=/usr/local/go/bin:$PATH' - insertafter: 'EOF' - regexp: 'export PATH=/usr/local/go/bin:$PATH' - state: present - when: ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat' - # dest dir depends on skopeo-containers package - - name: Copy redhat sigstore file to test host - copy: - remote_src: yes - src: /root/src/github.com/kubernetes-incubator/cri-o/test/redhat_sigstore.yaml - dest: /etc/containers/registries.d/registry.access.redhat.com.yaml - - name: Copy test policy.json file - copy: - remote_src: yes - src: /root/src/github.com/kubernetes-incubator/cri-o/test/policy.json - dest: /etc/containers/policy.json - - name: run integration tests RHEL - shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2 --storage-opt overlay2.override_kernel_check=1" make localintegration 2>&1 > testout.txt' - args: - chdir: /root/src/github.com/kubernetes-incubator/cri-o - async: "{{ integration_timeout_seconds | int }}" - poll: 10 - ignore_errors: yes - when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' - - name: run integration tests RHEL with xunit results - shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2 --storage-opt overlay2.override_kernel_check=1" make localintegration' - args: - chdir: /root/src/github.com/kubernetes-incubator/cri-o - async: "{{ integration_timeout_seconds | int }}" - poll: 10 - ignore_errors: yes - when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS') and xunit - - name: run integration tests Fedora - shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2" make localintegration 2>&1 > testout.txt' - args: - chdir: /root/src/github.com/kubernetes-incubator/cri-o - async: "{{ integration_timeout_seconds | int }}" - poll: 10 - ignore_errors: yes - when: ansible_distribution == 'Fedora' - - name: run integration tests Fedora with xunit results - shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2" make localintegration' - args: - chdir: /root/src/github.com/kubernetes-incubator/cri-o - async: "{{ integration_timeout_seconds | int }}" - poll: 10 - ignore_errors: yes - when: (ansible_distribution == 'Fedora' and xunit) - - name: Make testing output directory - file: - path: /root/src/github.com/kubernetes-incubator/cri-o/reports - state: directory - ignore_errors: yes - when: xunit - - name: Move all xunit files into one dir to scp - shell: 'mv /root/src/github.com/kubernetes-incubator/cri-o/test/TestReport-bats*.xml /root/src/github.com/kubernetes-incubator/cri-o/reports/' - when: xunit - # XXX: kube tests from now on - - name: git clone k8s repo - git: - repo: https://github.com/runcom/kubernetes - dest: /root/src/k8s.io/kubernetes - # based on kube upstream v1.7.5 - version: cri-o-node-e2e-patched - force: yes - async: 600 - poll: 10 - - name: Add custom cluster service file for the e2e testing - copy: - dest: /etc/systemd/system/customcluster.service - content: | - [Unit] - After=network-online.target - Wants=network-online.target - [Service] - ExecStart=/usr/local/bin/createcluster.sh - User=root - [Install] - WantedBy=multi-user.target - - - name: Add create cluster background script for e2e testing - copy: - dest: /usr/local/bin/createcluster.sh - content: | - #!/bin/bash - - export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:/usr/local/go/bin:/root/go-tools/bin:/root/src/k8s.io/kubernetes/third_party/etcd - export GOPATH=/root/go-tools - cd /root/src/k8s.io/kubernetes - export CONTAINER_RUNTIME=remote - export CGROUP_DRIVER=systemd - export CONTAINER_RUNTIME_ENDPOINT='/var/run/crio.sock --runtime-request-timeout=5m' - export ALLOW_SECURITY_CONTEXT="," - export ALLOW_PRIVILEGED=1 - export DNS_SERVER_IP={{ ansible_default_ipv4.address }} - export API_HOST={{ ansible_default_ipv4.address }} - export API_HOST_IP={{ ansible_default_ipv4.address }} - export KUBE_ENABLE_CLUSTER_DNS=true - hack/local-up-cluster.sh - - name: Add path to bashrc files - lineinfile: - dest: /root/.bashrc - line: 'export PATH=/tmp/mybin:$PATH:/usr/local/go/bin:/root/go-tools/bin:/root/src/k8s.io/kubernetes/third_party/etcd' - insertafter: 'EOF' - regexp: 'export PATH=/tmp/mybin:\$PATH:/usr/local/go/bin:/root/go-tools/bin:/root/src/k8s.io/kubernetes/third_party/etcd' - state: present - - name: Add gopath to bashrc files - lineinfile: - dest: /root/.bashrc - line: 'export GOPATH=/root/go-tools' - insertafter: 'EOF' - regexp: 'export GOPATH=/root/go-tools' - state: present - - name: Source the bash rc file - shell: source /root/.bashrc - - name: Get godep - shell: go get -u github.com/tools/godep - async: 600 - poll: 10 - - name: Install etcd - command: hack/install-etcd.sh - args: - chdir: /root/src/k8s.io/kubernetes - async: 600 - poll: 10 - - name: Install necessary github go packages - shell: go get github.com/onsi/ginkgo/ginkgo ; go get github.com/onsi/gomega ; go get -u github.com/cloudflare/cfssl/cmd/... - args: - chdir: /root/src/k8s.io/kubernetes - async: 600 - poll: 10 - - name: Make testing output verbose so it can be converted to xunit - lineinfile: - dest: /root/src/k8s.io/kubernetes/hack/make-rules/test.sh - line: ' go test -v "${goflags[@]:+${goflags[@]}}" \' - regexp: ' go test \"\$' - state: present - - name: make and install CRI-O - shell: | - make install.tools && \ - make && \ - make install && \ - make install.systemd && \ - make install.config - args: - chdir: /root/src/github.com/kubernetes-incubator/cri-o - async: 600 - poll: 10 - - name: link runc - file: src=/usr/local/sbin/runc dest=/usr/bin/runc state=link - - name: run with overlay2 - replace: - regexp: 'storage_driver = ""' - replace: 'storage_driver = "overlay2"' - name: /etc/crio/crio.conf - backup: yes - - name: run with systemd cgroup manager - replace: - regexp: 'cgroup_manager = "cgroupfs"' - replace: 'cgroup_manager = "systemd"' - name: /etc/crio/crio.conf - backup: yes - - name: add docker.io default registry - lineinfile: - dest: /etc/crio/crio.conf - line: '"docker.io"' - insertafter: 'registries = \[' - regexp: 'docker\.io' - state: present - - name: add overlay2 storage opts on RHEL/CentOS - lineinfile: - dest: /etc/crio/crio.conf - line: '"overlay2.override_kernel_check=1"' - insertafter: 'storage_option = \[' - regexp: 'overlay2\.override_kernel_check=1' - state: present - when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' - - name: enable and start CRI-O - systemd: - name: crio - state: started - enabled: yes - daemon_reload: yes - async: 600 - poll: 10 - # see https://github.com/kubernetes-incubator/cri-o/issues/528 - - name: disable selinux for k8s conformance tests - shell: | - setenforce 0 - async: 600 - poll: 10 - - name: Go get the go-bindata file - shell: go get -u github.com/jteeuwen/go-bindata/go-bindata - args: - chdir: /root/src/k8s.io/kubernetes - async: 600 - poll: 10 - - name: Install necessary github go packages - shell: go get github.com/onsi/ginkgo/ginkgo ; go get github.com/onsi/gomega ; go get -u github.com/cloudflare/cfssl/cmd/... - args: - chdir: /root/src/k8s.io/kubernetes - async: 600 - poll: 10 - - name: Add path to bashrc files - lineinfile: - dest: /root/.bashrc - line: 'export PATH=$PATH:/root/src/k8s.io/kubernetes/third_party/etcd' - insertafter: 'EOF' - regexp: 'export PATH=\$PATH:/root/src/k8s.io/kubernetes/third_party/etcd' - state: present - - name: inject hostname into /etc/hosts - lineinfile: - dest: /etc/hosts - line: '{{ ansible_default_ipv4.address }} {{ ansible_nodename }}' - insertafter: 'EOF' - regexp: '{{ ansible_default_ipv4.address }}\s+{{ ansible_nodename }}' - state: present - - name: Flush the iptables - command: iptables -F - - name: Make clean - make: - target: clean - chdir: /root/src/k8s.io/kubernetes - - name: Make - make: - chdir: /root/src/k8s.io/kubernetes - - name: Set kubernetes_provider to be local - lineinfile: - dest: /root/.bashrc - line: 'export KUBERNETES_PROVIDER=local' - insertafter: 'EOF' - regexp: 'export KUBERNETES_PROVIDER=local' - state: present - - name: Set KUBECONFIG - lineinfile: - dest: /root/.bashrc - line: 'export KUBECONFIG=/var/run/kubernetes/admin.kubeconfig' - insertafter: 'EOF' - regexp: 'export KUBECONFIG=/var/run/kubernetes/admin.kubeconfig' - state: present - - name: Source the bash rc file - shell: source /root/.bashrc - - name: Make the create cluster script executable - file: - path: /usr/local/bin/createcluster.sh - mode: a+x - - name: Reload daemons to make custom cluster available - shell: systemctl daemon-reload - - name: Bring up the cluster - service: - name: customcluster.service - state: started - enabled: yes - - name: Give the cluster time to initialize - pause: minutes=5 - - name: Flush the iptables - command: iptables -F - # XXX: The task below is running kube e2e not kube node-e2e!!! - # The reason behind calling the file node-e2e.log is because the internal - # RH CI looks for a file named like that and would require otherwise a - # change to the CI itself (not doable in this playbook) to look for just - # "e2e.log".. - # So, we'll change it to "e2e.log", some day... - - name: run e2e tests - shell: | - go run hack/e2e.go -v --test -test_args="-host=https://{{ ansible_default_ipv4.address }}:6443 --ginkgo.focus=\[Conformance\]" 2>&1 > node-e2e.log - args: - chdir: /root/src/k8s.io/kubernetes - async: 7200 - poll: 10 - ignore_errors: yes diff --git a/contrib/test/integration/build/bats.yml b/contrib/test/integration/build/bats.yml new file mode 100644 index 00000000..b1621e40 --- /dev/null +++ b/contrib/test/integration/build/bats.yml @@ -0,0 +1,17 @@ +--- +- name: clone bats source repo + git: + repo: "https://github.com/sstephenson/bats.git" + dest: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats" + version: "{{ 'pull/161/head' if xunit else 'HEAD' }}" + +- name: install bats + command: "./install.sh /usr/local" + args: + chdir: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats" + +- name: link bats + file: + src: /usr/local/bin/bats + dest: /usr/bin/bats + state: link \ No newline at end of file diff --git a/contrib/test/integration/build/cri-o.yml b/contrib/test/integration/build/cri-o.yml new file mode 100644 index 00000000..b0ed43bf --- /dev/null +++ b/contrib/test/integration/build/cri-o.yml @@ -0,0 +1,78 @@ +--- +- name: stat the expected cri-o directory + stat: + path: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + register: dir_stat + +- name: expect cri-o to be cloned already + fail: + msg: "Expected cri-o to be cloned at {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o but it wasn't!" + when: not dir_stat.stat.exists + +- name: install cri-o tools + make: + target: install.tools + chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + +- name: build cri-o + make: + chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + +- name: install cri-o + make: + target: install + chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + +- name: install cri-o systemd files + make: + target: install.systemd + chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + +- name: install cri-o config + make: + target: install.config + chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + +- name: install configs + copy: + src: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/{{ item.src }}" + dest: "{{ item.dest }}" + remote_src: yes + with_items: + - src: contrib/cni/10-crio-bridge.conf + dest: /etc/cni/net.d/10-crio-bridge.conf + - src: contrib/cni/99-loopback.conf + dest: /etc/cni/net.d/99-loopback.conf + - src: test/redhat_sigstore.yaml + dest: /etc/containers/registries.d/registry.access.redhat.com.yaml + +- name: run with overlay2 + replace: + regexp: 'storage_driver = ""' + replace: 'storage_driver = "overlay2"' + name: /etc/crio/crio.conf + backup: yes + +- name: run with systemd cgroup manager + replace: + regexp: 'cgroup_manager = "cgroupfs"' + replace: 'cgroup_manager = "systemd"' + name: /etc/crio/crio.conf + backup: yes + +- name: add docker.io default registry + lineinfile: + dest: /etc/crio/crio.conf + line: '"docker.io"' + insertafter: 'registries = \[' + regexp: 'docker\.io' + state: present + +- name: add overlay2 storage opts on RHEL/CentOS + lineinfile: + dest: /etc/crio/crio.conf + line: '"overlay2.override_kernel_check=1"' + insertafter: 'storage_option = \[' + regexp: 'overlay2\.override_kernel_check=1' + state: present + when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' \ No newline at end of file diff --git a/contrib/test/integration/build/cri-tools.yml b/contrib/test/integration/build/cri-tools.yml new file mode 100644 index 00000000..a2c2b755 --- /dev/null +++ b/contrib/test/integration/build/cri-tools.yml @@ -0,0 +1,15 @@ +--- +- name: clone cri-tools source repo + git: + repo: "https://github.com/kubernetes-incubator/cri-tools.git" + dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-tools" + version: "16e6fe4d7199c5689db4630a9330e6a8a12cecd1" + +- name: install crictl + command: "go install github.com/kubernetes-incubator/cri-tools/cmd/crictl" + +- name: link crictl + file: + src: "{{ ansible_env.GOPATH }}/bin/crictl" + dest: /usr/bin/crictl + state: link \ No newline at end of file diff --git a/contrib/test/integration/build/kubernetes.yml b/contrib/test/integration/build/kubernetes.yml new file mode 100644 index 00000000..2f4dedee --- /dev/null +++ b/contrib/test/integration/build/kubernetes.yml @@ -0,0 +1,61 @@ +--- +- name: clone kubernetes source repo + git: + repo: "https://github.com/runcom/kubernetes.git" + dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + version: "cri-o-node-e2e-patched" + +- name: install etcd + command: "hack/install-etcd.sh" + args: + chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + +- name: build kubernetes + make: + chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + +- name: Add custom cluster service file for the e2e testing + copy: + dest: /etc/systemd/system/customcluster.service + content: | + [Unit] + After=network-online.target + Wants=network-online.target + [Service] + ExecStart=/usr/local/bin/createcluster.sh + User=root + [Install] + WantedBy=multi-user.target + +- name: Add create cluster background script for e2e testing + copy: + dest: /usr/local/bin/createcluster.sh + content: | + #!/bin/bash + + export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ ansible_env.GOPATH }}/bin:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/third_party/etcd:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/ + export CONTAINER_RUNTIME=remote + export CGROUP_DRIVER=systemd + export CONTAINER_RUNTIME_ENDPOINT='/var/run/crio.sock --runtime-request-timeout=5m' + export ALLOW_SECURITY_CONTEXT="," + export ALLOW_PRIVILEGED=1 + export DNS_SERVER_IP={{ ansible_eth0.ipv4.address }} + export API_HOST={{ ansible_eth0.ipv4.address }} + export API_HOST_IP={{ ansible_eth0.ipv4.address }} + export KUBE_ENABLE_CLUSTER_DNS=true + {{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/hack/local-up-cluster.sh + mode: "u=rwx,g=rwx,o=x" + +- name: Set kubernetes_provider to be local + lineinfile: + dest: /etc/environment + line: 'KUBERNETES_PROVIDER=local' + regexp: 'KUBERNETES_PROVIDER=' + state: present + +- name: Set KUBECONFIG + lineinfile: + dest: /etc/environment + line: 'KUBECONFIG=/var/run/kubernetes/admin.kubeconfig' + regexp: 'KUBECONFIG=' + state: present \ No newline at end of file diff --git a/contrib/test/integration/build/plugins.yml b/contrib/test/integration/build/plugins.yml new file mode 100644 index 00000000..29a5dc54 --- /dev/null +++ b/contrib/test/integration/build/plugins.yml @@ -0,0 +1,49 @@ +--- +- name: clone plugins source repo + git: + repo: "https://github.com/containernetworking/plugins.git" + dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + version: "dcf7368eeab15e2affc6256f0bb1e84dd46a34de" + +- name: build plugins + command: "./build.sh" + args: + chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + +- name: install plugins + copy: + src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/{{ item }}" + dest: "/opt/cni/bin" + mode: "o=rwx,g=rx,o=rx" + remote_src: yes + with_items: + - bridge + - dhcp + - flannel + - host-local + - ipvlan + - loopback + - macvlan + - ptp + - sample + - tuning + - vlan + +- name: clone runcom plugins source repo + git: + repo: "https://github.com/runcom/plugins.git" + dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + version: "custom-bridge" + force: yes + +- name: build plugins + command: "./build.sh" + args: + chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + +- name: install custom bridge + copy: + src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/bridge" + dest: "/opt/cni/bin/bridge-custom" + mode: "o=rwx,g=rx,o=rx" + remote_src: yes \ No newline at end of file diff --git a/contrib/test/integration/build/runc.yml b/contrib/test/integration/build/runc.yml new file mode 100644 index 00000000..b146e5b7 --- /dev/null +++ b/contrib/test/integration/build/runc.yml @@ -0,0 +1,21 @@ +--- +- name: clone runc source repo + git: + repo: "https://github.com/opencontainers/runc.git" + dest: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" + +- name: build runc + make: + params: BUILDTAGS="seccomp selinux" + chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" + +- name: install runc + make: + target: "install" + chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" + +- name: link runc + file: + src: /usr/local/sbin/runc + dest: /usr/bin/runc + state: link \ No newline at end of file diff --git a/contrib/test/integration/e2e.yml b/contrib/test/integration/e2e.yml new file mode 100644 index 00000000..e808c8f5 --- /dev/null +++ b/contrib/test/integration/e2e.yml @@ -0,0 +1,44 @@ +--- +- name: enable and start CRI-O + systemd: + name: crio + state: started + enabled: yes + daemon_reload: yes + +- name: update the server address for the custom cluster + lineinfile: + dest: /usr/local/bin/createcluster.sh + line: "export {{ item }}={{ ansible_eth0.ipv4.address }}" + regexp: "^export {{ item }}=" + state: present + with_items: + - DNS_SERVER_IP + - API_HOST + - API_HOST_IP + +- name: enable and start the custom cluster + systemd: + name: customcluster.service + state: started + enabled: yes + daemon_reload: yes + +- name: wait for the cluster to be running + command: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default" + register: kube_poll + until: kube_poll | succeeded + retries: 100 + delay: 30 + +- name: ensure directory exists for e2e reports + file: + path: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/artifacts" + state: directory + +- name: run e2e tests + shell: | + go run hack/e2e.go -v --test -test_args="-host=https://{{ ansible_default_ipv4.address }}:6443 --ginkgo.focus=\[Conformance\]" >e2e.log 2>&1 + args: + chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + ignore_errors: yes \ No newline at end of file diff --git a/contrib/test/integration/golang.yml b/contrib/test/integration/golang.yml new file mode 100644 index 00000000..bc6ef440 --- /dev/null +++ b/contrib/test/integration/golang.yml @@ -0,0 +1,40 @@ +--- +- name: set up GOPATH if it is not already set + lineinfile: + dest: /etc/environment + line: 'GOPATH={{ ansible_env.HOME }}/go' + regexp: 'GOPATH=' + state: present + when: ansible_env.GOPATH is not defined + +- name: ensure user profile exists + file: + path: "{{ ansible_user_dir }}/.profile" + state: touch + +- name: set up PATH for Go binaries + lineinfile: + dest: "{{ ansible_user_dir }}/.profile" + line: 'PATH={{ ansible_env.PATH }}:{{ ansible_env.GOPATH }}/bin' + regexp: '^PATH=' + state: present + +- name: set up directories + file: + path: "{{ item }}" + state: directory + with_items: + - "{{ ansible_env.GOPATH }}/src/github.com/containernetworking" + - "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator" + - "{{ ansible_env.GOPATH }}/src/github.com/k8s.io" + - "{{ ansible_env.GOPATH }}/src/github.com/sstephenson" + - "{{ ansible_env.GOPATH }}/src/github.com/opencontainers" + +- name: install Go tools and dependencies + shell: /usr/bin/go get -u "github.com/{{ item }}" + with_items: + - tools/godep + - onsi/ginkgo/ginkgo + - onsi/gomega + - cloudflare/cfssl/cmd/... + - jteeuwen/go-bindata/go-bindata \ No newline at end of file diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml new file mode 100644 index 00000000..f2e1ce66 --- /dev/null +++ b/contrib/test/integration/main.yml @@ -0,0 +1,56 @@ +- hosts: all + remote_user: root + vars: + xunit: false + tags: + - setup + tasks: + - name: set up the system + include: system.yml + + - name: install Golang tools + include: golang.yml + + - name: clone build and install bats + include: "build/bats.yml" + + - name: clone build and install cri-tools + include: "build/cri-tools.yml" + + - name: clone build and install kubernetes + include: "build/kubernetes.yml" + + - name: clone build and install runc + include: "build/runc.yml" + + - name: clone build and install networking plugins + include: "build/plugins.yml" + +- hosts: all + remote_user: root + vars: + xunit: false + tags: + - integration + - e2e + tasks: + - name: clone build and install cri-o + include: "build/cri-o.yml" + +- hosts: all + remote_user: root + vars: + xunit: false + tags: + - integration + tasks: + - name: run cri-o integration tests + include: test.yml + +- hosts: all + remote_user: root + tags: + - e2e + tasks: + - name: run k8s e2e tests + include: e2e.yml \ No newline at end of file diff --git a/contrib/test/integration/system.yml b/contrib/test/integration/system.yml new file mode 100644 index 00000000..d94b582d --- /dev/null +++ b/contrib/test/integration/system.yml @@ -0,0 +1,119 @@ +--- +- name: register a repo for Golang + yum_repository: + name: 'centos-paas-sig-openshift-origin37-rpms' + state: present + description: 'CentOS PaaS SIG Origin 3.7 Repository' + baseurl: 'https://cbs.centos.org/repos/paas7-openshift-origin37-candidate/x86_64/os/' + includepkgs: 'golang' + gpgcheck: no + sslverify: no + when: ansible_distribution in ['RedHat', 'CentOS'] + +- name: Make sure we have all required packages + package: + name: "{{ item }}" + state: present + with_items: + - btrfs-progs-devel + - container-selinux + - curl + - device-mapper-devel + - expect + - findutils + - gcc + - git + - glib2-devel + - glibc-devel + - glibc-static + - golang + - gpgme-devel + - hostname + - iproute + - iptables + - krb5-workstation + - libassuan-devel + - libffi-devel + - libgpg-error-devel + - libguestfs-tools + - libseccomp-devel + - libvirt-client + - libvirt-python + - libxml2-devel + - libxslt-devel + - make + - mlocate + - nfs-utils + - nmap-ncat + - npm + - oci-register-machine + - oci-systemd-hook + - oci-umount + - openssl + - openssl-devel + - ostree-devel + - pkgconfig + - python + - python2-boto + - python2-crypto + - python2-mock + - python-click + - python-devel + - python-virtualenv + - PyYAML + - redhat-rpm-config + - rpcbind + - rsync + - sed + - skopeo-containers + - socat + - tar + - wget + async: 600 + poll: 10 + +- name: Update all packages + package: + name: '*' + state: latest + async: 600 + poll: 10 + +- name: Setup swap to prevent kernel firing off the OOM killer + shell: | + truncate -s 8G /root/swap && \ + export SWAPDEV=$(losetup --show -f /root/swap | head -1) && \ + mkswap $SWAPDEV && \ + swapon $SWAPDEV && \ + swapon --show + +- name: ensure directories exist as needed + file: + path: "{{ item }}" + state: directory + with_items: + - /opt/cni/bin + - /etc/cni/net.d + +- name: set sysctl vm.overcommit_memory=1 for CentOS + sysctl: + name: vm.overcommit_memory + state: present + value: 1 + when: ansible_distribution == 'CentOS' + +- name: disable selinux see https://github.com/kubernetes-incubator/cri-o/issues/528 + selinux: + policy: targeted + state: permissive + +- name: inject hostname into /etc/hosts + lineinfile: + dest: /etc/hosts + line: '{{ ansible_default_ipv4.address }} {{ ansible_nodename }}' + insertafter: 'EOF' + regexp: '{{ ansible_default_ipv4.address }}\s+{{ ansible_nodename }}' + state: present + +- name: Flush the iptables + command: iptables -F \ No newline at end of file diff --git a/contrib/test/integration/test.yml b/contrib/test/integration/test.yml new file mode 100644 index 00000000..d9b9a2c3 --- /dev/null +++ b/contrib/test/integration/test.yml @@ -0,0 +1,50 @@ +--- +- name: Change test_runner.sh to use bats xunit output + lineinfile: + dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/test/test_runner.sh" + line: 'execute time bats --tap --junit $TESTS' + regexp: 'execute time bats --tap \$TESTS' + state: present + when: xunit + +- name: Make testing output verbose so it can be converted to xunit + lineinfile: + dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/hack/make-rules/test.sh" + line: ' go test -v "${goflags[@]:+${goflags[@]}}" \' + regexp: ' go test \"\$' + state: present + +- name: set extra storage options + set_fact: + extra_storage_opts: " --storage-opt overlay2.override_kernel_check=1" + when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' + +- name: set extra shell for non-xunit tests + set_fact: + extra_shell_suffix: " >testout.txt 2>&1" + when: not xunit + +- name: run integration tests + shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTS='--storage-driver=overlay2{{ extra_storage_opts | default('') }}' make localintegration{{ extra_shell_suffix }}" + args: + chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + ignore_errors: yes + register: integration_test + async: 5400 + poll: 30 + +- name: Make testing output directory + file: + path: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/reports" + state: directory + ignore_errors: yes + when: xunit + +- name: Move all xunit files into one dir to scp + shell: "mv {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/test/TestReport-bats*.xml {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/reports/" + when: xunit + +- name: ensure we fail on bad tests + fail: + msg: Integration tests failed! + when: "'not ok' in integration_test.stdout" \ No newline at end of file