Refactor the CRI-O test playbook to be more modular

Signed-off-by: Steve Kuznetsov <skuznets@redhat.com>

contrib/test/integration/build/bats.yml

@@ -0,0 +1,17 @@
+---
+- name: clone bats source repo
+  git:
+    repo: "https://github.com/sstephenson/bats.git"
+    dest: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats"
+    version: "{{ 'pull/161/head' if xunit else 'HEAD' }}"
+
+- name: install bats
+  command: "./install.sh /usr/local"
+  args:
+    chdir: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats"
+
+- name: link bats
+  file:
+    src: /usr/local/bin/bats
+    dest: /usr/bin/bats
+    state: link

contrib/test/integration/build/cri-o.yml

@@ -0,0 +1,78 @@
+---
+- name: stat the expected cri-o directory
+  stat:
+    path: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+  register: dir_stat
+
+- name: expect cri-o to be cloned already
+  fail:
+    msg: "Expected cri-o to be cloned at {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o but it wasn't!"
+  when: not dir_stat.stat.exists
+
+- name: install cri-o tools
+  make:
+    target: install.tools
+    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+
+- name: build cri-o
+  make:
+    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+
+- name: install cri-o
+  make:
+    target: install
+    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+
+- name: install cri-o systemd files
+  make:
+    target: install.systemd
+    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+
+- name: install cri-o config
+  make:
+    target: install.config
+    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+
+- name: install configs
+  copy:
+    src: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/{{ item.src }}"
+    dest: "{{ item.dest }}"
+    remote_src: yes
+  with_items:
+    - src: contrib/cni/10-crio-bridge.conf
+      dest: /etc/cni/net.d/10-crio-bridge.conf
+    - src: contrib/cni/99-loopback.conf
+      dest: /etc/cni/net.d/99-loopback.conf
+    - src: test/redhat_sigstore.yaml
+      dest: /etc/containers/registries.d/registry.access.redhat.com.yaml
+
+- name: run with overlay2
+  replace:
+    regexp: 'storage_driver = ""'
+    replace: 'storage_driver = "overlay2"'
+    name: /etc/crio/crio.conf
+    backup: yes
+
+- name: run with systemd cgroup manager
+  replace:
+    regexp: 'cgroup_manager = "cgroupfs"'
+    replace: 'cgroup_manager = "systemd"'
+    name: /etc/crio/crio.conf
+    backup: yes
+
+- name: add docker.io default registry
+  lineinfile:
+    dest: /etc/crio/crio.conf
+    line: '"docker.io"'
+    insertafter: 'registries = \['
+    regexp: 'docker\.io'
+    state: present
+
+- name: add overlay2 storage opts on RHEL/CentOS
+  lineinfile:
+    dest: /etc/crio/crio.conf
+    line: '"overlay2.override_kernel_check=1"'
+    insertafter: 'storage_option = \['
+    regexp: 'overlay2\.override_kernel_check=1'
+    state: present
+  when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'

contrib/test/integration/build/cri-tools.yml

@@ -0,0 +1,15 @@
+---
+- name: clone cri-tools source repo
+  git:
+    repo: "https://github.com/kubernetes-incubator/cri-tools.git"
+    dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-tools"
+    version: "16e6fe4d7199c5689db4630a9330e6a8a12cecd1"
+
+- name: install crictl
+  command: "go install github.com/kubernetes-incubator/cri-tools/cmd/crictl"
+
+- name: link crictl
+  file:
+    src: "{{ ansible_env.GOPATH }}/bin/crictl"
+    dest: /usr/bin/crictl
+    state: link

contrib/test/integration/build/kubernetes.yml

@@ -0,0 +1,61 @@
+---
+- name: clone kubernetes source repo
+  git:
+    repo: "https://github.com/runcom/kubernetes.git"
+    dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+    version: "cri-o-node-e2e-patched"
+
+- name: install etcd
+  command: "hack/install-etcd.sh"
+  args:
+    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+
+- name: build kubernetes
+  make:
+    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+
+- name: Add custom cluster service file for the e2e testing
+  copy:
+    dest: /etc/systemd/system/customcluster.service
+    content: |
+      [Unit]
+      After=network-online.target
+      Wants=network-online.target
+      [Service]
+      ExecStart=/usr/local/bin/createcluster.sh
+      User=root
+      [Install]
+      WantedBy=multi-user.target
+
+- name: Add create cluster background script for e2e testing
+  copy:
+    dest: /usr/local/bin/createcluster.sh
+    content: |
+      #!/bin/bash
+
+      export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ ansible_env.GOPATH }}/bin:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/third_party/etcd:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/
+      export CONTAINER_RUNTIME=remote
+      export CGROUP_DRIVER=systemd
+      export CONTAINER_RUNTIME_ENDPOINT='/var/run/crio.sock --runtime-request-timeout=5m'
+      export ALLOW_SECURITY_CONTEXT=","
+      export ALLOW_PRIVILEGED=1
+      export DNS_SERVER_IP={{ ansible_eth0.ipv4.address }}
+      export API_HOST={{ ansible_eth0.ipv4.address }}
+      export API_HOST_IP={{ ansible_eth0.ipv4.address }}
+      export KUBE_ENABLE_CLUSTER_DNS=true
+      {{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/hack/local-up-cluster.sh
+    mode: "u=rwx,g=rwx,o=x"
+
+- name: Set kubernetes_provider to be local
+  lineinfile:
+    dest: /etc/environment
+    line: 'KUBERNETES_PROVIDER=local'
+    regexp: 'KUBERNETES_PROVIDER='
+    state: present
+
+- name: Set KUBECONFIG
+  lineinfile:
+    dest: /etc/environment
+    line: 'KUBECONFIG=/var/run/kubernetes/admin.kubeconfig'
+    regexp: 'KUBECONFIG='
+    state: present

contrib/test/integration/build/plugins.yml

@@ -0,0 +1,49 @@
+---
+- name: clone plugins source repo
+  git:
+    repo: "https://github.com/containernetworking/plugins.git"
+    dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+    version: "dcf7368eeab15e2affc6256f0bb1e84dd46a34de"
+
+- name: build plugins
+  command: "./build.sh"
+  args:
+    chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+
+- name: install plugins
+  copy:
+    src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/{{ item }}"
+    dest: "/opt/cni/bin"
+    mode: "o=rwx,g=rx,o=rx"
+    remote_src: yes
+  with_items:
+    - bridge
+    - dhcp
+    - flannel
+    - host-local
+    - ipvlan
+    - loopback
+    - macvlan
+    - ptp
+    - sample
+    - tuning
+    - vlan
+
+- name: clone runcom plugins source repo
+  git:
+    repo: "https://github.com/runcom/plugins.git"
+    dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+    version: "custom-bridge"
+    force: yes
+
+- name: build plugins
+  command: "./build.sh"
+  args:
+    chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+
+- name: install custom bridge
+  copy:
+    src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/bridge"
+    dest: "/opt/cni/bin/bridge-custom"
+    mode: "o=rwx,g=rx,o=rx"
+    remote_src: yes

contrib/test/integration/build/runc.yml

@@ -0,0 +1,21 @@
+---
+- name: clone runc source repo
+  git:
+    repo: "https://github.com/opencontainers/runc.git"
+    dest: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
+
+- name: build runc
+  make:
+    params: BUILDTAGS="seccomp selinux"
+    chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
+
+- name: install runc
+  make:
+    target: "install"
+    chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
+
+- name: link runc
+  file:
+    src: /usr/local/sbin/runc
+    dest: /usr/bin/runc
+    state: link

contrib/test/integration/e2e.yml

@@ -0,0 +1,44 @@
+---
+- name: enable and start CRI-O
+  systemd:
+    name: crio
+    state: started
+    enabled: yes
+    daemon_reload: yes
+
+- name: update the server address for the custom cluster
+  lineinfile:
+    dest: /usr/local/bin/createcluster.sh
+    line: "export {{ item }}={{ ansible_eth0.ipv4.address }}"
+    regexp: "^export {{ item }}="
+    state: present
+  with_items:
+    - DNS_SERVER_IP
+    - API_HOST
+    - API_HOST_IP
+
+- name: enable and start the custom cluster
+  systemd:
+    name: customcluster.service
+    state: started
+    enabled: yes
+    daemon_reload: yes
+
+- name: wait for the cluster to be running
+  command: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default"
+  register: kube_poll
+  until: kube_poll | succeeded
+  retries: 100
+  delay: 30
+
+- name: ensure directory exists for e2e reports
+  file:
+    path: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/artifacts"
+    state: directory
+
+- name: run e2e tests
+  shell: |
+    go run hack/e2e.go -v --test -test_args="-host=https://{{ ansible_default_ipv4.address }}:6443 --ginkgo.focus=\[Conformance\]" >e2e.log 2>&1
+  args:
+    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+  ignore_errors: yes

contrib/test/integration/golang.yml

@@ -0,0 +1,40 @@
+---
+- name: set up GOPATH if it is not already set
+  lineinfile:
+    dest: /etc/environment
+    line: 'GOPATH={{ ansible_env.HOME }}/go'
+    regexp: 'GOPATH='
+    state: present
+  when: ansible_env.GOPATH is not defined
+
+- name: ensure user profile exists
+  file:
+    path: "{{ ansible_user_dir }}/.profile"
+    state: touch
+
+- name: set up PATH for Go binaries
+  lineinfile:
+    dest: "{{ ansible_user_dir }}/.profile"
+    line: 'PATH={{ ansible_env.PATH }}:{{ ansible_env.GOPATH }}/bin'
+    regexp: '^PATH='
+    state: present
+
+- name: set up directories
+  file:
+    path: "{{ item }}"
+    state: directory
+  with_items:
+    - "{{ ansible_env.GOPATH }}/src/github.com/containernetworking"
+    - "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator"
+    - "{{ ansible_env.GOPATH }}/src/github.com/k8s.io"
+    - "{{ ansible_env.GOPATH }}/src/github.com/sstephenson"
+    - "{{ ansible_env.GOPATH }}/src/github.com/opencontainers"
+
+- name: install Go tools and dependencies
+  shell: /usr/bin/go get -u "github.com/{{ item }}"
+  with_items:
+    - tools/godep
+    - onsi/ginkgo/ginkgo
+    - onsi/gomega
+    - cloudflare/cfssl/cmd/...
+    - jteeuwen/go-bindata/go-bindata

contrib/test/integration/main.yml

@@ -0,0 +1,56 @@
+- hosts: all
+  remote_user: root
+  vars:
+    xunit: false
+  tags:
+    - setup
+  tasks:
+    - name: set up the system
+      include: system.yml
+
+    - name: install Golang tools
+      include: golang.yml
+
+    - name: clone build and install bats
+      include: "build/bats.yml"
+
+    - name: clone build and install cri-tools
+      include: "build/cri-tools.yml"
+
+    - name: clone build and install kubernetes
+      include: "build/kubernetes.yml"
+
+    - name: clone build and install runc
+      include: "build/runc.yml"
+
+    - name: clone build and install networking plugins
+      include: "build/plugins.yml"
+
+- hosts: all
+  remote_user: root
+  vars:
+    xunit: false
+  tags:
+    - integration
+    - e2e
+  tasks:
+    - name: clone build and install cri-o
+      include: "build/cri-o.yml"
+
+- hosts: all
+  remote_user: root
+  vars:
+    xunit: false
+  tags:
+    - integration
+  tasks:
+    - name: run cri-o integration tests
+      include: test.yml
+
+- hosts: all
+  remote_user: root
+  tags:
+    - e2e
+  tasks:
+    - name: run k8s e2e tests
+      include: e2e.yml

contrib/test/integration/system.yml

@@ -0,0 +1,119 @@
+---
+- name: register a repo for Golang
+  yum_repository:
+    name: 'centos-paas-sig-openshift-origin37-rpms'
+    state: present
+    description: 'CentOS PaaS SIG Origin 3.7 Repository'
+    baseurl: 'https://cbs.centos.org/repos/paas7-openshift-origin37-candidate/x86_64/os/'
+    includepkgs: 'golang'
+    gpgcheck: no
+    sslverify: no
+  when: ansible_distribution in ['RedHat', 'CentOS']
+
+- name: Make sure we have all required packages
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - btrfs-progs-devel
+    - container-selinux
+    - curl
+    - device-mapper-devel
+    - expect
+    - findutils
+    - gcc
+    - git
+    - glib2-devel
+    - glibc-devel
+    - glibc-static
+    - golang
+    - gpgme-devel
+    - hostname
+    - iproute
+    - iptables
+    - krb5-workstation
+    - libassuan-devel
+    - libffi-devel
+    - libgpg-error-devel
+    - libguestfs-tools
+    - libseccomp-devel
+    - libvirt-client
+    - libvirt-python
+    - libxml2-devel
+    - libxslt-devel
+    - make
+    - mlocate
+    - nfs-utils
+    - nmap-ncat
+    - npm
+    - oci-register-machine
+    - oci-systemd-hook
+    - oci-umount
+    - openssl
+    - openssl-devel
+    - ostree-devel
+    - pkgconfig
+    - python
+    - python2-boto
+    - python2-crypto
+    - python2-mock
+    - python-click
+    - python-devel
+    - python-virtualenv
+    - PyYAML
+    - redhat-rpm-config
+    - rpcbind
+    - rsync
+    - sed
+    - skopeo-containers
+    - socat
+    - tar
+    - wget
+  async: 600
+  poll: 10
+
+- name: Update all packages
+  package:
+    name: '*'
+    state: latest
+  async: 600
+  poll: 10
+
+- name: Setup swap to prevent kernel firing off the OOM killer
+  shell: |
+    truncate -s 8G /root/swap && \
+    export SWAPDEV=$(losetup --show -f /root/swap | head -1) && \
+    mkswap $SWAPDEV && \
+    swapon $SWAPDEV && \
+    swapon --show
+
+- name: ensure directories exist as needed
+  file:
+    path: "{{ item }}"
+    state: directory
+  with_items:
+    - /opt/cni/bin
+    - /etc/cni/net.d
+
+- name: set sysctl vm.overcommit_memory=1 for CentOS
+  sysctl:
+    name: vm.overcommit_memory
+    state: present
+    value: 1
+  when: ansible_distribution == 'CentOS'
+
+- name: disable selinux see https://github.com/kubernetes-incubator/cri-o/issues/528
+  selinux:
+    policy: targeted
+    state: permissive
+
+- name: inject hostname into /etc/hosts
+  lineinfile:
+    dest: /etc/hosts
+    line: '{{ ansible_default_ipv4.address }} {{ ansible_nodename }}'
+    insertafter: 'EOF'
+    regexp: '{{ ansible_default_ipv4.address }}\s+{{ ansible_nodename }}'
+    state: present
+
+- name: Flush the iptables
+  command: iptables -F

contrib/test/integration/test.yml

@@ -0,0 +1,50 @@
+---
+- name: Change test_runner.sh to use bats xunit output
+  lineinfile:
+    dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/test/test_runner.sh"
+    line: 'execute time bats --tap --junit $TESTS'
+    regexp: 'execute time bats --tap \$TESTS'
+    state: present
+  when: xunit
+
+- name: Make testing output verbose so it can be converted to xunit
+  lineinfile:
+    dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/hack/make-rules/test.sh"
+    line: ' go test -v "${goflags[@]:+${goflags[@]}}" \'
+    regexp: ' go test \"\$'
+    state: present
+
+- name: set extra storage options
+  set_fact:
+    extra_storage_opts: " --storage-opt overlay2.override_kernel_check=1"
+  when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
+
+- name: set extra shell for non-xunit tests
+  set_fact:
+    extra_shell_suffix: " >testout.txt 2>&1"
+  when: not xunit
+
+- name: run integration tests
+  shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTS='--storage-driver=overlay2{{ extra_storage_opts | default('') }}' make localintegration{{ extra_shell_suffix }}"
+  args:
+    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+  ignore_errors: yes
+  register: integration_test
+  async: 5400
+  poll: 30
+
+- name: Make testing output directory
+  file:
+    path: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/reports"
+    state: directory
+  ignore_errors: yes
+  when: xunit
+
+- name: Move all xunit files into one dir to scp
+  shell: "mv {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/test/TestReport-bats*.xml {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/reports/"
+  when: xunit
+
+- name: ensure we fail on bad tests
+  fail:
+    msg: Integration tests failed!
+  when: "'not ok' in integration_test.stdout"