cri-o/contrib/test/crio-integration-playbook.yaml

## This playbook expects --extra-vars "commit=<commit>"
## and either --extra-vars "pullrequest=<PR #>" or
## --skip-tags pr.
##
## Required: --extra-vars extras_repo_baseurl=<HTTP or HTTPS>
- hosts: all
  remote_user: root
  gather_facts: False  # Ansible dependencies need installing first
  vars:
    xunit: false
    cni_commit: dcf7368eeab15e2affc6256f0bb1e84dd46a34de
    integration_timeout_seconds: "{{ 60 * 60 * 2 }}"
  pre_tasks:
    # Assert subscription stuff won't get in the way
    - assert:
        that:
          - 'extras_repo_baseurl | default(False)'
          - 'server_repo_baseurl | default(False)'
    - name: Install pre-requesites for ansible on all platforms
      raw: $(type -P dnf || type -P yum) install -y python libselinux-python yum-utils python3-pycurl
    # Gather facts
    - setup:
    # Just one more thing...
    - name: Install special ansible deps. for fedora
      raw: dnf install -y python2-dnf python3-dnf
      when: ansible_distribution == 'Fedora'
  tasks:
  - name: Enable EPEL on RHEL & CentOS
    yum_repository:
      name: epel
      description: "Extra Packages for Enterprise Linux"
      baseurl: "http://dl.fedoraproject.org/pub/epel/7/$basearch"
      metalink: "https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=$basearch"
      enabled: True
      gpgcheck: True
      gpgcakey: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7Server"
    when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS')
  - name: import EPEL GPG keys on RHEL & CentOS
    command: 'rpm --import {{ item }}'
    with_items:
      - https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7Server
      - https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
      - https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL
    when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS')
  - name: Enable required RHEL repositories
    yum_repository:
      name: "{{ item.key }}"
      description: Required repo. for RHEL
      baseurl: "{{ item.value }}"
      gpgcheck: False
      enabled: True
    with_dict:
      extras: '{{ extras_repo_baseurl | mandatory }}'
      server: '{{ server_repo_baseurl | mandatory }}'
    when: (ansible_distribution == 'RedHat')
  - name: Don't let old cache interfere with new efforts
    shell: $(type -P dnf || type -P yum) clean all && rm -rf /var/cache/{yum,dnf}
  - name: disable redhat yum plugin
    ini_file:
      path: "/etc/yum/pluginconf.d/subscription-manager.conf"
      section: main
      option: enabled
      value: 0
    when: (ansible_distribution == 'RedHat')
  - name: Update all packages
    yum:
      name: '*'
      state: latest
    async: 600
    poll: 10
    when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS')
    ignore_errors: true
  - name: Update all packages on Fedora
    dnf:
      name: '*'
      state: latest
    async: 600
    poll: 10
    when: ansible_distribution == 'Fedora'
  - name: Make sure we have all required packages
    yum:
      name: "{{ item }}"
      state: latest
    with_items:
      - btrfs-progs-devel
      - container-selinux
      - skopeo-containers
      - curl
      - device-mapper-devel
      - expect
      - findutils
      - gcc
      - git
      - glib2-devel
      - glibc-devel
      - glibc-static
      - golang
      - gpgme-devel
      - hostname
      - iproute
      - iptables
      - krb5-workstation
      - libassuan-devel
      - libffi-devel
      - libgpg-error-devel
      - libguestfs-tools
      - libseccomp-devel
      - libvirt-client
      - libvirt-python
      - libxml2-devel
      - libxslt-devel
      - make
      - mlocate
      - nfs-utils
      - nmap-ncat
      - npm
      - oci-register-machine
      - oci-systemd-hook
      - oci-umount
      - openssl
      - openssl-devel
      - ostree-devel
      - pkgconfig
      - python
      - python2-boto
      - python2-crypto
      - python2-mock
      - python-click
      - python-devel
      - python-virtualenv
      - PyYAML
      - redhat-rpm-config
      - rpcbind
      - rsync
      - sed
      - socat
    async: 600
    poll: 10
    when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS')
  - name: Make sure we have all required packages on Fedora
    dnf:
      name: "{{ item }}"
      state: latest
    with_items:
      - nmap-ncat
      - iptables
      - wget
      - git
      - make
      - gcc
      - tar
      - libseccomp-devel
      - golang
      - glib2-devel
      - glibc-static
      - container-selinux
      - btrfs-progs-devel
      - device-mapper-devel
      - ostree-devel
      - glibc-devel
      - gpgme-devel
      - libassuan-devel
      - libgpg-error-devel
      - pkgconfig
      - skopeo-containers
      - oci-systemd-hook
      - oci-register-machine
      - oci-umount
      - socat
    async: 600
    poll: 10
    when: ansible_distribution == 'Fedora'
  - name: Setup swap to prevent kernel firing off the OOM killer
    shell: |
      truncate -s 8G /root/swap && \
      export SWAPDEV=$(losetup --show -f /root/swap | head -1) && \
      mkswap $SWAPDEV && \
      swapon $SWAPDEV && \
      swapon --show      
  - name: Make testing directories to conform to testing standards
    file:
      path: "{{ item }}"
      state: directory
    with_items:
      - /root/src/github.com/kubernetes-incubator
      - /root/src/github.com/opencontainers
      - /opt/cni/bin
      - /etc/cni/net.d
      - /usr/local/go
  - name: install Golang upstream in CentOS
    shell: |
            curl -fsSL "https://golang.org/dl/go1.8.3.linux-amd64.tar.gz" \
            | tar -xzC /usr/local            
    when: ansible_distribution == 'CentOS'
  - name: Set custom Golang path for CentOS
    lineinfile:
      dest: /root/.bashrc
      line: 'export PATH=/usr/local/go/bin:$PATH'
      insertafter: 'EOF'
      regexp: 'export PATH=/usr/local/go/bin:$PATH'
      state: present
    when: ansible_distribution == 'CentOS'
  - name: set sysctl vm.overcommit_memory=1 for CentOS
    shell: |
            sysctl -w vm.overcommit_memory=1            
    when: ansible_distribution == 'CentOS'
  - name: disable selinux on CentOS :(
    shell: |
            setenforce 0            
    when: ansible_distribution == 'CentOS'
  - name: git clone bats repo
    git:
      repo: https://github.com/sstephenson/bats.git
      dest: /root/src/bats
    async: 600
    poll: 10
  - name: Fetch the xunit feature PR for bats
    shell: "git fetch origin +refs/pull/161/head:refs/remotes/origin/pr/161"
    args:
      chdir: /root/src/bats
    async: 600
    poll: 10
    when: xunit
  - name: Git checkout the xunit PR for bats
    shell: "git checkout origin/pr/161"
    args:
      chdir: /root/src/bats
    async: 600
    poll: 10
    when: xunit
  - name: git clone crictl repo
    git:
      repo: https://github.com/kubernetes-incubator/cri-tools
      dest: /root/src/github.com/kubernetes-incubator/cri-tools
      version: 16e6fe4d7199c5689db4630a9330e6a8a12cecd1
    async: 600
    poll: 10
  - name: git clone runc repo
    git:
      repo: https://github.com/opencontainers/runc
      dest: /root/src/github.com/opencontainers/runc
    async: 600
    poll: 10
  - name: git clone cri-o repo
    git:
      repo: https://github.com/kubernetes-incubator/cri-o
      dest: /root/src/github.com/kubernetes-incubator/cri-o
    async: 600
    poll: 10
  - name: git clone cni repo
    git:
      repo: https://github.com/containernetworking/plugins
      dest: /root/src/github.com/containernetworking/plugins
      version: "{{ cni_commit }}"
  - name: Git fetch the PR
    shell: "git fetch origin +refs/pull/{{ pullrequest }}/head:refs/remotes/origin/pr/{{ pullrequest }}"
    args:
      chdir: /root/src/github.com/kubernetes-incubator/cri-o
    tags:
      - pr
    async: 600
    poll: 10
  - name: Git checkout the commit into working branch
    shell: "git checkout {{ commit }}"
    args:
      chdir: /root/src/github.com/kubernetes-incubator/cri-o
    async: 600
    poll: 10
  - name: Install bats
    command: bats/install.sh /usr/local
    args:
      chdir: /root/src
  - name: Add go testing dir to bashrc files
    lineinfile:
      dest: /root/.bashrc
      line: 'export GOPATH=/root'
      insertafter: 'EOF'
      regexp: 'export GOPATH=/root'
      state: present
  - name: Source the bashrc file
    shell: source /root/.bashrc
  - name: Build cni networking
    shell: ./build.sh
    args:
      chdir: /root/src/github.com/containernetworking/plugins
  - name: cp bin to cni bin dir
    shell: cp /root/src/github.com/containernetworking/plugins/bin/* /opt/cni/bin
  - name: curl crio bridge conf file for cni networking
    get_url:
      url: https://raw.githubusercontent.com/kubernetes-incubator/cri-o/{{ commit }}/contrib/cni/10-crio-bridge.conf
      dest: /etc/cni/net.d/10-crio-bridge.conf
  - name: curl loopback conf for cni networking
    get_url:
      url: https://raw.githubusercontent.com/kubernetes-incubator/cri-o/{{ commit }}/contrib/cni/99-loopback.conf
      dest: /etc/cni/net.d/99-loopback.conf
  - name: make clean
    make:
      target: clean
      chdir: /root/src/github.com/opencontainers/runc
    async: 600
    poll: 10
  - name: make crictl
    shell: |
            go install github.com/kubernetes-incubator/cri-tools/cmd/crictl && \
            cp $GOPATH/bin/crictl /usr/bin/crictl            
    args:
      chdir: /root/src/github.com/kubernetes-incubator/cri-o/
  - name: make runc
    make:
      params: BUILDTAGS="seccomp selinux"
      chdir: /root/src/github.com/opencontainers/runc
    async: 600
    poll: 10
  - name: install runc
    make:
      target: install
      chdir: /root/src/github.com/opencontainers/runc
    async: 600
    poll: 10
  - name: Change test_runner.sh to use bats xunit output
    lineinfile:
      dest: /root/src/github.com/kubernetes-incubator/cri-o/test/test_runner.sh
      line: 'execute time bats --tap --junit $TESTS'
      regexp: 'execute time bats --tap \$TESTS'
      state: present
    when: xunit
  - name: git clone cni test repo
    git:
      repo: https://github.com/runcom/plugins
      dest: /root/src/github.com/containernetworking/plugins
      version: "custom-bridge"
      force: yes
  - name: Build cni test networking
    shell: ./build.sh
    args:
      chdir: /root/src/github.com/containernetworking/plugins
  - name: cp custom-bridge to opt bin
    shell: cp /root/src/github.com/containernetworking/plugins/bin/bridge /opt/cni/bin/bridge-custom
    # k8s builds with go1.8.x, rhel, fedora don't have it yet
  - name: install Golang upstream in Fedora/RHEL
    shell: |
            curl -fsSL "https://golang.org/dl/go1.8.3.linux-amd64.tar.gz" \
            | tar -xzC /usr/local            
    when: ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat'
  - name: Set custom Golang path for Fedora/RHEL
    lineinfile:
      dest: /root/.bashrc
      line: 'export PATH=/usr/local/go/bin:$PATH'
      insertafter: 'EOF'
      regexp: 'export PATH=/usr/local/go/bin:$PATH'
      state: present
    when: ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat'
  # dest dir depends on skopeo-containers package
  - name: Copy redhat sigstore file to test host
    copy:
      remote_src: yes
      src: /root/src/github.com/kubernetes-incubator/cri-o/test/redhat_sigstore.yaml
      dest: /etc/containers/registries.d/registry.access.redhat.com.yaml
  - name: Copy test policy.json file
    copy:
      remote_src: yes
      src: /root/src/github.com/kubernetes-incubator/cri-o/test/policy.json
      dest: /etc/containers/policy.json
  - name: run integration tests RHEL
    shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2 --storage-opt overlay2.override_kernel_check=1" make localintegration 2>&1 > testout.txt'
    args:
      chdir: /root/src/github.com/kubernetes-incubator/cri-o
    async: "{{ integration_timeout_seconds | int }}"
    poll: 10
    ignore_errors: yes
    when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
  - name: run integration tests RHEL with xunit results
    shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2 --storage-opt overlay2.override_kernel_check=1" make localintegration'
    args:
      chdir: /root/src/github.com/kubernetes-incubator/cri-o
    async: "{{ integration_timeout_seconds | int }}"
    poll: 10
    ignore_errors: yes
    when: (ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS') and xunit
  - name: run integration tests Fedora
    shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2" make localintegration 2>&1 > testout.txt'
    args:
      chdir: /root/src/github.com/kubernetes-incubator/cri-o
    async: "{{ integration_timeout_seconds | int }}"
    poll: 10
    ignore_errors: yes
    when: ansible_distribution == 'Fedora'
  - name: run integration tests Fedora with xunit results
    shell: 'CGROUP_MANAGER=cgroupfs STORAGE_OPTS="--storage-driver=overlay2" make localintegration'
    args:
      chdir: /root/src/github.com/kubernetes-incubator/cri-o
    async: "{{ integration_timeout_seconds | int }}"
    poll: 10
    ignore_errors: yes
    when: (ansible_distribution == 'Fedora' and xunit)
  - name: Make testing output directory
    file:
      path: /root/src/github.com/kubernetes-incubator/cri-o/reports
      state: directory
    ignore_errors: yes
    when: xunit
  - name: Move all xunit files into one dir to scp
    shell: 'mv /root/src/github.com/kubernetes-incubator/cri-o/test/TestReport-bats*.xml /root/src/github.com/kubernetes-incubator/cri-o/reports/'
    when: xunit
  # XXX: kube tests from now on
  - name: git clone k8s repo
    git:
      repo: https://github.com/runcom/kubernetes
      dest: /root/src/k8s.io/kubernetes
      # based on kube upstream v1.7.5
      version: cri-o-node-e2e-patched
      force: yes
    async: 600
    poll: 10
  - name: Add custom cluster service file for the e2e testing
    copy:
      dest: /etc/systemd/system/customcluster.service
      content: |
        [Unit]
        After=network-online.target
        Wants=network-online.target
        [Service]
        ExecStart=/usr/local/bin/createcluster.sh
        User=root
        [Install]
        WantedBy=multi-user.target        

  - name: Add create cluster background script for e2e testing
    copy:
      dest: /usr/local/bin/createcluster.sh
      content: |
        #!/bin/bash

        export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:/usr/local/go/bin:/root/go-tools/bin:/root/src/k8s.io/kubernetes/third_party/etcd
        export GOPATH=/root/go-tools
        cd /root/src/k8s.io/kubernetes
        export CONTAINER_RUNTIME=remote
        export CGROUP_DRIVER=systemd
        export CONTAINER_RUNTIME_ENDPOINT='/var/run/crio.sock --runtime-request-timeout=5m'
        export ALLOW_SECURITY_CONTEXT=","
        export ALLOW_PRIVILEGED=1
        export DNS_SERVER_IP={{ ansible_default_ipv4.address }}
        export API_HOST={{ ansible_default_ipv4.address }}
        export API_HOST_IP={{ ansible_default_ipv4.address }}
        export KUBE_ENABLE_CLUSTER_DNS=true
        hack/local-up-cluster.sh        
  - name: Add path to bashrc files
    lineinfile:
      dest: /root/.bashrc
      line: 'export PATH=/tmp/mybin:$PATH:/usr/local/go/bin:/root/go-tools/bin:/root/src/k8s.io/kubernetes/third_party/etcd'
      insertafter: 'EOF'
      regexp: 'export PATH=/tmp/mybin:\$PATH:/usr/local/go/bin:/root/go-tools/bin:/root/src/k8s.io/kubernetes/third_party/etcd'
      state: present
  - name: Add gopath to bashrc files
    lineinfile:
      dest: /root/.bashrc
      line: 'export GOPATH=/root/go-tools'
      insertafter: 'EOF'
      regexp: 'export GOPATH=/root/go-tools'
      state: present
  - name: Source the bash rc file
    shell: source /root/.bashrc
  - name: Get godep
    shell: go get -u github.com/tools/godep
    async: 600
    poll: 10
  - name: Install etcd
    command: hack/install-etcd.sh
    args:
      chdir: /root/src/k8s.io/kubernetes
    async: 600
    poll: 10
  - name: Install necessary github go packages
    shell: go get github.com/onsi/ginkgo/ginkgo ; go get github.com/onsi/gomega ; go get -u github.com/cloudflare/cfssl/cmd/...
    args:
      chdir: /root/src/k8s.io/kubernetes
    async: 600
    poll: 10
  - name: Make testing output verbose so it can be converted to xunit
    lineinfile:
      dest: /root/src/k8s.io/kubernetes/hack/make-rules/test.sh
      line: ' go test -v "${goflags[@]:+${goflags[@]}}" \'
      regexp: ' go test \"\$'
      state: present
  - name: make and install CRI-O
    shell: |
            make install.tools && \
            make && \
            make install && \
            make install.systemd && \
            make install.config            
    args:
      chdir: /root/src/github.com/kubernetes-incubator/cri-o
    async: 600
    poll: 10
  - name: link runc
    file: src=/usr/local/sbin/runc dest=/usr/bin/runc state=link
  - name: run with overlay2
    replace:
      regexp: 'storage_driver = ""'
      replace: 'storage_driver = "overlay2"'
      name: /etc/crio/crio.conf
      backup: yes
  - name: run with systemd cgroup manager
    replace:
      regexp: 'cgroup_manager = "cgroupfs"'
      replace: 'cgroup_manager = "systemd"'
      name: /etc/crio/crio.conf
      backup: yes
  - name: add docker.io default registry
    lineinfile:
      dest: /etc/crio/crio.conf
      line: '"docker.io"'
      insertafter: 'registries = \['
      regexp: 'docker\.io'
      state: present
  - name: add overlay2 storage opts on RHEL/CentOS
    lineinfile:
      dest: /etc/crio/crio.conf
      line: '"overlay2.override_kernel_check=1"'
      insertafter: 'storage_option = \['
      regexp: 'overlay2\.override_kernel_check=1'
      state: present
    when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
  - name: enable and start CRI-O
    systemd:
      name: crio
      state: started
      enabled: yes
      daemon_reload: yes
    async: 600
    poll: 10
  # see https://github.com/kubernetes-incubator/cri-o/issues/528
  - name: disable selinux for k8s conformance tests
    shell: |
            setenforce 0            
    async: 600
    poll: 10
  - name: Go get the go-bindata file
    shell: go get -u github.com/jteeuwen/go-bindata/go-bindata
    args:
      chdir: /root/src/k8s.io/kubernetes
    async: 600
    poll: 10
  - name: Install necessary github go packages
    shell: go get github.com/onsi/ginkgo/ginkgo ; go get github.com/onsi/gomega ; go get -u github.com/cloudflare/cfssl/cmd/...
    args:
      chdir: /root/src/k8s.io/kubernetes
    async: 600
    poll: 10
  - name: Add path to bashrc files
    lineinfile:
      dest: /root/.bashrc
      line: 'export PATH=$PATH:/root/src/k8s.io/kubernetes/third_party/etcd'
      insertafter: 'EOF'
      regexp: 'export PATH=\$PATH:/root/src/k8s.io/kubernetes/third_party/etcd'
      state: present
  - name: inject hostname into /etc/hosts
    lineinfile:
      dest: /etc/hosts
      line: '{{ ansible_default_ipv4.address }} {{ ansible_nodename }}'
      insertafter: 'EOF'
      regexp: '{{ ansible_default_ipv4.address }}\s+{{ ansible_nodename }}'
      state: present
  - name: Flush the iptables
    command: iptables -F
  - name: Make clean
    make:
      target: clean
      chdir: /root/src/k8s.io/kubernetes
  - name: Make
    make:
      chdir: /root/src/k8s.io/kubernetes
  - name: Set kubernetes_provider to be local
    lineinfile:
      dest: /root/.bashrc
      line: 'export KUBERNETES_PROVIDER=local'
      insertafter: 'EOF'
      regexp: 'export KUBERNETES_PROVIDER=local'
      state: present
  - name: Set KUBECONFIG
    lineinfile:
      dest: /root/.bashrc
      line: 'export KUBECONFIG=/var/run/kubernetes/admin.kubeconfig'
      insertafter: 'EOF'
      regexp: 'export KUBECONFIG=/var/run/kubernetes/admin.kubeconfig'
      state: present
  - name: Source the bash rc file
    shell: source /root/.bashrc
  - name: Make the create cluster script executable
    file:
      path: /usr/local/bin/createcluster.sh
      mode: a+x
  - name: Reload daemons to make custom cluster available
    shell: systemctl daemon-reload
  - name: Bring up the cluster
    service:
      name: customcluster.service
      state: started
      enabled: yes
  - name: Give the cluster time to initialize
    pause: minutes=5
  - name: Flush the iptables
    command: iptables -F
  # XXX: The task below is running kube e2e not kube node-e2e!!!
  #      The reason behind calling the file node-e2e.log is because the internal
  #      RH CI looks for a file named like that and would require otherwise a
  #      change to the CI itself (not doable in this playbook) to look for just
  #      "e2e.log"..
  #      So, we'll change it to "e2e.log", some day...
  - name: run e2e tests
    shell: |
      go run hack/e2e.go -v --test -test_args="-host=https://{{ ansible_default_ipv4.address }}:6443 --ginkgo.focus=\[Conformance\]" 2>&1 > node-e2e.log      
    args:
      chdir: /root/src/k8s.io/kubernetes
    async: 7200
    poll: 10
    ignore_errors: yes