Merge c457314b98 into c9b3d3df28

contrib/test/integration/README.md

@@ -1,21 +1,78 @@
 # Fedora and RHEL Integration and End-to-End Tests
 
 This directory contains playbooks to set up for and run the integration and
-end-to-end tests for CRI-O on RHEL and Fedora hosts. Two entrypoints exist:
+end-to-end tests for CRI-O on RHEL and Fedora hosts.  The expected entry-point
+is the ``main.yml`` Ansible playbook.
 
- - `main.yml`: sets up the machine and runs tests
+##Definitions:
- - `results.yml`: gathers test output to `/tmp/artifacts`
 
-When running `main.yml`, three tags are present:
+    Control-host:  The system from which the ``ansible-playbook`` or
+                   ``venv-ansible-playbook.sh`` command is executed.
 
- - `setup`: run all tasks to set up the system for testing
+    Subject-host(s): The target systems, on which actual playbook tasks are
- - `e2e`: build CRI-O from source and run Kubernetes node E2Es
+                     being carried out.
- - `integration`: build CRI-O from source and run the local integration suite
 
-The playbooks assume the following things about your system:
+##Topology:
 
- - on RHEL, the server and extras repos are configured and certs are present
+The control-host:
- - `ansible` is installed and the host is boot-strapped to allow `ansible` to run against it
+
- - the `$GOPATH` is set and present for all shells (*e.g.* written in `/etc/environment`)
+ - May be the subject.
- - CRI-O is checked out to the correct state at `${GOPATH}/src/github.com/kubernetes-incubator/cri-o`
+ - Is based on either RHEL/CentOS 6 (or later), or Fedora 24 (or later).
- - the user running the playbook has access to passwordless `sudo`
+ - Runs ``main.yml`` from within the cri-o repository already in the
+   desired state for testing.
+
+The subject-host(s):
+
+ - May be the control-host.
+ - May be executing the ``main.yml`` playbook against itself.
+ - If RHEL-like, has the ``server``, ``extras``, and ``EPEL`` repositories available
+   and enabled.
+ - Has remote password-less ssh configured for access by the control-host.
+ - When ssh-access is for a regular user, that user has password-less
+   sudo access to root.
+
+##Runtime Requirements:
+
+Execution of the ``main.yml`` playbook:
+
+ - Should occur through the ``cri-o/contrib/test/venv-ansible-playbook.sh`` wrapper.
+ - Execution may target localhost, or one or more subjects via standard Ansible
+   inventory arguments.
+ - Should use a combination (including none) of the following tags:
+
+     - ``setup``: Run all tasks to set up the system for testing. Final state must
+                  be self-contained and independent from other tags (i.e. support
+                  stage-caching).
+     - ``integration``: Assumes 'setup' previously completed successfully.
+                        May be executed from cached-state of ``setup``.
+                        Not required to execute coincident with other tags.
+                        Must build CRI-O from source and run the
+                        integration test suite.
+     - ``e2e``: Assumes 'setup' previously completed successfully.  May be executed
+                from cached-state of ``setup``. Not required to execute coincident with
+                other tags.  Must build CRI-O from source and run Kubernetes node
+                E2E tests.
+
+``cri-o/contrib/test/venv-ansible-playbook.sh`` Wrapper:
+
+ - May be executed on the control-host to both hide and version-lock playbook
+   execution dependencies, ansible and otherwise.
+ - Must accept all of the valid Ansible command-line options.
+ - Must sandbox dependencies under a python virtual environment ``.cri-o_venv``
+   with packages as specified in ``requirements.txt``.
+ - Requires the control-host has the following fundamental dependencies installed
+   (or equivalent): ``python2-virtualenv gcc openssl-devel
+   redhat-rpm-config libffi-devel python-devel libselinux-python rsync
+   yum-utils python3-pycurl python-simplejson``.
+
+For example:
+
+Given a populated '/path/to/inventory' file, a control-host could run:
+
+./venv-ansible-playbook.sh -i /path/to/inventory ./integration/main.yml
+
+-or-
+
+From a subject-host without an inventory:
+
+./venv-ansible-playbook.sh -i localhost, ./integration/main.yml

contrib/test/integration/ansible.cfg

@@ -57,11 +57,6 @@ gather_subset = network
 #host_key_checking = False
 host_key_checking = False
 
-# change the default callback
-#stdout_callback = skippy
-# enable additional callbacks
-#callback_whitelist = timer, mail
-
 # Determine whether includes in tasks and handlers are "static" by
 # default. As of 2.0, includes are dynamic by default. Setting these
 # values to True will make includes behave more like they did in the
@@ -165,7 +160,6 @@ deprecation_warnings = False
 # instead of shelling out to the git command.
 command_warnings = False
 
-
 # set plugin path directories here, separate with colons
 #action_plugins     = /usr/share/ansible/plugins/action
 #callback_plugins   = /usr/share/ansible/plugins/callback
@@ -219,7 +213,6 @@ nocolor = 0
 # When a playbook fails by default a .retry file will be created in ~/
 # You can disable this feature by setting retry_files_enabled to False
 # and you can change the location of the files by setting retry_files_save_path
-
 #retry_files_enabled = False
 retry_files_enabled = False
 
@@ -248,6 +241,7 @@ no_target_syslog = True
 # worker processes. At the default of 0, no compression
 # is used. This value must be an integer from 0 to 9.
 #var_compression_level = 9
+var_compression_level = 3
 
 # controls what compression method is used for new-style ansible modules when
 # they are sent to the remote system.  The compression types depend on having
@@ -298,6 +292,15 @@ ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o UserKnownHostsFile=/de
 # Example:
 # control_path = %(directory)s/%%h-%%r
 #control_path = %(directory)s/ansible-ssh-%%h-%%p-%%r
+# Using ssh's ControlPersist feature is desireable because of wide
+# compatibility and not needing to mess with /etc/sudoers
+# for pipelining (see below).  Unfortunately, in cloud environments,
+# auto-assigned VM hostnames tend to be rather longs.  Worse, in a CI
+# context, the default home-directory path may also be lengthy.  Fix
+# this to a short name, so Ansible doesn't fall back to opening new
+# connections for every task.
+control_path = /tmp/crio-%%n-%%p
+
 
 # Enabling pipelining reduces the number of SSH operations required to
 # execute a module on the remote server. This can result in a significant
@@ -308,7 +311,6 @@ ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o UserKnownHostsFile=/de
 # sudoers configurations that have requiretty (the default on many distros).
 #
 #pipelining = False
-pipelining=True
 
 # if True, make ansible use scp if the connection type is ssh
 # (default is sftp)

contrib/test/integration/build/bats.yml

@@ -3,12 +3,12 @@
 - name: clone bats source repo
   git:
     repo: "https://github.com/sstephenson/bats.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats"
+    dest: "{{ go_path }}/src/github.com/sstephenson/bats"
 
 - name: install bats
   command: "./install.sh /usr/local"
   args:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats"
+    chdir: "{{ go_path }}/src/github.com/sstephenson/bats"
 
 - name: link bats
   file:

contrib/test/integration/build/cri-o.yml

@@ -1,42 +1,42 @@
 ---
 
-- name: stat the expected cri-o directory
+- name: stat the expected cri-o directory and Makefile exists
   stat:
-    path: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    path: "{{ cri_o_dest_path }}/Makefile"
-  register: dir_stat
+  register: crio_stat
 
-- name: expect cri-o to be cloned already
+- name: Verify cri-o Makefile exists in expected location
   fail:
-    msg: "Expected cri-o to be cloned at {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o but it wasn't!"
+    msg: "Expected cri-o to be cloned at {{ cri_o_dest_path }}, but its 'Makefile' seems to be missing."
-  when: not dir_stat.stat.exists
+  when: not crio_stat.stat.exists or not crio_stat.stat.isreg
 
 - name: install cri-o tools
   make:
     target: install.tools
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
 
 - name: build cri-o
   make:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
 
 - name: install cri-o
   make:
     target: install
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
 
 - name: install cri-o systemd files
   make:
     target: install.systemd
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
 
 - name: install cri-o config
   make:
     target: install.config
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
 
 - name: install configs
   copy:
-    src: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/{{ item.src }}"
+    src: "{{ cri_o_dest_path }}/{{ item.src }}"
     dest: "{{ item.dest }}"
     remote_src: yes
   with_items:

contrib/test/integration/build/cri-tools.yml

@@ -3,7 +3,7 @@
 - name: clone cri-tools source repo
   git:
     repo: "https://github.com/kubernetes-incubator/cri-tools.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-tools"
+    dest: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-tools"
     version: "9ff5e8f78a4182ab8d5ba9bcccdda5f338600eab"
 
 - name: install crictl
@@ -11,6 +11,6 @@
 
 - name: link crictl
   file:
-    src: "{{ ansible_env.GOPATH }}/bin/crictl"
+    src: "{{ go_path }}/bin/crictl"
     dest: /usr/bin/crictl
     state: link

contrib/test/integration/build/kubernetes.yml

@@ -3,17 +3,17 @@
 - name: clone kubernetes source repo
   git:
     repo: "https://github.com/runcom/kubernetes.git"
-    dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+    dest: "{{ go_path }}/src/k8s.io/kubernetes"
     version: "cri-o-patched-1.8"
 
 - name: install etcd
   command: "hack/install-etcd.sh"
   args:
-    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+    chdir: "{{ go_path }}/src/k8s.io/kubernetes"
 
 - name: build kubernetes
   make:
-    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+    chdir: "{{ go_path }}/src/k8s.io/kubernetes"
 
 - name: Add custom cluster service file for the e2e testing
   copy:
@@ -23,7 +23,7 @@
       After=network-online.target
       Wants=network-online.target
       [Service]
-      WorkingDirectory={{ ansible_env.GOPATH }}/src/k8s.io/kubernetes
+      WorkingDirectory={{ go_path }}/src/k8s.io/kubernetes
       ExecStart=/usr/local/bin/createcluster.sh
       User=root
       [Install]
@@ -35,7 +35,7 @@
     content: |
       #!/bin/bash
 
-      export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ ansible_env.GOPATH }}/bin:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/third_party/etcd:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/
+      export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ go_path }}/bin:{{ go_path }}/src/k8s.io/kubernetes/third_party/etcd:{{ go_path }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/
       export CONTAINER_RUNTIME=remote
       export CGROUP_DRIVER=systemd
       export CONTAINER_RUNTIME_ENDPOINT='/var/run/crio.sock --runtime-request-timeout=5m'
@@ -47,17 +47,3 @@
       export KUBE_ENABLE_CLUSTER_DNS=true
       ./hack/local-up-cluster.sh
     mode: "u=rwx,g=rwx,o=x"
-
-- name: Set kubernetes_provider to be local
-  lineinfile:
-    dest: /etc/environment
-    line: 'KUBERNETES_PROVIDER=local'
-    regexp: 'KUBERNETES_PROVIDER='
-    state: present
-
-- name: Set KUBECONFIG
-  lineinfile:
-    dest: /etc/environment
-    line: 'KUBECONFIG=/var/run/kubernetes/admin.kubeconfig'
-    regexp: 'KUBECONFIG='
-    state: present

contrib/test/integration/build/plugins.yml

@@ -3,17 +3,17 @@
 - name: clone plugins source repo
   git:
     repo: "https://github.com/containernetworking/plugins.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+    dest: "{{ go_path }}/src/github.com/containernetworking/plugins"
     version: "dcf7368eeab15e2affc6256f0bb1e84dd46a34de"
 
 - name: build plugins
   command: "./build.sh"
   args:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+    chdir: "{{ go_path }}/src/github.com/containernetworking/plugins"
 
 - name: install plugins
   copy:
-    src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/{{ item }}"
+    src: "{{ go_path }}/src/github.com/containernetworking/plugins/bin/{{ item }}"
     dest: "/opt/cni/bin"
     mode: "o=rwx,g=rx,o=rx"
     remote_src: yes
@@ -33,18 +33,18 @@
 - name: clone runcom plugins source repo
   git:
     repo: "https://github.com/runcom/plugins.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+    dest: "{{ go_path }}/src/github.com/containernetworking/plugins"
     version: "custom-bridge"
     force: yes
 
 - name: build plugins
   command: "./build.sh"
   args:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+    chdir: "{{ go_path }}/src/github.com/containernetworking/plugins"
 
 - name: install custom bridge
   copy:
-    src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/bridge"
+    src: "{{ go_path }}/src/github.com/containernetworking/plugins/bin/bridge"
     dest: "/opt/cni/bin/bridge-custom"
     mode: "o=rwx,g=rx,o=rx"
     remote_src: yes

contrib/test/integration/build/runc.yml

@@ -3,18 +3,18 @@
 - name: clone runc source repo
   git:
     repo: "https://github.com/opencontainers/runc.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
+    dest: "{{ go_path }}/src/github.com/opencontainers/runc"
     version: "84a082bfef6f932de921437815355186db37aeb1"
 
 - name: build runc
   make:
     params: BUILDTAGS="seccomp selinux"
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
+    chdir: "{{ go_path }}/src/github.com/opencontainers/runc"
 
 - name: install runc
   make:
     target: "install"
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
+    chdir: "{{ go_path }}/src/github.com/opencontainers/runc"
 
 - name: link runc
   file:

contrib/test/integration/e2e.yml

@@ -29,7 +29,7 @@
     daemon_reload: yes
 
 - name: wait for the cluster to be running
-  command: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default"
+  command: "{{ go_path }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default"
   register: kube_poll
   until: kube_poll | succeeded
   retries: 100
@@ -51,10 +51,25 @@
             &> {{ artifacts }}/e2e.log
   # Fix vim syntax hilighting: "
 
-- name: disable SELinux
+- block:
-  command: setenforce 0
 
-- name: run e2e tests
+    - name: Disable swap during e2e tests
-  shell: "{{ e2e_shell_cmd | regex_replace('\\s+', ' ') }}"
+      command: 'swapoff -a'
-  args:
+      when: not e2e_swap_enabled
-    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+
+    - name: Disable selinux during e2e tests
+      command: 'setenforce 0'
+      when: not e2e_selinux_enabled
+
+    - name: run e2e tests
+      shell: "{{ e2e_shell_cmd | regex_replace('\\s+', ' ') }}"
+      args:
+        chdir: "{{ go_path }}/src/k8s.io/kubernetes"
+
+  always:
+
+    - name: Re-enable SELinux after e2e tsts
+      command: 'setenforce 1'
+
+    - name: Re-enalbe swap after e2e tests
+      command: 'swapon -a'

contrib/test/integration/github.yml

@@ -0,0 +1,27 @@
+---
+
+
+- name: Verify expectations
+  assert:
+    that:
+      - 'cri_o_dest_path is defined'
+      - 'cri_o_src_path is defined'
+
+- name: The cri-o repository directory exists
+  file:
+    path: "{{ cri_o_dest_path }}"
+    state: directory
+    mode: 0777
+
+- name: Synchronize cri-o from control-host to remote subject
+  synchronize:
+    archive: False
+    checksum: True
+    delete: True
+    dest: "{{ cri_o_dest_path }}/"
+    links: True
+    recursive: True
+    src: "{{ cri_o_src_path }}/"
+    times: True
+  # This task is excessively noisy, logging every change to every file :(
+  no_log: True

contrib/test/integration/golang.yml

@@ -16,28 +16,16 @@
     - gofmt
     - godoc
 
-- name: ensure user profile exists
-  file:
-    path: "{{ ansible_user_dir }}/.profile"
-    state: touch
-
-- name: set up PATH for Go toolchain and built binaries
-  lineinfile:
-    dest: "{{ ansible_user_dir }}/.profile"
-    line: 'PATH={{ ansible_env.PATH }}:{{ ansible_env.GOPATH }}/bin:/usr/local/go/bin'
-    regexp: '^PATH='
-    state: present
-
 - name: set up directories
   file:
-    path: "{{ item }}"
+    path: "{{ go_path }}/src/github.com/{{ item }}"
     state: directory
   with_items:
-    - "{{ ansible_env.GOPATH }}/src/github.com/containernetworking"
+    - "containernetworking"
-    - "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator"
+    - "kubernetes-incubator"
-    - "{{ ansible_env.GOPATH }}/src/github.com/k8s.io"
+    - "k8s.io"
-    - "{{ ansible_env.GOPATH }}/src/github.com/sstephenson"
+    - "sstephenson"
-    - "{{ ansible_env.GOPATH }}/src/github.com/opencontainers"
+    - "opencontainers"
 
 - name: install Go tools and dependencies
   shell: /usr/bin/go get -u "github.com/{{ item }}"

contrib/test/integration/main.yml

@@ -1,7 +1,53 @@
-- hosts: all
+---
-  remote_user: root
+
+- hosts: '{{ subjects | default("all") }}'
+  gather_facts: False  # Requires low-level ansible-dependencies
+  # Cannot use vars.yml - it references magic variables from setup module
+  tags:
+    - setup
+  tasks:
+    - name: Ansible setup-module dependencies are installed, ignoring errors (setup runs next).
+      raw: $(type -P dnf || type -P yum) install -y python2 python2-dnf libselinux-python
+      ignore_errors: True
+
+    - name: Gather only networking facts for speed
+      setup:
+        gather_subset: network
+
+    - name: Variables from vars.yml are hauled in after setup
+      include_vars: "{{ playbook_dir }}/vars.yml"
+
+    - name: Global environment are defined, but can be overriden on a task-by-task basis.
+      set_fact:
+        extra_storage_opts: >
+            {%- if ansible_distribution in ["RedHat", "CentOS"] -%}
+                "--storage-opt overlay.override_kernel_check=1"
+            {%- else -%}
+                ""
+            {%- endif -%}
+        environment_variables:
+            PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:{{ go_path }}/bin:/usr/local/go/bin"
+            GOPATH: "{{ go_path }}"
+            KUBERNETES_PROVIDER: "local"
+            KUBECONFIG: "/var/run/kubernetes/admin.kubeconfig"
+            CGROUP_MANAGER: "cgroupfs"
+            STORAGE_OPTS: '--storage-driver=overlay {{ extra_storage_opts | default("") | trim }}'
+
+
+- hosts: '{{ subjects | default("none") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
+  tags:
+    - setup
+  tasks:
+    - name: CRI-O source is available on every subject
+      include: github.yml
+
+
+- hosts: '{{ subjects | default("all") }}'
+  vars_files:
+    - "{{ playbook_dir }}/vars.yml"
+  environment: '{{ environment_variables }}'
   tags:
     - setup
   tasks:
@@ -26,33 +72,24 @@
     - name: clone build and install networking plugins
       include: "build/plugins.yml"
 
-- hosts: all
-  remote_user: root
-  vars_files:
-    - "{{ playbook_dir }}/vars.yml"
-  tags:
-    - integration
-    - e2e
-  tasks:
-    - name: clone build and install cri-o
-      include: "build/cri-o.yml"
 
-- hosts: all
+- hosts: '{{ subjects | default("all") }}'
-  remote_user: root
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
-  tags:
+  environment: '{{ environment_variables }}'
-    - integration
   tasks:
+
+    - name: Build and install cri-o
+      include: "build/cri-o.yml"
+      tags:
+        - always
+
     - name: run cri-o integration tests
       include: test.yml
+      tags:
+        - integration
 
-- hosts: all
-  remote_user: root
-  vars_files:
-    - "{{ playbook_dir }}/vars.yml"
-  tags:
-    - e2e
-  tasks:
     - name: run k8s e2e tests
       include: e2e.yml
+      tags:
+        - e2e

contrib/test/integration/results.yml

@@ -1,7 +1,7 @@
 ---
 # vim-syntax: ansible
 
-- hosts: '{{ hosts | default("all") }}'
+- hosts: '{{ subjects | default("all") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
   vars:

contrib/test/integration/swap.yml

@@ -0,0 +1,42 @@
+---
+
+- name: Obtain current state of swap
+  command: swapon --noheadings --show=NAME
+  register: swapon
+
+- name: Setup swap if none already, to prevent kernel firing off the OOM killer
+  block:
+
+    - name: A unique swapfile path is generated
+      command: mktemp --tmpdir=/root swapfile_XXX
+      register: swapfilepath
+
+    - name: Swap file path is buffered
+      set_fact:
+        swapfilepath: '{{ swapfilepath.stdout | trim }}'
+
+    - name: Set swap file permissions
+      file:
+        path: "{{ swapfilepath }}"
+        owner: root
+        group: root
+        mode: 0600
+
+    - name: Swapfile padded to swapfile_size & timed to help debug any performance problems
+      shell: 'time dd if=/dev/zero of={{ swapfilepath }} bs={{ swapfileGB }}M count=1024'
+
+    - name: Swap file is formatted
+      command: 'mkswap {{ swapfilepath }}'
+
+    - name: Write swap entry in fstab
+      mount:
+        path: none
+        src: "{{ swapfilepath }}"
+        fstype: swap
+        opts: sw
+        state: present
+
+    - name: Mount swap
+      command: "swapon -a"
+
+  when: not (swapon.stdout_lines | length)

contrib/test/integration/system.yml

@@ -1,5 +1,12 @@
 ---
 
+- name: Update all packages
+  package:
+    name: '*'
+    state: latest
+  async: 600
+  poll: 10
+
 - name: Make sure we have all required packages
   package:
     name: "{{ item }}"
@@ -25,6 +32,7 @@
     - libgpg-error-devel
     - libguestfs-tools
     - libseccomp-devel
+    - libselinux-python
     - libvirt-client
     - libvirt-python
     - libxml2-devel
@@ -40,6 +48,7 @@
     - openssl-devel
     - ostree-devel
     - pkgconfig
+    - policycoreutils-python
     - python
     - python2-boto
     - python2-crypto
@@ -54,7 +63,7 @@
     - socat
     - tar
     - wget
-  async: 600
+  async: '{{ 20 * 60 }}'
   poll: 10
 
 - name: Add Btrfs for Fedora
@@ -63,22 +72,11 @@
     state: present
   with_items:
    - btrfs-progs-devel
+   - python2-virtualenv
   when: ansible_distribution in ['Fedora']
 
-- name: Update all packages
+- name: Check / setup swap
-  package:
+  include: "swap.yml"
-    name: '*'
-    state: latest
-  async: 600
-  poll: 10
-
-- name: Setup swap to prevent kernel firing off the OOM killer
-  shell: |
-    truncate -s 8G /root/swap && \
-    export SWAPDEV=$(losetup --show -f /root/swap | head -1) && \
-    mkswap $SWAPDEV && \
-    swapon $SWAPDEV && \
-    swapon --show
 
 - name: ensure directories exist as needed
   file:
@@ -115,3 +113,12 @@
 - name: Update the kernel cmdline to include quota support
   command: grubby --update-kernel=ALL --args="rootflags=pquota"
   when: ansible_distribution in ['RedHat', 'CentOS']
+
+- name: Enforce specific SELinux types for files on this platform
+  sefcontext:
+    target: '{{ item.key }}'
+    setype: '{{ item.value[ansible_distribution] | default(item.value.default) }}'
+    state: present
+  when: item.value[ansible_distribution] is defined or
+        item.value.default is defined
+  with_dict: '{{ set_setypes | default({}) }}'

contrib/test/integration/test.yml

@@ -5,24 +5,37 @@
 
 - name: Make testing output verbose so it can be converted to xunit
   lineinfile:
-    dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/hack/make-rules/test.sh"
+    dest: "{{ go_path }}/src/k8s.io/kubernetes/hack/make-rules/test.sh"
     line: ' go test -v "${goflags[@]:+${goflags[@]}}" \'
     regexp: ' go test \"\$'
     state: present
 
-- name: set extra storage options
+- name: ensure directory exists for integration results
-  set_fact:
-    extra_storage_opts: " --storage-opt overlay.override_kernel_check=1"
-  when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
-
-- name: ensure directory exists for e2e reports
   file:
     path: "{{ artifacts }}"
     state: directory
 
-- name: run integration tests
+- block:
-  shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTIONS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration >& {{ artifacts }}/testout.txt"
+
-  args:
+    - name: Disable swap during integration tests
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+      command: 'swapoff -a'
-  async: 5400
+      when: not integration_swap_enabled
-  poll: 30
+
+    - name: Disable selinux during integration tests
+      command: 'setenforce 0'
+      when: not integration_selinux_enabled
+
+    - name: run integration tests
+      shell: "make localintegration >& {{ artifacts }}/testout.txt"
+      args:
+        chdir: "{{ cri_o_dest_path }}"
+      async: 5400
+      poll: 30
+
+  always:
+
+    - name: Re-enable SELinux after integration tsts
+      command: 'setenforce 1'
+
+    - name: Re-enalbe swap after integration tests
+      command: 'swapon -a'

contrib/test/integration/vars.yml

@@ -1,5 +1,31 @@
 ---
 
+# When swap setup is necessary, make it this size
+swapfileGB: 8
+
+# When False, turn off all swapping on the system during indicated test.
+integration_swap_enabled: False
+e2e_swap_enabled: True
+
+# When False, disable SELinux on the system only during
+# particular tests.
+integration_selinux_enabled: True
+e2e_selinux_enabled: False
+
+# Base directory for all go-related source, build, and install.
+go_path: "/go"
+
+# Absolute path on control-host where the cri-o source exists
+cri_o_src_path: "{{ playbook_dir }}/../../../"
+
+# Absolute path on subjects where cri-o source is expected
+cri_o_dest_path: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-o"
+
+# Mapping of filenames to ansible_distribution (or default), to SELinux types
+set_setypes:
+    /usr/local/bin/crio:
+        default: 'container_runtime_exec_t'
+
 # For results.yml Paths use rsync 'source' conventions
 artifacts: "/tmp/artifacts"  # Base-directory for collection
 crio_integration_filepath: "{{ artifacts }}/testout.txt"