From 369aca6dcf9ffd86e382f339beca21d901faf246 Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Fri, 3 Nov 2017 15:41:37 -0400
Subject: [PATCH 01/12] Fix /test ami fail on duplicate k8s checkout

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/main.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml
index ce4a206f..e095aeb4 100644
--- a/contrib/test/integration/main.yml
+++ b/contrib/test/integration/main.yml
@@ -17,9 +17,6 @@
     - name: clone build and install cri-tools
       include: "build/cri-tools.yml"
 
-    - name: clone build and install kubernetes
-      include: "build/kubernetes.yml"
-
     - name: clone build and install runc
       include: "build/runc.yml"
 

From 70b83fd641d96206b66153e08c11612dfaa3bd60 Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Fri, 3 Nov 2017 15:37:37 -0400
Subject: [PATCH 02/12] Fix e2e test dependency on building k8s

These two should not be tightly-coupled.  e.g. for RPM testing, the
packages will be built/installed, then e2e tests will run.  Having k8s
secondarily built/installed will seriously screw with rpm-building and
testing.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/e2e.yml  | 3 ---
 contrib/test/integration/main.yml | 3 +++
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/contrib/test/integration/e2e.yml b/contrib/test/integration/e2e.yml
index 41f92757..5c4d656e 100644
--- a/contrib/test/integration/e2e.yml
+++ b/contrib/test/integration/e2e.yml
@@ -1,8 +1,5 @@
 ---
 
-- name: clone build and install kubernetes
-  include: "build/kubernetes.yml"
-
 - name: enable and start CRI-O
   systemd:
     name: crio
diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml
index e095aeb4..f09d8d52 100644
--- a/contrib/test/integration/main.yml
+++ b/contrib/test/integration/main.yml
@@ -31,6 +31,9 @@
     - integration
     - e2e
   tasks:
+    - name: clone build and install kubernetes
+      include: "build/kubernetes.yml"
+
     - name: clone build and install cri-o
       include: "build/cri-o.yml"
 

From 91f3c13dd16548fde3144211a740bf7094f2f25f Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Tue, 24 Oct 2017 19:14:24 -0400
Subject: [PATCH 03/12] Revert Revert "Idempotent Swapping"

This puts back the better qualified Idempotent Swapping, but
adds two variables which control whether or not swapping is
enabled during testing.  This addresses a short-term issue of
occasionally failing integration tests under some scenarios,
but not others.

The integration OOM-test isn't properly failing because
the cgroup memory control doesn't account for swap usage
(by design) in ``limit_in_bytes``.

Fixing this for the long-term requires repairing the
test to also set ``memory.memsw.limit_in_bytes=0``
(in addition to memory.limit_in_bytes=5m).

N/B: Normally these things are passed down from k8s,
which is why the same fix isn't currently needed
for the e2e tests - hence the new variable is ``True`` by
default.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/e2e.yml    | 19 ++++++++++---
 contrib/test/integration/swap.yml   | 42 +++++++++++++++++++++++++++++
 contrib/test/integration/system.yml |  9 ++-----
 contrib/test/integration/test.yml   | 23 +++++++++++-----
 contrib/test/integration/vars.yml   |  8 ++++++
 5 files changed, 84 insertions(+), 17 deletions(-)
 create mode 100644 contrib/test/integration/swap.yml

diff --git a/contrib/test/integration/e2e.yml b/contrib/test/integration/e2e.yml
index 5c4d656e..52d0b750 100644
--- a/contrib/test/integration/e2e.yml
+++ b/contrib/test/integration/e2e.yml
@@ -51,7 +51,18 @@
 - name: disable SELinux
   command: setenforce 0
 
-- name: run e2e tests
-  shell: "{{ e2e_shell_cmd | regex_replace('\\s+', ' ') }}"
-  args:
-    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+- block:
+
+    - name: Disable swap during e2e tests
+      command: 'swapoff -a'
+      when: not e2e_swap_enabled
+
+    - name: run e2e tests
+      shell: "{{ e2e_shell_cmd | regex_replace('\\s+', ' ') }}"
+      args:
+        chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+
+  always:
+
+    - name: Re-enable swap after e2e tests
+      command: 'swapon -a'
diff --git a/contrib/test/integration/swap.yml b/contrib/test/integration/swap.yml
new file mode 100644
index 00000000..6777699c
--- /dev/null
+++ b/contrib/test/integration/swap.yml
@@ -0,0 +1,42 @@
+---
+
+- name: Obtain current state of swap
+  command: swapon --noheadings --show=NAME
+  register: swapon
+
+- name: Setup swap if none already, to prevent kernel firing off the OOM killer
+  block:
+
+    - name: A unique swapfile path is generated
+      command: mktemp --tmpdir=/root swapfile_XXX
+      register: swapfilepath
+
+    - name: Swap file path is buffered
+      set_fact:
+        swapfilepath: '{{ swapfilepath.stdout | trim }}'
+
+    - name: Set swap file permissions
+      file:
+        path: "{{ swapfilepath }}"
+        owner: root
+        group: root
+        mode: 0600
+
+    - name: Swapfile padded to swapfile_size & timed to help debug any performance problems
+      shell: 'time dd if=/dev/zero of={{ swapfilepath }} bs={{ swapfileGB }}M count=1024'
+
+    - name: Swap file is formatted
+      command: 'mkswap {{ swapfilepath }}'
+
+    - name: Write swap entry in fstab
+      mount:
+        path: none
+        src: "{{ swapfilepath }}"
+        fstype: swap
+        opts: sw
+        state: present
+
+    - name: Mount swap
+      command: "swapon -a"
+
+  when: not (swapon.stdout_lines | length)
diff --git a/contrib/test/integration/system.yml b/contrib/test/integration/system.yml
index d07ae0c8..fb0667c6 100644
--- a/contrib/test/integration/system.yml
+++ b/contrib/test/integration/system.yml
@@ -72,13 +72,8 @@
   async: 600
   poll: 10
 
-- name: Setup swap to prevent kernel firing off the OOM killer
-  shell: |
-    truncate -s 8G /root/swap && \
-    export SWAPDEV=$(losetup --show -f /root/swap | head -1) && \
-    mkswap $SWAPDEV && \
-    swapon $SWAPDEV && \
-    swapon --show
+- name: Check / setup swap
+  include: "swap.yml"
 
 - name: ensure directories exist as needed
   file:
diff --git a/contrib/test/integration/test.yml b/contrib/test/integration/test.yml
index 593e8a1c..87c5b2ad 100644
--- a/contrib/test/integration/test.yml
+++ b/contrib/test/integration/test.yml
@@ -20,9 +20,20 @@
     path: "{{ artifacts }}"
     state: directory
 
-- name: run integration tests
-  shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTIONS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration >& {{ artifacts }}/testout.txt"
-  args:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
-  async: 5400
-  poll: 30
+- block:
+
+    - name: Disable swap during integration tests
+      command: 'swapoff -a'
+      when: not integration_swap_enabled
+
+    - name: run integration tests
+      shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTIONS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration >& {{ artifacts }}/testout.txt"
+      args:
+        chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+      async: 5400
+      poll: 30
+
+  always:
+
+    - name: Re-enable swap after integration tests
+      command: 'swapon -a'
diff --git a/contrib/test/integration/vars.yml b/contrib/test/integration/vars.yml
index f1e5e2f7..05f6ec6a 100644
--- a/contrib/test/integration/vars.yml
+++ b/contrib/test/integration/vars.yml
@@ -1,5 +1,13 @@
 ---
 
+# When swap setup is necessary, make it this size
+swapfileGB: 8
+
+# When False, turn off all swapping on the system only during
+# particular tests.
+integration_swap_enabled: False
+e2e_swap_enabled: True
+
 # For results.yml Paths use rsync 'source' conventions
 artifacts: "/tmp/artifacts"  # Base-directory for collection
 crio_integration_filepath: "{{ artifacts }}/testout.txt"

From ab949957fa7d467c638146160f3a8784ee3a8d2e Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Tue, 24 Oct 2017 19:40:05 -0400
Subject: [PATCH 04/12] Add ability to enable/disable SELinux during tests

Add a pair of variables to control whether or not SELinux
is enabled during particular tests.  In all cases, make
sure it's re-enabled afterwards.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/e2e.yml  | 12 ++++++++----
 contrib/test/integration/test.yml |  9 ++++++++-
 contrib/test/integration/vars.yml |  5 +++++
 3 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/contrib/test/integration/e2e.yml b/contrib/test/integration/e2e.yml
index 52d0b750..5520b4f4 100644
--- a/contrib/test/integration/e2e.yml
+++ b/contrib/test/integration/e2e.yml
@@ -48,15 +48,16 @@
             &> {{ artifacts }}/e2e.log
   # Fix vim syntax hilighting: "
 
-- name: disable SELinux
-  command: setenforce 0
-
 - block:
 
     - name: Disable swap during e2e tests
       command: 'swapoff -a'
       when: not e2e_swap_enabled
 
+    - name: Disable selinux during e2e tests
+      command: 'setenforce 0'
+      when: not e2e_selinux_enabled
+
     - name: run e2e tests
       shell: "{{ e2e_shell_cmd | regex_replace('\\s+', ' ') }}"
       args:
@@ -64,5 +65,8 @@
 
   always:
 
-    - name: Re-enable swap after e2e tests
+    - name: Re-enable SELinux after e2e tsts
+      command: 'setenforce 1'
+
+    - name: Re-enalbe swap after e2e tests
       command: 'swapon -a'
diff --git a/contrib/test/integration/test.yml b/contrib/test/integration/test.yml
index 87c5b2ad..e9023e77 100644
--- a/contrib/test/integration/test.yml
+++ b/contrib/test/integration/test.yml
@@ -26,6 +26,10 @@
       command: 'swapoff -a'
       when: not integration_swap_enabled
 
+    - name: Disable selinux during integration tests
+      command: 'setenforce 0'
+      when: not integration_selinux_enabled
+
     - name: run integration tests
       shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTIONS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration >& {{ artifacts }}/testout.txt"
       args:
@@ -35,5 +39,8 @@
 
   always:
 
-    - name: Re-enable swap after integration tests
+    - name: Re-enable SELinux after integration tsts
+      command: 'setenforce 1'
+
+    - name: Re-enalbe swap after integration tests
       command: 'swapon -a'
diff --git a/contrib/test/integration/vars.yml b/contrib/test/integration/vars.yml
index 05f6ec6a..801eb1bc 100644
--- a/contrib/test/integration/vars.yml
+++ b/contrib/test/integration/vars.yml
@@ -8,6 +8,11 @@ swapfileGB: 8
 integration_swap_enabled: False
 e2e_swap_enabled: True
 
+# When False, disable SELinux on the system only during
+# particular tests.
+integration_selinux_enabled: True
+e2e_selinux_enabled: False
+
 # For results.yml Paths use rsync 'source' conventions
 artifacts: "/tmp/artifacts"  # Base-directory for collection
 crio_integration_filepath: "{{ artifacts }}/testout.txt"

From fe82f1b8aa6bddebcb1c45257e31c0436b600a9b Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Wed, 11 Oct 2017 15:15:19 -0400
Subject: [PATCH 05/12] Update all packages before installing new ones

Every now and again, a host will be in an initial state that prevents
installing new packages due to existing packages having some script or
obsoletes problem.  Avoid this by first updating all packages, then
installing new ones.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/system.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/contrib/test/integration/system.yml b/contrib/test/integration/system.yml
index fb0667c6..ea9280ed 100644
--- a/contrib/test/integration/system.yml
+++ b/contrib/test/integration/system.yml
@@ -1,5 +1,12 @@
 ---
 
+- name: Update all packages
+  package:
+    name: '*'
+    state: latest
+  async: 600
+  poll: 10
+
 - name: Make sure we have all required packages
   package:
     name: "{{ item }}"
@@ -65,13 +72,6 @@
    - btrfs-progs-devel
   when: ansible_distribution in ['Fedora']
 
-- name: Update all packages
-  package:
-    name: '*'
-    state: latest
-  async: 600
-  poll: 10
-
 - name: Check / setup swap
   include: "swap.yml"
 

From 8e37304a96440412f2fa815f569000603a6775e8 Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Wed, 4 Oct 2017 10:37:37 -0400
Subject: [PATCH 06/12] Support testing against remote subjects.

It's a severe anti-pattern for a playbook to assume execution always
on a specific host.  The normal/expected pattern is to execute from a
"control host", against an inventory of (possibly-remote) subjects.

This doesn't preclude the inventory from only ever containing
'localhost', it simply means the plays and tasks should not assume
the inventory contents.

This concept is one of the central design-pillars of Ansible's,
and tantamount to it's usefulness and flexibility.  However, in
practice (and by ``integration/readme.md``), plays
specify ``- hosts: all`` but assume inventory_hostname == 'localhost'
(always).

Fix both the playbooks and ``readme.md`` to remove this anti-pattern,
while also allowing the control-host to be the subject-host as needed.
This is accomplished by ensuring low-level Ansible dependencies are
always installed, and writing tasks for steps previously performed
externally (in the CI/automation machinery).

Also update ``readme.md`` to recommend execution occurs through
the ``venv-ansible-playbook.sh`` wrapper to ensure consistent, stable,
version-locked execution dependencies on the control-host.

Remove ``remote_user: root`` from main, since this is better left
to the inventory and command-line.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/README.md       | 54 ++++++++++++++++++------
 contrib/test/integration/build/cri-o.yml | 22 +++++-----
 contrib/test/integration/github.yml      | 27 ++++++++++++
 contrib/test/integration/main.yml        | 40 ++++++++++++++----
 contrib/test/integration/results.yml     |  2 +-
 contrib/test/integration/system.yml      | 15 +++++++
 contrib/test/integration/test.yml        |  6 +--
 contrib/test/integration/vars.yml        |  9 ++++
 8 files changed, 138 insertions(+), 37 deletions(-)
 create mode 100644 contrib/test/integration/github.yml

diff --git a/contrib/test/integration/README.md b/contrib/test/integration/README.md
index f13b8b92..c90a5eac 100644
--- a/contrib/test/integration/README.md
+++ b/contrib/test/integration/README.md
@@ -1,21 +1,49 @@
 # Fedora and RHEL Integration and End-to-End Tests
 
 This directory contains playbooks to set up for and run the integration and
-end-to-end tests for CRI-O on RHEL and Fedora hosts. Two entrypoints exist:
+end-to-end tests for CRI-O on RHEL and Fedora hosts.  The expected entry-point
+is the ``main.yml`` Ansible playbook.
 
- - `main.yml`: sets up the machine and runs tests
- - `results.yml`: gathers test output to `/tmp/artifacts`
+The control-host:
 
-When running `main.yml`, three tags are present:
+ - May be the subject.
+ - Is based on either RHEL/CentOS 6 (or later), or Fedora 24 (or later).
+ - Runs ``main.yml`` from within the cri-o repository already in the
+   desired state for testing.
 
- - `setup`: run all tasks to set up the system for testing
- - `e2e`: build CRI-O from source and run Kubernetes node E2Es
- - `integration`: build CRI-O from source and run the local integration suite
+The subject host(s):
 
-The playbooks assume the following things about your system:
+ - May be the control-host.
+ - May be executing the ``main.yml`` playbook against itself.
+ - If RHEL-like, has the ``server``, ``extras``, and ``EPEL`` repositories available
+   and enabled.
+ - Has remote password-less ssh configured for direct or sudo access to the root user.
 
- - on RHEL, the server and extras repos are configured and certs are present
- - `ansible` is installed and the host is boot-strapped to allow `ansible` to run against it
- - the `$GOPATH` is set and present for all shells (*e.g.* written in `/etc/environment`)
- - CRI-O is checked out to the correct state at `${GOPATH}/src/github.com/kubernetes-incubator/cri-o`
- - the user running the playbook has access to passwordless `sudo`
\ No newline at end of file
+Execution of the ``main.yml`` playbook:
+
+ - Should occur through the ``cri-o/contrib/test/venv-ansible-playbook.sh`` wrapper.
+ - Execution may target localhost, or one or more subjects via standard Ansible
+   inventory arguments.
+ - Should use a combination (including none) of the following tags:
+
+     - ``setup``: Run all tasks to set up the system for testing. Final state must
+                  be self-contained and independent from other tags (i.e. support
+                  stage-caching).
+     - ``integration``: Assumes 'setup' previously completed successfully.
+                        May be executed from cached-state of ``setup``.
+                        Not required to execute conicident with other tags.
+                        Must build CRI-O from source and run the
+                        integration test suite.
+     - ``e2e``: Assumes 'setup' previously completed successfully.  May be executed
+                from cached-state of ``setup``. Not required to execute conicident with
+                other tags.  Must build CRI-O from source and run Kubernetes node
+                E2E tests.
+
+``cri-o/contrib/test/venv-ansible-playbook.sh`` Wrapper:
+
+ - Must accepts all of the valid Ansible command-line options.
+ - Must use version-locked & hashed dependencies as written in ``requirements.txt``.
+ - Must fully sandbox it's own execution environment except for the following
+   required packages (or equivalent): ``python2-virtualenv gcc openssl-devel
+   redhat-rpm-config libffi-devel python-devel libselinux-python rsync
+   yum-utils python3-pycurl python-simplejson``.
diff --git a/contrib/test/integration/build/cri-o.yml b/contrib/test/integration/build/cri-o.yml
index fa025035..29172866 100644
--- a/contrib/test/integration/build/cri-o.yml
+++ b/contrib/test/integration/build/cri-o.yml
@@ -1,42 +1,42 @@
 ---
 
-- name: stat the expected cri-o directory
+- name: stat the expected cri-o directory and Makefile exists
   stat:
-    path: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    path: "{{ cri_o_dest_path }}/Makefile"
   register: dir_stat
 
-- name: expect cri-o to be cloned already
+- name: Verify cri-o Makefile exists in expected location
   fail:
-    msg: "Expected cri-o to be cloned at {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o but it wasn't!"
-  when: not dir_stat.stat.exists
+    msg: "Expected cri-o to be cloned at {{ cri_o_dest_path }} but it wasn't!"
+  when: not dir_stat.stat.exists or not dir_stat.stat.isreg
 
 - name: install cri-o tools
   make:
     target: install.tools
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
 
 - name: build cri-o
   make:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
 
 - name: install cri-o
   make:
     target: install
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
 
 - name: install cri-o systemd files
   make:
     target: install.systemd
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
 
 - name: install cri-o config
   make:
     target: install.config
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
 
 - name: install configs
   copy:
-    src: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/{{ item.src }}"
+    src: "{{ cri_o_dest_path }}/{{ item.src }}"
     dest: "{{ item.dest }}"
     remote_src: yes
   with_items:
diff --git a/contrib/test/integration/github.yml b/contrib/test/integration/github.yml
new file mode 100644
index 00000000..16aef9f4
--- /dev/null
+++ b/contrib/test/integration/github.yml
@@ -0,0 +1,27 @@
+---
+
+
+- name: Verify expectations
+  assert:
+    that:
+      - 'cri_o_dest_path is defined'
+      - 'cri_o_src_path is defined'
+
+- name: The cri-o repository directory exists
+  file:
+    path: "{{ cri_o_dest_path }}"
+    state: directory
+    mode: 0777
+
+- name: Synchronize cri-o from control-host to remote subject
+  synchronize:
+    archive: False
+    checksum: True
+    delete: True
+    dest: "{{ cri_o_dest_path }}/"
+    links: True
+    recursive: True
+    src: "{{ cri_o_src_path }}/"
+    times: True
+  # This task is excessively noisy, logging every change to every file :(
+  no_log: True
diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml
index f09d8d52..98a3eef1 100644
--- a/contrib/test/integration/main.yml
+++ b/contrib/test/integration/main.yml
@@ -1,5 +1,27 @@
-- hosts: all
-  remote_user: root
+---
+
+- hosts: '{{ subjects | default("all") }}'
+  gather_facts: False  # Requires low-level ansible-dependencies
+  tasks:
+    - name: Ensure low-level ansible-dependencies are installed
+      raw: $(type -P dnf || type -P yum) install -y python2 python2-dnf libselinux-python git rsync
+
+    - name: Gather only networking facts for speed
+      setup:
+        gather_subset: network
+
+
+- hosts: '{{ subjects | default("none") }}'
+  vars_files:
+    - "{{ playbook_dir }}/vars.yml"
+  tags:
+    - setup
+  tasks:
+    - name: CRI-O source is available on every subject
+      include: github.yml
+
+
+- hosts: '{{ subjects | default("all") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
   tags:
@@ -23,8 +45,8 @@
     - name: clone build and install networking plugins
       include: "build/plugins.yml"
 
-- hosts: all
-  remote_user: root
+
+- hosts: '{{ subjects | default("all") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
   tags:
@@ -33,12 +55,13 @@
   tasks:
     - name: clone build and install kubernetes
       include: "build/kubernetes.yml"
+      tags:
+        - e2e
 
-    - name: clone build and install cri-o
+    - name: Build and install cri-o
       include: "build/cri-o.yml"
 
-- hosts: all
-  remote_user: root
+- hosts: '{{ subjects | default("all") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
   tags:
@@ -47,8 +70,7 @@
     - name: run cri-o integration tests
       include: test.yml
 
-- hosts: all
-  remote_user: root
+- hosts: '{{ subjects | default("all") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
   tags:
diff --git a/contrib/test/integration/results.yml b/contrib/test/integration/results.yml
index c9a96abb..96b0f0b4 100644
--- a/contrib/test/integration/results.yml
+++ b/contrib/test/integration/results.yml
@@ -1,7 +1,7 @@
 ---
 # vim-syntax: ansible
 
-- hosts: '{{ hosts | default("all") }}'
+- hosts: '{{ subjects | default("all") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
   vars:
diff --git a/contrib/test/integration/system.yml b/contrib/test/integration/system.yml
index ea9280ed..c453c422 100644
--- a/contrib/test/integration/system.yml
+++ b/contrib/test/integration/system.yml
@@ -70,6 +70,7 @@
     state: present
   with_items:
    - btrfs-progs-devel
+   - python2-virtualenv
   when: ansible_distribution in ['Fedora']
 
 - name: Check / setup swap
@@ -110,3 +111,17 @@
 - name: Update the kernel cmdline to include quota support
   command: grubby --update-kernel=ALL --args="rootflags=pquota"
   when: ansible_distribution in ['RedHat', 'CentOS']
+
+- name: Configure the GOPATH environment variable for all users
+  blockinfile:
+    path: /etc/environment
+    block: "GOPATH={{ go_path }}"
+    create: True
+    follow: True
+
+- name: Reset the ansible connection to incorporate /etc/environment changes
+  meta: reset_connection
+
+- name: Refresh facts to incorporate /etc/environment changes
+  setup:
+    gather_subset: network
diff --git a/contrib/test/integration/test.yml b/contrib/test/integration/test.yml
index e9023e77..dfa2c30f 100644
--- a/contrib/test/integration/test.yml
+++ b/contrib/test/integration/test.yml
@@ -15,7 +15,7 @@
     extra_storage_opts: " --storage-opt overlay.override_kernel_check=1"
   when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
 
-- name: ensure directory exists for e2e reports
+- name: ensure directory exists for integration results
   file:
     path: "{{ artifacts }}"
     state: directory
@@ -31,9 +31,9 @@
       when: not integration_selinux_enabled
 
     - name: run integration tests
-      shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTIONS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration >& {{ artifacts }}/testout.txt"
+      shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration >& {{ artifacts }}/testout.txt"
       args:
-        chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+        chdir: "{{ cri_o_dest_path }}"
       async: 5400
       poll: 30
 
diff --git a/contrib/test/integration/vars.yml b/contrib/test/integration/vars.yml
index 801eb1bc..3362f2d3 100644
--- a/contrib/test/integration/vars.yml
+++ b/contrib/test/integration/vars.yml
@@ -13,6 +13,15 @@ e2e_swap_enabled: True
 integration_selinux_enabled: True
 e2e_selinux_enabled: False
 
+# Path to encode into /etc/environment on all hosts
+go_path: "/go"
+
+# Absolute path on control-host where the cri-o source exists
+cri_o_src_path: "{{ playbook_dir }}/../../../"
+
+# Absolute path on subjects where cri-o source is expected
+cri_o_dest_path: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-o"
+
 # For results.yml Paths use rsync 'source' conventions
 artifacts: "/tmp/artifacts"  # Base-directory for collection
 crio_integration_filepath: "{{ artifacts }}/testout.txt"

From 8964657140395b4aa0607abc05d63633395fec63 Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Mon, 16 Oct 2017 14:22:13 -0400
Subject: [PATCH 07/12] Avoid relying on subject environment-vars

There are so many ways/places they can change values on the host from
one moment to the next.  Yet as written, the value of ansible_env.GOPATH
is really only fixed at "fact gathering" time.  In other words, the
environment variable can change (even during a play), but won't be
noticed until possibly much later.  This can cause very strange things
to happen which aren't easy to debug.

Fix this by using established facts (variables), and continuously
establishing them as environment variables.  This way, if/when a task
fails, the value of the environment will be present w/in the failure
message instead of obscrured by the shell.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/build/bats.yml       |  4 ++--
 contrib/test/integration/build/cri-tools.yml  |  4 ++--
 contrib/test/integration/build/kubernetes.yml | 24 ++++---------------
 contrib/test/integration/build/plugins.yml    | 12 +++++-----
 contrib/test/integration/build/runc.yml       |  6 ++---
 contrib/test/integration/e2e.yml              |  4 ++--
 contrib/test/integration/golang.yml           | 24 +++++--------------
 contrib/test/integration/main.yml             |  7 ++++++
 contrib/test/integration/system.yml           | 14 -----------
 contrib/test/integration/test.yml             |  4 ++--
 contrib/test/integration/vars.yml             |  9 +++++++
 11 files changed, 44 insertions(+), 68 deletions(-)

diff --git a/contrib/test/integration/build/bats.yml b/contrib/test/integration/build/bats.yml
index d4ea19c6..ec2900b0 100644
--- a/contrib/test/integration/build/bats.yml
+++ b/contrib/test/integration/build/bats.yml
@@ -3,12 +3,12 @@
 - name: clone bats source repo
   git:
     repo: "https://github.com/sstephenson/bats.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats"
+    dest: "{{ go_path }}/src/github.com/sstephenson/bats"
 
 - name: install bats
   command: "./install.sh /usr/local"
   args:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats"
+    chdir: "{{ go_path }}/src/github.com/sstephenson/bats"
 
 - name: link bats
   file:
diff --git a/contrib/test/integration/build/cri-tools.yml b/contrib/test/integration/build/cri-tools.yml
index 9a117f3c..fad1dbfe 100644
--- a/contrib/test/integration/build/cri-tools.yml
+++ b/contrib/test/integration/build/cri-tools.yml
@@ -3,7 +3,7 @@
 - name: clone cri-tools source repo
   git:
     repo: "https://github.com/kubernetes-incubator/cri-tools.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-tools"
+    dest: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-tools"
     version: "9ff5e8f78a4182ab8d5ba9bcccdda5f338600eab"
 
 - name: install crictl
@@ -11,6 +11,6 @@
 
 - name: link crictl
   file:
-    src: "{{ ansible_env.GOPATH }}/bin/crictl"
+    src: "{{ go_path }}/bin/crictl"
     dest: /usr/bin/crictl
     state: link
diff --git a/contrib/test/integration/build/kubernetes.yml b/contrib/test/integration/build/kubernetes.yml
index f724230c..e8b19c57 100644
--- a/contrib/test/integration/build/kubernetes.yml
+++ b/contrib/test/integration/build/kubernetes.yml
@@ -3,17 +3,17 @@
 - name: clone kubernetes source repo
   git:
     repo: "https://github.com/runcom/kubernetes.git"
-    dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+    dest: "{{ go_path }}/src/k8s.io/kubernetes"
     version: "cri-o-patched-1.8"
 
 - name: install etcd
   command: "hack/install-etcd.sh"
   args:
-    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+    chdir: "{{ go_path }}/src/k8s.io/kubernetes"
 
 - name: build kubernetes
   make:
-    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+    chdir: "{{ go_path }}/src/k8s.io/kubernetes"
 
 - name: Add custom cluster service file for the e2e testing
   copy:
@@ -23,7 +23,7 @@
       After=network-online.target
       Wants=network-online.target
       [Service]
-      WorkingDirectory={{ ansible_env.GOPATH }}/src/k8s.io/kubernetes
+      WorkingDirectory={{ go_path }}/src/k8s.io/kubernetes
       ExecStart=/usr/local/bin/createcluster.sh
       User=root
       [Install]
@@ -35,7 +35,7 @@
     content: |
       #!/bin/bash
 
-      export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ ansible_env.GOPATH }}/bin:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/third_party/etcd:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/
+      export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ go_path }}/bin:{{ go_path }}/src/k8s.io/kubernetes/third_party/etcd:{{ go_path }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/
       export CONTAINER_RUNTIME=remote
       export CGROUP_DRIVER=systemd
       export CONTAINER_RUNTIME_ENDPOINT='/var/run/crio.sock --runtime-request-timeout=5m'
@@ -47,17 +47,3 @@
       export KUBE_ENABLE_CLUSTER_DNS=true
       ./hack/local-up-cluster.sh
     mode: "u=rwx,g=rwx,o=x"
-
-- name: Set kubernetes_provider to be local
-  lineinfile:
-    dest: /etc/environment
-    line: 'KUBERNETES_PROVIDER=local'
-    regexp: 'KUBERNETES_PROVIDER='
-    state: present
-
-- name: Set KUBECONFIG
-  lineinfile:
-    dest: /etc/environment
-    line: 'KUBECONFIG=/var/run/kubernetes/admin.kubeconfig'
-    regexp: 'KUBECONFIG='
-    state: present
diff --git a/contrib/test/integration/build/plugins.yml b/contrib/test/integration/build/plugins.yml
index e342a0b9..b344270a 100644
--- a/contrib/test/integration/build/plugins.yml
+++ b/contrib/test/integration/build/plugins.yml
@@ -3,17 +3,17 @@
 - name: clone plugins source repo
   git:
     repo: "https://github.com/containernetworking/plugins.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+    dest: "{{ go_path }}/src/github.com/containernetworking/plugins"
     version: "dcf7368eeab15e2affc6256f0bb1e84dd46a34de"
 
 - name: build plugins
   command: "./build.sh"
   args:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+    chdir: "{{ go_path }}/src/github.com/containernetworking/plugins"
 
 - name: install plugins
   copy:
-    src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/{{ item }}"
+    src: "{{ go_path }}/src/github.com/containernetworking/plugins/bin/{{ item }}"
     dest: "/opt/cni/bin"
     mode: "o=rwx,g=rx,o=rx"
     remote_src: yes
@@ -33,18 +33,18 @@
 - name: clone runcom plugins source repo
   git:
     repo: "https://github.com/runcom/plugins.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+    dest: "{{ go_path }}/src/github.com/containernetworking/plugins"
     version: "custom-bridge"
     force: yes
 
 - name: build plugins
   command: "./build.sh"
   args:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+    chdir: "{{ go_path }}/src/github.com/containernetworking/plugins"
 
 - name: install custom bridge
   copy:
-    src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/bridge"
+    src: "{{ go_path }}/src/github.com/containernetworking/plugins/bin/bridge"
     dest: "/opt/cni/bin/bridge-custom"
     mode: "o=rwx,g=rx,o=rx"
     remote_src: yes
diff --git a/contrib/test/integration/build/runc.yml b/contrib/test/integration/build/runc.yml
index 7bb0491d..1dd04f03 100644
--- a/contrib/test/integration/build/runc.yml
+++ b/contrib/test/integration/build/runc.yml
@@ -3,18 +3,18 @@
 - name: clone runc source repo
   git:
     repo: "https://github.com/opencontainers/runc.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
+    dest: "{{ go_path }}/src/github.com/opencontainers/runc"
     version: "84a082bfef6f932de921437815355186db37aeb1"
 
 - name: build runc
   make:
     params: BUILDTAGS="seccomp selinux"
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
+    chdir: "{{ go_path }}/src/github.com/opencontainers/runc"
 
 - name: install runc
   make:
     target: "install"
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
+    chdir: "{{ go_path }}/src/github.com/opencontainers/runc"
 
 - name: link runc
   file:
diff --git a/contrib/test/integration/e2e.yml b/contrib/test/integration/e2e.yml
index 5520b4f4..547006c4 100644
--- a/contrib/test/integration/e2e.yml
+++ b/contrib/test/integration/e2e.yml
@@ -26,7 +26,7 @@
     daemon_reload: yes
 
 - name: wait for the cluster to be running
-  command: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default"
+  command: "{{ go_path }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default"
   register: kube_poll
   until: kube_poll | succeeded
   retries: 100
@@ -61,7 +61,7 @@
     - name: run e2e tests
       shell: "{{ e2e_shell_cmd | regex_replace('\\s+', ' ') }}"
       args:
-        chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+        chdir: "{{ go_path }}/src/k8s.io/kubernetes"
 
   always:
 
diff --git a/contrib/test/integration/golang.yml b/contrib/test/integration/golang.yml
index 63e55697..39be5aa1 100644
--- a/contrib/test/integration/golang.yml
+++ b/contrib/test/integration/golang.yml
@@ -16,28 +16,16 @@
     - gofmt
     - godoc
 
-- name: ensure user profile exists
-  file:
-    path: "{{ ansible_user_dir }}/.profile"
-    state: touch
-
-- name: set up PATH for Go toolchain and built binaries
-  lineinfile:
-    dest: "{{ ansible_user_dir }}/.profile"
-    line: 'PATH={{ ansible_env.PATH }}:{{ ansible_env.GOPATH }}/bin:/usr/local/go/bin'
-    regexp: '^PATH='
-    state: present
-
 - name: set up directories
   file:
-    path: "{{ item }}"
+    path: "{{ go_path }}/{{ item }}"
     state: directory
   with_items:
-    - "{{ ansible_env.GOPATH }}/src/github.com/containernetworking"
-    - "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator"
-    - "{{ ansible_env.GOPATH }}/src/github.com/k8s.io"
-    - "{{ ansible_env.GOPATH }}/src/github.com/sstephenson"
-    - "{{ ansible_env.GOPATH }}/src/github.com/opencontainers"
+    - "src/github.com/containernetworking"
+    - "src/github.com/kubernetes-incubator"
+    - "src/github.com/k8s.io"
+    - "src/github.com/sstephenson"
+    - "src/github.com/opencontainers"
 
 - name: install Go tools and dependencies
   shell: /usr/bin/go get -u "github.com/{{ item }}"
diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml
index 98a3eef1..dc8d1c18 100644
--- a/contrib/test/integration/main.yml
+++ b/contrib/test/integration/main.yml
@@ -2,6 +2,9 @@
 
 - hosts: '{{ subjects | default("all") }}'
   gather_facts: False  # Requires low-level ansible-dependencies
+  # Cannot use vars.yml - it references magic variables from setup module
+  tags:
+    - setup
   tasks:
     - name: Ensure low-level ansible-dependencies are installed
       raw: $(type -P dnf || type -P yum) install -y python2 python2-dnf libselinux-python git rsync
@@ -24,6 +27,7 @@
 - hosts: '{{ subjects | default("all") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
+  environment: '{{ environment_variables }}'
   tags:
     - setup
   tasks:
@@ -49,6 +53,7 @@
 - hosts: '{{ subjects | default("all") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
+  environment: '{{ environment_variables }}'
   tags:
     - integration
     - e2e
@@ -64,6 +69,7 @@
 - hosts: '{{ subjects | default("all") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
+  environment: '{{ environment_variables }}'
   tags:
     - integration
   tasks:
@@ -73,6 +79,7 @@
 - hosts: '{{ subjects | default("all") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
+  environment: '{{ environment_variables }}'
   tags:
     - e2e
   tasks:
diff --git a/contrib/test/integration/system.yml b/contrib/test/integration/system.yml
index c453c422..c8dfdb3b 100644
--- a/contrib/test/integration/system.yml
+++ b/contrib/test/integration/system.yml
@@ -111,17 +111,3 @@
 - name: Update the kernel cmdline to include quota support
   command: grubby --update-kernel=ALL --args="rootflags=pquota"
   when: ansible_distribution in ['RedHat', 'CentOS']
-
-- name: Configure the GOPATH environment variable for all users
-  blockinfile:
-    path: /etc/environment
-    block: "GOPATH={{ go_path }}"
-    create: True
-    follow: True
-
-- name: Reset the ansible connection to incorporate /etc/environment changes
-  meta: reset_connection
-
-- name: Refresh facts to incorporate /etc/environment changes
-  setup:
-    gather_subset: network
diff --git a/contrib/test/integration/test.yml b/contrib/test/integration/test.yml
index dfa2c30f..da814b17 100644
--- a/contrib/test/integration/test.yml
+++ b/contrib/test/integration/test.yml
@@ -5,7 +5,7 @@
 
 - name: Make testing output verbose so it can be converted to xunit
   lineinfile:
-    dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/hack/make-rules/test.sh"
+    dest: "{{ go_path }}/src/k8s.io/kubernetes/hack/make-rules/test.sh"
     line: ' go test -v "${goflags[@]:+${goflags[@]}}" \'
     regexp: ' go test \"\$'
     state: present
@@ -31,7 +31,7 @@
       when: not integration_selinux_enabled
 
     - name: run integration tests
-      shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration >& {{ artifacts }}/testout.txt"
+      shell: "make localintegration >& {{ artifacts }}/testout.txt"
       args:
         chdir: "{{ cri_o_dest_path }}"
       async: 5400
diff --git a/contrib/test/integration/vars.yml b/contrib/test/integration/vars.yml
index 3362f2d3..3382caf6 100644
--- a/contrib/test/integration/vars.yml
+++ b/contrib/test/integration/vars.yml
@@ -22,6 +22,15 @@ cri_o_src_path: "{{ playbook_dir }}/../../../"
 # Absolute path on subjects where cri-o source is expected
 cri_o_dest_path: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-o"
 
+# Global environment variables to use for all tasks
+environment_variables:
+    PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:{{ go_path }}/bin:/usr/local/go/bin"
+    GOPATH: "{{ go_path }}"
+    KUBERNETES_PROVIDER: "local"
+    KUBECONFIG: "/var/run/kubernetes/admin.kubeconfig"
+    CGROUP_MANAGER: "cgroupfs"
+    STORAGE_OPTS: '--storage-driver=overlay{{ extra_storage_opts | default("") }}'
+
 # For results.yml Paths use rsync 'source' conventions
 artifacts: "/tmp/artifacts"  # Base-directory for collection
 crio_integration_filepath: "{{ artifacts }}/testout.txt"

From 51fed531398a2ef6b15a5cf4bc3dd12ae5ca2961 Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Wed, 18 Oct 2017 10:09:18 -0400
Subject: [PATCH 08/12] Consolidate plays/tasks/tags

Simplify use of play-level tagging on three separate plays (with one
task).  Instead, make them all the same play, and apply the tags at
the task level instead.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/main.yml | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)

diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml
index dc8d1c18..869c6626 100644
--- a/contrib/test/integration/main.yml
+++ b/contrib/test/integration/main.yml
@@ -54,9 +54,6 @@
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
   environment: '{{ environment_variables }}'
-  tags:
-    - integration
-    - e2e
   tasks:
     - name: clone build and install kubernetes
       include: "build/kubernetes.yml"
@@ -65,23 +62,15 @@
 
     - name: Build and install cri-o
       include: "build/cri-o.yml"
+      tags:
+        - always
 
-- hosts: '{{ subjects | default("all") }}'
-  vars_files:
-    - "{{ playbook_dir }}/vars.yml"
-  environment: '{{ environment_variables }}'
-  tags:
-    - integration
-  tasks:
     - name: run cri-o integration tests
       include: test.yml
+      tags:
+        - integration
 
-- hosts: '{{ subjects | default("all") }}'
-  vars_files:
-    - "{{ playbook_dir }}/vars.yml"
-  environment: '{{ environment_variables }}'
-  tags:
-    - e2e
-  tasks:
     - name: run k8s e2e tests
       include: e2e.yml
+      tags:
+        - e2e

From 4df075abd2877a5798fa461ffb8bcc659a1242e9 Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Thu, 19 Oct 2017 09:05:25 -0400
Subject: [PATCH 09/12] Improve control-host CPU performance

When running from a central host, where multiple other playbooks may
also be executing, CPU time quickly becomes the scaleability bottleneck.

* Reduce the vars compression level at the cost of network utilization.
This assumes the number of vars being transfered back/forth remains
reasonably low, where there wouldn't be much advantage from higher
compression anyway.

Another enhancement ``ControlPersist`` (for ssh) is apt to fall
back to opening new connections (slow) for every request under some
conditions.  This happens if the socket filename is too large (108
characters, including path) - a kernel limitation.

Unfortunately, in cloud environments, auto-assigned VM hostnames tend to
be rather large to avoid clashes.  Worse, in a CI environment, the default
home-directory path also tends to be lengthy for the same reason.

* Address this by sticking persistent-connection, background socket files
in '/tmp/cri-o' (avoid %d).  Also remove the username (%r) designation,
since this will almost always be the same user anyway.  The tradeoff
here is clashes between jobs against the same host (unlikely) and
weakened security on the control host (less important for CI jobs).

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/ansible.cfg | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/contrib/test/integration/ansible.cfg b/contrib/test/integration/ansible.cfg
index 92a13a5f..120419f5 100644
--- a/contrib/test/integration/ansible.cfg
+++ b/contrib/test/integration/ansible.cfg
@@ -57,11 +57,6 @@ gather_subset = network
 #host_key_checking = False
 host_key_checking = False
 
-# change the default callback
-#stdout_callback = skippy
-# enable additional callbacks
-#callback_whitelist = timer, mail
-
 # Determine whether includes in tasks and handlers are "static" by
 # default. As of 2.0, includes are dynamic by default. Setting these
 # values to True will make includes behave more like they did in the
@@ -165,7 +160,6 @@ deprecation_warnings = False
 # instead of shelling out to the git command.
 command_warnings = False
 
-
 # set plugin path directories here, separate with colons
 #action_plugins     = /usr/share/ansible/plugins/action
 #callback_plugins   = /usr/share/ansible/plugins/callback
@@ -219,7 +213,6 @@ nocolor = 0
 # When a playbook fails by default a .retry file will be created in ~/
 # You can disable this feature by setting retry_files_enabled to False
 # and you can change the location of the files by setting retry_files_save_path
-
 #retry_files_enabled = False
 retry_files_enabled = False
 
@@ -248,6 +241,7 @@ no_target_syslog = True
 # worker processes. At the default of 0, no compression
 # is used. This value must be an integer from 0 to 9.
 #var_compression_level = 9
+var_compression_level = 3
 
 # controls what compression method is used for new-style ansible modules when
 # they are sent to the remote system.  The compression types depend on having
@@ -298,6 +292,7 @@ ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o UserKnownHostsFile=/de
 # Example:
 # control_path = %(directory)s/%%h-%%r
 #control_path = %(directory)s/ansible-ssh-%%h-%%p-%%r
+control_path = /tmp/crio-%%n-%%p
 
 # Enabling pipelining reduces the number of SSH operations required to
 # execute a module on the remote server. This can result in a significant

From 8c4db4222a9fbe4c2dfde2e8df773ef588502a83 Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Thu, 19 Oct 2017 14:38:09 -0400
Subject: [PATCH 10/12] Increase package install timeout

Because networking.

Esp. for VMs pulling content from the CDNs, allow install to run longer.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/system.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/test/integration/system.yml b/contrib/test/integration/system.yml
index c8dfdb3b..c17e3c6d 100644
--- a/contrib/test/integration/system.yml
+++ b/contrib/test/integration/system.yml
@@ -61,7 +61,7 @@
     - socat
     - tar
     - wget
-  async: 600
+  async: '{{ 20 * 60 }}'
   poll: 10
 
 - name: Add Btrfs for Fedora

From f1817ab2aabf5d19512669df8906f861132b125d Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Tue, 31 Oct 2017 09:22:56 -0400
Subject: [PATCH 11/12] Updates based on PR #972 feedback

* When installing low-level dependencies, ignore any errors as
  consequences will be revealed in the subsequent task.
* Rename dir_stat -> crio_stat to better match usage.
* Add comment regarding ControlPersist in ansible.cfg.
* Simplify extra_storage_opts variable usage, consolidate definition.
* Reduce duplication of base-paths in golang.yml directory-setup loop.
* Clarify a few points and add a wrapper example to README.md

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/README.md       | 45 +++++++++++++++++++-----
 contrib/test/integration/ansible.cfg     |  9 ++++-
 contrib/test/integration/build/cri-o.yml |  6 ++--
 contrib/test/integration/golang.yml      | 12 +++----
 contrib/test/integration/main.yml        | 26 ++++++++++++--
 contrib/test/integration/test.yml        |  5 ---
 contrib/test/integration/vars.yml        | 14 ++------
 7 files changed, 79 insertions(+), 38 deletions(-)

diff --git a/contrib/test/integration/README.md b/contrib/test/integration/README.md
index c90a5eac..f7220005 100644
--- a/contrib/test/integration/README.md
+++ b/contrib/test/integration/README.md
@@ -4,6 +4,16 @@ This directory contains playbooks to set up for and run the integration and
 end-to-end tests for CRI-O on RHEL and Fedora hosts.  The expected entry-point
 is the ``main.yml`` Ansible playbook.
 
+##Definitions:
+
+    Control-host:  The system from which the ``ansible-playbook`` or
+                   ``venv-ansible-playbook.sh`` command is executed.
+
+    Subject-host(s): The target systems, on which actual playbook tasks are
+                     being carried out.
+
+##Topology:
+
 The control-host:
 
  - May be the subject.
@@ -11,13 +21,17 @@ The control-host:
  - Runs ``main.yml`` from within the cri-o repository already in the
    desired state for testing.
 
-The subject host(s):
+The subject-host(s):
 
  - May be the control-host.
  - May be executing the ``main.yml`` playbook against itself.
  - If RHEL-like, has the ``server``, ``extras``, and ``EPEL`` repositories available
    and enabled.
- - Has remote password-less ssh configured for direct or sudo access to the root user.
+ - Has remote password-less ssh configured for access by the control-host.
+ - When ssh-access is for a regular user, that user has password-less
+   sudo access to root.
+
+##Runtime Requirements:
 
 Execution of the ``main.yml`` playbook:
 
@@ -31,19 +45,34 @@ Execution of the ``main.yml`` playbook:
                   stage-caching).
      - ``integration``: Assumes 'setup' previously completed successfully.
                         May be executed from cached-state of ``setup``.
-                        Not required to execute conicident with other tags.
+                        Not required to execute coincident with other tags.
                         Must build CRI-O from source and run the
                         integration test suite.
      - ``e2e``: Assumes 'setup' previously completed successfully.  May be executed
-                from cached-state of ``setup``. Not required to execute conicident with
+                from cached-state of ``setup``. Not required to execute coincident with
                 other tags.  Must build CRI-O from source and run Kubernetes node
                 E2E tests.
 
 ``cri-o/contrib/test/venv-ansible-playbook.sh`` Wrapper:
 
- - Must accepts all of the valid Ansible command-line options.
- - Must use version-locked & hashed dependencies as written in ``requirements.txt``.
- - Must fully sandbox it's own execution environment except for the following
-   required packages (or equivalent): ``python2-virtualenv gcc openssl-devel
+ - May be executed on the control-host to both hide and version-lock playbook
+   execution dependencies, ansible and otherwise.
+ - Must accept all of the valid Ansible command-line options.
+ - Must sandbox dependencies under a python virtual environment ``.cri-o_venv``
+   with packages as specified in ``requirements.txt``.
+ - Requires the control-host has the following fundamental dependencies installed
+   (or equivalent): ``python2-virtualenv gcc openssl-devel
    redhat-rpm-config libffi-devel python-devel libselinux-python rsync
    yum-utils python3-pycurl python-simplejson``.
+
+For example:
+
+Given a populated '/path/to/inventory' file, a control-host could run:
+
+./venv-ansible-playbook.sh -i /path/to/inventory ./integration/main.yml
+
+-or-
+
+From a subject-host without an inventory:
+
+./venv-ansible-playbook.sh -i localhost, ./integration/main.yml
diff --git a/contrib/test/integration/ansible.cfg b/contrib/test/integration/ansible.cfg
index 120419f5..33adb106 100644
--- a/contrib/test/integration/ansible.cfg
+++ b/contrib/test/integration/ansible.cfg
@@ -292,8 +292,16 @@ ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o UserKnownHostsFile=/de
 # Example:
 # control_path = %(directory)s/%%h-%%r
 #control_path = %(directory)s/ansible-ssh-%%h-%%p-%%r
+# Using ssh's ControlPersist feature is desireable because of wide
+# compatibility and not needing to mess with /etc/sudoers
+# for pipelining (see below).  Unfortunately, in cloud environments,
+# auto-assigned VM hostnames tend to be rather longs.  Worse, in a CI
+# context, the default home-directory path may also be lengthy.  Fix
+# this to a short name, so Ansible doesn't fall back to opening new
+# connections for every task.
 control_path = /tmp/crio-%%n-%%p
 
+
 # Enabling pipelining reduces the number of SSH operations required to
 # execute a module on the remote server. This can result in a significant
 # performance improvement when enabled, however when using "sudo:" you must
@@ -303,7 +311,6 @@ control_path = /tmp/crio-%%n-%%p
 # sudoers configurations that have requiretty (the default on many distros).
 #
 #pipelining = False
-pipelining=True
 
 # if True, make ansible use scp if the connection type is ssh
 # (default is sftp)
diff --git a/contrib/test/integration/build/cri-o.yml b/contrib/test/integration/build/cri-o.yml
index 29172866..3c7b2c16 100644
--- a/contrib/test/integration/build/cri-o.yml
+++ b/contrib/test/integration/build/cri-o.yml
@@ -3,12 +3,12 @@
 - name: stat the expected cri-o directory and Makefile exists
   stat:
     path: "{{ cri_o_dest_path }}/Makefile"
-  register: dir_stat
+  register: crio_stat
 
 - name: Verify cri-o Makefile exists in expected location
   fail:
-    msg: "Expected cri-o to be cloned at {{ cri_o_dest_path }} but it wasn't!"
-  when: not dir_stat.stat.exists or not dir_stat.stat.isreg
+    msg: "Expected cri-o to be cloned at {{ cri_o_dest_path }}, but its 'Makefile' seems to be missing."
+  when: not crio_stat.stat.exists or not crio_stat.stat.isreg
 
 - name: install cri-o tools
   make:
diff --git a/contrib/test/integration/golang.yml b/contrib/test/integration/golang.yml
index 39be5aa1..037fe851 100644
--- a/contrib/test/integration/golang.yml
+++ b/contrib/test/integration/golang.yml
@@ -18,14 +18,14 @@
 
 - name: set up directories
   file:
-    path: "{{ go_path }}/{{ item }}"
+    path: "{{ go_path }}/src/github.com/{{ item }}"
     state: directory
   with_items:
-    - "src/github.com/containernetworking"
-    - "src/github.com/kubernetes-incubator"
-    - "src/github.com/k8s.io"
-    - "src/github.com/sstephenson"
-    - "src/github.com/opencontainers"
+    - "containernetworking"
+    - "kubernetes-incubator"
+    - "k8s.io"
+    - "sstephenson"
+    - "opencontainers"
 
 - name: install Go tools and dependencies
   shell: /usr/bin/go get -u "github.com/{{ item }}"
diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml
index 869c6626..c68cea23 100644
--- a/contrib/test/integration/main.yml
+++ b/contrib/test/integration/main.yml
@@ -4,15 +4,35 @@
   gather_facts: False  # Requires low-level ansible-dependencies
   # Cannot use vars.yml - it references magic variables from setup module
   tags:
-    - setup
+    - always
   tasks:
-    - name: Ensure low-level ansible-dependencies are installed
-      raw: $(type -P dnf || type -P yum) install -y python2 python2-dnf libselinux-python git rsync
+    - name: Ansible setup-module dependencies are installed, ignoring errors (setup runs next).
+      raw: $(type -P dnf || type -P yum) install -y python2 python2-dnf libselinux-python
+      ignore_errors: True
 
     - name: Gather only networking facts for speed
       setup:
         gather_subset: network
 
+    - name: Variables from vars.yml are hauled in after setup
+      include_vars: "{{ playbook_dir }}/vars.yml"
+
+    - name: Global environment are defined, but can be overriden on a task-by-task basis.
+      set_fact:
+        extra_storage_opts: >
+            {%- if ansible_distribution in ["RedHat", "CentOS"] -%}
+                "--storage-opt overlay.override_kernel_check=1"
+            {%- else -%}
+                ""
+            {%- endif -%}
+        environment_variables:
+            PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:{{ go_path }}/bin:/usr/local/go/bin"
+            GOPATH: "{{ go_path }}"
+            KUBERNETES_PROVIDER: "local"
+            KUBECONFIG: "/var/run/kubernetes/admin.kubeconfig"
+            CGROUP_MANAGER: "cgroupfs"
+            STORAGE_OPTS: '--storage-driver=overlay {{ extra_storage_opts | default("") | trim }}'
+
 
 - hosts: '{{ subjects | default("none") }}'
   vars_files:
diff --git a/contrib/test/integration/test.yml b/contrib/test/integration/test.yml
index da814b17..1834aa7c 100644
--- a/contrib/test/integration/test.yml
+++ b/contrib/test/integration/test.yml
@@ -10,11 +10,6 @@
     regexp: ' go test \"\$'
     state: present
 
-- name: set extra storage options
-  set_fact:
-    extra_storage_opts: " --storage-opt overlay.override_kernel_check=1"
-  when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
-
 - name: ensure directory exists for integration results
   file:
     path: "{{ artifacts }}"
diff --git a/contrib/test/integration/vars.yml b/contrib/test/integration/vars.yml
index 3382caf6..fa8665db 100644
--- a/contrib/test/integration/vars.yml
+++ b/contrib/test/integration/vars.yml
@@ -3,8 +3,7 @@
 # When swap setup is necessary, make it this size
 swapfileGB: 8
 
-# When False, turn off all swapping on the system only during
-# particular tests.
+# When False, turn off all swapping on the system during indicated test.
 integration_swap_enabled: False
 e2e_swap_enabled: True
 
@@ -13,7 +12,7 @@ e2e_swap_enabled: True
 integration_selinux_enabled: True
 e2e_selinux_enabled: False
 
-# Path to encode into /etc/environment on all hosts
+# Base directory for all go-related source, build, and install.
 go_path: "/go"
 
 # Absolute path on control-host where the cri-o source exists
@@ -22,15 +21,6 @@ cri_o_src_path: "{{ playbook_dir }}/../../../"
 # Absolute path on subjects where cri-o source is expected
 cri_o_dest_path: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-o"
 
-# Global environment variables to use for all tasks
-environment_variables:
-    PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:{{ go_path }}/bin:/usr/local/go/bin"
-    GOPATH: "{{ go_path }}"
-    KUBERNETES_PROVIDER: "local"
-    KUBECONFIG: "/var/run/kubernetes/admin.kubeconfig"
-    CGROUP_MANAGER: "cgroupfs"
-    STORAGE_OPTS: '--storage-driver=overlay{{ extra_storage_opts | default("") }}'
-
 # For results.yml Paths use rsync 'source' conventions
 artifacts: "/tmp/artifacts"  # Base-directory for collection
 crio_integration_filepath: "{{ artifacts }}/testout.txt"

From b738a361c7557b9cc35d8204bfce74541be87594 Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Fri, 22 Sep 2017 07:41:15 -0400
Subject: [PATCH 12/12] integration: parse results to canonical jUnit

At the end of the result collection playbook, run a small python script
over the collected artifacts.  Convert/combine them into a single
authorative results file for ci-automation consumption.  Where possible,
add additional identifying details about the source of individual
results.  This extra layer of processing also provides possibilities
for altering the meaning of PASS/SKIP/FAIL WRT specific
ongoing testing needs.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/README.md    |  17 +-
 contrib/test/integration/results.yml  | 117 ++++++----
 contrib/test/integration/vars.yml     |  40 +++-
 contrib/test/parse2junit.py           | 313 ++++++++++++++++++++++++++
 contrib/test/requirements.txt         |   4 +
 contrib/test/venv-ansible-playbook.sh |   8 +-
 6 files changed, 451 insertions(+), 48 deletions(-)
 create mode 100755 contrib/test/parse2junit.py

diff --git a/contrib/test/integration/README.md b/contrib/test/integration/README.md
index f7220005..aac1e5c1 100644
--- a/contrib/test/integration/README.md
+++ b/contrib/test/integration/README.md
@@ -2,7 +2,7 @@
 
 This directory contains playbooks to set up for and run the integration and
 end-to-end tests for CRI-O on RHEL and Fedora hosts.  The expected entry-point
-is the ``main.yml`` Ansible playbook.
+is the ``main.yml``.
 
 ##Definitions:
 
@@ -53,6 +53,21 @@ Execution of the ``main.yml`` playbook:
                 other tags.  Must build CRI-O from source and run Kubernetes node
                 E2E tests.
 
+Execution of the ``results.yml`` playbook:
+
+ - Assumes 'setup' previously completed successfully.
+ - Either ``integration``, ``e2e``, or other testing steps
+   must have completed (even if in failure).
+ - Must be the authorative collector and producer of results for the run,
+   whether or not the control-host is the subject.
+ - Must gather all important/relevant artifacts into a central location.
+ - Must not duplicate, rename, or obfuscate any other results or artifact files
+   from this run or any others.  Must not fail due to missing files or failed commands.
+ - May add test-run identification details so long as they don't interfear with
+   downstream processing or any of the above requirements.
+ - Must be executed using the ``venv-ansible-playbook.sh`` wrapper (b/c
+   ``junitparser`` requirement).
+
 ``cri-o/contrib/test/venv-ansible-playbook.sh`` Wrapper:
 
  - May be executed on the control-host to both hide and version-lock playbook
diff --git a/contrib/test/integration/results.yml b/contrib/test/integration/results.yml
index 96b0f0b4..b4ea3c67 100644
--- a/contrib/test/integration/results.yml
+++ b/contrib/test/integration/results.yml
@@ -4,59 +4,96 @@
 - hosts: '{{ subjects | default("all") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
-  vars:
-    _result_filepaths: []  # do not use
-    _dstfnbuff: []  # do not use
+  environment: '{{ environment_variables }}'
   tasks:
-    - name: The crio_integration_filepath is required
-      tags:
-          - integration
-      set_fact:
-       _result_filepaths: "{{ _result_filepaths + [crio_integration_filepath] }}"
-
-    - name: The crio_node_e2e_filepath is required
-      tags:
-          - e2e
-      set_fact:
-       _result_filepaths: "{{ _result_filepaths + [crio_node_e2e_filepath] }}"
 
     - name: Verify expectations
       assert:
         that:
-          - 'result_dest_basedir | default(False, True)'
-          - '_result_filepaths | default(False, True)'
-          - '_dstfnbuff == []'
-          - 'results_fetched is undefined'
+          # Combined "is defined" and "isn't blank" check
+          - 'artifacts | default("", True) | trim | length'
+          - 'generated_artifacts | default("", True) | trim | length'
+          - 'extra_artifact_filepaths is defined'
+          - 'parsed_artifacts is defined'
+          - 'canonical_junit is defined'
+          - 'playbook_dir ~ "/../parse2junit.py" | is_file'
 
-    - name: Results directory exists
+    - name: artifacts directory exists
       file:
-          path: "{{ result_dest_basedir }}"
+          path: "{{ artifacts }}"
           state: directory
-      delegate_to: localhost
 
-    - name: destination file paths are buffered for overwrite-checking and jUnit conversion
-      set_fact:
-          _dstfnbuff: >
-              {{ _dstfnbuff |
-                 union( [result_dest_basedir ~ "/" ~ inventory_hostname ~ "/" ~ item | basename] ) }}
-      with_items: '{{ _result_filepaths }}'
+    - name: Extra artifacts are collected, except missing or with clashing filenames
+      command: 'cp --no-clobber --verbose "{{ item }}" "{{ artifacts }}/"'
+      ignore_errors: True
+      with_items: '{{ extra_artifact_filepaths }}'
 
-    - name: Overwriting existing results assumed very very bad
-      fail:
-          msg: "Cowardly refusing to overwrite {{ item }}"
-      when: item | exists
-      delegate_to: localhost
-      with_items: '{{ _dstfnbuff }}'
+    - name: Generated artifacts directory exists
+      file:
+          path: "{{ artifacts }}/generated"
+          state: directory
 
-    # fetch module doesn't support directories
-    - name: Retrieve results from all hosts
+    - name: Generated artifacts are produced
+      shell: '{{ item.value }} || true &> {{ item.key }}'
+      args:
+        chdir: "{{ artifacts }}/generated"
+        creates: "{{ artifacts }}/generated/{{ item.key }}"
+      ignore_errors: True
+      with_dict: "{{ generated_artifacts }}"
+
+    - name: Subject produces a single canonical jUnit file by combining parsed_artifacts
+      script: '{{ playbook_dir }}/../parse2junit.py {{ parsed_artifacts | join(" ") }} "{{ canonical_junit }}"'
+      args:
+        chdir: "{{ artifacts }}"
+
+
+- hosts: '{{ control_host | default("none") }}'
+  vars_files:
+    - "{{ playbook_dir }}/vars.yml"
+  environment: '{{ environment_variables }}'
+  tasks:
+
+    - name: Verify expectations
+      assert:
+        that:
+          # Combined "is defined" and "isn't blank" check
+          - 'artifacts | default("", True) | trim | length'
+          - 'canonical_junit is defined'
+          - 'playbook_dir ~ "/../parse2junit.py" | is_file'
+
+    - name: A subdirectory exists for this subject's artifacts
+      file:
+        path: "{{ collection_dirpath }}"
+        state: directory
+
+    - name: Artifacts are retrieved from subjects
       synchronize:
-          checksum: True  # Don't rely on date/time being in sync
           archive: False  # Don't bother with permissions or times
+          checksum: True  # Don't rely on date/time being in sync
           copy_links: True  # We want files, not links to files
           recursive: True
           mode: pull
-          dest: '{{ result_dest_basedir }}/{{ inventory_hostname }}/'  # must end in /
-          src: '{{ item }}'
-      register: results_fetched
-      with_items: '{{ _result_filepaths }}'
+          dest: '{{ collection_dirpath }}'
+          src: '{{ artifacts }}'
+          rsync_opts: '--ignore-missing-args'
+      delegate_to: '{{ item }}'
+      with_inventory_hostnames:
+        - '{{ subjects | default("all:!localhost") }}'
+
+    - name: The paths of canonical_junit files from all subjects are found
+      find:
+        paths:
+            - '{{ collection_dirpath }}'
+        patterns: "{{ canonical_junit | basename }}"
+        recurse: True
+      register: result
+
+    - name: Found paths are joined together into a single string
+      set_fact:
+        result: '{{ result.files | map(attribute="path") | join(" ") }}'
+
+    - name: The control host produces a top-level junit, combining all subject's canonical_junits
+      script: '{{ playbook_dir }}/../parse2junit.py {{ result }} "./{{ canonical_junit | basename }}"'
+      args:
+        chdir: "{{ collection_dirpath }}"
+      when: result | trim | length
diff --git a/contrib/test/integration/vars.yml b/contrib/test/integration/vars.yml
index fa8665db..0c4ec523 100644
--- a/contrib/test/integration/vars.yml
+++ b/contrib/test/integration/vars.yml
@@ -23,7 +23,39 @@ cri_o_dest_path: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-o"
 
 # For results.yml Paths use rsync 'source' conventions
 artifacts: "/tmp/artifacts"  # Base-directory for collection
-crio_integration_filepath: "{{ artifacts }}/testout.txt"
-crio_node_e2e_filepath: "{{ artifacts }}/junit_01.xml"
-result_dest_basedir: '{{ lookup("env","WORKSPACE") |
-                         default(playbook_dir, True) }}/artifacts'
+
+# List of absolute paths to extra filenames to collect into {{ artifacts }}.
+# Non-existing files and any name-collisions will be skipped.
+extra_artifact_filepaths:
+    - "/go/src/k8s.io/kubernetes/e2e.log"
+    - "/tmp/kubelet.log"
+    - "/tmp/kube-apiserver.log"
+    - "/tmp/kube-controller-manager.log"
+    - "/tmp/kube-proxy.log"
+    - "/tmp/kube-proxy.yaml"
+    - "/tmp/kube-scheduler.log"
+
+# Mapping of generated artifact filenames and their commands.  All
+# are relative to {{ artifacts }}/generated/
+generated_artifacts:
+    installed_packages.log: '$(type -P dnf || type -P yum) list installed'
+    avc_denials.log: 'ausearch -m AVC -m SELINUX_ERR -m USER_AVC'
+    filesystem.info: 'df -h && sudo pvs && sudo vgs && sudo lvs'
+    pid1.journal: 'journalctl _PID=1 --no-pager --all --lines=all'
+    crio.service: 'journalctl --unit crio.service --no-pager --all --lines=all'
+    customcluster.service: 'journalctl --unit customcluster.service --no-pager --all --lines=all'
+    systemd-journald.service: 'journalctl --unit systemd-journald.service --no-pager --all --lines=all'
+
+# Use ``parse2junits.py`` on these artifact files
+# to produce the '{{ canonical_junit }}' file.
+parsed_artifacts:
+    - "./testout.txt"
+    - "./junit_01.xml"
+
+# jUnit artifact file for ``combine_junits.py`` output
+canonical_junit: "./junit_01.xml"
+
+# When subject != localhost, synchronize "{{ artifacts }}" from
+# all subjects into this directory on the control-host.
+collection_dirpath: '{{ lookup("env","WORKSPACE") |
+                        default(playbook_dir, True) }}/artifacts/{{ inventory_hostname }}'
diff --git a/contrib/test/parse2junit.py b/contrib/test/parse2junit.py
new file mode 100755
index 00000000..6ec35dcb
--- /dev/null
+++ b/contrib/test/parse2junit.py
@@ -0,0 +1,313 @@
+#!/usr/bin/env python2
+
+# encoding: utf-8
+
+# N/B: Assumes script was called from cri-o repository on the test subject,
+#      with a remote name of 'origin.  It's executing under the results.py
+#      playbook, which in turn was executed by venv-ansible-playbook.sh
+#      i.e. everything in requirements.txt is already available
+#
+# Also Requires:
+#    python 2.7+
+#    git
+
+import os
+import sys
+import argparse
+import re
+import contextlib
+import uuid
+from socket import gethostname
+import subprocess
+from tempfile import NamedTemporaryFile
+# Ref: https://github.com/gastlygem/junitparser
+import junitparser
+
+# Parser function suffixes and regex patterns of supported input filenames
+TEST_TYPE_FILE_RE = dict(integration=re.compile(r'testout\.txt'),
+                         e2e=re.compile(r'junit_\d+.xml'))
+INTEGRATION_TEST_COUNT_RE = re.compile(r'^(?P<start>\d+)\.\.(?P<end>\d+)')
+INTEGRATION_SKIP_RE = re.compile(r'^(?P<stat>ok|not ok) (?P<tno>\d+) # skip'
+                                 r' (?P<sreason>\(.+\)) (?P<desc>.+)')
+INTEGRATION_RESULT_RE = re.compile(r'^(?P<stat>ok|not ok) (?P<tno>\d+) (?P<desc>.+)')
+
+
+def d(msg):
+    if msg:
+        try:
+            sys.stderr.write('{}\n'.format(msg))
+            sys.stderr.flush()
+        except IOError:
+            pass
+
+
+@contextlib.contextmanager
+def if_match(line, regex):
+    # __enter__
+    match = regex.search(line)
+    if match:
+        yield match
+    else:
+        yield None
+    # __exit__
+    pass  # Do nothing
+
+
+def if_case_add(suite, line_parser, *parser_args, **parser_dargs):
+    case = line_parser(*parser_args, **parser_dargs)
+    if case:
+        suite.add_testcase(case)
+
+
+def parse_integration_line(line, classname):
+    name_fmt = "[CRI-O] [integration] #{} {}"
+    with if_match(line, INTEGRATION_SKIP_RE) as match:
+        if match:
+            name = name_fmt.format(match.group('tno'), match.group('desc'))
+            case = junitparser.TestCase(name)
+            case.classname = classname
+            case.result = junitparser.Skipped(message=match.group('sreason'))
+            case.system_err = match.group('stat')
+            return case
+    with if_match(line, INTEGRATION_RESULT_RE) as match:
+        if match:
+            name = name_fmt.format(match.group('tno'), match.group('desc'))
+            case = junitparser.TestCase(name)
+            case.classname = classname
+            case.system_err = match.group('stat')
+            if match.group('stat') == 'not ok':
+                # Can't think of anything better to put here
+                case.result = junitparser.Failed('not ok')
+            elif not match.group('stat') == 'ok':
+                case.result = junitparser.Error(match.group('stat'))
+            return case
+    return None
+
+
+# N/B: name suffix corresponds to key in TEST_TYPE_FILE_RE
+def parse_integration(input_file_path, hostname):
+    suite = junitparser.TestSuite('CRI-O Integration suite')
+    suite.hostname = hostname
+    suite_stdout = []
+    classname = 'CRI-O integration suite'
+    n_tests = -1  # No tests ran
+    d("    Processing integration results for {}".format(suite.hostname))
+    with open(input_file_path) as testout_txt:
+        for line in testout_txt:
+            line = line.strip()
+            suite_stdout.append(line)  # Basically a copy of the file
+            # n_tests must come first
+            with if_match(line, INTEGRATION_TEST_COUNT_RE) as match:
+                if match:
+                    n_tests = int(match.group('end')) - int(match.group('start')) + 1
+                    d("      Collecting results from {} tests".format(n_tests))
+                    break
+        if n_tests > 0:
+            for line in testout_txt:
+                line = line.strip()
+                suite_stdout.append(line)
+                if_case_add(suite, parse_integration_line,
+                            line=line, classname=classname)
+        else:
+            d("      Uh oh, no results found, skipping.")
+            return None
+    # TODO: No date/time recorded in file
+    #stat = os.stat(input_file_path)
+    #test_start = stat.st_mtime
+    #test_end = stat.st_atime
+    #duration = test_end - test_start
+    suite.time = 0
+    suite.add_property('stdout', '\n'.join(suite_stdout))
+
+    d("    Parsed {} integration test cases".format(len(suite)))
+    return suite
+
+
+def flatten_testsuites(testsuites):
+    # The jUnit format allows nesting testsuites, squash into a list for simplicity
+    if isinstance(testsuites, junitparser.TestSuite):
+        testsuite = testsuites  # for clarity
+        return [testsuite]
+    result = []
+    for testsuite in testsuites:
+        if isinstance(testsuite, junitparser.TestSuite):
+            result.append(testsuite)
+        elif isinstance(testsuite, junitparser.JUnitXml):
+            nested_suites = flatten_testsuites(testsuite)
+            if nested_suites:
+                result += nested_suites
+    return result
+
+
+def find_k8s_e2e_suite(testsuites):
+    testsuites = flatten_testsuites(testsuites)
+    for testsuite in testsuites:
+        if testsuite.name and 'Kubernetes e2e' in testsuite.name:
+            return testsuite
+        # Name could be None or wrong, check classnames of all tests
+        classnames = ['Kubernetes e2e' in x.classname.strip() for x in testsuite]
+        if all(classnames):
+            return testsuite
+    return None
+
+
+# N/B: name suffix corresponds to key in TEST_TYPE_FILE_RE
+def parse_e2e(input_file_path, hostname):
+    # Load junit_xx.xml file, update contents with more identifying info.
+    try:
+        testsuites = junitparser.JUnitXml.fromfile(input_file_path)
+        suite = find_k8s_e2e_suite(testsuites)
+    except junitparser.JUnitXmlError, xcept:
+        d("    Error parsing {}, skipping it.: {}".format(input_file_path, xcept))
+        return None
+    if not suite:
+        d("    Failed to find any e2e results in {}".format(input_file_path))
+        return None
+    if not suite.hostname:
+        suite.hostname = hostname
+    if not suite.name:
+        suite.name = 'Kubernetes e2e suite'
+    d("    Processing e2e results for {}".format(suite.hostname))
+    for testcase in suite:
+        if not testcase.classname:
+            d("    Adding missing classname to case {}".format(testcase.name))
+            testcase.classname = "Kubernetes e2e suite"
+    d("    Parsed {} e2e test cases".format(len(suite)))
+    if not suite.time:
+        stat = os.stat(input_file_path)
+        test_start = stat.st_ctime
+        test_end = stat.st_mtime
+        duration = test_end - test_start
+        if duration:
+            suite.time = duration
+    return testsuites  # Retain original structure
+
+def parse_test_output(ifps, results_name, hostname):
+    time_total = 0
+    testsuites = junitparser.JUnitXml(results_name)
+    # Cheat, lookup parser function name suffix from global namespace
+    _globals = globals()
+    for input_file_path in ifps:
+        if not os.path.isfile(input_file_path):
+            d(" The file {} doesn't appear to exist, skipping it.".format(input_file_path))
+            continue
+        parser = None
+        for tname, regex in TEST_TYPE_FILE_RE.items():
+            if regex.search(input_file_path):
+                parser = _globals.get('parse_{}'.format(tname))
+                break
+        else:
+            d(" Could not find parser to handle input"
+              " file {}, skipping.".format(input_file_path))
+            continue
+
+        d("  Parsing {} using {}".format(input_file_path, parser))
+        for parsed_testsuite in flatten_testsuites(parser(input_file_path, hostname)):
+            d("  Adding {} suite for {}".format(parsed_testsuite.name, parsed_testsuite.hostname))
+            testsuites.add_testsuite(parsed_testsuite)
+            if parsed_testsuite.time:
+                time_total += parsed_testsuite.time
+    testsuites.time = time_total
+    return testsuites
+
+def make_host_name():
+    subject = '{}'.format(gethostname())
+    # Origin-CI doesn't use very distinguishable hostnames :(
+    if 'openshiftdevel' in subject or 'ip-' in subject:
+        try:
+            with open('/etc/machine-id') as machineid:
+                subject = 'machine-id-{}'.format(machineid.read().strip())
+        except IOError:  # Worst-case, but we gotta pick sumpfin
+            subject = 'uuid-{}'.format(uuid.uuid4())
+    return subject
+
+def make_results_name(argv):
+    script_dir = os.path.dirname(argv[0])
+    spco = lambda cmd: subprocess.check_output(cmd.split(' '),
+                                               stderr=subprocess.STDOUT,
+                                               close_fds=True,
+                                               cwd=script_dir,
+                                               universal_newlines=True)
+    pr_no = None
+    head_id = None
+    try:
+        head_id = spco('git rev-parse HEAD')
+        for line in spco('git ls-remote origin refs/pull/[0-9]*/head').strip().splitlines():
+            cid, ref = line.strip().split(None, 1)
+            if head_id in cid:
+                pr_no = ref.strip().split('/')[2]
+                break
+    except subprocess.CalledProcessError:
+        pass
+
+    if pr_no:
+        return "CRI-O Pull Request {}".format(pr_no)
+    elif head_id:
+        return "CRI-O Commit {}".format(head_id[:8])
+    else:  # Worst-case, but we gotta pick sumpfin
+        return "CRI-O Run ID {}".format(uuid.uuid4())
+
+
+def main(argv):
+    reload(sys)
+    sys.setdefaultencoding('utf8')
+    parser = argparse.ArgumentParser(epilog='Note: The parent directory of input files is'
+                                            'assumed to be the test suite name')
+    parser.add_argument('-f', '--fqdn',
+                        help="Alternative hostname to add to results if none present",
+                        default=make_host_name())
+    parser.add_argument('-b', '--backup', action="store_true",
+                        help="If output file name matches any input file, backup with"
+                             " 'original_' prefix",
+                        default=False)
+    parser.add_argument('ifps', nargs='+',
+                        help='Input file paths to test output from {}.'
+                             ''.format(TEST_TYPE_FILE_RE.keys()))
+    parser.add_argument('ofp', nargs=1,
+                        default='-',
+                        help='Output file path for jUnit XML, or "-" for stdout')
+    options = parser.parse_args(argv[1:])
+    ofp = options.ofp[0]  # nargs==1 still puts it into a list
+    results_name = make_results_name(argv)
+
+    d("Parsing {} to {}".format(options.ifps, ofp))
+    d("Using results name: {} and hostname {}".format(results_name, options.fqdn))
+    # Parse all results
+    new_testsuites = parse_test_output(options.ifps, results_name, options.fqdn)
+
+    if not len(new_testsuites):
+        d("Uh oh, doesn't look like anything was processed.  Bailing out")
+        return None
+
+    d("Parsed {} suites".format(len(new_testsuites)))
+
+    # etree can't handle files w/o filenames :(
+    tmp = NamedTemporaryFile(suffix='.tmp', prefix=results_name, bufsize=1)
+    new_testsuites.write(tmp.name)
+    tmp.seek(0)
+    del new_testsuites  # close up any open files
+    if ofp == '-':
+        sys.stdout.write('\n{}\n'.format(tmp.read()))
+    else:
+        for ifp in options.ifps:
+            if not os.path.isfile(ofp):
+                break
+            if os.path.samefile(ifp, ofp):
+                if not options.backup:
+                    d("Warning {} will be will be combined with other input files."
+                      "".format(ofp))
+                    break
+                dirname = os.path.dirname(ofp)
+                basename = os.path.basename(ofp)
+                origname = 'original_{}'.format(basename)
+                os.rename(ofp, os.path.join(dirname, origname))
+                break
+        with open(ofp, 'w', 1) as output_file:
+            output_file.truncate(0)
+            output_file.flush()
+            d("Writing {}".format(ofp))
+            output_file.write(tmp.read())
+
+
+if __name__ == '__main__':
+    main(sys.argv)
diff --git a/contrib/test/requirements.txt b/contrib/test/requirements.txt
index 4dc4531b..1a6ebf4e 100644
--- a/contrib/test/requirements.txt
+++ b/contrib/test/requirements.txt
@@ -52,3 +52,7 @@ virtualenv==15.1.0 --hash=sha256:39d88b533b422825d644087a21e78c45cf5af0ef7a99a1f
                    --hash=sha256:02f8102c2436bb03b3ee6dede1919d1dac8a427541652e5ec95171ec8adbc93a
 
 pip==9.0.1 --hash=sha256:690b762c0a8460c303c089d5d0be034fb15a5ea2b75bdf565f40421f542fefb0
+
+future==0.16.0 --hash=sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb
+
+junitparser==1.0.0 --hash=sha256:5b0f0ffeef3548878b5ae2cac40b5b128ae18337e2a260a8265f5519b52c907c
diff --git a/contrib/test/venv-ansible-playbook.sh b/contrib/test/venv-ansible-playbook.sh
index 58704215..993873ef 100755
--- a/contrib/test/venv-ansible-playbook.sh
+++ b/contrib/test/venv-ansible-playbook.sh
@@ -13,8 +13,9 @@
 # All errors are fatal
 set -e
 
-SCRIPT_PATH=`realpath $(dirname $0)`
-REQUIREMENTS="$SCRIPT_PATH/requirements.txt"
+export SCRIPT_PATH=`realpath $(dirname $0)`
+export REQUIREMENTS="$SCRIPT_PATH/requirements.txt"
+export ANSIBLE_CONFIG="$SCRIPT_PATH/integration/ansible.cfg"
 
 echo
 
@@ -47,7 +48,8 @@ else
 fi
 
 # Create a directory to contain logs and test artifacts
-export ARTIFACTS=$(mkdir -pv $WORKSPACE/artifacts | tail -1 | cut -d \' -f 2)
+[ -n "$ARTIFACTS" ] || export ARTIFACTS="$WORKSPACE/artifacts"
+[ -d "$ARTIFACTS" ] || mkdir -pv "$ARTIFACTS"
 [ -d "$ARTIFACTS" ] || exit 3
 
 # All command failures from now on are fatal