From 9e273538a744ac244dc767f918547dd550444c55 Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Wed, 4 Oct 2017 10:37:37 -0400
Subject: [PATCH 1/4] Support testing against remote subjects.

It's a severe anti-pattern for a playbook to assume execution always
on a specific host.  The normal/expected pattern is to execute from a
"control host", against an inventory of (possibly-remote) subjects.

This doesn't preclude the inventory from only ever containing
'localhost', it simply means the plays and tasks should not assume
the inventory contents.

This concept is one of the central design-pillars of Ansible's,
and tantamount to it's usefulness and flexibility.  However, in
practice (and by ``integration/readme.md``), plays
specify ``- hosts: all`` but assume inventory_hostname == 'localhost'
(always).

Fix both the playbooks and ``readme.md`` to remove this anti-pattern,
while also allowing the control-host to be the subject-host as needed.
This is accomplished by ensuring low-level Ansible dependencies are
always installed, and writing tasks for steps previously performed
externally (in the CI/automation machinery).

Also update ``readme.md`` to recommend execution occurs through
the ``venv-ansible-playbook.sh`` wrapper to ensure consistent, stable,
version-locked execution dependencies on the control-host.

Remove ``remote_user: root`` from main, since this is better left
to the inventory and command-line.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/README.md       | 54 ++++++++++++++++++------
 contrib/test/integration/build/cri-o.yml | 22 +++++-----
 contrib/test/integration/github.yml      | 27 ++++++++++++
 contrib/test/integration/main.yml        | 40 ++++++++++++++----
 contrib/test/integration/results.yml     |  2 +-
 contrib/test/integration/system.yml      | 15 +++++++
 contrib/test/integration/test.yml        |  2 +-
 7 files changed, 127 insertions(+), 35 deletions(-)
 create mode 100644 contrib/test/integration/github.yml

diff --git a/contrib/test/integration/README.md b/contrib/test/integration/README.md
index f13b8b92..c90a5eac 100644
--- a/contrib/test/integration/README.md
+++ b/contrib/test/integration/README.md
@@ -1,21 +1,49 @@
 # Fedora and RHEL Integration and End-to-End Tests
 
 This directory contains playbooks to set up for and run the integration and
-end-to-end tests for CRI-O on RHEL and Fedora hosts. Two entrypoints exist:
+end-to-end tests for CRI-O on RHEL and Fedora hosts.  The expected entry-point
+is the ``main.yml`` Ansible playbook.
 
- - `main.yml`: sets up the machine and runs tests
- - `results.yml`: gathers test output to `/tmp/artifacts`
+The control-host:
 
-When running `main.yml`, three tags are present:
+ - May be the subject.
+ - Is based on either RHEL/CentOS 6 (or later), or Fedora 24 (or later).
+ - Runs ``main.yml`` from within the cri-o repository already in the
+   desired state for testing.
 
- - `setup`: run all tasks to set up the system for testing
- - `e2e`: build CRI-O from source and run Kubernetes node E2Es
- - `integration`: build CRI-O from source and run the local integration suite
+The subject host(s):
 
-The playbooks assume the following things about your system:
+ - May be the control-host.
+ - May be executing the ``main.yml`` playbook against itself.
+ - If RHEL-like, has the ``server``, ``extras``, and ``EPEL`` repositories available
+   and enabled.
+ - Has remote password-less ssh configured for direct or sudo access to the root user.
 
- - on RHEL, the server and extras repos are configured and certs are present
- - `ansible` is installed and the host is boot-strapped to allow `ansible` to run against it
- - the `$GOPATH` is set and present for all shells (*e.g.* written in `/etc/environment`)
- - CRI-O is checked out to the correct state at `${GOPATH}/src/github.com/kubernetes-incubator/cri-o`
- - the user running the playbook has access to passwordless `sudo`
\ No newline at end of file
+Execution of the ``main.yml`` playbook:
+
+ - Should occur through the ``cri-o/contrib/test/venv-ansible-playbook.sh`` wrapper.
+ - Execution may target localhost, or one or more subjects via standard Ansible
+   inventory arguments.
+ - Should use a combination (including none) of the following tags:
+
+     - ``setup``: Run all tasks to set up the system for testing. Final state must
+                  be self-contained and independent from other tags (i.e. support
+                  stage-caching).
+     - ``integration``: Assumes 'setup' previously completed successfully.
+                        May be executed from cached-state of ``setup``.
+                        Not required to execute conicident with other tags.
+                        Must build CRI-O from source and run the
+                        integration test suite.
+     - ``e2e``: Assumes 'setup' previously completed successfully.  May be executed
+                from cached-state of ``setup``. Not required to execute conicident with
+                other tags.  Must build CRI-O from source and run Kubernetes node
+                E2E tests.
+
+``cri-o/contrib/test/venv-ansible-playbook.sh`` Wrapper:
+
+ - Must accepts all of the valid Ansible command-line options.
+ - Must use version-locked & hashed dependencies as written in ``requirements.txt``.
+ - Must fully sandbox it's own execution environment except for the following
+   required packages (or equivalent): ``python2-virtualenv gcc openssl-devel
+   redhat-rpm-config libffi-devel python-devel libselinux-python rsync
+   yum-utils python3-pycurl python-simplejson``.
diff --git a/contrib/test/integration/build/cri-o.yml b/contrib/test/integration/build/cri-o.yml
index fa025035..29172866 100644
--- a/contrib/test/integration/build/cri-o.yml
+++ b/contrib/test/integration/build/cri-o.yml
@@ -1,42 +1,42 @@
 ---
 
-- name: stat the expected cri-o directory
+- name: stat the expected cri-o directory and Makefile exists
   stat:
-    path: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    path: "{{ cri_o_dest_path }}/Makefile"
   register: dir_stat
 
-- name: expect cri-o to be cloned already
+- name: Verify cri-o Makefile exists in expected location
   fail:
-    msg: "Expected cri-o to be cloned at {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o but it wasn't!"
-  when: not dir_stat.stat.exists
+    msg: "Expected cri-o to be cloned at {{ cri_o_dest_path }} but it wasn't!"
+  when: not dir_stat.stat.exists or not dir_stat.stat.isreg
 
 - name: install cri-o tools
   make:
     target: install.tools
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
 
 - name: build cri-o
   make:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
 
 - name: install cri-o
   make:
     target: install
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
 
 - name: install cri-o systemd files
   make:
     target: install.systemd
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
 
 - name: install cri-o config
   make:
     target: install.config
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
 
 - name: install configs
   copy:
-    src: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/{{ item.src }}"
+    src: "{{ cri_o_dest_path }}/{{ item.src }}"
     dest: "{{ item.dest }}"
     remote_src: yes
   with_items:
diff --git a/contrib/test/integration/github.yml b/contrib/test/integration/github.yml
new file mode 100644
index 00000000..16aef9f4
--- /dev/null
+++ b/contrib/test/integration/github.yml
@@ -0,0 +1,27 @@
+---
+
+
+- name: Verify expectations
+  assert:
+    that:
+      - 'cri_o_dest_path is defined'
+      - 'cri_o_src_path is defined'
+
+- name: The cri-o repository directory exists
+  file:
+    path: "{{ cri_o_dest_path }}"
+    state: directory
+    mode: 0777
+
+- name: Synchronize cri-o from control-host to remote subject
+  synchronize:
+    archive: False
+    checksum: True
+    delete: True
+    dest: "{{ cri_o_dest_path }}/"
+    links: True
+    recursive: True
+    src: "{{ cri_o_src_path }}/"
+    times: True
+  # This task is excessively noisy, logging every change to every file :(
+  no_log: True
diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml
index ce4a206f..343f03ad 100644
--- a/contrib/test/integration/main.yml
+++ b/contrib/test/integration/main.yml
@@ -1,5 +1,29 @@
-- hosts: all
-  remote_user: root
+---
+
+- hosts: '{{ subjects | default("localhost") }}'
+  gather_facts: False  # Requires low-level ansible-dependencies
+  tags:
+    - always
+  tasks:
+    - name: Ensure low-level ansible-dependencies are installed
+      raw: $(type -P dnf || type -P yum) install -y python2 python2-dnf libselinux-python git rsync
+
+    - name: Gather only networking facts for speed
+      setup:
+        gather_subset: network
+
+
+- hosts: '{{ subjects | default("none") }}'
+  vars_files:
+    - "{{ playbook_dir }}/vars.yml"
+  tags:
+    - setup
+  tasks:
+    - name: CRI-O source is available on every subject
+      include: github.yml
+
+
+- hosts: '{{ subjects | default("localhost") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
   tags:
@@ -26,19 +50,18 @@
     - name: clone build and install networking plugins
       include: "build/plugins.yml"
 
-- hosts: all
-  remote_user: root
+
+- hosts: '{{ subjects | default("localhost") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
   tags:
     - integration
     - e2e
   tasks:
-    - name: clone build and install cri-o
+    - name: Build and install cri-o
       include: "build/cri-o.yml"
 
-- hosts: all
-  remote_user: root
+- hosts: '{{ subjects | default("localhost") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
   tags:
@@ -47,8 +70,7 @@
     - name: run cri-o integration tests
       include: test.yml
 
-- hosts: all
-  remote_user: root
+- hosts: '{{ subjects | default("localhost") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
   tags:
diff --git a/contrib/test/integration/results.yml b/contrib/test/integration/results.yml
index c9a96abb..96b0f0b4 100644
--- a/contrib/test/integration/results.yml
+++ b/contrib/test/integration/results.yml
@@ -1,7 +1,7 @@
 ---
 # vim-syntax: ansible
 
-- hosts: '{{ hosts | default("all") }}'
+- hosts: '{{ subjects | default("all") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
   vars:
diff --git a/contrib/test/integration/system.yml b/contrib/test/integration/system.yml
index d07ae0c8..fd51368a 100644
--- a/contrib/test/integration/system.yml
+++ b/contrib/test/integration/system.yml
@@ -63,6 +63,7 @@
     state: present
   with_items:
    - btrfs-progs-devel
+   - python2-virtualenv
   when: ansible_distribution in ['Fedora']
 
 - name: Update all packages
@@ -115,3 +116,17 @@
 - name: Update the kernel cmdline to include quota support
   command: grubby --update-kernel=ALL --args="rootflags=pquota"
   when: ansible_distribution in ['RedHat', 'CentOS']
+
+- name: Configure the GOPATH environment variable for all users
+  blockinfile:
+    path: /etc/environment
+    block: "GOPATH={{ go_path }}"
+    create: True
+    follow: True
+
+- name: Reset the ansible connection to incorporate /etc/environment changes
+  meta: reset_connection
+
+- name: Refresh facts to incorporate /etc/environment changes
+  setup:
+    gather_subset: network
diff --git a/contrib/test/integration/test.yml b/contrib/test/integration/test.yml
index 593e8a1c..34fc917e 100644
--- a/contrib/test/integration/test.yml
+++ b/contrib/test/integration/test.yml
@@ -15,7 +15,7 @@
     extra_storage_opts: " --storage-opt overlay.override_kernel_check=1"
   when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
 
-- name: ensure directory exists for e2e reports
+- name: ensure directory exists for integration results
   file:
     path: "{{ artifacts }}"
     state: directory

From 0e752a64230de230b310ad92a47e4e6cc37607a3 Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Mon, 16 Oct 2017 14:22:13 -0400
Subject: [PATCH 2/4] Avoid relying on subject environment-vars

There are so many ways/places they can change values on the host from
one moment to the next.  Yet as written, the value of ansible_env.GOPATH
is really only fixed at "fact gathering" time.  In other words, the
environment variable can change (even during a play), but won't be
noticed until possibly much later.  This can cause very strange things
to happen which aren't easy to debug.

Fix this by using established facts (variables), and continuously
establishing them as environment variables.  This way, if/when a task
fails, the value of the environment will be present w/in the failure
message instead of obscrured by the shell.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/build/bats.yml       |  4 ++--
 contrib/test/integration/build/cri-tools.yml  |  4 ++--
 contrib/test/integration/build/kubernetes.yml | 24 ++++---------------
 contrib/test/integration/build/plugins.yml    | 12 +++++-----
 contrib/test/integration/build/runc.yml       |  6 ++---
 contrib/test/integration/e2e.yml              |  7 ++----
 contrib/test/integration/golang.yml           | 24 +++++--------------
 contrib/test/integration/main.yml             |  6 +++++
 contrib/test/integration/system.yml           | 14 -----------
 contrib/test/integration/test.yml             |  6 ++---
 contrib/test/integration/vars.yml             | 18 ++++++++++++++
 11 files changed, 53 insertions(+), 72 deletions(-)

diff --git a/contrib/test/integration/build/bats.yml b/contrib/test/integration/build/bats.yml
index d4ea19c6..ec2900b0 100644
--- a/contrib/test/integration/build/bats.yml
+++ b/contrib/test/integration/build/bats.yml
@@ -3,12 +3,12 @@
 - name: clone bats source repo
   git:
     repo: "https://github.com/sstephenson/bats.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats"
+    dest: "{{ go_path }}/src/github.com/sstephenson/bats"
 
 - name: install bats
   command: "./install.sh /usr/local"
   args:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats"
+    chdir: "{{ go_path }}/src/github.com/sstephenson/bats"
 
 - name: link bats
   file:
diff --git a/contrib/test/integration/build/cri-tools.yml b/contrib/test/integration/build/cri-tools.yml
index 9a117f3c..fad1dbfe 100644
--- a/contrib/test/integration/build/cri-tools.yml
+++ b/contrib/test/integration/build/cri-tools.yml
@@ -3,7 +3,7 @@
 - name: clone cri-tools source repo
   git:
     repo: "https://github.com/kubernetes-incubator/cri-tools.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-tools"
+    dest: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-tools"
     version: "9ff5e8f78a4182ab8d5ba9bcccdda5f338600eab"
 
 - name: install crictl
@@ -11,6 +11,6 @@
 
 - name: link crictl
   file:
-    src: "{{ ansible_env.GOPATH }}/bin/crictl"
+    src: "{{ go_path }}/bin/crictl"
     dest: /usr/bin/crictl
     state: link
diff --git a/contrib/test/integration/build/kubernetes.yml b/contrib/test/integration/build/kubernetes.yml
index f724230c..e8b19c57 100644
--- a/contrib/test/integration/build/kubernetes.yml
+++ b/contrib/test/integration/build/kubernetes.yml
@@ -3,17 +3,17 @@
 - name: clone kubernetes source repo
   git:
     repo: "https://github.com/runcom/kubernetes.git"
-    dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+    dest: "{{ go_path }}/src/k8s.io/kubernetes"
     version: "cri-o-patched-1.8"
 
 - name: install etcd
   command: "hack/install-etcd.sh"
   args:
-    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+    chdir: "{{ go_path }}/src/k8s.io/kubernetes"
 
 - name: build kubernetes
   make:
-    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+    chdir: "{{ go_path }}/src/k8s.io/kubernetes"
 
 - name: Add custom cluster service file for the e2e testing
   copy:
@@ -23,7 +23,7 @@
       After=network-online.target
       Wants=network-online.target
       [Service]
-      WorkingDirectory={{ ansible_env.GOPATH }}/src/k8s.io/kubernetes
+      WorkingDirectory={{ go_path }}/src/k8s.io/kubernetes
       ExecStart=/usr/local/bin/createcluster.sh
       User=root
       [Install]
@@ -35,7 +35,7 @@
     content: |
       #!/bin/bash
 
-      export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ ansible_env.GOPATH }}/bin:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/third_party/etcd:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/
+      export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ go_path }}/bin:{{ go_path }}/src/k8s.io/kubernetes/third_party/etcd:{{ go_path }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/
       export CONTAINER_RUNTIME=remote
       export CGROUP_DRIVER=systemd
       export CONTAINER_RUNTIME_ENDPOINT='/var/run/crio.sock --runtime-request-timeout=5m'
@@ -47,17 +47,3 @@
       export KUBE_ENABLE_CLUSTER_DNS=true
       ./hack/local-up-cluster.sh
     mode: "u=rwx,g=rwx,o=x"
-
-- name: Set kubernetes_provider to be local
-  lineinfile:
-    dest: /etc/environment
-    line: 'KUBERNETES_PROVIDER=local'
-    regexp: 'KUBERNETES_PROVIDER='
-    state: present
-
-- name: Set KUBECONFIG
-  lineinfile:
-    dest: /etc/environment
-    line: 'KUBECONFIG=/var/run/kubernetes/admin.kubeconfig'
-    regexp: 'KUBECONFIG='
-    state: present
diff --git a/contrib/test/integration/build/plugins.yml b/contrib/test/integration/build/plugins.yml
index e342a0b9..b344270a 100644
--- a/contrib/test/integration/build/plugins.yml
+++ b/contrib/test/integration/build/plugins.yml
@@ -3,17 +3,17 @@
 - name: clone plugins source repo
   git:
     repo: "https://github.com/containernetworking/plugins.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+    dest: "{{ go_path }}/src/github.com/containernetworking/plugins"
     version: "dcf7368eeab15e2affc6256f0bb1e84dd46a34de"
 
 - name: build plugins
   command: "./build.sh"
   args:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+    chdir: "{{ go_path }}/src/github.com/containernetworking/plugins"
 
 - name: install plugins
   copy:
-    src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/{{ item }}"
+    src: "{{ go_path }}/src/github.com/containernetworking/plugins/bin/{{ item }}"
     dest: "/opt/cni/bin"
     mode: "o=rwx,g=rx,o=rx"
     remote_src: yes
@@ -33,18 +33,18 @@
 - name: clone runcom plugins source repo
   git:
     repo: "https://github.com/runcom/plugins.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+    dest: "{{ go_path }}/src/github.com/containernetworking/plugins"
     version: "custom-bridge"
     force: yes
 
 - name: build plugins
   command: "./build.sh"
   args:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
+    chdir: "{{ go_path }}/src/github.com/containernetworking/plugins"
 
 - name: install custom bridge
   copy:
-    src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/bridge"
+    src: "{{ go_path }}/src/github.com/containernetworking/plugins/bin/bridge"
     dest: "/opt/cni/bin/bridge-custom"
     mode: "o=rwx,g=rx,o=rx"
     remote_src: yes
diff --git a/contrib/test/integration/build/runc.yml b/contrib/test/integration/build/runc.yml
index 7bb0491d..1dd04f03 100644
--- a/contrib/test/integration/build/runc.yml
+++ b/contrib/test/integration/build/runc.yml
@@ -3,18 +3,18 @@
 - name: clone runc source repo
   git:
     repo: "https://github.com/opencontainers/runc.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
+    dest: "{{ go_path }}/src/github.com/opencontainers/runc"
     version: "84a082bfef6f932de921437815355186db37aeb1"
 
 - name: build runc
   make:
     params: BUILDTAGS="seccomp selinux"
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
+    chdir: "{{ go_path }}/src/github.com/opencontainers/runc"
 
 - name: install runc
   make:
     target: "install"
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
+    chdir: "{{ go_path }}/src/github.com/opencontainers/runc"
 
 - name: link runc
   file:
diff --git a/contrib/test/integration/e2e.yml b/contrib/test/integration/e2e.yml
index 41f92757..c0df99ef 100644
--- a/contrib/test/integration/e2e.yml
+++ b/contrib/test/integration/e2e.yml
@@ -29,7 +29,7 @@
     daemon_reload: yes
 
 - name: wait for the cluster to be running
-  command: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default"
+  command: "{{ go_path }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default"
   register: kube_poll
   until: kube_poll | succeeded
   retries: 100
@@ -51,10 +51,7 @@
             &> {{ artifacts }}/e2e.log
   # Fix vim syntax hilighting: "
 
-- name: disable SELinux
-  command: setenforce 0
-
 - name: run e2e tests
   shell: "{{ e2e_shell_cmd | regex_replace('\\s+', ' ') }}"
   args:
-    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+    chdir: "{{ go_path }}/src/k8s.io/kubernetes"
diff --git a/contrib/test/integration/golang.yml b/contrib/test/integration/golang.yml
index 63e55697..39be5aa1 100644
--- a/contrib/test/integration/golang.yml
+++ b/contrib/test/integration/golang.yml
@@ -16,28 +16,16 @@
     - gofmt
     - godoc
 
-- name: ensure user profile exists
-  file:
-    path: "{{ ansible_user_dir }}/.profile"
-    state: touch
-
-- name: set up PATH for Go toolchain and built binaries
-  lineinfile:
-    dest: "{{ ansible_user_dir }}/.profile"
-    line: 'PATH={{ ansible_env.PATH }}:{{ ansible_env.GOPATH }}/bin:/usr/local/go/bin'
-    regexp: '^PATH='
-    state: present
-
 - name: set up directories
   file:
-    path: "{{ item }}"
+    path: "{{ go_path }}/{{ item }}"
     state: directory
   with_items:
-    - "{{ ansible_env.GOPATH }}/src/github.com/containernetworking"
-    - "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator"
-    - "{{ ansible_env.GOPATH }}/src/github.com/k8s.io"
-    - "{{ ansible_env.GOPATH }}/src/github.com/sstephenson"
-    - "{{ ansible_env.GOPATH }}/src/github.com/opencontainers"
+    - "src/github.com/containernetworking"
+    - "src/github.com/kubernetes-incubator"
+    - "src/github.com/k8s.io"
+    - "src/github.com/sstephenson"
+    - "src/github.com/opencontainers"
 
 - name: install Go tools and dependencies
   shell: /usr/bin/go get -u "github.com/{{ item }}"
diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml
index 343f03ad..2f511f3e 100644
--- a/contrib/test/integration/main.yml
+++ b/contrib/test/integration/main.yml
@@ -4,6 +4,8 @@
   gather_facts: False  # Requires low-level ansible-dependencies
   tags:
     - always
+  tags:
+    - setup
   tasks:
     - name: Ensure low-level ansible-dependencies are installed
       raw: $(type -P dnf || type -P yum) install -y python2 python2-dnf libselinux-python git rsync
@@ -26,6 +28,7 @@
 - hosts: '{{ subjects | default("localhost") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
+  environment: '{{ environment_variables }}'
   tags:
     - setup
   tasks:
@@ -54,6 +57,7 @@
 - hosts: '{{ subjects | default("localhost") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
+  environment: '{{ environment_variables }}'
   tags:
     - integration
     - e2e
@@ -64,6 +68,7 @@
 - hosts: '{{ subjects | default("localhost") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
+  environment: '{{ environment_variables }}'
   tags:
     - integration
   tasks:
@@ -73,6 +78,7 @@
 - hosts: '{{ subjects | default("localhost") }}'
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
+  environment: '{{ environment_variables }}'
   tags:
     - e2e
   tasks:
diff --git a/contrib/test/integration/system.yml b/contrib/test/integration/system.yml
index fd51368a..c84e0f59 100644
--- a/contrib/test/integration/system.yml
+++ b/contrib/test/integration/system.yml
@@ -116,17 +116,3 @@
 - name: Update the kernel cmdline to include quota support
   command: grubby --update-kernel=ALL --args="rootflags=pquota"
   when: ansible_distribution in ['RedHat', 'CentOS']
-
-- name: Configure the GOPATH environment variable for all users
-  blockinfile:
-    path: /etc/environment
-    block: "GOPATH={{ go_path }}"
-    create: True
-    follow: True
-
-- name: Reset the ansible connection to incorporate /etc/environment changes
-  meta: reset_connection
-
-- name: Refresh facts to incorporate /etc/environment changes
-  setup:
-    gather_subset: network
diff --git a/contrib/test/integration/test.yml b/contrib/test/integration/test.yml
index 34fc917e..0e8ec8a4 100644
--- a/contrib/test/integration/test.yml
+++ b/contrib/test/integration/test.yml
@@ -5,7 +5,7 @@
 
 - name: Make testing output verbose so it can be converted to xunit
   lineinfile:
-    dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/hack/make-rules/test.sh"
+    dest: "{{ go_path }}/src/k8s.io/kubernetes/hack/make-rules/test.sh"
     line: ' go test -v "${goflags[@]:+${goflags[@]}}" \'
     regexp: ' go test \"\$'
     state: present
@@ -21,8 +21,8 @@
     state: directory
 
 - name: run integration tests
-  shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTIONS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration >& {{ artifacts }}/testout.txt"
+  shell: "make localintegration >& {{ artifacts }}/testout.txt"
   args:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
+    chdir: "{{ cri_o_dest_path }}"
   async: 5400
   poll: 30
diff --git a/contrib/test/integration/vars.yml b/contrib/test/integration/vars.yml
index f1e5e2f7..3eebd62f 100644
--- a/contrib/test/integration/vars.yml
+++ b/contrib/test/integration/vars.yml
@@ -1,5 +1,23 @@
 ---
 
+# Path to encode into /etc/environment on all hosts
+go_path: "/go"
+
+# Absolute path on control-host where the cri-o source exists
+cri_o_src_path: "{{ playbook_dir }}/../../../"
+
+# Absolute path on subject-host where cri-o source is expected
+cri_o_dest_path: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-o"
+
+# Global environment variables to use for tasks on subject-hosts
+environment_variables:
+    PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:{{ go_path }}/bin:/usr/local/go/bin"
+    GOPATH: "{{ go_path }}"
+    KUBERNETES_PROVIDER: "local"
+    KUBECONFIG: "/var/run/kubernetes/admin.kubeconfig"
+    CGROUP_MANAGER: "cgroupfs"
+    STORAGE_OPTS: '--storage-driver=overlay{{ extra_storage_opts | default("") }}'
+
 # For results.yml Paths use rsync 'source' conventions
 artifacts: "/tmp/artifacts"  # Base-directory for collection
 crio_integration_filepath: "{{ artifacts }}/testout.txt"

From 5f80135d0ed8953d75fd15fd08eee149d81549f5 Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Wed, 18 Oct 2017 10:09:18 -0400
Subject: [PATCH 3/4] Consolidate plays/tasks/tags

Simplify use of play-level tagging on three separate plays (with one
task).  Instead, make them all the same play, and apply the tags at
the task level instead.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/main.yml | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)

diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml
index 2f511f3e..4f709b31 100644
--- a/contrib/test/integration/main.yml
+++ b/contrib/test/integration/main.yml
@@ -58,29 +58,18 @@
   vars_files:
     - "{{ playbook_dir }}/vars.yml"
   environment: '{{ environment_variables }}'
-  tags:
-    - integration
-    - e2e
   tasks:
     - name: Build and install cri-o
       include: "build/cri-o.yml"
+      tags:
+        - always
 
-- hosts: '{{ subjects | default("localhost") }}'
-  vars_files:
-    - "{{ playbook_dir }}/vars.yml"
-  environment: '{{ environment_variables }}'
-  tags:
-    - integration
-  tasks:
     - name: run cri-o integration tests
       include: test.yml
+      tags:
+        - integration
 
-- hosts: '{{ subjects | default("localhost") }}'
-  vars_files:
-    - "{{ playbook_dir }}/vars.yml"
-  environment: '{{ environment_variables }}'
-  tags:
-    - e2e
-  tasks:
     - name: run k8s e2e tests
       include: e2e.yml
+      tags:
+        - e2e

From 393a34a35a94f35662cb47b78ee697d36a66978b Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Tue, 31 Oct 2017 09:22:56 -0400
Subject: [PATCH 4/4] Updates based on PR #972 feedback

* When installing low-level dependencies, ignore any errors as
  consequences will be revealed in the subsequent task.
* Rename dir_stat -> crio_stat to better match usage.
* Simplify extra_storage_opts variable usage, consolidate definition.
* Reduce duplication of base-paths in golang.yml directory-setup loop.
* Clarify a few points and add a wrapper example to README.md

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/test/integration/README.md       | 45 +++++++++++++++++++-----
 contrib/test/integration/build/cri-o.yml |  6 ++--
 contrib/test/integration/golang.yml      | 12 +++----
 contrib/test/integration/main.yml        | 30 +++++++++++++---
 contrib/test/integration/test.yml        |  5 ---
 contrib/test/integration/vars.yml        | 11 +-----
 6 files changed, 73 insertions(+), 36 deletions(-)

diff --git a/contrib/test/integration/README.md b/contrib/test/integration/README.md
index c90a5eac..f7220005 100644
--- a/contrib/test/integration/README.md
+++ b/contrib/test/integration/README.md
@@ -4,6 +4,16 @@ This directory contains playbooks to set up for and run the integration and
 end-to-end tests for CRI-O on RHEL and Fedora hosts.  The expected entry-point
 is the ``main.yml`` Ansible playbook.
 
+##Definitions:
+
+    Control-host:  The system from which the ``ansible-playbook`` or
+                   ``venv-ansible-playbook.sh`` command is executed.
+
+    Subject-host(s): The target systems, on which actual playbook tasks are
+                     being carried out.
+
+##Topology:
+
 The control-host:
 
  - May be the subject.
@@ -11,13 +21,17 @@ The control-host:
  - Runs ``main.yml`` from within the cri-o repository already in the
    desired state for testing.
 
-The subject host(s):
+The subject-host(s):
 
  - May be the control-host.
  - May be executing the ``main.yml`` playbook against itself.
  - If RHEL-like, has the ``server``, ``extras``, and ``EPEL`` repositories available
    and enabled.
- - Has remote password-less ssh configured for direct or sudo access to the root user.
+ - Has remote password-less ssh configured for access by the control-host.
+ - When ssh-access is for a regular user, that user has password-less
+   sudo access to root.
+
+##Runtime Requirements:
 
 Execution of the ``main.yml`` playbook:
 
@@ -31,19 +45,34 @@ Execution of the ``main.yml`` playbook:
                   stage-caching).
      - ``integration``: Assumes 'setup' previously completed successfully.
                         May be executed from cached-state of ``setup``.
-                        Not required to execute conicident with other tags.
+                        Not required to execute coincident with other tags.
                         Must build CRI-O from source and run the
                         integration test suite.
      - ``e2e``: Assumes 'setup' previously completed successfully.  May be executed
-                from cached-state of ``setup``. Not required to execute conicident with
+                from cached-state of ``setup``. Not required to execute coincident with
                 other tags.  Must build CRI-O from source and run Kubernetes node
                 E2E tests.
 
 ``cri-o/contrib/test/venv-ansible-playbook.sh`` Wrapper:
 
- - Must accepts all of the valid Ansible command-line options.
- - Must use version-locked & hashed dependencies as written in ``requirements.txt``.
- - Must fully sandbox it's own execution environment except for the following
-   required packages (or equivalent): ``python2-virtualenv gcc openssl-devel
+ - May be executed on the control-host to both hide and version-lock playbook
+   execution dependencies, ansible and otherwise.
+ - Must accept all of the valid Ansible command-line options.
+ - Must sandbox dependencies under a python virtual environment ``.cri-o_venv``
+   with packages as specified in ``requirements.txt``.
+ - Requires the control-host has the following fundamental dependencies installed
+   (or equivalent): ``python2-virtualenv gcc openssl-devel
    redhat-rpm-config libffi-devel python-devel libselinux-python rsync
    yum-utils python3-pycurl python-simplejson``.
+
+For example:
+
+Given a populated '/path/to/inventory' file, a control-host could run:
+
+./venv-ansible-playbook.sh -i /path/to/inventory ./integration/main.yml
+
+-or-
+
+From a subject-host without an inventory:
+
+./venv-ansible-playbook.sh -i localhost, ./integration/main.yml
diff --git a/contrib/test/integration/build/cri-o.yml b/contrib/test/integration/build/cri-o.yml
index 29172866..3c7b2c16 100644
--- a/contrib/test/integration/build/cri-o.yml
+++ b/contrib/test/integration/build/cri-o.yml
@@ -3,12 +3,12 @@
 - name: stat the expected cri-o directory and Makefile exists
   stat:
     path: "{{ cri_o_dest_path }}/Makefile"
-  register: dir_stat
+  register: crio_stat
 
 - name: Verify cri-o Makefile exists in expected location
   fail:
-    msg: "Expected cri-o to be cloned at {{ cri_o_dest_path }} but it wasn't!"
-  when: not dir_stat.stat.exists or not dir_stat.stat.isreg
+    msg: "Expected cri-o to be cloned at {{ cri_o_dest_path }}, but its 'Makefile' seems to be missing."
+  when: not crio_stat.stat.exists or not crio_stat.stat.isreg
 
 - name: install cri-o tools
   make:
diff --git a/contrib/test/integration/golang.yml b/contrib/test/integration/golang.yml
index 39be5aa1..037fe851 100644
--- a/contrib/test/integration/golang.yml
+++ b/contrib/test/integration/golang.yml
@@ -18,14 +18,14 @@
 
 - name: set up directories
   file:
-    path: "{{ go_path }}/{{ item }}"
+    path: "{{ go_path }}/src/github.com/{{ item }}"
     state: directory
   with_items:
-    - "src/github.com/containernetworking"
-    - "src/github.com/kubernetes-incubator"
-    - "src/github.com/k8s.io"
-    - "src/github.com/sstephenson"
-    - "src/github.com/opencontainers"
+    - "containernetworking"
+    - "kubernetes-incubator"
+    - "k8s.io"
+    - "sstephenson"
+    - "opencontainers"
 
 - name: install Go tools and dependencies
   shell: /usr/bin/go get -u "github.com/{{ item }}"
diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml
index 4f709b31..caa64b12 100644
--- a/contrib/test/integration/main.yml
+++ b/contrib/test/integration/main.yml
@@ -4,16 +4,38 @@
   gather_facts: False  # Requires low-level ansible-dependencies
   tags:
     - always
-  tags:
-    - setup
   tasks:
-    - name: Ensure low-level ansible-dependencies are installed
-      raw: $(type -P dnf || type -P yum) install -y python2 python2-dnf libselinux-python git rsync
+    - name: Ansible setup-module dependencies are installed, ignoring errors (setup runs next).
+      raw: $(type -P dnf || type -P yum) install -y python2 python2-dnf libselinux-python
+      ignore_errors: True
 
     - name: Gather only networking facts for speed
       setup:
         gather_subset: network
 
+    - name: Variables from vars.yml are hauled in after setup
+      include_vars: "{{ playbook_dir }}/vars.yml"
+
+    - name: Global environment are defined, but can be overriden on a task-by-task basis.
+      set_fact:
+        extra_storage_opts: >
+            {%- if ansible_distribution in ["RedHat", "CentOS"] -%}
+                "--storage-opt overlay.override_kernel_check=1"
+            {%- else -%}
+                ""
+            {%- endif -%}
+        environment_variables:
+            PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:{{ go_path }}/bin:/usr/local/go/bin"
+            GOPATH: "{{ go_path }}"
+            KUBERNETES_PROVIDER: "local"
+            KUBECONFIG: "/var/run/kubernetes/admin.kubeconfig"
+            CGROUP_MANAGER: "cgroupfs"
+            # > is YAML for replace non-orphan newlines with single spaces
+            STORAGE_OPTS: >
+                --storage-driver=overlay
+                {% if ansible_distribution in ["RedHat", "CentOS"] %}
+                --storage-opt overlay.override_kernel_check=1
+                {% endif %}
 
 - hosts: '{{ subjects | default("none") }}'
   vars_files:
diff --git a/contrib/test/integration/test.yml b/contrib/test/integration/test.yml
index 0e8ec8a4..c097343b 100644
--- a/contrib/test/integration/test.yml
+++ b/contrib/test/integration/test.yml
@@ -10,11 +10,6 @@
     regexp: ' go test \"\$'
     state: present
 
-- name: set extra storage options
-  set_fact:
-    extra_storage_opts: " --storage-opt overlay.override_kernel_check=1"
-  when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
-
 - name: ensure directory exists for integration results
   file:
     path: "{{ artifacts }}"
diff --git a/contrib/test/integration/vars.yml b/contrib/test/integration/vars.yml
index 3eebd62f..c453c007 100644
--- a/contrib/test/integration/vars.yml
+++ b/contrib/test/integration/vars.yml
@@ -1,6 +1,6 @@
 ---
 
-# Path to encode into /etc/environment on all hosts
+# Base directory for all go-related source, build, and install.
 go_path: "/go"
 
 # Absolute path on control-host where the cri-o source exists
@@ -9,15 +9,6 @@ cri_o_src_path: "{{ playbook_dir }}/../../../"
 # Absolute path on subject-host where cri-o source is expected
 cri_o_dest_path: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-o"
 
-# Global environment variables to use for tasks on subject-hosts
-environment_variables:
-    PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:{{ go_path }}/bin:/usr/local/go/bin"
-    GOPATH: "{{ go_path }}"
-    KUBERNETES_PROVIDER: "local"
-    KUBECONFIG: "/var/run/kubernetes/admin.kubeconfig"
-    CGROUP_MANAGER: "cgroupfs"
-    STORAGE_OPTS: '--storage-driver=overlay{{ extra_storage_opts | default("") }}'
-
 # For results.yml Paths use rsync 'source' conventions
 artifacts: "/tmp/artifacts"  # Base-directory for collection
 crio_integration_filepath: "{{ artifacts }}/testout.txt"