Merge pull request #945 from cevich/just_collect_results

Write playbook to collect results from subjects

contrib/test/integration/ansible.cfg

@@ -0,0 +1,359 @@
+# config file for ansible -- http://ansible.com/
+# ==============================================
+
+# nearly all parameters can be overridden in ansible-playbook
+# or with command line flags. ansible will read ANSIBLE_CONFIG,
+# ansible.cfg in the current working directory, .ansible.cfg in
+# the home directory or /etc/ansible/ansible.cfg, whichever it
+# finds first
+
+[defaults]
+
+# some basic default values...
+
+#inventory       = inventory
+#library        = /usr/share/my_modules/
+#remote_tmp     = $HOME/.ansible/tmp
+#local_tmp      = .ansible/tmp
+#forks          = 5
+forks           = 10
+#poll_interval  = 15
+#sudo_user      = root
+#ask_sudo_pass = True
+ask_sudo_pass = False
+#ask_pass      = True
+ask_pass      = False
+#transport      = smart
+#remote_port    = 22
+#module_lang    = C
+#module_set_locale = True
+
+# plays will gather facts by default, which contain information about
+# the remote system.
+#
+# smart - gather by default, but don't regather if already gathered
+# implicit - gather by default, turn off with gather_facts: False
+# explicit - do not gather by default, must say gather_facts: True
+#gathering = implicit
+gathering = smart
+
+# by default retrieve all facts subsets
+# all - gather all subsets
+# network - gather min and network facts
+# hardware - gather hardware facts (longest facts to retrieve)
+# virtual - gather min and virtual facts
+# facter - import facts from facter
+# ohai - import facts from ohai
+# You can combine them using comma (ex: network,virtual)
+# You can negate them using ! (ex: !hardware,!facter,!ohai)
+# A minimal set of facts is always gathered.
+gather_subset = network
+
+# additional paths to search for roles in, colon separated
+# N/B: This depends on how ansible is called
+#roles_path    = $WORKSPACE/kommandir_workspace/roles
+
+# uncomment this to disable SSH key host checking
+#host_key_checking = False
+host_key_checking = False
+
+# change the default callback
+#stdout_callback = skippy
+# enable additional callbacks
+#callback_whitelist = timer, mail
+
+# Determine whether includes in tasks and handlers are "static" by
+# default. As of 2.0, includes are dynamic by default. Setting these
+# values to True will make includes behave more like they did in the
+# 1.x versions.
+task_includes_static = True
+handler_includes_static = True
+
+# change this for alternative sudo implementations
+#sudo_exe = sudo
+
+# What flags to pass to sudo
+# WARNING: leaving out the defaults might create unexpected behaviours
+#sudo_flags = -H -S -n
+
+# SSH timeout
+#timeout = 10
+
+# default user to use for playbooks if user is not specified
+# (/usr/bin/ansible will use current user as default)
+#remote_user = root
+remote_user = root
+
+# logging is off by default unless this path is defined
+# if so defined, consider logrotate
+log_path = $ARTIFACTS/main.log
+
+# default module name for /usr/bin/ansible
+#module_name = command
+
+# use this shell for commands executed under sudo
+# you may need to change this to bin/bash in rare instances
+# if sudo is constrained
+# executable = /bin/sh
+
+# if inventory variables overlap, does the higher precedence one win
+# or are hash values merged together?  The default is 'replace' but
+# this can also be set to 'merge'.
+hash_behaviour = replace
+
+# by default, variables from roles will be visible in the global variable
+# scope. To prevent this, the following option can be enabled, and only
+# tasks and handlers within the role will see the variables there
+private_role_vars = False
+
+# list any Jinja2 extensions to enable here:
+#jinja2_extensions = jinja2.ext.do,jinja2.ext.i18n
+
+# if set, always use this private key file for authentication, same as
+# if passing --private-key to ansible or ansible-playbook
+#private_key_file = /path/to/file
+
+# If set, configures the path to the Vault password file as an alternative to
+# specifying --vault-password-file on the command line.
+#vault_password_file = /path/to/vault_password_file
+
+# format of string {{ ansible_managed }} available within Jinja2
+# templates indicates to users editing templates files will be replaced.
+# replacing {file}, {host} and {uid} and strftime codes with proper values.
+#ansible_managed = Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S by {uid} on {host}
+# This short version is better used in templates as it won't flag the file as changed every run.
+#ansible_managed = Ansible managed: {file} on {host}
+
+# by default, ansible-playbook will display "Skipping [host]" if it determines a task
+# should not be run on a host.  Set this to "False" if you don't want to see these "Skipping"
+# messages. NOTE: the task header will still be shown regardless of whether or not the
+# task is skipped.
+#display_skipped_hosts = True
+display_skipped_hosts = False
+
+# by default, if a task in a playbook does not include a name: field then
+# ansible-playbook will construct a header that includes the task's action but
+# not the task's args.  This is a security feature because ansible cannot know
+# if the *module* considers an argument to be no_log at the time that the
+# header is printed.  If your environment doesn't have a problem securing
+# stdout from ansible-playbook (or you have manually specified no_log in your
+# playbook on all of the tasks where you have secret information) then you can
+# safely set this to True to get more informative messages.
+display_args_to_stdout = False
+
+# by default (as of 1.3), Ansible will raise errors when attempting to dereference
+# Jinja2 variables that are not set in templates or action lines. Uncomment this line
+# to revert the behavior to pre-1.3.
+#error_on_undefined_vars = False
+
+# by default (as of 1.6), Ansible may display warnings based on the configuration of the
+# system running ansible itself. This may include warnings about 3rd party packages or
+# other conditions that should be resolved if possible.
+# to disable these warnings, set the following value to False:
+system_warnings = False
+
+# by default (as of 1.4), Ansible may display deprecation warnings for language
+# features that should no longer be used and will be removed in future versions.
+# to disable these warnings, set the following value to False:
+deprecation_warnings = False
+
+# (as of 1.8), Ansible can optionally warn when usage of the shell and
+# command module appear to be simplified by using a default Ansible module
+# instead.  These warnings can be silenced by adjusting the following
+# setting or adding warn=yes or warn=no to the end of the command line
+# parameter string.  This will for example suggest using the git module
+# instead of shelling out to the git command.
+command_warnings = False
+
+
+# set plugin path directories here, separate with colons
+#action_plugins     = /usr/share/ansible/plugins/action
+#callback_plugins   = /usr/share/ansible/plugins/callback
+#connection_plugins = /usr/share/ansible/plugins/connection
+#lookup_plugins     = /usr/share/ansible/plugins/lookup
+#vars_plugins       = /usr/share/ansible/plugins/vars
+#filter_plugins     = /usr/share/ansible/plugins/filter
+#test_plugins       = /usr/share/ansible/plugins/test
+#strategy_plugins   = /usr/share/ansible/plugins/strategy
+
+# Most callbacks shipped with Ansible are disabled by default
+# and need to be whitelisted in your ansible.cfg file in order to function.
+callback_whitelist = default
+
+# by default callbacks are not loaded for /bin/ansible, enable this if you
+# want, for example, a notification or logging callback to also apply to
+# /bin/ansible runs
+#bin_ansible_callbacks = False
+
+
+# don't like cows?  that's unfortunate.
+# set to 1 if you don't want cowsay support or export ANSIBLE_NOCOWS=1
+#nocows = 1
+
+# set which cowsay stencil you'd like to use by default. When set to 'random',
+# a random stencil will be selected for each task. The selection will be filtered
+# against the `cow_whitelist` option below.
+#cow_selection = default
+#cow_selection = random
+
+# when using the 'random' option for cowsay, stencils will be restricted to this list.
+# it should be formatted as a comma-separated list with no spaces between names.
+# NOTE: line continuations here are for formatting purposes only, as the INI parser
+#       in python does not support them.
+#cow_whitelist=bud-frogs,bunny,cheese,daemon,default,dragon,elephant-in-snake,elephant,eyes,\
+#              hellokitty,kitty,luke-koala,meow,milk,moofasa,moose,ren,sheep,small,stegosaurus,\
+#              stimpy,supermilker,three-eyes,turkey,turtle,tux,udder,vader-koala,vader,www
+
+# don't like colors either?
+# set to 1 if you don't want colors, or export ANSIBLE_NOCOLOR=1
+nocolor = 0
+
+# if set to a persistent type (not 'memory', for example 'redis') fact values
+# from previous runs in Ansible will be stored.  This may be useful when
+# wanting to use, for example, IP information from one group of servers
+# without having to talk to them in the same playbook run to get their
+# current IP information.
+#fact_caching = memory
+
+# retry files
+# When a playbook fails by default a .retry file will be created in ~/
+# You can disable this feature by setting retry_files_enabled to False
+# and you can change the location of the files by setting retry_files_save_path
+
+#retry_files_enabled = False
+retry_files_enabled = False
+
+# squash actions
+# Ansible can optimise actions that call modules with list parameters
+# when looping. Instead of calling the module once per with_ item, the
+# module is called once with all items at once. Currently this only works
+# under limited circumstances, and only with parameters named 'name'.
+squash_actions = apk,apt,dnf,package,pacman,pkgng,yum,zypper
+
+# prevents logging of task data, off by default
+#no_log = False
+
+# prevents logging of tasks, but only on the targets, data is still logged on the master/controller
+no_target_syslog = True
+
+# controls whether Ansible will raise an error or warning if a task has no
+# choice but to create world readable temporary files to execute a module on
+# the remote machine.  This option is False by default for security.  Users may
+# turn this on to have behaviour more like Ansible prior to 2.1.x.  See
+# https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user
+# for more secure ways to fix this than enabling this option.
+#allow_world_readable_tmpfiles = False
+
+# controls the compression level of variables sent to
+# worker processes. At the default of 0, no compression
+# is used. This value must be an integer from 0 to 9.
+#var_compression_level = 9
+
+# controls what compression method is used for new-style ansible modules when
+# they are sent to the remote system.  The compression types depend on having
+# support compiled into both the controller's python and the client's python.
+# The names should match with the python Zipfile compression types:
+# * ZIP_STORED (no compression. available everywhere)
+# * ZIP_DEFLATED (uses zlib, the default)
+# These values may be set per host via the ansible_module_compression inventory
+# variable
+#module_compression = 'ZIP_DEFLATED'
+
+# This controls the cutoff point (in bytes) on --diff for files
+# set to 0 for unlimited (RAM may suffer!).
+#max_diff_size = 1048576
+
+[privilege_escalation]
+#become=True
+#become_method=sudo
+#become_user=root
+become_user=root
+#become_ask_pass=False
+
+[paramiko_connection]
+
+# uncomment this line to cause the paramiko connection plugin to not record new host
+# keys encountered.  Increases performance on new host additions.  Setting works independently of the
+# host key checking setting above.
+#record_host_keys=False
+
+# by default, Ansible requests a pseudo-terminal for commands executed under sudo. Uncomment this
+# line to disable this behaviour.
+#pty=False
+
+[ssh_connection]
+
+# ssh arguments to use
+# Leaving off ControlPersist will result in poor performance, so use
+# paramiko on older platforms rather than removing it
+ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o UserKnownHostsFile=/dev/null -o PreferredAuthentications=publickey -o ConnectTimeout=13
+
+# The path to use for the ControlPath sockets. This defaults to
+# "%(directory)s/ansible-ssh-%%h-%%p-%%r", however on some systems with
+# very long hostnames or very long path names (caused by long user names or
+# deeply nested home directories) this can exceed the character limit on
+# file socket names (108 characters for most platforms). In that case, you
+# may wish to shorten the string below.
+#
+# Example:
+# control_path = %(directory)s/%%h-%%r
+#control_path = %(directory)s/ansible-ssh-%%h-%%p-%%r
+
+# Enabling pipelining reduces the number of SSH operations required to
+# execute a module on the remote server. This can result in a significant
+# performance improvement when enabled, however when using "sudo:" you must
+# first disable 'requiretty' in /etc/sudoers
+#
+# By default, this option is disabled to preserve compatibility with
+# sudoers configurations that have requiretty (the default on many distros).
+#
+#pipelining = False
+pipelining=True
+
+# if True, make ansible use scp if the connection type is ssh
+# (default is sftp)
+#scp_if_ssh = True
+
+# if False, sftp will not use batch mode to transfer files. This may cause some
+# types of file transfer failures impossible to catch however, and should
+# only be disabled if your sftp version has problems with batch mode
+#sftp_batch_mode = False
+
+[accelerate]
+#accelerate_port = 5099
+#accelerate_timeout = 30
+#accelerate_connect_timeout = 5.0
+
+# The daemon timeout is measured in minutes. This time is measured
+# from the last activity to the accelerate daemon.
+#accelerate_daemon_timeout = 30
+
+# If set to yes, accelerate_multi_key will allow multiple
+# private keys to be uploaded to it, though each user must
+# have access to the system via SSH to add a new key. The default
+# is "no".
+#accelerate_multi_key = yes
+
+[selinux]
+# file systems that require special treatment when dealing with security context
+# the default behaviour that copies the existing context or uses the user default
+# needs to be changed to use the file system dependent context.
+#special_context_filesystems=nfs,vboxsf,fuse,ramfs
+
+# Set this to yes to allow libvirt_lxc connections to work without SELinux.
+#libvirt_lxc_noseclabel = yes
+
+[colors]
+#highlight = white
+#verbose = blue
+#warn = bright purple
+#error = red
+#debug = dark gray
+#deprecate = purple
+#skip = cyan
+#unreachable = red
+#ok = green
+#changed = yellow
+#diff_add = green
+#diff_remove = red
+#diff_lines = cyan

contrib/test/integration/build/bats.yml

@@ -1,4 +1,5 @@
 ---
+
 - name: clone bats source repo
   git:
     repo: "https://github.com/sstephenson/bats.git"
@@ -14,4 +15,4 @@
   file:
     src: /usr/local/bin/bats
     dest: /usr/bin/bats
-    state: link
+    state: link

contrib/test/integration/build/cri-o.yml

@@ -1,4 +1,5 @@
 ---
+
 - name: stat the expected cri-o directory
   stat:
     path: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
@@ -75,4 +76,4 @@
     insertafter: 'storage_option = \['
     regexp: 'overlay2\.override_kernel_check=1'
     state: present
-  when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
+  when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'

contrib/test/integration/build/cri-tools.yml

@@ -1,4 +1,5 @@
 ---
+
 - name: clone cri-tools source repo
   git:
     repo: "https://github.com/kubernetes-incubator/cri-tools.git"
@@ -12,4 +13,4 @@
   file:
     src: "{{ ansible_env.GOPATH }}/bin/crictl"
     dest: /usr/bin/crictl
-    state: link
+    state: link

contrib/test/integration/build/kubernetes.yml

@@ -1,4 +1,5 @@
 ---
+
 - name: clone kubernetes source repo
   git:
     repo: "https://github.com/runcom/kubernetes.git"
@@ -58,4 +59,4 @@
     dest: /etc/environment
     line: 'KUBECONFIG=/var/run/kubernetes/admin.kubeconfig'
     regexp: 'KUBECONFIG='
-    state: present
+    state: present

contrib/test/integration/build/plugins.yml

@@ -1,4 +1,5 @@
 ---
+
 - name: clone plugins source repo
   git:
     repo: "https://github.com/containernetworking/plugins.git"
@@ -46,4 +47,4 @@
     src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/bridge"
     dest: "/opt/cni/bin/bridge-custom"
     mode: "o=rwx,g=rx,o=rx"
-    remote_src: yes
+    remote_src: yes

contrib/test/integration/build/runc.yml

@@ -1,4 +1,5 @@
 ---
+
 - name: clone runc source repo
   git:
     repo: "https://github.com/opencontainers/runc.git"
@@ -18,4 +19,4 @@
   file:
     src: /usr/local/sbin/runc
     dest: /usr/bin/runc
-    state: link
+    state: link

contrib/test/integration/callback_plugins/default.py

@@ -0,0 +1,156 @@
+'''Plugin to override the default output logic.'''
+
+# upstream: https://gist.github.com/cliffano/9868180
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# For some reason this has to be done
+import imp
+import os
+
+ANSIBLE_PATH = imp.find_module('ansible')[1]
+DEFAULT_PATH = os.path.join(ANSIBLE_PATH, 'plugins/callback/default.py')
+DEFAULT_MODULE = imp.load_source(
+    'ansible.plugins.callback.default',
+    DEFAULT_PATH
+)
+
+try:
+    from ansible.plugins.callback import CallbackBase
+    BASECLASS = CallbackBase
+except ImportError: # < ansible 2.1
+    BASECLASS = DEFAULT_MODULE.CallbackModule
+
+
+class CallbackModule(DEFAULT_MODULE.CallbackModule):  # pylint: disable=too-few-public-methods,no-init
+    '''
+    Override for the default callback module.
+
+    Render std err/out outside of the rest of the result which it prints with
+    indentation.
+    '''
+    CALLBACK_VERSION = 2.0
+    CALLBACK_TYPE = 'stdout'
+    CALLBACK_NAME = 'default'
+
+    def __init__(self, *args, **kwargs):
+        # pylint: disable=non-parent-init-called
+        BASECLASS.__init__(self, *args, **kwargs)
+        self.failed_task = []
+        self.result_file = os.environ.get('AHT_RESULT_FILE')
+
+    def _dump_results(self, result):
+        '''Return the text to output for a result.'''
+        result['_ansible_verbose_always'] = True
+
+        save = {}
+        for key in ['stdout', 'stdout_lines', 'stderr', 'stderr_lines', 'msg']:
+            if key in result:
+                save[key] = result.pop(key)
+
+        output = BASECLASS._dump_results(self, result)  # pylint: disable=protected-access
+
+        for key in ['stdout', 'stderr', 'msg']:
+            if key in save and save[key]:
+                output += '\n\n%s:\n---\n%s\n---' % (key.upper(), save[key])
+
+        for key, value in save.items():
+            result[key] = value
+
+        return output
+
+    def v2_runner_on_unreachable(self, result):
+        self.failed_task = result
+
+        if self._play.strategy == 'free' and self._last_task_banner != result._task._uuid:
+            self._print_task_banner(result._task)
+
+        delegated_vars = result._result.get('_ansible_delegated_vars', None)
+        if delegated_vars:
+            self._display.display("fatal: [%s -> %s]: UNREACHABLE! => %s" % (result._host.get_name(), delegated_vars['ansible_host'], self._dump_results(result._result)), color=C.COLOR_UNREACHABLE)
+        else:
+            self._display.display("fatal: [%s]: UNREACHABLE! => %s" % (result._host.get_name(), self._dump_results(result._result)), color=C.COLOR_UNREACHABLE)
+
+    def v2_runner_on_failed(self,result, ignore_errors=False):
+        if ignore_errors is not True:
+            # Sets environment variable for test failures for use in playboks.
+            # Handlers tasks can conditionalize themselves using this variable
+            # to run only on failure.
+            os.environ["AHT_FAILURE"] = "1"
+
+            # Save last failure
+            self.failed_task = result
+
+        if self._play.strategy == 'free' and self._last_task_banner != result._task._uuid:
+            self._print_task_banner(result._task)
+
+        delegated_vars = result._result.get('_ansible_delegated_vars', None)
+        if 'exception' in result._result:
+            if self._display.verbosity < 3:
+                # extract just the actual error message from the exception text
+                error = result._result['exception'].strip().split('\n')[-1]
+                msg = "An exception occurred during task execution. To see the full traceback, use -vvv. The error was: %s" % error
+            else:
+                msg = "An exception occurred during task execution. The full traceback is:\n" + result._result['exception']
+
+            self._display.display(msg, color=C.COLOR_ERROR)
+
+        if result._task.loop and 'results' in result._result:
+            self._process_items(result)
+
+        else:
+            if delegated_vars:
+                self._display.display("fatal: [%s -> %s]: FAILED! => %s" % (result._host.get_name(), delegated_vars['ansible_host'], self._dump_results(result._result)), color=C.COLOR_ERROR)
+            else:
+                self._display.display("fatal: [%s]: FAILED! => %s" % (result._host.get_name(), self._dump_results(result._result)), color=C.COLOR_ERROR)
+
+        if ignore_errors:
+            self._display.display("...ignoring", color=C.COLOR_SKIP)
+
+    def v2_playbook_on_stats(self, stats):
+        self._display.banner("PLAY RECAP")
+
+        hosts = sorted(stats.processed.keys())
+        for h in hosts:
+            t = stats.summarize(h)
+
+            self._display.display(u"%s : %s %s %s %s" % (
+                hostcolor(h, t),
+                colorize(u'ok', t['ok'], C.COLOR_OK),
+                colorize(u'changed', t['changed'], C.COLOR_CHANGED),
+                colorize(u'unreachable', t['unreachable'], C.COLOR_UNREACHABLE),
+                colorize(u'failed', t['failures'], C.COLOR_ERROR)),
+                screen_only=True
+            )
+
+            self._display.display(u"%s : %s %s %s %s" % (
+                hostcolor(h, t, False),
+                colorize(u'ok', t['ok'], None),
+                colorize(u'changed', t['changed'], None),
+                colorize(u'unreachable', t['unreachable'], None),
+                colorize(u'failed', t['failures'], None)),
+                log_only=True
+            )
+
+        self._display.display("", screen_only=True)
+        # Save result to file if environment variable exists
+        if self.result_file is not None:
+            if self.failed_task:
+                with open(self.result_file, 'w') as f:
+                    f.write("PLAY: %s\n%s\n%s" % (self._play, \
+                                                  self.failed_task._task, \
+                                                  self._dump_results(self.failed_task._result)))
+            else:
+                open(self.result_file, 'w').close()

contrib/test/integration/e2e.yml

@@ -1,4 +1,5 @@
 ---
+
 - name: enable and start CRI-O
   systemd:
     name: crio
@@ -33,11 +34,21 @@
 
 - name: ensure directory exists for e2e reports
   file:
-    path: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/artifacts"
+    path: "{{ artifacts }}"
     state: directory
 
+- name: Buffer the e2e testing command to workaround Ansible YAML folding "feature"
+  set_fact:
+    e2e_shell_cmd: >
+        go run hack/e2e.go
+            --test
+            -test_args="-host=https://{{ ansible_default_ipv4.address }}:6443
+                        --ginkgo.focus=\[Conformance\]
+                        --report-dir={{ artifacts }}"
+            &> {{ artifacts }}/e2e.log
+  # Fix vim syntax hilighting: "
+
 - name: run e2e tests
-  shell: |
-    go run hack/e2e.go -v --test -test_args="-host=https://{{ ansible_default_ipv4.address }}:6443 --ginkgo.focus=\[Conformance\]" >e2e.log 2>&1
+  shell: "{{ e2e_shell_cmd | regex_replace('\\s+', ' ') }}"
   args:
-    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
+    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"

contrib/test/integration/golang.yml

@@ -1,4 +1,5 @@
 ---
+
 - name: set up GOPATH if it is not already set
   lineinfile:
     dest: /etc/environment
@@ -37,4 +38,4 @@
     - onsi/ginkgo/ginkgo
     - onsi/gomega
     - cloudflare/cfssl/cmd/...
-    - jteeuwen/go-bindata/go-bindata
+    - jteeuwen/go-bindata/go-bindata

contrib/test/integration/main.yml

@@ -1,7 +1,7 @@
 - hosts: all
   remote_user: root
-  vars:
-    xunit: false
+  vars_files:
+    - "{{ playbook_dir }}/vars.yml"
   tags:
     - setup
   tasks:
@@ -28,8 +28,8 @@
 
 - hosts: all
   remote_user: root
-  vars:
-    xunit: false
+  vars_files:
+    - "{{ playbook_dir }}/vars.yml"
   tags:
     - integration
     - e2e
@@ -39,8 +39,8 @@
 
 - hosts: all
   remote_user: root
-  vars:
-    xunit: false
+  vars_files:
+    - "{{ playbook_dir }}/vars.yml"
   tags:
     - integration
   tasks:
@@ -49,8 +49,10 @@
 
 - hosts: all
   remote_user: root
+  vars_files:
+    - "{{ playbook_dir }}/vars.yml"
   tags:
     - e2e
   tasks:
     - name: run k8s e2e tests
-      include: e2e.yml
+      include: e2e.yml

contrib/test/integration/results.yml

@@ -0,0 +1,62 @@
+---
+# vim-syntax: ansible
+
+- hosts: '{{ hosts | default("all") }}'
+  vars_files:
+    - "{{ playbook_dir }}/vars.yml"
+  vars:
+    _result_filepaths: []  # do not use
+    _dstfnbuff: []  # do not use
+  tasks:
+    - name: The crio_integration_filepath is required
+      tags:
+          - integration
+      set_fact:
+       _result_filepaths: "{{ _result_filepaths + [crio_integration_filepath] }}"
+
+    - name: The crio_node_e2e_filepath is required
+      tags:
+          - e2e
+      set_fact:
+       _result_filepaths: "{{ _result_filepaths + [crio_node_e2e_filepath] }}"
+
+    - name: Verify expectations
+      assert:
+        that:
+          - 'result_dest_basedir | default(False, True)'
+          - '_result_filepaths | default(False, True)'
+          - '_dstfnbuff == []'
+          - 'results_fetched is undefined'
+
+    - name: Results directory exists
+      file:
+          path: "{{ result_dest_basedir }}"
+          state: directory
+      delegate_to: localhost
+
+    - name: destination file paths are buffered for overwrite-checking and jUnit conversion
+      set_fact:
+          _dstfnbuff: >
+              {{ _dstfnbuff |
+                 union( [result_dest_basedir ~ "/" ~ inventory_hostname ~ "/" ~ item | basename] ) }}
+      with_items: '{{ _result_filepaths }}'
+
+    - name: Overwriting existing results assumed very very bad
+      fail:
+          msg: "Cowardly refusing to overwrite {{ item }}"
+      when: item | exists
+      delegate_to: localhost
+      with_items: '{{ _dstfnbuff }}'
+
+    # fetch module doesn't support directories
+    - name: Retrieve results from all hosts
+      synchronize:
+          checksum: True  # Don't rely on date/time being in sync
+          archive: False  # Don't bother with permissions or times
+          copy_links: True  # We want files, not links to files
+          recursive: True
+          mode: pull
+          dest: '{{ result_dest_basedir }}/{{ inventory_hostname }}/'  # must end in /
+          src: '{{ item }}'
+      register: results_fetched
+      with_items: '{{ _result_filepaths }}'

contrib/test/integration/system.yml

@@ -1,4 +1,5 @@
 ---
+
 - name: register a repo for Golang
   yum_repository:
     name: 'centos-paas-sig-openshift-origin37-rpms'
@@ -14,61 +15,7 @@
   package:
     name: "{{ item }}"
     state: present
-  with_items:
-    - btrfs-progs-devel
-    - container-selinux
-    - curl
-    - device-mapper-devel
-    - expect
-    - findutils
-    - gcc
-    - git
-    - glib2-devel
-    - glibc-devel
-    - glibc-static
-    - golang
-    - gpgme-devel
-    - hostname
-    - iproute
-    - iptables
-    - krb5-workstation
-    - libassuan-devel
-    - libffi-devel
-    - libgpg-error-devel
-    - libguestfs-tools
-    - libseccomp-devel
-    - libvirt-client
-    - libvirt-python
-    - libxml2-devel
-    - libxslt-devel
-    - make
-    - mlocate
-    - nfs-utils
-    - nmap-ncat
-    - npm
-    - oci-register-machine
-    - oci-systemd-hook
-    - oci-umount
-    - openssl
-    - openssl-devel
-    - ostree-devel
-    - pkgconfig
-    - python
-    - python2-boto
-    - python2-crypto
-    - python2-mock
-    - python-click
-    - python-devel
-    - python-virtualenv
-    - PyYAML
-    - redhat-rpm-config
-    - rpcbind
-    - rsync
-    - sed
-    - skopeo-containers
-    - socat
-    - tar
-    - wget
+  with_items: "{{ rpm_pkgs }}"
   async: 600
   poll: 10
 
@@ -116,4 +63,4 @@
     state: present
 
 - name: Flush the iptables
-  command: iptables -F
+  command: iptables -F

contrib/test/integration/test.yml

@@ -1,4 +1,5 @@
 ---
+
 - name: Change test_runner.sh to use bats xunit output
   lineinfile:
     dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/test/test_runner.sh"
@@ -21,7 +22,7 @@
 
 - name: set extra shell for non-xunit tests
   set_fact:
-    extra_shell_suffix: " >testout.txt 2>&1"
+    extra_shell_suffix: " &> {{ artifacts }}/testout.txt"
   when: not xunit
 
 - name: run integration tests
@@ -35,16 +36,16 @@
 
 - name: Make testing output directory
   file:
-    path: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/reports"
+    path: "{{ artifacts }}"
     state: directory
   ignore_errors: yes
   when: xunit
 
 - name: Move all xunit files into one dir to scp
-  shell: "mv {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/test/TestReport-bats*.xml {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/reports/"
+  shell: "mv {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/test/TestReport-bats*.xml {{ artifacts }}"
   when: xunit
 
 - name: ensure we fail on bad tests
   fail:
     msg: Integration tests failed!
-  when: "'not ok' in integration_test.stdout"
+  when: "'not ok' in integration_test.stdout"

contrib/test/integration/vars.yml

@@ -0,0 +1,67 @@
+---
+
+# Enable using BATS to output integration-test xunit output (FIXME: broken? Kill it?)
+xunit: false
+
+rpm_pkgs:
+    - btrfs-progs-devel
+    - container-selinux
+    - curl
+    - device-mapper-devel
+    - expect
+    - findutils
+    - gcc
+    - git
+    - glib2-devel
+    - glibc-devel
+    - glibc-static
+    - golang
+    - gpgme-devel
+    - hostname
+    - iproute
+    - iptables
+    - krb5-workstation
+    - libassuan-devel
+    - libffi-devel
+    - libgpg-error-devel
+    - libguestfs-tools
+    - libseccomp-devel
+    - libvirt-client
+    - libvirt-python
+    - libxml2-devel
+    - libxslt-devel
+    - make
+    - mlocate
+    - nfs-utils
+    - nmap-ncat
+    - npm
+    - oci-register-machine
+    - oci-systemd-hook
+    - oci-umount
+    - openssl
+    - openssl-devel
+    - ostree-devel
+    - pkgconfig
+    - python
+    - python2-boto
+    - python2-crypto
+    - python2-mock
+    - python-click
+    - python-devel
+    - python-virtualenv
+    - PyYAML
+    - redhat-rpm-config
+    - rpcbind
+    - rsync
+    - sed
+    - skopeo-containers
+    - socat
+    - tar
+    - wget
+
+# For results.yml Paths use rsync 'source' conventions
+artifacts: "/tmp/artifacts"  # Base-directory for collection
+crio_integration_filepath: "{{ artifacts }}/testout.txt"
+crio_node_e2e_filepath: "{{ artifacts }}/junit_01.xml"
+result_dest_basedir: '{{ lookup("env","WORKSPACE") |
+                         default(playbook_dir, True) }}/artifacts'

contrib/test/venv-ansible-playbook.sh

@@ -46,13 +46,18 @@ else
     trap 'rm -rf "$PIPCACHE"' EXIT
 fi
 
+# Create a directory to contain logs and test artifacts
+export ARTIFACTS=$(mkdir -pv $WORKSPACE/artifacts | tail -1 | cut -d \' -f 2)
+[ -d "$ARTIFACTS" ] || exit 3
+
 # All command failures from now on are fatal
 set -e
 echo
 echo "Bootstrapping trusted virtual environment, this may take a few minutes, depending on networking."
-echo "(logs:  \"$WORKSPACE/crio_venv_setup_log.txt\")"
+echo "(logs: \"$ARTIFACTS/crio_venv_setup_log.txt\")"
 echo
 
+
 (
     set -x
     cd "$WORKSPACE"
@@ -81,21 +86,21 @@ echo
     fi
     # Enter trusted virtualenv
     source ./.cri-o_venv/bin/activate
-    # Re-install from cache
-    pip install --force-reinstall --upgrade pip==9.0.1
+    # Upgrade stock-pip to support hashes
+    pip install --force-reinstall --cache-dir="$PIPCACHE" --upgrade pip==9.0.1
+    # Re-install from cache but validate all hashes (including on pip itself)
     pip --cache-dir="$PIPCACHE" install --require-hashes \
         --requirement "$SCRIPT_PATH/requirements.txt"
     # Remove temporary bootstrap virtualenv
     rm -rf ./.venvbootstrap
     # Exit trusted virtualenv
 
-) &> $WORKSPACE/crio_venv_setup_log.txt;
+) &> $ARTIFACTS/crio_venv_setup_log.txt;
 
 echo
 echo "Executing \"$WORKSPACE/.cri-o_venv/bin/ansible-playbook $@\""
 echo
 
 # Execute command-line arguments under virtualenv
-cd "$WORKSPACE"
-source ./.cri-o_venv/bin/activate
-./.cri-o_venv/bin/ansible-playbook $@
+source ${WORKSPACE}/.cri-o_venv/bin/activate
+${WORKSPACE}/.cri-o_venv/bin/ansible-playbook $@