integration: Collect subject results

Add a playbook to pull down the integration and e2e testing
logs/xml.  By default they will appear in a 'artifacts' subdirectory
of wherever the ``results.yml`` playbook lives. If the ``$WORKSPACE``
env. var is set and non-empty, the subdirectory will be created
there instead.

Inside the ``artifacts`` directory, further sub-directories are created,
one for each subject's Ansible inventory name.  Within those
sub-directories are all the collected logs from that host.  In this way,
automation may simply archive the entire 'artifacts' directory to
capture the important log files.

(Depends on PR #935)

Signed-off-by: Chris Evich <cevich@redhat.com>

contrib/test/integration/e2e.yml

@@ -41,14 +41,18 @@
     path: "{{ artifacts }}"
     state: directory
 
+- name: Buffer the e2e testing command to workaround Ansible YAML folding "feature"
+  set_fact:
+    e2e_shell_cmd: >
+        go run hack/e2e.go
+            --test
+            -test_args="-host=https://{{ ansible_default_ipv4.address }}:6443
+                        --ginkgo.focus=\[Conformance\]
+                        --report-dir={{ artifacts }}"
+            &> {{ artifacts }}/e2e.log
+  # Fix vim syntax hilighting: "
+
 - name: run e2e tests
-  shell: >
+  shell: "{{ e2e_shell_cmd | regex_replace('\\s+', ' ') }}"
-    go run hack/e2e.go \
-        --test \
-        -test_args="-host=https://{{ ansible_default_ipv4.address }}:6443 \
-                    --ginkgo.focus=\[Conformance\] \
-                    --report-dir={{ artifacts }}" \
-        &> {{ artifacts }}/e2e.log
-  # Fix " syntax hilighting
   args:
     chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"

contrib/test/integration/results.yml

@@ -0,0 +1,67 @@
+---
+# vim-syntax: ansible
+
+- hosts: '{{ hosts | default("all") }}'
+  vars:
+    # Paths use rsync 'source' conventions
+    crio_integration_filepath: "/root/src/github.com/kubernetes-incubator/cri-o/testout.txt"
+    # N/B: This needs coordination with e2e.yml
+    crio_node_e2e_filepath: "/root/go-tools/src/k8s.io/kubernetes/artifacts/"  # everything
+    # Treat empty env. var as "undefined and use default()
+    result_dest_basedir: '{{ lookup("env","WORKSPACE") |
+                             default(playbook_dir, True) }}/artifacts'
+    _result_filepaths: []  # do not use
+    _dstfnbuff: []  # do not use
+  tasks:
+    - name: The crio_integration_filepath is required
+      tags:
+          - integration
+      set_fact:
+       _result_filepaths: "{{ _result_filepaths + [crio_integration_filepath] }}"
+
+    - name: The crio_node_e2e_filepath is required
+      tags:
+          - e2e
+      set_fact:
+       _result_filepaths: "{{ _result_filepaths + [crio_node_e2e_filepath] }}"
+
+    - name: Verify expectations
+      assert:
+        that:
+          - 'result_dest_basedir | default(False, True)'
+          - '_result_filepaths | default(False, True)'
+          - '_dstfnbuff == []'
+          - 'results_fetched is undefined'
+
+    - name: Results directory exists
+      file:
+          path: "{{ result_dest_basedir }}"
+          state: directory
+      delegate_to: localhost
+
+    - name: destination file paths are buffered for overwrite-checking and jUnit conversion
+      set_fact:
+          _dstfnbuff: >
+              {{ _dstfnbuff |
+                 union( [result_dest_basedir ~ "/" ~ inventory_hostname ~ "/" ~ item | basename] ) }}
+      with_items: '{{ _result_filepaths }}'
+
+    - name: Overwriting existing results assumed very very bad
+      fail:
+          msg: "Cowardly refusing to overwrite {{ item }}"
+      when: item | exists
+      delegate_to: localhost
+      with_items: '{{ _dstfnbuff }}'
+
+    # fetch module doesn't support directories
+    - name: Retrieve results from all hosts
+      synchronize:
+          checksum: True  # Don't rely on date/time being in sync
+          archive: False  # Don't bother with permissions or times
+          copy_links: True  # We want files, not links to files
+          recursive: True
+          mode: pull
+          dest: '{{ result_dest_basedir }}/{{ inventory_hostname }}/'  # must end in /
+          src: '{{ item }}'
+      register: results_fetched
+      with_items: '{{ _result_filepaths }}'

contrib/test/venv-ansible-playbook.sh

@@ -46,14 +46,17 @@ else
     trap 'rm -rf "$PIPCACHE"' EXIT
 fi
 
+# Create a directory to contain logs and test artifacts
+export ARTIFACTS=$(mkdir -pv $WORKSPACE/artifacts | tail -1 | cut -d \' -f 2)
+[ -d "$ARTIFACTS" ] || exit 3
+
 # All command failures from now on are fatal
 set -e
 echo
 echo "Bootstrapping trusted virtual environment, this may take a few minutes, depending on networking."
-echo "(logs:  \"$WORKSPACE/crio_venv_setup_log.txt\")"
+echo "(logs: \"$ARTIFACTS/crio_venv_setup_log.txt\")"
 echo
 
-mkdir -p "$WORKSPACE/artifacts"
 
 (
     set -x
@@ -83,15 +86,16 @@ mkdir -p "$WORKSPACE/artifacts"
     fi
     # Enter trusted virtualenv
     source ./.cri-o_venv/bin/activate
-    # Re-install from cache
+    # Upgrade stock-pip to support hashes
-    pip install --force-reinstall --upgrade pip==9.0.1
+    pip install --force-reinstall --cache-dir="$PIPCACHE" --upgrade pip==9.0.1
+    # Re-install from cache but validate all hashes (including on pip itself)
     pip --cache-dir="$PIPCACHE" install --require-hashes \
         --requirement "$SCRIPT_PATH/requirements.txt"
     # Remove temporary bootstrap virtualenv
     rm -rf ./.venvbootstrap
     # Exit trusted virtualenv
 
-) &> $WORKSPACE/artifacts/crio_venv_setup_log.txt;
+) &> $ARTIFACTS/crio_venv_setup_log.txt;
 
 echo
 echo "Executing \"$WORKSPACE/.cri-o_venv/bin/ansible-playbook $@\""