Rewrite playbook for package-centric testing

Rather than maintaining yet-another-set-of-build-steps
in playbooks, simply re-use the inate tallent of the Makefile.

Separate out the build vs install vs testing steps clearly.  This
enables alternative packaging/installing tasks to be easily added later
for different environments.

Signed-off-by: Chris Evich <cevich@redhat.com>

contrib/test/.gitignore

@@ -0,0 +1,12 @@
+
+# Dropped by venv-cmd.sh
+.venvbootstrap
+.cache
+.venv
+artifacts
+
+# Dropped by python3
+__pycache__
+
+# Dropped by python2
+.pyc

contrib/test/README.md

@@ -0,0 +1,3 @@
+# Fedora and RHEL Integration and End-to-End Tests
+
+FIXME

contrib/test/group_vars/CentOS/yumrepos.yml

@@ -0,0 +1,12 @@
+---
+
+enable_rh_repos:
+    - base
+    - extras
+    - optional
+    - supplementary
+
+yum_repos:
+    - name: virt7-container-common-candidate
+      baseurl: https://cbs.centos.org/repos/virt7-container-common-candidate/x86_64/os/
+      gpgcheck: False

contrib/test/group_vars/RedHat/yumrepos.yml

@@ -0,0 +1,15 @@
+---
+
+disable_all_rh_repos: True  # Because: Special Reasons
+
+enable_rh_repos:
+    - rhel-7-server-rpms
+    - rhel-7-server-extras-rpms
+    - rhel-7-server-optional-rpms
+    - rhel-7-server-supplementary-rpms
+
+yum_repos:
+    # TODO: Workaround not having any released crio spec file for RHEL yet
+    - name: virt7-container-common-candidate
+      baseurl: https://cbs.centos.org/repos/virt7-container-common-candidate/x86_64/os/
+      gpgcheck: False

contrib/test/integration/README.md

@@ -1,21 +0,0 @@
-# Fedora and RHEL Integration and End-to-End Tests
-
-This directory contains playbooks to set up for and run the integration and
-end-to-end tests for CRI-O on RHEL and Fedora hosts. Two entrypoints exist:
-
- - `main.yml`: sets up the machine and runs tests
- - `results.yml`: gathers test output to `/tmp/artifacts`
-
-When running `main.yml`, three tags are present:
-
- - `setup`: run all tasks to set up the system for testing
- - `e2e`: build CRI-O from source and run Kubernetes node E2Es
- - `integration`: build CRI-O from source and run the local integration suite
-
-The playbooks assume the following things about your system:
-
- - on RHEL, the server and extras repos are configured and certs are present
- - `ansible` is installed and the host is boot-strapped to allow `ansible` to run against it
- - the `$GOPATH` is set and present for all shells (*e.g.* written in `/etc/environment`)
- - CRI-O is checked out to the correct state at `${GOPATH}/src/github.com/kubernetes-incubator/cri-o`
- - the user running the playbook has access to passwordless `sudo`

contrib/test/integration/build/bats.yml

@@ -1,17 +0,0 @@
----
-
-- name: clone bats source repo
-  git:
-    repo: "https://github.com/sstephenson/bats.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats"
-
-- name: install bats
-  command: "./install.sh /usr/local"
-  args:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats"
-
-- name: link bats
-  file:
-    src: /usr/local/bin/bats
-    dest: /usr/bin/bats
-    state: link

contrib/test/integration/build/cri-o.yml

@@ -1,79 +0,0 @@
----
-
-- name: stat the expected cri-o directory
-  stat:
-    path: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
-  register: dir_stat
-
-- name: expect cri-o to be cloned already
-  fail:
-    msg: "Expected cri-o to be cloned at {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o but it wasn't!"
-  when: not dir_stat.stat.exists
-
-- name: install cri-o tools
-  make:
-    target: install.tools
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
-
-- name: build cri-o
-  make:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
-
-- name: install cri-o
-  make:
-    target: install
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
-
-- name: install cri-o systemd files
-  make:
-    target: install.systemd
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
-
-- name: install cri-o config
-  make:
-    target: install.config
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
-
-- name: install configs
-  copy:
-    src: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/{{ item.src }}"
-    dest: "{{ item.dest }}"
-    remote_src: yes
-  with_items:
-    - src: contrib/cni/10-crio-bridge.conf
-      dest: /etc/cni/net.d/10-crio-bridge.conf
-    - src: contrib/cni/99-loopback.conf
-      dest: /etc/cni/net.d/99-loopback.conf
-    - src: test/redhat_sigstore.yaml
-      dest: /etc/containers/registries.d/registry.access.redhat.com.yaml
-
-- name: run with overlay
-  replace:
-    regexp: 'storage_driver = ""'
-    replace: 'storage_driver = "overlay"'
-    name: /etc/crio/crio.conf
-    backup: yes
-
-- name: run with systemd cgroup manager
-  replace:
-    regexp: 'cgroup_manager = "cgroupfs"'
-    replace: 'cgroup_manager = "systemd"'
-    name: /etc/crio/crio.conf
-    backup: yes
-
-- name: add docker.io default registry
-  lineinfile:
-    dest: /etc/crio/crio.conf
-    line: '"docker.io"'
-    insertafter: 'registries = \['
-    regexp: 'docker\.io'
-    state: present
-
-- name: add overlay storage opts on RHEL/CentOS
-  lineinfile:
-    dest: /etc/crio/crio.conf
-    line: '"overlay.override_kernel_check=1"'
-    insertafter: 'storage_option = \['
-    regexp: 'overlay\.override_kernel_check=1'
-    state: present
-  when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'

contrib/test/integration/build/cri-tools.yml

@@ -1,26 +0,0 @@
----
-
-- name: clone cri-tools source repo
-  git:
-    repo: "https://github.com/kubernetes-incubator/cri-tools.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-tools"
-    version: "{{ cri_tools_git_version }}"
-    force: "{{ force_clone | default(False) | bool}}"
-
-- name: install crictl
-  command: "/usr/bin/go install github.com/kubernetes-incubator/cri-tools/cmd/crictl"
-
-- name: install critest
-  command: "/usr/bin/go install github.com/kubernetes-incubator/cri-tools/cmd/critest"
-
-- name: link crictl
-  file:
-    src: "{{ ansible_env.GOPATH }}/bin/crictl"
-    dest: /usr/bin/crictl
-    state: link
-
-- name: link critest
-  file:
-    src: "{{ ansible_env.GOPATH }}/bin/critest"
-    dest: /usr/bin/critest
-    state: link

contrib/test/integration/build/kubernetes.yml

@@ -1,65 +0,0 @@
----
-
-- name: clone kubernetes source repo
-  git:
-    repo: "https://github.com/{{ k8s_github_fork }}/kubernetes.git"
-    dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
-    # based on kube v1.9.0-alpha.2, update as needed
-    version: "{{ k8s_git_version }}"
-    force: "{{ force_clone | default(False) | bool}}"
-
-- name: install etcd
-  command: "hack/install-etcd.sh"
-  args:
-    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
-
-- name: build kubernetes
-  make:
-    chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
-
-- name: Add custom cluster service file for the e2e testing
-  copy:
-    dest: /etc/systemd/system/customcluster.service
-    content: |
-      [Unit]
-      After=network-online.target
-      Wants=network-online.target
-      [Service]
-      WorkingDirectory={{ ansible_env.GOPATH }}/src/k8s.io/kubernetes
-      ExecStart=/usr/local/bin/createcluster.sh
-      User=root
-      [Install]
-      WantedBy=multi-user.target
-
-- name: Add create cluster background script for e2e testing
-  copy:
-    dest: /usr/local/bin/createcluster.sh
-    content: |
-      #!/bin/bash
-
-      export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ ansible_env.GOPATH }}/bin:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/third_party/etcd:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/
-      export CONTAINER_RUNTIME=remote
-      export CGROUP_DRIVER=systemd
-      export CONTAINER_RUNTIME_ENDPOINT='{{ crio_socket }} --runtime-request-timeout=5m'
-      export ALLOW_SECURITY_CONTEXT=","
-      export ALLOW_PRIVILEGED=1
-      export DNS_SERVER_IP={{ ansible_eth0.ipv4.address }}
-      export API_HOST={{ ansible_eth0.ipv4.address }}
-      export API_HOST_IP={{ ansible_eth0.ipv4.address }}
-      export KUBE_ENABLE_CLUSTER_DNS=true
-      ./hack/local-up-cluster.sh
-    mode: "u=rwx,g=rwx,o=x"
-
-- name: Set kubernetes_provider to be local
-  lineinfile:
-    dest: /etc/environment
-    line: 'KUBERNETES_PROVIDER=local'
-    regexp: 'KUBERNETES_PROVIDER='
-    state: present
-
-- name: Set KUBECONFIG
-  lineinfile:
-    dest: /etc/environment
-    line: 'KUBECONFIG=/var/run/kubernetes/admin.kubeconfig'
-    regexp: 'KUBECONFIG='
-    state: present

contrib/test/integration/build/plugins.yml

@@ -1,50 +0,0 @@
----
-
-- name: clone plugins source repo
-  git:
-    repo: "https://github.com/containernetworking/plugins.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
-    version: "dcf7368eeab15e2affc6256f0bb1e84dd46a34de"
-
-- name: build plugins
-  command: "./build.sh"
-  args:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
-
-- name: install plugins
-  copy:
-    src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/{{ item }}"
-    dest: "/opt/cni/bin"
-    mode: "o=rwx,g=rx,o=rx"
-    remote_src: yes
-  with_items:
-    - bridge
-    - dhcp
-    - flannel
-    - host-local
-    - ipvlan
-    - loopback
-    - macvlan
-    - ptp
-    - sample
-    - tuning
-    - vlan
-
-- name: clone runcom plugins source repo
-  git:
-    repo: "https://github.com/runcom/plugins.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
-    version: "custom-bridge"
-    force: yes
-
-- name: build plugins
-  command: "./build.sh"
-  args:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins"
-
-- name: install custom bridge
-  copy:
-    src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/bridge"
-    dest: "/opt/cni/bin/bridge-custom"
-    mode: "o=rwx,g=rx,o=rx"
-    remote_src: yes

contrib/test/integration/build/runc.yml

@@ -1,23 +0,0 @@
----
-
-- name: clone runc source repo
-  git:
-    repo: "https://github.com/opencontainers/runc.git"
-    dest: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
-    version: "84a082bfef6f932de921437815355186db37aeb1"
-
-- name: build runc
-  make:
-    params: BUILDTAGS="seccomp selinux"
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
-
-- name: install runc
-  make:
-    target: "install"
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
-
-- name: link runc
-  file:
-    src: /usr/local/sbin/runc
-    dest: /usr/bin/runc
-    state: link

contrib/test/integration/golang.yml

@@ -1,55 +0,0 @@
----
-
-- name: ensure Golang dir is empty first
-  file:
-    path: /usr/local/go
-    state: absent
-
-- name: fetch Golang
-  unarchive:
-    remote_src: yes
-    src: "https://storage.googleapis.com/golang/go{{ version }}.linux-amd64.tar.gz"
-    dest: /usr/local
-
-- name: link go toolchain
-  file:
-    src: "/usr/local/go/bin/{{ item }}"
-    dest: "/usr/bin/{{ item }}"
-    state: link
-  with_items:
-    - go
-    - gofmt
-    - godoc
-
-- name: ensure user profile exists
-  file:
-    path: "{{ ansible_user_dir }}/.profile"
-    state: touch
-
-- name: set up PATH for Go toolchain and built binaries
-  lineinfile:
-    dest: "{{ ansible_user_dir }}/.profile"
-    line: 'PATH={{ ansible_env.PATH }}:{{ ansible_env.GOPATH }}/bin:/usr/local/go/bin'
-    regexp: '^PATH='
-    state: present
-
-- name: set up directories
-  file:
-    path: "{{ item }}"
-    state: directory
-  with_items:
-    - "{{ ansible_env.GOPATH }}/src/github.com/containernetworking"
-    - "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator"
-    - "{{ ansible_env.GOPATH }}/src/github.com/k8s.io"
-    - "{{ ansible_env.GOPATH }}/src/github.com/sstephenson"
-    - "{{ ansible_env.GOPATH }}/src/github.com/opencontainers"
-
-- name: install Go tools and dependencies
-  shell: /usr/bin/go get -u "github.com/{{ item }}"
-  with_items:
-    - tools/godep
-    - onsi/ginkgo/ginkgo
-    - onsi/gomega
-    - cloudflare/cfssl/cmd/...
-    - jteeuwen/go-bindata/go-bindata
-    - cpuguy83/go-md2man

contrib/test/integration/main.yml

@@ -1,89 +0,0 @@
-- hosts: all
-  remote_user: root
-  vars_files:
-    - "{{ playbook_dir }}/vars.yml"
-  tags:
-    - setup
-  tasks:
-    - name: set up the system
-      include: system.yml
-
-    - name: install Golang tools
-      include: golang.yml
-      vars:
-        version: "1.8.4"
-
-    - name: clone build and install bats
-      include: "build/bats.yml"
-
-    - name: clone build and install cri-tools
-      include: "build/cri-tools.yml"
-      vars:
-        cri_tools_git_version: "b42fc3f364dd48f649d55926c34492beeb9b2e99"
-
-    - name: clone build and install kubernetes
-      include: "build/kubernetes.yml"
-      vars:
-        k8s_git_version: "cri-o-node-e2e-patched-logs"
-        k8s_github_fork: "runcom"
-        crio_socket: "/var/run/crio.sock"
-
-    - name: clone build and install runc
-      include: "build/runc.yml"
-
-    - name: clone build and install networking plugins
-      include: "build/plugins.yml"
-
-- hosts: all
-  remote_user: root
-  vars_files:
-    - "{{ playbook_dir }}/vars.yml"
-  tags:
-    - integration
-    - e2e
-  tasks:
-    - name: clone build and install cri-o
-      include: "build/cri-o.yml"
-
-- hosts: all
-  remote_user: root
-  vars_files:
-    - "{{ playbook_dir }}/vars.yml"
-  tags:
-    - integration
-  tasks:
-    - name: clone build and install cri-tools
-      include: "build/cri-tools.yml"
-      vars:
-        force_clone: True
-        cri_tools_git_version: "a9e38a4a000bc1a4052fb33de1c967b8cfe9ad40"
-    - name: run cri-o integration tests
-      include: test.yml
-
-- hosts: all
-  remote_user: root
-  vars_files:
-    - "{{ playbook_dir }}/vars.yml"
-  tags:
-    - e2e
-  tasks:
-    - name: install Golang tools
-      include: golang.yml
-      vars:
-          version: "1.9.2"
-    - name: setup critest
-      include: "build/cri-tools.yml"
-      vars:
-          force_clone: True
-          cri_tools_git_version: "a9e38a4a000bc1a4052fb33de1c967b8cfe9ad40"
-    - name: run critest validation and benchmarks
-      include: critest.yml
-    - name: clone build and install kubernetes
-      include: "build/kubernetes.yml"
-      vars:
-          force_clone: True
-          k8s_git_version: "release-1.9"
-          k8s_github_fork: "kubernetes"
-          crio_socket: "/var/run/crio/crio.sock"
-    - name: run k8s e2e tests
-      include: e2e.yml

contrib/test/integration/results.yml

@@ -1,62 +0,0 @@
----
-# vim-syntax: ansible
-
-- hosts: '{{ hosts | default("all") }}'
-  vars_files:
-    - "{{ playbook_dir }}/vars.yml"
-  vars:
-    _result_filepaths: []  # do not use
-    _dstfnbuff: []  # do not use
-  tasks:
-    - name: The crio_integration_filepath is required
-      tags:
-          - integration
-      set_fact:
-       _result_filepaths: "{{ _result_filepaths + [crio_integration_filepath] }}"
-
-    - name: The crio_node_e2e_filepath is required
-      tags:
-          - e2e
-      set_fact:
-       _result_filepaths: "{{ _result_filepaths + [crio_node_e2e_filepath] }}"
-
-    - name: Verify expectations
-      assert:
-        that:
-          - 'result_dest_basedir | default(False, True)'
-          - '_result_filepaths | default(False, True)'
-          - '_dstfnbuff == []'
-          - 'results_fetched is undefined'
-
-    - name: Results directory exists
-      file:
-          path: "{{ result_dest_basedir }}"
-          state: directory
-      delegate_to: localhost
-
-    - name: destination file paths are buffered for overwrite-checking and jUnit conversion
-      set_fact:
-          _dstfnbuff: >
-              {{ _dstfnbuff |
-                 union( [result_dest_basedir ~ "/" ~ inventory_hostname ~ "/" ~ item | basename] ) }}
-      with_items: '{{ _result_filepaths }}'
-
-    - name: Overwriting existing results assumed very very bad
-      fail:
-          msg: "Cowardly refusing to overwrite {{ item }}"
-      when: item | exists
-      delegate_to: localhost
-      with_items: '{{ _dstfnbuff }}'
-
-    # fetch module doesn't support directories
-    - name: Retrieve results from all hosts
-      synchronize:
-          checksum: True  # Don't rely on date/time being in sync
-          archive: False  # Don't bother with permissions or times
-          copy_links: True  # We want files, not links to files
-          recursive: True
-          mode: pull
-          dest: '{{ result_dest_basedir }}/{{ inventory_hostname }}/'  # must end in /
-          src: '{{ item }}'
-      register: results_fetched
-      with_items: '{{ _result_filepaths }}'

contrib/test/integration/system.yml

@@ -1,117 +0,0 @@
----
-
-- name: Make sure we have all required packages
-  package:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - container-selinux
-    - curl
-    - device-mapper-devel
-    - expect
-    - findutils
-    - gcc
-    - git
-    - glib2-devel
-    - glibc-devel
-    - glibc-static
-    - gpgme-devel
-    - hostname
-    - iproute
-    - iptables
-    - krb5-workstation
-    - libassuan-devel
-    - libffi-devel
-    - libgpg-error-devel
-    - libguestfs-tools
-    - libseccomp-devel
-    - libvirt-client
-    - libvirt-python
-    - libxml2-devel
-    - libxslt-devel
-    - make
-    - mlocate
-    - nfs-utils
-    - nmap-ncat
-    - oci-register-machine
-    - oci-systemd-hook
-    - oci-umount
-    - openssl
-    - openssl-devel
-    - ostree-devel
-    - pkgconfig
-    - python
-    - python2-boto
-    - python2-crypto
-    - python-devel
-    - python-virtualenv
-    - PyYAML
-    - redhat-rpm-config
-    - rpcbind
-    - rsync
-    - sed
-    - skopeo-containers
-    - socat
-    - tar
-    - wget
-  async: 600
-  poll: 10
-
-- name: Add Btrfs for Fedora
-  package:
-    name: "{{ item }}"
-    state: present
-  with_items:
-   - btrfs-progs-devel
-  when: ansible_distribution in ['Fedora']
-
-- name: Update all packages
-  package:
-    name: '*'
-    state: latest
-  async: 600
-  poll: 10
-
-- name: Setup swap to prevent kernel firing off the OOM killer
-  shell: |
-    truncate -s 8G /root/swap && \
-    export SWAPDEV=$(losetup --show -f /root/swap | head -1) && \
-    mkswap $SWAPDEV && \
-    swapon $SWAPDEV && \
-    swapon --show
-
-- name: ensure directories exist as needed
-  file:
-    path: "{{ item }}"
-    state: directory
-  with_items:
-    - /opt/cni/bin
-    - /etc/cni/net.d
-
-- name: set sysctl vm.overcommit_memory=1 for CentOS
-  sysctl:
-    name: vm.overcommit_memory
-    state: present
-    value: 1
-  when: ansible_distribution == 'CentOS'
-
-- name: inject hostname into /etc/hosts
-  lineinfile:
-    dest: /etc/hosts
-    line: '{{ ansible_default_ipv4.address }} {{ ansible_nodename }}'
-    insertafter: 'EOF'
-    regexp: '{{ ansible_default_ipv4.address }}\s+{{ ansible_nodename }}'
-    state: present
-
-- name: Flush the iptables
-  command: iptables -F
-
-- name: Enable localnet routing
-  command: sysctl -w net.ipv4.conf.all.route_localnet=1
-
-- name: Add masquerade for localhost
-  command: iptables -t nat -I POSTROUTING -s 127.0.0.1 ! -d 127.0.0.1 -j MASQUERADE
-
-- name: Update the kernel cmdline to include quota support
-  command: grubby --update-kernel=ALL --args="rootflags=pquota"
-  when: ansible_distribution in ['RedHat', 'CentOS']

contrib/test/integration/test.yml

@@ -1,25 +0,0 @@
----
-
-- name: Make testing output verbose so it can be converted to xunit
-  lineinfile:
-    dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/hack/make-rules/test.sh"
-    line: ' go test -v "${goflags[@]:+${goflags[@]}}" \'
-    regexp: ' go test \"\$'
-    state: present
-
-- name: set extra storage options
-  set_fact:
-    extra_storage_opts: " --storage-opt overlay.override_kernel_check=1"
-  when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
-
-- name: ensure directory exists for e2e reports
-  file:
-    path: "{{ artifacts }}"
-    state: directory
-
-- name: run integration tests
-  shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTIONS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration >& {{ artifacts }}/testout.txt"
-  args:
-    chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o"
-  async: 5400
-  poll: 30

contrib/test/integration/vars.yml

@@ -1,8 +0,0 @@
----
-
-# For results.yml Paths use rsync 'source' conventions
-artifacts: "/tmp/artifacts"  # Base-directory for collection
-crio_integration_filepath: "{{ artifacts }}/testout.txt"
-crio_node_e2e_filepath: "{{ artifacts }}/junit_01.xml"
-result_dest_basedir: '{{ lookup("env","WORKSPACE") |
-                         default(playbook_dir, True) }}/artifacts'

contrib/test/main.yml

@@ -0,0 +1,53 @@
+---
+
+- hosts: '{{ subjects | default("all") }}'
+  gather_facts: False  # requires ansible-dependencies
+  tags:  # FIXME: fudging past origin-ci use of tags
+    - e2e
+    - integration
+
+  roles:
+    - ansible_dependencies
+
+
+- hosts: '{{ subjects | default("all") }}'
+  gather_facts: True
+  gather_subset: network
+  vars_files:
+    - "{{ playbook_dir }}/vars.yml"
+
+  tags:  # FIXME: fudging past origin-ci use of tags
+    - e2e
+    - integration
+
+  pre_tasks:
+    - name: Subject's bring in group-vars based on their ansible_distribution
+      group_by:
+        key: "{{ ansible_distribution }}"
+
+  roles:
+    - role: yumrepos
+
+    - role: test_subject
+
+    - role: runscript
+      execute: 'setup.sh'
+
+    - role: runscript
+      execute: 'build.sh'
+      basedir: '{{ cri_o_dest_path }}'
+
+    - role: runscript
+      execute: "install.sh"
+      basedir: '{{ cri_o_dest_path }}'
+
+    - role: runscript
+      execute: "cri-o_smoke_test.sh"
+
+    - role: runscript
+      execute: "cri-o_integration_test.sh"
+      basedir: '{{ cri_o_dest_path }}'
+
+  post_tasks:
+    - name: Execute Kubernetes e2e tests
+      include: includes/e2e.yml

contrib/test/requirements.txt

@@ -1,54 +1,53 @@
-# Pip requirements file for Ansible-based integration-testing environment.
-# Intended to be utilized by venv-ansible-playbook.sh script
-#
 # N/B: Hashes are required here | versions frozen for stability
 
-ansible==2.3.1.0 --hash=sha256:cd4b8f53720fcd0c351156b840fdd15ecfbec22c951b5406ec503de49d40b9f5
+ansible==2.4.1.0 --hash=sha256:da61afb29cc5bd6bc4737a2da06e673fb6fccc3ae2685130d19ab3a8e404fb6a
 
-asn1crypto==0.22.0 --hash=sha256:d232509fefcfcdb9a331f37e9c9dc20441019ad927c7d2176cf18ed5da0ba097 \
+ansible-lint==3.4.17 --hash=sha256:9cebc110019f52a7dd66cb785d99d43b556f246c3046661b00c7bcfe74a9504d
-                   --hash=sha256:cbbadd640d3165ab24b06ef25d1dca09a3441611ac15f6a6b452474fdf0aed1a
 
-bcrypt==3.1.3 --hash=sha256:05b35b9842b009b44496fa5433ce462f69966291e50fbd471dbb427f399f748f \
+asn1crypto==0.23.0 --hash=sha256:654b7db3b120e23474e9a1e5e38d268c77e58a9e17d2cb595456c37309846494
-              --hash=sha256:6645c8d0ad845308de3eb9be98b6fd22a46ec5412bfc664a423e411cdd8f5488
 
-cffi==1.10.0 --hash=sha256:c49187260043bd4c1d6a52186f9774f17d9b1da0a406798ebf4bfc12da166ade \
+bcrypt==3.1.4 --hash=sha256:a005ed6163490988711ff732386b08effcbf8df62ae93dd1e5bda0714fad8afb \
-             --hash=sha256:b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5
+              --hash=sha256:2788c32673a2ad0062bea850ab73cffc0dba874db10d7a3682b6f2f280553f20 \
+              --hash=sha256:49e96267cd9be55a349fd74f9852eb9ae2c427cd7f6455d0f1765d7332292832 \
+              --hash=sha256:ae35dbcb6b011af6c840893b32399252d81ff57d52c13e12422e16b5fea1d0fb
 
-cryptography==1.9 --hash=sha256:5518337022718029e367d982642f3e3523541e098ad671672a90b82474c84882
+cffi==1.11.2 --hash=sha256:89829f5cfbcb5ad568a3d61bd23a8e33ad69b488d8f6a385e0097a4c20742a9b \
+             --hash=sha256:d7461ef8671ae40f991384bbc4a6b1b79f4e7175d8052584be44041996f46517 \
+             --hash=sha256:062c66dabc3faf8e0db1ca09a6b8e308846e5d35f43bed1a68c492b0d96ac171 \
+             --hash=sha256:5f96c92d5f5713ccb71e76dfa14cf819c59ecb9778e94bcb541e13e6d96d1ce5
 
-enum34==1.1.6 --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \
+cryptography==2.1.3 --hash=sha256:35eb35340fdc0b772301f9de985db8d732f3c79dbd647d06b9a8e4e111b53950 \
-              --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1
+                    --hash=sha256:1fc1c6ad9f04871399de407a4f0f555adba5c7ec68068fd27d7ceee9e493755c \
+                    --hash=sha256:2d72c8cd1e2be9942052b85b1481c74b2eb36780889696ce66afe602c04b9c67 \
+                    --hash=sha256:791e228b5df8f124bfa33384195864cb9f5420b619580258d9002f14e625312e
 
-idna==2.5 --hash=sha256:cc19709fd6d0cbfed39ea875d29ba6d4e22c0cebc510a76d6302a28385e8bb70 \
+enum34==1.1.6 --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a
-          --hash=sha256:3cb5ce08046c4e3a560fc02f138d0ac63e00f8ce5901a56b32ec8b7994082aab
 
-ipaddress==1.0.18 --hash=sha256:d34cf15d95ce9a734560f7400a8bd2ac2606f378e2a1d0eadbf1c98707e7c74a \
+idna==2.6 --hash=sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4
-                  --hash=sha256:5d8534c8e185f2d8a1fda1ef73f2c8f4b23264e8e30063feeb9511d492a413e1
 
-Jinja2==2.9.6 --hash=sha256:2231bace0dfd8d2bf1e5d7e41239c06c9e0ded46e70cc1094a0aa64b0afeb054 \
+ipaddress==1.0.18 --hash=sha256:5d8534c8e185f2d8a1fda1ef73f2c8f4b23264e8e30063feeb9511d492a413e1
-              --hash=sha256:ddaa01a212cd6d641401cb01b605f4a4d9f37bfc93043d7f760ec70fb99ff9ff
+
+Jinja2==2.10 --hash=sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd
 
 MarkupSafe==1.0 --hash=sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665
 
-paramiko==2.2.1 --hash=sha256:9c9402377ba8594889aab1e44a13b78eda685eb2145dc00b2353b4fbb25088cf \
+paramiko==2.4.0 --hash=sha256:8851e728e8b7590989e68e3936c48ee3ca4dad91d29e3d7ff0305b6c5fc582db
-                --hash=sha256:ff94ae65379914ec3c960de731381f49092057b6dd1d24d18842ead5a2eb2277
 
-pyasn1==0.2.3 --hash=sha256:0439b9bd518418260c2641a571f0e07fce4370cab13b68f19b5e023306c03cad \
+pyasn1==0.3.7 --hash=sha256:16e896433f84575f0636cd9aa8b24659689268a62e00f17235e1fc23c6b00b25
-              --hash=sha256:738c4ebd88a718e700ee35c8d129acce2286542daa80a82823a7073644f706ad
 
-pycparser==2.17 --hash=sha256:0aac31e917c24cb3357f5a4d5566f2cc91a19ca41862f6c3c22dc60a629673b6
+pycparser==2.18 --hash=sha256:99a8ca03e29851d96616ad0404b4aad7d9ee16f25c9f9708a11faf2810f7b226
 
 pycrypto==2.6.1 --hash=sha256:f2ce1e989b272cfcb677616763e0a2e7ec659effa67a88aa92b3a65528f60a3c
 
-PyNaCl==1.1.2 --hash=sha256:57314a7bad4bd39501dc622942f9921923673e52e126b0fc4f0214b5d25d619a \
+PyNaCl==1.2.0 --hash=sha256:8e194ea19c447c4caa94a84316412ad11cfb61f029d408fd4bdc1164ec694578 \
-              --hash=sha256:32f52b754abf07c319c04ce16905109cab44b0e7f7c79497431d3b2000f8af8c
+              --hash=sha256:b83e4232b43a52c8802234d575f992f82c1e9c466acd911983613a3823c4dc4e \
+              --hash=sha256:189410422028e7b0543dee6aca3da026bbd66bbad078143c46c5a3faf2733acb \
+              --hash=sha256:77c3b6d6fbf8b2137d41be9aed9eff30232287aeba00a6d353aa48fc9de4c55e
 
 PyYAML==3.12 --hash=sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab
 
-six==1.10.0 --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \
+six==1.11.0 --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb
-            --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a
 
-virtualenv==15.1.0 --hash=sha256:39d88b533b422825d644087a21e78c45cf5af0ef7a99a1fc9fbb7b481e5c85b0 \
+virtualenv==15.1.0 --hash=sha256:39d88b533b422825d644087a21e78c45cf5af0ef7a99a1fc9fbb7b481e5c85b0
-                   --hash=sha256:02f8102c2436bb03b3ee6dede1919d1dac8a427541652e5ec95171ec8adbc93a
 
 pip==9.0.1 --hash=sha256:690b762c0a8460c303c089d5d0be034fb15a5ea2b75bdf565f40421f542fefb0

contrib/test/roles/ansible_dependencies/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+
+# Required rpms for ansible to function, missing packages are ignored
+ansible_dependencies: >
+    python python2 libselinux-python policycoreutils-python
+    python2-dnf python3-dnf python-pycurl python2-pycurl
+    python3-pycurl python2-simplejson python-simplejson

contrib/test/roles/ansible_dependencies/tasks/main.yml

@@ -0,0 +1,14 @@
+---
+
+- name: Input expectations are verified
+  assert:
+    that:
+        - 'inventory_hostname != "localhost"'
+        - 'ansible_dependencies | default("", True) | trim | length'
+        # Contains fix for shell-module 'chdir' option not being passed through
+        - 'ansible_version.string | version_compare("2.4.1", ">=")'
+
+- name: Low-level Ansible setup module-dependencies are installed
+  raw: $(type -P dnf || type -P yum || echo ) install -y {{ ansible_dependencies }}
+  failed_when: False  # Any missing & required packages will show up as failures in setup
+  changed_when: True

contrib/test/roles/runscript/files

@@ -0,0 +1 @@
+../../scripts

contrib/test/roles/runscript/tasks/main.yml

@@ -0,0 +1,54 @@
+---
+
+- name: Input expectations are verified
+  assert:
+    that:
+        - 'execute | default("", True) | trim | length'
+        - 'artifacts | default("", True) | trim | length'
+        - 'role_path ~ "/files/" ~ execute | is_file'
+
+- name: The script base-directory path defaults to artifacts
+  set_fact:
+    basedir: "{{ artifacts }}"
+  when: basedir is undefined
+
+- name: Script base-directory exists
+  file:
+    path: "{{ basedir }}"
+    state: directory
+
+- name: Script artifacts directory exists
+  file:
+    path: "{{ artifacts }}"
+    state: directory
+
+- debug:
+    msg: 'Script {{ execute }} will be sent to subject-host, & run from {{ basedir }}) with arguments: {{ ansible_distribution }} {{ artifacts }}'
+
+- block:
+
+    - name: The script is executed
+      script: '{{ playbook_dir }}/scripts/{{ execute }} {{ ansible_distribution }}'
+      args:
+        chdir: "{{ basedir }}"
+        creates: "/var/tmp/{{ execute }}_done"
+      register: result
+
+    - name: The script's touchstone file is touched
+      file:
+        path: "/var/tmp/{{ execute }}_done"
+        state: touch
+      when: result | success
+
+  always:
+
+    - name: The script result is logged
+      blockinfile:
+        path: '{{ artifacts }}/{{ execute }}.txt'
+        marker: '# {mark} execution result of cri-o/contrib/test/scripts/{{ execute }} at {{ ansible_date_time.iso8601 }}'
+        block: '{{ lookup("template", role_path ~ "/templates/result_format.j2") }}'
+        create: true
+      when: '"stdout" in result'
+
+- fail:
+  when: result | failed

contrib/test/roles/runscript/templates/result_format.j2

@@ -0,0 +1,15 @@
+STDOUT:
+-------
+.
+{{ result.stdout }}
+.
+STDERR:
+-------
+.
+{{ result.stderr }}
+.
+EXIT:
+-----
+.
+{{ result.rc }}
+.

contrib/test/roles/test_subject/defaults/main.yml

@@ -0,0 +1,12 @@
+---
+
+update_install_timeout: "{{ 10 * 60 }}"
+
+common_rpms: "python-devel, redhat-rpm-config, rpm-build, yum-utils, iptables, git, rsync, wget"
+
+rhel_centos_rpms: "python34, python34-devel, python-virtualenv, gcc, openssl-devel, libffi-devel, libselinux-python, yum-utils, python-pycurl, bats"
+
+packages:
+    Fedora: "{{ common_rpms }}, python3-devel, btrfs-progs-devel"
+    RedHat: "{{ common_rpms }}, {{ rhel_centos_rpms }}"
+    CentOS: "{{ common_rpms }}, {{ rhel_centos_rpms }}"

contrib/test/roles/test_subject/tasks/main.yml

@@ -0,0 +1,77 @@
+---
+
+- name: Verify expectations
+  assert:
+    that:
+      - 'artifacts | default("", True) | trim | length'
+      - 'go_path | default("", True) | trim | length'
+      - 'cri_o_dest_path | default("", True) | trim | length'
+      - 'cri_o_src_path | default("", True) | trim | length'
+      - 'update_install_timeout | int > 0'
+      - 'packages[ansible_distribution] | default("", True) | trim | length'
+
+- name: All packages are updated
+  package:
+    name: '*'
+    state: latest
+  async: '{{ update_install_timeout }}'
+  poll: 5
+
+- name: Required packages are installed
+  package:
+    name: "{{ packages[ansible_distribution]  }}"
+    state: present
+  async: '{{ update_install_timeout }}'
+  poll: 5
+
+- name: Hostname exists in /etc/hosts
+  blockinfile:
+    dest: /etc/hosts
+    block: '{{ ansible_default_ipv4.address }} {{ ansible_nodename }}'
+    state: present
+
+- name: Directory exists for setup logs
+  file:
+    path: "{{ artifacts }}"
+    state: directory
+
+- name: The cri-o repository directory exists
+  file:
+    path: "{{ cri_o_dest_path }}"
+    state: directory
+    mode: 0777
+
+- name: Synchronize cri-o from control-host to remote subject
+  synchronize:
+    archive: False
+    checksum: True
+    delete: True
+    dest: "{{ cri_o_dest_path }}/"
+    links: True
+    recursive: True
+    src: "{{ cri_o_src_path }}/"
+    times: True
+  # This task is excessively noisy, logging every change to every file :(
+  no_log: True
+
+- name: Sysctl's are set to 1
+  sysctl:
+    name: '{{ item }}'
+    state: present
+    value: 1
+  with_items:
+    - "vm.overcommit_memory"
+    - "net.ipv4.conf.all.route_localnet"
+
+- name: Flush the iptables
+  iptables:
+    flush: True
+
+- name: Add masquerade for localhost
+  iptables:
+    table: "nat"
+    action: "insert"
+    chain: "POSTROUTING"
+    source: "127.0.0.1"
+    destination: "!127.0.0.1"
+    jump: "MASQUERADE"

contrib/test/roles/yumrepos/defaults/main.yml

@@ -0,0 +1,25 @@
+---
+
+# There are situations where default repositories are broken.
+# Setting this true causes ALL subscription-manager supplied repos
+# to be disabled.
+disable_all_rh_repos: False
+
+# This is the opposite of ``disable_all_rh_repos`` (above).  It's
+# a list of subscription-manager supplied repos to explicitly enable.
+enable_rh_repos: []
+
+# When neither of the above meet testing requirements, or additional
+# local repositories should be added, this specifies them.  Each item
+# in the list is a dictionary of arguments to the standard
+# ``yum_repository`` Ansible module.  e.g.
+#
+# yum_repos:
+#     - name: "My special Repo"
+#       baseurl: "https://my.special.repo.example.com/"
+#       gpgcheck: False
+#       includepkgs: rocket2moon
+#     - name: "Other special repo"
+#       baseurl: "https://you.get.the.idea"
+#
+yum_repos: []

contrib/test/roles/yumrepos/tasks/main.yml

@@ -0,0 +1,42 @@
+---
+
+- assert:
+    that:
+        - 'ansible_distribution in ["RedHat","Fedora","CentOS"]'
+        - >
+            disable_all_rh_repos in [True,False] or
+            enable_rh_repos | default() not in [None,'',{}] or
+            yum_repos | default() not in [None,'',{}]
+
+# There are situations where default repositories are broken
+- name: All RH repositories are disabled in subscription manager
+  command: /usr/sbin/subscription-manager repos --disable=*
+  register: result
+  # The goal is disabling all, ignore failure when no repos enabled.
+  failed_when: >
+    result | failed and
+    not (result.stdout | search("^Error: '\\*' does not match"))
+  when: ansible_distribution == "RedHat" and
+        "subscribed" in group_names and
+        disable_all_rh_repos | default(False)
+
+- name: Select RH repos are enabled
+  command: /usr/sbin/subscription-manager repos --enable={{ item }}
+  when: ansible_distribution == "RedHat"
+  with_items: '{{ enable_rh_repos | default([]) }}'
+
+- name: yum repositories are setup and enabled
+  yum_repository:
+    name: "{{ item.name }}"
+    baseurl: "{{ item.baseurl }}"
+    description: "Ansible added {{ item.name }} repo"
+    gpgcheck: "{{ item.gpgcheck | default(True) }}"
+    exclude: "{{ item.excludepkgs | default(omit) }}"
+    includepkgs: "{{ item.includepkgs | default(omit) }}"
+    metadata_expire: 900  # quarter-hour
+    protect: "{{ item.protect | default(False) }}"
+    # Subscription manager can't disable non-redhat repos.
+    enabled: True
+    state: present
+  when: item.name is defined
+  with_items: '{{ yum_repos | default([]) }}'

contrib/test/scripts/build.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -ex
+
+if [ "$1" == "RedHat" ] || [ "$1" == "CentOS" ] || [ "$1" == "Fedora" ]
+then
+    make clean-rpm
+    make test-rpm
+else
+    echo "Distro $1 not supported yet"
+    exit 1
+fi

contrib/test/scripts/cri-o_integration_test.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -ex
+
+# Set by 'runscript' role
+DISTRO="$1"
+ARTIFACTS="$2"
+
+# Make sure it's not running
+( systemctl is-active cri-o && systemctl stop cri-o ) || true
+
+# FIXME: This should use the installed integration-tests package (not avail. on RHEL yet)
+make test-binaries  # bin2img, copyimg, checkseccomp
+# Override defaults in test/helpers.bash
+export CRIO_ROOT=$(realpath "$PWD/..")
+export CRIO_BINARY='/usr/bin/crio'
+export CONMON_BINARY='/usr/libexec/crio/conmon'
+export PAUSE_BINARY='/usr/libexec/crio/pause'
+export CRIO_CNI_PLUGIN='/usr/libexec/cni/'
+
+if [ "$DISTRO" == "RedHat" ] || [ "$DISTRO" == "Fedora" ]
+then
+    export STORAGE_OPTIONS='--storage-driver=overlay --storage-opt overlay.override_kernel_check=1'
+else
+    export export STORAGE_OPTIONS='--storage-driver=overlay'
+fi
+
+./test/test_runner.sh | tee /tmp/artifacts/integration_results.txt"

contrib/test/scripts/cri-o_smoke_test.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -x
+
+# Restarting CRI-O service
+systemctl --no-pager restart cri-o
+
+# Dump the CRI-O service journal
+journalctl --unit cri-o --no-pager
+
+# Fail if CRI-O service is not active
+systemctl is-active cri-o || exit $?
+
+runc --version
+
+crioctl --version
+
+crioctl info
+
+crioctl runtimeversion
+
+crioctl image pull busybox

contrib/test/scripts/install.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -ex
+
+if [ "$1" == "RedHat" ] || [ "$1" == "Fedora" ] || [ "$1" == "CentOS" ]
+then
+    rm -f *.src.rpm;
+    $(type -P dnf || type -P yum) install -y $(find -name '*.rpm');
+else
+    echo "Distro $1 not supported yet"
+    exit 1
+fi

contrib/test/scripts/setup.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -ex
+
+if [ "$1" == "RedHat" ] || [ "$1" == "CentOS" ] || [ "$1" == "Fedora" ]
+then
+    iptables -F
+    sysctl -w net.ipv4.conf.all.route_localnet=1
+    iptables -t nat -I POSTROUTING -s 127.0.0.1 ! -d 127.0.0.1 -j MASQUERADE
+
+    if [ "$1" == "RedHat" ] || [ "$1" == "CentOS" ]
+    then
+        grubby --update-kernel=ALL --args="rootflags=pquota"
+    fi
+else
+    echo "Distro $1 not supported yet"
+    exit 1
+fi

contrib/test/vars.yml

@@ -0,0 +1,13 @@
+---
+
+# Base directory for all go-related source, build, and install.
+go_path: "/go"
+
+# Absolute path on control-host where the cri-o source exists
+cri_o_src_path: "{{ playbook_dir }}/../../"
+
+# Absolute path on subject-host where cri-o source is expected
+cri_o_dest_path: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-o"
+
+# For results.yml Paths use rsync 'source' conventions
+artifacts: "/tmp/artifacts"  # Base-directory for collection

contrib/test/venv-ansible-playbook.sh

@@ -1,106 +0,0 @@
-#!/bin/bash
-
-# example usage
-#   $ ./venv-ansible-playbook.sh \
-#               -i 192.168.169.170 \
-#               --private-key=/path/to/key \
-#               --extra-vars "pullrequest=42" \
-#               --extra-vars "commit=abcd1234" \
-#               --user root \
-#               --verbose \
-#               $PWD/crio-integration-playbook.yaml
-
-# All errors are fatal
-set -e
-
-SCRIPT_PATH=`realpath $(dirname $0)`
-REQUIREMENTS="$SCRIPT_PATH/requirements.txt"
-
-echo
-
-if ! type -P virtualenv &> /dev/null
-then
-    echo "Could not find required 'virtualenv' binary installed on system."
-    exit 1
-fi
-
-if [ "$#" -lt "1" ]
-then
-    echo "No ansible-playbook command-line options specified."
-    echo "usage: $0 -i whatever --private-key=something --extra-vars foo=bar playbook.yml"
-    exit 2
-fi
-
-# Avoid dirtying up repository, keep execution bits confined to a known location
-if [ -z "$WORKSPACE" ] || [ ! -d "$WORKSPACE" ]
-then
-    export WORKSPACE="$(mktemp -d)"
-    echo "Using temporary \$WORKSPACE=\"$WORKSPACE\" for execution environment."
-    echo "Directory will be removed upon exit.  Export this variable with path"
-    echo "to an existing directory to preserve contents."
-    trap 'rm -rf "$WORKSPACE"' EXIT
-else
-    echo "Using existing \$WORKSPACE=\"$WORKSPACE\" for execution environment."
-    echo "Directory will be left as-is upon exit."
-    # Don't recycle cache, next job may have different requirements
-    trap 'rm -rf "$PIPCACHE"' EXIT
-fi
-
-# Create a directory to contain logs and test artifacts
-export ARTIFACTS=$(mkdir -pv $WORKSPACE/artifacts | tail -1 | cut -d \' -f 2)
-[ -d "$ARTIFACTS" ] || exit 3
-
-# All command failures from now on are fatal
-set -e
-echo
-echo "Bootstrapping trusted virtual environment, this may take a few minutes, depending on networking."
-echo "(logs: \"$ARTIFACTS/crio_venv_setup_log.txt\")"
-echo
-
-
-(
-    set -x
-    cd "$WORKSPACE"
-    # When running more than once, make it fast by skipping the bootstrap
-    if [ ! -d "./.cri-o_venv" ]; then
-        # N/B: local system's virtualenv binary - uncontrolled version fixed below
-        virtualenv --no-site-packages --python=python2.7 ./.venvbootstrap
-        # Set up paths to install/operate out of $WORKSPACE/.venvbootstrap
-        source ./.venvbootstrap/bin/activate
-        # N/B: local system's pip binary - uncontrolled version fixed below
-        # pip may not support --cache-dir, force it's location into $WORKSPACE the ugly-way
-        OLD_HOME="$HOME"
-        export HOME="$WORKSPACE"
-        export PIPCACHE="$WORKSPACE/.cache/pip"
-        pip install --force-reinstall --upgrade pip==9.0.1
-        # Undo --cache-dir workaround
-        export HOME="$OLD_HOME"
-        # Install fixed, trusted, hashed versions of all requirements (including pip and virtualenv)
-        pip --cache-dir="$PIPCACHE" install --require-hashes \
-            --requirement "$SCRIPT_PATH/requirements.txt"
-
-        # Setup trusted virtualenv using hashed binary from requirements.txt
-        ./.venvbootstrap/bin/virtualenv --no-site-packages --python=python2.7 ./.cri-o_venv
-        # Exit untrusted virtualenv
-        deactivate
-    fi
-    # Enter trusted virtualenv
-    source ./.cri-o_venv/bin/activate
-    # Upgrade stock-pip to support hashes
-    pip install --force-reinstall --cache-dir="$PIPCACHE" --upgrade pip==9.0.1
-    # Re-install from cache but validate all hashes (including on pip itself)
-    pip --cache-dir="$PIPCACHE" install --require-hashes \
-        --requirement "$SCRIPT_PATH/requirements.txt"
-    # Remove temporary bootstrap virtualenv
-    rm -rf ./.venvbootstrap
-    # Exit trusted virtualenv
-
-) &> $ARTIFACTS/crio_venv_setup_log.txt;
-
-echo
-echo "Executing \"$WORKSPACE/.cri-o_venv/bin/ansible-playbook $@\""
-echo
-
-# Execute command-line arguments under virtualenv
-source ${WORKSPACE}/.cri-o_venv/bin/activate
-${WORKSPACE}/.cri-o_venv/bin/ansible-playbook $@

contrib/test/venv-cmd.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+
+# This wrapper-script reduces the number of python-dependencies needed to execute a command
+# and always executes from a fixed-version / verified environment. It only requires
+# the following (or equivilent) be installed:
+#
+#    python2-virtualenv gcc openssl-devel redhat-rpm-config libffi-devel
+#    python-devel python3-pycurl python-pycurl python2-simplejson util-linux
+#
+# Example usage (where ansible is NOT already installed)
+#
+#   $ ./venv-cmd ansible-playbook --version
+#
+# N/B: You may set $WORKSPACE and/or $ARTIFACTS to control where things are written
+
+# All errors are fatal
+set -e
+
+echo
+
+if [ "$#" -lt "1" ]
+then
+    echo "No command and command-line options specified."
+    echo "usage: $0 <COMMAND> [OPTIONS...]"
+    exit 3
+fi
+
+VENV_DIRNAME=".venv"
+LOCKTIMEOUT_MINUTES="10"
+SCRIPT_NAME=$(basename "$0")
+SCRIPT_DIR=$(dirname `realpath "$0"`)
+[ -n "$WORKSPACE" ] || export WORKSPACE="$SCRIPT_DIR"
+export WORKSPACE=$(realpath $WORKSPACE)
+mkdir -p "$WORKSPACE"
+REQUIREMENTS="$WORKSPACE/requirements.txt"
+
+# Confine this w/in the workspace
+export PIPCACHE="$WORKSPACE/.cache/pip"
+mkdir -p "$PIPCACHE"
+# Don't recycle cache, it may become polluted between runs
+trap 'rm -rf "$PIPCACHE" "$WORKSPACE/${VENV_DIRNAME}bootstrap"' EXIT
+
+[ -n "$ARTIFACTS" ] || export ARTIFACTS="$WORKSPACE/artifacts"
+export ARTIFACTS=$(realpath "$ARTIFACTS")
+mkdir -p "$ARTIFACTS"
+export LOGFILEPATH="$ARTIFACTS/$SCRIPT_NAME.log"
+
+# All command failures from now on are fatal
+set -e
+echo "Bootstrapping trusted virtual environment, this may take a few minutes, depending on networking."
+echo
+echo "-----> Log: \"$LOGFILEPATH\")"
+echo
+
+(
+  if ! flock --nonblock 42
+  then
+      echo "Another $SCRIPT_NAME virtual environment creation process is running."
+      echo "Waiting up to $LOCKTIMEOUT_MINUTES minutes for it to exit."
+      echo
+      if ! flock --timeout $[60 * LOCKTIMEOUT_MINUTES] 42
+      then
+          echo "Could not obtain lock on virtual environment creation"
+          echo
+          exit 9
+      fi
+  fi
+  echo "Virtual environment creation lock acquired"
+  echo
+  (
+    set -x
+    cd "$WORKSPACE"
+    # When running more than once, make it fast by skipping the bootstrap
+    if [ ! -d "./$VENV_DIRNAME" ] || [ ! -r "./$VENV_DIRNAME/.complete" ]; then
+        # N/B: local system's virtualenv binary - uncontrolled version fixed below
+        virtualenv --no-site-packages --python=python2 "./${VENV_DIRNAME}bootstrap"
+        python3 -m venv --copies "./${VENV_DIRNAME}bootstrap"
+        # Set up paths to install/operate out of $WORKSPACE/${VENV_DIRNAME}bootstrap
+        source "./${VENV_DIRNAME}bootstrap/bin/activate"
+        # N/B: local system's pip binary - uncontrolled version fixed below
+        # pip may not support --cache-dir, force it's location into $WORKSPACE the ugly-way
+        OLD_HOME="$HOME"
+        export HOME="$WORKSPACE"
+        pip install --force-reinstall --upgrade pip==9.0.1
+        # Undo --cache-dir workaround
+        export HOME="$OLD_HOME"
+        # Install fixed, trusted, hashed versions of all requirements (including pip and virtualenv)
+        pip --cache-dir="$PIPCACHE" install --force-reinstall --require-hashes \
+            --requirement "$SCRIPT_DIR/requirements.txt"
+        # Setup trusted virtualenv using hashed packages from requirements.txt
+        "./${VENV_DIRNAME}bootstrap/bin/virtualenv" --no-site-packages --python=python2 "./$VENV_DIRNAME"
+        "./${VENV_DIRNAME}bootstrap/bin/python3" -m venv --copies "./$VENV_DIRNAME"
+        # Exit untrusted virtualenv
+        deactivate
+    fi
+    # Enter trusted virtualenv
+    source "./$VENV_DIRNAME/bin/activate"
+    # Upgrade stock-pip to support hashes
+    "./$VENV_DIRNAME/bin/pip" install --force-reinstall --cache-dir="$PIPCACHE" --upgrade pip==9.0.1
+    # Re-install from cache but validate all hashes (including on pip itself)
+    "./$VENV_DIRNAME/bin/pip" --cache-dir="$PIPCACHE" install --require-hashes \
+        --requirement "$SCRIPT_DIR/requirements.txt"
+    [ -r "./$VENV_DIRNAME/.complete" ] || echo "Setup by: $@" > "./$VENV_DIRNAME/.complete"
+  ) &>> "$LOGFILEPATH"
+) 42>>"$LOGFILEPATH"
+
+# Since setup is complete, only kill the bootstrap on exit
+trap 'rm -rf "$WORKSPACE/${VENV_DIRNAME}bootstrap"' EXIT
+
+# Enter trusted virtualenv in this shell
+source "$WORKSPACE/$VENV_DIRNAME/bin/activate"
+echo "Executing $@"
+echo
+"$@"
+deactivate  # just in case