From 20ca436b8c384cb217f4d4ab11694a5d8ce4d431 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Wed, 4 Oct 2017 10:37:37 -0400 Subject: [PATCH] Support testing against remote subjects. It's a severe anti-pattern for a playbook to assume execution always on a specific host. The normal/expected pattern is to execute from a "control host", against an inventory of (possibly-remote) subjects. This doesn't preclude the inventory from only ever containing 'localhost', it simply means the plays and tasks should not assume the inventory contents. This concept is one of the central design-pillars of Ansible's, and tantamount to it's usefulness and flexibility. However, in practice (and by ``integration/readme.md``), plays specify ``- hosts: all`` but assume inventory_hostname == 'localhost' (always). Fix both the playbooks and ``readme.md`` to remove this anti-pattern, while also allowing the control-host to be the subject-host as needed. This is accomplished by ensuring low-level Ansible dependencies are always installed, and writing tasks for steps previously performed externally (in the CI/automation machinery). Also update ``readme.md`` to recommend execution occurs through the ``venv-ansible-playbook.sh`` wrapper to ensure consistent, stable, version-locked execution dependencies on the control-host. Remove ``remote_user: root`` from main, since this is better left to the inventory and command-line. Signed-off-by: Chris Evich --- contrib/test/integration/README.md | 54 ++++++++++++++++++------ contrib/test/integration/build/cri-o.yml | 22 +++++----- contrib/test/integration/github.yml | 27 ++++++++++++ contrib/test/integration/main.yml | 38 +++++++++++++---- contrib/test/integration/results.yml | 2 +- contrib/test/integration/system.yml | 15 +++++++ contrib/test/integration/test.yml | 6 +-- contrib/test/integration/vars.yml | 9 ++++ 8 files changed, 136 insertions(+), 37 deletions(-) create mode 100644 contrib/test/integration/github.yml diff --git a/contrib/test/integration/README.md b/contrib/test/integration/README.md index f13b8b92..c90a5eac 100644 --- a/contrib/test/integration/README.md +++ b/contrib/test/integration/README.md @@ -1,21 +1,49 @@ # Fedora and RHEL Integration and End-to-End Tests This directory contains playbooks to set up for and run the integration and -end-to-end tests for CRI-O on RHEL and Fedora hosts. Two entrypoints exist: +end-to-end tests for CRI-O on RHEL and Fedora hosts. The expected entry-point +is the ``main.yml`` Ansible playbook. - - `main.yml`: sets up the machine and runs tests - - `results.yml`: gathers test output to `/tmp/artifacts` +The control-host: -When running `main.yml`, three tags are present: + - May be the subject. + - Is based on either RHEL/CentOS 6 (or later), or Fedora 24 (or later). + - Runs ``main.yml`` from within the cri-o repository already in the + desired state for testing. - - `setup`: run all tasks to set up the system for testing - - `e2e`: build CRI-O from source and run Kubernetes node E2Es - - `integration`: build CRI-O from source and run the local integration suite +The subject host(s): -The playbooks assume the following things about your system: + - May be the control-host. + - May be executing the ``main.yml`` playbook against itself. + - If RHEL-like, has the ``server``, ``extras``, and ``EPEL`` repositories available + and enabled. + - Has remote password-less ssh configured for direct or sudo access to the root user. - - on RHEL, the server and extras repos are configured and certs are present - - `ansible` is installed and the host is boot-strapped to allow `ansible` to run against it - - the `$GOPATH` is set and present for all shells (*e.g.* written in `/etc/environment`) - - CRI-O is checked out to the correct state at `${GOPATH}/src/github.com/kubernetes-incubator/cri-o` - - the user running the playbook has access to passwordless `sudo` \ No newline at end of file +Execution of the ``main.yml`` playbook: + + - Should occur through the ``cri-o/contrib/test/venv-ansible-playbook.sh`` wrapper. + - Execution may target localhost, or one or more subjects via standard Ansible + inventory arguments. + - Should use a combination (including none) of the following tags: + + - ``setup``: Run all tasks to set up the system for testing. Final state must + be self-contained and independent from other tags (i.e. support + stage-caching). + - ``integration``: Assumes 'setup' previously completed successfully. + May be executed from cached-state of ``setup``. + Not required to execute conicident with other tags. + Must build CRI-O from source and run the + integration test suite. + - ``e2e``: Assumes 'setup' previously completed successfully. May be executed + from cached-state of ``setup``. Not required to execute conicident with + other tags. Must build CRI-O from source and run Kubernetes node + E2E tests. + +``cri-o/contrib/test/venv-ansible-playbook.sh`` Wrapper: + + - Must accepts all of the valid Ansible command-line options. + - Must use version-locked & hashed dependencies as written in ``requirements.txt``. + - Must fully sandbox it's own execution environment except for the following + required packages (or equivalent): ``python2-virtualenv gcc openssl-devel + redhat-rpm-config libffi-devel python-devel libselinux-python rsync + yum-utils python3-pycurl python-simplejson``. diff --git a/contrib/test/integration/build/cri-o.yml b/contrib/test/integration/build/cri-o.yml index fa025035..29172866 100644 --- a/contrib/test/integration/build/cri-o.yml +++ b/contrib/test/integration/build/cri-o.yml @@ -1,42 +1,42 @@ --- -- name: stat the expected cri-o directory +- name: stat the expected cri-o directory and Makefile exists stat: - path: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + path: "{{ cri_o_dest_path }}/Makefile" register: dir_stat -- name: expect cri-o to be cloned already +- name: Verify cri-o Makefile exists in expected location fail: - msg: "Expected cri-o to be cloned at {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o but it wasn't!" - when: not dir_stat.stat.exists + msg: "Expected cri-o to be cloned at {{ cri_o_dest_path }} but it wasn't!" + when: not dir_stat.stat.exists or not dir_stat.stat.isreg - name: install cri-o tools make: target: install.tools - chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + chdir: "{{ cri_o_dest_path }}" - name: build cri-o make: - chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + chdir: "{{ cri_o_dest_path }}" - name: install cri-o make: target: install - chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + chdir: "{{ cri_o_dest_path }}" - name: install cri-o systemd files make: target: install.systemd - chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + chdir: "{{ cri_o_dest_path }}" - name: install cri-o config make: target: install.config - chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + chdir: "{{ cri_o_dest_path }}" - name: install configs copy: - src: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/{{ item.src }}" + src: "{{ cri_o_dest_path }}/{{ item.src }}" dest: "{{ item.dest }}" remote_src: yes with_items: diff --git a/contrib/test/integration/github.yml b/contrib/test/integration/github.yml new file mode 100644 index 00000000..16aef9f4 --- /dev/null +++ b/contrib/test/integration/github.yml @@ -0,0 +1,27 @@ +--- + + +- name: Verify expectations + assert: + that: + - 'cri_o_dest_path is defined' + - 'cri_o_src_path is defined' + +- name: The cri-o repository directory exists + file: + path: "{{ cri_o_dest_path }}" + state: directory + mode: 0777 + +- name: Synchronize cri-o from control-host to remote subject + synchronize: + archive: False + checksum: True + delete: True + dest: "{{ cri_o_dest_path }}/" + links: True + recursive: True + src: "{{ cri_o_src_path }}/" + times: True + # This task is excessively noisy, logging every change to every file :( + no_log: True diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml index ce4a206f..6a60e0e6 100644 --- a/contrib/test/integration/main.yml +++ b/contrib/test/integration/main.yml @@ -1,5 +1,27 @@ -- hosts: all - remote_user: root +--- + +- hosts: '{{ subjects | default("all") }}' + gather_facts: False # Requires low-level ansible-dependencies + tasks: + - name: Ensure low-level ansible-dependencies are installed + raw: $(type -P dnf || type -P yum) install -y python2 python2-dnf libselinux-python git rsync + + - name: Gather only networking facts for speed + setup: + gather_subset: network + + +- hosts: '{{ subjects | default("none") }}' + vars_files: + - "{{ playbook_dir }}/vars.yml" + tags: + - setup + tasks: + - name: CRI-O source is available on every subject + include: github.yml + + +- hosts: '{{ subjects | default("all") }}' vars_files: - "{{ playbook_dir }}/vars.yml" tags: @@ -26,19 +48,18 @@ - name: clone build and install networking plugins include: "build/plugins.yml" -- hosts: all - remote_user: root + +- hosts: '{{ subjects | default("all") }}' vars_files: - "{{ playbook_dir }}/vars.yml" tags: - integration - e2e tasks: - - name: clone build and install cri-o + - name: Build and install cri-o include: "build/cri-o.yml" -- hosts: all - remote_user: root +- hosts: '{{ subjects | default("all") }}' vars_files: - "{{ playbook_dir }}/vars.yml" tags: @@ -47,8 +68,7 @@ - name: run cri-o integration tests include: test.yml -- hosts: all - remote_user: root +- hosts: '{{ subjects | default("all") }}' vars_files: - "{{ playbook_dir }}/vars.yml" tags: diff --git a/contrib/test/integration/results.yml b/contrib/test/integration/results.yml index c9a96abb..96b0f0b4 100644 --- a/contrib/test/integration/results.yml +++ b/contrib/test/integration/results.yml @@ -1,7 +1,7 @@ --- # vim-syntax: ansible -- hosts: '{{ hosts | default("all") }}' +- hosts: '{{ subjects | default("all") }}' vars_files: - "{{ playbook_dir }}/vars.yml" vars: diff --git a/contrib/test/integration/system.yml b/contrib/test/integration/system.yml index ea9280ed..c453c422 100644 --- a/contrib/test/integration/system.yml +++ b/contrib/test/integration/system.yml @@ -70,6 +70,7 @@ state: present with_items: - btrfs-progs-devel + - python2-virtualenv when: ansible_distribution in ['Fedora'] - name: Check / setup swap @@ -110,3 +111,17 @@ - name: Update the kernel cmdline to include quota support command: grubby --update-kernel=ALL --args="rootflags=pquota" when: ansible_distribution in ['RedHat', 'CentOS'] + +- name: Configure the GOPATH environment variable for all users + blockinfile: + path: /etc/environment + block: "GOPATH={{ go_path }}" + create: True + follow: True + +- name: Reset the ansible connection to incorporate /etc/environment changes + meta: reset_connection + +- name: Refresh facts to incorporate /etc/environment changes + setup: + gather_subset: network diff --git a/contrib/test/integration/test.yml b/contrib/test/integration/test.yml index e9023e77..dfa2c30f 100644 --- a/contrib/test/integration/test.yml +++ b/contrib/test/integration/test.yml @@ -15,7 +15,7 @@ extra_storage_opts: " --storage-opt overlay.override_kernel_check=1" when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' -- name: ensure directory exists for e2e reports +- name: ensure directory exists for integration results file: path: "{{ artifacts }}" state: directory @@ -31,9 +31,9 @@ when: not integration_selinux_enabled - name: run integration tests - shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTIONS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration >& {{ artifacts }}/testout.txt" + shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration >& {{ artifacts }}/testout.txt" args: - chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + chdir: "{{ cri_o_dest_path }}" async: 5400 poll: 30 diff --git a/contrib/test/integration/vars.yml b/contrib/test/integration/vars.yml index 801eb1bc..3362f2d3 100644 --- a/contrib/test/integration/vars.yml +++ b/contrib/test/integration/vars.yml @@ -13,6 +13,15 @@ e2e_swap_enabled: True integration_selinux_enabled: True e2e_selinux_enabled: False +# Path to encode into /etc/environment on all hosts +go_path: "/go" + +# Absolute path on control-host where the cri-o source exists +cri_o_src_path: "{{ playbook_dir }}/../../../" + +# Absolute path on subjects where cri-o source is expected +cri_o_dest_path: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-o" + # For results.yml Paths use rsync 'source' conventions artifacts: "/tmp/artifacts" # Base-directory for collection crio_integration_filepath: "{{ artifacts }}/testout.txt"