diff --git a/contrib/test/integration/README.md b/contrib/test/integration/README.md index f13b8b92..f7220005 100644 --- a/contrib/test/integration/README.md +++ b/contrib/test/integration/README.md @@ -1,21 +1,78 @@ # Fedora and RHEL Integration and End-to-End Tests This directory contains playbooks to set up for and run the integration and -end-to-end tests for CRI-O on RHEL and Fedora hosts. Two entrypoints exist: +end-to-end tests for CRI-O on RHEL and Fedora hosts. The expected entry-point +is the ``main.yml`` Ansible playbook. - - `main.yml`: sets up the machine and runs tests - - `results.yml`: gathers test output to `/tmp/artifacts` +##Definitions: -When running `main.yml`, three tags are present: + Control-host: The system from which the ``ansible-playbook`` or + ``venv-ansible-playbook.sh`` command is executed. - - `setup`: run all tasks to set up the system for testing - - `e2e`: build CRI-O from source and run Kubernetes node E2Es - - `integration`: build CRI-O from source and run the local integration suite + Subject-host(s): The target systems, on which actual playbook tasks are + being carried out. -The playbooks assume the following things about your system: +##Topology: - - on RHEL, the server and extras repos are configured and certs are present - - `ansible` is installed and the host is boot-strapped to allow `ansible` to run against it - - the `$GOPATH` is set and present for all shells (*e.g.* written in `/etc/environment`) - - CRI-O is checked out to the correct state at `${GOPATH}/src/github.com/kubernetes-incubator/cri-o` - - the user running the playbook has access to passwordless `sudo` \ No newline at end of file +The control-host: + + - May be the subject. + - Is based on either RHEL/CentOS 6 (or later), or Fedora 24 (or later). + - Runs ``main.yml`` from within the cri-o repository already in the + desired state for testing. + +The subject-host(s): + + - May be the control-host. + - May be executing the ``main.yml`` playbook against itself. + - If RHEL-like, has the ``server``, ``extras``, and ``EPEL`` repositories available + and enabled. + - Has remote password-less ssh configured for access by the control-host. + - When ssh-access is for a regular user, that user has password-less + sudo access to root. + +##Runtime Requirements: + +Execution of the ``main.yml`` playbook: + + - Should occur through the ``cri-o/contrib/test/venv-ansible-playbook.sh`` wrapper. + - Execution may target localhost, or one or more subjects via standard Ansible + inventory arguments. + - Should use a combination (including none) of the following tags: + + - ``setup``: Run all tasks to set up the system for testing. Final state must + be self-contained and independent from other tags (i.e. support + stage-caching). + - ``integration``: Assumes 'setup' previously completed successfully. + May be executed from cached-state of ``setup``. + Not required to execute coincident with other tags. + Must build CRI-O from source and run the + integration test suite. + - ``e2e``: Assumes 'setup' previously completed successfully. May be executed + from cached-state of ``setup``. Not required to execute coincident with + other tags. Must build CRI-O from source and run Kubernetes node + E2E tests. + +``cri-o/contrib/test/venv-ansible-playbook.sh`` Wrapper: + + - May be executed on the control-host to both hide and version-lock playbook + execution dependencies, ansible and otherwise. + - Must accept all of the valid Ansible command-line options. + - Must sandbox dependencies under a python virtual environment ``.cri-o_venv`` + with packages as specified in ``requirements.txt``. + - Requires the control-host has the following fundamental dependencies installed + (or equivalent): ``python2-virtualenv gcc openssl-devel + redhat-rpm-config libffi-devel python-devel libselinux-python rsync + yum-utils python3-pycurl python-simplejson``. + +For example: + +Given a populated '/path/to/inventory' file, a control-host could run: + +./venv-ansible-playbook.sh -i /path/to/inventory ./integration/main.yml + +-or- + +From a subject-host without an inventory: + +./venv-ansible-playbook.sh -i localhost, ./integration/main.yml diff --git a/contrib/test/integration/build/bats.yml b/contrib/test/integration/build/bats.yml index d4ea19c6..ec2900b0 100644 --- a/contrib/test/integration/build/bats.yml +++ b/contrib/test/integration/build/bats.yml @@ -3,12 +3,12 @@ - name: clone bats source repo git: repo: "https://github.com/sstephenson/bats.git" - dest: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats" + dest: "{{ go_path }}/src/github.com/sstephenson/bats" - name: install bats command: "./install.sh /usr/local" args: - chdir: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats" + chdir: "{{ go_path }}/src/github.com/sstephenson/bats" - name: link bats file: diff --git a/contrib/test/integration/build/cri-o.yml b/contrib/test/integration/build/cri-o.yml index fa025035..3c7b2c16 100644 --- a/contrib/test/integration/build/cri-o.yml +++ b/contrib/test/integration/build/cri-o.yml @@ -1,42 +1,42 @@ --- -- name: stat the expected cri-o directory +- name: stat the expected cri-o directory and Makefile exists stat: - path: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" - register: dir_stat + path: "{{ cri_o_dest_path }}/Makefile" + register: crio_stat -- name: expect cri-o to be cloned already +- name: Verify cri-o Makefile exists in expected location fail: - msg: "Expected cri-o to be cloned at {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o but it wasn't!" - when: not dir_stat.stat.exists + msg: "Expected cri-o to be cloned at {{ cri_o_dest_path }}, but its 'Makefile' seems to be missing." + when: not crio_stat.stat.exists or not crio_stat.stat.isreg - name: install cri-o tools make: target: install.tools - chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + chdir: "{{ cri_o_dest_path }}" - name: build cri-o make: - chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + chdir: "{{ cri_o_dest_path }}" - name: install cri-o make: target: install - chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + chdir: "{{ cri_o_dest_path }}" - name: install cri-o systemd files make: target: install.systemd - chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + chdir: "{{ cri_o_dest_path }}" - name: install cri-o config make: target: install.config - chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + chdir: "{{ cri_o_dest_path }}" - name: install configs copy: - src: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/{{ item.src }}" + src: "{{ cri_o_dest_path }}/{{ item.src }}" dest: "{{ item.dest }}" remote_src: yes with_items: diff --git a/contrib/test/integration/build/cri-tools.yml b/contrib/test/integration/build/cri-tools.yml index 9a117f3c..fad1dbfe 100644 --- a/contrib/test/integration/build/cri-tools.yml +++ b/contrib/test/integration/build/cri-tools.yml @@ -3,7 +3,7 @@ - name: clone cri-tools source repo git: repo: "https://github.com/kubernetes-incubator/cri-tools.git" - dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-tools" + dest: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-tools" version: "9ff5e8f78a4182ab8d5ba9bcccdda5f338600eab" - name: install crictl @@ -11,6 +11,6 @@ - name: link crictl file: - src: "{{ ansible_env.GOPATH }}/bin/crictl" + src: "{{ go_path }}/bin/crictl" dest: /usr/bin/crictl state: link diff --git a/contrib/test/integration/build/kubernetes.yml b/contrib/test/integration/build/kubernetes.yml index f724230c..e8b19c57 100644 --- a/contrib/test/integration/build/kubernetes.yml +++ b/contrib/test/integration/build/kubernetes.yml @@ -3,17 +3,17 @@ - name: clone kubernetes source repo git: repo: "https://github.com/runcom/kubernetes.git" - dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + dest: "{{ go_path }}/src/k8s.io/kubernetes" version: "cri-o-patched-1.8" - name: install etcd command: "hack/install-etcd.sh" args: - chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + chdir: "{{ go_path }}/src/k8s.io/kubernetes" - name: build kubernetes make: - chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + chdir: "{{ go_path }}/src/k8s.io/kubernetes" - name: Add custom cluster service file for the e2e testing copy: @@ -23,7 +23,7 @@ After=network-online.target Wants=network-online.target [Service] - WorkingDirectory={{ ansible_env.GOPATH }}/src/k8s.io/kubernetes + WorkingDirectory={{ go_path }}/src/k8s.io/kubernetes ExecStart=/usr/local/bin/createcluster.sh User=root [Install] @@ -35,7 +35,7 @@ content: | #!/bin/bash - export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ ansible_env.GOPATH }}/bin:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/third_party/etcd:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/ + export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ go_path }}/bin:{{ go_path }}/src/k8s.io/kubernetes/third_party/etcd:{{ go_path }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/ export CONTAINER_RUNTIME=remote export CGROUP_DRIVER=systemd export CONTAINER_RUNTIME_ENDPOINT='/var/run/crio.sock --runtime-request-timeout=5m' @@ -47,17 +47,3 @@ export KUBE_ENABLE_CLUSTER_DNS=true ./hack/local-up-cluster.sh mode: "u=rwx,g=rwx,o=x" - -- name: Set kubernetes_provider to be local - lineinfile: - dest: /etc/environment - line: 'KUBERNETES_PROVIDER=local' - regexp: 'KUBERNETES_PROVIDER=' - state: present - -- name: Set KUBECONFIG - lineinfile: - dest: /etc/environment - line: 'KUBECONFIG=/var/run/kubernetes/admin.kubeconfig' - regexp: 'KUBECONFIG=' - state: present diff --git a/contrib/test/integration/build/plugins.yml b/contrib/test/integration/build/plugins.yml index e342a0b9..b344270a 100644 --- a/contrib/test/integration/build/plugins.yml +++ b/contrib/test/integration/build/plugins.yml @@ -3,17 +3,17 @@ - name: clone plugins source repo git: repo: "https://github.com/containernetworking/plugins.git" - dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + dest: "{{ go_path }}/src/github.com/containernetworking/plugins" version: "dcf7368eeab15e2affc6256f0bb1e84dd46a34de" - name: build plugins command: "./build.sh" args: - chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + chdir: "{{ go_path }}/src/github.com/containernetworking/plugins" - name: install plugins copy: - src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/{{ item }}" + src: "{{ go_path }}/src/github.com/containernetworking/plugins/bin/{{ item }}" dest: "/opt/cni/bin" mode: "o=rwx,g=rx,o=rx" remote_src: yes @@ -33,18 +33,18 @@ - name: clone runcom plugins source repo git: repo: "https://github.com/runcom/plugins.git" - dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + dest: "{{ go_path }}/src/github.com/containernetworking/plugins" version: "custom-bridge" force: yes - name: build plugins command: "./build.sh" args: - chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + chdir: "{{ go_path }}/src/github.com/containernetworking/plugins" - name: install custom bridge copy: - src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/bridge" + src: "{{ go_path }}/src/github.com/containernetworking/plugins/bin/bridge" dest: "/opt/cni/bin/bridge-custom" mode: "o=rwx,g=rx,o=rx" remote_src: yes diff --git a/contrib/test/integration/build/runc.yml b/contrib/test/integration/build/runc.yml index 7bb0491d..1dd04f03 100644 --- a/contrib/test/integration/build/runc.yml +++ b/contrib/test/integration/build/runc.yml @@ -3,18 +3,18 @@ - name: clone runc source repo git: repo: "https://github.com/opencontainers/runc.git" - dest: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" + dest: "{{ go_path }}/src/github.com/opencontainers/runc" version: "84a082bfef6f932de921437815355186db37aeb1" - name: build runc make: params: BUILDTAGS="seccomp selinux" - chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" + chdir: "{{ go_path }}/src/github.com/opencontainers/runc" - name: install runc make: target: "install" - chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" + chdir: "{{ go_path }}/src/github.com/opencontainers/runc" - name: link runc file: diff --git a/contrib/test/integration/e2e.yml b/contrib/test/integration/e2e.yml index 41f92757..c0df99ef 100644 --- a/contrib/test/integration/e2e.yml +++ b/contrib/test/integration/e2e.yml @@ -29,7 +29,7 @@ daemon_reload: yes - name: wait for the cluster to be running - command: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default" + command: "{{ go_path }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default" register: kube_poll until: kube_poll | succeeded retries: 100 @@ -51,10 +51,7 @@ &> {{ artifacts }}/e2e.log # Fix vim syntax hilighting: " -- name: disable SELinux - command: setenforce 0 - - name: run e2e tests shell: "{{ e2e_shell_cmd | regex_replace('\\s+', ' ') }}" args: - chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + chdir: "{{ go_path }}/src/k8s.io/kubernetes" diff --git a/contrib/test/integration/github.yml b/contrib/test/integration/github.yml new file mode 100644 index 00000000..16aef9f4 --- /dev/null +++ b/contrib/test/integration/github.yml @@ -0,0 +1,27 @@ +--- + + +- name: Verify expectations + assert: + that: + - 'cri_o_dest_path is defined' + - 'cri_o_src_path is defined' + +- name: The cri-o repository directory exists + file: + path: "{{ cri_o_dest_path }}" + state: directory + mode: 0777 + +- name: Synchronize cri-o from control-host to remote subject + synchronize: + archive: False + checksum: True + delete: True + dest: "{{ cri_o_dest_path }}/" + links: True + recursive: True + src: "{{ cri_o_src_path }}/" + times: True + # This task is excessively noisy, logging every change to every file :( + no_log: True diff --git a/contrib/test/integration/golang.yml b/contrib/test/integration/golang.yml index 63e55697..037fe851 100644 --- a/contrib/test/integration/golang.yml +++ b/contrib/test/integration/golang.yml @@ -16,28 +16,16 @@ - gofmt - godoc -- name: ensure user profile exists - file: - path: "{{ ansible_user_dir }}/.profile" - state: touch - -- name: set up PATH for Go toolchain and built binaries - lineinfile: - dest: "{{ ansible_user_dir }}/.profile" - line: 'PATH={{ ansible_env.PATH }}:{{ ansible_env.GOPATH }}/bin:/usr/local/go/bin' - regexp: '^PATH=' - state: present - - name: set up directories file: - path: "{{ item }}" + path: "{{ go_path }}/src/github.com/{{ item }}" state: directory with_items: - - "{{ ansible_env.GOPATH }}/src/github.com/containernetworking" - - "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator" - - "{{ ansible_env.GOPATH }}/src/github.com/k8s.io" - - "{{ ansible_env.GOPATH }}/src/github.com/sstephenson" - - "{{ ansible_env.GOPATH }}/src/github.com/opencontainers" + - "containernetworking" + - "kubernetes-incubator" + - "k8s.io" + - "sstephenson" + - "opencontainers" - name: install Go tools and dependencies shell: /usr/bin/go get -u "github.com/{{ item }}" diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml index ce4a206f..caa64b12 100644 --- a/contrib/test/integration/main.yml +++ b/contrib/test/integration/main.yml @@ -1,7 +1,56 @@ -- hosts: all - remote_user: root +--- + +- hosts: '{{ subjects | default("localhost") }}' + gather_facts: False # Requires low-level ansible-dependencies + tags: + - always + tasks: + - name: Ansible setup-module dependencies are installed, ignoring errors (setup runs next). + raw: $(type -P dnf || type -P yum) install -y python2 python2-dnf libselinux-python + ignore_errors: True + + - name: Gather only networking facts for speed + setup: + gather_subset: network + + - name: Variables from vars.yml are hauled in after setup + include_vars: "{{ playbook_dir }}/vars.yml" + + - name: Global environment are defined, but can be overriden on a task-by-task basis. + set_fact: + extra_storage_opts: > + {%- if ansible_distribution in ["RedHat", "CentOS"] -%} + "--storage-opt overlay.override_kernel_check=1" + {%- else -%} + "" + {%- endif -%} + environment_variables: + PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:{{ go_path }}/bin:/usr/local/go/bin" + GOPATH: "{{ go_path }}" + KUBERNETES_PROVIDER: "local" + KUBECONFIG: "/var/run/kubernetes/admin.kubeconfig" + CGROUP_MANAGER: "cgroupfs" + # > is YAML for replace non-orphan newlines with single spaces + STORAGE_OPTS: > + --storage-driver=overlay + {% if ansible_distribution in ["RedHat", "CentOS"] %} + --storage-opt overlay.override_kernel_check=1 + {% endif %} + +- hosts: '{{ subjects | default("none") }}' vars_files: - "{{ playbook_dir }}/vars.yml" + tags: + - setup + tasks: + - name: CRI-O source is available on every subject + include: github.yml + + +- hosts: '{{ subjects | default("localhost") }}' + vars_files: + - "{{ playbook_dir }}/vars.yml" + environment: '{{ environment_variables }}' tags: - setup tasks: @@ -26,33 +75,23 @@ - name: clone build and install networking plugins include: "build/plugins.yml" -- hosts: all - remote_user: root - vars_files: - - "{{ playbook_dir }}/vars.yml" - tags: - - integration - - e2e - tasks: - - name: clone build and install cri-o - include: "build/cri-o.yml" -- hosts: all - remote_user: root +- hosts: '{{ subjects | default("localhost") }}' vars_files: - "{{ playbook_dir }}/vars.yml" - tags: - - integration + environment: '{{ environment_variables }}' tasks: + - name: Build and install cri-o + include: "build/cri-o.yml" + tags: + - always + - name: run cri-o integration tests include: test.yml + tags: + - integration -- hosts: all - remote_user: root - vars_files: - - "{{ playbook_dir }}/vars.yml" - tags: - - e2e - tasks: - name: run k8s e2e tests include: e2e.yml + tags: + - e2e diff --git a/contrib/test/integration/results.yml b/contrib/test/integration/results.yml index c9a96abb..96b0f0b4 100644 --- a/contrib/test/integration/results.yml +++ b/contrib/test/integration/results.yml @@ -1,7 +1,7 @@ --- # vim-syntax: ansible -- hosts: '{{ hosts | default("all") }}' +- hosts: '{{ subjects | default("all") }}' vars_files: - "{{ playbook_dir }}/vars.yml" vars: diff --git a/contrib/test/integration/system.yml b/contrib/test/integration/system.yml index d07ae0c8..c84e0f59 100644 --- a/contrib/test/integration/system.yml +++ b/contrib/test/integration/system.yml @@ -63,6 +63,7 @@ state: present with_items: - btrfs-progs-devel + - python2-virtualenv when: ansible_distribution in ['Fedora'] - name: Update all packages diff --git a/contrib/test/integration/test.yml b/contrib/test/integration/test.yml index 593e8a1c..c097343b 100644 --- a/contrib/test/integration/test.yml +++ b/contrib/test/integration/test.yml @@ -5,24 +5,19 @@ - name: Make testing output verbose so it can be converted to xunit lineinfile: - dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/hack/make-rules/test.sh" + dest: "{{ go_path }}/src/k8s.io/kubernetes/hack/make-rules/test.sh" line: ' go test -v "${goflags[@]:+${goflags[@]}}" \' regexp: ' go test \"\$' state: present -- name: set extra storage options - set_fact: - extra_storage_opts: " --storage-opt overlay.override_kernel_check=1" - when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' - -- name: ensure directory exists for e2e reports +- name: ensure directory exists for integration results file: path: "{{ artifacts }}" state: directory - name: run integration tests - shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTIONS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration >& {{ artifacts }}/testout.txt" + shell: "make localintegration >& {{ artifacts }}/testout.txt" args: - chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + chdir: "{{ cri_o_dest_path }}" async: 5400 poll: 30 diff --git a/contrib/test/integration/vars.yml b/contrib/test/integration/vars.yml index f1e5e2f7..c453c007 100644 --- a/contrib/test/integration/vars.yml +++ b/contrib/test/integration/vars.yml @@ -1,5 +1,14 @@ --- +# Base directory for all go-related source, build, and install. +go_path: "/go" + +# Absolute path on control-host where the cri-o source exists +cri_o_src_path: "{{ playbook_dir }}/../../../" + +# Absolute path on subject-host where cri-o source is expected +cri_o_dest_path: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-o" + # For results.yml Paths use rsync 'source' conventions artifacts: "/tmp/artifacts" # Base-directory for collection crio_integration_filepath: "{{ artifacts }}/testout.txt"