From 0e752a64230de230b310ad92a47e4e6cc37607a3 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Mon, 16 Oct 2017 14:22:13 -0400 Subject: [PATCH] Avoid relying on subject environment-vars There are so many ways/places they can change values on the host from one moment to the next. Yet as written, the value of ansible_env.GOPATH is really only fixed at "fact gathering" time. In other words, the environment variable can change (even during a play), but won't be noticed until possibly much later. This can cause very strange things to happen which aren't easy to debug. Fix this by using established facts (variables), and continuously establishing them as environment variables. This way, if/when a task fails, the value of the environment will be present w/in the failure message instead of obscrured by the shell. Signed-off-by: Chris Evich --- contrib/test/integration/build/bats.yml | 4 ++-- contrib/test/integration/build/cri-tools.yml | 4 ++-- contrib/test/integration/build/kubernetes.yml | 24 ++++--------------- contrib/test/integration/build/plugins.yml | 12 +++++----- contrib/test/integration/build/runc.yml | 6 ++--- contrib/test/integration/e2e.yml | 7 ++---- contrib/test/integration/golang.yml | 24 +++++-------------- contrib/test/integration/main.yml | 6 +++++ contrib/test/integration/system.yml | 14 ----------- contrib/test/integration/test.yml | 6 ++--- contrib/test/integration/vars.yml | 18 ++++++++++++++ 11 files changed, 53 insertions(+), 72 deletions(-) diff --git a/contrib/test/integration/build/bats.yml b/contrib/test/integration/build/bats.yml index d4ea19c6..ec2900b0 100644 --- a/contrib/test/integration/build/bats.yml +++ b/contrib/test/integration/build/bats.yml @@ -3,12 +3,12 @@ - name: clone bats source repo git: repo: "https://github.com/sstephenson/bats.git" - dest: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats" + dest: "{{ go_path }}/src/github.com/sstephenson/bats" - name: install bats command: "./install.sh /usr/local" args: - chdir: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats" + chdir: "{{ go_path }}/src/github.com/sstephenson/bats" - name: link bats file: diff --git a/contrib/test/integration/build/cri-tools.yml b/contrib/test/integration/build/cri-tools.yml index 9a117f3c..fad1dbfe 100644 --- a/contrib/test/integration/build/cri-tools.yml +++ b/contrib/test/integration/build/cri-tools.yml @@ -3,7 +3,7 @@ - name: clone cri-tools source repo git: repo: "https://github.com/kubernetes-incubator/cri-tools.git" - dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-tools" + dest: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-tools" version: "9ff5e8f78a4182ab8d5ba9bcccdda5f338600eab" - name: install crictl @@ -11,6 +11,6 @@ - name: link crictl file: - src: "{{ ansible_env.GOPATH }}/bin/crictl" + src: "{{ go_path }}/bin/crictl" dest: /usr/bin/crictl state: link diff --git a/contrib/test/integration/build/kubernetes.yml b/contrib/test/integration/build/kubernetes.yml index f724230c..e8b19c57 100644 --- a/contrib/test/integration/build/kubernetes.yml +++ b/contrib/test/integration/build/kubernetes.yml @@ -3,17 +3,17 @@ - name: clone kubernetes source repo git: repo: "https://github.com/runcom/kubernetes.git" - dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + dest: "{{ go_path }}/src/k8s.io/kubernetes" version: "cri-o-patched-1.8" - name: install etcd command: "hack/install-etcd.sh" args: - chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + chdir: "{{ go_path }}/src/k8s.io/kubernetes" - name: build kubernetes make: - chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + chdir: "{{ go_path }}/src/k8s.io/kubernetes" - name: Add custom cluster service file for the e2e testing copy: @@ -23,7 +23,7 @@ After=network-online.target Wants=network-online.target [Service] - WorkingDirectory={{ ansible_env.GOPATH }}/src/k8s.io/kubernetes + WorkingDirectory={{ go_path }}/src/k8s.io/kubernetes ExecStart=/usr/local/bin/createcluster.sh User=root [Install] @@ -35,7 +35,7 @@ content: | #!/bin/bash - export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ ansible_env.GOPATH }}/bin:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/third_party/etcd:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/ + export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ go_path }}/bin:{{ go_path }}/src/k8s.io/kubernetes/third_party/etcd:{{ go_path }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/ export CONTAINER_RUNTIME=remote export CGROUP_DRIVER=systemd export CONTAINER_RUNTIME_ENDPOINT='/var/run/crio.sock --runtime-request-timeout=5m' @@ -47,17 +47,3 @@ export KUBE_ENABLE_CLUSTER_DNS=true ./hack/local-up-cluster.sh mode: "u=rwx,g=rwx,o=x" - -- name: Set kubernetes_provider to be local - lineinfile: - dest: /etc/environment - line: 'KUBERNETES_PROVIDER=local' - regexp: 'KUBERNETES_PROVIDER=' - state: present - -- name: Set KUBECONFIG - lineinfile: - dest: /etc/environment - line: 'KUBECONFIG=/var/run/kubernetes/admin.kubeconfig' - regexp: 'KUBECONFIG=' - state: present diff --git a/contrib/test/integration/build/plugins.yml b/contrib/test/integration/build/plugins.yml index e342a0b9..b344270a 100644 --- a/contrib/test/integration/build/plugins.yml +++ b/contrib/test/integration/build/plugins.yml @@ -3,17 +3,17 @@ - name: clone plugins source repo git: repo: "https://github.com/containernetworking/plugins.git" - dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + dest: "{{ go_path }}/src/github.com/containernetworking/plugins" version: "dcf7368eeab15e2affc6256f0bb1e84dd46a34de" - name: build plugins command: "./build.sh" args: - chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + chdir: "{{ go_path }}/src/github.com/containernetworking/plugins" - name: install plugins copy: - src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/{{ item }}" + src: "{{ go_path }}/src/github.com/containernetworking/plugins/bin/{{ item }}" dest: "/opt/cni/bin" mode: "o=rwx,g=rx,o=rx" remote_src: yes @@ -33,18 +33,18 @@ - name: clone runcom plugins source repo git: repo: "https://github.com/runcom/plugins.git" - dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + dest: "{{ go_path }}/src/github.com/containernetworking/plugins" version: "custom-bridge" force: yes - name: build plugins command: "./build.sh" args: - chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + chdir: "{{ go_path }}/src/github.com/containernetworking/plugins" - name: install custom bridge copy: - src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/bridge" + src: "{{ go_path }}/src/github.com/containernetworking/plugins/bin/bridge" dest: "/opt/cni/bin/bridge-custom" mode: "o=rwx,g=rx,o=rx" remote_src: yes diff --git a/contrib/test/integration/build/runc.yml b/contrib/test/integration/build/runc.yml index 7bb0491d..1dd04f03 100644 --- a/contrib/test/integration/build/runc.yml +++ b/contrib/test/integration/build/runc.yml @@ -3,18 +3,18 @@ - name: clone runc source repo git: repo: "https://github.com/opencontainers/runc.git" - dest: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" + dest: "{{ go_path }}/src/github.com/opencontainers/runc" version: "84a082bfef6f932de921437815355186db37aeb1" - name: build runc make: params: BUILDTAGS="seccomp selinux" - chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" + chdir: "{{ go_path }}/src/github.com/opencontainers/runc" - name: install runc make: target: "install" - chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" + chdir: "{{ go_path }}/src/github.com/opencontainers/runc" - name: link runc file: diff --git a/contrib/test/integration/e2e.yml b/contrib/test/integration/e2e.yml index 41f92757..c0df99ef 100644 --- a/contrib/test/integration/e2e.yml +++ b/contrib/test/integration/e2e.yml @@ -29,7 +29,7 @@ daemon_reload: yes - name: wait for the cluster to be running - command: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default" + command: "{{ go_path }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default" register: kube_poll until: kube_poll | succeeded retries: 100 @@ -51,10 +51,7 @@ &> {{ artifacts }}/e2e.log # Fix vim syntax hilighting: " -- name: disable SELinux - command: setenforce 0 - - name: run e2e tests shell: "{{ e2e_shell_cmd | regex_replace('\\s+', ' ') }}" args: - chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + chdir: "{{ go_path }}/src/k8s.io/kubernetes" diff --git a/contrib/test/integration/golang.yml b/contrib/test/integration/golang.yml index 63e55697..39be5aa1 100644 --- a/contrib/test/integration/golang.yml +++ b/contrib/test/integration/golang.yml @@ -16,28 +16,16 @@ - gofmt - godoc -- name: ensure user profile exists - file: - path: "{{ ansible_user_dir }}/.profile" - state: touch - -- name: set up PATH for Go toolchain and built binaries - lineinfile: - dest: "{{ ansible_user_dir }}/.profile" - line: 'PATH={{ ansible_env.PATH }}:{{ ansible_env.GOPATH }}/bin:/usr/local/go/bin' - regexp: '^PATH=' - state: present - - name: set up directories file: - path: "{{ item }}" + path: "{{ go_path }}/{{ item }}" state: directory with_items: - - "{{ ansible_env.GOPATH }}/src/github.com/containernetworking" - - "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator" - - "{{ ansible_env.GOPATH }}/src/github.com/k8s.io" - - "{{ ansible_env.GOPATH }}/src/github.com/sstephenson" - - "{{ ansible_env.GOPATH }}/src/github.com/opencontainers" + - "src/github.com/containernetworking" + - "src/github.com/kubernetes-incubator" + - "src/github.com/k8s.io" + - "src/github.com/sstephenson" + - "src/github.com/opencontainers" - name: install Go tools and dependencies shell: /usr/bin/go get -u "github.com/{{ item }}" diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml index 343f03ad..2f511f3e 100644 --- a/contrib/test/integration/main.yml +++ b/contrib/test/integration/main.yml @@ -4,6 +4,8 @@ gather_facts: False # Requires low-level ansible-dependencies tags: - always + tags: + - setup tasks: - name: Ensure low-level ansible-dependencies are installed raw: $(type -P dnf || type -P yum) install -y python2 python2-dnf libselinux-python git rsync @@ -26,6 +28,7 @@ - hosts: '{{ subjects | default("localhost") }}' vars_files: - "{{ playbook_dir }}/vars.yml" + environment: '{{ environment_variables }}' tags: - setup tasks: @@ -54,6 +57,7 @@ - hosts: '{{ subjects | default("localhost") }}' vars_files: - "{{ playbook_dir }}/vars.yml" + environment: '{{ environment_variables }}' tags: - integration - e2e @@ -64,6 +68,7 @@ - hosts: '{{ subjects | default("localhost") }}' vars_files: - "{{ playbook_dir }}/vars.yml" + environment: '{{ environment_variables }}' tags: - integration tasks: @@ -73,6 +78,7 @@ - hosts: '{{ subjects | default("localhost") }}' vars_files: - "{{ playbook_dir }}/vars.yml" + environment: '{{ environment_variables }}' tags: - e2e tasks: diff --git a/contrib/test/integration/system.yml b/contrib/test/integration/system.yml index fd51368a..c84e0f59 100644 --- a/contrib/test/integration/system.yml +++ b/contrib/test/integration/system.yml @@ -116,17 +116,3 @@ - name: Update the kernel cmdline to include quota support command: grubby --update-kernel=ALL --args="rootflags=pquota" when: ansible_distribution in ['RedHat', 'CentOS'] - -- name: Configure the GOPATH environment variable for all users - blockinfile: - path: /etc/environment - block: "GOPATH={{ go_path }}" - create: True - follow: True - -- name: Reset the ansible connection to incorporate /etc/environment changes - meta: reset_connection - -- name: Refresh facts to incorporate /etc/environment changes - setup: - gather_subset: network diff --git a/contrib/test/integration/test.yml b/contrib/test/integration/test.yml index 34fc917e..0e8ec8a4 100644 --- a/contrib/test/integration/test.yml +++ b/contrib/test/integration/test.yml @@ -5,7 +5,7 @@ - name: Make testing output verbose so it can be converted to xunit lineinfile: - dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/hack/make-rules/test.sh" + dest: "{{ go_path }}/src/k8s.io/kubernetes/hack/make-rules/test.sh" line: ' go test -v "${goflags[@]:+${goflags[@]}}" \' regexp: ' go test \"\$' state: present @@ -21,8 +21,8 @@ state: directory - name: run integration tests - shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTIONS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration >& {{ artifacts }}/testout.txt" + shell: "make localintegration >& {{ artifacts }}/testout.txt" args: - chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + chdir: "{{ cri_o_dest_path }}" async: 5400 poll: 30 diff --git a/contrib/test/integration/vars.yml b/contrib/test/integration/vars.yml index f1e5e2f7..3eebd62f 100644 --- a/contrib/test/integration/vars.yml +++ b/contrib/test/integration/vars.yml @@ -1,5 +1,23 @@ --- +# Path to encode into /etc/environment on all hosts +go_path: "/go" + +# Absolute path on control-host where the cri-o source exists +cri_o_src_path: "{{ playbook_dir }}/../../../" + +# Absolute path on subject-host where cri-o source is expected +cri_o_dest_path: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-o" + +# Global environment variables to use for tasks on subject-hosts +environment_variables: + PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:{{ go_path }}/bin:/usr/local/go/bin" + GOPATH: "{{ go_path }}" + KUBERNETES_PROVIDER: "local" + KUBECONFIG: "/var/run/kubernetes/admin.kubeconfig" + CGROUP_MANAGER: "cgroupfs" + STORAGE_OPTS: '--storage-driver=overlay{{ extra_storage_opts | default("") }}' + # For results.yml Paths use rsync 'source' conventions artifacts: "/tmp/artifacts" # Base-directory for collection crio_integration_filepath: "{{ artifacts }}/testout.txt"