From 25c49a80081dd58ec56128d01cae0faefaa0599b Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Mon, 16 Oct 2017 14:22:13 -0400 Subject: [PATCH] Avoid relying on subject environment-vars There are so many ways/places they can change values on the host from one moment to the next. Yet as written, the value of ansible_env.GOPATH is really only fixed at "fact gathering" time. In other words, the environment variable can change (even during a play), but won't be noticed until possibly much later. This can cause very strange things to happen which aren't easy to debug. Fix this by using established facts (variables), and continuously establishing them as environment variables. This way, if/when a task fails, the value of the environment will be present w/in the failure message instead of obscrured by the shell. Signed-off-by: Chris Evich --- contrib/test/integration/build/bats.yml | 4 ++-- contrib/test/integration/build/cri-tools.yml | 4 ++-- contrib/test/integration/build/kubernetes.yml | 24 ++++--------------- contrib/test/integration/build/plugins.yml | 12 +++++----- contrib/test/integration/build/runc.yml | 6 ++--- contrib/test/integration/e2e.yml | 4 ++-- contrib/test/integration/golang.yml | 24 +++++-------------- contrib/test/integration/main.yml | 7 ++++++ contrib/test/integration/system.yml | 14 ----------- contrib/test/integration/test.yml | 4 ++-- contrib/test/integration/vars.yml | 9 +++++++ 11 files changed, 44 insertions(+), 68 deletions(-) diff --git a/contrib/test/integration/build/bats.yml b/contrib/test/integration/build/bats.yml index d4ea19c6..ec2900b0 100644 --- a/contrib/test/integration/build/bats.yml +++ b/contrib/test/integration/build/bats.yml @@ -3,12 +3,12 @@ - name: clone bats source repo git: repo: "https://github.com/sstephenson/bats.git" - dest: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats" + dest: "{{ go_path }}/src/github.com/sstephenson/bats" - name: install bats command: "./install.sh /usr/local" args: - chdir: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats" + chdir: "{{ go_path }}/src/github.com/sstephenson/bats" - name: link bats file: diff --git a/contrib/test/integration/build/cri-tools.yml b/contrib/test/integration/build/cri-tools.yml index 9a117f3c..fad1dbfe 100644 --- a/contrib/test/integration/build/cri-tools.yml +++ b/contrib/test/integration/build/cri-tools.yml @@ -3,7 +3,7 @@ - name: clone cri-tools source repo git: repo: "https://github.com/kubernetes-incubator/cri-tools.git" - dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-tools" + dest: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-tools" version: "9ff5e8f78a4182ab8d5ba9bcccdda5f338600eab" - name: install crictl @@ -11,6 +11,6 @@ - name: link crictl file: - src: "{{ ansible_env.GOPATH }}/bin/crictl" + src: "{{ go_path }}/bin/crictl" dest: /usr/bin/crictl state: link diff --git a/contrib/test/integration/build/kubernetes.yml b/contrib/test/integration/build/kubernetes.yml index f724230c..e8b19c57 100644 --- a/contrib/test/integration/build/kubernetes.yml +++ b/contrib/test/integration/build/kubernetes.yml @@ -3,17 +3,17 @@ - name: clone kubernetes source repo git: repo: "https://github.com/runcom/kubernetes.git" - dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + dest: "{{ go_path }}/src/k8s.io/kubernetes" version: "cri-o-patched-1.8" - name: install etcd command: "hack/install-etcd.sh" args: - chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + chdir: "{{ go_path }}/src/k8s.io/kubernetes" - name: build kubernetes make: - chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + chdir: "{{ go_path }}/src/k8s.io/kubernetes" - name: Add custom cluster service file for the e2e testing copy: @@ -23,7 +23,7 @@ After=network-online.target Wants=network-online.target [Service] - WorkingDirectory={{ ansible_env.GOPATH }}/src/k8s.io/kubernetes + WorkingDirectory={{ go_path }}/src/k8s.io/kubernetes ExecStart=/usr/local/bin/createcluster.sh User=root [Install] @@ -35,7 +35,7 @@ content: | #!/bin/bash - export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ ansible_env.GOPATH }}/bin:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/third_party/etcd:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/ + export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ go_path }}/bin:{{ go_path }}/src/k8s.io/kubernetes/third_party/etcd:{{ go_path }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/ export CONTAINER_RUNTIME=remote export CGROUP_DRIVER=systemd export CONTAINER_RUNTIME_ENDPOINT='/var/run/crio.sock --runtime-request-timeout=5m' @@ -47,17 +47,3 @@ export KUBE_ENABLE_CLUSTER_DNS=true ./hack/local-up-cluster.sh mode: "u=rwx,g=rwx,o=x" - -- name: Set kubernetes_provider to be local - lineinfile: - dest: /etc/environment - line: 'KUBERNETES_PROVIDER=local' - regexp: 'KUBERNETES_PROVIDER=' - state: present - -- name: Set KUBECONFIG - lineinfile: - dest: /etc/environment - line: 'KUBECONFIG=/var/run/kubernetes/admin.kubeconfig' - regexp: 'KUBECONFIG=' - state: present diff --git a/contrib/test/integration/build/plugins.yml b/contrib/test/integration/build/plugins.yml index e342a0b9..b344270a 100644 --- a/contrib/test/integration/build/plugins.yml +++ b/contrib/test/integration/build/plugins.yml @@ -3,17 +3,17 @@ - name: clone plugins source repo git: repo: "https://github.com/containernetworking/plugins.git" - dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + dest: "{{ go_path }}/src/github.com/containernetworking/plugins" version: "dcf7368eeab15e2affc6256f0bb1e84dd46a34de" - name: build plugins command: "./build.sh" args: - chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + chdir: "{{ go_path }}/src/github.com/containernetworking/plugins" - name: install plugins copy: - src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/{{ item }}" + src: "{{ go_path }}/src/github.com/containernetworking/plugins/bin/{{ item }}" dest: "/opt/cni/bin" mode: "o=rwx,g=rx,o=rx" remote_src: yes @@ -33,18 +33,18 @@ - name: clone runcom plugins source repo git: repo: "https://github.com/runcom/plugins.git" - dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + dest: "{{ go_path }}/src/github.com/containernetworking/plugins" version: "custom-bridge" force: yes - name: build plugins command: "./build.sh" args: - chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + chdir: "{{ go_path }}/src/github.com/containernetworking/plugins" - name: install custom bridge copy: - src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/bridge" + src: "{{ go_path }}/src/github.com/containernetworking/plugins/bin/bridge" dest: "/opt/cni/bin/bridge-custom" mode: "o=rwx,g=rx,o=rx" remote_src: yes diff --git a/contrib/test/integration/build/runc.yml b/contrib/test/integration/build/runc.yml index 7bb0491d..1dd04f03 100644 --- a/contrib/test/integration/build/runc.yml +++ b/contrib/test/integration/build/runc.yml @@ -3,18 +3,18 @@ - name: clone runc source repo git: repo: "https://github.com/opencontainers/runc.git" - dest: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" + dest: "{{ go_path }}/src/github.com/opencontainers/runc" version: "84a082bfef6f932de921437815355186db37aeb1" - name: build runc make: params: BUILDTAGS="seccomp selinux" - chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" + chdir: "{{ go_path }}/src/github.com/opencontainers/runc" - name: install runc make: target: "install" - chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" + chdir: "{{ go_path }}/src/github.com/opencontainers/runc" - name: link runc file: diff --git a/contrib/test/integration/e2e.yml b/contrib/test/integration/e2e.yml index 1f7738f6..7eae7d4e 100644 --- a/contrib/test/integration/e2e.yml +++ b/contrib/test/integration/e2e.yml @@ -29,7 +29,7 @@ daemon_reload: yes - name: wait for the cluster to be running - command: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default" + command: "{{ go_path }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default" register: kube_poll until: kube_poll | succeeded retries: 100 @@ -64,7 +64,7 @@ - name: run e2e tests shell: "{{ e2e_shell_cmd | regex_replace('\\s+', ' ') }}" args: - chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + chdir: "{{ go_path }}/src/k8s.io/kubernetes" always: diff --git a/contrib/test/integration/golang.yml b/contrib/test/integration/golang.yml index 63e55697..39be5aa1 100644 --- a/contrib/test/integration/golang.yml +++ b/contrib/test/integration/golang.yml @@ -16,28 +16,16 @@ - gofmt - godoc -- name: ensure user profile exists - file: - path: "{{ ansible_user_dir }}/.profile" - state: touch - -- name: set up PATH for Go toolchain and built binaries - lineinfile: - dest: "{{ ansible_user_dir }}/.profile" - line: 'PATH={{ ansible_env.PATH }}:{{ ansible_env.GOPATH }}/bin:/usr/local/go/bin' - regexp: '^PATH=' - state: present - - name: set up directories file: - path: "{{ item }}" + path: "{{ go_path }}/{{ item }}" state: directory with_items: - - "{{ ansible_env.GOPATH }}/src/github.com/containernetworking" - - "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator" - - "{{ ansible_env.GOPATH }}/src/github.com/k8s.io" - - "{{ ansible_env.GOPATH }}/src/github.com/sstephenson" - - "{{ ansible_env.GOPATH }}/src/github.com/opencontainers" + - "src/github.com/containernetworking" + - "src/github.com/kubernetes-incubator" + - "src/github.com/k8s.io" + - "src/github.com/sstephenson" + - "src/github.com/opencontainers" - name: install Go tools and dependencies shell: /usr/bin/go get -u "github.com/{{ item }}" diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml index 6a60e0e6..140801ae 100644 --- a/contrib/test/integration/main.yml +++ b/contrib/test/integration/main.yml @@ -2,6 +2,9 @@ - hosts: '{{ subjects | default("all") }}' gather_facts: False # Requires low-level ansible-dependencies + # Cannot use vars.yml - it references magic variables from setup module + tags: + - setup tasks: - name: Ensure low-level ansible-dependencies are installed raw: $(type -P dnf || type -P yum) install -y python2 python2-dnf libselinux-python git rsync @@ -24,6 +27,7 @@ - hosts: '{{ subjects | default("all") }}' vars_files: - "{{ playbook_dir }}/vars.yml" + environment: '{{ environment_variables }}' tags: - setup tasks: @@ -52,6 +56,7 @@ - hosts: '{{ subjects | default("all") }}' vars_files: - "{{ playbook_dir }}/vars.yml" + environment: '{{ environment_variables }}' tags: - integration - e2e @@ -62,6 +67,7 @@ - hosts: '{{ subjects | default("all") }}' vars_files: - "{{ playbook_dir }}/vars.yml" + environment: '{{ environment_variables }}' tags: - integration tasks: @@ -71,6 +77,7 @@ - hosts: '{{ subjects | default("all") }}' vars_files: - "{{ playbook_dir }}/vars.yml" + environment: '{{ environment_variables }}' tags: - e2e tasks: diff --git a/contrib/test/integration/system.yml b/contrib/test/integration/system.yml index c453c422..c8dfdb3b 100644 --- a/contrib/test/integration/system.yml +++ b/contrib/test/integration/system.yml @@ -111,17 +111,3 @@ - name: Update the kernel cmdline to include quota support command: grubby --update-kernel=ALL --args="rootflags=pquota" when: ansible_distribution in ['RedHat', 'CentOS'] - -- name: Configure the GOPATH environment variable for all users - blockinfile: - path: /etc/environment - block: "GOPATH={{ go_path }}" - create: True - follow: True - -- name: Reset the ansible connection to incorporate /etc/environment changes - meta: reset_connection - -- name: Refresh facts to incorporate /etc/environment changes - setup: - gather_subset: network diff --git a/contrib/test/integration/test.yml b/contrib/test/integration/test.yml index dfa2c30f..da814b17 100644 --- a/contrib/test/integration/test.yml +++ b/contrib/test/integration/test.yml @@ -5,7 +5,7 @@ - name: Make testing output verbose so it can be converted to xunit lineinfile: - dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/hack/make-rules/test.sh" + dest: "{{ go_path }}/src/k8s.io/kubernetes/hack/make-rules/test.sh" line: ' go test -v "${goflags[@]:+${goflags[@]}}" \' regexp: ' go test \"\$' state: present @@ -31,7 +31,7 @@ when: not integration_selinux_enabled - name: run integration tests - shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration >& {{ artifacts }}/testout.txt" + shell: "make localintegration >& {{ artifacts }}/testout.txt" args: chdir: "{{ cri_o_dest_path }}" async: 5400 diff --git a/contrib/test/integration/vars.yml b/contrib/test/integration/vars.yml index 3362f2d3..3382caf6 100644 --- a/contrib/test/integration/vars.yml +++ b/contrib/test/integration/vars.yml @@ -22,6 +22,15 @@ cri_o_src_path: "{{ playbook_dir }}/../../../" # Absolute path on subjects where cri-o source is expected cri_o_dest_path: "{{ go_path }}/src/github.com/kubernetes-incubator/cri-o" +# Global environment variables to use for all tasks +environment_variables: + PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:{{ go_path }}/bin:/usr/local/go/bin" + GOPATH: "{{ go_path }}" + KUBERNETES_PROVIDER: "local" + KUBECONFIG: "/var/run/kubernetes/admin.kubeconfig" + CGROUP_MANAGER: "cgroupfs" + STORAGE_OPTS: '--storage-driver=overlay{{ extra_storage_opts | default("") }}' + # For results.yml Paths use rsync 'source' conventions artifacts: "/tmp/artifacts" # Base-directory for collection crio_integration_filepath: "{{ artifacts }}/testout.txt"