From 34fc624d3b8a59a5535ab0488c73aac5334f2ce9 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Sun, 3 Dec 2017 09:54:07 +0100 Subject: [PATCH] contrib: test: add CI system container We'll configure docker hub auto build to automatically rebuild the system container for the origin CI when we push changes to a given branch (master for this PR). Docker Hub will rebuild from master and tag docker.io/crio/cri-o:v3.9 to be used in the Origin CI. The same will be done for the following branches->tags: release-1.7 -> docker.io/crio/cri-o:v3.7 release-1.8 -> docker.io/crio/cri-o:v3.8 master -> docker.io/crio/cri-o:v3.9 The only moving bit will be the master branch, we need to re-configure docker hub auto builds whenever we branch out from master for the next CRI-O release. Signed-off-by: Antonio Murdaca --- contrib/test/system-container/Dockerfile | 54 +++ contrib/test/system-container/README.md | 57 +++ .../system-container/config.json.template | 427 ++++++++++++++++++ contrib/test/system-container/manifest.json | 10 + contrib/test/system-container/run.sh | 8 + .../test/system-container/service.template | 20 + contrib/test/system-container/set_mounts.sh | 7 + .../test/system-container/tmpfiles.template | 5 + 8 files changed, 588 insertions(+) create mode 100644 contrib/test/system-container/Dockerfile create mode 100644 contrib/test/system-container/README.md create mode 100644 contrib/test/system-container/config.json.template create mode 100644 contrib/test/system-container/manifest.json create mode 100755 contrib/test/system-container/run.sh create mode 100644 contrib/test/system-container/service.template create mode 100755 contrib/test/system-container/set_mounts.sh create mode 100644 contrib/test/system-container/tmpfiles.template diff --git a/contrib/test/system-container/Dockerfile b/contrib/test/system-container/Dockerfile new file mode 100644 index 00000000..8df93349 --- /dev/null +++ b/contrib/test/system-container/Dockerfile @@ -0,0 +1,54 @@ +FROM centos + +RUN yum-config-manager --nogpgcheck --add-repo https://cbs.centos.org/repos/virt7-container-common-candidate/x86_64/os/ && \ + yum install --disablerepo=extras --nogpgcheck --setopt=tsflags=nodocs -y \ + btrfs-progs-devel \ + device-mapper-devel \ + git \ + glib2-devel \ + glibc-devel \ + glibc-static \ + go \ + golang-github-cpuguy83-go-md2man \ + gpgme-devel \ + libassuan-devel \ + libgpg-error-devel \ + libseccomp-devel \ + libselinux-devel \ + ostree-devel \ + pkgconfig \ + make \ + skopeo-containers \ + # start runtime deps + runc \ + socat \ + iproute \ + iptables && \ + # end runtime deps + yum clean all + +ENV CRIO_PR "" +ENV CRIO_COMMIT release-1.8 +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone https://github.com/kubernetes-incubator/cri-o.git "$GOPATH/src/github.com/kubernetes-incubator/cri-o" \ + && cd "$GOPATH/src/github.com/kubernetes-incubator/cri-o" \ + && if [[ -n "$CRIO_PR" ]]; then git fetch origin "pull/$CRIO_PR/head:pr$CRIO_PR" && git checkout "pr$CRIO_PR"; else git checkout -q "$CRIO_COMMIT"; fi \ + && make binaries docs \ + && ./bin/crio --selinux=true --storage-driver=overlay --conmon /usr/libexec/crio/conmon --cni-plugin-dir /usr/libexec/cni --default-mounts /usr/share/rhel/secrets:/run/secrets --cgroup-manager=systemd config > crio.conf \ + && make install.config install.systemd PREFIX=/usr \ + && make install PREFIX=/usr \ + && rm -rf "$GOPATH" + +RUN mkdir -p /exports/hostfs/etc/crio /exports/hostfs/opt/cni/bin/ /exports/hostfs/var/lib/containers/storage/ && \ + cp /etc/crio/* /exports/hostfs/etc/crio && \ + if test -e /usr/libexec/cni; then cp -Lr /usr/libexec/cni/* /exports/hostfs/opt/cni/bin/; fi + +RUN sed -i '/storage_option =/s/.*/&\n"overlay.override_kernel_check=1",/' /exports/hostfs/etc/crio/crio.conf + +COPY manifest.json tmpfiles.template config.json.template service.template /exports/ + +COPY set_mounts.sh / +COPY run.sh /usr/bin/ + +CMD ["/usr/bin/run.sh"] diff --git a/contrib/test/system-container/README.md b/contrib/test/system-container/README.md new file mode 100644 index 00000000..dbe99769 --- /dev/null +++ b/contrib/test/system-container/README.md @@ -0,0 +1,57 @@ +# cri-o + +This is the cri-o daemon as a system container. + +## Building the image from source: + +``` +# git clone https://github.com/kubernetes-incubator/cri-o +# cd contrib/test/system-container +# docker build -t cri-o . +``` + +## Running the system container, with the atomic CLI: + +Pull from registry into ostree: + +``` +# atomic pull --storage ostree $REGISTRY/cri-o +``` + +Or alternatively, pull from local docker: + +``` +# atomic pull --storage ostree docker:cri-o:latest +``` + +Install the container: + +Currently we recommend using --system-package=no to avoid having rpmbuild create an rpm file +during installation. This flag will tell the atomic CLI to fall back to copying files to the +host instead. + +``` +# atomic install --system --system-package=no --name=crio ($REGISTRY)/cri-o +``` + +Start as a systemd service: + +``` +# systemctl start crio +``` + +Stopping the service + +``` +# systemctl stop crio +``` + +Removing the container + +``` +# atomic uninstall crio +``` + +## Binary version + +You can find the image automatically built as: docker.io/crio/cri-o diff --git a/contrib/test/system-container/config.json.template b/contrib/test/system-container/config.json.template new file mode 100644 index 00000000..785383d4 --- /dev/null +++ b/contrib/test/system-container/config.json.template @@ -0,0 +1,427 @@ +{ + "ociVersion": "1.0.0", + "platform": { + "arch": "amd64", + "os": "linux" + }, + "process": { + "args": [ + "/usr/bin/run.sh" + ], + "capabilities": { + "ambient": [ + "CAP_CHOWN", + "CAP_FOWNER", + "CAP_FSETID", + "CAP_KILL", + "CAP_SETGID", + "CAP_SETUID", + "CAP_SETPCAP", + "CAP_LINUX_IMMUTABLE", + "CAP_NET_BIND_SERVICE", + "CAP_NET_BROADCAST", + "CAP_NET_ADMIN", + "CAP_NET_RAW", + "CAP_IPC_LOCK", + "CAP_IPC_OWNER", + "CAP_SYS_MODULE", + "CAP_SYS_RAWIO", + "CAP_SYS_CHROOT", + "CAP_SYS_PTRACE", + "CAP_SYS_PACCT", + "CAP_SYS_ADMIN", + "CAP_SYS_BOOT", + "CAP_SYS_NICE", + "CAP_SYS_RESOURCE", + "CAP_SYS_TIME", + "CAP_SYS_TTY_CONFIG", + "CAP_MKNOD", + "CAP_LEASE", + "CAP_AUDIT_WRITE", + "CAP_AUDIT_CONTROL", + "CAP_SETFCAP", + "CAP_DAC_OVERRIDE", + "CAP_MAC_OVERRIDE", + "CAP_DAC_READ_SEARCH", + "CAP_MAC_ADMIN", + "CAP_SYSLOG", + "CAP_WAKE_ALARM", + "CAP_BLOCK_SUSPEND" + ], + "bounding": [ + "CAP_CHOWN", + "CAP_FOWNER", + "CAP_FSETID", + "CAP_KILL", + "CAP_SETGID", + "CAP_SETUID", + "CAP_SETPCAP", + "CAP_LINUX_IMMUTABLE", + "CAP_NET_BIND_SERVICE", + "CAP_NET_BROADCAST", + "CAP_NET_ADMIN", + "CAP_NET_RAW", + "CAP_IPC_LOCK", + "CAP_IPC_OWNER", + "CAP_SYS_MODULE", + "CAP_SYS_RAWIO", + "CAP_SYS_CHROOT", + "CAP_SYS_PTRACE", + "CAP_SYS_PACCT", + "CAP_SYS_ADMIN", + "CAP_SYS_BOOT", + "CAP_SYS_NICE", + "CAP_SYS_RESOURCE", + "CAP_SYS_TIME", + "CAP_SYS_TTY_CONFIG", + "CAP_MKNOD", + "CAP_LEASE", + "CAP_AUDIT_WRITE", + "CAP_AUDIT_CONTROL", + "CAP_SETFCAP", + "CAP_DAC_OVERRIDE", + "CAP_MAC_OVERRIDE", + "CAP_DAC_READ_SEARCH", + "CAP_MAC_ADMIN", + "CAP_SYSLOG", + "CAP_WAKE_ALARM", + "CAP_BLOCK_SUSPEND" + ], + "effective": [ + "CAP_CHOWN", + "CAP_FOWNER", + "CAP_FSETID", + "CAP_KILL", + "CAP_SETGID", + "CAP_SETUID", + "CAP_SETPCAP", + "CAP_LINUX_IMMUTABLE", + "CAP_NET_BIND_SERVICE", + "CAP_NET_BROADCAST", + "CAP_NET_ADMIN", + "CAP_NET_RAW", + "CAP_IPC_LOCK", + "CAP_IPC_OWNER", + "CAP_SYS_MODULE", + "CAP_SYS_RAWIO", + "CAP_SYS_CHROOT", + "CAP_SYS_PTRACE", + "CAP_SYS_PACCT", + "CAP_SYS_ADMIN", + "CAP_SYS_BOOT", + "CAP_SYS_NICE", + "CAP_SYS_RESOURCE", + "CAP_SYS_TIME", + "CAP_SYS_TTY_CONFIG", + "CAP_MKNOD", + "CAP_LEASE", + "CAP_AUDIT_WRITE", + "CAP_AUDIT_CONTROL", + "CAP_SETFCAP", + "CAP_DAC_OVERRIDE", + "CAP_MAC_OVERRIDE", + "CAP_DAC_READ_SEARCH", + "CAP_MAC_ADMIN", + "CAP_SYSLOG", + "CAP_WAKE_ALARM", + "CAP_BLOCK_SUSPEND" + ], + "inheritable": [ + "CAP_CHOWN", + "CAP_FOWNER", + "CAP_FSETID", + "CAP_KILL", + "CAP_SETGID", + "CAP_SETUID", + "CAP_SETPCAP", + "CAP_LINUX_IMMUTABLE", + "CAP_NET_BIND_SERVICE", + "CAP_NET_BROADCAST", + "CAP_NET_ADMIN", + "CAP_NET_RAW", + "CAP_IPC_LOCK", + "CAP_IPC_OWNER", + "CAP_SYS_MODULE", + "CAP_SYS_RAWIO", + "CAP_SYS_CHROOT", + "CAP_SYS_PTRACE", + "CAP_SYS_PACCT", + "CAP_SYS_ADMIN", + "CAP_SYS_BOOT", + "CAP_SYS_NICE", + "CAP_SYS_RESOURCE", + "CAP_SYS_TIME", + "CAP_SYS_TTY_CONFIG", + "CAP_MKNOD", + "CAP_LEASE", + "CAP_AUDIT_WRITE", + "CAP_AUDIT_CONTROL", + "CAP_SETFCAP", + "CAP_DAC_OVERRIDE", + "CAP_MAC_OVERRIDE", + "CAP_DAC_READ_SEARCH", + "CAP_MAC_ADMIN", + "CAP_SYSLOG", + "CAP_WAKE_ALARM", + "CAP_BLOCK_SUSPEND" + ], + "permitted": [ + "CAP_CHOWN", + "CAP_FOWNER", + "CAP_FSETID", + "CAP_KILL", + "CAP_SETGID", + "CAP_SETUID", + "CAP_SETPCAP", + "CAP_LINUX_IMMUTABLE", + "CAP_NET_BIND_SERVICE", + "CAP_NET_BROADCAST", + "CAP_NET_ADMIN", + "CAP_NET_RAW", + "CAP_IPC_LOCK", + "CAP_IPC_OWNER", + "CAP_SYS_MODULE", + "CAP_SYS_RAWIO", + "CAP_SYS_CHROOT", + "CAP_SYS_PTRACE", + "CAP_SYS_PACCT", + "CAP_SYS_ADMIN", + "CAP_SYS_BOOT", + "CAP_SYS_NICE", + "CAP_SYS_RESOURCE", + "CAP_SYS_TIME", + "CAP_SYS_TTY_CONFIG", + "CAP_MKNOD", + "CAP_LEASE", + "CAP_AUDIT_WRITE", + "CAP_AUDIT_CONTROL", + "CAP_SETFCAP", + "CAP_DAC_OVERRIDE", + "CAP_MAC_OVERRIDE", + "CAP_DAC_READ_SEARCH", + "CAP_MAC_ADMIN", + "CAP_SYSLOG", + "CAP_WAKE_ALARM", + "CAP_BLOCK_SUSPEND" + ] + }, + "selinuxLabel": "system_u:system_r:container_runtime_t:s0", + "cwd": "/", + "env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin:/root/go/bin", + "TERM=xterm", + "LOG_LEVEL=$LOG_LEVEL", + "NAME=$NAME" + ], + "noNewPrivileges": false, + "terminal": false, + "user": { + "gid": 0, + "uid": 0 + } + }, + "root": { + "path": "rootfs", + "readonly": true + }, + "hooks": {}, + "linux": { + "namespaces": [ + { + "type": "mount" + } + ], + "resources": { + "devices": [ + { + "access": "rwm", + "allow": true + } + ] + }, + "rootfsPropagation": "private" + }, + "mounts": [ + { + "destination": "/tmp", + "options": [ + "private", + "bind", + "rw", + "mode=755" + ], + "source": "/tmp", + "type": "bind" + }, + { + "destination": "/etc", + "options": [ + "rbind", + "rprivate", + "rw", + "mode=755" + ], + "source": "/etc", + "type": "bind" + }, + { + "destination": "/lib/modules", + "options": [ + "rbind", + "rprivate", + "rw", + "mode=755" + ], + "source": "/lib/modules", + "type": "bind" + }, + { + "destination": "/root", + "options": [ + "rbind", + "rprivate", + "rw", + "mode=755" + ], + "source": "/root", + "type": "bind" + }, + { + "destination": "/home", + "options": [ + "rbind", + "rprivate", + "rw", + "mode=755" + ], + "source": "/home", + "type": "bind" + }, + { + "destination": "/mnt", + "options": [ + "rbind", + "rw", + "rprivate", + "mode=755" + ], + "source": "/mnt", + "type": "bind" + }, + { + "type": "bind", + "source": "${RUN_DIRECTORY}", + "destination": "/run", + "options": [ + "rshared", + "rbind", + "rw", + "mode=755" + ] + }, + { + "type": "bind", + "source": "${RUN_DIRECTORY}/systemd", + "destination": "/run/systemd", + "options": [ + "rslave", + "bind", + "rw", + "mode=755" + ] + }, + { + "destination": "/var/log", + "options": [ + "rbind", + "rslave", + "rw" + ], + "source": "/var/log", + "type": "bind" + }, + { + "destination": "/var/lib", + "options": [ + "rbind", + "rprivate", + "rw" + ], + "source": "${STATE_DIRECTORY}", + "type": "bind" + }, + { + "destination": "/var/lib/containers/storage", + "options": [ + "rbind", + "rshared", + "rw" + ], + "source": "${VAR_LIB_CONTAINERS_STORAGE}", + "type": "bind" + }, + { + "destination": "/var/lib/origin", + "options": [ + "rshared", + "bind", + "rw" + ], + "source": "${VAR_LIB_ORIGIN}", + "type": "bind" + }, + { + "destination": "/var/lib/kubelet", + "options": [ + "rshared", + "bind", + "rw" + ], + "source": "${VAR_LIB_KUBE}", + "type": "bind" + }, + { + "destination": "/opt/cni", + "options": [ + "rbind", + "rprivate", + "ro", + "mode=755" + ], + "source": "${OPT_CNI}", + "type": "bind" + }, + { + "destination": "/dev", + "options": [ + "rprivate", + "rbind", + "rw", + "mode=755" + ], + "source": "/dev", + "type": "bind" + }, + { + "destination": "/sys", + "options": [ + "rprivate", + "rbind", + "rw", + "mode=755" + ], + "source": "/sys", + "type": "bind" + }, + { + "destination": "/proc", + "options": [ + "rbind", + "rw", + "mode=755" + ], + "source": "/proc", + "type": "proc" + } + ] +} diff --git a/contrib/test/system-container/manifest.json b/contrib/test/system-container/manifest.json new file mode 100644 index 00000000..38f4dc87 --- /dev/null +++ b/contrib/test/system-container/manifest.json @@ -0,0 +1,10 @@ +{ + "version": "1.0", + "defaultValues": { + "LOG_LEVEL" : "info", + "OPT_CNI" : "/opt/cni", + "VAR_LIB_CONTAINERS_STORAGE" : "/var/lib/containers/storage", + "VAR_LIB_ORIGIN" : "/var/lib/origin", + "VAR_LIB_KUBE" : "/var/lib/kubelet" + } +} diff --git a/contrib/test/system-container/run.sh b/contrib/test/system-container/run.sh new file mode 100755 index 00000000..5621a7ba --- /dev/null +++ b/contrib/test/system-container/run.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# Ensure that new process maintain this SELinux label +PID=$$ +LABEL=`tr -d '\000' < /proc/$PID/attr/current` +printf %s $LABEL > /proc/self/attr/exec + +exec /usr/bin/crio --log-level=$LOG_LEVEL diff --git a/contrib/test/system-container/service.template b/contrib/test/system-container/service.template new file mode 100644 index 00000000..4c08b39d --- /dev/null +++ b/contrib/test/system-container/service.template @@ -0,0 +1,20 @@ +[Unit] +Description=crio daemon +After=network.target + +[Service] +Type=notify +ExecStartPre=/bin/sh $DESTDIR/rootfs/set_mounts.sh +ExecStart=$EXEC_START +ExecStop=$EXEC_STOP +Restart=on-failure +WorkingDirectory=$DESTDIR +RuntimeDirectory=${NAME} +TasksMax=infinity +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +TimeoutStartSec=0 + +[Install] +WantedBy=multi-user.target diff --git a/contrib/test/system-container/set_mounts.sh b/contrib/test/system-container/set_mounts.sh new file mode 100755 index 00000000..c1f0c050 --- /dev/null +++ b/contrib/test/system-container/set_mounts.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +findmnt /var/lib/containers/storage > /dev/null || mount --rbind --make-shared /var/lib/containers/storage /var/lib/containers/storage +findmnt /var/lib/origin > /dev/null || mount --bind --make-shared /var/lib/origin /var/lib/origin +findmnt /var/lib/kubelet > /dev/null || mount --bind --make-shared /var/lib/kubelet /var/lib/kubelet +mount --make-shared /run +findmnt /run/systemd > /dev/null || mount --bind --make-rslave /run/systemd /run/systemd diff --git a/contrib/test/system-container/tmpfiles.template b/contrib/test/system-container/tmpfiles.template new file mode 100644 index 00000000..015919c4 --- /dev/null +++ b/contrib/test/system-container/tmpfiles.template @@ -0,0 +1,5 @@ +d ${RUN_DIRECTORY}/${NAME} - - - - - +d /etc/crio - - - - - +Z /etc/crio - - - - - +d ${STATE_DIRECTORY}/origin - - - - - +d ${STATE_DIRECTORY}/kubelet - - - - -