Merge pull request #1231 from runcom/lock-1.7

[release-1.7] lib,oci: drop stateLock when possible

.github/CODEOWNERS

@@ -0,0 +1,7 @@
+# GitHub code owners
+# See https://help.github.com/articles/about-codeowners/
+#
+# KEEP THIS FILE SORTED. Order is important. Last match takes precedence.
+
+*                                       @mrunalp @runcom
+pkg/storage/**                          @nalind @runcom @rhatdan

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,58 @@
+<!--
+If you are reporting a new issue, make sure that we do not have any duplicates
+already open. You can ensure this by searching the issue list for this
+repository. If there is a duplicate, please close your issue and add a comment
+to the existing issue instead.
+
+If you suspect your issue is a bug, please edit your issue description to
+include the BUG REPORT INFORMATION shown below. If you fail to provide this
+information within 7 days, we cannot debug your issue and will close it. We
+will, however, reopen it if you later provide the information.
+
+For more information about reporting issues, see
+https://github.com/kubernetes-incubator/cri-o/blob/master/CONTRIBUTING.md#reporting-issues
+
+---------------------------------------------------
+GENERAL SUPPORT INFORMATION
+---------------------------------------------------
+
+The GitHub issue tracker is for bug reports and feature requests.
+General support for **CRI-O** can be found at the following locations:
+
+- IRC - #cri-o channel on irc.freenode.org
+- Slack - kubernetes.slack.com #sig-node channel
+- Post a question on StackOverflow, using the CRI-O tag
+
+---------------------------------------------------
+BUG REPORT INFORMATION
+---------------------------------------------------
+Use the commands below to provide key information from your environment:
+You do NOT have to include this information if this is a FEATURE REQUEST
+-->
+
+**Description**
+
+<!--
+Briefly describe the problem you are having in a few paragraphs.
+-->
+
+**Steps to reproduce the issue:**
+1.
+2.
+3.
+
+**Describe the results you received:**
+
+
+**Describe the results you expected:**
+
+
+**Additional information you deem important (e.g. issue happens only occasionally):**
+
+**Output of `crio --version`:**
+
+```
+(paste your output here)
+```
+
+**Additional environment details (AWS, VirtualBox, physical, etc.):**

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,23 @@
+<!--
+Please make sure you've read and understood our contributing guidelines;
+https://github.com/kubernetes-incubator/cri-o/blob/master/CONTRIBUTING.md
+
+** Make sure all your commits include a signature generated with `git commit -s` **
+
+If this is a bug fix, make sure your description includes "fixes #xxxx", or
+"closes #xxxx"
+
+Please provide the following information:
+-->
+
+**- What I did**
+
+**- How I did it**
+
+**- How to verify it**
+
+**- Description for the changelog**
+<!--
+Write a short (one line) summary that describes the changes in this
+pull request for inclusion in the changelog:
+-->

.gitignore

@@ -1,17 +1,13 @@
 /.artifacts/
 /_output/
-/conmon/conmon
 /conmon/conmon.o
 /docs/*.[158]
 /docs/*.[158].gz
-/kpod
-/crioctl
-/crio
 /crio.conf
 *.o
 *.orig
-/pause/pause
 /pause/pause.o
+/bin/
 /test/bin2img/bin2img
 /test/checkseccomp/checkseccomp
 /test/copyimg/copyimg

.travis.yml

@@ -47,8 +47,6 @@ jobs:
       go: 1.9.x
     - script:
         - make .gitvalidation
-        - make gofmt
-        - make lint
         - make testunit
         - make docs
         - make

Dockerfile

@@ -97,7 +97,7 @@ RUN set -x \
        && rm -rf "$GOPATH"
 
 # Install crictl
-ENV CRICTL_COMMIT 16e6fe4d7199c5689db4630a9330e6a8a12cecd1
+ENV CRICTL_COMMIT b42fc3f364dd48f649d55926c34492beeb9b2e99
 RUN set -x \
        && export GOPATH="$(mktemp -d)" \
        && git clone https://github.com/kubernetes-incubator/cri-tools.git "$GOPATH/src/github.com/kubernetes-incubator/cri-tools" \

Makefile

@@ -11,7 +11,9 @@ LIBEXECDIR ?= ${PREFIX}/libexec
 MANDIR ?= ${PREFIX}/share/man
 ETCDIR ?= ${DESTDIR}/etc
 ETCDIR_CRIO ?= ${ETCDIR}/crio
-BUILDTAGS ?= selinux seccomp $(shell hack/btrfs_tag.sh) $(shell hack/libdm_tag.sh) $(shell hack/btrfs_installed_tag.sh)
+BUILDTAGS ?= seccomp $(shell hack/btrfs_tag.sh) $(shell hack/libdm_tag.sh) $(shell hack/btrfs_installed_tag.sh) $(shell hack/ostree_tag.sh) $(shell hack/selinux_tag.sh)
+CRICTL_CONFIG_DIR=${DESTDIR}/etc
+
 BASHINSTALLDIR=${PREFIX}/share/bash-completion/completions
 OCIUMOUNTINSTALLDIR=$(PREFIX)/share/oci-umount/oci-umount.d
 
@@ -22,7 +24,6 @@ COMMIT_NO := $(shell git rev-parse HEAD 2> /dev/null || true)
 GIT_COMMIT := $(if $(shell git status --porcelain --untracked-files=no),"${COMMIT_NO}-dirty","${COMMIT_NO}")
 BUILD_INFO := $(shell date +%s)
 
-VERSION := ${shell cat ./VERSION}
 KPOD_VERSION := ${shell cat ./KPOD_VERSION}
 
 # If GOPATH not specified, use one in the local directory
@@ -35,8 +36,11 @@ GOPKGBASEDIR := $(shell dirname "$(GOPKGDIR)")
 
 # Update VPATH so make finds .gopathok
 VPATH := $(VPATH):$(GOPATH)
-
-LDFLAGS := -ldflags '-X main.gitCommit=${GIT_COMMIT} -X main.buildInfo=${BUILD_INFO} -X main.version=${VERSION} -X main.kpodVersion=${KPOD_VERSION}'
+SHRINKFLAGS := -s -w
+BASE_LDFLAGS := ${SHRINKFLAGS} -X main.gitCommit=${GIT_COMMIT} -X main.buildInfo=${BUILD_INFO}
+KPOD_LDFLAGS := -X main.kpodVersion=${KPOD_VERSION}
+LDFLAGS := -ldflags '${BASE_LDFLAGS}'
+LDFLAGS_KPOD := -ldflags '${BASE_LDFLAGS} ${KPOD_LDFLAGS}'
 
 all: binaries crio.conf docs
 
@@ -46,7 +50,7 @@ help:
 	@echo "Usage: make <target>"
 	@echo
 	@echo " * 'install' - Install binaries to system locations"
-	@echo " * 'binaries' - Build crio, conmon and crioctl"
+	@echo " * 'binaries' - Build crio, conmon, pause, crioctl and kpod"
 	@echo " * 'integration' - Execute integration tests"
 	@echo " * 'clean' - Clean artifacts"
 	@echo " * 'lint' - Execute the source code linter"
@@ -73,25 +77,25 @@ pause:
 	$(MAKE) -C $@
 
 test/bin2img/bin2img: .gopathok $(wildcard test/bin2img/*.go)
-	$(GO) build $(LDFLAGS) -tags "$(BUILDTAGS)" -o $@ $(PROJECT)/test/bin2img
+	$(GO) build $(LDFLAGS) -tags "$(BUILDTAGS) containers_image_ostree_stub" -o $@ $(PROJECT)/test/bin2img
 
 test/copyimg/copyimg: .gopathok $(wildcard test/copyimg/*.go)
-	$(GO) build $(LDFLAGS) -tags "$(BUILDTAGS)" -o $@ $(PROJECT)/test/copyimg
+	$(GO) build $(LDFLAGS) -tags "$(BUILDTAGS) containers_image_ostree_stub" -o $@ $(PROJECT)/test/copyimg
 
 test/checkseccomp/checkseccomp: .gopathok $(wildcard test/checkseccomp/*.go)
-	$(GO) build $(LDFLAGS) -tags "$(BUILDTAGS)" -o $@ $(PROJECT)/test/checkseccomp
+	$(GO) build $(LDFLAGS) -tags "$(BUILDTAGS) containers_image_ostree_stub" -o $@ $(PROJECT)/test/checkseccomp
 
 crio: .gopathok $(shell hack/find-godeps.sh $(GOPKGDIR) cmd/crio $(PROJECT))
-	$(GO) build $(LDFLAGS) -tags "$(BUILDTAGS)" -o $@ $(PROJECT)/cmd/crio
+	$(GO) build $(LDFLAGS) -tags "$(BUILDTAGS) containers_image_ostree_stub" -o bin/$@ $(PROJECT)/cmd/crio
 
 crioctl: .gopathok $(shell hack/find-godeps.sh $(GOPKGDIR) cmd/crioctl $(PROJECT))
-	$(GO) build $(LDFLAGS) -tags "$(BUILDTAGS)" -o $@ $(PROJECT)/cmd/crioctl
+	$(GO) build $(LDFLAGS) -tags "$(BUILDTAGS) containers_image_ostree_stub" -o bin/$@ $(PROJECT)/cmd/crioctl
 
 kpod: .gopathok $(shell hack/find-godeps.sh $(GOPKGDIR) cmd/kpod $(PROJECT))
-	$(GO) build $(LDFLAGS) -tags "$(BUILDTAGS)" -o $@ $(PROJECT)/cmd/kpod
+	$(GO) build $(LDFLAGS_KPOD) -tags "$(BUILDTAGS)" -o bin/$@ $(PROJECT)/cmd/kpod
 
 crio.conf: crio
-	./crio --config="" config --default > crio.conf
+	./bin/crio --config="" config --default > crio.conf
 
 clean:
 ifneq ($(GOPATH),)
@@ -102,7 +106,7 @@ endif
 	rm -fr test/testdata/redis-image
 	find . -name \*~ -delete
 	find . -name \#\* -delete
-	rm -f crioctl crio kpod
+	rm -f bin/crioctl bin/crio bin/kpod
 	make -C conmon clean
 	make -C pause clean
 	rm -f test/bin2img/bin2img
@@ -121,10 +125,11 @@ integration: crioimage
 testunit:
 	$(GO) test -tags "$(BUILDTAGS)" -cover $(PACKAGES)
 
-localintegration: clean binaries
+localintegration: clean binaries test-binaries
 	./test/test_runner.sh ${TESTFLAGS}
 
-binaries: crio crioctl kpod conmon pause test/bin2img/bin2img test/copyimg/copyimg test/checkseccomp/checkseccomp
+binaries: crio conmon pause kpod crioctl
+test-binaries: test/bin2img/bin2img test/copyimg/copyimg test/checkseccomp/checkseccomp
 
 MANPAGES_MD := $(wildcard docs/*.md)
 MANPAGES    := $(MANPAGES_MD:%.md=%)
@@ -143,11 +148,11 @@ docs: $(MANPAGES)
 install: .gopathok install.bin install.man
 
 install.bin:
-	install ${SELINUXOPT} -D -m 755 crio $(BINDIR)/crio
-	install ${SELINUXOPT} -D -m 755 crioctl $(BINDIR)/crioctl
-	install ${SELINUXOPT} -D -m 755 kpod $(BINDIR)/kpod
-	install ${SELINUXOPT} -D -m 755 conmon/conmon $(LIBEXECDIR)/crio/conmon
-	install ${SELINUXOPT} -D -m 755 pause/pause $(LIBEXECDIR)/crio/pause
+	install ${SELINUXOPT} -D -m 755 bin/crio $(BINDIR)/crio
+	install ${SELINUXOPT} -D -m 755 bin/crioctl $(BINDIR)/crioctl
+	install ${SELINUXOPT} -D -m 755 bin/kpod $(BINDIR)/kpod
+	install ${SELINUXOPT} -D -m 755 bin/conmon $(LIBEXECDIR)/crio/conmon
+	install ${SELINUXOPT} -D -m 755 bin/pause $(LIBEXECDIR)/crio/pause
 
 install.man:
 	install ${SELINUXOPT} -d -m 755 $(MANDIR)/man1
@@ -161,6 +166,7 @@ install.config:
 	install ${SELINUXOPT} -D -m 644 crio.conf $(ETCDIR_CRIO)/crio.conf
 	install ${SELINUXOPT} -D -m 644 seccomp.json $(ETCDIR_CRIO)/seccomp.json
 	install ${SELINUXOPT} -D -m 644 crio-umount.conf $(OCIUMOUNTINSTALLDIR)/crio-umount.conf
+	install ${SELINUXOPT} -D -m 644 crictl.yaml $(CRICTL_CONFIG_DIR)
 
 install.completions:
 	install ${SELINUXOPT} -d -m 755 ${BASHINSTALLDIR}

README.md

@@ -4,7 +4,7 @@
 [![Build Status](https://img.shields.io/travis/kubernetes-incubator/cri-o.svg?maxAge=2592000&style=flat-square)](https://travis-ci.org/kubernetes-incubator/cri-o)
 [![Go Report Card](https://goreportcard.com/badge/github.com/kubernetes-incubator/cri-o?style=flat-square)](https://goreportcard.com/report/github.com/kubernetes-incubator/cri-o)
 
-### Status: Release Candidate 3
+### Status: Stable
 
 ## What is the scope of this project?
 
@@ -256,5 +256,4 @@ To run a full cluster, see [the instructions](kubernetes.md).
 1. Support for log management, networking integration using CNI, pluggable image/storage management (done)
 1. Support for exec/attach (done)
 1. Target fully automated kubernetes testing without failures [e2e status](https://github.com/kubernetes-incubator/cri-o/issues/533)
-1. Release 1.0
 1. Track upstream k8s releases

VERSION

@@ -1 +0,0 @@
-1.0.0-rc4-dev

cmd/crio/config.go

@@ -108,6 +108,10 @@ cgroup_manager = "{{ .CgroupManager }}"
 # hooks_dir_path is the oci hooks directory for automatically executed hooks
 hooks_dir_path = "{{ .HooksDirPath }}"
 
+# default_mounts is the mounts list to be mounted for the container when created
+default_mounts = [
+{{ range $mount := .DefaultMounts }}{{ printf "\t%q, \n" $mount }}{{ end }}]
+
 # pids_limit is the number of processes allowed in a container
 pids_limit = {{ .PidsLimit }}
 

cmd/crio/main.go

@@ -10,10 +10,12 @@ import (
 	"os/signal"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/containers/storage/pkg/reexec"
 	"github.com/kubernetes-incubator/cri-o/libkpod"
 	"github.com/kubernetes-incubator/cri-o/server"
+	"github.com/kubernetes-incubator/cri-o/version"
 	"github.com/opencontainers/selinux/go-selinux"
 	"github.com/sirupsen/logrus"
 	"github.com/soheilhy/cmux"
@@ -23,10 +25,6 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 )
 
-// This is populated by the Makefile from the VERSION file
-// in the repository
-var version = ""
-
 // gitCommit is the commit that the binary is being built from.
 // It will be populated by the Makefile.
 var gitCommit = ""
@@ -127,6 +125,9 @@ func mergeConfig(config *server.Config, ctx *cli.Context) error {
 	if ctx.GlobalIsSet("hooks-dir-path") {
 		config.HooksDirPath = ctx.GlobalString("hooks-dir-path")
 	}
+	if ctx.GlobalIsSet("default-mounts") {
+		config.DefaultMounts = ctx.GlobalStringSlice("default-mounts")
+	}
 	if ctx.GlobalIsSet("pids-limit") {
 		config.PidsLimit = ctx.GlobalInt64("pids-limit")
 	}
@@ -179,9 +180,7 @@ func main() {
 	app := cli.NewApp()
 
 	var v []string
-	if version != "" {
-		v = append(v, version)
-	}
+	v = append(v, version.Version)
 	if gitCommit != "" {
 		v = append(v, fmt.Sprintf("commit: %s", gitCommit))
 	}
@@ -322,6 +321,11 @@ func main() {
 			Value:  libkpod.DefaultHooksDirPath,
 			Hidden: true,
 		},
+		cli.StringSliceFlag{
+			Name:   "default-mounts",
+			Usage:  "add one or more default mount paths in the form host:container",
+			Hidden: true,
+		},
 		cli.BoolFlag{
 			Name:  "profile",
 			Usage: "enable pprof remote profiler on localhost:6060",
@@ -405,6 +409,16 @@ func main() {
 			}()
 		}
 
+		args := c.Args()
+		if len(args) > 0 {
+			for _, command := range app.Commands {
+				if args[0] == command.Name {
+					break
+				}
+			}
+			return fmt.Errorf("command %q not supported", args[0])
+		}
+
 		config := c.App.Metadata["config"].(*server.Config)
 
 		if !config.SELinux {
@@ -467,7 +481,8 @@ func main() {
 
 		infoMux := service.GetInfoMux()
 		srv := &http.Server{
-			Handler: infoMux,
+			Handler:     infoMux,
+			ReadTimeout: 5 * time.Second,
 		}
 
 		graceful := false

cmd/kpod/formats/formats.go

@@ -8,6 +8,7 @@ import (
 	"text/tabwriter"
 	"text/template"
 
+	"bytes"
 	"github.com/ghodss/yaml"
 	"github.com/pkg/errors"
 )
@@ -59,6 +60,16 @@ func (j JSONStructArray) Out() error {
 	if err != nil {
 		return err
 	}
+
+	// JSON returns a byte array with a literal null [110 117 108 108] in it
+	// if it is passed empty data.  We used bytes.Compare to see if that is
+	// the case.
+	if diff := bytes.Compare(data, []byte("null")); diff == 0 {
+		data = []byte("[]")
+	}
+
+	// If the we did get NULL back, we should spit out {} which is
+	// at least valid JSON for the consumer.
 	fmt.Printf("%s\n", data)
 	return nil
 }

cmd/kpod/ps.go

@@ -398,7 +398,9 @@ func getJSONOutput(containers []*libkpod.ContainerData, nSpace bool) (psOutput [
 
 func generatePsOutput(containers []*libkpod.ContainerData, server *libkpod.ContainerServer, opts psOptions) error {
 	containersOutput := getContainers(containers, opts)
-	if len(containersOutput) == 0 {
+	// In the case of JSON, we want to continue so we at least pass
+	// {} --valid JSON-- to the consumer
+	if len(containersOutput) == 0 && opts.format != formats.JSONString {
 		return nil
 	}
 

cmd/kpod/rm.go

@@ -6,6 +6,7 @@ import (
 	"github.com/kubernetes-incubator/cri-o/libkpod"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
+	"golang.org/x/net/context"
 )
 
 var (
@@ -53,7 +54,7 @@ func rmCmd(c *cli.Context) error {
 	force := c.Bool("force")
 
 	for _, container := range c.Args() {
-		id, err2 := server.Remove(container, force)
+		id, err2 := server.Remove(context.Background(), container, force)
 		if err2 != nil {
 			if err == nil {
 				err = err2

cmd/kpod/stop.go

@@ -7,6 +7,7 @@ import (
 	"github.com/kubernetes-incubator/cri-o/libkpod"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
+	"golang.org/x/net/context"
 )
 
 var (
@@ -61,7 +62,7 @@ func stopCmd(c *cli.Context) error {
 	}
 	var lastError error
 	for _, container := range c.Args() {
-		cid, err := server.ContainerStop(container, stopTimeout)
+		cid, err := server.ContainerStop(context.Background(), container, stopTimeout)
 		if err != nil {
 			if lastError != nil {
 				fmt.Fprintln(os.Stderr, lastError)

conmon/Makefile

@@ -5,8 +5,8 @@ override LIBS += $(shell pkg-config --libs glib-2.0)
 override CFLAGS += -std=c99 -Os -Wall -Wextra $(shell pkg-config --cflags glib-2.0)
 
 conmon: $(obj)
-	$(CC) -o $@ $^ $(CFLAGS) $(LIBS)
+	$(CC) -o ../bin/$@ $^ $(CFLAGS) $(LIBS)
 
 .PHONY: clean
 clean:
-	rm -f $(obj) conmon
+	rm -f $(obj) ../bin/conmon

contrib/system_containers/centos/Dockerfile

@@ -0,0 +1,29 @@
+FROM centos
+
+ENV VERSION=0 RELEASE=1 ARCH=x86_64
+LABEL com.redhat.component="cri-o" \
+      name="$FGC/cri-o" \
+      version="$VERSION" \
+      release="$RELEASE.$DISTTAG" \
+      architecture="$ARCH" \
+      usage="atomic install --system --system-package=no crio && systemctl start crio" \
+      summary="The cri-o daemon as a system container." \
+      maintainer="Yu Qi Zhang <jzehrarnyg@gmail.com>" \
+      atomic.type="system"
+
+RUN yum-config-manager --nogpgcheck --add-repo https://cbs.centos.org/repos/virt7-container-common-candidate/x86_64/os/ && \
+    yum install --disablerepo=extras --nogpgcheck --setopt=tsflags=nodocs -y iptables cri-o socat iproute runc && \
+    rpm -V iptables cri-o iproute runc && \
+    yum clean all && \
+    mkdir -p /exports/hostfs/etc/crio /exports/hostfs/opt/cni/bin/ /exports/hostfs/var/lib/containers/storage/ && \
+    cp /etc/crio/* /exports/hostfs/etc/crio && \
+    if test -e /usr/libexec/cni; then cp -Lr /usr/libexec/cni/* /exports/hostfs/opt/cni/bin/; fi
+
+RUN sed -i '/storage_option =/s/.*/&\n"overlay.override_kernel_check=1",/' /exports/hostfs/etc/crio/crio.conf
+
+COPY manifest.json tmpfiles.template config.json.template service.template /exports/
+
+COPY set_mounts.sh /
+COPY run.sh /usr/bin/
+
+CMD ["/usr/bin/run.sh"]

contrib/system_containers/centos/README.md

@@ -0,0 +1,57 @@
+# cri-o
+
+This is the cri-o daemon as a system container.
+
+## Building the image from source:
+
+```
+# git clone https://github.com/projectatomic/atomic-system-containers
+# cd atomic-system-containers/cri-o
+# docker build -t crio .
+```
+
+## Running the system container, with the atomic CLI:
+
+Pull from registry into ostree:
+
+```
+# atomic pull --storage ostree $REGISTRY/crio
+```
+
+Or alternatively, pull from local docker:
+
+```
+# atomic pull --storage ostree docker:crio:latest
+```
+
+Install the container:
+
+Currently we recommend using --system-package=no to avoid having rpmbuild create an rpm file
+during installation. This flag will tell the atomic CLI to fall back to copying files to the
+host instead.
+
+```
+# atomic install --system --system-package=no --name=crio ($REGISTRY)/crio
+```
+
+Start as a systemd service:
+
+```
+# systemctl start crio
+```
+
+Stopping the service
+
+```
+# systemctl stop crio
+```
+
+Removing the container
+
+```
+# atomic uninstall crio
+```
+
+## Binary version
+
+You can find the image automatically built as: registry.centos.org/projectatomic/cri-o:latest

contrib/system_containers/centos/cccp.yml

@@ -0,0 +1,41 @@
+# This is for the purpose of building containers on the CentOS Community Container
+# Pipeline. The containers are built, tested and delivered to registry.centos.org and
+# lifecycled as well. A corresponding entry must exist in the container index itself,
+# located at https://github.com/CentOS/container-index/tree/master/index.d
+# You can know more at the following links:
+# * https://github.com/CentOS/container-pipeline-service/blob/master/README.md
+# * https://github.com/CentOS/container-index/blob/master/README.rst
+# * https://wiki.centos.org/ContainerPipeline
+
+# This will be part of the name of the container. It should match the job-id in index entry
+job-id: cri-o
+
+#the following are optional, can be left blank
+#defaults, where applicable are filled in
+#nulecule-file   : nulecule
+
+# This flag tells the container pipeline to skip user defined tests on their container
+test-skip       : True
+
+# This is path of the script that initiates the user defined tests. It must be able to
+# return an exit code.
+test-script     : null
+
+# This is the path of custom build script.
+build-script    : null
+
+# This is the path of the custom delivery script
+delivery-script : null
+
+# This flag tells the pipeline to deliver this container to docker hub.
+docker-index    : True
+
+# This flag can be used to enable or disable the custom delivery
+custom-delivery : False
+
+# This flag can be used to enable or disable delivery of container to local registry
+local-delivery  : True
+
+Upstreams       :
+        - ref           :
+          url           :

contrib/system_containers/centos/config.json.template

@@ -0,0 +1,427 @@
+{
+    "ociVersion": "1.0.0",
+    "platform": {
+        "arch": "amd64",
+        "os": "linux"
+    },
+    "process": {
+        "args": [
+            "/usr/bin/run.sh"
+        ],
+        "capabilities": {
+            "ambient": [
+                "CAP_CHOWN",
+                "CAP_FOWNER",
+                "CAP_FSETID",
+                "CAP_KILL",
+                "CAP_SETGID",
+                "CAP_SETUID",
+                "CAP_SETPCAP",
+                "CAP_LINUX_IMMUTABLE",
+                "CAP_NET_BIND_SERVICE",
+                "CAP_NET_BROADCAST",
+                "CAP_NET_ADMIN",
+                "CAP_NET_RAW",
+                "CAP_IPC_LOCK",
+                "CAP_IPC_OWNER",
+                "CAP_SYS_MODULE",
+                "CAP_SYS_RAWIO",
+                "CAP_SYS_CHROOT",
+                "CAP_SYS_PTRACE",
+                "CAP_SYS_PACCT",
+                "CAP_SYS_ADMIN",
+                "CAP_SYS_BOOT",
+                "CAP_SYS_NICE",
+                "CAP_SYS_RESOURCE",
+                "CAP_SYS_TIME",
+                "CAP_SYS_TTY_CONFIG",
+                "CAP_MKNOD",
+                "CAP_LEASE",
+                "CAP_AUDIT_WRITE",
+                "CAP_AUDIT_CONTROL",
+                "CAP_SETFCAP",
+                "CAP_DAC_OVERRIDE",
+                "CAP_MAC_OVERRIDE",
+                "CAP_DAC_READ_SEARCH",
+                "CAP_MAC_ADMIN",
+                "CAP_SYSLOG",
+                "CAP_WAKE_ALARM",
+                "CAP_BLOCK_SUSPEND"
+            ],
+            "bounding": [
+                "CAP_CHOWN",
+                "CAP_FOWNER",
+                "CAP_FSETID",
+                "CAP_KILL",
+                "CAP_SETGID",
+                "CAP_SETUID",
+                "CAP_SETPCAP",
+                "CAP_LINUX_IMMUTABLE",
+                "CAP_NET_BIND_SERVICE",
+                "CAP_NET_BROADCAST",
+                "CAP_NET_ADMIN",
+                "CAP_NET_RAW",
+                "CAP_IPC_LOCK",
+                "CAP_IPC_OWNER",
+                "CAP_SYS_MODULE",
+                "CAP_SYS_RAWIO",
+                "CAP_SYS_CHROOT",
+                "CAP_SYS_PTRACE",
+                "CAP_SYS_PACCT",
+                "CAP_SYS_ADMIN",
+                "CAP_SYS_BOOT",
+                "CAP_SYS_NICE",
+                "CAP_SYS_RESOURCE",
+                "CAP_SYS_TIME",
+                "CAP_SYS_TTY_CONFIG",
+                "CAP_MKNOD",
+                "CAP_LEASE",
+                "CAP_AUDIT_WRITE",
+                "CAP_AUDIT_CONTROL",
+                "CAP_SETFCAP",
+                "CAP_DAC_OVERRIDE",
+                "CAP_MAC_OVERRIDE",
+                "CAP_DAC_READ_SEARCH",
+                "CAP_MAC_ADMIN",
+                "CAP_SYSLOG",
+                "CAP_WAKE_ALARM",
+                "CAP_BLOCK_SUSPEND"
+            ],
+            "effective": [
+                "CAP_CHOWN",
+                "CAP_FOWNER",
+                "CAP_FSETID",
+                "CAP_KILL",
+                "CAP_SETGID",
+                "CAP_SETUID",
+                "CAP_SETPCAP",
+                "CAP_LINUX_IMMUTABLE",
+                "CAP_NET_BIND_SERVICE",
+                "CAP_NET_BROADCAST",
+                "CAP_NET_ADMIN",
+                "CAP_NET_RAW",
+                "CAP_IPC_LOCK",
+                "CAP_IPC_OWNER",
+                "CAP_SYS_MODULE",
+                "CAP_SYS_RAWIO",
+                "CAP_SYS_CHROOT",
+                "CAP_SYS_PTRACE",
+                "CAP_SYS_PACCT",
+                "CAP_SYS_ADMIN",
+                "CAP_SYS_BOOT",
+                "CAP_SYS_NICE",
+                "CAP_SYS_RESOURCE",
+                "CAP_SYS_TIME",
+                "CAP_SYS_TTY_CONFIG",
+                "CAP_MKNOD",
+                "CAP_LEASE",
+                "CAP_AUDIT_WRITE",
+                "CAP_AUDIT_CONTROL",
+                "CAP_SETFCAP",
+                "CAP_DAC_OVERRIDE",
+                "CAP_MAC_OVERRIDE",
+                "CAP_DAC_READ_SEARCH",
+                "CAP_MAC_ADMIN",
+                "CAP_SYSLOG",
+                "CAP_WAKE_ALARM",
+                "CAP_BLOCK_SUSPEND"
+            ],
+            "inheritable": [
+                "CAP_CHOWN",
+                "CAP_FOWNER",
+                "CAP_FSETID",
+                "CAP_KILL",
+                "CAP_SETGID",
+                "CAP_SETUID",
+                "CAP_SETPCAP",
+                "CAP_LINUX_IMMUTABLE",
+                "CAP_NET_BIND_SERVICE",
+                "CAP_NET_BROADCAST",
+                "CAP_NET_ADMIN",
+                "CAP_NET_RAW",
+                "CAP_IPC_LOCK",
+                "CAP_IPC_OWNER",
+                "CAP_SYS_MODULE",
+                "CAP_SYS_RAWIO",
+                "CAP_SYS_CHROOT",
+                "CAP_SYS_PTRACE",
+                "CAP_SYS_PACCT",
+                "CAP_SYS_ADMIN",
+                "CAP_SYS_BOOT",
+                "CAP_SYS_NICE",
+                "CAP_SYS_RESOURCE",
+                "CAP_SYS_TIME",
+                "CAP_SYS_TTY_CONFIG",
+                "CAP_MKNOD",
+                "CAP_LEASE",
+                "CAP_AUDIT_WRITE",
+                "CAP_AUDIT_CONTROL",
+                "CAP_SETFCAP",
+                "CAP_DAC_OVERRIDE",
+                "CAP_MAC_OVERRIDE",
+                "CAP_DAC_READ_SEARCH",
+                "CAP_MAC_ADMIN",
+                "CAP_SYSLOG",
+                "CAP_WAKE_ALARM",
+                "CAP_BLOCK_SUSPEND"
+            ],
+            "permitted": [
+                "CAP_CHOWN",
+                "CAP_FOWNER",
+                "CAP_FSETID",
+                "CAP_KILL",
+                "CAP_SETGID",
+                "CAP_SETUID",
+                "CAP_SETPCAP",
+                "CAP_LINUX_IMMUTABLE",
+                "CAP_NET_BIND_SERVICE",
+                "CAP_NET_BROADCAST",
+                "CAP_NET_ADMIN",
+                "CAP_NET_RAW",
+                "CAP_IPC_LOCK",
+                "CAP_IPC_OWNER",
+                "CAP_SYS_MODULE",
+                "CAP_SYS_RAWIO",
+                "CAP_SYS_CHROOT",
+                "CAP_SYS_PTRACE",
+                "CAP_SYS_PACCT",
+                "CAP_SYS_ADMIN",
+                "CAP_SYS_BOOT",
+                "CAP_SYS_NICE",
+                "CAP_SYS_RESOURCE",
+                "CAP_SYS_TIME",
+                "CAP_SYS_TTY_CONFIG",
+                "CAP_MKNOD",
+                "CAP_LEASE",
+                "CAP_AUDIT_WRITE",
+                "CAP_AUDIT_CONTROL",
+                "CAP_SETFCAP",
+                "CAP_DAC_OVERRIDE",
+                "CAP_MAC_OVERRIDE",
+                "CAP_DAC_READ_SEARCH",
+                "CAP_MAC_ADMIN",
+                "CAP_SYSLOG",
+                "CAP_WAKE_ALARM",
+                "CAP_BLOCK_SUSPEND"
+            ]
+        },
+        "selinuxLabel": "system_u:system_r:container_runtime_t:s0",
+        "cwd": "/",
+        "env": [
+            "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin:/root/go/bin",
+            "TERM=xterm",
+            "LOG_LEVEL=$LOG_LEVEL",
+            "NAME=$NAME"
+        ],
+        "noNewPrivileges": false,
+        "terminal": false,
+        "user": {
+            "gid": 0,
+            "uid": 0
+        }
+    },
+    "root": {
+        "path": "rootfs",
+        "readonly": true
+    },
+    "hooks": {},
+    "linux": {
+        "namespaces": [
+            {
+                "type": "mount"
+            }
+        ],
+        "resources": {
+            "devices": [
+                {
+                    "access": "rwm",
+                    "allow": true
+                }
+            ]
+        },
+        "rootfsPropagation": "private"
+    },
+    "mounts": [
+        {
+            "destination": "/tmp",
+            "options": [
+                "private",
+                "bind",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/tmp",
+            "type": "bind"
+        },
+        {
+            "destination": "/etc",
+            "options": [
+                "rbind",
+                "rprivate",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/etc",
+            "type": "bind"
+        },
+        {
+            "destination": "/lib/modules",
+            "options": [
+                "rbind",
+                "rprivate",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/lib/modules",
+            "type": "bind"
+        },
+        {
+            "destination": "/root",
+            "options": [
+                "rbind",
+                "rprivate",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/root",
+            "type": "bind"
+        },
+        {
+            "destination": "/home",
+            "options": [
+                "rbind",
+                "rprivate",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/home",
+            "type": "bind"
+        },
+        {
+            "destination": "/mnt",
+            "options": [
+                "rbind",
+                "rw",
+                "rprivate",
+                "mode=755"
+            ],
+            "source": "/mnt",
+            "type": "bind"
+        },
+	{
+	    "type": "bind",
+	    "source": "${RUN_DIRECTORY}",
+	    "destination": "/run",
+	    "options": [
+		"rshared",
+		"rbind",
+		"rw",
+		"mode=755"
+	    ]
+	},
+	{
+	    "type": "bind",
+	    "source": "${RUN_DIRECTORY}/systemd",
+	    "destination": "/run/systemd",
+	    "options": [
+                "rslave",
+                "bind",
+                "rw",
+                "mode=755"
+	    ]
+	},
+        {
+            "destination": "/var/log",
+            "options": [
+                "rbind",
+                "rslave",
+                "rw"
+            ],
+            "source": "/var/log",
+            "type": "bind"
+        },
+        {
+            "destination": "/var/lib",
+            "options": [
+                "rbind",
+                "rprivate",
+                "rw"
+            ],
+            "source": "${STATE_DIRECTORY}",
+            "type": "bind"
+        },
+        {
+            "destination": "/var/lib/containers/storage",
+            "options": [
+                "rbind",
+                "rshared",
+                "rw"
+            ],
+            "source": "${VAR_LIB_CONTAINERS_STORAGE}",
+            "type": "bind"
+        },
+        {
+            "destination": "/var/lib/origin",
+            "options": [
+                "rshared",
+                "bind",
+                "rw"
+            ],
+            "source": "${VAR_LIB_ORIGIN}",
+            "type": "bind"
+        },
+        {
+            "destination": "/var/lib/kubelet",
+            "options": [
+                "rshared",
+                "bind",
+                "rw"
+            ],
+            "source": "${VAR_LIB_KUBE}",
+            "type": "bind"
+        },
+        {
+            "destination": "/opt/cni",
+            "options": [
+                "rbind",
+                "rprivate",
+                "ro",
+                "mode=755"
+            ],
+            "source": "${OPT_CNI}",
+            "type": "bind"
+        },
+        {
+            "destination": "/dev",
+            "options": [
+                "rprivate",
+                "rbind",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/dev",
+            "type": "bind"
+        },
+        {
+            "destination": "/sys",
+            "options": [
+                "rprivate",
+                "rbind",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/sys",
+            "type": "bind"
+        },
+        {
+            "destination": "/proc",
+            "options": [
+                "rbind",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/proc",
+            "type": "proc"
+        }
+    ]
+}

contrib/system_containers/centos/manifest.json

@@ -0,0 +1,10 @@
+{
+    "version": "1.0",
+    "defaultValues": {
+        "LOG_LEVEL" : "info",
+        "OPT_CNI" : "/opt/cni",
+        "VAR_LIB_CONTAINERS_STORAGE" : "/var/lib/containers/storage",
+        "VAR_LIB_ORIGIN" : "/var/lib/origin",
+        "VAR_LIB_KUBE" : "/var/lib/kubelet"
+    }
+}

contrib/system_containers/centos/run.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Ensure that new process maintain this SELinux label
+PID=$$
+LABEL=`tr -d '\000' < /proc/$PID/attr/current`
+printf %s $LABEL > /proc/self/attr/exec
+
+exec /usr/bin/crio --log-level=$LOG_LEVEL

contrib/system_containers/centos/service.template

@@ -0,0 +1,20 @@
+[Unit]
+Description=crio daemon
+After=network.target
+
+[Service]
+Type=notify
+ExecStartPre=/bin/sh $DESTDIR/rootfs/set_mounts.sh
+ExecStart=$EXEC_START
+ExecStop=$EXEC_STOP
+Restart=on-failure
+WorkingDirectory=$DESTDIR
+RuntimeDirectory=${NAME}
+TasksMax=infinity
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+TimeoutStartSec=0
+
+[Install]
+WantedBy=multi-user.target

contrib/system_containers/centos/set_mounts.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+findmnt /var/lib/containers/storage > /dev/null || mount --rbind --make-shared /var/lib/containers/storage /var/lib/containers/storage
+findmnt /var/lib/origin > /dev/null || mount --bind --make-shared /var/lib/origin /var/lib/origin
+findmnt /var/lib/kubelet > /dev/null || mount --bind --make-shared /var/lib/kubelet /var/lib/kubelet
+mount --make-shared /run
+findmnt /run/systemd > /dev/null || mount --bind --make-rslave /run/systemd /run/systemd

contrib/system_containers/centos/tmpfiles.template

@@ -0,0 +1,5 @@
+d    ${RUN_DIRECTORY}/${NAME}               -        -           -       - -
+d    /etc/crio - - - - -
+Z    /etc/crio - - - - -
+d    ${STATE_DIRECTORY}/origin               -        -           -       - -
+d    ${STATE_DIRECTORY}/kubelet              -        -           -       - -

contrib/system_containers/fedora/Dockerfile

@@ -0,0 +1,30 @@
+FROM registry.fedoraproject.org/fedora:27
+
+ENV VERSION=0 RELEASE=1 ARCH=x86_64
+LABEL com.redhat.component="cri-o" \
+      name="$FGC/cri-o" \
+      version="$VERSION" \
+      release="$RELEASE.$DISTTAG" \
+      architecture="$ARCH" \
+      usage="atomic install --system --system-package=no crio && systemctl start crio" \
+      summary="The cri-o daemon as a system container." \
+      maintainer="Yu Qi Zhang <jzehrarnyg@gmail.com>" \
+      atomic.type="system"
+
+COPY README.md /
+
+RUN dnf install --enablerepo=updates-testing --setopt=tsflags=nodocs -y iptables cri-o socat iproute runc && \
+    rpm -V iptables cri-o iproute runc && \
+    dnf clean all && \
+    mkdir -p /exports/hostfs/etc/crio /exports/hostfs/opt/cni/bin/ /exports/hostfs/var/lib/containers/storage/ && \
+    cp /etc/crio/* /exports/hostfs/etc/crio && \
+    if test -e /usr/libexec/cni; then cp -Lr /usr/libexec/cni/* /exports/hostfs/opt/cni/bin/; fi
+
+RUN sed -i '/storage_option =/s/.*/&\n"overlay.override_kernel_check=1",/' /exports/hostfs/etc/crio/crio.conf
+
+COPY manifest.json tmpfiles.template config.json.template service.template /exports/
+
+COPY set_mounts.sh /
+COPY run.sh /usr/bin/
+
+CMD ["/usr/bin/run.sh"]

contrib/system_containers/fedora/README.md

@@ -0,0 +1,53 @@
+# cri-o
+
+This is the cri-o daemon as a system container.
+
+## Building the image from source:
+
+```
+# git clone https://github.com/projectatomic/atomic-system-containers
+# cd atomic-system-containers/cri-o
+# docker build -t crio .
+```
+
+## Running the system container, with the atomic CLI:
+
+Pull from registry into ostree:
+
+```
+# atomic pull --storage ostree $REGISTRY/crio
+```
+
+Or alternatively, pull from local docker:
+
+```
+# atomic pull --storage ostree docker:crio:latest
+```
+
+Install the container:
+
+Currently we recommend using --system-package=no to avoid having rpmbuild create an rpm file
+during installation. This flag will tell the atomic CLI to fall back to copying files to the
+host instead.
+
+```
+# atomic install --system --system-package=no --name=crio ($REGISTRY)/crio
+```
+
+Start as a systemd service:
+
+```
+# systemctl start crio
+```
+
+Stopping the service
+
+```
+# systemctl stop crio
+```
+
+Removing the container
+
+```
+# atomic uninstall crio
+```

contrib/system_containers/fedora/config.json.template

@@ -0,0 +1,432 @@
+{
+    "ociVersion": "1.0.0",
+    "platform": {
+        "arch": "amd64",
+        "os": "linux"
+    },
+    "process": {
+        "args": [
+            "/usr/bin/run.sh"
+        ],
+        "selinuxLabel": "system_u:system_r:container_runtime_t:s0",
+        "capabilities": {
+            "ambient": [
+                "CAP_CHOWN",
+                "CAP_FOWNER",
+                "CAP_FSETID",
+                "CAP_KILL",
+                "CAP_SETGID",
+                "CAP_SETUID",
+                "CAP_SETPCAP",
+                "CAP_LINUX_IMMUTABLE",
+                "CAP_NET_BIND_SERVICE",
+                "CAP_NET_BROADCAST",
+                "CAP_NET_ADMIN",
+                "CAP_NET_RAW",
+                "CAP_IPC_LOCK",
+                "CAP_IPC_OWNER",
+                "CAP_SYS_MODULE",
+                "CAP_SYS_RAWIO",
+                "CAP_SYS_CHROOT",
+                "CAP_SYS_PTRACE",
+                "CAP_SYS_PACCT",
+                "CAP_SYS_ADMIN",
+                "CAP_SYS_BOOT",
+                "CAP_SYS_NICE",
+                "CAP_SYS_RESOURCE",
+                "CAP_SYS_TIME",
+                "CAP_SYS_TTY_CONFIG",
+                "CAP_MKNOD",
+                "CAP_LEASE",
+                "CAP_AUDIT_WRITE",
+                "CAP_AUDIT_CONTROL",
+                "CAP_SETFCAP",
+                "CAP_DAC_OVERRIDE",
+                "CAP_MAC_OVERRIDE",
+                "CAP_DAC_READ_SEARCH",
+                "CAP_MAC_ADMIN",
+                "CAP_SYSLOG",
+                "CAP_WAKE_ALARM",
+                "CAP_BLOCK_SUSPEND",
+                "CAP_AUDIT_READ"
+            ],
+            "bounding": [
+                "CAP_CHOWN",
+                "CAP_FOWNER",
+                "CAP_FSETID",
+                "CAP_KILL",
+                "CAP_SETGID",
+                "CAP_SETUID",
+                "CAP_SETPCAP",
+                "CAP_LINUX_IMMUTABLE",
+                "CAP_NET_BIND_SERVICE",
+                "CAP_NET_BROADCAST",
+                "CAP_NET_ADMIN",
+                "CAP_NET_RAW",
+                "CAP_IPC_LOCK",
+                "CAP_IPC_OWNER",
+                "CAP_SYS_MODULE",
+                "CAP_SYS_RAWIO",
+                "CAP_SYS_CHROOT",
+                "CAP_SYS_PTRACE",
+                "CAP_SYS_PACCT",
+                "CAP_SYS_ADMIN",
+                "CAP_SYS_BOOT",
+                "CAP_SYS_NICE",
+                "CAP_SYS_RESOURCE",
+                "CAP_SYS_TIME",
+                "CAP_SYS_TTY_CONFIG",
+                "CAP_MKNOD",
+                "CAP_LEASE",
+                "CAP_AUDIT_WRITE",
+                "CAP_AUDIT_CONTROL",
+                "CAP_SETFCAP",
+                "CAP_DAC_OVERRIDE",
+                "CAP_MAC_OVERRIDE",
+                "CAP_DAC_READ_SEARCH",
+                "CAP_MAC_ADMIN",
+                "CAP_SYSLOG",
+                "CAP_WAKE_ALARM",
+                "CAP_BLOCK_SUSPEND",
+                "CAP_AUDIT_READ"
+            ],
+            "effective": [
+                "CAP_CHOWN",
+                "CAP_FOWNER",
+                "CAP_FSETID",
+                "CAP_KILL",
+                "CAP_SETGID",
+                "CAP_SETUID",
+                "CAP_SETPCAP",
+                "CAP_LINUX_IMMUTABLE",
+                "CAP_NET_BIND_SERVICE",
+                "CAP_NET_BROADCAST",
+                "CAP_NET_ADMIN",
+                "CAP_NET_RAW",
+                "CAP_IPC_LOCK",
+                "CAP_IPC_OWNER",
+                "CAP_SYS_MODULE",
+                "CAP_SYS_RAWIO",
+                "CAP_SYS_CHROOT",
+                "CAP_SYS_PTRACE",
+                "CAP_SYS_PACCT",
+                "CAP_SYS_ADMIN",
+                "CAP_SYS_BOOT",
+                "CAP_SYS_NICE",
+                "CAP_SYS_RESOURCE",
+                "CAP_SYS_TIME",
+                "CAP_SYS_TTY_CONFIG",
+                "CAP_MKNOD",
+                "CAP_LEASE",
+                "CAP_AUDIT_WRITE",
+                "CAP_AUDIT_CONTROL",
+                "CAP_SETFCAP",
+                "CAP_DAC_OVERRIDE",
+                "CAP_MAC_OVERRIDE",
+                "CAP_DAC_READ_SEARCH",
+                "CAP_MAC_ADMIN",
+                "CAP_SYSLOG",
+                "CAP_WAKE_ALARM",
+                "CAP_BLOCK_SUSPEND",
+                "CAP_AUDIT_READ"
+            ],
+            "inheritable": [
+                "CAP_CHOWN",
+                "CAP_FOWNER",
+                "CAP_FSETID",
+                "CAP_KILL",
+                "CAP_SETGID",
+                "CAP_SETUID",
+                "CAP_SETPCAP",
+                "CAP_LINUX_IMMUTABLE",
+                "CAP_NET_BIND_SERVICE",
+                "CAP_NET_BROADCAST",
+                "CAP_NET_ADMIN",
+                "CAP_NET_RAW",
+                "CAP_IPC_LOCK",
+                "CAP_IPC_OWNER",
+                "CAP_SYS_MODULE",
+                "CAP_SYS_RAWIO",
+                "CAP_SYS_CHROOT",
+                "CAP_SYS_PTRACE",
+                "CAP_SYS_PACCT",
+                "CAP_SYS_ADMIN",
+                "CAP_SYS_BOOT",
+                "CAP_SYS_NICE",
+                "CAP_SYS_RESOURCE",
+                "CAP_SYS_TIME",
+                "CAP_SYS_TTY_CONFIG",
+                "CAP_MKNOD",
+                "CAP_LEASE",
+                "CAP_AUDIT_WRITE",
+                "CAP_AUDIT_CONTROL",
+                "CAP_SETFCAP",
+                "CAP_DAC_OVERRIDE",
+                "CAP_MAC_OVERRIDE",
+                "CAP_DAC_READ_SEARCH",
+                "CAP_MAC_ADMIN",
+                "CAP_SYSLOG",
+                "CAP_WAKE_ALARM",
+                "CAP_BLOCK_SUSPEND",
+                "CAP_AUDIT_READ"
+            ],
+            "permitted": [
+                "CAP_CHOWN",
+                "CAP_FOWNER",
+                "CAP_FSETID",
+                "CAP_KILL",
+                "CAP_SETGID",
+                "CAP_SETUID",
+                "CAP_SETPCAP",
+                "CAP_LINUX_IMMUTABLE",
+                "CAP_NET_BIND_SERVICE",
+                "CAP_NET_BROADCAST",
+                "CAP_NET_ADMIN",
+                "CAP_NET_RAW",
+                "CAP_IPC_LOCK",
+                "CAP_IPC_OWNER",
+                "CAP_SYS_MODULE",
+                "CAP_SYS_RAWIO",
+                "CAP_SYS_CHROOT",
+                "CAP_SYS_PTRACE",
+                "CAP_SYS_PACCT",
+                "CAP_SYS_ADMIN",
+                "CAP_SYS_BOOT",
+                "CAP_SYS_NICE",
+                "CAP_SYS_RESOURCE",
+                "CAP_SYS_TIME",
+                "CAP_SYS_TTY_CONFIG",
+                "CAP_MKNOD",
+                "CAP_LEASE",
+                "CAP_AUDIT_WRITE",
+                "CAP_AUDIT_CONTROL",
+                "CAP_SETFCAP",
+                "CAP_DAC_OVERRIDE",
+                "CAP_MAC_OVERRIDE",
+                "CAP_DAC_READ_SEARCH",
+                "CAP_MAC_ADMIN",
+                "CAP_SYSLOG",
+                "CAP_WAKE_ALARM",
+                "CAP_BLOCK_SUSPEND",
+                "CAP_AUDIT_READ"
+            ]
+        },
+        "cwd": "/",
+        "env": [
+            "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin:/root/go/bin",
+            "TERM=xterm",
+            "LOG_LEVEL=$LOG_LEVEL",
+            "NAME=$NAME"
+        ],
+        "noNewPrivileges": false,
+        "terminal": false,
+        "user": {
+            "gid": 0,
+            "uid": 0
+        }
+    },
+    "root": {
+        "path": "rootfs",
+        "readonly": true
+    },
+    "hooks": {},
+    "linux": {
+        "namespaces": [
+            {
+                "type": "mount"
+            }
+        ],
+        "resources": {
+            "devices": [
+                {
+                    "access": "rwm",
+                    "allow": true
+                }
+            ]
+        },
+        "rootfsPropagation": "private"
+    },
+    "mounts": [
+        {
+            "destination": "/tmp",
+            "options": [
+                "private",
+                "bind",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/tmp",
+            "type": "bind"
+        },
+        {
+            "destination": "/etc",
+            "options": [
+                "rbind",
+                "rprivate",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/etc",
+            "type": "bind"
+        },
+        {
+            "destination": "/lib/modules",
+            "options": [
+                "rbind",
+                "rprivate",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/lib/modules",
+            "type": "bind"
+        },
+        {
+            "destination": "/root",
+            "options": [
+                "rbind",
+                "rprivate",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/root",
+            "type": "bind"
+        },
+        {
+            "destination": "/home",
+            "options": [
+                "rbind",
+                "rprivate",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/home",
+            "type": "bind"
+        },
+        {
+            "destination": "/mnt",
+            "options": [
+                "rbind",
+                "rw",
+                "rprivate",
+                "mode=755"
+            ],
+            "source": "/mnt",
+            "type": "bind"
+        },
+	{
+	    "type": "bind",
+	    "source": "${RUN_DIRECTORY}",
+	    "destination": "/run",
+	    "options": [
+		"rshared",
+		"rbind",
+		"rw",
+		"mode=755"
+	    ]
+	},
+	{
+	    "type": "bind",
+	    "source": "${RUN_DIRECTORY}/systemd",
+	    "destination": "/run/systemd",
+	    "options": [
+                "rslave",
+                "bind",
+                "rw",
+                "mode=755"
+	    ]
+	},
+        {
+            "destination": "/var/log",
+            "options": [
+                "rbind",
+                "rslave",
+                "rw"
+            ],
+            "source": "/var/log",
+            "type": "bind"
+        },
+        {
+            "destination": "/var/lib",
+            "options": [
+                "rbind",
+                "rprivate",
+                "rw"
+            ],
+            "source": "${STATE_DIRECTORY}",
+            "type": "bind"
+        },
+        {
+            "destination": "/var/lib/containers/storage",
+            "options": [
+                "rbind",
+                "rshared",
+                "rw"
+            ],
+            "source": "${VAR_LIB_CONTAINERS_STORAGE}",
+            "type": "bind"
+        },
+        {
+            "destination": "/var/lib/origin",
+            "options": [
+                "rshared",
+                "bind",
+                "rw"
+            ],
+            "source": "${VAR_LIB_ORIGIN}",
+            "type": "bind"
+        },
+        {
+            "destination": "/var/lib/kubelet",
+            "options": [
+                "rshared",
+                "bind",
+                "rw"
+            ],
+            "source": "${VAR_LIB_KUBE}",
+            "type": "bind"
+        },
+        {
+            "destination": "/opt/cni",
+            "options": [
+                "rbind",
+                "rprivate",
+                "ro",
+                "mode=755"
+            ],
+            "source": "${OPT_CNI}",
+            "type": "bind"
+        },
+        {
+            "destination": "/dev",
+            "options": [
+                "rprivate",
+                "rbind",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/dev",
+            "type": "bind"
+        },
+        {
+            "destination": "/sys",
+            "options": [
+                "rprivate",
+                "rbind",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/sys",
+            "type": "bind"
+        },
+        {
+            "destination": "/proc",
+            "options": [
+                "rbind",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/proc",
+            "type": "proc"
+        }
+    ]
+}

contrib/system_containers/fedora/manifest.json

@@ -0,0 +1,10 @@
+{
+    "version": "1.0",
+    "defaultValues": {
+        "LOG_LEVEL" : "info",
+        "OPT_CNI" : "/opt/cni",
+        "VAR_LIB_CONTAINERS_STORAGE" : "/var/lib/containers/storage",
+        "VAR_LIB_ORIGIN" : "/var/lib/origin",
+        "VAR_LIB_KUBE" : "/var/lib/kubelet"
+    }
+}

contrib/system_containers/fedora/run.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Ensure that new process maintain this SELinux label
+PID=$$
+LABEL=`tr -d '\000' < /proc/$PID/attr/current`
+printf %s $LABEL > /proc/self/attr/exec
+
+exec /usr/bin/crio --log-level=$LOG_LEVEL

contrib/system_containers/fedora/service.template

@@ -0,0 +1,20 @@
+[Unit]
+Description=crio daemon
+After=network.target
+
+[Service]
+Type=notify
+ExecStartPre=/bin/sh $DESTDIR/rootfs/set_mounts.sh
+ExecStart=$EXEC_START
+ExecStop=$EXEC_STOP
+Restart=on-failure
+WorkingDirectory=$DESTDIR
+RuntimeDirectory=${NAME}
+TasksMax=infinity
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+TimeoutStartSec=0
+
+[Install]
+WantedBy=multi-user.target

contrib/system_containers/fedora/set_mounts.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+findmnt /var/lib/containers/storage > /dev/null || mount --rbind --make-shared /var/lib/containers/storage /var/lib/containers/storage
+findmnt /var/lib/origin > /dev/null || mount --bind --make-shared /var/lib/origin /var/lib/origin
+findmnt /var/lib/kubelet > /dev/null || mount --bind --make-shared /var/lib/kubelet /var/lib/kubelet
+mount --make-shared /run
+findmnt /run/systemd > /dev/null || mount --bind --make-rslave /run/systemd /run/systemd

contrib/system_containers/fedora/tmpfiles.template

@@ -0,0 +1,5 @@
+d    ${RUN_DIRECTORY}/${NAME}               -        -           -       - -
+d    /etc/crio - - - - -
+Z    /etc/crio - - - - -
+d    ${STATE_DIRECTORY}/origin               -        -           -       - -
+d    ${STATE_DIRECTORY}/kubelet              -        -           -       - -

contrib/system_containers/rhel/Dockerfile

@@ -0,0 +1,41 @@
+#oit## This file is managed by the OpenShift Image Tool
+#oit## by the OpenShift Continuous Delivery team.
+#oit##
+#oit## Any yum repos listed in this file will effectively be ignored during CD builds.
+#oit## Yum repos must be enabled in the oit configuration files.
+#oit## Some aspects of this file may be managed programmatically. For example, the image name, labels (version,
+#oit## release, and other), and the base FROM. Changes made directly in distgit may be lost during the next
+#oit## reconciliation.
+#oit##
+FROM rhel7:7-released
+
+RUN \
+    yum install --setopt=tsflags=nodocs -y socat iptables cri-o iproute runc skopeo-containers container-selinux && \
+    rpm -V socat iptables cri-o iproute runc skopeo-containers container-selinux && \
+    yum clean all && \
+    mkdir -p /exports/hostfs/etc/crio /exports/hostfs/opt/cni/bin/ /exports/hostfs/var/lib/containers/storage/ && \
+    cp /etc/crio/* /exports/hostfs/etc/crio && \
+    if test -e /usr/libexec/cni; then cp -Lr /usr/libexec/cni/* /exports/hostfs/opt/cni/bin/; fi
+
+COPY manifest.json tmpfiles.template config.json.template service.template /exports/
+
+COPY set_mounts.sh /
+COPY run.sh /usr/bin/
+
+CMD ["/usr/bin/run.sh"]
+
+LABEL \
+        com.redhat.component="cri-o-docker" \
+        io.k8s.description="CRI-O is an implementation of the Kubernetes CRI. It is a lightweight, OCI-compliant runtime that is native to kubernetes. CRI-O supports OCI container images and can pull from any container registry." \
+        maintainer="Jhon Honce <jhonce@redhat.com>" \
+        name="openshift3/cri-o" \
+        License="GPLv2+" \
+        io.k8s.display-name="CRI-O" \
+        summary="OCI-based implementation of Kubernetes Container Runtime Interface" \
+        release="0.13.0.0" \
+        version="v3.8.0" \
+        architecture="x86_64" \
+        usage="atomic install --system --system-package=no crio && systemctl start crio" \
+        vendor="Red Hat" \
+        io.openshift.tags="cri-o system rhel7" \
+        atomic.type="system"

contrib/system_containers/rhel/config.json.template

@@ -0,0 +1,422 @@
+{
+  "ociVersion": "1.0.0",
+  "platform": {
+    "arch": "amd64",
+    "os": "linux"
+  },
+  "process": {
+    "args": [
+      "/usr/bin/run.sh"
+    ],
+    "capabilities": {
+      "ambient": [
+        "CAP_CHOWN",
+        "CAP_FOWNER",
+        "CAP_FSETID",
+        "CAP_KILL",
+        "CAP_SETGID",
+        "CAP_SETUID",
+        "CAP_SETPCAP",
+        "CAP_LINUX_IMMUTABLE",
+        "CAP_NET_BIND_SERVICE",
+        "CAP_NET_BROADCAST",
+        "CAP_NET_ADMIN",
+        "CAP_NET_RAW",
+        "CAP_IPC_LOCK",
+        "CAP_IPC_OWNER",
+        "CAP_SYS_MODULE",
+        "CAP_SYS_RAWIO",
+        "CAP_SYS_CHROOT",
+        "CAP_SYS_PTRACE",
+        "CAP_SYS_PACCT",
+        "CAP_SYS_ADMIN",
+        "CAP_SYS_BOOT",
+        "CAP_SYS_NICE",
+        "CAP_SYS_RESOURCE",
+        "CAP_SYS_TIME",
+        "CAP_SYS_TTY_CONFIG",
+        "CAP_MKNOD",
+        "CAP_LEASE",
+        "CAP_AUDIT_WRITE",
+        "CAP_AUDIT_CONTROL",
+        "CAP_SETFCAP",
+        "CAP_DAC_OVERRIDE",
+        "CAP_MAC_OVERRIDE",
+        "CAP_DAC_READ_SEARCH",
+        "CAP_MAC_ADMIN",
+        "CAP_SYSLOG",
+        "CAP_WAKE_ALARM",
+        "CAP_BLOCK_SUSPEND"
+      ],
+      "bounding": [
+        "CAP_CHOWN",
+        "CAP_FOWNER",
+        "CAP_FSETID",
+        "CAP_KILL",
+        "CAP_SETGID",
+        "CAP_SETUID",
+        "CAP_SETPCAP",
+        "CAP_LINUX_IMMUTABLE",
+        "CAP_NET_BIND_SERVICE",
+        "CAP_NET_BROADCAST",
+        "CAP_NET_ADMIN",
+        "CAP_NET_RAW",
+        "CAP_IPC_LOCK",
+        "CAP_IPC_OWNER",
+        "CAP_SYS_MODULE",
+        "CAP_SYS_RAWIO",
+        "CAP_SYS_CHROOT",
+        "CAP_SYS_PTRACE",
+        "CAP_SYS_PACCT",
+        "CAP_SYS_ADMIN",
+        "CAP_SYS_BOOT",
+        "CAP_SYS_NICE",
+        "CAP_SYS_RESOURCE",
+        "CAP_SYS_TIME",
+        "CAP_SYS_TTY_CONFIG",
+        "CAP_MKNOD",
+        "CAP_LEASE",
+        "CAP_AUDIT_WRITE",
+        "CAP_AUDIT_CONTROL",
+        "CAP_SETFCAP",
+        "CAP_DAC_OVERRIDE",
+        "CAP_MAC_OVERRIDE",
+        "CAP_DAC_READ_SEARCH",
+        "CAP_MAC_ADMIN",
+        "CAP_SYSLOG",
+        "CAP_WAKE_ALARM",
+        "CAP_BLOCK_SUSPEND"
+      ],
+      "effective": [
+        "CAP_CHOWN",
+        "CAP_FOWNER",
+        "CAP_FSETID",
+        "CAP_KILL",
+        "CAP_SETGID",
+        "CAP_SETUID",
+        "CAP_SETPCAP",
+        "CAP_LINUX_IMMUTABLE",
+        "CAP_NET_BIND_SERVICE",
+        "CAP_NET_BROADCAST",
+        "CAP_NET_ADMIN",
+        "CAP_NET_RAW",
+        "CAP_IPC_LOCK",
+        "CAP_IPC_OWNER",
+        "CAP_SYS_MODULE",
+        "CAP_SYS_RAWIO",
+        "CAP_SYS_CHROOT",
+        "CAP_SYS_PTRACE",
+        "CAP_SYS_PACCT",
+        "CAP_SYS_ADMIN",
+        "CAP_SYS_BOOT",
+        "CAP_SYS_NICE",
+        "CAP_SYS_RESOURCE",
+        "CAP_SYS_TIME",
+        "CAP_SYS_TTY_CONFIG",
+        "CAP_MKNOD",
+        "CAP_LEASE",
+        "CAP_AUDIT_WRITE",
+        "CAP_AUDIT_CONTROL",
+        "CAP_SETFCAP",
+        "CAP_DAC_OVERRIDE",
+        "CAP_MAC_OVERRIDE",
+        "CAP_DAC_READ_SEARCH",
+        "CAP_MAC_ADMIN",
+        "CAP_SYSLOG",
+        "CAP_WAKE_ALARM",
+        "CAP_BLOCK_SUSPEND"
+      ],
+      "inheritable": [
+        "CAP_CHOWN",
+        "CAP_FOWNER",
+        "CAP_FSETID",
+        "CAP_KILL",
+        "CAP_SETGID",
+        "CAP_SETUID",
+        "CAP_SETPCAP",
+        "CAP_LINUX_IMMUTABLE",
+        "CAP_NET_BIND_SERVICE",
+        "CAP_NET_BROADCAST",
+        "CAP_NET_ADMIN",
+        "CAP_NET_RAW",
+        "CAP_IPC_LOCK",
+        "CAP_IPC_OWNER",
+        "CAP_SYS_MODULE",
+        "CAP_SYS_RAWIO",
+        "CAP_SYS_CHROOT",
+        "CAP_SYS_PTRACE",
+        "CAP_SYS_PACCT",
+        "CAP_SYS_ADMIN",
+        "CAP_SYS_BOOT",
+        "CAP_SYS_NICE",
+        "CAP_SYS_RESOURCE",
+        "CAP_SYS_TIME",
+        "CAP_SYS_TTY_CONFIG",
+        "CAP_MKNOD",
+        "CAP_LEASE",
+        "CAP_AUDIT_WRITE",
+        "CAP_AUDIT_CONTROL",
+        "CAP_SETFCAP",
+        "CAP_DAC_OVERRIDE",
+        "CAP_MAC_OVERRIDE",
+        "CAP_DAC_READ_SEARCH",
+        "CAP_MAC_ADMIN",
+        "CAP_SYSLOG",
+        "CAP_WAKE_ALARM",
+        "CAP_BLOCK_SUSPEND"
+      ],
+      "permitted": [
+        "CAP_CHOWN",
+        "CAP_FOWNER",
+        "CAP_FSETID",
+        "CAP_KILL",
+        "CAP_SETGID",
+        "CAP_SETUID",
+        "CAP_SETPCAP",
+        "CAP_LINUX_IMMUTABLE",
+        "CAP_NET_BIND_SERVICE",
+        "CAP_NET_BROADCAST",
+        "CAP_NET_ADMIN",
+        "CAP_NET_RAW",
+        "CAP_IPC_LOCK",
+        "CAP_IPC_OWNER",
+        "CAP_SYS_MODULE",
+        "CAP_SYS_RAWIO",
+        "CAP_SYS_CHROOT",
+        "CAP_SYS_PTRACE",
+        "CAP_SYS_PACCT",
+        "CAP_SYS_ADMIN",
+        "CAP_SYS_BOOT",
+        "CAP_SYS_NICE",
+        "CAP_SYS_RESOURCE",
+        "CAP_SYS_TIME",
+        "CAP_SYS_TTY_CONFIG",
+        "CAP_MKNOD",
+        "CAP_LEASE",
+        "CAP_AUDIT_WRITE",
+        "CAP_AUDIT_CONTROL",
+        "CAP_SETFCAP",
+        "CAP_DAC_OVERRIDE",
+        "CAP_MAC_OVERRIDE",
+        "CAP_DAC_READ_SEARCH",
+        "CAP_MAC_ADMIN",
+        "CAP_SYSLOG",
+        "CAP_WAKE_ALARM",
+        "CAP_BLOCK_SUSPEND"
+      ]
+    },
+    "selinuxLabel": "system_u:system_r:container_runtime_t:s0",
+    "cwd": "/",
+    "env": [
+      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin:/root/go/bin",
+      "TERM=xterm",
+      "LOG_LEVEL=$LOG_LEVEL",
+      "NAME=$NAME"
+    ],
+    "noNewPrivileges": false,
+    "terminal": false,
+    "user": {
+      "gid": 0,
+      "uid": 0
+    }
+  },
+  "root": {
+    "path": "rootfs",
+    "readonly": true
+  },
+  "hooks": {},
+  "linux": {
+    "namespaces": [{
+      "type": "mount"
+    }],
+    "resources": {
+      "devices": [{
+        "access": "rwm",
+        "allow": true
+      }]
+    },
+    "rootfsPropagation": "private"
+  },
+  "mounts": [{
+      "destination": "/tmp",
+      "options": [
+        "private",
+        "bind",
+        "rw",
+        "mode=755"
+      ],
+      "source": "/tmp",
+      "type": "bind"
+    },
+    {
+      "destination": "/etc",
+      "options": [
+        "rbind",
+        "rprivate",
+        "rw",
+        "mode=755"
+      ],
+      "source": "/etc",
+      "type": "bind"
+    },
+    {
+      "destination": "/lib/modules",
+      "options": [
+        "rbind",
+        "rprivate",
+        "rw",
+        "mode=755"
+      ],
+      "source": "/lib/modules",
+      "type": "bind"
+    },
+    {
+      "destination": "/root",
+      "options": [
+        "rbind",
+        "rprivate",
+        "rw",
+        "mode=755"
+      ],
+      "source": "/root",
+      "type": "bind"
+    },
+    {
+      "destination": "/home",
+      "options": [
+        "rbind",
+        "rprivate",
+        "rw",
+        "mode=755"
+      ],
+      "source": "/home",
+      "type": "bind"
+    },
+    {
+      "destination": "/mnt",
+      "options": [
+        "rbind",
+        "rw",
+        "rprivate",
+        "mode=755"
+      ],
+      "source": "/mnt",
+      "type": "bind"
+    },
+    {
+      "type": "bind",
+      "source": "${RUN_DIRECTORY}",
+      "destination": "/run",
+      "options": [
+        "rshared",
+        "rbind",
+        "rw",
+        "mode=755"
+      ]
+    },
+    {
+      "type": "bind",
+      "source": "${RUN_DIRECTORY}/systemd",
+      "destination": "/run/systemd",
+      "options": [
+        "rslave",
+        "bind",
+        "rw",
+        "mode=755"
+      ]
+    },
+    {
+      "destination": "/var/log",
+      "options": [
+        "rbind",
+        "rslave",
+        "rw"
+      ],
+      "source": "/var/log",
+      "type": "bind"
+    },
+    {
+      "destination": "/var/lib",
+      "options": [
+        "rbind",
+        "rprivate",
+        "rw"
+      ],
+      "source": "${STATE_DIRECTORY}",
+      "type": "bind"
+    },
+    {
+      "destination": "/var/lib/containers/storage",
+      "options": [
+        "rbind",
+        "rshared",
+        "rw"
+      ],
+      "source": "${VAR_LIB_CONTAINERS_STORAGE}",
+      "type": "bind"
+    },
+    {
+      "destination": "/var/lib/origin",
+      "options": [
+        "rshared",
+        "bind",
+        "rw"
+      ],
+      "source": "${VAR_LIB_ORIGIN}",
+      "type": "bind"
+    },
+    {
+      "destination": "/var/lib/kubelet",
+      "options": [
+        "rshared",
+        "bind",
+        "rw"
+      ],
+      "source": "${VAR_LIB_KUBE}",
+      "type": "bind"
+    },
+    {
+      "destination": "/opt/cni",
+      "options": [
+        "rbind",
+        "rprivate",
+        "ro",
+        "mode=755"
+      ],
+      "source": "${OPT_CNI}",
+      "type": "bind"
+    },
+    {
+      "destination": "/dev",
+      "options": [
+        "rprivate",
+        "rbind",
+        "rw",
+        "mode=755"
+      ],
+      "source": "/dev",
+      "type": "bind"
+    },
+    {
+      "destination": "/sys",
+      "options": [
+        "rprivate",
+        "rbind",
+        "rw",
+        "mode=755"
+      ],
+      "source": "/sys",
+      "type": "bind"
+    },
+    {
+      "destination": "/proc",
+      "options": [
+        "rbind",
+        "rw",
+        "mode=755"
+      ],
+      "source": "/proc",
+      "type": "proc"
+    }
+  ]
+}

contrib/system_containers/rhel/help.md

@@ -0,0 +1,37 @@
+% CRI-O (1) Container Image Pages
+% Jhon Honce
+% September 7, 2017
+
+# NAME
+cri-o - OCI-based implementation of Kubernetes Container Runtime Interface
+
+# DESCRIPTION
+CRI-O is an implementation of the Kubernetes CRI. It is a lightweight, OCI-compliant runtime that is native to kubernetes. CRI-O supports OCI container images and can pull from any container registry.
+
+You can find more information on the CRI-O project at <https://github.com/kubernetes-incubator/cri-o/>
+
+# USAGE
+Pull from local docker and install system container:
+
+```
+# atomic pull --storage ostree docker:openshift3/cri-o:latest
+# atomic install --system --system-package=no --name cri-o openshift3/cri-o
+```
+
+Start and enable as a systemd service:
+```
+# systemctl enable --now cri-o
+```
+
+Stopping the service
+```
+# systemctl stop cri-o
+```
+
+Removing the container
+```
+# atomic uninstall cri-o
+```
+
+# SEE ALSO
+man systemd(1)

contrib/system_containers/rhel/manifest.json

@@ -0,0 +1,10 @@
+{
+  "version": "1.0",
+  "defaultValues": {
+    "LOG_LEVEL": "info",
+    "OPT_CNI": "/opt/cni",
+    "VAR_LIB_CONTAINERS_STORAGE": "/var/lib/containers/storage",
+    "VAR_LIB_ORIGIN": "/var/lib/origin",
+    "VAR_LIB_KUBE": "/var/lib/kubelet"
+  }
+}

contrib/system_containers/rhel/run.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Ensure that new process maintain this SELinux label
+PID=$$
+LABEL=`tr -d '\000' < /proc/$PID/attr/current`
+printf %s $LABEL > /proc/self/attr/exec
+
+exec /usr/bin/crio --log-level=$LOG_LEVEL

contrib/system_containers/rhel/service.template

@@ -0,0 +1,20 @@
+[Unit]
+Description=crio daemon
+After=network.target
+
+[Service]
+Type=notify
+ExecStartPre=/bin/sh $DESTDIR/rootfs/set_mounts.sh
+ExecStart=$EXEC_START
+ExecStop=$EXEC_STOP
+Restart=on-failure
+WorkingDirectory=$DESTDIR
+RuntimeDirectory=${NAME}
+TasksMax=infinity
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+TimeoutStartSec=0
+
+[Install]
+WantedBy=multi-user.target

contrib/system_containers/rhel/set_mounts.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+findmnt /var/lib/containers/storage > /dev/null || mount --rbind --make-shared /var/lib/containers/storage /var/lib/containers/storage
+findmnt /var/lib/origin > /dev/null || mount --bind --make-shared /var/lib/origin /var/lib/origin
+findmnt /var/lib/kubelet > /dev/null || mount --bind --make-shared /var/lib/kubelet /var/lib/kubelet
+mount --make-shared /run
+findmnt /run/systemd > /dev/null || mount --bind --make-rslave /run/systemd /run/systemd

contrib/system_containers/rhel/tmpfiles.template

@@ -0,0 +1,5 @@
+d    ${RUN_DIRECTORY}/${NAME}               -        -           -       - -
+d    /etc/crio - - - - -
+Z    /etc/crio - - - - -
+d    ${STATE_DIRECTORY}/origin               -        -           -       - -
+d    ${STATE_DIRECTORY}/kubelet              -        -           -       - -

contrib/systemd/crio.service

@@ -12,7 +12,7 @@ ExecStart=/usr/local/bin/crio \
           $CRIO_STORAGE_OPTIONS \
           $CRIO_NETWORK_OPTIONS
 ExecReload=/bin/kill -s HUP $MAINPID
-TasksMax=8192
+TasksMax=infinity
 LimitNOFILE=1048576
 LimitNPROC=1048576
 LimitCORE=infinity

contrib/test/integration/build/cri-tools.yml

@@ -4,7 +4,7 @@
   git:
     repo: "https://github.com/kubernetes-incubator/cri-tools.git"
     dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-tools"
-    version: "16e6fe4d7199c5689db4630a9330e6a8a12cecd1"
+    version: "b42fc3f364dd48f649d55926c34492beeb9b2e99"
 
 - name: install crictl
   command: "/usr/bin/go install github.com/kubernetes-incubator/cri-tools/cmd/crictl"

contrib/test/integration/build/kubernetes.yml

@@ -4,7 +4,7 @@
   git:
     repo: "https://github.com/runcom/kubernetes.git"
     dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes"
-    version: "cri-o-node-e2e-patched"
+    version: "cri-o-node-e2e-patched-logs"
 
 - name: install etcd
   command: "hack/install-etcd.sh"

contrib/test/integration/e2e.yml

@@ -42,7 +42,7 @@
     e2e_shell_cmd: >
         /usr/bin/go run hack/e2e.go
             --test
-            -test_args="-host=https://{{ ansible_default_ipv4.address }}:6443
+            --test_args="-host=https://{{ ansible_default_ipv4.address }}:6443
                         --ginkgo.focus=\[Conformance\]
                         --report-dir={{ artifacts }}"
             &> {{ artifacts }}/e2e.log

contrib/test/integration/system.yml

@@ -108,4 +108,4 @@
 
 - name: Update the kernel cmdline to include quota support
   command: grubby --update-kernel=ALL --args="rootflags=pquota"
-  when: ansible_distribution in ['RedHat', 'CentOS']
+  when: ansible_distribution in ['RedHat', 'CentOS']

crictl.yaml

@@ -0,0 +1 @@
+runtime-endpoint: /var/run/crio.sock

docs/crio.conf.5.md

@@ -105,6 +105,9 @@ Example:
 **no_pivot**=*true*|*false*
   Instructs the runtime to not use pivot_root, but instead use MS_MOVE
 
+**default_mounts**=[]
+  List of mount points, in the form host:container, to be mounted in every container
+
 ## CRIO.IMAGE TABLE
 
 **default_transport**

hack/ostree_tag.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+if ! pkg-config ostree-1 2> /dev/null ; then
+	echo containers_image_ostree_stub
+fi

hack/selinux_tag.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+if pkg-config libselinux 2> /dev/null ; then
+	echo selinux
+fi

libkpod/config.go

@@ -145,6 +145,10 @@ type RuntimeConfig struct {
 	// HooksDirPath location of oci hooks config files
 	HooksDirPath string `toml:"hooks_dir_path"`
 
+	// DefaultMounts is the list of mounts to be mounted for each container
+	// The format of each mount is "host-path:container-path"
+	DefaultMounts []string `toml:"default_mounts"`
+
 	// Hooks List of hooks to run with container
 	Hooks map[string]HookParams
 

libkpod/container_server.go

@@ -19,6 +19,7 @@ import (
 	"github.com/kubernetes-incubator/cri-o/pkg/storage"
 	"github.com/opencontainers/runc/libcontainer"
 	rspec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/selinux/go-selinux"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -167,7 +168,8 @@ func New(config *Config) (*ContainerServer, error) {
 		state: &containerServerState{
 			containers:      oci.NewMemoryStore(),
 			infraContainers: oci.NewMemoryStore(),
-			sandboxes:       make(map[string]*sandbox.Sandbox),
+			sandboxes:       sandbox.NewMemoryStore(),
+			processLevels:   make(map[string]int),
 		},
 		config: config,
 	}, nil
@@ -388,6 +390,7 @@ func (c *ContainerServer) LoadSandbox(id string) error {
 	if err != nil {
 		return err
 	}
+	scontainer.SetSpec(&m)
 	scontainer.SetMountPoint(m.Annotations[annotations.MountPoint])
 
 	if m.Annotations[annotations.Volumes] != "" {
@@ -511,6 +514,7 @@ func (c *ContainerServer) LoadContainer(id string) error {
 	if err != nil {
 		return err
 	}
+	ctr.SetSpec(&m)
 	ctr.SetMountPoint(m.Annotations[annotations.MountPoint])
 
 	c.ContainerStateFromDisk(ctr)
@@ -608,68 +612,53 @@ func (c *ContainerServer) Shutdown() error {
 type containerServerState struct {
 	containers      oci.ContainerStorer
 	infraContainers oci.ContainerStorer
-	sandboxes       map[string]*sandbox.Sandbox
+	sandboxes       sandbox.Storer
+	// processLevels The number of sandboxes using the same SELinux MCS level. Need to release MCS Level, when count reaches 0
+	processLevels map[string]int
 }
 
 // AddContainer adds a container to the container state store
 func (c *ContainerServer) AddContainer(ctr *oci.Container) {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
-	sandbox := c.state.sandboxes[ctr.Sandbox()]
+	sandbox := c.state.sandboxes.Get(ctr.Sandbox())
 	sandbox.AddContainer(ctr)
 	c.state.containers.Add(ctr.ID(), ctr)
 }
 
 // AddInfraContainer adds a container to the container state store
 func (c *ContainerServer) AddInfraContainer(ctr *oci.Container) {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
 	c.state.infraContainers.Add(ctr.ID(), ctr)
 }
 
 // GetContainer returns a container by its ID
 func (c *ContainerServer) GetContainer(id string) *oci.Container {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
 	return c.state.containers.Get(id)
 }
 
 // GetInfraContainer returns a container by its ID
 func (c *ContainerServer) GetInfraContainer(id string) *oci.Container {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
 	return c.state.infraContainers.Get(id)
 }
 
 // HasContainer checks if a container exists in the state
 func (c *ContainerServer) HasContainer(id string) bool {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
-	ctr := c.state.containers.Get(id)
-	return ctr != nil
+	return c.state.containers.Get(id) != nil
 }
 
 // RemoveContainer removes a container from the container state store
 func (c *ContainerServer) RemoveContainer(ctr *oci.Container) {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
 	sbID := ctr.Sandbox()
-	sb := c.state.sandboxes[sbID]
+	sb := c.state.sandboxes.Get(sbID)
 	sb.RemoveContainer(ctr)
 	c.state.containers.Delete(ctr.ID())
 }
 
 // RemoveInfraContainer removes a container from the container state store
 func (c *ContainerServer) RemoveInfraContainer(ctr *oci.Container) {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
 	c.state.infraContainers.Delete(ctr.ID())
 }
 
 // listContainers returns a list of all containers stored by the server state
 func (c *ContainerServer) listContainers() []*oci.Container {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
 	return c.state.containers.List()
 }
 
@@ -693,54 +682,52 @@ func (c *ContainerServer) ListContainers(filters ...func(*oci.Container) bool) (
 
 // AddSandbox adds a sandbox to the sandbox state store
 func (c *ContainerServer) AddSandbox(sb *sandbox.Sandbox) {
+	c.state.sandboxes.Add(sb.ID(), sb)
+
 	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
-	c.state.sandboxes[sb.ID()] = sb
+	c.state.processLevels[selinux.NewContext(sb.ProcessLabel())["level"]]++
+	c.stateLock.Unlock()
 }
 
 // GetSandbox returns a sandbox by its ID
 func (c *ContainerServer) GetSandbox(id string) *sandbox.Sandbox {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
-	return c.state.sandboxes[id]
+	return c.state.sandboxes.Get(id)
 }
 
 // GetSandboxContainer returns a sandbox's infra container
 func (c *ContainerServer) GetSandboxContainer(id string) *oci.Container {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
-	sb, ok := c.state.sandboxes[id]
-	if !ok {
-		return nil
-	}
+	sb := c.state.sandboxes.Get(id)
 	return sb.InfraContainer()
 }
 
 // HasSandbox checks if a sandbox exists in the state
 func (c *ContainerServer) HasSandbox(id string) bool {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
-	_, ok := c.state.sandboxes[id]
-	return ok
+	return c.state.sandboxes.Get(id) != nil
 }
 
 // RemoveSandbox removes a sandbox from the state store
 func (c *ContainerServer) RemoveSandbox(id string) {
+	sb := c.state.sandboxes.Get(id)
+	processLabel := sb.ProcessLabel()
+	level := selinux.NewContext(processLabel)["level"]
+
 	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
-	delete(c.state.sandboxes, id)
+	pl, ok := c.state.processLevels[level]
+	if ok {
+		c.state.processLevels[level] = pl - 1
+		if c.state.processLevels[level] == 0 {
+			label.ReleaseLabel(processLabel)
+			delete(c.state.processLevels, level)
+		}
+	}
+	c.stateLock.Unlock()
+
+	c.state.sandboxes.Delete(id)
 }
 
 // ListSandboxes lists all sandboxes in the state store
 func (c *ContainerServer) ListSandboxes() []*sandbox.Sandbox {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
-	sbArray := make([]*sandbox.Sandbox, 0, len(c.state.sandboxes))
-	for _, sb := range c.state.sandboxes {
-		sbArray = append(sbArray, sb)
-	}
-
-	return sbArray
+	return c.state.sandboxes.List()
 }
 
 // LibcontainerStats gets the stats for the container with the given id from runc/libcontainer

libkpod/remove.go

@@ -6,10 +6,11 @@ import (
 
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/pkg/errors"
+	"golang.org/x/net/context"
 )
 
 // Remove removes a container
-func (c *ContainerServer) Remove(container string, force bool) (string, error) {
+func (c *ContainerServer) Remove(ctx context.Context, container string, force bool) (string, error) {
 	ctr, err := c.LookupContainer(container)
 	if err != nil {
 		return "", err
@@ -22,7 +23,7 @@ func (c *ContainerServer) Remove(container string, force bool) (string, error) {
 		return "", errors.Errorf("cannot remove paused container %s", ctrID)
 	case oci.ContainerStateCreated, oci.ContainerStateRunning:
 		if force {
-			_, err = c.ContainerStop(container, -1)
+			_, err = c.ContainerStop(ctx, container, 10)
 			if err != nil {
 				return "", errors.Wrapf(err, "unable to stop container %s", ctrID)
 			}

libkpod/sandbox/history.go

@@ -0,0 +1,31 @@
+package sandbox
+
+import "sort"
+
+// History is a convenience type for storing a list of sandboxes,
+// sorted by creation date in descendant order.
+type History []*Sandbox
+
+// Len returns the number of sandboxes in the history.
+func (history *History) Len() int {
+	return len(*history)
+}
+
+// Less compares two sandboxes and returns true if the second one
+// was created before the first one.
+func (history *History) Less(i, j int) bool {
+	sandboxes := *history
+	// FIXME: state access should be serialized
+	return sandboxes[j].created.Before(sandboxes[i].created)
+}
+
+// Swap switches sandboxes i and j positions in the history.
+func (history *History) Swap(i, j int) {
+	sandboxes := *history
+	sandboxes[i], sandboxes[j] = sandboxes[j], sandboxes[i]
+}
+
+// sort orders the history by creation date in descendant order.
+func (history *History) sort() {
+	sort.Sort(history)
+}

libkpod/sandbox/memory_store.go

@@ -0,0 +1,93 @@
+package sandbox
+
+import "sync"
+
+// memoryStore implements a Store in memory.
+type memoryStore struct {
+	s map[string]*Sandbox
+	sync.RWMutex
+}
+
+// NewMemoryStore initializes a new memory store.
+func NewMemoryStore() Storer {
+	return &memoryStore{
+		s: make(map[string]*Sandbox),
+	}
+}
+
+// Add appends a new sandbox to the memory store.
+// It overrides the id if it existed before.
+func (c *memoryStore) Add(id string, cont *Sandbox) {
+	c.Lock()
+	c.s[id] = cont
+	c.Unlock()
+}
+
+// Get returns a sandbox from the store by id.
+func (c *memoryStore) Get(id string) *Sandbox {
+	var res *Sandbox
+	c.RLock()
+	res = c.s[id]
+	c.RUnlock()
+	return res
+}
+
+// Delete removes a sandbox from the store by id.
+func (c *memoryStore) Delete(id string) {
+	c.Lock()
+	delete(c.s, id)
+	c.Unlock()
+}
+
+// List returns a sorted list of sandboxes from the store.
+// The sandboxes are ordered by creation date.
+func (c *memoryStore) List() []*Sandbox {
+	sandboxes := History(c.all())
+	sandboxes.sort()
+	return sandboxes
+}
+
+// Size returns the number of sandboxes in the store.
+func (c *memoryStore) Size() int {
+	c.RLock()
+	defer c.RUnlock()
+	return len(c.s)
+}
+
+// First returns the first sandbox found in the store by a given filter.
+func (c *memoryStore) First(filter StoreFilter) *Sandbox {
+	for _, cont := range c.all() {
+		if filter(cont) {
+			return cont
+		}
+	}
+	return nil
+}
+
+// ApplyAll calls the reducer function with every sandbox in the store.
+// This operation is asynchronous in the memory store.
+// NOTE: Modifications to the store MUST NOT be done by the StoreReducer.
+func (c *memoryStore) ApplyAll(apply StoreReducer) {
+	wg := new(sync.WaitGroup)
+	for _, cont := range c.all() {
+		wg.Add(1)
+		go func(sandbox *Sandbox) {
+			apply(sandbox)
+			wg.Done()
+		}(cont)
+	}
+
+	wg.Wait()
+}
+
+func (c *memoryStore) all() []*Sandbox {
+	c.RLock()
+	sandboxes := make([]*Sandbox, 0, len(c.s))
+	for _, cont := range c.s {
+		sandboxes = append(sandboxes, cont)
+	}
+	c.RUnlock()
+	return sandboxes
+}
+
+var _ Storer = &memoryStore{}

libkpod/sandbox/sandbox.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"time"
 
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/docker/docker/pkg/mount"
@@ -156,7 +157,8 @@ type Sandbox struct {
 	portMappings   []*hostport.PortMapping
 	stopped        bool
 	// ipv4 or ipv6 cache
-	ip string
+	ip      string
+	created time.Time
 }
 
 const (
@@ -201,6 +203,7 @@ func New(id, namespace, name, kubeName, logDir string, labels, annotations map[s
 	sb.resolvPath = resolvPath
 	sb.hostname = hostname
 	sb.portMappings = portMappings
+	sb.created = time.Now()
 
 	return sb, nil
 }

libkpod/sandbox/store.go

@@ -0,0 +1,27 @@
+package sandbox
+
+// StoreFilter defines a function to filter
+// sandboxes in the store.
+type StoreFilter func(*Sandbox) bool
+
+// StoreReducer defines a function to
+// manipulate sandboxes in the store
+type StoreReducer func(*Sandbox)
+
+// Storer defines an interface that any container store must implement.
+type Storer interface {
+	// Add appends a new sandbox to the store.
+	Add(string, *Sandbox)
+	// Get returns a sandbox from the store by the identifier it was stored with.
+	Get(string) *Sandbox
+	// Delete removes a sandbox from the store by the identifier it was stored with.
+	Delete(string)
+	// List returns a list of sandboxes from the store.
+	List() []*Sandbox
+	// Size returns the number of sandboxes in the store.
+	Size() int
+	// First returns the first sandbox found in the store by a given filter.
+	First(StoreFilter) *Sandbox
+	// ApplyAll calls the reducer function with every sandbox in the store.
+	ApplyAll(StoreReducer)
+}

libkpod/stop.go

@@ -3,10 +3,11 @@ package libkpod
 import (
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/pkg/errors"
+	"golang.org/x/net/context"
 )
 
 // ContainerStop stops a running container with a grace period (i.e., timeout).
-func (c *ContainerServer) ContainerStop(container string, timeout int64) (string, error) {
+func (c *ContainerServer) ContainerStop(ctx context.Context, container string, timeout int64) (string, error) {
 	ctr, err := c.LookupContainer(container)
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to find container %s", container)
@@ -20,7 +21,7 @@ func (c *ContainerServer) ContainerStop(container string, timeout int64) (string
 		return "", errors.Errorf("cannot stop paused container %s", ctrID)
 	default:
 		if cStatus.Status != oci.ContainerStateStopped {
-			if err := c.runtime.StopContainer(ctr, timeout); err != nil {
+			if err := c.runtime.StopContainer(ctx, ctr, timeout); err != nil {
 				return "", errors.Wrapf(err, "failed to stop container %s", ctrID)
 			}
 			if err := c.storageRuntimeServer.StopContainer(ctrID); err != nil {

oci/container.go

@@ -48,6 +48,7 @@ type Container struct {
 	imageRef   string
 	volumes    []ContainerVolume
 	mountPoint string
+	spec       *specs.Spec
 }
 
 // ContainerVolume is a bind mount for the container.
@@ -99,6 +100,16 @@ func NewContainer(id string, name string, bundlePath string, logPath string, net
 	return c, nil
 }
 
+// SetSpec loads the OCI spec in the container struct
+func (c *Container) SetSpec(s *specs.Spec) {
+	c.spec = s
+}
+
+// Spec returns a copy of the spec for the container
+func (c *Container) Spec() specs.Spec {
+	return *c.spec
+}
+
 // GetStopSignal returns the container's own stop signal configured from the
 // image configuration or the default one.
 func (c *Container) GetStopSignal() string {

oci/memory_store.go

@@ -25,8 +25,9 @@ func (c *memoryStore) Add(id string, cont *Container) {
 
 // Get returns a container from the store by id.
 func (c *memoryStore) Get(id string) *Container {
+	var res *Container
 	c.RLock()
-	res := c.s[id]
+	res = c.s[id]
 	c.RUnlock()
 	return res
 }

oci/oci.go

@@ -17,6 +17,7 @@ import (
 	"github.com/kubernetes-incubator/cri-o/utils"
 	rspec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
 	"golang.org/x/sys/unix"
 	kwait "k8s.io/apimachinery/pkg/util/wait"
 )
@@ -39,6 +40,10 @@ const (
 	SystemdCgroupsManager = "systemd"
 	// ContainerExitsDir is the location of container exit dirs
 	ContainerExitsDir = "/var/run/crio/exits"
+
+	// killContainerTimeout is the timeout that we wait for the container to
+	// be SIGKILLed.
+	killContainerTimeout = 2 * time.Minute
 )
 
 // New creates a new Runtime with options provided
@@ -224,11 +229,12 @@ func (r *Runtime) CreateContainer(c *Container, cgroupParent string) (err error)
 		if err != nil {
 			logrus.Warnf("Failed to add conmon to cgroupfs sandbox cgroup: %v", err)
 		} else {
-			// XXX: this defer does nothing as the cgroup can't be deleted cause
-			// it contains the conmon pid in tasks
-			// we need to remove this defer and delete the cgroup once conmon exits
-			// maybe need a conmon monitor?
-			defer control.Delete()
+			// Here we should defer a crio-connmon- cgroup hierarchy deletion, but it will
+			// always fail as conmon's pid is still there.
+			// Fortunately, kubelet takes care of deleting this for us, so the leak will
+			// only happens in corner case where one does a manual deletion of the container
+			// through e.g. runc. This should be handled by implementing a conmon monitoring
+			// routine that does the cgroup cleanup once conmon is terminated.
 			if err := control.Add(cgroups.Process{Pid: cmd.Process.Pid}); err != nil {
 				logrus.Warnf("Failed to add conmon to cgroupfs sandbox cgroup: %v", err)
 			}
@@ -406,15 +412,6 @@ func (r *Runtime) ExecSync(c *Container, command []string, timeout int64) (resp
 		os.RemoveAll(logPath)
 	}()
 
-	f, err := ioutil.TempFile("", "exec-process")
-	if err != nil {
-		return nil, ExecSyncError{
-			ExitCode: -1,
-			Err:      err,
-		}
-	}
-	defer os.RemoveAll(f.Name())
-
 	var args []string
 	args = append(args, "-c", c.id)
 	args = append(args, "-r", r.Path(c))
@@ -429,27 +426,16 @@ func (r *Runtime) ExecSync(c *Container, command []string, timeout int64) (resp
 	}
 	args = append(args, "-l", logPath)
 
-	pspec := rspec.Process{
-		Env:  r.conmonEnv,
-		Args: command,
-		Cwd:  "/",
-	}
-	processJSON, err := json.Marshal(pspec)
+	processFile, err := PrepareProcessExec(c, command, false)
 	if err != nil {
 		return nil, ExecSyncError{
 			ExitCode: -1,
 			Err:      err,
 		}
 	}
+	defer os.RemoveAll(processFile.Name())
 
-	if err := ioutil.WriteFile(f.Name(), processJSON, 0644); err != nil {
-		return nil, ExecSyncError{
-			ExitCode: -1,
-			Err:      err,
-		}
-	}
-
-	args = append(args, "--exec-process-spec", f.Name())
+	args = append(args, "--exec-process-spec", processFile.Name())
 
 	cmd := exec.Command(r.conmonPath, args...)
 
@@ -541,25 +527,7 @@ func (r *Runtime) ExecSync(c *Container, command []string, timeout int64) (resp
 	}, nil
 }
 
-// StopContainer stops a container. Timeout is given in seconds.
-func (r *Runtime) StopContainer(c *Container, timeout int64) error {
-	c.opLock.Lock()
-	defer c.opLock.Unlock()
-
-	// Check if the process is around before sending a signal
-	err := unix.Kill(c.state.Pid, 0)
-	if err == unix.ESRCH {
-		c.state.Finished = time.Now()
-		return nil
-	}
-
-	if err := utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, r.Path(c), "kill", c.id, c.GetStopSignal()); err != nil {
-		return fmt.Errorf("failed to stop container %s, %v", c.id, err)
-	}
-	if timeout == -1 {
-		// default 10 seconds delay
-		timeout = 10
-	}
+func waitContainerStop(ctx context.Context, c *Container, timeout time.Duration) error {
 	done := make(chan struct{})
 	// we could potentially re-use "done" channel to exit the loop on timeout
 	// but we use another channel "chControl" so that we won't never incur in the
@@ -587,7 +555,10 @@ func (r *Runtime) StopContainer(c *Container, timeout int64) error {
 	select {
 	case <-done:
 		return nil
-	case <-time.After(time.Duration(timeout) * time.Second):
+	case <-ctx.Done():
+		close(chControl)
+		return ctx.Err()
+	case <-time.After(timeout):
 		close(chControl)
 		err := unix.Kill(c.state.Pid, unix.SIGKILL)
 		if err != nil && err != unix.ESRCH {
@@ -596,10 +567,39 @@ func (r *Runtime) StopContainer(c *Container, timeout int64) error {
 	}
 
 	c.state.Finished = time.Now()
-
 	return nil
 }
 
+// StopContainer stops a container. Timeout is given in seconds.
+func (r *Runtime) StopContainer(ctx context.Context, c *Container, timeout int64) error {
+	c.opLock.Lock()
+	defer c.opLock.Unlock()
+
+	// Check if the process is around before sending a signal
+	err := unix.Kill(c.state.Pid, 0)
+	if err == unix.ESRCH {
+		c.state.Finished = time.Now()
+		return nil
+	}
+
+	if timeout > 0 {
+		if err := utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, r.Path(c), "kill", c.id, c.GetStopSignal()); err != nil {
+			return fmt.Errorf("failed to stop container %s, %v", c.id, err)
+		}
+		err = waitContainerStop(ctx, c, time.Duration(timeout)*time.Second)
+		if err == nil {
+			return nil
+		}
+		logrus.Warnf("Stop container %q timed out: %v", c.ID(), err)
+	}
+
+	if err := utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, r.Path(c), "kill", "--all", c.id, "KILL"); err != nil {
+		return fmt.Errorf("failed to stop container %s, %v", c.id, err)
+	}
+
+	return waitContainerStop(ctx, c, killContainerTimeout)
+}
+
 // DeleteContainer deletes a container.
 func (r *Runtime) DeleteContainer(c *Container) error {
 	c.opLock.Lock()
@@ -724,3 +724,27 @@ func (r *Runtime) UnpauseContainer(c *Container) error {
 	_, err := utils.ExecCmd(r.Path(c), "resume", c.id)
 	return err
 }
+
+// PrepareProcessExec returns the path of the process.json used in runc exec -p
+// caller is responsible to close the returned *os.File if needed.
+func PrepareProcessExec(c *Container, cmd []string, tty bool) (*os.File, error) {
+	f, err := ioutil.TempFile("", "exec-process-")
+	if err != nil {
+		return nil, err
+	}
+
+	pspec := c.Spec().Process
+	pspec.Args = cmd
+	if tty {
+		pspec.Terminal = true
+	}
+	processJSON, err := json.Marshal(pspec)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := ioutil.WriteFile(f.Name(), processJSON, 0644); err != nil {
+		return nil, err
+	}
+	return f, nil
+}

pause/Makefile

@@ -5,9 +5,9 @@ override LIBS +=
 override CFLAGS += -std=c99 -Os -Wall -Wextra -static
 
 pause: $(obj)
-	$(CC) -o $@ $^ $(CFLAGS) $(LIBS)
-	strip $@
+	$(CC) -o ../bin/$@ $^ $(CFLAGS) $(LIBS)
+	strip ../bin/$@
 
 .PHONY: clean
 clean:
-	rm -f $(obj) pause
+	rm -f $(obj) ../bin/pause

pkg/storage/image.go

@@ -2,10 +2,8 @@ package storage
 
 import (
 	"errors"
-	"fmt"
 	"net"
 	"path/filepath"
-	"regexp"
 	"strings"
 
 	"github.com/containers/image/copy"
@@ -16,7 +14,12 @@ import (
 	"github.com/containers/image/transports/alltransports"
 	"github.com/containers/image/types"
 	"github.com/containers/storage"
-	distreference "github.com/docker/distribution/reference"
+	digest "github.com/opencontainers/go-digest"
+)
+
+var (
+	// ErrCannotParseImageID is returned when we try to ResolveNames for an image ID
+	ErrCannotParseImageID = errors.New("cannot parse an image ID")
 )
 
 // ImageResult wraps a subset of information about an image: its ID, its names,
@@ -25,6 +28,10 @@ type ImageResult struct {
 	ID    string
 	Names []string
 	Size  *uint64
+	// TODO(runcom): this is an hack for https://github.com/kubernetes-incubator/cri-o/pull/1136
+	// drop this when we have proper image IDs (as in, image IDs should be just
+	// the config blog digest which is stable across same images).
+	ConfigDigest digest.Digest
 }
 
 type indexInfo struct {
@@ -47,6 +54,9 @@ type ImageServer interface {
 	ListImages(systemContext *types.SystemContext, filter string) ([]ImageResult, error)
 	// ImageStatus returns status of an image which matches the filter.
 	ImageStatus(systemContext *types.SystemContext, filter string) (*ImageResult, error)
+	// PrepareImage returns an Image where the config digest can be grabbed
+	// for further analysis. Call Close() on the resulting image.
+	PrepareImage(systemContext *types.SystemContext, imageName string, options *copy.Options) (types.Image, error)
 	// PullImage imports an image from the specified location.
 	PullImage(systemContext *types.SystemContext, imageName string, options *copy.Options) (types.ImageReference, error)
 	// RemoveImage deletes the specified image.
@@ -146,14 +156,16 @@ func (svc *imageService) ImageStatus(systemContext *types.SystemContext, nameOrI
 	if err != nil {
 		return nil, err
 	}
+	defer img.Close()
 	size := imageSize(img)
-	img.Close()
 
-	return &ImageResult{
-		ID:    image.ID,
-		Names: image.Names,
-		Size:  size,
-	}, nil
+	res := &ImageResult{
+		ID:           image.ID,
+		Names:        image.Names,
+		Size:         size,
+		ConfigDigest: img.ConfigInfo().Digest,
+	}
+	return res, nil
 }
 
 func imageSize(img types.Image) *uint64 {
@@ -165,7 +177,7 @@ func imageSize(img types.Image) *uint64 {
 }
 
 func (svc *imageService) CanPull(imageName string, options *copy.Options) (bool, error) {
-	srcRef, err := svc.prepareImage(imageName, options)
+	srcRef, err := svc.prepareReference(imageName, options)
 	if err != nil {
 		return false, err
 	}
@@ -182,9 +194,9 @@ func (svc *imageService) CanPull(imageName string, options *copy.Options) (bool,
 	return true, nil
 }
 
-// prepareImage creates an image reference from an image string and set options
+// prepareReference creates an image reference from an image string and set options
 // for the source context
-func (svc *imageService) prepareImage(imageName string, options *copy.Options) (types.ImageReference, error) {
+func (svc *imageService) prepareReference(imageName string, options *copy.Options) (types.ImageReference, error) {
 	if imageName == "" {
 		return nil, storage.ErrNotAnImage
 	}
@@ -212,6 +224,18 @@ func (svc *imageService) prepareImage(imageName string, options *copy.Options) (
 	return srcRef, nil
 }
 
+func (svc *imageService) PrepareImage(systemContext *types.SystemContext, imageName string, options *copy.Options) (types.Image, error) {
+	if options == nil {
+		options = &copy.Options{}
+	}
+
+	srcRef, err := svc.prepareReference(imageName, options)
+	if err != nil {
+		return nil, err
+	}
+	return srcRef.NewImage(systemContext)
+}
+
 func (svc *imageService) PullImage(systemContext *types.SystemContext, imageName string, options *copy.Options) (types.ImageReference, error) {
 	policy, err := signature.DefaultPolicy(systemContext)
 	if err != nil {
@@ -225,7 +249,7 @@ func (svc *imageService) PullImage(systemContext *types.SystemContext, imageName
 		options = &copy.Options{}
 	}
 
-	srcRef, err := svc.prepareImage(imageName, options)
+	srcRef, err := svc.prepareReference(imageName, options)
 	if err != nil {
 		return nil, err
 	}
@@ -307,113 +331,27 @@ func (svc *imageService) isSecureIndex(indexName string) bool {
 	return true
 }
 
-func isValidHostname(hostname string) bool {
-	return hostname != "" && !strings.Contains(hostname, "/") &&
-		(strings.Contains(hostname, ".") ||
-			strings.Contains(hostname, ":") || hostname == "localhost")
-}
-
-func isReferenceFullyQualified(reposName reference.Named) bool {
-	indexName, _, _ := splitReposName(reposName)
-	return indexName != ""
-}
-
-const (
-	// defaultHostname is the default built-in hostname
-	defaultHostname = "docker.io"
-	// legacyDefaultHostname is automatically converted to DefaultHostname
-	legacyDefaultHostname = "index.docker.io"
-	// defaultRepoPrefix is the prefix used for default repositories in default host
-	defaultRepoPrefix = "library/"
-)
-
-// splitReposName breaks a reposName into an index name and remote name
-func splitReposName(reposName reference.Named) (indexName string, remoteName reference.Named, err error) {
-	var remoteNameStr string
-	indexName, remoteNameStr = distreference.SplitHostname(reposName)
-	if !isValidHostname(indexName) {
-		// This is a Docker Index repos (ex: samalba/hipache or ubuntu)
-		// 'docker.io'
-		indexName = ""
-		remoteName = reposName
-	} else {
-		remoteName, err = withName(remoteNameStr)
-	}
-	return
-}
-
-func validateName(name string) error {
-	if err := validateID(strings.TrimPrefix(name, defaultHostname+"/")); err == nil {
-		return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name)
-	}
-	return nil
-}
-
-var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)
-
-// validateID checks whether an ID string is a valid image ID.
-func validateID(id string) error {
-	if ok := validHex.MatchString(id); !ok {
-		return fmt.Errorf("image ID %q is invalid", id)
-	}
-	return nil
-}
-
-// withName returns a named object representing the given string. If the input
-// is invalid ErrReferenceInvalidFormat will be returned.
-func withName(name string) (reference.Named, error) {
-	name, err := normalize(name)
-	if err != nil {
-		return nil, err
-	}
-	if err := validateName(name); err != nil {
-		return nil, err
-	}
-	r, err := distreference.WithName(name)
-	return r, err
-}
-
-// splitHostname splits a repository name to hostname and remotename string.
-// If no valid hostname is found, empty string will be returned as a resulting
-// hostname. Repository name needs to be already validated before.
-func splitHostname(name string) (hostname, remoteName string) {
+func splitDockerDomain(name string) (domain, remainder string) {
 	i := strings.IndexRune(name, '/')
 	if i == -1 || (!strings.ContainsAny(name[:i], ".:") && name[:i] != "localhost") {
-		hostname, remoteName = "", name
+		domain, remainder = "", name
 	} else {
-		hostname, remoteName = name[:i], name[i+1:]
-	}
-	if hostname == legacyDefaultHostname {
-		hostname = defaultHostname
-	}
-	if hostname == defaultHostname && !strings.ContainsRune(remoteName, '/') {
-		remoteName = defaultRepoPrefix + remoteName
+		domain, remainder = name[:i], name[i+1:]
 	}
 	return
 }
 
-// normalize returns a repository name in its normalized form, meaning it
-// will contain library/ prefix for official images.
-func normalize(name string) (string, error) {
-	host, remoteName := splitHostname(name)
-	if strings.ToLower(remoteName) != remoteName {
-		return "", errors.New("invalid reference format: repository name must be lowercase")
-	}
-	if host == defaultHostname {
-		if strings.HasPrefix(remoteName, defaultRepoPrefix) {
-			remoteName = strings.TrimPrefix(remoteName, defaultRepoPrefix)
-		}
-		return host + "/" + remoteName, nil
-	}
-	return name, nil
-}
-
 func (svc *imageService) ResolveNames(imageName string) ([]string, error) {
-	r, err := reference.ParseNormalizedNamed(imageName)
+	// This to prevent any image ID to go through this routine
+	_, err := reference.ParseNormalizedNamed(imageName)
 	if err != nil {
+		if strings.Contains(err.Error(), "cannot specify 64-byte hexadecimal strings") {
+			return nil, ErrCannotParseImageID
+		}
 		return nil, err
 	}
-	if isReferenceFullyQualified(r) {
+	domain, remainder := splitDockerDomain(imageName)
+	if domain != "" {
 		// this means the image is already fully qualified
 		return []string{imageName}, nil
 	}
@@ -425,10 +363,13 @@ func (svc *imageService) ResolveNames(imageName string) ([]string, error) {
 	// this means we got an image in the form of "busybox"
 	// we need to use additional registries...
 	// normalize the unqualified image to be domain/repo/image...
-	_, rest := splitDomain(r.Name())
 	images := []string{}
 	for _, r := range svc.registries {
-		images = append(images, filepath.Join(r, rest))
+		rem := remainder
+		if r == "docker.io" && !strings.ContainsRune(remainder, '/') {
+			rem = "library/" + rem
+		}
+		images = append(images, filepath.Join(r, rem))
 	}
 	return images, nil
 }

pkg/storage/image_regexp.go

@@ -1,125 +0,0 @@
-package storage
-
-// This is a fork of docker/distribution code to be used when manipulating image
-// references.
-// DO NOT EDIT THIS FILE.
-
-import "regexp"
-
-var (
-	// alphaNumericRegexp defines the alpha numeric atom, typically a
-	// component of names. This only allows lower case characters and digits.
-	alphaNumericRegexp = match(`[a-z0-9]+`)
-
-	// separatorRegexp defines the separators allowed to be embedded in name
-	// components. This allow one period, one or two underscore and multiple
-	// dashes.
-	separatorRegexp = match(`(?:[._]|__|[-]*)`)
-
-	// nameComponentRegexp restricts registry path component names to start
-	// with at least one letter or number, with following parts able to be
-	// separated by one period, one or two underscore and multiple dashes.
-	nameComponentRegexp = expression(
-		alphaNumericRegexp,
-		optional(repeated(separatorRegexp, alphaNumericRegexp)))
-
-	// domainComponentRegexp restricts the registry domain component of a
-	// repository name to start with a component as defined by domainRegexp
-	// and followed by an optional port.
-	domainComponentRegexp = match(`(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])`)
-
-	// domainRegexp defines the structure of potential domain components
-	// that may be part of image names. This is purposely a subset of what is
-	// allowed by DNS to ensure backwards compatibility with Docker image
-	// names.
-	domainRegexp = expression(
-		domainComponentRegexp,
-		optional(repeated(literal(`.`), domainComponentRegexp)),
-		optional(literal(`:`), match(`[0-9]+`)))
-
-	// NameRegexp is the format for the name component of references. The
-	// regexp has capturing groups for the domain and name part omitting
-	// the separating forward slash from either.
-	NameRegexp = expression(
-		optional(domainRegexp, literal(`/`)),
-		nameComponentRegexp,
-		optional(repeated(literal(`/`), nameComponentRegexp)))
-
-	// anchoredNameRegexp is used to parse a name value, capturing the
-	// domain and trailing components.
-	anchoredNameRegexp = anchored(
-		optional(capture(domainRegexp), literal(`/`)),
-		capture(nameComponentRegexp,
-			optional(repeated(literal(`/`), nameComponentRegexp))))
-
-	// IdentifierRegexp is the format for string identifier used as a
-	// content addressable identifier using sha256. These identifiers
-	// are like digests without the algorithm, since sha256 is used.
-	IdentifierRegexp = match(`([a-f0-9]{64})`)
-
-	// ShortIdentifierRegexp is the format used to represent a prefix
-	// of an identifier. A prefix may be used to match a sha256 identifier
-	// within a list of trusted identifiers.
-	ShortIdentifierRegexp = match(`([a-f0-9]{6,64})`)
-)
-
-// match compiles the string to a regular expression.
-var match = regexp.MustCompile
-
-// literal compiles s into a literal regular expression, escaping any regexp
-// reserved characters.
-func literal(s string) *regexp.Regexp {
-	re := match(regexp.QuoteMeta(s))
-
-	if _, complete := re.LiteralPrefix(); !complete {
-		panic("must be a literal")
-	}
-
-	return re
-}
-
-func splitDomain(name string) (string, string) {
-	match := anchoredNameRegexp.FindStringSubmatch(name)
-	if len(match) != 3 {
-		return "", name
-	}
-	return match[1], match[2]
-}
-
-// expression defines a full expression, where each regular expression must
-// follow the previous.
-func expression(res ...*regexp.Regexp) *regexp.Regexp {
-	var s string
-	for _, re := range res {
-		s += re.String()
-	}
-
-	return match(s)
-}
-
-// optional wraps the expression in a non-capturing group and makes the
-// production optional.
-func optional(res ...*regexp.Regexp) *regexp.Regexp {
-	return match(group(expression(res...)).String() + `?`)
-}
-
-// repeated wraps the regexp in a non-capturing group to get one or more
-// matches.
-func repeated(res ...*regexp.Regexp) *regexp.Regexp {
-	return match(group(expression(res...)).String() + `+`)
-}
-
-// group wraps the regexp in a non-capturing group.
-func group(res ...*regexp.Regexp) *regexp.Regexp {
-	return match(`(?:` + expression(res...).String() + `)`)
-}
-
-// capture wraps the expression in a capturing group.
-func capture(res ...*regexp.Regexp) *regexp.Regexp {
-	return match(`(` + expression(res...).String() + `)`)
-}
-
-// anchored anchors the regular expression by adding start and end delimiters.
-func anchored(res ...*regexp.Regexp) *regexp.Regexp {
-	return match(`^` + expression(res...).String() + `$`)
-}

pkg/storage/image_test.go

@@ -0,0 +1,84 @@
+package storage
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestResolveNames(t *testing.T) {
+	cases := []struct {
+		name                 string
+		additionalRegistries []string
+		imageName            string
+		expected             []string
+		err                  bool
+		errContains          string
+	}{
+		{
+			name:                 "test unqualified images get correctly qualified in order and correct tag",
+			additionalRegistries: []string{"testregistry.com", "registry.access.redhat.com", "docker.io"},
+			imageName:            "openshift3/ose-deployer:sometag",
+			expected:             []string{"testregistry.com/openshift3/ose-deployer:sometag", "registry.access.redhat.com/openshift3/ose-deployer:sometag", "docker.io/openshift3/ose-deployer:sometag"},
+			err:                  false,
+		},
+		{
+			name:                 "test unqualified images get correctly qualified in order and correct digest",
+			additionalRegistries: []string{"testregistry.com", "registry.access.redhat.com", "docker.io"},
+			imageName:            "openshift3/ose-deployer@sha256:dc5f67a48da730d67bf4bfb8824ea8a51be26711de090d6d5a1ffff2723168a3",
+			expected:             []string{"testregistry.com/openshift3/ose-deployer@sha256:dc5f67a48da730d67bf4bfb8824ea8a51be26711de090d6d5a1ffff2723168a3", "registry.access.redhat.com/openshift3/ose-deployer@sha256:dc5f67a48da730d67bf4bfb8824ea8a51be26711de090d6d5a1ffff2723168a3", "docker.io/openshift3/ose-deployer@sha256:dc5f67a48da730d67bf4bfb8824ea8a51be26711de090d6d5a1ffff2723168a3"},
+			err:                  false,
+		},
+		{
+			name:                 "test unqualified images get correctly qualified in order",
+			additionalRegistries: []string{"testregistry.com", "registry.access.redhat.com", "docker.io"},
+			imageName:            "openshift3/ose-deployer:latest",
+			expected:             []string{"testregistry.com/openshift3/ose-deployer:latest", "registry.access.redhat.com/openshift3/ose-deployer:latest", "docker.io/openshift3/ose-deployer:latest"},
+			err:                  false,
+		},
+		{
+			name:                 "test unqualified images get correctly qualified from official library",
+			additionalRegistries: []string{"testregistry.com", "registry.access.redhat.com", "docker.io"},
+			imageName:            "nginx:latest",
+			expected:             []string{"testregistry.com/nginx:latest", "registry.access.redhat.com/nginx:latest", "docker.io/library/nginx:latest"},
+			err:                  false,
+		},
+		{
+			name:                 "test qualified images returns just qualified",
+			additionalRegistries: []string{"testregistry.com", "registry.access.redhat.com", "docker.io"},
+			imageName:            "mypersonalregistry.com/nginx:latest",
+			expected:             []string{"mypersonalregistry.com/nginx:latest"},
+			err:                  false,
+		},
+		{
+			name:      "test we don't have names w/o registries",
+			imageName: "openshift3/ose-deployer:latest",
+			err:       true,
+		},
+		{
+			name:        "test we cannot resolve names from an image ID",
+			imageName:   "6ad733544a6317992a6fac4eb19fe1df577d4dec7529efec28a5bd0edad0fd30",
+			err:         true,
+			errContains: "cannot parse an image ID",
+		},
+	}
+	for _, c := range cases {
+		svc := &imageService{
+			registries: c.additionalRegistries,
+		}
+		names, err := svc.ResolveNames(c.imageName)
+		if !c.err {
+			require.NoError(t, err, c.name)
+			if !reflect.DeepEqual(names, c.expected) {
+				t.Fatalf("Exepected: %v, Got: %v: %q", c.expected, names, c.name)
+			}
+		} else {
+			require.Error(t, err, c.name)
+			if c.errContains != "" {
+				assert.Contains(t, err.Error(), c.errContains)
+			}
+		}
+	}
+}

server/container_attach.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/kubernetes-incubator/cri-o/utils"
@@ -25,10 +26,15 @@ const (
 )
 
 // Attach prepares a streaming endpoint to attach to a running container.
-func (s *Server) Attach(ctx context.Context, req *pb.AttachRequest) (*pb.AttachResponse, error) {
+func (s *Server) Attach(ctx context.Context, req *pb.AttachRequest) (resp *pb.AttachResponse, err error) {
+	const operation = "attach"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
 	logrus.Debugf("AttachRequest %+v", req)
 
-	resp, err := s.GetAttach(req)
+	resp, err = s.GetAttach(req)
 	if err != nil {
 		return nil, fmt.Errorf("unable to prepare attach endpoint")
 	}

server/container_create.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -46,29 +47,54 @@ const (
 	defaultSystemdParent  = "system.slice"
 )
 
-func addOCIBindMounts(mountLabel string, containerConfig *pb.ContainerConfig, specgen *generate.Generator) ([]oci.ContainerVolume, error) {
+type orderedMounts []rspec.Mount
+
+// Len returns the number of mounts. Used in sorting.
+func (m orderedMounts) Len() int {
+	return len(m)
+}
+
+// Less returns true if the number of parts (a/b/c would be 3 parts) in the
+// mount indexed by parameter 1 is less than that of the mount indexed by
+// parameter 2. Used in sorting.
+func (m orderedMounts) Less(i, j int) bool {
+	return m.parts(i) < m.parts(j)
+}
+
+// Swap swaps two items in an array of mounts. Used in sorting
+func (m orderedMounts) Swap(i, j int) {
+	m[i], m[j] = m[j], m[i]
+}
+
+// parts returns the number of parts in the destination of a mount. Used in sorting.
+func (m orderedMounts) parts(i int) int {
+	return strings.Count(filepath.Clean(m[i].Destination), string(os.PathSeparator))
+}
+
+func addOCIBindMounts(mountLabel string, containerConfig *pb.ContainerConfig, specgen *generate.Generator) ([]oci.ContainerVolume, []rspec.Mount, error) {
 	volumes := []oci.ContainerVolume{}
+	ociMounts := []rspec.Mount{}
 	mounts := containerConfig.GetMounts()
 	for _, mount := range mounts {
 		dest := mount.ContainerPath
 		if dest == "" {
-			return nil, fmt.Errorf("Mount.ContainerPath is empty")
+			return nil, nil, fmt.Errorf("Mount.ContainerPath is empty")
 		}
 
 		src := mount.HostPath
 		if src == "" {
-			return nil, fmt.Errorf("Mount.HostPath is empty")
+			return nil, nil, fmt.Errorf("Mount.HostPath is empty")
 		}
 
 		if _, err := os.Stat(src); err != nil && os.IsNotExist(err) {
 			if err1 := os.MkdirAll(src, 0644); err1 != nil {
-				return nil, fmt.Errorf("Failed to mkdir %s: %s", src, err)
+				return nil, nil, fmt.Errorf("Failed to mkdir %s: %s", src, err)
 			}
 		}
 
 		src, err := resolveSymbolicLink(src)
 		if err != nil {
-			return nil, fmt.Errorf("failed to resolve symlink %q: %v", src, err)
+			return nil, nil, fmt.Errorf("failed to resolve symlink %q: %v", src, err)
 		}
 
 		options := []string{"rw"}
@@ -80,7 +106,7 @@ func addOCIBindMounts(mountLabel string, containerConfig *pb.ContainerConfig, sp
 		if mount.SelinuxRelabel {
 			// Need a way in kubernetes to determine if the volume is shared or private
 			if err := label.Relabel(src, mountLabel, true); err != nil && err != unix.ENOTSUP {
-				return nil, fmt.Errorf("relabel failed %s: %v", src, err)
+				return nil, nil, fmt.Errorf("relabel failed %s: %v", src, err)
 			}
 		}
 
@@ -90,45 +116,55 @@ func addOCIBindMounts(mountLabel string, containerConfig *pb.ContainerConfig, sp
 			Readonly:      mount.Readonly,
 		})
 
-		specgen.AddBindMount(src, dest, options)
+		ociMounts = append(ociMounts, rspec.Mount{
+			Source:      src,
+			Destination: dest,
+			Options:     options,
+		})
 	}
 
-	return volumes, nil
+	return volumes, ociMounts, nil
 }
 
-func addImageVolumes(rootfs string, s *Server, containerInfo *storage.ContainerInfo, specgen *generate.Generator, mountLabel string) error {
+func addImageVolumes(rootfs string, s *Server, containerInfo *storage.ContainerInfo, specgen *generate.Generator, mountLabel string) ([]rspec.Mount, error) {
+	mounts := []rspec.Mount{}
 	for dest := range containerInfo.Config.Config.Volumes {
 		fp, err := symlink.FollowSymlinkInScope(filepath.Join(rootfs, dest), rootfs)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		switch s.config.ImageVolumes {
 		case libkpod.ImageVolumesMkdir:
 			if err1 := os.MkdirAll(fp, 0644); err1 != nil {
-				return err1
+				return nil, err1
 			}
 		case libkpod.ImageVolumesBind:
 			volumeDirName := stringid.GenerateNonCryptoID()
 			src := filepath.Join(containerInfo.RunDir, "mounts", volumeDirName)
 			if err1 := os.MkdirAll(src, 0644); err1 != nil {
-				return err1
+				return nil, err1
 			}
 			// Label the source with the sandbox selinux mount label
 			if mountLabel != "" {
 				if err1 := label.Relabel(src, mountLabel, true); err1 != nil && err1 != unix.ENOTSUP {
-					return fmt.Errorf("relabel failed %s: %v", src, err1)
+					return nil, fmt.Errorf("relabel failed %s: %v", src, err1)
 				}
 			}
 
 			logrus.Debugf("Adding bind mounted volume: %s to %s", src, dest)
-			specgen.AddBindMount(src, dest, []string{"rw"})
+			mounts = append(mounts, rspec.Mount{
+				Source:      src,
+				Destination: dest,
+				Options:     []string{"rw"},
+			})
+
 		case libkpod.ImageVolumesIgnore:
 			logrus.Debugf("Ignoring volume %v", dest)
 		default:
 			logrus.Fatalf("Unrecognized image volumes setting")
 		}
 	}
-	return nil
+	return mounts, nil
 }
 
 // resolveSymbolicLink resolves a possbile symlink path. If the path is a symlink, returns resolved
@@ -384,8 +420,23 @@ func ensureSaneLogPath(logPath string) error {
 	return nil
 }
 
+// addSecretsBindMounts mounts user defined secrets to the container
+func addSecretsBindMounts(mountLabel, ctrRunDir string, defaultMounts []string, specgen generate.Generator) ([]rspec.Mount, error) {
+	containerMounts := specgen.Spec().Mounts
+	mounts, err := secretMounts(defaultMounts, mountLabel, ctrRunDir, containerMounts)
+	if err != nil {
+		return nil, err
+	}
+	return mounts, nil
+}
+
 // CreateContainer creates a new container in specified PodSandbox
 func (s *Server) CreateContainer(ctx context.Context, req *pb.CreateContainerRequest) (res *pb.CreateContainerResponse, err error) {
+	const operation = "create_container"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
 	logrus.Debugf("CreateContainerRequest %+v", req)
 
 	s.updateLock.RLock()
@@ -549,7 +600,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 		}
 	}
 
-	containerVolumes, err := addOCIBindMounts(mountLabel, containerConfig, &specgen)
+	containerVolumes, ociMounts, err := addOCIBindMounts(mountLabel, containerConfig, &specgen)
 	if err != nil {
 		return nil, err
 	}
@@ -569,6 +620,10 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 
 	labels := containerConfig.GetLabels()
 
+	if err := validateLabels(labels); err != nil {
+		return nil, err
+	}
+
 	metadata := containerConfig.GetMetadata()
 
 	kubeAnnotations := containerConfig.GetAnnotations()
@@ -733,6 +788,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 				"/proc/timer_list",
 				"/proc/timer_stats",
 				"/proc/sched_debug",
+				"/proc/scsi",
 				"/sys/firmware",
 			} {
 				specgen.AddLinuxMaskedPaths(mp)
@@ -756,10 +812,20 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 	logrus.Debugf("pod container state %+v", podInfraState)
 
 	ipcNsPath := fmt.Sprintf("/proc/%d/ns/ipc", podInfraState.Pid)
-	if err := specgen.AddOrReplaceLinuxNamespace("ipc", ipcNsPath); err != nil {
+	if err := specgen.AddOrReplaceLinuxNamespace(string(rspec.IPCNamespace), ipcNsPath); err != nil {
 		return nil, err
 	}
 
+	utsNsPath := fmt.Sprintf("/proc/%d/ns/uts", podInfraState.Pid)
+	if err := specgen.AddOrReplaceLinuxNamespace(string(rspec.UTSNamespace), utsNsPath); err != nil {
+		return nil, err
+	}
+
+	// Do not share pid ns for now
+	if containerConfig.GetLinux().GetSecurityContext().GetNamespaceOptions().GetHostPid() {
+		specgen.RemoveLinuxNamespace(string(rspec.PIDNamespace))
+	}
+
 	netNsPath := sb.NetNsPath()
 	if netNsPath == "" {
 		// The sandbox does not have a permanent namespace,
@@ -767,7 +833,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 		netNsPath = fmt.Sprintf("/proc/%d/ns/net", podInfraState.Pid)
 	}
 
-	if err := specgen.AddOrReplaceLinuxNamespace("network", netNsPath); err != nil {
+	if err := specgen.AddOrReplaceLinuxNamespace(string(rspec.NetworkNamespace), netNsPath); err != nil {
 		return nil, err
 	}
 
@@ -782,8 +848,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 	}
 	images, err := s.StorageImageServer().ResolveNames(image)
 	if err != nil {
-		// This means we got an image ID
-		if strings.Contains(err.Error(), "cannot specify 64-byte hexadecimal strings") {
+		if err == storage.ErrCannotParseImageID {
 			images = append(images, image)
 		} else {
 			return nil, err
@@ -856,7 +921,9 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 		specgen.AddBindMount("/etc/hosts", "/etc/hosts", options)
 	}
 
+	// Set hostname and add env for hostname
 	specgen.SetHostname(sb.Hostname())
+	specgen.AddProcessEnv("HOSTNAME", sb.Hostname())
 
 	specgen.AddAnnotation(annotations.Name, containerName)
 	specgen.AddAnnotation(annotations.ContainerID, containerID)
@@ -928,7 +995,8 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 	}
 
 	// Add image volumes
-	if err := addImageVolumes(mountPoint, s, &containerInfo, &specgen, mountLabel); err != nil {
+	volumeMounts, err := addImageVolumes(mountPoint, s, &containerInfo, &specgen, mountLabel)
+	if err != nil {
 		return nil, err
 	}
 
@@ -938,30 +1006,46 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 	}
 	specgen.SetProcessArgs(processArgs)
 
-	// Add environment variables from CRI and image config
-	envs := containerConfig.GetEnvs()
-	if envs != nil {
-		for _, item := range envs {
-			key := item.Key
-			value := item.Value
-			if key == "" {
+	envs := []string{}
+	if containerConfig.GetEnvs() == nil && containerImageConfig != nil {
+		envs = containerImageConfig.Config.Env
+	} else {
+		for _, item := range containerConfig.GetEnvs() {
+			if item.GetKey() == "" {
 				continue
 			}
-			specgen.AddProcessEnv(key, value)
+			envs = append(envs, item.GetKey()+"="+item.GetValue())
+		}
+		if containerImageConfig != nil {
+			for _, imageEnv := range containerImageConfig.Config.Env {
+				var found bool
+				parts := strings.SplitN(imageEnv, "=", 2)
+				if len(parts) != 2 {
+					continue
+				}
+				imageEnvKey := parts[0]
+				if imageEnvKey == "" {
+					continue
+				}
+				for _, kubeEnv := range envs {
+					kubeEnvKey := strings.SplitN(kubeEnv, "=", 2)[0]
+					if kubeEnvKey == "" {
+						continue
+					}
+					if imageEnvKey == kubeEnvKey {
+						found = true
+						break
+					}
+				}
+				if !found {
+					envs = append(envs, imageEnv)
+				}
+			}
 		}
 	}
-	if containerImageConfig != nil {
-		for _, item := range containerImageConfig.Config.Env {
-			parts := strings.SplitN(item, "=", 2)
-			if len(parts) != 2 {
-				return nil, fmt.Errorf("invalid env from image: %s", item)
-			}
-
-			if parts[0] == "" {
-				continue
-			}
-			specgen.AddProcessEnv(parts[0], parts[1])
-		}
+	for _, e := range envs {
+		parts := strings.SplitN(e, "=", 2)
+		specgen.AddProcessEnv(parts[0], parts[1])
 	}
 
 	// Set working directory
@@ -978,6 +1062,32 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 		containerCwd = runtimeCwd
 	}
 	specgen.SetProcessCwd(containerCwd)
+	if err := setupWorkingDirectory(mountPoint, mountLabel, containerCwd); err != nil {
+		if err1 := s.StorageRuntimeServer().StopContainer(containerID); err1 != nil {
+			return nil, fmt.Errorf("can't umount container after cwd error %v: %v", err, err1)
+		}
+		return nil, err
+	}
+
+	var secretMounts []rspec.Mount
+	if len(s.config.DefaultMounts) > 0 {
+		var err error
+		secretMounts, err = addSecretsBindMounts(mountLabel, containerInfo.RunDir, s.config.DefaultMounts, specgen)
+		if err != nil {
+			return nil, fmt.Errorf("failed to mount secrets: %v", err)
+		}
+	}
+
+	mounts := []rspec.Mount{}
+	mounts = append(mounts, ociMounts...)
+	mounts = append(mounts, volumeMounts...)
+	mounts = append(mounts, secretMounts...)
+
+	sort.Sort(orderedMounts(mounts))
+
+	for _, m := range mounts {
+		specgen.AddBindMount(m.Source, m.Destination, m.Options)
+	}
 
 	if err := s.setupOCIHooks(&specgen, sb, containerConfig, processArgs[0]); err != nil {
 		return nil, err
@@ -1013,6 +1123,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 	if err != nil {
 		return nil, err
 	}
+	container.SetSpec(specgen.Spec())
 	container.SetMountPoint(mountPoint)
 
 	for _, cv := range containerVolumes {
@@ -1135,3 +1246,19 @@ func clearReadOnly(m *rspec.Mount) {
 	}
 	m.Options = opt
 }
+
+func setupWorkingDirectory(rootfs, mountLabel, containerCwd string) error {
+	fp, err := symlink.FollowSymlinkInScope(filepath.Join(rootfs, containerCwd), rootfs)
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(fp, 0755); err != nil {
+		return err
+	}
+	if mountLabel != "" {
+		if err1 := label.Relabel(fp, mountLabel, true); err1 != nil && err1 != unix.ENOTSUP {
+			return fmt.Errorf("relabel failed %s: %v", fp, err1)
+		}
+	}
+	return nil
+}

server/container_exec.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"time"
 
 	"github.com/docker/docker/pkg/pools"
 	"github.com/kubernetes-incubator/cri-o/oci"
@@ -18,10 +19,16 @@ import (
 )
 
 // Exec prepares a streaming endpoint to execute a command in the container.
-func (s *Server) Exec(ctx context.Context, req *pb.ExecRequest) (*pb.ExecResponse, error) {
+func (s *Server) Exec(ctx context.Context, req *pb.ExecRequest) (resp *pb.ExecResponse, err error) {
+	const operation = "exec"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
+
 	logrus.Debugf("ExecRequest %+v", req)
 
-	resp, err := s.GetExec(req)
+	resp, err = s.GetExec(req)
 	if err != nil {
 		return nil, fmt.Errorf("unable to prepare exec endpoint")
 	}
@@ -46,12 +53,15 @@ func (ss streamService) Exec(containerID string, cmd []string, stdin io.Reader,
 		return fmt.Errorf("container is not created or running")
 	}
 
-	args := []string{"exec"}
-	if tty {
-		args = append(args, "-t")
+	processFile, err := oci.PrepareProcessExec(c, cmd, tty)
+	if err != nil {
+		return err
 	}
+	defer os.RemoveAll(processFile.Name())
+
+	args := []string{"exec"}
+	args = append(args, "--process", processFile.Name())
 	args = append(args, c.ID())
-	args = append(args, cmd...)
 	execCmd := exec.Command(ss.runtimeServer.Runtime().Path(c), args...)
 	var cmdErr error
 	if tty {

server/container_execsync.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/sirupsen/logrus"
@@ -10,7 +11,12 @@ import (
 )
 
 // ExecSync runs a command in a container synchronously.
-func (s *Server) ExecSync(ctx context.Context, req *pb.ExecSyncRequest) (*pb.ExecSyncResponse, error) {
+func (s *Server) ExecSync(ctx context.Context, req *pb.ExecSyncRequest) (resp *pb.ExecSyncResponse, err error) {
+	const operation = "exec_sync"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
 	logrus.Debugf("ExecSyncRequest %+v", req)
 	c, err := s.GetContainerFromRequest(req.ContainerId)
 	if err != nil {
@@ -35,7 +41,7 @@ func (s *Server) ExecSync(ctx context.Context, req *pb.ExecSyncRequest) (*pb.Exe
 	if err != nil {
 		return nil, err
 	}
-	resp := &pb.ExecSyncResponse{
+	resp = &pb.ExecSyncResponse{
 		Stdout:   execResp.Stdout,
 		Stderr:   execResp.Stderr,
 		ExitCode: execResp.ExitCode,

server/container_list.go

@@ -1,6 +1,8 @@
 package server
 
 import (
+	"time"
+
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
@@ -27,41 +29,49 @@ func filterContainer(c *pb.Container, filter *pb.ContainerFilter) bool {
 }
 
 // ListContainers lists all containers by filters.
-func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersRequest) (*pb.ListContainersResponse, error) {
+func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersRequest) (resp *pb.ListContainersResponse, err error) {
+	const operation = "list_containers"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
 	logrus.Debugf("ListContainersRequest %+v", req)
+
 	var ctrs []*pb.Container
-	filter := req.Filter
+	filter := req.GetFilter()
 	ctrList, err := s.ContainerServer.ListContainers()
 	if err != nil {
 		return nil, err
 	}
 
-	// Filter using container id and pod id first.
-	if filter.Id != "" {
-		id, err := s.CtrIDIndex().Get(filter.Id)
-		if err != nil {
-			return nil, err
-		}
-		c := s.ContainerServer.GetContainer(id)
-		if c != nil {
-			if filter.PodSandboxId != "" {
-				if c.Sandbox() == filter.PodSandboxId {
-					ctrList = []*oci.Container{c}
-				} else {
-					ctrList = []*oci.Container{}
-				}
-
-			} else {
-				ctrList = []*oci.Container{c}
+	if filter != nil {
+		// Filter using container id and pod id first.
+		if filter.Id != "" {
+			id, err := s.CtrIDIndex().Get(filter.Id)
+			if err != nil {
+				return nil, err
 			}
-		}
-	} else {
-		if filter.PodSandboxId != "" {
-			pod := s.ContainerServer.GetSandbox(filter.PodSandboxId)
-			if pod == nil {
-				ctrList = []*oci.Container{}
-			} else {
-				ctrList = pod.Containers().List()
+			c := s.ContainerServer.GetContainer(id)
+			if c != nil {
+				if filter.PodSandboxId != "" {
+					if c.Sandbox() == filter.PodSandboxId {
+						ctrList = []*oci.Container{c}
+					} else {
+						ctrList = []*oci.Container{}
+					}
+
+				} else {
+					ctrList = []*oci.Container{c}
+				}
+			}
+		} else {
+			if filter.PodSandboxId != "" {
+				pod := s.ContainerServer.GetSandbox(filter.PodSandboxId)
+				if pod == nil {
+					ctrList = []*oci.Container{}
+				} else {
+					ctrList = pod.Containers().List()
+				}
 			}
 		}
 	}
@@ -101,7 +111,7 @@ func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersReque
 		}
 	}
 
-	resp := &pb.ListContainersResponse{
+	resp = &pb.ListContainersResponse{
 		Containers: ctrs,
 	}
 	logrus.Debugf("ListContainersResponse: %+v", resp)

server/container_portforward.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"os/exec"
 	"strings"
+	"time"
 
 	"github.com/docker/docker/pkg/pools"
 	"github.com/kubernetes-incubator/cri-o/oci"
@@ -15,11 +16,15 @@ import (
 )
 
 // PortForward prepares a streaming endpoint to forward ports from a PodSandbox.
-func (s *Server) PortForward(ctx context.Context, req *pb.PortForwardRequest) (*pb.PortForwardResponse, error) {
+func (s *Server) PortForward(ctx context.Context, req *pb.PortForwardRequest) (resp *pb.PortForwardResponse, err error) {
+	const operation = "port_forward"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
 	logrus.Debugf("PortForwardRequest %+v", req)
 
-	resp, err := s.GetPortForward(req)
-
+	resp, err = s.GetPortForward(req)
 	if err != nil {
 		return nil, fmt.Errorf("unable to prepare portforward endpoint")
 	}

server/container_remove.go

@@ -1,6 +1,8 @@
 package server
 
 import (
+	"time"
+
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
@@ -8,13 +10,20 @@ import (
 
 // RemoveContainer removes the container. If the container is running, the container
 // should be force removed.
-func (s *Server) RemoveContainer(ctx context.Context, req *pb.RemoveContainerRequest) (*pb.RemoveContainerResponse, error) {
-	_, err := s.ContainerServer.Remove(req.ContainerId, true)
+func (s *Server) RemoveContainer(ctx context.Context, req *pb.RemoveContainerRequest) (resp *pb.RemoveContainerResponse, err error) {
+	const operation = "remove_container"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
+	logrus.Debugf("RemoveContainerRequest: %+v", req)
+
+	_, err = s.ContainerServer.Remove(ctx, req.ContainerId, true)
 	if err != nil {
 		return nil, err
 	}
 
-	resp := &pb.RemoveContainerResponse{}
+	resp = &pb.RemoveContainerResponse{}
 	logrus.Debugf("RemoveContainerResponse: %+v", resp)
 	return resp, nil
 }

server/container_start.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/sirupsen/logrus"
@@ -10,7 +11,12 @@ import (
 )
 
 // StartContainer starts the container.
-func (s *Server) StartContainer(ctx context.Context, req *pb.StartContainerRequest) (*pb.StartContainerResponse, error) {
+func (s *Server) StartContainer(ctx context.Context, req *pb.StartContainerRequest) (resp *pb.StartContainerResponse, err error) {
+	const operation = "start_container"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
 	logrus.Debugf("StartContainerRequest %+v", req)
 	c, err := s.GetContainerFromRequest(req.ContainerId)
 	if err != nil {
@@ -37,7 +43,7 @@ func (s *Server) StartContainer(ctx context.Context, req *pb.StartContainerReque
 		return nil, fmt.Errorf("failed to start container %s: %v", c.ID(), err)
 	}
 
-	resp := &pb.StartContainerResponse{}
+	resp = &pb.StartContainerResponse{}
 	logrus.Debugf("StartContainerResponse %+v", resp)
 	return resp, nil
 }

server/container_stats.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"fmt"
+	"time"
 
 	"golang.org/x/net/context"
 	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
@@ -9,6 +10,11 @@ import (
 
 // ContainerStats returns stats of the container. If the container does not
 // exist, the call returns an error.
-func (s *Server) ContainerStats(ctx context.Context, req *pb.ContainerStatsRequest) (*pb.ContainerStatsResponse, error) {
+func (s *Server) ContainerStats(ctx context.Context, req *pb.ContainerStatsRequest) (resp *pb.ContainerStatsResponse, err error) {
+	const operation = "container_stats"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
 	return nil, fmt.Errorf("not implemented")
 }

server/container_stats_list.go

@@ -2,12 +2,18 @@ package server
 
 import (
 	"fmt"
+	"time"
 
 	"golang.org/x/net/context"
 	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 )
 
 // ListContainerStats returns stats of all running containers.
-func (s *Server) ListContainerStats(ctx context.Context, req *pb.ListContainerStatsRequest) (*pb.ListContainerStatsResponse, error) {
+func (s *Server) ListContainerStats(ctx context.Context, req *pb.ListContainerStatsRequest) (resp *pb.ListContainerStatsResponse, err error) {
+	const operation = "list_container_stats"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
 	return nil, fmt.Errorf("not implemented")
 }

server/container_status.go

@@ -1,6 +1,8 @@
 package server
 
 import (
+	"time"
+
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
@@ -14,7 +16,12 @@ const (
 )
 
 // ContainerStatus returns status of the container.
-func (s *Server) ContainerStatus(ctx context.Context, req *pb.ContainerStatusRequest) (*pb.ContainerStatusResponse, error) {
+func (s *Server) ContainerStatus(ctx context.Context, req *pb.ContainerStatusRequest) (resp *pb.ContainerStatusResponse, err error) {
+	const operation = "container_status"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
 	logrus.Debugf("ContainerStatusRequest %+v", req)
 	c, err := s.GetContainerFromRequest(req.ContainerId)
 	if err != nil {
@@ -22,7 +29,7 @@ func (s *Server) ContainerStatus(ctx context.Context, req *pb.ContainerStatusReq
 	}
 
 	containerID := c.ID()
-	resp := &pb.ContainerStatusResponse{
+	resp = &pb.ContainerStatusResponse{
 		Status: &pb.ContainerStatus{
 			Id:          containerID,
 			Metadata:    c.Metadata(),

server/container_stop.go

@@ -1,19 +1,28 @@
 package server
 
 import (
+	"time"
+
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 )
 
 // StopContainer stops a running container with a grace period (i.e., timeout).
-func (s *Server) StopContainer(ctx context.Context, req *pb.StopContainerRequest) (*pb.StopContainerResponse, error) {
-	_, err := s.ContainerServer.ContainerStop(req.ContainerId, req.Timeout)
+func (s *Server) StopContainer(ctx context.Context, req *pb.StopContainerRequest) (resp *pb.StopContainerResponse, err error) {
+	const operation = "stop_container"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
+	logrus.Debugf("StopContainerRequest %+v", req)
+
+	_, err = s.ContainerServer.ContainerStop(ctx, req.ContainerId, req.Timeout)
 	if err != nil {
 		return nil, err
 	}
 
-	resp := &pb.StopContainerResponse{}
+	resp = &pb.StopContainerResponse{}
 	logrus.Debugf("StopContainerResponse %s: %+v", req.ContainerId, resp)
 	return resp, nil
 }

server/container_updateruntimeconfig.go

@@ -1,11 +1,19 @@
 package server
 
 import (
+	"time"
+
 	"golang.org/x/net/context"
 	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 )
 
 // UpdateRuntimeConfig updates the configuration of a running container.
-func (s *Server) UpdateRuntimeConfig(ctx context.Context, req *pb.UpdateRuntimeConfigRequest) (*pb.UpdateRuntimeConfigResponse, error) {
+func (s *Server) UpdateRuntimeConfig(ctx context.Context, req *pb.UpdateRuntimeConfigRequest) (resp *pb.UpdateRuntimeConfigResponse, err error) {
+	const operation = "update_runtime_config"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
+
 	return &pb.UpdateRuntimeConfigResponse{}, nil
 }

server/image_fs_info.go

@@ -2,12 +2,19 @@ package server
 
 import (
 	"fmt"
+	"time"
 
 	"golang.org/x/net/context"
 	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 )
 
 // ImageFsInfo returns information of the filesystem that is used to store images.
-func (s *Server) ImageFsInfo(ctx context.Context, req *pb.ImageFsInfoRequest) (*pb.ImageFsInfoResponse, error) {
+func (s *Server) ImageFsInfo(ctx context.Context, req *pb.ImageFsInfoRequest) (resp *pb.ImageFsInfoResponse, err error) {
+	const operation = "image_fs_info"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
+
 	return nil, fmt.Errorf("not implemented")
 }

server/image_list.go

@@ -1,13 +1,21 @@
 package server
 
 import (
+	"time"
+
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 )
 
 // ListImages lists existing images.
-func (s *Server) ListImages(ctx context.Context, req *pb.ListImagesRequest) (*pb.ListImagesResponse, error) {
+func (s *Server) ListImages(ctx context.Context, req *pb.ListImagesRequest) (resp *pb.ListImagesResponse, err error) {
+	const operation = "list_images"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
+
 	logrus.Debugf("ListImagesRequest: %+v", req)
 	filter := ""
 	reqFilter := req.GetFilter()
@@ -21,21 +29,21 @@ func (s *Server) ListImages(ctx context.Context, req *pb.ListImagesRequest) (*pb
 	if err != nil {
 		return nil, err
 	}
-	response := pb.ListImagesResponse{}
+	resp = &pb.ListImagesResponse{}
 	for _, result := range results {
 		if result.Size != nil {
-			response.Images = append(response.Images, &pb.Image{
+			resp.Images = append(resp.Images, &pb.Image{
 				Id:       result.ID,
 				RepoTags: result.Names,
 				Size_:    *result.Size,
 			})
 		} else {
-			response.Images = append(response.Images, &pb.Image{
+			resp.Images = append(resp.Images, &pb.Image{
 				Id:       result.ID,
 				RepoTags: result.Names,
 			})
 		}
 	}
-	logrus.Debugf("ListImagesResponse: %+v", response)
-	return &response, nil
+	logrus.Debugf("ListImagesResponse: %+v", resp)
+	return resp, nil
 }

server/image_pull.go

@@ -3,16 +3,24 @@ package server
 import (
 	"encoding/base64"
 	"strings"
+	"time"
 
 	"github.com/containers/image/copy"
 	"github.com/containers/image/types"
+	"github.com/kubernetes-incubator/cri-o/pkg/storage"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 )
 
 // PullImage pulls a image with authentication config.
-func (s *Server) PullImage(ctx context.Context, req *pb.PullImageRequest) (*pb.PullImageResponse, error) {
+func (s *Server) PullImage(ctx context.Context, req *pb.PullImageRequest) (resp *pb.PullImageResponse, err error) {
+	const operation = "pull_image"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
+
 	logrus.Debugf("PullImageRequest: %+v", req)
 	// TODO: what else do we need here? (Signatures when the story isn't just pulling from docker://)
 	image := ""
@@ -24,7 +32,6 @@ func (s *Server) PullImage(ctx context.Context, req *pb.PullImageRequest) (*pb.P
 	var (
 		images []string
 		pulled string
-		err    error
 	)
 	images, err = s.StorageImageServer().ResolveNames(image)
 	if err != nil {
@@ -67,11 +74,23 @@ func (s *Server) PullImage(ctx context.Context, req *pb.PullImageRequest) (*pb.P
 		}
 
 		// let's be smart, docker doesn't repull if image already exists.
-		_, err = s.StorageImageServer().ImageStatus(s.ImageContext(), img)
+		var storedImage *storage.ImageResult
+		storedImage, err = s.StorageImageServer().ImageStatus(s.ImageContext(), img)
 		if err == nil {
-			logrus.Debugf("image %s already in store, skipping pull", img)
-			pulled = img
-			break
+			tmpImg, err := s.StorageImageServer().PrepareImage(s.ImageContext(), img, options)
+			if err == nil {
+				tmpImgConfigDigest := tmpImg.ConfigInfo().Digest
+				if tmpImgConfigDigest.String() == "" {
+					// this means we are playing with a schema1 image, in which
+					// case, we're going to repull the image in any case
+					logrus.Debugf("image config digest is empty, re-pulling image")
+				} else if tmpImgConfigDigest.String() == storedImage.ConfigDigest.String() {
+					logrus.Debugf("image %s already in store, skipping pull", img)
+					pulled = img
+					break
+				}
+			}
+			logrus.Debugf("image in store has different ID, re-pulling %s", img)
 		}
 
 		_, err = s.StorageImageServer().PullImage(s.ImageContext(), img, options)
@@ -85,7 +104,7 @@ func (s *Server) PullImage(ctx context.Context, req *pb.PullImageRequest) (*pb.P
 	if pulled == "" && err != nil {
 		return nil, err
 	}
-	resp := &pb.PullImageResponse{
+	resp = &pb.PullImageResponse{
 		ImageRef: pulled,
 	}
 	logrus.Debugf("PullImageResponse: %+v", resp)

server/image_remove.go

@@ -2,15 +2,22 @@ package server
 
 import (
 	"fmt"
-	"strings"
+	"time"
 
+	"github.com/kubernetes-incubator/cri-o/pkg/storage"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 )
 
 // RemoveImage removes the image.
-func (s *Server) RemoveImage(ctx context.Context, req *pb.RemoveImageRequest) (*pb.RemoveImageResponse, error) {
+func (s *Server) RemoveImage(ctx context.Context, req *pb.RemoveImageRequest) (resp *pb.RemoveImageResponse, err error) {
+	const operation = "remove_image"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
+
 	logrus.Debugf("RemoveImageRequest: %+v", req)
 	image := ""
 	img := req.GetImage()
@@ -22,13 +29,11 @@ func (s *Server) RemoveImage(ctx context.Context, req *pb.RemoveImageRequest) (*
 	}
 	var (
 		images  []string
-		err     error
 		deleted bool
 	)
 	images, err = s.StorageImageServer().ResolveNames(image)
 	if err != nil {
-		// This means we got an image ID
-		if strings.Contains(err.Error(), "cannot specify 64-byte hexadecimal strings") {
+		if err == storage.ErrCannotParseImageID {
 			images = append(images, image)
 		} else {
 			return nil, err
@@ -46,7 +51,7 @@ func (s *Server) RemoveImage(ctx context.Context, req *pb.RemoveImageRequest) (*
 	if !deleted && err != nil {
 		return nil, err
 	}
-	resp := &pb.RemoveImageResponse{}
+	resp = &pb.RemoveImageResponse{}
 	logrus.Debugf("RemoveImageResponse: %+v", resp)
 	return resp, nil
 }

server/image_status.go

@@ -2,9 +2,10 @@ package server
 
 import (
 	"fmt"
-	"strings"
+	"time"
 
 	"github.com/containers/storage"
+	pkgstorage "github.com/kubernetes-incubator/cri-o/pkg/storage"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
@@ -12,7 +13,13 @@ import (
 )
 
 // ImageStatus returns the status of the image.
-func (s *Server) ImageStatus(ctx context.Context, req *pb.ImageStatusRequest) (*pb.ImageStatusResponse, error) {
+func (s *Server) ImageStatus(ctx context.Context, req *pb.ImageStatusRequest) (resp *pb.ImageStatusResponse, err error) {
+	const operation = "image_status"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
+
 	logrus.Debugf("ImageStatusRequest: %+v", req)
 	image := ""
 	img := req.GetImage()
@@ -24,8 +31,7 @@ func (s *Server) ImageStatus(ctx context.Context, req *pb.ImageStatusRequest) (*
 	}
 	images, err := s.StorageImageServer().ResolveNames(image)
 	if err != nil {
-		// This means we got an image ID
-		if strings.Contains(err.Error(), "cannot specify 64-byte hexadecimal strings") {
+		if err == pkgstorage.ErrCannotParseImageID {
 			images = append(images, image)
 		} else {
 			return nil, err
@@ -40,7 +46,7 @@ func (s *Server) ImageStatus(ctx context.Context, req *pb.ImageStatusRequest) (*
 		}
 		return nil, err
 	}
-	resp := &pb.ImageStatusResponse{
+	resp = &pb.ImageStatusResponse{
 		Image: &pb.Image{
 			Id:       status.ID,
 			RepoTags: status.Names,

server/inspect.go

@@ -48,7 +48,7 @@ func (s *Server) getContainerInfo(id string, getContainerFunc func(id string) *o
 	return types.ContainerInfo{
 		Name:            ctr.Name(),
 		Pid:             ctrState.Pid,
-		Image:           ctr.Image(),
+		Image:           ctr.ImageName(),
 		CreatedTime:     ctrState.Created.UnixNano(),
 		Labels:          ctr.Labels(),
 		Annotations:     ctr.Annotations(),

server/inspect_test.go

@@ -96,7 +96,7 @@ func TestGetContainerInfo(t *testing.T) {
 		t.Fatalf("expected same created time %d, got %d", created.UnixNano(), ci.CreatedTime)
 	}
 	if ci.Pid != 42 {
-		t.Fatalf("expected pid 42, got %s", ci.Pid)
+		t.Fatalf("expected pid 42, got %v", ci.Pid)
 	}
 	if ci.Name != "testname" {
 		t.Fatalf("expected name testname, got %s", ci.Name)
@@ -114,7 +114,7 @@ func TestGetContainerInfo(t *testing.T) {
 		t.Fatalf("expected sandbox to be testsandboxid, got %s", ci.Sandbox)
 	}
 	if ci.IP != "1.1.1.42" {
-		t.Fatal("expected ip 1.1.1.42, got %s", ci.IP)
+		t.Fatalf("expected ip 1.1.1.42, got %s", ci.IP)
 	}
 	if len(ci.Annotations) == 0 {
 		t.Fatal("annotations are empty")

server/metrics/metrics.go

@@ -0,0 +1,70 @@
+package metrics
+
+import (
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	// CRIOOperationsKey is the key for CRI-O operation metrics.
+	CRIOOperationsKey = "crio_operations"
+	// CRIOOperationsLatencyKey is the key for the operation latency metrics.
+	CRIOOperationsLatencyKey = "crio_operations_latency_microseconds"
+	// CRIOOperationsErrorsKey is the key for the operation error metrics.
+	CRIOOperationsErrorsKey = "crio_operations_errors"
+
+	// TODO(runcom):
+	// timeouts
+
+	subsystem = "container_runtime"
+)
+
+var (
+	// CRIOOperations collects operation counts by operation type.
+	CRIOOperations = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: subsystem,
+			Name:      CRIOOperationsKey,
+			Help:      "Cumulative number of CRI-O operations by operation type.",
+		},
+		[]string{"operation_type"},
+	)
+	// CRIOOperationsLatency collects operation latency numbers by operation
+	// type.
+	CRIOOperationsLatency = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Subsystem: subsystem,
+			Name:      CRIOOperationsLatencyKey,
+			Help:      "Latency in microseconds of CRI-O operations. Broken down by operation type.",
+		},
+		[]string{"operation_type"},
+	)
+	// CRIOOperationsErrors collects operation errors by operation
+	// type.
+	CRIOOperationsErrors = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: subsystem,
+			Name:      CRIOOperationsErrorsKey,
+			Help:      "Cumulative number of CRI-O operation errors by operation type.",
+		},
+		[]string{"operation_type"},
+	)
+)
+
+var registerMetrics sync.Once
+
+// Register all metrics
+func Register() {
+	registerMetrics.Do(func() {
+		prometheus.MustRegister(CRIOOperations)
+		prometheus.MustRegister(CRIOOperationsLatency)
+		prometheus.MustRegister(CRIOOperationsErrors)
+	})
+}
+
+// SinceInMicroseconds gets the time since the specified start in microseconds.
+func SinceInMicroseconds(start time.Time) float64 {
+	return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds())
+}

server/runtime_status.go

@@ -1,12 +1,19 @@
 package server
 
 import (
+	"time"
+
 	"golang.org/x/net/context"
 	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 )
 
 // Status returns the status of the runtime
-func (s *Server) Status(ctx context.Context, req *pb.StatusRequest) (*pb.StatusResponse, error) {
+func (s *Server) Status(ctx context.Context, req *pb.StatusRequest) (resp *pb.StatusResponse, err error) {
+	const operation = "status"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
 
 	// Deal with Runtime conditions
 	runtimeReady, err := s.Runtime().RuntimeReady()
@@ -22,7 +29,7 @@ func (s *Server) Status(ctx context.Context, req *pb.StatusRequest) (*pb.StatusR
 	runtimeReadyConditionString := pb.RuntimeReady
 	networkReadyConditionString := pb.NetworkReady
 
-	resp := &pb.StatusResponse{
+	resp = &pb.StatusResponse{
 		Status: &pb.RuntimeStatus{
 			Conditions: []*pb.RuntimeCondition{
 				{

server/sandbox_list.go

@@ -1,6 +1,8 @@
 package server
 
 import (
+	"time"
+
 	"github.com/kubernetes-incubator/cri-o/libkpod/sandbox"
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/sirupsen/logrus"
@@ -28,7 +30,13 @@ func filterSandbox(p *pb.PodSandbox, filter *pb.PodSandboxFilter) bool {
 }
 
 // ListPodSandbox returns a list of SandBoxes.
-func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxRequest) (*pb.ListPodSandboxResponse, error) {
+func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxRequest) (resp *pb.ListPodSandboxResponse, err error) {
+	const operation = "list_pod_sandbox"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
+
 	logrus.Debugf("ListPodSandboxRequest %+v", req)
 	var pods []*pb.PodSandbox
 	var podList []*sandbox.Sandbox
@@ -82,7 +90,7 @@ func (s *Server) ListPodSandbox(ctx context.Context, req *pb.ListPodSandboxReque
 		}
 	}
 
-	resp := &pb.ListPodSandboxResponse{
+	resp = &pb.ListPodSandboxResponse{
 		Items: pods,
 	}
 	logrus.Debugf("ListPodSandboxResponse %+v", resp)

server/sandbox_remove.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/containers/storage"
 	"github.com/kubernetes-incubator/cri-o/libkpod/sandbox"
@@ -15,7 +16,13 @@ import (
 
 // RemovePodSandbox deletes the sandbox. If there are any running containers in the
 // sandbox, they should be force deleted.
-func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxRequest) (*pb.RemovePodSandboxResponse, error) {
+func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxRequest) (resp *pb.RemovePodSandboxResponse, err error) {
+	const operation = "remove_pod_sandbox"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
+
 	logrus.Debugf("RemovePodSandboxRequest %+v", req)
 	sb, err := s.getPodSandboxFromRequest(req.PodSandboxId)
 	if err != nil {
@@ -27,7 +34,7 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR
 		// the the CRI interface which expects to not error out in not found
 		// cases.
 
-		resp := &pb.RemovePodSandboxResponse{}
+		resp = &pb.RemovePodSandboxResponse{}
 		logrus.Warnf("could not get sandbox %s, it's probably been removed already: %v", req.PodSandboxId, err)
 		return resp, nil
 	}
@@ -41,7 +48,7 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR
 		if !sb.Stopped() {
 			cState := s.Runtime().ContainerStatus(c)
 			if cState.Status == oci.ContainerStateCreated || cState.Status == oci.ContainerStateRunning {
-				if err := s.Runtime().StopContainer(c, -1); err != nil {
+				if err := s.Runtime().StopContainer(ctx, c, 10); err != nil {
 					// Assume container is already stopped
 					logrus.Warnf("failed to stop container %s: %v", c.Name(), err)
 				}
@@ -92,7 +99,7 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR
 		return nil, fmt.Errorf("failed to delete pod sandbox %s from index: %v", sb.ID(), err)
 	}
 
-	resp := &pb.RemovePodSandboxResponse{}
+	resp = &pb.RemovePodSandboxResponse{}
 	logrus.Debugf("RemovePodSandboxResponse %+v", resp)
 	return resp, nil
 }

server/sandbox_run.go

@@ -16,6 +16,7 @@ import (
 	"github.com/kubernetes-incubator/cri-o/libkpod/sandbox"
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/kubernetes-incubator/cri-o/pkg/annotations"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/generate"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
@@ -94,6 +95,12 @@ var (
 
 // RunPodSandbox creates and runs a pod-level sandbox.
 func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest) (resp *pb.RunPodSandboxResponse, err error) {
+	const operation = "run_pod_sandbox"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
+
 	s.updateLock.RLock()
 	defer s.updateLock.RUnlock()
 
@@ -220,6 +227,10 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
 	// add labels
 	labels := req.GetConfig().GetLabels()
 
+	if err := validateLabels(labels); err != nil {
+		return nil, err
+	}
+
 	// Add special container name label for the infra container
 	labelsJSON := []byte{}
 	if labels != nil {
@@ -424,7 +435,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
 
 	// set up namespaces
 	if hostNetwork {
-		err = g.RemoveLinuxNamespace("network")
+		err = g.RemoveLinuxNamespace(string(runtimespec.NetworkNamespace))
 		if err != nil {
 			return nil, err
 		}
@@ -445,21 +456,21 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
 		}()
 
 		// Pass the created namespace path to the runtime
-		err = g.AddOrReplaceLinuxNamespace("network", sb.NetNsPath())
+		err = g.AddOrReplaceLinuxNamespace(string(runtimespec.NetworkNamespace), sb.NetNsPath())
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	if namespaceOptions.HostPid {
-		err = g.RemoveLinuxNamespace("pid")
+	if securityContext.GetNamespaceOptions().GetHostPid() {
+		err = g.RemoveLinuxNamespace(string(runtimespec.PIDNamespace))
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	if namespaceOptions.HostIpc {
-		err = g.RemoveLinuxNamespace("ipc")
+	if securityContext.GetNamespaceOptions().GetHostIpc() {
+		err = g.RemoveLinuxNamespace(string(runtimespec.IPCNamespace))
 		if err != nil {
 			return nil, err
 		}
@@ -492,6 +503,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
 	if err != nil {
 		return nil, err
 	}
+	container.SetSpec(g.Spec())
 	container.SetMountPoint(mountPoint)
 
 	sb.SetInfraContainer(container)

server/sandbox_status.go

@@ -1,6 +1,8 @@
 package server
 
 import (
+	"time"
+
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
@@ -8,7 +10,13 @@ import (
 )
 
 // PodSandboxStatus returns the Status of the PodSandbox.
-func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusRequest) (*pb.PodSandboxStatusResponse, error) {
+func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusRequest) (resp *pb.PodSandboxStatusResponse, err error) {
+	const operation = "pod_sandbox_status"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
+
 	logrus.Debugf("PodSandboxStatusRequest %+v", req)
 	sb, err := s.getPodSandboxFromRequest(req.PodSandboxId)
 	if err != nil {
@@ -24,7 +32,7 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR
 	}
 
 	sandboxID := sb.ID()
-	resp := &pb.PodSandboxStatusResponse{
+	resp = &pb.PodSandboxStatusResponse{
 		Status: &pb.PodSandboxStatus{
 			Id:          sandboxID,
 			CreatedAt:   podInfraContainer.CreatedAt().UnixNano(),

server/sandbox_stop.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/containers/storage"
 	"github.com/docker/docker/pkg/mount"
@@ -18,7 +19,13 @@ import (
 
 // StopPodSandbox stops the sandbox. If there are any running containers in the
 // sandbox, they should be force terminated.
-func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxRequest) (*pb.StopPodSandboxResponse, error) {
+func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxRequest) (resp *pb.StopPodSandboxResponse, err error) {
+	const operation = "stop_pod_sandbox"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
+
 	logrus.Debugf("StopPodSandboxRequest %+v", req)
 	sb, err := s.getPodSandboxFromRequest(req.PodSandboxId)
 	if err != nil {
@@ -30,14 +37,14 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque
 		// the the CRI interface which expects to not error out in not found
 		// cases.
 
-		resp := &pb.StopPodSandboxResponse{}
+		resp = &pb.StopPodSandboxResponse{}
 		logrus.Warnf("could not get sandbox %s, it's probably been stopped already: %v", req.PodSandboxId, err)
 		logrus.Debugf("StopPodSandboxResponse %s: %+v", req.PodSandboxId, resp)
 		return resp, nil
 	}
 
 	if sb.Stopped() {
-		resp := &pb.StopPodSandboxResponse{}
+		resp = &pb.StopPodSandboxResponse{}
 		logrus.Debugf("StopPodSandboxResponse %s: %+v", sb.ID(), resp)
 		return resp, nil
 	}
@@ -56,7 +63,7 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque
 	for _, c := range containers {
 		cStatus := s.Runtime().ContainerStatus(c)
 		if cStatus.Status != oci.ContainerStateStopped {
-			if err := s.Runtime().StopContainer(c, -1); err != nil {
+			if err := s.Runtime().StopContainer(ctx, c, 10); err != nil {
 				return nil, fmt.Errorf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err)
 			}
 			if c.ID() == podInfraContainer.ID() {
@@ -95,7 +102,7 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque
 	}
 
 	sb.SetStopped()
-	resp := &pb.StopPodSandboxResponse{}
+	resp = &pb.StopPodSandboxResponse{}
 	logrus.Debugf("StopPodSandboxResponse %s: %+v", sb.ID(), resp)
 	return resp, nil
 }

server/secrets.go

@@ -0,0 +1,162 @@
+package server
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	rspec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/selinux/go-selinux/label"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// SecretData info
+type SecretData struct {
+	Name string
+	Data []byte
+}
+
+// SaveTo saves secret data to given directory
+func (s SecretData) SaveTo(dir string) error {
+	path := filepath.Join(dir, s.Name)
+	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil && !os.IsExist(err) {
+		return err
+	}
+	return ioutil.WriteFile(path, s.Data, 0700)
+}
+
+func readAll(root, prefix string) ([]SecretData, error) {
+	path := filepath.Join(root, prefix)
+
+	data := []SecretData{}
+
+	files, err := ioutil.ReadDir(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return data, nil
+		}
+
+		return nil, err
+	}
+
+	for _, f := range files {
+		fileData, err := readFile(root, filepath.Join(prefix, f.Name()))
+		if err != nil {
+			// If the file did not exist, might be a dangling symlink
+			// Ignore the error
+			if os.IsNotExist(err) {
+				continue
+			}
+			return nil, err
+		}
+		data = append(data, fileData...)
+	}
+
+	return data, nil
+}
+
+func readFile(root, name string) ([]SecretData, error) {
+	path := filepath.Join(root, name)
+
+	s, err := os.Stat(path)
+	if err != nil {
+		return nil, err
+	}
+
+	if s.IsDir() {
+		dirData, err := readAll(root, name)
+		if err != nil {
+			return nil, err
+		}
+		return dirData, nil
+	}
+	bytes, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	return []SecretData{{Name: name, Data: bytes}}, nil
+}
+
+// getHostAndCtrDir separates the host:container paths
+func getMountsMap(path string) (string, string, error) {
+	arr := strings.SplitN(path, ":", 2)
+	if len(arr) == 2 {
+		return arr[0], arr[1], nil
+	}
+	return "", "", errors.Errorf("unable to get host and container dir")
+}
+
+func getHostSecretData(hostDir string) ([]SecretData, error) {
+	var allSecrets []SecretData
+	hostSecrets, err := readAll(hostDir, "")
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to read secrets from %q", hostDir)
+	}
+	return append(allSecrets, hostSecrets...), nil
+}
+
+// secretMount copies the contents of host directory to container directory
+// and returns a list of mounts
+func secretMounts(defaultMountsPaths []string, mountLabel, containerWorkingDir string, runtimeMounts []rspec.Mount) ([]rspec.Mount, error) {
+	var mounts []rspec.Mount
+	for _, path := range defaultMountsPaths {
+		hostDir, ctrDir, err := getMountsMap(path)
+		if err != nil {
+			return nil, err
+		}
+		// skip if the hostDir path doesn't exist
+		if _, err := os.Stat(hostDir); os.IsNotExist(err) {
+			logrus.Warnf("%q doesn't exist, skipping", hostDir)
+			continue
+		}
+
+		ctrDirOnHost := filepath.Join(containerWorkingDir, ctrDir)
+		// skip if ctrDir has already been mounted by caller
+		if isAlreadyMounted(runtimeMounts, ctrDir) {
+			logrus.Warnf("%q has already been mounted; cannot override mount", ctrDir)
+			continue
+		}
+
+		if err := os.RemoveAll(ctrDirOnHost); err != nil {
+			return nil, fmt.Errorf("remove container directory failed: %v", err)
+		}
+
+		if err := os.MkdirAll(ctrDirOnHost, 0755); err != nil {
+			return nil, fmt.Errorf("making container directory failed: %v", err)
+		}
+
+		hostDir, err = resolveSymbolicLink(hostDir)
+		if err != nil {
+			return nil, err
+		}
+
+		data, err := getHostSecretData(hostDir)
+		if err != nil {
+			return nil, errors.Wrapf(err, "getting host secret data failed")
+		}
+		for _, s := range data {
+			s.SaveTo(ctrDirOnHost)
+		}
+		label.Relabel(ctrDirOnHost, mountLabel, false)
+
+		m := rspec.Mount{
+			Source:      ctrDirOnHost,
+			Destination: ctrDir,
+		}
+
+		mounts = append(mounts, m)
+	}
+	return mounts, nil
+}
+
+func isAlreadyMounted(mounts []rspec.Mount, mountPath string) bool {
+	for _, mount := range mounts {
+		if mount.Destination == mountPath {
+			return true
+		}
+	}
+	return false
+}

server/server.go

@@ -20,6 +20,7 @@ import (
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/kubernetes-incubator/cri-o/pkg/storage"
 	"github.com/kubernetes-incubator/cri-o/server/apparmor"
+	"github.com/kubernetes-incubator/cri-o/server/metrics"
 	"github.com/kubernetes-incubator/cri-o/server/seccomp"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
@@ -35,8 +36,7 @@ import (
 )
 
 const (
-	runtimeAPIVersion = "v1alpha1"
-	shutdownFile      = "/var/lib/crio/crio.shutdown"
+	shutdownFile = "/var/lib/crio/crio.shutdown"
 )
 
 func isTrue(annotaton string) bool {
@@ -350,6 +350,7 @@ func (s *Server) getPodSandboxFromRequest(podSandboxID string) (*sandbox.Sandbox
 // CreateMetricsEndpoint creates a /metrics endpoint
 // for prometheus monitoring
 func (s *Server) CreateMetricsEndpoint() (*http.ServeMux, error) {
+	metrics.Register()
 	mux := &http.ServeMux{}
 	mux.Handle("/metrics", prometheus.Handler())
 	return mux, nil
@@ -419,6 +420,7 @@ func (s *Server) StartExitMonitor() {
 	}()
 	if err := watcher.Add(s.config.ContainerExitsDir); err != nil {
 		logrus.Errorf("watcher.Add(%q) failed: %s", s.config.ContainerExitsDir, err)
+		close(done)
 	}
 	<-done
 }

server/utils.go

@@ -5,9 +5,11 @@ import (
 	"io"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/cri-o/ocicni/pkg/ocicni"
 	"github.com/kubernetes-incubator/cri-o/libkpod/sandbox"
+	"github.com/kubernetes-incubator/cri-o/server/metrics"
 	"github.com/opencontainers/runtime-tools/validate"
 	"github.com/syndtr/gocapability/capability"
 )
@@ -16,6 +18,8 @@ const (
 	// According to http://man7.org/linux/man-pages/man5/resolv.conf.5.html:
 	// "The search list is currently limited to six domains with a total of 256 characters."
 	maxDNSSearches = 6
+
+	maxLabelSize = 4096
 )
 
 func copyFile(src, dest string) error {
@@ -181,3 +185,28 @@ func getOCICapabilitiesList() []string {
 	}
 	return caps
 }
+
+func recordOperation(operation string, start time.Time) {
+	metrics.CRIOOperations.WithLabelValues(operation).Inc()
+	metrics.CRIOOperationsLatency.WithLabelValues(operation).Observe(metrics.SinceInMicroseconds(start))
+}
+
+// recordError records error for metric if an error occurred.
+func recordError(operation string, err error) {
+	if err != nil {
+		// TODO(runcom): handle timeout from ctx as well
+		metrics.CRIOOperationsErrors.WithLabelValues(operation).Inc()
+	}
+}
+
+func validateLabels(labels map[string]string) error {
+	for k, v := range labels {
+		if (len(k) + len(v)) > maxLabelSize {
+			if len(k) > 10 {
+				k = k[:10]
+			}
+			return fmt.Errorf("label key and value greater than maximum size (%d bytes), key: %s", maxLabelSize, k)
+		}
+	}
+	return nil
+}

server/version.go

@@ -1,29 +1,35 @@
 package server
 
 import (
+	"time"
+
+	"github.com/kubernetes-incubator/cri-o/version"
 	"golang.org/x/net/context"
 	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 )
 
-// Version returns the runtime name, runtime version and runtime API version
-func (s *Server) Version(ctx context.Context, req *pb.VersionRequest) (*pb.VersionResponse, error) {
-
-	runtimeVersion, err := s.Runtime().Version()
-	if err != nil {
-		return nil, err
-	}
-
+const (
+	// kubeAPIVersion is the api version of kubernetes.
 	// TODO: Track upstream code. For now it expects 0.1.0
-	version := "0.1.0"
+	kubeAPIVersion = "0.1.0"
+	// containerName is the name prepended in kubectl describe->Container ID:
+	// cri-o://<CONTAINER_ID>
+	containerName     = "cri-o"
+	runtimeAPIVersion = "v1alpha1"
+)
 
-	// taking const address
-	rav := runtimeAPIVersion
-	runtimeName := s.Runtime().Name()
+// Version returns the runtime name, runtime version and runtime API version
+func (s *Server) Version(ctx context.Context, req *pb.VersionRequest) (resp *pb.VersionResponse, err error) {
+	const operation = "version"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
 
 	return &pb.VersionResponse{
-		Version:           version,
-		RuntimeName:       runtimeName,
-		RuntimeVersion:    runtimeVersion,
-		RuntimeApiVersion: rav,
+		Version:           kubeAPIVersion,
+		RuntimeName:       containerName,
+		RuntimeVersion:    version.Version,
+		RuntimeApiVersion: runtimeAPIVersion,
 	}, nil
 }

test/apparmor.bats

@@ -27,7 +27,6 @@ function teardown() {
     echo "$output"
     [ "$status" -eq 0 ]
     ctr_id="$output"
-    [ "$status" -eq 0 ]
     run crioctl ctr execsync --id "$ctr_id" touch test.txt
     echo "$output"
     [ "$status" -eq 0 ]
@@ -60,7 +59,6 @@ function teardown() {
     echo "$output"
     [ "$status" -eq 0 ]
     ctr_id="$output"
-    [ "$status" -eq 0 ]
     run crioctl ctr execsync --id "$ctr_id" touch test.txt
     echo "$output"
     [ "$status" -ne 0 ]
@@ -94,7 +92,6 @@ function teardown() {
     echo "$output"
     [ "$status" -eq 0 ]
     ctr_id="$output"
-    [ "$status" -eq 0 ]
     run crioctl ctr execsync --id "$ctr_id" touch test.txt
     echo "$output"
     [ "$status" -ne 0 ]
@@ -156,7 +153,6 @@ function teardown() {
     echo "$output"
     [ "$status" -eq 0 ]
     ctr_id="$output"
-    [ "$status" -eq 0 ]
     run crioctl ctr execsync --id "$ctr_id" touch test.txt
     echo "$output"
     [ "$status" -eq 0 ]

test/command.bats

@@ -0,0 +1,12 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "crio commands" {
+	run ${CRIO_BINARY} --config /dev/null config > /dev/null
+	echo "$output"
+	[ "$status" -eq 0 ]
+	run ${CRIO_BINARY} badoption > /dev/null
+	echo "$output"
+	[ "$status" -ne 0 ]
+}

test/ctr.bats

@@ -560,6 +560,7 @@ function teardown() {
 	run crioctl ctr execsync --id "$ctr_id" --timeout 1 sleep 10
 	echo "$output"
 	[[ "$output" =~ "command timed out" ]]
+	[ "$status" -ne 0 ]
 	run crioctl pod stop --id "$pod_id"
 	echo "$output"
 	[ "$status" -eq 0 ]
@@ -599,6 +600,31 @@ function teardown() {
 	stop_crio
 }
 
+@test "ctr hostname env" {
+	start_crio
+	run crioctl pod run --config "$TESTDATA"/sandbox_config.json
+	echo "$output"
+	[ "$status" -eq 0 ]
+	pod_id="$output"
+	run crioctl ctr create --config "$TESTDATA"/container_config.json --pod "$pod_id"
+	echo "$output"
+	[ "$status" -eq 0 ]
+	ctr_id="$output"
+	run crioctl ctr execsync --id "$ctr_id" env
+	echo "$output"
+	[ "$status" -eq 0 ]
+	[[ "$output" =~ "HOSTNAME" ]]
+	run crioctl pod stop --id "$pod_id"
+	echo "$output"
+	[ "$status" -eq 0 ]
+	run crioctl pod remove --id "$pod_id"
+	echo "$output"
+	[ "$status" -eq 0 ]
+	cleanup_ctrs
+	cleanup_pods
+	stop_crio
+}
+
 @test "ctr execsync failure" {
 	start_crio
 	run crioctl pod run --config "$TESTDATA"/sandbox_config.json
@@ -766,10 +792,16 @@ function teardown() {
 	echo "$output"
 	[ "$status" -eq 0 ]
 	# Wait for container to OOM
-	run sleep 100
-	run crioctl ctr status --id "$ctr_id"
-	echo "$output"
-	[ "$status" -eq 0 ]
+	attempt=0
+	while [ $attempt -le 100 ]; do
+		attempt=$((attempt+1))
+		run crioctl ctr status --id "$ctr_id"
+		echo "$output"
+		if [[ "$output" =~ "OOMKilled" ]]; then
+			break
+		fi
+		sleep 10
+	done
 	[[ "$output" =~ "OOMKilled" ]]
 	run crioctl pod stop --id "$pod_id"
 	echo "$output"
@@ -864,3 +896,59 @@ function teardown() {
 	cleanup_pods
 	stop_crio
 }
+
+@test "ctr correctly setup working directory" {
+	start_crio
+	run crioctl pod run --config "$TESTDATA"/sandbox_config.json
+	echo "$output"
+	[ "$status" -eq 0 ]
+	pod_id="$output"
+	notexistcwd=$(cat "$TESTDATA"/container_config.json | python -c 'import json,sys;obj=json.load(sys.stdin);obj["working_dir"] = "/thisshouldntexistatall"; json.dump(obj, sys.stdout)')
+	echo "$notexistcwd" > "$TESTDIR"/container_cwd_notexist.json
+	run crioctl ctr create --config "$TESTDIR"/container_cwd_notexist.json --pod "$pod_id"
+	echo "$output"
+	[ "$status" -eq 0 ]
+	ctr_id="$output"
+	run crioctl ctr start --id "$ctr_id"
+	echo "$output"
+	[ "$status" -eq 0 ]
+
+	filecwd=$(cat "$TESTDATA"/container_config.json | python -c 'import json,sys;obj=json.load(sys.stdin);obj["working_dir"] = "/etc/passwd"; obj["metadata"]["name"] = "container2"; json.dump(obj, sys.stdout)')
+	echo "$filecwd" > "$TESTDIR"/container_cwd_file.json
+	run crioctl ctr create --config "$TESTDIR"/container_cwd_file.json --pod "$pod_id"
+	echo "$output"
+	[ "$status" -ne 0 ]
+	ctr_id="$output"
+	[[ "$output" =~ "not a directory" ]]
+
+	cleanup_ctrs
+	cleanup_pods
+	stop_crio
+}
+
+@test "ctr execsync conflicting with conmon env" {
+	start_crio
+	run crictl runs "$TESTDATA"/sandbox_config.json
+	echo "$output"
+	[ "$status" -eq 0 ]
+	pod_id="$output"
+	run crictl create "$pod_id" "$TESTDATA"/container_redis_env_custom.json "$TESTDATA"/sandbox_config.json
+	echo "$output"
+	[ "$status" -eq 0 ]
+	ctr_id="$output"
+	run crictl start "$ctr_id"
+	echo "$output"
+	[ "$status" -eq 0 ]
+	run crictl exec "$ctr_id" env
+	echo "$output"
+	echo "$status"
+	[ "$status" -eq 0 ]
+	[[ "$output" =~ "acustompathinpath" ]]
+	run crictl exec --sync "$ctr_id" env
+	echo "$output"
+	[ "$status" -eq 0 ]
+	[[ "$output" =~ "acustompathinpath" ]]
+	cleanup_ctrs
+	cleanup_pods
+	stop_crio
+}

test/default_mounts.bats

@@ -0,0 +1,69 @@
+#!/usr/bin/env bats
+
+load helpers
+
+IMAGE="redis:alpine"
+
+function teardown() {
+	cleanup_test
+}
+
+@test "bind secrets mounts to container" {
+    start_crio
+    run crioctl pod run --config "$TESTDATA"/sandbox_config.json
+    echo "$output"
+    [ "$status" -eq 0 ]
+    pod_id="$output"
+    run crioctl image pull "$IMAGE"
+    [ "$status" -eq 0 ]
+    run crioctl ctr create --config "$TESTDATA"/container_redis.json --pod "$pod_id"
+    echo "$output"
+    [ "$status" -eq 0 ]
+    ctr_id="$output"
+    run crioctl ctr execsync --id "$ctr_id" cat /proc/mounts
+    echo "$output"
+    [ "$status" -eq 0 ]
+    mount_info="$output"
+    run grep /container/path1 <<< "$mount_info"
+    echo "$output"
+    [ "$status" -eq 0 ]
+    cleanup_ctrs
+    cleanup_pods
+    stop_crio
+}
+
+@test "default mounts correctly sorted with other mounts" {
+    start_crio
+    run crioctl pod run --config "$TESTDATA"/sandbox_config.json
+    echo "$output"
+    [ "$status" -eq 0 ]
+    pod_id="$output"
+    run crioctl image pull "$IMAGE"
+    [ "$status" -eq 0 ]
+    host_path="$TESTDIR"/clash
+    mkdir "$host_path"
+    echo "clashing..." > "$host_path"/clashing.txt
+    sed -e "s,%HPATH%,$host_path,g" "$TESTDATA"/container_redis_default_mounts.json > "$TESTDIR"/defmounts_pre.json
+    sed -e 's,%CPATH%,\/container\/path1\/clash,g' "$TESTDIR"/defmounts_pre.json > "$TESTDIR"/defmounts.json
+    run crioctl ctr create --config "$TESTDIR"/defmounts.json --pod "$pod_id"
+    echo "$output"
+    [ "$status" -eq 0 ]
+    ctr_id="$output"
+    run crioctl ctr execsync --id "$ctr_id" ls -la /container/path1/clash
+    echo "$output"
+    [ "$status" -eq 0 ]
+    run crioctl ctr execsync --id "$ctr_id" cat /container/path1/clash/clashing.txt
+    echo "$output"
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "clashing..." ]]
+    run crioctl ctr execsync --id "$ctr_id" ls -la /container/path1
+    echo "$output"
+    [ "$status" -eq 0 ]
+    run crioctl ctr execsync --id "$ctr_id" cat /container/path1/test.txt
+    echo "$output"
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "Testing secrets mounts!" ]]
+    cleanup_ctrs
+    cleanup_pods
+    stop_crio
+}